From ea5ca90c32b1a4f218a63a1f0d05409caebe1bba Mon Sep 17 00:00:00 2001 From: Spaction <33352702+Spaction@users.noreply.github.com> Date: Thu, 21 Mar 2024 00:01:11 -0400 Subject: [PATCH 01/10] Update NpmDriver.java Include Version compare for NpmBuildInfoExtractor --- .../jfrog/build/extractor/npm/NpmDriver.java | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/NpmDriver.java b/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/NpmDriver.java index 3a30f4ddf..fcf34c152 100644 --- a/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/NpmDriver.java +++ b/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/NpmDriver.java @@ -97,6 +97,32 @@ public JsonNode list(File workingDirectory, List extraArgs) throws IOExc } } + public int compareVersionTo(File workingDirectory, String compareVersion){ + String currentVersion = this.version(workingDirectory); + + List currentComponents = Arrays.stream(currentVersion.split("\\.")) + .map(Integer::parseInt) + .collect(Collectors.toList()); + List compareComponents = Arrays.stream(compareVersion.split("\\.")) + .map(Integer::parseInt) + .collect(Collectors.toList()); + + int currentSize = currentComponents.size(); + int compareSize = compareComponents.size(); + int maxLength = Math.max(currentSize, compareSize); + + for (int i = 0; i < maxLength; i++){ + int currentComp = i < currentSize ? currentComponents.get(i) : 0; + int compareComp = i < compareSize ? compareComponents.get(i) : 0; + + int comp = compareComp - currentComp; + if(comp != 0){ + return comp; + } + } + return 0; + } + public String version(File workingDirectory) throws IOException, InterruptedException { return runCommand(workingDirectory, new String[]{"--version"}, Collections.emptyList()).getRes(); } @@ -131,4 +157,4 @@ private CommandResults runCommand(File workingDirectory, String[] args, List Date: Thu, 21 Mar 2024 00:52:34 -0400 Subject: [PATCH 02/10] Update NpmBuildInfoExtractor.java Added update to Auth prop for newer NPM versions --- .../npm/extractor/NpmBuildInfoExtractor.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractor.java b/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractor.java index 1aad88869..25e0d2bc8 100644 --- a/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractor.java +++ b/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractor.java @@ -189,9 +189,22 @@ private void createTempNpmrc(Path workingDir, List commandArgs) throws I npmrcBuilder.append("proxy = ").append(this.npmProxy).append("\n"); } + //Update Auth property for newer npm versions + if(this.npmDrive.compareVersionTo(workingDir, "8.19") >= 0) + String authProp = npmAuth.getProperty("_auth"); + + String newAuthKey = artifactoryManage.getUrl(); + if (!StringUtils.endsWith(newAuthKey, "/")) { + newAuthKey += "/"; + } + newAuthKey += "api/npm/:_auth"; + newAuthKey = newAuthKey.replaceAll("^http(s)?:","") + ":_auth"; + npmAuth.setProperty(newAuthKey, authProp); + npmAuth.remove("_auth"); + } + // Save npm auth npmAuth.forEach((key, value) -> npmrcBuilder.append(key).append("=").append(value).append("\n")); - // Write npmrc file try (FileWriter fileWriter = new FileWriter(npmrcPath.toFile()); BufferedWriter bufferedWriter = new BufferedWriter(fileWriter)) { From 5d0010cfa3756659c363ac213daa2829431d0a80 Mon Sep 17 00:00:00 2001 From: Spaction <33352702+Spaction@users.noreply.github.com> Date: Thu, 21 Mar 2024 03:16:38 -0400 Subject: [PATCH 03/10] Corrections Info corrections --- .../jfrog/build/extractor/npm/NpmDriver.java | 4 ++-- .../npm/extractor/NpmBuildInfoExtractor.java | 22 ++++++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/NpmDriver.java b/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/NpmDriver.java index fcf34c152..d06c03d60 100644 --- a/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/NpmDriver.java +++ b/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/NpmDriver.java @@ -97,8 +97,8 @@ public JsonNode list(File workingDirectory, List extraArgs) throws IOExc } } - public int compareVersionTo(File workingDirectory, String compareVersion){ - String currentVersion = this.version(workingDirectory); + public int compareVersionTo(File workingDirectory, String compareVersion) throws IOException, InterruptedException { + String currentVersion = version(workingDirectory); List currentComponents = Arrays.stream(currentVersion.split("\\.")) .map(Integer::parseInt) diff --git a/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractor.java b/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractor.java index 25e0d2bc8..4aa520e47 100644 --- a/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractor.java +++ b/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractor.java @@ -190,17 +190,19 @@ private void createTempNpmrc(Path workingDir, List commandArgs) throws I } //Update Auth property for newer npm versions - if(this.npmDrive.compareVersionTo(workingDir, "8.19") >= 0) - String authProp = npmAuth.getProperty("_auth"); - - String newAuthKey = artifactoryManage.getUrl(); - if (!StringUtils.endsWith(newAuthKey, "/")) { - newAuthKey += "/"; + if( this.npmDriver.compareVersionTo(workingDir.toFile(), "8.19") >= 0){ + try (ArtifactoryManager artifactoryManager = artifactoryManagerBuilder.build()) { + String authProp = npmAuth.getProperty("_auth"); + + String newAuthKey = artifactoryManager.getUrl(); + if (!StringUtils.endsWith(newAuthKey, "/")) { + newAuthKey += "/"; + } + newAuthKey += "api/npm/:_auth"; + newAuthKey = newAuthKey.replaceAll("^http(s)?:","") + ":_auth"; + npmAuth.setProperty(newAuthKey, authProp); + npmAuth.remove("_auth"); } - newAuthKey += "api/npm/:_auth"; - newAuthKey = newAuthKey.replaceAll("^http(s)?:","") + ":_auth"; - npmAuth.setProperty(newAuthKey, authProp); - npmAuth.remove("_auth"); } // Save npm auth From 155f2ff179db97162936b265a102327055418f4d Mon Sep 17 00:00:00 2001 From: Spaction <33352702+Spaction@users.noreply.github.com> Date: Fri, 22 Mar 2024 00:15:40 -0400 Subject: [PATCH 04/10] Refactor and include tests --- .github/workflows/integrationTests.yml | 56 +++++- .gitignore | 5 + .../jfrog/build/extractor/npm/NpmDriver.java | 26 --- .../npm/extractor/NpmBuildInfoExtractor.java | 59 +++++-- .../extractor/NpmBuildInfoExtractorTest.java | 164 +++++++++++++++++- 5 files changed, 266 insertions(+), 44 deletions(-) diff --git a/.github/workflows/integrationTests.yml b/.github/workflows/integrationTests.yml index 9d40694dd..6b178467d 100644 --- a/.github/workflows/integrationTests.yml +++ b/.github/workflows/integrationTests.yml @@ -262,7 +262,7 @@ jobs: npm: needs: Pretest - name: npm (${{ matrix.os }}) + name: npm (${{ matrix.os }}) < 8.19 strategy: fail-fast: false matrix: @@ -314,6 +314,60 @@ jobs: env: # We use localhost because npm does not support authenticating with registries started with 127.0.0.1 BITESTS_PLATFORM_URL: http://localhost:8081 + npm: + needs: Pretest + name: npm (${{ matrix.os }}) >= 8.19 + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest, macos-latest, windows-latest ] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Install npm + uses: actions/setup-node@v3 + with: + node-version: "20" + - name: Config list + run: npm config ls -l + - name: Install Java + uses: actions/setup-java@v3 + with: + java-version: "8" + distribution: "temurin" + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: 1.19.x + + - name: Cache local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Setup Artifactory + run: | + go install github.com/jfrog/jfrog-testing-infra/local-rt-setup@latest + ~/go/bin/local-rt-setup + env: + RTLIC: ${{secrets.RTLIC}} + GOPROXY: direct + + # Run tests + - name: Run Tests + uses: gradle/gradle-build-action@v2 + with: + arguments: clean build-info-extractor-npm:test + env: + # We use localhost because npm does not support authenticating with registries started with 127.0.0.1 + BITESTS_PLATFORM_URL: http://localhost:8081 NuGet: needs: Pretest diff --git a/.gitignore b/.gitignore index 68c37531f..6cd28e3e1 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,8 @@ atlassian-*.xml /itest/src/test/resources/org/jfrog/build/cache /build-info-extractor/src/test/resources/artifactory-bi.properties local.properties +*.class +/*/bin/test/org/jfrog/build +/*/bin/test/*/snapshots/*.xml +/*/bin/test/*/settings/build-info*.json +/*/bin/test/*/pipLog.txt \ No newline at end of file diff --git a/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/NpmDriver.java b/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/NpmDriver.java index d06c03d60..9de094cd6 100644 --- a/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/NpmDriver.java +++ b/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/NpmDriver.java @@ -97,32 +97,6 @@ public JsonNode list(File workingDirectory, List extraArgs) throws IOExc } } - public int compareVersionTo(File workingDirectory, String compareVersion) throws IOException, InterruptedException { - String currentVersion = version(workingDirectory); - - List currentComponents = Arrays.stream(currentVersion.split("\\.")) - .map(Integer::parseInt) - .collect(Collectors.toList()); - List compareComponents = Arrays.stream(compareVersion.split("\\.")) - .map(Integer::parseInt) - .collect(Collectors.toList()); - - int currentSize = currentComponents.size(); - int compareSize = compareComponents.size(); - int maxLength = Math.max(currentSize, compareSize); - - for (int i = 0; i < maxLength; i++){ - int currentComp = i < currentSize ? currentComponents.get(i) : 0; - int compareComp = i < compareSize ? compareComponents.get(i) : 0; - - int comp = compareComp - currentComp; - if(comp != 0){ - return comp; - } - } - return 0; - } - public String version(File workingDirectory) throws IOException, InterruptedException { return runCommand(workingDirectory, new String[]{"--version"}, Collections.emptyList()).getRes(); } diff --git a/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractor.java b/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractor.java index 4aa520e47..530effbf5 100644 --- a/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractor.java +++ b/build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractor.java @@ -5,6 +5,7 @@ import org.jfrog.build.api.builder.ModuleType; import org.jfrog.build.api.util.Log; import org.jfrog.build.client.ProxyConfiguration; +import org.jfrog.build.client.Version; import org.jfrog.build.extractor.BuildInfoExtractor; import org.jfrog.build.extractor.builder.ModuleBuilder; import org.jfrog.build.extractor.ci.BuildInfo; @@ -189,21 +190,8 @@ private void createTempNpmrc(Path workingDir, List commandArgs) throws I npmrcBuilder.append("proxy = ").append(this.npmProxy).append("\n"); } - //Update Auth property for newer npm versions - if( this.npmDriver.compareVersionTo(workingDir.toFile(), "8.19") >= 0){ - try (ArtifactoryManager artifactoryManager = artifactoryManagerBuilder.build()) { - String authProp = npmAuth.getProperty("_auth"); - - String newAuthKey = artifactoryManager.getUrl(); - if (!StringUtils.endsWith(newAuthKey, "/")) { - newAuthKey += "/"; - } - newAuthKey += "api/npm/:_auth"; - newAuthKey = newAuthKey.replaceAll("^http(s)?:","") + ":_auth"; - npmAuth.setProperty(newAuthKey, authProp); - npmAuth.remove("_auth"); - } - } + // Update Auth property for newer npm versions + handleNpmCompatibility(npmAuth, workingDir); // Save npm auth npmAuth.forEach((key, value) -> npmrcBuilder.append(key).append("=").append(value).append("\n")); @@ -215,6 +203,47 @@ private void createTempNpmrc(Path workingDir, List commandArgs) throws I } } + /** + * Handles when Npm is at least version 8.19 the Auth related settings needing to be scoped to a specific registry. + * Results in transforming: + * + * old NPMRC of : + * + * registry=http://NO-NO-Repo/ + * _auth={{AuthString}} + * + * + * into the new NPMRC: + * + * registry=http://NO-NO-Repo/ + * //NO-NO-Repo/:_auth={{AuthString}} + * + */ + private void handleNpmCompatibility(Properties npmAuth, Path workingDir) throws IOException, InterruptedException{ + Version npmVersion = new Version(this.npmDriver.version(workingDir.toFile())); + if(npmVersion.isAtLeast(new Version("8.19"))){ + logger.debug("NPM version at least 8.19"); + try (ArtifactoryManager artifactoryManager = artifactoryManagerBuilder.build()) { + String newAuthKey = artifactoryManager.getUrl(); + if (!StringUtils.endsWith(newAuthKey, "/")) { + newAuthKey += "/"; + } + newAuthKey += "api/npm/:"; + newAuthKey = newAuthKey.replaceAll("^http(s)?:",""); + + String[] checkList = { "_auth","_authToken","username","_password", "email", "certfile", "keyfile"}; + for(String propKey: checkList){ + String prop = npmAuth.getProperty(propKey); + if(prop != null){ + logger.debug("Found "+ propKey +", replacing with " + newAuthKey + propKey); + npmAuth.setProperty(newAuthKey+propKey, prop); + npmAuth.remove(propKey); + } + } + } + } + } + /** * Adds an array-value config to a StringBuilder of .npmrc file, in the following format: * key[] = value diff --git a/build-info-extractor-npm/src/test/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractorTest.java b/build-info-extractor-npm/src/test/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractorTest.java index d90761197..6cf185a65 100644 --- a/build-info-extractor-npm/src/test/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractorTest.java +++ b/build-info-extractor-npm/src/test/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractorTest.java @@ -1,26 +1,64 @@ package org.jfrog.build.extractor.npm.extractor; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.jfrog.build.IntegrationTestsBase; import org.jfrog.build.extractor.builder.BuildInfoBuilder; import org.jfrog.build.extractor.builder.DependencyBuilder; import org.jfrog.build.extractor.builder.ModuleBuilder; import org.jfrog.build.extractor.ci.BuildInfo; import org.jfrog.build.extractor.ci.Dependency; import org.jfrog.build.extractor.ci.Module; +import org.jfrog.build.extractor.npm.NpmDriver; +import org.jfrog.build.extractor.npm.types.NpmProject; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.testng.Assert.*; import static org.jfrog.build.extractor.npm.extractor.NpmBuildInfoExtractor.getDependenciesMapFromBuild; -import static org.testng.Assert.assertEquals; @Test -public class NpmBuildInfoExtractorTest { +public class NpmBuildInfoExtractorTest extends IntegrationTestsBase { + + private static final String TEST_SPACE = "npm_test_space"; + private static final File tempWorkspace = new File(System.getProperty("java.io.tmpdir"), TEST_SPACE); + private static final Path PROJECTS_ROOT = Paths.get(".").toAbsolutePath().normalize().resolve(Paths.get("src", "test", "resources", "org", "jfrog", "build", "extractor")); + + + private static final String NPM_LOCAL_REPO = "build-info-tests-npm-local"; + private static final Set DEV_SCOPE = Stream.of("dev").collect(Collectors.toSet()); + private static final Set PROD_SCOPE = Stream.of("prod").collect(Collectors.toSet()); + private static final Set DEV_PROD_SCOPE = Stream.of("prod", "dev").collect(Collectors.toSet()); + + @AfterMethod + protected void cleanup() throws IOException { + FileUtils.deleteDirectory(tempWorkspace); + } + @BeforeMethod + protected void init2() throws IOException { + FileUtils.forceMkdir(tempWorkspace); + } @DataProvider private Object[][] getDependenciesMapFromBuildProvider() { return new Object[][]{ @@ -122,4 +160,126 @@ private Module createTestModule(String id, List dependencies) { new DependencyBuilder().id("mod2dep1:2.1.0").sha1("sha1-mod2dep1").md5("md5-mod2dep1").build(), new DependencyBuilder().id("mod2dep2:2.2.0").sha1("sha1-mod2dep2").md5("md5-mod2dep2").build() }; + + private enum Project { + // Dependencies + ASGARD("asgard", "jfrog-asgard", "jfrog-asgard", "2.0.0", "a1fc28aa8733a161fa92d03379b71468d19292cd", "2fb7c420d2119831bc38559138d3444e"), + MIDGARD("midgard", "jfrog-midgard", "jfrog-midgard", "1.0.0", "547b8c7bb019863cc26438ef36e9b2d33668a626", "82f1558593727a7c89fb0b91859dab26"), + ALFHEIM("alfheim", "jfrog-alfheim", "jfrog-alfheim", "3.5.2", "f5592b523d2693649a94bbc2377cc653607a4053", "93e19985bb1c7c815abef052b67be244"), + SVARTALFHEIM("svartalfheim", "jfrog-svartalfheim", "jfrog-svartalfheim", "0.5.0", "473a5e001c67d716b8c9993245bd0ba2010c7374", "b1678118e32908b8e57f26fef1a23473"), + + // Test projects + A("a", "NpmExtractorTest Project A", "package-name1", "v0.0.1", "", ""), + B("b", "NpmExtractorTest-Project-B", "package-name2", "0.0.2", "", ""), + C("c", "NpmExtractorTestProjectC", "package-name3", "=0.0.3", "", ""); + + private final File projectOrigin; + private final String targetDir; + private final String name; + private final String version; + private final String sha1; + private final String md5; + + Project(String sourceDir, String targetDir, String name, String version, String sha1, String md5) { + this.projectOrigin = PROJECTS_ROOT.resolve(sourceDir).toFile(); + this.targetDir = targetDir; + this.name = name; + this.version = version; + this.sha1 = sha1; + this.md5 = md5; + } + + private String getModuleId() { + return String.format("%s:%s", name, version); + } + + private String getPackedFileName() { + return String.format("%s-%s.tgz", name, version); + } + + private String getDependencyId() { + return String.format("%s:%s", name, version); + } + + private Dependency toDependency(String[][] requestedBy, Set scope) { + return new DependencyBuilder().id(getDependencyId()) + .sha1(sha1) + .md5(md5) + .scopes(scope) + .requestedBy(requestedBy) + .build(); + } + + private String getRemotePath() { + return String.format("%s/-", name); + } + + private String getTargetPath() { + return String.format("%s/%s", getRemotePath(), getPackedFileName()); + } + } + + + @DataProvider + private Object[][] npmCiProvider() { + Dependency[] expectedDepsStep1 = new Dependency[]{Project.ASGARD.toDependency(new String[][]{{"package-name1:v0.0.1"}}, PROD_SCOPE), Project.SVARTALFHEIM.toDependency(new String[][]{{"package-name1:v0.0.1"}}, PROD_SCOPE)}; + Dependency[] expectedDepsStep2 = new Dependency[]{Project.ASGARD.toDependency(new String[][]{{"jfrog-midgard:1.0.0", "@jscope/package-name2:0.0.2"}}, DEV_SCOPE), Project.MIDGARD.toDependency(new String[][]{{"@jscope/package-name2:0.0.2"}}, DEV_SCOPE), Project.ALFHEIM.toDependency(new String[][]{{"jfrog-midgard:1.0.0", "@jscope/package-name2:0.0.2"}}, DEV_SCOPE)}; + Dependency[] expectedDepsStep3 = new Dependency[]{Project.ASGARD.toDependency(new String[][]{{"jfrog-midgard:1.0.0", "package-name3:=0.0.3"}, {"package-name3:=0.0.3"}}, DEV_PROD_SCOPE), Project.MIDGARD.toDependency(new String[][]{{"package-name3:=0.0.3"}}, DEV_SCOPE), Project.ALFHEIM.toDependency(new String[][]{{"jfrog-midgard:1.0.0", "package-name3:=0.0.3"}}, DEV_SCOPE), Project.SVARTALFHEIM.toDependency(new String[][]{{"package-name3:=0.0.3"}}, PROD_SCOPE)}; + Dependency[] expectedDepsStep4 = new Dependency[]{Project.ASGARD.toDependency(new String[][]{{"package-name3:=0.0.3"}}, PROD_SCOPE), Project.SVARTALFHEIM.toDependency(new String[][]{{"package-name3:=0.0.3"}}, PROD_SCOPE)}; + return new Object[][]{ + {Project.A, expectedDepsStep1, "", true}, + {Project.B, expectedDepsStep2, "", true}, + {Project.B, new Dependency[]{}, "--production", false}, + {Project.C, expectedDepsStep3, "", true}, + {Project.C, expectedDepsStep4, "--only=production", true} + }; + } + + @SuppressWarnings("unused") + @Test(dataProvider = "npmCiProvider") + public void npmCiTest(Project project, Dependency[] expectedDependencies, String args, boolean packageJsonPath) { + runNpmTest(project, expectedDependencies, args, packageJsonPath, true); + } + + private void runNpmTest(Project project, Dependency[] expectedDependencies, String args, boolean packageJsonPath, boolean isNpmCi) { + args += " --verbose --no-audit"; + Path projectDir = null; + try { + // Prepare. + projectDir = createProjectDir(project); + Path path = packageJsonPath ? projectDir.resolve("package.json") : projectDir; + if (isNpmCi) { + // Run npm install to generate package-lock.json file. + new NpmInstallCi(artifactoryManagerBuilder, localRepo1, args, log, path, null, null, null, false, null).execute(); + } + + NpmDriver driver = new NpmDriver(null); + List commandArgs = StringUtils.isBlank(args) ? new ArrayList<>() : Arrays.asList(args.trim().split("\\s+")); + NpmProject proj = new NpmProject(commandArgs, localRepo1, path, isNpmCi); + // Execute command. + + NpmBuildInfoExtractor buildExtractor = new NpmBuildInfoExtractor(artifactoryManagerBuilder, driver, log, null, null,null); + BuildInfo buildInfo = buildExtractor.extract(proj); + + // Validate. + assertEquals(buildInfo.getModules().size(), 1); + Module module = buildInfo.getModules().get(0); + assertEquals(module.getType(), "npm"); + assertEquals(module.getId(), project.getModuleId()); + assertEqualsNoOrder(module.getDependencies().toArray(), expectedDependencies); + } catch (Exception e) { + fail(ExceptionUtils.getStackTrace(e)); + } finally { + if (projectDir != null) { + FileUtils.deleteQuietly(projectDir.toFile()); + } + } + } + + private Path createProjectDir(Project project) throws IOException { + File projectDir = Files.createTempDirectory(project.targetDir).toFile(); + FileUtils.copyDirectory(project.projectOrigin, projectDir); + return projectDir.toPath(); + } + } From 664ca29174d4a6edfbe0a51d9db63590f357c009 Mon Sep 17 00:00:00 2001 From: Spaction <33352702+Spaction@users.noreply.github.com> Date: Fri, 22 Mar 2024 00:33:28 -0400 Subject: [PATCH 05/10] Update integrationTests.yml --- .github/workflows/integrationTests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integrationTests.yml b/.github/workflows/integrationTests.yml index 6b178467d..1bc03a813 100644 --- a/.github/workflows/integrationTests.yml +++ b/.github/workflows/integrationTests.yml @@ -259,7 +259,7 @@ jobs: uses: gradle/gradle-build-action@v2 with: arguments: clean build-info-extractor-maven3:test - + npm: needs: Pretest name: npm (${{ matrix.os }}) < 8.19 @@ -314,7 +314,8 @@ jobs: env: # We use localhost because npm does not support authenticating with registries started with 127.0.0.1 BITESTS_PLATFORM_URL: http://localhost:8081 - npm: + + npm_9: needs: Pretest name: npm (${{ matrix.os }}) >= 8.19 strategy: From c740ba8246ab73d70898a3bf1654e26097bd10d7 Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Wed, 22 May 2024 17:28:15 +0300 Subject: [PATCH 06/10] merge npm tests configuration --- .github/workflows/integrationTests.yml | 63 +++----------------------- 1 file changed, 6 insertions(+), 57 deletions(-) diff --git a/.github/workflows/integrationTests.yml b/.github/workflows/integrationTests.yml index c6e171f9b..29eaf8409 100644 --- a/.github/workflows/integrationTests.yml +++ b/.github/workflows/integrationTests.yml @@ -272,11 +272,13 @@ jobs: npm: needs: Pretest - name: npm (${{ matrix.os }}) < 8.19 + name: npm (${{ matrix.os }}) - node ${{ matrix.node }} strategy: fail-fast: false matrix: os: [ ubuntu-latest, macos-13, windows-latest ] + # Testing both npm < 8.19 and npm >= 8.19 + node: [ 15, 20 ] runs-on: ${{ matrix.os }} steps: - name: Checkout code @@ -287,9 +289,11 @@ jobs: - name: Install npm uses: actions/setup-node@v3 with: - node-version: "15" + node-version: matrix.node + - name: Config list run: npm config ls -l + - name: Install Java uses: actions/setup-java@v3 with: @@ -327,61 +331,6 @@ jobs: # We use localhost because npm does not support authenticating with registries started with 127.0.0.1 BITESTS_PLATFORM_URL: http://localhost:8081 - npm_9: - needs: Pretest - name: npm (${{ matrix.os }}) >= 8.19 - strategy: - fail-fast: false - matrix: - os: [ ubuntu-latest, macos-latest, windows-latest ] - runs-on: ${{ matrix.os }} - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: Install npm - uses: actions/setup-node@v3 - with: - node-version: "20" - - name: Config list - run: npm config ls -l - - name: Install Java - uses: actions/setup-java@v3 - with: - java-version: "8" - distribution: "temurin" - - name: Install Go - uses: actions/setup-go@v3 - with: - go-version: 1.19.x - - - name: Cache local Maven repository - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - - name: Setup Artifactory - run: | - go install github.com/jfrog/jfrog-testing-infra/local-rt-setup@latest - ~/go/bin/local-rt-setup - env: - RTLIC: ${{secrets.RTLIC}} - GOPROXY: direct - - # Run tests - - name: Run Tests - uses: gradle/gradle-build-action@v2 - with: - arguments: clean build-info-extractor-npm:test - env: - # We use localhost because npm does not support authenticating with registries started with 127.0.0.1 - BITESTS_PLATFORM_URL: http://localhost:8081 - NuGet: needs: Pretest name: NuGet (${{ matrix.os }}) From 8764f66bd14b07309a43e30f40d50d76a0fdb4a7 Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Wed, 22 May 2024 18:22:13 +0300 Subject: [PATCH 07/10] fix repo init --- .github/workflows/integrationTests.yml | 2 +- .../npm/extractor/NpmBuildInfoExtractorTest.java | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integrationTests.yml b/.github/workflows/integrationTests.yml index 29eaf8409..fab400f7e 100644 --- a/.github/workflows/integrationTests.yml +++ b/.github/workflows/integrationTests.yml @@ -289,7 +289,7 @@ jobs: - name: Install npm uses: actions/setup-node@v3 with: - node-version: matrix.node + node-version: ${{ matrix.node }} - name: Config list run: npm config ls -l diff --git a/build-info-extractor-npm/src/test/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractorTest.java b/build-info-extractor-npm/src/test/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractorTest.java index 6cf185a65..b5ce5fb97 100644 --- a/build-info-extractor-npm/src/test/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractorTest.java +++ b/build-info-extractor-npm/src/test/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractorTest.java @@ -49,7 +49,13 @@ public class NpmBuildInfoExtractorTest extends IntegrationTestsBase { private static final Set DEV_SCOPE = Stream.of("dev").collect(Collectors.toSet()); private static final Set PROD_SCOPE = Stream.of("prod").collect(Collectors.toSet()); private static final Set DEV_PROD_SCOPE = Stream.of("prod", "dev").collect(Collectors.toSet()); - + + public NpmBuildInfoExtractorTest() { + localRepo1 = getKeyWithTimestamp(NPM_LOCAL_REPO); + remoteRepo = ""; + virtualRepo = ""; + } + @AfterMethod protected void cleanup() throws IOException { FileUtils.deleteDirectory(tempWorkspace); From f0f8328b64c4b3bc55c3f537122aa4726a14a2f6 Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Wed, 22 May 2024 18:52:44 +0300 Subject: [PATCH 08/10] run updated workflow --- .github/workflows/integrationTests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integrationTests.yml b/.github/workflows/integrationTests.yml index fab400f7e..68753253d 100644 --- a/.github/workflows/integrationTests.yml +++ b/.github/workflows/integrationTests.yml @@ -4,7 +4,7 @@ on: branches: - master # Triggers the workflow on labeled PRs only. - pull_request_target: + pull_request: # TODO undo change here before merging. types: [ labeled ] # Ensures that only the latest commit is running for each PR at a time. concurrency: From 78ec5610a1f23a07c4ef1bac00e6f05e27cc59e2 Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Wed, 22 May 2024 18:57:52 +0300 Subject: [PATCH 09/10] Revert "run updated workflow" This reverts commit f0f8328b64c4b3bc55c3f537122aa4726a14a2f6. --- .github/workflows/integrationTests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integrationTests.yml b/.github/workflows/integrationTests.yml index 68753253d..fab400f7e 100644 --- a/.github/workflows/integrationTests.yml +++ b/.github/workflows/integrationTests.yml @@ -4,7 +4,7 @@ on: branches: - master # Triggers the workflow on labeled PRs only. - pull_request: # TODO undo change here before merging. + pull_request_target: types: [ labeled ] # Ensures that only the latest commit is running for each PR at a time. concurrency: From 3b0f5fee90902e98b65639ede257bf4bc9affb47 Mon Sep 17 00:00:00 2001 From: Spaction Date: Mon, 27 May 2024 23:05:50 -0400 Subject: [PATCH 10/10] Update NpmBuildInfoExtractorTest.java Correct Unit Test --- .../extractor/NpmBuildInfoExtractorTest.java | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/build-info-extractor-npm/src/test/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractorTest.java b/build-info-extractor-npm/src/test/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractorTest.java index b5ce5fb97..b8466b94f 100644 --- a/build-info-extractor-npm/src/test/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractorTest.java +++ b/build-info-extractor-npm/src/test/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractorTest.java @@ -10,10 +10,13 @@ import org.jfrog.build.extractor.ci.BuildInfo; import org.jfrog.build.extractor.ci.Dependency; import org.jfrog.build.extractor.ci.Module; +import org.jfrog.build.extractor.clientConfiguration.deploy.DeployDetails; +import org.jfrog.build.extractor.clientConfiguration.util.DependenciesDownloaderHelper; import org.jfrog.build.extractor.npm.NpmDriver; import org.jfrog.build.extractor.npm.types.NpmProject; import org.testng.Assert; import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -225,6 +228,23 @@ private String getTargetPath() { } } + @BeforeClass + private void setUp() throws IOException { + deployTestDependencies(Project.ASGARD, Project.MIDGARD, Project.ALFHEIM, Project.SVARTALFHEIM); + } + + private void deployTestDependencies(Project... projects) throws IOException { + for (Project project : projects) { + DeployDetails deployDetails = new DeployDetails.Builder() + .file(project.projectOrigin.toPath().resolve(project.getPackedFileName()).toFile()) + .targetRepository(localRepo1) + .artifactPath(project.getTargetPath()) + .packageType(DeployDetails.PackageType.NPM) + .build(); + artifactoryManager.upload(deployDetails); + } + } + @DataProvider private Object[][] npmCiProvider() { @@ -244,12 +264,13 @@ private Object[][] npmCiProvider() { @SuppressWarnings("unused") @Test(dataProvider = "npmCiProvider") public void npmCiTest(Project project, Dependency[] expectedDependencies, String args, boolean packageJsonPath) { - runNpmTest(project, expectedDependencies, args, packageJsonPath, true); + runNpmTest(project, expectedDependencies, args, packageJsonPath); } - private void runNpmTest(Project project, Dependency[] expectedDependencies, String args, boolean packageJsonPath, boolean isNpmCi) { + private void runNpmTest(Project project, Dependency[] expectedDependencies, String args, boolean packageJsonPath) { args += " --verbose --no-audit"; Path projectDir = null; + boolean isNpmCi = true; try { // Prepare. projectDir = createProjectDir(project);