Add support for SNAPSHOT dependencies layer (#584)

* Add support for SNAPSHOT dependencies layer

jib-core/src/integration-test/java/com/google/cloud/tools/jib/builder/BuildStepsIntegrationTest.java

@@ -54,7 +54,13 @@ public class BuildStepsIntegrationTest {
 
   @Before
   public void setUp() throws IOException, URISyntaxException {
-    sourceFilesConfiguration = new TestSourceFilesConfiguration();
+    sourceFilesConfiguration =
+        TestSourceFilesConfiguration.builder()
+            .withClasses()
+            .withDependencies()
+            .withSnapshotDependencies()
+            .withResources()
+            .build();
   }
 
   @Test

jib-core/src/main/java/com/google/cloud/tools/jib/builder/SourceFilesConfiguration.java

@@ -35,6 +35,12 @@ public interface SourceFilesConfiguration {
    */
   ImmutableList<Path> getDependenciesFiles();
 
+  /**
+   * @return the source files for snapshot dependencies. These files should be in a deterministic
+   *     order
+   */
+  ImmutableList<Path> getSnapshotDependenciesFiles();
+
   /**
    * @return the source files for the resources layer. These files should be in a deterministic
    *     order.
@@ -48,7 +54,7 @@ public interface SourceFilesConfiguration {
 
   /**
    * @return the Unix-style path where the dependencies source files are placed in the container
-   *     filesystem. Must end with slash.
+   *     filesystem. Must end with slash. This includes both regular and snapshot dependencies.
    */
   String getDependenciesPathOnImage();
 

jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java

@@ -86,6 +86,20 @@ static ImmutableList<BuildAndCacheApplicationLayerStep> makeList(
                           .build(),
                       cache));
 
+      // Adds a snapshot dependencies layer, if snapshot files present.
+      if (!sourceFilesConfiguration.getSnapshotDependenciesFiles().isEmpty()) {
+        buildLayerStepsBuilder.add(
+            new BuildAndCacheApplicationLayerStep(
+                "snapshot-dependencies",
+                listeningExecutorService,
+                buildConfiguration,
+                LayerConfiguration.builder()
+                    .addEntry(
+                        sourceFilesConfiguration.getSnapshotDependenciesFiles(),
+                        sourceFilesConfiguration.getDependenciesPathOnImage())
+                    .build(),
+                cache));
+      }
       // Adds the extra layer to be built, if configured.
       if (buildConfiguration.getExtraFilesLayerConfiguration() != null) {
         buildLayerStepsBuilder.add(

jib-core/src/main/java/com/google/cloud/tools/jib/docker/DockerContextGenerator.java

@@ -136,14 +136,18 @@ public void generate(Path targetDirectory) throws IOException {
 
     // Creates the directories.
     Path dependenciesDir = targetDirectory.resolve("libs");
+    Path snapshotDependenciesDir = targetDirectory.resolve("snapshot-libs");
     Path resourcesDIr = targetDirectory.resolve("resources");
     Path classesDir = targetDirectory.resolve("classes");
     Files.createDirectory(dependenciesDir);
+    Files.createDirectories(snapshotDependenciesDir);
     Files.createDirectory(resourcesDIr);
     Files.createDirectory(classesDir);
 
     // Copies dependencies.
     FileOperations.copy(sourceFilesConfiguration.getDependenciesFiles(), dependenciesDir);
+    FileOperations.copy(
+        sourceFilesConfiguration.getSnapshotDependenciesFiles(), snapshotDependenciesDir);
     FileOperations.copy(sourceFilesConfiguration.getResourcesFiles(), resourcesDIr);
     FileOperations.copy(sourceFilesConfiguration.getClassesFiles(), classesDir);
 
@@ -179,6 +183,8 @@ String makeDockerfile() throws JsonProcessingException {
         .append(Preconditions.checkNotNull(baseImage))
         .append("\n\nCOPY libs ")
         .append(sourceFilesConfiguration.getDependenciesPathOnImage())
+        .append("\nCOPY snapshot-libs ")
+        .append(sourceFilesConfiguration.getDependenciesPathOnImage())
         .append("\nCOPY resources ")
         .append(sourceFilesConfiguration.getResourcesPathOnImage())
         .append("\nCOPY classes ")

jib-core/src/test/java/com/google/cloud/tools/jib/builder/TestSourceFilesConfiguration.java

@@ -28,23 +28,82 @@
 /** Implementation of {@link SourceFilesConfiguration} that uses test resources. */
 public class TestSourceFilesConfiguration implements SourceFilesConfiguration {
 
+  public static class Builder {
+    private ImmutableList<Path> dependenciesSourceFiles = ImmutableList.of();
+    private ImmutableList<Path> snapshotDependenciesSourceFiles = ImmutableList.of();
+    private ImmutableList<Path> resourcesSourceFiles = ImmutableList.of();
+    private ImmutableList<Path> classesSourceFiles = ImmutableList.of();
+
+    public Builder withDependencies() throws IOException, URISyntaxException {
+      dependenciesSourceFiles = getFilesList("application/dependencies");
+      return this;
+    }
+
+    public Builder withSnapshotDependencies() throws IOException, URISyntaxException {
+      snapshotDependenciesSourceFiles = getFilesList("application/snapshot-dependencies");
+      return this;
+    }
+
+    public Builder withResources() throws IOException, URISyntaxException {
+      resourcesSourceFiles = getFilesList("application/resources");
+      return this;
+    }
+
+    public Builder withClasses() throws IOException, URISyntaxException {
+      classesSourceFiles = getFilesList("application/classes");
+      return this;
+    }
+
+    public TestSourceFilesConfiguration build() {
+      return new TestSourceFilesConfiguration(
+          dependenciesSourceFiles,
+          snapshotDependenciesSourceFiles,
+          resourcesSourceFiles,
+          classesSourceFiles);
+    }
+
+    /** Lists the files in the {@code resourcePath} resources directory. */
+    private ImmutableList<Path> getFilesList(String resourcePath)
+        throws URISyntaxException, IOException {
+      try (Stream<Path> fileStream =
+          Files.list(Paths.get(Resources.getResource(resourcePath).toURI()))) {
+        return fileStream.collect(ImmutableList.toImmutableList());
+      }
+    }
+  }
+
+  public static Builder builder() {
+    return new Builder();
+  }
+
   private static final String EXTRACTION_PATH = "/some/extraction/path/";
 
   private final ImmutableList<Path> dependenciesSourceFiles;
+  private final ImmutableList<Path> snapshotDependenciesSourceFiles;
   private final ImmutableList<Path> resourcesSourceFiles;
   private final ImmutableList<Path> classesSourceFiles;
 
-  public TestSourceFilesConfiguration() throws URISyntaxException, IOException {
-    dependenciesSourceFiles = getFilesList("application/dependencies");
-    resourcesSourceFiles = getFilesList("application/resources");
-    classesSourceFiles = getFilesList("application/classes");
+  private TestSourceFilesConfiguration(
+      ImmutableList<Path> dependenciesSourceFiles,
+      ImmutableList<Path> snapshotDependenciesSourceFiles,
+      ImmutableList<Path> resourcesSourceFiles,
+      ImmutableList<Path> classesSourceFiles) {
+    this.dependenciesSourceFiles = dependenciesSourceFiles;
+    this.snapshotDependenciesSourceFiles = snapshotDependenciesSourceFiles;
+    this.resourcesSourceFiles = resourcesSourceFiles;
+    this.classesSourceFiles = classesSourceFiles;
   }
 
   @Override
   public ImmutableList<Path> getDependenciesFiles() {
     return dependenciesSourceFiles;
   }
 
+  @Override
+  public ImmutableList<Path> getSnapshotDependenciesFiles() {
+    return snapshotDependenciesSourceFiles;
+  }
+
   @Override
   public ImmutableList<Path> getResourcesFiles() {
     return resourcesSourceFiles;
@@ -69,13 +128,4 @@ public String getResourcesPathOnImage() {
   public String getClassesPathOnImage() {
     return EXTRACTION_PATH + "classes/";
   }
-
-  /** Lists the files in the {@code resourcePath} resources directory. */
-  private ImmutableList<Path> getFilesList(String resourcePath)
-      throws URISyntaxException, IOException {
-    try (Stream<Path> fileStream =
-        Files.list(Paths.get(Resources.getResource(resourcePath).toURI()))) {
-      return fileStream.collect(ImmutableList.toImmutableList());
-    }
-  }
 }

jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStepTest.java

@@ -18,6 +18,7 @@
 
 import com.google.cloud.tools.jib.async.NonBlockingSteps;
 import com.google.cloud.tools.jib.builder.BuildConfiguration;
+import com.google.cloud.tools.jib.builder.SourceFilesConfiguration;
 import com.google.cloud.tools.jib.builder.TestBuildLogger;
 import com.google.cloud.tools.jib.builder.TestSourceFilesConfiguration;
 import com.google.cloud.tools.jib.cache.Cache;
@@ -37,6 +38,7 @@
 import java.nio.file.Paths;
 import java.util.concurrent.ExecutionException;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
@@ -49,30 +51,35 @@
 @RunWith(MockitoJUnitRunner.class)
 public class BuildAndCacheApplicationLayerStepTest {
 
+  private static final String EXTRA_FILES_LAYER_EXTRACTION_PATH = "/extra";
+
   @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();
 
   @Mock private BuildConfiguration mockBuildConfiguration;
+  private Path temporaryCacheDirectory;
 
-  @Test
-  public void testRun()
-      throws LayerPropertyNotFoundException, IOException, CacheMetadataCorruptedException,
-          URISyntaxException, ExecutionException {
+  @Before
+  public void setUp() throws IOException, URISyntaxException {
     Mockito.when(mockBuildConfiguration.getBuildLogger()).thenReturn(new TestBuildLogger());
-    TestSourceFilesConfiguration testSourceFilesConfiguration = new TestSourceFilesConfiguration();
-    Path temporaryCacheDirectory = temporaryFolder.newFolder().toPath();
+    temporaryCacheDirectory = temporaryFolder.newFolder().toPath();
+  }
 
-    // Adds an extra file layer.
+  private ImmutableList<Path> configureExtraFilesLayer() throws URISyntaxException {
     ImmutableList<Path> extraFilesLayerSourceFiles =
         ImmutableList.of(
             Paths.get(Resources.getResource("fileA").toURI()),
             Paths.get(Resources.getResource("fileB").toURI()));
-    String extraFilesLayerExtractionPath = "/extra";
     Mockito.when(mockBuildConfiguration.getExtraFilesLayerConfiguration())
         .thenReturn(
             LayerConfiguration.builder()
-                .addEntry(extraFilesLayerSourceFiles, extraFilesLayerExtractionPath)
+                .addEntry(extraFilesLayerSourceFiles, EXTRA_FILES_LAYER_EXTRACTION_PATH)
                 .build());
+    return extraFilesLayerSourceFiles;
+  }
 
+  private ImageLayers<CachedLayerWithMetadata> configureAvailableLayers(
+      SourceFilesConfiguration testSourceFilesConfiguration)
+      throws CacheMetadataCorruptedException, IOException, ExecutionException {
     ImageLayers.Builder<CachedLayerWithMetadata> applicationLayersBuilder = ImageLayers.builder();
     ImageLayers<CachedLayerWithMetadata> applicationLayers;
 
@@ -91,8 +98,30 @@ public void testRun()
 
       applicationLayers = applicationLayersBuilder.build();
       cache.addCachedLayersWithMetadataToMetadata(applicationLayers.getLayers());
-      Assert.assertEquals(4, applicationLayers.size());
     }
+    return applicationLayers;
+  }
+
+  @Test
+  public void testRun()
+      throws LayerPropertyNotFoundException, IOException, CacheMetadataCorruptedException,
+          URISyntaxException, ExecutionException {
+
+    TestSourceFilesConfiguration testSourceFilesConfiguration =
+        TestSourceFilesConfiguration.builder()
+            .withDependencies()
+            .withSnapshotDependencies()
+            .withClasses()
+            .withResources()
+            .build();
+
+    // Adds an extra file layer.
+    ImmutableList<Path> extraFilesLayerSourceFiles = configureExtraFilesLayer();
+
+    // Populate the cache
+    ImageLayers<CachedLayerWithMetadata> applicationLayers =
+        configureAvailableLayers(testSourceFilesConfiguration);
+    Assert.assertEquals(5, applicationLayers.size());
 
     // Re-initialize cache with the updated metadata.
     Cache cache = Cache.init(temporaryCacheDirectory);
@@ -102,6 +131,11 @@ public void testRun()
             new LayerEntry(
                 testSourceFilesConfiguration.getDependenciesFiles(),
                 testSourceFilesConfiguration.getDependenciesPathOnImage()));
+    ImmutableList<LayerEntry> snapshotDependenciesLayerEntry =
+        ImmutableList.of(
+            new LayerEntry(
+                testSourceFilesConfiguration.getSnapshotDependenciesFiles(),
+                testSourceFilesConfiguration.getDependenciesPathOnImage()));
     ImmutableList<LayerEntry> resourcesLayerEntry =
         ImmutableList.of(
             new LayerEntry(
@@ -113,7 +147,8 @@ public void testRun()
                 testSourceFilesConfiguration.getClassesFiles(),
                 testSourceFilesConfiguration.getClassesPathOnImage()));
     ImmutableList<LayerEntry> extraFilesLayerEntry =
-        ImmutableList.of(new LayerEntry(extraFilesLayerSourceFiles, extraFilesLayerExtractionPath));
+        ImmutableList.of(
+            new LayerEntry(extraFilesLayerSourceFiles, EXTRA_FILES_LAYER_EXTRACTION_PATH));
 
     // Verifies that the cached layers are up-to-date.
     CacheReader cacheReader = new CacheReader(cache);
@@ -128,6 +163,11 @@ public void testRun()
         cacheReader.getUpToDateLayerByLayerEntries(classesLayerEntry).getBlobDescriptor());
     Assert.assertEquals(
         applicationLayers.get(3).getBlobDescriptor(),
+        cacheReader
+            .getUpToDateLayerByLayerEntries(snapshotDependenciesLayerEntry)
+            .getBlobDescriptor());
+    Assert.assertEquals(
+        applicationLayers.get(4).getBlobDescriptor(),
         cacheReader.getUpToDateLayerByLayerEntries(extraFilesLayerEntry).getBlobDescriptor());
 
     // Verifies that the cache reader gets the same layers as the newest application layers.
@@ -139,6 +179,66 @@ public void testRun()
     Assert.assertEquals(
         applicationLayers.get(2).getContentFile(), cacheReader.getLayerFile(classesLayerEntry));
     Assert.assertEquals(
-        applicationLayers.get(3).getContentFile(), cacheReader.getLayerFile(extraFilesLayerEntry));
+        applicationLayers.get(3).getContentFile(),
+        cacheReader.getLayerFile(snapshotDependenciesLayerEntry));
+    Assert.assertEquals(
+        applicationLayers.get(4).getContentFile(), cacheReader.getLayerFile(extraFilesLayerEntry));
+  }
+
+  @Test
+  public void testRun_emptyLayersIgnored()
+      throws IOException, URISyntaxException, CacheMetadataCorruptedException, ExecutionException {
+
+    TestSourceFilesConfiguration testSourceFilesConfiguration =
+        TestSourceFilesConfiguration.builder()
+            .withDependencies()
+            .withClasses()
+            .withResources()
+            .build();
+
+    // Populate the cache
+    ImageLayers<CachedLayerWithMetadata> applicationLayers =
+        configureAvailableLayers(testSourceFilesConfiguration);
+    Assert.assertEquals(3, applicationLayers.size());
+
+    ImmutableList<LayerEntry> dependenciesLayerEntry =
+        ImmutableList.of(
+            new LayerEntry(
+                testSourceFilesConfiguration.getDependenciesFiles(),
+                testSourceFilesConfiguration.getDependenciesPathOnImage()));
+    ImmutableList<LayerEntry> resourcesLayerEntry =
+        ImmutableList.of(
+            new LayerEntry(
+                testSourceFilesConfiguration.getResourcesFiles(),
+                testSourceFilesConfiguration.getResourcesPathOnImage()));
+    ImmutableList<LayerEntry> classesLayerEntry =
+        ImmutableList.of(
+            new LayerEntry(
+                testSourceFilesConfiguration.getClassesFiles(),
+                testSourceFilesConfiguration.getClassesPathOnImage()));
+
+    // Re-initialize cache with the updated metadata.
+    Cache cache = Cache.init(temporaryCacheDirectory);
+
+    // Verifies that the cached layers are up-to-date.
+    CacheReader cacheReader = new CacheReader(cache);
+    Assert.assertEquals(
+        applicationLayers.get(0).getBlobDescriptor(),
+        cacheReader.getUpToDateLayerByLayerEntries(dependenciesLayerEntry).getBlobDescriptor());
+    Assert.assertEquals(
+        applicationLayers.get(1).getBlobDescriptor(),
+        cacheReader.getUpToDateLayerByLayerEntries(resourcesLayerEntry).getBlobDescriptor());
+    Assert.assertEquals(
+        applicationLayers.get(2).getBlobDescriptor(),
+        cacheReader.getUpToDateLayerByLayerEntries(classesLayerEntry).getBlobDescriptor());
+
+    // Verifies that the cache reader gets the same layers as the newest application layers.
+    Assert.assertEquals(
+        applicationLayers.get(0).getContentFile(),
+        cacheReader.getLayerFile(dependenciesLayerEntry));
+    Assert.assertEquals(
+        applicationLayers.get(1).getContentFile(), cacheReader.getLayerFile(resourcesLayerEntry));
+    Assert.assertEquals(
+        applicationLayers.get(2).getContentFile(), cacheReader.getLayerFile(classesLayerEntry));
   }
 }