diff --git a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/builder/BuildStepsIntegrationTest.java b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/builder/BuildStepsIntegrationTest.java index e5a6685e0e..66bc62e8ed 100644 --- a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/builder/BuildStepsIntegrationTest.java +++ b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/builder/BuildStepsIntegrationTest.java @@ -17,11 +17,9 @@ package com.google.cloud.tools.jib.builder; import com.google.cloud.tools.jib.Command; -import com.google.cloud.tools.jib.cache.CacheDirectoryCreationException; -import com.google.cloud.tools.jib.cache.CacheDirectoryNotOwnedException; -import com.google.cloud.tools.jib.cache.CacheMetadataCorruptedException; -import com.google.cloud.tools.jib.cache.Caches; import com.google.cloud.tools.jib.configuration.BuildConfiguration; +import com.google.cloud.tools.jib.configuration.CacheConfiguration; +import com.google.cloud.tools.jib.configuration.CacheDirectoryCreationException; import com.google.cloud.tools.jib.configuration.ContainerConfiguration; import com.google.cloud.tools.jib.configuration.ImageConfiguration; import com.google.cloud.tools.jib.configuration.LayerConfiguration; @@ -123,10 +121,10 @@ public void setUp() throws IOException, URISyntaxException { @Test public void testSteps_forBuildToDockerRegistry() - throws IOException, InterruptedException, CacheMetadataCorruptedException, ExecutionException, - CacheDirectoryNotOwnedException, CacheDirectoryCreationException { + throws IOException, InterruptedException, ExecutionException, + CacheDirectoryCreationException { BuildSteps buildImageSteps = - getBuildSteps( + BuildSteps.forBuildToDockerRegistry( getBuildConfigurationBuilder( ImageReference.of("gcr.io", "distroless/java", "latest"), ImageReference.of("localhost:5000", "testimage", "testtag")) @@ -148,10 +146,10 @@ public void testSteps_forBuildToDockerRegistry() @Test public void testSteps_forBuildToDockerRegistry_multipleTags() - throws IOException, InterruptedException, CacheMetadataCorruptedException, ExecutionException, - CacheDirectoryNotOwnedException, CacheDirectoryCreationException { + throws IOException, InterruptedException, ExecutionException, + CacheDirectoryCreationException { BuildSteps buildImageSteps = - getBuildSteps( + BuildSteps.forBuildToDockerRegistry( getBuildConfigurationBuilder( ImageReference.of("gcr.io", "distroless/java", "latest"), ImageReference.of("localhost:5000", "testimage", "testtag")) @@ -188,10 +186,9 @@ public void testSteps_forBuildToDockerRegistry_multipleTags() @Test public void testSteps_forBuildToDockerRegistry_dockerHubBaseImage() - throws InvalidImageReferenceException, IOException, InterruptedException, ExecutionException, - CacheDirectoryCreationException, CacheMetadataCorruptedException, - CacheDirectoryNotOwnedException { - getBuildSteps( + throws InvalidImageReferenceException, IOException, InterruptedException, + CacheDirectoryCreationException, ExecutionException { + BuildSteps.forBuildToDockerRegistry( getBuildConfigurationBuilder( ImageReference.parse("openjdk:8-jre-alpine"), ImageReference.of("localhost:5000", "testimage", "testtag")) @@ -206,19 +203,15 @@ public void testSteps_forBuildToDockerRegistry_dockerHubBaseImage() @Test public void testSteps_forBuildToDockerDaemon() - throws IOException, InterruptedException, CacheMetadataCorruptedException, ExecutionException, - CacheDirectoryNotOwnedException, CacheDirectoryCreationException { + throws IOException, InterruptedException, ExecutionException, + CacheDirectoryCreationException { String imageReference = "testdocker"; BuildConfiguration buildConfiguration = getBuildConfigurationBuilder( ImageReference.of("gcr.io", "distroless/java", "latest"), ImageReference.of(null, imageReference, null)) .build(); - Path cacheDirectory = temporaryFolder.newFolder().toPath(); - BuildSteps.forBuildToDockerDaemon( - buildConfiguration, - new Caches.Initializer(cacheDirectory, false, cacheDirectory, false)) - .run(); + BuildSteps.forBuildToDockerDaemon(buildConfiguration).run(); assertDockerInspect(imageReference); Assert.assertEquals( @@ -227,8 +220,8 @@ public void testSteps_forBuildToDockerDaemon() @Test public void testSteps_forBuildToDockerDaemon_multipleTags() - throws IOException, InterruptedException, CacheMetadataCorruptedException, ExecutionException, - CacheDirectoryNotOwnedException, CacheDirectoryCreationException { + throws IOException, InterruptedException, ExecutionException, + CacheDirectoryCreationException { String imageReference = "testdocker"; BuildConfiguration buildConfiguration = getBuildConfigurationBuilder( @@ -236,11 +229,7 @@ public void testSteps_forBuildToDockerDaemon_multipleTags() ImageReference.of(null, imageReference, null)) .setAdditionalTargetImageTags(ImmutableSet.of("testtag2", "testtag3")) .build(); - Path cacheDirectory = temporaryFolder.newFolder().toPath(); - BuildSteps.forBuildToDockerDaemon( - buildConfiguration, - new Caches.Initializer(cacheDirectory, false, cacheDirectory, false)) - .run(); + BuildSteps.forBuildToDockerDaemon(buildConfiguration).run(); assertDockerInspect(imageReference); Assert.assertEquals( @@ -257,34 +246,23 @@ public void testSteps_forBuildToDockerDaemon_multipleTags() @Test public void testSteps_forBuildToTarball() - throws IOException, InterruptedException, CacheMetadataCorruptedException, ExecutionException, - CacheDirectoryNotOwnedException, CacheDirectoryCreationException { + throws IOException, InterruptedException, ExecutionException, + CacheDirectoryCreationException { BuildConfiguration buildConfiguration = getBuildConfigurationBuilder( ImageReference.of("gcr.io", "distroless/java", "latest"), ImageReference.of(null, "testtar", null)) .build(); Path outputPath = temporaryFolder.newFolder().toPath().resolve("test.tar"); - Path cacheDirectory = temporaryFolder.newFolder().toPath(); - BuildSteps.forBuildToTar( - outputPath, - buildConfiguration, - new Caches.Initializer(cacheDirectory, false, cacheDirectory, false)) - .run(); + BuildSteps.forBuildToTar(outputPath, buildConfiguration).run(); new Command("docker", "load", "--input", outputPath.toString()).run(); Assert.assertEquals( "Hello, world. An argument.\n", new Command("docker", "run", "--rm", "testtar").run()); } - private BuildSteps getBuildSteps(BuildConfiguration buildConfiguration) throws IOException { - Path cacheDirectory = temporaryFolder.newFolder().toPath(); - return BuildSteps.forBuildToDockerRegistry( - buildConfiguration, new Caches.Initializer(cacheDirectory, false, cacheDirectory, false)); - } - private BuildConfiguration.Builder getBuildConfigurationBuilder( - ImageReference baseImage, ImageReference targetImage) { + ImageReference baseImage, ImageReference targetImage) throws IOException { ImageConfiguration baseImageConfiguration = ImageConfiguration.builder(baseImage).build(); ImageConfiguration targetImageConfiguration = ImageConfiguration.builder(targetImage).build(); ContainerConfiguration containerConfiguration = @@ -298,10 +276,13 @@ private BuildConfiguration.Builder getBuildConfigurationBuilder( ExposedPortsParser.parse(Arrays.asList("1000", "2000-2002/tcp", "3000/udp"))) .setLabels(ImmutableMap.of("key1", "value1", "key2", "value2")) .build(); + Path cacheDirectory = temporaryFolder.newFolder().toPath(); return BuildConfiguration.builder() .setBaseImageConfiguration(baseImageConfiguration) .setTargetImageConfiguration(targetImageConfiguration) .setContainerConfiguration(containerConfiguration) + .setBaseImageLayersCacheConfiguration(CacheConfiguration.forPath(cacheDirectory)) + .setApplicationLayersCacheConfiguration(CacheConfiguration.forPath(cacheDirectory)) .setAllowInsecureRegistries(true) .setLayerConfigurations(fakeLayerConfigurations) .setToolName("jib-integration-test"); diff --git a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java index bd76ef03e6..3529608f1b 100644 --- a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java +++ b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java @@ -22,7 +22,6 @@ import com.google.cloud.tools.jib.image.json.V21ManifestTemplate; import com.google.common.io.ByteStreams; import java.io.IOException; -import java.io.OutputStream; import java.security.DigestException; import org.hamcrest.CoreMatchers; import org.junit.Assert; @@ -30,7 +29,6 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.mockito.Mockito; /** Integration tests for {@link BlobPuller}. */ public class BlobPullerIntegrationTest { @@ -56,28 +54,31 @@ public void testPull() throws IOException, RegistryException, InterruptedExcepti // Pulls a layer BLOB of the busybox image. CountingDigestOutputStream layerOutputStream = new CountingDigestOutputStream(ByteStreams.nullOutputStream()); - registryClient.pullBlob(realDigest, layerOutputStream); + registryClient.pullBlob(realDigest).writeTo(layerOutputStream); Assert.assertEquals(realDigest, layerOutputStream.toBlobDescriptor().getDigest()); } @Test - public void testPull_unknownBlob() - throws RegistryException, IOException, DigestException, InterruptedException { + public void testPull_unknownBlob() throws IOException, DigestException, InterruptedException { localRegistry.pullAndPushToLocal("busybox", "busybox"); DescriptorDigest nonexistentDigest = DescriptorDigest.fromHash( "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + RegistryClient registryClient = + RegistryClient.factory(EVENT_DISPATCHER, "localhost:5000", "busybox") + .setAllowInsecureRegistries(true) + .newRegistryClient(); + try { - RegistryClient registryClient = - RegistryClient.factory(EVENT_DISPATCHER, "localhost:5000", "busybox") - .setAllowInsecureRegistries(true) - .newRegistryClient(); - registryClient.pullBlob(nonexistentDigest, Mockito.mock(OutputStream.class)); + registryClient.pullBlob(nonexistentDigest).writeTo(ByteStreams.nullOutputStream()); Assert.fail("Trying to pull nonexistent blob should have errored"); - } catch (RegistryErrorException ex) { + } catch (IOException ex) { + if (!(ex.getCause() instanceof RegistryErrorException)) { + throw ex; + } Assert.assertThat( ex.getMessage(), CoreMatchers.containsString( diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/api/JibContainerBuilder.java b/jib-core/src/main/java/com/google/cloud/tools/jib/api/JibContainerBuilder.java index 796a7f0b8c..9386f12b86 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/api/JibContainerBuilder.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/api/JibContainerBuilder.java @@ -18,6 +18,7 @@ // TODO: Move to com.google.cloud.tools.jib once that package is cleaned up. import com.google.cloud.tools.jib.configuration.BuildConfiguration; +import com.google.cloud.tools.jib.configuration.CacheDirectoryCreationException; import com.google.cloud.tools.jib.configuration.ContainerConfiguration; import com.google.cloud.tools.jib.configuration.LayerConfiguration; import com.google.cloud.tools.jib.configuration.Port; @@ -302,7 +303,8 @@ public JibContainerBuilder addLabel(String key, String value) { } @VisibleForTesting - BuildConfiguration toBuildConfiguration(TargetImage targetImage) { + BuildConfiguration toBuildConfiguration(TargetImage targetImage) + throws IOException, CacheDirectoryCreationException { BuildConfiguration.Builder buildConfigurationBuilder = BuildConfiguration.builder(); buildConfigurationBuilder diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/BuildSteps.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/BuildSteps.java index d6c6e9f185..e6702b080a 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/BuildSteps.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/BuildSteps.java @@ -17,14 +17,8 @@ package com.google.cloud.tools.jib.builder; import com.google.cloud.tools.jib.builder.steps.StepsRunner; -import com.google.cloud.tools.jib.cache.Cache; -import com.google.cloud.tools.jib.cache.CacheDirectoryCreationException; -import com.google.cloud.tools.jib.cache.CacheDirectoryNotOwnedException; -import com.google.cloud.tools.jib.cache.CacheMetadataCorruptedException; -import com.google.cloud.tools.jib.cache.Caches; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.event.events.LogEvent; -import java.io.IOException; import java.nio.file.Path; import java.util.concurrent.ExecutionException; @@ -46,15 +40,12 @@ private interface StepsRunnerConsumer { * All the steps to build an image to a Docker registry. * * @param buildConfiguration the configuration parameters for the build - * @param cachesInitializer the {@link Caches.Initializer} used to setup the cache * @return a new {@link BuildSteps} for building to a registry */ - public static BuildSteps forBuildToDockerRegistry( - BuildConfiguration buildConfiguration, Caches.Initializer cachesInitializer) { + public static BuildSteps forBuildToDockerRegistry(BuildConfiguration buildConfiguration) { return new BuildSteps( DESCRIPTION_FOR_DOCKER_REGISTRY, buildConfiguration, - cachesInitializer, stepsRunner -> stepsRunner .runRetrieveTargetRegistryCredentialsStep() @@ -75,15 +66,12 @@ public static BuildSteps forBuildToDockerRegistry( * All the steps to build to Docker daemon * * @param buildConfiguration the configuration parameters for the build - * @param cachesInitializer the {@link Caches.Initializer} used to setup the cache * @return a new {@link BuildSteps} for building to a Docker daemon */ - public static BuildSteps forBuildToDockerDaemon( - BuildConfiguration buildConfiguration, Caches.Initializer cachesInitializer) { + public static BuildSteps forBuildToDockerDaemon(BuildConfiguration buildConfiguration) { return new BuildSteps( DESCRIPTION_FOR_DOCKER_DAEMON, buildConfiguration, - cachesInitializer, stepsRunner -> stepsRunner .runPullBaseImageStep() @@ -100,17 +88,12 @@ public static BuildSteps forBuildToDockerDaemon( * * @param outputPath the path to output the tarball to * @param buildConfiguration the configuration parameters for the build - * @param cachesInitializer the {@link Caches.Initializer} used to setup the cache * @return a new {@link BuildSteps} for building a tarball */ - public static BuildSteps forBuildToTar( - Path outputPath, - BuildConfiguration buildConfiguration, - Caches.Initializer cachesInitializer) { + public static BuildSteps forBuildToTar(Path outputPath, BuildConfiguration buildConfiguration) { return new BuildSteps( DESCRIPTION_FOR_TARBALL, buildConfiguration, - cachesInitializer, stepsRunner -> stepsRunner .runPullBaseImageStep() @@ -124,21 +107,19 @@ public static BuildSteps forBuildToTar( private final String description; private final BuildConfiguration buildConfiguration; - private final Caches.Initializer cachesInitializer; private final StepsRunnerConsumer stepsRunnerConsumer; /** * @param description a description of what the steps do + * @param buildConfiguration the configuration parameters for the build * @param stepsRunnerConsumer accepts a {@link StepsRunner} by running the necessary steps */ private BuildSteps( String description, BuildConfiguration buildConfiguration, - Caches.Initializer cachesInitializer, StepsRunnerConsumer stepsRunnerConsumer) { this.description = description; this.buildConfiguration = buildConfiguration; - this.cachesInitializer = cachesInitializer; this.stepsRunnerConsumer = stepsRunnerConsumer; } @@ -146,26 +127,12 @@ public BuildConfiguration getBuildConfiguration() { return buildConfiguration; } - public void run() - throws InterruptedException, ExecutionException, CacheMetadataCorruptedException, IOException, - CacheDirectoryNotOwnedException, CacheDirectoryCreationException { + public void run() throws InterruptedException, ExecutionException { buildConfiguration.getEventDispatcher().dispatch(LogEvent.lifecycle("")); try (TimerEventDispatcher ignored = new TimerEventDispatcher(buildConfiguration.getEventDispatcher(), description)) { - try (Caches caches = cachesInitializer.init()) { - Cache baseImageLayersCache = caches.getBaseCache(); - Cache applicationLayersCache = caches.getApplicationCache(); - - StepsRunner stepsRunner = - new StepsRunner(buildConfiguration, baseImageLayersCache, applicationLayersCache); - stepsRunnerConsumer.accept(stepsRunner); - - // Writes the cached layers to the cache metadata. - baseImageLayersCache.addCachedLayersToMetadata(stepsRunner.getCachedBaseImageLayers()); - applicationLayersCache.addCachedLayersWithMetadataToMetadata( - stepsRunner.getCachedApplicationLayers()); - } + stepsRunnerConsumer.accept(new StepsRunner(buildConfiguration)); } if (buildConfiguration.getContainerConfiguration() != null) { diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java index 4ac67806ff..4e26811127 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java @@ -17,16 +17,15 @@ package com.google.cloud.tools.jib.builder.steps; import com.google.cloud.tools.jib.async.AsyncStep; +import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; -import com.google.cloud.tools.jib.cache.Cache; -import com.google.cloud.tools.jib.cache.CacheMetadataCorruptedException; -import com.google.cloud.tools.jib.cache.CacheReader; -import com.google.cloud.tools.jib.cache.CacheWriter; -import com.google.cloud.tools.jib.cache.CachedLayerWithMetadata; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.configuration.LayerConfiguration; import com.google.cloud.tools.jib.event.events.LogEvent; import com.google.cloud.tools.jib.image.ReproducibleLayerBuilder; +import com.google.cloud.tools.jib.ncache.Cache; +import com.google.cloud.tools.jib.ncache.CacheCorruptedException; +import com.google.cloud.tools.jib.ncache.CacheEntry; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -35,8 +34,7 @@ import java.util.concurrent.Callable; /** Builds and caches application layers. */ -class BuildAndCacheApplicationLayerStep - implements AsyncStep, Callable { +class BuildAndCacheApplicationLayerStep implements AsyncStep, Callable { private static final String DESCRIPTION = "Building application layers"; @@ -45,9 +43,7 @@ class BuildAndCacheApplicationLayerStep * classes layers. Optionally adds an extra layer if configured to do so. */ static ImmutableList makeList( - ListeningExecutorService listeningExecutorService, - BuildConfiguration buildConfiguration, - Cache cache) { + ListeningExecutorService listeningExecutorService, BuildConfiguration buildConfiguration) { try (TimerEventDispatcher ignored = new TimerEventDispatcher(buildConfiguration.getEventDispatcher(), DESCRIPTION)) { ImmutableList.Builder buildAndCacheApplicationLayerSteps = @@ -63,8 +59,7 @@ static ImmutableList makeList( layerConfiguration.getName(), listeningExecutorService, buildConfiguration, - layerConfiguration, - cache)); + layerConfiguration)); } return buildAndCacheApplicationLayerSteps.build(); } @@ -73,56 +68,52 @@ static ImmutableList makeList( private final String layerType; private final BuildConfiguration buildConfiguration; private final LayerConfiguration layerConfiguration; - private final Cache cache; - private final ListenableFuture listenableFuture; + private final ListenableFuture listenableFuture; private BuildAndCacheApplicationLayerStep( String layerType, ListeningExecutorService listeningExecutorService, BuildConfiguration buildConfiguration, - LayerConfiguration layerConfiguration, - Cache cache) { + LayerConfiguration layerConfiguration) { this.layerType = layerType; this.buildConfiguration = buildConfiguration; this.layerConfiguration = layerConfiguration; - this.cache = cache; listenableFuture = listeningExecutorService.submit(this); } @Override - public ListenableFuture getFuture() { + public ListenableFuture getFuture() { return listenableFuture; } @Override - public CachedLayerWithMetadata call() throws IOException, CacheMetadataCorruptedException { + public CacheEntry call() throws IOException, CacheCorruptedException { String description = "Building " + layerType + " layer"; buildConfiguration.getEventDispatcher().dispatch(LogEvent.lifecycle(description + "...")); try (TimerEventDispatcher ignored = new TimerEventDispatcher(buildConfiguration.getEventDispatcher(), description)) { + Cache cache = buildConfiguration.getApplicationLayersCache(); + // Don't build the layer if it exists already. - Optional optionalCachedLayer = - new CacheReader(cache) - .getUpToDateLayerByLayerEntries(layerConfiguration.getLayerEntries()); - if (optionalCachedLayer.isPresent()) { - return optionalCachedLayer.get(); + Optional optionalCacheEntry = + cache.retrieve(layerConfiguration.getLayerEntries()); + if (optionalCacheEntry.isPresent()) { + return optionalCacheEntry.get(); } - CachedLayerWithMetadata cachedLayer = - new CacheWriter(cache) - .writeLayer(new ReproducibleLayerBuilder(layerConfiguration.getLayerEntries())); + Blob layerBlob = new ReproducibleLayerBuilder(layerConfiguration.getLayerEntries()).build(); + CacheEntry cacheEntry = + cache.writeUncompressedLayer(layerBlob, layerConfiguration.getLayerEntries()); buildConfiguration .getEventDispatcher() - .dispatch( - LogEvent.debug( - description + " built " + cachedLayer.getBlobDescriptor().getDigest())); + .dispatch(LogEvent.debug(description + " built " + cacheEntry.getLayerDigest())); - return cachedLayer; + return cacheEntry; } } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildImageStep.java index 1c87ecaa66..3592280450 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildImageStep.java @@ -18,14 +18,18 @@ import com.google.cloud.tools.jib.async.AsyncStep; import com.google.cloud.tools.jib.async.NonBlockingSteps; +import com.google.cloud.tools.jib.blob.Blob; +import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; -import com.google.cloud.tools.jib.cache.CachedLayer; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.configuration.ContainerConfiguration; +import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.Layer; import com.google.cloud.tools.jib.image.LayerPropertyNotFoundException; import com.google.cloud.tools.jib.image.json.HistoryEntry; +import com.google.cloud.tools.jib.ncache.CacheEntry; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -38,17 +42,38 @@ /** Builds a model {@link Image}. */ class BuildImageStep - implements AsyncStep>>, Callable>> { + implements AsyncStep>>, Callable>> { private static final String DESCRIPTION = "Building container configuration"; + @VisibleForTesting + static Layer cacheEntryToLayer(CacheEntry cacheEntry) { + return new Layer() { + + @Override + public Blob getBlob() throws LayerPropertyNotFoundException { + return cacheEntry.getLayerBlob(); + } + + @Override + public BlobDescriptor getBlobDescriptor() throws LayerPropertyNotFoundException { + return new BlobDescriptor(cacheEntry.getLayerSize(), cacheEntry.getLayerDigest()); + } + + @Override + public DescriptorDigest getDiffId() throws LayerPropertyNotFoundException { + return cacheEntry.getLayerDiffId(); + } + }; + } + private final BuildConfiguration buildConfiguration; private final PullBaseImageStep pullBaseImageStep; private final PullAndCacheBaseImageLayersStep pullAndCacheBaseImageLayersStep; private final ImmutableList buildAndCacheApplicationLayerSteps; private final ListeningExecutorService listeningExecutorService; - private final ListenableFuture>> listenableFuture; + private final ListenableFuture>> listenableFuture; BuildImageStep( ListeningExecutorService listeningExecutorService, @@ -69,12 +94,12 @@ class BuildImageStep } @Override - public ListenableFuture>> getFuture() { + public ListenableFuture>> getFuture() { return listenableFuture; } @Override - public AsyncStep> call() throws ExecutionException { + public AsyncStep> call() throws ExecutionException { List> dependencies = new ArrayList<>(); for (PullAndCacheBaseImageLayerStep pullAndCacheBaseImageLayerStep : @@ -85,18 +110,18 @@ public AsyncStep> call() throws ExecutionException { buildAndCacheApplicationLayerSteps) { dependencies.add(buildAndCacheApplicationLayerStep.getFuture()); } - ListenableFuture> future = + ListenableFuture> future = Futures.whenAllSucceed(dependencies) - .call(this::afterCachedLayersSteps, listeningExecutorService); + .call(this::afterCacheEntrySteps, listeningExecutorService); return () -> future; } - private Image afterCachedLayersSteps() + private Image afterCacheEntrySteps() throws ExecutionException, LayerPropertyNotFoundException { try (TimerEventDispatcher ignored = new TimerEventDispatcher(buildConfiguration.getEventDispatcher(), DESCRIPTION)) { // Constructs the image. - Image.Builder imageBuilder = Image.builder(); + Image.Builder imageBuilder = Image.builder(); Image baseImage = NonBlockingSteps.get(pullBaseImageStep).getBaseImage(); ContainerConfiguration containerConfiguration = buildConfiguration.getContainerConfiguration(); @@ -105,7 +130,8 @@ private Image afterCachedLayersSteps() List baseImageLayers = NonBlockingSteps.get(pullAndCacheBaseImageLayersStep); for (PullAndCacheBaseImageLayerStep pullAndCacheBaseImageLayerStep : baseImageLayers) { - imageBuilder.addLayer(NonBlockingSteps.get(pullAndCacheBaseImageLayerStep)); + imageBuilder.addLayer( + cacheEntryToLayer(NonBlockingSteps.get(pullAndCacheBaseImageLayerStep))); } // Passthrough config and count non-empty history entries @@ -136,7 +162,8 @@ private Image afterCachedLayersSteps() // Add built layers/configuration for (BuildAndCacheApplicationLayerStep buildAndCacheApplicationLayerStep : buildAndCacheApplicationLayerSteps) { - imageBuilder.addLayer(NonBlockingSteps.get(buildAndCacheApplicationLayerStep)); + imageBuilder.addLayer( + cacheEntryToLayer(NonBlockingSteps.get(buildAndCacheApplicationLayerStep))); imageBuilder.addHistory( HistoryEntry.builder() .setCreationTimestamp(layerCreationTime) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/LoadDockerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/LoadDockerStep.java index adedfab16a..04b6aca189 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/LoadDockerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/LoadDockerStep.java @@ -18,13 +18,13 @@ import com.google.cloud.tools.jib.async.AsyncStep; import com.google.cloud.tools.jib.async.NonBlockingSteps; -import com.google.cloud.tools.jib.cache.CachedLayer; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.docker.DockerClient; import com.google.cloud.tools.jib.docker.ImageToTarballTranslator; import com.google.cloud.tools.jib.event.events.LogEvent; import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.ImageReference; +import com.google.cloud.tools.jib.image.Layer; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -86,7 +86,7 @@ public Void call() throws ExecutionException, InterruptedException { private Void afterPushBaseImageLayerFuturesFuture() throws ExecutionException, InterruptedException, IOException { - Image image = NonBlockingSteps.get(NonBlockingSteps.get(buildImageStep)); + Image image = NonBlockingSteps.get(NonBlockingSteps.get(buildImageStep)); ImageReference targetImageReference = buildConfiguration.getTargetImageConfiguration().getImage(); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java index 33dbc760d9..3ac1856109 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java @@ -18,42 +18,37 @@ import com.google.cloud.tools.jib.async.AsyncStep; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; -import com.google.cloud.tools.jib.cache.Cache; -import com.google.cloud.tools.jib.cache.CacheReader; -import com.google.cloud.tools.jib.cache.CacheWriter; -import com.google.cloud.tools.jib.cache.CachedLayer; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.http.Authorization; import com.google.cloud.tools.jib.image.DescriptorDigest; +import com.google.cloud.tools.jib.ncache.Cache; +import com.google.cloud.tools.jib.ncache.CacheCorruptedException; +import com.google.cloud.tools.jib.ncache.CacheEntry; import com.google.cloud.tools.jib.registry.RegistryClient; -import com.google.cloud.tools.jib.registry.RegistryException; -import com.google.common.io.CountingOutputStream; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import java.io.IOException; +import java.util.Optional; import java.util.concurrent.Callable; import javax.annotation.Nullable; /** Pulls and caches a single base image layer. */ -class PullAndCacheBaseImageLayerStep implements AsyncStep, Callable { +class PullAndCacheBaseImageLayerStep implements AsyncStep, Callable { private static final String DESCRIPTION = "Pulling base image layer %s"; private final BuildConfiguration buildConfiguration; - private final Cache cache; private final DescriptorDigest layerDigest; private final @Nullable Authorization pullAuthorization; - private final ListenableFuture listenableFuture; + private final ListenableFuture listenableFuture; PullAndCacheBaseImageLayerStep( ListeningExecutorService listeningExecutorService, BuildConfiguration buildConfiguration, - Cache cache, DescriptorDigest layerDigest, @Nullable Authorization pullAuthorization) { this.buildConfiguration = buildConfiguration; - this.cache = cache; this.layerDigest = layerDigest; this.pullAuthorization = pullAuthorization; @@ -61,32 +56,29 @@ class PullAndCacheBaseImageLayerStep implements AsyncStep, Callable } @Override - public ListenableFuture getFuture() { + public ListenableFuture getFuture() { return listenableFuture; } @Override - public CachedLayer call() throws IOException, RegistryException { + public CacheEntry call() throws IOException, CacheCorruptedException { try (TimerEventDispatcher ignored = new TimerEventDispatcher( buildConfiguration.getEventDispatcher(), String.format(DESCRIPTION, layerDigest))) { + Cache cache = buildConfiguration.getBaseImageLayersCache(); + + // Checks if the layer already exists in the cache. + Optional optionalCacheEntry = cache.retrieve(layerDigest); + if (optionalCacheEntry.isPresent()) { + return optionalCacheEntry.get(); + } + RegistryClient registryClient = buildConfiguration .newBaseImageRegistryClientFactory() .setAuthorization(pullAuthorization) .newRegistryClient(); - - // Checks if the layer already exists in the cache. - CachedLayer cachedLayer = new CacheReader(cache).getLayer(layerDigest); - if (cachedLayer != null) { - return cachedLayer; - } - - CacheWriter cacheWriter = new CacheWriter(cache); - CountingOutputStream layerOutputStream = cacheWriter.getLayerOutputStream(layerDigest); - registryClient.pullBlob(layerDigest, layerOutputStream); - layerOutputStream.close(); - return cacheWriter.getCachedLayer(layerOutputStream.getCount(), layerDigest); + return cache.writeCompressedLayer(registryClient.pullBlob(layerDigest)); } } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayersStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayersStep.java index f3632d324c..6b07112f13 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayersStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayersStep.java @@ -20,7 +20,6 @@ import com.google.cloud.tools.jib.async.NonBlockingSteps; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; import com.google.cloud.tools.jib.builder.steps.PullBaseImageStep.BaseImageWithAuthorization; -import com.google.cloud.tools.jib.cache.Cache; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.image.Layer; import com.google.cloud.tools.jib.image.LayerPropertyNotFoundException; @@ -39,7 +38,6 @@ class PullAndCacheBaseImageLayersStep private static final String DESCRIPTION = "Setting up base image caching"; private final BuildConfiguration buildConfiguration; - private final Cache cache; private final PullBaseImageStep pullBaseImageStep; private final ListeningExecutorService listeningExecutorService; @@ -48,11 +46,9 @@ class PullAndCacheBaseImageLayersStep PullAndCacheBaseImageLayersStep( ListeningExecutorService listeningExecutorService, BuildConfiguration buildConfiguration, - Cache cache, PullBaseImageStep pullBaseImageStep) { this.listeningExecutorService = listeningExecutorService; this.buildConfiguration = buildConfiguration; - this.cache = cache; this.pullBaseImageStep = pullBaseImageStep; listenableFuture = @@ -79,7 +75,6 @@ public ImmutableList call() new PullAndCacheBaseImageLayerStep( listeningExecutorService, buildConfiguration, - cache, layer.getBlobDescriptor().getDigest(), pullBaseImageStepResult.getBaseImageAuthorization())); } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java index 2c61c0feab..344a2f2e15 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java @@ -47,9 +47,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import javax.annotation.Nullable; @@ -213,12 +211,10 @@ private Image pullBaseImage(@Nullable Authorization registryAuthorization + Blobs.writeToString(JsonTemplateMapper.toBlob(v22ManifestTemplate))); } - ByteArrayOutputStream containerConfigurationOutputStream = new ByteArrayOutputStream(); - registryClient.pullBlob( - v22ManifestTemplate.getContainerConfiguration().getDigest(), - containerConfigurationOutputStream); String containerConfigurationString = - new String(containerConfigurationOutputStream.toByteArray(), StandardCharsets.UTF_8); + Blobs.writeToString( + registryClient.pullBlob( + v22ManifestTemplate.getContainerConfiguration().getDigest())); ContainerConfigurationTemplate containerConfigurationTemplate = JsonTemplateMapper.readJson( diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushContainerConfigurationStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushContainerConfigurationStep.java index 8514f87905..04de287a0e 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushContainerConfigurationStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushContainerConfigurationStep.java @@ -21,10 +21,9 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; -import com.google.cloud.tools.jib.cache.CachedLayer; import com.google.cloud.tools.jib.configuration.BuildConfiguration; -import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; import com.google.cloud.tools.jib.image.Image; +import com.google.cloud.tools.jib.image.Layer; import com.google.cloud.tools.jib.image.json.ImageToJsonTranslator; import com.google.common.io.ByteStreams; import com.google.common.util.concurrent.Futures; @@ -79,14 +78,11 @@ private PushBlobStep afterBuildConfigurationFutureFuture() throws ExecutionException, IOException { try (TimerEventDispatcher ignored = new TimerEventDispatcher(buildConfiguration.getEventDispatcher(), DESCRIPTION)) { - Image image = NonBlockingSteps.get(NonBlockingSteps.get(buildImageStep)); + Image image = NonBlockingSteps.get(NonBlockingSteps.get(buildImageStep)); Blob containerConfigurationBlob = new ImageToJsonTranslator(image).getContainerConfigurationBlob(); - CountingDigestOutputStream digestOutputStream = - new CountingDigestOutputStream(ByteStreams.nullOutputStream()); - containerConfigurationBlob.writeTo(digestOutputStream); - - BlobDescriptor blobDescriptor = digestOutputStream.toBlobDescriptor(); + BlobDescriptor blobDescriptor = + containerConfigurationBlob.writeTo(ByteStreams.nullOutputStream()); return new PushBlobStep( listeningExecutorService, diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayersStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayersStep.java index 76f1764bc1..9a4c896c7d 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayersStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayersStep.java @@ -18,9 +18,10 @@ import com.google.cloud.tools.jib.async.AsyncStep; import com.google.cloud.tools.jib.async.NonBlockingSteps; +import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; -import com.google.cloud.tools.jib.cache.CachedLayer; import com.google.cloud.tools.jib.configuration.BuildConfiguration; +import com.google.cloud.tools.jib.ncache.CacheEntry; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -36,8 +37,8 @@ class PushLayersStep private final BuildConfiguration buildConfiguration; private final AuthenticatePushStep authenticatePushStep; - private final AsyncStep>> - cachedLayerStepsStep; + private final AsyncStep>> + cacheEntryStep; private final ListeningExecutorService listeningExecutorService; private final ListenableFuture>> listenableFuture; @@ -46,16 +47,15 @@ class PushLayersStep ListeningExecutorService listeningExecutorService, BuildConfiguration buildConfiguration, AuthenticatePushStep authenticatePushStep, - AsyncStep>> - cachedLayerStepsStep) { + AsyncStep>> + cacheEntryStep) { this.listeningExecutorService = listeningExecutorService; this.buildConfiguration = buildConfiguration; this.authenticatePushStep = authenticatePushStep; - this.cachedLayerStepsStep = cachedLayerStepsStep; + this.cacheEntryStep = cacheEntryStep; listenableFuture = - Futures.whenAllSucceed(cachedLayerStepsStep.getFuture()) - .call(this, listeningExecutorService); + Futures.whenAllSucceed(cacheEntryStep.getFuture()).call(this, listeningExecutorService); } @Override @@ -67,12 +67,12 @@ public ListenableFuture>> getFuture() { public ImmutableList> call() throws ExecutionException { try (TimerEventDispatcher ignored = new TimerEventDispatcher(buildConfiguration.getEventDispatcher(), DESCRIPTION)) { - ImmutableList> cachedLayerSteps = - NonBlockingSteps.get(cachedLayerStepsStep); + ImmutableList> cacheEntry = + NonBlockingSteps.get(cacheEntryStep); // Constructs a PushBlobStep for each layer. ImmutableList.Builder> pushBlobStepsBuilder = ImmutableList.builder(); - for (AsyncStep cachedLayerStep : cachedLayerSteps) { + for (AsyncStep cachedLayerStep : cacheEntry) { ListenableFuture pushBlobStepFuture = Futures.whenAllSucceed(cachedLayerStep.getFuture()) .call(() -> makePushBlobStep(cachedLayerStep), listeningExecutorService); @@ -83,14 +83,14 @@ public ImmutableList> call() throws ExecutionException { } } - private PushBlobStep makePushBlobStep(AsyncStep cachedLayerStep) + private PushBlobStep makePushBlobStep(AsyncStep cacheEntryStep) throws ExecutionException { - CachedLayer cachedLayer = NonBlockingSteps.get(cachedLayerStep); + CacheEntry cacheEntry = NonBlockingSteps.get(cacheEntryStep); return new PushBlobStep( listeningExecutorService, buildConfiguration, authenticatePushStep, - cachedLayer.getBlobDescriptor(), - cachedLayer.getBlob()); + new BlobDescriptor(cacheEntry.getLayerSize(), cacheEntry.getLayerDigest()), + cacheEntry.getLayerBlob()); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index 642b81964c..f2c8577441 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -17,10 +17,6 @@ package com.google.cloud.tools.jib.builder.steps; import com.google.cloud.tools.jib.async.AsyncSteps; -import com.google.cloud.tools.jib.async.NonBlockingSteps; -import com.google.cloud.tools.jib.cache.Cache; -import com.google.cloud.tools.jib.cache.CachedLayer; -import com.google.cloud.tools.jib.cache.CachedLayerWithMetadata; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.global.JibSystemProperties; import com.google.common.base.Preconditions; @@ -28,10 +24,8 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import java.nio.file.Path; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -47,8 +41,6 @@ public class StepsRunner { private final ListeningExecutorService listeningExecutorService; private final BuildConfiguration buildConfiguration; - private final Cache baseLayersCache; - private final Cache applicationLayersCache; @Nullable private RetrieveRegistryCredentialsStep retrieveTargetRegistryCredentialsStep; @Nullable private AuthenticatePushStep authenticatePushStep; @@ -66,11 +58,8 @@ public class StepsRunner { @Nullable private LoadDockerStep loadDockerStep; @Nullable private WriteTarFileStep writeTarFileStep; - public StepsRunner( - BuildConfiguration buildConfiguration, Cache baseLayersCache, Cache applicationLayersCache) { + public StepsRunner(BuildConfiguration buildConfiguration) { this.buildConfiguration = buildConfiguration; - this.baseLayersCache = baseLayersCache; - this.applicationLayersCache = applicationLayersCache; ExecutorService executorService = JibSystemProperties.isSerializedExecutionEnabled() @@ -105,7 +94,6 @@ public StepsRunner runPullAndCacheBaseImageLayersStep() { new PullAndCacheBaseImageLayersStep( listeningExecutorService, buildConfiguration, - baseLayersCache, Preconditions.checkNotNull(pullBaseImageStep)); return this; } @@ -122,8 +110,7 @@ public StepsRunner runPushBaseImageLayersStep() { public StepsRunner runBuildAndCacheApplicationLayerSteps() { buildAndCacheApplicationLayerSteps = - BuildAndCacheApplicationLayerStep.makeList( - listeningExecutorService, buildConfiguration, applicationLayersCache); + BuildAndCacheApplicationLayerStep.makeList(listeningExecutorService, buildConfiguration); return this; } @@ -225,36 +212,4 @@ public void waitOnLoadDockerStep() throws ExecutionException, InterruptedExcepti public void waitOnWriteTarFileStep() throws ExecutionException, InterruptedException { Preconditions.checkNotNull(writeTarFileStep).getFuture().get(); } - - /** - * @return the layers cached by {@link #pullAndCacheBaseImageLayersStep} - * @throws ExecutionException if {@link #pullAndCacheBaseImageLayersStep} threw an exception - * during execution - */ - public List getCachedBaseImageLayers() throws ExecutionException { - ImmutableList pullAndCacheBaseImageLayerSteps = - NonBlockingSteps.get(Preconditions.checkNotNull(pullAndCacheBaseImageLayersStep)); - - List cachedLayers = new ArrayList<>(pullAndCacheBaseImageLayerSteps.size()); - for (PullAndCacheBaseImageLayerStep pullAndCacheBaseImageLayerStep : - pullAndCacheBaseImageLayerSteps) { - cachedLayers.add(NonBlockingSteps.get(pullAndCacheBaseImageLayerStep)); - } - return cachedLayers; - } - - /** - * @return the layers cached by {@link #buildAndCacheApplicationLayerSteps} - * @throws ExecutionException if {@link #buildAndCacheApplicationLayerSteps} threw an exception - * during execution - */ - public List getCachedApplicationLayers() throws ExecutionException { - List cachedLayersWithMetadata = - new ArrayList<>(Preconditions.checkNotNull(buildAndCacheApplicationLayerSteps).size()); - for (BuildAndCacheApplicationLayerStep buildAndCacheApplicationLayerStep : - buildAndCacheApplicationLayerSteps) { - cachedLayersWithMetadata.add(NonBlockingSteps.get(buildAndCacheApplicationLayerStep)); - } - return cachedLayersWithMetadata; - } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/WriteTarFileStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/WriteTarFileStep.java index 6417ba05c7..beacb3623f 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/WriteTarFileStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/WriteTarFileStep.java @@ -18,12 +18,12 @@ import com.google.cloud.tools.jib.async.AsyncStep; import com.google.cloud.tools.jib.async.NonBlockingSteps; -import com.google.cloud.tools.jib.cache.CachedLayer; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.docker.ImageToTarballTranslator; import com.google.cloud.tools.jib.event.events.LogEvent; import com.google.cloud.tools.jib.filesystem.FileOperations; import com.google.cloud.tools.jib.image.Image; +import com.google.cloud.tools.jib.image.Layer; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -90,7 +90,7 @@ public Void call() throws ExecutionException, InterruptedException { } private Void afterPushBaseImageLayerFuturesFuture() throws ExecutionException, IOException { - Image image = NonBlockingSteps.get(NonBlockingSteps.get(buildImageStep)); + Image image = NonBlockingSteps.get(NonBlockingSteps.get(buildImageStep)); // Build the image to a tarball buildConfiguration diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/Cache.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/Cache.java deleted file mode 100644 index e9bfd08954..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/Cache.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright 2017 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache; - -import com.google.cloud.tools.jib.blob.Blobs; -import com.google.cloud.tools.jib.cache.json.CacheMetadataTemplate; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; -import com.google.common.annotations.VisibleForTesting; -import java.io.Closeable; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.NotDirectoryException; -import java.nio.file.Path; -import java.util.List; - -/** Manages a cache. Implementation is thread-safe. */ -public class Cache implements Closeable { - - /** - * Initializes a cache with a directory. This also loads the cache metadata if it exists in the - * directory. - * - * @param cacheDirectory the directory to use for the cache. - * @return the initialized cache. - * @throws NotDirectoryException if {@code cacheDirectory} is not a directory. - * @throws CacheMetadataCorruptedException if loading the cache metadata fails. - */ - public static Cache init(Path cacheDirectory) - throws NotDirectoryException, CacheMetadataCorruptedException { - if (!Files.isDirectory(cacheDirectory)) { - throw new NotDirectoryException("The cache can only write to a directory"); - } - CacheMetadata cacheMetadata = loadCacheMetadata(cacheDirectory); - - return new Cache(cacheDirectory, cacheMetadata); - } - - private static CacheMetadata loadCacheMetadata(Path cacheDirectory) - throws CacheMetadataCorruptedException { - Path cacheMetadataJsonFile = cacheDirectory.resolve(CacheFiles.METADATA_FILENAME); - - if (!Files.exists(cacheMetadataJsonFile)) { - return CacheMetadata.builder().build(); - } - - try { - CacheMetadataTemplate cacheMetadataJson = - JsonTemplateMapper.readJsonFromFileWithLock( - cacheMetadataJsonFile, CacheMetadataTemplate.class); - return CacheMetadataTranslator.fromTemplate(cacheMetadataJson, cacheDirectory); - } catch (IOException ex) { - // The cache metadata is probably corrupted. - throw new CacheMetadataCorruptedException(ex); - } - } - - /** The path to the root of the cache. */ - private final Path cacheDirectory; - - /** The metadata that corresponds to the cache at {@link #cacheDirectory}. */ - private final CacheMetadata cacheMetadata; - - /** Builds the updated cache metadata to save back to the cache. */ - private final CacheMetadata.Builder cacheMetadataBuilder; - - private Cache(Path cacheDirectory, CacheMetadata cacheMetadata) { - this.cacheDirectory = cacheDirectory; - this.cacheMetadata = cacheMetadata; - cacheMetadataBuilder = cacheMetadata.newAppendingBuilder(); - } - - /** - * Finishes the use of the cache by flushing any unsaved changes. - * - * @throws IOException if saving the cache metadata fails. - */ - @Override - public void close() throws IOException { - saveCacheMetadata(cacheDirectory); - } - - /** - * Adds the cached layer to the cache metadata. This is NOT thread-safe. - * - * @param cachedLayers the layers to add - */ - public void addCachedLayersToMetadata(List cachedLayers) { - for (CachedLayer cachedLayer : cachedLayers) { - cacheMetadataBuilder.addLayer(new CachedLayerWithMetadata(cachedLayer, null)); - } - } - - /** - * Adds the cached layer to the cache metadata. This is NOT thread-safe. - * - * @param cachedLayersWithMetadata the layers to add - */ - public void addCachedLayersWithMetadataToMetadata( - List cachedLayersWithMetadata) { - for (CachedLayerWithMetadata cachedLayerWithMetadata : cachedLayersWithMetadata) { - cacheMetadataBuilder.addLayer(cachedLayerWithMetadata); - } - } - - @VisibleForTesting - Path getCacheDirectory() { - return cacheDirectory; - } - - @VisibleForTesting - CacheMetadata getMetadata() { - return cacheMetadata; - } - - @VisibleForTesting - CacheMetadata getUpdatedMetadata() { - return cacheMetadataBuilder.build(); - } - - /** Saves the updated cache metadata back to the cache. */ - private void saveCacheMetadata(Path cacheDirectory) throws IOException { - Path cacheMetadataJsonFile = cacheDirectory.resolve(CacheFiles.METADATA_FILENAME); - - CacheMetadataTemplate cacheMetadataJson = - CacheMetadataTranslator.toTemplate(cacheMetadataBuilder.build()); - - Blobs.writeToFileWithLock(JsonTemplateMapper.toBlob(cacheMetadataJson), cacheMetadataJsonFile); - } -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheDirectoryNotOwnedException.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheDirectoryNotOwnedException.java deleted file mode 100644 index df9ff6b5ef..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheDirectoryNotOwnedException.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache; - -import java.nio.file.Path; - -/** - * Thrown when trying to use a directory as {@link Cache}, but the directory might be used by other - * applications. - */ -public class CacheDirectoryNotOwnedException extends Exception { - - private final Path cacheDirectory; - - /** Initializes with the cache directory that was unsafe to use. */ - CacheDirectoryNotOwnedException(Path cacheDirectory) { - super(); - - this.cacheDirectory = cacheDirectory; - } - - public Path getCacheDirectory() { - return cacheDirectory; - } -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheFiles.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheFiles.java deleted file mode 100644 index 49b3e86530..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheFiles.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2017 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache; - -import com.google.cloud.tools.jib.image.DescriptorDigest; -import java.nio.file.Path; - -/** Methods for getting static cache filename properties. */ -class CacheFiles { - - /** Increment the version prefix when the metadata format changes. */ - static final String METADATA_FILENAME = "metadata-v3.json"; - - private static final String LAYER_FILE_EXTENSION = ".tar.gz"; - - static Path getMetadataFile(Path cacheDirectory) { - return cacheDirectory.resolve(METADATA_FILENAME); - } - - /** - * Gets the path to the file for a layer digest. The file is {@code [cache directory]/[layer - * hash].tar.gz}. - */ - static Path getLayerFile(Path cacheDirectory, DescriptorDigest layerDigest) { - return cacheDirectory.resolve(layerDigest.getHash() + LAYER_FILE_EXTENSION); - } - - private CacheFiles() {} -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheMetadata.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheMetadata.java deleted file mode 100644 index f502e6369d..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheMetadata.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright 2017 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache; - -import com.google.cloud.tools.jib.image.ImageLayers; -import com.google.cloud.tools.jib.image.LayerEntry; -import com.google.cloud.tools.jib.image.LayerPropertyNotFoundException; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; -import java.util.Iterator; -import javax.annotation.Nullable; - -/** - * The cache stores all the layer BLOBs as separate files and the cache metadata contains - * information about each layer BLOB. - */ -class CacheMetadata { - - /** Builds a {@link CacheMetadata}. */ - static class Builder { - - private final ImageLayers.Builder layersBuilder = - ImageLayers.builder().removeDuplicates(); - - private Builder(ImageLayers initialLayers) { - layersBuilder.addAll(initialLayers); - } - - private Builder() {} - - /** - * Adds a layer. This method is NOT thread-safe. - * - * @param layer the layer to add - */ - Builder addLayer(CachedLayerWithMetadata layer) { - layersBuilder.add(layer); - return this; - } - - CacheMetadata build() { - return new CacheMetadata(layersBuilder.build()); - } - } - - /** Can be used to filter layers in the metadata. */ - static class LayerFilter { - - /** - * Checks if the layer entries matches the metadata layer entries. - * - * @param layerEntries the layer entries to check - * @param metadataEntries the metadata entries to match against - * @return {@code true} if the layer entries match the metadata entries; {@code false} otherwise - */ - @VisibleForTesting - static boolean doLayerEntriesMatchMetadataEntries( - ImmutableList layerEntries, - ImmutableList metadataEntries) { - // Checks the layer entries are the same as the metadata layer entries. - if (layerEntries.size() != metadataEntries.size()) { - return false; - } - - // Pairwise-compares all the layer entries with the metadata layer entries. - Iterator layerEntriesIterator = layerEntries.iterator(); - Iterator metadataEntriesIterator = - metadataEntries.iterator(); - while (layerEntriesIterator.hasNext() && metadataEntriesIterator.hasNext()) { - LayerEntry layerEntry = layerEntriesIterator.next(); - LayerMetadata.LayerMetadataEntry metadataEntry = metadataEntriesIterator.next(); - - boolean areSourceFilesEqual = - layerEntry - .getAbsoluteSourceFileString() - .equals(metadataEntry.getAbsoluteSourceFileString()); - boolean areExtractionPathsEqual = - layerEntry - .getAbsoluteExtractionPathString() - .equals(metadataEntry.getAbsoluteExtractionPathString()); - if (!areSourceFilesEqual || !areExtractionPathsEqual) { - return false; - } - } - return true; - } - - private final ImageLayers layers; - - @Nullable private ImmutableList layerEntries; - - private LayerFilter(ImageLayers layers) { - this.layers = layers; - } - - /** Filters to a certain list of {@link LayerEntry}s. */ - LayerFilter byLayerEntries(ImmutableList layerEntries) { - this.layerEntries = layerEntries; - return this; - } - - /** Applies the filters to the metadata layers. */ - ImageLayers filter() throws CacheMetadataCorruptedException { - try { - ImageLayers.Builder filteredLayersBuilder = ImageLayers.builder(); - - for (CachedLayerWithMetadata layer : layers) { - if (layerEntries != null) { - if (layer.getMetadata() == null) { - // There is no metadata, so it doesn't pass the filter. - continue; - } - - if (!doLayerEntriesMatchMetadataEntries( - layerEntries, layer.getMetadata().getEntries())) { - // The layer entries do not match. - continue; - } - } - - filteredLayersBuilder.add(layer); - } - - return filteredLayersBuilder.build(); - - } catch (LayerPropertyNotFoundException ex) { - throw new CacheMetadataCorruptedException(ex); - } - } - } - - static Builder builder() { - return new Builder(); - } - - private final ImageLayers layers; - - private CacheMetadata(ImageLayers layers) { - this.layers = layers; - } - - ImageLayers getLayers() { - return layers; - } - - LayerFilter filterLayers() { - return new LayerFilter(layers); - } - - /** @return a {@link Builder} starts with all the layers in this metadata */ - Builder newAppendingBuilder() { - return new Builder(layers); - } -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheMetadataCorruptedException.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheMetadataCorruptedException.java deleted file mode 100644 index c92590a68f..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheMetadataCorruptedException.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2017 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache; - -/** - * Exception thrown when the cache metadata failed to load correctly, indicating that it is probably - * corrupted. - */ -public class CacheMetadataCorruptedException extends Exception { - - CacheMetadataCorruptedException(Throwable cause) { - super(cause); - } - - public CacheMetadataCorruptedException(String message) { - super(message); - } -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheMetadataTranslator.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheMetadataTranslator.java deleted file mode 100644 index dd0cf93f84..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheMetadataTranslator.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright 2017 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache; - -import com.google.cloud.tools.jib.blob.BlobDescriptor; -import com.google.cloud.tools.jib.cache.json.CacheMetadataLayerObjectTemplate; -import com.google.cloud.tools.jib.cache.json.CacheMetadataLayerPropertiesObjectTemplate; -import com.google.cloud.tools.jib.cache.json.CacheMetadataLayerPropertiesObjectTemplate.LayerEntryTemplate; -import com.google.cloud.tools.jib.cache.json.CacheMetadataTemplate; -import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; -import com.google.cloud.tools.jib.image.LayerEntry; -import com.google.common.collect.ImmutableList; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; - -/** Translates {@link CacheMetadata} to and from {@link CacheMetadataTemplate}. */ -public class CacheMetadataTranslator { - - /** Translates {@link CacheMetadataTemplate} to {@link CacheMetadata}. */ - static CacheMetadata fromTemplate(CacheMetadataTemplate template, Path cacheDirectory) - throws CacheMetadataCorruptedException { - CacheMetadata.Builder cacheMetadataBuilder = CacheMetadata.builder(); - - // Converts each layer object in the template to a cache metadata layer. - for (CacheMetadataLayerObjectTemplate layerObjectTemplate : template.getLayers()) { - if (layerObjectTemplate.getDigest() == null || layerObjectTemplate.getDiffId() == null) { - throw new CacheMetadataCorruptedException( - "Cannot translate cache metadata layer without a digest or diffId"); - } - - Path layerContentFile = - CacheFiles.getLayerFile(cacheDirectory, layerObjectTemplate.getDigest()); - - // Gets the properties for a layer. Properties only exist for application layers. - CacheMetadataLayerPropertiesObjectTemplate propertiesObjectTemplate = - layerObjectTemplate.getProperties(); - - // Constructs the cache metadata layer from a cached layer and layer metadata. - LayerMetadata layerMetadata = null; - if (propertiesObjectTemplate != null) { - // Converts the layer entry templates to layer entries. - ImmutableList.Builder layerEntries = - ImmutableList.builderWithExpectedSize( - propertiesObjectTemplate.getLayerEntries().size()); - for (LayerEntryTemplate layerEntryTemplate : propertiesObjectTemplate.getLayerEntries()) { - if (layerEntryTemplate.getSourceFileString() == null - || layerEntryTemplate.getExtractionPathString() == null) { - throw new CacheMetadataCorruptedException( - "Cannot translate cache metadata layer entry without source files or extraction path"); - } - layerEntries.add( - new LayerEntry( - Paths.get(layerEntryTemplate.getSourceFileString()), - AbsoluteUnixPath.get(layerEntryTemplate.getExtractionPathString()))); - } - - layerMetadata = - LayerMetadata.from( - layerEntries.build(), propertiesObjectTemplate.getLastModifiedTime()); - } - - CachedLayer cachedLayer = - new CachedLayer( - layerContentFile, - new BlobDescriptor(layerObjectTemplate.getSize(), layerObjectTemplate.getDigest()), - layerObjectTemplate.getDiffId()); - - CachedLayerWithMetadata cachedLayerWithMetadata = - new CachedLayerWithMetadata(cachedLayer, layerMetadata); - cacheMetadataBuilder.addLayer(cachedLayerWithMetadata); - } - - return cacheMetadataBuilder.build(); - } - - /** Translates {@link CacheMetadata} to {@link CacheMetadataTemplate}. */ - static CacheMetadataTemplate toTemplate(CacheMetadata cacheMetadata) { - CacheMetadataTemplate template = new CacheMetadataTemplate(); - - for (CachedLayerWithMetadata cachedLayerWithMetadata : cacheMetadata.getLayers()) { - CacheMetadataLayerObjectTemplate layerObjectTemplate = - new CacheMetadataLayerObjectTemplate() - .setSize(cachedLayerWithMetadata.getBlobDescriptor().getSize()) - .setDigest(cachedLayerWithMetadata.getBlobDescriptor().getDigest()) - .setDiffId(cachedLayerWithMetadata.getDiffId()); - - if (cachedLayerWithMetadata.getMetadata() != null) { - // Constructs the layer entry templates to add to the layer object template. - ImmutableList metadataEntries = - cachedLayerWithMetadata.getMetadata().getEntries(); - List layerEntryTemplates = new ArrayList<>(metadataEntries.size()); - for (LayerMetadata.LayerMetadataEntry metadataEntry : metadataEntries) { - layerEntryTemplates.add( - new LayerEntryTemplate( - metadataEntry.getAbsoluteSourceFileString(), - metadataEntry.getAbsoluteExtractionPathString())); - } - - layerObjectTemplate.setProperties( - new CacheMetadataLayerPropertiesObjectTemplate() - .setLayerEntries(layerEntryTemplates) - .setLastModifiedTime(cachedLayerWithMetadata.getMetadata().getLastModifiedTime())); - } - - template.addLayer(layerObjectTemplate); - } - - return template; - } -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheReader.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheReader.java deleted file mode 100644 index 3eb00cb623..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheReader.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache; - -import com.google.cloud.tools.jib.filesystem.DirectoryWalker; -import com.google.cloud.tools.jib.image.DescriptorDigest; -import com.google.cloud.tools.jib.image.ImageLayers; -import com.google.cloud.tools.jib.image.LayerEntry; -import com.google.common.collect.ImmutableList; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.FileTime; -import java.time.Instant; -import java.util.List; -import java.util.Optional; -import javax.annotation.Nullable; - -/** Reads image content from the cache. */ -public class CacheReader { - - /** - * Gets the last modified time for the file at {@code path}. If {@code path} is a directory, then - * gets the latest modified time of its subfiles. - * - * @param path the file to check - * @return the last modified time - * @throws IOException if checking the last modified time fails - */ - private static FileTime getLastModifiedTime(Path path) throws IOException { - if (Files.isDirectory(path)) { - List subFiles = new DirectoryWalker(path).walk(); - FileTime maxLastModifiedTime = FileTime.from(Instant.MIN); - - // Finds the max last modified time for the subfiles. - for (Path subFilePath : subFiles) { - FileTime subFileLastModifiedTime = Files.getLastModifiedTime(subFilePath); - if (subFileLastModifiedTime.compareTo(maxLastModifiedTime) > 0) { - maxLastModifiedTime = subFileLastModifiedTime; - } - } - - return maxLastModifiedTime; - } - - return Files.getLastModifiedTime(path); - } - - private final Cache cache; - - public CacheReader(Cache cache) { - this.cache = cache; - } - - /** - * @param layerDigest the layer digest of the layer to get. - * @return the cached layer with digest {@code layerDigest}, or {@code null} if not found. - */ - @Nullable - public CachedLayer getLayer(DescriptorDigest layerDigest) { - return cache.getMetadata().getLayers().get(layerDigest); - } - - /** - * Finds the file that stores the content BLOB for an application layer. - * - * @param layerEntries the entries for the layer content - * @return the newest cached layer file that matches the {@code layerType} and {@code - * sourceFiles}, or {@code null} if there is no match. - * @throws CacheMetadataCorruptedException if getting the cache metadata fails. - */ - @Nullable - public Path getLayerFile(ImmutableList layerEntries) - throws CacheMetadataCorruptedException { - CacheMetadata cacheMetadata = cache.getMetadata(); - ImageLayers cachedLayers = - cacheMetadata.filterLayers().byLayerEntries(layerEntries).filter(); - - // Finds the newest cached layer for the layer type. - FileTime newestLastModifiedTime = FileTime.from(Instant.MIN); - - Path newestLayerFile = null; - for (CachedLayerWithMetadata cachedLayer : cachedLayers) { - if (cachedLayer.getMetadata() == null) { - throw new IllegalStateException("Layers with sourceFiles should have metadata"); - } - - FileTime cachedLayerLastModifiedTime = cachedLayer.getMetadata().getLastModifiedTime(); - if (cachedLayerLastModifiedTime.compareTo(newestLastModifiedTime) > 0) { - newestLastModifiedTime = cachedLayerLastModifiedTime; - newestLayerFile = cachedLayer.getContentFile(); - } - } - - return newestLayerFile; - } - - /** - * Gets an up-to-date layer that is built from the {@code layerEntries}. - * - *

The method returns the first up-to-date layer found. This is safe because the source files - * will not have been modified since creation of any up-to-date layer (ie. all up-to-date layers - * should have the same file contents). - * - * @param layerEntries the layer's content entries - * @return an up-to-date layer containing the source files. - * @throws IOException if reading the source files fails. - * @throws CacheMetadataCorruptedException if reading the cache metadata fails. - */ - public Optional getUpToDateLayerByLayerEntries( - ImmutableList layerEntries) throws IOException, CacheMetadataCorruptedException { - // Grabs all the layers that have matching source files. - ImageLayers cachedLayersWithSourceFiles = - cache.getMetadata().filterLayers().byLayerEntries(layerEntries).filter(); - if (cachedLayersWithSourceFiles.isEmpty()) { - return Optional.empty(); - } - - // Determines the latest modification time for the source files. - FileTime sourceFilesLastModifiedTime = FileTime.from(Instant.MIN); - for (LayerEntry layerEntry : layerEntries) { - FileTime lastModifiedTime = getLastModifiedTime(layerEntry.getSourceFile()); - if (lastModifiedTime.compareTo(sourceFilesLastModifiedTime) > 0) { - sourceFilesLastModifiedTime = lastModifiedTime; - } - } - - // Checks if at least one of the matched layers is up-to-date. - for (CachedLayerWithMetadata cachedLayer : cachedLayersWithSourceFiles) { - if (cachedLayer.getMetadata() == null) { - throw new IllegalStateException("Layers with sourceFiles should have metadata"); - } - - if (sourceFilesLastModifiedTime.compareTo(cachedLayer.getMetadata().getLastModifiedTime()) - <= 0) { - // This layer is an up-to-date layer. - return Optional.of(cachedLayer); - } - } - - return Optional.empty(); - } -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheWriter.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheWriter.java deleted file mode 100644 index 9aa1538684..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheWriter.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache; - -import com.google.cloud.tools.jib.blob.BlobDescriptor; -import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; -import com.google.cloud.tools.jib.image.DescriptorDigest; -import com.google.cloud.tools.jib.image.ReproducibleLayerBuilder; -import com.google.cloud.tools.jib.image.UnwrittenLayer; -import com.google.common.io.ByteStreams; -import com.google.common.io.CountingOutputStream; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.FileTime; -import java.time.Instant; -import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; - -/** Writes {@link UnwrittenLayer}s to the cache. */ -public class CacheWriter { - - private final Cache cache; - - public CacheWriter(Cache cache) { - this.cache = cache; - } - - /** - * Builds an {@link UnwrittenLayer} from a {@link ReproducibleLayerBuilder} and compresses and - * writes the {@link UnwrittenLayer}'s uncompressed layer content BLOB to cache. - * - * @param reproducibleLayerBuilder the layer builder - * @return the cached layer with accompanying {@link LayerMetadata} - * @throws IOException if writing the layer to file fails - */ - public CachedLayerWithMetadata writeLayer(ReproducibleLayerBuilder reproducibleLayerBuilder) - throws IOException { - UnwrittenLayer unwrittenLayer = reproducibleLayerBuilder.build(); - - // Writes to a temporary file first because the UnwrittenLayer needs to be written first to - // obtain its digest. - Path tempLayerFile = Files.createTempFile(cache.getCacheDirectory(), null, null); - // TODO: Find a way to do this with java.nio.file - tempLayerFile.toFile().deleteOnExit(); - - // Writes the UnwrittenLayer layer BLOB to a file to convert into a CachedLayer. - try (CountingDigestOutputStream compressedDigestOutputStream = - new CountingDigestOutputStream( - new BufferedOutputStream(Files.newOutputStream(tempLayerFile)))) { - // Writes the layer with GZIP compression. The original bytes are captured as the layer's - // diff ID and the bytes outputted from the GZIP compression are captured as the layer's - // content descriptor. - GZIPOutputStream compressorStream = new GZIPOutputStream(compressedDigestOutputStream); - DescriptorDigest diffId = unwrittenLayer.getBlob().writeTo(compressorStream).getDigest(); - - // The GZIPOutputStream must be closed in order to write out the remaining compressed data. - compressorStream.close(); - BlobDescriptor compressedBlobDescriptor = compressedDigestOutputStream.toBlobDescriptor(); - - // Renames the temporary layer file to the correct filename. - Path layerFile = getLayerFile(compressedBlobDescriptor.getDigest()); - try { - Files.move(tempLayerFile, layerFile); - } catch (FileAlreadyExistsException ignored) { - // If the file already exists, we skip renaming and use the existing file. This happens if a - // new layer happens to have the same content as a previously-cached layer. - // - // Do not attempt to remove the try-catch block with the idea of checking file existence - // before moving; there can be concurrent file moves. - } - - CachedLayer cachedLayer = new CachedLayer(layerFile, compressedBlobDescriptor, diffId); - LayerMetadata layerMetadata = - LayerMetadata.from( - reproducibleLayerBuilder.getLayerEntries(), FileTime.from(Instant.now())); - return new CachedLayerWithMetadata(cachedLayer, layerMetadata); - } - } - - /** - * @param layerDigest the written layer's digest. - * @return the {@link CountingOutputStream} to write to to cache a layer with the specified - * compressed digest. - * @throws IOException if writing to the layer file's output stream fails. - */ - public CountingOutputStream getLayerOutputStream(DescriptorDigest layerDigest) - throws IOException { - Path layerFile = getLayerFile(layerDigest); - return new CountingOutputStream(new BufferedOutputStream(Files.newOutputStream(layerFile))); - } - - /** - * Gets the layer that was cached by writing the contents to a file in the cache. The returned - * {@link CachedLayer} should be added to the cache metadata. - * - * @param layerSize the size of the layer - * @param layerDigest the digest of the layer to retrieve - * @return a {@link CachedLayer} from a layer digest and the {@link CountingOutputStream} the - * layer BLOB was written to - * @throws IOException if closing the output stream or getting the layer diff ID fails - */ - public CachedLayer getCachedLayer(long layerSize, DescriptorDigest layerDigest) - throws IOException { - Path layerFile = getLayerFile(layerDigest); - - return new CachedLayer( - layerFile, new BlobDescriptor(layerSize, layerDigest), getDiffId(layerFile)); - } - - /** - * @param compressedDigest the compressed digest of the layer to find. - * @return the path to the file for the layer with the specified compressed digest. - */ - private Path getLayerFile(DescriptorDigest compressedDigest) { - return CacheFiles.getLayerFile(cache.getCacheDirectory(), compressedDigest); - } - - /** - * @param layerFile the layer content file. - * @return the layer diff ID by decompressing the layer content file. - * @throws IOException if reading the layer file fails. - */ - private DescriptorDigest getDiffId(Path layerFile) throws IOException { - CountingDigestOutputStream diffIdCaptureOutputStream = - new CountingDigestOutputStream(ByteStreams.nullOutputStream()); - try (InputStream fileInputStream = new BufferedInputStream(Files.newInputStream(layerFile)); - GZIPInputStream decompressorStream = new GZIPInputStream(fileInputStream)) { - ByteStreams.copy(decompressorStream, diffIdCaptureOutputStream); - } - return diffIdCaptureOutputStream.toBlobDescriptor().getDigest(); - } -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CachedLayer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CachedLayer.java deleted file mode 100644 index cf0d33a199..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CachedLayer.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright 2017 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache; - -import com.google.cloud.tools.jib.blob.Blob; -import com.google.cloud.tools.jib.blob.BlobDescriptor; -import com.google.cloud.tools.jib.blob.Blobs; -import com.google.cloud.tools.jib.image.DescriptorDigest; -import com.google.cloud.tools.jib.image.Layer; -import java.nio.file.Path; - -/** - * A {@link Layer} that has been written out to a cache and has its file-backed content BLOB, - * digest, size, and diff ID. - */ -public class CachedLayer implements Layer { - - private final Path contentFile; - private final BlobDescriptor blobDescriptor; - private final DescriptorDigest diffId; - - /** - * Initializes the layer with its file-backed content BLOB, content descriptor (digest and size), - * and diff ID. The {@code blobDescriptor} and {@code diffId} must match the BLOB stored in - * the file - no checks are made at runtime. - * - * @param contentFile the file with the layer's content BLOB - * @param blobDescriptor the content descriptor for the layer's content BLOB - * @param diffId the diff ID for the layer - * @see Layer - */ - public CachedLayer(Path contentFile, BlobDescriptor blobDescriptor, DescriptorDigest diffId) { - this.contentFile = contentFile; - this.blobDescriptor = blobDescriptor; - this.diffId = diffId; - } - - public Path getContentFile() { - return contentFile; - } - - @Override - public Blob getBlob() { - return Blobs.from(contentFile); - } - - @Override - public BlobDescriptor getBlobDescriptor() { - return blobDescriptor; - } - - @Override - public DescriptorDigest getDiffId() { - return diffId; - } - - @Override - public boolean equals(Object other) { - if (other == this) { - return true; - } - if (!(other instanceof CachedLayer)) { - return false; - } - CachedLayer otherLayer = (CachedLayer) other; - return getBlobDescriptor().getDigest().equals(otherLayer.getBlobDescriptor().getDigest()); - } - - @Override - public int hashCode() { - return getBlobDescriptor().getDigest().hashCode(); - } -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CachedLayerWithMetadata.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CachedLayerWithMetadata.java deleted file mode 100644 index c087c78766..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CachedLayerWithMetadata.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2017 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache; - -import javax.annotation.Nullable; - -/** A {@link CachedLayer} with a last modified time. */ -public class CachedLayerWithMetadata extends CachedLayer { - - /** Extra layer properties for application layers. */ - @Nullable private final LayerMetadata metadata; - - public CachedLayerWithMetadata(CachedLayer cachedLayer, @Nullable LayerMetadata metadata) { - super(cachedLayer.getContentFile(), cachedLayer.getBlobDescriptor(), cachedLayer.getDiffId()); - - this.metadata = metadata; - } - - @Nullable - LayerMetadata getMetadata() { - return metadata; - } -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/Caches.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/Caches.java deleted file mode 100644 index 3c5a5294f3..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/Caches.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright 2017 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache; - -import com.google.common.annotations.VisibleForTesting; -import java.io.Closeable; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; - -/** - * Manages both the base image layers cache and the application image layers cache. - * - *

In general, the cache for base image layers should be shared between projects, while the cache - * for the application image layers should be specific to a single project. - */ -public class Caches implements Closeable { - - /** Initializes a {@link Caches} with directory paths. */ - public static class Initializer { - - /** A file to store in the default base image layers cache to check ownership by Jib. */ - private static final String OWNERSHIP_FILE_NAME = ".jib"; - - /** - * Ensures ownership of {@code cacheDirectory} by checking for the existence of {@link - * #OWNERSHIP_FILE_NAME}. - * - *

This is a safety check to make sure we are not writing to a directory not created by Jib. - */ - @VisibleForTesting - static void ensureOwnership(Path cacheDirectory) - throws CacheDirectoryNotOwnedException, CacheDirectoryCreationException { - Path ownershipFile = cacheDirectory.resolve(OWNERSHIP_FILE_NAME); - - if (Files.exists(cacheDirectory)) { - // Checks for the ownership file. - if (!Files.exists(ownershipFile)) { - throw new CacheDirectoryNotOwnedException(cacheDirectory); - } - - } else { - try { - // Creates the cache directory and ownership file. - Files.createDirectories(cacheDirectory); - Files.createFile(ownershipFile); - - } catch (IOException ex) { - throw new CacheDirectoryCreationException(cacheDirectory, ex); - } - } - } - - private final Path baseImageLayersCacheDirectory; - private final boolean shouldEnsureOwnershipOfBaseImageLayersCacheDirectory; - private final Path applicationLayersCacheDirectory; - private final boolean shouldEnsureOwnershipOfApplicationLayersCacheDirectory; - - /** - * @param baseImageLayersCacheDirectory cache for the application image layers - usually not - * local to the application project - * @param shouldEnsureOwnershipOfBaseImageLayersCacheDirectory if {@code true}, ensures the base - * image layers cache directory is safe to write to - * @param applicationLayersCacheDirectory cache for the application image layers - usually local - * to the application project - * @param shouldEnsureOwnershipOfApplicationLayersCacheDirectory if {@code true}, ensures the - * base image layers cache directory is safe to write to - */ - public Initializer( - Path baseImageLayersCacheDirectory, - boolean shouldEnsureOwnershipOfBaseImageLayersCacheDirectory, - Path applicationLayersCacheDirectory, - boolean shouldEnsureOwnershipOfApplicationLayersCacheDirectory) { - this.baseImageLayersCacheDirectory = baseImageLayersCacheDirectory; - this.shouldEnsureOwnershipOfBaseImageLayersCacheDirectory = - shouldEnsureOwnershipOfBaseImageLayersCacheDirectory; - this.applicationLayersCacheDirectory = applicationLayersCacheDirectory; - this.shouldEnsureOwnershipOfApplicationLayersCacheDirectory = - shouldEnsureOwnershipOfApplicationLayersCacheDirectory; - } - - public Caches init() - throws CacheMetadataCorruptedException, CacheDirectoryNotOwnedException, - CacheDirectoryCreationException, IOException { - if (shouldEnsureOwnershipOfBaseImageLayersCacheDirectory) { - ensureOwnership(baseImageLayersCacheDirectory); - } - if (shouldEnsureOwnershipOfApplicationLayersCacheDirectory) { - ensureOwnership(applicationLayersCacheDirectory); - } - - return new Caches(baseImageLayersCacheDirectory, applicationLayersCacheDirectory); - } - } - - private final Cache baseCache; - private final Cache applicationCache; - - /** Instantiate with {@link Initializer#init}. */ - private Caches(Path baseCacheDirectory, Path applicationCacheDirectory) - throws CacheMetadataCorruptedException, IOException { - applicationCache = Cache.init(applicationCacheDirectory); - - // Ensures that only one Cache is initialized if using the same directory. - if (Files.isSameFile(baseCacheDirectory, applicationCacheDirectory)) { - baseCache = applicationCache; - } else { - baseCache = Cache.init(baseCacheDirectory); - } - } - - public Cache getBaseCache() { - return baseCache; - } - - public Cache getApplicationCache() { - return applicationCache; - } - - @Override - public void close() throws IOException { - applicationCache.close(); - - if (baseCache != applicationCache) { - baseCache.close(); - } - } -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/LayerMetadata.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/LayerMetadata.java deleted file mode 100644 index d629fa7d70..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/LayerMetadata.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2017 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache; - -import com.google.cloud.tools.jib.image.LayerEntry; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; -import java.nio.file.attribute.FileTime; - -/** - * Metadata about an application layer stored in the cache. This is part of the {@link - * CacheMetadata}. - */ -class LayerMetadata { - - /** Entry into the layer metadata. */ - static class LayerMetadataEntry { - - /** The source file path string, in Unix form. The path should be an absolute path. */ - private final String absoluteSourceFileString; - - /** The extraction path string, in Unix form. The path should be an absolute path. */ - private final String absoluteExtractionPathString; - - String getAbsoluteSourceFileString() { - return absoluteSourceFileString; - } - - String getAbsoluteExtractionPathString() { - return absoluteExtractionPathString; - } - - @VisibleForTesting - LayerMetadataEntry(String absoluteSourceFileString, String absoluteExtractionPathString) { - this.absoluteSourceFileString = absoluteSourceFileString; - this.absoluteExtractionPathString = absoluteExtractionPathString; - } - } - - static LayerMetadata from(ImmutableList layerEntries, FileTime lastModifiedTime) { - ImmutableList.Builder entries = - ImmutableList.builderWithExpectedSize(layerEntries.size()); - - for (LayerEntry layerEntry : layerEntries) { - entries.add( - new LayerMetadataEntry( - layerEntry.getAbsoluteSourceFileString(), - layerEntry.getAbsoluteExtractionPathString())); - } - - return new LayerMetadata(entries.build(), lastModifiedTime); - } - - /** The entries that define the layer contents. */ - private ImmutableList entries; - - /** The last time the layer was constructed. */ - private final FileTime lastModifiedTime; - - LayerMetadata(ImmutableList entries, FileTime lastModifiedTime) { - this.entries = entries; - this.lastModifiedTime = lastModifiedTime; - } - - ImmutableList getEntries() { - return entries; - } - - FileTime getLastModifiedTime() { - return lastModifiedTime; - } - - @VisibleForTesting - void setEntries(ImmutableList layerMetadataEntries) { - entries = layerMetadataEntries; - } -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/json/CacheMetadataLayerObjectTemplate.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/json/CacheMetadataLayerObjectTemplate.java deleted file mode 100644 index 3c1b982a3a..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/json/CacheMetadataLayerObjectTemplate.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright 2017 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache.json; - -import com.google.cloud.tools.jib.image.DescriptorDigest; -import com.google.cloud.tools.jib.image.Layer; -import com.google.cloud.tools.jib.json.JsonTemplate; -import javax.annotation.Nullable; - -/** - * Inner JSON template for storing metadata about a layer in the cache as part of {@link - * CacheMetadataTemplate}. - * - * @see CacheMetadataTemplate for example - */ -public class CacheMetadataLayerObjectTemplate implements JsonTemplate { - - /** The reference to the layer. */ - private final ReferenceObject reference = new ReferenceObject(); - - /** Additional properties for the layer. */ - @Nullable private CacheMetadataLayerPropertiesObjectTemplate properties; - - /** - * The reference for a layer consists of its size (in bytes), digest, and diff ID. - * - * @see Layer for details - */ - private static class ReferenceObject implements JsonTemplate { - - private long size; - @Nullable private DescriptorDigest digest; - @Nullable private DescriptorDigest diffId; - } - - public long getSize() { - return reference.size; - } - - @Nullable - public DescriptorDigest getDigest() { - return reference.digest; - } - - @Nullable - public DescriptorDigest getDiffId() { - return reference.diffId; - } - - @Nullable - public CacheMetadataLayerPropertiesObjectTemplate getProperties() { - return properties; - } - - public CacheMetadataLayerObjectTemplate setSize(long size) { - reference.size = size; - return this; - } - - public CacheMetadataLayerObjectTemplate setDigest(DescriptorDigest digest) { - reference.digest = digest; - return this; - } - - public CacheMetadataLayerObjectTemplate setDiffId(DescriptorDigest diffId) { - reference.diffId = diffId; - return this; - } - - public CacheMetadataLayerObjectTemplate setProperties( - CacheMetadataLayerPropertiesObjectTemplate properties) { - this.properties = properties; - return this; - } -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/json/CacheMetadataLayerPropertiesObjectTemplate.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/json/CacheMetadataLayerPropertiesObjectTemplate.java deleted file mode 100644 index f6de5a10a1..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/json/CacheMetadataLayerPropertiesObjectTemplate.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2017 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache.json; - -import com.google.cloud.tools.jib.json.JsonTemplate; -import java.nio.file.attribute.FileTime; -import java.util.Collections; -import java.util.List; -import javax.annotation.Nullable; - -/** - * Inner JSON template for extra properties for an application layer, as part of {@link - * CacheMetadataLayerObjectTemplate}. - */ -public class CacheMetadataLayerPropertiesObjectTemplate implements JsonTemplate { - - /** Represents a pair of source files and extraction path. */ - public static class LayerEntryTemplate implements JsonTemplate { - - /** The path to the source file for this layer entry. */ - @Nullable private String sourceFile; - - /** The intended path to extract the source file to in the container. */ - @Nullable private String extractionPath; - - @Nullable - public String getSourceFileString() { - return sourceFile; - } - - @Nullable - public String getExtractionPathString() { - return extractionPath; - } - - public LayerEntryTemplate(String absoluteSourceFile, String absoluteExtractionPath) { - sourceFile = absoluteSourceFile; - extractionPath = absoluteExtractionPath; - } - - /** For Jackson JSON templating. */ - public LayerEntryTemplate() {} - } - - /** The content entries for the layer. */ - private List layerEntries = Collections.emptyList(); - - /** The last time the layer was constructed. */ - private long lastModifiedTime; - - public List getLayerEntries() { - return layerEntries; - } - - public FileTime getLastModifiedTime() { - return FileTime.fromMillis(lastModifiedTime); - } - - public CacheMetadataLayerPropertiesObjectTemplate setLayerEntries( - List layerEntries) { - this.layerEntries = layerEntries; - return this; - } - - public CacheMetadataLayerPropertiesObjectTemplate setLastModifiedTime(FileTime lastModifiedTime) { - this.lastModifiedTime = lastModifiedTime.toMillis(); - return this; - } -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/json/CacheMetadataTemplate.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/json/CacheMetadataTemplate.java deleted file mode 100644 index 4cee1f87e9..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/json/CacheMetadataTemplate.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright 2017 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache.json; - -import com.google.cloud.tools.jib.json.JsonTemplate; -import java.util.ArrayList; -import java.util.List; - -/** - * JSON template for storing metadata about the cache. - * - *

Example: - * - * 

{@code
- * {
- *   "layers": [
- *     {
- *       // This is a base image layer.
- *       "reference": {
- *         "size": 631,
- *         "digest": "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef",
- *         "diffId": "sha256:b56ae66c29370df48e7377c8f9baa744a3958058a766793f821dadcb144a4647"
- *       }
- *     },
- *     ...
- *     {
- *       // This is an application layer (it has properties).
- *       "reference": {
- *         "size": 223,
- *         "digest": "sha256:8c662931926fa990b41da3c9f42663a537ccd498130030f9149173a0493832ad",
- *         "diffId": "sha256:a3f3e99c29370df48e7377c8f9baa744a3958058a766793f821dadcb144a8372"
- *       }
- *       "properties": {
- *         "layerEntries": [
- *           {
- *             "sourceFile": "build/classes",
- *             "extractionPath": "/app/classes"
- *           },
- *           ...
- *         ],
- *         "lastModifiedTime": 255073580723571
- *       }
- *     },
- *     ...
- *   ]
- * }
- * }
- */ -public class CacheMetadataTemplate implements JsonTemplate { - - private final List layers = new ArrayList<>(); - - public List getLayers() { - return layers; - } - - public CacheMetadataTemplate addLayer(CacheMetadataLayerObjectTemplate layer) { - layers.add(layer); - return this; - } -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/configuration/BuildConfiguration.java b/jib-core/src/main/java/com/google/cloud/tools/jib/configuration/BuildConfiguration.java index a1a3c96310..20cfb6aa3e 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/configuration/BuildConfiguration.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/configuration/BuildConfiguration.java @@ -20,9 +20,12 @@ import com.google.cloud.tools.jib.event.events.LogEvent; import com.google.cloud.tools.jib.image.json.BuildableManifestTemplate; import com.google.cloud.tools.jib.image.json.V22ManifestTemplate; +import com.google.cloud.tools.jib.ncache.Cache; import com.google.cloud.tools.jib.registry.RegistryClient; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -187,8 +190,10 @@ public Builder setEventDispatcher(EventDispatcher eventDispatcher) { * Builds a new {@link BuildConfiguration} using the parameters passed into the builder. * * @return the corresponding build configuration + * @throws IOException if an I/O exception occurs + * @throws CacheDirectoryCreationException if failed to create the configured cache directories */ - public BuildConfiguration build() { + public BuildConfiguration build() throws IOException, CacheDirectoryCreationException { // Validates the parameters. List errorMessages = new ArrayList<>(); if (baseImageConfiguration == null) { @@ -211,13 +216,21 @@ public BuildConfiguration build() { + "' does not use a specific image digest - build may not be reproducible")); } + if (baseImageLayersCacheConfiguration == null) { + baseImageLayersCacheConfiguration = + CacheConfiguration.forDefaultUserLevelCacheDirectory(); + } + if (applicationLayersCacheConfiguration == null) { + applicationLayersCacheConfiguration = CacheConfiguration.makeTemporary(); + } + return new BuildConfiguration( baseImageConfiguration, targetImageConfiguration, additionalTargetImageTags, containerConfiguration, - applicationLayersCacheConfiguration, - baseImageLayersCacheConfiguration, + Cache.withDirectory(baseImageLayersCacheConfiguration.getCacheDirectory()), + Cache.withDirectory(applicationLayersCacheConfiguration.getCacheDirectory()), targetFormat, allowInsecureRegistries, layerConfigurations, @@ -235,6 +248,18 @@ public BuildConfiguration build() { throw new IllegalStateException(); } } + + @Nullable + @VisibleForTesting + CacheConfiguration getBaseImageLayersCacheConfiguration() { + return baseImageLayersCacheConfiguration; + } + + @Nullable + @VisibleForTesting + CacheConfiguration getApplicationLayersCacheConfiguration() { + return applicationLayersCacheConfiguration; + } } /** @@ -250,8 +275,8 @@ public static Builder builder() { private final ImageConfiguration targetImageConfiguration; private final ImmutableSet additionalTargetImageTags; @Nullable private final ContainerConfiguration containerConfiguration; - @Nullable private final CacheConfiguration applicationLayersCacheConfiguration; - @Nullable private final CacheConfiguration baseImageLayersCacheConfiguration; + private final Cache baseImageLayersCache; + private final Cache applicationLayersCache; private Class targetFormat; private final boolean allowInsecureRegistries; private final ImmutableList layerConfigurations; @@ -264,8 +289,8 @@ private BuildConfiguration( ImageConfiguration targetImageConfiguration, ImmutableSet additionalTargetImageTags, @Nullable ContainerConfiguration containerConfiguration, - @Nullable CacheConfiguration applicationLayersCacheConfiguration, - @Nullable CacheConfiguration baseImageLayersCacheConfiguration, + Cache baseImageLayersCache, + Cache applicationLayersCache, Class targetFormat, boolean allowInsecureRegistries, ImmutableList layerConfigurations, @@ -275,8 +300,8 @@ private BuildConfiguration( this.targetImageConfiguration = targetImageConfiguration; this.additionalTargetImageTags = additionalTargetImageTags; this.containerConfiguration = containerConfiguration; - this.applicationLayersCacheConfiguration = applicationLayersCacheConfiguration; - this.baseImageLayersCacheConfiguration = baseImageLayersCacheConfiguration; + this.baseImageLayersCache = baseImageLayersCache; + this.applicationLayersCache = applicationLayersCache; this.targetFormat = targetFormat; this.allowInsecureRegistries = allowInsecureRegistries; this.layerConfigurations = layerConfigurations; @@ -318,23 +343,20 @@ public EventDispatcher getEventDispatcher() { } /** - * Gets the location of the cache for storing application layers. + * Gets the {@link Cache} for base image layers. * - * @return the application layers {@link CacheConfiguration}, or {@code null} if not set + * @return the {@link Cache} for base image layers */ - @Nullable - public CacheConfiguration getApplicationLayersCacheConfiguration() { - return applicationLayersCacheConfiguration; + public Cache getBaseImageLayersCache() { + return baseImageLayersCache; } - /** - * Gets the location of the cache for storing base image layers. + * Gets the {@link Cache} for application layers. * - * @return the base image layers {@link CacheConfiguration}, or {@code null} if not set + * @return the {@link Cache} for application layers */ - @Nullable - public CacheConfiguration getBaseImageLayersCacheConfiguration() { - return baseImageLayersCacheConfiguration; + public Cache getApplicationLayersCache() { + return applicationLayersCache; } /** diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/configuration/CacheConfiguration.java b/jib-core/src/main/java/com/google/cloud/tools/jib/configuration/CacheConfiguration.java index f8250d918c..ca976a270c 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/configuration/CacheConfiguration.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/configuration/CacheConfiguration.java @@ -16,7 +16,6 @@ package com.google.cloud.tools.jib.configuration; -import com.google.cloud.tools.jib.cache.CacheDirectoryCreationException; import com.google.cloud.tools.jib.filesystem.UserCacheHome; import com.google.common.annotations.VisibleForTesting; import java.io.IOException; @@ -77,6 +76,7 @@ public static CacheConfiguration forDefaultUserLevelCacheDirectory() { } private final Path cacheDirectory; + // TODO: Deprecate private final boolean shouldEnsureOwnership; private CacheConfiguration(Path cacheDirectory, boolean shouldEnsureOwnership) { diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheDirectoryCreationException.java b/jib-core/src/main/java/com/google/cloud/tools/jib/configuration/CacheDirectoryCreationException.java similarity index 74% rename from jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheDirectoryCreationException.java rename to jib-core/src/main/java/com/google/cloud/tools/jib/configuration/CacheDirectoryCreationException.java index f15ae06189..cd103021b4 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheDirectoryCreationException.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/configuration/CacheDirectoryCreationException.java @@ -14,19 +14,13 @@ * the License. */ -package com.google.cloud.tools.jib.cache; +package com.google.cloud.tools.jib.configuration; -import java.nio.file.Path; - -/** Thrown when a directory to be used as the {@link Cache} could not be created. */ +/** Thrown when a directory to be used as the cache could not be created. */ public class CacheDirectoryCreationException extends Exception { private static final String MESSAGE = "Could not create cache directory"; - CacheDirectoryCreationException(Path cacheDirectory, Throwable cause) { - super(MESSAGE + ": " + cacheDirectory, cause); - } - public CacheDirectoryCreationException(Throwable cause) { super(MESSAGE, cause); } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageToTarballTranslator.java b/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageToTarballTranslator.java index c5e407dd82..edc6083fe7 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageToTarballTranslator.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageToTarballTranslator.java @@ -18,34 +18,36 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.Blobs; -import com.google.cloud.tools.jib.cache.CachedLayer; import com.google.cloud.tools.jib.docker.json.DockerLoadManifestEntryTemplate; import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.ImageReference; +import com.google.cloud.tools.jib.image.Layer; import com.google.cloud.tools.jib.image.json.ImageToJsonTranslator; import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.cloud.tools.jib.tar.TarStreamBuilder; import java.io.IOException; -import java.nio.file.Path; import java.util.Collections; -import org.apache.commons.compress.archivers.tar.TarArchiveEntry; /** Translates an {@link Image} to a tarball that can be loaded into Docker. */ public class ImageToTarballTranslator { /** File name for the container configuration in the tarball. */ private static final String CONTAINER_CONFIGURATION_JSON_FILE_NAME = "config.json"; + /** File name for the manifest in the tarball. */ private static final String MANIFEST_JSON_FILE_NAME = "manifest.json"; - private final Image image; + /** File name extension for the layer content files. */ + private static final String LAYER_FILE_EXTENSION = ".tar.gz"; + + private final Image image; /** * Instantiate with an {@link Image}. * * @param image the image to convert into a tarball. */ - public ImageToTarballTranslator(Image image) { + public ImageToTarballTranslator(Image image) { this.image = image; } @@ -54,12 +56,11 @@ public Blob toTarballBlob(ImageReference imageReference) throws IOException { DockerLoadManifestEntryTemplate manifestTemplate = new DockerLoadManifestEntryTemplate(); // Adds all the layers to the tarball and manifest. - for (CachedLayer layer : image.getLayers()) { - Path layerContentFile = layer.getContentFile(); - String layerName = layerContentFile.getFileName().toString(); + for (Layer layer : image.getLayers()) { + String layerName = layer.getBlobDescriptor().getDigest().getHash() + LAYER_FILE_EXTENSION; - tarStreamBuilder.addTarArchiveEntry( - new TarArchiveEntry(layerContentFile.toFile(), layerName)); + tarStreamBuilder.addBlobEntry( + layer.getBlob(), layer.getBlobDescriptor().getSize(), layerName); manifestTemplate.addLayerFile(layerName); } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/ReproducibleLayerBuilder.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/ReproducibleLayerBuilder.java index 505228f739..f30db1aaff 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/image/ReproducibleLayerBuilder.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/image/ReproducibleLayerBuilder.java @@ -16,6 +16,7 @@ package com.google.cloud.tools.jib.image; +import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.tar.TarStreamBuilder; import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; @@ -30,9 +31,9 @@ import org.apache.commons.compress.archivers.tar.TarArchiveEntry; /** - * Builds a reproducible {@link UnwrittenLayer} from files. The reproducibility is implemented by - * strips out all non-reproducible elements (modification time, group ID, user ID, user name, and - * group name) from name-sorted tar archive entries. + * Builds a reproducible layer {@link Blob} from files. The reproducibility is implemented by strips + * out all non-reproducible elements (modification time, group ID, user ID, user name, and group + * name) from name-sorted tar archive entries. */ public class ReproducibleLayerBuilder { @@ -88,11 +89,11 @@ public ReproducibleLayerBuilder(ImmutableList layerEntries) { } /** - * Builds and returns the layer. + * Builds and returns the layer {@link Blob}. * * @return the new layer */ - public UnwrittenLayer build() { + public Blob build() { UniqueTarArchiveEntries uniqueTarArchiveEntries = new UniqueTarArchiveEntries(); // Adds all the layer entries as tar entries. @@ -125,10 +126,6 @@ public UnwrittenLayer build() { tarStreamBuilder.addTarArchiveEntry(entry); } - return new UnwrittenLayer(tarStreamBuilder.toBlob()); - } - - public ImmutableList getLayerEntries() { - return layerEntries; + return tarStreamBuilder.toBlob(); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/UnwrittenLayer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/UnwrittenLayer.java deleted file mode 100644 index 126d4c9bf5..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/image/UnwrittenLayer.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.image; - -import com.google.cloud.tools.jib.blob.Blob; -import com.google.cloud.tools.jib.blob.BlobDescriptor; -import com.google.cloud.tools.jib.cache.CachedLayer; - -/** - * A layer that has not been written out and only has the unwritten content {@link Blob}. Once - * written, this layer becomes a {@link CachedLayer}. - */ -public class UnwrittenLayer implements Layer { - - private final Blob uncompressedBlob; - - /** - * Initializes with the uncompressed {@link Blob} of the layer content. - * - * @param uncompressedBlob the uncompressed {@link Blob} of the layer content - */ - public UnwrittenLayer(Blob uncompressedBlob) { - this.uncompressedBlob = uncompressedBlob; - } - - /** Gets the uncompressed layer content BLOB. */ - @Override - public Blob getBlob() { - return uncompressedBlob; - } - - @Override - public BlobDescriptor getBlobDescriptor() throws LayerPropertyNotFoundException { - throw new LayerPropertyNotFoundException("Blob descriptor not available for unwritten layer"); - } - - @Override - public DescriptorDigest getDiffId() throws LayerPropertyNotFoundException { - throw new LayerPropertyNotFoundException("Diff ID not available for unwritten layer"); - } -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java index 88ca26f703..bf8e4987d5 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java @@ -18,10 +18,10 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; -import com.google.cloud.tools.jib.cache.CachedLayer; import com.google.cloud.tools.jib.configuration.Port; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.Image; +import com.google.cloud.tools.jib.image.Layer; import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -90,14 +90,14 @@ static ImmutableList environmentMapToList(@Nullable Map .collect(ImmutableList.toImmutableList()); } - private final Image image; + private final Image image; /** * Instantiate with an {@link Image}. * * @param image the image to translate. */ - public ImageToJsonTranslator(Image image) { + public ImageToJsonTranslator(Image image) { this.image = image; } @@ -111,7 +111,7 @@ public Blob getContainerConfigurationBlob() { ContainerConfigurationTemplate template = new ContainerConfigurationTemplate(); // Adds the layer diff IDs. - for (CachedLayer layer : image.getLayers()) { + for (Layer layer : image.getLayers()) { template.addLayerDiffId(layer.getDiffId()); } @@ -168,7 +168,7 @@ public T getManifestTemplate( template.setContainerConfiguration(containerConfigurationSize, containerConfigurationDigest); // Adds the layers. - for (CachedLayer layer : image.getLayers()) { + for (Layer layer : image.getLayers()) { template.addLayer( layer.getBlobDescriptor().getSize(), layer.getBlobDescriptor().getDigest()); } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryClient.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryClient.java index acb5939bf0..6ed8f62f6f 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryClient.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryClient.java @@ -18,6 +18,7 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; +import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; import com.google.cloud.tools.jib.event.EventDispatcher; import com.google.cloud.tools.jib.global.JibSystemProperties; @@ -30,7 +31,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import java.io.IOException; -import java.io.OutputStream; import java.net.URL; import javax.annotation.Nullable; @@ -238,20 +238,24 @@ public BlobDescriptor checkBlob(DescriptorDigest blobDigest) } /** - * Downloads the BLOB to a file. + * Gets the BLOB referenced by {@code blobDigest}. Note that the BLOB is only pulled when it is + * written out. * * @param blobDigest the digest of the BLOB to download - * @param destinationOutputStream the {@link OutputStream} to write the BLOB to * @return a {@link Blob} backed by the file at {@code destPath}. The file at {@code destPath} * must exist for {@link Blob} to be valid. - * @throws IOException if communicating with the endpoint fails - * @throws RegistryException if communicating with the endpoint fails */ - public Void pullBlob(DescriptorDigest blobDigest, OutputStream destinationOutputStream) - throws RegistryException, IOException { - BlobPuller blobPuller = - new BlobPuller(registryEndpointRequestProperties, blobDigest, destinationOutputStream); - return callRegistryEndpoint(blobPuller); + public Blob pullBlob(DescriptorDigest blobDigest) { + return Blobs.from( + outputStream -> { + try { + callRegistryEndpoint( + new BlobPuller(registryEndpointRequestProperties, blobDigest, outputStream)); + + } catch (RegistryException ex) { + throw new IOException(ex); + } + }); } /** diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/tar/TarStreamBuilder.java b/jib-core/src/main/java/com/google/cloud/tools/jib/tar/TarStreamBuilder.java index 832bd04de5..8e38af8bb8 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/tar/TarStreamBuilder.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/tar/TarStreamBuilder.java @@ -77,6 +77,20 @@ public void addByteEntry(byte[] contents, String name) { archiveMap.put(entry, Blobs.from(outputStream -> outputStream.write(contents))); } + /** + * Adds a blob to the archive. Note that this should be used with non-file {@link Blob}s; for + * adding files to the archive, use {@link #addTarArchiveEntry}. + * + * @param blob the {@link Blob} to add to the tarball + * @param size the size (in bytes) of {@code blob} + * @param name the name of the entry (i.e. filename) + */ + public void addBlobEntry(Blob blob, long size, String name) { + TarArchiveEntry entry = new TarArchiveEntry(name); + entry.setSize(size); + archiveMap.put(entry, blob); + } + /** @return a new {@link Blob} that can stream the uncompressed tarball archive BLOB. */ public Blob toBlob() { return Blobs.from(this::writeEntriesAsTarArchive); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/api/JibContainerBuilderTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/api/JibContainerBuilderTest.java index f6f5554bf1..c087c71d07 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/api/JibContainerBuilderTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/api/JibContainerBuilderTest.java @@ -17,6 +17,7 @@ package com.google.cloud.tools.jib.api; import com.google.cloud.tools.jib.configuration.BuildConfiguration; +import com.google.cloud.tools.jib.configuration.CacheDirectoryCreationException; import com.google.cloud.tools.jib.configuration.ContainerConfiguration; import com.google.cloud.tools.jib.configuration.LayerConfiguration; import com.google.cloud.tools.jib.configuration.Port; @@ -26,6 +27,7 @@ import com.google.cloud.tools.jib.image.InvalidImageReferenceException; import com.google.cloud.tools.jib.registry.credentials.CredentialRetrievalException; import com.google.common.collect.ImmutableMap; +import java.io.IOException; import java.util.Arrays; import org.junit.Assert; import org.junit.Test; @@ -91,7 +93,8 @@ public void testToContainerConfiguration_add() { @Test public void testToBuildConfiguration() - throws InvalidImageReferenceException, CredentialRetrievalException { + throws InvalidImageReferenceException, CredentialRetrievalException, IOException, + CacheDirectoryCreationException { RegistryImage baseImage = RegistryImage.named("base/image").addCredentialRetriever(mockCredentialRetriever); RegistryImage targetImage = diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStepTest.java index bf98ec892b..9adf7b0615 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStepTest.java @@ -17,17 +17,19 @@ package com.google.cloud.tools.jib.builder.steps; import com.google.cloud.tools.jib.async.NonBlockingSteps; -import com.google.cloud.tools.jib.cache.Cache; -import com.google.cloud.tools.jib.cache.CacheMetadataCorruptedException; -import com.google.cloud.tools.jib.cache.CacheReader; -import com.google.cloud.tools.jib.cache.CachedLayerWithMetadata; +import com.google.cloud.tools.jib.blob.Blob; +import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.configuration.LayerConfiguration; import com.google.cloud.tools.jib.event.EventDispatcher; import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; import com.google.cloud.tools.jib.image.ImageLayers; +import com.google.cloud.tools.jib.image.Layer; import com.google.cloud.tools.jib.image.LayerEntry; import com.google.cloud.tools.jib.image.LayerPropertyNotFoundException; +import com.google.cloud.tools.jib.ncache.Cache; +import com.google.cloud.tools.jib.ncache.CacheCorruptedException; +import com.google.cloud.tools.jib.ncache.CacheEntry; import com.google.common.collect.ImmutableList; import com.google.common.io.Resources; import com.google.common.util.concurrent.MoreExecutors; @@ -76,11 +78,16 @@ private static LayerConfiguration makeLayerConfiguration( } } + private static void assertBlobsEqual(Blob expectedBlob, Blob blob) throws IOException { + Assert.assertArrayEquals(Blobs.writeToByteArray(expectedBlob), Blobs.writeToByteArray(blob)); + } + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @Mock private BuildConfiguration mockBuildConfiguration; + + private Cache cache; @Mock private EventDispatcher mockEventDispatcher; - private Path temporaryCacheDirectory; private LayerConfiguration fakeDependenciesLayerConfiguration; private LayerConfiguration fakeSnapshotDependenciesLayerConfiguration; @@ -110,35 +117,34 @@ public void setUp() throws IOException, URISyntaxException { EXTRA_FILES_LAYER_EXTRACTION_PATH.resolve("fileB")) .build(); emptyLayerConfiguration = LayerConfiguration.builder().build(); + + cache = Cache.withDirectory(temporaryFolder.newFolder().toPath()); + Mockito.when(mockBuildConfiguration.getEventDispatcher()).thenReturn(mockEventDispatcher); - temporaryCacheDirectory = temporaryFolder.newFolder().toPath(); + Mockito.when(mockBuildConfiguration.getApplicationLayersCache()).thenReturn(cache); } - private ImageLayers buildFakeLayersToCache() - throws CacheMetadataCorruptedException, IOException, ExecutionException { - ImageLayers.Builder applicationLayersBuilder = ImageLayers.builder(); - ImageLayers applicationLayers; + private ImageLayers buildFakeLayersToCache() throws ExecutionException { + ImageLayers.Builder applicationLayersBuilder = ImageLayers.builder(); - try (Cache cache = Cache.init(temporaryCacheDirectory)) { - ImmutableList buildAndCacheApplicationLayerSteps = - BuildAndCacheApplicationLayerStep.makeList( - MoreExecutors.newDirectExecutorService(), mockBuildConfiguration, cache); + ImmutableList buildAndCacheApplicationLayerSteps = + BuildAndCacheApplicationLayerStep.makeList( + MoreExecutors.newDirectExecutorService(), mockBuildConfiguration); - for (BuildAndCacheApplicationLayerStep buildAndCacheApplicationLayerStep : - buildAndCacheApplicationLayerSteps) { - applicationLayersBuilder.add(NonBlockingSteps.get(buildAndCacheApplicationLayerStep)); - } - - applicationLayers = applicationLayersBuilder.build(); - cache.addCachedLayersWithMetadataToMetadata(applicationLayers.getLayers()); + for (BuildAndCacheApplicationLayerStep buildAndCacheApplicationLayerStep : + buildAndCacheApplicationLayerSteps) { + applicationLayersBuilder.add( + BuildImageStep.cacheEntryToLayer( + NonBlockingSteps.get(buildAndCacheApplicationLayerStep))); } - return applicationLayers; + + return applicationLayersBuilder.build(); } @Test public void testRun() - throws LayerPropertyNotFoundException, IOException, CacheMetadataCorruptedException, - ExecutionException { + throws LayerPropertyNotFoundException, IOException, ExecutionException, + CacheCorruptedException { ImmutableList fakeLayerConfigurations = ImmutableList.of( fakeDependenciesLayerConfiguration, @@ -150,12 +156,9 @@ public void testRun() .thenReturn(fakeLayerConfigurations); // Populates the cache. - ImageLayers applicationLayers = buildFakeLayersToCache(); + ImageLayers applicationLayers = buildFakeLayersToCache(); Assert.assertEquals(5, applicationLayers.size()); - // Re-initialize cache with the updated metadata. - Cache cache = Cache.init(temporaryCacheDirectory); - ImmutableList dependenciesLayerEntries = fakeLayerConfigurations.get(0).getLayerEntries(); ImmutableList snapshotDependenciesLayerEntries = @@ -167,58 +170,46 @@ public void testRun() ImmutableList extraFilesLayerEntries = fakeLayerConfigurations.get(4).getLayerEntries(); + CacheEntry dependenciesCacheEntry = + cache.retrieve(dependenciesLayerEntries).orElseThrow(AssertionError::new); + CacheEntry snapshotDependenciesCacheEntry = + cache.retrieve(snapshotDependenciesLayerEntries).orElseThrow(AssertionError::new); + CacheEntry resourcesCacheEntry = + cache.retrieve(resourcesLayerEntries).orElseThrow(AssertionError::new); + CacheEntry classesCacheEntry = + cache.retrieve(classesLayerEntries).orElseThrow(AssertionError::new); + CacheEntry extraFilesCacheEntry = + cache.retrieve(extraFilesLayerEntries).orElseThrow(AssertionError::new); + // Verifies that the cached layers are up-to-date. - CacheReader cacheReader = new CacheReader(cache); Assert.assertEquals( - applicationLayers.get(0).getBlobDescriptor(), - cacheReader - .getUpToDateLayerByLayerEntries(dependenciesLayerEntries) - .orElseThrow(AssertionError::new) - .getBlobDescriptor()); + applicationLayers.get(0).getBlobDescriptor().getDigest(), + dependenciesCacheEntry.getLayerDigest()); Assert.assertEquals( - applicationLayers.get(1).getBlobDescriptor(), - cacheReader - .getUpToDateLayerByLayerEntries(snapshotDependenciesLayerEntries) - .orElseThrow(AssertionError::new) - .getBlobDescriptor()); + applicationLayers.get(1).getBlobDescriptor().getDigest(), + snapshotDependenciesCacheEntry.getLayerDigest()); Assert.assertEquals( - applicationLayers.get(2).getBlobDescriptor(), - cacheReader - .getUpToDateLayerByLayerEntries(resourcesLayerEntries) - .orElseThrow(AssertionError::new) - .getBlobDescriptor()); + applicationLayers.get(2).getBlobDescriptor().getDigest(), + resourcesCacheEntry.getLayerDigest()); Assert.assertEquals( - applicationLayers.get(3).getBlobDescriptor(), - cacheReader - .getUpToDateLayerByLayerEntries(classesLayerEntries) - .orElseThrow(AssertionError::new) - .getBlobDescriptor()); + applicationLayers.get(3).getBlobDescriptor().getDigest(), + classesCacheEntry.getLayerDigest()); Assert.assertEquals( - applicationLayers.get(4).getBlobDescriptor(), - cacheReader - .getUpToDateLayerByLayerEntries(extraFilesLayerEntries) - .orElseThrow(AssertionError::new) - .getBlobDescriptor()); + applicationLayers.get(4).getBlobDescriptor().getDigest(), + extraFilesCacheEntry.getLayerDigest()); // Verifies that the cache reader gets the same layers as the newest application layers. - Assert.assertEquals( - applicationLayers.get(0).getContentFile(), - cacheReader.getLayerFile(dependenciesLayerEntries)); - Assert.assertEquals( - applicationLayers.get(1).getContentFile(), - cacheReader.getLayerFile(snapshotDependenciesLayerEntries)); - Assert.assertEquals( - applicationLayers.get(2).getContentFile(), cacheReader.getLayerFile(resourcesLayerEntries)); - Assert.assertEquals( - applicationLayers.get(3).getContentFile(), cacheReader.getLayerFile(classesLayerEntries)); - Assert.assertEquals( - applicationLayers.get(4).getContentFile(), - cacheReader.getLayerFile(extraFilesLayerEntries)); + assertBlobsEqual(applicationLayers.get(0).getBlob(), dependenciesCacheEntry.getLayerBlob()); + assertBlobsEqual( + applicationLayers.get(1).getBlob(), snapshotDependenciesCacheEntry.getLayerBlob()); + assertBlobsEqual(applicationLayers.get(2).getBlob(), resourcesCacheEntry.getLayerBlob()); + assertBlobsEqual(applicationLayers.get(3).getBlob(), classesCacheEntry.getLayerBlob()); + assertBlobsEqual(applicationLayers.get(4).getBlob(), extraFilesCacheEntry.getLayerBlob()); } @Test public void testRun_emptyLayersIgnored() - throws IOException, CacheMetadataCorruptedException, ExecutionException { + throws IOException, ExecutionException, CacheCorruptedException { ImmutableList fakeLayerConfigurations = ImmutableList.of( fakeDependenciesLayerConfiguration, @@ -230,7 +221,7 @@ public void testRun_emptyLayersIgnored() .thenReturn(fakeLayerConfigurations); // Populates the cache. - ImageLayers applicationLayers = buildFakeLayersToCache(); + ImageLayers applicationLayers = buildFakeLayersToCache(); Assert.assertEquals(3, applicationLayers.size()); ImmutableList dependenciesLayerEntries = @@ -240,39 +231,27 @@ public void testRun_emptyLayersIgnored() ImmutableList classesLayerEntries = fakeLayerConfigurations.get(3).getLayerEntries(); - // Re-initialize cache with the updated metadata. - Cache cache = Cache.init(temporaryCacheDirectory); + CacheEntry dependenciesCacheEntry = + cache.retrieve(dependenciesLayerEntries).orElseThrow(AssertionError::new); + CacheEntry resourcesCacheEntry = + cache.retrieve(resourcesLayerEntries).orElseThrow(AssertionError::new); + CacheEntry classesCacheEntry = + cache.retrieve(classesLayerEntries).orElseThrow(AssertionError::new); // Verifies that the cached layers are up-to-date. - CacheReader cacheReader = new CacheReader(cache); Assert.assertEquals( - applicationLayers.get(0).getBlobDescriptor(), - cacheReader - .getUpToDateLayerByLayerEntries(dependenciesLayerEntries) - .orElseThrow(AssertionError::new) - .getBlobDescriptor()); + applicationLayers.get(0).getBlobDescriptor().getDigest(), + dependenciesCacheEntry.getLayerDigest()); Assert.assertEquals( - applicationLayers.get(1).getBlobDescriptor(), - cacheReader - .getUpToDateLayerByLayerEntries(resourcesLayerEntries) - .orElseThrow(AssertionError::new) - .getBlobDescriptor()); + applicationLayers.get(1).getBlobDescriptor().getDigest(), + resourcesCacheEntry.getLayerDigest()); Assert.assertEquals( - applicationLayers.get(2).getBlobDescriptor(), - cacheReader - .getUpToDateLayerByLayerEntries(classesLayerEntries) - .orElseThrow(AssertionError::new) - .getBlobDescriptor()); + applicationLayers.get(2).getBlobDescriptor().getDigest(), + classesCacheEntry.getLayerDigest()); // Verifies that the cache reader gets the same layers as the newest application layers. - Assert.assertEquals( - applicationLayers.get(0).getContentFile(), - cacheReader.getLayerFile(fakeLayerConfigurations.get(0).getLayerEntries())); - Assert.assertEquals( - applicationLayers.get(1).getContentFile(), - cacheReader.getLayerFile(fakeLayerConfigurations.get(2).getLayerEntries())); - Assert.assertEquals( - applicationLayers.get(2).getContentFile(), - cacheReader.getLayerFile(fakeLayerConfigurations.get(3).getLayerEntries())); + assertBlobsEqual(applicationLayers.get(0).getBlob(), dependenciesCacheEntry.getLayerBlob()); + assertBlobsEqual(applicationLayers.get(1).getBlob(), resourcesCacheEntry.getLayerBlob()); + assertBlobsEqual(applicationLayers.get(2).getBlob(), classesCacheEntry.getLayerBlob()); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java index c0a400af8c..e23a135206 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java @@ -16,9 +16,8 @@ package com.google.cloud.tools.jib.builder.steps; -import com.google.cloud.tools.jib.blob.BlobDescriptor; -import com.google.cloud.tools.jib.cache.CachedLayer; -import com.google.cloud.tools.jib.cache.CachedLayerWithMetadata; +import com.google.cloud.tools.jib.blob.Blob; +import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.configuration.ContainerConfiguration; import com.google.cloud.tools.jib.event.EventDispatcher; @@ -26,13 +25,14 @@ import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.Layer; import com.google.cloud.tools.jib.image.json.HistoryEntry; +import com.google.cloud.tools.jib.ncache.CacheEntry; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.MoreExecutors; -import java.nio.file.Paths; import java.security.DigestException; import java.time.Instant; +import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import org.junit.Assert; @@ -64,11 +64,33 @@ public void setUp() throws DigestException { testDescriptorDigest = DescriptorDigest.fromHash( "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); - CachedLayerWithMetadata testCachedLayer = - new CachedLayerWithMetadata( - new CachedLayer( - Paths.get(""), new BlobDescriptor(testDescriptorDigest), testDescriptorDigest), - null); + CacheEntry testCacheEntry = + new CacheEntry() { + @Override + public DescriptorDigest getLayerDigest() { + return testDescriptorDigest; + } + + @Override + public DescriptorDigest getLayerDiffId() { + return testDescriptorDigest; + } + + @Override + public long getLayerSize() { + return 0; + } + + @Override + public Blob getLayerBlob() { + return Blobs.from("ignored"); + } + + @Override + public Optional getMetadataBlob() { + return Optional.empty(); + } + }; Mockito.when(mockBuildConfiguration.getEventDispatcher()).thenReturn(mockEventDispatcher); Mockito.when(mockBuildConfiguration.getContainerConfiguration()) @@ -104,7 +126,7 @@ public void setUp() throws DigestException { .addHistory(emptyLayerHistory) .build(); Mockito.when(mockPullAndCacheBaseImageLayerStep.getFuture()) - .thenReturn(Futures.immediateFuture(testCachedLayer)); + .thenReturn(Futures.immediateFuture(testCacheEntry)); Mockito.when(mockPullAndCacheBaseImageLayersStep.getFuture()) .thenReturn( Futures.immediateFuture( @@ -117,7 +139,7 @@ public void setUp() throws DigestException { Futures.immediateFuture( new PullBaseImageStep.BaseImageWithAuthorization(baseImage, null))); Mockito.when(mockBuildAndCacheApplicationLayerStep.getFuture()) - .thenReturn(Futures.immediateFuture(testCachedLayer)); + .thenReturn(Futures.immediateFuture(testCacheEntry)); } @Test @@ -132,7 +154,7 @@ public void test_validateAsyncDependencies() throws ExecutionException, Interrup mockBuildAndCacheApplicationLayerStep, mockBuildAndCacheApplicationLayerStep, mockBuildAndCacheApplicationLayerStep)); - Image image = buildImageStep.getFuture().get().getFuture().get(); + Image image = buildImageStep.getFuture().get().getFuture().get(); Assert.assertEquals( testDescriptorDigest, image.getLayers().asList().get(0).getBlobDescriptor().getDigest()); } @@ -154,7 +176,7 @@ public void test_propagateBaseImageConfiguration() mockBuildAndCacheApplicationLayerStep, mockBuildAndCacheApplicationLayerStep, mockBuildAndCacheApplicationLayerStep)); - Image image = buildImageStep.getFuture().get().getFuture().get(); + Image image = buildImageStep.getFuture().get().getFuture().get(); Assert.assertEquals( ImmutableMap.of("BASE_ENV", "BASE_ENV_VALUE", "MY_ENV", "MY_ENV_VALUE"), image.getEnvironment()); @@ -180,7 +202,7 @@ public void test_generateHistoryObjects() throws ExecutionException, Interrupted mockBuildAndCacheApplicationLayerStep, mockBuildAndCacheApplicationLayerStep, mockBuildAndCacheApplicationLayerStep)); - Image image = buildImageStep.getFuture().get().getFuture().get(); + Image image = buildImageStep.getFuture().get().getFuture().get(); // Make sure history is as expected HistoryEntry expectedAddedBaseLayerHistory = diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/RetrieveRegistryCredentialsStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/RetrieveRegistryCredentialsStepTest.java index a8c652c62a..f076be98ca 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/RetrieveRegistryCredentialsStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/RetrieveRegistryCredentialsStepTest.java @@ -17,6 +17,7 @@ package com.google.cloud.tools.jib.builder.steps; import com.google.cloud.tools.jib.configuration.BuildConfiguration; +import com.google.cloud.tools.jib.configuration.CacheDirectoryCreationException; import com.google.cloud.tools.jib.configuration.ImageConfiguration; import com.google.cloud.tools.jib.configuration.credentials.Credential; import com.google.cloud.tools.jib.configuration.credentials.CredentialRetriever; @@ -25,6 +26,7 @@ import com.google.cloud.tools.jib.image.ImageReference; import com.google.cloud.tools.jib.registry.credentials.CredentialRetrievalException; import com.google.common.util.concurrent.ListeningExecutorService; +import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -44,7 +46,8 @@ public class RetrieveRegistryCredentialsStepTest { @Mock private ListeningExecutorService mockListeningExecutorService; @Test - public void testCall_retrieved() throws CredentialRetrievalException { + public void testCall_retrieved() + throws CredentialRetrievalException, IOException, CacheDirectoryCreationException { BuildConfiguration buildConfiguration = makeFakeBuildConfiguration( Arrays.asList( @@ -67,7 +70,8 @@ public void testCall_retrieved() throws CredentialRetrievalException { } @Test - public void testCall_none() throws CredentialRetrievalException { + public void testCall_none() + throws CredentialRetrievalException, IOException, CacheDirectoryCreationException { BuildConfiguration buildConfiguration = makeFakeBuildConfiguration( Arrays.asList(Optional::empty, Optional::empty), Collections.emptyList()); @@ -89,7 +93,7 @@ public void testCall_none() throws CredentialRetrievalException { } @Test - public void testCall_exception() { + public void testCall_exception() throws IOException, CacheDirectoryCreationException { CredentialRetrievalException credentialRetrievalException = Mockito.mock(CredentialRetrievalException.class); BuildConfiguration buildConfiguration = @@ -111,7 +115,8 @@ public void testCall_exception() { private BuildConfiguration makeFakeBuildConfiguration( List baseCredentialRetrievers, - List targetCredentialRetrievers) { + List targetCredentialRetrievers) + throws IOException, CacheDirectoryCreationException { ImageReference baseImage = ImageReference.of("baseregistry", "ignored", null); ImageReference targetImage = ImageReference.of("targetregistry", "ignored", null); return BuildConfiguration.builder() diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheFilesTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheFilesTest.java deleted file mode 100644 index 3df1f2ce55..0000000000 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheFilesTest.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2017 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache; - -import com.google.cloud.tools.jib.image.DescriptorDigest; -import java.nio.file.Path; -import java.security.DigestException; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.ArgumentCaptor; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; - -/** Tests for {@link CacheFiles}. */ -@RunWith(MockitoJUnitRunner.class) -public class CacheFilesTest { - - @Mock private Path mockPath; - - @Test - public void testGetMetadataFile() { - ArgumentCaptor fileNameCaptor = ArgumentCaptor.forClass(String.class); - - Mockito.when(mockPath.resolve(fileNameCaptor.capture())).thenReturn(mockPath); - - Path metadataFile = CacheFiles.getMetadataFile(mockPath); - - Assert.assertEquals("metadata-v3.json", fileNameCaptor.getValue()); - Assert.assertEquals(mockPath, metadataFile); - } - - @Test - public void testGetLayerFile() throws DigestException { - DescriptorDigest layerDigest = - DescriptorDigest.fromDigest( - "sha256:8c662931926fa990b41da3c9f42663a537ccd498130030f9149173a0493832ad"); - - ArgumentCaptor fileNameCaptor = ArgumentCaptor.forClass(String.class); - - Mockito.when(mockPath.resolve(fileNameCaptor.capture())).thenReturn(mockPath); - - Path layerFile = CacheFiles.getLayerFile(mockPath, layerDigest); - - Assert.assertEquals( - "8c662931926fa990b41da3c9f42663a537ccd498130030f9149173a0493832ad.tar.gz", - fileNameCaptor.getValue()); - Assert.assertEquals(mockPath, layerFile); - } -} diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheMetadataTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheMetadataTest.java deleted file mode 100644 index f93292bb8c..0000000000 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheMetadataTest.java +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Copyright 2017 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache; - -import com.google.cloud.tools.jib.blob.BlobDescriptor; -import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; -import com.google.cloud.tools.jib.image.DescriptorDigest; -import com.google.cloud.tools.jib.image.ImageLayers; -import com.google.cloud.tools.jib.image.LayerEntry; -import com.google.common.collect.ImmutableList; -import java.nio.file.Paths; -import java.nio.file.attribute.FileTime; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; - -/** Tests for {@link CacheMetadata}. */ -@RunWith(MockitoJUnitRunner.class) -public class CacheMetadataTest { - - private static CachedLayer mockCachedLayer() { - CachedLayer mockCachedLayer = Mockito.mock(CachedLayer.class); - BlobDescriptor mockBlobDescriptor = Mockito.mock(BlobDescriptor.class); - DescriptorDigest mockDescriptorDigest = Mockito.mock(DescriptorDigest.class); - Mockito.when(mockCachedLayer.getBlobDescriptor()).thenReturn(mockBlobDescriptor); - Mockito.when(mockBlobDescriptor.getDigest()).thenReturn(mockDescriptorDigest); - return mockCachedLayer; - } - - @Mock private LayerEntry mockLayerEntry; - - @Test - public void testAddLayer() { - CachedLayerWithMetadata testCachedLayerWithMetadata = - new CachedLayerWithMetadata(mockCachedLayer(), Mockito.mock(LayerMetadata.class)); - - CacheMetadata cacheMetadata = - CacheMetadata.builder().addLayer(testCachedLayerWithMetadata).build(); - - Assert.assertEquals( - Collections.singletonList(testCachedLayerWithMetadata), - cacheMetadata.getLayers().getLayers()); - } - - @Test - public void testFilter_doLayerEntriesMatchMetadataEntries_mismatchSize() { - Assert.assertFalse( - CacheMetadata.LayerFilter.doLayerEntriesMatchMetadataEntries( - ImmutableList.of(mockLayerEntry), ImmutableList.of())); - } - - @Test - public void testFilter_doLayerEntriesMatchMetadataEntries_extractionPath() { - ImmutableList metadataEntries = - LayerMetadata.from( - ImmutableList.of( - new LayerEntry( - Paths.get("anotherSourceFile"), - AbsoluteUnixPath.get("/another/extraction/path"))), - FileTime.fromMillis(0)) - .getEntries(); - - Assert.assertFalse( - CacheMetadata.LayerFilter.doLayerEntriesMatchMetadataEntries( - ImmutableList.of( - new LayerEntry(Paths.get("sourceFile"), AbsoluteUnixPath.get("/extraction/path"))), - metadataEntries)); - Assert.assertTrue( - CacheMetadata.LayerFilter.doLayerEntriesMatchMetadataEntries( - ImmutableList.of( - new LayerEntry( - Paths.get("anotherSourceFile"), - AbsoluteUnixPath.get("/another/extraction/path"))), - metadataEntries)); - } - - @Test - public void testFilter_doLayerEntriesMatchMetadataEntries_pass() { - LayerEntry layerEntry1 = - new LayerEntry(Paths.get("sourceFile1"), AbsoluteUnixPath.get("/extraction/path")); - LayerEntry layerEntry2 = - new LayerEntry(Paths.get("sourceFile2"), AbsoluteUnixPath.get("/extraction/path")); - LayerEntry layerEntry3 = - new LayerEntry(Paths.get("sourceFile3"), AbsoluteUnixPath.get("/another/extraction/path")); - LayerEntry layerEntry4 = - new LayerEntry(Paths.get("sourceFile4"), AbsoluteUnixPath.get("/another/extraction/path")); - - ImmutableList layerEntries = - ImmutableList.of(layerEntry1, layerEntry2, layerEntry3, layerEntry4); - ImmutableList metadataEntries = - LayerMetadata.from( - ImmutableList.of(layerEntry1, layerEntry2, layerEntry3, layerEntry4), - FileTime.fromMillis(0)) - .getEntries(); - - Assert.assertTrue( - CacheMetadata.LayerFilter.doLayerEntriesMatchMetadataEntries( - layerEntries, metadataEntries)); - } - - @Test - public void testFilter_bySourceFiles() throws CacheMetadataCorruptedException { - List mockLayers = - Stream.generate(CacheMetadataTest::mockCachedLayer).limit(6).collect(Collectors.toList()); - - LayerEntry fakeLayerEntry1 = - new LayerEntry(Paths.get("some/source/file"), AbsoluteUnixPath.get("/extraction/path")); - LayerEntry fakeLayerEntry2 = - new LayerEntry( - Paths.get("some/source/directory"), AbsoluteUnixPath.get("/extraction/path")); - - LayerMetadata fakeExpectedSourceFilesClassesLayerMetadata = - LayerMetadata.from( - ImmutableList.of(fakeLayerEntry1, fakeLayerEntry2), FileTime.fromMillis(0)); - LayerMetadata fakeExpectedSourceFilesResourcesLayerMetadata = - LayerMetadata.from( - ImmutableList.of(fakeLayerEntry1, fakeLayerEntry2), FileTime.fromMillis(0)); - LayerMetadata fakeOtherSourceFilesLayerMetadata = - LayerMetadata.from( - ImmutableList.of( - new LayerEntry( - Paths.get("not/the/same/source/file"), - AbsoluteUnixPath.get("/extraction/path"))), - FileTime.fromMillis(0)); - LayerMetadata fakeEmptySourceFilesLayerMetadata = - LayerMetadata.from(ImmutableList.of(), FileTime.fromMillis(0)); - - List cachedLayers = - Arrays.asList( - new CachedLayerWithMetadata(mockLayers.get(0), fakeOtherSourceFilesLayerMetadata), - new CachedLayerWithMetadata( - mockLayers.get(1), fakeExpectedSourceFilesResourcesLayerMetadata), - new CachedLayerWithMetadata(mockLayers.get(2), fakeOtherSourceFilesLayerMetadata), - new CachedLayerWithMetadata(mockLayers.get(3), fakeEmptySourceFilesLayerMetadata), - new CachedLayerWithMetadata( - mockLayers.get(4), fakeExpectedSourceFilesClassesLayerMetadata), - new CachedLayerWithMetadata( - mockLayers.get(5), fakeExpectedSourceFilesResourcesLayerMetadata)); - - CacheMetadata.Builder cacheMetadataBuilder = CacheMetadata.builder(); - for (CachedLayerWithMetadata cachedLayer : cachedLayers) { - cacheMetadataBuilder.addLayer(cachedLayer); - } - CacheMetadata cacheMetadata = cacheMetadataBuilder.build(); - - ImageLayers filteredLayers = - cacheMetadata - .filterLayers() - .byLayerEntries(ImmutableList.of(fakeLayerEntry1, fakeLayerEntry2)) - .filter(); - - Assert.assertEquals(3, filteredLayers.size()); - Assert.assertEquals( - fakeExpectedSourceFilesResourcesLayerMetadata, filteredLayers.get(0).getMetadata()); - Assert.assertEquals( - fakeExpectedSourceFilesClassesLayerMetadata, filteredLayers.get(1).getMetadata()); - Assert.assertEquals( - fakeExpectedSourceFilesResourcesLayerMetadata, filteredLayers.get(2).getMetadata()); - } - - @Test - public void testFilter_byNoEntries() throws CacheMetadataCorruptedException { - List mockLayers = - Stream.generate(CacheMetadataTest::mockCachedLayer).limit(2).collect(Collectors.toList()); - - LayerEntry fakeLayerEntry = - new LayerEntry(Paths.get("some/source/file"), AbsoluteUnixPath.get("/extraction/path")); - - LayerMetadata fakeSourceFilesLayerMetadata = - LayerMetadata.from(ImmutableList.of(fakeLayerEntry), FileTime.fromMillis(0)); - LayerMetadata fakeNoEntriesLayerMetadata = - new LayerMetadata(ImmutableList.of(), FileTime.fromMillis(0)); - - List cachedLayers = - Arrays.asList( - new CachedLayerWithMetadata(mockLayers.get(0), fakeSourceFilesLayerMetadata), - new CachedLayerWithMetadata(mockLayers.get(1), fakeNoEntriesLayerMetadata)); - - CacheMetadata.Builder cacheMetadataBuilder = CacheMetadata.builder(); - for (CachedLayerWithMetadata cachedLayer : cachedLayers) { - cacheMetadataBuilder.addLayer(cachedLayer); - } - CacheMetadata cacheMetadata = cacheMetadataBuilder.build(); - - ImageLayers filteredLayers = - cacheMetadata.filterLayers().byLayerEntries(ImmutableList.of()).filter(); - - Assert.assertEquals(1, filteredLayers.size()); - Assert.assertEquals(fakeNoEntriesLayerMetadata, filteredLayers.get(0).getMetadata()); - } - - @Test - public void testFilter_byEmptySourceFiles() throws CacheMetadataCorruptedException { - List mockLayers = - Stream.generate(CacheMetadataTest::mockCachedLayer).limit(2).collect(Collectors.toList()); - - LayerEntry fakeLayerEntry = - new LayerEntry(Paths.get("some/source/file"), AbsoluteUnixPath.get("/extraction/path")); - - LayerMetadata fakeSourceFilesLayerMetadata = - LayerMetadata.from(ImmutableList.of(fakeLayerEntry), FileTime.fromMillis(0)); - LayerMetadata fakeEmptySourceFilesLayerMetadata = - LayerMetadata.from(ImmutableList.of(), FileTime.fromMillis(0)); - - List cachedLayers = - Arrays.asList( - new CachedLayerWithMetadata(mockLayers.get(0), fakeSourceFilesLayerMetadata), - new CachedLayerWithMetadata(mockLayers.get(1), fakeEmptySourceFilesLayerMetadata)); - - CacheMetadata.Builder cacheMetadataBuilder = CacheMetadata.builder(); - for (CachedLayerWithMetadata cachedLayer : cachedLayers) { - cacheMetadataBuilder.addLayer(cachedLayer); - } - CacheMetadata cacheMetadata = cacheMetadataBuilder.build(); - - ImageLayers filteredLayers = - cacheMetadata.filterLayers().byLayerEntries(ImmutableList.of()).filter(); - - Assert.assertEquals(1, filteredLayers.size()); - Assert.assertEquals(fakeEmptySourceFilesLayerMetadata, filteredLayers.get(0).getMetadata()); - } -} diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheMetadataTranslatorTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheMetadataTranslatorTest.java deleted file mode 100644 index 50548f1bde..0000000000 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheMetadataTranslatorTest.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright 2017 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache; - -import com.google.cloud.tools.jib.blob.BlobDescriptor; -import com.google.cloud.tools.jib.cache.json.CacheMetadataTemplate; -import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; -import com.google.cloud.tools.jib.image.DescriptorDigest; -import com.google.cloud.tools.jib.image.LayerEntry; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; -import com.google.common.collect.ImmutableList; -import com.google.common.io.Resources; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.FileTime; -import java.security.DigestException; -import java.util.List; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; - -/** Tests for {@link CacheMetadataTranslator}. */ -@RunWith(MockitoJUnitRunner.class) -public class CacheMetadataTranslatorTest { - - private static final String CLASSES_LAYER_SOURCE_FILE = "/some/source/path"; - private static final AbsoluteUnixPath CLASSES_LAYER_EXTRACTION_PATH = - AbsoluteUnixPath.get("/some/extraction/path"); - private static final FileTime CLASSES_LAYER_LAST_MODIFIED_TIME = - FileTime.fromMillis(255073580723571L); - - @Mock private Path mockPath; - - private BlobDescriptor baseLayerBlobDescriptor; - private DescriptorDigest baseLayerDiffId; - private BlobDescriptor classesLayerBlobDescriptor; - private DescriptorDigest classesLayerDiffId; - - @Before - public void setUp() throws DigestException { - baseLayerBlobDescriptor = - new BlobDescriptor( - 631, - DescriptorDigest.fromDigest( - "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef")); - baseLayerDiffId = - DescriptorDigest.fromDigest( - "sha256:b56ae66c29370df48e7377c8f9baa744a3958058a766793f821dadcb144a4647"); - classesLayerBlobDescriptor = - new BlobDescriptor( - 223, - DescriptorDigest.fromDigest( - "sha256:8c662931926fa990b41da3c9f42663a537ccd498130030f9149173a0493832ad")); - classesLayerDiffId = - DescriptorDigest.fromDigest( - "sha256:a3f3e99c29370df48e7377c8f9baa744a3958058a766793f821dadcb144a8372"); - } - - @Test - public void testFromTemplate() - throws URISyntaxException, IOException, CacheMetadataCorruptedException { - Path fakePath = Paths.get("fake/path"); - - // Loads the expected JSON string. - Path jsonFile = Paths.get(Resources.getResource("json/metadata-v3.json").toURI()); - - // Deserializes into a metadata JSON object. - CacheMetadataTemplate metadataTemplate = - JsonTemplateMapper.readJsonFromFile(jsonFile, CacheMetadataTemplate.class); - - CacheMetadata cacheMetadata = CacheMetadataTranslator.fromTemplate(metadataTemplate, fakePath); - - List layers = cacheMetadata.getLayers().getLayers(); - - // Checks that the base layer was translated correctly. - CachedLayerWithMetadata baseLayer = layers.get(0); - Assert.assertEquals( - CacheFiles.getLayerFile(fakePath, baseLayerBlobDescriptor.getDigest()), - baseLayer.getContentFile()); - Assert.assertEquals(baseLayerBlobDescriptor, baseLayer.getBlobDescriptor()); - Assert.assertEquals(baseLayerDiffId, baseLayer.getDiffId()); - - // Checks that the classses layer was translated correctly. - CachedLayerWithMetadata classesLayer = layers.get(1); - Assert.assertEquals( - CacheFiles.getLayerFile(fakePath, classesLayerBlobDescriptor.getDigest()), - classesLayer.getContentFile()); - Assert.assertEquals(classesLayerBlobDescriptor, classesLayer.getBlobDescriptor()); - Assert.assertEquals(classesLayerDiffId, classesLayer.getDiffId()); - Assert.assertNotNull(classesLayer.getMetadata()); - Assert.assertEquals( - CLASSES_LAYER_SOURCE_FILE, - classesLayer.getMetadata().getEntries().get(0).getAbsoluteSourceFileString()); - Assert.assertEquals( - CLASSES_LAYER_EXTRACTION_PATH.toString(), - classesLayer.getMetadata().getEntries().get(0).getAbsoluteExtractionPathString()); - Assert.assertEquals( - CLASSES_LAYER_LAST_MODIFIED_TIME, classesLayer.getMetadata().getLastModifiedTime()); - } - - @Test - public void testToTemplate() throws URISyntaxException, IOException { - Path jsonFile = Paths.get(Resources.getResource("json/metadata-v3.json").toURI()); - String expectedJson = new String(Files.readAllBytes(jsonFile), StandardCharsets.UTF_8); - - CachedLayer baseCachedLayer = - new CachedLayer(mockPath, baseLayerBlobDescriptor, baseLayerDiffId); - CachedLayerWithMetadata baseLayer = new CachedLayerWithMetadata(baseCachedLayer, null); - - CachedLayer classesCachedLayer = - new CachedLayer(mockPath, classesLayerBlobDescriptor, classesLayerDiffId); - LayerMetadata classesLayerMetadata = - LayerMetadata.from( - ImmutableList.of( - new LayerEntry( - Paths.get(CLASSES_LAYER_SOURCE_FILE), CLASSES_LAYER_EXTRACTION_PATH)), - CLASSES_LAYER_LAST_MODIFIED_TIME); - CachedLayerWithMetadata classesLayer = - new CachedLayerWithMetadata(classesCachedLayer, classesLayerMetadata); - - CacheMetadata cacheMetadata = - CacheMetadata.builder().addLayer(baseLayer).addLayer(classesLayer).build(); - - CacheMetadataTemplate cacheMetadataTemplate = CacheMetadataTranslator.toTemplate(cacheMetadata); - - // Serializes the JSON object. - ByteArrayOutputStream jsonStream = new ByteArrayOutputStream(); - JsonTemplateMapper.toBlob(cacheMetadataTemplate).writeTo(jsonStream); - - Assert.assertEquals(expectedJson, jsonStream.toString()); - } -} diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheReaderTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheReaderTest.java deleted file mode 100644 index 9157dcc343..0000000000 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheReaderTest.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache; - -import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; -import com.google.cloud.tools.jib.filesystem.DirectoryWalker; -import com.google.cloud.tools.jib.image.DescriptorDigest; -import com.google.cloud.tools.jib.image.ImageLayers; -import com.google.cloud.tools.jib.image.LayerEntry; -import com.google.common.collect.ImmutableList; -import com.google.common.io.Resources; -import java.io.IOException; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.FileTime; -import java.security.DigestException; -import java.util.Comparator; -import java.util.Optional; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -/** Tests for {@link CacheReader}. */ -public class CacheReaderTest { - - // TODO: Replace with filesystem.DirectoryWalker. - private static void copyDirectory(Path source, Path destination) throws IOException { - new DirectoryWalker(source) - .filter(path -> !path.equals(source)) - .walk( - path -> { - Path newPath = destination.resolve(source.relativize(path)); - Files.copy(path, newPath); - }); - } - - @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - - private Path testCacheFolder; - - @Before - public void setUp() throws IOException, URISyntaxException { - testCacheFolder = temporaryFolder.newFolder().toPath(); - - // Copies the test resource cache to the temporary test cache folder. - Path resourceCache = Paths.get(Resources.getResource("cache").toURI()); - copyDirectory(resourceCache, testCacheFolder); - } - - @Test - public void testAreBaseImageLayersCached() - throws DigestException, CacheMetadataCorruptedException, IOException { - try (Cache cache = Cache.init(testCacheFolder)) { - CacheReader cacheReader = new CacheReader(cache); - Assert.assertNotNull( - cacheReader.getLayer( - DescriptorDigest.fromDigest( - "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef"))); - Assert.assertNull( - cacheReader.getLayer( - DescriptorDigest.fromDigest( - "sha256:6f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef"))); - Assert.assertNotNull( - cacheReader.getLayer( - DescriptorDigest.fromDigest( - "sha256:8c662931926fa990b41da3c9f42663a537ccd498130030f9149173a0493832ad"))); - } - } - - @Test - public void testGetLayerFile() throws CacheMetadataCorruptedException, IOException { - Path expectedFile = - testCacheFolder.resolve( - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.tar.gz"); - - try (Cache cache = Cache.init(testCacheFolder)) { - CacheReader cacheReader = new CacheReader(cache); - - Assert.assertEquals( - expectedFile, - cacheReader.getLayerFile( - ImmutableList.of( - new LayerEntry( - Paths.get("/some/source/path"), - AbsoluteUnixPath.get("/some/extraction/path"))))); - Assert.assertNull(cacheReader.getLayerFile(ImmutableList.of())); - } - } - - @Test - public void testGetUpToDateLayerBySourceFiles() - throws URISyntaxException, IOException, CacheMetadataCorruptedException { - // The two last modified times to use. Must be in thousands as most file time granularity is in - // seconds. - FileTime olderLastModifiedTime = FileTime.fromMillis(1000); - FileTime newerLastModifiedTime = FileTime.fromMillis(2000); - - // Copies test files to a modifiable temporary folder. - Path resourceSourceFilesPath = Paths.get(Resources.getResource("layer").toURI()); - Path testSourceFilesPath = temporaryFolder.newFolder().toPath(); - copyDirectory(resourceSourceFilesPath, testSourceFilesPath); - - // Gets the contents of the test source files. - ImmutableList testSourceFiles = new DirectoryWalker(testSourceFilesPath).walk(); - - // Walk the files in reverse order so that the subfiles are changed before the parent - // directories are. - ImmutableList paths = - ImmutableList.sortedCopyOf(Comparator.reverseOrder(), testSourceFiles); - for (Path path : paths) { - Files.setLastModifiedTime(path, olderLastModifiedTime); - } - - // Sets the metadata source file to the new temporary folder. - CachedLayerWithMetadata classesCachedLayer; - try (Cache cache = Cache.init(testCacheFolder)) { - ImageLayers cachedLayers = - cache.getMetadata().filterLayers().filter(); - - Assert.assertEquals(3, cachedLayers.size()); - classesCachedLayer = cachedLayers.get(2); - - Assert.assertNotNull(classesCachedLayer.getMetadata()); - classesCachedLayer - .getMetadata() - .setEntries( - testSourceFiles - .stream() - .map( - testSourceFile -> - new LayerMetadata.LayerMetadataEntry( - testSourceFile.toString(), - AbsoluteUnixPath.get("/some/extraction/path") - .resolve(testSourceFilesPath.relativize(testSourceFile)) - .toString())) - .collect(ImmutableList.toImmutableList())); - } - - try (Cache cache = Cache.init(testCacheFolder)) { - CacheReader cacheReader = new CacheReader(cache); - - ImmutableList upToDateLayerEntries = - testSourceFiles - .stream() - .map( - testSourceFile -> - new LayerEntry( - testSourceFile, - AbsoluteUnixPath.get("/some/extraction/path") - .resolve(testSourceFilesPath.relativize(testSourceFile)))) - .collect(ImmutableList.toImmutableList()); - - Optional optionalUpToDateLayer = - cacheReader.getUpToDateLayerByLayerEntries(upToDateLayerEntries); - Assert.assertEquals( - classesCachedLayer.getBlobDescriptor(), - optionalUpToDateLayer.orElseThrow(AssertionError::new).getBlobDescriptor()); - - // Changes a file and checks that the change is detected. - Files.setLastModifiedTime( - testSourceFilesPath.resolve("a").resolve("b").resolve("bar"), newerLastModifiedTime); - Assert.assertFalse( - cacheReader.getUpToDateLayerByLayerEntries(upToDateLayerEntries).isPresent()); - Assert.assertFalse( - cacheReader - .getUpToDateLayerByLayerEntries( - testSourceFiles - .stream() - .map( - testSourceFile -> - new LayerEntry( - testSourceFile, - AbsoluteUnixPath.get("/another/extraction/path") - .resolve(testSourceFilesPath.relativize(testSourceFile)))) - .collect(ImmutableList.toImmutableList())) - .isPresent()); - - // Any non-cached directory should be deemed modified. - Assert.assertFalse( - cacheReader - .getUpToDateLayerByLayerEntries( - new DirectoryWalker(resourceSourceFilesPath) - .walk() - .stream() - .map( - resourceSourceFile -> - new LayerEntry( - resourceSourceFile, - AbsoluteUnixPath.get("/some/extraction/path") - .resolve( - resourceSourceFilesPath.relativize(resourceSourceFile)))) - .collect(ImmutableList.toImmutableList())) - .isPresent()); - } - } -} diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java deleted file mode 100644 index 8d6888a7d0..0000000000 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright 2017 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache; - -import com.google.cloud.tools.jib.blob.BlobDescriptor; -import com.google.cloud.tools.jib.cache.LayerMetadata.LayerMetadataEntry; -import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; -import com.google.cloud.tools.jib.image.DescriptorDigest; -import com.google.cloud.tools.jib.image.LayerEntry; -import com.google.common.collect.ImmutableList; -import com.google.common.io.Resources; -import java.io.IOException; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.NotDirectoryException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.FileTime; -import java.security.DigestException; -import java.time.Instant; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -/** Tests for {@link Cache}. */ -public class CacheTest { - - @Rule public TemporaryFolder temporaryCacheDirectory = new TemporaryFolder(); - - @Test - public void testInit_empty() throws IOException, CacheMetadataCorruptedException { - Path cacheDirectory = temporaryCacheDirectory.newFolder().toPath(); - - Cache cache = Cache.init(cacheDirectory); - Assert.assertEquals(0, cache.getMetadata().getLayers().getLayers().size()); - } - - @Test - public void testInit_notDirectory() throws CacheMetadataCorruptedException, IOException { - Path tempFile = temporaryCacheDirectory.newFile().toPath(); - - try { - Cache.init(tempFile); - Assert.fail("Cache should not be able to initialize on non-directory"); - - } catch (NotDirectoryException ex) { - Assert.assertEquals("The cache can only write to a directory", ex.getMessage()); - } - } - - @Test - public void testInit_withMetadata() - throws URISyntaxException, IOException, CacheMetadataCorruptedException { - Path cacheDirectory = temporaryCacheDirectory.newFolder().toPath(); - - Path resourceMetadataJsonPath = - Paths.get(Resources.getResource("json/metadata-v3.json").toURI()); - Path testMetadataJsonPath = cacheDirectory.resolve(CacheFiles.METADATA_FILENAME); - Files.copy(resourceMetadataJsonPath, testMetadataJsonPath); - - try (Cache cache = Cache.init(cacheDirectory)) { - Assert.assertEquals(2, cache.getMetadata().getLayers().getLayers().size()); - } - - Assert.assertArrayEquals( - Files.readAllBytes(resourceMetadataJsonPath), Files.readAllBytes(testMetadataJsonPath)); - } - - @Test - public void test_saveMetadata_noDuplicates() - throws IOException, CacheMetadataCorruptedException, DigestException, URISyntaxException { - Path cacheDirectory = temporaryCacheDirectory.newFolder().toPath(); - - Path resourceMetadataJsonPath = - Paths.get(Resources.getResource("json/metadata-v3.json").toURI()); - Path testMetadataJsonPath = cacheDirectory.resolve(CacheFiles.METADATA_FILENAME); - Files.copy(resourceMetadataJsonPath, testMetadataJsonPath); - - DescriptorDigest descriptorDigest1 = - DescriptorDigest.fromHash( - "8c662931926fa990b41da3c9f42663a537ccd498130030f9149173a0493832ad"); - DescriptorDigest descriptorDigest2 = - DescriptorDigest.fromHash( - "6f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef"); - - LayerEntry layerEntry1 = - new LayerEntry(Paths.get("some", "file"), AbsoluteUnixPath.get("/extraction/path/1")); - LayerEntry layerEntry2 = - new LayerEntry( - Paths.get("some", "other", "file"), AbsoluteUnixPath.get("/extraction/path/1")); - LayerEntry layerEntry3 = - new LayerEntry(Paths.get("another", "file"), AbsoluteUnixPath.get("/extraction/path/2")); - LayerEntry layerEntry4 = - new LayerEntry( - Paths.get("yet", "another", "file"), AbsoluteUnixPath.get("/extraction/path/2")); - - LayerMetadata layerMetadata1 = - LayerMetadata.from( - ImmutableList.of(layerEntry1, layerEntry2, layerEntry3, layerEntry4), - FileTime.from(Instant.now())); - LayerMetadata layerMetadata2 = - LayerMetadata.from( - ImmutableList.of(layerEntry3, layerEntry4), FileTime.from(Instant.EPOCH)); - - DescriptorDigest mockDiffId = - DescriptorDigest.fromHash( - "91e0cae00b86c289b33fee303a807ae72dd9f0315c16b74e6ab0cdbe9d996c10"); - - // Layers ABA. - List cachedLayersWithMetadata = - Arrays.asList( - new CachedLayerWithMetadata( - new CachedLayer( - Paths.get("nonexistent"), new BlobDescriptor(descriptorDigest1), mockDiffId), - layerMetadata1), - new CachedLayerWithMetadata( - new CachedLayer( - Paths.get("nonexistent"), new BlobDescriptor(descriptorDigest2), mockDiffId), - layerMetadata2), - new CachedLayerWithMetadata( - new CachedLayer( - Paths.get("nonexistent"), new BlobDescriptor(descriptorDigest1), mockDiffId), - layerMetadata2)); - - // Saves the new layers to the cache metadata. - try (Cache cache = Cache.init(cacheDirectory)) { - cache.addCachedLayersWithMetadataToMetadata(cachedLayersWithMetadata); - } - - // Reload the cache and check that all digests are unique. - try (Cache cache = Cache.init(cacheDirectory)) { - Set encounteredDigests = new HashSet<>(); - for (CachedLayerWithMetadata layer : cache.getMetadata().getLayers()) { - DescriptorDigest layerDigest = layer.getBlobDescriptor().getDigest(); - Assert.assertFalse(encounteredDigests.contains(layerDigest)); - encounteredDigests.add(layerDigest); - } - - // The layer metadata for layer with digest descriptorDigest1 should be layerMetadata2. - CachedLayerWithMetadata descriptorDigest1Layer = - cache.getMetadata().getLayers().get(descriptorDigest1); - Assert.assertNotNull(descriptorDigest1Layer); - LayerMetadata layerMetadata = descriptorDigest1Layer.getMetadata(); - Assert.assertNotNull(layerMetadata); - Assert.assertEquals(2, layerMetadata.getEntries().size()); - Assert.assertEquals(FileTime.from(Instant.EPOCH), layerMetadata.getLastModifiedTime()); - Assert.assertEquals( - ImmutableList.of( - layerEntry3.getAbsoluteSourceFileString(), layerEntry4.getAbsoluteSourceFileString()), - layerMetadata - .getEntries() - .stream() - .map(LayerMetadataEntry::getAbsoluteSourceFileString) - .collect(ImmutableList.toImmutableList())); - Assert.assertEquals( - ImmutableList.of( - layerEntry3.getAbsoluteExtractionPathString(), - layerEntry4.getAbsoluteExtractionPathString()), - layerMetadata - .getEntries() - .stream() - .map(LayerMetadataEntry::getAbsoluteExtractionPathString) - .collect(ImmutableList.toImmutableList())); - } - } -} diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheWriterTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheWriterTest.java deleted file mode 100644 index ced3a601ce..0000000000 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheWriterTest.java +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache; - -import com.google.cloud.tools.jib.blob.Blob; -import com.google.cloud.tools.jib.blob.BlobDescriptor; -import com.google.cloud.tools.jib.blob.Blobs; -import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; -import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; -import com.google.cloud.tools.jib.image.DescriptorDigest; -import com.google.cloud.tools.jib.image.LayerEntry; -import com.google.cloud.tools.jib.image.ReproducibleLayerBuilder; -import com.google.cloud.tools.jib.image.UnwrittenLayer; -import com.google.common.collect.ImmutableList; -import com.google.common.io.ByteStreams; -import com.google.common.io.CountingOutputStream; -import com.google.common.io.Resources; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Collections; -import java.util.stream.Stream; -import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.mockito.Mockito; - -/** Tests for {@link CacheWriter}. */ -public class CacheWriterTest { - - private static boolean isTarGz(Path path) { - return path.toString().endsWith(".tar.gz"); - } - - @Rule public final TemporaryFolder temporaryCacheDirectory = new TemporaryFolder(); - - private Cache testCache; - private CacheWriter cacheWriter; - - private Path resourceBlob; - - private static class ExpectedLayer { - - private final BlobDescriptor blobDescriptor; - private final DescriptorDigest diffId; - private final Blob blob; - - private ExpectedLayer(BlobDescriptor blobDescriptor, DescriptorDigest diffId, Blob blob) { - this.blobDescriptor = blobDescriptor; - this.diffId = diffId; - this.blob = blob; - } - } - - @Before - public void setUp() throws CacheMetadataCorruptedException, IOException, URISyntaxException { - Path cacheDirectory = temporaryCacheDirectory.newFolder().toPath(); - - testCache = Cache.init(cacheDirectory); - // Writes resourceBlob as a layer to the cache. - cacheWriter = new CacheWriter(testCache); - - resourceBlob = Paths.get(Resources.getResource("blobA").toURI()); - } - - @Test - public void testWriteLayer_unwritten() throws IOException { - UnwrittenLayer unwrittenLayer = new UnwrittenLayer(Blobs.from(resourceBlob)); - - ReproducibleLayerBuilder mockReproducibleLayerBuilder = - Mockito.mock(ReproducibleLayerBuilder.class); - Mockito.when(mockReproducibleLayerBuilder.build()).thenReturn(unwrittenLayer); - Mockito.when(mockReproducibleLayerBuilder.getLayerEntries()) - .thenReturn( - ImmutableList.of( - new LayerEntry( - Paths.get("/some/source/file"), - AbsoluteUnixPath.get("/some/extraction/path")))); - - CachedLayerWithMetadata cachedLayerWithMetadata = - cacheWriter.writeLayer(mockReproducibleLayerBuilder); - testCache.addCachedLayersWithMetadataToMetadata( - Collections.singletonList(cachedLayerWithMetadata)); - - LayerMetadata layerMetadata = testCache.getUpdatedMetadata().getLayers().get(0).getMetadata(); - Assert.assertNotNull(layerMetadata); - Assert.assertEquals(1, layerMetadata.getEntries().size()); - Assert.assertEquals( - "/some/source/file", layerMetadata.getEntries().get(0).getAbsoluteSourceFileString()); - Assert.assertEquals( - "/some/extraction/path", - layerMetadata.getEntries().get(0).getAbsoluteExtractionPathString()); - - verifyCachedLayerIsExpected(getExpectedLayer(), cachedLayerWithMetadata); - } - - // Windows file overwrite issue: https://github.com/GoogleContainerTools/jib/issues/719 - @Test - public void testWriteLayer_doesNotOverwriteExistingTarGz() - throws IOException, InterruptedException { - // Writes resourceBlob as a layer to the cache. - UnwrittenLayer unwrittenLayer = new UnwrittenLayer(Blobs.from(resourceBlob)); - - ReproducibleLayerBuilder layerBuilder = Mockito.mock(ReproducibleLayerBuilder.class); - Mockito.when(layerBuilder.build()).thenReturn(unwrittenLayer); - LayerEntry layerEntry = - new LayerEntry( - Paths.get("some/source/file"), AbsoluteUnixPath.get("/some/extraction/path")); - Mockito.when(layerBuilder.getLayerEntries()).thenReturn(ImmutableList.of(layerEntry)); - - cacheWriter.writeLayer(layerBuilder); - Assert.assertEquals(1, getTarGzCountInCache()); - long tarGzModifiedTime = getTarGzModifiedTimeInCache(); - - Thread.sleep(1000); // to have different modified time - cacheWriter.writeLayer(layerBuilder); - Assert.assertEquals(1, getTarGzCountInCache()); - Assert.assertEquals(tarGzModifiedTime, getTarGzModifiedTimeInCache()); - } - - private long getTarGzCountInCache() throws IOException { - try (Stream stream = Files.walk(temporaryCacheDirectory.getRoot().toPath())) { - return stream.filter(CacheWriterTest::isTarGz).count(); - } - } - - private long getTarGzModifiedTimeInCache() throws IOException { - try (Stream fileStream = Files.walk(temporaryCacheDirectory.getRoot().toPath())) { - return fileStream - .filter(CacheWriterTest::isTarGz) - .findFirst() - .orElseThrow(AssertionError::new) - .toFile() - .lastModified(); - } - } - - @Test - public void testGetLayerOutputStream() throws IOException { - ExpectedLayer expectedLayer = getExpectedLayer(); - - CountingOutputStream layerOutputStream = - cacheWriter.getLayerOutputStream(expectedLayer.blobDescriptor.getDigest()); - expectedLayer.blob.writeTo(layerOutputStream); - layerOutputStream.close(); - CachedLayer cachedLayer = - cacheWriter.getCachedLayer( - layerOutputStream.getCount(), expectedLayer.blobDescriptor.getDigest()); - testCache.addCachedLayersToMetadata(Collections.singletonList(cachedLayer)); - - CachedLayerWithMetadata layerInMetadata = testCache.getUpdatedMetadata().getLayers().get(0); - Assert.assertNull(layerInMetadata.getMetadata()); - - verifyCachedLayerIsExpected(expectedLayer, cachedLayer); - } - - /** - * @return the expected layer to test against, represented by the {@code resourceBlob} resource - * file - */ - private ExpectedLayer getExpectedLayer() throws IOException { - // Gets the expected content descriptor, diff ID, and compressed BLOB. - ByteArrayOutputStream compressedBlobOutputStream = new ByteArrayOutputStream(); - CountingDigestOutputStream compressedDigestOutputStream = - new CountingDigestOutputStream(compressedBlobOutputStream); - CountingDigestOutputStream uncompressedDigestOutputStream; - try (GZIPOutputStream compressorStream = new GZIPOutputStream(compressedDigestOutputStream)) { - uncompressedDigestOutputStream = new CountingDigestOutputStream(compressorStream); - byte[] expectedBlobABytes = Files.readAllBytes(resourceBlob); - uncompressedDigestOutputStream.write(expectedBlobABytes); - } - - BlobDescriptor expectedBlobADescriptor = compressedDigestOutputStream.toBlobDescriptor(); - DescriptorDigest expectedBlobADiffId = - uncompressedDigestOutputStream.toBlobDescriptor().getDigest(); - - ByteArrayInputStream compressedBlobInputStream = - new ByteArrayInputStream(compressedBlobOutputStream.toByteArray()); - Blob blob = Blobs.from(compressedBlobInputStream); - - return new ExpectedLayer(expectedBlobADescriptor, expectedBlobADiffId, blob); - } - - private void verifyCachedLayerIsExpected(ExpectedLayer expectedLayer, CachedLayer cachedLayer) - throws IOException { - // Reads the cached layer back. - Path compressedBlobFile = cachedLayer.getContentFile(); - - try (GZIPInputStream in = new GZIPInputStream(Files.newInputStream(compressedBlobFile))) { - byte[] decompressedBytes = ByteStreams.toByteArray(in); - byte[] expectedBlobABytes = Files.readAllBytes(resourceBlob); - Assert.assertArrayEquals(expectedBlobABytes, decompressedBytes); - Assert.assertEquals( - expectedLayer.blobDescriptor.getSize(), cachedLayer.getBlobDescriptor().getSize()); - Assert.assertEquals( - expectedLayer.blobDescriptor.getDigest(), cachedLayer.getBlobDescriptor().getDigest()); - Assert.assertEquals(expectedLayer.blobDescriptor, cachedLayer.getBlobDescriptor()); - Assert.assertEquals(expectedLayer.diffId, cachedLayer.getDiffId()); - } - } -} diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CachedLayerTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CachedLayerTest.java deleted file mode 100644 index 38effaba14..0000000000 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CachedLayerTest.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright 2017 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache; - -import com.google.cloud.tools.jib.blob.Blob; -import com.google.cloud.tools.jib.blob.BlobDescriptor; -import com.google.cloud.tools.jib.image.DescriptorDigest; -import com.google.common.io.Resources; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; - -/** Tests for {@link CachedLayer}. */ -@RunWith(MockitoJUnitRunner.class) -public class CachedLayerTest { - - @Mock private Path mockPath; - @Mock private BlobDescriptor mockBlobDescriptor; - @Mock private DescriptorDigest mockDiffId; - - @Test - public void testNew() { - CachedLayer layer = new CachedLayer(mockPath, mockBlobDescriptor, mockDiffId); - - Assert.assertEquals(mockPath, layer.getContentFile()); - Assert.assertEquals(mockBlobDescriptor, layer.getBlobDescriptor()); - Assert.assertEquals(mockDiffId, layer.getDiffId()); - } - - @Test - public void testGetBlob() throws URISyntaxException, IOException { - Path fileA = Paths.get(Resources.getResource("fileA").toURI()); - String expectedFileAString = new String(Files.readAllBytes(fileA), StandardCharsets.UTF_8); - - CachedLayer cachedLayer = new CachedLayer(fileA, mockBlobDescriptor, mockDiffId); - - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - Blob fileBlob = cachedLayer.getBlob(); - fileBlob.writeTo(outputStream); - - Assert.assertEquals( - expectedFileAString, new String(outputStream.toByteArray(), StandardCharsets.UTF_8)); - Assert.assertEquals(mockBlobDescriptor, cachedLayer.getBlobDescriptor()); - Assert.assertEquals(mockDiffId, cachedLayer.getDiffId()); - } -} diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CachedLayerWithMetadataTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CachedLayerWithMetadataTest.java deleted file mode 100644 index eed8db7262..0000000000 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CachedLayerWithMetadataTest.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2017 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache; - -import org.junit.Assert; -import org.junit.Test; -import org.mockito.Mockito; - -/** Tests for {@link CachedLayerWithMetadata}. */ -public class CachedLayerWithMetadataTest { - - @Test - public void testNew() { - LayerMetadata mockLayerMetadata = Mockito.mock(LayerMetadata.class); - CachedLayerWithMetadata cachedLayerWithMetadata = - new CachedLayerWithMetadata(Mockito.mock(CachedLayer.class), mockLayerMetadata); - Assert.assertEquals(mockLayerMetadata, cachedLayerWithMetadata.getMetadata()); - } -} diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CachesTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CachesTest.java deleted file mode 100644 index ca6290ad23..0000000000 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CachesTest.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2017 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -/** Tests for {@link Caches}. */ -public class CachesTest { - - @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - - @Test - public void testInitializer() - throws CacheMetadataCorruptedException, IOException, CacheDirectoryNotOwnedException, - CacheDirectoryCreationException { - Path tempBaseCacheDirectory = temporaryFolder.newFolder().toPath(); - Path tempApplicationCacheDirectory = temporaryFolder.newFolder().toPath(); - - try (Caches caches = - new Caches.Initializer(tempBaseCacheDirectory, false, tempApplicationCacheDirectory, false) - .init()) { - Assert.assertEquals(tempBaseCacheDirectory, caches.getBaseCache().getCacheDirectory()); - Assert.assertEquals( - tempApplicationCacheDirectory, caches.getApplicationCache().getCacheDirectory()); - } - - // Checks that the caches were closed (metadata.json saved). - Assert.assertTrue(Files.exists(tempBaseCacheDirectory.resolve(CacheFiles.METADATA_FILENAME))); - Assert.assertTrue( - Files.exists(tempApplicationCacheDirectory.resolve(CacheFiles.METADATA_FILENAME))); - } - - @Test - public void testEnsureOwnership_notOwned() throws IOException, CacheDirectoryCreationException { - Path cacheDirectory = temporaryFolder.newFolder().toPath(); - - try { - Caches.Initializer.ensureOwnership(cacheDirectory); - Assert.fail("Expected CacheDirectoryNotOwnedException to be thrown"); - - } catch (CacheDirectoryNotOwnedException ex) { - Assert.assertEquals(cacheDirectory, ex.getCacheDirectory()); - } - } - - @Test - public void testEnsureOwnership_create() - throws IOException, CacheDirectoryNotOwnedException, CacheDirectoryCreationException { - Path cacheDirectory = temporaryFolder.newFolder().toPath(); - Path nonexistentDirectory = cacheDirectory.resolve("somefolder"); - - Caches.Initializer.ensureOwnership(nonexistentDirectory); - } -} diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/json/CacheMetadataTemplateTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/json/CacheMetadataTemplateTest.java deleted file mode 100644 index 6458b1962d..0000000000 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/json/CacheMetadataTemplateTest.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright 2017 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.cache.json; - -import com.google.cloud.tools.jib.cache.json.CacheMetadataLayerPropertiesObjectTemplate.LayerEntryTemplate; -import com.google.cloud.tools.jib.image.DescriptorDigest; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; -import com.google.common.io.Resources; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.FileTime; -import java.security.DigestException; -import java.util.Collections; -import java.util.List; -import org.junit.Assert; -import org.junit.Test; - -/** Tests for {@link CacheMetadataTemplate}. */ -public class CacheMetadataTemplateTest { - - @Test - public void testToJson() throws URISyntaxException, IOException, DigestException { - // Loads the expected JSON string. - Path jsonFile = Paths.get(Resources.getResource("json/metadata-v3.json").toURI()); - String expectedJson = new String(Files.readAllBytes(jsonFile), StandardCharsets.UTF_8); - - CacheMetadataTemplate cacheMetadataTemplate = new CacheMetadataTemplate(); - - // Adds a base layer. - CacheMetadataLayerObjectTemplate baseLayerTemplate = - new CacheMetadataLayerObjectTemplate() - .setSize(631) - .setDigest( - DescriptorDigest.fromDigest( - "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef")) - .setDiffId( - DescriptorDigest.fromDigest( - "sha256:b56ae66c29370df48e7377c8f9baa744a3958058a766793f821dadcb144a4647")); - - // Adds an application layer. - CacheMetadataLayerPropertiesObjectTemplate propertiesTemplate = - new CacheMetadataLayerPropertiesObjectTemplate() - .setLayerEntries( - Collections.singletonList( - new LayerEntryTemplate("/some/source/path", "/some/extraction/path"))) - .setLastModifiedTime(FileTime.fromMillis(255073580723571L)); - CacheMetadataLayerObjectTemplate classesLayerTemplate = - new CacheMetadataLayerObjectTemplate() - .setSize(223) - .setDigest( - DescriptorDigest.fromDigest( - "sha256:8c662931926fa990b41da3c9f42663a537ccd498130030f9149173a0493832ad")) - .setDiffId( - DescriptorDigest.fromDigest( - "sha256:a3f3e99c29370df48e7377c8f9baa744a3958058a766793f821dadcb144a8372")) - .setProperties(propertiesTemplate); - - cacheMetadataTemplate.addLayer(baseLayerTemplate).addLayer(classesLayerTemplate); - - // Serializes the JSON object. - ByteArrayOutputStream jsonStream = new ByteArrayOutputStream(); - JsonTemplateMapper.toBlob(cacheMetadataTemplate).writeTo(jsonStream); - - Assert.assertEquals(expectedJson, jsonStream.toString()); - } - - @Test - public void testFromJson() throws URISyntaxException, IOException, DigestException { - // Loads the expected JSON string. - Path jsonFile = Paths.get(Resources.getResource("json/metadata-v3.json").toURI()); - - // Deserializes into a metadata JSON object. - CacheMetadataTemplate metadataTemplate = - JsonTemplateMapper.readJsonFromFile(jsonFile, CacheMetadataTemplate.class); - - List layers = metadataTemplate.getLayers(); - - Assert.assertEquals(2, layers.size()); - - // Checks the first layer is correct. - CacheMetadataLayerObjectTemplate baseLayerTemplate = layers.get(0); - Assert.assertEquals(631, baseLayerTemplate.getSize()); - Assert.assertEquals( - DescriptorDigest.fromDigest( - "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef"), - baseLayerTemplate.getDigest()); - Assert.assertEquals( - DescriptorDigest.fromDigest( - "sha256:b56ae66c29370df48e7377c8f9baa744a3958058a766793f821dadcb144a4647"), - baseLayerTemplate.getDiffId()); - Assert.assertNull(baseLayerTemplate.getProperties()); - - // Checks the second layer is correct. - CacheMetadataLayerObjectTemplate classesLayerTemplate = layers.get(1); - Assert.assertEquals(223, classesLayerTemplate.getSize()); - Assert.assertEquals( - DescriptorDigest.fromDigest( - "sha256:8c662931926fa990b41da3c9f42663a537ccd498130030f9149173a0493832ad"), - classesLayerTemplate.getDigest()); - Assert.assertEquals( - DescriptorDigest.fromDigest( - "sha256:a3f3e99c29370df48e7377c8f9baa744a3958058a766793f821dadcb144a8372"), - classesLayerTemplate.getDiffId()); - Assert.assertNotNull(classesLayerTemplate.getProperties()); - Assert.assertEquals(1, classesLayerTemplate.getProperties().getLayerEntries().size()); - Assert.assertEquals( - "/some/source/path", - classesLayerTemplate.getProperties().getLayerEntries().get(0).getSourceFileString()); - Assert.assertEquals( - "/some/extraction/path", - classesLayerTemplate.getProperties().getLayerEntries().get(0).getExtractionPathString()); - Assert.assertEquals( - FileTime.fromMillis(255073580723571L), - classesLayerTemplate.getProperties().getLastModifiedTime()); - } -} diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/configuration/BuildConfigurationTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/configuration/BuildConfigurationTest.java index 2966837231..d0e4738dbd 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/configuration/BuildConfigurationTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/configuration/BuildConfigurationTest.java @@ -26,6 +26,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import java.io.IOException; import java.nio.file.Paths; import java.time.Instant; import java.util.Arrays; @@ -143,10 +144,10 @@ public void testBuilder() throws Exception { Assert.assertEquals(expectedTargetFormat, buildConfiguration.getTargetFormat()); Assert.assertEquals( expectedApplicationLayersCacheConfiguration, - buildConfiguration.getApplicationLayersCacheConfiguration()); + buildConfigurationBuilder.getApplicationLayersCacheConfiguration()); Assert.assertEquals( expectedBaseImageLayersCacheConfiguration, - buildConfiguration.getBaseImageLayersCacheConfiguration()); + buildConfigurationBuilder.getBaseImageLayersCacheConfiguration()); Assert.assertTrue(buildConfiguration.getAllowInsecureRegistries()); Assert.assertEquals(expectedLayerConfigurations, buildConfiguration.getLayerConfigurations()); Assert.assertEquals( @@ -155,7 +156,7 @@ public void testBuilder() throws Exception { } @Test - public void testBuilder_default() { + public void testBuilder_default() throws IOException, CacheDirectoryCreationException { // These are required and don't have defaults. String expectedBaseImageServerUrl = "someserver"; String expectedBaseImageName = "baseimage"; @@ -174,16 +175,22 @@ public void testBuilder_default() { ImageReference.of( expectedTargetServerUrl, expectedTargetImageName, expectedTargetTag)) .build(); - BuildConfiguration buildConfiguration = + BuildConfiguration.Builder buildConfigurationBuilder = BuildConfiguration.builder() .setBaseImageConfiguration(baseImageConfiguration) - .setTargetImageConfiguration(targetImageConfiguration) - .build(); + .setTargetImageConfiguration(targetImageConfiguration); + BuildConfiguration buildConfiguration = buildConfigurationBuilder.build(); Assert.assertEquals(ImmutableSet.of("targettag"), buildConfiguration.getAllTargetImageTags()); Assert.assertEquals(V22ManifestTemplate.class, buildConfiguration.getTargetFormat()); - Assert.assertNull(buildConfiguration.getApplicationLayersCacheConfiguration()); - Assert.assertNull(buildConfiguration.getBaseImageLayersCacheConfiguration()); + Assert.assertNotNull(buildConfigurationBuilder.getApplicationLayersCacheConfiguration()); + Assert.assertNotEquals( + CacheConfiguration.forDefaultUserLevelCacheDirectory().getCacheDirectory(), + buildConfigurationBuilder.getApplicationLayersCacheConfiguration().getCacheDirectory()); + Assert.assertNotNull(buildConfigurationBuilder.getBaseImageLayersCacheConfiguration()); + Assert.assertEquals( + CacheConfiguration.forDefaultUserLevelCacheDirectory().getCacheDirectory(), + buildConfigurationBuilder.getBaseImageLayersCacheConfiguration().getCacheDirectory()); Assert.assertNull(buildConfiguration.getContainerConfiguration()); Assert.assertFalse(buildConfiguration.getAllowInsecureRegistries()); Assert.assertEquals(Collections.emptyList(), buildConfiguration.getLayerConfigurations()); @@ -191,7 +198,7 @@ public void testBuilder_default() { } @Test - public void testBuilder_missingValues() { + public void testBuilder_missingValues() throws IOException, CacheDirectoryCreationException { // Target image is missing try { BuildConfiguration.builder() diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/configuration/CacheConfigurationTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/configuration/CacheConfigurationTest.java index 61383c2ac1..b994c249b0 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/configuration/CacheConfigurationTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/configuration/CacheConfigurationTest.java @@ -16,7 +16,6 @@ package com.google.cloud.tools.jib.configuration; -import com.google.cloud.tools.jib.cache.CacheDirectoryCreationException; import java.nio.file.Files; import java.nio.file.Paths; import org.junit.Assert; diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/DockerClientTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/DockerClientTest.java index 52a709a3a6..138d13d07d 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/DockerClientTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/DockerClientTest.java @@ -155,14 +155,14 @@ public void testLoad_stdoutFail() throws InterruptedException { Mockito.when(mockProcess.getOutputStream()).thenReturn(ByteStreams.nullOutputStream()); Mockito.when(mockProcess.getInputStream()) - .thenReturn(new ByteArrayInputStream("failed".getBytes(StandardCharsets.UTF_8))); + .thenReturn(new ByteArrayInputStream("ignored".getBytes(StandardCharsets.UTF_8))); try { testDockerClient.load(Blobs.from("jib")); Assert.fail("Process should have failed"); } catch (IOException ex) { - Assert.assertEquals("'docker load' command failed with output: failed", ex.getMessage()); + Assert.assertEquals("'docker load' command failed with output: ignored", ex.getMessage()); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageToTarballTranslatorTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageToTarballTranslatorTest.java index 73e10168af..a15a7f33ff 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageToTarballTranslatorTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageToTarballTranslatorTest.java @@ -19,12 +19,12 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.blob.Blobs; -import com.google.cloud.tools.jib.cache.CachedLayer; import com.google.cloud.tools.jib.docker.json.DockerLoadManifestEntryTemplate; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.ImageReference; import com.google.cloud.tools.jib.image.InvalidImageReferenceException; +import com.google.cloud.tools.jib.image.Layer; import com.google.cloud.tools.jib.image.LayerPropertyNotFoundException; import com.google.cloud.tools.jib.image.json.ContainerConfigurationTemplate; import com.google.cloud.tools.jib.json.JsonTemplateMapper; @@ -35,6 +35,7 @@ import java.io.InputStreamReader; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.security.DigestException; @@ -42,16 +43,26 @@ import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.junit.Assert; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; /** Tests for {@link ImageToTarballTranslator}. */ +@RunWith(MockitoJUnitRunner.class) public class ImageToTarballTranslatorTest { + @Mock private Layer mockLayer1; + @Mock private Layer mockLayer2; + @Test public void testToTarballBlob() throws InvalidImageReferenceException, IOException, URISyntaxException, LayerPropertyNotFoundException, DigestException { Path fileA = Paths.get(Resources.getResource("fileA").toURI()); Path fileB = Paths.get(Resources.getResource("fileB").toURI()); + long fileASize = Files.size(fileA); + long fileBSize = Files.size(fileB); DescriptorDigest fakeDigestA = DescriptorDigest.fromHash( @@ -60,11 +71,15 @@ public void testToTarballBlob() DescriptorDigest.fromHash( "5994471abb01112afcc18159f6cc74b4f511b99806da59b3caf5a9c173cacfc6"); - Image testImage = - Image.builder() - .addLayer(new CachedLayer(fileA, new BlobDescriptor(fakeDigestA), fakeDigestA)) - .addLayer(new CachedLayer(fileB, new BlobDescriptor(fakeDigestB), fakeDigestB)) - .build(); + Mockito.when(mockLayer1.getBlob()).thenReturn(Blobs.from(fileA)); + Mockito.when(mockLayer1.getBlobDescriptor()) + .thenReturn(new BlobDescriptor(fileASize, fakeDigestA)); + Mockito.when(mockLayer1.getDiffId()).thenReturn(fakeDigestA); + Mockito.when(mockLayer2.getBlob()).thenReturn(Blobs.from(fileB)); + Mockito.when(mockLayer2.getBlobDescriptor()) + .thenReturn(new BlobDescriptor(fileBSize, fakeDigestB)); + Mockito.when(mockLayer2.getDiffId()).thenReturn(fakeDigestB); + Image testImage = Image.builder().addLayer(mockLayer1).addLayer(mockLayer2).build(); Blob tarballBlob = new ImageToTarballTranslator(testImage).toTarballBlob(ImageReference.parse("my/image:tag")); @@ -75,7 +90,7 @@ public void testToTarballBlob() new TarArchiveInputStream(tarballBytesStream)) { // Verifies layer with fileA was added. TarArchiveEntry headerFileALayer = tarArchiveInputStream.getNextTarEntry(); - Assert.assertEquals("fileA", headerFileALayer.getName()); + Assert.assertEquals(fakeDigestA.getHash() + ".tar.gz", headerFileALayer.getName()); String fileAString = CharStreams.toString( new InputStreamReader(tarArchiveInputStream, StandardCharsets.UTF_8)); @@ -83,7 +98,7 @@ public void testToTarballBlob() // Verifies layer with fileB was added. TarArchiveEntry headerFileBLayer = tarArchiveInputStream.getNextTarEntry(); - Assert.assertEquals("fileB", headerFileBLayer.getName()); + Assert.assertEquals(fakeDigestB.getHash() + ".tar.gz", headerFileBLayer.getName()); String fileBString = CharStreams.toString( new InputStreamReader(tarArchiveInputStream, StandardCharsets.UTF_8)); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/image/ImageLayersTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/image/ImageLayersTest.java index 3346f786ba..2c9c6ae23c 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/image/ImageLayersTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/image/ImageLayersTest.java @@ -17,7 +17,6 @@ package com.google.cloud.tools.jib.image; import com.google.cloud.tools.jib.blob.BlobDescriptor; -import com.google.cloud.tools.jib.cache.CachedLayer; import java.util.Arrays; import java.util.List; import org.hamcrest.CoreMatchers; @@ -33,10 +32,10 @@ @RunWith(MockitoJUnitRunner.class) public class ImageLayersTest { - @Mock private CachedLayer mockCachedLayer; + @Mock private Layer mockLayer; @Mock private ReferenceLayer mockReferenceLayer; @Mock private DigestOnlyLayer mockDigestOnlyLayer; - @Mock private UnwrittenLayer mockUnwrittenLayer; + @Mock private Layer mockLayer2; @Before public void setUpFakes() throws LayerPropertyNotFoundException { @@ -44,28 +43,27 @@ public void setUpFakes() throws LayerPropertyNotFoundException { DescriptorDigest mockDescriptorDigest2 = Mockito.mock(DescriptorDigest.class); DescriptorDigest mockDescriptorDigest3 = Mockito.mock(DescriptorDigest.class); - BlobDescriptor cachedLayerBlobDescriptor = new BlobDescriptor(0, mockDescriptorDigest1); + BlobDescriptor layerBlobDescriptor = new BlobDescriptor(0, mockDescriptorDigest1); BlobDescriptor referenceLayerBlobDescriptor = new BlobDescriptor(0, mockDescriptorDigest2); BlobDescriptor referenceNoDiffIdLayerBlobDescriptor = new BlobDescriptor(0, mockDescriptorDigest3); - // Intentionally the same digest as the mockCachedLayer. - BlobDescriptor unwrittenLayerBlobDescriptor = new BlobDescriptor(0, mockDescriptorDigest1); + // Intentionally the same digest as the mockLayer. + BlobDescriptor anotherBlobDescriptor = new BlobDescriptor(0, mockDescriptorDigest1); - Mockito.when(mockCachedLayer.getBlobDescriptor()).thenReturn(cachedLayerBlobDescriptor); + Mockito.when(mockLayer.getBlobDescriptor()).thenReturn(layerBlobDescriptor); Mockito.when(mockReferenceLayer.getBlobDescriptor()).thenReturn(referenceLayerBlobDescriptor); Mockito.when(mockDigestOnlyLayer.getBlobDescriptor()) .thenReturn(referenceNoDiffIdLayerBlobDescriptor); - Mockito.when(mockUnwrittenLayer.getBlobDescriptor()).thenReturn(unwrittenLayerBlobDescriptor); + Mockito.when(mockLayer2.getBlobDescriptor()).thenReturn(anotherBlobDescriptor); } @Test public void testAddLayer_success() throws LayerPropertyNotFoundException { - List expectedLayers = - Arrays.asList(mockCachedLayer, mockReferenceLayer, mockDigestOnlyLayer); + List expectedLayers = Arrays.asList(mockLayer, mockReferenceLayer, mockDigestOnlyLayer); ImageLayers imageLayers = ImageLayers.builder() - .add(mockCachedLayer) + .add(mockLayer) .add(mockReferenceLayer) .add(mockDigestOnlyLayer) .build(); @@ -77,20 +75,15 @@ public void testAddLayer_success() throws LayerPropertyNotFoundException { public void testAddLayer_maintainDuplicates() throws LayerPropertyNotFoundException { // must maintain duplicate List expectedLayers = - Arrays.asList( - mockCachedLayer, - mockReferenceLayer, - mockDigestOnlyLayer, - mockUnwrittenLayer, - mockCachedLayer); + Arrays.asList(mockLayer, mockReferenceLayer, mockDigestOnlyLayer, mockLayer2, mockLayer); ImageLayers imageLayers = ImageLayers.builder() - .add(mockCachedLayer) + .add(mockLayer) .add(mockReferenceLayer) .add(mockDigestOnlyLayer) - .add(mockUnwrittenLayer) - .add(mockCachedLayer) + .add(mockLayer2) + .add(mockLayer) .build(); Assert.assertEquals(expectedLayers, imageLayers.getLayers()); @@ -100,16 +93,16 @@ public void testAddLayer_maintainDuplicates() throws LayerPropertyNotFoundExcept public void testAddLayer_removeDuplicates() throws LayerPropertyNotFoundException { // remove duplicates: last layer should be kept List expectedLayers = - Arrays.asList(mockReferenceLayer, mockDigestOnlyLayer, mockUnwrittenLayer, mockCachedLayer); + Arrays.asList(mockReferenceLayer, mockDigestOnlyLayer, mockLayer2, mockLayer); ImageLayers imageLayers = ImageLayers.builder() .removeDuplicates() - .add(mockCachedLayer) + .add(mockLayer) .add(mockReferenceLayer) .add(mockDigestOnlyLayer) - .add(mockUnwrittenLayer) - .add(mockCachedLayer) + .add(mockLayer2) + .add(mockLayer) .build(); Assert.assertEquals(expectedLayers, imageLayers.getLayers()); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/image/LayerTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/image/LayerTest.java index 3e70fae524..0faa3bb480 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/image/LayerTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/image/LayerTest.java @@ -16,7 +16,6 @@ package com.google.cloud.tools.jib.image; -import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; import org.junit.Assert; import org.junit.Test; @@ -28,32 +27,10 @@ @RunWith(MockitoJUnitRunner.class) public class LayerTest { - @Mock private Blob mockUncompressedBlob; @Mock private DescriptorDigest mockDescriptorDigest; @Mock private BlobDescriptor mockBlobDescriptor; @Mock private DescriptorDigest mockDiffId; - @Test - public void testNew_unwritten() throws LayerPropertyNotFoundException { - Layer layer = new UnwrittenLayer(mockUncompressedBlob); - - Assert.assertEquals(mockUncompressedBlob, layer.getBlob()); - - try { - layer.getBlobDescriptor(); - Assert.fail("Blob descriptor should not be available for unwritten layer"); - } catch (LayerPropertyNotFoundException ex) { - Assert.assertEquals("Blob descriptor not available for unwritten layer", ex.getMessage()); - } - - try { - layer.getDiffId(); - Assert.fail("Diff ID should not be available for unwritten layer"); - } catch (LayerPropertyNotFoundException ex) { - Assert.assertEquals("Diff ID not available for unwritten layer", ex.getMessage()); - } - } - @Test public void testNew_reference() throws LayerPropertyNotFoundException { Layer layer = new ReferenceLayer(mockBlobDescriptor, mockDiffId); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/image/ReproducibleLayerBuilderTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/image/ReproducibleLayerBuilderTest.java index cccc80491a..829739ccaa 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/image/ReproducibleLayerBuilderTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/image/ReproducibleLayerBuilderTest.java @@ -106,11 +106,11 @@ public void testBuild() throws URISyntaxException, IOException { .getLayerEntries()); // Writes the layer tar to a temporary file. - UnwrittenLayer unwrittenLayer = layerBuilder.build(); + Blob unwrittenBlob = layerBuilder.build(); Path temporaryFile = temporaryFolder.newFile().toPath(); try (OutputStream temporaryFileOutputStream = new BufferedOutputStream(Files.newOutputStream(temporaryFile))) { - unwrittenLayer.getBlob().writeTo(temporaryFileOutputStream); + unwrittenBlob.writeTo(temporaryFileOutputStream); } // Reads the file back. @@ -166,15 +166,13 @@ public void testToBlob_reproducibility() throws IOException { ImmutableList.of( new LayerEntry(fileA1, AbsoluteUnixPath.get("/somewhere/fileA")), new LayerEntry(fileB1, AbsoluteUnixPath.get("/somewhere/fileB")))) - .build() - .getBlob(); + .build(); Blob reproduced = new ReproducibleLayerBuilder( ImmutableList.of( new LayerEntry(fileB2, AbsoluteUnixPath.get("/somewhere/fileB")), new LayerEntry(fileA2, AbsoluteUnixPath.get("/somewhere/fileA")))) - .build() - .getBlob(); + .build(); byte[] layerContent = Blobs.writeToByteArray(layer); byte[] reproducedLayerContent = Blobs.writeToByteArray(reproduced); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslatorTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslatorTest.java index d3ec7777d1..edc8e40bb8 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslatorTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslatorTest.java @@ -18,10 +18,11 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; -import com.google.cloud.tools.jib.cache.CachedLayer; +import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.configuration.Port; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.Image; +import com.google.cloud.tools.jib.image.Layer; import com.google.cloud.tools.jib.image.LayerPropertyNotFoundException; import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.collect.ImmutableList; @@ -43,7 +44,6 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.mockito.Mockito; /** Tests for {@link ImageToJsonTranslator}. */ public class ImageToJsonTranslatorTest { @@ -52,7 +52,7 @@ public class ImageToJsonTranslatorTest { @Before public void setUp() throws DigestException, LayerPropertyNotFoundException { - Image.Builder testImageBuilder = Image.builder(); + Image.Builder testImageBuilder = Image.builder(); testImageBuilder.setCreated(Instant.ofEpochSecond(20)); testImageBuilder.addEnvironmentVariable("VAR1", "VAL1"); @@ -67,9 +67,24 @@ public void setUp() throws DigestException, LayerPropertyNotFoundException { DescriptorDigest fakeDigest = DescriptorDigest.fromDigest( "sha256:8c662931926fa990b41da3c9f42663a537ccd498130030f9149173a0493832ad"); - CachedLayer fakeLayer = - new CachedLayer(Mockito.mock(Path.class), new BlobDescriptor(1000, fakeDigest), fakeDigest); - testImageBuilder.addLayer(fakeLayer); + testImageBuilder.addLayer( + new Layer() { + + @Override + public Blob getBlob() throws LayerPropertyNotFoundException { + return Blobs.from("ignored"); + } + + @Override + public BlobDescriptor getBlobDescriptor() throws LayerPropertyNotFoundException { + return new BlobDescriptor(1000, fakeDigest); + } + + @Override + public DescriptorDigest getDiffId() throws LayerPropertyNotFoundException { + return fakeDigest; + } + }); testImageBuilder.addHistory( HistoryEntry.builder() .setCreationTimestamp(Instant.EPOCH) diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/tar/TarStreamBuilderTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/tar/TarStreamBuilderTest.java index 2ea09946a2..4486e9d514 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/tar/TarStreamBuilderTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/tar/TarStreamBuilderTest.java @@ -17,14 +17,13 @@ package com.google.cloud.tools.jib.tar; import com.google.cloud.tools.jib.blob.Blob; +import com.google.cloud.tools.jib.blob.Blobs; import com.google.common.io.ByteStreams; -import com.google.common.io.CharStreams; import com.google.common.io.Resources; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; import java.io.OutputStream; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; @@ -100,6 +99,8 @@ public void testToBlob_stringsAndTarArchiveEntriesWithCompression() throws IOExc public void testToBlob_multiByte() throws IOException { testTarStreamBuilder.addByteEntry("日本語".getBytes(StandardCharsets.UTF_8), "test"); testTarStreamBuilder.addByteEntry("asdf".getBytes(StandardCharsets.UTF_8), "crepecake"); + testTarStreamBuilder.addBlobEntry( + Blobs.from("jib"), "jib".getBytes(StandardCharsets.UTF_8).length, "jib"); Blob blob = testTarStreamBuilder.toBlob(); // Writes the BLOB and captures the output. @@ -116,15 +117,18 @@ public void testToBlob_multiByte() throws IOException { // Verify multi-byte characters are written/read correctly TarArchiveEntry headerFile = tarArchiveInputStream.getNextTarEntry(); Assert.assertEquals("test", headerFile.getName()); - String fileString = - CharStreams.toString(new InputStreamReader(tarArchiveInputStream, StandardCharsets.UTF_8)); - Assert.assertEquals("日本語", fileString); + Assert.assertEquals( + "日本語", new String(ByteStreams.toByteArray(tarArchiveInputStream), StandardCharsets.UTF_8)); headerFile = tarArchiveInputStream.getNextTarEntry(); Assert.assertEquals("crepecake", headerFile.getName()); - fileString = - CharStreams.toString(new InputStreamReader(tarArchiveInputStream, StandardCharsets.UTF_8)); - Assert.assertEquals("asdf", fileString); + Assert.assertEquals( + "asdf", new String(ByteStreams.toByteArray(tarArchiveInputStream), StandardCharsets.UTF_8)); + + headerFile = tarArchiveInputStream.getNextTarEntry(); + Assert.assertEquals("jib", headerFile.getName()); + Assert.assertEquals( + "jib", new String(ByteStreams.toByteArray(tarArchiveInputStream), StandardCharsets.UTF_8)); Assert.assertNull(tarArchiveInputStream.getNextTarEntry()); } diff --git a/jib-gradle-plugin/src/main/java/com/google/cloud/tools/jib/gradle/BuildDockerTask.java b/jib-gradle-plugin/src/main/java/com/google/cloud/tools/jib/gradle/BuildDockerTask.java index c4e0dfbd7a..ef48ff9395 100644 --- a/jib-gradle-plugin/src/main/java/com/google/cloud/tools/jib/gradle/BuildDockerTask.java +++ b/jib-gradle-plugin/src/main/java/com/google/cloud/tools/jib/gradle/BuildDockerTask.java @@ -16,8 +16,8 @@ package com.google.cloud.tools.jib.gradle; -import com.google.cloud.tools.jib.cache.CacheDirectoryCreationException; import com.google.cloud.tools.jib.configuration.BuildConfiguration; +import com.google.cloud.tools.jib.configuration.CacheDirectoryCreationException; import com.google.cloud.tools.jib.configuration.ImageConfiguration; import com.google.cloud.tools.jib.docker.DockerClient; import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; @@ -28,6 +28,7 @@ import com.google.cloud.tools.jib.plugins.common.ConfigurationPropertyValidator; import com.google.cloud.tools.jib.plugins.common.HelpfulSuggestions; import com.google.common.base.Preconditions; +import java.io.IOException; import javax.annotation.Nullable; import org.gradle.api.DefaultTask; import org.gradle.api.GradleException; @@ -65,7 +66,9 @@ public void setTargetImage(String targetImage) { } @TaskAction - public void buildDocker() throws InvalidImageReferenceException { + public void buildDocker() + throws InvalidImageReferenceException, IOException, CacheDirectoryCreationException, + BuildStepsExecutionException { if (!new DockerClient().isDockerInstalled()) { throw new GradleException( HelpfulSuggestions.forDockerNotInstalled(HELPFUL_SUGGESTIONS_PREFIX)); @@ -112,13 +115,7 @@ public void buildDocker() throws InvalidImageReferenceException { .setTargetImageReference(buildConfiguration.getTargetImageConfiguration().getImage()) .build(); - // Uses a directory in the Gradle build cache as the Jib cache. - try { - BuildStepsRunner.forBuildToDockerDaemon(buildConfiguration).build(helpfulSuggestions); - - } catch (CacheDirectoryCreationException | BuildStepsExecutionException ex) { - throw new GradleException(ex.getMessage(), ex.getCause()); - } + BuildStepsRunner.forBuildToDockerDaemon(buildConfiguration).build(helpfulSuggestions); } @Override diff --git a/jib-gradle-plugin/src/main/java/com/google/cloud/tools/jib/gradle/BuildImageTask.java b/jib-gradle-plugin/src/main/java/com/google/cloud/tools/jib/gradle/BuildImageTask.java index e18073839c..ec85cb6a57 100644 --- a/jib-gradle-plugin/src/main/java/com/google/cloud/tools/jib/gradle/BuildImageTask.java +++ b/jib-gradle-plugin/src/main/java/com/google/cloud/tools/jib/gradle/BuildImageTask.java @@ -16,8 +16,8 @@ package com.google.cloud.tools.jib.gradle; -import com.google.cloud.tools.jib.cache.CacheDirectoryCreationException; import com.google.cloud.tools.jib.configuration.BuildConfiguration; +import com.google.cloud.tools.jib.configuration.CacheDirectoryCreationException; import com.google.cloud.tools.jib.configuration.ImageConfiguration; import com.google.cloud.tools.jib.configuration.credentials.Credential; import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; @@ -31,6 +31,7 @@ import com.google.cloud.tools.jib.plugins.common.HelpfulSuggestions; import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import java.io.IOException; import java.util.Optional; import javax.annotation.Nullable; import org.gradle.api.DefaultTask; @@ -69,7 +70,9 @@ public void setTargetImage(String targetImage) { } @TaskAction - public void buildImage() throws InvalidImageReferenceException { + public void buildImage() + throws InvalidImageReferenceException, IOException, CacheDirectoryCreationException, + BuildStepsExecutionException { // Asserts required @Input parameters are not null. Preconditions.checkNotNull(jibExtension); AbsoluteUnixPath appRoot = PluginConfigurationProcessor.getAppRootChecked(jibExtension); @@ -133,12 +136,7 @@ public void buildImage() throws InvalidImageReferenceException { .setTargetImageHasConfiguredCredentials(optionalToCredential.isPresent()) .build(); - try { - BuildStepsRunner.forBuildImage(buildConfiguration).build(helpfulSuggestions); - - } catch (CacheDirectoryCreationException | BuildStepsExecutionException ex) { - throw new GradleException(ex.getMessage(), ex.getCause()); - } + BuildStepsRunner.forBuildImage(buildConfiguration).build(helpfulSuggestions); } @Override diff --git a/jib-gradle-plugin/src/main/java/com/google/cloud/tools/jib/gradle/BuildTarTask.java b/jib-gradle-plugin/src/main/java/com/google/cloud/tools/jib/gradle/BuildTarTask.java index 703a864f50..e62af1974a 100644 --- a/jib-gradle-plugin/src/main/java/com/google/cloud/tools/jib/gradle/BuildTarTask.java +++ b/jib-gradle-plugin/src/main/java/com/google/cloud/tools/jib/gradle/BuildTarTask.java @@ -16,8 +16,8 @@ package com.google.cloud.tools.jib.gradle; -import com.google.cloud.tools.jib.cache.CacheDirectoryCreationException; import com.google.cloud.tools.jib.configuration.BuildConfiguration; +import com.google.cloud.tools.jib.configuration.CacheDirectoryCreationException; import com.google.cloud.tools.jib.configuration.ImageConfiguration; import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; import com.google.cloud.tools.jib.image.ImageReference; @@ -27,10 +27,10 @@ import com.google.cloud.tools.jib.plugins.common.ConfigurationPropertyValidator; import com.google.cloud.tools.jib.plugins.common.HelpfulSuggestions; import com.google.common.base.Preconditions; +import java.io.IOException; import java.nio.file.Paths; import javax.annotation.Nullable; import org.gradle.api.DefaultTask; -import org.gradle.api.GradleException; import org.gradle.api.file.FileCollection; import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.Nested; @@ -97,7 +97,9 @@ private String getTargetPath() { } @TaskAction - public void buildTar() throws InvalidImageReferenceException { + public void buildTar() + throws InvalidImageReferenceException, BuildStepsExecutionException, IOException, + CacheDirectoryCreationException { // Asserts required @Input parameters are not null. Preconditions.checkNotNull(jibExtension); AbsoluteUnixPath appRoot = PluginConfigurationProcessor.getAppRootChecked(jibExtension); @@ -137,14 +139,8 @@ public void buildTar() throws InvalidImageReferenceException { .setTargetImageReference(buildConfiguration.getTargetImageConfiguration().getImage()) .build(); - // Uses a directory in the Gradle build cache as the Jib cache. - try { - BuildStepsRunner.forBuildTar(Paths.get(getTargetPath()), buildConfiguration) - .build(helpfulSuggestions); - - } catch (CacheDirectoryCreationException | BuildStepsExecutionException ex) { - throw new GradleException(ex.getMessage(), ex.getCause()); - } + BuildStepsRunner.forBuildTar(Paths.get(getTargetPath()), buildConfiguration) + .build(helpfulSuggestions); } @Override diff --git a/jib-maven-plugin/src/main/java/com/google/cloud/tools/jib/maven/BuildDockerMojo.java b/jib-maven-plugin/src/main/java/com/google/cloud/tools/jib/maven/BuildDockerMojo.java index af2240bf9a..8174195e0c 100644 --- a/jib-maven-plugin/src/main/java/com/google/cloud/tools/jib/maven/BuildDockerMojo.java +++ b/jib-maven-plugin/src/main/java/com/google/cloud/tools/jib/maven/BuildDockerMojo.java @@ -16,8 +16,8 @@ package com.google.cloud.tools.jib.maven; -import com.google.cloud.tools.jib.cache.CacheDirectoryCreationException; import com.google.cloud.tools.jib.configuration.BuildConfiguration; +import com.google.cloud.tools.jib.configuration.CacheDirectoryCreationException; import com.google.cloud.tools.jib.configuration.ImageConfiguration; import com.google.cloud.tools.jib.docker.DockerClient; import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; @@ -28,6 +28,7 @@ import com.google.cloud.tools.jib.plugins.common.ConfigurationPropertyValidator; import com.google.cloud.tools.jib.plugins.common.HelpfulSuggestions; import com.google.common.annotations.VisibleForTesting; +import java.io.IOException; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugins.annotations.Mojo; import org.apache.maven.plugins.annotations.ResolutionScope; @@ -100,9 +101,10 @@ public void execute() throws MojoExecutionException { BuildStepsRunner.forBuildToDockerDaemon(buildConfiguration).build(helpfulSuggestions); getLog().info(""); - } catch (CacheDirectoryCreationException - | BuildStepsExecutionException - | InvalidImageReferenceException ex) { + } catch (CacheDirectoryCreationException | InvalidImageReferenceException | IOException ex) { + throw new MojoExecutionException(ex.getMessage(), ex); + + } catch (BuildStepsExecutionException ex) { throw new MojoExecutionException(ex.getMessage(), ex.getCause()); } } diff --git a/jib-maven-plugin/src/main/java/com/google/cloud/tools/jib/maven/BuildImageMojo.java b/jib-maven-plugin/src/main/java/com/google/cloud/tools/jib/maven/BuildImageMojo.java index a88b5c11a3..fde58f5827 100644 --- a/jib-maven-plugin/src/main/java/com/google/cloud/tools/jib/maven/BuildImageMojo.java +++ b/jib-maven-plugin/src/main/java/com/google/cloud/tools/jib/maven/BuildImageMojo.java @@ -16,8 +16,8 @@ package com.google.cloud.tools.jib.maven; -import com.google.cloud.tools.jib.cache.CacheDirectoryCreationException; import com.google.cloud.tools.jib.configuration.BuildConfiguration; +import com.google.cloud.tools.jib.configuration.CacheDirectoryCreationException; import com.google.cloud.tools.jib.configuration.ImageConfiguration; import com.google.cloud.tools.jib.configuration.credentials.Credential; import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; @@ -31,6 +31,7 @@ import com.google.cloud.tools.jib.plugins.common.HelpfulSuggestions; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; +import java.io.IOException; import java.util.Arrays; import java.util.Optional; import org.apache.maven.plugin.MojoExecutionException; @@ -122,32 +123,35 @@ public void execute() throws MojoExecutionException, MojoFailureException { .setCredentialRetrievers(defaultCredentialRetrievers.asList()) .build(); - BuildConfiguration buildConfiguration = - pluginConfigurationProcessor - .getBuildConfigurationBuilder() - .setBaseImageConfiguration( - pluginConfigurationProcessor.getBaseImageConfigurationBuilder().build()) - .setTargetImageConfiguration(targetImageConfiguration) - .setAdditionalTargetImageTags(getTargetImageAdditionalTags()) - .setContainerConfiguration( - pluginConfigurationProcessor.getContainerConfigurationBuilder().build()) - .setTargetFormat(ImageFormat.valueOf(getFormat()).getManifestTemplateClass()) - .build(); - - HelpfulSuggestions helpfulSuggestions = - new MavenHelpfulSuggestionsBuilder(HELPFUL_SUGGESTIONS_PREFIX, this) - .setBaseImageReference(buildConfiguration.getBaseImageConfiguration().getImage()) - .setBaseImageHasConfiguredCredentials( - pluginConfigurationProcessor.isBaseImageCredentialPresent()) - .setTargetImageReference(buildConfiguration.getTargetImageConfiguration().getImage()) - .setTargetImageHasConfiguredCredentials(optionalToCredential.isPresent()) - .build(); - try { + BuildConfiguration buildConfiguration = + pluginConfigurationProcessor + .getBuildConfigurationBuilder() + .setBaseImageConfiguration( + pluginConfigurationProcessor.getBaseImageConfigurationBuilder().build()) + .setTargetImageConfiguration(targetImageConfiguration) + .setAdditionalTargetImageTags(getTargetImageAdditionalTags()) + .setContainerConfiguration( + pluginConfigurationProcessor.getContainerConfigurationBuilder().build()) + .setTargetFormat(ImageFormat.valueOf(getFormat()).getManifestTemplateClass()) + .build(); + + HelpfulSuggestions helpfulSuggestions = + new MavenHelpfulSuggestionsBuilder(HELPFUL_SUGGESTIONS_PREFIX, this) + .setBaseImageReference(buildConfiguration.getBaseImageConfiguration().getImage()) + .setBaseImageHasConfiguredCredentials( + pluginConfigurationProcessor.isBaseImageCredentialPresent()) + .setTargetImageReference(buildConfiguration.getTargetImageConfiguration().getImage()) + .setTargetImageHasConfiguredCredentials(optionalToCredential.isPresent()) + .build(); + BuildStepsRunner.forBuildImage(buildConfiguration).build(helpfulSuggestions); getLog().info(""); - } catch (CacheDirectoryCreationException | BuildStepsExecutionException ex) { + } catch (CacheDirectoryCreationException | IOException ex) { + throw new MojoExecutionException(ex.getMessage(), ex); + + } catch (BuildStepsExecutionException ex) { throw new MojoExecutionException(ex.getMessage(), ex.getCause()); } } diff --git a/jib-maven-plugin/src/main/java/com/google/cloud/tools/jib/maven/BuildTarMojo.java b/jib-maven-plugin/src/main/java/com/google/cloud/tools/jib/maven/BuildTarMojo.java index 5ea2adf1d3..214cefa755 100644 --- a/jib-maven-plugin/src/main/java/com/google/cloud/tools/jib/maven/BuildTarMojo.java +++ b/jib-maven-plugin/src/main/java/com/google/cloud/tools/jib/maven/BuildTarMojo.java @@ -16,8 +16,8 @@ package com.google.cloud.tools.jib.maven; -import com.google.cloud.tools.jib.cache.CacheDirectoryCreationException; import com.google.cloud.tools.jib.configuration.BuildConfiguration; +import com.google.cloud.tools.jib.configuration.CacheDirectoryCreationException; import com.google.cloud.tools.jib.configuration.ImageConfiguration; import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; import com.google.cloud.tools.jib.image.ImageReference; @@ -27,6 +27,7 @@ import com.google.cloud.tools.jib.plugins.common.ConfigurationPropertyValidator; import com.google.cloud.tools.jib.plugins.common.HelpfulSuggestions; import com.google.common.annotations.VisibleForTesting; +import java.io.IOException; import java.nio.file.Paths; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugins.annotations.Mojo; @@ -99,9 +100,10 @@ public void execute() throws MojoExecutionException { .build(helpfulSuggestions); getLog().info(""); - } catch (CacheDirectoryCreationException - | BuildStepsExecutionException - | InvalidImageReferenceException ex) { + } catch (CacheDirectoryCreationException | InvalidImageReferenceException | IOException ex) { + throw new MojoExecutionException(ex.getMessage(), ex); + + } catch (BuildStepsExecutionException ex) { throw new MojoExecutionException(ex.getMessage(), ex.getCause()); } } diff --git a/jib-plugins-common/src/main/java/com/google/cloud/tools/jib/plugins/common/BuildStepsRunner.java b/jib-plugins-common/src/main/java/com/google/cloud/tools/jib/plugins/common/BuildStepsRunner.java index 1df8208c16..93b92fb40f 100644 --- a/jib-plugins-common/src/main/java/com/google/cloud/tools/jib/plugins/common/BuildStepsRunner.java +++ b/jib-plugins-common/src/main/java/com/google/cloud/tools/jib/plugins/common/BuildStepsRunner.java @@ -19,12 +19,7 @@ import com.google.api.client.http.HttpResponseException; import com.google.api.client.http.HttpStatusCodes; import com.google.cloud.tools.jib.builder.BuildSteps; -import com.google.cloud.tools.jib.cache.CacheDirectoryCreationException; -import com.google.cloud.tools.jib.cache.CacheDirectoryNotOwnedException; -import com.google.cloud.tools.jib.cache.CacheMetadataCorruptedException; -import com.google.cloud.tools.jib.cache.Caches.Initializer; import com.google.cloud.tools.jib.configuration.BuildConfiguration; -import com.google.cloud.tools.jib.configuration.CacheConfiguration; import com.google.cloud.tools.jib.configuration.LayerConfiguration; import com.google.cloud.tools.jib.event.EventDispatcher; import com.google.cloud.tools.jib.event.events.LogEvent; @@ -37,7 +32,6 @@ import com.google.cloud.tools.jib.registry.RegistryUnauthorizedException; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Verify; -import java.io.IOException; import java.net.UnknownHostException; import java.nio.file.Path; import java.util.StringJoiner; @@ -84,13 +78,10 @@ private static String buildMessageWithTargetImageReferences( * * @param buildConfiguration the configuration parameters for the build * @return a {@link BuildStepsRunner} for building to a registry - * @throws CacheDirectoryCreationException if the {@code cacheDirectory} could not be created */ - public static BuildStepsRunner forBuildImage(BuildConfiguration buildConfiguration) - throws CacheDirectoryCreationException { + public static BuildStepsRunner forBuildImage(BuildConfiguration buildConfiguration) { return new BuildStepsRunner( - BuildSteps.forBuildToDockerRegistry( - buildConfiguration, getCacheInitializer(buildConfiguration)), + BuildSteps.forBuildToDockerRegistry(buildConfiguration), buildMessageWithTargetImageReferences( buildConfiguration, STARTUP_MESSAGE_PREFIX_FOR_DOCKER_REGISTRY, "..."), buildMessageWithTargetImageReferences( @@ -102,13 +93,10 @@ buildConfiguration, getCacheInitializer(buildConfiguration)), * * @param buildConfiguration the configuration parameters for the build * @return a {@link BuildStepsRunner} for building to a Docker daemon - * @throws CacheDirectoryCreationException if the {@code cacheDirectory} could not be created */ - public static BuildStepsRunner forBuildToDockerDaemon(BuildConfiguration buildConfiguration) - throws CacheDirectoryCreationException { + public static BuildStepsRunner forBuildToDockerDaemon(BuildConfiguration buildConfiguration) { return new BuildStepsRunner( - BuildSteps.forBuildToDockerDaemon( - buildConfiguration, getCacheInitializer(buildConfiguration)), + BuildSteps.forBuildToDockerDaemon(buildConfiguration), buildMessageWithTargetImageReferences( buildConfiguration, STARTUP_MESSAGE_PREFIX_FOR_DOCKER_DAEMON, "..."), buildMessageWithTargetImageReferences( @@ -121,37 +109,15 @@ buildConfiguration, getCacheInitializer(buildConfiguration)), * @param outputPath the path to output the tarball to * @param buildConfiguration the configuration parameters for the build * @return a {@link BuildStepsRunner} for building a tarball - * @throws CacheDirectoryCreationException if the {@code cacheDirectory} could not be created */ - public static BuildStepsRunner forBuildTar(Path outputPath, BuildConfiguration buildConfiguration) - throws CacheDirectoryCreationException { + public static BuildStepsRunner forBuildTar( + Path outputPath, BuildConfiguration buildConfiguration) { return new BuildStepsRunner( - BuildSteps.forBuildToTar( - outputPath, buildConfiguration, getCacheInitializer(buildConfiguration)), + BuildSteps.forBuildToTar(outputPath, buildConfiguration), String.format(STARTUP_MESSAGE_FORMAT_FOR_TARBALL, outputPath.toString()), String.format(SUCCESS_MESSAGE_FORMAT_FOR_TARBALL, outputPath.toString())); } - // TODO: Move this up to somewhere where defaults for cache location are provided and ownership is - // checked rather than in Caches.Initializer. - private static Initializer getCacheInitializer(BuildConfiguration buildConfiguration) - throws CacheDirectoryCreationException { - CacheConfiguration applicationLayersCacheConfiguration = - buildConfiguration.getApplicationLayersCacheConfiguration() == null - ? CacheConfiguration.makeTemporary() - : buildConfiguration.getApplicationLayersCacheConfiguration(); - CacheConfiguration baseImageLayersCacheConfiguration = - buildConfiguration.getBaseImageLayersCacheConfiguration() == null - ? CacheConfiguration.forDefaultUserLevelCacheDirectory() - : buildConfiguration.getBaseImageLayersCacheConfiguration(); - - return new Initializer( - baseImageLayersCacheConfiguration.getCacheDirectory(), - applicationLayersCacheConfiguration.shouldEnsureOwnership(), - applicationLayersCacheConfiguration.getCacheDirectory(), - applicationLayersCacheConfiguration.shouldEnsureOwnership()); - } - private static void handleRegistryUnauthorizedException( RegistryUnauthorizedException registryUnauthorizedException, HelpfulSuggestions helpfulSuggestions) @@ -227,10 +193,6 @@ public void build(HelpfulSuggestions helpfulSuggestions) throws BuildStepsExecut eventDispatcher.dispatch(LogEvent.lifecycle("")); eventDispatcher.dispatch(LogEvent.lifecycle(successMessage)); - } catch (CacheMetadataCorruptedException cacheMetadataCorruptedException) { - throw new BuildStepsExecutionException( - helpfulSuggestions.forCacheNeedsClean(), cacheMetadataCorruptedException); - } catch (ExecutionException executionException) { Throwable exceptionDuringBuildSteps = executionException.getCause(); @@ -279,21 +241,9 @@ public void build(HelpfulSuggestions helpfulSuggestions) throws BuildStepsExecut helpfulSuggestions.none(), executionException.getCause()); } - } catch (InterruptedException | IOException | CacheDirectoryCreationException ex) { + } catch (InterruptedException ex) { // TODO: Add more suggestions for various build failures. throw new BuildStepsExecutionException(helpfulSuggestions.none(), ex); - - } catch (CacheDirectoryNotOwnedException ex) { - String helpfulSuggestion = - helpfulSuggestions.forCacheDirectoryNotOwned(ex.getCacheDirectory()); - CacheConfiguration applicationLayersCacheConfiguration = - buildSteps.getBuildConfiguration().getApplicationLayersCacheConfiguration(); - if (applicationLayersCacheConfiguration != null - && ex.getCacheDirectory() - .equals(applicationLayersCacheConfiguration.getCacheDirectory())) { - helpfulSuggestion = helpfulSuggestions.forCacheNeedsClean(); - } - throw new BuildStepsExecutionException(helpfulSuggestion, ex); } } } diff --git a/jib-plugins-common/src/test/java/com/google/cloud/tools/jib/plugins/common/BuildStepsRunnerTest.java b/jib-plugins-common/src/test/java/com/google/cloud/tools/jib/plugins/common/BuildStepsRunnerTest.java index ea500e9f2c..f2b79a5a0e 100644 --- a/jib-plugins-common/src/test/java/com/google/cloud/tools/jib/plugins/common/BuildStepsRunnerTest.java +++ b/jib-plugins-common/src/test/java/com/google/cloud/tools/jib/plugins/common/BuildStepsRunnerTest.java @@ -19,11 +19,7 @@ import com.google.api.client.http.HttpResponseException; import com.google.api.client.http.HttpStatusCodes; import com.google.cloud.tools.jib.builder.BuildSteps; -import com.google.cloud.tools.jib.cache.CacheDirectoryCreationException; -import com.google.cloud.tools.jib.cache.CacheDirectoryNotOwnedException; -import com.google.cloud.tools.jib.cache.CacheMetadataCorruptedException; import com.google.cloud.tools.jib.configuration.BuildConfiguration; -import com.google.cloud.tools.jib.configuration.CacheConfiguration; import com.google.cloud.tools.jib.configuration.LayerConfiguration; import com.google.cloud.tools.jib.event.EventDispatcher; import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; @@ -32,9 +28,7 @@ import com.google.cloud.tools.jib.registry.RegistryCredentialsNotSentException; import com.google.cloud.tools.jib.registry.RegistryUnauthorizedException; import com.google.common.collect.ImmutableList; -import java.io.IOException; import java.net.UnknownHostException; -import java.nio.file.Path; import java.nio.file.Paths; import java.util.concurrent.ExecutionException; import org.apache.http.conn.HttpHostConnectException; @@ -105,28 +99,9 @@ public void testBuildImage_pass() throws BuildStepsExecutionException { testBuildImageStepsRunner.build(TEST_HELPFUL_SUGGESTIONS); } - @Test - public void testBuildImage_cacheMetadataCorruptedException() - throws InterruptedException, ExecutionException, CacheMetadataCorruptedException, IOException, - CacheDirectoryNotOwnedException, CacheDirectoryCreationException { - CacheMetadataCorruptedException mockCacheMetadataCorruptedException = - Mockito.mock(CacheMetadataCorruptedException.class); - Mockito.doThrow(mockCacheMetadataCorruptedException).when(mockBuildSteps).run(); - - try { - testBuildImageStepsRunner.build(TEST_HELPFUL_SUGGESTIONS); - Assert.fail("buildImage should have thrown an exception"); - - } catch (BuildStepsExecutionException ex) { - Assert.assertEquals(TEST_HELPFUL_SUGGESTIONS.forCacheNeedsClean(), ex.getMessage()); - Assert.assertEquals(mockCacheMetadataCorruptedException, ex.getCause()); - } - } - @Test public void testBuildImage_executionException_httpHostConnectException() - throws InterruptedException, ExecutionException, CacheMetadataCorruptedException, IOException, - CacheDirectoryNotOwnedException, CacheDirectoryCreationException { + throws InterruptedException, ExecutionException { HttpHostConnectException mockHttpHostConnectException = Mockito.mock(HttpHostConnectException.class); Mockito.when(mockExecutionException.getCause()).thenReturn(mockHttpHostConnectException); @@ -144,8 +119,7 @@ public void testBuildImage_executionException_httpHostConnectException() @Test public void testBuildImage_executionException_unknownHostException() - throws InterruptedException, ExecutionException, CacheMetadataCorruptedException, IOException, - CacheDirectoryNotOwnedException, CacheDirectoryCreationException { + throws InterruptedException, ExecutionException { UnknownHostException mockUnknownHostException = Mockito.mock(UnknownHostException.class); Mockito.when(mockExecutionException.getCause()).thenReturn(mockUnknownHostException); Mockito.doThrow(mockExecutionException).when(mockBuildSteps).run(); @@ -162,8 +136,7 @@ public void testBuildImage_executionException_unknownHostException() @Test public void testBuildImage_executionException_insecureRegistryException() - throws InterruptedException, ExecutionException, CacheDirectoryNotOwnedException, - CacheMetadataCorruptedException, IOException, CacheDirectoryCreationException { + throws InterruptedException, ExecutionException { InsecureRegistryException mockInsecureRegistryException = Mockito.mock(InsecureRegistryException.class); Mockito.when(mockExecutionException.getCause()).thenReturn(mockInsecureRegistryException); @@ -181,8 +154,7 @@ public void testBuildImage_executionException_insecureRegistryException() @Test public void testBuildImage_executionException_registryUnauthorizedException_statusCodeForbidden() - throws InterruptedException, ExecutionException, CacheMetadataCorruptedException, IOException, - CacheDirectoryNotOwnedException, CacheDirectoryCreationException { + throws InterruptedException, ExecutionException { Mockito.when(mockRegistryUnauthorizedException.getHttpResponseException()) .thenReturn(mockHttpResponseException); Mockito.when(mockRegistryUnauthorizedException.getImageReference()) @@ -207,8 +179,7 @@ public void testBuildImage_executionException_registryUnauthorizedException_stat @Test public void testBuildImage_executionException_registryUnauthorizedException_noCredentials() - throws InterruptedException, ExecutionException, CacheMetadataCorruptedException, IOException, - CacheDirectoryNotOwnedException, CacheDirectoryCreationException { + throws InterruptedException, ExecutionException { Mockito.when(mockRegistryUnauthorizedException.getHttpResponseException()) .thenReturn(mockHttpResponseException); Mockito.when(mockRegistryUnauthorizedException.getRegistry()).thenReturn("someregistry"); @@ -232,8 +203,7 @@ public void testBuildImage_executionException_registryUnauthorizedException_noCr @Test public void testBuildImage_executionException_registryCredentialsNotSentException() - throws InterruptedException, ExecutionException, CacheMetadataCorruptedException, IOException, - CacheDirectoryNotOwnedException, CacheDirectoryCreationException { + throws InterruptedException, ExecutionException { Mockito.when(mockExecutionException.getCause()) .thenReturn(mockRegistryCredentialsNotSentException); Mockito.doThrow(mockExecutionException).when(mockBuildSteps).run(); @@ -250,8 +220,7 @@ public void testBuildImage_executionException_registryCredentialsNotSentExceptio @Test public void testBuildImage_executionException_other() - throws InterruptedException, ExecutionException, CacheMetadataCorruptedException, IOException, - CacheDirectoryNotOwnedException, CacheDirectoryCreationException { + throws InterruptedException, ExecutionException { Throwable throwable = new Throwable(); Mockito.when(mockExecutionException.getCause()).thenReturn(throwable); Mockito.doThrow(mockExecutionException).when(mockBuildSteps).run(); @@ -265,73 +234,4 @@ public void testBuildImage_executionException_other() Assert.assertEquals(throwable, ex.getCause()); } } - - @Test - public void testBuildImage_otherException() - throws InterruptedException, ExecutionException, CacheMetadataCorruptedException, IOException, - CacheDirectoryNotOwnedException, CacheDirectoryCreationException { - IOException ioException = new IOException(); - Mockito.doThrow(ioException).when(mockBuildSteps).run(); - - try { - testBuildImageStepsRunner.build(TEST_HELPFUL_SUGGESTIONS); - Assert.fail("buildImage should have thrown an exception"); - - } catch (BuildStepsExecutionException ex) { - Assert.assertEquals(TEST_HELPFUL_SUGGESTIONS.none(), ex.getMessage()); - Assert.assertEquals(ioException, ex.getCause()); - } - } - - @Test - public void testBuildImage_cacheDirectoryNotOwnedException_needsClean() - throws InterruptedException, ExecutionException, CacheDirectoryNotOwnedException, - CacheMetadataCorruptedException, IOException, CacheDirectoryCreationException { - Path expectedCacheDirectory = Paths.get("some/path"); - - CacheDirectoryNotOwnedException mockCacheDirectoryNotOwnedException = - Mockito.mock(CacheDirectoryNotOwnedException.class); - Mockito.when(mockCacheDirectoryNotOwnedException.getCacheDirectory()) - .thenReturn(expectedCacheDirectory); - Mockito.doThrow(mockCacheDirectoryNotOwnedException).when(mockBuildSteps).run(); - - Mockito.when(mockBuildConfiguration.getApplicationLayersCacheConfiguration()) - .thenReturn(CacheConfiguration.forPath(expectedCacheDirectory)); - - try { - testBuildImageStepsRunner.build(TEST_HELPFUL_SUGGESTIONS); - Assert.fail("buildImage should have thrown an exception"); - - } catch (BuildStepsExecutionException ex) { - Assert.assertEquals(TEST_HELPFUL_SUGGESTIONS.forCacheNeedsClean(), ex.getMessage()); - Assert.assertEquals(mockCacheDirectoryNotOwnedException, ex.getCause()); - } - } - - @Test - public void testBuildImage_cacheDirectoryNotOwnedException() - throws InterruptedException, ExecutionException, CacheDirectoryNotOwnedException, - CacheMetadataCorruptedException, IOException, CacheDirectoryCreationException { - Path expectedCacheDirectory = Paths.get("some/path"); - - CacheDirectoryNotOwnedException mockCacheDirectoryNotOwnedException = - Mockito.mock(CacheDirectoryNotOwnedException.class); - Mockito.when(mockCacheDirectoryNotOwnedException.getCacheDirectory()) - .thenReturn(expectedCacheDirectory); - Mockito.doThrow(mockCacheDirectoryNotOwnedException).when(mockBuildSteps).run(); - - Mockito.when(mockBuildConfiguration.getApplicationLayersCacheConfiguration()) - .thenReturn(CacheConfiguration.forPath(Paths.get("another/path"))); - - try { - testBuildImageStepsRunner.build(TEST_HELPFUL_SUGGESTIONS); - Assert.fail("buildImage should have thrown an exception"); - - } catch (BuildStepsExecutionException ex) { - Assert.assertEquals( - TEST_HELPFUL_SUGGESTIONS.forCacheDirectoryNotOwned(expectedCacheDirectory), - ex.getMessage()); - Assert.assertEquals(mockCacheDirectoryNotOwnedException, ex.getCause()); - } - } }