From 0688f288a88d93e2684d19552c2cfb98a84e0987 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Fri, 19 Apr 2019 11:35:13 -0400 Subject: [PATCH 01/39] Decouple Blob --- .../jib/builder/steps/LoadDockerStep.java | 3 +-- .../jib/builder/steps/WriteTarFileStep.java | 4 +-- .../cloud/tools/jib/docker/DockerClient.java | 8 +++--- .../jib/docker/ImageToTarballTranslator.java | 11 +++++--- .../tools/jib/docker/DockerClientTest.java | 27 ++++++++++++++----- .../docker/ImageToTarballTranslatorTest.java | 19 ++++++------- 6 files changed, 46 insertions(+), 26 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/LoadDockerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/LoadDockerStep.java index 02bf62b57b..27f2bb3a8f 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/LoadDockerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/LoadDockerStep.java @@ -106,8 +106,7 @@ private BuildResult afterPushBaseImageLayerFuturesFuture() .getEventDispatcher() .dispatch( LogEvent.debug( - dockerClient.load( - new ImageToTarballTranslator(image).toTarballBlob(targetImageReference)))); + dockerClient.load(new ImageToTarballTranslator(image, targetImageReference)))); // Tags the image with all the additional tags, skipping the one 'docker load' already loaded. for (String tag : buildConfiguration.getAllTargetImageTags()) { diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/WriteTarFileStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/WriteTarFileStep.java index da3089a01c..1f97c6c778 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/WriteTarFileStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/WriteTarFileStep.java @@ -103,8 +103,8 @@ private BuildResult writeTarFile() throws ExecutionException, IOException { Files.createDirectories(outputPath.getParent()); try (OutputStream outputStream = new BufferedOutputStream(FileOperations.newLockingOutputStream(outputPath))) { - new ImageToTarballTranslator(image) - .toTarballBlob(buildConfiguration.getTargetImageConfiguration().getImage()) + new ImageToTarballTranslator( + image, buildConfiguration.getTargetImageConfiguration().getImage()) .writeTo(outputStream); } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/docker/DockerClient.java b/jib-core/src/main/java/com/google/cloud/tools/jib/docker/DockerClient.java index af1c4df2a5..d466d04cc2 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/docker/DockerClient.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/docker/DockerClient.java @@ -16,7 +16,6 @@ package com.google.cloud.tools.jib.docker; -import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.image.ImageReference; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; @@ -164,18 +163,19 @@ public static boolean isDockerInstalled(Path dockerExecutable) { * * @see https://docs.docker.com/engine/reference/commandline/load - * @param imageTarballBlob the built container tarball. + * @param imageTarball the built container tarball. * @return stdout from {@code docker}. * @throws InterruptedException if the 'docker load' process is interrupted. * @throws IOException if streaming the blob to 'docker load' fails. */ - public String load(Blob imageTarballBlob) throws InterruptedException, IOException { + public String load(ImageToTarballTranslator imageTarball) + throws InterruptedException, IOException { // Runs 'docker load'. Process dockerProcess = docker("load"); try (OutputStream stdin = dockerProcess.getOutputStream()) { try { - imageTarballBlob.writeTo(stdin); + imageTarball.writeTo(stdin); } catch (IOException ex) { // Tries to read from stderr. diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageToTarballTranslator.java b/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageToTarballTranslator.java index c87e876a12..cf98ddbe3f 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageToTarballTranslator.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageToTarballTranslator.java @@ -26,6 +26,7 @@ import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.cloud.tools.jib.tar.TarStreamBuilder; import java.io.IOException; +import java.io.OutputStream; import java.util.Collections; /** Translates an {@link Image} to a tarball that can be loaded into Docker. */ @@ -42,16 +43,20 @@ public class ImageToTarballTranslator { private final Image image; + private final ImageReference imageReference; + /** * Instantiate with an {@link Image}. * * @param image the image to convert into a tarball + * @param imageReference image reference to set in the manifest */ - public ImageToTarballTranslator(Image image) { + public ImageToTarballTranslator(Image image, ImageReference imageReference) { this.image = image; + this.imageReference = imageReference; } - public Blob toTarballBlob(ImageReference imageReference) throws IOException { + public void writeTo(OutputStream out) throws IOException { TarStreamBuilder tarStreamBuilder = new TarStreamBuilder(); DockerLoadManifestEntryTemplate manifestTemplate = new DockerLoadManifestEntryTemplate(); @@ -77,6 +82,6 @@ public Blob toTarballBlob(ImageReference imageReference) throws IOException { JsonTemplateMapper.toBlob(Collections.singletonList(manifestTemplate))), MANIFEST_JSON_FILE_NAME); - return tarStreamBuilder.toBlob(); + tarStreamBuilder.toBlob().writeTo(out); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/DockerClientTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/DockerClientTest.java index b1ccf8c0f7..c151a72a10 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/DockerClientTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/DockerClientTest.java @@ -16,7 +16,6 @@ package com.google.cloud.tools.jib.docker; -import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.image.ImageReference; import com.google.cloud.tools.jib.image.InvalidImageReferenceException; import com.google.common.collect.ImmutableMap; @@ -38,22 +37,32 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; +import org.mockito.AdditionalAnswers; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.VoidAnswer1; /** Tests for {@link DockerClient}. */ @RunWith(MockitoJUnitRunner.class) public class DockerClientTest { - @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); @Mock private ProcessBuilder mockProcessBuilder; @Mock private Process mockProcess; + @Mock private ImageToTarballTranslator imageTarball; @Before public void setUp() throws IOException { Mockito.when(mockProcessBuilder.start()).thenReturn(mockProcess); + + Mockito.doAnswer( + AdditionalAnswers.answerVoid( + (VoidAnswer1) + out -> out.write("jib".getBytes(StandardCharsets.UTF_8)))) + .when(imageTarball) + .writeTo(Mockito.any(OutputStream.class)); } @Test @@ -79,7 +88,13 @@ public void testLoad() throws IOException, InterruptedException { Mockito.when(mockProcess.getInputStream()) .thenReturn(new ByteArrayInputStream("output".getBytes(StandardCharsets.UTF_8))); - String output = testDockerClient.load(Blobs.from("jib")); + Mockito.doAnswer( + AdditionalAnswers.answerVoid( + (VoidAnswer1) + out -> out.write("jib".getBytes(StandardCharsets.UTF_8)))) + .when(imageTarball) + .writeTo(Mockito.any(OutputStream.class)); + String output = testDockerClient.load(imageTarball); Assert.assertEquals( "jib", new String(byteArrayOutputStream.toByteArray(), StandardCharsets.UTF_8)); @@ -103,7 +118,7 @@ public void write(int b) throws IOException { .thenReturn(new ByteArrayInputStream("error".getBytes(StandardCharsets.UTF_8))); try { - testDockerClient.load(Blobs.from("jib")); + testDockerClient.load(imageTarball); Assert.fail("Write should have failed"); } catch (IOException ex) { @@ -136,7 +151,7 @@ public int read() throws IOException { }); try { - testDockerClient.load(Blobs.from("jib")); + testDockerClient.load(imageTarball); Assert.fail("Write should have failed"); } catch (IOException ex) { @@ -156,7 +171,7 @@ public void testLoad_stdoutFail() throws InterruptedException { .thenReturn(new ByteArrayInputStream("error".getBytes(StandardCharsets.UTF_8))); try { - testDockerClient.load(Blobs.from("jib")); + testDockerClient.load(imageTarball); Assert.fail("Process should have failed"); } catch (IOException ex) { diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageToTarballTranslatorTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageToTarballTranslatorTest.java index 0759ef9e18..4edaab2cf9 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageToTarballTranslatorTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageToTarballTranslatorTest.java @@ -16,7 +16,6 @@ package com.google.cloud.tools.jib.docker; -import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.docker.json.DockerLoadManifestEntryTemplate; @@ -31,9 +30,10 @@ import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.io.CharStreams; import com.google.common.io.Resources; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStreamReader; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -57,7 +57,7 @@ public class ImageToTarballTranslatorTest { @Mock private Layer mockLayer2; @Test - public void testToTarballBlob() + public void testWriteTo() throws InvalidImageReferenceException, IOException, URISyntaxException, LayerPropertyNotFoundException, DigestException { Path fileA = Paths.get(Resources.getResource("core/fileA").toURI()); @@ -83,13 +83,14 @@ public void testToTarballBlob() Image testImage = Image.builder(V22ManifestTemplate.class).addLayer(mockLayer1).addLayer(mockLayer2).build(); - Blob tarballBlob = - new ImageToTarballTranslator(testImage).toTarballBlob(ImageReference.parse("my/image:tag")); + ImageToTarballTranslator imageToTarball = + new ImageToTarballTranslator(testImage, ImageReference.parse("my/image:tag")); + + try (PipedInputStream in = new PipedInputStream(); + PipedOutputStream out = new PipedOutputStream(in); + TarArchiveInputStream tarArchiveInputStream = new TarArchiveInputStream(in)) { + imageToTarball.writeTo(out); - try (ByteArrayInputStream tarballBytesStream = - new ByteArrayInputStream(Blobs.writeToByteArray(tarballBlob)); - TarArchiveInputStream tarArchiveInputStream = - new TarArchiveInputStream(tarballBytesStream)) { // Verifies layer with fileA was added. TarArchiveEntry headerFileALayer = tarArchiveInputStream.getNextTarEntry(); Assert.assertEquals(fakeDigestA.getHash() + ".tar.gz", headerFileALayer.getName()); From ec340f310ffd0ad885ffd92147a9c5733bc54d21 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Fri, 19 Apr 2019 11:44:56 -0400 Subject: [PATCH 02/39] Rename ImageToTarTranslator --- .../jib/builder/steps/LoadDockerStep.java | 5 ++--- .../jib/builder/steps/WriteTarFileStep.java | 5 ++--- .../cloud/tools/jib/docker/DockerClient.java | 3 +-- ...arballTranslator.java => ImageTarball.java} | 4 ++-- .../tools/jib/docker/DockerClientTest.java | 2 +- ...slatorTest.java => ImageToTarballTest.java} | 18 +++++++++--------- 6 files changed, 17 insertions(+), 20 deletions(-) rename jib-core/src/main/java/com/google/cloud/tools/jib/docker/{ImageToTarballTranslator.java => ImageTarball.java} (96%) rename jib-core/src/test/java/com/google/cloud/tools/jib/docker/{ImageToTarballTranslatorTest.java => ImageToTarballTest.java} (91%) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/LoadDockerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/LoadDockerStep.java index 27f2bb3a8f..dc15cddcfb 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/LoadDockerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/LoadDockerStep.java @@ -23,7 +23,7 @@ import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.docker.DockerClient; -import com.google.cloud.tools.jib.docker.ImageToTarballTranslator; +import com.google.cloud.tools.jib.docker.ImageTarball; import com.google.cloud.tools.jib.event.events.LogEvent; import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.ImageReference; @@ -105,8 +105,7 @@ private BuildResult afterPushBaseImageLayerFuturesFuture() buildConfiguration .getEventDispatcher() .dispatch( - LogEvent.debug( - dockerClient.load(new ImageToTarballTranslator(image, targetImageReference)))); + LogEvent.debug(dockerClient.load(new ImageTarball(image, targetImageReference)))); // Tags the image with all the additional tags, skipping the one 'docker load' already loaded. for (String tag : buildConfiguration.getAllTargetImageTags()) { diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/WriteTarFileStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/WriteTarFileStep.java index 1f97c6c778..5a19c07ce5 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/WriteTarFileStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/WriteTarFileStep.java @@ -22,7 +22,7 @@ import com.google.cloud.tools.jib.builder.BuildStepType; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.configuration.BuildConfiguration; -import com.google.cloud.tools.jib.docker.ImageToTarballTranslator; +import com.google.cloud.tools.jib.docker.ImageTarball; import com.google.cloud.tools.jib.event.events.LogEvent; import com.google.cloud.tools.jib.filesystem.FileOperations; import com.google.cloud.tools.jib.image.Image; @@ -103,8 +103,7 @@ private BuildResult writeTarFile() throws ExecutionException, IOException { Files.createDirectories(outputPath.getParent()); try (OutputStream outputStream = new BufferedOutputStream(FileOperations.newLockingOutputStream(outputPath))) { - new ImageToTarballTranslator( - image, buildConfiguration.getTargetImageConfiguration().getImage()) + new ImageTarball(image, buildConfiguration.getTargetImageConfiguration().getImage()) .writeTo(outputStream); } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/docker/DockerClient.java b/jib-core/src/main/java/com/google/cloud/tools/jib/docker/DockerClient.java index d466d04cc2..ccf8292b7b 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/docker/DockerClient.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/docker/DockerClient.java @@ -168,8 +168,7 @@ public static boolean isDockerInstalled(Path dockerExecutable) { * @throws InterruptedException if the 'docker load' process is interrupted. * @throws IOException if streaming the blob to 'docker load' fails. */ - public String load(ImageToTarballTranslator imageTarball) - throws InterruptedException, IOException { + public String load(ImageTarball imageTarball) throws InterruptedException, IOException { // Runs 'docker load'. Process dockerProcess = docker("load"); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageToTarballTranslator.java b/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageTarball.java similarity index 96% rename from jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageToTarballTranslator.java rename to jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageTarball.java index cf98ddbe3f..becda0e6df 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageToTarballTranslator.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageTarball.java @@ -30,7 +30,7 @@ import java.util.Collections; /** Translates an {@link Image} to a tarball that can be loaded into Docker. */ -public class ImageToTarballTranslator { +public class ImageTarball { /** File name for the container configuration in the tarball. */ private static final String CONTAINER_CONFIGURATION_JSON_FILE_NAME = "config.json"; @@ -51,7 +51,7 @@ public class ImageToTarballTranslator { * @param image the image to convert into a tarball * @param imageReference image reference to set in the manifest */ - public ImageToTarballTranslator(Image image, ImageReference imageReference) { + public ImageTarball(Image image, ImageReference imageReference) { this.image = image; this.imageReference = imageReference; } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/DockerClientTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/DockerClientTest.java index c151a72a10..29dfd942c1 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/DockerClientTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/DockerClientTest.java @@ -51,7 +51,7 @@ public class DockerClientTest { @Mock private ProcessBuilder mockProcessBuilder; @Mock private Process mockProcess; - @Mock private ImageToTarballTranslator imageTarball; + @Mock private ImageTarball imageTarball; @Before public void setUp() throws IOException { diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageToTarballTranslatorTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageToTarballTest.java similarity index 91% rename from jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageToTarballTranslatorTest.java rename to jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageToTarballTest.java index 4edaab2cf9..a36809c99c 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageToTarballTranslatorTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageToTarballTest.java @@ -30,10 +30,10 @@ import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.io.CharStreams; import com.google.common.io.Resources; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStreamReader; -import java.io.PipedInputStream; -import java.io.PipedOutputStream; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -49,9 +49,9 @@ import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; -/** Tests for {@link ImageToTarballTranslator}. */ +/** Tests for {@link ImageToTarball}. */ @RunWith(MockitoJUnitRunner.class) -public class ImageToTarballTranslatorTest { +public class ImageToTarballTest { @Mock private Layer mockLayer1; @Mock private Layer mockLayer2; @@ -83,12 +83,12 @@ public void testWriteTo() Image testImage = Image.builder(V22ManifestTemplate.class).addLayer(mockLayer1).addLayer(mockLayer2).build(); - ImageToTarballTranslator imageToTarball = - new ImageToTarballTranslator(testImage, ImageReference.parse("my/image:tag")); + ImageTarball imageToTarball = new ImageTarball(testImage, ImageReference.parse("my/image:tag")); - try (PipedInputStream in = new PipedInputStream(); - PipedOutputStream out = new PipedOutputStream(in); - TarArchiveInputStream tarArchiveInputStream = new TarArchiveInputStream(in)) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + imageToTarball.writeTo(out); + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + try (TarArchiveInputStream tarArchiveInputStream = new TarArchiveInputStream(in)) { imageToTarball.writeTo(out); // Verifies layer with fileA was added. From 305f25460c8ad0045068a6b10715e4270566dcd3 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Fri, 19 Apr 2019 12:00:41 -0400 Subject: [PATCH 03/39] Rename test class --- .../docker/{ImageToTarballTest.java => ImageTarballTest.java} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename jib-core/src/test/java/com/google/cloud/tools/jib/docker/{ImageToTarballTest.java => ImageTarballTest.java} (98%) diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageToTarballTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageTarballTest.java similarity index 98% rename from jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageToTarballTest.java rename to jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageTarballTest.java index a36809c99c..d98885f1a9 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageToTarballTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageTarballTest.java @@ -49,9 +49,9 @@ import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; -/** Tests for {@link ImageToTarball}. */ +/** Tests for {@link ImageTarball}. */ @RunWith(MockitoJUnitRunner.class) -public class ImageToTarballTest { +public class ImageTarballTest { @Mock private Layer mockLayer1; @Mock private Layer mockLayer2; From 02b42ad4064556b141b838b333ad14730262465e Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Fri, 19 Apr 2019 12:12:38 -0400 Subject: [PATCH 04/39] Decouple Blob from TarStreamBuilder --- .../google/cloud/tools/jib/docker/ImageTarball.java | 2 +- .../tools/jib/image/ReproducibleLayerBuilder.java | 3 ++- .../google/cloud/tools/jib/tar/TarStreamBuilder.java | 11 +++-------- .../cloud/tools/jib/tar/TarStreamBuilderTest.java | 12 +++--------- 4 files changed, 9 insertions(+), 19 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageTarball.java b/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageTarball.java index becda0e6df..f7ad06a162 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageTarball.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageTarball.java @@ -82,6 +82,6 @@ public void writeTo(OutputStream out) throws IOException { JsonTemplateMapper.toBlob(Collections.singletonList(manifestTemplate))), MANIFEST_JSON_FILE_NAME); - tarStreamBuilder.toBlob().writeTo(out); + tarStreamBuilder.writeAsTarArchiveTo(out); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/ReproducibleLayerBuilder.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/ReproducibleLayerBuilder.java index cbc1fec78f..66f3c3af47 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/image/ReproducibleLayerBuilder.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/image/ReproducibleLayerBuilder.java @@ -17,6 +17,7 @@ package com.google.cloud.tools.jib.image; import com.google.cloud.tools.jib.blob.Blob; +import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.tar.TarStreamBuilder; import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; @@ -132,6 +133,6 @@ public Blob build() { tarStreamBuilder.addTarArchiveEntry(entry); } - return tarStreamBuilder.toBlob(); + return Blobs.from(outputStream -> tarStreamBuilder.writeAsTarArchiveTo(outputStream)); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/tar/TarStreamBuilder.java b/jib-core/src/main/java/com/google/cloud/tools/jib/tar/TarStreamBuilder.java index 8e38af8bb8..c67e85ae77 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/tar/TarStreamBuilder.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/tar/TarStreamBuilder.java @@ -38,12 +38,12 @@ public class TarStreamBuilder { /** * Writes each entry in the filesystem to the tarball archive stream. * - * @param tarByteStream the stream to write to. + * @param out the stream to write to. * @throws IOException if building the tarball fails. */ - private void writeEntriesAsTarArchive(OutputStream tarByteStream) throws IOException { + public void writeAsTarArchiveTo(OutputStream out) throws IOException { try (TarArchiveOutputStream tarArchiveOutputStream = - new TarArchiveOutputStream(tarByteStream, StandardCharsets.UTF_8.name())) { + new TarArchiveOutputStream(out, StandardCharsets.UTF_8.name())) { // Enables PAX extended headers to support long file names. tarArchiveOutputStream.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); for (Map.Entry entry : archiveMap.entrySet()) { @@ -90,9 +90,4 @@ public void addBlobEntry(Blob blob, long size, String name) { entry.setSize(size); archiveMap.put(entry, blob); } - - /** @return a new {@link Blob} that can stream the uncompressed tarball archive BLOB. */ - public Blob toBlob() { - return Blobs.from(this::writeEntriesAsTarArchive); - } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/tar/TarStreamBuilderTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/tar/TarStreamBuilderTest.java index f119b892dc..6555c69c70 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/tar/TarStreamBuilderTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/tar/TarStreamBuilderTest.java @@ -16,7 +16,6 @@ package com.google.cloud.tools.jib.tar; -import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.Blobs; import com.google.common.io.ByteStreams; import com.google.common.io.Resources; @@ -101,12 +100,11 @@ public void testToBlob_multiByte() throws IOException { testTarStreamBuilder.addByteEntry("asdf".getBytes(StandardCharsets.UTF_8), "crepecake"); testTarStreamBuilder.addBlobEntry( Blobs.from("jib"), "jib".getBytes(StandardCharsets.UTF_8).length, "jib"); - Blob blob = testTarStreamBuilder.toBlob(); // Writes the BLOB and captures the output. ByteArrayOutputStream tarByteOutputStream = new ByteArrayOutputStream(); OutputStream compressorStream = new GZIPOutputStream(tarByteOutputStream); - blob.writeTo(compressorStream); + testTarStreamBuilder.writeAsTarArchiveTo(compressorStream); // Rearrange the output into input for verification. ByteArrayInputStream byteArrayInputStream = @@ -173,12 +171,10 @@ private void setUpWithStringsAndTarEntries() { /** Creates a compressed blob from the TarStreamBuilder and verifies it. */ private void verifyBlobWithCompression() throws IOException { - Blob blob = testTarStreamBuilder.toBlob(); - // Writes the BLOB and captures the output. ByteArrayOutputStream tarByteOutputStream = new ByteArrayOutputStream(); OutputStream compressorStream = new GZIPOutputStream(tarByteOutputStream); - blob.writeTo(compressorStream); + testTarStreamBuilder.writeAsTarArchiveTo(compressorStream); // Rearrange the output into input for verification. ByteArrayInputStream byteArrayInputStream = @@ -190,11 +186,9 @@ private void verifyBlobWithCompression() throws IOException { /** Creates an uncompressed blob from the TarStreamBuilder and verifies it. */ private void verifyBlobWithoutCompression() throws IOException { - Blob blob = testTarStreamBuilder.toBlob(); - // Writes the BLOB and captures the output. ByteArrayOutputStream tarByteOutputStream = new ByteArrayOutputStream(); - blob.writeTo(tarByteOutputStream); + testTarStreamBuilder.writeAsTarArchiveTo(tarByteOutputStream); // Rearrange the output into input for verification. ByteArrayInputStream byteArrayInputStream = From 0ff040a6be0a445ae74f54910bd67a5fc00268ed Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Fri, 19 Apr 2019 12:17:06 -0400 Subject: [PATCH 05/39] empty commit to stalled Kokoro From ac3ef64c48dfe69ca3147b8a059f31c0176ad65c Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Fri, 19 Apr 2019 12:20:11 -0400 Subject: [PATCH 06/39] Clean up --- .../com/google/cloud/tools/jib/docker/DockerClientTest.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/DockerClientTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/DockerClientTest.java index 29dfd942c1..230c2a9a38 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/DockerClientTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/DockerClientTest.java @@ -88,12 +88,6 @@ public void testLoad() throws IOException, InterruptedException { Mockito.when(mockProcess.getInputStream()) .thenReturn(new ByteArrayInputStream("output".getBytes(StandardCharsets.UTF_8))); - Mockito.doAnswer( - AdditionalAnswers.answerVoid( - (VoidAnswer1) - out -> out.write("jib".getBytes(StandardCharsets.UTF_8)))) - .when(imageTarball) - .writeTo(Mockito.any(OutputStream.class)); String output = testDockerClient.load(imageTarball); Assert.assertEquals( From 447573d6584cd574ddc39704656bc5e6451ce154 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Fri, 19 Apr 2019 14:50:11 -0400 Subject: [PATCH 07/39] more intuitive code --- .../cloud/tools/jib/tar/TarStreamBuilder.java | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/tar/TarStreamBuilder.java b/jib-core/src/main/java/com/google/cloud/tools/jib/tar/TarStreamBuilder.java index c67e85ae77..db12686a45 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/tar/TarStreamBuilder.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/tar/TarStreamBuilder.java @@ -30,10 +30,21 @@ public class TarStreamBuilder { /** - * Maps from {@link TarArchiveEntry} to a {@link Blob}. The order of the entries is the order they - * belong in the tarball. + * Represents contents of a tar entry. It may represent "yet-to-be-realized" contents; for + * example, loading the actual contents from a file may happen only when writing the contents into + * an output stream. */ - private final LinkedHashMap archiveMap = new LinkedHashMap<>(); + @FunctionalInterface + private interface Contents { + + void writeTo(OutputStream out) throws IOException; + } + + /** + * Maps from {@link TarArchiveEntry} to a {@link Contents}. The order of the entries is the order + * they belong in the tarball. + */ + private final LinkedHashMap archiveMap = new LinkedHashMap<>(); /** * Writes each entry in the filesystem to the tarball archive stream. @@ -46,7 +57,7 @@ public void writeAsTarArchiveTo(OutputStream out) throws IOException { new TarArchiveOutputStream(out, StandardCharsets.UTF_8.name())) { // Enables PAX extended headers to support long file names. tarArchiveOutputStream.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); - for (Map.Entry entry : archiveMap.entrySet()) { + for (Map.Entry entry : archiveMap.entrySet()) { tarArchiveOutputStream.putArchiveEntry(entry.getKey()); entry.getValue().writeTo(tarArchiveOutputStream); tarArchiveOutputStream.closeArchiveEntry(); @@ -60,13 +71,17 @@ public void writeAsTarArchiveTo(OutputStream out) throws IOException { * @param entry the {@link TarArchiveEntry} */ public void addTarArchiveEntry(TarArchiveEntry entry) { - archiveMap.put( - entry, entry.isFile() ? Blobs.from(entry.getFile().toPath()) : Blobs.from(ignored -> {})); + if (!entry.isFile()) { + archiveMap.put(entry, ignored -> {}); + } else { + Blob fileBlob = Blobs.from(entry.getFile().toPath()); + archiveMap.put(entry, outputStream -> fileBlob.writeTo(outputStream)); + } } /** - * Adds a blob to the archive. Note that this should be used with raw bytes and not file contents; - * for adding files to the archive, use {@link #addTarArchiveEntry}. + * Adds byte contents to the archive. Note that this should be used with raw bytes and not file + * contents; for adding files to the archive, use {@link #addTarArchiveEntry}. * * @param contents the bytes to add to the tarball * @param name the name of the entry (i.e. filename) @@ -74,7 +89,7 @@ public void addTarArchiveEntry(TarArchiveEntry entry) { public void addByteEntry(byte[] contents, String name) { TarArchiveEntry entry = new TarArchiveEntry(name); entry.setSize(contents.length); - archiveMap.put(entry, Blobs.from(outputStream -> outputStream.write(contents))); + archiveMap.put(entry, outputStream -> outputStream.write(contents)); } /** @@ -88,6 +103,6 @@ public void addByteEntry(byte[] contents, String name) { public void addBlobEntry(Blob blob, long size, String name) { TarArchiveEntry entry = new TarArchiveEntry(name); entry.setSize(size); - archiveMap.put(entry, blob); + archiveMap.put(entry, outputStream -> blob.writeTo(outputStream)); } } From 8098dc3ab2f6fb3741458d08a85c07fa6c9305fb Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Fri, 19 Apr 2019 17:23:19 -0400 Subject: [PATCH 08/39] wip --- .../registry/BlobPullerIntegrationTest.java | 3 +- .../cloud/tools/jib/blob/BlobDescriptor.java | 23 ---- .../google/cloud/tools/jib/blob/FileBlob.java | 5 +- .../cloud/tools/jib/blob/InputStreamBlob.java | 3 +- .../cloud/tools/jib/blob/StringBlob.java | 5 +- .../cloud/tools/jib/blob/WriterBlob.java | 8 +- .../tools/jib/builder/steps/BuildResult.java | 7 +- .../jib/builder/steps/PushImageStep.java | 8 +- .../jib/cache/DefaultCacheStorageWriter.java | 7 +- .../tools/jib/cache/LayerEntriesSelector.java | 7 +- .../cloud/tools/jib/docker/ImageTarball.java | 8 +- .../jib/hash/CountingDigestOutputStream.java | 43 ++++--- .../cloud/tools/jib/hash/DigestUtil.java | 109 ++++++++++++++++++ .../tools/jib/json/JsonTemplateMapper.java | 18 +++ .../cloud/tools/jib/tar/TarStreamBuilder.java | 2 +- .../google/cloud/tools/jib/blob/BlobTest.java | 10 +- .../cache/DefaultCacheStorageWriterTest.java | 6 +- .../tools/jib/docker/ImageTarballTest.java | 1 + .../hash/CountingDigestOutputStreamTest.java | 8 +- .../tools/jib/registry/BlobPullerTest.java | 2 +- 20 files changed, 192 insertions(+), 91 deletions(-) create mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java diff --git a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java index 1368525e41..c140fa43b4 100644 --- a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java +++ b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java @@ -69,8 +69,7 @@ public void testPull() throws IOException, RegistryException, InterruptedExcepti .writeTo(layerOutputStream); Assert.assertTrue(expectedSize.sum() > 0); Assert.assertEquals(expectedSize.sum(), totalByteCount.sum()); - - Assert.assertEquals(realDigest, layerOutputStream.toBlobDescriptor().getDigest()); + Assert.assertEquals(realDigest, layerOutputStream.getDigest()); } @Test diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/BlobDescriptor.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/BlobDescriptor.java index 0b067cc912..ff59209998 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/BlobDescriptor.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/BlobDescriptor.java @@ -16,12 +16,7 @@ package com.google.cloud.tools.jib.blob; -import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; import com.google.cloud.tools.jib.image.DescriptorDigest; -import com.google.common.io.ByteStreams; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; /** Contains properties describing a BLOB, including its digest and possibly its size (in bytes). */ public class BlobDescriptor { @@ -31,24 +26,6 @@ public class BlobDescriptor { /** The size of the BLOB (in bytes). Negative if unknown. */ private final long size; - /** - * Creates a new {@link BlobDescriptor} from the contents of an {@link InputStream} while piping - * to an {@link OutputStream}. Does not close either streams. - * - * @param inputStream the stream to read the contents from - * @param outputStream the {@link OutputStream} to pipe to - * @return a {@link BlobDescriptor} of the piped contents - * @throws IOException if reading from or writing to the streams fails - */ - static BlobDescriptor fromPipe(InputStream inputStream, OutputStream outputStream) - throws IOException { - CountingDigestOutputStream countingDigestOutputStream = - new CountingDigestOutputStream(outputStream); - ByteStreams.copy(inputStream, countingDigestOutputStream); - countingDigestOutputStream.flush(); - return countingDigestOutputStream.toBlobDescriptor(); - } - public BlobDescriptor(long size, DescriptorDigest digest) { this.size = size; this.digest = digest; diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/FileBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/FileBlob.java index 5c978c2dfb..7748b9bddd 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/FileBlob.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/FileBlob.java @@ -16,6 +16,7 @@ package com.google.cloud.tools.jib.blob; +import com.google.cloud.tools.jib.hash.DigestUtil; import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; @@ -34,8 +35,8 @@ class FileBlob implements Blob { @Override public BlobDescriptor writeTo(OutputStream outputStream) throws IOException { - try (InputStream fileStream = new BufferedInputStream(Files.newInputStream(file))) { - return BlobDescriptor.fromPipe(fileStream, outputStream); + try (InputStream fileIn = new BufferedInputStream(Files.newInputStream(file))) { + return DigestUtil.computeDigest(fileIn, outputStream); } } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/InputStreamBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/InputStreamBlob.java index 46273b691f..5e5e172f8c 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/InputStreamBlob.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/InputStreamBlob.java @@ -16,6 +16,7 @@ package com.google.cloud.tools.jib.blob; +import com.google.cloud.tools.jib.hash.DigestUtil; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -39,7 +40,7 @@ public BlobDescriptor writeTo(OutputStream outputStream) throws IOException { throw new IllegalStateException("Cannot rewrite Blob backed by an InputStream"); } try (InputStream inputStream = this.inputStream) { - return BlobDescriptor.fromPipe(inputStream, outputStream); + return DigestUtil.computeDigest(inputStream, outputStream); } finally { isWritten = true; diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/StringBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/StringBlob.java index 62e07e1297..9f61014666 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/StringBlob.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/StringBlob.java @@ -16,6 +16,7 @@ package com.google.cloud.tools.jib.blob; +import com.google.cloud.tools.jib.hash.DigestUtil; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -33,9 +34,9 @@ class StringBlob implements Blob { @Override public BlobDescriptor writeTo(OutputStream outputStream) throws IOException { - try (InputStream stringInputStream = + try (InputStream stringIn = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8))) { - return BlobDescriptor.fromPipe(stringInputStream, outputStream); + return DigestUtil.computeDigest(stringIn, outputStream); } } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterBlob.java index c47dc3e037..fe92ec5df8 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterBlob.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterBlob.java @@ -16,7 +16,7 @@ package com.google.cloud.tools.jib.blob; -import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; +import com.google.cloud.tools.jib.hash.DigestUtil; import java.io.IOException; import java.io.OutputStream; @@ -31,10 +31,6 @@ class WriterBlob implements Blob { @Override public BlobDescriptor writeTo(OutputStream outputStream) throws IOException { - CountingDigestOutputStream countingDigestOutputStream = - new CountingDigestOutputStream(outputStream); - writer.writeTo(countingDigestOutputStream); - countingDigestOutputStream.flush(); - return countingDigestOutputStream.toBlobDescriptor(); + return DigestUtil.computeDigest(out -> writer.writeTo(out), outputStream); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildResult.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildResult.java index 383b055760..a8597d7bff 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildResult.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildResult.java @@ -17,12 +17,12 @@ package com.google.cloud.tools.jib.builder.steps; import com.google.cloud.tools.jib.blob.BlobDescriptor; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.Layer; import com.google.cloud.tools.jib.image.json.BuildableManifestTemplate; import com.google.cloud.tools.jib.image.json.ImageToJsonTranslator; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.io.ByteStreams; import java.io.IOException; import java.util.Objects; @@ -49,10 +49,7 @@ static BuildResult fromImage( BuildableManifestTemplate manifestTemplate = imageToJsonTranslator.getManifestTemplate( targetFormat, containerConfigurationBlobDescriptor); - DescriptorDigest imageDigest = - JsonTemplateMapper.toBlob(manifestTemplate) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DescriptorDigest imageDigest = DigestUtil.computeJsonDigest(manifestTemplate); DescriptorDigest imageId = containerConfigurationBlobDescriptor.getDigest(); return new BuildResult(imageDigest, imageId); } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java index 4d7821b2ec..e44bb7980f 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java @@ -25,13 +25,12 @@ import com.google.cloud.tools.jib.builder.TimerEventDispatcher; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.event.events.LogEvent; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.json.BuildableManifestTemplate; import com.google.cloud.tools.jib.image.json.ImageToJsonTranslator; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.cloud.tools.jib.registry.RegistryClient; import com.google.common.collect.ImmutableSet; -import com.google.common.io.ByteStreams; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -165,10 +164,7 @@ private BuildResult afterAllPushed() })); } - DescriptorDigest imageDigest = - JsonTemplateMapper.toBlob(manifestTemplate) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DescriptorDigest imageDigest = DigestUtil.computeJsonDigest(manifestTemplate); DescriptorDigest imageId = containerConfigurationBlobDescriptor.getDigest(); BuildResult result = new BuildResult(imageDigest, imageId); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriter.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriter.java index c826ce0465..011ccf82d3 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriter.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriter.java @@ -97,7 +97,7 @@ private static DescriptorDigest getDiffIdByDecompressingFile(Path compressedFile GZIPInputStream decompressorStream = new GZIPInputStream(fileInputStream)) { ByteStreams.copy(decompressorStream, diffIdCaptureOutputStream); } - return diffIdCaptureOutputStream.toBlobDescriptor().getDigest(); + return diffIdCaptureOutputStream.getDigest(); } } @@ -257,9 +257,8 @@ private WrittenLayer writeUncompressedLayerBlobToDirectory( // The GZIPOutputStream must be closed in order to write out the remaining compressed data. compressorStream.close(); - BlobDescriptor compressedBlobDescriptor = compressedDigestOutputStream.toBlobDescriptor(); - DescriptorDigest layerDigest = compressedBlobDescriptor.getDigest(); - long layerSize = compressedBlobDescriptor.getSize(); + DescriptorDigest layerDigest = compressedDigestOutputStream.getDigest(); + long layerSize = compressedDigestOutputStream.getBytesHahsed(); // Renames the temporary layer file to the correct filename. Path layerFile = diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/LayerEntriesSelector.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/LayerEntriesSelector.java index b47011fff9..dd5c18cb1c 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/LayerEntriesSelector.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/LayerEntriesSelector.java @@ -16,13 +16,12 @@ package com.google.cloud.tools.jib.cache; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.LayerEntry; import com.google.cloud.tools.jib.json.JsonTemplate; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; -import com.google.common.io.ByteStreams; import java.io.IOException; import java.nio.file.Files; import java.time.Instant; @@ -142,9 +141,7 @@ static List toSortedJsonTemplates(List layerEntr */ static DescriptorDigest generateSelector(ImmutableList layerEntries) throws IOException { - return JsonTemplateMapper.toBlob(toSortedJsonTemplates(layerEntries)) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + return DigestUtil.computeJsonDigest(toSortedJsonTemplates(layerEntries)); } private LayerEntriesSelector() {} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageTarball.java b/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageTarball.java index f7ad06a162..7843afbe48 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageTarball.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageTarball.java @@ -25,6 +25,7 @@ import com.google.cloud.tools.jib.image.json.ImageToJsonTranslator; import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.cloud.tools.jib.tar.TarStreamBuilder; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; import java.util.Collections; @@ -77,10 +78,9 @@ public void writeTo(OutputStream out) throws IOException { // Adds the manifest to tarball. manifestTemplate.setRepoTags(imageReference.toStringWithTag()); - tarStreamBuilder.addByteEntry( - Blobs.writeToByteArray( - JsonTemplateMapper.toBlob(Collections.singletonList(manifestTemplate))), - MANIFEST_JSON_FILE_NAME); + ByteArrayOutputStream outStream = new ByteArrayOutputStream(); + JsonTemplateMapper.writeTo(Collections.singletonList(manifestTemplate), outStream); + tarStreamBuilder.addByteEntry(outStream.toByteArray(), MANIFEST_JSON_FILE_NAME); tarStreamBuilder.writeAsTarArchiveTo(out); } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStream.java b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStream.java index b2e872c2a2..59343b8a48 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStream.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStream.java @@ -16,22 +16,27 @@ package com.google.cloud.tools.jib.hash; -import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.image.DescriptorDigest; +import com.google.common.base.Verify; import java.io.IOException; import java.io.OutputStream; import java.security.DigestException; import java.security.DigestOutputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import javax.annotation.Nullable; /** A {@link DigestOutputStream} that also keeps track of the total number of bytes written. */ public class CountingDigestOutputStream extends DigestOutputStream { private static final String SHA_256_ALGORITHM = "SHA-256"; - /** Keeps track of the total number of bytes appended. */ - private long totalBytes = 0; + private long bytesSoFar = 0; + + /** The total number of bytes used to compute a digest. Resets when {@link computeDigest) is called. */ + private long bytesHashed; + + @Nullable private DescriptorDigest descriptorDigest; /** * Wraps the {@code outputStream}. @@ -49,12 +54,10 @@ public CountingDigestOutputStream(OutputStream outputStream) { } /** - * Builds a {@link BlobDescriptor} with the hash and size of the bytes written. The buffer resets - * after this method is called, so this method should only be called once per BlobDescriptor. - * - * @return the built {@link BlobDescriptor}. + * Computes the hash and size of the bytes written. The buffer resets after this method is called, + * so this method should only be called once per computation. */ - public BlobDescriptor toBlobDescriptor() { + public void computeDigest() { try { byte[] hashedBytes = digest.digest(); @@ -65,8 +68,9 @@ public BlobDescriptor toBlobDescriptor() { } String hash = stringBuilder.toString(); - DescriptorDigest digest = DescriptorDigest.fromHash(hash); - return new BlobDescriptor(totalBytes, digest); + bytesHashed = bytesSoFar; + descriptorDigest = DescriptorDigest.fromHash(hash); + bytesSoFar = 0; } catch (DigestException ex) { throw new RuntimeException("SHA-256 algorithm produced invalid hash: " + ex.getMessage(), ex); @@ -74,19 +78,30 @@ public BlobDescriptor toBlobDescriptor() { } /** @return the total number of bytes that were hashed */ - public long getTotalBytes() { - return totalBytes; + public long getBytesHahsed() { + if (descriptorDigest == null) { + computeDigest(); + } + return bytesHashed; + } + + /** @return the digest hash */ + public DescriptorDigest getDigest() { + if (descriptorDigest == null) { + computeDigest(); + } + return Verify.verifyNotNull(descriptorDigest); } @Override public void write(byte[] data, int offset, int length) throws IOException { super.write(data, offset, length); - totalBytes += length; + bytesSoFar += length; } @Override public void write(int singleByte) throws IOException { super.write(singleByte); - totalBytes++; + bytesSoFar++; } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java new file mode 100644 index 0000000000..4570f90d45 --- /dev/null +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java @@ -0,0 +1,109 @@ +/* + * Copyright 2017 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.tools.jib.hash; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.cloud.tools.jib.blob.Blob; +import com.google.cloud.tools.jib.blob.BlobDescriptor; +import com.google.cloud.tools.jib.image.DescriptorDigest; +import com.google.cloud.tools.jib.json.JsonTemplate; +import com.google.common.io.ByteStreams; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.List; + +/** Utility class for input/output streams. */ +public class DigestUtil { + + /** + * Represents contents. It may represent "yet-to-be-realized" contents; for example, loading the + * actual contents from a file may happen only when writing the contents into an output stream. + */ + @FunctionalInterface + public static interface Contents { + + void writeTo(OutputStream out) throws IOException; + } + + public static DescriptorDigest computeJsonDigest(JsonTemplate template) throws IOException { + return computeJsonDigest((Object) template); + } + + public static DescriptorDigest computeJsonDigest(List templates) + throws IOException { + return computeJsonDigest((Object) templates); + } + + private static DescriptorDigest computeJsonDigest(Object jsonObject) throws IOException { + Contents contents = outStream -> new ObjectMapper().writeValue(outStream, jsonObject); + return computeDigest(contents).getDigest(); + } + + public static BlobDescriptor computeDigest(Blob blob) throws IOException { + return computeDigest(out -> blob.writeTo(out), ByteStreams.nullOutputStream()); + } + + /** + * Computes the digest by consuming the contents. + * + * @param contents the contents for which the digest is computed + * @return computed digest and bytes consumed + * @throws IOException if reading fails + */ + public static BlobDescriptor computeDigest(Contents contents) throws IOException { + return computeDigest(contents, ByteStreams.nullOutputStream()); + } + + public static BlobDescriptor computeDigest(InputStream inStream) throws IOException { + return computeDigest(inStream, ByteStreams.nullOutputStream()); + } + + /** + * Computes the digest by consuming the contents of an {@link InputStream} and optionally copying + * it to an {@link OutputStream}. Returns the computed digested along with the bytes consumed to + * compute the digest. Does not close either stream. + * + * @param inStream the stream to read the contents from + * @param outStream the stream to which the contents are copied + * @return computed digest and bytes consumed + * @throws IOException if reading from or writing fails + */ + public static BlobDescriptor computeDigest(InputStream inStream, OutputStream outStream) + throws IOException { + Contents contents = anyOutSteam -> ByteStreams.copy(inStream, anyOutSteam); + return computeDigest(contents, outStream); + } + + /** + * Computes the digest by consuming the contents and optionally copying it to an {@link + * OutputStream}. Returns the computed digested along with the bytes consumed. Does not close the + * stream. + * + * @param contents the contents for which the digest is computed + * @param outStream the stream to which the contents are copied + * @return computed digest and bytes consumed + * @throws IOException if reading from or writing fails + */ + public static BlobDescriptor computeDigest(Contents contents, OutputStream outStream) + throws IOException { + CountingDigestOutputStream digestOutStream = new CountingDigestOutputStream(outStream); + contents.writeTo(digestOutStream); + digestOutStream.flush(); + return new BlobDescriptor(digestOutStream.getBytesHahsed(), digestOutStream.getDigest()); + } +} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java b/jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java index 2ff9e4dbe5..cff06f72b5 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java @@ -16,12 +16,15 @@ package com.google.cloud.tools.jib.json; +import com.fasterxml.jackson.core.JsonGenerationException; +import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.type.CollectionType; import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.Blobs; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.nio.file.Files; @@ -127,6 +130,21 @@ public static Blob toBlob(JsonTemplate template) { return toBlob((Object) template); } + public static void writeTo(JsonTemplate template, OutputStream out) + throws JsonGenerationException, JsonMappingException, IOException { + writeTo((Object) template, out); + } + + public static void writeTo(List templates, OutputStream out) + throws JsonGenerationException, JsonMappingException, IOException { + writeTo((Object) templates, out); + } + + private static void writeTo(Object template, OutputStream out) + throws JsonGenerationException, JsonMappingException, IOException { + objectMapper.writeValue(out, template); + } + /** * Convert a list of {@link JsonTemplate} to a {@link Blob} of the JSON string. * diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/tar/TarStreamBuilder.java b/jib-core/src/main/java/com/google/cloud/tools/jib/tar/TarStreamBuilder.java index db12686a45..df691ba03a 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/tar/TarStreamBuilder.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/tar/TarStreamBuilder.java @@ -35,7 +35,7 @@ public class TarStreamBuilder { * an output stream. */ @FunctionalInterface - private interface Contents { + private static interface Contents { void writeTo(OutputStream out) throws IOException; } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java index 623068664d..1646bd659c 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java @@ -16,7 +16,7 @@ package com.google.cloud.tools.jib.blob; -import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.common.io.Resources; import java.io.ByteArrayInputStream; @@ -32,7 +32,6 @@ import java.nio.file.StandardOpenOption; import org.junit.Assert; import org.junit.Test; -import org.mockito.Mockito; /** Tests for {@link Blob}. */ public class BlobTest { @@ -107,10 +106,9 @@ private void verifyBlobWriteTo(String expected, Blob blob) throws IOException { byte[] expectedBytes = expected.getBytes(StandardCharsets.UTF_8); Assert.assertEquals(expectedBytes.length, blobDescriptor.getSize()); - CountingDigestOutputStream countingDigestOutputStream = - new CountingDigestOutputStream(Mockito.mock(OutputStream.class)); - countingDigestOutputStream.write(expectedBytes); - DescriptorDigest expectedDigest = countingDigestOutputStream.toBlobDescriptor().getDigest(); + ; + DescriptorDigest expectedDigest = + DigestUtil.computeDigest(new ByteArrayInputStream(expectedBytes)).getDigest(); Assert.assertEquals(expectedDigest, blobDescriptor.getDigest()); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriterTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriterTest.java index 58b1c382b0..59bcd8b54e 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriterTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriterTest.java @@ -20,6 +20,7 @@ import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.common.io.ByteStreams; import java.io.ByteArrayInputStream; @@ -42,10 +43,7 @@ private static DescriptorDigest getDigest(Blob blob) throws IOException { } private static BlobDescriptor getCompressedBlobDescriptor(Blob blob) throws IOException { - CountingDigestOutputStream compressedDigestOutputStream = - new CountingDigestOutputStream(ByteStreams.nullOutputStream()); - compress(blob).writeTo(compressedDigestOutputStream); - return compressedDigestOutputStream.toBlobDescriptor(); + return DigestUtil.computeDigest(compress(blob)); } private static Blob compress(Blob blob) { diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageTarballTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageTarballTest.java index d98885f1a9..77695d019b 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageTarballTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageTarballTest.java @@ -121,6 +121,7 @@ public void testWriteTo() String manifestJson = CharStreams.toString( new InputStreamReader(tarArchiveInputStream, StandardCharsets.UTF_8)); + System.out.println(manifestJson); JsonTemplateMapper.readListOfJson(manifestJson, DockerLoadManifestEntryTemplate.class); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStreamTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStreamTest.java index a2b1b40f7e..b4901f2615 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStreamTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStreamTest.java @@ -16,7 +16,6 @@ package com.google.cloud.tools.jib.hash; -import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteStreams; @@ -57,10 +56,9 @@ public void test_smokeTest() throws IOException, DigestException { InputStream toHashInputStream = new ByteArrayInputStream(bytesToHash); ByteStreams.copy(toHashInputStream, countingDigestOutputStream); - BlobDescriptor expectedBlobDescriptor = - new BlobDescriptor(bytesToHash.length, DescriptorDigest.fromHash(expectedHash)); - Assert.assertEquals(expectedBlobDescriptor, countingDigestOutputStream.toBlobDescriptor()); - Assert.assertEquals(bytesToHash.length, countingDigestOutputStream.getTotalBytes()); + Assert.assertEquals( + DescriptorDigest.fromHash(expectedHash), countingDigestOutputStream.getDigest()); + Assert.assertEquals(bytesToHash.length, countingDigestOutputStream.getBytesHahsed()); } } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java index e0eb9311cd..a21f4bd725 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java @@ -87,7 +87,7 @@ public void testHandleResponse() throws IOException, UnexpectedBlobDigestExcepti Assert.assertEquals( "some BLOB content", new String(layerContentOutputStream.toByteArray(), StandardCharsets.UTF_8)); - Assert.assertEquals(testBlobDigest, layerOutputStream.toBlobDescriptor().getDigest()); + Assert.assertEquals(testBlobDigest, layerOutputStream.getDigest()); Assert.assertEquals("some BLOB content".length(), byteCount.sum()); } From 181dbade38f922ba520de62ea23165d4b4af6e40 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Fri, 19 Apr 2019 20:13:32 -0400 Subject: [PATCH 09/39] wip --- .../registry/BlobPullerIntegrationTest.java | 14 +++---- .../ManifestPusherIntegrationTest.java | 11 ++--- .../google/cloud/tools/jib/blob/Blobs.java | 5 +++ .../google/cloud/tools/jib/blob/JsonBlob.java | 37 +++++++++++++++++ .../jib/builder/steps/PullBaseImageStep.java | 2 +- .../cloud/tools/jib/hash/DigestUtil.java | 22 +++++----- .../jib/image/json/ImageToJsonTranslator.java | 4 +- .../tools/jib/json/JsonTemplateMapper.java | 40 ++++++++----------- .../tools/jib/registry/ManifestPusher.java | 11 ++--- .../google/cloud/tools/jib/blob/BlobTest.java | 1 - .../cloud/tools/jib/cache/CacheTest.java | 20 +++------- .../cache/DefaultCacheStorageWriterTest.java | 11 ++--- .../jib/cache/LayerEntriesSelectorTest.java | 12 ++---- .../tools/jib/docker/ImageTarballTest.java | 2 - .../json/DockerLoadManifestTemplateTest.java | 3 +- .../ContainerConfigurationTemplateTest.java | 6 +-- .../image/json/ImageToJsonTranslatorTest.java | 5 +-- .../image/json/OCIManifestTemplateTest.java | 6 +-- .../image/json/V22ManifestTemplateTest.java | 6 +-- .../jib/json/JsonTemplateMapperTest.java | 11 +---- .../tools/jib/registry/BlobCheckerTest.java | 7 ++-- .../jib/registry/ManifestPusherTest.java | 22 +++------- .../jib/maven/MavenProjectProperties.java | 2 + 23 files changed, 115 insertions(+), 145 deletions(-) create mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/blob/JsonBlob.java diff --git a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java index c140fa43b4..4b38eafca1 100644 --- a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java +++ b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java @@ -16,8 +16,9 @@ package com.google.cloud.tools.jib.registry; +import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.event.EventDispatcher; -import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.http.TestBlobProgressListener; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.json.V21ManifestTemplate; @@ -54,22 +55,19 @@ public void testPull() throws IOException, RegistryException, InterruptedExcepti DescriptorDigest realDigest = manifestTemplate.getLayerDigests().get(0); // Pulls a layer BLOB of the busybox image. - CountingDigestOutputStream layerOutputStream = - new CountingDigestOutputStream(ByteStreams.nullOutputStream()); LongAdder totalByteCount = new LongAdder(); LongAdder expectedSize = new LongAdder(); - registryClient - .pullBlob( + Blob pulledBlob = + registryClient.pullBlob( realDigest, size -> { Assert.assertEquals(0, expectedSize.sum()); expectedSize.add(size); }, - new TestBlobProgressListener(totalByteCount::add)) - .writeTo(layerOutputStream); + new TestBlobProgressListener(totalByteCount::add)); Assert.assertTrue(expectedSize.sum() > 0); Assert.assertEquals(expectedSize.sum(), totalByteCount.sum()); - Assert.assertEquals(realDigest, layerOutputStream.getDigest()); + Assert.assertEquals(realDigest, DigestUtil.computeDigest(pulledBlob).getDigest()); } @Test diff --git a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPusherIntegrationTest.java b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPusherIntegrationTest.java index 67a444f4c7..7f7800e542 100644 --- a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPusherIntegrationTest.java +++ b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPusherIntegrationTest.java @@ -21,12 +21,11 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.event.EventDispatcher; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.http.TestBlobProgressListener; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.json.ManifestTemplate; import com.google.cloud.tools.jib.image.json.V22ManifestTemplate; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; -import com.google.common.io.ByteStreams; import java.io.IOException; import java.security.DigestException; import org.junit.Assert; @@ -114,11 +113,7 @@ public void testPush() V22ManifestTemplate manifestTemplateByDigest = registryClient.pullManifest(imageDigest.toString(), V22ManifestTemplate.class); Assert.assertEquals( - JsonTemplateMapper.toBlob(manifestTemplate) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(), - JsonTemplateMapper.toBlob(manifestTemplateByDigest) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest()); + DigestUtil.computeJsonDigest(manifestTemplate), + DigestUtil.computeJsonDigest(manifestTemplateByDigest)); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java index 2d493d607f..142f4134fb 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java @@ -17,6 +17,7 @@ package com.google.cloud.tools.jib.blob; import com.google.cloud.tools.jib.filesystem.FileOperations; +import com.google.cloud.tools.jib.json.JsonTemplate; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; @@ -35,6 +36,10 @@ public static Blob from(Path file) { return new FileBlob(file); } + public static Blob from(JsonTemplate template) { + return new JsonBlob(template); + } + /** * Creates a {@link StringBlob} with UTF-8 encoding. * diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/JsonBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/JsonBlob.java new file mode 100644 index 0000000000..83dd4c5314 --- /dev/null +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/JsonBlob.java @@ -0,0 +1,37 @@ +/* + * Copyright 2019 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.tools.jib.blob; + +import com.google.cloud.tools.jib.hash.DigestUtil; +import com.google.cloud.tools.jib.json.JsonTemplate; +import java.io.IOException; +import java.io.OutputStream; + +/** A {@link Blob} that holds {@link JsonTemplate}. */ +class JsonBlob implements Blob { + + private final JsonTemplate template; + + JsonBlob(JsonTemplate template) { + this.template = template; + } + + @Override + public BlobDescriptor writeTo(OutputStream outputStream) throws IOException { + return DigestUtil.computeDigest(template, outputStream); + } +} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java index 1e96d4a310..a7d8773357 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java @@ -237,7 +237,7 @@ private Image pullBaseImage( || v22ManifestTemplate.getContainerConfiguration().getDigest() == null) { throw new UnknownManifestFormatException( "Invalid container configuration in Docker V2.2 manifest: \n" - + Blobs.writeToString(JsonTemplateMapper.toBlob(v22ManifestTemplate))); + + JsonTemplateMapper.toUtf8String(v22ManifestTemplate)); } DescriptorDigest containerConfigurationDigest = diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java index 4570f90d45..efc3bdbd76 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java @@ -37,21 +37,29 @@ public class DigestUtil { @FunctionalInterface public static interface Contents { + static Contents fromJson(Object jsonObject) { + return outStream -> new ObjectMapper().writeValue(outStream, jsonObject); + } + void writeTo(OutputStream out) throws IOException; } public static DescriptorDigest computeJsonDigest(JsonTemplate template) throws IOException { - return computeJsonDigest((Object) template); + return computeDigest(template, ByteStreams.nullOutputStream()).getDigest(); } public static DescriptorDigest computeJsonDigest(List templates) throws IOException { - return computeJsonDigest((Object) templates); + return computeDigest(Contents.fromJson(templates), ByteStreams.nullOutputStream()).getDigest(); } - private static DescriptorDigest computeJsonDigest(Object jsonObject) throws IOException { - Contents contents = outStream -> new ObjectMapper().writeValue(outStream, jsonObject); - return computeDigest(contents).getDigest(); + public static BlobDescriptor computeDigest(JsonTemplate template, OutputStream outStream) + throws IOException { + return computeDigest(Contents.fromJson(template), outStream); + } + + public static BlobDescriptor computeDigest(InputStream inStream) throws IOException { + return computeDigest(inStream, ByteStreams.nullOutputStream()); } public static BlobDescriptor computeDigest(Blob blob) throws IOException { @@ -69,10 +77,6 @@ public static BlobDescriptor computeDigest(Contents contents) throws IOException return computeDigest(contents, ByteStreams.nullOutputStream()); } - public static BlobDescriptor computeDigest(InputStream inStream) throws IOException { - return computeDigest(inStream, ByteStreams.nullOutputStream()); - } - /** * Computes the digest by consuming the contents of an {@link InputStream} and optionally copying * it to an {@link OutputStream}. Returns the computed digested along with the bytes consumed to diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java index bb504ffeac..8a8f06c5e9 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java @@ -18,13 +18,13 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; +import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.configuration.DockerHealthCheck; import com.google.cloud.tools.jib.configuration.Port; import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.Layer; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -189,7 +189,7 @@ public Blob getContainerConfigurationBlob() { } // Serializes into JSON. - return JsonTemplateMapper.toBlob(template); + return Blobs.from(template); } /** diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java b/jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java index cff06f72b5..c5f1c5f501 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java @@ -20,8 +20,7 @@ import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.type.CollectionType; -import com.google.cloud.tools.jib.blob.Blob; -import com.google.cloud.tools.jib.blob.Blobs; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -120,14 +119,14 @@ public static List readListOfJson( return objectMapper.readValue(jsonString, listType); } - /** - * Convert a {@link JsonTemplate} to a {@link Blob} of the JSON string. - * - * @param template the JSON template to convert - * @return a {@link Blob} of the JSON string - */ - public static Blob toBlob(JsonTemplate template) { - return toBlob((Object) template); + public static String toUtf8String(JsonTemplate template) + throws JsonGenerationException, JsonMappingException, IOException { + return toUtf8String((Object) template); + } + + public static String toUtf8String(List templates) + throws JsonGenerationException, JsonMappingException, IOException { + return toUtf8String((Object) templates); } public static void writeTo(JsonTemplate template, OutputStream out) @@ -140,23 +139,16 @@ public static void writeTo(List templates, OutputStream writeTo((Object) templates, out); } - private static void writeTo(Object template, OutputStream out) + private static String toUtf8String(Object template) throws JsonGenerationException, JsonMappingException, IOException { - objectMapper.writeValue(out, template); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writeTo(template, out); + return out.toString("UTF-8"); } - /** - * Convert a list of {@link JsonTemplate} to a {@link Blob} of the JSON string. - * - * @param templates the list of JSON templates to convert - * @return a {@link Blob} of the JSON string - */ - public static Blob toBlob(List templates) { - return toBlob((Object) templates); - } - - private static Blob toBlob(Object template) { - return Blobs.from(outputStream -> objectMapper.writeValue(outputStream, template)); + private static void writeTo(Object template, OutputStream out) + throws JsonGenerationException, JsonMappingException, IOException { + objectMapper.writeValue(out, template); } private JsonTemplateMapper() {} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPusher.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPusher.java index 87834575ba..a0cf547579 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPusher.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPusher.java @@ -18,14 +18,14 @@ import com.google.api.client.http.HttpMethods; import com.google.api.client.http.HttpResponseException; +import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.event.EventDispatcher; import com.google.cloud.tools.jib.event.events.LogEvent; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.http.BlobHttpContent; import com.google.cloud.tools.jib.http.Response; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.json.BuildableManifestTemplate; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; -import com.google.common.io.ByteStreams; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; @@ -82,7 +82,7 @@ private static String makeUnexpectedImageDigestWarning( @Override public BlobHttpContent getContent() { return new BlobHttpContent( - JsonTemplateMapper.toBlob(manifestTemplate), + Blobs.from(manifestTemplate), manifestTemplate.getManifestMediaType(), // TODO: Consider giving progress on manifest push as well? null); @@ -125,10 +125,7 @@ public DescriptorDigest handleHttpResponseException(HttpResponseException httpRe @Override public DescriptorDigest handleResponse(Response response) throws IOException { // Checks if the image digest is as expected. - DescriptorDigest expectedDigest = - JsonTemplateMapper.toBlob(manifestTemplate) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DescriptorDigest expectedDigest = DigestUtil.computeJsonDigest(manifestTemplate); List receivedDigests = response.getHeader(RESPONSE_DIGEST_HEADER); if (receivedDigests.size() == 1) { diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java index 1646bd659c..3f84a4b8f3 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java @@ -106,7 +106,6 @@ private void verifyBlobWriteTo(String expected, Blob blob) throws IOException { byte[] expectedBytes = expected.getBytes(StandardCharsets.UTF_8); Assert.assertEquals(expectedBytes.length, blobDescriptor.getSize()); - ; DescriptorDigest expectedDigest = DigestUtil.computeDigest(new ByteArrayInputStream(expectedBytes)).getDigest(); Assert.assertEquals(expectedDigest, blobDescriptor.getDigest()); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java index 16d3c1f0ab..f8dde2a44a 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java @@ -19,6 +19,7 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.LayerEntry; import com.google.common.collect.ImmutableList; @@ -74,17 +75,6 @@ private static Blob decompress(Blob blob) { }); } - /** - * Gets the digest of {@code blob}. - * - * @param blob the {@link Blob} - * @return the {@link DescriptorDigest} of {@code blob} - * @throws IOException if an I/O exception occurs - */ - private static DescriptorDigest digestOf(Blob blob) throws IOException { - return blob.writeTo(ByteStreams.nullOutputStream()).getDigest(); - } - /** * Gets the size of {@code blob}. * @@ -122,8 +112,8 @@ public void setUp() throws IOException { Files.createFile(directory.resolve("another/source/file")); layerBlob1 = Blobs.from("layerBlob1"); - layerDigest1 = digestOf(compress(layerBlob1)); - layerDiffId1 = digestOf(layerBlob1); + layerDigest1 = DigestUtil.computeDigest(compress(layerBlob1)).getDigest(); + layerDiffId1 = DigestUtil.computeDigest(layerBlob1).getDigest(); layerSize1 = sizeOf(compress(layerBlob1)); layerEntries1 = ImmutableList.of( @@ -134,8 +124,8 @@ public void setUp() throws IOException { AbsoluteUnixPath.get("/another/extraction/path"))); layerBlob2 = Blobs.from("layerBlob2"); - layerDigest2 = digestOf(compress(layerBlob2)); - layerDiffId2 = digestOf(layerBlob2); + layerDigest2 = DigestUtil.computeDigest(compress(layerBlob2)).getDigest(); + layerDiffId2 = DigestUtil.computeDigest(layerBlob2).getDigest(); layerSize2 = sizeOf(compress(layerBlob2)); layerEntries2 = ImmutableList.of(); } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriterTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriterTest.java index 59bcd8b54e..b1cf458d01 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriterTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriterTest.java @@ -19,10 +19,8 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.blob.Blobs; -import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.image.DescriptorDigest; -import com.google.common.io.ByteStreams; import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.file.Files; @@ -38,10 +36,6 @@ /** Tests for {@link DefaultCacheStorageWriter}. */ public class DefaultCacheStorageWriterTest { - private static DescriptorDigest getDigest(Blob blob) throws IOException { - return blob.writeTo(new CountingDigestOutputStream(ByteStreams.nullOutputStream())).getDigest(); - } - private static BlobDescriptor getCompressedBlobDescriptor(Blob blob) throws IOException { return DigestUtil.computeDigest(compress(blob)); } @@ -83,7 +77,8 @@ public void testWrite_compressed() throws IOException { public void testWrite_uncompressed() throws IOException { Blob uncompressedLayerBlob = Blobs.from("uncompressedLayerBlob"); DescriptorDigest layerDigest = getCompressedBlobDescriptor(uncompressedLayerBlob).getDigest(); - DescriptorDigest selector = getDigest(Blobs.from("selector")); + + DescriptorDigest selector = DigestUtil.computeDigest(Blobs.from("selector")).getDigest(); CachedLayer cachedLayer = new DefaultCacheStorageWriter(defaultCacheStorageFiles) @@ -100,7 +95,7 @@ public void testWrite_uncompressed() throws IOException { private void verifyCachedLayer(CachedLayer cachedLayer, Blob uncompressedLayerBlob) throws IOException { BlobDescriptor layerBlobDescriptor = getCompressedBlobDescriptor(uncompressedLayerBlob); - DescriptorDigest layerDiffId = getDigest(uncompressedLayerBlob); + DescriptorDigest layerDiffId = DigestUtil.computeDigest(uncompressedLayerBlob).getDigest(); // Verifies cachedLayer is correct. Assert.assertEquals(layerBlobDescriptor.getDigest(), cachedLayer.getDigest()); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/LayerEntriesSelectorTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/LayerEntriesSelectorTest.java index b8181d2e55..7337c9fc2b 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/LayerEntriesSelectorTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/LayerEntriesSelectorTest.java @@ -19,11 +19,10 @@ import com.google.cloud.tools.jib.cache.LayerEntriesSelector.LayerEntryTemplate; import com.google.cloud.tools.jib.configuration.FilePermissions; import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.LayerEntry; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.collect.ImmutableList; -import com.google.common.io.ByteStreams; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -108,10 +107,7 @@ public void testToSortedJsonTemplates() throws IOException { @Test public void testGenerateSelector_empty() throws IOException { - DescriptorDigest expectedSelector = - JsonTemplateMapper.toBlob(ImmutableList.of()) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DescriptorDigest expectedSelector = DigestUtil.computeJsonDigest(ImmutableList.of()); Assert.assertEquals( expectedSelector, LayerEntriesSelector.generateSelector(ImmutableList.of())); } @@ -119,9 +115,7 @@ public void testGenerateSelector_empty() throws IOException { @Test public void testGenerateSelector() throws IOException { DescriptorDigest expectedSelector = - JsonTemplateMapper.toBlob(toLayerEntryTemplates(inOrderLayerEntries)) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DigestUtil.computeJsonDigest(toLayerEntryTemplates(inOrderLayerEntries)); Assert.assertEquals( expectedSelector, LayerEntriesSelector.generateSelector(outOfOrderLayerEntries)); } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageTarballTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageTarballTest.java index 77695d019b..146bd780c8 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageTarballTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/ImageTarballTest.java @@ -89,7 +89,6 @@ public void testWriteTo() imageToTarball.writeTo(out); ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); try (TarArchiveInputStream tarArchiveInputStream = new TarArchiveInputStream(in)) { - imageToTarball.writeTo(out); // Verifies layer with fileA was added. TarArchiveEntry headerFileALayer = tarArchiveInputStream.getNextTarEntry(); @@ -121,7 +120,6 @@ public void testWriteTo() String manifestJson = CharStreams.toString( new InputStreamReader(tarArchiveInputStream, StandardCharsets.UTF_8)); - System.out.println(manifestJson); JsonTemplateMapper.readListOfJson(manifestJson, DockerLoadManifestEntryTemplate.class); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/json/DockerLoadManifestTemplateTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/json/DockerLoadManifestTemplateTest.java index 8669f6467f..2f45ed5f53 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/json/DockerLoadManifestTemplateTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/json/DockerLoadManifestTemplateTest.java @@ -16,7 +16,6 @@ package com.google.cloud.tools.jib.docker.json; -import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.image.ImageReference; import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.io.Resources; @@ -48,6 +47,6 @@ public void testToJson() throws URISyntaxException, IOException { template.addLayerFile("layer3.tar.gz"); List loadManifest = Collections.singletonList(template); - Assert.assertEquals(expectedJson, Blobs.writeToString(JsonTemplateMapper.toBlob(loadManifest))); + Assert.assertEquals(expectedJson, JsonTemplateMapper.toUtf8String(loadManifest)); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ContainerConfigurationTemplateTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ContainerConfigurationTemplateTest.java index 98366aacaa..e0e5423b16 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ContainerConfigurationTemplateTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ContainerConfigurationTemplateTest.java @@ -22,7 +22,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSortedMap; import com.google.common.io.Resources; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; @@ -91,10 +90,7 @@ public void testToJson() throws IOException, URISyntaxException, DigestException .build()); // Serializes the JSON object. - ByteArrayOutputStream jsonStream = new ByteArrayOutputStream(); - JsonTemplateMapper.toBlob(containerConfigJson).writeTo(jsonStream); - - Assert.assertEquals(expectedJson, jsonStream.toString()); + Assert.assertEquals(expectedJson, JsonTemplateMapper.toUtf8String(containerConfigJson)); } @Test diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslatorTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslatorTest.java index f7a5f68524..f7d333c3c6 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslatorTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslatorTest.java @@ -190,9 +190,6 @@ private void testGetManifest( T manifestTemplate = imageToJsonTranslator.getManifestTemplate(manifestTemplateClass, blobDescriptor); - ByteArrayOutputStream jsonStream = new ByteArrayOutputStream(); - JsonTemplateMapper.toBlob(manifestTemplate).writeTo(jsonStream); - - Assert.assertEquals(expectedJson, new String(jsonStream.toByteArray(), StandardCharsets.UTF_8)); + Assert.assertEquals(expectedJson, JsonTemplateMapper.toUtf8String(manifestTemplate)); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/OCIManifestTemplateTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/OCIManifestTemplateTest.java index 40ff8e4fc1..7c7defd779 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/OCIManifestTemplateTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/OCIManifestTemplateTest.java @@ -19,7 +19,6 @@ import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.io.Resources; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; @@ -53,10 +52,7 @@ public void testToJson() throws DigestException, IOException, URISyntaxException "4945ba5011739b0b98c4a41afe224e417f47c7c99b2ce76830999c9a0861b236")); // Serializes the JSON object. - ByteArrayOutputStream jsonStream = new ByteArrayOutputStream(); - JsonTemplateMapper.toBlob(manifestJson).writeTo(jsonStream); - - Assert.assertEquals(expectedJson, jsonStream.toString()); + Assert.assertEquals(expectedJson, JsonTemplateMapper.toUtf8String(manifestJson)); } @Test diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/V22ManifestTemplateTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/V22ManifestTemplateTest.java index 58eb1a3486..c8d0ba8d55 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/V22ManifestTemplateTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/V22ManifestTemplateTest.java @@ -19,7 +19,6 @@ import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.io.Resources; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; @@ -53,10 +52,7 @@ public void testToJson() throws DigestException, IOException, URISyntaxException "4945ba5011739b0b98c4a41afe224e417f47c7c99b2ce76830999c9a0861b236")); // Serializes the JSON object. - ByteArrayOutputStream jsonStream = new ByteArrayOutputStream(); - JsonTemplateMapper.toBlob(manifestJson).writeTo(jsonStream); - - Assert.assertEquals(expectedJson, jsonStream.toString()); + Assert.assertEquals(expectedJson, JsonTemplateMapper.toUtf8String(manifestJson)); } @Test diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/json/JsonTemplateMapperTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/json/JsonTemplateMapperTest.java index 1da54bad3d..4d2ea955c5 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/json/JsonTemplateMapperTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/json/JsonTemplateMapperTest.java @@ -18,7 +18,6 @@ import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.common.io.Resources; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; @@ -85,10 +84,7 @@ public void testWriteJson() throws DigestException, IOException, URISyntaxExcept "sha256:d38f571aa1c11e3d516e0ef7e513e7308ccbeb869770cb8c4319d63b10a0075e")); testJson.list = Arrays.asList(innerObject1, innerObject2); - ByteArrayOutputStream jsonStream = new ByteArrayOutputStream(); - JsonTemplateMapper.toBlob(testJson).writeTo(jsonStream); - - Assert.assertEquals(expectedJson, jsonStream.toString()); + Assert.assertEquals(expectedJson, JsonTemplateMapper.toUtf8String(testJson)); } @Test @@ -155,9 +151,6 @@ public void testToBlob_listOfJson() throws IOException, URISyntaxException { String jsonString = new String(Files.readAllBytes(jsonFile), StandardCharsets.UTF_8); List listOfJson = JsonTemplateMapper.readListOfJson(jsonString, TestJson.class); - ByteArrayOutputStream jsonStream = new ByteArrayOutputStream(); - JsonTemplateMapper.toBlob(listOfJson).writeTo(jsonStream); - - Assert.assertEquals(jsonString, jsonStream.toString()); + Assert.assertEquals(jsonString, JsonTemplateMapper.toUtf8String(listOfJson)); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobCheckerTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobCheckerTest.java index 2ec250c417..a45833830e 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobCheckerTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobCheckerTest.java @@ -19,7 +19,6 @@ import com.google.api.client.http.HttpResponseException; import com.google.api.client.http.HttpStatusCodes; import com.google.cloud.tools.jib.blob.BlobDescriptor; -import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.http.Response; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.json.JsonTemplateMapper; @@ -92,7 +91,7 @@ public void testHandleHttpResponseException() throws IOException, RegistryErrorE new ErrorResponseTemplate() .addError(new ErrorEntryTemplate(ErrorCodes.BLOB_UNKNOWN.name(), "some message")); Mockito.when(mockHttpResponseException.getContent()) - .thenReturn(Blobs.writeToString(JsonTemplateMapper.toBlob(emptyErrorResponseTemplate))); + .thenReturn(JsonTemplateMapper.toUtf8String(emptyErrorResponseTemplate)); BlobDescriptor blobDescriptor = testBlobChecker.handleHttpResponseException(mockHttpResponseException); @@ -112,7 +111,7 @@ public void testHandleHttpResponseException_hasOtherErrors() .addError(new ErrorEntryTemplate(ErrorCodes.BLOB_UNKNOWN.name(), "some message")) .addError(new ErrorEntryTemplate(ErrorCodes.MANIFEST_UNKNOWN.name(), "some message")); Mockito.when(mockHttpResponseException.getContent()) - .thenReturn(Blobs.writeToString(JsonTemplateMapper.toBlob(emptyErrorResponseTemplate))); + .thenReturn(JsonTemplateMapper.toUtf8String(emptyErrorResponseTemplate)); try { testBlobChecker.handleHttpResponseException(mockHttpResponseException); @@ -132,7 +131,7 @@ public void testHandleHttpResponseException_notBlobUnknown() ErrorResponseTemplate emptyErrorResponseTemplate = new ErrorResponseTemplate(); Mockito.when(mockHttpResponseException.getContent()) - .thenReturn(Blobs.writeToString(JsonTemplateMapper.toBlob(emptyErrorResponseTemplate))); + .thenReturn(JsonTemplateMapper.toUtf8String(emptyErrorResponseTemplate)); try { testBlobChecker.handleHttpResponseException(mockHttpResponseException); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPusherTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPusherTest.java index 57b8fdb0f8..c15ca8dfa9 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPusherTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPusherTest.java @@ -20,12 +20,12 @@ import com.google.api.client.http.HttpResponseException; import com.google.cloud.tools.jib.event.EventDispatcher; import com.google.cloud.tools.jib.event.events.LogEvent; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.http.BlobHttpContent; import com.google.cloud.tools.jib.http.Response; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.json.V22ManifestTemplate; import com.google.cloud.tools.jib.json.JsonTemplateMapper; -import com.google.common.io.ByteStreams; import com.google.common.io.Resources; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -90,10 +90,7 @@ public void testGetContent() throws IOException { @Test public void testHandleResponse_valid() throws IOException { - DescriptorDigest expectedDigest = - JsonTemplateMapper.toBlob(fakeManifestTemplate) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DescriptorDigest expectedDigest = DigestUtil.computeJsonDigest(fakeManifestTemplate); Mockito.when(mockResponse.getHeader("Docker-Content-Digest")) .thenReturn(Collections.singletonList(expectedDigest.toString())); Assert.assertEquals(expectedDigest, testManifestPusher.handleResponse(mockResponse)); @@ -101,10 +98,7 @@ public void testHandleResponse_valid() throws IOException { @Test public void testHandleResponse_noDigest() throws IOException { - DescriptorDigest expectedDigest = - JsonTemplateMapper.toBlob(fakeManifestTemplate) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DescriptorDigest expectedDigest = DigestUtil.computeJsonDigest(fakeManifestTemplate); Mockito.when(mockResponse.getHeader("Docker-Content-Digest")) .thenReturn(Collections.emptyList()); @@ -115,10 +109,7 @@ public void testHandleResponse_noDigest() throws IOException { @Test public void testHandleResponse_multipleDigests() throws IOException { - DescriptorDigest expectedDigest = - JsonTemplateMapper.toBlob(fakeManifestTemplate) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DescriptorDigest expectedDigest = DigestUtil.computeJsonDigest(fakeManifestTemplate); Mockito.when(mockResponse.getHeader("Docker-Content-Digest")) .thenReturn(Arrays.asList("too", "many")); @@ -130,10 +121,7 @@ public void testHandleResponse_multipleDigests() throws IOException { @Test public void testHandleResponse_invalidDigest() throws IOException { - DescriptorDigest expectedDigest = - JsonTemplateMapper.toBlob(fakeManifestTemplate) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DescriptorDigest expectedDigest = DigestUtil.computeJsonDigest(fakeManifestTemplate); Mockito.when(mockResponse.getHeader("Docker-Content-Digest")) .thenReturn(Collections.singletonList("not valid")); diff --git a/jib-maven-plugin/src/main/java/com/google/cloud/tools/jib/maven/MavenProjectProperties.java b/jib-maven-plugin/src/main/java/com/google/cloud/tools/jib/maven/MavenProjectProperties.java index ad34a38c44..677be7e69e 100644 --- a/jib-maven-plugin/src/main/java/com/google/cloud/tools/jib/maven/MavenProjectProperties.java +++ b/jib-maven-plugin/src/main/java/com/google/cloud/tools/jib/maven/MavenProjectProperties.java @@ -82,6 +82,8 @@ static MavenProjectProperties getForProject( * checks for a property defined in the POM, then returns null if neither are defined. * * @param propertyName the name of the system property + * @param project Maven project + * @param session Maven session * @return the value of the system property, or null if not defined */ @Nullable From efd85efeeda2ab4577d528c05d76c8e231da9c0d Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Mon, 29 Apr 2019 17:06:53 -0400 Subject: [PATCH 10/39] Revert TarStreamBuilder --- .../cloud/tools/jib/tar/TarStreamBuilder.java | 35 ++++++------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/tar/TarStreamBuilder.java b/jib-core/src/main/java/com/google/cloud/tools/jib/tar/TarStreamBuilder.java index d37c4cb2ac..c67e85ae77 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/tar/TarStreamBuilder.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/tar/TarStreamBuilder.java @@ -30,21 +30,10 @@ public class TarStreamBuilder { /** - * Represents contents of a tar entry. It may represent "yet-to-be-realized" contents; for - * example, loading the actual contents from a file may happen only when writing the contents into - * an output stream. + * Maps from {@link TarArchiveEntry} to a {@link Blob}. The order of the entries is the order they + * belong in the tarball. */ - @FunctionalInterface - private static interface WritableContents { - - void writeTo(OutputStream out) throws IOException; - } - - /** - * Maps from {@link TarArchiveEntry} to a {@link WritableContents}. The order of the entries is - * the order they belong in the tarball. - */ - private final LinkedHashMap archiveMap = new LinkedHashMap<>(); + private final LinkedHashMap archiveMap = new LinkedHashMap<>(); /** * Writes each entry in the filesystem to the tarball archive stream. @@ -57,7 +46,7 @@ public void writeAsTarArchiveTo(OutputStream out) throws IOException { new TarArchiveOutputStream(out, StandardCharsets.UTF_8.name())) { // Enables PAX extended headers to support long file names. tarArchiveOutputStream.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); - for (Map.Entry entry : archiveMap.entrySet()) { + for (Map.Entry entry : archiveMap.entrySet()) { tarArchiveOutputStream.putArchiveEntry(entry.getKey()); entry.getValue().writeTo(tarArchiveOutputStream); tarArchiveOutputStream.closeArchiveEntry(); @@ -71,17 +60,13 @@ public void writeAsTarArchiveTo(OutputStream out) throws IOException { * @param entry the {@link TarArchiveEntry} */ public void addTarArchiveEntry(TarArchiveEntry entry) { - if (!entry.isFile()) { - archiveMap.put(entry, ignored -> {}); - } else { - Blob fileBlob = Blobs.from(entry.getFile().toPath()); - archiveMap.put(entry, outputStream -> fileBlob.writeTo(outputStream)); - } + archiveMap.put( + entry, entry.isFile() ? Blobs.from(entry.getFile().toPath()) : Blobs.from(ignored -> {})); } /** - * Adds byte contents to the archive. Note that this should be used with raw bytes and not file - * contents; for adding files to the archive, use {@link #addTarArchiveEntry}. + * Adds a blob to the archive. Note that this should be used with raw bytes and not file contents; + * for adding files to the archive, use {@link #addTarArchiveEntry}. * * @param contents the bytes to add to the tarball * @param name the name of the entry (i.e. filename) @@ -89,7 +74,7 @@ public void addTarArchiveEntry(TarArchiveEntry entry) { public void addByteEntry(byte[] contents, String name) { TarArchiveEntry entry = new TarArchiveEntry(name); entry.setSize(contents.length); - archiveMap.put(entry, outputStream -> outputStream.write(contents)); + archiveMap.put(entry, Blobs.from(outputStream -> outputStream.write(contents))); } /** @@ -103,6 +88,6 @@ public void addByteEntry(byte[] contents, String name) { public void addBlobEntry(Blob blob, long size, String name) { TarArchiveEntry entry = new TarArchiveEntry(name); entry.setSize(size); - archiveMap.put(entry, outputStream -> blob.writeTo(outputStream)); + archiveMap.put(entry, blob); } } From 21067f798490ac7fa568937727a3e63f7252f221 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Mon, 29 Apr 2019 18:04:13 -0400 Subject: [PATCH 11/39] wip --- .../google/cloud/tools/jib/blob/Blobs.java | 4 +- ...{BlobWriter.java => WritableContents.java} | 2 +- ...rBlob.java => WriterableContentsBlob.java} | 12 ++--- .../cloud/tools/jib/hash/DigestUtil.java | 51 ++++++++----------- .../google/cloud/tools/jib/blob/BlobTest.java | 4 +- 5 files changed, 31 insertions(+), 42 deletions(-) rename jib-core/src/main/java/com/google/cloud/tools/jib/blob/{BlobWriter.java => WritableContents.java} (96%) rename jib-core/src/main/java/com/google/cloud/tools/jib/blob/{WriterBlob.java => WriterableContentsBlob.java} (69%) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java index 142f4134fb..445647cf28 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java @@ -50,8 +50,8 @@ public static Blob from(String content) { return new StringBlob(content); } - public static Blob from(BlobWriter writer) { - return new WriterBlob(writer); + public static Blob from(WritableContents writable) { + return new WritableContentsBlob(writable); } /** diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/BlobWriter.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WritableContents.java similarity index 96% rename from jib-core/src/main/java/com/google/cloud/tools/jib/blob/BlobWriter.java rename to jib-core/src/main/java/com/google/cloud/tools/jib/blob/WritableContents.java index acf393a524..3745bab65b 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/BlobWriter.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WritableContents.java @@ -24,7 +24,7 @@ * of the number of times it is called. */ @FunctionalInterface -public interface BlobWriter { +public interface WritableContents { void writeTo(OutputStream outputStream) throws IOException; } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterableContentsBlob.java similarity index 69% rename from jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterBlob.java rename to jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterableContentsBlob.java index fe92ec5df8..725067dd62 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterBlob.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterableContentsBlob.java @@ -20,17 +20,17 @@ import java.io.IOException; import java.io.OutputStream; -/** A {@link Blob} that writes with a {@link BlobWriter} function and hashes the bytes. */ -class WriterBlob implements Blob { +/** A {@link Blob} that writes with a {@link WritableContents} function and hashes the bytes. */ +class WritableContentsBlob implements Blob { - private final BlobWriter writer; + private final WritableContents writableContents; - WriterBlob(BlobWriter writer) { - this.writer = writer; + WritableContentsBlob(WritableContents writableContents) { + this.writableContents = writableContents; } @Override public BlobDescriptor writeTo(OutputStream outputStream) throws IOException { - return DigestUtil.computeDigest(out -> writer.writeTo(out), outputStream); + return DigestUtil.computeDigest(writableContents, outputStream); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java index efc3bdbd76..6f0f7cc7d0 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java @@ -16,11 +16,12 @@ package com.google.cloud.tools.jib.hash; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; +import com.google.cloud.tools.jib.blob.WritableContents; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.json.JsonTemplate; +import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.io.ByteStreams; import java.io.IOException; import java.io.InputStream; @@ -30,42 +31,26 @@ /** Utility class for input/output streams. */ public class DigestUtil { - /** - * Represents contents. It may represent "yet-to-be-realized" contents; for example, loading the - * actual contents from a file may happen only when writing the contents into an output stream. - */ - @FunctionalInterface - public static interface Contents { - - static Contents fromJson(Object jsonObject) { - return outStream -> new ObjectMapper().writeValue(outStream, jsonObject); - } - - void writeTo(OutputStream out) throws IOException; - } - public static DescriptorDigest computeJsonDigest(JsonTemplate template) throws IOException { return computeDigest(template, ByteStreams.nullOutputStream()).getDigest(); } public static DescriptorDigest computeJsonDigest(List templates) throws IOException { - return computeDigest(Contents.fromJson(templates), ByteStreams.nullOutputStream()).getDigest(); + WritableContents contents = contentsOut -> JsonTemplateMapper.writeTo(templates, contentsOut); + return computeDigest(contents, ByteStreams.nullOutputStream()).getDigest(); } - public static BlobDescriptor computeDigest(JsonTemplate template, OutputStream outStream) + public static BlobDescriptor computeDigest(JsonTemplate template, OutputStream optionalOutStream) throws IOException { - return computeDigest(Contents.fromJson(template), outStream); + WritableContents contents = contentsOut -> JsonTemplateMapper.writeTo(template, contentsOut); + return computeDigest(contents, optionalOutStream); } public static BlobDescriptor computeDigest(InputStream inStream) throws IOException { return computeDigest(inStream, ByteStreams.nullOutputStream()); } - public static BlobDescriptor computeDigest(Blob blob) throws IOException { - return computeDigest(out -> blob.writeTo(out), ByteStreams.nullOutputStream()); - } - /** * Computes the digest by consuming the contents. * @@ -73,7 +58,7 @@ public static BlobDescriptor computeDigest(Blob blob) throws IOException { * @return computed digest and bytes consumed * @throws IOException if reading fails */ - public static BlobDescriptor computeDigest(Contents contents) throws IOException { + public static BlobDescriptor computeDigest(WritableContents contents) throws IOException { return computeDigest(contents, ByteStreams.nullOutputStream()); } @@ -83,14 +68,14 @@ public static BlobDescriptor computeDigest(Contents contents) throws IOException * compute the digest. Does not close either stream. * * @param inStream the stream to read the contents from - * @param outStream the stream to which the contents are copied + * @param optionalOutStream the stream to which the contents are copied * @return computed digest and bytes consumed * @throws IOException if reading from or writing fails */ - public static BlobDescriptor computeDigest(InputStream inStream, OutputStream outStream) + public static BlobDescriptor computeDigest(InputStream inStream, OutputStream optionalOutStream) throws IOException { - Contents contents = anyOutSteam -> ByteStreams.copy(inStream, anyOutSteam); - return computeDigest(contents, outStream); + WritableContents contents = contentsOut -> ByteStreams.copy(inStream, contentsOut); + return computeDigest(contents, optionalOutStream); } /** @@ -99,15 +84,19 @@ public static BlobDescriptor computeDigest(InputStream inStream, OutputStream ou * stream. * * @param contents the contents for which the digest is computed - * @param outStream the stream to which the contents are copied + * @param optionalOutStream the stream to which the contents are copied * @return computed digest and bytes consumed * @throws IOException if reading from or writing fails */ - public static BlobDescriptor computeDigest(Contents contents, OutputStream outStream) - throws IOException { - CountingDigestOutputStream digestOutStream = new CountingDigestOutputStream(outStream); + public static BlobDescriptor computeDigest( + WritableContents contents, OutputStream optionalOutStream) throws IOException { + CountingDigestOutputStream digestOutStream = new CountingDigestOutputStream(optionalOutStream); contents.writeTo(digestOutStream); digestOutStream.flush(); return new BlobDescriptor(digestOutStream.getBytesHahsed(), digestOutStream.getDigest()); } + + public static BlobDescriptor computeDigest(Blob blob) throws IOException { + return blob.writeTo(ByteStreams.nullOutputStream()); + } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java index 3f84a4b8f3..6124b573f5 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java @@ -60,10 +60,10 @@ public void testFromString() throws IOException { public void testFromBlobWriter() throws IOException { String expected = "crepecake"; - BlobWriter writer = + WritableContents writableContents = outputStream -> outputStream.write(expected.getBytes(StandardCharsets.UTF_8)); - verifyBlobWriteTo(expected, Blobs.from(writer)); + verifyBlobWriteTo(expected, Blobs.from(writableContents)); } @Test From ec4a67faa2abf94513547d2cb6d6b40bf0764cba Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 30 Apr 2019 10:54:29 -0400 Subject: [PATCH 12/39] Remove Blob dep from DigestUtil --- .../jib/registry/BlobPullerIntegrationTest.java | 3 +-- .../tools/jib/blob/WriterableContentsBlob.java | 2 +- .../google/cloud/tools/jib/hash/DigestUtil.java | 5 ----- .../google/cloud/tools/jib/cache/CacheTest.java | 13 ++++++++----- .../jib/cache/DefaultCacheStorageWriterTest.java | 15 +++++++-------- 5 files changed, 17 insertions(+), 21 deletions(-) diff --git a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java index 4b38eafca1..8fa4a24344 100644 --- a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java +++ b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java @@ -18,7 +18,6 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.event.EventDispatcher; -import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.http.TestBlobProgressListener; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.json.V21ManifestTemplate; @@ -67,7 +66,7 @@ public void testPull() throws IOException, RegistryException, InterruptedExcepti new TestBlobProgressListener(totalByteCount::add)); Assert.assertTrue(expectedSize.sum() > 0); Assert.assertEquals(expectedSize.sum(), totalByteCount.sum()); - Assert.assertEquals(realDigest, DigestUtil.computeDigest(pulledBlob).getDigest()); + Assert.assertEquals(realDigest, pulledBlob.writeTo(ByteStreams.nullOutputStream()).getDigest()); } @Test diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterableContentsBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterableContentsBlob.java index 725067dd62..dc45430534 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterableContentsBlob.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterableContentsBlob.java @@ -20,7 +20,7 @@ import java.io.IOException; import java.io.OutputStream; -/** A {@link Blob} that writes with a {@link WritableContents} function and hashes the bytes. */ +/** A {@link Blob} that holds {@link WritableContents}. */ class WritableContentsBlob implements Blob { private final WritableContents writableContents; diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java index 6f0f7cc7d0..a101801012 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java @@ -16,7 +16,6 @@ package com.google.cloud.tools.jib.hash; -import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.blob.WritableContents; import com.google.cloud.tools.jib.image.DescriptorDigest; @@ -95,8 +94,4 @@ public static BlobDescriptor computeDigest( digestOutStream.flush(); return new BlobDescriptor(digestOutStream.getBytesHahsed(), digestOutStream.getDigest()); } - - public static BlobDescriptor computeDigest(Blob blob) throws IOException { - return blob.writeTo(ByteStreams.nullOutputStream()); - } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java index f8dde2a44a..755a48715e 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java @@ -19,7 +19,6 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; -import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.LayerEntry; import com.google.common.collect.ImmutableList; @@ -89,6 +88,10 @@ private static long sizeOf(Blob blob) throws IOException { return countingOutputStream.getCount(); } + private static DescriptorDigest digestOf(Blob blob) throws IOException { + return blob.writeTo(ByteStreams.nullOutputStream()).getDigest(); + } + @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); private Blob layerBlob1; @@ -112,8 +115,8 @@ public void setUp() throws IOException { Files.createFile(directory.resolve("another/source/file")); layerBlob1 = Blobs.from("layerBlob1"); - layerDigest1 = DigestUtil.computeDigest(compress(layerBlob1)).getDigest(); - layerDiffId1 = DigestUtil.computeDigest(layerBlob1).getDigest(); + layerDigest1 = digestOf(compress(layerBlob1)); + layerDiffId1 = digestOf(layerBlob1); layerSize1 = sizeOf(compress(layerBlob1)); layerEntries1 = ImmutableList.of( @@ -124,8 +127,8 @@ public void setUp() throws IOException { AbsoluteUnixPath.get("/another/extraction/path"))); layerBlob2 = Blobs.from("layerBlob2"); - layerDigest2 = DigestUtil.computeDigest(compress(layerBlob2)).getDigest(); - layerDiffId2 = DigestUtil.computeDigest(layerBlob2).getDigest(); + layerDigest2 = digestOf(compress(layerBlob2)); + layerDiffId2 = digestOf(layerBlob2); layerSize2 = sizeOf(compress(layerBlob2)); layerEntries2 = ImmutableList.of(); } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriterTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriterTest.java index dac9b1427a..ef1e9086a1 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriterTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriterTest.java @@ -19,7 +19,6 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.blob.Blobs; -import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.ImageReference; import com.google.cloud.tools.jib.image.InvalidImageReferenceException; @@ -28,6 +27,7 @@ import com.google.cloud.tools.jib.image.json.V21ManifestTemplate; import com.google.cloud.tools.jib.image.json.V22ManifestTemplate; import com.google.cloud.tools.jib.json.JsonTemplateMapper; +import com.google.common.io.ByteStreams; import java.io.ByteArrayInputStream; import java.io.IOException; import java.net.URISyntaxException; @@ -45,8 +45,8 @@ /** Tests for {@link DefaultCacheStorageWriter}. */ public class DefaultCacheStorageWriterTest { - private static BlobDescriptor getCompressedBlobDescriptor(Blob blob) throws IOException { - return DigestUtil.computeDigest(compress(blob)); + private static BlobDescriptor digestOf(Blob blob) throws IOException { + return blob.writeTo(ByteStreams.nullOutputStream()); } private static Blob compress(Blob blob) { @@ -87,9 +87,8 @@ public void testWrite_compressed() throws IOException { @Test public void testWrite_uncompressed() throws IOException { Blob uncompressedLayerBlob = Blobs.from("uncompressedLayerBlob"); - DescriptorDigest layerDigest = getCompressedBlobDescriptor(uncompressedLayerBlob).getDigest(); - - DescriptorDigest selector = DigestUtil.computeDigest(Blobs.from("selector")).getDigest(); + DescriptorDigest layerDigest = digestOf(compress(uncompressedLayerBlob)).getDigest(); + DescriptorDigest selector = digestOf(Blobs.from("selector")).getDigest(); CachedLayer cachedLayer = new DefaultCacheStorageWriter(defaultCacheStorageFiles) @@ -162,8 +161,8 @@ public void testWriteMetadata_v22() private void verifyCachedLayer(CachedLayer cachedLayer, Blob uncompressedLayerBlob) throws IOException { - BlobDescriptor layerBlobDescriptor = getCompressedBlobDescriptor(uncompressedLayerBlob); - DescriptorDigest layerDiffId = DigestUtil.computeDigest(uncompressedLayerBlob).getDigest(); + BlobDescriptor layerBlobDescriptor = digestOf(compress(uncompressedLayerBlob)); + DescriptorDigest layerDiffId = digestOf(uncompressedLayerBlob).getDigest(); // Verifies cachedLayer is correct. Assert.assertEquals(layerBlobDescriptor.getDigest(), cachedLayer.getDigest()); From 2cf3c4bc17699e13e1bb9c903420845a4d0c84ca Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 30 Apr 2019 11:19:07 -0400 Subject: [PATCH 13/39] Move WritableContents to hash package --- .../tools/jib/registry/BlobPullerIntegrationTest.java | 2 +- .../java/com/google/cloud/tools/jib/blob/Blobs.java | 1 + .../cloud/tools/jib/blob/WriterableContentsBlob.java | 1 + .../com/google/cloud/tools/jib/hash/DigestUtil.java | 9 +++++++-- .../tools/jib/{blob => hash}/WritableContents.java | 10 ++++++---- .../java/com/google/cloud/tools/jib/blob/BlobTest.java | 3 ++- 6 files changed, 18 insertions(+), 8 deletions(-) rename jib-core/src/main/java/com/google/cloud/tools/jib/{blob => hash}/WritableContents.java (66%) diff --git a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java index 8fa4a24344..4c46ceca3e 100644 --- a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java +++ b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java @@ -64,9 +64,9 @@ public void testPull() throws IOException, RegistryException, InterruptedExcepti expectedSize.add(size); }, new TestBlobProgressListener(totalByteCount::add)); + Assert.assertEquals(realDigest, pulledBlob.writeTo(ByteStreams.nullOutputStream()).getDigest()); Assert.assertTrue(expectedSize.sum() > 0); Assert.assertEquals(expectedSize.sum(), totalByteCount.sum()); - Assert.assertEquals(realDigest, pulledBlob.writeTo(ByteStreams.nullOutputStream()).getDigest()); } @Test diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java index 13d9d78300..cb621b232f 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java @@ -17,6 +17,7 @@ package com.google.cloud.tools.jib.blob; import com.google.cloud.tools.jib.filesystem.FileOperations; +import com.google.cloud.tools.jib.hash.WritableContents; import com.google.cloud.tools.jib.json.JsonTemplate; import java.io.ByteArrayOutputStream; import java.io.IOException; diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterableContentsBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterableContentsBlob.java index dc45430534..3c6f843293 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterableContentsBlob.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterableContentsBlob.java @@ -17,6 +17,7 @@ package com.google.cloud.tools.jib.blob; import com.google.cloud.tools.jib.hash.DigestUtil; +import com.google.cloud.tools.jib.hash.WritableContents; import java.io.IOException; import java.io.OutputStream; diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java index a101801012..6688d7096b 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java @@ -17,7 +17,6 @@ package com.google.cloud.tools.jib.hash; import com.google.cloud.tools.jib.blob.BlobDescriptor; -import com.google.cloud.tools.jib.blob.WritableContents; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.json.JsonTemplate; import com.google.cloud.tools.jib.json.JsonTemplateMapper; @@ -27,7 +26,13 @@ import java.io.OutputStream; import java.util.List; -/** Utility class for input/output streams. */ +/** + * Utility class for computing a digest for various inputs while optionally writing to an output + * stream. + */ +// Note: intentionally this class does not depend on Blob, as Blob classes depend on this class. +// TODO: BlobDescriptor is merely a tuple of (size, digest). Rename BlobDescriptor to something +// more general. public class DigestUtil { public static DescriptorDigest computeJsonDigest(JsonTemplate template) throws IOException { diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WritableContents.java b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/WritableContents.java similarity index 66% rename from jib-core/src/main/java/com/google/cloud/tools/jib/blob/WritableContents.java rename to jib-core/src/main/java/com/google/cloud/tools/jib/hash/WritableContents.java index 3745bab65b..5fc4fae624 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WritableContents.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/WritableContents.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Google LLC. + * Copyright 2019 Google LLC. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -14,14 +14,16 @@ * the License. */ -package com.google.cloud.tools.jib.blob; +package com.google.cloud.tools.jib.hash; import java.io.IOException; import java.io.OutputStream; /** - * This function writes the contents of a BLOB. This function should write the same BLOB regardless - * of the number of times it is called. + * As a function, writes some contents to an output stream. As a class, represents contents that can + * be written to an output stream. This may be "unrealized-before-write" contents; for example, a + * file may be open and read for input contents only when this function is called to write to an + * output stream. */ @FunctionalInterface public interface WritableContents { diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java index 6124b573f5..721b36b257 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java @@ -17,6 +17,7 @@ package com.google.cloud.tools.jib.blob; import com.google.cloud.tools.jib.hash.DigestUtil; +import com.google.cloud.tools.jib.hash.WritableContents; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.common.io.Resources; import java.io.ByteArrayInputStream; @@ -57,7 +58,7 @@ public void testFromString() throws IOException { } @Test - public void testFromBlobWriter() throws IOException { + public void testFromWritableContents() throws IOException { String expected = "crepecake"; WritableContents writableContents = From e5bb5e9c6480db003a50ace4295337bf19d2e3b1 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 30 Apr 2019 11:55:43 -0400 Subject: [PATCH 14/39] More refactoring --- .../jib/hash/CountingDigestOutputStream.java | 10 +++++----- .../google/cloud/tools/jib/hash/DigestUtil.java | 10 +++++----- .../google/cloud/tools/jib/cache/CacheTest.java | 15 +++++++++++---- .../jib/cache/DefaultCacheStorageWriterTest.java | 10 +++++----- 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStream.java b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStream.java index 59343b8a48..12cf3ebd65 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStream.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStream.java @@ -34,7 +34,7 @@ public class CountingDigestOutputStream extends DigestOutputStream { private long bytesSoFar = 0; /** The total number of bytes used to compute a digest. Resets when {@link computeDigest) is called. */ - private long bytesHashed; + private long bytesHashed = 0; @Nullable private DescriptorDigest descriptorDigest; @@ -54,8 +54,8 @@ public CountingDigestOutputStream(OutputStream outputStream) { } /** - * Computes the hash and size of the bytes written. The buffer resets after this method is called, - * so this method should only be called once per computation. + * Computes the hash and remembers the size of the bytes written to compute the hash. The buffer + * resets after this method is called, so this method should only be called once per computation. */ public void computeDigest() { try { @@ -77,7 +77,7 @@ public void computeDigest() { } } - /** @return the total number of bytes that were hashed */ + /** @return the number of bytes written and used to compute the most recent digest */ public long getBytesHahsed() { if (descriptorDigest == null) { computeDigest(); @@ -85,7 +85,7 @@ public long getBytesHahsed() { return bytesHashed; } - /** @return the digest hash */ + /** @return the most recently computed digest hash */ public DescriptorDigest getDigest() { if (descriptorDigest == null) { computeDigest(); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java index 6688d7096b..946083c69a 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java @@ -68,8 +68,8 @@ public static BlobDescriptor computeDigest(WritableContents contents) throws IOE /** * Computes the digest by consuming the contents of an {@link InputStream} and optionally copying - * it to an {@link OutputStream}. Returns the computed digested along with the bytes consumed to - * compute the digest. Does not close either stream. + * it to an {@link OutputStream}. Returns the computed digest along with the size of the bytes + * consumed to compute the digest. Does not close either stream. * * @param inStream the stream to read the contents from * @param optionalOutStream the stream to which the contents are copied @@ -84,10 +84,10 @@ public static BlobDescriptor computeDigest(InputStream inStream, OutputStream op /** * Computes the digest by consuming the contents and optionally copying it to an {@link - * OutputStream}. Returns the computed digested along with the bytes consumed. Does not close the - * stream. + * OutputStream}. Returns the computed digest along with the size of the bytes consumed to compute + * the digest. Does not close the stream. * - * @param contents the contents for which the digest is computed + * @param contents the contents to compute digest for * @param optionalOutStream the stream to which the contents are copied * @return computed digest and bytes consumed * @throws IOException if reading from or writing fails diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java index 755a48715e..16d3c1f0ab 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java @@ -74,6 +74,17 @@ private static Blob decompress(Blob blob) { }); } + /** + * Gets the digest of {@code blob}. + * + * @param blob the {@link Blob} + * @return the {@link DescriptorDigest} of {@code blob} + * @throws IOException if an I/O exception occurs + */ + private static DescriptorDigest digestOf(Blob blob) throws IOException { + return blob.writeTo(ByteStreams.nullOutputStream()).getDigest(); + } + /** * Gets the size of {@code blob}. * @@ -88,10 +99,6 @@ private static long sizeOf(Blob blob) throws IOException { return countingOutputStream.getCount(); } - private static DescriptorDigest digestOf(Blob blob) throws IOException { - return blob.writeTo(ByteStreams.nullOutputStream()).getDigest(); - } - @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); private Blob layerBlob1; diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriterTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriterTest.java index ef1e9086a1..981a98cf8c 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriterTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriterTest.java @@ -45,7 +45,7 @@ /** Tests for {@link DefaultCacheStorageWriter}. */ public class DefaultCacheStorageWriterTest { - private static BlobDescriptor digestOf(Blob blob) throws IOException { + private static BlobDescriptor getDigest(Blob blob) throws IOException { return blob.writeTo(ByteStreams.nullOutputStream()); } @@ -87,8 +87,8 @@ public void testWrite_compressed() throws IOException { @Test public void testWrite_uncompressed() throws IOException { Blob uncompressedLayerBlob = Blobs.from("uncompressedLayerBlob"); - DescriptorDigest layerDigest = digestOf(compress(uncompressedLayerBlob)).getDigest(); - DescriptorDigest selector = digestOf(Blobs.from("selector")).getDigest(); + DescriptorDigest layerDigest = getDigest(compress(uncompressedLayerBlob)).getDigest(); + DescriptorDigest selector = getDigest(Blobs.from("selector")).getDigest(); CachedLayer cachedLayer = new DefaultCacheStorageWriter(defaultCacheStorageFiles) @@ -161,8 +161,8 @@ public void testWriteMetadata_v22() private void verifyCachedLayer(CachedLayer cachedLayer, Blob uncompressedLayerBlob) throws IOException { - BlobDescriptor layerBlobDescriptor = digestOf(compress(uncompressedLayerBlob)); - DescriptorDigest layerDiffId = digestOf(uncompressedLayerBlob).getDigest(); + BlobDescriptor layerBlobDescriptor = getDigest(compress(uncompressedLayerBlob)); + DescriptorDigest layerDiffId = getDigest(uncompressedLayerBlob).getDigest(); // Verifies cachedLayer is correct. Assert.assertEquals(layerBlobDescriptor.getDigest(), cachedLayer.getDigest()); From 8b823279095f13cb9ba9919f6174bcebcbaed565 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 30 Apr 2019 13:00:56 -0400 Subject: [PATCH 15/39] Decouple Blob from http Response --- .../google/cloud/tools/jib/http/Response.java | 9 ++++---- .../cloud/tools/jib/registry/BlobPuller.java | 4 +++- .../tools/jib/registry/ManifestPuller.java | 2 +- .../jib/registry/RegistryAuthenticator.java | 2 +- .../cloud/tools/jib/http/ResponseTest.java | 7 +++--- .../jib/http/WithServerConnectionTest.java | 14 +++++++++--- .../tools/jib/registry/BlobPullerTest.java | 20 +++++++++-------- .../jib/registry/ManifestPullerTest.java | 22 ++++++++++++++----- .../registry/RegistryEndpointCallerTest.java | 12 +++++++--- 9 files changed, 60 insertions(+), 32 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/http/Response.java b/jib-core/src/main/java/com/google/cloud/tools/jib/http/Response.java index a9f66d235b..3c7f895586 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/http/Response.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/http/Response.java @@ -18,10 +18,9 @@ import com.google.api.client.http.GenericUrl; import com.google.api.client.http.HttpResponse; -import com.google.cloud.tools.jib.blob.Blob; -import com.google.cloud.tools.jib.blob.Blobs; import com.google.common.net.HttpHeaders; import java.io.IOException; +import java.io.InputStream; import java.util.List; /** Holds an HTTP response. */ @@ -65,11 +64,11 @@ public long getContentLength() throws NumberFormatException { } /** - * @return the HTTP response body as a {@link Blob}. + * @return the HTTP response body as an {@link InputStream}. * @throws IOException if getting the HTTP response content fails. */ - public Blob getBody() throws IOException { - return Blobs.from(httpResponse.getContent()); + public InputStream getBody() throws IOException { + return httpResponse.getContent(); } /** @return the original request URL */ diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java index b0534dab42..b7eed4b3ec 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java @@ -18,6 +18,7 @@ import com.google.api.client.http.HttpMethods; import com.google.cloud.tools.jib.blob.BlobDescriptor; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.http.BlobHttpContent; import com.google.cloud.tools.jib.http.BlobProgressListener; import com.google.cloud.tools.jib.http.ListenableCountingOutputStream; @@ -70,7 +71,8 @@ public Void handleResponse(Response response) throws IOException, UnexpectedBlob destinationOutputStream, blobProgressListener::handleByteCount, blobProgressListener.getDelayBetweenCallbacks())) { - BlobDescriptor receivedBlobDescriptor = response.getBody().writeTo(outputStream); + BlobDescriptor receivedBlobDescriptor = + DigestUtil.computeDigest(response.getBody(), outputStream); if (!blobDigest.equals(receivedBlobDescriptor.getDigest())) { throw new UnexpectedBlobDigestException( diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPuller.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPuller.java index a5ba631c57..2d961094a2 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPuller.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPuller.java @@ -79,7 +79,7 @@ public List getAccept() { /** Parses the response body into a {@link ManifestTemplate}. */ @Override public T handleResponse(Response response) throws IOException, UnknownManifestFormatException { - return getManifestTemplateFromJson(Blobs.writeToString(response.getBody())); + return getManifestTemplateFromJson(Blobs.writeToString(Blobs.from(response.getBody()))); } @Override diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryAuthenticator.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryAuthenticator.java index 0058a23062..7f24f65a92 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryAuthenticator.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryAuthenticator.java @@ -323,7 +323,7 @@ private Authorization authenticate(String scope) throws RegistryAuthenticationFa Request request = requestBuilder.build(); Response response = isOAuth2Auth() ? connection.post(request) : connection.get(request); - String responseString = Blobs.writeToString(response.getBody()); + String responseString = Blobs.writeToString(Blobs.from(response.getBody())); AuthenticationResponseTemplate responseJson = JsonTemplateMapper.readJson(responseString, AuthenticationResponseTemplate.class); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/http/ResponseTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/http/ResponseTest.java index bde361410e..b75af784bc 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/http/ResponseTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/http/ResponseTest.java @@ -17,10 +17,11 @@ package com.google.cloud.tools.jib.http; import com.google.api.client.http.HttpResponse; -import com.google.cloud.tools.jib.blob.Blob; +import com.google.common.io.ByteStreams; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.charset.StandardCharsets; import org.junit.Assert; import org.junit.Test; @@ -44,10 +45,10 @@ public void testGetContent() throws IOException { Mockito.when(httpResponseMock.getContent()).thenReturn(responseInputStream); Response response = new Response(httpResponseMock); - Blob responseStream = response.getBody(); + InputStream responseStream = response.getBody(); ByteArrayOutputStream responseOutputStream = new ByteArrayOutputStream(); - responseStream.writeTo(responseOutputStream); + ByteStreams.copy(responseStream, responseOutputStream); Assert.assertEquals( expectedResponse, new String(responseOutputStream.toByteArray(), StandardCharsets.UTF_8)); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/http/WithServerConnectionTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/http/WithServerConnectionTest.java index f870911d26..39855fc6ec 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/http/WithServerConnectionTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/http/WithServerConnectionTest.java @@ -16,8 +16,10 @@ package com.google.cloud.tools.jib.http; -import com.google.cloud.tools.jib.blob.Blobs; +import com.google.common.io.ByteStreams; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.net.URISyntaxException; import java.net.URL; import java.security.GeneralSecurityException; @@ -28,6 +30,12 @@ /** Tests for {@link Connection} using an actual local server. */ public class WithServerConnectionTest { + private static String inputStreamToUtf8String(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ByteStreams.copy(in, out); + return out.toString("UTF-8"); + } + @Test public void testGet() throws IOException, InterruptedException, GeneralSecurityException, URISyntaxException { @@ -37,7 +45,7 @@ public void testGet() Response response = connection.send("GET", new Request.Builder().build()); Assert.assertEquals(200, response.getStatusCode()); - Assert.assertEquals("Hello World!", Blobs.writeToString(response.getBody())); + Assert.assertEquals("Hello World!", inputStreamToUtf8String(response.getBody())); } } @@ -81,7 +89,7 @@ public void testInsecureConnection() Response response = connection.send("GET", new Request.Builder().build()); Assert.assertEquals(200, response.getStatusCode()); - Assert.assertEquals("Hello World!", Blobs.writeToString(response.getBody())); + Assert.assertEquals("Hello World!", inputStreamToUtf8String(response.getBody())); } } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java index a21f4bd725..8980af070d 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java @@ -16,15 +16,15 @@ package com.google.cloud.tools.jib.registry; -import com.google.cloud.tools.jib.blob.Blob; -import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.http.Response; import com.google.cloud.tools.jib.http.TestBlobProgressListener; import com.google.cloud.tools.jib.image.DescriptorDigest; -import com.google.common.io.ByteStreams; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.net.MalformedURLException; import java.net.URL; import java.nio.charset.StandardCharsets; @@ -68,12 +68,13 @@ public void setUpFakes() throws DigestException { @Test public void testHandleResponse() throws IOException, UnexpectedBlobDigestException { - Blob testBlob = Blobs.from("some BLOB content"); - DescriptorDigest testBlobDigest = testBlob.writeTo(ByteStreams.nullOutputStream()).getDigest(); + InputStream blobContent = + new ByteArrayInputStream("some BLOB content".getBytes(StandardCharsets.UTF_8)); + DescriptorDigest testBlobDigest = DigestUtil.computeDigest(blobContent).getDigest(); Response mockResponse = Mockito.mock(Response.class); Mockito.when(mockResponse.getContentLength()).thenReturn((long) "some BLOB content".length()); - Mockito.when(mockResponse.getBody()).thenReturn(testBlob); + Mockito.when(mockResponse.getBody()).thenReturn(blobContent); LongAdder byteCount = new LongAdder(); BlobPuller blobPuller = @@ -93,11 +94,12 @@ public void testHandleResponse() throws IOException, UnexpectedBlobDigestExcepti @Test public void testHandleResponse_unexpectedDigest() throws IOException { - Blob testBlob = Blobs.from("some BLOB content"); - DescriptorDigest testBlobDigest = testBlob.writeTo(ByteStreams.nullOutputStream()).getDigest(); + InputStream blobContent = + new ByteArrayInputStream("some BLOB content".getBytes(StandardCharsets.UTF_8)); + DescriptorDigest testBlobDigest = DigestUtil.computeDigest(blobContent).getDigest(); Response mockResponse = Mockito.mock(Response.class); - Mockito.when(mockResponse.getBody()).thenReturn(testBlob); + Mockito.when(mockResponse.getBody()).thenReturn(blobContent); try { testBlobPuller.handleResponse(mockResponse); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPullerTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPullerTest.java index 05747193e3..0c5a37fdd4 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPullerTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPullerTest.java @@ -16,7 +16,6 @@ package com.google.cloud.tools.jib.registry; -import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.http.Response; import com.google.cloud.tools.jib.image.json.ManifestTemplate; import com.google.cloud.tools.jib.image.json.OCIManifestTemplate; @@ -24,10 +23,14 @@ import com.google.cloud.tools.jib.image.json.V21ManifestTemplate; import com.google.cloud.tools.jib.image.json.V22ManifestTemplate; import com.google.common.io.Resources; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; @@ -44,6 +47,10 @@ @RunWith(MockitoJUnitRunner.class) public class ManifestPullerTest { + private static InputStream stringToInputStreamUtf8(String string) { + return new ByteArrayInputStream(string.getBytes(StandardCharsets.UTF_8)); + } + @Mock private Response mockResponse; private final RegistryEndpointRequestProperties fakeRegistryEndpointRequestProperties = @@ -56,8 +63,9 @@ public class ManifestPullerTest { public void testHandleResponse_v21() throws URISyntaxException, IOException, UnknownManifestFormatException { Path v21ManifestFile = Paths.get(Resources.getResource("core/json/v21manifest.json").toURI()); + InputStream v21Manifest = new ByteArrayInputStream(Files.readAllBytes(v21ManifestFile)); - Mockito.when(mockResponse.getBody()).thenReturn(Blobs.from(v21ManifestFile)); + Mockito.when(mockResponse.getBody()).thenReturn(v21Manifest); ManifestTemplate manifestTemplate = new ManifestPuller<>( fakeRegistryEndpointRequestProperties, "test-image-tag", V21ManifestTemplate.class) @@ -70,8 +78,9 @@ public void testHandleResponse_v21() public void testHandleResponse_v22() throws URISyntaxException, IOException, UnknownManifestFormatException { Path v22ManifestFile = Paths.get(Resources.getResource("core/json/v22manifest.json").toURI()); + InputStream v22Manifest = new ByteArrayInputStream(Files.readAllBytes(v22ManifestFile)); - Mockito.when(mockResponse.getBody()).thenReturn(Blobs.from(v22ManifestFile)); + Mockito.when(mockResponse.getBody()).thenReturn(v22Manifest); ManifestTemplate manifestTemplate = new ManifestPuller<>( fakeRegistryEndpointRequestProperties, "test-image-tag", V22ManifestTemplate.class) @@ -82,7 +91,7 @@ public void testHandleResponse_v22() @Test public void testHandleResponse_noSchemaVersion() throws IOException { - Mockito.when(mockResponse.getBody()).thenReturn(Blobs.from("{}")); + Mockito.when(mockResponse.getBody()).thenReturn(stringToInputStreamUtf8("{}")); try { testManifestPuller.handleResponse(mockResponse); Assert.fail("An empty manifest should throw an error"); @@ -95,7 +104,7 @@ public void testHandleResponse_noSchemaVersion() throws IOException { @Test public void testHandleResponse_invalidSchemaVersion() throws IOException { Mockito.when(mockResponse.getBody()) - .thenReturn(Blobs.from("{\"schemaVersion\":\"not valid\"}")); + .thenReturn(stringToInputStreamUtf8("{\"schemaVersion\":\"not valid\"}")); try { testManifestPuller.handleResponse(mockResponse); Assert.fail("A non-integer schemaVersion should throw an error"); @@ -107,7 +116,8 @@ public void testHandleResponse_invalidSchemaVersion() throws IOException { @Test public void testHandleResponse_unknownSchemaVersion() throws IOException { - Mockito.when(mockResponse.getBody()).thenReturn(Blobs.from("{\"schemaVersion\":0}")); + Mockito.when(mockResponse.getBody()) + .thenReturn(stringToInputStreamUtf8("{\"schemaVersion\":0}")); try { testManifestPuller.handleResponse(mockResponse); Assert.fail("An unknown manifest schemaVersion should throw an error"); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/RegistryEndpointCallerTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/RegistryEndpointCallerTest.java index a01711d047..5600a9cc3d 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/RegistryEndpointCallerTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/RegistryEndpointCallerTest.java @@ -20,7 +20,6 @@ import com.google.api.client.http.HttpResponse; import com.google.api.client.http.HttpResponseException; import com.google.api.client.http.HttpStatusCodes; -import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.event.EventDispatcher; import com.google.cloud.tools.jib.event.events.LogEvent; import com.google.cloud.tools.jib.global.JibSystemProperties; @@ -29,11 +28,15 @@ import com.google.cloud.tools.jib.http.Connection; import com.google.cloud.tools.jib.http.MockConnection; import com.google.cloud.tools.jib.http.Response; +import com.google.common.io.ByteStreams; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.ConnectException; import java.net.MalformedURLException; import java.net.SocketException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.List; import java.util.function.Function; @@ -83,7 +86,9 @@ public List getAccept() { @Nullable @Override public String handleResponse(Response response) throws IOException { - return Blobs.writeToString(response.getBody()); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + ByteStreams.copy(response.getBody(), outputStream); + return outputStream.toString("UTF-8"); } @Override @@ -122,7 +127,8 @@ public void setUp() throws IOException { Mockito.when(mockConnectionFactory.apply(Mockito.any())).thenReturn(mockConnection); Mockito.when(mockInsecureConnectionFactory.apply(Mockito.any())) .thenReturn(mockInsecureConnection); - Mockito.when(mockResponse.getBody()).thenReturn(Blobs.from("body")); + Mockito.when(mockResponse.getBody()) + .thenReturn(new ByteArrayInputStream("body".getBytes(StandardCharsets.UTF_8))); } @After From 873ada8ab7fac887d73ace1d711ead750b3cd60a Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 30 Apr 2019 14:10:31 -0400 Subject: [PATCH 16/39] wip --- .../builder/steps/AuthenticatePushStep.java | 3 +- .../jib/builder/steps/PullBaseImageStep.java | 3 +- .../cloud/tools/jib/http/Authorization.java | 29 ++++++++++ .../cloud/tools/jib/http/Authorizations.java | 53 ------------------- .../tools/jib/registry/ManifestPuller.java | 8 ++- .../jib/registry/RegistryAuthenticator.java | 12 +++-- .../cloud/tools/jib/http/ConnectionTest.java | 2 +- .../cloud/tools/jib/http/ResponseTest.java | 14 ++--- .../jib/http/WithServerConnectionTest.java | 17 +++--- .../registry/RegistryEndpointCallerTest.java | 13 +++-- 10 files changed, 61 insertions(+), 93 deletions(-) delete mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/http/Authorizations.java diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java index 036e0fcd43..89cef660e0 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java @@ -25,7 +25,6 @@ import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.configuration.credentials.Credential; import com.google.cloud.tools.jib.http.Authorization; -import com.google.cloud.tools.jib.http.Authorizations; import com.google.cloud.tools.jib.registry.RegistryAuthenticationFailedException; import com.google.cloud.tools.jib.registry.RegistryAuthenticator; import com.google.cloud.tools.jib.registry.RegistryException; @@ -102,7 +101,7 @@ public Authorization call() return (registryCredential == null || registryCredential.isOAuth2RefreshToken()) ? null - : Authorizations.withBasicCredentials( + : Authorization.withBasicCredentials( registryCredential.getUsername(), registryCredential.getPassword()); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java index a7d8773357..e45c6783fd 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java @@ -29,7 +29,6 @@ import com.google.cloud.tools.jib.event.events.LogEvent; import com.google.cloud.tools.jib.event.events.ProgressEvent; import com.google.cloud.tools.jib.http.Authorization; -import com.google.cloud.tools.jib.http.Authorizations; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.Layer; @@ -160,7 +159,7 @@ public BaseImageWithAuthorization call() Authorization registryAuthorization = registryCredential == null || registryCredential.isOAuth2RefreshToken() ? null - : Authorizations.withBasicCredentials( + : Authorization.withBasicCredentials( registryCredential.getUsername(), registryCredential.getPassword()); try { diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/http/Authorization.java b/jib-core/src/main/java/com/google/cloud/tools/jib/http/Authorization.java index 0bb13e407d..17e7bb9f20 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/http/Authorization.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/http/Authorization.java @@ -16,6 +16,8 @@ package com.google.cloud.tools.jib.http; +import com.google.api.client.util.Base64; +import java.nio.charset.StandardCharsets; import java.util.Objects; /** @@ -27,6 +29,33 @@ */ public class Authorization { + /** + * @param token the token + * @return an {@link Authorization} with a {@code Bearer} token + */ + public static Authorization withBearerToken(String token) { + return new Authorization("Bearer", token); + } + + /** + * @param username the username + * @param secret the secret + * @return an {@link Authorization} with a {@code Basic} credentials + */ + public static Authorization withBasicCredentials(String username, String secret) { + String credentials = username + ":" + secret; + String token = Base64.encodeBase64String(credentials.getBytes(StandardCharsets.UTF_8)); + return new Authorization("Basic", token); + } + + /** + * @param token the token + * @return an {@link Authorization} with a base64-encoded {@code username:password} string + */ + public static Authorization withBasicToken(String token) { + return new Authorization("Basic", token); + } + private final String scheme; private final String token; diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/http/Authorizations.java b/jib-core/src/main/java/com/google/cloud/tools/jib/http/Authorizations.java deleted file mode 100644 index e8ae835a23..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/http/Authorizations.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2017 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.http; - -import com.google.api.client.util.Base64; -import java.nio.charset.StandardCharsets; - -/** Static initializers for {@link Authorization}. */ -public class Authorizations { - - /** - * @param token the token - * @return an {@link Authorization} with a {@code Bearer} token - */ - public static Authorization withBearerToken(String token) { - return new Authorization("Bearer", token); - } - - /** - * @param username the username - * @param secret the secret - * @return an {@link Authorization} with a {@code Basic} credentials - */ - public static Authorization withBasicCredentials(String username, String secret) { - String credentials = username + ":" + secret; - String token = Base64.encodeBase64String(credentials.getBytes(StandardCharsets.UTF_8)); - return new Authorization("Basic", token); - } - - /** - * @param token the token - * @return an {@link Authorization} with a base64-encoded {@code username:password} string - */ - public static Authorization withBasicToken(String token) { - return new Authorization("Basic", token); - } - - private Authorizations() {} -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPuller.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPuller.java index 2d961094a2..615a4a9500 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPuller.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPuller.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.api.client.http.HttpMethods; -import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.http.BlobHttpContent; import com.google.cloud.tools.jib.http.Response; import com.google.cloud.tools.jib.image.json.ManifestTemplate; @@ -28,9 +27,12 @@ import com.google.cloud.tools.jib.image.json.V21ManifestTemplate; import com.google.cloud.tools.jib.image.json.V22ManifestTemplate; import com.google.cloud.tools.jib.json.JsonTemplateMapper; +import com.google.common.io.CharStreams; import java.io.IOException; +import java.io.InputStreamReader; import java.net.MalformedURLException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -79,7 +81,9 @@ public List getAccept() { /** Parses the response body into a {@link ManifestTemplate}. */ @Override public T handleResponse(Response response) throws IOException, UnknownManifestFormatException { - return getManifestTemplateFromJson(Blobs.writeToString(Blobs.from(response.getBody()))); + String jsonString = + CharStreams.toString(new InputStreamReader(response.getBody(), StandardCharsets.UTF_8)); + return getManifestTemplateFromJson(jsonString); } @Override diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryAuthenticator.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryAuthenticator.java index 7f24f65a92..0ee7d52cc1 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryAuthenticator.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryAuthenticator.java @@ -22,7 +22,6 @@ import com.google.cloud.tools.jib.event.EventDispatcher; import com.google.cloud.tools.jib.global.JibSystemProperties; import com.google.cloud.tools.jib.http.Authorization; -import com.google.cloud.tools.jib.http.Authorizations; import com.google.cloud.tools.jib.http.BlobHttpContent; import com.google.cloud.tools.jib.http.Connection; import com.google.cloud.tools.jib.http.Request; @@ -31,10 +30,13 @@ import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Verify; +import com.google.common.io.CharStreams; import com.google.common.net.MediaType; import java.io.IOException; +import java.io.InputStreamReader; import java.net.MalformedURLException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.annotation.Nullable; @@ -317,13 +319,13 @@ private Authorization authenticate(String scope) throws RegistryAuthenticationFa new BlobHttpContent(Blobs.from(parameters), MediaType.FORM_DATA.toString(), null)); } else if (credential != null) { requestBuilder.setAuthorization( - Authorizations.withBasicCredentials( - credential.getUsername(), credential.getPassword())); + Authorization.withBasicCredentials(credential.getUsername(), credential.getPassword())); } Request request = requestBuilder.build(); Response response = isOAuth2Auth() ? connection.post(request) : connection.get(request); - String responseString = Blobs.writeToString(Blobs.from(response.getBody())); + String responseString = + CharStreams.toString(new InputStreamReader(response.getBody(), StandardCharsets.UTF_8)); AuthenticationResponseTemplate responseJson = JsonTemplateMapper.readJson(responseString, AuthenticationResponseTemplate.class); @@ -337,7 +339,7 @@ private Authorization authenticate(String scope) throws RegistryAuthenticationFa + "; parameters: " + getAuthRequestParameters(scope)); } - return Authorizations.withBearerToken(responseJson.getToken()); + return Authorization.withBearerToken(responseJson.getToken()); } catch (IOException ex) { throw new RegistryAuthenticationFailedException( diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/http/ConnectionTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/http/ConnectionTest.java index 3732146904..1659c63eea 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/http/ConnectionTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/http/ConnectionTest.java @@ -114,7 +114,7 @@ private void setUpMocksAndFakes(Integer httpTimeout) throws IOException { Blobs.from("crepecake"), "fake.content.type", new TestBlobProgressListener(byteCount -> totalByteCount += byteCount))) - .setAuthorization(Authorizations.withBasicCredentials("fake-username", "fake-secret")) + .setAuthorization(Authorization.withBasicCredentials("fake-username", "fake-secret")) .setHttpTimeout(httpTimeout) .build(); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/http/ResponseTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/http/ResponseTest.java index b75af784bc..91865a270a 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/http/ResponseTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/http/ResponseTest.java @@ -19,9 +19,7 @@ import com.google.api.client.http.HttpResponse; import com.google.common.io.ByteStreams; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.InputStream; import java.nio.charset.StandardCharsets; import org.junit.Assert; import org.junit.Test; @@ -38,19 +36,13 @@ public class ResponseTest { @Test public void testGetContent() throws IOException { - String expectedResponse = "crepecake\nis\ngood!"; - ByteArrayInputStream responseInputStream = - new ByteArrayInputStream(expectedResponse.getBytes(StandardCharsets.UTF_8)); + byte[] expectedResponse = "crepecake\nis\ngood!".getBytes(StandardCharsets.UTF_8); + ByteArrayInputStream responseInputStream = new ByteArrayInputStream(expectedResponse); Mockito.when(httpResponseMock.getContent()).thenReturn(responseInputStream); Response response = new Response(httpResponseMock); - InputStream responseStream = response.getBody(); - ByteArrayOutputStream responseOutputStream = new ByteArrayOutputStream(); - ByteStreams.copy(responseStream, responseOutputStream); - - Assert.assertEquals( - expectedResponse, new String(responseOutputStream.toByteArray(), StandardCharsets.UTF_8)); + Assert.assertEquals(expectedResponse, ByteStreams.toByteArray(response.getBody())); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/http/WithServerConnectionTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/http/WithServerConnectionTest.java index 39855fc6ec..8ab952cab8 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/http/WithServerConnectionTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/http/WithServerConnectionTest.java @@ -17,11 +17,10 @@ package com.google.cloud.tools.jib.http; import com.google.common.io.ByteStreams; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.InputStream; import java.net.URISyntaxException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.security.GeneralSecurityException; import javax.net.ssl.SSLException; import org.junit.Assert; @@ -30,12 +29,6 @@ /** Tests for {@link Connection} using an actual local server. */ public class WithServerConnectionTest { - private static String inputStreamToUtf8String(InputStream in) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - ByteStreams.copy(in, out); - return out.toString("UTF-8"); - } - @Test public void testGet() throws IOException, InterruptedException, GeneralSecurityException, URISyntaxException { @@ -45,7 +38,9 @@ public void testGet() Response response = connection.send("GET", new Request.Builder().build()); Assert.assertEquals(200, response.getStatusCode()); - Assert.assertEquals("Hello World!", inputStreamToUtf8String(response.getBody())); + Assert.assertEquals( + "Hello World!".getBytes(StandardCharsets.UTF_8), + ByteStreams.toByteArray(response.getBody())); } } @@ -89,7 +84,9 @@ public void testInsecureConnection() Response response = connection.send("GET", new Request.Builder().build()); Assert.assertEquals(200, response.getStatusCode()); - Assert.assertEquals("Hello World!", inputStreamToUtf8String(response.getBody())); + Assert.assertEquals( + "Hello World!".getBytes(StandardCharsets.UTF_8), + ByteStreams.toByteArray(response.getBody())); } } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/RegistryEndpointCallerTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/RegistryEndpointCallerTest.java index 5600a9cc3d..72db4a16f6 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/RegistryEndpointCallerTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/RegistryEndpointCallerTest.java @@ -23,15 +23,15 @@ import com.google.cloud.tools.jib.event.EventDispatcher; import com.google.cloud.tools.jib.event.events.LogEvent; import com.google.cloud.tools.jib.global.JibSystemProperties; -import com.google.cloud.tools.jib.http.Authorizations; +import com.google.cloud.tools.jib.http.Authorization; import com.google.cloud.tools.jib.http.BlobHttpContent; import com.google.cloud.tools.jib.http.Connection; import com.google.cloud.tools.jib.http.MockConnection; import com.google.cloud.tools.jib.http.Response; -import com.google.common.io.ByteStreams; +import com.google.common.io.CharStreams; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStreamReader; import java.net.ConnectException; import java.net.MalformedURLException; import java.net.SocketException; @@ -86,9 +86,8 @@ public List getAccept() { @Nullable @Override public String handleResponse(Response response) throws IOException { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - ByteStreams.copy(response.getBody(), outputStream); - return outputStream.toString("UTF-8"); + return CharStreams.toString( + new InputStreamReader(response.getBody(), StandardCharsets.UTF_8)); } @Override @@ -641,7 +640,7 @@ private RegistryEndpointCaller createRegistryEndpointCaller( "userAgent", (port == -1) ? "apiRouteBase" : ("apiRouteBase:" + port), new TestRegistryEndpointProvider(), - Authorizations.withBasicToken("token"), + Authorization.withBasicToken("token"), new RegistryEndpointRequestProperties("serverUrl", "imageName"), allowInsecure, mockConnectionFactory, From a647b80f1f07751c212e4e759cb052c2f482b620 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 30 Apr 2019 14:10:44 -0400 Subject: [PATCH 17/39] Decouple Blob from http Request --- .../com/google/cloud/tools/jib/http/Request.java | 13 +++++++------ .../tools/jib/http/TestBlobProgressListener.java | 7 +------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/http/Request.java b/jib-core/src/main/java/com/google/cloud/tools/jib/http/Request.java index e6ef54f7f0..88631a5546 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/http/Request.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/http/Request.java @@ -16,6 +16,7 @@ package com.google.cloud.tools.jib.http; +import com.google.api.client.http.HttpContent; import com.google.api.client.http.HttpHeaders; import java.util.List; import javax.annotation.Nullable; @@ -27,7 +28,7 @@ public class Request { private final HttpHeaders headers; /** The HTTP request body. */ - @Nullable private final BlobHttpContent body; + @Nullable private final HttpContent body; /** HTTP connection and read timeout. */ @Nullable private final Integer httpTimeout; @@ -35,7 +36,7 @@ public class Request { public static class Builder { private final HttpHeaders headers = new HttpHeaders().setAccept("*/*"); - @Nullable private BlobHttpContent body; + @Nullable private HttpContent body; @Nullable private Integer httpTimeout; public Request build() { @@ -92,11 +93,11 @@ public Builder setHttpTimeout(@Nullable Integer httpTimeout) { /** * Sets the body and its corresponding {@code Content-Type} header. * - * @param blobHttpContent the body content + * @param httpContent the body content * @return this */ - public Builder setBody(@Nullable BlobHttpContent blobHttpContent) { - this.body = blobHttpContent; + public Builder setBody(@Nullable HttpContent httpContent) { + this.body = httpContent; return this; } } @@ -116,7 +117,7 @@ HttpHeaders getHeaders() { } @Nullable - BlobHttpContent getHttpContent() { + HttpContent getHttpContent() { return body; } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/http/TestBlobProgressListener.java b/jib-core/src/test/java/com/google/cloud/tools/jib/http/TestBlobProgressListener.java index e0e24d0874..3fd4a0adfe 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/http/TestBlobProgressListener.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/http/TestBlobProgressListener.java @@ -23,7 +23,7 @@ * Test implementation of {@link BlobProgressListener} that always forwards to a {@link Consumer * }. */ -public class TestBlobProgressListener implements BlobProgressListener, Consumer { +public class TestBlobProgressListener implements BlobProgressListener { private final Consumer byteCountConsumer; @@ -40,9 +40,4 @@ public void handleByteCount(long byteCount) { public Duration getDelayBetweenCallbacks() { return Duration.ofSeconds(-1); } - - @Override - public void accept(Long byteCount) { - handleByteCount(byteCount); - } } From 8f963375b3ce9c378e89a6759a1fefa20a413d4e Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 30 Apr 2019 18:07:41 -0400 Subject: [PATCH 18/39] test --- .../registry/BlobPullerIntegrationTest.java | 5 +- .../registry/BlobPusherIntegrationTest.java | 5 +- .../ManifestPusherIntegrationTest.java | 6 +- .../ProgressEventDispatcherContainer.java | 12 +-- .../tools/jib/builder/steps/PushBlobStep.java | 26 +----- .../jib/event/progress/DelayedConsumer.java | 81 +++++++++++++++++++ .../cloud/tools/jib/http/BlobHttpContent.java | 20 ++--- .../tools/jib/http/BlobProgressListener.java | 37 --------- .../http/ListenableCountingOutputStream.java | 59 +++----------- .../cloud/tools/jib/registry/BlobPuller.java | 15 ++-- .../cloud/tools/jib/registry/BlobPusher.java | 14 ++-- .../tools/jib/registry/RegistryClient.java | 14 ++-- .../cloud/tools/jib/http/ConnectionTest.java | 9 +-- .../ListenableCountingOutputStreamTest.java | 10 ++- .../jib/http/TestBlobProgressListener.java | 43 ---------- .../tools/jib/registry/BlobPullerTest.java | 5 +- .../tools/jib/registry/BlobPusherTest.java | 31 ++----- 17 files changed, 149 insertions(+), 243 deletions(-) create mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/DelayedConsumer.java delete mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobProgressListener.java delete mode 100644 jib-core/src/test/java/com/google/cloud/tools/jib/http/TestBlobProgressListener.java diff --git a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java index 4c46ceca3e..063052fe31 100644 --- a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java +++ b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java @@ -18,7 +18,6 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.event.EventDispatcher; -import com.google.cloud.tools.jib.http.TestBlobProgressListener; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.json.V21ManifestTemplate; import com.google.common.io.ByteStreams; @@ -63,7 +62,7 @@ public void testPull() throws IOException, RegistryException, InterruptedExcepti Assert.assertEquals(0, expectedSize.sum()); expectedSize.add(size); }, - new TestBlobProgressListener(totalByteCount::add)); + totalByteCount::add); Assert.assertEquals(realDigest, pulledBlob.writeTo(ByteStreams.nullOutputStream()).getDigest()); Assert.assertTrue(expectedSize.sum() > 0); Assert.assertEquals(expectedSize.sum(), totalByteCount.sum()); @@ -83,7 +82,7 @@ public void testPull_unknownBlob() throws IOException, DigestException, Interrup try { registryClient - .pullBlob(nonexistentDigest, ignored -> {}, new TestBlobProgressListener(ignored -> {})) + .pullBlob(nonexistentDigest, ignored -> {}, ignored -> {}) .writeTo(ByteStreams.nullOutputStream()); Assert.fail("Trying to pull nonexistent blob should have errored"); diff --git a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPusherIntegrationTest.java b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPusherIntegrationTest.java index 3d0ef643e0..fb737ae18a 100644 --- a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPusherIntegrationTest.java +++ b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPusherIntegrationTest.java @@ -19,7 +19,6 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.event.EventDispatcher; -import com.google.cloud.tools.jib.http.TestBlobProgressListener; import com.google.cloud.tools.jib.image.DescriptorDigest; import java.io.IOException; import java.security.DigestException; @@ -47,8 +46,6 @@ public void testPush() RegistryClient.factory(EVENT_DISPATCHER, "localhost:5000", "testimage") .setAllowInsecureRegistries(true) .newRegistryClient(); - Assert.assertFalse( - registryClient.pushBlob( - testBlobDigest, testBlob, null, new TestBlobProgressListener(ignored -> {}))); + Assert.assertFalse(registryClient.pushBlob(testBlobDigest, testBlob, null, ignored -> {})); } } diff --git a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPusherIntegrationTest.java b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPusherIntegrationTest.java index 7f7800e542..366b80bcfb 100644 --- a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPusherIntegrationTest.java +++ b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPusherIntegrationTest.java @@ -22,7 +22,6 @@ import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.event.EventDispatcher; import com.google.cloud.tools.jib.hash.DigestUtil; -import com.google.cloud.tools.jib.http.TestBlobProgressListener; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.json.ManifestTemplate; import com.google.cloud.tools.jib.image.json.V22ManifestTemplate; @@ -87,14 +86,13 @@ public void testPush() .setAllowInsecureRegistries(true) .newRegistryClient(); Assert.assertFalse( - registryClient.pushBlob( - testLayerBlobDigest, testLayerBlob, null, new TestBlobProgressListener(ignored -> {}))); + registryClient.pushBlob(testLayerBlobDigest, testLayerBlob, null, ignored -> {})); Assert.assertFalse( registryClient.pushBlob( testContainerConfigurationBlobDigest, testContainerConfigurationBlob, null, - new TestBlobProgressListener(ignored -> {}))); + ignored -> {})); // Pushes the manifest. DescriptorDigest imageDigest = registryClient.pushManifest(expectedManifestTemplate, "latest"); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ProgressEventDispatcherContainer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ProgressEventDispatcherContainer.java index d1ff257010..ae73314ea8 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ProgressEventDispatcherContainer.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ProgressEventDispatcherContainer.java @@ -18,10 +18,9 @@ import com.google.cloud.tools.jib.builder.BuildStepType; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; -import com.google.cloud.tools.jib.http.BlobProgressListener; import com.google.common.base.Preconditions; import java.io.Closeable; -import java.time.Duration; +import java.util.function.Consumer; import javax.annotation.Nullable; /** @@ -32,7 +31,7 @@ * response headers are received, only after which can the {@link ProgressEventDispatcher} be * created. */ -class ProgressEventDispatcherContainer implements BlobProgressListener, Closeable { +class ProgressEventDispatcherContainer implements Consumer, Closeable { private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; private final String description; @@ -49,16 +48,11 @@ class ProgressEventDispatcherContainer implements BlobProgressListener, Closeabl } @Override - public void handleByteCount(long byteCount) { + public void accept(Long byteCount) { Preconditions.checkNotNull(progressEventDispatcher); progressEventDispatcher.dispatchProgress(byteCount); } - @Override - public Duration getDelayBetweenCallbacks() { - return Duration.ofMillis(100); - } - @Override public void close() { Preconditions.checkNotNull(progressEventDispatcher); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java index cdc996549a..078ddfc233 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java @@ -26,38 +26,17 @@ import com.google.cloud.tools.jib.builder.TimerEventDispatcher; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.event.events.LogEvent; -import com.google.cloud.tools.jib.http.BlobProgressListener; import com.google.cloud.tools.jib.registry.RegistryClient; import com.google.cloud.tools.jib.registry.RegistryException; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import java.io.IOException; -import java.time.Duration; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; /** Pushes a BLOB to the target registry. */ class PushBlobStep implements AsyncStep, Callable { - private static class ForwardingProgressListener implements BlobProgressListener { - - private final ProgressEventDispatcher progressEventDispatcher; - - private ForwardingProgressListener(ProgressEventDispatcher progressEventDispatcher) { - this.progressEventDispatcher = progressEventDispatcher; - } - - @Override - public void handleByteCount(long byteCount) { - progressEventDispatcher.dispatchProgress(byteCount); - } - - @Override - public Duration getDelayBetweenCallbacks() { - return Duration.ofMillis(100); - } - } - private static final String DESCRIPTION = "Pushing BLOB "; private final BuildConfiguration buildConfiguration; @@ -122,10 +101,7 @@ public BlobDescriptor call() throws IOException, RegistryException, ExecutionExc // todo: leverage cross-repository mounts registryClient.pushBlob( - blobDescriptor.getDigest(), - blob, - null, - new ForwardingProgressListener(progressEventDispatcher)); + blobDescriptor.getDigest(), blob, null, progressEventDispatcher::dispatchProgress); return blobDescriptor; } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/DelayedConsumer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/DelayedConsumer.java new file mode 100644 index 0000000000..810238faf2 --- /dev/null +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/DelayedConsumer.java @@ -0,0 +1,81 @@ +/* + * Copyright 2019 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.tools.jib.event.progress; + +import java.time.Duration; +import java.time.Instant; +import java.util.function.BinaryOperator; +import java.util.function.Consumer; +import java.util.function.Supplier; + +/** + * Wraps a {@link Consumer} so that multiple consume calls within a short time are blocked and + * delayed into a single call. + */ +public class DelayedConsumer implements Consumer { + + private final Consumer originalConsumer; + + /** Delay between each call to {@link #byteCountConsumer}. */ + private final Duration delayBetweenCallbacks; + + /** + * Binary operator that will "add up" multiple delayed values. Used to accumulate past values in + * case delays happen so that callback is called once with the "added" value after the delay. + */ + private final BinaryOperator adder; + + /** Returns the current {@link Instant}. */ + private final Supplier getNow; + + /** Last time {@link #byteCountConsumer} was called. */ + private Instant previousCallback; + + private Consumer consumer; + + /** Wraps a consumer with the delay of 100 ms. */ + public DelayedConsumer(Consumer callback, BinaryOperator adder) { + this(callback, adder, Duration.ofMillis(100), Instant::now); + } + + public DelayedConsumer( + Consumer consumer, + BinaryOperator adder, + Duration delayBetweenCallbacks, + Supplier getNow) { + this.originalConsumer = consumer; + this.consumer = consumer; + this.adder = adder; + this.delayBetweenCallbacks = delayBetweenCallbacks; + this.getNow = getNow; + + previousCallback = getNow.get(); + } + + @Override + public void accept(T value) { + Instant now = getNow.get(); + if (previousCallback.plus(delayBetweenCallbacks).isBefore(now)) { + previousCallback = now; + consumer.accept(value); + consumer = originalConsumer; + } else { + Consumer currentConsumer = consumer; + consumer = nextValue -> currentConsumer.accept(adder.apply(nextValue, value)); + } + } +} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobHttpContent.java b/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobHttpContent.java index ad1fe85e73..2bca5bbedc 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobHttpContent.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobHttpContent.java @@ -18,8 +18,10 @@ import com.google.api.client.http.HttpContent; import com.google.cloud.tools.jib.blob.Blob; +import com.google.cloud.tools.jib.event.progress.DelayedConsumer; import java.io.IOException; import java.io.OutputStream; +import java.util.function.Consumer; import javax.annotation.Nullable; /** {@link Blob}-backed {@link HttpContent}. */ @@ -27,13 +29,13 @@ public class BlobHttpContent implements HttpContent { private final Blob blob; private final String contentType; - @Nullable private final BlobProgressListener blobProgressListener; + @Nullable private final Consumer writtenByteCountConsumer; public BlobHttpContent( - Blob blob, String contentType, @Nullable BlobProgressListener blobProgressListener) { + Blob blob, String contentType, @Nullable Consumer writtenByteCountConsumer) { this.blob = blob; this.contentType = contentType; - this.blobProgressListener = blobProgressListener; + this.writtenByteCountConsumer = writtenByteCountConsumer; } @Override @@ -54,13 +56,11 @@ public boolean retrySupported() { @Override public void writeTo(OutputStream outputStream) throws IOException { - outputStream = - blobProgressListener == null - ? outputStream - : new ListenableCountingOutputStream( - outputStream, - blobProgressListener::handleByteCount, - blobProgressListener.getDelayBetweenCallbacks()); + if (writtenByteCountConsumer != null) { + Consumer delayedConsumer = + new DelayedConsumer<>(writtenByteCountConsumer, (a, b) -> a + b); + outputStream = new ListenableCountingOutputStream(outputStream, delayedConsumer); + } blob.writeTo(outputStream); outputStream.flush(); } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobProgressListener.java b/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobProgressListener.java deleted file mode 100644 index 499e36a920..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobProgressListener.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.http; - -import java.time.Duration; - -/** Listens to BLOb send/receive progress. */ -public interface BlobProgressListener { - - /** - * Callback to call with count of newly-processed bytes. - * - * @param byteCount sent/received byte count - */ - void handleByteCount(long byteCount); - - /** - * Gets the minimum delay between calls to {@link #handleByteCount}. - * - * @return the delay between callbacks - */ - Duration getDelayBetweenCallbacks(); -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStream.java b/jib-core/src/main/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStream.java index fcc77470a4..635f3443b9 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStream.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStream.java @@ -16,18 +16,11 @@ package com.google.cloud.tools.jib.http; -import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.io.OutputStream; -import java.time.Duration; -import java.time.Instant; import java.util.function.Consumer; -import java.util.function.Supplier; -/** - * Counts the number of bytes written and reports the count to a callback. The count is reported - * with certain time delays to avoid calling the callback too often. - */ +/** Counts the number of bytes written and reports the count to a callback. */ public class ListenableCountingOutputStream extends OutputStream { /** The underlying {@link OutputStream} to wrap and forward bytes to. */ @@ -36,92 +29,58 @@ public class ListenableCountingOutputStream extends OutputStream { /** Receives a count of bytes written since the last call. */ private final Consumer byteCountConsumer; - /** Delay between each call to {@link #byteCountConsumer}. */ - private final Duration delayBetweenCallbacks; - - /** Supplies the current {@link Instant}. */ - private final Supplier instantSupplier; - /** Number of bytes to provide to {@link #byteCountConsumer}. */ private long byteCount = 0; - /** Last time {@link #byteCountConsumer} was called. */ - private Instant previousCallback; - /** * Wraps the {@code underlyingOutputStream} to count the bytes written. * * @param underlyingOutputStream the wrapped {@link OutputStream} * @param byteCountConsumer the byte count {@link Consumer} - * @param delayBetweenCallbacks the minimum delay between each call to {@link #byteCountConsumer} */ public ListenableCountingOutputStream( - OutputStream underlyingOutputStream, - Consumer byteCountConsumer, - Duration delayBetweenCallbacks) { - this(underlyingOutputStream, byteCountConsumer, delayBetweenCallbacks, Instant::now); - } - - @VisibleForTesting - ListenableCountingOutputStream( - OutputStream underlyingOutputStream, - Consumer byteCountConsumer, - Duration delayBetweenCallbacks, - Supplier instantSupplier) { + OutputStream underlyingOutputStream, Consumer byteCountConsumer) { this.underlyingOutputStream = underlyingOutputStream; this.byteCountConsumer = byteCountConsumer; - this.delayBetweenCallbacks = delayBetweenCallbacks; - this.instantSupplier = instantSupplier; - - previousCallback = instantSupplier.get(); } @Override public void write(int singleByte) throws IOException { underlyingOutputStream.write(singleByte); - countBytesWritten(1); + addAndCallByteCountConsumer(1); } @Override public void write(byte[] byteArray) throws IOException { underlyingOutputStream.write(byteArray); - countBytesWritten(byteArray.length); + addAndCallByteCountConsumer(byteArray.length); } @Override public void write(byte byteArray[], int offset, int length) throws IOException { underlyingOutputStream.write(byteArray, offset, length); - countBytesWritten(length); + addAndCallByteCountConsumer(length); } @Override public void flush() throws IOException { underlyingOutputStream.flush(); - callByteCountConsumer(instantSupplier.get()); + addAndCallByteCountConsumer(0); } @Override public void close() throws IOException { underlyingOutputStream.close(); - callByteCountConsumer(instantSupplier.get()); - } - - private void countBytesWritten(int byteCount) { - this.byteCount += byteCount; - - Instant now = instantSupplier.get(); - if (previousCallback.plus(delayBetweenCallbacks).isBefore(now)) { - callByteCountConsumer(now); - } + addAndCallByteCountConsumer(0); } - private void callByteCountConsumer(Instant now) { + private void addAndCallByteCountConsumer(int written) { + this.byteCount += written; if (byteCount == 0) { return; } byteCountConsumer.accept(byteCount); byteCount = 0; - previousCallback = now; } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java index b7eed4b3ec..faaffbc56f 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java @@ -18,9 +18,9 @@ import com.google.api.client.http.HttpMethods; import com.google.cloud.tools.jib.blob.BlobDescriptor; +import com.google.cloud.tools.jib.event.progress.DelayedConsumer; import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.http.BlobHttpContent; -import com.google.cloud.tools.jib.http.BlobProgressListener; import com.google.cloud.tools.jib.http.ListenableCountingOutputStream; import com.google.cloud.tools.jib.http.Response; import com.google.cloud.tools.jib.image.DescriptorDigest; @@ -47,30 +47,29 @@ class BlobPuller implements RegistryEndpointProvider { private final OutputStream destinationOutputStream; private final Consumer blobSizeConsumer; - private final BlobProgressListener blobProgressListener; + private final Consumer writtenByteCountConsumer; BlobPuller( RegistryEndpointRequestProperties registryEndpointRequestProperties, DescriptorDigest blobDigest, OutputStream destinationOutputStream, Consumer blobSizeConsumer, - BlobProgressListener blobProgressListener) { + Consumer writtenByteCountConsumer) { this.registryEndpointRequestProperties = registryEndpointRequestProperties; this.blobDigest = blobDigest; this.destinationOutputStream = destinationOutputStream; this.blobSizeConsumer = blobSizeConsumer; - this.blobProgressListener = blobProgressListener; + this.writtenByteCountConsumer = writtenByteCountConsumer; } @Override public Void handleResponse(Response response) throws IOException, UnexpectedBlobDigestException { blobSizeConsumer.accept(response.getContentLength()); + Consumer delayedConsumer = + new DelayedConsumer<>(writtenByteCountConsumer, (a, b) -> a + b); try (OutputStream outputStream = - new ListenableCountingOutputStream( - destinationOutputStream, - blobProgressListener::handleByteCount, - blobProgressListener.getDelayBetweenCallbacks())) { + new ListenableCountingOutputStream(destinationOutputStream, delayedConsumer)) { BlobDescriptor receivedBlobDescriptor = DigestUtil.computeDigest(response.getBody(), outputStream); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPusher.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPusher.java index 8cb6fb1e23..61f474f937 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPusher.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPusher.java @@ -20,7 +20,6 @@ import com.google.api.client.http.HttpMethods; import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.http.BlobHttpContent; -import com.google.cloud.tools.jib.http.BlobProgressListener; import com.google.cloud.tools.jib.http.Response; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.common.net.MediaType; @@ -29,6 +28,7 @@ import java.net.URL; import java.util.Collections; import java.util.List; +import java.util.function.Consumer; import javax.annotation.Nullable; /** @@ -112,12 +112,12 @@ public String getActionDescription() { private class Writer implements RegistryEndpointProvider { private final URL location; - private final BlobProgressListener blobProgressListener; + private final Consumer writtenByteCountConsumer; @Nullable @Override public BlobHttpContent getContent() { - return new BlobHttpContent(blob, MediaType.OCTET_STREAM.toString(), blobProgressListener); + return new BlobHttpContent(blob, MediaType.OCTET_STREAM.toString(), writtenByteCountConsumer); } @Override @@ -147,9 +147,9 @@ public String getActionDescription() { return BlobPusher.this.getActionDescription(); } - private Writer(URL location, BlobProgressListener blobProgressListener) { + private Writer(URL location, Consumer writtenByteCountConsumer) { this.location = location; - this.blobProgressListener = blobProgressListener; + this.writtenByteCountConsumer = writtenByteCountConsumer; } } @@ -219,8 +219,8 @@ RegistryEndpointProvider initializer() { * @param blobProgressListener the listener for {@link Blob} push progress * @return a {@link RegistryEndpointProvider} for writing the BLOB to an upload location */ - RegistryEndpointProvider writer(URL location, BlobProgressListener blobProgressListener) { - return new Writer(location, blobProgressListener); + RegistryEndpointProvider writer(URL location, Consumer writtenByteCountConsumer) { + return new Writer(location, writtenByteCountConsumer); } /** diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryClient.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryClient.java index 7adcdacec9..0adf970e6f 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryClient.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryClient.java @@ -24,7 +24,6 @@ import com.google.cloud.tools.jib.event.EventDispatcher; import com.google.cloud.tools.jib.global.JibSystemProperties; import com.google.cloud.tools.jib.http.Authorization; -import com.google.cloud.tools.jib.http.BlobProgressListener; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.json.BuildableManifestTemplate; import com.google.cloud.tools.jib.image.json.ManifestTemplate; @@ -247,13 +246,14 @@ public BlobDescriptor checkBlob(DescriptorDigest blobDigest) * * @param blobDigest the digest of the BLOB to download * @param blobSizeConsumer callback to receive the total size of the BLOb to pull - * @param blobProgressListener listener for progress of the pull + * @param writtenByteCountConsumer listens on byte count written to an output stream during the + * pull * @return a {@link Blob} */ public Blob pullBlob( DescriptorDigest blobDigest, Consumer blobSizeConsumer, - BlobProgressListener blobProgressListener) { + Consumer writtenByteCountConsumer) { return Blobs.from( outputStream -> { try { @@ -263,7 +263,7 @@ public Blob pullBlob( blobDigest, outputStream, blobSizeConsumer, - blobProgressListener)); + writtenByteCountConsumer)); } catch (RegistryException ex) { throw new IOException(ex); @@ -279,7 +279,7 @@ public Blob pullBlob( * @param blob the BLOB to push * @param sourceRepository if pushing to the same registry then the source image, or {@code null} * otherwise; used to optimize the BLOB push - * @param blobProgressListener listener for BLOb push progress + * @param writtenByteCountConsumer listens on byte count written to the registry during the push * @return {@code true} if the BLOB already exists on the registry and pushing was skipped; false * if the BLOB was pushed * @throws IOException if communicating with the endpoint fails @@ -289,7 +289,7 @@ public boolean pushBlob( DescriptorDigest blobDigest, Blob blob, @Nullable String sourceRepository, - BlobProgressListener blobProgressListener) + Consumer writtenByteCountConsumer) throws IOException, RegistryException { BlobPusher blobPusher = new BlobPusher(registryEndpointRequestProperties, blobDigest, blob, sourceRepository); @@ -311,7 +311,7 @@ public boolean pushBlob( // PATCH with BLOB URL putLocation = - callRegistryEndpoint(blobPusher.writer(patchLocation, blobProgressListener)); + callRegistryEndpoint(blobPusher.writer(patchLocation, writtenByteCountConsumer)); Preconditions.checkNotNull(putLocation); timerEventDispatcher2.lap("pushBlob PUT " + blobDigest); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/http/ConnectionTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/http/ConnectionTest.java index 1659c63eea..a4e8d2850a 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/http/ConnectionTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/http/ConnectionTest.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.concurrent.atomic.LongAdder; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -55,11 +56,11 @@ private interface SendFunction { ArgumentCaptor.forClass(BlobHttpContent.class); private final GenericUrl fakeUrl = new GenericUrl("http://crepecake/fake/url"); + private final LongAdder totalByteCount = new LongAdder(); + private Request fakeRequest; private HttpResponse mockHttpResponse; - private long totalByteCount = 0; - @InjectMocks private final Connection testConnection = Connection.getConnectionFactory().apply(fakeUrl.toURL()); @@ -111,9 +112,7 @@ private void setUpMocksAndFakes(Integer httpTimeout) throws IOException { .setUserAgent("fake user agent") .setBody( new BlobHttpContent( - Blobs.from("crepecake"), - "fake.content.type", - new TestBlobProgressListener(byteCount -> totalByteCount += byteCount))) + Blobs.from("crepecake"), "fake.content.type", totalByteCount::add)) .setAuthorization(Authorization.withBasicCredentials("fake-username", "fake-secret")) .setHttpTimeout(httpTimeout) .build(); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStreamTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStreamTest.java index 6f3f126897..460aace01e 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStreamTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStreamTest.java @@ -16,6 +16,7 @@ package com.google.cloud.tools.jib.http; +import com.google.cloud.tools.jib.event.progress.DelayedConsumer; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.time.Duration; @@ -38,8 +39,7 @@ public void testCallback_correctSequence() throws IOException { List byteCounts = new ArrayList<>(); try (ListenableCountingOutputStream listenableCountingOutputStream = - new ListenableCountingOutputStream( - byteArrayOutputStream, byteCounts::add, Duration.ofSeconds(-1))) { + new ListenableCountingOutputStream(byteArrayOutputStream, byteCounts::add)) { listenableCountingOutputStream.write(0); listenableCountingOutputStream.write(new byte[] {1, 2, 3}); listenableCountingOutputStream.write(new byte[] {1, 2, 3, 4, 5}, 3, 2); @@ -60,12 +60,16 @@ public void testDelay() throws IOException { try (ListenableCountingOutputStream listenableCountingOutputStream = new ListenableCountingOutputStream( - byteArrayOutputStream, byteCounts::add, Duration.ofSeconds(3), instantQueue::remove)) { + byteArrayOutputStream, + new DelayedConsumer<>( + byteCounts::add, (a, b) -> a + b, Duration.ofSeconds(3), instantQueue::remove))) { instantQueue.add(Instant.EPOCH); listenableCountingOutputStream.write(100); instantQueue.add(Instant.EPOCH); listenableCountingOutputStream.write(new byte[] {101, 102, 103}); + System.out.println(instantQueue); instantQueue.add(Instant.EPOCH.plusSeconds(4)); + System.out.println(instantQueue); listenableCountingOutputStream.write(new byte[] {104, 105, 106}); instantQueue.add(Instant.EPOCH.plusSeconds(10)); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/http/TestBlobProgressListener.java b/jib-core/src/test/java/com/google/cloud/tools/jib/http/TestBlobProgressListener.java deleted file mode 100644 index 3fd4a0adfe..0000000000 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/http/TestBlobProgressListener.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.http; - -import java.time.Duration; -import java.util.function.Consumer; - -/** - * Test implementation of {@link BlobProgressListener} that always forwards to a {@link Consumer - * }. - */ -public class TestBlobProgressListener implements BlobProgressListener { - - private final Consumer byteCountConsumer; - - public TestBlobProgressListener(Consumer byteCountConsumer) { - this.byteCountConsumer = byteCountConsumer; - } - - @Override - public void handleByteCount(long byteCount) { - byteCountConsumer.accept(byteCount); - } - - @Override - public Duration getDelayBetweenCallbacks() { - return Duration.ofSeconds(-1); - } -} diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java index 8980af070d..27e20e00cd 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java @@ -19,7 +19,6 @@ import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.http.Response; -import com.google.cloud.tools.jib.http.TestBlobProgressListener; import com.google.cloud.tools.jib.image.DescriptorDigest; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -63,7 +62,7 @@ public void setUpFakes() throws DigestException { fakeDigest, layerOutputStream, ignored -> {}, - new TestBlobProgressListener(ignored -> {})); + ignored -> {}); } @Test @@ -83,7 +82,7 @@ public void testHandleResponse() throws IOException, UnexpectedBlobDigestExcepti testBlobDigest, layerOutputStream, size -> Assert.assertEquals("some BLOB content".length(), size.longValue()), - new TestBlobProgressListener(byteCount::add)); + byteCount::add); blobPuller.handleResponse(mockResponse); Assert.assertEquals( "some BLOB content", diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPusherTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPusherTest.java index 9066005762..2c44c32d63 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPusherTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPusherTest.java @@ -21,7 +21,6 @@ import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.http.BlobHttpContent; import com.google.cloud.tools.jib.http.Response; -import com.google.cloud.tools.jib.http.TestBlobProgressListener; import com.google.cloud.tools.jib.image.DescriptorDigest; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -164,8 +163,7 @@ public void testInitializer_getActionDescription() { @Test public void testWriter_getContent() throws IOException { LongAdder byteCount = new LongAdder(); - BlobHttpContent body = - testBlobPusher.writer(mockURL, new TestBlobProgressListener(byteCount::add)).getContent(); + BlobHttpContent body = testBlobPusher.writer(mockURL, byteCount::add).getContent(); Assert.assertNotNull(body); Assert.assertEquals("application/octet-stream", body.getType()); @@ -180,12 +178,7 @@ public void testWriter_getContent() throws IOException { @Test public void testWriter_GetAccept() { - Assert.assertEquals( - 0, - testBlobPusher - .writer(mockURL, new TestBlobProgressListener(ignored -> {})) - .getAccept() - .size()); + Assert.assertEquals(0, testBlobPusher.writer(mockURL, ignored -> {}).getAccept().size()); } @Test @@ -196,37 +189,25 @@ public void testWriter_handleResponse() throws IOException, RegistryException { Mockito.when(mockResponse.getRequestUrl()).thenReturn(requestUrl); Assert.assertEquals( new URL("https://somenewurl/location"), - testBlobPusher - .writer(mockURL, new TestBlobProgressListener(ignored -> {})) - .handleResponse(mockResponse)); + testBlobPusher.writer(mockURL, ignored -> {}).handleResponse(mockResponse)); } @Test public void testWriter_getApiRoute() throws MalformedURLException { URL fakeUrl = new URL("http://someurl"); - Assert.assertEquals( - fakeUrl, - testBlobPusher - .writer(fakeUrl, new TestBlobProgressListener(ignored -> {})) - .getApiRoute("")); + Assert.assertEquals(fakeUrl, testBlobPusher.writer(fakeUrl, ignored -> {}).getApiRoute("")); } @Test public void testWriter_getHttpMethod() { - Assert.assertEquals( - "PATCH", - testBlobPusher - .writer(mockURL, new TestBlobProgressListener(ignored -> {})) - .getHttpMethod()); + Assert.assertEquals("PATCH", testBlobPusher.writer(mockURL, ignored -> {}).getHttpMethod()); } @Test public void testWriter_getActionDescription() { Assert.assertEquals( "push BLOB for someServerUrl/someImageName with digest " + fakeDescriptorDigest, - testBlobPusher - .writer(mockURL, new TestBlobProgressListener(ignored -> {})) - .getActionDescription()); + testBlobPusher.writer(mockURL, ignored -> {}).getActionDescription()); } @Test From d3bfc7a815e7271fc648604be9411cd6bf787a79 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 30 Apr 2019 18:09:48 -0400 Subject: [PATCH 19/39] Reorg and refactor --- .../registry/BlobPullerIntegrationTest.java | 14 +-- .../ManifestPusherIntegrationTest.java | 11 +- .../cloud/tools/jib/blob/BlobDescriptor.java | 23 ---- .../google/cloud/tools/jib/blob/Blobs.java | 10 +- .../google/cloud/tools/jib/blob/FileBlob.java | 5 +- .../cloud/tools/jib/blob/InputStreamBlob.java | 3 +- .../google/cloud/tools/jib/blob/JsonBlob.java | 37 +++++++ .../cloud/tools/jib/blob/StringBlob.java | 5 +- ...rBlob.java => WriterableContentsBlob.java} | 19 ++-- .../tools/jib/builder/steps/BuildResult.java | 7 +- .../jib/builder/steps/PullBaseImageStep.java | 2 +- .../jib/builder/steps/PushImageStep.java | 8 +- .../jib/cache/DefaultCacheStorageWriter.java | 11 +- .../tools/jib/cache/LayerEntriesSelector.java | 7 +- .../cloud/tools/jib/docker/ImageTarball.java | 8 +- .../jib/hash/CountingDigestOutputStream.java | 45 +++++--- .../cloud/tools/jib/hash/DigestUtil.java | 102 ++++++++++++++++++ .../WritableContents.java} | 12 ++- .../google/cloud/tools/jib/http/Request.java | 13 +-- .../jib/image/json/ImageToJsonTranslator.java | 4 +- .../tools/jib/json/JsonTemplateMapper.java | 50 +++++---- .../tools/jib/registry/ManifestPusher.java | 11 +- .../google/cloud/tools/jib/blob/BlobTest.java | 16 ++- .../cache/DefaultCacheStorageWriterTest.java | 20 ++-- .../jib/cache/LayerEntriesSelectorTest.java | 12 +-- .../json/DockerLoadManifestTemplateTest.java | 3 +- .../hash/CountingDigestOutputStreamTest.java | 8 +- .../jib/http/TestBlobProgressListener.java | 7 +- .../ContainerConfigurationTemplateTest.java | 6 +- .../image/json/ImageToJsonTranslatorTest.java | 5 +- .../image/json/OCIManifestTemplateTest.java | 6 +- .../image/json/V22ManifestTemplateTest.java | 6 +- .../jib/json/JsonTemplateMapperTest.java | 11 +- .../tools/jib/registry/BlobCheckerTest.java | 7 +- .../tools/jib/registry/BlobPullerTest.java | 2 +- .../jib/registry/ManifestPusherTest.java | 22 +--- 36 files changed, 304 insertions(+), 234 deletions(-) create mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/blob/JsonBlob.java rename jib-core/src/main/java/com/google/cloud/tools/jib/blob/{WriterBlob.java => WriterableContentsBlob.java} (59%) create mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java rename jib-core/src/main/java/com/google/cloud/tools/jib/{blob/BlobWriter.java => hash/WritableContents.java} (63%) diff --git a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java index 1368525e41..4c46ceca3e 100644 --- a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java +++ b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java @@ -16,8 +16,8 @@ package com.google.cloud.tools.jib.registry; +import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.event.EventDispatcher; -import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; import com.google.cloud.tools.jib.http.TestBlobProgressListener; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.json.V21ManifestTemplate; @@ -54,23 +54,19 @@ public void testPull() throws IOException, RegistryException, InterruptedExcepti DescriptorDigest realDigest = manifestTemplate.getLayerDigests().get(0); // Pulls a layer BLOB of the busybox image. - CountingDigestOutputStream layerOutputStream = - new CountingDigestOutputStream(ByteStreams.nullOutputStream()); LongAdder totalByteCount = new LongAdder(); LongAdder expectedSize = new LongAdder(); - registryClient - .pullBlob( + Blob pulledBlob = + registryClient.pullBlob( realDigest, size -> { Assert.assertEquals(0, expectedSize.sum()); expectedSize.add(size); }, - new TestBlobProgressListener(totalByteCount::add)) - .writeTo(layerOutputStream); + new TestBlobProgressListener(totalByteCount::add)); + Assert.assertEquals(realDigest, pulledBlob.writeTo(ByteStreams.nullOutputStream()).getDigest()); Assert.assertTrue(expectedSize.sum() > 0); Assert.assertEquals(expectedSize.sum(), totalByteCount.sum()); - - Assert.assertEquals(realDigest, layerOutputStream.toBlobDescriptor().getDigest()); } @Test diff --git a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPusherIntegrationTest.java b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPusherIntegrationTest.java index 67a444f4c7..7f7800e542 100644 --- a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPusherIntegrationTest.java +++ b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPusherIntegrationTest.java @@ -21,12 +21,11 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.event.EventDispatcher; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.http.TestBlobProgressListener; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.json.ManifestTemplate; import com.google.cloud.tools.jib.image.json.V22ManifestTemplate; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; -import com.google.common.io.ByteStreams; import java.io.IOException; import java.security.DigestException; import org.junit.Assert; @@ -114,11 +113,7 @@ public void testPush() V22ManifestTemplate manifestTemplateByDigest = registryClient.pullManifest(imageDigest.toString(), V22ManifestTemplate.class); Assert.assertEquals( - JsonTemplateMapper.toBlob(manifestTemplate) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(), - JsonTemplateMapper.toBlob(manifestTemplateByDigest) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest()); + DigestUtil.computeJsonDigest(manifestTemplate), + DigestUtil.computeJsonDigest(manifestTemplateByDigest)); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/BlobDescriptor.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/BlobDescriptor.java index 0b067cc912..ff59209998 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/BlobDescriptor.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/BlobDescriptor.java @@ -16,12 +16,7 @@ package com.google.cloud.tools.jib.blob; -import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; import com.google.cloud.tools.jib.image.DescriptorDigest; -import com.google.common.io.ByteStreams; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; /** Contains properties describing a BLOB, including its digest and possibly its size (in bytes). */ public class BlobDescriptor { @@ -31,24 +26,6 @@ public class BlobDescriptor { /** The size of the BLOB (in bytes). Negative if unknown. */ private final long size; - /** - * Creates a new {@link BlobDescriptor} from the contents of an {@link InputStream} while piping - * to an {@link OutputStream}. Does not close either streams. - * - * @param inputStream the stream to read the contents from - * @param outputStream the {@link OutputStream} to pipe to - * @return a {@link BlobDescriptor} of the piped contents - * @throws IOException if reading from or writing to the streams fails - */ - static BlobDescriptor fromPipe(InputStream inputStream, OutputStream outputStream) - throws IOException { - CountingDigestOutputStream countingDigestOutputStream = - new CountingDigestOutputStream(outputStream); - ByteStreams.copy(inputStream, countingDigestOutputStream); - countingDigestOutputStream.flush(); - return countingDigestOutputStream.toBlobDescriptor(); - } - public BlobDescriptor(long size, DescriptorDigest digest) { this.size = size; this.digest = digest; diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java index 4cec2f9b7e..cb621b232f 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java @@ -17,6 +17,8 @@ package com.google.cloud.tools.jib.blob; import com.google.cloud.tools.jib.filesystem.FileOperations; +import com.google.cloud.tools.jib.hash.WritableContents; +import com.google.cloud.tools.jib.json.JsonTemplate; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; @@ -36,6 +38,10 @@ public static Blob from(Path file) { return new FileBlob(file); } + public static Blob from(JsonTemplate template) { + return new JsonBlob(template); + } + /** * Creates a {@link StringBlob} with UTF-8 encoding. * @@ -46,8 +52,8 @@ public static Blob from(String content) { return new StringBlob(content); } - public static Blob from(BlobWriter writer) { - return new WriterBlob(writer); + public static Blob from(WritableContents writable) { + return new WritableContentsBlob(writable); } /** diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/FileBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/FileBlob.java index 5c978c2dfb..7748b9bddd 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/FileBlob.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/FileBlob.java @@ -16,6 +16,7 @@ package com.google.cloud.tools.jib.blob; +import com.google.cloud.tools.jib.hash.DigestUtil; import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; @@ -34,8 +35,8 @@ class FileBlob implements Blob { @Override public BlobDescriptor writeTo(OutputStream outputStream) throws IOException { - try (InputStream fileStream = new BufferedInputStream(Files.newInputStream(file))) { - return BlobDescriptor.fromPipe(fileStream, outputStream); + try (InputStream fileIn = new BufferedInputStream(Files.newInputStream(file))) { + return DigestUtil.computeDigest(fileIn, outputStream); } } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/InputStreamBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/InputStreamBlob.java index 46273b691f..5e5e172f8c 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/InputStreamBlob.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/InputStreamBlob.java @@ -16,6 +16,7 @@ package com.google.cloud.tools.jib.blob; +import com.google.cloud.tools.jib.hash.DigestUtil; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -39,7 +40,7 @@ public BlobDescriptor writeTo(OutputStream outputStream) throws IOException { throw new IllegalStateException("Cannot rewrite Blob backed by an InputStream"); } try (InputStream inputStream = this.inputStream) { - return BlobDescriptor.fromPipe(inputStream, outputStream); + return DigestUtil.computeDigest(inputStream, outputStream); } finally { isWritten = true; diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/JsonBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/JsonBlob.java new file mode 100644 index 0000000000..83dd4c5314 --- /dev/null +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/JsonBlob.java @@ -0,0 +1,37 @@ +/* + * Copyright 2019 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.tools.jib.blob; + +import com.google.cloud.tools.jib.hash.DigestUtil; +import com.google.cloud.tools.jib.json.JsonTemplate; +import java.io.IOException; +import java.io.OutputStream; + +/** A {@link Blob} that holds {@link JsonTemplate}. */ +class JsonBlob implements Blob { + + private final JsonTemplate template; + + JsonBlob(JsonTemplate template) { + this.template = template; + } + + @Override + public BlobDescriptor writeTo(OutputStream outputStream) throws IOException { + return DigestUtil.computeDigest(template, outputStream); + } +} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/StringBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/StringBlob.java index 62e07e1297..9f61014666 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/StringBlob.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/StringBlob.java @@ -16,6 +16,7 @@ package com.google.cloud.tools.jib.blob; +import com.google.cloud.tools.jib.hash.DigestUtil; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -33,9 +34,9 @@ class StringBlob implements Blob { @Override public BlobDescriptor writeTo(OutputStream outputStream) throws IOException { - try (InputStream stringInputStream = + try (InputStream stringIn = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8))) { - return BlobDescriptor.fromPipe(stringInputStream, outputStream); + return DigestUtil.computeDigest(stringIn, outputStream); } } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterableContentsBlob.java similarity index 59% rename from jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterBlob.java rename to jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterableContentsBlob.java index c47dc3e037..3c6f843293 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterBlob.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterableContentsBlob.java @@ -16,25 +16,22 @@ package com.google.cloud.tools.jib.blob; -import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; +import com.google.cloud.tools.jib.hash.DigestUtil; +import com.google.cloud.tools.jib.hash.WritableContents; import java.io.IOException; import java.io.OutputStream; -/** A {@link Blob} that writes with a {@link BlobWriter} function and hashes the bytes. */ -class WriterBlob implements Blob { +/** A {@link Blob} that holds {@link WritableContents}. */ +class WritableContentsBlob implements Blob { - private final BlobWriter writer; + private final WritableContents writableContents; - WriterBlob(BlobWriter writer) { - this.writer = writer; + WritableContentsBlob(WritableContents writableContents) { + this.writableContents = writableContents; } @Override public BlobDescriptor writeTo(OutputStream outputStream) throws IOException { - CountingDigestOutputStream countingDigestOutputStream = - new CountingDigestOutputStream(outputStream); - writer.writeTo(countingDigestOutputStream); - countingDigestOutputStream.flush(); - return countingDigestOutputStream.toBlobDescriptor(); + return DigestUtil.computeDigest(writableContents, outputStream); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildResult.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildResult.java index 383b055760..a8597d7bff 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildResult.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildResult.java @@ -17,12 +17,12 @@ package com.google.cloud.tools.jib.builder.steps; import com.google.cloud.tools.jib.blob.BlobDescriptor; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.Layer; import com.google.cloud.tools.jib.image.json.BuildableManifestTemplate; import com.google.cloud.tools.jib.image.json.ImageToJsonTranslator; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.io.ByteStreams; import java.io.IOException; import java.util.Objects; @@ -49,10 +49,7 @@ static BuildResult fromImage( BuildableManifestTemplate manifestTemplate = imageToJsonTranslator.getManifestTemplate( targetFormat, containerConfigurationBlobDescriptor); - DescriptorDigest imageDigest = - JsonTemplateMapper.toBlob(manifestTemplate) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DescriptorDigest imageDigest = DigestUtil.computeJsonDigest(manifestTemplate); DescriptorDigest imageId = containerConfigurationBlobDescriptor.getDigest(); return new BuildResult(imageDigest, imageId); } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java index 1e96d4a310..a7d8773357 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java @@ -237,7 +237,7 @@ private Image pullBaseImage( || v22ManifestTemplate.getContainerConfiguration().getDigest() == null) { throw new UnknownManifestFormatException( "Invalid container configuration in Docker V2.2 manifest: \n" - + Blobs.writeToString(JsonTemplateMapper.toBlob(v22ManifestTemplate))); + + JsonTemplateMapper.toUtf8String(v22ManifestTemplate)); } DescriptorDigest containerConfigurationDigest = diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java index 4d7821b2ec..e44bb7980f 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java @@ -25,13 +25,12 @@ import com.google.cloud.tools.jib.builder.TimerEventDispatcher; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.event.events.LogEvent; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.json.BuildableManifestTemplate; import com.google.cloud.tools.jib.image.json.ImageToJsonTranslator; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.cloud.tools.jib.registry.RegistryClient; import com.google.common.collect.ImmutableSet; -import com.google.common.io.ByteStreams; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -165,10 +164,7 @@ private BuildResult afterAllPushed() })); } - DescriptorDigest imageDigest = - JsonTemplateMapper.toBlob(manifestTemplate) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DescriptorDigest imageDigest = DigestUtil.computeJsonDigest(manifestTemplate); DescriptorDigest imageId = containerConfigurationBlobDescriptor.getDigest(); BuildResult result = new BuildResult(imageDigest, imageId); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriter.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriter.java index 0768103421..f164751c8a 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriter.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriter.java @@ -105,7 +105,7 @@ private static DescriptorDigest getDiffIdByDecompressingFile(Path compressedFile GZIPInputStream decompressorStream = new GZIPInputStream(fileInputStream)) { ByteStreams.copy(decompressorStream, diffIdCaptureOutputStream); } - return diffIdCaptureOutputStream.toBlobDescriptor().getDigest(); + return diffIdCaptureOutputStream.getDigest(); } } @@ -121,7 +121,9 @@ private static void writeMetadata(JsonTemplate jsonTemplate, Path destination) throws IOException { Path temporaryFile = Files.createTempFile(destination.getParent(), null, null); temporaryFile.toFile().deleteOnExit(); - Blobs.writeToFile(JsonTemplateMapper.toBlob(jsonTemplate), temporaryFile); + try (OutputStream outputStream = Files.newOutputStream(temporaryFile)) { + JsonTemplateMapper.writeTo(jsonTemplate, outputStream); + } // Attempts an atomic move first, and falls back to non-atomic if the file system does not // support atomic moves. @@ -333,9 +335,8 @@ private WrittenLayer writeUncompressedLayerBlobToDirectory( // The GZIPOutputStream must be closed in order to write out the remaining compressed data. compressorStream.close(); - BlobDescriptor compressedBlobDescriptor = compressedDigestOutputStream.toBlobDescriptor(); - DescriptorDigest layerDigest = compressedBlobDescriptor.getDigest(); - long layerSize = compressedBlobDescriptor.getSize(); + DescriptorDigest layerDigest = compressedDigestOutputStream.getDigest(); + long layerSize = compressedDigestOutputStream.getBytesHahsed(); // Renames the temporary layer file to the correct filename. Path layerFile = diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/LayerEntriesSelector.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/LayerEntriesSelector.java index b47011fff9..dd5c18cb1c 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/LayerEntriesSelector.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/LayerEntriesSelector.java @@ -16,13 +16,12 @@ package com.google.cloud.tools.jib.cache; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.LayerEntry; import com.google.cloud.tools.jib.json.JsonTemplate; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; -import com.google.common.io.ByteStreams; import java.io.IOException; import java.nio.file.Files; import java.time.Instant; @@ -142,9 +141,7 @@ static List toSortedJsonTemplates(List layerEntr */ static DescriptorDigest generateSelector(ImmutableList layerEntries) throws IOException { - return JsonTemplateMapper.toBlob(toSortedJsonTemplates(layerEntries)) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + return DigestUtil.computeJsonDigest(toSortedJsonTemplates(layerEntries)); } private LayerEntriesSelector() {} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageTarball.java b/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageTarball.java index f7ad06a162..7843afbe48 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageTarball.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageTarball.java @@ -25,6 +25,7 @@ import com.google.cloud.tools.jib.image.json.ImageToJsonTranslator; import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.cloud.tools.jib.tar.TarStreamBuilder; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; import java.util.Collections; @@ -77,10 +78,9 @@ public void writeTo(OutputStream out) throws IOException { // Adds the manifest to tarball. manifestTemplate.setRepoTags(imageReference.toStringWithTag()); - tarStreamBuilder.addByteEntry( - Blobs.writeToByteArray( - JsonTemplateMapper.toBlob(Collections.singletonList(manifestTemplate))), - MANIFEST_JSON_FILE_NAME); + ByteArrayOutputStream outStream = new ByteArrayOutputStream(); + JsonTemplateMapper.writeTo(Collections.singletonList(manifestTemplate), outStream); + tarStreamBuilder.addByteEntry(outStream.toByteArray(), MANIFEST_JSON_FILE_NAME); tarStreamBuilder.writeAsTarArchiveTo(out); } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStream.java b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStream.java index b2e872c2a2..12cf3ebd65 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStream.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStream.java @@ -16,22 +16,27 @@ package com.google.cloud.tools.jib.hash; -import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.image.DescriptorDigest; +import com.google.common.base.Verify; import java.io.IOException; import java.io.OutputStream; import java.security.DigestException; import java.security.DigestOutputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import javax.annotation.Nullable; /** A {@link DigestOutputStream} that also keeps track of the total number of bytes written. */ public class CountingDigestOutputStream extends DigestOutputStream { private static final String SHA_256_ALGORITHM = "SHA-256"; - /** Keeps track of the total number of bytes appended. */ - private long totalBytes = 0; + private long bytesSoFar = 0; + + /** The total number of bytes used to compute a digest. Resets when {@link computeDigest) is called. */ + private long bytesHashed = 0; + + @Nullable private DescriptorDigest descriptorDigest; /** * Wraps the {@code outputStream}. @@ -49,12 +54,10 @@ public CountingDigestOutputStream(OutputStream outputStream) { } /** - * Builds a {@link BlobDescriptor} with the hash and size of the bytes written. The buffer resets - * after this method is called, so this method should only be called once per BlobDescriptor. - * - * @return the built {@link BlobDescriptor}. + * Computes the hash and remembers the size of the bytes written to compute the hash. The buffer + * resets after this method is called, so this method should only be called once per computation. */ - public BlobDescriptor toBlobDescriptor() { + public void computeDigest() { try { byte[] hashedBytes = digest.digest(); @@ -65,28 +68,40 @@ public BlobDescriptor toBlobDescriptor() { } String hash = stringBuilder.toString(); - DescriptorDigest digest = DescriptorDigest.fromHash(hash); - return new BlobDescriptor(totalBytes, digest); + bytesHashed = bytesSoFar; + descriptorDigest = DescriptorDigest.fromHash(hash); + bytesSoFar = 0; } catch (DigestException ex) { throw new RuntimeException("SHA-256 algorithm produced invalid hash: " + ex.getMessage(), ex); } } - /** @return the total number of bytes that were hashed */ - public long getTotalBytes() { - return totalBytes; + /** @return the number of bytes written and used to compute the most recent digest */ + public long getBytesHahsed() { + if (descriptorDigest == null) { + computeDigest(); + } + return bytesHashed; + } + + /** @return the most recently computed digest hash */ + public DescriptorDigest getDigest() { + if (descriptorDigest == null) { + computeDigest(); + } + return Verify.verifyNotNull(descriptorDigest); } @Override public void write(byte[] data, int offset, int length) throws IOException { super.write(data, offset, length); - totalBytes += length; + bytesSoFar += length; } @Override public void write(int singleByte) throws IOException { super.write(singleByte); - totalBytes++; + bytesSoFar++; } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java new file mode 100644 index 0000000000..946083c69a --- /dev/null +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java @@ -0,0 +1,102 @@ +/* + * Copyright 2017 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.tools.jib.hash; + +import com.google.cloud.tools.jib.blob.BlobDescriptor; +import com.google.cloud.tools.jib.image.DescriptorDigest; +import com.google.cloud.tools.jib.json.JsonTemplate; +import com.google.cloud.tools.jib.json.JsonTemplateMapper; +import com.google.common.io.ByteStreams; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.List; + +/** + * Utility class for computing a digest for various inputs while optionally writing to an output + * stream. + */ +// Note: intentionally this class does not depend on Blob, as Blob classes depend on this class. +// TODO: BlobDescriptor is merely a tuple of (size, digest). Rename BlobDescriptor to something +// more general. +public class DigestUtil { + + public static DescriptorDigest computeJsonDigest(JsonTemplate template) throws IOException { + return computeDigest(template, ByteStreams.nullOutputStream()).getDigest(); + } + + public static DescriptorDigest computeJsonDigest(List templates) + throws IOException { + WritableContents contents = contentsOut -> JsonTemplateMapper.writeTo(templates, contentsOut); + return computeDigest(contents, ByteStreams.nullOutputStream()).getDigest(); + } + + public static BlobDescriptor computeDigest(JsonTemplate template, OutputStream optionalOutStream) + throws IOException { + WritableContents contents = contentsOut -> JsonTemplateMapper.writeTo(template, contentsOut); + return computeDigest(contents, optionalOutStream); + } + + public static BlobDescriptor computeDigest(InputStream inStream) throws IOException { + return computeDigest(inStream, ByteStreams.nullOutputStream()); + } + + /** + * Computes the digest by consuming the contents. + * + * @param contents the contents for which the digest is computed + * @return computed digest and bytes consumed + * @throws IOException if reading fails + */ + public static BlobDescriptor computeDigest(WritableContents contents) throws IOException { + return computeDigest(contents, ByteStreams.nullOutputStream()); + } + + /** + * Computes the digest by consuming the contents of an {@link InputStream} and optionally copying + * it to an {@link OutputStream}. Returns the computed digest along with the size of the bytes + * consumed to compute the digest. Does not close either stream. + * + * @param inStream the stream to read the contents from + * @param optionalOutStream the stream to which the contents are copied + * @return computed digest and bytes consumed + * @throws IOException if reading from or writing fails + */ + public static BlobDescriptor computeDigest(InputStream inStream, OutputStream optionalOutStream) + throws IOException { + WritableContents contents = contentsOut -> ByteStreams.copy(inStream, contentsOut); + return computeDigest(contents, optionalOutStream); + } + + /** + * Computes the digest by consuming the contents and optionally copying it to an {@link + * OutputStream}. Returns the computed digest along with the size of the bytes consumed to compute + * the digest. Does not close the stream. + * + * @param contents the contents to compute digest for + * @param optionalOutStream the stream to which the contents are copied + * @return computed digest and bytes consumed + * @throws IOException if reading from or writing fails + */ + public static BlobDescriptor computeDigest( + WritableContents contents, OutputStream optionalOutStream) throws IOException { + CountingDigestOutputStream digestOutStream = new CountingDigestOutputStream(optionalOutStream); + contents.writeTo(digestOutStream); + digestOutStream.flush(); + return new BlobDescriptor(digestOutStream.getBytesHahsed(), digestOutStream.getDigest()); + } +} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/BlobWriter.java b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/WritableContents.java similarity index 63% rename from jib-core/src/main/java/com/google/cloud/tools/jib/blob/BlobWriter.java rename to jib-core/src/main/java/com/google/cloud/tools/jib/hash/WritableContents.java index acf393a524..5fc4fae624 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/BlobWriter.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/WritableContents.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Google LLC. + * Copyright 2019 Google LLC. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -14,17 +14,19 @@ * the License. */ -package com.google.cloud.tools.jib.blob; +package com.google.cloud.tools.jib.hash; import java.io.IOException; import java.io.OutputStream; /** - * This function writes the contents of a BLOB. This function should write the same BLOB regardless - * of the number of times it is called. + * As a function, writes some contents to an output stream. As a class, represents contents that can + * be written to an output stream. This may be "unrealized-before-write" contents; for example, a + * file may be open and read for input contents only when this function is called to write to an + * output stream. */ @FunctionalInterface -public interface BlobWriter { +public interface WritableContents { void writeTo(OutputStream outputStream) throws IOException; } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/http/Request.java b/jib-core/src/main/java/com/google/cloud/tools/jib/http/Request.java index e6ef54f7f0..88631a5546 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/http/Request.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/http/Request.java @@ -16,6 +16,7 @@ package com.google.cloud.tools.jib.http; +import com.google.api.client.http.HttpContent; import com.google.api.client.http.HttpHeaders; import java.util.List; import javax.annotation.Nullable; @@ -27,7 +28,7 @@ public class Request { private final HttpHeaders headers; /** The HTTP request body. */ - @Nullable private final BlobHttpContent body; + @Nullable private final HttpContent body; /** HTTP connection and read timeout. */ @Nullable private final Integer httpTimeout; @@ -35,7 +36,7 @@ public class Request { public static class Builder { private final HttpHeaders headers = new HttpHeaders().setAccept("*/*"); - @Nullable private BlobHttpContent body; + @Nullable private HttpContent body; @Nullable private Integer httpTimeout; public Request build() { @@ -92,11 +93,11 @@ public Builder setHttpTimeout(@Nullable Integer httpTimeout) { /** * Sets the body and its corresponding {@code Content-Type} header. * - * @param blobHttpContent the body content + * @param httpContent the body content * @return this */ - public Builder setBody(@Nullable BlobHttpContent blobHttpContent) { - this.body = blobHttpContent; + public Builder setBody(@Nullable HttpContent httpContent) { + this.body = httpContent; return this; } } @@ -116,7 +117,7 @@ HttpHeaders getHeaders() { } @Nullable - BlobHttpContent getHttpContent() { + HttpContent getHttpContent() { return body; } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java index bb504ffeac..8a8f06c5e9 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java @@ -18,13 +18,13 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; +import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.configuration.DockerHealthCheck; import com.google.cloud.tools.jib.configuration.Port; import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.Layer; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -189,7 +189,7 @@ public Blob getContainerConfigurationBlob() { } // Serializes into JSON. - return JsonTemplateMapper.toBlob(template); + return Blobs.from(template); } /** diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java b/jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java index 2ff9e4dbe5..c5f1c5f501 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java @@ -16,12 +16,14 @@ package com.google.cloud.tools.jib.json; +import com.fasterxml.jackson.core.JsonGenerationException; +import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.type.CollectionType; -import com.google.cloud.tools.jib.blob.Blob; -import com.google.cloud.tools.jib.blob.Blobs; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.nio.file.Files; @@ -117,28 +119,36 @@ public static List readListOfJson( return objectMapper.readValue(jsonString, listType); } - /** - * Convert a {@link JsonTemplate} to a {@link Blob} of the JSON string. - * - * @param template the JSON template to convert - * @return a {@link Blob} of the JSON string - */ - public static Blob toBlob(JsonTemplate template) { - return toBlob((Object) template); + public static String toUtf8String(JsonTemplate template) + throws JsonGenerationException, JsonMappingException, IOException { + return toUtf8String((Object) template); } - /** - * Convert a list of {@link JsonTemplate} to a {@link Blob} of the JSON string. - * - * @param templates the list of JSON templates to convert - * @return a {@link Blob} of the JSON string - */ - public static Blob toBlob(List templates) { - return toBlob((Object) templates); + public static String toUtf8String(List templates) + throws JsonGenerationException, JsonMappingException, IOException { + return toUtf8String((Object) templates); + } + + public static void writeTo(JsonTemplate template, OutputStream out) + throws JsonGenerationException, JsonMappingException, IOException { + writeTo((Object) template, out); + } + + public static void writeTo(List templates, OutputStream out) + throws JsonGenerationException, JsonMappingException, IOException { + writeTo((Object) templates, out); + } + + private static String toUtf8String(Object template) + throws JsonGenerationException, JsonMappingException, IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writeTo(template, out); + return out.toString("UTF-8"); } - private static Blob toBlob(Object template) { - return Blobs.from(outputStream -> objectMapper.writeValue(outputStream, template)); + private static void writeTo(Object template, OutputStream out) + throws JsonGenerationException, JsonMappingException, IOException { + objectMapper.writeValue(out, template); } private JsonTemplateMapper() {} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPusher.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPusher.java index 87834575ba..a0cf547579 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPusher.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPusher.java @@ -18,14 +18,14 @@ import com.google.api.client.http.HttpMethods; import com.google.api.client.http.HttpResponseException; +import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.event.EventDispatcher; import com.google.cloud.tools.jib.event.events.LogEvent; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.http.BlobHttpContent; import com.google.cloud.tools.jib.http.Response; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.json.BuildableManifestTemplate; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; -import com.google.common.io.ByteStreams; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; @@ -82,7 +82,7 @@ private static String makeUnexpectedImageDigestWarning( @Override public BlobHttpContent getContent() { return new BlobHttpContent( - JsonTemplateMapper.toBlob(manifestTemplate), + Blobs.from(manifestTemplate), manifestTemplate.getManifestMediaType(), // TODO: Consider giving progress on manifest push as well? null); @@ -125,10 +125,7 @@ public DescriptorDigest handleHttpResponseException(HttpResponseException httpRe @Override public DescriptorDigest handleResponse(Response response) throws IOException { // Checks if the image digest is as expected. - DescriptorDigest expectedDigest = - JsonTemplateMapper.toBlob(manifestTemplate) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DescriptorDigest expectedDigest = DigestUtil.computeJsonDigest(manifestTemplate); List receivedDigests = response.getHeader(RESPONSE_DIGEST_HEADER); if (receivedDigests.size() == 1) { diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java index 623068664d..721b36b257 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java @@ -16,7 +16,8 @@ package com.google.cloud.tools.jib.blob; -import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; +import com.google.cloud.tools.jib.hash.DigestUtil; +import com.google.cloud.tools.jib.hash.WritableContents; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.common.io.Resources; import java.io.ByteArrayInputStream; @@ -32,7 +33,6 @@ import java.nio.file.StandardOpenOption; import org.junit.Assert; import org.junit.Test; -import org.mockito.Mockito; /** Tests for {@link Blob}. */ public class BlobTest { @@ -58,13 +58,13 @@ public void testFromString() throws IOException { } @Test - public void testFromBlobWriter() throws IOException { + public void testFromWritableContents() throws IOException { String expected = "crepecake"; - BlobWriter writer = + WritableContents writableContents = outputStream -> outputStream.write(expected.getBytes(StandardCharsets.UTF_8)); - verifyBlobWriteTo(expected, Blobs.from(writer)); + verifyBlobWriteTo(expected, Blobs.from(writableContents)); } @Test @@ -107,10 +107,8 @@ private void verifyBlobWriteTo(String expected, Blob blob) throws IOException { byte[] expectedBytes = expected.getBytes(StandardCharsets.UTF_8); Assert.assertEquals(expectedBytes.length, blobDescriptor.getSize()); - CountingDigestOutputStream countingDigestOutputStream = - new CountingDigestOutputStream(Mockito.mock(OutputStream.class)); - countingDigestOutputStream.write(expectedBytes); - DescriptorDigest expectedDigest = countingDigestOutputStream.toBlobDescriptor().getDigest(); + DescriptorDigest expectedDigest = + DigestUtil.computeDigest(new ByteArrayInputStream(expectedBytes)).getDigest(); Assert.assertEquals(expectedDigest, blobDescriptor.getDigest()); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriterTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriterTest.java index 53e9c78612..981a98cf8c 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriterTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriterTest.java @@ -19,7 +19,6 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.blob.Blobs; -import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.ImageReference; import com.google.cloud.tools.jib.image.InvalidImageReferenceException; @@ -46,15 +45,8 @@ /** Tests for {@link DefaultCacheStorageWriter}. */ public class DefaultCacheStorageWriterTest { - private static DescriptorDigest getDigest(Blob blob) throws IOException { - return blob.writeTo(new CountingDigestOutputStream(ByteStreams.nullOutputStream())).getDigest(); - } - - private static BlobDescriptor getCompressedBlobDescriptor(Blob blob) throws IOException { - CountingDigestOutputStream compressedDigestOutputStream = - new CountingDigestOutputStream(ByteStreams.nullOutputStream()); - compress(blob).writeTo(compressedDigestOutputStream); - return compressedDigestOutputStream.toBlobDescriptor(); + private static BlobDescriptor getDigest(Blob blob) throws IOException { + return blob.writeTo(ByteStreams.nullOutputStream()); } private static Blob compress(Blob blob) { @@ -95,8 +87,8 @@ public void testWrite_compressed() throws IOException { @Test public void testWrite_uncompressed() throws IOException { Blob uncompressedLayerBlob = Blobs.from("uncompressedLayerBlob"); - DescriptorDigest layerDigest = getCompressedBlobDescriptor(uncompressedLayerBlob).getDigest(); - DescriptorDigest selector = getDigest(Blobs.from("selector")); + DescriptorDigest layerDigest = getDigest(compress(uncompressedLayerBlob)).getDigest(); + DescriptorDigest selector = getDigest(Blobs.from("selector")).getDigest(); CachedLayer cachedLayer = new DefaultCacheStorageWriter(defaultCacheStorageFiles) @@ -169,8 +161,8 @@ public void testWriteMetadata_v22() private void verifyCachedLayer(CachedLayer cachedLayer, Blob uncompressedLayerBlob) throws IOException { - BlobDescriptor layerBlobDescriptor = getCompressedBlobDescriptor(uncompressedLayerBlob); - DescriptorDigest layerDiffId = getDigest(uncompressedLayerBlob); + BlobDescriptor layerBlobDescriptor = getDigest(compress(uncompressedLayerBlob)); + DescriptorDigest layerDiffId = getDigest(uncompressedLayerBlob).getDigest(); // Verifies cachedLayer is correct. Assert.assertEquals(layerBlobDescriptor.getDigest(), cachedLayer.getDigest()); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/LayerEntriesSelectorTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/LayerEntriesSelectorTest.java index b8181d2e55..7337c9fc2b 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/LayerEntriesSelectorTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/LayerEntriesSelectorTest.java @@ -19,11 +19,10 @@ import com.google.cloud.tools.jib.cache.LayerEntriesSelector.LayerEntryTemplate; import com.google.cloud.tools.jib.configuration.FilePermissions; import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.LayerEntry; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.collect.ImmutableList; -import com.google.common.io.ByteStreams; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -108,10 +107,7 @@ public void testToSortedJsonTemplates() throws IOException { @Test public void testGenerateSelector_empty() throws IOException { - DescriptorDigest expectedSelector = - JsonTemplateMapper.toBlob(ImmutableList.of()) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DescriptorDigest expectedSelector = DigestUtil.computeJsonDigest(ImmutableList.of()); Assert.assertEquals( expectedSelector, LayerEntriesSelector.generateSelector(ImmutableList.of())); } @@ -119,9 +115,7 @@ public void testGenerateSelector_empty() throws IOException { @Test public void testGenerateSelector() throws IOException { DescriptorDigest expectedSelector = - JsonTemplateMapper.toBlob(toLayerEntryTemplates(inOrderLayerEntries)) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DigestUtil.computeJsonDigest(toLayerEntryTemplates(inOrderLayerEntries)); Assert.assertEquals( expectedSelector, LayerEntriesSelector.generateSelector(outOfOrderLayerEntries)); } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/json/DockerLoadManifestTemplateTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/json/DockerLoadManifestTemplateTest.java index 8669f6467f..2f45ed5f53 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/json/DockerLoadManifestTemplateTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/json/DockerLoadManifestTemplateTest.java @@ -16,7 +16,6 @@ package com.google.cloud.tools.jib.docker.json; -import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.image.ImageReference; import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.io.Resources; @@ -48,6 +47,6 @@ public void testToJson() throws URISyntaxException, IOException { template.addLayerFile("layer3.tar.gz"); List loadManifest = Collections.singletonList(template); - Assert.assertEquals(expectedJson, Blobs.writeToString(JsonTemplateMapper.toBlob(loadManifest))); + Assert.assertEquals(expectedJson, JsonTemplateMapper.toUtf8String(loadManifest)); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStreamTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStreamTest.java index a2b1b40f7e..b4901f2615 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStreamTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStreamTest.java @@ -16,7 +16,6 @@ package com.google.cloud.tools.jib.hash; -import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteStreams; @@ -57,10 +56,9 @@ public void test_smokeTest() throws IOException, DigestException { InputStream toHashInputStream = new ByteArrayInputStream(bytesToHash); ByteStreams.copy(toHashInputStream, countingDigestOutputStream); - BlobDescriptor expectedBlobDescriptor = - new BlobDescriptor(bytesToHash.length, DescriptorDigest.fromHash(expectedHash)); - Assert.assertEquals(expectedBlobDescriptor, countingDigestOutputStream.toBlobDescriptor()); - Assert.assertEquals(bytesToHash.length, countingDigestOutputStream.getTotalBytes()); + Assert.assertEquals( + DescriptorDigest.fromHash(expectedHash), countingDigestOutputStream.getDigest()); + Assert.assertEquals(bytesToHash.length, countingDigestOutputStream.getBytesHahsed()); } } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/http/TestBlobProgressListener.java b/jib-core/src/test/java/com/google/cloud/tools/jib/http/TestBlobProgressListener.java index e0e24d0874..3fd4a0adfe 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/http/TestBlobProgressListener.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/http/TestBlobProgressListener.java @@ -23,7 +23,7 @@ * Test implementation of {@link BlobProgressListener} that always forwards to a {@link Consumer * }. */ -public class TestBlobProgressListener implements BlobProgressListener, Consumer { +public class TestBlobProgressListener implements BlobProgressListener { private final Consumer byteCountConsumer; @@ -40,9 +40,4 @@ public void handleByteCount(long byteCount) { public Duration getDelayBetweenCallbacks() { return Duration.ofSeconds(-1); } - - @Override - public void accept(Long byteCount) { - handleByteCount(byteCount); - } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ContainerConfigurationTemplateTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ContainerConfigurationTemplateTest.java index 98366aacaa..e0e5423b16 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ContainerConfigurationTemplateTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ContainerConfigurationTemplateTest.java @@ -22,7 +22,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSortedMap; import com.google.common.io.Resources; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; @@ -91,10 +90,7 @@ public void testToJson() throws IOException, URISyntaxException, DigestException .build()); // Serializes the JSON object. - ByteArrayOutputStream jsonStream = new ByteArrayOutputStream(); - JsonTemplateMapper.toBlob(containerConfigJson).writeTo(jsonStream); - - Assert.assertEquals(expectedJson, jsonStream.toString()); + Assert.assertEquals(expectedJson, JsonTemplateMapper.toUtf8String(containerConfigJson)); } @Test diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslatorTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslatorTest.java index f7a5f68524..f7d333c3c6 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslatorTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslatorTest.java @@ -190,9 +190,6 @@ private void testGetManifest( T manifestTemplate = imageToJsonTranslator.getManifestTemplate(manifestTemplateClass, blobDescriptor); - ByteArrayOutputStream jsonStream = new ByteArrayOutputStream(); - JsonTemplateMapper.toBlob(manifestTemplate).writeTo(jsonStream); - - Assert.assertEquals(expectedJson, new String(jsonStream.toByteArray(), StandardCharsets.UTF_8)); + Assert.assertEquals(expectedJson, JsonTemplateMapper.toUtf8String(manifestTemplate)); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/OCIManifestTemplateTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/OCIManifestTemplateTest.java index 40ff8e4fc1..7c7defd779 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/OCIManifestTemplateTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/OCIManifestTemplateTest.java @@ -19,7 +19,6 @@ import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.io.Resources; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; @@ -53,10 +52,7 @@ public void testToJson() throws DigestException, IOException, URISyntaxException "4945ba5011739b0b98c4a41afe224e417f47c7c99b2ce76830999c9a0861b236")); // Serializes the JSON object. - ByteArrayOutputStream jsonStream = new ByteArrayOutputStream(); - JsonTemplateMapper.toBlob(manifestJson).writeTo(jsonStream); - - Assert.assertEquals(expectedJson, jsonStream.toString()); + Assert.assertEquals(expectedJson, JsonTemplateMapper.toUtf8String(manifestJson)); } @Test diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/V22ManifestTemplateTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/V22ManifestTemplateTest.java index 58eb1a3486..c8d0ba8d55 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/V22ManifestTemplateTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/V22ManifestTemplateTest.java @@ -19,7 +19,6 @@ import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.io.Resources; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; @@ -53,10 +52,7 @@ public void testToJson() throws DigestException, IOException, URISyntaxException "4945ba5011739b0b98c4a41afe224e417f47c7c99b2ce76830999c9a0861b236")); // Serializes the JSON object. - ByteArrayOutputStream jsonStream = new ByteArrayOutputStream(); - JsonTemplateMapper.toBlob(manifestJson).writeTo(jsonStream); - - Assert.assertEquals(expectedJson, jsonStream.toString()); + Assert.assertEquals(expectedJson, JsonTemplateMapper.toUtf8String(manifestJson)); } @Test diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/json/JsonTemplateMapperTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/json/JsonTemplateMapperTest.java index 1da54bad3d..4d2ea955c5 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/json/JsonTemplateMapperTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/json/JsonTemplateMapperTest.java @@ -18,7 +18,6 @@ import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.common.io.Resources; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; @@ -85,10 +84,7 @@ public void testWriteJson() throws DigestException, IOException, URISyntaxExcept "sha256:d38f571aa1c11e3d516e0ef7e513e7308ccbeb869770cb8c4319d63b10a0075e")); testJson.list = Arrays.asList(innerObject1, innerObject2); - ByteArrayOutputStream jsonStream = new ByteArrayOutputStream(); - JsonTemplateMapper.toBlob(testJson).writeTo(jsonStream); - - Assert.assertEquals(expectedJson, jsonStream.toString()); + Assert.assertEquals(expectedJson, JsonTemplateMapper.toUtf8String(testJson)); } @Test @@ -155,9 +151,6 @@ public void testToBlob_listOfJson() throws IOException, URISyntaxException { String jsonString = new String(Files.readAllBytes(jsonFile), StandardCharsets.UTF_8); List listOfJson = JsonTemplateMapper.readListOfJson(jsonString, TestJson.class); - ByteArrayOutputStream jsonStream = new ByteArrayOutputStream(); - JsonTemplateMapper.toBlob(listOfJson).writeTo(jsonStream); - - Assert.assertEquals(jsonString, jsonStream.toString()); + Assert.assertEquals(jsonString, JsonTemplateMapper.toUtf8String(listOfJson)); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobCheckerTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobCheckerTest.java index 2ec250c417..a45833830e 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobCheckerTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobCheckerTest.java @@ -19,7 +19,6 @@ import com.google.api.client.http.HttpResponseException; import com.google.api.client.http.HttpStatusCodes; import com.google.cloud.tools.jib.blob.BlobDescriptor; -import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.http.Response; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.json.JsonTemplateMapper; @@ -92,7 +91,7 @@ public void testHandleHttpResponseException() throws IOException, RegistryErrorE new ErrorResponseTemplate() .addError(new ErrorEntryTemplate(ErrorCodes.BLOB_UNKNOWN.name(), "some message")); Mockito.when(mockHttpResponseException.getContent()) - .thenReturn(Blobs.writeToString(JsonTemplateMapper.toBlob(emptyErrorResponseTemplate))); + .thenReturn(JsonTemplateMapper.toUtf8String(emptyErrorResponseTemplate)); BlobDescriptor blobDescriptor = testBlobChecker.handleHttpResponseException(mockHttpResponseException); @@ -112,7 +111,7 @@ public void testHandleHttpResponseException_hasOtherErrors() .addError(new ErrorEntryTemplate(ErrorCodes.BLOB_UNKNOWN.name(), "some message")) .addError(new ErrorEntryTemplate(ErrorCodes.MANIFEST_UNKNOWN.name(), "some message")); Mockito.when(mockHttpResponseException.getContent()) - .thenReturn(Blobs.writeToString(JsonTemplateMapper.toBlob(emptyErrorResponseTemplate))); + .thenReturn(JsonTemplateMapper.toUtf8String(emptyErrorResponseTemplate)); try { testBlobChecker.handleHttpResponseException(mockHttpResponseException); @@ -132,7 +131,7 @@ public void testHandleHttpResponseException_notBlobUnknown() ErrorResponseTemplate emptyErrorResponseTemplate = new ErrorResponseTemplate(); Mockito.when(mockHttpResponseException.getContent()) - .thenReturn(Blobs.writeToString(JsonTemplateMapper.toBlob(emptyErrorResponseTemplate))); + .thenReturn(JsonTemplateMapper.toUtf8String(emptyErrorResponseTemplate)); try { testBlobChecker.handleHttpResponseException(mockHttpResponseException); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java index e0eb9311cd..a21f4bd725 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java @@ -87,7 +87,7 @@ public void testHandleResponse() throws IOException, UnexpectedBlobDigestExcepti Assert.assertEquals( "some BLOB content", new String(layerContentOutputStream.toByteArray(), StandardCharsets.UTF_8)); - Assert.assertEquals(testBlobDigest, layerOutputStream.toBlobDescriptor().getDigest()); + Assert.assertEquals(testBlobDigest, layerOutputStream.getDigest()); Assert.assertEquals("some BLOB content".length(), byteCount.sum()); } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPusherTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPusherTest.java index 57b8fdb0f8..c15ca8dfa9 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPusherTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPusherTest.java @@ -20,12 +20,12 @@ import com.google.api.client.http.HttpResponseException; import com.google.cloud.tools.jib.event.EventDispatcher; import com.google.cloud.tools.jib.event.events.LogEvent; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.http.BlobHttpContent; import com.google.cloud.tools.jib.http.Response; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.json.V22ManifestTemplate; import com.google.cloud.tools.jib.json.JsonTemplateMapper; -import com.google.common.io.ByteStreams; import com.google.common.io.Resources; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -90,10 +90,7 @@ public void testGetContent() throws IOException { @Test public void testHandleResponse_valid() throws IOException { - DescriptorDigest expectedDigest = - JsonTemplateMapper.toBlob(fakeManifestTemplate) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DescriptorDigest expectedDigest = DigestUtil.computeJsonDigest(fakeManifestTemplate); Mockito.when(mockResponse.getHeader("Docker-Content-Digest")) .thenReturn(Collections.singletonList(expectedDigest.toString())); Assert.assertEquals(expectedDigest, testManifestPusher.handleResponse(mockResponse)); @@ -101,10 +98,7 @@ public void testHandleResponse_valid() throws IOException { @Test public void testHandleResponse_noDigest() throws IOException { - DescriptorDigest expectedDigest = - JsonTemplateMapper.toBlob(fakeManifestTemplate) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DescriptorDigest expectedDigest = DigestUtil.computeJsonDigest(fakeManifestTemplate); Mockito.when(mockResponse.getHeader("Docker-Content-Digest")) .thenReturn(Collections.emptyList()); @@ -115,10 +109,7 @@ public void testHandleResponse_noDigest() throws IOException { @Test public void testHandleResponse_multipleDigests() throws IOException { - DescriptorDigest expectedDigest = - JsonTemplateMapper.toBlob(fakeManifestTemplate) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DescriptorDigest expectedDigest = DigestUtil.computeJsonDigest(fakeManifestTemplate); Mockito.when(mockResponse.getHeader("Docker-Content-Digest")) .thenReturn(Arrays.asList("too", "many")); @@ -130,10 +121,7 @@ public void testHandleResponse_multipleDigests() throws IOException { @Test public void testHandleResponse_invalidDigest() throws IOException { - DescriptorDigest expectedDigest = - JsonTemplateMapper.toBlob(fakeManifestTemplate) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DescriptorDigest expectedDigest = DigestUtil.computeJsonDigest(fakeManifestTemplate); Mockito.when(mockResponse.getHeader("Docker-Content-Digest")) .thenReturn(Collections.singletonList("not valid")); From 11d7b60ff323fafd7ee2589acddb893ecea7d9ed Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 30 Apr 2019 19:12:09 -0400 Subject: [PATCH 20/39] wip --- .../jib/event/progress/DelayedConsumer.java | 33 +++++++++++++------ .../cloud/tools/jib/http/BlobHttpContent.java | 20 +++++------ .../cloud/tools/jib/registry/BlobPuller.java | 8 ++--- .../tools/jib/registry/ManifestPusher.java | 6 ++-- .../jib/registry/RegistryAuthenticator.java | 2 +- .../cloud/tools/jib/http/ConnectionTest.java | 2 +- .../ListenableCountingOutputStreamTest.java | 12 +++---- .../cloud/tools/jib/http/ResponseTest.java | 2 +- .../jib/http/WithServerConnectionTest.java | 4 +-- 9 files changed, 48 insertions(+), 41 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/DelayedConsumer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/DelayedConsumer.java index 810238faf2..daba5e67d0 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/DelayedConsumer.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/DelayedConsumer.java @@ -16,19 +16,22 @@ package com.google.cloud.tools.jib.event.progress; +import java.io.Closeable; +import java.io.IOException; import java.time.Duration; import java.time.Instant; import java.util.function.BinaryOperator; import java.util.function.Consumer; import java.util.function.Supplier; +import javax.annotation.Nullable; /** * Wraps a {@link Consumer} so that multiple consume calls within a short time are blocked and * delayed into a single call. */ -public class DelayedConsumer implements Consumer { +public class DelayedConsumer implements Consumer, Closeable { - private final Consumer originalConsumer; + private final Consumer consumer; /** Delay between each call to {@link #byteCountConsumer}. */ private final Duration delayBetweenCallbacks; @@ -45,9 +48,14 @@ public class DelayedConsumer implements Consumer { /** Last time {@link #byteCountConsumer} was called. */ private Instant previousCallback; - private Consumer consumer; + @Nullable private T valueSoFar; - /** Wraps a consumer with the delay of 100 ms. */ + /** + * Wraps a consumer with the delay of 100 ms. + * + * @param callback {@link Consumer} callback to wrap + * @param adder adds up multiple delayed values + */ public DelayedConsumer(Consumer callback, BinaryOperator adder) { this(callback, adder, Duration.ofMillis(100), Instant::now); } @@ -57,7 +65,6 @@ public DelayedConsumer( BinaryOperator adder, Duration delayBetweenCallbacks, Supplier getNow) { - this.originalConsumer = consumer; this.consumer = consumer; this.adder = adder; this.delayBetweenCallbacks = delayBetweenCallbacks; @@ -68,14 +75,20 @@ public DelayedConsumer( @Override public void accept(T value) { + valueSoFar = valueSoFar == null ? value : adder.apply(valueSoFar, value); + Instant now = getNow.get(); if (previousCallback.plus(delayBetweenCallbacks).isBefore(now)) { + consumer.accept(valueSoFar); previousCallback = now; - consumer.accept(value); - consumer = originalConsumer; - } else { - Consumer currentConsumer = consumer; - consumer = nextValue -> currentConsumer.accept(adder.apply(nextValue, value)); + valueSoFar = null; + } + } + + @Override + public void close() throws IOException { + if (valueSoFar != null) { + consumer.accept(valueSoFar); } } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobHttpContent.java b/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobHttpContent.java index 2bca5bbedc..3ec702f826 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobHttpContent.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobHttpContent.java @@ -22,17 +22,19 @@ import java.io.IOException; import java.io.OutputStream; import java.util.function.Consumer; -import javax.annotation.Nullable; /** {@link Blob}-backed {@link HttpContent}. */ public class BlobHttpContent implements HttpContent { private final Blob blob; private final String contentType; - @Nullable private final Consumer writtenByteCountConsumer; + private final Consumer writtenByteCountConsumer; - public BlobHttpContent( - Blob blob, String contentType, @Nullable Consumer writtenByteCountConsumer) { + public BlobHttpContent(Blob blob, String contentType) { + this(blob, contentType, ignored -> {}); + } + + public BlobHttpContent(Blob blob, String contentType, Consumer writtenByteCountConsumer) { this.blob = blob; this.contentType = contentType; this.writtenByteCountConsumer = writtenByteCountConsumer; @@ -56,12 +58,10 @@ public boolean retrySupported() { @Override public void writeTo(OutputStream outputStream) throws IOException { - if (writtenByteCountConsumer != null) { - Consumer delayedConsumer = - new DelayedConsumer<>(writtenByteCountConsumer, (a, b) -> a + b); - outputStream = new ListenableCountingOutputStream(outputStream, delayedConsumer); + try (DelayedConsumer delayedConsumer = + new DelayedConsumer<>(writtenByteCountConsumer, (a, b) -> a + b)) { + blob.writeTo(new ListenableCountingOutputStream(outputStream, delayedConsumer)); + outputStream.flush(); } - blob.writeTo(outputStream); - outputStream.flush(); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java index faaffbc56f..8ac3204530 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java @@ -66,10 +66,10 @@ class BlobPuller implements RegistryEndpointProvider { public Void handleResponse(Response response) throws IOException, UnexpectedBlobDigestException { blobSizeConsumer.accept(response.getContentLength()); - Consumer delayedConsumer = - new DelayedConsumer<>(writtenByteCountConsumer, (a, b) -> a + b); - try (OutputStream outputStream = - new ListenableCountingOutputStream(destinationOutputStream, delayedConsumer)) { + try (DelayedConsumer delayedByteConsumer = + new DelayedConsumer<>(writtenByteCountConsumer, (a, b) -> a + b); + OutputStream outputStream = + new ListenableCountingOutputStream(destinationOutputStream, delayedByteConsumer)) { BlobDescriptor receivedBlobDescriptor = DigestUtil.computeDigest(response.getBody(), outputStream); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPusher.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPusher.java index a0cf547579..ce554c0896 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPusher.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPusher.java @@ -81,11 +81,9 @@ private static String makeUnexpectedImageDigestWarning( @Override public BlobHttpContent getContent() { + // TODO: Consider giving progress on manifest push as well? return new BlobHttpContent( - Blobs.from(manifestTemplate), - manifestTemplate.getManifestMediaType(), - // TODO: Consider giving progress on manifest push as well? - null); + Blobs.from(manifestTemplate), manifestTemplate.getManifestMediaType()); } @Override diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryAuthenticator.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryAuthenticator.java index 0ee7d52cc1..1857660c7a 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryAuthenticator.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryAuthenticator.java @@ -316,7 +316,7 @@ private Authorization authenticate(String scope) throws RegistryAuthenticationFa if (isOAuth2Auth()) { String parameters = getAuthRequestParameters(scope); requestBuilder.setBody( - new BlobHttpContent(Blobs.from(parameters), MediaType.FORM_DATA.toString(), null)); + new BlobHttpContent(Blobs.from(parameters), MediaType.FORM_DATA.toString())); } else if (credential != null) { requestBuilder.setAuthorization( Authorization.withBasicCredentials(credential.getUsername(), credential.getPassword())); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/http/ConnectionTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/http/ConnectionTest.java index a4e8d2850a..9c880b5eca 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/http/ConnectionTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/http/ConnectionTest.java @@ -158,6 +158,6 @@ private void testSend(String httpMethod, SendFunction sendFunction) throws IOExc Assert.assertEquals( "crepecake", new String(byteArrayOutputStream.toByteArray(), StandardCharsets.UTF_8)); - Assert.assertEquals("crepecake".length(), totalByteCount); + Assert.assertEquals("crepecake".length(), totalByteCount.longValue()); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStreamTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStreamTest.java index 460aace01e..1fed293f6c 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStreamTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStreamTest.java @@ -58,18 +58,16 @@ public void testDelay() throws IOException { Queue instantQueue = new ArrayDeque<>(); instantQueue.add(Instant.EPOCH); - try (ListenableCountingOutputStream listenableCountingOutputStream = - new ListenableCountingOutputStream( - byteArrayOutputStream, + try (DelayedConsumer byteCounter = new DelayedConsumer<>( - byteCounts::add, (a, b) -> a + b, Duration.ofSeconds(3), instantQueue::remove))) { + byteCounts::add, (a, b) -> a + b, Duration.ofSeconds(3), instantQueue::remove); + ListenableCountingOutputStream listenableCountingOutputStream = + new ListenableCountingOutputStream(byteArrayOutputStream, byteCounter)) { instantQueue.add(Instant.EPOCH); listenableCountingOutputStream.write(100); instantQueue.add(Instant.EPOCH); listenableCountingOutputStream.write(new byte[] {101, 102, 103}); - System.out.println(instantQueue); instantQueue.add(Instant.EPOCH.plusSeconds(4)); - System.out.println(instantQueue); listenableCountingOutputStream.write(new byte[] {104, 105, 106}); instantQueue.add(Instant.EPOCH.plusSeconds(10)); @@ -79,8 +77,6 @@ public void testDelay() throws IOException { listenableCountingOutputStream.write(new byte[] {109}); instantQueue.add(Instant.EPOCH.plusSeconds(13)); listenableCountingOutputStream.write(new byte[] {0, 110}, 1, 1); - - instantQueue.add(Instant.EPOCH.plusSeconds(14)); } Assert.assertEquals(Arrays.asList(7L, 2L, 2L), byteCounts); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/http/ResponseTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/http/ResponseTest.java index 91865a270a..0cba195cd0 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/http/ResponseTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/http/ResponseTest.java @@ -43,6 +43,6 @@ public void testGetContent() throws IOException { Response response = new Response(httpResponseMock); - Assert.assertEquals(expectedResponse, ByteStreams.toByteArray(response.getBody())); + Assert.assertArrayEquals(expectedResponse, ByteStreams.toByteArray(response.getBody())); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/http/WithServerConnectionTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/http/WithServerConnectionTest.java index 8ab952cab8..52dd89eb78 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/http/WithServerConnectionTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/http/WithServerConnectionTest.java @@ -38,7 +38,7 @@ public void testGet() Response response = connection.send("GET", new Request.Builder().build()); Assert.assertEquals(200, response.getStatusCode()); - Assert.assertEquals( + Assert.assertArrayEquals( "Hello World!".getBytes(StandardCharsets.UTF_8), ByteStreams.toByteArray(response.getBody())); } @@ -84,7 +84,7 @@ public void testInsecureConnection() Response response = connection.send("GET", new Request.Builder().build()); Assert.assertEquals(200, response.getStatusCode()); - Assert.assertEquals( + Assert.assertArrayEquals( "Hello World!".getBytes(StandardCharsets.UTF_8), ByteStreams.toByteArray(response.getBody())); } From 52c64554b762ad266c8a336a73dd8d7a31433ad3 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Wed, 1 May 2019 11:58:29 -0400 Subject: [PATCH 21/39] done --- .../cloud/tools/jib/http/BlobHttpContent.java | 12 +++++----- .../http/ListenableCountingOutputStream.java | 24 +++++++++---------- .../cloud/tools/jib/registry/BlobPuller.java | 20 ++++++++-------- .../cloud/tools/jib/registry/BlobPusher.java | 12 +++++----- .../tools/jib/registry/RegistryClient.java | 18 +++++++------- .../tools/jib/registry/BlobPullerTest.java | 2 ++ 6 files changed, 45 insertions(+), 43 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobHttpContent.java b/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobHttpContent.java index 3ec702f826..bf08a135fa 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobHttpContent.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobHttpContent.java @@ -28,16 +28,16 @@ public class BlobHttpContent implements HttpContent { private final Blob blob; private final String contentType; - private final Consumer writtenByteCountConsumer; + private final Consumer writtenByteCountListener; public BlobHttpContent(Blob blob, String contentType) { this(blob, contentType, ignored -> {}); } - public BlobHttpContent(Blob blob, String contentType, Consumer writtenByteCountConsumer) { + public BlobHttpContent(Blob blob, String contentType, Consumer writtenByteCountListener) { this.blob = blob; this.contentType = contentType; - this.writtenByteCountConsumer = writtenByteCountConsumer; + this.writtenByteCountListener = writtenByteCountListener; } @Override @@ -58,9 +58,9 @@ public boolean retrySupported() { @Override public void writeTo(OutputStream outputStream) throws IOException { - try (DelayedConsumer delayedConsumer = - new DelayedConsumer<>(writtenByteCountConsumer, (a, b) -> a + b)) { - blob.writeTo(new ListenableCountingOutputStream(outputStream, delayedConsumer)); + try (DelayedConsumer delayedCountListener = + new DelayedConsumer<>(writtenByteCountListener, (a, b) -> a + b)) { + blob.writeTo(new ListenableCountingOutputStream(outputStream, delayedCountListener)); outputStream.flush(); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStream.java b/jib-core/src/main/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStream.java index 635f3443b9..e151c151c8 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStream.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStream.java @@ -27,60 +27,60 @@ public class ListenableCountingOutputStream extends OutputStream { private final OutputStream underlyingOutputStream; /** Receives a count of bytes written since the last call. */ - private final Consumer byteCountConsumer; + private final Consumer byteCountListener; - /** Number of bytes to provide to {@link #byteCountConsumer}. */ + /** Number of bytes to provide to {@link #byteCountListener}. */ private long byteCount = 0; /** * Wraps the {@code underlyingOutputStream} to count the bytes written. * * @param underlyingOutputStream the wrapped {@link OutputStream} - * @param byteCountConsumer the byte count {@link Consumer} + * @param byteCountListener the byte count {@link Consumer} */ public ListenableCountingOutputStream( - OutputStream underlyingOutputStream, Consumer byteCountConsumer) { + OutputStream underlyingOutputStream, Consumer byteCountListener) { this.underlyingOutputStream = underlyingOutputStream; - this.byteCountConsumer = byteCountConsumer; + this.byteCountListener = byteCountListener; } @Override public void write(int singleByte) throws IOException { underlyingOutputStream.write(singleByte); - addAndCallByteCountConsumer(1); + countAndCallListener(1); } @Override public void write(byte[] byteArray) throws IOException { underlyingOutputStream.write(byteArray); - addAndCallByteCountConsumer(byteArray.length); + countAndCallListener(byteArray.length); } @Override public void write(byte byteArray[], int offset, int length) throws IOException { underlyingOutputStream.write(byteArray, offset, length); - addAndCallByteCountConsumer(length); + countAndCallListener(length); } @Override public void flush() throws IOException { underlyingOutputStream.flush(); - addAndCallByteCountConsumer(0); + countAndCallListener(0); } @Override public void close() throws IOException { underlyingOutputStream.close(); - addAndCallByteCountConsumer(0); + countAndCallListener(0); } - private void addAndCallByteCountConsumer(int written) { + private void countAndCallListener(int written) { this.byteCount += written; if (byteCount == 0) { return; } - byteCountConsumer.accept(byteCount); + byteCountListener.accept(byteCount); byteCount = 0; } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java index 8ac3204530..8cd8a949f2 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java @@ -46,30 +46,30 @@ class BlobPuller implements RegistryEndpointProvider { */ private final OutputStream destinationOutputStream; - private final Consumer blobSizeConsumer; - private final Consumer writtenByteCountConsumer; + private final Consumer blobSizeListener; + private final Consumer writtenByteCountListener; BlobPuller( RegistryEndpointRequestProperties registryEndpointRequestProperties, DescriptorDigest blobDigest, OutputStream destinationOutputStream, - Consumer blobSizeConsumer, - Consumer writtenByteCountConsumer) { + Consumer blobSizeListener, + Consumer writtenByteCountListener) { this.registryEndpointRequestProperties = registryEndpointRequestProperties; this.blobDigest = blobDigest; this.destinationOutputStream = destinationOutputStream; - this.blobSizeConsumer = blobSizeConsumer; - this.writtenByteCountConsumer = writtenByteCountConsumer; + this.blobSizeListener = blobSizeListener; + this.writtenByteCountListener = writtenByteCountListener; } @Override public Void handleResponse(Response response) throws IOException, UnexpectedBlobDigestException { - blobSizeConsumer.accept(response.getContentLength()); + blobSizeListener.accept(response.getContentLength()); - try (DelayedConsumer delayedByteConsumer = - new DelayedConsumer<>(writtenByteCountConsumer, (a, b) -> a + b); + try (DelayedConsumer delayedCountListener = + new DelayedConsumer<>(writtenByteCountListener, (a, b) -> a + b); OutputStream outputStream = - new ListenableCountingOutputStream(destinationOutputStream, delayedByteConsumer)) { + new ListenableCountingOutputStream(destinationOutputStream, delayedCountListener)) { BlobDescriptor receivedBlobDescriptor = DigestUtil.computeDigest(response.getBody(), outputStream); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPusher.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPusher.java index 61f474f937..7e21d94a80 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPusher.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPusher.java @@ -112,12 +112,12 @@ public String getActionDescription() { private class Writer implements RegistryEndpointProvider { private final URL location; - private final Consumer writtenByteCountConsumer; + private final Consumer writtenByteCountListener; @Nullable @Override public BlobHttpContent getContent() { - return new BlobHttpContent(blob, MediaType.OCTET_STREAM.toString(), writtenByteCountConsumer); + return new BlobHttpContent(blob, MediaType.OCTET_STREAM.toString(), writtenByteCountListener); } @Override @@ -147,9 +147,9 @@ public String getActionDescription() { return BlobPusher.this.getActionDescription(); } - private Writer(URL location, Consumer writtenByteCountConsumer) { + private Writer(URL location, Consumer writtenByteCountListener) { this.location = location; - this.writtenByteCountConsumer = writtenByteCountConsumer; + this.writtenByteCountListener = writtenByteCountListener; } } @@ -219,8 +219,8 @@ RegistryEndpointProvider initializer() { * @param blobProgressListener the listener for {@link Blob} push progress * @return a {@link RegistryEndpointProvider} for writing the BLOB to an upload location */ - RegistryEndpointProvider writer(URL location, Consumer writtenByteCountConsumer) { - return new Writer(location, writtenByteCountConsumer); + RegistryEndpointProvider writer(URL location, Consumer writtenByteCountListener) { + return new Writer(location, writtenByteCountListener); } /** diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryClient.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryClient.java index 0adf970e6f..6fc02728a8 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryClient.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryClient.java @@ -245,15 +245,15 @@ public BlobDescriptor checkBlob(DescriptorDigest blobDigest) * written out. * * @param blobDigest the digest of the BLOB to download - * @param blobSizeConsumer callback to receive the total size of the BLOb to pull - * @param writtenByteCountConsumer listens on byte count written to an output stream during the + * @param blobSizeListener callback to receive the total size of the BLOb to pull + * @param writtenByteCountListener listens on byte count written to an output stream during the * pull * @return a {@link Blob} */ public Blob pullBlob( DescriptorDigest blobDigest, - Consumer blobSizeConsumer, - Consumer writtenByteCountConsumer) { + Consumer blobSizeListener, + Consumer writtenByteCountListener) { return Blobs.from( outputStream -> { try { @@ -262,8 +262,8 @@ public Blob pullBlob( registryEndpointRequestProperties, blobDigest, outputStream, - blobSizeConsumer, - writtenByteCountConsumer)); + blobSizeListener, + writtenByteCountListener)); } catch (RegistryException ex) { throw new IOException(ex); @@ -279,7 +279,7 @@ public Blob pullBlob( * @param blob the BLOB to push * @param sourceRepository if pushing to the same registry then the source image, or {@code null} * otherwise; used to optimize the BLOB push - * @param writtenByteCountConsumer listens on byte count written to the registry during the push + * @param writtenByteCountListener listens on byte count written to the registry during the push * @return {@code true} if the BLOB already exists on the registry and pushing was skipped; false * if the BLOB was pushed * @throws IOException if communicating with the endpoint fails @@ -289,7 +289,7 @@ public boolean pushBlob( DescriptorDigest blobDigest, Blob blob, @Nullable String sourceRepository, - Consumer writtenByteCountConsumer) + Consumer writtenByteCountListener) throws IOException, RegistryException { BlobPusher blobPusher = new BlobPusher(registryEndpointRequestProperties, blobDigest, blob, sourceRepository); @@ -311,7 +311,7 @@ public boolean pushBlob( // PATCH with BLOB URL putLocation = - callRegistryEndpoint(blobPusher.writer(patchLocation, writtenByteCountConsumer)); + callRegistryEndpoint(blobPusher.writer(patchLocation, writtenByteCountListener)); Preconditions.checkNotNull(putLocation); timerEventDispatcher2.lap("pushBlob PUT " + blobDigest); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java index 27e20e00cd..242219a2b6 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java @@ -70,6 +70,7 @@ public void testHandleResponse() throws IOException, UnexpectedBlobDigestExcepti InputStream blobContent = new ByteArrayInputStream("some BLOB content".getBytes(StandardCharsets.UTF_8)); DescriptorDigest testBlobDigest = DigestUtil.computeDigest(blobContent).getDigest(); + blobContent.reset(); Response mockResponse = Mockito.mock(Response.class); Mockito.when(mockResponse.getContentLength()).thenReturn((long) "some BLOB content".length()); @@ -96,6 +97,7 @@ public void testHandleResponse_unexpectedDigest() throws IOException { InputStream blobContent = new ByteArrayInputStream("some BLOB content".getBytes(StandardCharsets.UTF_8)); DescriptorDigest testBlobDigest = DigestUtil.computeDigest(blobContent).getDigest(); + blobContent.reset(); Response mockResponse = Mockito.mock(Response.class); Mockito.when(mockResponse.getBody()).thenReturn(blobContent); From f293268184ad92bdd7857748c2f9da2416d44b0f Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Wed, 1 May 2019 17:02:13 -0400 Subject: [PATCH 22/39] Fix wrong filename --- .../{WriterableContentsBlob.java => WritableContentsBlob.java} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename jib-core/src/main/java/com/google/cloud/tools/jib/blob/{WriterableContentsBlob.java => WritableContentsBlob.java} (100%) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterableContentsBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WritableContentsBlob.java similarity index 100% rename from jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterableContentsBlob.java rename to jib-core/src/main/java/com/google/cloud/tools/jib/blob/WritableContentsBlob.java From e4461b4a0f3e38fcd94e00d116fd06874f1f2b17 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Wed, 1 May 2019 17:10:37 -0400 Subject: [PATCH 23/39] Remove more Blob indirection --- .../google/cloud/tools/jib/blob/Blobs.java | 31 ------------------- .../tools/jib/cache/CacheStorageWriter.java | 6 +++- .../google/cloud/tools/jib/blob/BlobTest.java | 30 ------------------ .../jib/cache/CacheStorageReaderTest.java | 9 ++++-- 4 files changed, 11 insertions(+), 65 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java index cb621b232f..643572f529 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java @@ -16,15 +16,12 @@ package com.google.cloud.tools.jib.blob; -import com.google.cloud.tools.jib.filesystem.FileOperations; import com.google.cloud.tools.jib.hash.WritableContents; import com.google.cloud.tools.jib.json.JsonTemplate; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.nio.file.Path; /** Static methods for {@link Blob}. */ @@ -80,33 +77,5 @@ public static byte[] writeToByteArray(Blob blob) throws IOException { return byteArrayOutputStream.toByteArray(); } - /** - * Writes the BLOB to a file. - * - * @param blob the BLOB to to write - * @param blobFile the file to write to - * @return the {@link BlobDescriptor} of the written BLOB - * @throws IOException if writing the BLOB fails - */ - public static BlobDescriptor writeToFile(Blob blob, Path blobFile) throws IOException { - try (OutputStream outputStream = Files.newOutputStream(blobFile)) { - return blob.writeTo(outputStream); - } - } - - /** - * Writes the BLOB to a file with an exclusive lock. - * - * @param blob the BLOB to to write - * @param blobFile the file to write to - * @return the {@link BlobDescriptor} of the written BLOB - * @throws IOException if writing the BLOB fails - */ - public static BlobDescriptor writeToFileWithLock(Blob blob, Path blobFile) throws IOException { - try (OutputStream outputStream = FileOperations.newLockingOutputStream(blobFile)) { - return blob.writeTo(outputStream); - } - } - private Blobs() {} } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java index 7a55c63a25..97848c00d1 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java @@ -19,6 +19,7 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.blob.Blobs; +import com.google.cloud.tools.jib.filesystem.FileOperations; import com.google.cloud.tools.jib.filesystem.LockFile; import com.google.cloud.tools.jib.filesystem.TemporaryDirectory; import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; @@ -36,6 +37,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.FileSystemException; import java.nio.file.Files; @@ -364,7 +366,9 @@ private void writeSelector(DescriptorDigest selector, DescriptorDigest layerDige // Writes the selector to a temporary file and then moves the file to the intended location. Path temporarySelectorFile = Files.createTempFile(null, null); temporarySelectorFile.toFile().deleteOnExit(); - Blobs.writeToFileWithLock(Blobs.from(layerDigest.getHash()), temporarySelectorFile); + try (OutputStream fileOut = FileOperations.newLockingOutputStream(temporarySelectorFile)) { + fileOut.write(layerDigest.getHash().getBytes(StandardCharsets.UTF_8)); + } // Attempts an atomic move first, and falls back to non-atomic if the file system does not // support atomic moves. diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java index 721b36b257..edc0431433 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java @@ -30,7 +30,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; import org.junit.Assert; import org.junit.Test; @@ -67,35 +66,6 @@ public void testFromWritableContents() throws IOException { verifyBlobWriteTo(expected, Blobs.from(writableContents)); } - @Test - public void testWriteToFileWithLock_newFile() throws IOException { - String expected = "crepecake"; - Path blobFile = Files.createTempFile("blob", "bin"); - Assert.assertTrue(Files.deleteIfExists(blobFile)); // ensure it doesn't exist - - Blobs.writeToFileWithLock(Blobs.from(expected), blobFile); - - Assert.assertTrue(Files.exists(blobFile)); - verifyBlobWriteTo(expected, Blobs.from(blobFile)); - } - - @Test - public void testWriteToFileWithLock_existingFile() throws IOException { - Path blobFile = Files.createTempFile("blob", "bin"); - // write out more bytes to ensure properly truncated - byte[] dataBytes = new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - Files.write(blobFile, dataBytes, StandardOpenOption.WRITE); - Assert.assertTrue(Files.exists(blobFile)); - Assert.assertEquals(10, Files.size(blobFile)); - - String expected = "crepecake"; - Blobs.writeToFileWithLock(Blobs.from(expected), blobFile); - - Assert.assertTrue(Files.exists(blobFile)); - Assert.assertEquals(9, Files.size(blobFile)); - verifyBlobWriteTo(expected, Blobs.from(blobFile)); - } - /** Checks that the {@link Blob} streams the expected string. */ private void verifyBlobWriteTo(String expected, Blob blob) throws IOException { OutputStream outputStream = new ByteArrayOutputStream(); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java index f3205759ce..2d74e6c391 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java @@ -16,10 +16,11 @@ package com.google.cloud.tools.jib.cache; -import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.image.DescriptorDigest; import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.security.DigestException; @@ -89,9 +90,11 @@ public void testRetrieve() throws IOException, CacheCorruptedException { // Creates the test layer directory. DescriptorDigest layerDigest = layerDigest1; DescriptorDigest layerDiffId = layerDigest2; - Blob layerBlob = Blobs.from("layerBlob"); Files.createDirectories(cacheStorageFiles.getLayerDirectory(layerDigest)); - Blobs.writeToFileWithLock(layerBlob, cacheStorageFiles.getLayerFile(layerDigest, layerDiffId)); + try (OutputStream out = + Files.newOutputStream(cacheStorageFiles.getLayerFile(layerDigest, layerDiffId))) { + out.write("layerBlob".getBytes(StandardCharsets.UTF_8)); + } // Checks that the CachedLayer is retrieved correctly. Optional optionalCachedLayer = cacheStorageReader.retrieve(layerDigest); From 40f66662bf42e61ba7dec9a33206055e1da01935 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Wed, 1 May 2019 19:07:08 -0400 Subject: [PATCH 24/39] Rename --- ...patcherContainer.java => ProgressCounter.java} | 15 +++++++-------- .../steps/PullAndCacheBaseImageLayerStep.java | 8 ++++---- .../jib/builder/steps/PullBaseImageStep.java | 8 ++++---- 3 files changed, 15 insertions(+), 16 deletions(-) rename jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/{ProgressEventDispatcherContainer.java => ProgressCounter.java} (82%) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ProgressEventDispatcherContainer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ProgressCounter.java similarity index 82% rename from jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ProgressEventDispatcherContainer.java rename to jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ProgressCounter.java index ae73314ea8..e04e9bebe4 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ProgressEventDispatcherContainer.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ProgressCounter.java @@ -20,7 +20,6 @@ import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.common.base.Preconditions; import java.io.Closeable; -import java.util.function.Consumer; import javax.annotation.Nullable; /** @@ -31,14 +30,14 @@ * response headers are received, only after which can the {@link ProgressEventDispatcher} be * created. */ -class ProgressEventDispatcherContainer implements Consumer, Closeable { +class ProgressEventDispatcherWrapper implements Closeable { private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; private final String description; private final BuildStepType type; @Nullable private ProgressEventDispatcher progressEventDispatcher; - ProgressEventDispatcherContainer( + ProgressEventDispatcherWrapper( ProgressEventDispatcher.Factory progressEventDispatcherFactory, String description, BuildStepType type) { @@ -47,10 +46,9 @@ class ProgressEventDispatcherContainer implements Consumer, Closeable { this.type = type; } - @Override - public void accept(Long byteCount) { + void dispatchProgress(long progressUnits) { Preconditions.checkNotNull(progressEventDispatcher); - progressEventDispatcher.dispatchProgress(byteCount); + progressEventDispatcher.dispatchProgress(progressUnits); } @Override @@ -59,8 +57,9 @@ public void close() { progressEventDispatcher.close(); } - void initializeWithBlobSize(long blobSize) { + void setProgressTarget(long allocationUnits) { Preconditions.checkState(progressEventDispatcher == null); - progressEventDispatcher = progressEventDispatcherFactory.create(type, description, blobSize); + progressEventDispatcher = + progressEventDispatcherFactory.create(type, description, allocationUnits); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java index 9c10f3060d..3cd884b320 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java @@ -90,16 +90,16 @@ public CachedLayer call() throws IOException, CacheCorruptedException { .setAuthorization(pullAuthorization) .newRegistryClient(); - try (ProgressEventDispatcherContainer progressEventDispatcherContainer = - new ProgressEventDispatcherContainer( + try (ProgressEventDispatcherWrapper progressEventDispatcherWrapper = + new ProgressEventDispatcherWrapper( progressEventDispatcher.newChildProducer(), "pulling base image layer " + layerDigest, BuildStepType.PULL_AND_CACHE_BASE_IMAGE_LAYER)) { return cache.writeCompressedLayer( registryClient.pullBlob( layerDigest, - progressEventDispatcherContainer::initializeWithBlobSize, - progressEventDispatcherContainer)); + progressEventDispatcherWrapper::setProgressTarget, + progressEventDispatcherWrapper::dispatchProgress)); } } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java index e45c6783fd..fb7fabab05 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java @@ -242,8 +242,8 @@ private Image pullBaseImage( DescriptorDigest containerConfigurationDigest = v22ManifestTemplate.getContainerConfiguration().getDigest(); - try (ProgressEventDispatcherContainer progressEventDispatcherContainer = - new ProgressEventDispatcherContainer( + try (ProgressEventDispatcherWrapper progressEventDispatcherWrapper = + new ProgressEventDispatcherWrapper( progressEventDispatcher.newChildProducer(), "pull container configuration " + containerConfigurationDigest, BuildStepType.PULL_BASE_IMAGE)) { @@ -251,8 +251,8 @@ private Image pullBaseImage( Blobs.writeToString( registryClient.pullBlob( containerConfigurationDigest, - progressEventDispatcherContainer::initializeWithBlobSize, - progressEventDispatcherContainer)); + progressEventDispatcherWrapper::setProgressTarget, + progressEventDispatcherWrapper::dispatchProgress)); ContainerConfigurationTemplate containerConfigurationTemplate = JsonTemplateMapper.readJson( From d6f02daa6e617c19674af56eee9c5c2517794c83 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Thu, 2 May 2019 10:04:42 -0400 Subject: [PATCH 25/39] Rename --- .../{ProgressCounter.java => ProgressEventDispatcherWrapper.java} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/{ProgressCounter.java => ProgressEventDispatcherWrapper.java} (100%) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ProgressCounter.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ProgressEventDispatcherWrapper.java similarity index 100% rename from jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ProgressCounter.java rename to jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ProgressEventDispatcherWrapper.java From ea31cfc5139860994125d00d6900b6afc103f667 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Mon, 6 May 2019 19:22:13 -0400 Subject: [PATCH 26/39] private --- .../java/com/google/cloud/tools/jib/http/Authorization.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/http/Authorization.java b/jib-core/src/main/java/com/google/cloud/tools/jib/http/Authorization.java index 17e7bb9f20..a7dfc8e8e7 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/http/Authorization.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/http/Authorization.java @@ -59,7 +59,7 @@ public static Authorization withBasicToken(String token) { private final String scheme; private final String token; - Authorization(String scheme, String token) { + private Authorization(String scheme, String token) { this.scheme = scheme; this.token = token; } From f27ff67a61d5847ffc736f0b82b7ab23a255902b Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 7 May 2019 10:35:09 -0400 Subject: [PATCH 27/39] Refactor --- .../tools/jib/builder/steps/BuildResult.java | 5 +-- .../steps/PushContainerConfigurationStep.java | 14 ++++---- .../tools/jib/cache/CacheStorageWriter.java | 7 ++-- .../cloud/tools/jib/docker/ImageTarball.java | 17 +++++---- .../jib/hash/CountingDigestOutputStream.java | 36 +++++-------------- .../cloud/tools/jib/hash/DigestUtil.java | 2 +- .../google/cloud/tools/jib/hash/Digests.java | 6 +++- .../jib/image/json/ImageToJsonTranslator.java | 7 ++-- .../tools/jib/json/JsonTemplateMapper.java | 18 +++++++++- .../jib/cache/CacheStorageReaderTest.java | 4 +-- .../cloud/tools/jib/cache/CacheTest.java | 12 ++----- .../hash/CountingDigestOutputStreamTest.java | 7 ++-- .../image/json/ImageToJsonTranslatorTest.java | 17 ++++----- .../tools/jib/registry/BlobPullerTest.java | 2 +- 14 files changed, 70 insertions(+), 84 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildResult.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildResult.java index 836cbe1cc2..aa2c2c6032 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildResult.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildResult.java @@ -22,7 +22,6 @@ import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.json.BuildableManifestTemplate; import com.google.cloud.tools.jib.image.json.ImageToJsonTranslator; -import com.google.common.io.ByteStreams; import java.io.IOException; import java.util.Objects; @@ -41,9 +40,7 @@ static BuildResult fromImage(Image image, Class te return computeDigest(contents, ByteStreams.nullOutputStream()).getDigest(); } + public static BlobDescriptor computeDigest(JsonTemplate template) throws IOException { + return computeDigest(template, ByteStreams.nullOutputStream()); + } + public static BlobDescriptor computeDigest(JsonTemplate template, OutputStream optionalOutStream) throws IOException { WritableContents contents = contentsOut -> JsonTemplateMapper.writeTo(template, contentsOut); @@ -97,6 +101,6 @@ public static BlobDescriptor computeDigest( CountingDigestOutputStream digestOutStream = new CountingDigestOutputStream(optionalOutStream); contents.writeTo(digestOutStream); digestOutStream.flush(); - return new BlobDescriptor(digestOutStream.getBytesHahsed(), digestOutStream.getDigest()); + return digestOutStream.computeDigest(); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java index c0a07bd298..a8e1b7ead1 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java @@ -18,13 +18,13 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; -import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.configuration.DockerHealthCheck; import com.google.cloud.tools.jib.configuration.Port; import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.Layer; +import com.google.cloud.tools.jib.json.JsonTemplate; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -145,7 +145,7 @@ public ImageToJsonTranslator(Image image) { * * @return the container configuration {@link Blob} */ - public Blob getContainerConfigurationBlob() { + public JsonTemplate getContainerConfiguration() { // Set up the JSON template. ContainerConfigurationTemplate template = new ContainerConfigurationTemplate(); @@ -188,8 +188,7 @@ public Blob getContainerConfigurationBlob() { template.setContainerHealthCheckRetries(healthCheck.getRetries().orElse(null)); } - // Serializes into JSON. - return Blobs.from(template); + return template; } /** diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java b/jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java index c5f1c5f501..fe462059f6 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java @@ -26,6 +26,7 @@ import java.io.OutputStream; import java.nio.channels.Channels; import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; @@ -129,6 +130,16 @@ public static String toUtf8String(List templates) return toUtf8String((Object) templates); } + public static byte[] toByteArray(JsonTemplate template) + throws JsonGenerationException, JsonMappingException, IOException { + return toByteArray((Object) template); + } + + public static byte[] toByteArray(List templates) + throws JsonGenerationException, JsonMappingException, IOException { + return toByteArray((Object) templates); + } + public static void writeTo(JsonTemplate template, OutputStream out) throws JsonGenerationException, JsonMappingException, IOException { writeTo((Object) template, out); @@ -141,9 +152,14 @@ public static void writeTo(List templates, OutputStream private static String toUtf8String(Object template) throws JsonGenerationException, JsonMappingException, IOException { + return new String(toByteArray(template), StandardCharsets.UTF_8); + } + + private static byte[] toByteArray(Object template) + throws JsonGenerationException, JsonMappingException, IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); writeTo(template, out); - return out.toString("UTF-8"); + return out.toByteArray(); } private static void writeTo(Object template, OutputStream out) diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java index 25b0345605..709d116b9b 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java @@ -215,7 +215,7 @@ public void testSelect_invalidLayerDigest() throws IOException { DescriptorDigest selector = layerDigest1; Path selectorFile = cacheStorageFiles.getSelectorFile(selector); Files.createDirectories(selectorFile.getParent()); - Files.write(selectorFile, Blobs.writeToByteArray(Blobs.from("not a valid layer digest"))); + Files.write(selectorFile, "not a valid layer digest".getBytes()); try { cacheStorageReader.select(selector); @@ -241,7 +241,7 @@ public void testSelect() throws IOException, CacheCorruptedException { DescriptorDigest selector = layerDigest1; Path selectorFile = cacheStorageFiles.getSelectorFile(selector); Files.createDirectories(selectorFile.getParent()); - Files.write(selectorFile, Blobs.writeToByteArray(Blobs.from(layerDigest2.getHash()))); + Files.write(selectorFile, layerDigest2.getHash().getBytes()); Optional selectedLayerDigest = cacheStorageReader.select(selector); Assert.assertTrue(selectedLayerDigest.isPresent()); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java index 16d3c1f0ab..9ab7f9e0eb 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java @@ -62,16 +62,10 @@ private static Blob compress(Blob blob) { * * @param blob the {@link Blob} to decompress * @return the decompressed {@link Blob} + * @throws IOException if an I/O exception occurs */ - private static Blob decompress(Blob blob) { - return Blobs.from( - outputStream -> { - ByteArrayInputStream compressedInputStream = - new ByteArrayInputStream(Blobs.writeToByteArray(blob)); - try (GZIPInputStream decompressorStream = new GZIPInputStream(compressedInputStream)) { - ByteStreams.copy(decompressorStream, outputStream); - } - }); + private static Blob decompress(Blob blob) throws IOException { + return Blobs.from(new GZIPInputStream(new ByteArrayInputStream(Blobs.writeToByteArray(blob)))); } /** diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStreamTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStreamTest.java index b4901f2615..d58fad282d 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStreamTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStreamTest.java @@ -16,6 +16,7 @@ package com.google.cloud.tools.jib.hash; +import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteStreams; @@ -56,9 +57,9 @@ public void test_smokeTest() throws IOException, DigestException { InputStream toHashInputStream = new ByteArrayInputStream(bytesToHash); ByteStreams.copy(toHashInputStream, countingDigestOutputStream); - Assert.assertEquals( - DescriptorDigest.fromHash(expectedHash), countingDigestOutputStream.getDigest()); - Assert.assertEquals(bytesToHash.length, countingDigestOutputStream.getBytesHahsed()); + BlobDescriptor blobDescriptor = countingDigestOutputStream.computeDigest(); + Assert.assertEquals(DescriptorDigest.fromHash(expectedHash), blobDescriptor.getDigest()); + Assert.assertEquals(bytesToHash.length, blobDescriptor.getSize()); } } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslatorTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslatorTest.java index 51045a449a..e2d5584062 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslatorTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslatorTest.java @@ -22,18 +22,18 @@ import com.google.cloud.tools.jib.configuration.DockerHealthCheck; import com.google.cloud.tools.jib.configuration.Port; import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; +import com.google.cloud.tools.jib.hash.Digests; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.Layer; import com.google.cloud.tools.jib.image.LayerPropertyNotFoundException; +import com.google.cloud.tools.jib.json.JsonTemplate; import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedMap; -import com.google.common.io.ByteStreams; import com.google.common.io.Resources; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; @@ -127,13 +127,9 @@ public void testGetContainerConfiguration() String expectedJson = new String(Files.readAllBytes(jsonFile), StandardCharsets.UTF_8); // Translates the image to the container configuration and writes the JSON string. - Blob containerConfigurationBlob = imageToJsonTranslator.getContainerConfigurationBlob(); + JsonTemplate containerConfiguration = imageToJsonTranslator.getContainerConfiguration(); - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - containerConfigurationBlob.writeTo(byteArrayOutputStream); - - Assert.assertEquals( - expectedJson, new String(byteArrayOutputStream.toByteArray(), StandardCharsets.UTF_8)); + Assert.assertEquals(expectedJson, JsonTemplateMapper.toUtf8String(containerConfiguration)); } @Test @@ -184,9 +180,8 @@ private void testGetManifest( String expectedJson = new String(Files.readAllBytes(jsonFile), StandardCharsets.UTF_8); // Translates the image to the manifest and writes the JSON string. - Blob containerConfigurationBlob = imageToJsonTranslator.getContainerConfigurationBlob(); - BlobDescriptor blobDescriptor = - containerConfigurationBlob.writeTo(ByteStreams.nullOutputStream()); + JsonTemplate containerConfiguration = imageToJsonTranslator.getContainerConfiguration(); + BlobDescriptor blobDescriptor = Digests.computeDigest(containerConfiguration); T manifestTemplate = imageToJsonTranslator.getManifestTemplate(manifestTemplateClass, blobDescriptor); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java index 242219a2b6..ab357d7504 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java @@ -88,7 +88,7 @@ public void testHandleResponse() throws IOException, UnexpectedBlobDigestExcepti Assert.assertEquals( "some BLOB content", new String(layerContentOutputStream.toByteArray(), StandardCharsets.UTF_8)); - Assert.assertEquals(testBlobDigest, layerOutputStream.getDigest()); + Assert.assertEquals(testBlobDigest, layerOutputStream.computeDigest().getDigest()); Assert.assertEquals("some BLOB content".length(), byteCount.sum()); } From f2be62e258b1de6c5cc8b2fed3803d4539c8f2f8 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 7 May 2019 11:00:15 -0400 Subject: [PATCH 28/39] Clean up --- .../cloud/tools/jib/hash/DigestUtil.java | 102 ------------------ .../cloud/tools/jib/registry/BlobPuller.java | 4 +- .../jib/cache/CacheStorageReaderTest.java | 4 +- .../tools/jib/registry/BlobPullerTest.java | 6 +- 4 files changed, 7 insertions(+), 109 deletions(-) delete mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java deleted file mode 100644 index 4fbea344c6..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2017 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.hash; - -import com.google.cloud.tools.jib.blob.BlobDescriptor; -import com.google.cloud.tools.jib.image.DescriptorDigest; -import com.google.cloud.tools.jib.json.JsonTemplate; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; -import com.google.common.io.ByteStreams; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.List; - -/** - * Utility class for computing a digest for various inputs while optionally writing to an output - * stream. - */ -// Note: intentionally this class does not depend on Blob, as Blob classes depend on this class. -// TODO: BlobDescriptor is merely a tuple of (size, digest). Rename BlobDescriptor to something -// more general. -public class DigestUtil { - - public static DescriptorDigest computeJsonDigest(JsonTemplate template) throws IOException { - return computeDigest(template, ByteStreams.nullOutputStream()).getDigest(); - } - - public static DescriptorDigest computeJsonDigest(List templates) - throws IOException { - WritableContents contents = contentsOut -> JsonTemplateMapper.writeTo(templates, contentsOut); - return computeDigest(contents, ByteStreams.nullOutputStream()).getDigest(); - } - - public static BlobDescriptor computeDigest(JsonTemplate template, OutputStream optionalOutStream) - throws IOException { - WritableContents contents = contentsOut -> JsonTemplateMapper.writeTo(template, contentsOut); - return computeDigest(contents, optionalOutStream); - } - - public static BlobDescriptor computeDigest(InputStream inStream) throws IOException { - return computeDigest(inStream, ByteStreams.nullOutputStream()); - } - - /** - * Computes the digest by consuming the contents. - * - * @param contents the contents for which the digest is computed - * @return computed digest and bytes consumed - * @throws IOException if reading fails - */ - public static BlobDescriptor computeDigest(WritableContents contents) throws IOException { - return computeDigest(contents, ByteStreams.nullOutputStream()); - } - - /** - * Computes the digest by consuming the contents of an {@link InputStream} and optionally copying - * it to an {@link OutputStream}. Returns the computed digest along with the size of the bytes - * consumed to compute the digest. Does not close either stream. - * - * @param inStream the stream to read the contents from - * @param optionalOutStream the stream to which the contents are copied - * @return computed digest and bytes consumed - * @throws IOException if reading from or writing fails - */ - public static BlobDescriptor computeDigest(InputStream inStream, OutputStream optionalOutStream) - throws IOException { - WritableContents contents = contentsOut -> ByteStreams.copy(inStream, contentsOut); - return computeDigest(contents, optionalOutStream); - } - - /** - * Computes the digest by consuming the contents and optionally copying it to an {@link - * OutputStream}. Returns the computed digest along with the size of the bytes consumed to compute - * the digest. Does not close the stream. - * - * @param contents the contents to compute digest for - * @param optionalOutStream the stream to which the contents are copied - * @return computed digest and bytes consumed - * @throws IOException if reading from or writing fails - */ - public static BlobDescriptor computeDigest( - WritableContents contents, OutputStream optionalOutStream) throws IOException { - CountingDigestOutputStream digestOutStream = new CountingDigestOutputStream(optionalOutStream); - contents.writeTo(digestOutStream); - digestOutStream.flush(); - return digestOutStream.computeDigest(); - } -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java index 8cd8a949f2..5a5e9a0d43 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java @@ -19,7 +19,7 @@ import com.google.api.client.http.HttpMethods; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.event.progress.DelayedConsumer; -import com.google.cloud.tools.jib.hash.DigestUtil; +import com.google.cloud.tools.jib.hash.Digests; import com.google.cloud.tools.jib.http.BlobHttpContent; import com.google.cloud.tools.jib.http.ListenableCountingOutputStream; import com.google.cloud.tools.jib.http.Response; @@ -71,7 +71,7 @@ public Void handleResponse(Response response) throws IOException, UnexpectedBlob OutputStream outputStream = new ListenableCountingOutputStream(destinationOutputStream, delayedCountListener)) { BlobDescriptor receivedBlobDescriptor = - DigestUtil.computeDigest(response.getBody(), outputStream); + Digests.computeDigest(response.getBody(), outputStream); if (!blobDigest.equals(receivedBlobDescriptor.getDigest())) { throw new UnexpectedBlobDigestException( diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java index 709d116b9b..1d3ecdc058 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java @@ -215,7 +215,7 @@ public void testSelect_invalidLayerDigest() throws IOException { DescriptorDigest selector = layerDigest1; Path selectorFile = cacheStorageFiles.getSelectorFile(selector); Files.createDirectories(selectorFile.getParent()); - Files.write(selectorFile, "not a valid layer digest".getBytes()); + Files.write(selectorFile, "not a valid layer digest".getBytes(StandardCharsets.UTF_8)); try { cacheStorageReader.select(selector); @@ -241,7 +241,7 @@ public void testSelect() throws IOException, CacheCorruptedException { DescriptorDigest selector = layerDigest1; Path selectorFile = cacheStorageFiles.getSelectorFile(selector); Files.createDirectories(selectorFile.getParent()); - Files.write(selectorFile, layerDigest2.getHash().getBytes()); + Files.write(selectorFile, layerDigest2.getHash().getBytes(StandardCharsets.UTF_8)); Optional selectedLayerDigest = cacheStorageReader.select(selector); Assert.assertTrue(selectedLayerDigest.isPresent()); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java index ab357d7504..7e655c6d09 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java @@ -17,7 +17,7 @@ package com.google.cloud.tools.jib.registry; import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; -import com.google.cloud.tools.jib.hash.DigestUtil; +import com.google.cloud.tools.jib.hash.Digests; import com.google.cloud.tools.jib.http.Response; import com.google.cloud.tools.jib.image.DescriptorDigest; import java.io.ByteArrayInputStream; @@ -69,7 +69,7 @@ public void setUpFakes() throws DigestException { public void testHandleResponse() throws IOException, UnexpectedBlobDigestException { InputStream blobContent = new ByteArrayInputStream("some BLOB content".getBytes(StandardCharsets.UTF_8)); - DescriptorDigest testBlobDigest = DigestUtil.computeDigest(blobContent).getDigest(); + DescriptorDigest testBlobDigest = Digests.computeDigest(blobContent).getDigest(); blobContent.reset(); Response mockResponse = Mockito.mock(Response.class); @@ -96,7 +96,7 @@ public void testHandleResponse() throws IOException, UnexpectedBlobDigestExcepti public void testHandleResponse_unexpectedDigest() throws IOException { InputStream blobContent = new ByteArrayInputStream("some BLOB content".getBytes(StandardCharsets.UTF_8)); - DescriptorDigest testBlobDigest = DigestUtil.computeDigest(blobContent).getDigest(); + DescriptorDigest testBlobDigest = Digests.computeDigest(blobContent).getDigest(); blobContent.reset(); Response mockResponse = Mockito.mock(Response.class); From e75f8e0eb11a31feaf87348c2a6f0858d9945bc3 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 7 May 2019 11:10:16 -0400 Subject: [PATCH 29/39] Fix Javadocs --- .../cloud/tools/jib/hash/CountingDigestOutputStream.java | 2 ++ .../cloud/tools/jib/image/json/ImageToJsonTranslator.java | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStream.java b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStream.java index d2df5d50af..46416540da 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStream.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStream.java @@ -51,6 +51,8 @@ public CountingDigestOutputStream(OutputStream outputStream) { * Computes the hash and returns it along with the size of the bytes written to compute the hash. * The buffer resets after this method is called, so this method should only be called once per * computation. + * + * @return the computed hash and the size of the bytes consumed */ public BlobDescriptor computeDigest() { try { diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java index a8e1b7ead1..3b069ac8ea 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java @@ -193,8 +193,8 @@ public JsonTemplate getContainerConfiguration() { /** * Gets the manifest as a JSON template. The {@code containerConfigurationBlobDescriptor} must be - * the [@link BlobDescriptor} obtained by writing out the container configuration {@link Blob} - * returned from {@link #getContainerConfigurationBlob()}. + * the {@link BlobDescriptor} obtained by writing out the container configuration JSON returned + * from {@link #getContainerConfiguration()}. * * @param child type of {@link BuildableManifestTemplate}. * @param manifestTemplateClass the JSON template to translate the image to. From 523b4f7744ebba9f63601bd7ec87c210b7e3d0a0 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 7 May 2019 11:22:11 -0400 Subject: [PATCH 30/39] Update Javadocs --- .../jib/event/progress/DelayedConsumer.java | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/DelayedConsumer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/DelayedConsumer.java index daba5e67d0..b9008a31cc 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/DelayedConsumer.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/DelayedConsumer.java @@ -26,27 +26,28 @@ import javax.annotation.Nullable; /** - * Wraps a {@link Consumer} so that multiple consume calls within a short time are blocked and - * delayed into a single call. + * Wraps a {@link Consumer} so that multiple consume calls ({@link #accept}) within a short period + * of time are merged into a single later call. */ public class DelayedConsumer implements Consumer, Closeable { private final Consumer consumer; - /** Delay between each call to {@link #byteCountConsumer}. */ + /** Delay between each call to the underlying {@link #accept}. */ private final Duration delayBetweenCallbacks; - /** - * Binary operator that will "add up" multiple delayed values. Used to accumulate past values in - * case delays happen so that callback is called once with the "added" value after the delay. - */ - private final BinaryOperator adder; + /** Last time the underlying {@link #accept} was called. */ + private Instant previousCallback; - /** Returns the current {@link Instant}. */ + /** "Clock" that returns the current {@link Instant}. */ private final Supplier getNow; - /** Last time {@link #byteCountConsumer} was called. */ - private Instant previousCallback; + /** + * Binary operator to be used to merge ("add up") multiple delayed values. Used to accumulate past + * values in case delays happen so that callback is called once with the "added" value after the + * delay. + */ + private final BinaryOperator valueAdder; @Nullable private T valueSoFar; @@ -54,19 +55,19 @@ public class DelayedConsumer implements Consumer, Closeable { * Wraps a consumer with the delay of 100 ms. * * @param callback {@link Consumer} callback to wrap - * @param adder adds up multiple delayed values + * @param valueAdder merger to add up multiple delayed values */ - public DelayedConsumer(Consumer callback, BinaryOperator adder) { - this(callback, adder, Duration.ofMillis(100), Instant::now); + public DelayedConsumer(Consumer callback, BinaryOperator valueAdder) { + this(callback, valueAdder, Duration.ofMillis(100), Instant::now); } public DelayedConsumer( Consumer consumer, - BinaryOperator adder, + BinaryOperator valueAdder, Duration delayBetweenCallbacks, Supplier getNow) { this.consumer = consumer; - this.adder = adder; + this.valueAdder = valueAdder; this.delayBetweenCallbacks = delayBetweenCallbacks; this.getNow = getNow; @@ -75,7 +76,7 @@ public DelayedConsumer( @Override public void accept(T value) { - valueSoFar = valueSoFar == null ? value : adder.apply(valueSoFar, value); + valueSoFar = valueSoFar == null ? value : valueAdder.apply(valueSoFar, value); Instant now = getNow.get(); if (previousCallback.plus(delayBetweenCallbacks).isBefore(now)) { From c74330c82ae39a0406fbef202f58b868f762250a Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 7 May 2019 11:32:15 -0400 Subject: [PATCH 31/39] Rename --- .../java/com/google/cloud/tools/jib/http/BlobHttpContent.java | 2 +- .../java/com/google/cloud/tools/jib/registry/BlobPuller.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobHttpContent.java b/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobHttpContent.java index bf08a135fa..3caacca721 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobHttpContent.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobHttpContent.java @@ -59,7 +59,7 @@ public boolean retrySupported() { @Override public void writeTo(OutputStream outputStream) throws IOException { try (DelayedConsumer delayedCountListener = - new DelayedConsumer<>(writtenByteCountListener, (a, b) -> a + b)) { + new DelayedConsumer<>(writtenByteCountListener, (count1, count2) -> count1 + count2)) { blob.writeTo(new ListenableCountingOutputStream(outputStream, delayedCountListener)); outputStream.flush(); } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java index 5a5e9a0d43..b0dcfbdf49 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java @@ -67,7 +67,7 @@ public Void handleResponse(Response response) throws IOException, UnexpectedBlob blobSizeListener.accept(response.getContentLength()); try (DelayedConsumer delayedCountListener = - new DelayedConsumer<>(writtenByteCountListener, (a, b) -> a + b); + new DelayedConsumer<>(writtenByteCountListener, (count1, count2) -> count1 + count2); OutputStream outputStream = new ListenableCountingOutputStream(destinationOutputStream, delayedCountListener)) { BlobDescriptor receivedBlobDescriptor = From b8640afd2e6eaead3978832dac4924400b959634 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 14 May 2019 15:56:41 -0400 Subject: [PATCH 32/39] Push up throttling decision --- .../steps/PullAndCacheBaseImageLayerStep.java | 4 ++-- .../jib/builder/steps/PullBaseImageStep.java | 4 ++-- .../tools/jib/builder/steps/PushBlobStep.java | 9 +++++--- ...ottledProgressEventDispatcherWrapper.java} | 22 +++++++++++++------ ...edConsumer.java => ThrottledConsumer.java} | 12 +++++----- .../cloud/tools/jib/http/BlobHttpContent.java | 8 ++----- .../cloud/tools/jib/registry/BlobPuller.java | 7 ++---- .../ListenableCountingOutputStreamTest.java | 6 ++--- 8 files changed, 38 insertions(+), 34 deletions(-) rename jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/{ProgressEventDispatcherWrapper.java => ThrottledProgressEventDispatcherWrapper.java} (69%) rename jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/{DelayedConsumer.java => ThrottledConsumer.java} (89%) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java index 77e886db8c..eb1df521b2 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java @@ -95,8 +95,8 @@ public CachedLayer call() throws IOException, CacheCorruptedException { .setAuthorization(pullAuthorization) .newRegistryClient(); - try (ProgressEventDispatcherWrapper progressEventDispatcherWrapper = - new ProgressEventDispatcherWrapper( + try (ThrottledProgressEventDispatcherWrapper progressEventDispatcherWrapper = + new ThrottledProgressEventDispatcherWrapper( progressEventDispatcher.newChildProducer(), "pulling base image layer " + layerDigest, BuildStepType.PULL_AND_CACHE_BASE_IMAGE_LAYER)) { diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java index 85d078dc47..dd77943b65 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java @@ -253,8 +253,8 @@ private Image pullBaseImage( DescriptorDigest containerConfigurationDigest = buildableManifestTemplate.getContainerConfiguration().getDigest(); - try (ProgressEventDispatcherWrapper progressEventDispatcherWrapper = - new ProgressEventDispatcherWrapper( + try (ThrottledProgressEventDispatcherWrapper progressEventDispatcherWrapper = + new ThrottledProgressEventDispatcherWrapper( progressEventDispatcher.newChildProducer(), "pull container configuration " + containerConfigurationDigest, BuildStepType.PULL_BASE_IMAGE)) { diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java index 2ab6f7d9ff..b857597aa3 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java @@ -27,6 +27,7 @@ import com.google.cloud.tools.jib.builder.TimerEventDispatcher; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.event.events.LogEvent; +import com.google.cloud.tools.jib.event.progress.ThrottledConsumer; import com.google.cloud.tools.jib.registry.RegistryClient; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -84,7 +85,10 @@ public BlobDescriptor call() throws IOException, RegistryException, ExecutionExc blobDescriptor.getSize()); TimerEventDispatcher ignored = new TimerEventDispatcher( - buildConfiguration.getEventDispatcher(), DESCRIPTION + blobDescriptor)) { + buildConfiguration.getEventDispatcher(), DESCRIPTION + blobDescriptor); + ThrottledConsumer throttledProgressReporter = + new ThrottledConsumer<>( + progressEventDispatcher::dispatchProgress, (unit1, unit2) -> unit1 + unit2)) { RegistryClient registryClient = buildConfiguration .newTargetImageRegistryClientFactory() @@ -100,8 +104,7 @@ public BlobDescriptor call() throws IOException, RegistryException, ExecutionExc } // todo: leverage cross-repository mounts - registryClient.pushBlob( - blobDescriptor.getDigest(), blob, null, progressEventDispatcher::dispatchProgress); + registryClient.pushBlob(blobDescriptor.getDigest(), blob, null, throttledProgressReporter); return blobDescriptor; } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ProgressEventDispatcherWrapper.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ThrottledProgressEventDispatcherWrapper.java similarity index 69% rename from jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ProgressEventDispatcherWrapper.java rename to jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ThrottledProgressEventDispatcherWrapper.java index e04e9bebe4..690eba5893 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ProgressEventDispatcherWrapper.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ThrottledProgressEventDispatcherWrapper.java @@ -18,26 +18,29 @@ import com.google.cloud.tools.jib.builder.BuildStepType; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; +import com.google.cloud.tools.jib.event.progress.ThrottledConsumer; import com.google.common.base.Preconditions; import java.io.Closeable; import javax.annotation.Nullable; /** - * Contains a {@link ProgressEventDispatcher}. This class is mutable and should only be used within - * a local context. + * Contains a {@link ProgressEventDispatcher} and throttles dispatching progress events with the + * default delay used by {@link ThrottledConsumer}. This class is mutable and should only be used + * within a local context. * *

This class is necessary because the total BLOb size (allocation units) is not known until the * response headers are received, only after which can the {@link ProgressEventDispatcher} be * created. */ -class ProgressEventDispatcherWrapper implements Closeable { +class ThrottledProgressEventDispatcherWrapper implements Closeable { private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; private final String description; private final BuildStepType type; @Nullable private ProgressEventDispatcher progressEventDispatcher; + @Nullable private ThrottledConsumer throttledDispatcher; - ProgressEventDispatcherWrapper( + ThrottledProgressEventDispatcherWrapper( ProgressEventDispatcher.Factory progressEventDispatcherFactory, String description, BuildStepType type) { @@ -46,14 +49,16 @@ class ProgressEventDispatcherWrapper implements Closeable { this.type = type; } - void dispatchProgress(long progressUnits) { - Preconditions.checkNotNull(progressEventDispatcher); - progressEventDispatcher.dispatchProgress(progressUnits); + public void dispatchProgress(Long progressUnits) { + Preconditions.checkNotNull(throttledDispatcher); + throttledDispatcher.accept(progressUnits); } @Override public void close() { Preconditions.checkNotNull(progressEventDispatcher); + Preconditions.checkNotNull(throttledDispatcher); + throttledDispatcher.close(); progressEventDispatcher.close(); } @@ -61,5 +66,8 @@ void setProgressTarget(long allocationUnits) { Preconditions.checkState(progressEventDispatcher == null); progressEventDispatcher = progressEventDispatcherFactory.create(type, description, allocationUnits); + throttledDispatcher = + new ThrottledConsumer<>( + progressEventDispatcher::dispatchProgress, (unit1, unit2) -> unit1 + unit2); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/DelayedConsumer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/ThrottledConsumer.java similarity index 89% rename from jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/DelayedConsumer.java rename to jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/ThrottledConsumer.java index b9008a31cc..53433bc2ef 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/DelayedConsumer.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/ThrottledConsumer.java @@ -17,7 +17,6 @@ package com.google.cloud.tools.jib.event.progress; import java.io.Closeable; -import java.io.IOException; import java.time.Duration; import java.time.Instant; import java.util.function.BinaryOperator; @@ -29,7 +28,7 @@ * Wraps a {@link Consumer} so that multiple consume calls ({@link #accept}) within a short period * of time are merged into a single later call. */ -public class DelayedConsumer implements Consumer, Closeable { +public class ThrottledConsumer implements Consumer, Closeable { private final Consumer consumer; @@ -57,11 +56,11 @@ public class DelayedConsumer implements Consumer, Closeable { * @param callback {@link Consumer} callback to wrap * @param valueAdder merger to add up multiple delayed values */ - public DelayedConsumer(Consumer callback, BinaryOperator valueAdder) { + public ThrottledConsumer(Consumer callback, BinaryOperator valueAdder) { this(callback, valueAdder, Duration.ofMillis(100), Instant::now); } - public DelayedConsumer( + public ThrottledConsumer( Consumer consumer, BinaryOperator valueAdder, Duration delayBetweenCallbacks, @@ -79,7 +78,8 @@ public void accept(T value) { valueSoFar = valueSoFar == null ? value : valueAdder.apply(valueSoFar, value); Instant now = getNow.get(); - if (previousCallback.plus(delayBetweenCallbacks).isBefore(now)) { + Instant nextFireTime = previousCallback.plus(delayBetweenCallbacks); + if (now.isAfter(nextFireTime)) { consumer.accept(valueSoFar); previousCallback = now; valueSoFar = null; @@ -87,7 +87,7 @@ public void accept(T value) { } @Override - public void close() throws IOException { + public void close() { if (valueSoFar != null) { consumer.accept(valueSoFar); } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobHttpContent.java b/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobHttpContent.java index 3caacca721..f8d9e9f838 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobHttpContent.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobHttpContent.java @@ -18,7 +18,6 @@ import com.google.api.client.http.HttpContent; import com.google.cloud.tools.jib.blob.Blob; -import com.google.cloud.tools.jib.event.progress.DelayedConsumer; import java.io.IOException; import java.io.OutputStream; import java.util.function.Consumer; @@ -58,10 +57,7 @@ public boolean retrySupported() { @Override public void writeTo(OutputStream outputStream) throws IOException { - try (DelayedConsumer delayedCountListener = - new DelayedConsumer<>(writtenByteCountListener, (count1, count2) -> count1 + count2)) { - blob.writeTo(new ListenableCountingOutputStream(outputStream, delayedCountListener)); - outputStream.flush(); - } + blob.writeTo(new ListenableCountingOutputStream(outputStream, writtenByteCountListener)); + outputStream.flush(); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java index 13b0a2876d..aebe846e53 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java @@ -19,7 +19,6 @@ import com.google.api.client.http.HttpMethods; import com.google.cloud.tools.jib.api.DescriptorDigest; import com.google.cloud.tools.jib.blob.BlobDescriptor; -import com.google.cloud.tools.jib.event.progress.DelayedConsumer; import com.google.cloud.tools.jib.hash.Digests; import com.google.cloud.tools.jib.http.BlobHttpContent; import com.google.cloud.tools.jib.http.ListenableCountingOutputStream; @@ -66,10 +65,8 @@ class BlobPuller implements RegistryEndpointProvider { public Void handleResponse(Response response) throws IOException, UnexpectedBlobDigestException { blobSizeListener.accept(response.getContentLength()); - try (DelayedConsumer delayedCountListener = - new DelayedConsumer<>(writtenByteCountListener, (count1, count2) -> count1 + count2); - OutputStream outputStream = - new ListenableCountingOutputStream(destinationOutputStream, delayedCountListener)) { + try (OutputStream outputStream = + new ListenableCountingOutputStream(destinationOutputStream, writtenByteCountListener)) { BlobDescriptor receivedBlobDescriptor = Digests.computeDigest(response.getBody(), outputStream); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStreamTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStreamTest.java index 1fed293f6c..f7393d11cd 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStreamTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStreamTest.java @@ -16,7 +16,7 @@ package com.google.cloud.tools.jib.http; -import com.google.cloud.tools.jib.event.progress.DelayedConsumer; +import com.google.cloud.tools.jib.event.progress.ThrottledConsumer; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.time.Duration; @@ -58,8 +58,8 @@ public void testDelay() throws IOException { Queue instantQueue = new ArrayDeque<>(); instantQueue.add(Instant.EPOCH); - try (DelayedConsumer byteCounter = - new DelayedConsumer<>( + try (ThrottledConsumer byteCounter = + new ThrottledConsumer<>( byteCounts::add, (a, b) -> a + b, Duration.ofSeconds(3), instantQueue::remove); ListenableCountingOutputStream listenableCountingOutputStream = new ListenableCountingOutputStream(byteArrayOutputStream, byteCounter)) { From 3fba22b300fd7c4462691e0e7077b7ebbea63bab Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Wed, 15 May 2019 14:10:01 -0400 Subject: [PATCH 33/39] Rename "with..." to "from..." --- .../cloud/tools/jib/builder/steps/AuthenticatePushStep.java | 2 +- .../cloud/tools/jib/builder/steps/PullBaseImageStep.java | 2 +- .../java/com/google/cloud/tools/jib/http/Authorization.java | 6 +++--- .../cloud/tools/jib/registry/RegistryAuthenticator.java | 4 ++-- .../com/google/cloud/tools/jib/http/ConnectionTest.java | 2 +- .../tools/jib/registry/RegistryEndpointCallerTest.java | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java index a1f3da1f50..47e560c5cb 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java @@ -101,7 +101,7 @@ public Authorization call() return (registryCredential == null || registryCredential.isOAuth2RefreshToken()) ? null - : Authorization.withBasicCredentials( + : Authorization.fromBasicCredentials( registryCredential.getUsername(), registryCredential.getPassword()); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java index dd77943b65..d0b177339e 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java @@ -165,7 +165,7 @@ public BaseImageWithAuthorization call() Authorization registryAuthorization = registryCredential == null || registryCredential.isOAuth2RefreshToken() ? null - : Authorization.withBasicCredentials( + : Authorization.fromBasicCredentials( registryCredential.getUsername(), registryCredential.getPassword()); try { diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/http/Authorization.java b/jib-core/src/main/java/com/google/cloud/tools/jib/http/Authorization.java index a7dfc8e8e7..7c764f4f2f 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/http/Authorization.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/http/Authorization.java @@ -33,7 +33,7 @@ public class Authorization { * @param token the token * @return an {@link Authorization} with a {@code Bearer} token */ - public static Authorization withBearerToken(String token) { + public static Authorization fromBearerToken(String token) { return new Authorization("Bearer", token); } @@ -42,7 +42,7 @@ public static Authorization withBearerToken(String token) { * @param secret the secret * @return an {@link Authorization} with a {@code Basic} credentials */ - public static Authorization withBasicCredentials(String username, String secret) { + public static Authorization fromBasicCredentials(String username, String secret) { String credentials = username + ":" + secret; String token = Base64.encodeBase64String(credentials.getBytes(StandardCharsets.UTF_8)); return new Authorization("Basic", token); @@ -52,7 +52,7 @@ public static Authorization withBasicCredentials(String username, String secret) * @param token the token * @return an {@link Authorization} with a base64-encoded {@code username:password} string */ - public static Authorization withBasicToken(String token) { + public static Authorization fromBasicToken(String token) { return new Authorization("Basic", token); } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryAuthenticator.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryAuthenticator.java index 77cc994f9d..8530ef1280 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryAuthenticator.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryAuthenticator.java @@ -322,7 +322,7 @@ private Authorization authenticate(String scope) throws RegistryAuthenticationFa new BlobHttpContent(Blobs.from(parameters), MediaType.FORM_DATA.toString())); } else if (credential != null) { requestBuilder.setAuthorization( - Authorization.withBasicCredentials(credential.getUsername(), credential.getPassword())); + Authorization.fromBasicCredentials(credential.getUsername(), credential.getPassword())); } Request request = requestBuilder.build(); @@ -342,7 +342,7 @@ private Authorization authenticate(String scope) throws RegistryAuthenticationFa + "; parameters: " + getAuthRequestParameters(scope)); } - return Authorization.withBearerToken(responseJson.getToken()); + return Authorization.fromBearerToken(responseJson.getToken()); } catch (IOException ex) { throw new RegistryAuthenticationFailedException( diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/http/ConnectionTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/http/ConnectionTest.java index 9c880b5eca..a7e26c2eb2 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/http/ConnectionTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/http/ConnectionTest.java @@ -113,7 +113,7 @@ private void setUpMocksAndFakes(Integer httpTimeout) throws IOException { .setBody( new BlobHttpContent( Blobs.from("crepecake"), "fake.content.type", totalByteCount::add)) - .setAuthorization(Authorization.withBasicCredentials("fake-username", "fake-secret")) + .setAuthorization(Authorization.fromBasicCredentials("fake-username", "fake-secret")) .setHttpTimeout(httpTimeout) .build(); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/RegistryEndpointCallerTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/RegistryEndpointCallerTest.java index 5f6a90709f..642949eb3f 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/RegistryEndpointCallerTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/RegistryEndpointCallerTest.java @@ -643,7 +643,7 @@ private RegistryEndpointCaller createRegistryEndpointCaller( "userAgent", (port == -1) ? "apiRouteBase" : ("apiRouteBase:" + port), new TestRegistryEndpointProvider(), - Authorization.withBasicToken("token"), + Authorization.fromBasicToken("token"), new RegistryEndpointRequestProperties("serverUrl", "imageName"), allowInsecure, mockConnectionFactory, From 6f9ea9b38824b9dbfa68c0696b1e0128c81d2d0e Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Wed, 15 May 2019 18:08:03 -0400 Subject: [PATCH 34/39] Specialize ThrottledConsumer --- .../tools/jib/builder/steps/PushBlobStep.java | 7 ++-- ...rottledProgressEventDispatcherWrapper.java | 8 ++--- ...nsumer.java => ThrottledLongConsumer.java} | 34 ++++++------------- .../ListenableCountingOutputStreamTest.java | 8 ++--- 4 files changed, 21 insertions(+), 36 deletions(-) rename jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/{ThrottledConsumer.java => ThrottledLongConsumer.java} (63%) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java index b857597aa3..23f0753d73 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java @@ -27,7 +27,7 @@ import com.google.cloud.tools.jib.builder.TimerEventDispatcher; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.event.events.LogEvent; -import com.google.cloud.tools.jib.event.progress.ThrottledConsumer; +import com.google.cloud.tools.jib.event.progress.ThrottledLongConsumer; import com.google.cloud.tools.jib.registry.RegistryClient; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -86,9 +86,8 @@ public BlobDescriptor call() throws IOException, RegistryException, ExecutionExc TimerEventDispatcher ignored = new TimerEventDispatcher( buildConfiguration.getEventDispatcher(), DESCRIPTION + blobDescriptor); - ThrottledConsumer throttledProgressReporter = - new ThrottledConsumer<>( - progressEventDispatcher::dispatchProgress, (unit1, unit2) -> unit1 + unit2)) { + ThrottledLongConsumer throttledProgressReporter = + new ThrottledLongConsumer(progressEventDispatcher::dispatchProgress)) { RegistryClient registryClient = buildConfiguration .newTargetImageRegistryClientFactory() diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ThrottledProgressEventDispatcherWrapper.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ThrottledProgressEventDispatcherWrapper.java index 690eba5893..de975898e4 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ThrottledProgressEventDispatcherWrapper.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ThrottledProgressEventDispatcherWrapper.java @@ -18,7 +18,7 @@ import com.google.cloud.tools.jib.builder.BuildStepType; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; -import com.google.cloud.tools.jib.event.progress.ThrottledConsumer; +import com.google.cloud.tools.jib.event.progress.ThrottledLongConsumer; import com.google.common.base.Preconditions; import java.io.Closeable; import javax.annotation.Nullable; @@ -38,7 +38,7 @@ class ThrottledProgressEventDispatcherWrapper implements Closeable { private final String description; private final BuildStepType type; @Nullable private ProgressEventDispatcher progressEventDispatcher; - @Nullable private ThrottledConsumer throttledDispatcher; + @Nullable private ThrottledLongConsumer throttledDispatcher; ThrottledProgressEventDispatcherWrapper( ProgressEventDispatcher.Factory progressEventDispatcherFactory, @@ -66,8 +66,6 @@ void setProgressTarget(long allocationUnits) { Preconditions.checkState(progressEventDispatcher == null); progressEventDispatcher = progressEventDispatcherFactory.create(type, description, allocationUnits); - throttledDispatcher = - new ThrottledConsumer<>( - progressEventDispatcher::dispatchProgress, (unit1, unit2) -> unit1 + unit2); + throttledDispatcher = new ThrottledLongConsumer(progressEventDispatcher::dispatchProgress); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/ThrottledConsumer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/ThrottledLongConsumer.java similarity index 63% rename from jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/ThrottledConsumer.java rename to jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/ThrottledLongConsumer.java index 53433bc2ef..afcb4aeb35 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/ThrottledConsumer.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/ThrottledLongConsumer.java @@ -19,18 +19,17 @@ import java.io.Closeable; import java.time.Duration; import java.time.Instant; -import java.util.function.BinaryOperator; import java.util.function.Consumer; import java.util.function.Supplier; import javax.annotation.Nullable; /** - * Wraps a {@link Consumer} so that multiple consume calls ({@link #accept}) within a short period - * of time are merged into a single later call. + * Wraps a {@link Consumer} so that multiple consume calls ({@link #accept}) within a short + * period of time are merged into a single later call with the value accumulated up to that point. */ -public class ThrottledConsumer implements Consumer, Closeable { +public class ThrottledLongConsumer implements Consumer, Closeable { - private final Consumer consumer; + private final Consumer consumer; /** Delay between each call to the underlying {@link #accept}. */ private final Duration delayBetweenCallbacks; @@ -41,14 +40,7 @@ public class ThrottledConsumer implements Consumer, Closeable { /** "Clock" that returns the current {@link Instant}. */ private final Supplier getNow; - /** - * Binary operator to be used to merge ("add up") multiple delayed values. Used to accumulate past - * values in case delays happen so that callback is called once with the "added" value after the - * delay. - */ - private final BinaryOperator valueAdder; - - @Nullable private T valueSoFar; + @Nullable private Long valueSoFar; /** * Wraps a consumer with the delay of 100 ms. @@ -56,17 +48,13 @@ public class ThrottledConsumer implements Consumer, Closeable { * @param callback {@link Consumer} callback to wrap * @param valueAdder merger to add up multiple delayed values */ - public ThrottledConsumer(Consumer callback, BinaryOperator valueAdder) { - this(callback, valueAdder, Duration.ofMillis(100), Instant::now); + public ThrottledLongConsumer(Consumer callback) { + this(callback, Duration.ofMillis(100), Instant::now); } - public ThrottledConsumer( - Consumer consumer, - BinaryOperator valueAdder, - Duration delayBetweenCallbacks, - Supplier getNow) { + public ThrottledLongConsumer( + Consumer consumer, Duration delayBetweenCallbacks, Supplier getNow) { this.consumer = consumer; - this.valueAdder = valueAdder; this.delayBetweenCallbacks = delayBetweenCallbacks; this.getNow = getNow; @@ -74,8 +62,8 @@ public ThrottledConsumer( } @Override - public void accept(T value) { - valueSoFar = valueSoFar == null ? value : valueAdder.apply(valueSoFar, value); + public void accept(Long value) { + valueSoFar = valueSoFar == null ? value : valueSoFar + value; Instant now = getNow.get(); Instant nextFireTime = previousCallback.plus(delayBetweenCallbacks); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStreamTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStreamTest.java index f7393d11cd..9ca2af5030 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStreamTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStreamTest.java @@ -16,7 +16,7 @@ package com.google.cloud.tools.jib.http; -import com.google.cloud.tools.jib.event.progress.ThrottledConsumer; +import com.google.cloud.tools.jib.event.progress.ThrottledLongConsumer; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.time.Duration; @@ -58,9 +58,9 @@ public void testDelay() throws IOException { Queue instantQueue = new ArrayDeque<>(); instantQueue.add(Instant.EPOCH); - try (ThrottledConsumer byteCounter = - new ThrottledConsumer<>( - byteCounts::add, (a, b) -> a + b, Duration.ofSeconds(3), instantQueue::remove); + try (ThrottledLongConsumer byteCounter = + new ThrottledLongConsumer( + byteCounts::add, Duration.ofSeconds(3), instantQueue::remove); ListenableCountingOutputStream listenableCountingOutputStream = new ListenableCountingOutputStream(byteArrayOutputStream, byteCounter)) { instantQueue.add(Instant.EPOCH); From 70d07477e4767f5bc13d0717bc1dd483e2dd6e10 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Wed, 15 May 2019 18:14:01 -0400 Subject: [PATCH 35/39] Fix Javadoc --- .../cloud/tools/jib/event/progress/ThrottledLongConsumer.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/ThrottledLongConsumer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/ThrottledLongConsumer.java index afcb4aeb35..c3c2b138c7 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/ThrottledLongConsumer.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/ThrottledLongConsumer.java @@ -24,7 +24,7 @@ import javax.annotation.Nullable; /** - * Wraps a {@link Consumer} so that multiple consume calls ({@link #accept}) within a short + * Wraps a {@code Consumer} so that multiple consume calls ({@link #accept}) within a short * period of time are merged into a single later call with the value accumulated up to that point. */ public class ThrottledLongConsumer implements Consumer, Closeable { @@ -46,7 +46,6 @@ public class ThrottledLongConsumer implements Consumer, Closeable { * Wraps a consumer with the delay of 100 ms. * * @param callback {@link Consumer} callback to wrap - * @param valueAdder merger to add up multiple delayed values */ public ThrottledLongConsumer(Consumer callback) { this(callback, Duration.ofMillis(100), Instant::now); From 998d56a02c77269fdbd510ef62f7103faee69044 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Thu, 16 May 2019 00:18:40 -0400 Subject: [PATCH 36/39] ListenableCountingOutputStream -> NotifyingOutputStream --- .../cloud/tools/jib/http/BlobHttpContent.java | 2 +- ...Stream.java => NotifyingOutputStream.java} | 4 +-- .../cloud/tools/jib/registry/BlobPuller.java | 4 +-- ...st.java => NotifyingOutputStreamTest.java} | 30 +++++++++---------- 4 files changed, 20 insertions(+), 20 deletions(-) rename jib-core/src/main/java/com/google/cloud/tools/jib/http/{ListenableCountingOutputStream.java => NotifyingOutputStream.java} (95%) rename jib-core/src/test/java/com/google/cloud/tools/jib/http/{ListenableCountingOutputStreamTest.java => NotifyingOutputStreamTest.java} (69%) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobHttpContent.java b/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobHttpContent.java index f8d9e9f838..4c185cc30e 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobHttpContent.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/http/BlobHttpContent.java @@ -57,7 +57,7 @@ public boolean retrySupported() { @Override public void writeTo(OutputStream outputStream) throws IOException { - blob.writeTo(new ListenableCountingOutputStream(outputStream, writtenByteCountListener)); + blob.writeTo(new NotifyingOutputStream(outputStream, writtenByteCountListener)); outputStream.flush(); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStream.java b/jib-core/src/main/java/com/google/cloud/tools/jib/http/NotifyingOutputStream.java similarity index 95% rename from jib-core/src/main/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStream.java rename to jib-core/src/main/java/com/google/cloud/tools/jib/http/NotifyingOutputStream.java index e151c151c8..8b7bf1f4be 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStream.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/http/NotifyingOutputStream.java @@ -21,7 +21,7 @@ import java.util.function.Consumer; /** Counts the number of bytes written and reports the count to a callback. */ -public class ListenableCountingOutputStream extends OutputStream { +public class NotifyingOutputStream extends OutputStream { /** The underlying {@link OutputStream} to wrap and forward bytes to. */ private final OutputStream underlyingOutputStream; @@ -38,7 +38,7 @@ public class ListenableCountingOutputStream extends OutputStream { * @param underlyingOutputStream the wrapped {@link OutputStream} * @param byteCountListener the byte count {@link Consumer} */ - public ListenableCountingOutputStream( + public NotifyingOutputStream( OutputStream underlyingOutputStream, Consumer byteCountListener) { this.underlyingOutputStream = underlyingOutputStream; this.byteCountListener = byteCountListener; diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java index aebe846e53..04dd6b4bc3 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/BlobPuller.java @@ -21,7 +21,7 @@ import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.hash.Digests; import com.google.cloud.tools.jib.http.BlobHttpContent; -import com.google.cloud.tools.jib.http.ListenableCountingOutputStream; +import com.google.cloud.tools.jib.http.NotifyingOutputStream; import com.google.cloud.tools.jib.http.Response; import java.io.IOException; import java.io.OutputStream; @@ -66,7 +66,7 @@ public Void handleResponse(Response response) throws IOException, UnexpectedBlob blobSizeListener.accept(response.getContentLength()); try (OutputStream outputStream = - new ListenableCountingOutputStream(destinationOutputStream, writtenByteCountListener)) { + new NotifyingOutputStream(destinationOutputStream, writtenByteCountListener)) { BlobDescriptor receivedBlobDescriptor = Digests.computeDigest(response.getBody(), outputStream); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStreamTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/http/NotifyingOutputStreamTest.java similarity index 69% rename from jib-core/src/test/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStreamTest.java rename to jib-core/src/test/java/com/google/cloud/tools/jib/http/NotifyingOutputStreamTest.java index 9ca2af5030..63776df664 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/http/ListenableCountingOutputStreamTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/http/NotifyingOutputStreamTest.java @@ -29,8 +29,8 @@ import org.junit.Assert; import org.junit.Test; -/** Tests for {@link ListenableCountingOutputStream}. */ -public class ListenableCountingOutputStreamTest { +/** Tests for {@link NotifyingOutputStream}. */ +public class NotifyingOutputStreamTest { @Test public void testCallback_correctSequence() throws IOException { @@ -38,11 +38,11 @@ public void testCallback_correctSequence() throws IOException { List byteCounts = new ArrayList<>(); - try (ListenableCountingOutputStream listenableCountingOutputStream = - new ListenableCountingOutputStream(byteArrayOutputStream, byteCounts::add)) { - listenableCountingOutputStream.write(0); - listenableCountingOutputStream.write(new byte[] {1, 2, 3}); - listenableCountingOutputStream.write(new byte[] {1, 2, 3, 4, 5}, 3, 2); + try (NotifyingOutputStream notifyingOutputStream = + new NotifyingOutputStream(byteArrayOutputStream, byteCounts::add)) { + notifyingOutputStream.write(0); + notifyingOutputStream.write(new byte[] {1, 2, 3}); + notifyingOutputStream.write(new byte[] {1, 2, 3, 4, 5}, 3, 2); } Assert.assertEquals(Arrays.asList(1L, 3L, 2L), byteCounts); @@ -61,22 +61,22 @@ public void testDelay() throws IOException { try (ThrottledLongConsumer byteCounter = new ThrottledLongConsumer( byteCounts::add, Duration.ofSeconds(3), instantQueue::remove); - ListenableCountingOutputStream listenableCountingOutputStream = - new ListenableCountingOutputStream(byteArrayOutputStream, byteCounter)) { + NotifyingOutputStream notifyingOutputStream = + new NotifyingOutputStream(byteArrayOutputStream, byteCounter)) { instantQueue.add(Instant.EPOCH); - listenableCountingOutputStream.write(100); + notifyingOutputStream.write(100); instantQueue.add(Instant.EPOCH); - listenableCountingOutputStream.write(new byte[] {101, 102, 103}); + notifyingOutputStream.write(new byte[] {101, 102, 103}); instantQueue.add(Instant.EPOCH.plusSeconds(4)); - listenableCountingOutputStream.write(new byte[] {104, 105, 106}); + notifyingOutputStream.write(new byte[] {104, 105, 106}); instantQueue.add(Instant.EPOCH.plusSeconds(10)); - listenableCountingOutputStream.write(new byte[] {107, 108}); + notifyingOutputStream.write(new byte[] {107, 108}); instantQueue.add(Instant.EPOCH.plusSeconds(10)); - listenableCountingOutputStream.write(new byte[] {109}); + notifyingOutputStream.write(new byte[] {109}); instantQueue.add(Instant.EPOCH.plusSeconds(13)); - listenableCountingOutputStream.write(new byte[] {0, 110}, 1, 1); + notifyingOutputStream.write(new byte[] {0, 110}, 1, 1); } Assert.assertEquals(Arrays.asList(7L, 2L, 2L), byteCounts); From 70be4b2405ebbac267fa6fbeb58addf805d27346 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Thu, 16 May 2019 00:21:19 -0400 Subject: [PATCH 37/39] optionalOutStream -> outStream --- .../google/cloud/tools/jib/hash/Digests.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/Digests.java b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/Digests.java index 1c9bb1121c..606282e8b7 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/Digests.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/Digests.java @@ -49,10 +49,10 @@ public static BlobDescriptor computeDigest(JsonTemplate template) throws IOExcep return computeDigest(template, ByteStreams.nullOutputStream()); } - public static BlobDescriptor computeDigest(JsonTemplate template, OutputStream optionalOutStream) + public static BlobDescriptor computeDigest(JsonTemplate template, OutputStream outStream) throws IOException { WritableContents contents = contentsOut -> JsonTemplateMapper.writeTo(template, contentsOut); - return computeDigest(contents, optionalOutStream); + return computeDigest(contents, outStream); } public static BlobDescriptor computeDigest(InputStream inStream) throws IOException { @@ -71,34 +71,34 @@ public static BlobDescriptor computeDigest(WritableContents contents) throws IOE } /** - * Computes the digest by consuming the contents of an {@link InputStream} and optionally copying - * it to an {@link OutputStream}. Returns the computed digest along with the size of the bytes - * consumed to compute the digest. Does not close either stream. + * Computes the digest by consuming the contents of an {@link InputStream} while copying it to an + * {@link OutputStream}. Returns the computed digest along with the size of the bytes consumed to + * compute the digest. Does not close either stream. * * @param inStream the stream to read the contents from - * @param optionalOutStream the stream to which the contents are copied + * @param outStream the stream to which the contents are copied * @return computed digest and bytes consumed * @throws IOException if reading from or writing fails */ - public static BlobDescriptor computeDigest(InputStream inStream, OutputStream optionalOutStream) + public static BlobDescriptor computeDigest(InputStream inStream, OutputStream outStream) throws IOException { WritableContents contents = contentsOut -> ByteStreams.copy(inStream, contentsOut); - return computeDigest(contents, optionalOutStream); + return computeDigest(contents, outStream); } /** - * Computes the digest by consuming the contents and optionally copying it to an {@link - * OutputStream}. Returns the computed digest along with the size of the bytes consumed to compute - * the digest. Does not close the stream. + * Computes the digest by consuming the contents while copying it to an {@link OutputStream}. + * Returns the computed digest along with the size of the bytes consumed to compute the digest. + * Does not close the stream. * * @param contents the contents to compute digest for - * @param optionalOutStream the stream to which the contents are copied + * @param outStream the stream to which the contents are copied * @return computed digest and bytes consumed * @throws IOException if reading from or writing fails */ - public static BlobDescriptor computeDigest( - WritableContents contents, OutputStream optionalOutStream) throws IOException { - CountingDigestOutputStream digestOutStream = new CountingDigestOutputStream(optionalOutStream); + public static BlobDescriptor computeDigest(WritableContents contents, OutputStream outStream) + throws IOException { + CountingDigestOutputStream digestOutStream = new CountingDigestOutputStream(outStream); contents.writeTo(digestOutStream); digestOutStream.flush(); return digestOutStream.computeDigest(); From d6842e6f4e75788b52d1c918bdfe085d67c632be Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Thu, 16 May 2019 00:25:28 -0400 Subject: [PATCH 38/39] ThrottledLongConsumer -> ThrottledAccumulatingConsumer --- .../google/cloud/tools/jib/builder/steps/PushBlobStep.java | 6 +++--- .../steps/ThrottledProgressEventDispatcherWrapper.java | 7 ++++--- ...ongConsumer.java => ThrottledAccumulatingConsumer.java} | 0 .../cloud/tools/jib/http/NotifyingOutputStreamTest.java | 6 +++--- 4 files changed, 10 insertions(+), 9 deletions(-) rename jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/{ThrottledLongConsumer.java => ThrottledAccumulatingConsumer.java} (100%) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java index 23f0753d73..0528127c16 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java @@ -27,7 +27,7 @@ import com.google.cloud.tools.jib.builder.TimerEventDispatcher; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.event.events.LogEvent; -import com.google.cloud.tools.jib.event.progress.ThrottledLongConsumer; +import com.google.cloud.tools.jib.event.progress.ThrottledAccumulatingConsumer; import com.google.cloud.tools.jib.registry.RegistryClient; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -86,8 +86,8 @@ public BlobDescriptor call() throws IOException, RegistryException, ExecutionExc TimerEventDispatcher ignored = new TimerEventDispatcher( buildConfiguration.getEventDispatcher(), DESCRIPTION + blobDescriptor); - ThrottledLongConsumer throttledProgressReporter = - new ThrottledLongConsumer(progressEventDispatcher::dispatchProgress)) { + ThrottledAccumulatingConsumer throttledProgressReporter = + new ThrottledAccumulatingConsumer(progressEventDispatcher::dispatchProgress)) { RegistryClient registryClient = buildConfiguration .newTargetImageRegistryClientFactory() diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ThrottledProgressEventDispatcherWrapper.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ThrottledProgressEventDispatcherWrapper.java index de975898e4..4d518f7788 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ThrottledProgressEventDispatcherWrapper.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ThrottledProgressEventDispatcherWrapper.java @@ -18,7 +18,7 @@ import com.google.cloud.tools.jib.builder.BuildStepType; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; -import com.google.cloud.tools.jib.event.progress.ThrottledLongConsumer; +import com.google.cloud.tools.jib.event.progress.ThrottledAccumulatingConsumer; import com.google.common.base.Preconditions; import java.io.Closeable; import javax.annotation.Nullable; @@ -38,7 +38,7 @@ class ThrottledProgressEventDispatcherWrapper implements Closeable { private final String description; private final BuildStepType type; @Nullable private ProgressEventDispatcher progressEventDispatcher; - @Nullable private ThrottledLongConsumer throttledDispatcher; + @Nullable private ThrottledAccumulatingConsumer throttledDispatcher; ThrottledProgressEventDispatcherWrapper( ProgressEventDispatcher.Factory progressEventDispatcherFactory, @@ -66,6 +66,7 @@ void setProgressTarget(long allocationUnits) { Preconditions.checkState(progressEventDispatcher == null); progressEventDispatcher = progressEventDispatcherFactory.create(type, description, allocationUnits); - throttledDispatcher = new ThrottledLongConsumer(progressEventDispatcher::dispatchProgress); + throttledDispatcher = + new ThrottledAccumulatingConsumer(progressEventDispatcher::dispatchProgress); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/ThrottledLongConsumer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/ThrottledAccumulatingConsumer.java similarity index 100% rename from jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/ThrottledLongConsumer.java rename to jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/ThrottledAccumulatingConsumer.java diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/http/NotifyingOutputStreamTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/http/NotifyingOutputStreamTest.java index 63776df664..facf1cdd76 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/http/NotifyingOutputStreamTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/http/NotifyingOutputStreamTest.java @@ -16,7 +16,7 @@ package com.google.cloud.tools.jib.http; -import com.google.cloud.tools.jib.event.progress.ThrottledLongConsumer; +import com.google.cloud.tools.jib.event.progress.ThrottledAccumulatingConsumer; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.time.Duration; @@ -58,8 +58,8 @@ public void testDelay() throws IOException { Queue instantQueue = new ArrayDeque<>(); instantQueue.add(Instant.EPOCH); - try (ThrottledLongConsumer byteCounter = - new ThrottledLongConsumer( + try (ThrottledAccumulatingConsumer byteCounter = + new ThrottledAccumulatingConsumer( byteCounts::add, Duration.ofSeconds(3), instantQueue::remove); NotifyingOutputStream notifyingOutputStream = new NotifyingOutputStream(byteArrayOutputStream, byteCounter)) { From c5fedc7b38a85e1ff5271e8eb08b16966f529d30 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Thu, 16 May 2019 00:29:54 -0400 Subject: [PATCH 39/39] ThrottledLongConsumer -> ThrottledAccumulatingConsumer --- .../jib/event/progress/ThrottledAccumulatingConsumer.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/ThrottledAccumulatingConsumer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/ThrottledAccumulatingConsumer.java index c3c2b138c7..65140b2ddc 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/ThrottledAccumulatingConsumer.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/event/progress/ThrottledAccumulatingConsumer.java @@ -27,7 +27,7 @@ * Wraps a {@code Consumer} so that multiple consume calls ({@link #accept}) within a short * period of time are merged into a single later call with the value accumulated up to that point. */ -public class ThrottledLongConsumer implements Consumer, Closeable { +public class ThrottledAccumulatingConsumer implements Consumer, Closeable { private final Consumer consumer; @@ -47,11 +47,11 @@ public class ThrottledLongConsumer implements Consumer, Closeable { * * @param callback {@link Consumer} callback to wrap */ - public ThrottledLongConsumer(Consumer callback) { + public ThrottledAccumulatingConsumer(Consumer callback) { this(callback, Duration.ofMillis(100), Instant::now); } - public ThrottledLongConsumer( + public ThrottledAccumulatingConsumer( Consumer consumer, Duration delayBetweenCallbacks, Supplier getNow) { this.consumer = consumer; this.delayBetweenCallbacks = delayBetweenCallbacks;