From d3bfc7a815e7271fc648604be9411cd6bf787a79 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 30 Apr 2019 18:09:48 -0400 Subject: [PATCH 1/4] Reorg and refactor --- .../registry/BlobPullerIntegrationTest.java | 14 +-- .../ManifestPusherIntegrationTest.java | 11 +- .../cloud/tools/jib/blob/BlobDescriptor.java | 23 ---- .../google/cloud/tools/jib/blob/Blobs.java | 10 +- .../google/cloud/tools/jib/blob/FileBlob.java | 5 +- .../cloud/tools/jib/blob/InputStreamBlob.java | 3 +- .../google/cloud/tools/jib/blob/JsonBlob.java | 37 +++++++ .../cloud/tools/jib/blob/StringBlob.java | 5 +- ...rBlob.java => WriterableContentsBlob.java} | 19 ++-- .../tools/jib/builder/steps/BuildResult.java | 7 +- .../jib/builder/steps/PullBaseImageStep.java | 2 +- .../jib/builder/steps/PushImageStep.java | 8 +- .../jib/cache/DefaultCacheStorageWriter.java | 11 +- .../tools/jib/cache/LayerEntriesSelector.java | 7 +- .../cloud/tools/jib/docker/ImageTarball.java | 8 +- .../jib/hash/CountingDigestOutputStream.java | 45 +++++--- .../cloud/tools/jib/hash/DigestUtil.java | 102 ++++++++++++++++++ .../WritableContents.java} | 12 ++- .../google/cloud/tools/jib/http/Request.java | 13 +-- .../jib/image/json/ImageToJsonTranslator.java | 4 +- .../tools/jib/json/JsonTemplateMapper.java | 50 +++++---- .../tools/jib/registry/ManifestPusher.java | 11 +- .../google/cloud/tools/jib/blob/BlobTest.java | 16 ++- .../cache/DefaultCacheStorageWriterTest.java | 20 ++-- .../jib/cache/LayerEntriesSelectorTest.java | 12 +-- .../json/DockerLoadManifestTemplateTest.java | 3 +- .../hash/CountingDigestOutputStreamTest.java | 8 +- .../jib/http/TestBlobProgressListener.java | 7 +- .../ContainerConfigurationTemplateTest.java | 6 +- .../image/json/ImageToJsonTranslatorTest.java | 5 +- .../image/json/OCIManifestTemplateTest.java | 6 +- .../image/json/V22ManifestTemplateTest.java | 6 +- .../jib/json/JsonTemplateMapperTest.java | 11 +- .../tools/jib/registry/BlobCheckerTest.java | 7 +- .../tools/jib/registry/BlobPullerTest.java | 2 +- .../jib/registry/ManifestPusherTest.java | 22 +--- 36 files changed, 304 insertions(+), 234 deletions(-) create mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/blob/JsonBlob.java rename jib-core/src/main/java/com/google/cloud/tools/jib/blob/{WriterBlob.java => WriterableContentsBlob.java} (59%) create mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java rename jib-core/src/main/java/com/google/cloud/tools/jib/{blob/BlobWriter.java => hash/WritableContents.java} (63%) diff --git a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java index 1368525e41..4c46ceca3e 100644 --- a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java +++ b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java @@ -16,8 +16,8 @@ package com.google.cloud.tools.jib.registry; +import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.event.EventDispatcher; -import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; import com.google.cloud.tools.jib.http.TestBlobProgressListener; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.json.V21ManifestTemplate; @@ -54,23 +54,19 @@ public void testPull() throws IOException, RegistryException, InterruptedExcepti DescriptorDigest realDigest = manifestTemplate.getLayerDigests().get(0); // Pulls a layer BLOB of the busybox image. - CountingDigestOutputStream layerOutputStream = - new CountingDigestOutputStream(ByteStreams.nullOutputStream()); LongAdder totalByteCount = new LongAdder(); LongAdder expectedSize = new LongAdder(); - registryClient - .pullBlob( + Blob pulledBlob = + registryClient.pullBlob( realDigest, size -> { Assert.assertEquals(0, expectedSize.sum()); expectedSize.add(size); }, - new TestBlobProgressListener(totalByteCount::add)) - .writeTo(layerOutputStream); + new TestBlobProgressListener(totalByteCount::add)); + Assert.assertEquals(realDigest, pulledBlob.writeTo(ByteStreams.nullOutputStream()).getDigest()); Assert.assertTrue(expectedSize.sum() > 0); Assert.assertEquals(expectedSize.sum(), totalByteCount.sum()); - - Assert.assertEquals(realDigest, layerOutputStream.toBlobDescriptor().getDigest()); } @Test diff --git a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPusherIntegrationTest.java b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPusherIntegrationTest.java index 67a444f4c7..7f7800e542 100644 --- a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPusherIntegrationTest.java +++ b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPusherIntegrationTest.java @@ -21,12 +21,11 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.event.EventDispatcher; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.http.TestBlobProgressListener; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.json.ManifestTemplate; import com.google.cloud.tools.jib.image.json.V22ManifestTemplate; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; -import com.google.common.io.ByteStreams; import java.io.IOException; import java.security.DigestException; import org.junit.Assert; @@ -114,11 +113,7 @@ public void testPush() V22ManifestTemplate manifestTemplateByDigest = registryClient.pullManifest(imageDigest.toString(), V22ManifestTemplate.class); Assert.assertEquals( - JsonTemplateMapper.toBlob(manifestTemplate) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(), - JsonTemplateMapper.toBlob(manifestTemplateByDigest) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest()); + DigestUtil.computeJsonDigest(manifestTemplate), + DigestUtil.computeJsonDigest(manifestTemplateByDigest)); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/BlobDescriptor.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/BlobDescriptor.java index 0b067cc912..ff59209998 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/BlobDescriptor.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/BlobDescriptor.java @@ -16,12 +16,7 @@ package com.google.cloud.tools.jib.blob; -import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; import com.google.cloud.tools.jib.image.DescriptorDigest; -import com.google.common.io.ByteStreams; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; /** Contains properties describing a BLOB, including its digest and possibly its size (in bytes). */ public class BlobDescriptor { @@ -31,24 +26,6 @@ public class BlobDescriptor { /** The size of the BLOB (in bytes). Negative if unknown. */ private final long size; - /** - * Creates a new {@link BlobDescriptor} from the contents of an {@link InputStream} while piping - * to an {@link OutputStream}. Does not close either streams. - * - * @param inputStream the stream to read the contents from - * @param outputStream the {@link OutputStream} to pipe to - * @return a {@link BlobDescriptor} of the piped contents - * @throws IOException if reading from or writing to the streams fails - */ - static BlobDescriptor fromPipe(InputStream inputStream, OutputStream outputStream) - throws IOException { - CountingDigestOutputStream countingDigestOutputStream = - new CountingDigestOutputStream(outputStream); - ByteStreams.copy(inputStream, countingDigestOutputStream); - countingDigestOutputStream.flush(); - return countingDigestOutputStream.toBlobDescriptor(); - } - public BlobDescriptor(long size, DescriptorDigest digest) { this.size = size; this.digest = digest; diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java index 4cec2f9b7e..cb621b232f 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java @@ -17,6 +17,8 @@ package com.google.cloud.tools.jib.blob; import com.google.cloud.tools.jib.filesystem.FileOperations; +import com.google.cloud.tools.jib.hash.WritableContents; +import com.google.cloud.tools.jib.json.JsonTemplate; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; @@ -36,6 +38,10 @@ public static Blob from(Path file) { return new FileBlob(file); } + public static Blob from(JsonTemplate template) { + return new JsonBlob(template); + } + /** * Creates a {@link StringBlob} with UTF-8 encoding. * @@ -46,8 +52,8 @@ public static Blob from(String content) { return new StringBlob(content); } - public static Blob from(BlobWriter writer) { - return new WriterBlob(writer); + public static Blob from(WritableContents writable) { + return new WritableContentsBlob(writable); } /** diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/FileBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/FileBlob.java index 5c978c2dfb..7748b9bddd 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/FileBlob.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/FileBlob.java @@ -16,6 +16,7 @@ package com.google.cloud.tools.jib.blob; +import com.google.cloud.tools.jib.hash.DigestUtil; import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; @@ -34,8 +35,8 @@ class FileBlob implements Blob { @Override public BlobDescriptor writeTo(OutputStream outputStream) throws IOException { - try (InputStream fileStream = new BufferedInputStream(Files.newInputStream(file))) { - return BlobDescriptor.fromPipe(fileStream, outputStream); + try (InputStream fileIn = new BufferedInputStream(Files.newInputStream(file))) { + return DigestUtil.computeDigest(fileIn, outputStream); } } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/InputStreamBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/InputStreamBlob.java index 46273b691f..5e5e172f8c 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/InputStreamBlob.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/InputStreamBlob.java @@ -16,6 +16,7 @@ package com.google.cloud.tools.jib.blob; +import com.google.cloud.tools.jib.hash.DigestUtil; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -39,7 +40,7 @@ public BlobDescriptor writeTo(OutputStream outputStream) throws IOException { throw new IllegalStateException("Cannot rewrite Blob backed by an InputStream"); } try (InputStream inputStream = this.inputStream) { - return BlobDescriptor.fromPipe(inputStream, outputStream); + return DigestUtil.computeDigest(inputStream, outputStream); } finally { isWritten = true; diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/JsonBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/JsonBlob.java new file mode 100644 index 0000000000..83dd4c5314 --- /dev/null +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/JsonBlob.java @@ -0,0 +1,37 @@ +/* + * Copyright 2019 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.tools.jib.blob; + +import com.google.cloud.tools.jib.hash.DigestUtil; +import com.google.cloud.tools.jib.json.JsonTemplate; +import java.io.IOException; +import java.io.OutputStream; + +/** A {@link Blob} that holds {@link JsonTemplate}. */ +class JsonBlob implements Blob { + + private final JsonTemplate template; + + JsonBlob(JsonTemplate template) { + this.template = template; + } + + @Override + public BlobDescriptor writeTo(OutputStream outputStream) throws IOException { + return DigestUtil.computeDigest(template, outputStream); + } +} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/StringBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/StringBlob.java index 62e07e1297..9f61014666 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/StringBlob.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/StringBlob.java @@ -16,6 +16,7 @@ package com.google.cloud.tools.jib.blob; +import com.google.cloud.tools.jib.hash.DigestUtil; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -33,9 +34,9 @@ class StringBlob implements Blob { @Override public BlobDescriptor writeTo(OutputStream outputStream) throws IOException { - try (InputStream stringInputStream = + try (InputStream stringIn = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8))) { - return BlobDescriptor.fromPipe(stringInputStream, outputStream); + return DigestUtil.computeDigest(stringIn, outputStream); } } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterableContentsBlob.java similarity index 59% rename from jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterBlob.java rename to jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterableContentsBlob.java index c47dc3e037..3c6f843293 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterBlob.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterableContentsBlob.java @@ -16,25 +16,22 @@ package com.google.cloud.tools.jib.blob; -import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; +import com.google.cloud.tools.jib.hash.DigestUtil; +import com.google.cloud.tools.jib.hash.WritableContents; import java.io.IOException; import java.io.OutputStream; -/** A {@link Blob} that writes with a {@link BlobWriter} function and hashes the bytes. */ -class WriterBlob implements Blob { +/** A {@link Blob} that holds {@link WritableContents}. */ +class WritableContentsBlob implements Blob { - private final BlobWriter writer; + private final WritableContents writableContents; - WriterBlob(BlobWriter writer) { - this.writer = writer; + WritableContentsBlob(WritableContents writableContents) { + this.writableContents = writableContents; } @Override public BlobDescriptor writeTo(OutputStream outputStream) throws IOException { - CountingDigestOutputStream countingDigestOutputStream = - new CountingDigestOutputStream(outputStream); - writer.writeTo(countingDigestOutputStream); - countingDigestOutputStream.flush(); - return countingDigestOutputStream.toBlobDescriptor(); + return DigestUtil.computeDigest(writableContents, outputStream); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildResult.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildResult.java index 383b055760..a8597d7bff 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildResult.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildResult.java @@ -17,12 +17,12 @@ package com.google.cloud.tools.jib.builder.steps; import com.google.cloud.tools.jib.blob.BlobDescriptor; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.Layer; import com.google.cloud.tools.jib.image.json.BuildableManifestTemplate; import com.google.cloud.tools.jib.image.json.ImageToJsonTranslator; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.io.ByteStreams; import java.io.IOException; import java.util.Objects; @@ -49,10 +49,7 @@ static BuildResult fromImage( BuildableManifestTemplate manifestTemplate = imageToJsonTranslator.getManifestTemplate( targetFormat, containerConfigurationBlobDescriptor); - DescriptorDigest imageDigest = - JsonTemplateMapper.toBlob(manifestTemplate) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DescriptorDigest imageDigest = DigestUtil.computeJsonDigest(manifestTemplate); DescriptorDigest imageId = containerConfigurationBlobDescriptor.getDigest(); return new BuildResult(imageDigest, imageId); } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java index 1e96d4a310..a7d8773357 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java @@ -237,7 +237,7 @@ private Image pullBaseImage( || v22ManifestTemplate.getContainerConfiguration().getDigest() == null) { throw new UnknownManifestFormatException( "Invalid container configuration in Docker V2.2 manifest: \n" - + Blobs.writeToString(JsonTemplateMapper.toBlob(v22ManifestTemplate))); + + JsonTemplateMapper.toUtf8String(v22ManifestTemplate)); } DescriptorDigest containerConfigurationDigest = diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java index 4d7821b2ec..e44bb7980f 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java @@ -25,13 +25,12 @@ import com.google.cloud.tools.jib.builder.TimerEventDispatcher; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.event.events.LogEvent; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.json.BuildableManifestTemplate; import com.google.cloud.tools.jib.image.json.ImageToJsonTranslator; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.cloud.tools.jib.registry.RegistryClient; import com.google.common.collect.ImmutableSet; -import com.google.common.io.ByteStreams; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -165,10 +164,7 @@ private BuildResult afterAllPushed() })); } - DescriptorDigest imageDigest = - JsonTemplateMapper.toBlob(manifestTemplate) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DescriptorDigest imageDigest = DigestUtil.computeJsonDigest(manifestTemplate); DescriptorDigest imageId = containerConfigurationBlobDescriptor.getDigest(); BuildResult result = new BuildResult(imageDigest, imageId); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriter.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriter.java index 0768103421..f164751c8a 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriter.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriter.java @@ -105,7 +105,7 @@ private static DescriptorDigest getDiffIdByDecompressingFile(Path compressedFile GZIPInputStream decompressorStream = new GZIPInputStream(fileInputStream)) { ByteStreams.copy(decompressorStream, diffIdCaptureOutputStream); } - return diffIdCaptureOutputStream.toBlobDescriptor().getDigest(); + return diffIdCaptureOutputStream.getDigest(); } } @@ -121,7 +121,9 @@ private static void writeMetadata(JsonTemplate jsonTemplate, Path destination) throws IOException { Path temporaryFile = Files.createTempFile(destination.getParent(), null, null); temporaryFile.toFile().deleteOnExit(); - Blobs.writeToFile(JsonTemplateMapper.toBlob(jsonTemplate), temporaryFile); + try (OutputStream outputStream = Files.newOutputStream(temporaryFile)) { + JsonTemplateMapper.writeTo(jsonTemplate, outputStream); + } // Attempts an atomic move first, and falls back to non-atomic if the file system does not // support atomic moves. @@ -333,9 +335,8 @@ private WrittenLayer writeUncompressedLayerBlobToDirectory( // The GZIPOutputStream must be closed in order to write out the remaining compressed data. compressorStream.close(); - BlobDescriptor compressedBlobDescriptor = compressedDigestOutputStream.toBlobDescriptor(); - DescriptorDigest layerDigest = compressedBlobDescriptor.getDigest(); - long layerSize = compressedBlobDescriptor.getSize(); + DescriptorDigest layerDigest = compressedDigestOutputStream.getDigest(); + long layerSize = compressedDigestOutputStream.getBytesHahsed(); // Renames the temporary layer file to the correct filename. Path layerFile = diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/LayerEntriesSelector.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/LayerEntriesSelector.java index b47011fff9..dd5c18cb1c 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/LayerEntriesSelector.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/LayerEntriesSelector.java @@ -16,13 +16,12 @@ package com.google.cloud.tools.jib.cache; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.LayerEntry; import com.google.cloud.tools.jib.json.JsonTemplate; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; -import com.google.common.io.ByteStreams; import java.io.IOException; import java.nio.file.Files; import java.time.Instant; @@ -142,9 +141,7 @@ static List toSortedJsonTemplates(List layerEntr */ static DescriptorDigest generateSelector(ImmutableList layerEntries) throws IOException { - return JsonTemplateMapper.toBlob(toSortedJsonTemplates(layerEntries)) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + return DigestUtil.computeJsonDigest(toSortedJsonTemplates(layerEntries)); } private LayerEntriesSelector() {} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageTarball.java b/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageTarball.java index f7ad06a162..7843afbe48 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageTarball.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/docker/ImageTarball.java @@ -25,6 +25,7 @@ import com.google.cloud.tools.jib.image.json.ImageToJsonTranslator; import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.cloud.tools.jib.tar.TarStreamBuilder; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; import java.util.Collections; @@ -77,10 +78,9 @@ public void writeTo(OutputStream out) throws IOException { // Adds the manifest to tarball. manifestTemplate.setRepoTags(imageReference.toStringWithTag()); - tarStreamBuilder.addByteEntry( - Blobs.writeToByteArray( - JsonTemplateMapper.toBlob(Collections.singletonList(manifestTemplate))), - MANIFEST_JSON_FILE_NAME); + ByteArrayOutputStream outStream = new ByteArrayOutputStream(); + JsonTemplateMapper.writeTo(Collections.singletonList(manifestTemplate), outStream); + tarStreamBuilder.addByteEntry(outStream.toByteArray(), MANIFEST_JSON_FILE_NAME); tarStreamBuilder.writeAsTarArchiveTo(out); } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStream.java b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStream.java index b2e872c2a2..12cf3ebd65 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStream.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStream.java @@ -16,22 +16,27 @@ package com.google.cloud.tools.jib.hash; -import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.image.DescriptorDigest; +import com.google.common.base.Verify; import java.io.IOException; import java.io.OutputStream; import java.security.DigestException; import java.security.DigestOutputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import javax.annotation.Nullable; /** A {@link DigestOutputStream} that also keeps track of the total number of bytes written. */ public class CountingDigestOutputStream extends DigestOutputStream { private static final String SHA_256_ALGORITHM = "SHA-256"; - /** Keeps track of the total number of bytes appended. */ - private long totalBytes = 0; + private long bytesSoFar = 0; + + /** The total number of bytes used to compute a digest. Resets when {@link computeDigest) is called. */ + private long bytesHashed = 0; + + @Nullable private DescriptorDigest descriptorDigest; /** * Wraps the {@code outputStream}. @@ -49,12 +54,10 @@ public CountingDigestOutputStream(OutputStream outputStream) { } /** - * Builds a {@link BlobDescriptor} with the hash and size of the bytes written. The buffer resets - * after this method is called, so this method should only be called once per BlobDescriptor. - * - * @return the built {@link BlobDescriptor}. + * Computes the hash and remembers the size of the bytes written to compute the hash. The buffer + * resets after this method is called, so this method should only be called once per computation. */ - public BlobDescriptor toBlobDescriptor() { + public void computeDigest() { try { byte[] hashedBytes = digest.digest(); @@ -65,28 +68,40 @@ public BlobDescriptor toBlobDescriptor() { } String hash = stringBuilder.toString(); - DescriptorDigest digest = DescriptorDigest.fromHash(hash); - return new BlobDescriptor(totalBytes, digest); + bytesHashed = bytesSoFar; + descriptorDigest = DescriptorDigest.fromHash(hash); + bytesSoFar = 0; } catch (DigestException ex) { throw new RuntimeException("SHA-256 algorithm produced invalid hash: " + ex.getMessage(), ex); } } - /** @return the total number of bytes that were hashed */ - public long getTotalBytes() { - return totalBytes; + /** @return the number of bytes written and used to compute the most recent digest */ + public long getBytesHahsed() { + if (descriptorDigest == null) { + computeDigest(); + } + return bytesHashed; + } + + /** @return the most recently computed digest hash */ + public DescriptorDigest getDigest() { + if (descriptorDigest == null) { + computeDigest(); + } + return Verify.verifyNotNull(descriptorDigest); } @Override public void write(byte[] data, int offset, int length) throws IOException { super.write(data, offset, length); - totalBytes += length; + bytesSoFar += length; } @Override public void write(int singleByte) throws IOException { super.write(singleByte); - totalBytes++; + bytesSoFar++; } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java new file mode 100644 index 0000000000..946083c69a --- /dev/null +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java @@ -0,0 +1,102 @@ +/* + * Copyright 2017 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.tools.jib.hash; + +import com.google.cloud.tools.jib.blob.BlobDescriptor; +import com.google.cloud.tools.jib.image.DescriptorDigest; +import com.google.cloud.tools.jib.json.JsonTemplate; +import com.google.cloud.tools.jib.json.JsonTemplateMapper; +import com.google.common.io.ByteStreams; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.List; + +/** + * Utility class for computing a digest for various inputs while optionally writing to an output + * stream. + */ +// Note: intentionally this class does not depend on Blob, as Blob classes depend on this class. +// TODO: BlobDescriptor is merely a tuple of (size, digest). Rename BlobDescriptor to something +// more general. +public class DigestUtil { + + public static DescriptorDigest computeJsonDigest(JsonTemplate template) throws IOException { + return computeDigest(template, ByteStreams.nullOutputStream()).getDigest(); + } + + public static DescriptorDigest computeJsonDigest(List templates) + throws IOException { + WritableContents contents = contentsOut -> JsonTemplateMapper.writeTo(templates, contentsOut); + return computeDigest(contents, ByteStreams.nullOutputStream()).getDigest(); + } + + public static BlobDescriptor computeDigest(JsonTemplate template, OutputStream optionalOutStream) + throws IOException { + WritableContents contents = contentsOut -> JsonTemplateMapper.writeTo(template, contentsOut); + return computeDigest(contents, optionalOutStream); + } + + public static BlobDescriptor computeDigest(InputStream inStream) throws IOException { + return computeDigest(inStream, ByteStreams.nullOutputStream()); + } + + /** + * Computes the digest by consuming the contents. + * + * @param contents the contents for which the digest is computed + * @return computed digest and bytes consumed + * @throws IOException if reading fails + */ + public static BlobDescriptor computeDigest(WritableContents contents) throws IOException { + return computeDigest(contents, ByteStreams.nullOutputStream()); + } + + /** + * Computes the digest by consuming the contents of an {@link InputStream} and optionally copying + * it to an {@link OutputStream}. Returns the computed digest along with the size of the bytes + * consumed to compute the digest. Does not close either stream. + * + * @param inStream the stream to read the contents from + * @param optionalOutStream the stream to which the contents are copied + * @return computed digest and bytes consumed + * @throws IOException if reading from or writing fails + */ + public static BlobDescriptor computeDigest(InputStream inStream, OutputStream optionalOutStream) + throws IOException { + WritableContents contents = contentsOut -> ByteStreams.copy(inStream, contentsOut); + return computeDigest(contents, optionalOutStream); + } + + /** + * Computes the digest by consuming the contents and optionally copying it to an {@link + * OutputStream}. Returns the computed digest along with the size of the bytes consumed to compute + * the digest. Does not close the stream. + * + * @param contents the contents to compute digest for + * @param optionalOutStream the stream to which the contents are copied + * @return computed digest and bytes consumed + * @throws IOException if reading from or writing fails + */ + public static BlobDescriptor computeDigest( + WritableContents contents, OutputStream optionalOutStream) throws IOException { + CountingDigestOutputStream digestOutStream = new CountingDigestOutputStream(optionalOutStream); + contents.writeTo(digestOutStream); + digestOutStream.flush(); + return new BlobDescriptor(digestOutStream.getBytesHahsed(), digestOutStream.getDigest()); + } +} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/BlobWriter.java b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/WritableContents.java similarity index 63% rename from jib-core/src/main/java/com/google/cloud/tools/jib/blob/BlobWriter.java rename to jib-core/src/main/java/com/google/cloud/tools/jib/hash/WritableContents.java index acf393a524..5fc4fae624 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/BlobWriter.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/WritableContents.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Google LLC. + * Copyright 2019 Google LLC. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -14,17 +14,19 @@ * the License. */ -package com.google.cloud.tools.jib.blob; +package com.google.cloud.tools.jib.hash; import java.io.IOException; import java.io.OutputStream; /** - * This function writes the contents of a BLOB. This function should write the same BLOB regardless - * of the number of times it is called. + * As a function, writes some contents to an output stream. As a class, represents contents that can + * be written to an output stream. This may be "unrealized-before-write" contents; for example, a + * file may be open and read for input contents only when this function is called to write to an + * output stream. */ @FunctionalInterface -public interface BlobWriter { +public interface WritableContents { void writeTo(OutputStream outputStream) throws IOException; } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/http/Request.java b/jib-core/src/main/java/com/google/cloud/tools/jib/http/Request.java index e6ef54f7f0..88631a5546 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/http/Request.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/http/Request.java @@ -16,6 +16,7 @@ package com.google.cloud.tools.jib.http; +import com.google.api.client.http.HttpContent; import com.google.api.client.http.HttpHeaders; import java.util.List; import javax.annotation.Nullable; @@ -27,7 +28,7 @@ public class Request { private final HttpHeaders headers; /** The HTTP request body. */ - @Nullable private final BlobHttpContent body; + @Nullable private final HttpContent body; /** HTTP connection and read timeout. */ @Nullable private final Integer httpTimeout; @@ -35,7 +36,7 @@ public class Request { public static class Builder { private final HttpHeaders headers = new HttpHeaders().setAccept("*/*"); - @Nullable private BlobHttpContent body; + @Nullable private HttpContent body; @Nullable private Integer httpTimeout; public Request build() { @@ -92,11 +93,11 @@ public Builder setHttpTimeout(@Nullable Integer httpTimeout) { /** * Sets the body and its corresponding {@code Content-Type} header. * - * @param blobHttpContent the body content + * @param httpContent the body content * @return this */ - public Builder setBody(@Nullable BlobHttpContent blobHttpContent) { - this.body = blobHttpContent; + public Builder setBody(@Nullable HttpContent httpContent) { + this.body = httpContent; return this; } } @@ -116,7 +117,7 @@ HttpHeaders getHeaders() { } @Nullable - BlobHttpContent getHttpContent() { + HttpContent getHttpContent() { return body; } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java index bb504ffeac..8a8f06c5e9 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java @@ -18,13 +18,13 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; +import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.configuration.DockerHealthCheck; import com.google.cloud.tools.jib.configuration.Port; import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.Layer; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -189,7 +189,7 @@ public Blob getContainerConfigurationBlob() { } // Serializes into JSON. - return JsonTemplateMapper.toBlob(template); + return Blobs.from(template); } /** diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java b/jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java index 2ff9e4dbe5..c5f1c5f501 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java @@ -16,12 +16,14 @@ package com.google.cloud.tools.jib.json; +import com.fasterxml.jackson.core.JsonGenerationException; +import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.type.CollectionType; -import com.google.cloud.tools.jib.blob.Blob; -import com.google.cloud.tools.jib.blob.Blobs; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.nio.file.Files; @@ -117,28 +119,36 @@ public static List readListOfJson( return objectMapper.readValue(jsonString, listType); } - /** - * Convert a {@link JsonTemplate} to a {@link Blob} of the JSON string. - * - * @param template the JSON template to convert - * @return a {@link Blob} of the JSON string - */ - public static Blob toBlob(JsonTemplate template) { - return toBlob((Object) template); + public static String toUtf8String(JsonTemplate template) + throws JsonGenerationException, JsonMappingException, IOException { + return toUtf8String((Object) template); } - /** - * Convert a list of {@link JsonTemplate} to a {@link Blob} of the JSON string. - * - * @param templates the list of JSON templates to convert - * @return a {@link Blob} of the JSON string - */ - public static Blob toBlob(List templates) { - return toBlob((Object) templates); + public static String toUtf8String(List templates) + throws JsonGenerationException, JsonMappingException, IOException { + return toUtf8String((Object) templates); + } + + public static void writeTo(JsonTemplate template, OutputStream out) + throws JsonGenerationException, JsonMappingException, IOException { + writeTo((Object) template, out); + } + + public static void writeTo(List templates, OutputStream out) + throws JsonGenerationException, JsonMappingException, IOException { + writeTo((Object) templates, out); + } + + private static String toUtf8String(Object template) + throws JsonGenerationException, JsonMappingException, IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writeTo(template, out); + return out.toString("UTF-8"); } - private static Blob toBlob(Object template) { - return Blobs.from(outputStream -> objectMapper.writeValue(outputStream, template)); + private static void writeTo(Object template, OutputStream out) + throws JsonGenerationException, JsonMappingException, IOException { + objectMapper.writeValue(out, template); } private JsonTemplateMapper() {} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPusher.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPusher.java index 87834575ba..a0cf547579 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPusher.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPusher.java @@ -18,14 +18,14 @@ import com.google.api.client.http.HttpMethods; import com.google.api.client.http.HttpResponseException; +import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.event.EventDispatcher; import com.google.cloud.tools.jib.event.events.LogEvent; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.http.BlobHttpContent; import com.google.cloud.tools.jib.http.Response; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.json.BuildableManifestTemplate; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; -import com.google.common.io.ByteStreams; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; @@ -82,7 +82,7 @@ private static String makeUnexpectedImageDigestWarning( @Override public BlobHttpContent getContent() { return new BlobHttpContent( - JsonTemplateMapper.toBlob(manifestTemplate), + Blobs.from(manifestTemplate), manifestTemplate.getManifestMediaType(), // TODO: Consider giving progress on manifest push as well? null); @@ -125,10 +125,7 @@ public DescriptorDigest handleHttpResponseException(HttpResponseException httpRe @Override public DescriptorDigest handleResponse(Response response) throws IOException { // Checks if the image digest is as expected. - DescriptorDigest expectedDigest = - JsonTemplateMapper.toBlob(manifestTemplate) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DescriptorDigest expectedDigest = DigestUtil.computeJsonDigest(manifestTemplate); List receivedDigests = response.getHeader(RESPONSE_DIGEST_HEADER); if (receivedDigests.size() == 1) { diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java index 623068664d..721b36b257 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java @@ -16,7 +16,8 @@ package com.google.cloud.tools.jib.blob; -import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; +import com.google.cloud.tools.jib.hash.DigestUtil; +import com.google.cloud.tools.jib.hash.WritableContents; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.common.io.Resources; import java.io.ByteArrayInputStream; @@ -32,7 +33,6 @@ import java.nio.file.StandardOpenOption; import org.junit.Assert; import org.junit.Test; -import org.mockito.Mockito; /** Tests for {@link Blob}. */ public class BlobTest { @@ -58,13 +58,13 @@ public void testFromString() throws IOException { } @Test - public void testFromBlobWriter() throws IOException { + public void testFromWritableContents() throws IOException { String expected = "crepecake"; - BlobWriter writer = + WritableContents writableContents = outputStream -> outputStream.write(expected.getBytes(StandardCharsets.UTF_8)); - verifyBlobWriteTo(expected, Blobs.from(writer)); + verifyBlobWriteTo(expected, Blobs.from(writableContents)); } @Test @@ -107,10 +107,8 @@ private void verifyBlobWriteTo(String expected, Blob blob) throws IOException { byte[] expectedBytes = expected.getBytes(StandardCharsets.UTF_8); Assert.assertEquals(expectedBytes.length, blobDescriptor.getSize()); - CountingDigestOutputStream countingDigestOutputStream = - new CountingDigestOutputStream(Mockito.mock(OutputStream.class)); - countingDigestOutputStream.write(expectedBytes); - DescriptorDigest expectedDigest = countingDigestOutputStream.toBlobDescriptor().getDigest(); + DescriptorDigest expectedDigest = + DigestUtil.computeDigest(new ByteArrayInputStream(expectedBytes)).getDigest(); Assert.assertEquals(expectedDigest, blobDescriptor.getDigest()); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriterTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriterTest.java index 53e9c78612..981a98cf8c 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriterTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/DefaultCacheStorageWriterTest.java @@ -19,7 +19,6 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.blob.Blobs; -import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.ImageReference; import com.google.cloud.tools.jib.image.InvalidImageReferenceException; @@ -46,15 +45,8 @@ /** Tests for {@link DefaultCacheStorageWriter}. */ public class DefaultCacheStorageWriterTest { - private static DescriptorDigest getDigest(Blob blob) throws IOException { - return blob.writeTo(new CountingDigestOutputStream(ByteStreams.nullOutputStream())).getDigest(); - } - - private static BlobDescriptor getCompressedBlobDescriptor(Blob blob) throws IOException { - CountingDigestOutputStream compressedDigestOutputStream = - new CountingDigestOutputStream(ByteStreams.nullOutputStream()); - compress(blob).writeTo(compressedDigestOutputStream); - return compressedDigestOutputStream.toBlobDescriptor(); + private static BlobDescriptor getDigest(Blob blob) throws IOException { + return blob.writeTo(ByteStreams.nullOutputStream()); } private static Blob compress(Blob blob) { @@ -95,8 +87,8 @@ public void testWrite_compressed() throws IOException { @Test public void testWrite_uncompressed() throws IOException { Blob uncompressedLayerBlob = Blobs.from("uncompressedLayerBlob"); - DescriptorDigest layerDigest = getCompressedBlobDescriptor(uncompressedLayerBlob).getDigest(); - DescriptorDigest selector = getDigest(Blobs.from("selector")); + DescriptorDigest layerDigest = getDigest(compress(uncompressedLayerBlob)).getDigest(); + DescriptorDigest selector = getDigest(Blobs.from("selector")).getDigest(); CachedLayer cachedLayer = new DefaultCacheStorageWriter(defaultCacheStorageFiles) @@ -169,8 +161,8 @@ public void testWriteMetadata_v22() private void verifyCachedLayer(CachedLayer cachedLayer, Blob uncompressedLayerBlob) throws IOException { - BlobDescriptor layerBlobDescriptor = getCompressedBlobDescriptor(uncompressedLayerBlob); - DescriptorDigest layerDiffId = getDigest(uncompressedLayerBlob); + BlobDescriptor layerBlobDescriptor = getDigest(compress(uncompressedLayerBlob)); + DescriptorDigest layerDiffId = getDigest(uncompressedLayerBlob).getDigest(); // Verifies cachedLayer is correct. Assert.assertEquals(layerBlobDescriptor.getDigest(), cachedLayer.getDigest()); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/LayerEntriesSelectorTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/LayerEntriesSelectorTest.java index b8181d2e55..7337c9fc2b 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/LayerEntriesSelectorTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/LayerEntriesSelectorTest.java @@ -19,11 +19,10 @@ import com.google.cloud.tools.jib.cache.LayerEntriesSelector.LayerEntryTemplate; import com.google.cloud.tools.jib.configuration.FilePermissions; import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.LayerEntry; -import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.collect.ImmutableList; -import com.google.common.io.ByteStreams; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -108,10 +107,7 @@ public void testToSortedJsonTemplates() throws IOException { @Test public void testGenerateSelector_empty() throws IOException { - DescriptorDigest expectedSelector = - JsonTemplateMapper.toBlob(ImmutableList.of()) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DescriptorDigest expectedSelector = DigestUtil.computeJsonDigest(ImmutableList.of()); Assert.assertEquals( expectedSelector, LayerEntriesSelector.generateSelector(ImmutableList.of())); } @@ -119,9 +115,7 @@ public void testGenerateSelector_empty() throws IOException { @Test public void testGenerateSelector() throws IOException { DescriptorDigest expectedSelector = - JsonTemplateMapper.toBlob(toLayerEntryTemplates(inOrderLayerEntries)) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DigestUtil.computeJsonDigest(toLayerEntryTemplates(inOrderLayerEntries)); Assert.assertEquals( expectedSelector, LayerEntriesSelector.generateSelector(outOfOrderLayerEntries)); } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/json/DockerLoadManifestTemplateTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/json/DockerLoadManifestTemplateTest.java index 8669f6467f..2f45ed5f53 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/docker/json/DockerLoadManifestTemplateTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/docker/json/DockerLoadManifestTemplateTest.java @@ -16,7 +16,6 @@ package com.google.cloud.tools.jib.docker.json; -import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.image.ImageReference; import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.io.Resources; @@ -48,6 +47,6 @@ public void testToJson() throws URISyntaxException, IOException { template.addLayerFile("layer3.tar.gz"); List loadManifest = Collections.singletonList(template); - Assert.assertEquals(expectedJson, Blobs.writeToString(JsonTemplateMapper.toBlob(loadManifest))); + Assert.assertEquals(expectedJson, JsonTemplateMapper.toUtf8String(loadManifest)); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStreamTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStreamTest.java index a2b1b40f7e..b4901f2615 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStreamTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/hash/CountingDigestOutputStreamTest.java @@ -16,7 +16,6 @@ package com.google.cloud.tools.jib.hash; -import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteStreams; @@ -57,10 +56,9 @@ public void test_smokeTest() throws IOException, DigestException { InputStream toHashInputStream = new ByteArrayInputStream(bytesToHash); ByteStreams.copy(toHashInputStream, countingDigestOutputStream); - BlobDescriptor expectedBlobDescriptor = - new BlobDescriptor(bytesToHash.length, DescriptorDigest.fromHash(expectedHash)); - Assert.assertEquals(expectedBlobDescriptor, countingDigestOutputStream.toBlobDescriptor()); - Assert.assertEquals(bytesToHash.length, countingDigestOutputStream.getTotalBytes()); + Assert.assertEquals( + DescriptorDigest.fromHash(expectedHash), countingDigestOutputStream.getDigest()); + Assert.assertEquals(bytesToHash.length, countingDigestOutputStream.getBytesHahsed()); } } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/http/TestBlobProgressListener.java b/jib-core/src/test/java/com/google/cloud/tools/jib/http/TestBlobProgressListener.java index e0e24d0874..3fd4a0adfe 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/http/TestBlobProgressListener.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/http/TestBlobProgressListener.java @@ -23,7 +23,7 @@ * Test implementation of {@link BlobProgressListener} that always forwards to a {@link Consumer * }. */ -public class TestBlobProgressListener implements BlobProgressListener, Consumer { +public class TestBlobProgressListener implements BlobProgressListener { private final Consumer byteCountConsumer; @@ -40,9 +40,4 @@ public void handleByteCount(long byteCount) { public Duration getDelayBetweenCallbacks() { return Duration.ofSeconds(-1); } - - @Override - public void accept(Long byteCount) { - handleByteCount(byteCount); - } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ContainerConfigurationTemplateTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ContainerConfigurationTemplateTest.java index 98366aacaa..e0e5423b16 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ContainerConfigurationTemplateTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ContainerConfigurationTemplateTest.java @@ -22,7 +22,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSortedMap; import com.google.common.io.Resources; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; @@ -91,10 +90,7 @@ public void testToJson() throws IOException, URISyntaxException, DigestException .build()); // Serializes the JSON object. - ByteArrayOutputStream jsonStream = new ByteArrayOutputStream(); - JsonTemplateMapper.toBlob(containerConfigJson).writeTo(jsonStream); - - Assert.assertEquals(expectedJson, jsonStream.toString()); + Assert.assertEquals(expectedJson, JsonTemplateMapper.toUtf8String(containerConfigJson)); } @Test diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslatorTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslatorTest.java index f7a5f68524..f7d333c3c6 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslatorTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslatorTest.java @@ -190,9 +190,6 @@ private void testGetManifest( T manifestTemplate = imageToJsonTranslator.getManifestTemplate(manifestTemplateClass, blobDescriptor); - ByteArrayOutputStream jsonStream = new ByteArrayOutputStream(); - JsonTemplateMapper.toBlob(manifestTemplate).writeTo(jsonStream); - - Assert.assertEquals(expectedJson, new String(jsonStream.toByteArray(), StandardCharsets.UTF_8)); + Assert.assertEquals(expectedJson, JsonTemplateMapper.toUtf8String(manifestTemplate)); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/OCIManifestTemplateTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/OCIManifestTemplateTest.java index 40ff8e4fc1..7c7defd779 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/OCIManifestTemplateTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/OCIManifestTemplateTest.java @@ -19,7 +19,6 @@ import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.io.Resources; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; @@ -53,10 +52,7 @@ public void testToJson() throws DigestException, IOException, URISyntaxException "4945ba5011739b0b98c4a41afe224e417f47c7c99b2ce76830999c9a0861b236")); // Serializes the JSON object. - ByteArrayOutputStream jsonStream = new ByteArrayOutputStream(); - JsonTemplateMapper.toBlob(manifestJson).writeTo(jsonStream); - - Assert.assertEquals(expectedJson, jsonStream.toString()); + Assert.assertEquals(expectedJson, JsonTemplateMapper.toUtf8String(manifestJson)); } @Test diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/V22ManifestTemplateTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/V22ManifestTemplateTest.java index 58eb1a3486..c8d0ba8d55 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/V22ManifestTemplateTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/V22ManifestTemplateTest.java @@ -19,7 +19,6 @@ import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.io.Resources; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; @@ -53,10 +52,7 @@ public void testToJson() throws DigestException, IOException, URISyntaxException "4945ba5011739b0b98c4a41afe224e417f47c7c99b2ce76830999c9a0861b236")); // Serializes the JSON object. - ByteArrayOutputStream jsonStream = new ByteArrayOutputStream(); - JsonTemplateMapper.toBlob(manifestJson).writeTo(jsonStream); - - Assert.assertEquals(expectedJson, jsonStream.toString()); + Assert.assertEquals(expectedJson, JsonTemplateMapper.toUtf8String(manifestJson)); } @Test diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/json/JsonTemplateMapperTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/json/JsonTemplateMapperTest.java index 1da54bad3d..4d2ea955c5 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/json/JsonTemplateMapperTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/json/JsonTemplateMapperTest.java @@ -18,7 +18,6 @@ import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.common.io.Resources; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; @@ -85,10 +84,7 @@ public void testWriteJson() throws DigestException, IOException, URISyntaxExcept "sha256:d38f571aa1c11e3d516e0ef7e513e7308ccbeb869770cb8c4319d63b10a0075e")); testJson.list = Arrays.asList(innerObject1, innerObject2); - ByteArrayOutputStream jsonStream = new ByteArrayOutputStream(); - JsonTemplateMapper.toBlob(testJson).writeTo(jsonStream); - - Assert.assertEquals(expectedJson, jsonStream.toString()); + Assert.assertEquals(expectedJson, JsonTemplateMapper.toUtf8String(testJson)); } @Test @@ -155,9 +151,6 @@ public void testToBlob_listOfJson() throws IOException, URISyntaxException { String jsonString = new String(Files.readAllBytes(jsonFile), StandardCharsets.UTF_8); List listOfJson = JsonTemplateMapper.readListOfJson(jsonString, TestJson.class); - ByteArrayOutputStream jsonStream = new ByteArrayOutputStream(); - JsonTemplateMapper.toBlob(listOfJson).writeTo(jsonStream); - - Assert.assertEquals(jsonString, jsonStream.toString()); + Assert.assertEquals(jsonString, JsonTemplateMapper.toUtf8String(listOfJson)); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobCheckerTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobCheckerTest.java index 2ec250c417..a45833830e 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobCheckerTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobCheckerTest.java @@ -19,7 +19,6 @@ import com.google.api.client.http.HttpResponseException; import com.google.api.client.http.HttpStatusCodes; import com.google.cloud.tools.jib.blob.BlobDescriptor; -import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.http.Response; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.json.JsonTemplateMapper; @@ -92,7 +91,7 @@ public void testHandleHttpResponseException() throws IOException, RegistryErrorE new ErrorResponseTemplate() .addError(new ErrorEntryTemplate(ErrorCodes.BLOB_UNKNOWN.name(), "some message")); Mockito.when(mockHttpResponseException.getContent()) - .thenReturn(Blobs.writeToString(JsonTemplateMapper.toBlob(emptyErrorResponseTemplate))); + .thenReturn(JsonTemplateMapper.toUtf8String(emptyErrorResponseTemplate)); BlobDescriptor blobDescriptor = testBlobChecker.handleHttpResponseException(mockHttpResponseException); @@ -112,7 +111,7 @@ public void testHandleHttpResponseException_hasOtherErrors() .addError(new ErrorEntryTemplate(ErrorCodes.BLOB_UNKNOWN.name(), "some message")) .addError(new ErrorEntryTemplate(ErrorCodes.MANIFEST_UNKNOWN.name(), "some message")); Mockito.when(mockHttpResponseException.getContent()) - .thenReturn(Blobs.writeToString(JsonTemplateMapper.toBlob(emptyErrorResponseTemplate))); + .thenReturn(JsonTemplateMapper.toUtf8String(emptyErrorResponseTemplate)); try { testBlobChecker.handleHttpResponseException(mockHttpResponseException); @@ -132,7 +131,7 @@ public void testHandleHttpResponseException_notBlobUnknown() ErrorResponseTemplate emptyErrorResponseTemplate = new ErrorResponseTemplate(); Mockito.when(mockHttpResponseException.getContent()) - .thenReturn(Blobs.writeToString(JsonTemplateMapper.toBlob(emptyErrorResponseTemplate))); + .thenReturn(JsonTemplateMapper.toUtf8String(emptyErrorResponseTemplate)); try { testBlobChecker.handleHttpResponseException(mockHttpResponseException); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java index e0eb9311cd..a21f4bd725 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/BlobPullerTest.java @@ -87,7 +87,7 @@ public void testHandleResponse() throws IOException, UnexpectedBlobDigestExcepti Assert.assertEquals( "some BLOB content", new String(layerContentOutputStream.toByteArray(), StandardCharsets.UTF_8)); - Assert.assertEquals(testBlobDigest, layerOutputStream.toBlobDescriptor().getDigest()); + Assert.assertEquals(testBlobDigest, layerOutputStream.getDigest()); Assert.assertEquals("some BLOB content".length(), byteCount.sum()); } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPusherTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPusherTest.java index 57b8fdb0f8..c15ca8dfa9 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPusherTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPusherTest.java @@ -20,12 +20,12 @@ import com.google.api.client.http.HttpResponseException; import com.google.cloud.tools.jib.event.EventDispatcher; import com.google.cloud.tools.jib.event.events.LogEvent; +import com.google.cloud.tools.jib.hash.DigestUtil; import com.google.cloud.tools.jib.http.BlobHttpContent; import com.google.cloud.tools.jib.http.Response; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.json.V22ManifestTemplate; import com.google.cloud.tools.jib.json.JsonTemplateMapper; -import com.google.common.io.ByteStreams; import com.google.common.io.Resources; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -90,10 +90,7 @@ public void testGetContent() throws IOException { @Test public void testHandleResponse_valid() throws IOException { - DescriptorDigest expectedDigest = - JsonTemplateMapper.toBlob(fakeManifestTemplate) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DescriptorDigest expectedDigest = DigestUtil.computeJsonDigest(fakeManifestTemplate); Mockito.when(mockResponse.getHeader("Docker-Content-Digest")) .thenReturn(Collections.singletonList(expectedDigest.toString())); Assert.assertEquals(expectedDigest, testManifestPusher.handleResponse(mockResponse)); @@ -101,10 +98,7 @@ public void testHandleResponse_valid() throws IOException { @Test public void testHandleResponse_noDigest() throws IOException { - DescriptorDigest expectedDigest = - JsonTemplateMapper.toBlob(fakeManifestTemplate) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DescriptorDigest expectedDigest = DigestUtil.computeJsonDigest(fakeManifestTemplate); Mockito.when(mockResponse.getHeader("Docker-Content-Digest")) .thenReturn(Collections.emptyList()); @@ -115,10 +109,7 @@ public void testHandleResponse_noDigest() throws IOException { @Test public void testHandleResponse_multipleDigests() throws IOException { - DescriptorDigest expectedDigest = - JsonTemplateMapper.toBlob(fakeManifestTemplate) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DescriptorDigest expectedDigest = DigestUtil.computeJsonDigest(fakeManifestTemplate); Mockito.when(mockResponse.getHeader("Docker-Content-Digest")) .thenReturn(Arrays.asList("too", "many")); @@ -130,10 +121,7 @@ public void testHandleResponse_multipleDigests() throws IOException { @Test public void testHandleResponse_invalidDigest() throws IOException { - DescriptorDigest expectedDigest = - JsonTemplateMapper.toBlob(fakeManifestTemplate) - .writeTo(ByteStreams.nullOutputStream()) - .getDigest(); + DescriptorDigest expectedDigest = DigestUtil.computeJsonDigest(fakeManifestTemplate); Mockito.when(mockResponse.getHeader("Docker-Content-Digest")) .thenReturn(Collections.singletonList("not valid")); From f293268184ad92bdd7857748c2f9da2416d44b0f Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Wed, 1 May 2019 17:02:13 -0400 Subject: [PATCH 2/4] Fix wrong filename --- .../{WriterableContentsBlob.java => WritableContentsBlob.java} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename jib-core/src/main/java/com/google/cloud/tools/jib/blob/{WriterableContentsBlob.java => WritableContentsBlob.java} (100%) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterableContentsBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WritableContentsBlob.java similarity index 100% rename from jib-core/src/main/java/com/google/cloud/tools/jib/blob/WriterableContentsBlob.java rename to jib-core/src/main/java/com/google/cloud/tools/jib/blob/WritableContentsBlob.java From e4461b4a0f3e38fcd94e00d116fd06874f1f2b17 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Wed, 1 May 2019 17:10:37 -0400 Subject: [PATCH 3/4] Remove more Blob indirection --- .../google/cloud/tools/jib/blob/Blobs.java | 31 ------------------- .../tools/jib/cache/CacheStorageWriter.java | 6 +++- .../google/cloud/tools/jib/blob/BlobTest.java | 30 ------------------ .../jib/cache/CacheStorageReaderTest.java | 9 ++++-- 4 files changed, 11 insertions(+), 65 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java index cb621b232f..643572f529 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java @@ -16,15 +16,12 @@ package com.google.cloud.tools.jib.blob; -import com.google.cloud.tools.jib.filesystem.FileOperations; import com.google.cloud.tools.jib.hash.WritableContents; import com.google.cloud.tools.jib.json.JsonTemplate; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.nio.file.Path; /** Static methods for {@link Blob}. */ @@ -80,33 +77,5 @@ public static byte[] writeToByteArray(Blob blob) throws IOException { return byteArrayOutputStream.toByteArray(); } - /** - * Writes the BLOB to a file. - * - * @param blob the BLOB to to write - * @param blobFile the file to write to - * @return the {@link BlobDescriptor} of the written BLOB - * @throws IOException if writing the BLOB fails - */ - public static BlobDescriptor writeToFile(Blob blob, Path blobFile) throws IOException { - try (OutputStream outputStream = Files.newOutputStream(blobFile)) { - return blob.writeTo(outputStream); - } - } - - /** - * Writes the BLOB to a file with an exclusive lock. - * - * @param blob the BLOB to to write - * @param blobFile the file to write to - * @return the {@link BlobDescriptor} of the written BLOB - * @throws IOException if writing the BLOB fails - */ - public static BlobDescriptor writeToFileWithLock(Blob blob, Path blobFile) throws IOException { - try (OutputStream outputStream = FileOperations.newLockingOutputStream(blobFile)) { - return blob.writeTo(outputStream); - } - } - private Blobs() {} } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java index 7a55c63a25..97848c00d1 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java @@ -19,6 +19,7 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.blob.Blobs; +import com.google.cloud.tools.jib.filesystem.FileOperations; import com.google.cloud.tools.jib.filesystem.LockFile; import com.google.cloud.tools.jib.filesystem.TemporaryDirectory; import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; @@ -36,6 +37,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.FileSystemException; import java.nio.file.Files; @@ -364,7 +366,9 @@ private void writeSelector(DescriptorDigest selector, DescriptorDigest layerDige // Writes the selector to a temporary file and then moves the file to the intended location. Path temporarySelectorFile = Files.createTempFile(null, null); temporarySelectorFile.toFile().deleteOnExit(); - Blobs.writeToFileWithLock(Blobs.from(layerDigest.getHash()), temporarySelectorFile); + try (OutputStream fileOut = FileOperations.newLockingOutputStream(temporarySelectorFile)) { + fileOut.write(layerDigest.getHash().getBytes(StandardCharsets.UTF_8)); + } // Attempts an atomic move first, and falls back to non-atomic if the file system does not // support atomic moves. diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java index 721b36b257..edc0431433 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java @@ -30,7 +30,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; import org.junit.Assert; import org.junit.Test; @@ -67,35 +66,6 @@ public void testFromWritableContents() throws IOException { verifyBlobWriteTo(expected, Blobs.from(writableContents)); } - @Test - public void testWriteToFileWithLock_newFile() throws IOException { - String expected = "crepecake"; - Path blobFile = Files.createTempFile("blob", "bin"); - Assert.assertTrue(Files.deleteIfExists(blobFile)); // ensure it doesn't exist - - Blobs.writeToFileWithLock(Blobs.from(expected), blobFile); - - Assert.assertTrue(Files.exists(blobFile)); - verifyBlobWriteTo(expected, Blobs.from(blobFile)); - } - - @Test - public void testWriteToFileWithLock_existingFile() throws IOException { - Path blobFile = Files.createTempFile("blob", "bin"); - // write out more bytes to ensure properly truncated - byte[] dataBytes = new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - Files.write(blobFile, dataBytes, StandardOpenOption.WRITE); - Assert.assertTrue(Files.exists(blobFile)); - Assert.assertEquals(10, Files.size(blobFile)); - - String expected = "crepecake"; - Blobs.writeToFileWithLock(Blobs.from(expected), blobFile); - - Assert.assertTrue(Files.exists(blobFile)); - Assert.assertEquals(9, Files.size(blobFile)); - verifyBlobWriteTo(expected, Blobs.from(blobFile)); - } - /** Checks that the {@link Blob} streams the expected string. */ private void verifyBlobWriteTo(String expected, Blob blob) throws IOException { OutputStream outputStream = new ByteArrayOutputStream(); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java index f3205759ce..2d74e6c391 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java @@ -16,10 +16,11 @@ package com.google.cloud.tools.jib.cache; -import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.image.DescriptorDigest; import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.security.DigestException; @@ -89,9 +90,11 @@ public void testRetrieve() throws IOException, CacheCorruptedException { // Creates the test layer directory. DescriptorDigest layerDigest = layerDigest1; DescriptorDigest layerDiffId = layerDigest2; - Blob layerBlob = Blobs.from("layerBlob"); Files.createDirectories(cacheStorageFiles.getLayerDirectory(layerDigest)); - Blobs.writeToFileWithLock(layerBlob, cacheStorageFiles.getLayerFile(layerDigest, layerDiffId)); + try (OutputStream out = + Files.newOutputStream(cacheStorageFiles.getLayerFile(layerDigest, layerDiffId))) { + out.write("layerBlob".getBytes(StandardCharsets.UTF_8)); + } // Checks that the CachedLayer is retrieved correctly. Optional optionalCachedLayer = cacheStorageReader.retrieve(layerDigest); From 005f1d8cd0a53759a339e03a61378d49425fe033 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Mon, 6 May 2019 17:01:59 -0400 Subject: [PATCH 4/4] DigestUtil --> Digests --- .../jib/registry/ManifestPusherIntegrationTest.java | 6 +++--- .../java/com/google/cloud/tools/jib/blob/FileBlob.java | 4 ++-- .../google/cloud/tools/jib/blob/InputStreamBlob.java | 4 ++-- .../java/com/google/cloud/tools/jib/blob/JsonBlob.java | 4 ++-- .../com/google/cloud/tools/jib/blob/StringBlob.java | 4 ++-- .../cloud/tools/jib/blob/WritableContentsBlob.java | 4 ++-- .../cloud/tools/jib/builder/steps/BuildResult.java | 4 ++-- .../cloud/tools/jib/builder/steps/PushImageStep.java | 4 ++-- .../cloud/tools/jib/cache/LayerEntriesSelector.java | 4 ++-- .../tools/jib/hash/{DigestUtil.java => Digests.java} | 2 +- .../cloud/tools/jib/registry/ManifestPusher.java | 4 ++-- .../java/com/google/cloud/tools/jib/blob/BlobTest.java | 4 ++-- .../tools/jib/cache/LayerEntriesSelectorTest.java | 6 +++--- .../cloud/tools/jib/registry/ManifestPusherTest.java | 10 +++++----- 14 files changed, 32 insertions(+), 32 deletions(-) rename jib-core/src/main/java/com/google/cloud/tools/jib/hash/{DigestUtil.java => Digests.java} (99%) diff --git a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPusherIntegrationTest.java b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPusherIntegrationTest.java index 7f7800e542..6be922efad 100644 --- a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPusherIntegrationTest.java +++ b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPusherIntegrationTest.java @@ -21,7 +21,7 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.event.EventDispatcher; -import com.google.cloud.tools.jib.hash.DigestUtil; +import com.google.cloud.tools.jib.hash.Digests; import com.google.cloud.tools.jib.http.TestBlobProgressListener; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.json.ManifestTemplate; @@ -113,7 +113,7 @@ public void testPush() V22ManifestTemplate manifestTemplateByDigest = registryClient.pullManifest(imageDigest.toString(), V22ManifestTemplate.class); Assert.assertEquals( - DigestUtil.computeJsonDigest(manifestTemplate), - DigestUtil.computeJsonDigest(manifestTemplateByDigest)); + Digests.computeJsonDigest(manifestTemplate), + Digests.computeJsonDigest(manifestTemplateByDigest)); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/FileBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/FileBlob.java index 7748b9bddd..7392f14116 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/FileBlob.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/FileBlob.java @@ -16,7 +16,7 @@ package com.google.cloud.tools.jib.blob; -import com.google.cloud.tools.jib.hash.DigestUtil; +import com.google.cloud.tools.jib.hash.Digests; import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; @@ -36,7 +36,7 @@ class FileBlob implements Blob { @Override public BlobDescriptor writeTo(OutputStream outputStream) throws IOException { try (InputStream fileIn = new BufferedInputStream(Files.newInputStream(file))) { - return DigestUtil.computeDigest(fileIn, outputStream); + return Digests.computeDigest(fileIn, outputStream); } } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/InputStreamBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/InputStreamBlob.java index 5e5e172f8c..a0f709d2d0 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/InputStreamBlob.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/InputStreamBlob.java @@ -16,7 +16,7 @@ package com.google.cloud.tools.jib.blob; -import com.google.cloud.tools.jib.hash.DigestUtil; +import com.google.cloud.tools.jib.hash.Digests; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -40,7 +40,7 @@ public BlobDescriptor writeTo(OutputStream outputStream) throws IOException { throw new IllegalStateException("Cannot rewrite Blob backed by an InputStream"); } try (InputStream inputStream = this.inputStream) { - return DigestUtil.computeDigest(inputStream, outputStream); + return Digests.computeDigest(inputStream, outputStream); } finally { isWritten = true; diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/JsonBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/JsonBlob.java index 83dd4c5314..5d463a5b27 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/JsonBlob.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/JsonBlob.java @@ -16,7 +16,7 @@ package com.google.cloud.tools.jib.blob; -import com.google.cloud.tools.jib.hash.DigestUtil; +import com.google.cloud.tools.jib.hash.Digests; import com.google.cloud.tools.jib.json.JsonTemplate; import java.io.IOException; import java.io.OutputStream; @@ -32,6 +32,6 @@ class JsonBlob implements Blob { @Override public BlobDescriptor writeTo(OutputStream outputStream) throws IOException { - return DigestUtil.computeDigest(template, outputStream); + return Digests.computeDigest(template, outputStream); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/StringBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/StringBlob.java index 9f61014666..5c5c29539f 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/StringBlob.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/StringBlob.java @@ -16,7 +16,7 @@ package com.google.cloud.tools.jib.blob; -import com.google.cloud.tools.jib.hash.DigestUtil; +import com.google.cloud.tools.jib.hash.Digests; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -36,7 +36,7 @@ class StringBlob implements Blob { public BlobDescriptor writeTo(OutputStream outputStream) throws IOException { try (InputStream stringIn = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8))) { - return DigestUtil.computeDigest(stringIn, outputStream); + return Digests.computeDigest(stringIn, outputStream); } } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WritableContentsBlob.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WritableContentsBlob.java index 3c6f843293..1cddad04b2 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WritableContentsBlob.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/WritableContentsBlob.java @@ -16,7 +16,7 @@ package com.google.cloud.tools.jib.blob; -import com.google.cloud.tools.jib.hash.DigestUtil; +import com.google.cloud.tools.jib.hash.Digests; import com.google.cloud.tools.jib.hash.WritableContents; import java.io.IOException; import java.io.OutputStream; @@ -32,6 +32,6 @@ class WritableContentsBlob implements Blob { @Override public BlobDescriptor writeTo(OutputStream outputStream) throws IOException { - return DigestUtil.computeDigest(writableContents, outputStream); + return Digests.computeDigest(writableContents, outputStream); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildResult.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildResult.java index edf2ba032b..836cbe1cc2 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildResult.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildResult.java @@ -17,7 +17,7 @@ package com.google.cloud.tools.jib.builder.steps; import com.google.cloud.tools.jib.blob.BlobDescriptor; -import com.google.cloud.tools.jib.hash.DigestUtil; +import com.google.cloud.tools.jib.hash.Digests; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.json.BuildableManifestTemplate; @@ -47,7 +47,7 @@ static BuildResult fromImage(Image image, Class toSortedJsonTemplates(List layerEntr */ static DescriptorDigest generateSelector(ImmutableList layerEntries) throws IOException { - return DigestUtil.computeJsonDigest(toSortedJsonTemplates(layerEntries)); + return Digests.computeJsonDigest(toSortedJsonTemplates(layerEntries)); } private LayerEntriesSelector() {} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/Digests.java similarity index 99% rename from jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java rename to jib-core/src/main/java/com/google/cloud/tools/jib/hash/Digests.java index 946083c69a..e5b0afca0d 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/hash/DigestUtil.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/hash/Digests.java @@ -33,7 +33,7 @@ // Note: intentionally this class does not depend on Blob, as Blob classes depend on this class. // TODO: BlobDescriptor is merely a tuple of (size, digest). Rename BlobDescriptor to something // more general. -public class DigestUtil { +public class Digests { public static DescriptorDigest computeJsonDigest(JsonTemplate template) throws IOException { return computeDigest(template, ByteStreams.nullOutputStream()).getDigest(); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPusher.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPusher.java index a0cf547579..eb5c9dc15f 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPusher.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPusher.java @@ -21,7 +21,7 @@ import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.event.EventDispatcher; import com.google.cloud.tools.jib.event.events.LogEvent; -import com.google.cloud.tools.jib.hash.DigestUtil; +import com.google.cloud.tools.jib.hash.Digests; import com.google.cloud.tools.jib.http.BlobHttpContent; import com.google.cloud.tools.jib.http.Response; import com.google.cloud.tools.jib.image.DescriptorDigest; @@ -125,7 +125,7 @@ public DescriptorDigest handleHttpResponseException(HttpResponseException httpRe @Override public DescriptorDigest handleResponse(Response response) throws IOException { // Checks if the image digest is as expected. - DescriptorDigest expectedDigest = DigestUtil.computeJsonDigest(manifestTemplate); + DescriptorDigest expectedDigest = Digests.computeJsonDigest(manifestTemplate); List receivedDigests = response.getHeader(RESPONSE_DIGEST_HEADER); if (receivedDigests.size() == 1) { diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java index edc0431433..e31f075dc5 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/blob/BlobTest.java @@ -16,7 +16,7 @@ package com.google.cloud.tools.jib.blob; -import com.google.cloud.tools.jib.hash.DigestUtil; +import com.google.cloud.tools.jib.hash.Digests; import com.google.cloud.tools.jib.hash.WritableContents; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.common.io.Resources; @@ -78,7 +78,7 @@ private void verifyBlobWriteTo(String expected, Blob blob) throws IOException { Assert.assertEquals(expectedBytes.length, blobDescriptor.getSize()); DescriptorDigest expectedDigest = - DigestUtil.computeDigest(new ByteArrayInputStream(expectedBytes)).getDigest(); + Digests.computeDigest(new ByteArrayInputStream(expectedBytes)).getDigest(); Assert.assertEquals(expectedDigest, blobDescriptor.getDigest()); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/LayerEntriesSelectorTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/LayerEntriesSelectorTest.java index 7337c9fc2b..3089ce42eb 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/LayerEntriesSelectorTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/LayerEntriesSelectorTest.java @@ -19,7 +19,7 @@ import com.google.cloud.tools.jib.cache.LayerEntriesSelector.LayerEntryTemplate; import com.google.cloud.tools.jib.configuration.FilePermissions; import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; -import com.google.cloud.tools.jib.hash.DigestUtil; +import com.google.cloud.tools.jib.hash.Digests; import com.google.cloud.tools.jib.image.DescriptorDigest; import com.google.cloud.tools.jib.image.LayerEntry; import com.google.common.collect.ImmutableList; @@ -107,7 +107,7 @@ public void testToSortedJsonTemplates() throws IOException { @Test public void testGenerateSelector_empty() throws IOException { - DescriptorDigest expectedSelector = DigestUtil.computeJsonDigest(ImmutableList.of()); + DescriptorDigest expectedSelector = Digests.computeJsonDigest(ImmutableList.of()); Assert.assertEquals( expectedSelector, LayerEntriesSelector.generateSelector(ImmutableList.of())); } @@ -115,7 +115,7 @@ public void testGenerateSelector_empty() throws IOException { @Test public void testGenerateSelector() throws IOException { DescriptorDigest expectedSelector = - DigestUtil.computeJsonDigest(toLayerEntryTemplates(inOrderLayerEntries)); + Digests.computeJsonDigest(toLayerEntryTemplates(inOrderLayerEntries)); Assert.assertEquals( expectedSelector, LayerEntriesSelector.generateSelector(outOfOrderLayerEntries)); } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPusherTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPusherTest.java index c15ca8dfa9..da0bae510f 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPusherTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPusherTest.java @@ -20,7 +20,7 @@ import com.google.api.client.http.HttpResponseException; import com.google.cloud.tools.jib.event.EventDispatcher; import com.google.cloud.tools.jib.event.events.LogEvent; -import com.google.cloud.tools.jib.hash.DigestUtil; +import com.google.cloud.tools.jib.hash.Digests; import com.google.cloud.tools.jib.http.BlobHttpContent; import com.google.cloud.tools.jib.http.Response; import com.google.cloud.tools.jib.image.DescriptorDigest; @@ -90,7 +90,7 @@ public void testGetContent() throws IOException { @Test public void testHandleResponse_valid() throws IOException { - DescriptorDigest expectedDigest = DigestUtil.computeJsonDigest(fakeManifestTemplate); + DescriptorDigest expectedDigest = Digests.computeJsonDigest(fakeManifestTemplate); Mockito.when(mockResponse.getHeader("Docker-Content-Digest")) .thenReturn(Collections.singletonList(expectedDigest.toString())); Assert.assertEquals(expectedDigest, testManifestPusher.handleResponse(mockResponse)); @@ -98,7 +98,7 @@ public void testHandleResponse_valid() throws IOException { @Test public void testHandleResponse_noDigest() throws IOException { - DescriptorDigest expectedDigest = DigestUtil.computeJsonDigest(fakeManifestTemplate); + DescriptorDigest expectedDigest = Digests.computeJsonDigest(fakeManifestTemplate); Mockito.when(mockResponse.getHeader("Docker-Content-Digest")) .thenReturn(Collections.emptyList()); @@ -109,7 +109,7 @@ public void testHandleResponse_noDigest() throws IOException { @Test public void testHandleResponse_multipleDigests() throws IOException { - DescriptorDigest expectedDigest = DigestUtil.computeJsonDigest(fakeManifestTemplate); + DescriptorDigest expectedDigest = Digests.computeJsonDigest(fakeManifestTemplate); Mockito.when(mockResponse.getHeader("Docker-Content-Digest")) .thenReturn(Arrays.asList("too", "many")); @@ -121,7 +121,7 @@ public void testHandleResponse_multipleDigests() throws IOException { @Test public void testHandleResponse_invalidDigest() throws IOException { - DescriptorDigest expectedDigest = DigestUtil.computeJsonDigest(fakeManifestTemplate); + DescriptorDigest expectedDigest = Digests.computeJsonDigest(fakeManifestTemplate); Mockito.when(mockResponse.getHeader("Docker-Content-Digest")) .thenReturn(Collections.singletonList("not valid"));