Reorg and refactor repo - continued (#1684)

* Decouple Blob
* Rename ImageToTarTranslator
* Decouple Blob from TarStreamBuilder
* Remove Blob dep from DigestUtil
* Move WritableContents to hash package
* Decouple Blob from http Response
* Decouple Blob from http Request
* Remove more Blob indirection
* Push up throttling decision
* Rename "with..." to "from..."
* Specialize ThrottledConsumer
* ListenableCountingOutputStream -> NotifyingOutputStream
* optionalOutStream -> outStream
* ThrottledLongConsumer -> ThrottledAccumulatingConsumer

jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPullerIntegrationTest.java

@@ -20,7 +20,6 @@
 import com.google.cloud.tools.jib.api.RegistryException;
 import com.google.cloud.tools.jib.blob.Blob;
 import com.google.cloud.tools.jib.event.EventDispatcher;
-import com.google.cloud.tools.jib.http.TestBlobProgressListener;
 import com.google.cloud.tools.jib.image.json.V21ManifestTemplate;
 import com.google.common.io.ByteStreams;
 import java.io.IOException;
@@ -64,7 +63,7 @@ public void testPull() throws IOException, RegistryException, InterruptedExcepti
               Assert.assertEquals(0, expectedSize.sum());
               expectedSize.add(size);
             },
-            new TestBlobProgressListener(totalByteCount::add));
+            totalByteCount::add);
     Assert.assertEquals(realDigest, pulledBlob.writeTo(ByteStreams.nullOutputStream()).getDigest());
     Assert.assertTrue(expectedSize.sum() > 0);
     Assert.assertEquals(expectedSize.sum(), totalByteCount.sum());
@@ -84,7 +83,7 @@ public void testPull_unknownBlob() throws IOException, DigestException, Interrup
 
     try {
       registryClient
-          .pullBlob(nonexistentDigest, ignored -> {}, new TestBlobProgressListener(ignored -> {}))
+          .pullBlob(nonexistentDigest, ignored -> {}, ignored -> {})
           .writeTo(ByteStreams.nullOutputStream());
       Assert.fail("Trying to pull nonexistent blob should have errored");
 

jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/BlobPusherIntegrationTest.java

@@ -21,7 +21,6 @@
 import com.google.cloud.tools.jib.blob.Blob;
 import com.google.cloud.tools.jib.blob.Blobs;
 import com.google.cloud.tools.jib.event.EventDispatcher;
-import com.google.cloud.tools.jib.http.TestBlobProgressListener;
 import java.io.IOException;
 import java.security.DigestException;
 import org.junit.Assert;
@@ -48,8 +47,6 @@ public void testPush()
         RegistryClient.factory(EVENT_DISPATCHER, "localhost:5000", "testimage")
             .setAllowInsecureRegistries(true)
             .newRegistryClient();
-    Assert.assertFalse(
-        registryClient.pushBlob(
-            testBlobDigest, testBlob, null, new TestBlobProgressListener(ignored -> {})));
+    Assert.assertFalse(registryClient.pushBlob(testBlobDigest, testBlob, null, ignored -> {}));
   }
 }

jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPusherIntegrationTest.java

@@ -24,7 +24,6 @@
 import com.google.cloud.tools.jib.blob.Blobs;
 import com.google.cloud.tools.jib.event.EventDispatcher;
 import com.google.cloud.tools.jib.hash.Digests;
-import com.google.cloud.tools.jib.http.TestBlobProgressListener;
 import com.google.cloud.tools.jib.image.json.ManifestTemplate;
 import com.google.cloud.tools.jib.image.json.V22ManifestTemplate;
 import java.io.IOException;
@@ -88,14 +87,13 @@ public void testPush()
             .setAllowInsecureRegistries(true)
             .newRegistryClient();
     Assert.assertFalse(
-        registryClient.pushBlob(
-            testLayerBlobDigest, testLayerBlob, null, new TestBlobProgressListener(ignored -> {})));
+        registryClient.pushBlob(testLayerBlobDigest, testLayerBlob, null, ignored -> {}));
     Assert.assertFalse(
         registryClient.pushBlob(
             testContainerConfigurationBlobDigest,
             testContainerConfigurationBlob,
             null,
-            new TestBlobProgressListener(ignored -> {})));
+            ignored -> {}));
 
     // Pushes the manifest.
     DescriptorDigest imageDigest = registryClient.pushManifest(expectedManifestTemplate, "latest");

jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java

@@ -28,7 +28,6 @@
 import com.google.cloud.tools.jib.configuration.BuildConfiguration;
 import com.google.cloud.tools.jib.configuration.credentials.Credential;
 import com.google.cloud.tools.jib.http.Authorization;
-import com.google.cloud.tools.jib.http.Authorizations;
 import com.google.cloud.tools.jib.registry.RegistryAuthenticator;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.ListeningExecutorService;
@@ -102,7 +101,7 @@ public Authorization call()
 
     return (registryCredential == null || registryCredential.isOAuth2RefreshToken())
         ? null
-        : Authorizations.withBasicCredentials(
+        : Authorization.fromBasicCredentials(
             registryCredential.getUsername(), registryCredential.getPassword());
   }
 }

jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildResult.java

@@ -22,7 +22,6 @@
 import com.google.cloud.tools.jib.image.Image;
 import com.google.cloud.tools.jib.image.json.BuildableManifestTemplate;
 import com.google.cloud.tools.jib.image.json.ImageToJsonTranslator;
-import com.google.common.io.ByteStreams;
 import java.io.IOException;
 import java.util.Objects;
 
@@ -41,9 +40,7 @@ static BuildResult fromImage(Image image, Class<? extends BuildableManifestTempl
       throws IOException {
     ImageToJsonTranslator imageToJsonTranslator = new ImageToJsonTranslator(image);
     BlobDescriptor containerConfigurationBlobDescriptor =
-        imageToJsonTranslator
-            .getContainerConfigurationBlob()
-            .writeTo(ByteStreams.nullOutputStream());
+        Digests.computeDigest(imageToJsonTranslator.getContainerConfiguration());
     BuildableManifestTemplate manifestTemplate =
         imageToJsonTranslator.getManifestTemplate(
             targetFormat, containerConfigurationBlobDescriptor);

jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java

@@ -95,16 +95,16 @@ public CachedLayer call() throws IOException, CacheCorruptedException {
               .setAuthorization(pullAuthorization)
               .newRegistryClient();
 
-      try (ProgressEventDispatcherContainer progressEventDispatcherContainer =
-          new ProgressEventDispatcherContainer(
+      try (ThrottledProgressEventDispatcherWrapper progressEventDispatcherWrapper =
+          new ThrottledProgressEventDispatcherWrapper(
               progressEventDispatcher.newChildProducer(),
               "pulling base image layer " + layerDigest,
               BuildStepType.PULL_AND_CACHE_BASE_IMAGE_LAYER)) {
         return cache.writeCompressedLayer(
             registryClient.pullBlob(
                 layerDigest,
-                progressEventDispatcherContainer::initializeWithBlobSize,
-                progressEventDispatcherContainer));
+                progressEventDispatcherWrapper::setProgressTarget,
+                progressEventDispatcherWrapper::dispatchProgress));
       }
     }
   }

jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java

@@ -36,7 +36,6 @@
 import com.google.cloud.tools.jib.event.events.LogEvent;
 import com.google.cloud.tools.jib.event.events.ProgressEvent;
 import com.google.cloud.tools.jib.http.Authorization;
-import com.google.cloud.tools.jib.http.Authorizations;
 import com.google.cloud.tools.jib.image.Image;
 import com.google.cloud.tools.jib.image.LayerCountMismatchException;
 import com.google.cloud.tools.jib.image.LayerPropertyNotFoundException;
@@ -163,7 +162,7 @@ public BaseImageWithAuthorization call()
         Authorization registryAuthorization =
             registryCredential == null || registryCredential.isOAuth2RefreshToken()
                 ? null
-                : Authorizations.withBasicCredentials(
+                : Authorization.fromBasicCredentials(
                     registryCredential.getUsername(), registryCredential.getPassword());
 
         try {
@@ -251,17 +250,17 @@ private Image pullBaseImage(
         DescriptorDigest containerConfigurationDigest =
             buildableManifestTemplate.getContainerConfiguration().getDigest();
 
-        try (ProgressEventDispatcherContainer progressEventDispatcherContainer =
-            new ProgressEventDispatcherContainer(
+        try (ThrottledProgressEventDispatcherWrapper progressEventDispatcherWrapper =
+            new ThrottledProgressEventDispatcherWrapper(
                 progressEventDispatcher.newChildProducer(),
                 "pull container configuration " + containerConfigurationDigest,
                 BuildStepType.PULL_BASE_IMAGE)) {
           String containerConfigurationString =
               Blobs.writeToString(
                   registryClient.pullBlob(
                       containerConfigurationDigest,
-                      progressEventDispatcherContainer::initializeWithBlobSize,
-                      progressEventDispatcherContainer));
+                      progressEventDispatcherWrapper::setProgressTarget,
+                      progressEventDispatcherWrapper::dispatchProgress));
 
           ContainerConfigurationTemplate containerConfigurationTemplate =
               JsonTemplateMapper.readJson(

jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java

@@ -27,37 +27,17 @@
 import com.google.cloud.tools.jib.builder.TimerEventDispatcher;
 import com.google.cloud.tools.jib.configuration.BuildConfiguration;
 import com.google.cloud.tools.jib.event.events.LogEvent;
-import com.google.cloud.tools.jib.http.BlobProgressListener;
+import com.google.cloud.tools.jib.event.progress.ThrottledAccumulatingConsumer;
 import com.google.cloud.tools.jib.registry.RegistryClient;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.ListeningExecutorService;
 import java.io.IOException;
-import java.time.Duration;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 
 /** Pushes a BLOB to the target registry. */
 class PushBlobStep implements AsyncStep<BlobDescriptor>, Callable<BlobDescriptor> {
 
-  private static class ForwardingProgressListener implements BlobProgressListener {
-
-    private final ProgressEventDispatcher progressEventDispatcher;
-
-    private ForwardingProgressListener(ProgressEventDispatcher progressEventDispatcher) {
-      this.progressEventDispatcher = progressEventDispatcher;
-    }
-
-    @Override
-    public void handleByteCount(long byteCount) {
-      progressEventDispatcher.dispatchProgress(byteCount);
-    }
-
-    @Override
-    public Duration getDelayBetweenCallbacks() {
-      return Duration.ofMillis(100);
-    }
-  }
-
   private static final String DESCRIPTION = "Pushing BLOB ";
 
   private final BuildConfiguration buildConfiguration;
@@ -105,7 +85,9 @@ public BlobDescriptor call() throws IOException, RegistryException, ExecutionExc
                 blobDescriptor.getSize());
         TimerEventDispatcher ignored =
             new TimerEventDispatcher(
-                buildConfiguration.getEventDispatcher(), DESCRIPTION + blobDescriptor)) {
+                buildConfiguration.getEventDispatcher(), DESCRIPTION + blobDescriptor);
+        ThrottledAccumulatingConsumer throttledProgressReporter =
+            new ThrottledAccumulatingConsumer(progressEventDispatcher::dispatchProgress)) {
       RegistryClient registryClient =
           buildConfiguration
               .newTargetImageRegistryClientFactory()
@@ -121,11 +103,7 @@ public BlobDescriptor call() throws IOException, RegistryException, ExecutionExc
       }
 
       // todo: leverage cross-repository mounts
-      registryClient.pushBlob(
-          blobDescriptor.getDigest(),
-          blob,
-          null,
-          new ForwardingProgressListener(progressEventDispatcher));
+      registryClient.pushBlob(blobDescriptor.getDigest(), blob, null, throttledProgressReporter);
 
       return blobDescriptor;
     }

jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushContainerConfigurationStep.java

@@ -19,15 +19,16 @@
 import com.google.cloud.tools.jib.async.AsyncDependencies;
 import com.google.cloud.tools.jib.async.AsyncStep;
 import com.google.cloud.tools.jib.async.NonBlockingSteps;
-import com.google.cloud.tools.jib.blob.Blob;
 import com.google.cloud.tools.jib.blob.BlobDescriptor;
+import com.google.cloud.tools.jib.blob.Blobs;
 import com.google.cloud.tools.jib.builder.BuildStepType;
 import com.google.cloud.tools.jib.builder.ProgressEventDispatcher;
 import com.google.cloud.tools.jib.builder.TimerEventDispatcher;
 import com.google.cloud.tools.jib.configuration.BuildConfiguration;
+import com.google.cloud.tools.jib.hash.Digests;
 import com.google.cloud.tools.jib.image.Image;
 import com.google.cloud.tools.jib.image.json.ImageToJsonTranslator;
-import com.google.common.io.ByteStreams;
+import com.google.cloud.tools.jib.json.JsonTemplate;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.ListeningExecutorService;
 import java.io.IOException;
@@ -90,18 +91,17 @@ private PushBlobStep afterBuildConfigurationFutureFuture()
         TimerEventDispatcher ignored =
             new TimerEventDispatcher(buildConfiguration.getEventDispatcher(), DESCRIPTION)) {
       Image image = NonBlockingSteps.get(NonBlockingSteps.get(buildImageStep));
-      Blob containerConfigurationBlob =
-          new ImageToJsonTranslator(image).getContainerConfigurationBlob();
-      BlobDescriptor blobDescriptor =
-          containerConfigurationBlob.writeTo(ByteStreams.nullOutputStream());
+      JsonTemplate containerConfiguration =
+          new ImageToJsonTranslator(image).getContainerConfiguration();
+      BlobDescriptor blobDescriptor = Digests.computeDigest(containerConfiguration);
 
       return new PushBlobStep(
           listeningExecutorService,
           buildConfiguration,
           progressEventDispatcher.newChildProducer(),
           authenticatePushStep,
           blobDescriptor,
-          containerConfigurationBlob,
+          Blobs.from(containerConfiguration),
           BuildStepType.PUSH_CONTAINER_CONFIGURATION);
     }
   }

jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ProgressEventDispatcherContainer.java → jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ThrottledProgressEventDispatcherWrapper.java

@@ -18,28 +18,29 @@
 
 import com.google.cloud.tools.jib.builder.BuildStepType;
 import com.google.cloud.tools.jib.builder.ProgressEventDispatcher;
-import com.google.cloud.tools.jib.http.BlobProgressListener;
+import com.google.cloud.tools.jib.event.progress.ThrottledAccumulatingConsumer;
 import com.google.common.base.Preconditions;
 import java.io.Closeable;
-import java.time.Duration;
 import javax.annotation.Nullable;
 
 /**
- * Contains a {@link ProgressEventDispatcher}. This class is mutable and should only be used within
- * a local context.
+ * Contains a {@link ProgressEventDispatcher} and throttles dispatching progress events with the
+ * default delay used by {@link ThrottledConsumer}. This class is mutable and should only be used
+ * within a local context.
  *
  * <p>This class is necessary because the total BLOb size (allocation units) is not known until the
  * response headers are received, only after which can the {@link ProgressEventDispatcher} be
  * created.
  */
-class ProgressEventDispatcherContainer implements BlobProgressListener, Closeable {
+class ThrottledProgressEventDispatcherWrapper implements Closeable {
 
   private final ProgressEventDispatcher.Factory progressEventDispatcherFactory;
   private final String description;
   private final BuildStepType type;
   @Nullable private ProgressEventDispatcher progressEventDispatcher;
+  @Nullable private ThrottledAccumulatingConsumer throttledDispatcher;
 
-  ProgressEventDispatcherContainer(
+  ThrottledProgressEventDispatcherWrapper(
       ProgressEventDispatcher.Factory progressEventDispatcherFactory,
       String description,
       BuildStepType type) {
@@ -48,25 +49,24 @@ class ProgressEventDispatcherContainer implements BlobProgressListener, Closeabl
     this.type = type;
   }
 
-  @Override
-  public void handleByteCount(long byteCount) {
-    Preconditions.checkNotNull(progressEventDispatcher);
-    progressEventDispatcher.dispatchProgress(byteCount);
-  }
-
-  @Override
-  public Duration getDelayBetweenCallbacks() {
-    return Duration.ofMillis(100);
+  public void dispatchProgress(Long progressUnits) {
+    Preconditions.checkNotNull(throttledDispatcher);
+    throttledDispatcher.accept(progressUnits);
   }
 
   @Override
   public void close() {
     Preconditions.checkNotNull(progressEventDispatcher);
+    Preconditions.checkNotNull(throttledDispatcher);
+    throttledDispatcher.close();
     progressEventDispatcher.close();
   }
 
-  void initializeWithBlobSize(long blobSize) {
+  void setProgressTarget(long allocationUnits) {
     Preconditions.checkState(progressEventDispatcher == null);
-    progressEventDispatcher = progressEventDispatcherFactory.create(type, description, blobSize);
+    progressEventDispatcher =
+        progressEventDispatcherFactory.create(type, description, allocationUnits);
+    throttledDispatcher =
+        new ThrottledAccumulatingConsumer(progressEventDispatcher::dispatchProgress);
   }
 }

jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java

@@ -108,7 +108,7 @@ private static DescriptorDigest getDiffIdByDecompressingFile(Path compressedFile
           GZIPInputStream decompressorStream = new GZIPInputStream(fileInputStream)) {
         ByteStreams.copy(decompressorStream, diffIdCaptureOutputStream);
       }
-      return diffIdCaptureOutputStream.getDigest();
+      return diffIdCaptureOutputStream.computeDigest().getDigest();
     }
   }
 
@@ -337,8 +337,9 @@ private WrittenLayer writeUncompressedLayerBlobToDirectory(
 
       // The GZIPOutputStream must be closed in order to write out the remaining compressed data.
       compressorStream.close();
-      DescriptorDigest layerDigest = compressedDigestOutputStream.getDigest();
-      long layerSize = compressedDigestOutputStream.getBytesHahsed();
+      BlobDescriptor blobDescriptor = compressedDigestOutputStream.computeDigest();
+      DescriptorDigest layerDigest = blobDescriptor.getDigest();
+      long layerSize = blobDescriptor.getSize();
 
       // Renames the temporary layer file to the correct filename.
       Path layerFile = layerDirectory.resolve(cacheStorageFiles.getLayerFilename(layerDiffId));