Skip to content

Commit

Permalink
GoogleContainerTools#3158 - [Jib core] Tar archives with same content…
Browse files Browse the repository at this point in the history 
…s are not reproducible
  • Loading branch information
@davidtron
davidtron committed Apr 8, 2021
1 parent f4d6757 commit 438e2dc
Show file tree
Hide file tree
Showing 3 changed files with 38 additions and 18 deletions.
26 changes: 18 additions & 8 deletions jib-core/src/main/java/com/google/cloud/tools/jib/image/ImageTarball.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.Collections;

/** Translates an {@link Image} to a tarball that can be loaded into Docker. */
Expand All @@ -48,6 +49,7 @@ public class ImageTarball {
private final Image image;
private final ImageReference imageReference;
private final ImmutableSet<String> allTargetImageTags;
private final Instant creationTime = Instant.EPOCH;

/**
* Instantiate with an {@link Image}.
Expand Down Expand Up @@ -88,7 +90,8 @@ private void ociWriteTo(OutputStream out) throws IOException {
DescriptorDigest digest = layer.getBlobDescriptor().getDigest();
long size = layer.getBlobDescriptor().getSize();

tarStreamBuilder.addBlobEntry(layer.getBlob(), size, "blobs/sha256/" + digest.getHash());
tarStreamBuilder.addBlobEntry(
layer.getBlob(), size, "blobs/sha256/" + digest.getHash(), creationTime);
manifest.addLayer(size, digest);
}

Expand All @@ -99,21 +102,26 @@ private void ociWriteTo(OutputStream out) throws IOException {
manifest.setContainerConfiguration(configDescriptor.getSize(), configDescriptor.getDigest());
tarStreamBuilder.addByteEntry(
JsonTemplateMapper.toByteArray(containerConfiguration),
"blobs/sha256/" + configDescriptor.getDigest().getHash());
"blobs/sha256/" + configDescriptor.getDigest().getHash(),
creationTime);

// Adds the manifest to the tarball
BlobDescriptor manifestDescriptor = Digests.computeDigest(manifest);
tarStreamBuilder.addByteEntry(
JsonTemplateMapper.toByteArray(manifest),
"blobs/sha256/" + manifestDescriptor.getDigest().getHash());
"blobs/sha256/" + manifestDescriptor.getDigest().getHash(),
creationTime);

// Adds the oci-layout and index.json
tarStreamBuilder.addByteEntry(
"{\"imageLayoutVersion\": \"1.0.0\"}".getBytes(StandardCharsets.UTF_8), "oci-layout");
"{\"imageLayoutVersion\": \"1.0.0\"}".getBytes(StandardCharsets.UTF_8),
"oci-layout",
creationTime);
OciIndexTemplate index = new OciIndexTemplate();
// TODO: figure out how to tag with allTargetImageTags
index.addManifest(manifestDescriptor, imageReference.toStringWithQualifier());
tarStreamBuilder.addByteEntry(JsonTemplateMapper.toByteArray(index), "index.json");
tarStreamBuilder.addByteEntry(
JsonTemplateMapper.toByteArray(index), "index.json", creationTime);

tarStreamBuilder.writeAsTarArchiveTo(out);
}
Expand All @@ -127,7 +135,7 @@ private void dockerWriteTo(OutputStream out) throws IOException {
String layerName = layer.getBlobDescriptor().getDigest().getHash() + LAYER_FILE_EXTENSION;

tarStreamBuilder.addBlobEntry(
layer.getBlob(), layer.getBlobDescriptor().getSize(), layerName);
layer.getBlob(), layer.getBlobDescriptor().getSize(), layerName, creationTime);
manifestTemplate.addLayerFile(layerName);
}

Expand All @@ -136,15 +144,17 @@ private void dockerWriteTo(OutputStream out) throws IOException {
new ImageToJsonTranslator(image).getContainerConfiguration();
tarStreamBuilder.addByteEntry(
JsonTemplateMapper.toByteArray(containerConfiguration),
CONTAINER_CONFIGURATION_JSON_FILE_NAME);
CONTAINER_CONFIGURATION_JSON_FILE_NAME,
creationTime);

// Adds the manifest to tarball.
for (String tag : allTargetImageTags) {
manifestTemplate.addRepoTag(imageReference.withQualifier(tag).toStringWithQualifier());
}
tarStreamBuilder.addByteEntry(
JsonTemplateMapper.toByteArray(Collections.singletonList(manifestTemplate)),
MANIFEST_JSON_FILE_NAME);
MANIFEST_JSON_FILE_NAME,
creationTime);

tarStreamBuilder.writeAsTarArchiveTo(out);
}
Expand Down
9 changes: 7 additions & 2 deletions jib-core/src/main/java/com/google/cloud/tools/jib/tar/TarStreamBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.LinkedHashMap;
import java.util.Map;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
Expand Down Expand Up @@ -71,10 +72,12 @@ public void addTarArchiveEntry(TarArchiveEntry entry) {
*
* @param contents the bytes to add to the tarball
* @param name the name of the entry (i.e. filename)
* @param modTime the time the entry is created
*/
public void addByteEntry(byte[] contents, String name) {
public void addByteEntry(byte[] contents, String name, Instant modTime) {
TarArchiveEntry entry = new TarArchiveEntry(name);
entry.setSize(contents.length);
entry.setModTime(modTime.getEpochSecond());
archiveMap.put(entry, Blobs.from(outputStream -> outputStream.write(contents)));
}

Expand All @@ -85,10 +88,12 @@ public void addByteEntry(byte[] contents, String name) {
* @param blob the {@link Blob} to add to the tarball
* @param size the size (in bytes) of {@code blob}
* @param name the name of the entry (i.e. filename)
* @param modTime the time the entry is created
*/
public void addBlobEntry(Blob blob, long size, String name) {
public void addBlobEntry(Blob blob, long size, String name, Instant modTime) {
TarArchiveEntry entry = new TarArchiveEntry(name);
entry.setSize(size);
entry.setModTime(modTime.getEpochSecond());
archiveMap.put(entry, blob);
}
}
21 changes: 13 additions & 8 deletions jib-core/src/test/java/com/google/cloud/tools/jib/tar/TarStreamBuilderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Instant;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
Expand All @@ -46,6 +47,7 @@ public class TarStreamBuilderTest {
private byte[] fileAContents;
private byte[] fileBContents;
private TarStreamBuilder testTarStreamBuilder = new TarStreamBuilder();
private final Instant creationTime = Instant.EPOCH;

@Before
public void setup() throws URISyntaxException, IOException {
Expand Down Expand Up @@ -96,10 +98,11 @@ public void testToBlob_stringsAndTarArchiveEntriesWithCompression() throws IOExc

@Test
public void testToBlob_multiByte() throws IOException {
testTarStreamBuilder.addByteEntry("日本語".getBytes(StandardCharsets.UTF_8), "test");
testTarStreamBuilder.addByteEntry("asdf".getBytes(StandardCharsets.UTF_8), "crepecake");
testTarStreamBuilder.addByteEntry("日本語".getBytes(StandardCharsets.UTF_8), "test", creationTime);
testTarStreamBuilder.addByteEntry(
"asdf".getBytes(StandardCharsets.UTF_8), "crepecake", creationTime);
testTarStreamBuilder.addBlobEntry(
Blobs.from("jib"), "jib".getBytes(StandardCharsets.UTF_8).length, "jib");
Blobs.from("jib"), "jib".getBytes(StandardCharsets.UTF_8).length, "jib", creationTime);

// Writes the BLOB and captures the output.
ByteArrayOutputStream tarByteOutputStream = new ByteArrayOutputStream();
Expand Down Expand Up @@ -148,25 +151,27 @@ private void setUpWithTarEntries() {
/** Creates a TarStreamBuilder using Strings. */
private void setUpWithStrings() {
// Prepares a test TarStreamBuilder.
testTarStreamBuilder.addByteEntry(fileAContents, "some/path/to/resourceFileA");
testTarStreamBuilder.addByteEntry(fileBContents, "crepecake");
testTarStreamBuilder.addByteEntry(fileAContents, "some/path/to/resourceFileA", creationTime);
testTarStreamBuilder.addByteEntry(fileBContents, "crepecake", creationTime);
testTarStreamBuilder.addTarArchiveEntry(
new TarArchiveEntry(directoryA.toFile(), "some/path/to"));
testTarStreamBuilder.addByteEntry(
fileAContents,
"some/really/long/path/that/exceeds/100/characters/abcdefghijklmnopqrstuvwxyz0123456789012345678901234567890");
"some/really/long/path/that/exceeds/100/characters/abcdefghijklmnopqrstuvwxyz0123456789012345678901234567890",
creationTime);
}

/** Creates a TarStreamBuilder using Strings and TarArchiveEntries. */
private void setUpWithStringsAndTarEntries() {
// Prepares a test TarStreamBuilder.
testTarStreamBuilder.addByteEntry(fileAContents, "some/path/to/resourceFileA");
testTarStreamBuilder.addByteEntry(fileAContents, "some/path/to/resourceFileA", creationTime);
testTarStreamBuilder.addTarArchiveEntry(new TarArchiveEntry(fileB.toFile(), "crepecake"));
testTarStreamBuilder.addTarArchiveEntry(
new TarArchiveEntry(directoryA.toFile(), "some/path/to"));
testTarStreamBuilder.addByteEntry(
fileAContents,
"some/really/long/path/that/exceeds/100/characters/abcdefghijklmnopqrstuvwxyz0123456789012345678901234567890");
"some/really/long/path/that/exceeds/100/characters/abcdefghijklmnopqrstuvwxyz0123456789012345678901234567890",
creationTime);
}

/** Creates a compressed blob from the TarStreamBuilder and verifies it. */
Expand Down

0 comments on commit 438e2dc

Please sign in to comment.