Skip to content

Commit

Permalink
New cache API / refactor code
Browse files Browse the repository at this point in the history
  • Loading branch information
@chanseokoh
chanseokoh committed Aug 24, 2020
1 parent b3a0c89 commit b92f1ff
Show file tree
Hide file tree
Showing 7 changed files with 124 additions and 124 deletions.
68 changes: 35 additions & 33 deletions jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,49 +219,57 @@ private List<Image> pullBaseImages(
throws IOException, RegistryException, LayerPropertyNotFoundException,
LayerCountMismatchException, BadContainerConfigurationFormatException {
Cache cache = buildContext.getBaseImageLayersCache();
EventHandlers eventHandlers = buildContext.getEventHandlers();
ImageConfiguration baseImageConfig = buildContext.getBaseImageConfiguration();

ManifestAndDigest<?> manifestAndDigest =
registryClient.pullManifest(baseImageConfig.getImageQualifier());
ManifestTemplate manifestTemplate = manifestAndDigest.getManifest();
eventHandlers.dispatch(
LogEvent.lifecycle("Using base image with digest: " + manifestAndDigest.getDigest()));

ManifestTemplate manifestTemplate = manifestAndDigest.getManifest();
if (manifestTemplate instanceof V21ManifestTemplate) {
V21ManifestTemplate v21Manifest = (V21ManifestTemplate) manifestTemplate;
cache.writeMetadata(baseImageConfig.getImage(), v21Manifest);
return Collections.singletonList(JsonToImageTranslator.toImage(v21Manifest));
}

ManifestTemplate manifestList = null;
List<BuildableManifestTemplate> manifests = new ArrayList<>();
List<ContainerConfigurationTemplate> containerConfigs = new ArrayList<>();
// If a manifest list, search for the manifests matching the given platforms.
if (manifestTemplate instanceof V22ManifestListTemplate) {
manifestList = manifestTemplate;
for (Platform platform : buildContext.getContainerConfiguration().getPlatforms()) {
manifestAndDigest =
obtainPlatformSpecificImageManifest(
registryClient, (V22ManifestListTemplate) manifestList, platform);

manifests.add((BuildableManifestTemplate) manifestAndDigest.getManifest());
containerConfigs.add(
pullContainerConfigJson(manifestAndDigest, registryClient, progressEventDispatcher));
}

} else {
} else if (manifestTemplate instanceof BuildableManifestTemplate) {
// V22ManifestTemplate or OciManifestTemplate
// TODO: support OciIndexTemplate once AbstractManifestPuller starts to accept it.
manifests = Collections.singletonList((BuildableManifestTemplate) manifestTemplate);
containerConfigs =
Collections.singletonList(
pullContainerConfigJson(manifestAndDigest, registryClient, progressEventDispatcher));
BuildableManifestTemplate imageManifest = (BuildableManifestTemplate) manifestTemplate;
ContainerConfigurationTemplate containerConfig =
pullContainerConfigJson(manifestAndDigest, registryClient, progressEventDispatcher);
cache.writeMetadata(baseImageConfig.getImage(), imageManifest, containerConfig);
return Collections.singletonList(
JsonToImageTranslator.toImage(imageManifest, containerConfig));
}

cache.writeMetadata(baseImageConfig.getImage(), manifestList, manifests, containerConfigs);
// TODO: support OciIndexTemplate once AbstractManifestPuller starts to accept it.
Verify.verify(manifestTemplate instanceof V22ManifestListTemplate);

List<ManifestAndConfigTemplate> manifestsAndConfigs = new ArrayList<>();
ImmutableList.Builder<Image> images = ImmutableList.builder();
for (int i = 0; i < manifests.size(); i++) {
images.add(JsonToImageTranslator.toImage(manifests.get(i), containerConfigs.get(i)));
// If a manifest list, search for the manifests matching the given platforms.
for (Platform platform : buildContext.getContainerConfiguration().getPlatforms()) {
String message = "Searching for architecture=%s, os=%s in the base image manifest list";
eventHandlers.dispatch(
LogEvent.info(String.format(message, platform.getArchitecture(), platform.getOs())));

ManifestAndDigest<?> imageManifestAndDigest =
obtainPlatformSpecificImageManifest(
registryClient, (V22ManifestListTemplate) manifestTemplate, platform);

BuildableManifestTemplate imageManifest =
(BuildableManifestTemplate) imageManifestAndDigest.getManifest();
ContainerConfigurationTemplate containerConfig =
pullContainerConfigJson(imageManifestAndDigest, registryClient, progressEventDispatcher);

manifestsAndConfigs.add(new ManifestAndConfigTemplate(imageManifest, containerConfig));
images.add(JsonToImageTranslator.toImage(imageManifest, containerConfig));
}

cache.writeMetadata(
baseImageConfig.getImage(),
new ImageMetadataTemplate(manifestTemplate /* manifest list */, manifestsAndConfigs));
return images.build();
}

Expand All @@ -277,9 +285,6 @@ ManifestAndDigest<?> obtainPlatformSpecificImageManifest(
Platform platform)
throws IOException, RegistryException {
EventHandlers eventHandlers = buildContext.getEventHandlers();
String message = "Searching for architecture=%s, os=%s in the base image manifest list";
eventHandlers.dispatch(
LogEvent.info(String.format(message, platform.getArchitecture(), platform.getOs())));

List<String> digests =
manifestListTemplate.getDigestsForPlatform(platform.getArchitecture(), platform.getOs());
Expand Down Expand Up @@ -323,9 +328,6 @@ private ContainerConfigurationTemplate pullContainerConfigJson(
(BuildableManifestTemplate) manifestAndDigest.getManifest();
Preconditions.checkArgument(manifest.getSchemaVersion() == 2);

EventHandlers eventHandlers = buildContext.getEventHandlers();
eventHandlers.dispatch(
LogEvent.lifecycle("Using base image with digest: " + manifestAndDigest.getDigest()));
if (manifest.getContainerConfiguration() == null
|| manifest.getContainerConfiguration().getDigest() == null) {
throw new UnknownManifestFormatException(
Expand Down
43 changes: 29 additions & 14 deletions jib-core/src/main/java/com/google/cloud/tools/jib/cache/Cache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,15 @@
import com.google.cloud.tools.jib.image.json.BuildableManifestTemplate;
import com.google.cloud.tools.jib.image.json.ContainerConfigurationTemplate;
import com.google.cloud.tools.jib.image.json.ImageMetadataTemplate;
import com.google.cloud.tools.jib.image.json.ManifestTemplate;
import com.google.cloud.tools.jib.image.json.ManifestAndConfigTemplate;
import com.google.cloud.tools.jib.image.json.V21ManifestTemplate;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import javax.annotation.Nullable;
import javax.annotation.concurrent.Immutable;

/**
Expand Down Expand Up @@ -74,32 +73,48 @@ private Cache(CacheStorageFiles cacheStorageFiles) {
* an image reference.
*
* @param imageReference the image reference to save the metadata for
* @param manifestList the V2.2 manifest list or OCI image index. Can be null.
* @param manifests the V2.2 or OCI manifests
* @param containerConfigurations the container configurations
* @param metadata the image metadata
* @throws IOException if an I/O exception occurs
*/
public void writeMetadata(ImageReference imageReference, ImageMetadataTemplate metadata)
throws IOException {
cacheStorageWriter.writeMetadata(imageReference, metadata);
}

/**
* Saves a schema 2 manifest for an image reference. This is a simple wrapper around {@link
* #writeMetadata(ImageReference, ImageMetadataTemplate)} to save a single manifest without a
* manifest list.
*
* @param imageReference the image reference to save the manifest for
* @param manifest the V2.2 or OCI manifest
* @param containerConfiguration the container configuration
* @throws IOException if an I/O exception occurs
*/
public void writeMetadata(
ImageReference imageReference,
@Nullable ManifestTemplate manifestList,
List<BuildableManifestTemplate> manifests,
List<ContainerConfigurationTemplate> containerConfigurations)
BuildableManifestTemplate manifest,
ContainerConfigurationTemplate containerConfiguration)
throws IOException {
Preconditions.checkArgument(manifests.size() == containerConfigurations.size());
cacheStorageWriter.writeMetadata(
imageReference, manifestList, manifests, containerConfigurations);
List<ManifestAndConfigTemplate> singleton =
Collections.singletonList(new ManifestAndConfigTemplate(manifest, containerConfiguration));
cacheStorageWriter.writeMetadata(imageReference, new ImageMetadataTemplate(null, singleton));
}

/**
* Saves a V2.1 image manifest.
* Saves a V2.1 image manifest. This is a simple wrapper around {@link
* #writeMetadata(ImageReference, ImageMetadataTemplate)} to save a single manifest without a
* manifest list.
*
* @param imageReference the image reference to save the manifest for
* @param manifestTemplate the V2.1 manifest
* @throws IOException if an I/O exception occurs
*/
public void writeMetadata(ImageReference imageReference, V21ManifestTemplate manifestTemplate)
throws IOException {
cacheStorageWriter.writeMetadata(imageReference, manifestTemplate);
List<ManifestAndConfigTemplate> singleton =
Collections.singletonList(new ManifestAndConfigTemplate(manifestTemplate, null));
cacheStorageWriter.writeMetadata(imageReference, new ImageMetadataTemplate(null, singleton));
}

/**
Expand Down
74 changes: 30 additions & 44 deletions jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,9 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import javax.annotation.Nullable;
Expand All @@ -72,6 +71,32 @@ private WrittenLayer(
}
}

private static void verifyImageMetadata(ImageMetadataTemplate metadata) {
Predicate<ManifestAndConfigTemplate> isManifestNull = pair -> pair.getManifest() == null;
Predicate<ManifestAndConfigTemplate> isConfigNull = pair -> pair.getConfig() == null;

List<ManifestAndConfigTemplate> manifestsAndConfigs = metadata.getManifestsAndConfigs();
Preconditions.checkArgument(!manifestsAndConfigs.isEmpty(), "no manifests given");
Preconditions.checkArgument(
manifestsAndConfigs.stream().noneMatch(isManifestNull), "null manifest(s)");
Preconditions.checkArgument(
metadata.getManifestList() != null || manifestsAndConfigs.size() == 1,
"manifest list missing while multiple manifests given");

ManifestTemplate firstManifest = manifestsAndConfigs.get(0).getManifest();
if (firstManifest instanceof V21ManifestTemplate) {
Preconditions.checkArgument(
metadata.getManifestList() == null, "manifest list given for schema 1");
Preconditions.checkArgument(
isConfigNull.test(manifestsAndConfigs.get(0)), "container config given for schema 1");
} else if (firstManifest instanceof BuildableManifestTemplate) {
Preconditions.checkArgument(
manifestsAndConfigs.stream().noneMatch(isConfigNull), "null config(s)");
} else {
throw new IllegalArgumentException("Unknown manifest type: " + firstManifest);
}
}

/**
* Attempts to move {@code source} to {@code destination}. If {@code destination} already exists,
* this does nothing. Attempts an atomic move first, and falls back to non-atomic if the
Expand Down Expand Up @@ -306,53 +331,14 @@ CachedLayer writeTarLayer(DescriptorDigest diffId, Blob compressedBlob) throws I
* an image reference.
*
* @param imageReference the image reference to store the metadata for
* @param manifestList the V2.2 manifest list or OCI image index. Can be null.
* @param manifests the V2.2 or OCI manifests
* @param containerConfigurations the container configurations
*/
void writeMetadata(
ImageReference imageReference,
@Nullable ManifestTemplate manifestList,
List<BuildableManifestTemplate> manifests,
List<ContainerConfigurationTemplate> containerConfigurations)
throws IOException {
Preconditions.checkArgument(
manifests.size() == containerConfigurations.size(),
"manifests and containerConfigurations should be of same size");
for (BuildableManifestTemplate manifest : manifests) {
Preconditions.checkNotNull(manifest.getContainerConfiguration());
Preconditions.checkNotNull(manifest.getContainerConfiguration().getDigest());
}

Path imageDirectory = cacheStorageFiles.getImageDirectory(imageReference);
Files.createDirectories(imageDirectory);

List<ManifestAndConfigTemplate> manifestsAndConfigs = new ArrayList<>();
for (int i = 0; i < manifests.size(); i++) {
manifestsAndConfigs.add(
new ManifestAndConfigTemplate(manifests.get(i), containerConfigurations.get(i)));
}
try (LockFile ignored = LockFile.lock(imageDirectory.resolve("lock"))) {
writeJsonTemplate(
new ImageMetadataTemplate(manifestList, manifestsAndConfigs),
imageDirectory.resolve("manifests_configs.json"));
}
}

/**
* Writes a V2.1 manifest for a given image reference.
*
* @param imageReference the image reference to store the metadata for
* @param manifest the manifest
* @param metadata the image metadata
*/
void writeMetadata(ImageReference imageReference, V21ManifestTemplate manifest)
void writeMetadata(ImageReference imageReference, ImageMetadataTemplate metadata)
throws IOException {
verifyImageMetadata(metadata);
Path imageDirectory = cacheStorageFiles.getImageDirectory(imageReference);
Files.createDirectories(imageDirectory);

ImageMetadataTemplate metadata =
new ImageMetadataTemplate(
null, Collections.singletonList(new ManifestAndConfigTemplate(manifest, null)));
try (LockFile ignored = LockFile.lock(imageDirectory.resolve("lock"))) {
writeJsonTemplate(metadata, imageDirectory.resolve("manifests_configs.json"));
}
Expand Down
12 changes: 12 additions & 0 deletions jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ManifestAndConfigTemplate.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
/** Stores a manifest and container config. */
public class ManifestAndConfigTemplate implements JsonTemplate {

@Nullable private String manifestDigest;

@JsonTypeInfo(
use = JsonTypeInfo.Id.CLASS,
include = JsonTypeInfo.As.PROPERTY,
Expand All @@ -50,6 +52,16 @@ public ManifestAndConfigTemplate(
this.config = config;
}

/**
* Gets the digest of the manifest.
*
* @return the digest
*/
@Nullable
public String getManifestDigest() {
return manifestDigest;
}

/**
* Gets the manifest.
*
Expand Down
2 changes: 1 addition & 1 deletion jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStepTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ public void testCall_mismatchedPlatformOfCachedManifest_baseImageDigest()
}

@Test
public void testObtainPlatformSpecificImageManifest() throws IOException, RegistryException {
public void testLookUpPlatformSpecificImageManifest() throws IOException, RegistryException {
String manifestListJson =
" {\n"
+ " \"schemaVersion\": 2,\n"
Expand Down
9 changes: 2 additions & 7 deletions jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.mockito.Mockito;

/** Tests for {@link CacheStorageReader}. */
public class CacheStorageReaderTest {
Expand Down Expand Up @@ -578,13 +579,7 @@ public void testVerifyImageMetadata_schema2ManifestsCorrupted() {
public void testVerifyImageMetadata_unknownManifestType() {
ManifestAndConfigTemplate manifestAndConfig =
new ManifestAndConfigTemplate(
new ManifestTemplate() {
@Override
public int getSchemaVersion() {
return 987;
}
},
new ContainerConfigurationTemplate());
Mockito.mock(ManifestTemplate.class), new ContainerConfigurationTemplate());
ImageMetadataTemplate metadata =
new ImageMetadataTemplate(null, Arrays.asList(manifestAndConfig));
try {
Expand Down
Loading

0 comments on commit b92f1ff

Please sign in to comment.