Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for SNAPSHOT dependencies layer #584

Merged
merged 4 commits into from
Jul 12, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,13 @@ public class BuildStepsIntegrationTest {

@Before
public void setUp() throws IOException, URISyntaxException {
sourceFilesConfiguration = new TestSourceFilesConfiguration();
sourceFilesConfiguration =
TestSourceFilesConfiguration.builder()
.withClasses()
.withDependencies()
.withSnapshotDependencies()
.withResources()
.build();
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,12 @@ public interface SourceFilesConfiguration {
*/
ImmutableList<Path> getDependenciesFiles();

/**
* @return the source files for snapshot dependencies. These files should be in a deterministic
* order
*/
ImmutableList<Path> getSnapshotDependenciesFiles();

/**
* @return the source files for the resources layer. These files should be in a deterministic
* order.
Expand All @@ -48,7 +54,7 @@ public interface SourceFilesConfiguration {

/**
* @return the Unix-style path where the dependencies source files are placed in the container
* filesystem. Must end with slash.
* filesystem. Must end with slash. This includes both regular and snapshot dependencies.
*/
String getDependenciesPathOnImage();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,20 @@ static ImmutableList<BuildAndCacheApplicationLayerStep> makeList(
.build(),
cache));

// Adds a snapshot dependencies layer, if snapshot files present.
if (!sourceFilesConfiguration.getSnapshotDependenciesFiles().isEmpty()) {
buildLayerStepsBuilder.add(
new BuildAndCacheApplicationLayerStep(
"snapshot-dependencies",
listeningExecutorService,
buildConfiguration,
LayerConfiguration.builder()
.addEntry(
sourceFilesConfiguration.getSnapshotDependenciesFiles(),
sourceFilesConfiguration.getDependenciesPathOnImage())
.build(),
cache));
}
// Adds the extra layer to be built, if configured.
if (buildConfiguration.getExtraFilesLayerConfiguration() != null) {
buildLayerStepsBuilder.add(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,14 +136,18 @@ public void generate(Path targetDirectory) throws IOException {

// Creates the directories.
Path dependenciesDir = targetDirectory.resolve("libs");
Path snapshotDependenciesDir = targetDirectory.resolve("snapshot-libs");
Path resourcesDIr = targetDirectory.resolve("resources");
Path classesDir = targetDirectory.resolve("classes");
Files.createDirectory(dependenciesDir);
Files.createDirectories(snapshotDependenciesDir);
Files.createDirectory(resourcesDIr);
Files.createDirectory(classesDir);

// Copies dependencies.
FileOperations.copy(sourceFilesConfiguration.getDependenciesFiles(), dependenciesDir);
FileOperations.copy(
sourceFilesConfiguration.getSnapshotDependenciesFiles(), snapshotDependenciesDir);
FileOperations.copy(sourceFilesConfiguration.getResourcesFiles(), resourcesDIr);
FileOperations.copy(sourceFilesConfiguration.getClassesFiles(), classesDir);

Expand Down Expand Up @@ -179,6 +183,8 @@ String makeDockerfile() throws JsonProcessingException {
.append(Preconditions.checkNotNull(baseImage))
.append("\n\nCOPY libs ")
.append(sourceFilesConfiguration.getDependenciesPathOnImage())
.append("\nCOPY snapshot-libs ")
.append(sourceFilesConfiguration.getDependenciesPathOnImage())
.append("\nCOPY resources ")
.append(sourceFilesConfiguration.getResourcesPathOnImage())
.append("\nCOPY classes ")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,23 +28,82 @@
/** Implementation of {@link SourceFilesConfiguration} that uses test resources. */
public class TestSourceFilesConfiguration implements SourceFilesConfiguration {

public static class Builder {
private ImmutableList<Path> dependenciesSourceFiles = ImmutableList.of();
private ImmutableList<Path> snapshotDependenciesSourceFiles = ImmutableList.of();
private ImmutableList<Path> resourcesSourceFiles = ImmutableList.of();
private ImmutableList<Path> classesSourceFiles = ImmutableList.of();

public Builder withDependencies() throws IOException, URISyntaxException {
dependenciesSourceFiles = getFilesList("application/dependencies");
return this;
}

public Builder withSnapshotDependencies() throws IOException, URISyntaxException {
snapshotDependenciesSourceFiles = getFilesList("application/snapshot-dependencies");
return this;
}

public Builder withResources() throws IOException, URISyntaxException {
resourcesSourceFiles = getFilesList("application/resources");
return this;
}

public Builder withClasses() throws IOException, URISyntaxException {
classesSourceFiles = getFilesList("application/classes");
return this;
}

public TestSourceFilesConfiguration build() {
return new TestSourceFilesConfiguration(
dependenciesSourceFiles,
snapshotDependenciesSourceFiles,
resourcesSourceFiles,
classesSourceFiles);
}

/** Lists the files in the {@code resourcePath} resources directory. */
private ImmutableList<Path> getFilesList(String resourcePath)
throws URISyntaxException, IOException {
try (Stream<Path> fileStream =
Files.list(Paths.get(Resources.getResource(resourcePath).toURI()))) {
return fileStream.collect(ImmutableList.toImmutableList());
}
}
}

public static Builder builder() {
return new Builder();
}

private static final String EXTRACTION_PATH = "/some/extraction/path/";

private final ImmutableList<Path> dependenciesSourceFiles;
private final ImmutableList<Path> snapshotDependenciesSourceFiles;
private final ImmutableList<Path> resourcesSourceFiles;
private final ImmutableList<Path> classesSourceFiles;

public TestSourceFilesConfiguration() throws URISyntaxException, IOException {
dependenciesSourceFiles = getFilesList("application/dependencies");
resourcesSourceFiles = getFilesList("application/resources");
classesSourceFiles = getFilesList("application/classes");
private TestSourceFilesConfiguration(
ImmutableList<Path> dependenciesSourceFiles,
ImmutableList<Path> snapshotDependenciesSourceFiles,
ImmutableList<Path> resourcesSourceFiles,
ImmutableList<Path> classesSourceFiles) {
this.dependenciesSourceFiles = dependenciesSourceFiles;
this.snapshotDependenciesSourceFiles = snapshotDependenciesSourceFiles;
this.resourcesSourceFiles = resourcesSourceFiles;
this.classesSourceFiles = classesSourceFiles;
}

@Override
public ImmutableList<Path> getDependenciesFiles() {
return dependenciesSourceFiles;
}

@Override
public ImmutableList<Path> getSnapshotDependenciesFiles() {
return snapshotDependenciesSourceFiles;
}

@Override
public ImmutableList<Path> getResourcesFiles() {
return resourcesSourceFiles;
Expand All @@ -69,13 +128,4 @@ public String getResourcesPathOnImage() {
public String getClassesPathOnImage() {
return EXTRACTION_PATH + "classes/";
}

/** Lists the files in the {@code resourcePath} resources directory. */
private ImmutableList<Path> getFilesList(String resourcePath)
throws URISyntaxException, IOException {
try (Stream<Path> fileStream =
Files.list(Paths.get(Resources.getResource(resourcePath).toURI()))) {
return fileStream.collect(ImmutableList.toImmutableList());
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import com.google.cloud.tools.jib.async.NonBlockingSteps;
import com.google.cloud.tools.jib.builder.BuildConfiguration;
import com.google.cloud.tools.jib.builder.SourceFilesConfiguration;
import com.google.cloud.tools.jib.builder.TestBuildLogger;
import com.google.cloud.tools.jib.builder.TestSourceFilesConfiguration;
import com.google.cloud.tools.jib.cache.Cache;
Expand All @@ -37,6 +38,7 @@
import java.nio.file.Paths;
import java.util.concurrent.ExecutionException;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
Expand All @@ -49,30 +51,35 @@
@RunWith(MockitoJUnitRunner.class)
public class BuildAndCacheApplicationLayerStepTest {

private static final String EXTRA_FILES_LAYER_EXTRACTION_PATH = "/extra";

@Rule public TemporaryFolder temporaryFolder = new TemporaryFolder();

@Mock private BuildConfiguration mockBuildConfiguration;
private Path temporaryCacheDirectory;

@Test
public void testRun()
throws LayerPropertyNotFoundException, IOException, CacheMetadataCorruptedException,
URISyntaxException, ExecutionException {
@Before
public void setUp() throws IOException, URISyntaxException {
Mockito.when(mockBuildConfiguration.getBuildLogger()).thenReturn(new TestBuildLogger());
TestSourceFilesConfiguration testSourceFilesConfiguration = new TestSourceFilesConfiguration();
Path temporaryCacheDirectory = temporaryFolder.newFolder().toPath();
temporaryCacheDirectory = temporaryFolder.newFolder().toPath();
}

// Adds an extra file layer.
private ImmutableList<Path> configureExtraFilesLayer() throws URISyntaxException {
ImmutableList<Path> extraFilesLayerSourceFiles =
ImmutableList.of(
Paths.get(Resources.getResource("fileA").toURI()),
Paths.get(Resources.getResource("fileB").toURI()));
String extraFilesLayerExtractionPath = "/extra";
Mockito.when(mockBuildConfiguration.getExtraFilesLayerConfiguration())
.thenReturn(
LayerConfiguration.builder()
.addEntry(extraFilesLayerSourceFiles, extraFilesLayerExtractionPath)
.addEntry(extraFilesLayerSourceFiles, EXTRA_FILES_LAYER_EXTRACTION_PATH)
.build());
return extraFilesLayerSourceFiles;
}

private ImageLayers<CachedLayerWithMetadata> configureAvailableLayers(
SourceFilesConfiguration testSourceFilesConfiguration)
throws CacheMetadataCorruptedException, IOException, ExecutionException {
ImageLayers.Builder<CachedLayerWithMetadata> applicationLayersBuilder = ImageLayers.builder();
ImageLayers<CachedLayerWithMetadata> applicationLayers;

Expand All @@ -91,8 +98,30 @@ public void testRun()

applicationLayers = applicationLayersBuilder.build();
cache.addCachedLayersWithMetadataToMetadata(applicationLayers.getLayers());
Assert.assertEquals(4, applicationLayers.size());
}
return applicationLayers;
}

@Test
public void testRun()
throws LayerPropertyNotFoundException, IOException, CacheMetadataCorruptedException,
URISyntaxException, ExecutionException {

TestSourceFilesConfiguration testSourceFilesConfiguration =
TestSourceFilesConfiguration.builder()
.withDependencies()
.withSnapshotDependencies()
.withClasses()
.withResources()
.build();

// Adds an extra file layer.
ImmutableList<Path> extraFilesLayerSourceFiles = configureExtraFilesLayer();

// Populate the cache
ImageLayers<CachedLayerWithMetadata> applicationLayers =
configureAvailableLayers(testSourceFilesConfiguration);
Assert.assertEquals(5, applicationLayers.size());

// Re-initialize cache with the updated metadata.
Cache cache = Cache.init(temporaryCacheDirectory);
Expand All @@ -102,6 +131,11 @@ public void testRun()
new LayerEntry(
testSourceFilesConfiguration.getDependenciesFiles(),
testSourceFilesConfiguration.getDependenciesPathOnImage()));
ImmutableList<LayerEntry> snapshotDependenciesLayerEntry =
ImmutableList.of(
new LayerEntry(
testSourceFilesConfiguration.getSnapshotDependenciesFiles(),
testSourceFilesConfiguration.getDependenciesPathOnImage()));
ImmutableList<LayerEntry> resourcesLayerEntry =
ImmutableList.of(
new LayerEntry(
Expand All @@ -113,7 +147,8 @@ public void testRun()
testSourceFilesConfiguration.getClassesFiles(),
testSourceFilesConfiguration.getClassesPathOnImage()));
ImmutableList<LayerEntry> extraFilesLayerEntry =
ImmutableList.of(new LayerEntry(extraFilesLayerSourceFiles, extraFilesLayerExtractionPath));
ImmutableList.of(
new LayerEntry(extraFilesLayerSourceFiles, EXTRA_FILES_LAYER_EXTRACTION_PATH));

// Verifies that the cached layers are up-to-date.
CacheReader cacheReader = new CacheReader(cache);
Expand All @@ -128,6 +163,11 @@ public void testRun()
cacheReader.getUpToDateLayerByLayerEntries(classesLayerEntry).getBlobDescriptor());
Assert.assertEquals(
applicationLayers.get(3).getBlobDescriptor(),
cacheReader
.getUpToDateLayerByLayerEntries(snapshotDependenciesLayerEntry)
.getBlobDescriptor());
Assert.assertEquals(
applicationLayers.get(4).getBlobDescriptor(),
cacheReader.getUpToDateLayerByLayerEntries(extraFilesLayerEntry).getBlobDescriptor());

// Verifies that the cache reader gets the same layers as the newest application layers.
Expand All @@ -139,6 +179,66 @@ public void testRun()
Assert.assertEquals(
applicationLayers.get(2).getContentFile(), cacheReader.getLayerFile(classesLayerEntry));
Assert.assertEquals(
applicationLayers.get(3).getContentFile(), cacheReader.getLayerFile(extraFilesLayerEntry));
applicationLayers.get(3).getContentFile(),
cacheReader.getLayerFile(snapshotDependenciesLayerEntry));
Assert.assertEquals(
applicationLayers.get(4).getContentFile(), cacheReader.getLayerFile(extraFilesLayerEntry));
}

@Test
public void testRun_emptyLayersIgnored()
throws IOException, URISyntaxException, CacheMetadataCorruptedException, ExecutionException {

TestSourceFilesConfiguration testSourceFilesConfiguration =
TestSourceFilesConfiguration.builder()
.withDependencies()
.withClasses()
.withResources()
.build();

// Populate the cache
ImageLayers<CachedLayerWithMetadata> applicationLayers =
configureAvailableLayers(testSourceFilesConfiguration);
Assert.assertEquals(3, applicationLayers.size());

ImmutableList<LayerEntry> dependenciesLayerEntry =
ImmutableList.of(
new LayerEntry(
testSourceFilesConfiguration.getDependenciesFiles(),
testSourceFilesConfiguration.getDependenciesPathOnImage()));
ImmutableList<LayerEntry> resourcesLayerEntry =
ImmutableList.of(
new LayerEntry(
testSourceFilesConfiguration.getResourcesFiles(),
testSourceFilesConfiguration.getResourcesPathOnImage()));
ImmutableList<LayerEntry> classesLayerEntry =
ImmutableList.of(
new LayerEntry(
testSourceFilesConfiguration.getClassesFiles(),
testSourceFilesConfiguration.getClassesPathOnImage()));

// Re-initialize cache with the updated metadata.
Cache cache = Cache.init(temporaryCacheDirectory);

// Verifies that the cached layers are up-to-date.
CacheReader cacheReader = new CacheReader(cache);
Assert.assertEquals(
applicationLayers.get(0).getBlobDescriptor(),
cacheReader.getUpToDateLayerByLayerEntries(dependenciesLayerEntry).getBlobDescriptor());
Assert.assertEquals(
applicationLayers.get(1).getBlobDescriptor(),
cacheReader.getUpToDateLayerByLayerEntries(resourcesLayerEntry).getBlobDescriptor());
Assert.assertEquals(
applicationLayers.get(2).getBlobDescriptor(),
cacheReader.getUpToDateLayerByLayerEntries(classesLayerEntry).getBlobDescriptor());

// Verifies that the cache reader gets the same layers as the newest application layers.
Assert.assertEquals(
applicationLayers.get(0).getContentFile(),
cacheReader.getLayerFile(dependenciesLayerEntry));
Assert.assertEquals(
applicationLayers.get(1).getContentFile(), cacheReader.getLayerFile(resourcesLayerEntry));
Assert.assertEquals(
applicationLayers.get(2).getContentFile(), cacheReader.getLayerFile(classesLayerEntry));
}
}
Loading