diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 0569527742612d..6ae11febbf5bc7 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -279,6 +279,7 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { if (repoOptions.repositoryDownloaderRetries >= 0) { downloadManager.setRetries(repoOptions.repositoryDownloaderRetries); } + downloadManager.setUrlsAsDefaultCanonicalId(repoOptions.urlsAsDefaultCanonicalId); repositoryCache.setHardlink(repoOptions.useHardlinks); if (repoOptions.experimentalScaleTimeouts > 0.0) { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java index 5f28b960fe798d..6a0c5c6f550014 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java @@ -215,6 +215,19 @@ public class RepositoryOptions extends OptionsBase { + " to escalate it to a resolution failure.") public CheckDirectDepsMode checkDirectDependencies; + @Option( + name = "experimental_repository_cache_urls_as_default_canonical_id", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.BAZEL_CLIENT_OPTIONS, + effectTags = {OptionEffectTag.LOADING_AND_ANALYSIS}, + metadataTags = {OptionMetadataTag.EXPERIMENTAL}, + help = + "If true, use a string derived from the URLs of repository downloads as the canonical_id " + + "if not specified. This causes a change in the URLs to result in a redownload even " + + "if the cache contains a download with the same hash. This can be used to verify " + + "that URL changes don't result in broken repositories being masked by the cache.") + public boolean urlsAsDefaultCanonicalId; + /** An enum for specifying different modes for checking direct dependency accuracy. */ public enum CheckDirectDepsMode { OFF, // Don't check direct dependency accuracy. diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java index 00aaee646e7eac..399872fbdaa461 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java @@ -35,6 +35,7 @@ import java.net.URL; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import javax.annotation.Nullable; /** @@ -51,6 +52,7 @@ public class DownloadManager { private final Downloader downloader; private boolean disableDownload = false; private int retries = 0; + private boolean urlsAsDefaultCanonicalId; public DownloadManager(RepositoryCache repositoryCache, Downloader downloader) { this.repositoryCache = repositoryCache; @@ -74,6 +76,10 @@ public void setRetries(int retries) { this.retries = retries; } + public void setUrlsAsDefaultCanonicalId(boolean urlsAsDefaultCanonicalId) { + this.urlsAsDefaultCanonicalId = urlsAsDefaultCanonicalId; + } + /** * Downloads file to disk and returns path. * @@ -108,6 +114,10 @@ public Path download( throw new InterruptedException(); } + if (Strings.isNullOrEmpty(canonicalId) && urlsAsDefaultCanonicalId) { + canonicalId = originalUrls.stream().map(URL::toExternalForm).collect(Collectors.joining(" ")); + } + List rewrittenUrls = originalUrls; Map> rewrittenAuthHeaders = authHeaders; diff --git a/src/test/shell/bazel/bazel_repository_cache_test.sh b/src/test/shell/bazel/bazel_repository_cache_test.sh index 11b9eedfdc606a..28235a0557c272 100755 --- a/src/test/shell/bazel/bazel_repository_cache_test.sh +++ b/src/test/shell/bazel/bazel_repository_cache_test.sh @@ -465,4 +465,68 @@ EOF expect_log "Error downloading" } +function test_break_url() { + setup_repository + + bazel fetch --repository_cache="$repo_cache_dir" //zoo:breeding-program >& $TEST_log \ + || echo "Expected fetch to succeed" + + shutdown_server + + bazel fetch --repository_cache="$repo_cache_dir" //zoo:breeding-program >& $TEST_log \ + || echo "Expected fetch to succeed" + + # Break url in WORKSPACE + rm WORKSPACE + cat >> $(create_workspace_with_default_repos WORKSPACE) <& $TEST_log \ + || echo "Expected fetch to succeed" +} + +function test_experimental_repository_cache_urls_as_default_canonical_id() { + setup_repository + + bazel fetch --repository_cache="$repo_cache_dir" \ + --experimental_repository_cache_urls_as_default_canonical_id \ + //zoo:breeding-program >& $TEST_log \ + || echo "Expected fetch to succeed" + + shutdown_server + + bazel fetch --repository_cache="$repo_cache_dir" \ + --experimental_repository_cache_urls_as_default_canonical_id \ + //zoo:breeding-program >& $TEST_log \ + || echo "Expected fetch to succeed" + + # Break url in WORKSPACE + rm WORKSPACE + cat >> $(create_workspace_with_default_repos WORKSPACE) <& $TEST_log \ + && fail "expected failure" || : +} + run_suite "repository cache tests"