diff --git a/site/en/build/bzlmod.md b/site/en/build/bzlmod.md index c784cea929ac6c..fdb0864c8d4125 100644 --- a/site/en/build/bzlmod.md +++ b/site/en/build/bzlmod.md @@ -221,9 +221,10 @@ version. Notably, it does *not* need to serve the source archives itself. An index registry must follow the format below: -* `/bazel_registry.json`: A JSON file containing metadata for the registry. - Currently, it only has one key, `mirrors`, specifying the list of mirrors to - use for source archives. +* `/bazel_registry.json`: A JSON file containing metadata for the registry like: + * `mirrors`, specifying the list of mirrors to use for source archives. + * `module_base_path`, specifying the base path for modules with + `local_repository` type in the `source.json` file. * `/modules`: A directory containing a subdirectory for each module in this registry. * `/modules/$MODULE`: A directory containing a subdirectory for each version @@ -243,18 +244,29 @@ An index registry must follow the format below: * `/modules/$MODULE/$VERSION`: A directory containing the following files: * `MODULE.bazel`: The `MODULE.bazel` file of this module version. * `source.json`: A JSON file containing information on how to fetch the - source of this module version, with the following fields: - * `url`: The URL of the source archive. - * `integrity`: The - [Subresource Integrity](https://w3c.github.io/webappsec-subresource-integrity/#integrity-metadata-description){: .external} - checksum of the archive. - * `strip_prefix`: A directory prefix to strip when extracting the - source archive. - * `patches`: A list of strings, each of which names a patch file to - apply to the extracted archive. The patch files are located under - the `/modules/$MODULE/$VERSION/patches` directory. - * `patch_strip`: Same as the `--strip` argument of Unix patch. - * `patches/`: An optional directory containing patch files. + source of this module version. + * The default type is "archive" with the following fields: + * `url`: The URL of the source archive. + * `integrity`: The + [Subresource Integrity](https://w3c.github.io/webappsec-subresource-integrity/#integrity-metadata-description){: .external} + checksum of the archive. + * `strip_prefix`: A directory prefix to strip when extracting the + source archive. + * `patches`: A list of strings, each of which names a patch file to + apply to the extracted archive. The patch files are located under + the `/modules/$MODULE/$VERSION/patches` directory. + * `patch_strip`: Same as the `--strip` argument of Unix patch. + * The type can be changed to use a local path with these fields: + * `type`: `local_path` + * `path`: The local path to the repo, calculated as following: + * If path is an absolute path, will be used as it is. + * If path is a relative path and `module_base_path` is an absolute path, + path is resolved to `/` + * If path and `module_base_path` are both relative paths, path is + resolved to `//`. + Registry must be hosted locally and used by `--registry=file://`. + Otherwise, Bazel will throw an error. + * `patches/`: An optional directory containing patch files, only used when `source.json` has "archive" type. ### Bazel Central Registry {:#bazel-central-registry} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index ae357a4c8064e1..360390c4f17508 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -74,6 +74,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/events", + "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", "//third_party:gson", "//third_party:guava", ], diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java index 4f885ba8a13ef4..869ae3bcea7a50 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java @@ -24,6 +24,7 @@ import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.events.ExtendedEventHandler; +import com.google.devtools.build.lib.vfs.PathFragment; import com.google.gson.FieldNamingPolicy; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -97,15 +98,18 @@ public Optional getModuleFile(ModuleKey key, ExtendedEventHandler eventH /** Represents fields available in {@code bazel_registry.json} for the registry. */ private static class BazelRegistryJson { String[] mirrors; + String moduleBasePath; } /** Represents fields available in {@code source.json} for each version of a module. */ private static class SourceJson { + String type = "archive"; URL url; String integrity; String stripPrefix; Map patches; int patchStrip; + String path; } /** @@ -147,6 +151,51 @@ public RepoSpec getRepoSpec( throw new FileNotFoundException( String.format("Module %s's source information not found in registry %s", key, getUrl())); } + + String type = sourceJson.get().type; + switch (type) { + case "archive": + return createArchiveRepoSpec(sourceJson, bazelRegistryJson, key, repoName); + case "local_path": + return createLocalPathRepoSpec(sourceJson, bazelRegistryJson, key, repoName); + default: + throw new IOException(String.format("Invalid source type for module %s", key)); + } + } + + private RepoSpec createLocalPathRepoSpec( + Optional sourceJson, + Optional bazelRegistryJson, + ModuleKey key, + RepositoryName repoName) + throws IOException { + String path = sourceJson.get().path; + if (!PathFragment.isAbsolute(path)) { + String moduleBase = bazelRegistryJson.get().moduleBasePath; + path = moduleBase + "/" + path; + if (!PathFragment.isAbsolute(moduleBase)) { + if (uri.getScheme().equals("file")) { + path = uri.getPath() + "/" + path; + } else { + throw new IOException(String.format("Provided non local registry for module %s", key)); + } + } + } + + return RepoSpec.builder() + .setRuleClassName("local_repository") + .setAttributes( + ImmutableMap.of( + "name", repoName.getName(), "path", PathFragment.create(path).toString())) + .build(); + } + + private RepoSpec createArchiveRepoSpec( + Optional sourceJson, + Optional bazelRegistryJson, + ModuleKey key, + RepositoryName repoName) + throws IOException { URL sourceUrl = sourceJson.get().url; if (sourceUrl == null) { throw new IOException(String.format("Missing source URL for module %s", key)); diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java index fc66fc0c1c9bb4..d9ae5eff654b8e 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java @@ -115,7 +115,7 @@ public void testFileUrl() throws Exception { } @Test - public void testGetRepoSpec() throws Exception { + public void testGetArchiveRepoSpec() throws Exception { server.serve( "/bazel_registry.json", "{", @@ -183,6 +183,28 @@ public void testGetRepoSpec() throws Exception { .build()); } + @Test + public void testGetLocalPathRepoSpec() throws Exception { + server.serve("/bazel_registry.json", "{", " \"module_base_path\": \"/hello/foo\"", "}"); + server.serve( + "/modules/foo/1.0/source.json", + "{", + " \"type\": \"local_path\",", + " \"path\": \"../bar/project_x\"", + "}"); + server.start(); + + Registry registry = registryFactory.getRegistryWithUrl(server.getUrl()); + assertThat( + registry.getRepoSpec( + createModuleKey("foo", "1.0"), RepositoryName.create("foorepo"), reporter)) + .isEqualTo( + RepoSpec.builder() + .setRuleClassName("local_repository") + .setAttributes(ImmutableMap.of("name", "foorepo", "path", "/hello/bar/project_x")) + .build()); + } + @Test public void testGetRepoInvalidRegistryJsonSpec() throws Exception { server.serve("/bazel_registry.json", "", "", "", ""); diff --git a/src/test/py/bazel/bzlmod/bazel_module_test.py b/src/test/py/bazel/bzlmod/bazel_module_test.py index 58ce6fd85ce889..ad741ff7f032d8 100644 --- a/src/test/py/bazel/bzlmod/bazel_module_test.py +++ b/src/test/py/bazel/bzlmod/bazel_module_test.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os import pathlib import tempfile @@ -531,6 +532,46 @@ def testAllowedYankedDepsSuccessMix(self): env_add={'BZLMOD_ALLOW_YANKED_VERSIONS': 'yanked2@1.0'}, allow_failure=False) + def setUpProjectWithLocalRegistryModule(self, dep_name, dep_version, + module_base_path): + bazel_registry = { + 'module_base_path': module_base_path, + } + with self.main_registry.root.joinpath('bazel_registry.json').open('w') as f: + json.dump(bazel_registry, f, indent=4, sort_keys=True) + + self.main_registry.generateCcSource(dep_name, dep_version) + self.main_registry.createLocalPathModule(dep_name, dep_version, + dep_name + '/' + dep_version) + + self.ScratchFile('main.cc', [ + '#include "%s.h"' % dep_name, + 'int main() {', + ' hello_%s("main function");' % dep_name, + '}', + ]) + self.ScratchFile('MODULE.bazel', [ + 'bazel_dep(name = "%s", version = "%s")' % (dep_name, dep_version), + ]) + self.ScratchFile('BUILD', [ + 'cc_binary(', + ' name = "main",', + ' srcs = ["main.cc"],', + ' deps = ["@%s//:lib_%s"],' % (dep_name, dep_name), + ')', + ]) + self.ScratchFile('WORKSPACE', []) + + def testLocalRepoInSourceJsonAbsoluteBasePath(self): + self.setUpProjectWithLocalRegistryModule('sss', '1.3', + str(self.main_registry.projects)) + _, stdout, _ = self.RunBazel(['run', '//:main'], allow_failure=False) + self.assertIn('main function => sss@1.3', stdout) + + def testLocalRepoInSourceJsonRelativeBasePath(self): + self.setUpProjectWithLocalRegistryModule('sss', '1.3', 'projects') + _, stdout, _ = self.RunBazel(['run', '//:main'], allow_failure=False) + self.assertIn('main function => sss@1.3', stdout) if __name__ == '__main__': unittest.main() diff --git a/src/test/py/bazel/bzlmod/test_utils.py b/src/test/py/bazel/bzlmod/test_utils.py index 9de182a9b8e1d2..786c1a30879fba 100644 --- a/src/test/py/bazel/bzlmod/test_utils.py +++ b/src/test/py/bazel/bzlmod/test_utils.py @@ -267,3 +267,25 @@ def addMetadata(self, json.dump(metadata, f, indent=4, sort_keys=True) return self + + def createLocalPathModule(self, name, version, path): + """Add a local module into the registry.""" + module_dir = self.root.joinpath('modules', name, version) + module_dir.mkdir(parents=True, exist_ok=True) + + # Create source.json & copy patch files to the registry + source = { + 'type': 'local_path', + 'path': path, + } + + scratchFile( + module_dir.joinpath('MODULE.bazel'), [ + 'module(', + ' name = "%s",' % name, + ' version = "%s",' % version, + ')', + ]) + + with module_dir.joinpath('source.json').open('w') as f: + json.dump(source, f, indent=4, sort_keys=True)