From 3cadd2680953bd27771714aca12135845df45744 Mon Sep 17 00:00:00 2001 From: Ed Schouten Date: Mon, 31 May 2021 12:40:13 +0000 Subject: [PATCH] Remote Output Service: place bazel-out/ on a FUSE file system Builds that yield large output files can generate lots of network bandwidth when remote execution is used. To combat this, we have flags such as --remote_download_minimal to disable downloading of output files. Unfortunately, this makes it hard to perform ad hoc exploration of build outputs. In an attempt to solve this, this change adds an option to Bazel named --remote_output_service. When enabled, Bazel effectively gives up the responsibility of managing a bazel-out/ directory. Instead, it calls into a gRPC service to request a directory and creates a symlink that points to it. Smart implementations of this gRPC service may use things like FUSE to let this replacement bazel-out/ directory be lazy-loading, thereby reducing network bandwidth significantly. In order to create lazy-loading files and directories, Bazel can call into a BatchCreate() RPC that takes a list of Output{File,Directory,Symlink} messages, similar to REv2's ActionResult. This call is also used to create runfiles directories by providing fictive instances of OutputSymlink. To prevent Bazel from reading the contents of files stored in the FUSE file system (which would cause network I/O), the protocol offers a BatchStat() call that can return information such as the REv2 digest. Though this is redundant with --unix_digest_hash_attribute_name, there are a couple of reasons why I added this feature: 1. For non-Linux operating systems, it may make more sense to use NFSv4 instead of FUSE (i.e., running a virtual NFS daemon on localhost). Even though RFC 8276 adds support for extended attributes to NFSv4, not all operating systems implement it. 2. It addresses the security/hermeticity concerns that people raised when this feature was added. There is no way to add extended attributes to files that can't be tampered with (as a non-root user), while using gRPC solves that problem inherently. 3. Callers of Bazel's BatchStat.batchStat() may generate many system calls successively. This causes a large number of context switches between Bazel and the FUSE daemon. Using gRPC seems to be cheaper. By requiring that the output path returned by the gRPC service is writable, no-remote actions can still run as before, both with sandboxing enabled and disabled. The only difference is that it will use space on the gRPC service side, as opposed to the user's home directory (though the gRPC service may continue to store data in the user's home directory. I have a server implementation is written in Go on top of Buildbarn's storage and file system layer. My plan is to release the code for this service as soon as I've got a 'thumbs up' on the overall approach. Change-Id: I320586d7d153627fa2425a54e3cc505befe886c0 --- .../build/lib/buildtool/ExecutionTool.java | 2 + .../lib/remote/ActionResultDownloader.java | 8 + .../google/devtools/build/lib/remote/BUILD | 2 + .../lib/remote/GrpcRemoteOutputService.java | 652 ++++++++++++++++++ .../remote/RemoteActionContextProvider.java | 29 +- .../lib/remote/RemoteExecutionService.java | 32 +- .../build/lib/remote/RemoteModule.java | 82 ++- .../build/lib/remote/RemoteOutputService.java | 3 +- .../lib/remote/options/RemoteOptions.java | 36 + .../devtools/build/lib/vfs/OutputService.java | 4 +- src/main/protobuf/BUILD | 28 + src/main/protobuf/failure_details.proto | 2 + src/main/protobuf/remote_output_service.proto | 320 +++++++++ .../remote/RemoteExecutionServiceTest.java | 3 +- .../lib/remote/RemoteSpawnCacheTest.java | 3 +- .../lib/remote/RemoteSpawnRunnerTest.java | 6 +- ...SpawnRunnerWithGrpcRemoteExecutorTest.java | 3 +- .../OutputsInvalidationIntegrationTest.java | 12 +- 18 files changed, 1197 insertions(+), 30 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/remote/ActionResultDownloader.java create mode 100644 src/main/java/com/google/devtools/build/lib/remote/GrpcRemoteOutputService.java create mode 100644 src/main/protobuf/remote_output_service.proto diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java index 4ba1473c618ef7..1503850454eab9 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java @@ -294,6 +294,7 @@ public void prepareForExecution(UUID buildId) try (SilentCloseable c = Profiler.instance().profile("outputService.startBuild")) { modifiedOutputFiles = outputService.startBuild( + getExecRoot(), env.getDirectories().getRelativeOutputPath(), env.getReporter(), buildId, request.getBuildOptions().finalizeActions); } } else { @@ -380,6 +381,7 @@ void executeBuild( try (SilentCloseable c = Profiler.instance().profile("outputService.startBuild")) { modifiedOutputFiles = outputService.startBuild( + getExecRoot(), env.getDirectories().getRelativeOutputPath(), env.getReporter(), buildId, request.getBuildOptions().finalizeActions); } } else { diff --git a/src/main/java/com/google/devtools/build/lib/remote/ActionResultDownloader.java b/src/main/java/com/google/devtools/build/lib/remote/ActionResultDownloader.java new file mode 100644 index 00000000000000..36cf6434d274af --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/remote/ActionResultDownloader.java @@ -0,0 +1,8 @@ +package com.google.devtools.build.lib.remote; + +import build.bazel.remote.execution.v2.ActionResult; +import com.google.common.util.concurrent.ListenableFuture; + +public interface ActionResultDownloader { + public ListenableFuture downloadActionResult(ActionResult actionResult); +} diff --git a/src/main/java/com/google/devtools/build/lib/remote/BUILD b/src/main/java/com/google/devtools/build/lib/remote/BUILD index 04a36e0b999904..b6c00d7bc5c4de 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/BUILD +++ b/src/main/java/com/google/devtools/build/lib/remote/BUILD @@ -117,6 +117,8 @@ java_library( "//src/main/java/com/google/devtools/build/skyframe", "//src/main/java/com/google/devtools/common/options", "//src/main/protobuf:failure_details_java_proto", + "//src/main/protobuf:remote_output_service_java_grpc", + "//src/main/protobuf:remote_output_service_java_proto", "//src/main/protobuf:spawn_java_proto", "//third_party:auth", "//third_party:caffeine", diff --git a/src/main/java/com/google/devtools/build/lib/remote/GrpcRemoteOutputService.java b/src/main/java/com/google/devtools/build/lib/remote/GrpcRemoteOutputService.java new file mode 100644 index 00000000000000..259280deefe20f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/remote/GrpcRemoteOutputService.java @@ -0,0 +1,652 @@ +package com.google.devtools.build.lib.remote; + +import static com.google.common.collect.Iterators.filter; +import static com.google.common.collect.Iterators.partition; +import static java.lang.String.format; + +import build.bazel.remote.execution.v2.ActionResult; +import build.bazel.remote.execution.v2.Digest; +import build.bazel.remote.execution.v2.DigestFunction; +import build.bazel.remote.execution.v2.OutputDirectory; +import build.bazel.remote.execution.v2.OutputFile; +import build.bazel.remote.execution.v2.OutputSymlink; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.UnmodifiableIterator; +import com.google.common.flogger.GoogleLogger; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionInputMap; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactPathResolver; +import com.google.devtools.build.lib.actions.BuildFailedException; +import com.google.devtools.build.lib.actions.cache.MetadataHandler; +import com.google.devtools.build.lib.actions.EnvironmentalExecException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.FilesetOutputSymlink; +import com.google.devtools.build.lib.actions.LostInputsActionExecutionException; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.remote.RemoteOutputServiceGrpc.RemoteOutputServiceBlockingStub; +import com.google.devtools.build.lib.remote.RemoteOutputServiceGrpc.RemoteOutputServiceFutureStub; +import com.google.devtools.build.lib.remote.RemoteOutputServiceProto.BatchCreateRequest; +import com.google.devtools.build.lib.remote.RemoteOutputServiceProto.BatchStatRequest; +import com.google.devtools.build.lib.remote.RemoteOutputServiceProto.BatchStatResponse; +import com.google.devtools.build.lib.remote.RemoteOutputServiceProto.CleanRequest; +import com.google.devtools.build.lib.remote.RemoteOutputServiceProto.FileStatus; +import com.google.devtools.build.lib.remote.RemoteOutputServiceProto.FinalizeBuildRequest; +import com.google.devtools.build.lib.remote.RemoteOutputServiceProto.InitialOutputPathContents; +import com.google.devtools.build.lib.remote.RemoteOutputServiceProto.StartBuildRequest; +import com.google.devtools.build.lib.remote.RemoteOutputServiceProto.StartBuildResponse; +import com.google.devtools.build.lib.remote.RemoteOutputServiceProto.StatResponse; +import com.google.devtools.build.lib.remote.util.DigestUtil; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.vfs.BatchStat; +import com.google.devtools.build.lib.vfs.DelegateFileSystem; +import com.google.devtools.build.lib.vfs.DigestHashFunction; +import com.google.devtools.build.lib.vfs.FileStatusWithDigest; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.ModifiedFileSet; +import com.google.devtools.build.lib.vfs.OutputService; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.Root; +import com.google.devtools.build.skyframe.SkyFunction.Environment; +import io.grpc.Status.Code; +import io.grpc.StatusRuntimeException; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.util.Collection; +import java.util.concurrent.ExecutionException; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import javax.annotation.Nullable; + +public class GrpcRemoteOutputService implements OutputService, ActionResultDownloader { + private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); + + // State that needs to be retained across invocations of Bazel. The + // current build ID is placed inside, because it ensures that cached + // instances of Path that use GrpcFileSystem always generate RPCs to + // the Remote Output Service using the current build ID; not the one + // under which they were created. + public static class StateAcrossBuilds { + @Nullable private UUID currentBuildId; + @Nullable private UUID previousBuildId; + } + + private final StateAcrossBuilds stateAcrossBuilds; + private final ReferenceCountedChannel channel; + private final String outputBaseId; + private final PathFragment outputPathPrefix; + private final String instanceName; + private final DigestFunction.Value digestFunction; + private final Digest emptyDigest; + private final RemoteRetrier retrier; + + private PathFragment currentRelativeOutputPath; + private boolean currentBuildSuccessful; + + public GrpcRemoteOutputService(StateAcrossBuilds stateAcrossBuilds, ReferenceCountedChannel channel, String outputBaseId, PathFragment outputPathPrefix, String instanceName, DigestUtil digestUtil, RemoteRetrier retrier) { + this.stateAcrossBuilds = stateAcrossBuilds; + this.channel = channel; + this.outputBaseId = outputBaseId; + this.outputPathPrefix = outputPathPrefix; + this.instanceName = instanceName; + this.digestFunction = digestUtil.getDigestFunction(); + this.emptyDigest = digestUtil.compute(new byte[] {}); + this.retrier = retrier; + } + + + @Override + public String getFilesSystemName() { + return "GrpcRemoteOutputService"; + } + + @Override + public ModifiedFileSet startBuild( + Path execRoot, String relativeOutputPath, + EventHandler eventHandler, UUID buildId, boolean finalizeActions) + throws BuildFailedException, AbruptExitException, InterruptedException { + // Notify the remote output service that the build is about to + // start. The remote output service will return the directory in + // which it wants us to let the build take place. + // + // Make the output service aware of the location of the output path + // from our perspective and any symbolic links that will point to + // it. This allows the service to properly resolve symbolic links + // containing absolute paths as part of BatchStat() calls. + StartBuildRequest.Builder builder = StartBuildRequest.newBuilder(); + builder.setOutputBaseId(outputBaseId); + builder.setBuildId(buildId.toString()); + builder.setInstanceName(instanceName); + builder.setDigestFunction(digestFunction); + builder.setOutputPathPrefix(outputPathPrefix.toString()); + Path originalOutputPath = execRoot.getRelative(relativeOutputPath); + builder.putOutputPathAliases(originalOutputPath.toString(), "."); + StartBuildRequest request = builder.build(); + StartBuildResponse response; + try { + response = retrier.execute( + () -> + channel.withChannelBlocking( + channel -> RemoteOutputServiceGrpc.newBlockingStub(channel) + .startBuild(request))); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + // Replace the output path with a symbolic link pointing to the + // directory managed by the remote output service. + PathFragment outputPath = outputPathPrefix.getRelative(response.getOutputPathSuffix()); + try { + try { + originalOutputPath.deleteTree(); + } catch (FileNotFoundException e) {} + originalOutputPath.createSymbolicLink(outputPath); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + stateAcrossBuilds.currentBuildId = buildId; + currentRelativeOutputPath = PathFragment.create(relativeOutputPath); + + if (stateAcrossBuilds.previousBuildId == null || !response.hasInitialOutputPathContents()) { + // Either Bazel or the remote output service has performed no + // build before. + return ModifiedFileSet.EVERYTHING_MODIFIED; + } + InitialOutputPathContents initialContents = response.getInitialOutputPathContents(); + if (!initialContents.getBuildId().equals(stateAcrossBuilds.previousBuildId.toString())) { + // Bazel and the remote output service disagree on the build ID of + // the previous build. + return ModifiedFileSet.EVERYTHING_MODIFIED; + } + // Bazel and the remote output service agree on the build ID of the + // previous build. Return the set of paths that have been modified. + // + // TODO: Do these paths need to be relative to the exec root or the + // output path? Assume the exec root for now. + return ModifiedFileSet.builder() + .modifyAll( + Iterables.transform( + initialContents.getModifiedPathsList(), + (p) -> currentRelativeOutputPath.getRelative(p))) + .build(); + } + + @Override + public void finalizeBuild(boolean buildSuccessful) + throws BuildFailedException, AbruptExitException, InterruptedException { + currentBuildSuccessful = buildSuccessful; + } + + public void sendFinalizeBuild() { + if (stateAcrossBuilds.currentBuildId != null) { + FinalizeBuildRequest.Builder builder = FinalizeBuildRequest.newBuilder(); + builder.setBuildId(stateAcrossBuilds.currentBuildId.toString()); + builder.setBuildSuccessful(currentBuildSuccessful); + + try { + channel.withChannelBlocking( + channel -> RemoteOutputServiceGrpc.newBlockingStub(channel) + .finalizeBuild(builder.build())); + stateAcrossBuilds.previousBuildId = stateAcrossBuilds.currentBuildId; + } catch (InterruptedException e) { + stateAcrossBuilds.previousBuildId = null; + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof InterruptedException) { + stateAcrossBuilds.previousBuildId = null; + } else { + Throwables.throwIfUnchecked(cause); + throw new UncheckedIOException(new IOException(e)); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } finally { + stateAcrossBuilds.currentBuildId = null; + currentRelativeOutputPath = null; + } + } + } + + @Override + public void finalizeAction(Action action, MetadataHandler metadataHandler) + throws IOException, EnvironmentalExecException, InterruptedException { + // TODO: Would this be the right place to call into the remote + // output service to check whether any I/O errors occurred? If so, + // we should likely let createActionFileSystem() call into the + // remote output service to start capturing I/O errors. + } + + private String fixupExecRootPath(PathFragment path) { + return path.relativeTo(currentRelativeOutputPath).toString(); + } + + private static abstract class DumbFileStatus implements FileStatusWithDigest { + @Override + public long getSize() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long getLastModifiedTime() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long getLastChangeTime() throws IOException { + return -1; + } + + @Override + public long getNodeId() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] getDigest() throws IOException { + throw new UnsupportedOperationException(); + } + } + + private static class RegularFileStatus extends DumbFileStatus { + private final long size; + private final byte[] digest; + + RegularFileStatus(long size, byte[] digest) { + this.size = size; + this.digest = digest; + } + + @Override + public boolean isFile() { + return true; + } + + @Override + public boolean isDirectory() { + return false; + } + + @Override + public boolean isSymbolicLink() { + return false; + } + + @Override + public boolean isSpecialFile() { + return false; + } + + @Override + public long getSize() throws IOException { + return size; + } + + @Override + public byte[] getDigest() throws IOException { + return digest; + } + } + + private static class DirectoryFileStatus extends DumbFileStatus { + private final long lastModifiedTime; + + DirectoryFileStatus(long lastModifiedTime) { + this.lastModifiedTime = lastModifiedTime; + } + + @Override + public boolean isFile() { + return false; + } + + @Override + public boolean isDirectory() { + return true; + } + + @Override + public boolean isSymbolicLink() { + return false; + } + + @Override + public boolean isSpecialFile() { + return false; + } + + @Override + public long getLastModifiedTime() throws IOException { + return this.lastModifiedTime; + } + + } + + private static class SymlinkFileStatus extends DumbFileStatus { + @Override + public boolean isFile() { + return false; + } + + @Override + public boolean isDirectory() { + return false; + } + + @Override + public boolean isSymbolicLink() { + return true; + } + + @Override + public boolean isSpecialFile() { + return false; + } + } + + private class GrpcBatchStat implements BatchStat { + + public List batchStat(boolean includeDigest, + boolean includeLinks, + Iterable paths) + throws IOException, InterruptedException { + // TODO: Do we need to partition the input, just like in + // createSymlinkTree(), or is input already guaranteed to be + // bounded in size? + BatchStatRequest.Builder builder = BatchStatRequest.newBuilder(); + builder.setBuildId(stateAcrossBuilds.currentBuildId.toString()); + builder.setIncludeFileDigest(includeDigest); + for (PathFragment path : paths) { + builder.addPaths(fixupExecRootPath(path)); + } + + BatchStatResponse responses; + try { + responses = channel.withChannelBlocking( + channel -> RemoteOutputServiceGrpc.newBlockingStub(channel) + .batchStat(builder.build())); + } catch (StatusRuntimeException e) { + throw new IOException(e); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + Throwables.propagateIfPossible(cause, IOException.class, InterruptedException.class); + throw new RuntimeException(e); + } + return Lists.newArrayList( + Iterables.transform( + responses.getResponsesList(), + (response) -> { + if (!response.hasFileStatus()) { + // File not found. + return null; + } + FileStatus fileStatus = response.getFileStatus(); + if (fileStatus.hasFile()) { + Digest digest = fileStatus.getFile().getDigest(); + return new RegularFileStatus(digest.getSizeBytes(), DigestUtil.toBinaryDigest(digest)); + } + if (fileStatus.hasDirectory()) { + return new DirectoryFileStatus(fileStatus.getDirectory().getLastModifiedTime().getSeconds()); + } + if (fileStatus.hasSymlink()) { + return new SymlinkFileStatus(); + } + throw new UnsupportedOperationException(); + })); + } + } + + @Override + public BatchStat getBatchStatter() { + return new GrpcBatchStat(); + } + + @Override + public boolean canCreateSymlinkTree() { + return true; + } + + @Override + public void createSymlinkTree(Map symlinks, PathFragment symlinkTreeRoot) + throws ExecException, InterruptedException { + // The provided set of symbolic links may be too large to provide to + // the remote output service at once. Partition the symbolic links + // in groups of 1000, so that BatchCreateRequest messages remain + // small enough. + UnmodifiableIterator>> symlinkBatchIterator = + partition(symlinks.entrySet().iterator(), 1000); + boolean cleanPathPrefix = true; + while (symlinkBatchIterator.hasNext()) { + List> symlinksBatch = symlinkBatchIterator.next(); + BatchCreateRequest.Builder builder = BatchCreateRequest.newBuilder(); + builder.setBuildId(stateAcrossBuilds.currentBuildId.toString()); + builder.setPathPrefix(symlinkTreeRoot.relativeTo(currentRelativeOutputPath).toString()); + // During the first iteration, we should ensure that any existing + // contents of the symlink tree directory are removed. This may be + // triggered by setting the 'clean_path_prefix' option. + builder.setCleanPathPrefix(cleanPathPrefix); + for (Map.Entry symlink : symlinksBatch) { + PathFragment target = symlink.getValue(); + if (target == null) { + // No target indicates an empty file needs to be created. + OutputFile.Builder fileBuilder = builder.addFilesBuilder(); + fileBuilder.setPath(symlink.getKey().toString()); + fileBuilder.setDigest(emptyDigest); + fileBuilder.setIsExecutable(true); + } else { + OutputSymlink.Builder symlinkBuilder = builder.addSymlinksBuilder(); + symlinkBuilder.setPath(symlink.getKey().toString()); + symlinkBuilder.setTarget(symlink.getValue().toString()); + } + } + try { + channel.withChannelBlocking( + channel -> RemoteOutputServiceGrpc.newBlockingStub(channel) + .batchCreate(builder.build())); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + Throwables.propagateIfPossible(cause, InterruptedException.class); + throw new RuntimeException(e); + } + cleanPathPrefix = false; + } + } + + @Override + public void clean() throws ExecException, InterruptedException { + CleanRequest.Builder builder = CleanRequest.newBuilder(); + builder.setOutputBaseId(outputBaseId); + try { + channel.withChannelBlocking( + channel -> RemoteOutputServiceGrpc.newBlockingStub(channel) + .clean(builder.build())); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + Throwables.propagateIfPossible(cause, InterruptedException.class); + throw new RuntimeException(e); + } + } + + @Override + public ActionFileSystemType actionFileSystemType() { + return ActionFileSystemType.REMOTE_FILE_SYSTEM; + } + + private class GrpcFileSystem extends DelegateFileSystem { + private final PathFragment originalOutputPath; + + public GrpcFileSystem(FileSystem sourceDelegate, PathFragment originalOutputPath) { + super(sourceDelegate); + this.originalOutputPath = originalOutputPath; + } + + @Override + protected byte[] getFastDigest(PathFragment path) throws IOException { + // Don't attempt to compute digests ourselves. Call into the + // remote output service to request the digest. The service may be + // able to return the digest from its bookkeeping, as opposed to + // reading the actual file contents. + BatchStatRequest.Builder builder = BatchStatRequest.newBuilder(); + builder.setBuildId(stateAcrossBuilds.currentBuildId.toString()); + builder.setIncludeFileDigest(true); + builder.setFollowSymlinks(true); + PathFragment relativePath; + try { + relativePath = path.relativeTo(originalOutputPath); + } catch (IllegalArgumentException e) { + // Path is outside the output path. Send it to the regular file system. + return super.getFastDigest(path); + } + builder.addPaths(relativePath.toString()); + + BatchStatResponse responses; + try { + responses = channel.withChannelBlocking( + channel -> RemoteOutputServiceGrpc.newBlockingStub(channel) + .batchStat(builder.build())); + } catch (InterruptedException e) { + return null; + } catch (StatusRuntimeException e) { + if (e.getStatus().getCode() == Code.CANCELLED && Thread.currentThread().isInterrupted()) { + // The current thread is interrupted, meaning that gRPC calls + // into the remote output service are cancelled automatically. + // Return null, indicating that a fast digest is not + // available. + // + // Throwing an IOException in this case may cause other parts + // of Bazel to throw InconsistentFilesystemExceptions, which + // leads to spurious build error messages. + return null; + } + throw new IOException(e); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof InterruptedException) { + return null; + } + Throwables.propagateIfPossible(cause, IOException.class); + throw new RuntimeException(e); + } + if (responses.getResponsesCount() != 1) { + throw new IOException("Remote output service returned an unexpected number of stat responses"); + } + StatResponse response = responses.getResponses(0); + if (!response.hasFileStatus()) { + // Output service reports that the file does not exist. + throw new FileNotFoundException(); + } + FileStatus fileStatus = response.getFileStatus(); + if (fileStatus.hasFile()) { + FileStatus.File regularFileStatus = fileStatus.getFile(); + return DigestUtil.toBinaryDigest(regularFileStatus.getDigest()); + } + if (fileStatus.hasExternal()) { + // Path is a symbolic link that points to a location outside of + // the output path. The output service is unable to give a + // digest. We must compute it ourselves. + return super.getFastDigest( + originalOutputPath.getRelative(fileStatus.getExternal().getNextPath())); + } + throw new IOException(format("Remote output service did not return the status of file %s", path)); + } + } + + @Override + @Nullable + public FileSystem createActionFileSystem( + FileSystem sourceDelegate, + PathFragment execRootFragment, + String relativeOutputPath, + ImmutableList sourceRoots, + ActionInputMap inputArtifactData, + Iterable outputArtifacts, + boolean trackFailedRemoteReads) { + return new GrpcFileSystem(sourceDelegate, execRootFragment.getRelative(relativeOutputPath)); + } + + @Override + public void checkActionFileSystemForLostInputs(FileSystem actionFileSystem, Action action) + throws LostInputsActionExecutionException { + // There is no need to provide an implementation for this function, + // as StartBuild() on the server is supposed to call + // FindMissingBlobs() for all remotely tracked files. This ensures + // that they cannot disappear during the build. + } + + public ListenableFuture downloadActionResult(ActionResult actionResult) { + // Request that all outputs of the action are created. Do make sure + // to remove the "bazel-out/" prefix from all paths, as that part of + // the path is not managed by the remote output service. + BatchCreateRequest.Builder builder = BatchCreateRequest.newBuilder(); + builder.setBuildId(stateAcrossBuilds.currentBuildId.toString()); + for (OutputFile file : actionResult.getOutputFilesList()) { + // As there is no guarantee that permissions on files created by + // the remote output service can be modified, make sure that all + // downloaded files are marked executable. + // TODO: Do we only want to do this selectively? + builder.addFilesBuilder() + .mergeFrom(file) + .setPath(fixupExecRootPath(PathFragment.create(file.getPath()))) + .setIsExecutable(true); + } + for (OutputDirectory directory : actionResult.getOutputDirectoriesList()) { + builder.addDirectoriesBuilder() + .mergeFrom(directory) + .setPath(fixupExecRootPath(PathFragment.create(directory.getPath()))); + } + for (OutputSymlink symlink : Iterables.concat(actionResult.getOutputFileSymlinksList(), actionResult.getOutputDirectorySymlinksList())) { + builder.addSymlinksBuilder() + .mergeFrom(symlink) + .setPath(fixupExecRootPath(PathFragment.create(symlink.getPath()))); + } + + return Futures.catchingAsync( + Futures.transform( + channel.withChannelFuture( + channel -> + RemoteOutputServiceGrpc.newFutureStub(channel) + .batchCreate(builder.build())), + (result) -> null, + MoreExecutors.directExecutor()), + StatusRuntimeException.class, + (sre) -> Futures.immediateFailedFuture(new IOException(sre)), + MoreExecutors.directExecutor()); + } + + @Override + public boolean supportsPathResolverForArtifactValues() { + return true; + } + + @Override + public ArtifactPathResolver createPathResolverForArtifactValues( + PathFragment execRoot, + String relativeOutputPath, + FileSystem fileSystem, + ImmutableList pathEntries, + ActionInputMap actionInputMap, + Map> expandedArtifacts, + Map> filesets) { + FileSystem remoteFileSystem = + new GrpcFileSystem(fileSystem, execRoot.getRelative(relativeOutputPath)); + return ArtifactPathResolver.createPathResolver(remoteFileSystem, fileSystem.getPath(execRoot)); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java index 8666631621bcbf..d621ce6beb942b 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java @@ -45,6 +45,7 @@ final class RemoteActionContextProvider { @Nullable private final ListeningScheduledExecutorService retryScheduler; private final DigestUtil digestUtil; @Nullable private final Path logDir; + @Nullable private final ActionResultDownloader actionResultDownloader; private TempPathGenerator tempPathGenerator; private RemoteExecutionService remoteExecutionService; @Nullable private RemoteActionInputFetcher actionInputFetcher; @@ -56,7 +57,8 @@ private RemoteActionContextProvider( @Nullable RemoteExecutionClient remoteExecutor, @Nullable ListeningScheduledExecutorService retryScheduler, DigestUtil digestUtil, - @Nullable Path logDir) { + @Nullable Path logDir, + @Nullable ActionResultDownloader actionResultDownloader) { this.executor = executor; this.env = Preconditions.checkNotNull(env, "env"); this.remoteCache = remoteCache; @@ -64,6 +66,7 @@ private RemoteActionContextProvider( this.retryScheduler = retryScheduler; this.digestUtil = digestUtil; this.logDir = logDir; + this.actionResultDownloader = actionResultDownloader; } public static RemoteActionContextProvider createForPlaceholder( @@ -77,7 +80,8 @@ public static RemoteActionContextProvider createForPlaceholder( /*remoteExecutor=*/ null, retryScheduler, digestUtil, - /*logDir=*/ null); + /*logDir=*/ null, + /*actionResultDownloader=*/ null); } public static RemoteActionContextProvider createForRemoteCaching( @@ -85,7 +89,8 @@ public static RemoteActionContextProvider createForRemoteCaching( CommandEnvironment env, RemoteCache remoteCache, ListeningScheduledExecutorService retryScheduler, - DigestUtil digestUtil) { + DigestUtil digestUtil, + ActionResultDownloader actionResultDownloader) { return new RemoteActionContextProvider( executor, env, @@ -93,7 +98,8 @@ public static RemoteActionContextProvider createForRemoteCaching( /*remoteExecutor=*/ null, retryScheduler, digestUtil, - /*logDir=*/ null); + /*logDir=*/ null, + actionResultDownloader); } public static RemoteActionContextProvider createForRemoteExecution( @@ -103,9 +109,17 @@ public static RemoteActionContextProvider createForRemoteExecution( RemoteExecutionClient remoteExecutor, ListeningScheduledExecutorService retryScheduler, DigestUtil digestUtil, - Path logDir) { + Path logDir, + ActionResultDownloader actionResultDownloader) { return new RemoteActionContextProvider( - executor, env, remoteCache, remoteExecutor, retryScheduler, digestUtil, logDir); + executor, + env, + remoteCache, + remoteExecutor, + retryScheduler, + digestUtil, + logDir, + actionResultDownloader); } private RemotePathResolver createRemotePathResolver() { @@ -155,7 +169,8 @@ private RemoteExecutionService getRemoteExecutionService() { remoteCache, remoteExecutor, tempPathGenerator, - captureCorruptedOutputsDir); + captureCorruptedOutputsDir, + actionResultDownloader); env.getEventBus().register(remoteExecutionService); } diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java index 506b6da393a9d0..4fb1960df293ce 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java @@ -165,6 +165,7 @@ public class RemoteExecutionService { private final RemoteOptions remoteOptions; @Nullable private final RemoteCache remoteCache; @Nullable private final RemoteExecutionClient remoteExecutor; + @Nullable private final ActionResultDownloader actionResultDownloader; private final TempPathGenerator tempPathGenerator; @Nullable private final Path captureCorruptedOutputsDir; private final Cache> merkleTreeCache; @@ -189,7 +190,8 @@ public RemoteExecutionService( @Nullable RemoteCache remoteCache, @Nullable RemoteExecutionClient remoteExecutor, TempPathGenerator tempPathGenerator, - @Nullable Path captureCorruptedOutputsDir) { + @Nullable Path captureCorruptedOutputsDir, + @Nullable ActionResultDownloader actionResultDownloader) { this.reporter = reporter; this.verboseFailures = verboseFailures; this.execRoot = execRoot; @@ -200,6 +202,7 @@ public RemoteExecutionService( this.remoteOptions = remoteOptions; this.remoteCache = remoteCache; this.remoteExecutor = remoteExecutor; + this.actionResultDownloader = actionResultDownloader; Caffeine merkleTreeCacheBuilder = Caffeine.newBuilder().softValues(); // remoteMerkleTreesCacheSize = 0 means limitless. @@ -1083,6 +1086,33 @@ public InMemoryOutput downloadOutputs(RemoteAction action, RemoteActionResult re checkState(!shutdown.get(), "shutdown"); checkNotNull(remoteCache, "remoteCache can't be null"); + if (actionResultDownloader != null) { + // We have a remote output service process that can do the + // downloading for us. + ActionResult actionResult = result.actionResult; + List> downloads = new ArrayList<>(); + downloads.add(actionResultDownloader.downloadActionResult(actionResult)); + + FileOutErr outErr = action.getSpawnExecutionContext().getFileOutErr(); + FileOutErr tmpOutErr = outErr.childOutErr(); + downloads.addAll( + remoteCache.downloadOutErr( + action.getRemoteActionExecutionContext(), actionResult, tmpOutErr)); + + try { + waitForBulkTransfer(downloads, /* cancelRemainingOnInterrupt=*/ true); + if (tmpOutErr != null) { + FileOutErr.dump(tmpOutErr, outErr); + } + } finally { + if (tmpOutErr != null) { + tmpOutErr.clearOut(); + tmpOutErr.clearErr(); + } + } + return null; + } + ProgressStatusListener progressStatusListener = action.getSpawnExecutionContext()::report; RemoteActionExecutionContext context = action.getRemoteActionExecutionContext(); if (result.executeResponse != null) { diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java index eb8a970384575c..69963bd7be5591 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java @@ -14,6 +14,8 @@ package com.google.devtools.build.lib.remote; +import static com.google.common.hash.Hashing.md5; +import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.concurrent.TimeUnit.SECONDS; import build.bazel.remote.execution.v2.ActionCacheUpdateCapabilities; @@ -100,6 +102,7 @@ import com.google.devtools.build.lib.vfs.OutputPermissions; import com.google.devtools.build.lib.vfs.OutputService; import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.common.options.OptionsBase; import com.google.devtools.common.options.OptionsParsingResult; import io.grpc.CallCredentials; @@ -142,7 +145,9 @@ public final class RemoteModule extends BlazeModule { @Nullable private RemoteActionInputFetcher actionInputFetcher; @Nullable private ToplevelArtifactsDownloader toplevelArtifactsDownloader; @Nullable private RemoteOptions remoteOptions; - @Nullable private RemoteOutputService remoteOutputService; + @Nullable private OutputService remoteOutputService; + @Nullable private GrpcRemoteOutputService.StateAcrossBuilds remoteOutputServiceStateAcrossBuilds = + new GrpcRemoteOutputService.StateAcrossBuilds(); @Nullable private TempPathGenerator tempPathGenerator; @Nullable private BlockWaitingModule blockWaitingModule; @Nullable private ImmutableList patternsToDownload; @@ -281,7 +286,12 @@ private void initHttpAndDiskCache( new RemoteCache(HTTP_AND_DISK_CACHE_CAPABILITIES, cacheClient, remoteOptions, digestUtil); actionContextProvider = RemoteActionContextProvider.createForRemoteCaching( - executorService, env, remoteCache, /* retryScheduler= */ null, digestUtil); + executorService, + env, + remoteCache, + /* retryScheduler= */ null, + digestUtil, + /* actionResultDownloader= */ null); } @Override @@ -617,6 +627,46 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { cacheChannel.retain(), callCredentialsProvider, remoteOptions, retrier, digestUtil); cacheChannel.release(); + ActionResultDownloader actionResultDownloader = null; + Preconditions.checkState(remoteOutputService == null, "remoteOutputService must be null"); + if (remoteOptions.remoteOutputService != null) { + if (!remoteOptions.remoteOutputsMode.downloadAllOutputs()) { + throw createOptionsExitException( + "If --remote_output_service is specified, --remote_download_outputs must be set to \"all\"", + FailureDetails.RemoteOptions.Code.OUTPUT_SERVICE_WITH_INCOMPATIBLE_REMOTE_OUTPUTS_MODE); + } + String outputBaseId = remoteOptions.remoteOutputServiceOutputBaseId; + if (Strings.isNullOrEmpty(outputBaseId)) { + outputBaseId = DigestUtil.hashCodeToString(md5().hashString(env.getWorkspace().toString(), UTF_8)); + } + String outputPathPrefix = remoteOptions.remoteOutputServiceOutputPathPrefix; + if (Strings.isNullOrEmpty(outputPathPrefix)) { + throw createOptionsExitException( + "If --remote_output_service is specified, --remote_output_service_output_path_prefix must be set as well", + FailureDetails.RemoteOptions.Code.OUTPUT_SERVICE_WITHOUT_OUTPUT_PATH_REFIX); + } + + ReferenceCountedChannel channel = + new ReferenceCountedChannel( + new GoogleChannelConnectionFactory( + channelFactory, + remoteOptions.remoteOutputService, + remoteOptions.remoteProxy, + authAndTlsOptions, + ImmutableList.of(), + maxConcurrencyPerConnection)); + GrpcRemoteOutputService grpcRemoteOutputService = new GrpcRemoteOutputService( + remoteOutputServiceStateAcrossBuilds, + channel, + outputBaseId, + PathFragment.create(outputPathPrefix), + remoteOptions.remoteInstanceName, + digestUtil, + retrier); + remoteOutputService = grpcRemoteOutputService; + actionResultDownloader = grpcRemoteOutputService; + } + if (enableRemoteExecution) { if (enableDiskCache) { try { @@ -672,7 +722,8 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { remoteExecutor, retryScheduler, digestUtil, - logDir); + logDir, + actionResultDownloader); repositoryRemoteExecutorFactoryDelegate.init( new RemoteRepositoryRemoteExecutorFactory( remoteCache, @@ -704,7 +755,12 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { cacheCapabilities.getCacheCapabilities(), cacheClient, remoteOptions, digestUtil); actionContextProvider = RemoteActionContextProvider.createForRemoteCaching( - executorService, env, remoteCache, retryScheduler, digestUtil); + executorService, + env, + remoteCache, + retryScheduler, + digestUtil, + actionResultDownloader); } buildEventArtifactUploaderFactoryDelegate.init( @@ -854,7 +910,6 @@ public void afterCommand() { actionInputFetcher = null; toplevelArtifactsDownloader = null; remoteOptions = null; - remoteOutputService = null; tempPathGenerator = null; rpcLogFile = null; patternsToDownload = null; @@ -890,6 +945,16 @@ private static void afterCommandTask( } } + @Override + public void commandComplete() { + if (remoteOutputService != null) { + if (remoteOutputService instanceof GrpcRemoteOutputService) { + ((GrpcRemoteOutputService) remoteOutputService).sendFinalizeBuild(); + } + remoteOutputService = null; + } + } + @Override public void registerSpawnStrategies( SpawnStrategyRegistry.Builder registryBuilder, CommandEnvironment env) { @@ -1020,16 +1085,15 @@ public ActionInput getActionInput(Path path) { env.getSkyframeExecutor().getEvaluator(), env.getBlazeWorkspace().getPersistentActionCache()); - remoteOutputService.setActionInputFetcher(actionInputFetcher); - remoteOutputService.setLeaseService(leaseService); + ((RemoteOutputService) remoteOutputService).setActionInputFetcher(actionInputFetcher); + ((RemoteOutputService) remoteOutputService).setLeaseService(leaseService); env.getEventBus().register(remoteOutputService); } } @Override public OutputService getOutputService() { - Preconditions.checkState(remoteOutputService == null, "remoteOutputService must be null"); - if (remoteOptions != null + if (remoteOutputService == null && remoteOptions != null && !remoteOptions.remoteOutputsMode.downloadAllOutputs() && actionContextProvider.getRemoteCache() != null) { remoteOutputService = new RemoteOutputService(); diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteOutputService.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteOutputService.java index c1b9a6d73b465e..78777bc019686f 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteOutputService.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteOutputService.java @@ -104,7 +104,8 @@ public String getFilesSystemName() { @Override public ModifiedFileSet startBuild( - EventHandler eventHandler, UUID buildId, boolean finalizeActions) throws AbruptExitException { + Path execRoot, String relativeOutputPath, + EventHandler eventHandler, UUID buildId, boolean finalizeActions) { return ModifiedFileSet.EVERYTHING_MODIFIED; } diff --git a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java index 092d5cafdd31ff..bd6d2600687812 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java +++ b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java @@ -732,6 +732,42 @@ public RemoteOutputsStrategyConverter() { + " seconds.") public Duration remoteFailureWindowInterval; + @Option( + name = "remote_output_service", + defaultValue = "null", + documentationCategory = OptionDocumentationCategory.REMOTE, + effectTags = {OptionEffectTag.UNKNOWN}, + help = + "A URI of a remote output service. A remote output service is a daemon that runs next " + + "to Bazel, managing the contents of the bazel-out/ directory. It may use systems " + + "like FUSE to add features such as snapshotting and lazy loading of objects stored " + + "in a remote Content Addressable Storage.") + public String remoteOutputService; + + @Option( + name = "remote_output_service_output_path_prefix", + defaultValue = "null", + documentationCategory = OptionDocumentationCategory.REMOTE, + effectTags = {OptionEffectTag.UNKNOWN}, + help = + "The path at which all output path directories are visible that are managed by the " + + "remote output service.") + public String remoteOutputServiceOutputPathPrefix; + + @Option( + name = "remote_output_service_output_base_id", + defaultValue = "null", + documentationCategory = OptionDocumentationCategory.REMOTE, + effectTags = {OptionEffectTag.UNKNOWN}, + help = + "The identifier of the workspace in which the build takes place. This value must be " + + "unique for every workspace built through the same remote output service, as it " + + "allows the remote output service to track state for multiple workspaces in " + + "parallel. By default, it is set to the MD5 sum of the path of the workspace. " + + "It may be necessary to override this value if the file system is virtualized " + + "and multiple workspaces are placed at the same path.") + public String remoteOutputServiceOutputBaseId; + // The below options are not configurable by users, only tests. // This is part of the effort to reduce the overall number of flags. diff --git a/src/main/java/com/google/devtools/build/lib/vfs/OutputService.java b/src/main/java/com/google/devtools/build/lib/vfs/OutputService.java index 3fb78d67eb008f..366bc2937aba50 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/OutputService.java +++ b/src/main/java/com/google/devtools/build/lib/vfs/OutputService.java @@ -118,7 +118,9 @@ public default boolean shouldTrustRemoteArtifacts() { * @throws BuildFailedException if build preparation failed * @throws InterruptedException */ - ModifiedFileSet startBuild(EventHandler eventHandler, UUID buildId, boolean finalizeActions) + ModifiedFileSet startBuild( + Path execRoot, String relativeOutputPath, + EventHandler eventHandler, UUID buildId, boolean finalizeActions) throws BuildFailedException, AbruptExitException, InterruptedException; /** Flush and wait for in-progress downloads. */ diff --git a/src/main/protobuf/BUILD b/src/main/protobuf/BUILD index 809860f5474424..906eb01e782e1d 100644 --- a/src/main/protobuf/BUILD +++ b/src/main/protobuf/BUILD @@ -225,6 +225,32 @@ java_library_srcs( deps = [":remote_execution_log_java_proto"], ) +proto_library( + name = "remote_output_service_proto", + srcs = ["remote_output_service.proto"], + deps = [ + "@com_google_protobuf//:empty_proto", + "@com_google_protobuf//:timestamp_proto", + "@remoteapis//:build_bazel_remote_execution_v2_remote_execution_proto", + ], +) + +java_proto_library( + name = "remote_output_service_java_proto", + deps = [":remote_output_service_proto"], +) + +java_grpc_library( + name = "remote_output_service_java_grpc", + srcs = [":remote_output_service_proto"], + deps = [":remote_output_service_java_proto"], +) + +java_library_srcs( + name = "remote_output_service_java_proto_srcs", + deps = [":remote_output_service_java_proto"], +) + proto_library( name = "spawn_proto", srcs = ["spawn.proto"], @@ -280,6 +306,8 @@ filegroup( ":option_filters_java_proto_srcs", ":profile_java_proto_srcs", ":remote_execution_log_java_proto_srcs", + ":remote_output_service_java_grpc_srcs", + ":remote_output_service_java_proto_srcs", ":spawn_java_proto_srcs", ":xcode_java_proto_srcs", ], diff --git a/src/main/protobuf/failure_details.proto b/src/main/protobuf/failure_details.proto index ebce5159a339ab..b52d6b57955f71 100644 --- a/src/main/protobuf/failure_details.proto +++ b/src/main/protobuf/failure_details.proto @@ -295,6 +295,8 @@ message RemoteOptions { CREDENTIALS_WRITE_FAILURE = 3 [(metadata) = { exit_code: 36 }]; DOWNLOADER_WITHOUT_GRPC_CACHE = 4 [(metadata) = { exit_code: 2 }]; EXECUTION_WITH_INVALID_CACHE = 5 [(metadata) = { exit_code: 2 }]; + OUTPUT_SERVICE_WITHOUT_OUTPUT_PATH_REFIX = 6 [(metadata) = { exit_code: 2 }]; + OUTPUT_SERVICE_WITH_INCOMPATIBLE_REMOTE_OUTPUTS_MODE = 7 [(metadata) = { exit_code: 2 }]; } Code code = 1; diff --git a/src/main/protobuf/remote_output_service.proto b/src/main/protobuf/remote_output_service.proto new file mode 100644 index 00000000000000..e8dce0ae7bd7c2 --- /dev/null +++ b/src/main/protobuf/remote_output_service.proto @@ -0,0 +1,320 @@ +// Copyright 2021 The Bazel Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package remote_output_service; + +import "build/bazel/remote/execution/v2/remote_execution.proto"; +import "google/protobuf/empty.proto"; +import "google/protobuf/timestamp.proto"; + +option java_package = "com.google.devtools.build.lib.remote"; +option java_outer_classname = "RemoteOutputServiceProto"; +option go_package = "remoteoutputservice"; + +// The Remote Output Service may be used by users of the Remote +// Execution API to construct a directory on the local system that +// contains all output files of a build. +// +// Primitive implementations of this API may simply download files from +// the Content Addressable Storage (CAS) and store them at their +// designated location. Complex implementations may use a pseudo file +// system (e.g., FUSE) to support deduplication, lazy loading and +// snapshotting. +// +// Details: +// https://github.com/bazelbuild/proposals/blob/master/designs/2021-02-09-remote-output-service.md +// https://groups.google.com/g/remote-execution-apis/c/qOSWWwBLPzo +// https://groups.google.com/g/bazel-dev/c/lKzENsNd1Do +service RemoteOutputService { + // Methods that can be invoked at any point in time. + + // Clean all data associated with a single output path, so that the + // next invocation of StartBuild() yields an empty output path. This + // may be implemented in a way that's faster than removing all of the + // files from the file system manually. + rpc Clean(CleanRequest) returns (google.protobuf.Empty); + + // Signal that a new build is about to start. + // + // The client uses this call to obtain a directory where outputs of + // the build may be stored, called the output path. Based on the + // parameters provided, the remote output service may provide an empty + // output path, or one that has contents from a previous build of the + // same workspace. + // + // In case the output path contains data from a previous build, the + // remote output service is responsible for calling + // ContentAddressableStorage.FindMissingBlobs() for all of the objects + // that are stored remotely. This ensures that these objects don't + // disappear from the Content Addressable Storage while the build is + // running. Any files that are absent must be removed from the output + // path and reported through InitialOutputPathContents.modified_paths. + rpc StartBuild(StartBuildRequest) returns (StartBuildResponse); + + // Methods that can only be invoked during a build. + + // Create one or more files, directories or symbolic links in the + // output path. + rpc BatchCreate(BatchCreateRequest) returns (google.protobuf.Empty); + + // Obtain the status of one or more files, directories or symbolic + // links that are stored in the input path. + rpc BatchStat(BatchStatRequest) returns (BatchStatResponse); + + // Signal that a build has been completed. + rpc FinalizeBuild(FinalizeBuildRequest) returns (google.protobuf.Empty); +} + +message CleanRequest { + // The output base identifier that was provided to + // StartBuildRequest.output_base_id whose data needs to be removed. + string output_base_id = 1; +} + +message StartBuildRequest { + // A client-chosen value that uniquely identifies the workspace for + // which the build is being started. This value must be set to ensure + // that the remote output service is capable of managing builds for + // distinct workspaces concurrently. + // + // This value must be a valid filename for the operating system on + // which the remote output service and client are being executed. This + // allows the remote output service to create one subdirectory per + // project that needs to be built. + // + // By default, Bazel sets this value to the MD5 sum of the absolute + // path of the workspace directory. This is generally sufficient, + // though a more complex scheme may necessary in case the file system + // namespace is virtualized. + // + // Starting a build finalizes any previous build with the same + // output_base_id that has not been finalized yet. + string output_base_id = 1; + + // A client-chosen value that uniquely identifies this build. This + // value must be provided to most other methods to ensure that + // operations are targeted against the right output path. + string build_id = 2; + + // The instance name that the client uses when communicating with the + // remote execution system. The remote output service uses this value + // when loading objects from the Content Addressable Storage. + string instance_name = 3; + + // The digest function that the client uses when communicating with + // the remote execution system. The remote output service uses this + // value to ensure that FileStatus responses contain digests that were + // computed with right digest function. + build.bazel.remote.execution.v2.DigestFunction.Value digest_function = 4; + + // The absolute path at which the remote output service exposes its + // output paths, as seen from the perspective of the client. + // + // This value needs to be provided by the client, because file system + // namespace virtualization may cause this directory to appear at a + // location that differs from the one used by the service. + // + // The purpose of this field is to ensure that the remote output + // service is capable of expanding symbolic links containing absolute + // paths. + string output_path_prefix = 5; + + // A map of paths on the system that will become symbolic links + // pointing to locations inside the output path. Similar to + // output_path_prefix, this option is used to ensure the remote output + // service is capable of expanding symbolic links. + // + // Map keys are absolute paths, while map values are paths that are + // relative to the output path. + map output_path_aliases = 6; +} + +message InitialOutputPathContents { + // The identifier of a previously finalized build whose results are + // stored in the output path. + string build_id = 1; + + // Paths that have been modified or removed since the build finalized. + // + // If the remote output service freezes the contents of the output + // path between builds, this field can be left empty. + repeated string modified_paths = 2; +} + +message StartBuildResponse { + // If set, the contents of the output path are almost entirely + // identical on the results of a previous build. This information may + // be used by the client to prevent unnecessary scanning of the file + // system. + // + // Servers can leave this field unset in case the contents of the + // output path are empty, not based on a previous build, if no + // tracking of this information is performed, or if the number of + // changes made to the output path is too large to be expressed. + InitialOutputPathContents initial_output_path_contents = 1; + + // A relative path that the client must append to + // StartBuildRequest.output_path_prefix to obtain the full path at + // which outputs of the build are stored. + // + // If the remote output service is incapable of storing the output of + // multiple builds, this string may be left empty. + string output_path_suffix = 2; +} + +message BatchCreateRequest { + // The identifier of the build. The remote output service uses this to + // determine which output path needs to be modified. + string build_id = 1; + + // A path relative to the root of the output path where files, + // symbolic links and directories need to be created. + string path_prefix = 2; + + // Whether the contents of the path prefix should be removed prior to + // creating the specified files. + bool clean_path_prefix = 3; + + // Files that need to be downloaded from the Content Addressable + // Storage. + // + // Any missing parent directories, including those in path_prefix, are + // created as well. If any of the parents refer to a non-directory + // file, they are replaced by an empty directory. If a file or + // directory already exists at the provided path, it is replaced. + // + // This means that symbolic links are not followed when evaluating + // path_prefix and OutputFile.path. + repeated build.bazel.remote.execution.v2.OutputFile files = 4; + + // Symbolic links that need to be created. + // + // Any missing parent directories, including those in path_prefix, are + // created as well. If any of the parents refer to a non-directory + // file, they are replaced by an empty directory. If a file or + // directory already exists at the provided path, it is replaced. + // + // This means that symbolic links are not followed when evaluating + // path_prefix and OutputSymlink.path. + repeated build.bazel.remote.execution.v2.OutputSymlink symlinks = 5; + + // Directories that need to be downloaded from the Content Addressable + // Storage. + // + // Any missing parent directories, including those in path_prefix, are + // created as well. If any of the parents refer to a non-directory + // file, they are replaced by an empty directory. Any file or + // directory that already exists at the provided path is replaced. + // + // This means that symbolic links are not followed when evaluating + // path_prefix and OutputDirectory.path. + repeated build.bazel.remote.execution.v2.OutputDirectory directories = 6; +} + +message BatchStatRequest { + // The identifier of the build. The remote output service uses this to + // determine which output path needs to be inspected. + string build_id = 1; + + // In case the path corresponds to a regular file, include the hash + // and size of the file in the response. + bool include_file_digest = 2; + + // In case the path corresponds to a symbolic link, include the target + // of the symbolic link in the response. + bool include_symlink_target = 3; + + // If the last component of the path corresponds to a symbolic link, + // return the status of the file at the target location. + // + // Symbolic links encountered before the last component of the path + // are always expanded, regardless of the value of this option. + bool follow_symlinks = 4; + + // Paths whose status needs to be obtained. + repeated string paths = 5; +} + +message BatchStatResponse { + // The status response for each of the requested paths, using the same + // order as requested. This means that this list has the same length + // as BatchStatRequest.paths. + repeated StatResponse responses = 1; +} + +message StatResponse { + // The status of the file. If the file corresponding with the + // requested path does not exist, this field will be null. + FileStatus file_status = 1; +} + +message FileStatus { + message File { + // The hash and size of the file. This field is only set when + // BatchStatRequest.include_file_digest is set. + build.bazel.remote.execution.v2.Digest digest = 1; + } + + message Symlink { + // The target of the symbolic link. This field is only set when + // BatchStatRequest.include_symlink_target is set. + string target = 1; + } + + message Directory { + // The time at which the directory contents were last modified. + google.protobuf.Timestamp last_modified_time = 1; + } + + message External { + // The path relative to the root of the output path where the file + // is located. This path is absolute, or it is relative, starting + // with "../". + // + // The client can use this field to obtain the file status manually. + string next_path = 1; + } + + oneof file_type { + // The path resolves to a regular file. + File file = 1; + + // The path resolves to a symbolic link. + // + // This field may not be set if BatchStatRequest.follow_symlinks is + // set to true. + Symlink symlink = 2; + + // The path resolves to a directory. + Directory directory = 3; + + // The path resolves to a location outside the output path. The + // remote output service is unable to determine whether any file + // exists at the resulting path, and can therefore not obtain its + // status. + External external = 4; + } +} + +message FinalizeBuildRequest { + // The identifier of the build that should be finalized. + string build_id = 1; + + // Whether the build completed successfully. The remote output service + // may, for example, use this option to apply different retention + // policies that take the outcome of the build into account. + bool build_successful = 2; +} diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java index fcc728e7cf7814..db9160627f0b90 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java @@ -2100,6 +2100,7 @@ private RemoteExecutionService newRemoteExecutionService(RemoteOptions remoteOpt cache, executor, tempPathGenerator, - null); + /*captureCorruptedOutputsDir=*/ null, + /*actionResultDownloader=*/ null); } } diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java index f0a6fefd5eb023..3f51a135411694 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java @@ -248,7 +248,8 @@ private RemoteSpawnCache remoteSpawnCacheWithOptions(RemoteOptions options) { remoteCache, null, tempPathGenerator, - /* captureCorruptedOutputsDir= */ null)); + /* captureCorruptedOutputsDir= */ null, + /* actionResultDownloader= */ null)); return new RemoteSpawnCache(execRoot, options, /* verboseFailures=*/ true, service); } diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java index b479751e86fab3..869d781669ed2e 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java @@ -1048,7 +1048,8 @@ private void testParamFilesAreMaterializedForFlag(String flag) throws Exception cache, executor, tempPathGenerator, - /* captureCorruptedOutputsDir= */ null); + /*captureCorruptedOutputsDir=*/ null, + /*actionResultDownloader=*/ null); RemoteSpawnRunner runner = new RemoteSpawnRunner( execRoot, @@ -1585,7 +1586,8 @@ private RemoteSpawnRunner newSpawnRunner( cache, executor, tempPathGenerator, - /*captureCorruptedOutputsDir=*/ null)); + /*captureCorruptedOutputsDir=*/ null, + /*actionResultDownloader=*/ null)); return new RemoteSpawnRunner( execRoot, diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java index ee128fe43cfdb0..3bde4086ceceff 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerWithGrpcRemoteExecutorTest.java @@ -328,7 +328,8 @@ public int maxConcurrency() { remoteCache, executor, tempPathGenerator, - /* captureCorruptedOutputsDir= */ null); + /* captureCorruptedOutputsDir= */ null, + /* actionResultDownloader= */ null); client = new RemoteSpawnRunner( execRoot, diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/OutputsInvalidationIntegrationTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/OutputsInvalidationIntegrationTest.java index bb4441ff70b97a..b30469cbb0d578 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/OutputsInvalidationIntegrationTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/OutputsInvalidationIntegrationTest.java @@ -55,7 +55,7 @@ public void prepareOutputServiceMock() throws BuildFailedException, AbruptExitException, InterruptedException, IOException { when(outputService.actionFileSystemType()).thenReturn(ActionFileSystemType.DISABLED); when(outputService.getFilesSystemName()).thenReturn("fileSystemName"); - when(outputService.startBuild(any(), any(), anyBoolean())) + when(outputService.startBuild(any(), any(), any(), any(), anyBoolean())) .thenReturn(ModifiedFileSet.EVERYTHING_MODIFIED); } @@ -86,7 +86,7 @@ public void nothingModified_doesntInvalidateAnyActions(@TestParameter boolean de delete(getOnlyOutput("//foo")); } - when(outputService.startBuild(any(), any(), anyBoolean())) + when(outputService.startBuild(any(), any(), any(), any(), anyBoolean())) .thenReturn(ModifiedFileSet.NOTHING_MODIFIED); events.collector().clear(); buildTarget("//foo"); @@ -124,7 +124,7 @@ public void identicalOutputs_doesntInvalidateAnyActions( buildTarget("//foo"); MoreAsserts.assertContainsEvent(events.collector(), "Executing genrule //foo:foo"); - when(outputService.startBuild(any(), any(), anyBoolean())) + when(outputService.startBuild(any(), any(), any(), any(), anyBoolean())) .thenReturn(modification.modifiedFileSet(getOnlyOutput("//foo"))); events.collector().clear(); buildTarget("//foo"); @@ -140,7 +140,7 @@ public void noCheckOutputFiles_ignoresModifiedFiles( buildTarget("//foo"); MoreAsserts.assertContainsEvent(events.collector(), "Executing genrule //foo:foo"); - when(outputService.startBuild(any(), any(), anyBoolean())) + when(outputService.startBuild(any(), any(), any(), any(), anyBoolean())) .thenReturn(modification.modifiedFileSet(getOnlyOutput("//foo"))); events.collector().clear(); buildTarget("//foo"); @@ -162,7 +162,7 @@ public void everythingModified_invalidatesAllActions( MoreAsserts.assertContainsEvent(events.collector(), "Executing genrule //foo:foo"); delete(getOnlyOutput("//foo")); - when(outputService.startBuild(any(), any(), anyBoolean())) + when(outputService.startBuild(any(), any(), any(), any(), anyBoolean())) .thenReturn( everythingDeleted ? ModifiedFileSet.EVERYTHING_DELETED @@ -185,7 +185,7 @@ public void outputFileModified_invalidatesOnlyAffectedAction() throws Exception Artifact fooOut = getOnlyOutput("//foo"); delete(fooOut); - when(outputService.startBuild(any(), any(), anyBoolean())).thenReturn(modifiedFileSet(fooOut)); + when(outputService.startBuild(any(), any(), any(), any(), anyBoolean())).thenReturn(modifiedFileSet(fooOut)); events.collector().clear(); buildTarget("//foo:all");