diff --git a/src/main/java/com/google/devtools/build/lib/actions/Artifact.java b/src/main/java/com/google/devtools/build/lib/actions/Artifact.java
index c5e79a90ee5962..2f779d5fc3cf6c 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/Artifact.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/Artifact.java
@@ -28,6 +28,7 @@
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Streams;
+import com.google.devtools.build.docgen.annot.DocCategory;
import com.google.devtools.build.lib.actions.ArtifactRoot.RootType;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.LabelConstants;
@@ -42,6 +43,7 @@
import com.google.devtools.build.lib.skyframe.serialization.VisibleForSerialization;
import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant;
import com.google.devtools.build.lib.starlarkbuildapi.FileApi;
+import com.google.devtools.build.lib.starlarkbuildapi.FileRootApi;
import com.google.devtools.build.lib.util.FileType;
import com.google.devtools.build.lib.util.FileTypeSet;
import com.google.devtools.build.lib.util.HashCodes;
@@ -62,9 +64,11 @@
import java.util.Set;
import java.util.function.UnaryOperator;
import javax.annotation.Nullable;
+import net.starlark.java.annot.StarlarkBuiltin;
import net.starlark.java.eval.EvalException;
import net.starlark.java.eval.Printer;
import net.starlark.java.eval.Starlark;
+import net.starlark.java.eval.StarlarkSemantics;
/**
* An Artifact represents a file used by the build system, whether it's a source file or a derived
@@ -582,17 +586,23 @@ public SpecialArtifact getParent() {
/**
* Returns the directory name of this artifact, similar to dirname(1).
*
- *
The directory name is always a relative path to the execution directory.
+ *
The directory name is always a relative path to the execution directory.
*/
- @Override
public final String getDirname() {
+ return getDirname(execPath);
+ }
+
+ @Override
+ public final String getDirnameForStarlark(StarlarkSemantics semantics) {
+ return getDirname(PathMapper.loadFrom(semantics).map(execPath));
+ }
+
+ private static String getDirname(PathFragment execPath) {
PathFragment parent = execPath.getParentDirectory();
return (parent == null) ? "/" : parent.getSafePathString();
}
- /**
- * Returns the base file name of this artifact, similar to basename(1).
- */
+ /** Returns the base file name of this artifact, similar to basename(1). */
@Override
public final String getFilename() {
return execPath.getBaseName();
@@ -649,16 +659,37 @@ public final Label getOwner() {
* package-path entries (for source Artifacts), or one of the bin, genfiles or includes dirs (for
* derived Artifacts). It will always be an ancestor of getPath().
*/
- @Override
public final ArtifactRoot getRoot() {
return root;
}
+ @Override
+ public final FileRootApi getRootForStarlark(StarlarkSemantics semantics) {
+ // It would *not* be correct to just apply PathMapper#map to the exec path of the root: The
+ // root part of the mapped exec path of this artifact may depend on its complete exec path as
+ // well as on e.g. the digest of the artifact.
+ PathFragment mappedExecPath = PathMapper.loadFrom(semantics).map(execPath);
+ if (mappedExecPath.equals(execPath)) {
+ return root;
+ }
+ // PathMapper#map never changes the root-relative part of the exec path, so we can remove that
+ // suffix to get the mapped root part.
+ int rootRelativeSegmentCount = execPath.segmentCount() - root.getExecPath().segmentCount();
+ PathFragment mappedRootExecPath =
+ mappedExecPath.subFragment(0, mappedExecPath.segmentCount() - rootRelativeSegmentCount);
+ return new MappedArtifactRoot(mappedRootExecPath);
+ }
+
@Override
public final PathFragment getExecPath() {
return execPath;
}
+ @Override
+ public String getExecPathStringForStarlark(StarlarkSemantics semantics) {
+ return PathMapper.loadFrom(semantics).getMappedExecPathString(this);
+ }
+
/**
* Returns the relative path to this artifact relative to its root. It makes no guarantees as to
* the semantic meaning or the completeness of the returned path value. In other words, no
@@ -1641,4 +1672,61 @@ public String toString() {
return MoreObjects.toStringHelper(this).add("artifact", artifact.toDebugString()).toString();
}
}
+
+ /** A {@link FileRootApi} obtained by applying a {@link PathMapper} to an {@link ArtifactRoot}. */
+ @StarlarkBuiltin(
+ name = "mapped_root",
+ category = DocCategory.BUILTIN,
+ doc = "A root for files that have been subject to path mapping")
+ private static final class MappedArtifactRoot
+ implements FileRootApi, Comparable {
+ private final PathFragment mappedRootExecPath;
+
+ public MappedArtifactRoot(PathFragment mappedRootExecPath) {
+ this.mappedRootExecPath = mappedRootExecPath;
+ }
+
+ @Override
+ public String getExecPathString() {
+ return mappedRootExecPath.getPathString();
+ }
+
+ @Override
+ public int compareTo(MappedArtifactRoot otherRoot) {
+ return mappedRootExecPath.compareTo(otherRoot.mappedRootExecPath);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ // Per the contract of PathMapper#map, mapped roots never have exec paths that are equal to
+ // exec paths of non-mapped roots, that is, of instances of ArtifactRoot. Thus, it is correct
+ // for both equals implementations to return false if the other object is not an instance of
+ // the respective class.
+ if (!(obj instanceof MappedArtifactRoot)) {
+ return false;
+ }
+ MappedArtifactRoot other = (MappedArtifactRoot) obj;
+ return mappedRootExecPath.equals(other.mappedRootExecPath);
+ }
+
+ @Override
+ public int hashCode() {
+ return mappedRootExecPath.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return mappedRootExecPath + " [mapped]";
+ }
+
+ @Override
+ public void repr(Printer printer) {
+ printer.append("");
+ }
+
+ @Override
+ public boolean isImmutable() {
+ return true;
+ }
+ }
}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/BUILD b/src/main/java/com/google/devtools/build/lib/actions/BUILD
index 2037280420f728..db269e9c9b0531 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/actions/BUILD
@@ -307,6 +307,7 @@ java_library(
"ArtifactResolver.java",
"ArtifactRoot.java",
"Artifacts.java",
+ "PathMapper.java",
],
deps = [
":action_lookup_data",
@@ -316,6 +317,7 @@ java_library(
":fileset_output_symlink",
":package_roots",
":path_strippable",
+ "//src/main/java/com/google/devtools/build/docgen/annot",
"//src/main/java/com/google/devtools/build/lib/cmdline",
"//src/main/java/com/google/devtools/build/lib/collect/nestedset",
"//src/main/java/com/google/devtools/build/lib/concurrent",
@@ -334,6 +336,7 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
"//src/main/java/com/google/devtools/build/skyframe:execution_phase_skykey",
"//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
+ "//src/main/java/net/starlark/java/annot",
"//src/main/java/net/starlark/java/eval",
"//src/main/protobuf:failure_details_java_proto",
"//third_party:guava",
diff --git a/src/main/java/com/google/devtools/build/lib/actions/PathMapper.java b/src/main/java/com/google/devtools/build/lib/actions/PathMapper.java
index 0e7dcc546dd531..51b8fe63380eba 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/PathMapper.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/PathMapper.java
@@ -17,14 +17,20 @@
import com.google.devtools.build.lib.actions.CommandLineItem.ExceptionlessMapFn;
import com.google.devtools.build.lib.actions.CommandLineItem.MapFn;
import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.errorprone.annotations.CheckReturnValue;
import java.util.List;
+import java.util.Objects;
import javax.annotation.Nullable;
+import net.starlark.java.eval.StarlarkSemantics;
/**
* Support for mapping config parts of exec paths in an action's command line as well as when
* staging its inputs and outputs for execution, with the aim of making the resulting {@link Spawn}
* more cacheable.
*
+ *
Implementations must be pure functions of the set of inputs (including paths and potentially
+ * content) of a given action.
+ *
*
Actions that want to support path mapping should use {@link
* com.google.devtools.build.lib.analysis.actions.PathMappers}.
*
@@ -32,23 +38,77 @@
* com.google.devtools.build.lib.analysis.actions.StrippingPathMapper}, which removes the config
* part (e.g. "k8-fastbuild") from exec paths to allow for cross-configuration cache hits.
*/
-public interface PathMapper {
+public abstract class PathMapper {
/**
* Returns the exec path with the path mapping applied.
*
+ *
An exec path typically consists of a root part (such as "bazel-out/k8-opt/bin"
+ * or "") and a root-relative part (such as "path/to/pkg/file").
+ *
+ *
Overrides must satisfy the following properties:
+ *
+ *
+ *
The root-relative part of the path must not be modified.
+ *
If the path is modified, the new root part must be different from any possible valid root
+ * part of an unmapped path.
+ *
+ *
*
Path mappers may return paths with different roots for two paths that have the same root
* (e.g., they may map an artifact at {@code bazel-out/k8-fastbuild/bin/pkg/foo} to {@code
* bazel-out//bin/pkg/foo}). Paths of artifacts that should share the same
* parent directory, such as runfiles or tree artifact files, should thus be derived from the
* mapped path of their parent.
*/
- PathFragment map(PathFragment execPath);
+ public abstract PathFragment map(PathFragment execPath);
/** Returns the exec path of the input with the path mapping applied. */
- default String getMappedExecPathString(ActionInput artifact) {
+ public String getMappedExecPathString(ActionInput artifact) {
return map(artifact.getExecPath()).getPathString();
}
+ /** A {@link PathMapper} that doesn't change paths. */
+ public static final PathMapper NOOP =
+ new PathMapper() {
+ @Override
+ public PathFragment map(PathFragment execPath) {
+ return execPath;
+ }
+ };
+
+ private static final StarlarkSemantics.Key SEMANTICS_KEY =
+ new StarlarkSemantics.Key<>("path_mapper", PathMapper.NOOP);
+
+ /**
+ * Retrieve the {@link PathMapper} instance stored in the given {@link StarlarkSemantics} via
+ * {@link #storeIn(StarlarkSemantics)}.
+ */
+ public static PathMapper loadFrom(StarlarkSemantics semantics) {
+ return semantics.get(SEMANTICS_KEY);
+ }
+
+ /**
+ * Creates a new {@link StarlarkSemantics} instance which causes all Starlark threads using it to
+ * automatically apply this {@link PathMapper} to all struct fields of {@link
+ * com.google.devtools.build.lib.starlarkbuildapi.FileApi}.
+ *
+ *
This is meant to be used when evaluating user-defined callbacks to Starlark variants of
+ * custom command lines that are evaluated during the execution phase.
+ *
+ *
Since any unmapped path appearing in a command line will prevent cross-configuration cache
+ * hits, this mapping is applied automatically instead of requiring users to explicitly map all
+ * paths themselves. As an added benefit, this allows actions to opt into path mapping without
+ * actual changes to their command line code.
+ */
+ @CheckReturnValue
+ public StarlarkSemantics storeIn(StarlarkSemantics semantics) {
+ // Since PathMapper#equals returns true for all instances of the same class, every non-noop
+ // instance is different from the default value for the key and thus persisted. Furthermore,
+ // since this causes the resulting semantics to compare equal if they PathMapper class
+ // agrees, there is only a single cache entry for it in Starlark's CallUtils cache for Starlark
+ // methods.
+ return semantics.toBuilder().set(SEMANTICS_KEY, this).build();
+ }
+
/**
* We don't yet have a Starlark API for mapping paths in command lines. Simple Starlark calls like
* {@code args.add(arg_name, file_path} are automatically handled. But calls that involve custom
@@ -57,7 +117,7 @@ default String getMappedExecPathString(ActionInput artifact) {
*
This method allows implementations to hard-code support for specific command line entries
* for specific Starlark actions.
*/
- default List mapCustomStarlarkArgs(List args) {
+ public List mapCustomStarlarkArgs(List args) {
return args;
}
@@ -74,7 +134,7 @@ default List mapCustomStarlarkArgs(List args) {
*
*
By default, this method returns {@link MapFn#DEFAULT}.
*/
- default ExceptionlessMapFn