Implement Hermetic sandbox with support for hardlinks

src/main/java/com/google/devtools/build/lib/actions/FileContentsProxy.java

@@ -20,27 +20,33 @@
 
 /**
  * In case we can't get a fast digest from the filesystem, we store this metadata as a proxy to the
- * file contents. Currently it is a pair of a relevant timestamp and a "node id". On Linux the
- * former is the ctime and the latter is the inode number. We might want to add the device number in
+ * file contents. Currently it is two timestamps and a "node id". On Linux we
+ * use both ctime and mtime and inode number. We might want to add the device number in
  * the future.
  *
- * <p>For a Linux example of why mtime alone is insufficient, note that 'mv' preserves timestamps.
+ * <p>For a Linux example of why mtime alone is insufficient, note that 'mv' preserves mtime.
  * So if files 'a' and 'b' initially have the same timestamp, then we would think 'b' is unchanged
  * after the user executes `mv a b` between two builds.
+ *
+ * <p>On Linux we also need mtime for hardlinking sandbox, since updating the inode reference counter
+ * preserves mtime, but updates ctime. isModified() call can be used to compare two FileContentsProxys
+ * of hardlinked files.
  */
 public final class FileContentsProxy {
   private final long ctime;
+  private final long mtime;
   private final long nodeId;
 
-  private FileContentsProxy(long ctime, long nodeId) {
+  public FileContentsProxy(long ctime, long mtime, long nodeId) {
     this.ctime = ctime;
+    this.mtime = mtime;
     this.nodeId = nodeId;
   }
 
   public static FileContentsProxy create(FileStatus stat) throws IOException {
     // Note: there are file systems that return mtime for this call instead of ctime, such as the
     // WindowsFileSystem.
-    return new FileContentsProxy(stat.getLastChangeTime(), stat.getNodeId());
+    return new FileContentsProxy(stat.getLastChangeTime(), stat.getLastModifiedTime(), stat.getNodeId());
   }
 
   @Override
@@ -54,16 +60,31 @@ public boolean equals(Object other) {
     }
 
     FileContentsProxy that = (FileContentsProxy) other;
-    return ctime == that.ctime && nodeId == that.nodeId;
+    return ctime == that.ctime && mtime == that.mtime && nodeId == that.nodeId;
+  }
+
+  /**
+   * Can be used when hardlink reference counter changes
+   * should not be considered a file modification.
+   * Is only comparing mtime and not ctime and is therefore
+   * not detecting changed metadata like permission.
+   */
+  public boolean isModified(FileContentsProxy other) {
+    if (other == this) {
+      return false;
+    }
+    // true if nodeId are different or inode has a new mtime
+    return nodeId != other.nodeId || mtime != other.mtime;
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(ctime, nodeId);
+    return Objects.hash(ctime, mtime, nodeId);
   }
 
   void addToFingerprint(Fingerprint fp) {
     fp.addLong(ctime);
+    fp.addLong(mtime);
     fp.addLong(nodeId);
   }
 
@@ -73,6 +94,6 @@ public String toString() {
   }
 
   public String prettyPrint() {
-    return String.format("ctime of %d and nodeId of %d", ctime, nodeId);
+    return String.format("ctime of %d and mtime of %d and nodeId of %d", ctime, mtime, nodeId);
   }
 }

src/main/java/com/google/devtools/build/lib/sandbox/AbstractSandboxSpawnRunner.java

@@ -131,6 +131,9 @@ private SpawnResult runSpawn(
       try (SilentCloseable c = Profiler.instance().profile("subprocess.run")) {
         result = run(originalSpawn, sandbox, context.getTimeout(), outErr);
       }
+      try (SilentCloseable c = Profiler.instance().profile("sandbox.verifyPostCondition")) {
+        verifyPostCondition(originalSpawn, sandbox, context);
+      }
 
       context.lockOutputFiles();
       try (SilentCloseable c = Profiler.instance().profile("sandbox.copyOutputs")) {
@@ -148,6 +151,11 @@ private SpawnResult runSpawn(
       }
     }
   }
+  /**
+   * Override this method if you need to run a post condition after the action has executed
+   */
+  public void verifyPostCondition(Spawn originalSpawn, SandboxedSpawn sandbox,
+      SpawnExecutionContext context) throws IOException, ForbiddenActionInputException {}
 
   private String makeFailureMessage(Spawn originalSpawn, SandboxedSpawn sandbox) {
     if (sandboxOptions.sandboxDebug) {

src/main/java/com/google/devtools/build/lib/sandbox/BUILD

@@ -18,6 +18,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib:runtime",
         "//src/main/java/com/google/devtools/build/lib/actions",
         "//src/main/java/com/google/devtools/build/lib/actions:artifacts",
+        "//src/main/java/com/google/devtools/build/lib/actions:file_metadata",
         "//src/main/java/com/google/devtools/build/lib/actions:execution_requirements",
         "//src/main/java/com/google/devtools/build/lib/actions:localhost_capacity",
         "//src/main/java/com/google/devtools/build/lib/analysis:blaze_directories",

src/main/java/com/google/devtools/build/lib/sandbox/HardlinkedSandboxedSpawn.java

@@ -0,0 +1,100 @@
+// Copyright 2016 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.sandbox;
+
+import com.google.common.flogger.GoogleLogger;
+import com.google.devtools.build.lib.exec.TreeDeleter;
+import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxInputs;
+import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxOutputs;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.Symlinks;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Creates an execRoot for a Spawn that contains input files as hardlinks to their original
+ * destination.
+ */
+public class HardlinkedSandboxedSpawn extends AbstractContainerizingSandboxedSpawn {
+  private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();
+  private boolean sandboxDebug = false;
+  public HardlinkedSandboxedSpawn(
+      Path sandboxPath,
+      Path sandboxExecRoot,
+      List<String> arguments,
+      Map<String, String> environment,
+      SandboxInputs inputs,
+      SandboxOutputs outputs,
+      Set<Path> writableDirs,
+      TreeDeleter treeDeleter,
+      @Nullable Path statisticsPath,
+      boolean sandboxDebug) {
+    super(
+        sandboxPath,
+        sandboxExecRoot,
+        arguments,
+        environment,
+        inputs,
+        outputs,
+        writableDirs,
+        treeDeleter,
+        statisticsPath);
+    this.sandboxDebug = sandboxDebug;
+  }
+
+  @Override
+  protected void copyFile(Path source, Path target) throws IOException {
+    hardLinkRecursive(source, target);
+  }
+
+  /**
+   * Recursively creates hardlinks for all files in @param source path, in @param target path.
+   * Symlinks are resolved. If files is located on another disk, hardlink will fail
+   * and a copy will be made instead.
+   * Throws IllegalArgumentException if source path is a subdirectory of target path.
+   */
+  private void hardLinkRecursive(Path source, Path target) throws IOException {
+    if (source.isSymbolicLink()) {
+      source = source.resolveSymbolicLinks();
+    }
+
+    if (source.isFile(Symlinks.NOFOLLOW)) {
+      try {
+        source.createHardLink(target);
+      } catch (IOException e) {
+        if (sandboxDebug) {
+          logger.atInfo().log("File %s could not be hardlinked, file will be copied instead.", source);
+        }
+        FileSystemUtils.copyFile(source, target);
+      }
+    } else if (source.isDirectory()) {
+      if (source.startsWith(target)) {
+        throw new IllegalArgumentException(source + " is a subdirectory of " + target);
+      }
+      target.createDirectory();
+      Collection<Path> entries = source.getDirectoryEntries();
+      for (Path entry : entries) {
+        Path toPath = target.getChild(entry.getBaseName());
+        hardLinkRecursive(entry, toPath);
+      }
+    }
+  }
+}

src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxUtil.java

@@ -57,7 +57,7 @@ public static CommandLineBuilder commandLineBuilder(
   public static class CommandLineBuilder {
     private final Path linuxSandboxPath;
     private final List<String> commandArguments;
-
+    private Path hermeticSandboxPath;
     private Path workingDirectory;
     private Duration timeout;
     private Duration killDelay;
@@ -79,6 +79,13 @@ private CommandLineBuilder(Path linuxSandboxPath, List<String> commandArguments)
       this.commandArguments = commandArguments;
     }
 
+    /** Sets the sandbox path to chroot to, required for the hermetic linux sandbox to figure out
+        where the working directory is. */
+    public CommandLineBuilder setHermeticSandboxPath(Path sandboxPath) {
+      this.hermeticSandboxPath = sandboxPath;
+      return this;
+    }
+
     /** Sets the working directory to use, if any. */
     public CommandLineBuilder setWorkingDirectory(Path workingDirectory) {
       this.workingDirectory = workingDirectory;
@@ -221,6 +228,9 @@ public ImmutableList<String> build() {
       if (statisticsPath != null) {
         commandLineBuilder.add("-S", statisticsPath.getPathString());
       }
+      if (hermeticSandboxPath != null) {
+        commandLineBuilder.add("-h", hermeticSandboxPath.getPathString());
+      }
       if (useFakeHostname) {
         commandLineBuilder.add("-H");
       }

src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunner.java

@@ -19,12 +19,16 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Maps;
 import com.google.common.io.ByteStreams;
+import com.google.devtools.build.lib.actions.ActionInput;
 import com.google.devtools.build.lib.actions.ExecException;
 import com.google.devtools.build.lib.actions.ExecutionRequirements;
+import com.google.devtools.build.lib.actions.FileArtifactValue;
+import com.google.devtools.build.lib.actions.FileContentsProxy;
 import com.google.devtools.build.lib.actions.ForbiddenActionInputException;
 import com.google.devtools.build.lib.actions.Spawn;
 import com.google.devtools.build.lib.actions.Spawns;
 import com.google.devtools.build.lib.actions.UserExecException;
+import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
 import com.google.devtools.build.lib.analysis.BlazeDirectories;
 import com.google.devtools.build.lib.exec.TreeDeleter;
 import com.google.devtools.build.lib.exec.local.LocalEnvProvider;
@@ -38,17 +42,19 @@
 import com.google.devtools.build.lib.shell.Command;
 import com.google.devtools.build.lib.shell.CommandException;
 import com.google.devtools.build.lib.util.OS;
+import com.google.devtools.build.lib.vfs.FileStatus;
 import com.google.devtools.build.lib.vfs.FileSystem;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.PathFragment;
 import com.google.devtools.build.lib.vfs.Symlinks;
+
+import javax.annotation.Nullable;
 import java.io.File;
 import java.io.IOException;
 import java.time.Duration;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.SortedMap;
-import javax.annotation.Nullable;
 
 /** Spawn runner that uses linux sandboxing APIs to execute a local subprocess. */
 final class LinuxSandboxedSpawnRunner extends AbstractSandboxSpawnRunner {
@@ -229,6 +235,19 @@ spawn, getSandboxOptions().defaultSandboxAllowNetwork)))
           sandboxfsMapSymlinkTargets,
           treeDeleter,
           statisticsPath);
+    } else if (getSandboxOptions().useHermetic) {
+      commandLineBuilder.setHermeticSandboxPath(sandboxPath);
+      return new HardlinkedSandboxedSpawn(
+          sandboxPath,
+          sandboxExecRoot,
+          commandLineBuilder.build(),
+          environment,
+          inputs,
+          outputs,
+          writableDirs,
+          treeDeleter,
+          statisticsPath,
+          getSandboxOptions().sandboxDebug);
     } else {
       return new SymlinkedSandboxedSpawn(
           sandboxPath,
@@ -355,6 +374,42 @@ private void validateBindMounts(SortedMap<Path, Path> bindMounts) throws UserExe
       }
     }
   }
+  @Override
+  public void verifyPostCondition(
+      Spawn originalSpawn, SandboxedSpawn sandbox, SpawnExecutionContext context) throws IOException, ForbiddenActionInputException {
+    if(getSandboxOptions().useHermetic){
+      checkForConcurrentModifications(context);
+    }
+  }
+
+  private void checkForConcurrentModifications(SpawnExecutionContext context) throws IOException, ForbiddenActionInputException {
+    for (ActionInput input : (context.getInputMapping(PathFragment.EMPTY_FRAGMENT).values())) {
+      if (input instanceof VirtualActionInput) {
+        continue;
+      }
+
+      FileArtifactValue metadata = context.getMetadataProvider().getMetadata(input);
+      Path path = execRoot.getRelative(input.getExecPath());
+
+      try {
+        if (wasModifiedSinceDigest(metadata.getContentsProxy(), path)) {
+          throw new IOException("input dependency " + path + " was modified during execution.");
+        }
+      } catch (UnsupportedOperationException e) {
+        throw new IOException(
+          "input dependency " + path + " could not be checked for modifications during execution.",
+          e);
+      }
+    }
+  }
+
+  private boolean wasModifiedSinceDigest(FileContentsProxy proxy, Path path) throws IOException {
+      if (proxy == null) {
+        return false;
+      }
+      FileStatus stat = path.statIfFound(Symlinks.FOLLOW);
+      return stat == null || !stat.isFile() || proxy.isModified(FileContentsProxy.create(stat));
+  }
 
   @Override
   public void cleanupSandboxBase(Path sandboxBase, TreeDeleter treeDeleter) throws IOException {

src/main/java/com/google/devtools/build/lib/sandbox/SandboxOptions.java

@@ -371,6 +371,19 @@ public ImmutableSet<Path> getInaccessiblePaths(FileSystem fs) {
               + " avoid unnecessary setup costs.")
   public boolean reuseSandboxDirectories;
 
+  @Option(
+      name = "experimental_use_hermetic_linux_sandbox",
+      defaultValue = "false",
+      documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY,
+      effectTags = {OptionEffectTag.EXECUTION},
+      help =
+        "If set to true, do not mount root, only mount whats provided with "
+            + "sandbox_add_mount_pair. Input files will be hardlinked to the sandbox instead of "
+            + "symlinked to from the sandbox. "
+            + "If action input files are located on a filesystem different from the sandbox, "
+            + "then the input files will be copied instead.")
+  public boolean useHermetic;
+
   /** Converter for the number of threads used for asynchronous tree deletion. */
   public static final class AsyncTreeDeletesConverter extends ResourceConverter {
     public AsyncTreeDeletesConverter() {