Skip to content

Commit

Permalink
when writing to local disk cache, open files later in order to avoid …
Browse files Browse the repository at this point in the history
…"too many open files"

Re-use the existing "LazyFileOutputStream" in DiskAndRemoteCacheClient.java
in order to avoid "Too many open files".

Resolves bazelbuild#13435

Closes bazelbuild#13574.

PiperOrigin-RevId: 379892227
  • Loading branch information
jablin authored and coeuvre committed Jul 16, 2021
1 parent 97d7b4c commit ae56b54
Show file tree
Hide file tree
Showing 3 changed files with 72 additions and 56 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
import com.google.devtools.build.lib.remote.RemoteCache.ActionResultMetadata.DirectoryMetadata;
import com.google.devtools.build.lib.remote.RemoteCache.ActionResultMetadata.FileMetadata;
import com.google.devtools.build.lib.remote.RemoteCache.ActionResultMetadata.SymlinkMetadata;
import com.google.devtools.build.lib.remote.common.LazyFileOutputStream;
import com.google.devtools.build.lib.remote.common.OutputDigestMismatchException;
import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
import com.google.devtools.build.lib.remote.common.RemoteActionFileArtifactValue;
Expand Down Expand Up @@ -1099,57 +1100,6 @@ private static FailureDetail createFailureDetail(String message, Code detailedCo
.build();
}

/**
* Creates an {@link OutputStream} that isn't actually opened until the first data is written.
* This is useful to only have as many open file descriptors as necessary at a time to avoid
* running into system limits.
*/
private static class LazyFileOutputStream extends OutputStream {

private final Path path;
private OutputStream out;

public LazyFileOutputStream(Path path) {
this.path = path;
}

@Override
public void write(byte[] b) throws IOException {
ensureOpen();
out.write(b);
}

@Override
public void write(byte[] b, int off, int len) throws IOException {
ensureOpen();
out.write(b, off, len);
}

@Override
public void write(int b) throws IOException {
ensureOpen();
out.write(b);
}

@Override
public void flush() throws IOException {
ensureOpen();
out.flush();
}

@Override
public void close() throws IOException {
ensureOpen();
out.close();
}

private void ensureOpen() throws IOException {
if (out == null) {
out = path.getOutputStream();
}
}
}

/** In-memory representation of action result metadata. */
static class ActionResultMetadata {

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
// Copyright 2021 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.remote.common;

import com.google.devtools.build.lib.vfs.Path;
import java.io.IOException;
import java.io.OutputStream;

/**
* Creates an {@link OutputStream} that isn't actually opened until the first data is written. This
* is useful to only have as many open file descriptors as necessary at a time to avoid running into
* system limits.
*/
public class LazyFileOutputStream extends OutputStream {

private final Path path;
private OutputStream out;

public LazyFileOutputStream(Path path) {
this.path = path;
}

@Override
public void write(byte[] b) throws IOException {
ensureOpen();
out.write(b);
}

@Override
public void write(byte[] b, int off, int len) throws IOException {
ensureOpen();
out.write(b, off, len);
}

@Override
public void write(int b) throws IOException {
ensureOpen();
out.write(b);
}

@Override
public void flush() throws IOException {
ensureOpen();
out.flush();
}

@Override
public void close() throws IOException {
ensureOpen();
out.close();
}

private void ensureOpen() throws IOException {
if (out == null) {
out = path.getOutputStream();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.devtools.build.lib.remote.common.LazyFileOutputStream;
import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
import com.google.devtools.build.lib.remote.common.RemoteCacheClient;
import com.google.devtools.build.lib.remote.options.RemoteOptions;
Expand Down Expand Up @@ -142,11 +143,7 @@ public ListenableFuture<Void> downloadBlob(

Path tempPath = newTempPath();
final OutputStream tempOut;
try {
tempOut = tempPath.getOutputStream();
} catch (IOException e) {
return Futures.immediateFailedFuture(e);
}
tempOut = new LazyFileOutputStream(tempPath);

if (!options.incompatibleRemoteResultsIgnoreDisk || options.remoteAcceptCached) {
ListenableFuture<Void> download =
Expand Down

0 comments on commit ae56b54

Please sign in to comment.