Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Escape colons in globs before sending to the daemon #6141

Merged
merged 4 commits into from
Oct 12, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 25 additions & 3 deletions cli/internal/daemonclient/daemonclient.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ package daemonclient
import (
"context"
"path/filepath"
"runtime"
"strings"

"github.com/vercel/turbo/cli/internal/daemon/connector"
"github.com/vercel/turbo/cli/internal/fs/hash"
Expand Down Expand Up @@ -32,12 +34,32 @@ func New(client *connector.Client) *DaemonClient {
}
}

// formats a repo-relative glob to unix format with ':' characters handled.
// On windows, ':' is an invalid path character, but you can, and Turborepo does,
// read to and write from files that contain alternate data streams denoted by ':'.
// In the case of windows and an alternate data stream, we want change notifications just
// for the root file. Note that since ':' denotes a data stream for a _file_, it cannot
// appear in a directory name. Thus, if we find one, we know it's in the filename.
// See https://learn.microsoft.com/en-us/sysinternals/downloads/streams
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm hoping you can't delete data streams from a file individually. i.e. Do we need to worry about the situation where build:schema and build:types point to the same log file, but only the types stream gets deleted and we miss it's removal since the file is still there?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Based on the -d flag of the CLI tool I'm guessing the answer is yes. Obviously non-blocking, but fun.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can delete individual streams, we get a modification event for the file, which is sufficient for our purposes.

func formatRepoRelativeGlob(input string) string {
unixInput := filepath.ToSlash(input)
if runtime.GOOS == "windows" {
colonIndex := strings.Index(input, ":")
if colonIndex > -1 {
// we found an alternate data stream
unixInput = unixInput[:colonIndex]
}
return unixInput
}
return strings.ReplaceAll(unixInput, ":", "\\:")
}

// GetChangedOutputs implements runcache.OutputWatcher.GetChangedOutputs
func (d *DaemonClient) GetChangedOutputs(ctx context.Context, hash string, repoRelativeOutputGlobs []string) ([]string, int, error) {
// The daemon expects globs to be unix paths
var outputGlobs []string
for _, outputGlob := range repoRelativeOutputGlobs {
outputGlobs = append(outputGlobs, filepath.ToSlash(outputGlob))
outputGlobs = append(outputGlobs, formatRepoRelativeGlob(outputGlob))
}
resp, err := d.client.GetChangedOutputs(ctx, &turbodprotocol.GetChangedOutputsRequest{
Hash: hash,
Expand All @@ -55,10 +77,10 @@ func (d *DaemonClient) NotifyOutputsWritten(ctx context.Context, hash string, re
var inclusions []string
var exclusions []string
for _, inclusion := range repoRelativeOutputGlobs.Inclusions {
inclusions = append(inclusions, filepath.ToSlash(inclusion))
inclusions = append(inclusions, formatRepoRelativeGlob(inclusion))
}
for _, exclusion := range repoRelativeOutputGlobs.Exclusions {
exclusions = append(exclusions, filepath.ToSlash(exclusion))
exclusions = append(exclusions, formatRepoRelativeGlob(exclusion))
}
_, err := d.client.NotifyOutputsWritten(ctx, &turbodprotocol.NotifyOutputsWrittenRequest{
Hash: hash,
Expand Down
23 changes: 23 additions & 0 deletions cli/internal/daemonclient/daemonclient_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package daemonclient

import (
"path/filepath"
"runtime"
"testing"
)

func TestFormatRepoRelativeGlob(t *testing.T) {
rawGlob := filepath.Join("some", ".turbo", "turbo-foo:bar.log")
// Note that we expect unix slashes whether or not we are on Windows
var expected string
if runtime.GOOS == "windows" {
expected = "some/.turbo/turbo-foo"
} else {
expected = "some/.turbo/turbo-foo\\:bar.log"
}

result := formatRepoRelativeGlob(rawGlob)
if result != expected {
t.Errorf("formatRepoRelativeGlob(%v) got %v, want %v", rawGlob, result, expected)
}
}
1 change: 0 additions & 1 deletion cli/internal/runcache/runcache.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,6 @@ func (tc *TaskCache) RestoreOutputs(ctx context.Context, prefixedUI *cli.Prefixe

if err != nil {
progressLogger.Warn(fmt.Sprintf("Failed to check if we can skip restoring outputs for %v: %v. Proceeding to check cache", tc.pt.TaskID, err))
prefixedUI.Warn(ui.Dim(fmt.Sprintf("Failed to check if we can skip restoring outputs for %v: %v. Proceeding to check cache", tc.pt.TaskID, err)))
changedOutputGlobs = tc.repoRelativeGlobs.Inclusions
}

Expand Down
11 changes: 7 additions & 4 deletions crates/turborepo-filewatch/src/globwatcher.rs
Original file line number Diff line number Diff line change
Expand Up @@ -593,7 +593,12 @@ mod test {

let glob_watcher = GlobWatcher::new(&repo_root, cookie_jar, watcher.subscribe());

// On windows, we expect different sanitization before the
// globs are passed in, due to alternative data streams in files.
#[cfg(windows)]
let raw_includes = &["my-pkg/.next/next-file"];
#[cfg(not(windows))]
let raw_includes = &["my-pkg/.next/next-file\\:build"];
let raw_excludes: [&str; 0] = [];
let globs = GlobSet {
include: make_includes(raw_includes),
Expand All @@ -618,10 +623,8 @@ mod test {
assert!(results.is_empty());

// Change the watched file
repo_root
.join_components(&["my-pkg", ".next", "next-file"])
.create_with_contents("hello")
.unwrap();
let watched_file = repo_root.join_components(&["my-pkg", ".next", "next-file:build"]);
watched_file.create_with_contents("hello").unwrap();
let results = glob_watcher
.get_changed_globs(hash.clone(), candidates.clone())
.await
Expand Down
Loading