From 646f4f33b5fd42b85a45b93504bd3e7a936ec1c7 Mon Sep 17 00:00:00 2001 From: Tom Payne Date: Fri, 2 Jul 2021 20:46:27 +0200 Subject: [PATCH 1/2] Add literal_ attribute to make more filenames representable --- docs/REFERENCE.md | 5 + internal/chezmoi/attr.go | 24 +++- internal/chezmoi/attr_test.go | 138 +++++++++++++++++----- internal/chezmoi/chezmoi.go | 9 +- internal/cmd/testdata/scripts/literal.txt | 27 +++++ 5 files changed, 164 insertions(+), 39 deletions(-) create mode 100644 internal/cmd/testdata/scripts/literal.txt diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index 912209ced21..37cca29fb49 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -387,6 +387,7 @@ to as "attributes": | `encrypted_` | Encrypt the file in the source state. | | `exact_` | Remove anything not managed by chezmoi. | | `executable_`| Add executable permissions to the target file. | +| `literal_` | Stop parsing prefix attributes. | | `modify_` | Treat the contents as a script that modifies an existing file. | | `once_` | Run script once. | | `private_` | Remove all group and world permissions from the target file or directory. | @@ -409,6 +410,10 @@ prefixes is important. | Script | File | `run_`, `once_`, `before_` or `after_` | `.tmpl` | | Symbolic link | File | `symlink_`, `dot_`, | `.tmpl` | +The `literal_` prefix can appear anywhere and stop prefix attribute parsing. +This permits filenames that would otherwise conflict with chezmoi's attributes +to be represented. + In addition, if the source file is encrypted, the suffix `.age` (when age encryption is used) or `.asc` (when gpg encryption is used) is stripped. These suffixes can be overridden with the `age.suffix` and `gpg.suffix` configuration diff --git a/internal/chezmoi/attr.go b/internal/chezmoi/attr.go index d378e1bfeae..98b16c046cf 100644 --- a/internal/chezmoi/attr.go +++ b/internal/chezmoi/attr.go @@ -54,8 +54,11 @@ func parseDirAttr(sourceName string) DirAttr { name = mustTrimPrefix(name, privatePrefix) private = true } - if strings.HasPrefix(name, dotPrefix) { + switch { + case strings.HasPrefix(name, dotPrefix): name = "." + mustTrimPrefix(name, dotPrefix) + case strings.HasPrefix(name, literalPrefix): + name = name[len(literalPrefix):] } return DirAttr{ TargetName: name, @@ -73,9 +76,12 @@ func (da DirAttr) SourceName() string { if da.Private { sourceName += privatePrefix } - if strings.HasPrefix(da.TargetName, ".") { + switch { + case strings.HasPrefix(da.TargetName, "."): sourceName += dotPrefix + mustTrimPrefix(da.TargetName, ".") - } else { + case dirPrefixRegexp.MatchString(da.TargetName): + sourceName += literalPrefix + da.TargetName + default: sourceName += da.TargetName } return sourceName @@ -166,8 +172,11 @@ func parseFileAttr(sourceName, encryptedSuffix string) FileAttr { executable = true } } - if strings.HasPrefix(name, dotPrefix) { + switch { + case strings.HasPrefix(name, dotPrefix): name = "." + mustTrimPrefix(name, dotPrefix) + case strings.HasPrefix(name, literalPrefix): + name = name[len(literalPrefix):] } if encrypted { name = strings.TrimSuffix(name, encryptedSuffix) @@ -239,9 +248,12 @@ func (fa FileAttr) SourceName(encryptedSuffix string) string { case SourceFileTypeSymlink: sourceName = symlinkPrefix } - if strings.HasPrefix(fa.TargetName, ".") { + switch { + case strings.HasPrefix(fa.TargetName, "."): sourceName += dotPrefix + mustTrimPrefix(fa.TargetName, ".") - } else { + case filePrefixRegexp.MatchString(fa.TargetName): + sourceName += literalPrefix + fa.TargetName + default: sourceName += fa.TargetName } if fa.Template { diff --git a/internal/chezmoi/attr_test.go b/internal/chezmoi/attr_test.go index a3bf469ecc7..7cb5e2c384a 100644 --- a/internal/chezmoi/attr_test.go +++ b/internal/chezmoi/attr_test.go @@ -18,6 +18,7 @@ func TestDirAttr(t *testing.T) { ".dir", "dir.tmpl", "dir", + "exact_dir", "empty_dir", "encrypted_dir", "executable_dir", @@ -39,8 +40,51 @@ func TestDirAttr(t *testing.T) { } } +func TestDirAttrLiteral(t *testing.T) { + for _, tc := range []struct { + sourceName string + dirAttr DirAttr + }{ + { + sourceName: "exact_dir", + dirAttr: DirAttr{ + TargetName: "dir", + Exact: true, + }, + }, + { + sourceName: "literal_exact_dir", + dirAttr: DirAttr{ + TargetName: "exact_dir", + }, + }, + { + sourceName: "literal_literal_dir", + dirAttr: DirAttr{ + TargetName: "literal_dir", + }, + }, + } { + t.Run(tc.sourceName, func(t *testing.T) { + assert.Equal(t, tc.sourceName, tc.dirAttr.SourceName()) + assert.Equal(t, tc.dirAttr, parseDirAttr(tc.sourceName)) + }) + } +} + func TestFileAttr(t *testing.T) { var fas []FileAttr + targetNames := []string{ + ".name", + "create_name", + "dot_name", + "exact_name", + "literal_name", + "modify_name", + "name", + "run_name", + "symlink_name", + } require.NoError(t, combinator.Generate(&fas, struct { Type SourceFileTargetType TargetName []string @@ -49,12 +93,8 @@ func TestFileAttr(t *testing.T) { Private []bool Template []bool }{ - Type: SourceFileTypeCreate, - TargetName: []string{ - ".name", - "exact_name", - "name", - }, + Type: SourceFileTypeCreate, + TargetName: []string{}, Encrypted: []bool{false, true}, Executable: []bool{false, true}, Private: []bool{false, true}, @@ -69,12 +109,8 @@ func TestFileAttr(t *testing.T) { Private []bool Template []bool }{ - Type: SourceFileTypeFile, - TargetName: []string{ - ".name", - "exact_name", - "name", - }, + Type: SourceFileTypeFile, + TargetName: targetNames, Empty: []bool{false, true}, Encrypted: []bool{false, true}, Executable: []bool{false, true}, @@ -88,12 +124,8 @@ func TestFileAttr(t *testing.T) { Private []bool Template []bool }{ - Type: SourceFileTypeModify, - TargetName: []string{ - ".name", - "exact_name", - "name", - }, + Type: SourceFileTypeModify, + TargetName: targetNames, Executable: []bool{false, true}, Private: []bool{false, true}, Template: []bool{false, true}, @@ -104,25 +136,17 @@ func TestFileAttr(t *testing.T) { Once []bool Order []int }{ - Type: SourceFileTypeScript, - TargetName: []string{ - ".name", - "exact_name", - "name", - }, - Once: []bool{false, true}, - Order: []int{-1, 0, 1}, + Type: SourceFileTypeScript, + TargetName: targetNames, + Once: []bool{false, true}, + Order: []int{-1, 0, 1}, })) require.NoError(t, combinator.Generate(&fas, struct { Type SourceFileTargetType TargetName []string }{ - Type: SourceFileTypeSymlink, - TargetName: []string{ - ".name", - "exact_name", - "name", - }, + Type: SourceFileTypeSymlink, + TargetName: targetNames, })) for _, fa := range fas { actualSourceName := fa.SourceName("") @@ -154,3 +178,53 @@ func TestFileAttrEncryptedSuffix(t *testing.T) { assert.Equal(t, tc.expectedTargetName, fa.TargetName) } } + +func TestFileAttrLiteral(t *testing.T) { + for _, tc := range []struct { + sourceName string + encryptedSuffix string + fileAttr FileAttr + }{ + { + sourceName: "dot_file", + fileAttr: FileAttr{ + TargetName: ".file", + Type: SourceFileTypeFile, + }, + }, + { + sourceName: "literal_dot_file", + fileAttr: FileAttr{ + TargetName: "dot_file", + Type: SourceFileTypeFile, + }, + }, + { + sourceName: "literal_literal_file", + fileAttr: FileAttr{ + TargetName: "literal_file", + Type: SourceFileTypeFile, + }, + }, + { + sourceName: "run_once_script", + fileAttr: FileAttr{ + TargetName: "script", + Type: SourceFileTypeScript, + Once: true, + }, + }, + { + sourceName: "run_literal_once_script", + fileAttr: FileAttr{ + TargetName: "once_script", + Type: SourceFileTypeScript, + }, + }, + } { + t.Run(tc.sourceName, func(t *testing.T) { + assert.Equal(t, tc.fileAttr, parseFileAttr(tc.sourceName, tc.encryptedSuffix)) + assert.Equal(t, tc.sourceName, tc.fileAttr.SourceName(tc.encryptedSuffix)) + }) + } +} diff --git a/internal/chezmoi/chezmoi.go b/internal/chezmoi/chezmoi.go index b158931eebf..970fd6f8a7f 100644 --- a/internal/chezmoi/chezmoi.go +++ b/internal/chezmoi/chezmoi.go @@ -7,6 +7,7 @@ import ( "fmt" "io/fs" "path/filepath" + "regexp" "strings" ) @@ -21,7 +22,7 @@ var ( Umask = fs.FileMode(0) ) -// Suffixes and prefixes. +// Prefixes and suffixes. const ( ignorePrefix = "." afterPrefix = "after_" @@ -32,6 +33,7 @@ const ( encryptedPrefix = "encrypted_" exactPrefix = "exact_" executablePrefix = "executable_" + literalPrefix = "literal_" modifyPrefix = "modify_" oncePrefix = "once_" privatePrefix = "private_" @@ -51,6 +53,11 @@ const ( versionName = Prefix + "version" ) +var ( + dirPrefixRegexp = regexp.MustCompile(`\A(dot|exact|literal|private)_`) + filePrefixRegexp = regexp.MustCompile(`\A(after|before|create|dot|empty|encrypted|executable|literal|modify|once|private|run|symlink)_`) +) + // knownPrefixedFiles is a set of known filenames with the .chezmoi prefix. var knownPrefixedFiles = newStringSet( Prefix+".json"+TemplateSuffix, diff --git a/internal/cmd/testdata/scripts/literal.txt b/internal/cmd/testdata/scripts/literal.txt new file mode 100644 index 00000000000..c414912bd99 --- /dev/null +++ b/internal/cmd/testdata/scripts/literal.txt @@ -0,0 +1,27 @@ +# test that chezmoi add can add files that look like files in the source state +chezmoi add $HOME${/}dot_file $HOME${/}run_script $HOME${/}symlink_symlink +rm $HOME/dot_file $HOME/run_script $HOME/symlink_symlink +chezmoi apply --force +! exists $HOME/.file +! stdout . +! exists $HOME/symlink +cmp $HOME/dot_file golden/dot_file +cmp $HOME/run_script golden/run_script +cmp $HOME/symlink_symlink golden/symlink_symlink + +-- home/user/dot_file -- +# contents of dot_file +-- home/user/run_script -- +#!/bin/sh + +echo contents of run_script +-- home/user/symlink_symlink -- +# contents of symlink_symlink +-- golden/dot_file -- +# contents of dot_file +-- golden/run_script -- +#!/bin/sh + +echo contents of run_script +-- golden/symlink_symlink -- +# contents of symlink_symlink From d5064563691ca21099a9002f3bf0e64e7c6bfa72 Mon Sep 17 00:00:00 2001 From: Tom Payne Date: Fri, 2 Jul 2021 21:42:41 +0200 Subject: [PATCH 2/2] Add .literal suffix to make all filenames representable --- docs/FAQ.md | 15 ++------ docs/REFERENCE.md | 13 ++++--- internal/chezmoi/attr.go | 11 +++++- internal/chezmoi/attr_test.go | 46 ++++++++++++++++++++++- internal/chezmoi/chezmoi.go | 2 + internal/cmd/testdata/scripts/literal.txt | 10 ++++- 6 files changed, 75 insertions(+), 22 deletions(-) diff --git a/docs/FAQ.md b/docs/FAQ.md index c1d796e95ab..3d8c2301c3f 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -197,21 +197,12 @@ better than just having the correct dotfile contents. There are a number of criticisms of how chezmoi's source state is represented on disk: -1. The source file naming system cannot handle all possible filenames. -2. Not all possible file permissions can be represented. -3. The long source file names are weird and verbose. -4. Everything is in a single directory, which can end up containing many entries. +1. Not all possible file permissions can be represented. +2. The long source file names are weird and verbose. +3. Everything is in a single directory, which can end up containing many entries. chezmoi's source state representation is a deliberate, practical compromise. -Certain target filenames, for example `~/dot_example`, are incompatible with -chezmoi's -[attributes](https://github.com/twpayne/chezmoi/blob/master/docs/REFERENCE.md#source-state-attributes) -used in the source state. In practice, dotfile filenames are unlikely to -conflict with chezmoi's attributes. If this does cause a genuine problem for -you, please [open an issue on -GitHub](https://github.com/twpayne/chezmoi/issues/new/choose). - The `dot_` attribute makes it transparent which dotfiles are managed by chezmoi and which files are ignored by chezmoi. chezmoi ignores all files and directories that start with `.` so no special whitelists are needed for version diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index 37cca29fb49..a07cb9f18c7 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -394,9 +394,10 @@ to as "attributes": | `run_` | Treat the contents as a script to run. | | `symlink_` | Create a symlink instead of a regular file. | -| Suffix | Effect | -| ------- | ---------------------------------------------------- | -| `.tmpl` | Treat the contents of the source file as a template. | +| Suffix | Effect | +| ---------- | ---------------------------------------------------- | +| `.literal` | Stop parsing suffix attributes. | +| `.tmpl` | Treat the contents of the source file as a template. | Different target types allow different prefixes and suffixes. The order of prefixes is important. @@ -410,9 +411,9 @@ prefixes is important. | Script | File | `run_`, `once_`, `before_` or `after_` | `.tmpl` | | Symbolic link | File | `symlink_`, `dot_`, | `.tmpl` | -The `literal_` prefix can appear anywhere and stop prefix attribute parsing. -This permits filenames that would otherwise conflict with chezmoi's attributes -to be represented. +The `literal_` prefix and `.literal` suffix can appear anywhere and stop +attribute parsing. This permits filenames that would otherwise conflict with +chezmoi's attributes to be represented. In addition, if the source file is encrypted, the suffix `.age` (when age encryption is used) or `.asc` (when gpg encryption is used) is stripped. These diff --git a/internal/chezmoi/attr.go b/internal/chezmoi/attr.go index 98b16c046cf..4f35a562959 100644 --- a/internal/chezmoi/attr.go +++ b/internal/chezmoi/attr.go @@ -181,9 +181,15 @@ func parseFileAttr(sourceName, encryptedSuffix string) FileAttr { if encrypted { name = strings.TrimSuffix(name, encryptedSuffix) } - if strings.HasSuffix(name, TemplateSuffix) { + switch { + case strings.HasSuffix(name, literalSuffix): + name = mustTrimSuffix(name, literalSuffix) + case strings.HasSuffix(name, TemplateSuffix): name = mustTrimSuffix(name, TemplateSuffix) template = true + if strings.HasSuffix(name, literalSuffix) { + name = mustTrimSuffix(name, literalSuffix) + } } return FileAttr{ TargetName: name, @@ -256,6 +262,9 @@ func (fa FileAttr) SourceName(encryptedSuffix string) string { default: sourceName += fa.TargetName } + if fileSuffixRegexp.MatchString(fa.TargetName) { + sourceName += literalSuffix + } if fa.Template { sourceName += TemplateSuffix } diff --git a/internal/chezmoi/attr_test.go b/internal/chezmoi/attr_test.go index 7cb5e2c384a..c9eba98af1c 100644 --- a/internal/chezmoi/attr_test.go +++ b/internal/chezmoi/attr_test.go @@ -80,10 +80,13 @@ func TestFileAttr(t *testing.T) { "dot_name", "exact_name", "literal_name", + "literal_name", "modify_name", + "name.literal", "name", "run_name", "symlink_name", + "template.tmpl", } require.NoError(t, combinator.Generate(&fas, struct { Type SourceFileTargetType @@ -184,6 +187,7 @@ func TestFileAttrLiteral(t *testing.T) { sourceName string encryptedSuffix string fileAttr FileAttr + nonCanonical bool }{ { sourceName: "dot_file", @@ -221,10 +225,50 @@ func TestFileAttrLiteral(t *testing.T) { Type: SourceFileTypeScript, }, }, + { + sourceName: "file.literal", + fileAttr: FileAttr{ + TargetName: "file", + Type: SourceFileTypeFile, + }, + nonCanonical: true, + }, + { + sourceName: "file.literal.literal", + fileAttr: FileAttr{ + TargetName: "file.literal", + Type: SourceFileTypeFile, + }, + }, + { + sourceName: "file.tmpl", + fileAttr: FileAttr{ + TargetName: "file", + Type: SourceFileTypeFile, + Template: true, + }, + }, + { + sourceName: "file.tmpl.literal", + fileAttr: FileAttr{ + TargetName: "file.tmpl", + Type: SourceFileTypeFile, + }, + }, + { + sourceName: "file.tmpl.literal.tmpl", + fileAttr: FileAttr{ + TargetName: "file.tmpl", + Type: SourceFileTypeFile, + Template: true, + }, + }, } { t.Run(tc.sourceName, func(t *testing.T) { assert.Equal(t, tc.fileAttr, parseFileAttr(tc.sourceName, tc.encryptedSuffix)) - assert.Equal(t, tc.sourceName, tc.fileAttr.SourceName(tc.encryptedSuffix)) + if !tc.nonCanonical { + assert.Equal(t, tc.sourceName, tc.fileAttr.SourceName(tc.encryptedSuffix)) + } }) } } diff --git a/internal/chezmoi/chezmoi.go b/internal/chezmoi/chezmoi.go index 970fd6f8a7f..aaa7ccefa5b 100644 --- a/internal/chezmoi/chezmoi.go +++ b/internal/chezmoi/chezmoi.go @@ -39,6 +39,7 @@ const ( privatePrefix = "private_" runPrefix = "run_" symlinkPrefix = "symlink_" + literalSuffix = ".literal" TemplateSuffix = ".tmpl" ) @@ -56,6 +57,7 @@ const ( var ( dirPrefixRegexp = regexp.MustCompile(`\A(dot|exact|literal|private)_`) filePrefixRegexp = regexp.MustCompile(`\A(after|before|create|dot|empty|encrypted|executable|literal|modify|once|private|run|symlink)_`) + fileSuffixRegexp = regexp.MustCompile(`\.(literal|tmpl)\z`) ) // knownPrefixedFiles is a set of known filenames with the .chezmoi prefix. diff --git a/internal/cmd/testdata/scripts/literal.txt b/internal/cmd/testdata/scripts/literal.txt index c414912bd99..9f3e8e76b24 100644 --- a/internal/cmd/testdata/scripts/literal.txt +++ b/internal/cmd/testdata/scripts/literal.txt @@ -1,13 +1,15 @@ # test that chezmoi add can add files that look like files in the source state -chezmoi add $HOME${/}dot_file $HOME${/}run_script $HOME${/}symlink_symlink -rm $HOME/dot_file $HOME/run_script $HOME/symlink_symlink +chezmoi add $HOME${/}dot_file $HOME${/}run_script $HOME${/}symlink_symlink $HOME${/}template.tmpl +rm $HOME/dot_file $HOME/run_script $HOME/symlink_symlink $HOME/template.tmpl chezmoi apply --force ! exists $HOME/.file ! stdout . ! exists $HOME/symlink +! exists $HOME/template cmp $HOME/dot_file golden/dot_file cmp $HOME/run_script golden/run_script cmp $HOME/symlink_symlink golden/symlink_symlink +cmp $HOME/template.tmpl golden/template.tmpl -- home/user/dot_file -- # contents of dot_file @@ -17,6 +19,8 @@ cmp $HOME/symlink_symlink golden/symlink_symlink echo contents of run_script -- home/user/symlink_symlink -- # contents of symlink_symlink +-- home/user/template.tmpl -- +# contents of template.tmpl -- golden/dot_file -- # contents of dot_file -- golden/run_script -- @@ -25,3 +29,5 @@ echo contents of run_script echo contents of run_script -- golden/symlink_symlink -- # contents of symlink_symlink +-- golden/template.tmpl -- +# contents of template.tmpl