Fix fsutil.Shred and other cleanup (#2063)

RELEASE_NOTES=n/a

Signed-off-by: Dominik Schulz <dominik.schulz@gauner.org>

internal/backend/crypto.go

@@ -76,7 +76,7 @@ func DetectCrypto(ctx context.Context, storage Storage) (Crypto, error) {
 
 	for _, be := range CryptoRegistry.Prioritized() {
 		debug.Log("Trying %s for %s", be, storage)
-		if err := be.Handles(storage); err != nil {
+		if err := be.Handles(ctx, storage); err != nil {
 			debug.Log("failed to use crypto %s for %s", be, storage)
 			continue
 		}

internal/backend/crypto/age/loader.go

@@ -24,8 +24,7 @@ func (l loader) New(ctx context.Context) (backend.Crypto, error) {
 	return New()
 }
 
-func (l loader) Handles(s backend.Storage) error {
-	ctx := context.TODO()
+func (l loader) Handles(ctx context.Context, s backend.Storage) error {
 	if s.Exists(ctx, OldIDFile) || s.Exists(ctx, OldKeyring) {
 		if err := migrate(ctx, s); err != nil {
 			out.Errorf(ctx, "Failed to migrate age backend: %s", err)

internal/backend/crypto/gpg/cli/loader.go

@@ -30,8 +30,8 @@ func (l loader) New(ctx context.Context) (backend.Crypto, error) {
 	})
 }
 
-func (l loader) Handles(s backend.Storage) error {
-	if s.Exists(context.TODO(), IDFile) {
+func (l loader) Handles(ctx context.Context, s backend.Storage) error {
+	if s.Exists(ctx, IDFile) {
 		return nil
 	}
 	return fmt.Errorf("not supported")

internal/backend/crypto/plain/backend.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/gopasspw/gopass/internal/backend/crypto/gpg"
+	"github.com/gopasspw/gopass/pkg/debug"
 
 	"github.com/blang/semver/v4"
 )
@@ -108,7 +109,7 @@ func (m *Mocker) ImportPublicKey(context.Context, []byte) error {
 
 // Version returns dummy version info
 func (m *Mocker) Version(context.Context) semver.Version {
-	return semver.Version{}
+	return debug.ModuleVersion("github.com/gopasspw/gopass/internal/backend/crypto/plain")
 }
 
 // Binary always returns 'gpg'

internal/backend/crypto/plain/loader.go

@@ -24,8 +24,8 @@ func (l loader) New(ctx context.Context) (backend.Crypto, error) {
 	return New(), nil
 }
 
-func (l loader) Handles(s backend.Storage) error {
-	if s.Exists(context.TODO(), IDFile) {
+func (l loader) Handles(ctx context.Context, s backend.Storage) error {
+	if s.Exists(ctx, IDFile) {
 		return nil
 	}
 	return fmt.Errorf("not supported")

internal/backend/registry.go

@@ -27,7 +27,7 @@ type CryptoLoader interface {
 	fmt.Stringer
 	Prioritized
 	New(context.Context) (Crypto, error)
-	Handles(Storage) error
+	Handles(context.Context, Storage) error
 }
 
 // StorageLoader is the interface for creating a new storage backend.
@@ -37,7 +37,7 @@ type StorageLoader interface {
 	New(context.Context, string) (Storage, error)
 	Init(context.Context, string) (Storage, error)
 	Clone(context.Context, string, string) (Storage, error)
-	Handles(string) error
+	Handles(context.Context, string) error
 }
 
 // NewRegistry returns a new registry

internal/backend/registry_test.go

@@ -22,7 +22,7 @@ func (l fakeCryptoLoader) String() string {
 	return "fakecryptoloader"
 }
 
-func (l fakeCryptoLoader) Handles(_ backend.Storage) error {
+func (l fakeCryptoLoader) Handles(_ context.Context, _ backend.Storage) error {
 	return nil
 }
 

internal/backend/storage.go

@@ -67,7 +67,7 @@ func DetectStorage(ctx context.Context, path string) (Storage, error) {
 
 	for _, be := range StorageRegistry.Prioritized() {
 		debug.Log("Trying %s for %s", be, path)
-		if err := be.Handles(path); err != nil {
+		if err := be.Handles(ctx, path); err != nil {
 			debug.Log("failed to use %s for %s: %s", be, path, err)
 			continue
 		}

internal/backend/storage/fs/fsck_test.go

@@ -20,7 +20,7 @@ func TestFsck(t *testing.T) {
 	l := &loader{}
 	s, err := l.Init(ctx, path)
 	assert.NoError(t, err)
-	assert.NoError(t, l.Handles(path))
+	assert.NoError(t, l.Handles(ctx, path))
 
 	for _, fn := range []string{
 		filepath.Join(path, ".plain-ids"),

internal/backend/storage/fs/loader.go

@@ -41,7 +41,7 @@ func (l loader) Clone(ctx context.Context, repo, path string) (backend.Storage,
 	return l.New(ctx, path)
 }
 
-func (l loader) Handles(path string) error {
+func (l loader) Handles(ctx context.Context, path string) error {
 	if fsutil.IsDir(path) {
 		return nil
 	}

internal/backend/storage/fs/rcs_test.go

@@ -23,7 +23,7 @@ func TestRCS(t *testing.T) {
 	assert.NoError(t, g.Cmd(ctx, "foo", "bar"))
 	assert.Error(t, g.Init(ctx, "foo", "bar"))
 	assert.NoError(t, g.InitConfig(ctx, "foo", "bar"))
-	assert.Equal(t, g.Version(ctx), semver.Version{Minor: 1})
+	assert.Equal(t, true, g.Version(ctx).EQ(semver.Version{}), "Version eq 0.0.0")
 	assert.Equal(t, "fs", g.Name())
 	assert.NoError(t, g.AddRemote(ctx, "foo", "bar"))
 	revs, err := g.Revisions(ctx, "foo")

internal/backend/storage/fs/store.go

@@ -176,12 +176,12 @@ func (s *Store) Name() string {
 
 // Version returns the version of this backend
 func (s *Store) Version(context.Context) semver.Version {
-	return semver.Version{Minor: 1}
+	return debug.ModuleVersion("github.com/gopasspw/gopass/internal/backend/fs")
 }
 
 // String implements fmt.Stringer
 func (s *Store) String() string {
-	return fmt.Sprintf("fs(v0.1.0,path:%s)", s.path)
+	return fmt.Sprintf("fs(%s,path:%s)", s.Version(context.TODO()).String(), s.path)
 }
 
 // Path returns the path to this storage

internal/backend/storage/gitfs/git.go

@@ -58,7 +58,7 @@ func New(path string) (*Git, error) {
 
 // Clone clones an existing git repo and returns a new cli based git backend
 // configured for this clone repo
-func Clone(ctx context.Context, repo, path string) (*Git, error) {
+func Clone(ctx context.Context, repo, path, userName, userEmail string) (*Git, error) {
 	g := &Git{
 		fs: fs.New(path),
 	}
@@ -67,7 +67,7 @@ func Clone(ctx context.Context, repo, path string) (*Git, error) {
 	}
 
 	// initialize the local git config
-	if err := g.InitConfig(ctx, "", ""); err != nil {
+	if err := g.InitConfig(ctx, userName, userEmail); err != nil {
 		return g, fmt.Errorf("failed to configure git: %s", err)
 	}
 	out.Printf(ctx, "git configured at %s", g.fs.Path())

internal/backend/storage/gitfs/git_test.go

@@ -66,7 +66,7 @@ func TestGit(t *testing.T) {
 	})
 
 	t.Run("clone existing repo", func(t *testing.T) {
-		git, err := Clone(ctx, gitdir, gitdir2)
+		git, err := Clone(ctx, gitdir, gitdir2, "", "")
 		require.NoError(t, err)
 		require.NotNil(t, git)
 		assert.Equal(t, "git", git.Name())

internal/backend/storage/gitfs/loader.go

@@ -31,15 +31,15 @@ func (l loader) Open(ctx context.Context, path string) (backend.Storage, error)
 
 // Clone implements backend.RCSLoader
 func (l loader) Clone(ctx context.Context, repo, path string) (backend.Storage, error) {
-	return Clone(ctx, repo, path)
+	return Clone(ctx, repo, path, termio.DetectName(ctx, nil), termio.DetectEmail(ctx, nil))
 }
 
 // Init implements backend.RCSLoader
 func (l loader) Init(ctx context.Context, path string) (backend.Storage, error) {
 	return Init(ctx, path, termio.DetectName(ctx, nil), termio.DetectEmail(ctx, nil))
 }
 
-func (l loader) Handles(path string) error {
+func (l loader) Handles(ctx context.Context, path string) error {
 	if !fsutil.IsDir(filepath.Join(path, ".git")) {
 		return fmt.Errorf("no .git")
 	}

internal/out/print.go

@@ -42,7 +42,11 @@ func newline(ctx context.Context) string {
 
 // Print prints the given string
 func Print(ctx context.Context, arg any) {
-	Printf(ctx, "%s", arg)
+	if ctxutil.IsHidden(ctx) {
+		return
+	}
+	debug.LogN(1, "%s", arg)
+	fmt.Fprintf(Stdout, Prefix(ctx)+"%s"+newline(ctx), arg)
 }
 
 // Printf formats and prints the given string
@@ -56,7 +60,11 @@ func Printf(ctx context.Context, format string, args ...any) {
 
 // Notice prints the string with an exclamation mark
 func Notice(ctx context.Context, arg any) {
-	Noticef(ctx, "%s", arg)
+	if ctxutil.IsHidden(ctx) {
+		return
+	}
+	debug.LogN(1, "NOTICE: %s", arg)
+	fmt.Fprintf(Stdout, Prefix(ctx)+"⚠ %s"+newline(ctx), arg)
 }
 
 // Noticef prints the string with an exclamation mark in front
@@ -70,7 +78,11 @@ func Noticef(ctx context.Context, format string, args ...any) {
 
 // Error prints the string with a red cross in front
 func Error(ctx context.Context, arg any) {
-	Errorf(ctx, "%s", arg)
+	if ctxutil.IsHidden(ctx) {
+		return
+	}
+	debug.LogN(1, "ERROR: %s", arg)
+	fmt.Fprint(Stderr, color.RedString(Prefix(ctx)+"❌ %s"+newline(ctx), arg))
 }
 
 // Errorf prints the string in red to stderr
@@ -84,7 +96,11 @@ func Errorf(ctx context.Context, format string, args ...any) {
 
 // OK prints the string with a green checkmark in front
 func OK(ctx context.Context, arg any) {
-	OKf(ctx, "%s", arg)
+	if ctxutil.IsHidden(ctx) {
+		return
+	}
+	debug.LogN(1, "OK: %s", arg)
+	fmt.Fprintf(Stdout, Prefix(ctx)+"✅ %s"+newline(ctx), arg)
 }
 
 // OKf prints the string in with an OK checkmark in front
@@ -98,7 +114,11 @@ func OKf(ctx context.Context, format string, args ...any) {
 
 // Warning prints the string with a warning sign in front
 func Warning(ctx context.Context, arg any) {
-	Warningf(ctx, "%s", arg)
+	if ctxutil.IsHidden(ctx) {
+		return
+	}
+	debug.LogN(1, "WARNING: %s", arg)
+	fmt.Fprint(Stderr, color.YellowString(Prefix(ctx)+"⚠ %s"+newline(ctx), arg))
 }
 
 // Warningf prints the string in yellow to stderr and prepends a warning sign

internal/store/leaf/rcs_test.go

@@ -28,7 +28,7 @@ func TestGit(t *testing.T) {
 	require.NotNil(t, s.Storage())
 	require.Equal(t, "fs", s.Storage().Name())
 	assert.NoError(t, s.Storage().InitConfig(ctx, "foo", "bar@baz.com"))
-	assert.Equal(t, semver.Version{Minor: 1}, s.Storage().Version(ctx))
+	assert.Equal(t, semver.Version{}, s.Storage().Version(ctx))
 	assert.NoError(t, s.Storage().AddRemote(ctx, "foo", "bar"))
 	// RCS ops not supported by the fs backend
 	assert.Error(t, s.Storage().Pull(ctx, "origin", "master"))

internal/store/leaf/store.go

@@ -29,17 +29,17 @@ func Init(ctx context.Context, alias, path string) (*Store, error) {
 
 	st, err := backend.InitStorage(ctx, backend.GetStorageBackend(ctx), path)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to initialize storage for %s at %s: %w", alias, path, err)
 	}
 	s.storage = st
-	debug.Log("Storage initialized")
+	debug.Log("Storage for %s => %s initialized as %s", alias, path, st.Name())
 
 	crypto, err := backend.NewCrypto(ctx, backend.GetCryptoBackend(ctx))
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to initialize crypto for %s at %s: %w", alias, path, err)
 	}
 	s.crypto = crypto
-	debug.Log("Crypto initialized")
+	debug.Log("Crypto for %s => %s initialized as %s", alias, path, crypto.Name())
 
 	return s, nil
 }
@@ -57,15 +57,14 @@ func New(ctx context.Context, alias, path string) (*Store, error) {
 	if err := s.initStorageBackend(ctx); err != nil {
 		return nil, fmt.Errorf("failed to init storage backend: %w", err)
 	}
-	debug.Log("Storage initialized")
+	debug.Log("Storage for %s => %s initialized as %s", alias, path, s.storage)
 
 	// init crypto backend
 	if err := s.initCryptoBackend(ctx); err != nil {
 		return nil, fmt.Errorf("failed to init crypto backend: %w", err)
 	}
-	debug.Log("Crypto initialized")
+	debug.Log("Crypto for %s => %s initialized as %s", alias, path, s.crypto)
 
-	debug.Log("Instantiated %s at %s - storage: %+#v - crypto: %+#v", alias, path, s.storage, s.crypto)
 	return s, nil
 }
 
@@ -106,6 +105,8 @@ func (s *Store) idFiles(ctx context.Context) []string {
 		return nil
 	}
 
+	// we need to transform the list of files into a list of id files so we can't use
+	// set.SortedFiltered as it doesn't support transformations
 	idfs := make([]string, 0, len(files))
 	for _, f := range files {
 		if strings.HasPrefix(filepath.Base(f), ".") {
@@ -121,7 +122,7 @@ func (s *Store) idFiles(ctx context.Context) []string {
 	return set.Sorted(idfs)
 }
 
-// Equals returns true if this.storage has the same on-disk path as the other
+// Equals returns true if this storage has the same on-disk path as the other
 func (s *Store) Equals(other *Store) bool {
 	if other == nil {
 		return false

internal/store/mockstore/inmem/store.go

@@ -14,7 +14,7 @@ import (
 	"golang.org/x/exp/maps"
 )
 
-// InMem is a in-memory store
+// InMem is a thread-safe in-memory store
 type InMem struct {
 	sync.Mutex
 	data map[string][]byte
@@ -23,7 +23,7 @@ type InMem struct {
 // New creates a new mock
 func New() *InMem {
 	return &InMem{
-		data: make(map[string][]byte, 10),
+		data: make(map[string][]byte, 128),
 	}
 }
 
@@ -32,10 +32,17 @@ func (m *InMem) Get(ctx context.Context, name string) ([]byte, error) {
 	m.Lock()
 	defer m.Unlock()
 
+	if m.data == nil {
+		return nil, fmt.Errorf("entry not found")
+	}
+
 	sec, found := m.data[name]
 	if !found {
+		// not found
 		return nil, fmt.Errorf("entry not found")
 	}
+
+	// found
 	return sec, nil
 }
 

internal/store/root/convert.go

@@ -14,6 +14,7 @@ func (r *Store) Convert(ctx context.Context, name string, cryptoBe backend.Crypt
 	if err != nil {
 		return err
 	}
+
 	debug.Log("converting %s to crypto: %s, rcs: %s, storage: %s", name, cryptoBe, storageBe)
 	if err := sub.Convert(ctx, cryptoBe, storageBe, move); err != nil {
 		return err

internal/store/root/fsck.go

@@ -21,12 +21,16 @@ func (s *Store) Fsck(ctx context.Context, path string) error {
 			continue
 		}
 		path = strings.TrimPrefix(path, alias+"/")
+
+		// check sub store
 		debug.Log("Checking %s", alias)
 		if err := sub.Fsck(ctx, path); err != nil {
 			out.Errorf(ctx, "fsck failed on sub store %s: %s", alias, err)
 			result = multierror.Append(result, err)
 		}
 	}
+
+	// check root store
 	if err := s.store.Fsck(ctx, path); err != nil {
 		out.Errorf(ctx, "fsck failed on root store: %s", err)
 		result = multierror.Append(result, err)

internal/store/root/init.go

@@ -19,6 +19,7 @@ func (r *Store) IsInitialized(ctx context.Context) (bool, error) {
 			return false, fmt.Errorf("failed to initialize stores: %w", err)
 		}
 	}
+
 	debug.Log("root store is initialized")
 	return r.store.IsInitialized(ctx), nil
 }
@@ -32,6 +33,7 @@ func (r *Store) Init(ctx context.Context, alias, path string, ids ...string) err
 	if !backend.HasStorageBackend(ctx) {
 		ctx = backend.WithStorageBackend(ctx, backend.GitFS)
 	}
+
 	sub, err := leaf.New(ctx, alias, path)
 	if err != nil {
 		return fmt.Errorf("failed to instantiate new sub store: %w", err)