Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added support for sync.include and sync.exclude sections #671

Merged
merged 9 commits into from
Aug 18, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions bundle/bundle.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ import (
"github.com/hashicorp/terraform-exec/tfexec"
)

const internalFolder = ".internal"

type Bundle struct {
Config config.Root

Expand Down Expand Up @@ -155,6 +157,38 @@ func (b *Bundle) CacheDir(paths ...string) (string, error) {
return dir, nil
}

// This directory is used to store and automaticaly sync internal bundle files, such as, f.e
// notebook trampoline files for Python wheel and etc.
func (b *Bundle) InternalDir() (string, error) {
cacheDir, err := b.CacheDir()
if err != nil {
return "", err
}

dir := filepath.Join(cacheDir, internalFolder)
err = os.MkdirAll(dir, 0700)
if err != nil {
return dir, err
}

return dir, nil
}

// GetSyncIncludePatterns returns a list of user defined includes
// And also adds InternalDir folder to include list for sync command
// so this folder is always synced
func (b *Bundle) GetSyncIncludePatterns() ([]string, error) {
internalDir, err := b.InternalDir()
if err != nil {
return nil, err
}
internalDirRel, err := filepath.Rel(b.Config.Path, internalDir)
if err != nil {
return nil, err
}
return append(b.Config.Sync.Include, filepath.ToSlash(filepath.Join(internalDirRel, "*.*"))), nil
}

func (b *Bundle) GitRepository() (*git.Repository, error) {
rootPath, err := folders.FindDirWithLeaf(b.Config.Path, ".git")
if err != nil {
Expand Down
3 changes: 3 additions & 0 deletions bundle/config/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,9 @@ type Root struct {

// DEPRECATED. Left for backward compatibility with Targets
Environments map[string]*Target `json:"environments,omitempty"`

// Sync section specifies options for files synchronization
Sync Sync `json:"sync"`
}

func Load(path string) (*Root, error) {
Expand Down
13 changes: 13 additions & 0 deletions bundle/config/sync.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package config

type Sync struct {
// Include contains a list of globs evaluated relative to the bundle root path
// to explicitly include files that were excluded by the user's gitignore.
Include []string `json:"include,omitempty"`

// Exclude contains a list of globs evaluated relative to the bundle root path
// to explicitly exclude files that were included by
// 1) the default that observes the user's gitignore, or
// 2) the `Include` field above.
Exclude []string `json:"exclude,omitempty"`
}
12 changes: 10 additions & 2 deletions bundle/deploy/files/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,17 @@ func getSync(ctx context.Context, b *bundle.Bundle) (*sync.Sync, error) {
return nil, fmt.Errorf("cannot get bundle cache directory: %w", err)
}

includes, err := b.GetSyncIncludePatterns()
if err != nil {
return nil, fmt.Errorf("cannot get list of sync includes: %w", err)
}

opts := sync.SyncOptions{
LocalPath: b.Config.Path,
RemotePath: b.Config.Workspace.FilesPath,
LocalPath: b.Config.Path,
RemotePath: b.Config.Workspace.FilesPath,
Include: includes,
Exclude: b.Config.Sync.Exclude,

Full: false,
CurrentUser: b.Config.Workspace.CurrentUser.User,

Expand Down
7 changes: 7 additions & 0 deletions cmd/bundle/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,16 @@ func (f *syncFlags) syncOptionsFromBundle(cmd *cobra.Command, b *bundle.Bundle)
return nil, fmt.Errorf("cannot get bundle cache directory: %w", err)
}

includes, err := b.GetSyncIncludePatterns()
if err != nil {
return nil, fmt.Errorf("cannot get list of sync includes: %w", err)
}

opts := sync.SyncOptions{
LocalPath: b.Config.Path,
RemotePath: b.Config.Workspace.FilesPath,
Include: includes,
Exclude: b.Config.Sync.Exclude,
Full: f.full,
PollInterval: f.interval,

Expand Down
7 changes: 7 additions & 0 deletions cmd/sync/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,16 @@ func (f *syncFlags) syncOptionsFromBundle(cmd *cobra.Command, args []string, b *
return nil, fmt.Errorf("cannot get bundle cache directory: %w", err)
}

includes, err := b.GetSyncIncludePatterns()
if err != nil {
return nil, fmt.Errorf("cannot get list of sync includes: %w", err)
}

opts := sync.SyncOptions{
LocalPath: b.Config.Path,
RemotePath: b.Config.Workspace.FilesPath,
Include: includes,
Exclude: b.Config.Sync.Exclude,
Full: f.full,
PollInterval: f.interval,

Expand Down
49 changes: 49 additions & 0 deletions libs/fileset/glob.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package fileset

import (
"io/fs"
"os"
"path/filepath"
)

type GlobSet struct {
root string
patterns []string
pietern marked this conversation as resolved.
Show resolved Hide resolved
}

func NewGlobSet(root string, includes []string) (*GlobSet, error) {
absRoot, err := filepath.Abs(root)
if err != nil {
return nil, err
}
for k := range includes {
includes[k] = filepath.Join(absRoot, filepath.FromSlash(includes[k]))
}
return &GlobSet{absRoot, includes}, nil
}

// Return all files which matches defined glob patterns
func (s *GlobSet) All() ([]File, error) {
files := make([]File, 0)
for _, pattern := range s.patterns {
matches, err := filepath.Glob(pattern)
andrewnester marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return files, err
}

for _, match := range matches {
matchRel, err := filepath.Rel(s.root, match)
if err != nil {
return files, err
}

stat, err := os.Stat(match)
if err != nil {
return files, err
}
files = append(files, File{fs.FileInfoToDirEntry(stat), match, matchRel})
}
}

return files, nil
}
65 changes: 65 additions & 0 deletions libs/fileset/glob_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
package fileset

import (
"io/fs"
"os"
"path/filepath"
"slices"
"testing"

"github.com/stretchr/testify/require"
)

func TestGlobFileset(t *testing.T) {
cwd, err := os.Getwd()
require.NoError(t, err)
root := filepath.Join(cwd, "..", "filer")

entries, err := os.ReadDir(root)
require.NoError(t, err)

g, err := NewGlobSet(root, []string{
"./*.go",
})
require.NoError(t, err)

files, err := g.All()
require.NoError(t, err)

require.Equal(t, len(files), len(entries))
for _, f := range files {
exists := slices.ContainsFunc(entries, func(de fs.DirEntry) bool {
return de.Name() == f.Name()
})
require.True(t, exists)
}

g, err = NewGlobSet(root, []string{
"./*.js",
})
require.NoError(t, err)

files, err = g.All()
require.NoError(t, err)
require.Equal(t, len(files), 0)
}

func TestGlobFilesetWithRelativeRoot(t *testing.T) {
root := filepath.Join("..", "filer")

entries, err := os.ReadDir(root)
require.NoError(t, err)

g, err := NewGlobSet(root, []string{
"./*.go",
})
require.NoError(t, err)

files, err := g.All()
require.NoError(t, err)

require.Equal(t, len(files), len(entries))
for _, f := range files {
require.True(t, filepath.IsAbs(f.Absolute))
}
}
75 changes: 75 additions & 0 deletions libs/set/set.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
package set

import (
"fmt"

"golang.org/x/exp/maps"
pietern marked this conversation as resolved.
Show resolved Hide resolved
)

type hashFunc[T any] func(a T) string

// Set struct represents set data structure
type Set[T any] struct {
key hashFunc[T]
data map[string]T
}

// NewSetFromF initialise a new set with initial values and a hash function
// to define uniqueness of value
func NewSetFromF[T any](values []T, f hashFunc[T]) *Set[T] {
s := &Set[T]{
key: f,
data: make(map[string]T),
}

for _, v := range values {
s.Add(v)
}

return s
}

// NewSetF initialise a new empty and a hash function
// to define uniqueness of value
func NewSetF[T any](f hashFunc[T]) *Set[T] {
return NewSetFromF([]T{}, f)
}

// NewSetFrom initialise a new set with initial values which are comparable
func NewSetFrom[T comparable](values []T) *Set[T] {
return NewSetFromF(values, func(item T) string {
return fmt.Sprintf("%v", item)
})
}

// NewSetFrom initialise a new empty set for comparable values
func NewSet[T comparable]() *Set[T] {
return NewSetFrom([]T{})
}

func (s *Set[T]) addOne(item T) {
s.data[s.key(item)] = item
}

// Add one or multiple items to set
func (s *Set[T]) Add(items ...T) {
for _, i := range items {
s.addOne(i)
}
}

// Remove an item from set. No-op if the item does not exist
func (s *Set[T]) Remove(item T) {
delete(s.data, s.key(item))
}

// Indicates if the item exists in the set
func (s *Set[T]) Has(item T) bool {
_, ok := s.data[s.key(item)]
return ok
}

// Returns an iterable slice of values from set
func (s *Set[T]) Iter() []T {
return maps.Values(s.data)
}
Loading