Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added support for artifacts building for bundles #583

Merged
merged 13 commits into from
Jul 25, 2023
167 changes: 167 additions & 0 deletions bundle/artifacts/artifacts.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
package artifacts

import (
"context"
"crypto/sha256"
"encoding/base64"
"errors"
"fmt"
"os"
"path"

"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/artifacts/whl"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/databricks-sdk-go/service/workspace"
)

type mutatorFactory = func(name string) bundle.Mutator

var buildMutators map[config.ArtifactType]mutatorFactory = map[config.ArtifactType]mutatorFactory{
config.ArtifactPythonWheel: whl.Build,
}

var uploadMutators map[config.ArtifactType]mutatorFactory = map[config.ArtifactType]mutatorFactory{}

func getBuildMutator(t config.ArtifactType, name string) bundle.Mutator {
mutatorFactory, ok := buildMutators[t]
if !ok {
mutatorFactory = BasicBuild
}

return mutatorFactory(name)
}

func getUploadMutator(t config.ArtifactType, name string) bundle.Mutator {
mutatorFactory, ok := uploadMutators[t]
if !ok {
mutatorFactory = BasicUpload
}

return mutatorFactory(name)
}

// Basic Build defines a general build mutator which builds artifact based on artifact.BuildCommand
type basicBuild struct {
name string
}

func BasicBuild(name string) bundle.Mutator {
return &basicBuild{name: name}
}

func (m *basicBuild) Name() string {
return fmt.Sprintf("artifacts.Build(%s)", m.name)
}

func (m *basicBuild) Apply(ctx context.Context, b *bundle.Bundle) error {
artifact, ok := b.Config.Artifacts[m.name]
if !ok {
return fmt.Errorf("artifact doesn't exist: %s", m.name)
}

cmdio.LogString(ctx, fmt.Sprintf("artifacts.Build(%s): Building...", m.name))

out, err := artifact.Build(ctx)
if err != nil {
return fmt.Errorf("artifacts.Build(%s): %w, output: %s", m.name, err, out)
}
cmdio.LogString(ctx, fmt.Sprintf("artifacts.Build(%s): Build succeeded", m.name))

return nil
}

// Basic Upload defines a general upload mutator which uploads artifact as a library to workspace
type basicUpload struct {
name string
}

func BasicUpload(name string) bundle.Mutator {
return &basicUpload{name: name}
}

func (m *basicUpload) Name() string {
return fmt.Sprintf("artifacts.Build(%s)", m.name)
}

func (m *basicUpload) Apply(ctx context.Context, b *bundle.Bundle) error {
artifact, ok := b.Config.Artifacts[m.name]
if !ok {
return fmt.Errorf("artifact doesn't exist: %s", m.name)
}

if len(artifact.Files) == 0 {
return fmt.Errorf("artifact source is not configured: %s", m.name)
}

err := uploadArtifact(ctx, artifact, b)
if err != nil {
return fmt.Errorf("artifacts.Upload(%s): %w", m.name, err)
}

return nil
}

func uploadArtifact(ctx context.Context, a *config.Artifact, b *bundle.Bundle) error {
for i := range a.Files {
f := &a.Files[i]
if f.NeedsUpload() {
filename := path.Base(f.Source)
cmdio.LogString(ctx, fmt.Sprintf("artifacts.Upload(%s): Uploading...", filename))
remotePath, err := uploadArtifactFile(ctx, f.Source, b)
if err != nil {
return err
}
cmdio.LogString(ctx, fmt.Sprintf("artifacts.Upload(%s): Upload succeeded", filename))

f.RemotePath = remotePath
}
}

a.NormalisePaths()
return nil
}

// Function to upload artifact file to Workspace
func uploadArtifactFile(ctx context.Context, file string, b *bundle.Bundle) (string, error) {
raw, err := os.ReadFile(file)
if err != nil {
return "", fmt.Errorf("unable to read %s: %w", file, errors.Unwrap(err))
}

uploadPath, err := getUploadBasePath(b)
if err != nil {
return "", err
}

fileHash := sha256.Sum256(raw)
remotePath := path.Join(uploadPath, fmt.Sprintf("%x", fileHash), path.Base(file))
// Make sure target directory exists.
err = b.WorkspaceClient().Workspace.MkdirsByPath(ctx, path.Dir(remotePath))
if err != nil {
return "", fmt.Errorf("unable to create directory for %s: %w", remotePath, err)
}

// Import to workspace.
err = b.WorkspaceClient().Workspace.Import(ctx, workspace.Import{
Path: remotePath,
Overwrite: true,
Format: workspace.ImportFormatAuto,
Content: base64.StdEncoding.EncodeToString(raw),
})
if err != nil {
return "", fmt.Errorf("unable to import %s: %w", remotePath, err)
}

return remotePath, nil
}

func getUploadBasePath(b *bundle.Bundle) (string, error) {
artifactPath := b.Config.Workspace.ArtifactsPath
if artifactPath == "" {
return "", fmt.Errorf("remote artifact path not configured")
}

return path.Join(artifactPath, ".internal"), nil
}
22 changes: 18 additions & 4 deletions bundle/artifacts/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ package artifacts
import (
"context"
"fmt"
"path/filepath"

"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/artifacts/notebook"
)

func BuildAll() bundle.Mutator {
Expand Down Expand Up @@ -33,9 +33,23 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) error {
return fmt.Errorf("artifact doesn't exist: %s", m.name)
}

if artifact.Notebook != nil {
return bundle.Apply(ctx, b, notebook.Build(m.name))
if len(artifact.Files) == 0 && artifact.BuildCommand == "" {
return fmt.Errorf("artifact %s misconfigured: 'files' or 'build' property is required", m.name)
}

return nil
// If artifact file is explicitly defined, skip building the artifact
if len(artifact.Files) != 0 {
return nil
}

// If artifact path is not provided, use bundle root dir
if artifact.Path == "" {
artifact.Path = b.Config.Path
}

if !filepath.IsAbs(artifact.Path) {
artifact.Path = filepath.Join(b.Config.Path, artifact.Path)
}

return bundle.Apply(ctx, b, getBuildMutator(artifact.Type, m.name))
}
81 changes: 0 additions & 81 deletions bundle/artifacts/notebook/build.go

This file was deleted.

29 changes: 0 additions & 29 deletions bundle/artifacts/notebook/marker.go

This file was deleted.

60 changes: 0 additions & 60 deletions bundle/artifacts/notebook/upload.go

This file was deleted.

Loading