From 105a3dfc42a52f267c1c4dac493c18240fb8e78b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 12 Aug 2024 16:00:20 +0200 Subject: [PATCH 1/7] Add prompt when a pipeline recreation happens --- bundle/phases/deploy.go | 85 +++++++++++++------ bundle/phases/deploy_test.go | 67 +++++++++++++++ cmd/root/progress_logger.go | 6 ++ .../databricks_template_schema.json | 8 ++ .../template/databricks.yml.tmpl | 25 ++++++ .../bundles/recreate_pipeline/template/nb.sql | 2 + internal/bundle/deploy_test.go | 38 ++++++++- 7 files changed, 205 insertions(+), 26 deletions(-) create mode 100644 bundle/phases/deploy_test.go create mode 100644 internal/bundle/bundles/recreate_pipeline/databricks_template_schema.json create mode 100644 internal/bundle/bundles/recreate_pipeline/template/databricks.yml.tmpl create mode 100644 internal/bundle/bundles/recreate_pipeline/template/nb.sql diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 6929f74baf..91e3b98788 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -19,55 +19,90 @@ import ( "github.com/databricks/cli/bundle/scripts" "github.com/databricks/cli/libs/cmdio" terraformlib "github.com/databricks/cli/libs/terraform" + tfjson "github.com/hashicorp/terraform-json" ) -func approvalForUcSchemaDelete(ctx context.Context, b *bundle.Bundle) (bool, error) { - tf := b.Terraform - if tf == nil { - return false, fmt.Errorf("terraform not initialized") - } - - // read plan file - plan, err := tf.ShowPlanFile(ctx, b.Plan.Path) - if err != nil { - return false, err - } - - actions := make([]terraformlib.Action, 0) - for _, rc := range plan.ResourceChanges { - // We only care about destructive actions on UC schema resources. - if rc.Type != "databricks_schema" { +func parseTerraformActions(changes []*tfjson.ResourceChange, toInclude func(typ string, actions tfjson.Actions) bool) []terraformlib.Action { + res := make([]terraformlib.Action, 0) + for _, rc := range changes { + if !toInclude(rc.Type, rc.Change.Actions) { continue } var actionType terraformlib.ActionType - switch { case rc.Change.Actions.Delete(): actionType = terraformlib.ActionTypeDelete case rc.Change.Actions.Replace(): actionType = terraformlib.ActionTypeRecreate default: - // We don't need a prompt for non-destructive actions like creating - // or updating a schema. + // No use case for other action types yet. continue } - actions = append(actions, terraformlib.Action{ + res = append(res, terraformlib.Action{ Action: actionType, ResourceType: rc.Type, ResourceName: rc.Name, }) } - // No restricted actions planned. No need for approval. - if len(actions) == 0 { + return res +} + +func approvalForUcSchemaDelete(ctx context.Context, b *bundle.Bundle) (bool, error) { + tf := b.Terraform + if tf == nil { + return false, fmt.Errorf("terraform not initialized") + } + + // read plan file + plan, err := tf.ShowPlanFile(ctx, b.Plan.Path) + if err != nil { + return false, err + } + + schemaActions := parseTerraformActions(plan.ResourceChanges, func(typ string, actions tfjson.Actions) bool { + // Filter in only UC schema resources. + if typ != "databricks_schema" { + return false + } + + // We only display prompts for destructive actions like deleting or + // recreating a schema. + return actions.Delete() || actions.Replace() + }) + + dltActions := parseTerraformActions(plan.ResourceChanges, func(typ string, actions tfjson.Actions) bool { + // Filter in only DLT pipeline resources. + if typ != "databricks_pipeline" { + return false + } + + // Recreating DLT pipeline leads to metadata loss and for a transient period + // the underling tables will be unavailable. + return actions.Replace() + }) + + // We don't need to display any prompts in this case. + if len(dltActions) == 0 && len(schemaActions) == 0 { return true, nil } - cmdio.LogString(ctx, "The following UC schemas will be deleted or recreated. Any underlying data may be lost:") - for _, action := range actions { - cmdio.Log(ctx, action) + // One or more UC schema resources will be deleted or recreated. + if len(schemaActions) != 0 { + cmdio.LogString(ctx, "The following UC schemas will be deleted or recreated. Any underlying data may be lost:") + for _, action := range schemaActions { + cmdio.Log(ctx, action) + } + } + + // One or more DLT pipelines is being recreated. + if len(dltActions) != 0 { + cmdio.LogString(ctx, "The following DLT pipelines will be recreated. Underlying tables will be unavailable for a transient period, until the newly recreated pipeline is run once successfully. History of previous pipeline update runs will be lost as a result of the recreation:") + for _, action := range dltActions { + cmdio.Log(ctx, action) + } } if b.AutoApprove { diff --git a/bundle/phases/deploy_test.go b/bundle/phases/deploy_test.go new file mode 100644 index 0000000000..e00370b380 --- /dev/null +++ b/bundle/phases/deploy_test.go @@ -0,0 +1,67 @@ +package phases + +import ( + "testing" + + terraformlib "github.com/databricks/cli/libs/terraform" + tfjson "github.com/hashicorp/terraform-json" + "github.com/stretchr/testify/assert" +) + +func TestParseTerraformActions(t *testing.T) { + changes := []*tfjson.ResourceChange{ + { + Type: "databricks_pipeline", + Change: &tfjson.Change{ + Actions: tfjson.Actions{tfjson.ActionCreate}, + }, + Name: "create pipeline", + }, + { + Type: "databricks_pipeline", + Change: &tfjson.Change{ + Actions: tfjson.Actions{tfjson.ActionDelete}, + }, + Name: "delete pipeline", + }, + { + Type: "databricks_pipeline", + Change: &tfjson.Change{ + Actions: tfjson.Actions{tfjson.ActionDelete, tfjson.ActionCreate}, + }, + Name: "recreate pipeline", + }, + { + Type: "databricks_whatever", + Change: &tfjson.Change{ + Actions: tfjson.Actions{tfjson.ActionDelete, tfjson.ActionCreate}, + }, + Name: "recreate whatever", + }, + } + + res := parseTerraformActions(changes, func(typ string, actions tfjson.Actions) bool { + if typ != "databricks_pipeline" { + return false + } + + if actions.Delete() || actions.Replace() { + return true + } + + return false + }) + + assert.Equal(t, []terraformlib.Action{ + { + Action: terraformlib.ActionTypeDelete, + ResourceType: "databricks_pipeline", + ResourceName: "delete pipeline", + }, + { + Action: terraformlib.ActionTypeRecreate, + ResourceType: "databricks_pipeline", + ResourceName: "recreate pipeline", + }, + }, res) +} diff --git a/cmd/root/progress_logger.go b/cmd/root/progress_logger.go index c05ecb0437..7d6a1fa46d 100644 --- a/cmd/root/progress_logger.go +++ b/cmd/root/progress_logger.go @@ -29,6 +29,12 @@ func (f *progressLoggerFlag) resolveModeDefault(format flags.ProgressLogFormat) } func (f *progressLoggerFlag) initializeContext(ctx context.Context) (context.Context, error) { + // No need to initialize the logger if it's already set in the context. This + // happens in unit tests where the logger is setup as a fixture. + if _, ok := cmdio.FromContext(ctx); ok { + return ctx, nil + } + if f.log.level.String() != "disabled" && f.log.file.String() == "stderr" && f.ProgressLogFormat == flags.ModeInplace { return nil, fmt.Errorf("inplace progress logging cannot be used when log-file is stderr") diff --git a/internal/bundle/bundles/recreate_pipeline/databricks_template_schema.json b/internal/bundle/bundles/recreate_pipeline/databricks_template_schema.json new file mode 100644 index 0000000000..762f4470c2 --- /dev/null +++ b/internal/bundle/bundles/recreate_pipeline/databricks_template_schema.json @@ -0,0 +1,8 @@ +{ + "properties": { + "unique_id": { + "type": "string", + "description": "Unique ID for the schema and pipeline names" + } + } +} diff --git a/internal/bundle/bundles/recreate_pipeline/template/databricks.yml.tmpl b/internal/bundle/bundles/recreate_pipeline/template/databricks.yml.tmpl new file mode 100644 index 0000000000..10350f13e9 --- /dev/null +++ b/internal/bundle/bundles/recreate_pipeline/template/databricks.yml.tmpl @@ -0,0 +1,25 @@ +bundle: + name: "bundle-playground" + +variables: + catalog: + description: The catalog the DLT pipeline should use. + default: main + + +resources: + pipelines: + foo: + name: test-pipeline-{{.unique_id}} + libraries: + - notebook: + path: ./nb.sql + development: true + catalog: ${var.catalog} + +include: + - "*.yml" + +targets: + development: + default: true diff --git a/internal/bundle/bundles/recreate_pipeline/template/nb.sql b/internal/bundle/bundles/recreate_pipeline/template/nb.sql new file mode 100644 index 0000000000..199ff50788 --- /dev/null +++ b/internal/bundle/bundles/recreate_pipeline/template/nb.sql @@ -0,0 +1,2 @@ +-- Databricks notebook source +select 1 diff --git a/internal/bundle/deploy_test.go b/internal/bundle/deploy_test.go index 3da885705d..6e4896939e 100644 --- a/internal/bundle/deploy_test.go +++ b/internal/bundle/deploy_test.go @@ -119,7 +119,43 @@ func TestAccBundleDeployUcSchemaFailsWithoutAutoApprove(t *testing.T) { t.Setenv("BUNDLE_ROOT", bundleRoot) t.Setenv("TERM", "dumb") c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "deploy", "--force-lock") - stdout, _, err := c.Run() + stdout, stderr, err := c.Run() + + assert.EqualError(t, err, root.ErrAlreadyPrinted.Error()) + assert.Contains(t, stderr.String(), "The following UC schemas will be deleted or recreated. Any underlying data may be lost:\n delete schema bar") + assert.Contains(t, stdout.String(), "the deployment requires destructive actions, but current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed") +} + +func TestAccBundlePipelineRecreateWithoutAutoApprove(t *testing.T) { + ctx, wt := acc.UcWorkspaceTest(t) + w := wt.W + uniqueId := uuid.New().String() + + bundleRoot, err := initTestTemplate(t, ctx, "recreate_pipeline", map[string]any{ + "unique_id": uniqueId, + }) + require.NoError(t, err) + + err = deployBundle(t, ctx, bundleRoot) + require.NoError(t, err) + + t.Cleanup(func() { + destroyBundle(t, ctx, bundleRoot) + }) + + // Assert the pipeline is created + pipelineName := "test-pipeline-" + uniqueId + pipeline, err := w.Pipelines.GetByName(ctx, pipelineName) + require.NoError(t, err) + require.Equal(t, pipelineName, pipeline.Name) + + // Redeploy the bundle, pointing the DLT pipeline to a different UC catalog. + t.Setenv("BUNDLE_ROOT", bundleRoot) + t.Setenv("TERM", "dumb") + c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "deploy", "--force-lock", "--var=\"catalog=whatever\"") + stdout, stderr, err := c.Run() + assert.EqualError(t, err, root.ErrAlreadyPrinted.Error()) + assert.Contains(t, stderr.String(), "The following DLT pipelines will be recreated. Underlying tables will be unavailable for a transient period, until the newly recreated pipeline is run once successfully. History of previous pipeline update runs will be lost as a result of the recreation:\n recreate pipeline foo") assert.Contains(t, stdout.String(), "the deployment requires destructive actions, but current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed") } From d2707273720ec22f5880f25e1d80751569a07617 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 12 Aug 2024 16:13:18 +0200 Subject: [PATCH 2/7] grammer nit --- bundle/phases/deploy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 91e3b98788..23d14f1674 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -99,7 +99,7 @@ func approvalForUcSchemaDelete(ctx context.Context, b *bundle.Bundle) (bool, err // One or more DLT pipelines is being recreated. if len(dltActions) != 0 { - cmdio.LogString(ctx, "The following DLT pipelines will be recreated. Underlying tables will be unavailable for a transient period, until the newly recreated pipeline is run once successfully. History of previous pipeline update runs will be lost as a result of the recreation:") + cmdio.LogString(ctx, "The following DLT pipelines will be recreated. Underlying tables will be unavailable for a transient period until the newly recreated pipelines are run once successfully. History of previous pipeline update runs will be lost because of recreation:") for _, action := range dltActions { cmdio.Log(ctx, action) } From 27af70eb81d7bd4df1d894f788d78268a105bfb7 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 12 Aug 2024 16:16:25 +0200 Subject: [PATCH 3/7] rename --- bundle/phases/deploy.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 23d14f1674..855040a5f9 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -50,7 +50,7 @@ func parseTerraformActions(changes []*tfjson.ResourceChange, toInclude func(typ return res } -func approvalForUcSchemaDelete(ctx context.Context, b *bundle.Bundle) (bool, error) { +func approvalForDeploy(ctx context.Context, b *bundle.Bundle) (bool, error) { tf := b.Terraform if tf == nil { return false, fmt.Errorf("terraform not initialized") @@ -161,7 +161,7 @@ func Deploy() bundle.Mutator { terraform.CheckRunningResource(), terraform.Plan(terraform.PlanGoal("deploy")), bundle.If( - approvalForUcSchemaDelete, + approvalForDeploy, deployCore, bundle.LogString("Deployment cancelled!"), ), From d6c564d531923fb9a13adbdc0187d57d74efdd7e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 27 Aug 2024 20:20:36 +0200 Subject: [PATCH 4/7] add the why --- bundle/phases/deploy.go | 2 +- internal/bundle/deploy_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 6d32f9ca95..3dc477ca2b 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -99,7 +99,7 @@ func approvalForDeploy(ctx context.Context, b *bundle.Bundle) (bool, error) { // One or more DLT pipelines is being recreated. if len(dltActions) != 0 { - cmdio.LogString(ctx, "The following DLT pipelines will be recreated. Underlying tables will be unavailable for a transient period until the newly recreated pipelines are run once successfully. History of previous pipeline update runs will be lost because of recreation:") + cmdio.LogString(ctx, "The following DLT pipelines will be recreated. This commonly happens when the `catalog` or the `storage` specified for aq DLT pipeline is changed. Underlying tables will be unavailable for a transient period until the newly recreated pipelines are run once successfully. History of previous pipeline update runs will be lost because of recreation:") for _, action := range dltActions { cmdio.Log(ctx, action) } diff --git a/internal/bundle/deploy_test.go b/internal/bundle/deploy_test.go index 34a937bfc3..332962eafe 100644 --- a/internal/bundle/deploy_test.go +++ b/internal/bundle/deploy_test.go @@ -157,7 +157,7 @@ func TestAccBundlePipelineRecreateWithoutAutoApprove(t *testing.T) { stdout, stderr, err := c.Run() assert.EqualError(t, err, root.ErrAlreadyPrinted.Error()) - assert.Contains(t, stderr.String(), "The following DLT pipelines will be recreated. Underlying tables will be unavailable for a transient period, until the newly recreated pipeline is run once successfully. History of previous pipeline update runs will be lost as a result of the recreation:\n recreate pipeline foo") + assert.Contains(t, stderr.String(), "The following DLT pipelines will be recreated. This commonly happens when the `catalog` or the `storage` specified for a DLT pipeline is changed. Underlying tables will be unavailable for a transient period until the newly recreated pipelines are run once successfully. History of previous pipeline update runs will be lost because of recreation:\n recreate pipeline foo") assert.Contains(t, stdout.String(), "the deployment requires destructive actions, but current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed") } From 7e3a453baf7a0c185f1ae9054584dedf7d6bb249 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 30 Aug 2024 11:40:04 +0200 Subject: [PATCH 5/7] fix test --- bundle/phases/deploy.go | 4 ++-- internal/bundle/deploy_test.go | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 3dc477ca2b..1ce799832c 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -81,7 +81,7 @@ func approvalForDeploy(ctx context.Context, b *bundle.Bundle) (bool, error) { // Recreating DLT pipeline leads to metadata loss and for a transient period // the underling tables will be unavailable. - return actions.Replace() + return actions.Replace() || actions.Delete() }) // We don't need to display any prompts in this case. @@ -99,7 +99,7 @@ func approvalForDeploy(ctx context.Context, b *bundle.Bundle) (bool, error) { // One or more DLT pipelines is being recreated. if len(dltActions) != 0 { - cmdio.LogString(ctx, "The following DLT pipelines will be recreated. This commonly happens when the `catalog` or the `storage` specified for aq DLT pipeline is changed. Underlying tables will be unavailable for a transient period until the newly recreated pipelines are run once successfully. History of previous pipeline update runs will be lost because of recreation:") + cmdio.LogString(ctx, "This action will result in the deletion or recreation of the following DLT Pipelines along with the Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the Pipelines will restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline properties such as the `catalog` or `storage` are changed:") for _, action := range dltActions { cmdio.Log(ctx, action) } diff --git a/internal/bundle/deploy_test.go b/internal/bundle/deploy_test.go index 332962eafe..635d7f6fe4 100644 --- a/internal/bundle/deploy_test.go +++ b/internal/bundle/deploy_test.go @@ -157,7 +157,10 @@ func TestAccBundlePipelineRecreateWithoutAutoApprove(t *testing.T) { stdout, stderr, err := c.Run() assert.EqualError(t, err, root.ErrAlreadyPrinted.Error()) - assert.Contains(t, stderr.String(), "The following DLT pipelines will be recreated. This commonly happens when the `catalog` or the `storage` specified for a DLT pipeline is changed. Underlying tables will be unavailable for a transient period until the newly recreated pipelines are run once successfully. History of previous pipeline update runs will be lost because of recreation:\n recreate pipeline foo") + + fmt.Println("expected: ", "This action will result in the deletion or recreation of the following DLT Pipelines along with the Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the Pipelines will restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline properties such as the `catalog` or `storage` are changed::\n recreate pipeline foo") + fmt.Println("stderr: ", stderr.String()) + assert.Contains(t, stderr.String(), "This action will result in the deletion or recreation of the following DLT Pipelines along with the Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the Pipelines will restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline properties such as the `catalog` or `storage` are changed:\n recreate pipeline foo") assert.Contains(t, stdout.String(), "the deployment requires destructive actions, but current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed") } From d9ed8ebeaea6cb0e29c27ee9eb2b4070f7156daf Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 30 Aug 2024 11:55:07 +0200 Subject: [PATCH 6/7] add multiline message --- bundle/phases/deploy.go | 7 ++++++- internal/bundle/deploy_test.go | 6 +++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 1ce799832c..49544227ec 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -99,7 +99,12 @@ func approvalForDeploy(ctx context.Context, b *bundle.Bundle) (bool, error) { // One or more DLT pipelines is being recreated. if len(dltActions) != 0 { - cmdio.LogString(ctx, "This action will result in the deletion or recreation of the following DLT Pipelines along with the Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the Pipelines will restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline properties such as the `catalog` or `storage` are changed:") + msg := ` +This action will result in the deletion or recreation of the following DLT Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the Pipelines will +restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline +properties such as the 'catalog' or 'storage' are changed:` + cmdio.LogString(ctx, msg) for _, action := range dltActions { cmdio.Log(ctx, action) } diff --git a/internal/bundle/deploy_test.go b/internal/bundle/deploy_test.go index 635d7f6fe4..3660831922 100644 --- a/internal/bundle/deploy_test.go +++ b/internal/bundle/deploy_test.go @@ -160,7 +160,11 @@ func TestAccBundlePipelineRecreateWithoutAutoApprove(t *testing.T) { fmt.Println("expected: ", "This action will result in the deletion or recreation of the following DLT Pipelines along with the Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the Pipelines will restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline properties such as the `catalog` or `storage` are changed::\n recreate pipeline foo") fmt.Println("stderr: ", stderr.String()) - assert.Contains(t, stderr.String(), "This action will result in the deletion or recreation of the following DLT Pipelines along with the Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the Pipelines will restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline properties such as the `catalog` or `storage` are changed:\n recreate pipeline foo") + assert.Contains(t, stderr.String(), `This action will result in the deletion or recreation of the following DLT Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the Pipelines will +restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline +properties such as the 'catalog' or 'storage' are changed: + recreate pipeline foo`) assert.Contains(t, stdout.String(), "the deployment requires destructive actions, but current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed") } From 2f7fa9760136eb197ed2de397898dbe81a8cbd52 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 2 Sep 2024 15:09:20 +0200 Subject: [PATCH 7/7] add test for pipeline delete as well --- internal/bundle/deploy_test.go | 52 ++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/internal/bundle/deploy_test.go b/internal/bundle/deploy_test.go index 3660831922..736c880dbc 100644 --- a/internal/bundle/deploy_test.go +++ b/internal/bundle/deploy_test.go @@ -127,6 +127,55 @@ func TestAccBundleDeployUcSchemaFailsWithoutAutoApprove(t *testing.T) { assert.Contains(t, stdout.String(), "the deployment requires destructive actions, but current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed") } +func TestAccBundlePipelineDeleteWithoutAutoApprove(t *testing.T) { + ctx, wt := acc.WorkspaceTest(t) + w := wt.W + + nodeTypeId := internal.GetNodeTypeId(env.Get(ctx, "CLOUD_ENV")) + uniqueId := uuid.New().String() + bundleRoot, err := initTestTemplate(t, ctx, "deploy_then_remove_resources", map[string]any{ + "unique_id": uniqueId, + "node_type_id": nodeTypeId, + "spark_version": defaultSparkVersion, + }) + require.NoError(t, err) + + // deploy pipeline + err = deployBundle(t, ctx, bundleRoot) + require.NoError(t, err) + + // assert pipeline is created + pipelineName := "test-bundle-pipeline-" + uniqueId + pipeline, err := w.Pipelines.GetByName(ctx, pipelineName) + require.NoError(t, err) + assert.Equal(t, pipeline.Name, pipelineName) + + // assert job is created + jobName := "test-bundle-job-" + uniqueId + job, err := w.Jobs.GetBySettingsName(ctx, jobName) + require.NoError(t, err) + assert.Equal(t, job.Settings.Name, jobName) + + // delete resources.yml + err = os.Remove(filepath.Join(bundleRoot, "resources.yml")) + require.NoError(t, err) + + // Redeploy the bundle. Expect it to fail because deleting the pipeline requires --auto-approve. + t.Setenv("BUNDLE_ROOT", bundleRoot) + t.Setenv("TERM", "dumb") + c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "deploy", "--force-lock") + stdout, stderr, err := c.Run() + + assert.EqualError(t, err, root.ErrAlreadyPrinted.Error()) + assert.Contains(t, stderr.String(), `This action will result in the deletion or recreation of the following DLT Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the Pipelines will +restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline +properties such as the 'catalog' or 'storage' are changed: + delete pipeline bar`) + assert.Contains(t, stdout.String(), "the deployment requires destructive actions, but current console does not support prompting. Please specify --auto-approve if you would like to skip prompts and proceed") + +} + func TestAccBundlePipelineRecreateWithoutAutoApprove(t *testing.T) { ctx, wt := acc.UcWorkspaceTest(t) w := wt.W @@ -157,9 +206,6 @@ func TestAccBundlePipelineRecreateWithoutAutoApprove(t *testing.T) { stdout, stderr, err := c.Run() assert.EqualError(t, err, root.ErrAlreadyPrinted.Error()) - - fmt.Println("expected: ", "This action will result in the deletion or recreation of the following DLT Pipelines along with the Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the Pipelines will restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline properties such as the `catalog` or `storage` are changed::\n recreate pipeline foo") - fmt.Println("stderr: ", stderr.String()) assert.Contains(t, stderr.String(), `This action will result in the deletion or recreation of the following DLT Pipelines along with the Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the Pipelines will restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline