Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable job queueing by default #1385

Merged
merged 9 commits into from
Apr 22, 2024
38 changes: 38 additions & 0 deletions bundle/config/mutator/default_queueing.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package mutator

import (
"context"

"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/databricks-sdk-go/service/jobs"
)

type defaultQueueing struct{}

func DefaultQueueing() bundle.Mutator {
return &defaultQueueing{}
}

func (m *defaultQueueing) Name() string {
return "DefaultQueueing"
}

// Enable queueing for jobs by default, following the behavior from API 2.2+.
// As of 2024-04, we're still using API 2.1 which has queueing disabled by default.
// This mutator makes sure queueing is enabled by default before we can adopt API 2.2.
func (m *defaultQueueing) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
r := b.Config.Resources
for i := range r.Jobs {
if r.Jobs[i].JobSettings == nil {
r.Jobs[i].JobSettings = &jobs.JobSettings{}
}
if r.Jobs[i].Queue != nil {
continue
}
r.Jobs[i].Queue = &jobs.QueueSettings{
Enabled: true,
}
}
return nil
}
95 changes: 95 additions & 0 deletions bundle/config/mutator/default_queueing_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
package mutator

import (
"context"
"testing"

"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/databricks-sdk-go/service/jobs"
"github.com/stretchr/testify/assert"
)

func TestDefaultQueueing(t *testing.T) {
m := DefaultQueueing()
assert.IsType(t, &defaultQueueing{}, m)
}

func TestDefaultQueueingName(t *testing.T) {
m := DefaultQueueing()
assert.Equal(t, "DefaultQueueing", m.Name())
}

func TestDefaultQueueingApplyNoJobs(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Resources: config.Resources{},
},
}
d := bundle.Apply(context.Background(), b, DefaultQueueing())
assert.Len(t, d, 0)
assert.Len(t, b.Config.Resources.Jobs, 0)
}

func TestDefaultQueueingApplyJobsAlreadyEnabled(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Resources: config.Resources{
Jobs: map[string]*resources.Job{
"job": {
JobSettings: &jobs.JobSettings{
Queue: &jobs.QueueSettings{Enabled: true},
},
},
},
},
},
}
d := bundle.Apply(context.Background(), b, DefaultQueueing())
assert.Len(t, d, 0)
assert.True(t, b.Config.Resources.Jobs["job"].Queue.Enabled)
}

func TestDefaultQueueingApplyEnableQueueing(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Resources: config.Resources{
Jobs: map[string]*resources.Job{
"job": {},
},
},
},
}
d := bundle.Apply(context.Background(), b, DefaultQueueing())
assert.Len(t, d, 0)
assert.NotNil(t, b.Config.Resources.Jobs["job"].Queue)
assert.True(t, b.Config.Resources.Jobs["job"].Queue.Enabled)
}

func TestDefaultQueueingApplyWithMultipleJobs(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Resources: config.Resources{
Jobs: map[string]*resources.Job{
"job1": {
JobSettings: &jobs.JobSettings{
Queue: &jobs.QueueSettings{Enabled: false},
},
},
"job2": {},
"job3": {
JobSettings: &jobs.JobSettings{
Queue: &jobs.QueueSettings{Enabled: true},
},
},
},
},
},
}
d := bundle.Apply(context.Background(), b, DefaultQueueing())
assert.Len(t, d, 0)
assert.False(t, b.Config.Resources.Jobs["job1"].Queue.Enabled)
assert.True(t, b.Config.Resources.Jobs["job2"].Queue.Enabled)
assert.True(t, b.Config.Resources.Jobs["job3"].Queue.Enabled)
}
1 change: 1 addition & 0 deletions bundle/phases/initialize.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ func Initialize() bundle.Mutator {
mutator.SetRunAs(),
mutator.OverrideCompute(),
mutator.ProcessTargetMode(),
mutator.DefaultQueueing(),
mutator.ExpandPipelineGlobPaths(),
mutator.TranslatePaths(),
python.WrapperWarning(),
Expand Down