From 88a4a0a075e82c612ffabcd6408f420bf0bb512a Mon Sep 17 00:00:00 2001 From: Travis Raines <571832+rainest@users.noreply.github.com> Date: Wed, 13 Dec 2023 14:54:23 -0800 Subject: [PATCH 01/13] chore: migrate to new io function --- pkg/konnect/error.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/konnect/error.go b/pkg/konnect/error.go index 1c12d16..991f8c5 100644 --- a/pkg/konnect/error.go +++ b/pkg/konnect/error.go @@ -4,7 +4,7 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "net/http" ) @@ -13,7 +13,7 @@ func hasError(res *http.Response) error { return nil } - body, _ := ioutil.ReadAll(res.Body) // TODO error in error? + body, _ := io.ReadAll(res.Body) // TODO error in error? return &APIError{ httpCode: res.StatusCode, message: messageFromBody(body), From c91b477b0ec52185e24c4530c58f14d506b71405 Mon Sep 17 00:00:00 2001 From: Travis Raines <571832+rainest@users.noreply.github.com> Date: Wed, 13 Dec 2023 14:54:42 -0800 Subject: [PATCH 02/13] tmp: add notes to differ prints --- pkg/diff/diff.go | 41 ++++++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/pkg/diff/diff.go b/pkg/diff/diff.go index 8730e82..b2071ed 100644 --- a/pkg/diff/diff.go +++ b/pkg/diff/diff.go @@ -126,9 +126,13 @@ func NewSyncer(opts SyncerOpts) (*Syncer, error) { noMaskValues: opts.NoMaskValues, - createPrintln: opts.CreatePrintln, - updatePrintln: opts.UpdatePrintln, - deletePrintln: opts.DeletePrintln, + // TODO From review of existing code in https://github.com/Kong/go-database-reconciler/pull/30, these *Println + // allow you to _bring your own output formatter_ These are likely not used in practice (neither KIC nor deck + // set them) and can probably be removed. Doing so is technically a breaking change, however. + createPrintln: opts.CreatePrintln, + updatePrintln: opts.UpdatePrintln, + deletePrintln: opts.DeletePrintln, + includeLicenses: opts.IncludeLicenses, isKonnect: opts.IsKonnect, } @@ -338,7 +342,7 @@ func (sc *Syncer) wait() { } // Run starts a diff and invokes d for every diff. -func (sc *Syncer) Run(ctx context.Context, parallelism int, d Do) []error { +func (sc *Syncer) Run(ctx context.Context, parallelism int, action Do) []error { if parallelism < 1 { return append([]error{}, fmt.Errorf("parallelism can not be negative")) } @@ -355,7 +359,7 @@ func (sc *Syncer) Run(ctx context.Context, parallelism int, d Do) []error { wg.Add(parallelism) for i := 0; i < parallelism; i++ { go func() { - err := sc.eventLoop(ctx, d) + err := sc.eventLoop(ctx, action) if err != nil { sc.errChan <- err } @@ -408,7 +412,8 @@ func (sc *Syncer) Run(ctx context.Context, parallelism int, d Do) []error { // Do is the worker function to sync the diff type Do func(a crud.Event) (crud.Arg, error) -func (sc *Syncer) eventLoop(ctx context.Context, d Do) error { +// NOTE TRC part of the return path +func (sc *Syncer) eventLoop(ctx context.Context, action Do) error { for event := range sc.eventChan { // Stop if program is terminated select { @@ -417,7 +422,7 @@ func (sc *Syncer) eventLoop(ctx context.Context, d Do) error { default: } - err := sc.handleEvent(ctx, d, event) + err := sc.handleEvent(ctx, action, event) sc.eventCompleted() if err != nil { return err @@ -426,9 +431,10 @@ func (sc *Syncer) eventLoop(ctx context.Context, d Do) error { return nil } -func (sc *Syncer) handleEvent(ctx context.Context, d Do, event crud.Event) error { +// NOTE TRC part of the return path. this actually runs the +func (sc *Syncer) handleEvent(ctx context.Context, action Do, event crud.Event) error { err := backoff.Retry(func() error { - res, err := d(event) + res, err := action(event) if err != nil { err = fmt.Errorf("while processing event: %w", err) @@ -490,6 +496,14 @@ func generateDiffString(e crud.Event, isDelete bool, noMaskValues bool) (string, return diffString, err } +// NOTE TRC Solve is the entry point for both the local and konnect sync commands +// those command functions already output other text fwiw +// although they can iterate over a returned op set and print info for each, this does +// introduce a delay in output. Solve() currently prints each action as it takes them, +// whereas the returned set would be printed in a batch at the end. unsure if this should +// matter in practice, but probably not. we could introduce an event channel and separate +// goroutine to handle synch output, but probably not worth it + // Solve generates a diff and walks the graph. func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOut bool) (Stats, []error, EntityChanges, @@ -516,6 +530,9 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOu Deleting: []EntityState{}, } + // NOTE TRC the length makes it confusing to read, but the code below _isn't being run here_, it's an anon func + // arg to Run(), which parallelizes it. However, because it's defined in Solve()'s scope, the output created above + // is available in aggregate and contains most of the content we need already errs := sc.Run(ctx, parallelism, func(e crud.Event) (crud.Arg, error) { var err error var result crud.Arg @@ -527,9 +544,12 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOu } item := EntityState{ Body: objDiff, + // NOTE TRC currently used directly Name: c.Console(), + // NOTE TRC current prints use the kind directly, but it doesn't matter, it's just a string alias anyway Kind: string(e.Kind), } + // NOTE TRC currently we emit lines here, need to collect objects instead switch e.Op { case crud.Create: if isJSONOut { @@ -538,6 +558,7 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOu sc.createPrintln("creating", e.Kind, c.Console()) } case crud.Update: + // TODO TRC this is not currently available in the item EntityState diffString, err := generateDiffString(e, false, sc.noMaskValues) if err != nil { return nil, err @@ -579,6 +600,8 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOu // record operation in both: diff and sync commands recordOp(e.Op) + // TODO TRC our existing return is a complete object and error. probably need to return some sort of processed + // event struct return result, nil }) return stats, errs, output From 5ebb3ff4c33b62b0dc6d713d37c7b7a894789678 Mon Sep 17 00:00:00 2001 From: Travis Raines <571832+rainest@users.noreply.github.com> Date: Wed, 13 Dec 2023 15:18:21 -0800 Subject: [PATCH 03/13] feat: remove diff prints and add diff field Remove the event prints from diff in favor of only returning the information currently used for JSON ouptut. Add a Diff field for EntityState to hold the update diff string. --- pkg/diff/diff.go | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/pkg/diff/diff.go b/pkg/diff/diff.go index b2071ed..f3ec965 100644 --- a/pkg/diff/diff.go +++ b/pkg/diff/diff.go @@ -23,6 +23,7 @@ type EntityState struct { Name string `json:"name"` Kind string `json:"kind"` Body any `json:"body"` + Diff string `json:"-"` } type Summary struct { @@ -552,28 +553,17 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOu // NOTE TRC currently we emit lines here, need to collect objects instead switch e.Op { case crud.Create: - if isJSONOut { - output.Creating = append(output.Creating, item) - } else { - sc.createPrintln("creating", e.Kind, c.Console()) - } + output.Creating = append(output.Creating, item) case crud.Update: // TODO TRC this is not currently available in the item EntityState diffString, err := generateDiffString(e, false, sc.noMaskValues) if err != nil { return nil, err } - if isJSONOut { - output.Updating = append(output.Updating, item) - } else { - sc.updatePrintln("updating", e.Kind, c.Console(), diffString) - } + item.Diff = diffString + output.Updating = append(output.Updating, item) case crud.Delete: - if isJSONOut { - output.Deleting = append(output.Deleting, item) - } else { - sc.deletePrintln("deleting", e.Kind, c.Console()) - } + output.Deleting = append(output.Deleting, item) default: panic("unknown operation " + e.Op.String()) } From a7004fd2df36e226a92e88fb9840b7ef1b39b4d3 Mon Sep 17 00:00:00 2001 From: Travis Raines <571832+rainest@users.noreply.github.com> Date: Wed, 13 Dec 2023 15:23:14 -0800 Subject: [PATCH 04/13] feat: remove syncer print helpers --- pkg/diff/diff.go | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/pkg/diff/diff.go b/pkg/diff/diff.go index f3ec965..0a4c5ff 100644 --- a/pkg/diff/diff.go +++ b/pkg/diff/diff.go @@ -10,7 +10,6 @@ import ( "time" "github.com/cenkalti/backoff/v4" - "github.com/kong/go-database-reconciler/pkg/cprint" "github.com/kong/go-database-reconciler/pkg/crud" "github.com/kong/go-database-reconciler/pkg/konnect" "github.com/kong/go-database-reconciler/pkg/state" @@ -78,10 +77,6 @@ type Syncer struct { silenceWarnings bool stageDelaySec int - createPrintln func(a ...interface{}) - updatePrintln func(a ...interface{}) - deletePrintln func(a ...interface{}) - kongClient *kong.Client konnectClient *konnect.Client @@ -107,10 +102,6 @@ type SyncerOpts struct { IncludeLicenses bool IsKonnect bool - - CreatePrintln func(a ...interface{}) - UpdatePrintln func(a ...interface{}) - DeletePrintln func(a ...interface{}) } // NewSyncer constructs a Syncer. @@ -127,13 +118,6 @@ func NewSyncer(opts SyncerOpts) (*Syncer, error) { noMaskValues: opts.NoMaskValues, - // TODO From review of existing code in https://github.com/Kong/go-database-reconciler/pull/30, these *Println - // allow you to _bring your own output formatter_ These are likely not used in practice (neither KIC nor deck - // set them) and can probably be removed. Doing so is technically a breaking change, however. - createPrintln: opts.CreatePrintln, - updatePrintln: opts.UpdatePrintln, - deletePrintln: opts.DeletePrintln, - includeLicenses: opts.IncludeLicenses, isKonnect: opts.IsKonnect, } @@ -142,16 +126,6 @@ func NewSyncer(opts SyncerOpts) (*Syncer, error) { s.includeLicenses = false } - if s.createPrintln == nil { - s.createPrintln = cprint.CreatePrintln - } - if s.updatePrintln == nil { - s.updatePrintln = cprint.UpdatePrintln - } - if s.deletePrintln == nil { - s.deletePrintln = cprint.DeletePrintln - } - err := s.init() if err != nil { return nil, err From b79a770ab196cc655d846bfc7304ad81536d5ea6 Mon Sep 17 00:00:00 2001 From: Travis Raines <571832+rainest@users.noreply.github.com> Date: Wed, 13 Dec 2023 16:36:33 -0800 Subject: [PATCH 05/13] feat: lock the syncer log --- pkg/diff/diff.go | 52 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/pkg/diff/diff.go b/pkg/diff/diff.go index 0a4c5ff..dfc7e55 100644 --- a/pkg/diff/diff.go +++ b/pkg/diff/diff.go @@ -86,6 +86,9 @@ type Syncer struct { includeLicenses bool isKonnect bool + + eventLog EntityChanges + eventLogMutex sync.Mutex } type SyncerOpts struct { @@ -180,6 +183,13 @@ func (sc *Syncer) init() error { sc.processor.MustRegister(crud.Kind(entityType), entity.CRUDActions()) sc.entityDiffers[entityType] = entity.Differ() } + + sc.eventLog = EntityChanges{ + Creating: []EntityState{}, + Updating: []EntityState{}, + Deleting: []EntityState{}, + } + return nil } @@ -316,6 +326,34 @@ func (sc *Syncer) wait() { } } +// LogCreate adds a create action to the event log. +func (sc *Syncer) LogCreate(state EntityState) { + sc.eventLogMutex.Lock() + defer sc.eventLogMutex.Unlock() + sc.eventLog.Creating = append(sc.eventLog.Creating, state) +} + +// LogUpdate adds an update action to the event log. +func (sc *Syncer) LogUpdate(state EntityState) { + sc.eventLogMutex.Lock() + defer sc.eventLogMutex.Unlock() + sc.eventLog.Updating = append(sc.eventLog.Updating, state) +} + +// LogDelete adds a delete action to the event log. +func (sc *Syncer) LogDelete(state EntityState) { + sc.eventLogMutex.Lock() + defer sc.eventLogMutex.Unlock() + sc.eventLog.Deleting = append(sc.eventLog.Deleting, state) +} + +// GetEventLog returns the syncer event log. +func (sc *Syncer) GetEventLog() EntityChanges { + sc.eventLogMutex.Lock() + defer sc.eventLogMutex.Unlock() + return sc.eventLog +} + // Run starts a diff and invokes d for every diff. func (sc *Syncer) Run(ctx context.Context, parallelism int, action Do) []error { if parallelism < 1 { @@ -499,12 +537,6 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOu } } - output := EntityChanges{ - Creating: []EntityState{}, - Updating: []EntityState{}, - Deleting: []EntityState{}, - } - // NOTE TRC the length makes it confusing to read, but the code below _isn't being run here_, it's an anon func // arg to Run(), which parallelizes it. However, because it's defined in Solve()'s scope, the output created above // is available in aggregate and contains most of the content we need already @@ -527,7 +559,7 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOu // NOTE TRC currently we emit lines here, need to collect objects instead switch e.Op { case crud.Create: - output.Creating = append(output.Creating, item) + sc.LogCreate(item) case crud.Update: // TODO TRC this is not currently available in the item EntityState diffString, err := generateDiffString(e, false, sc.noMaskValues) @@ -535,9 +567,9 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOu return nil, err } item.Diff = diffString - output.Updating = append(output.Updating, item) + sc.LogUpdate(item) case crud.Delete: - output.Deleting = append(output.Deleting, item) + sc.LogDelete(item) default: panic("unknown operation " + e.Op.String()) } @@ -568,5 +600,5 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOu // event struct return result, nil }) - return stats, errs, output + return stats, errs, sc.GetEventLog() } From 5f17ace61f513cd887d635facc9c95e33b6bb2b9 Mon Sep 17 00:00:00 2001 From: Travis Raines <571832+rainest@users.noreply.github.com> Date: Thu, 14 Dec 2023 17:00:11 -0800 Subject: [PATCH 06/13] feat: replace prints with state warnings --- pkg/file/builder.go | 14 ++++++++++++-- pkg/types/basicauth.go | 13 ------------- pkg/utils/types.go | 3 +++ pkg/utils/utils.go | 7 +++---- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/pkg/file/builder.go b/pkg/file/builder.go index 9d1eee5..8423ad3 100644 --- a/pkg/file/builder.go +++ b/pkg/file/builder.go @@ -6,6 +6,7 @@ import ( "fmt" "reflect" "sort" + "sync" "github.com/blang/semver/v4" "github.com/kong/go-database-reconciler/pkg/konnect" @@ -14,7 +15,11 @@ import ( "github.com/kong/go-kong/kong" ) -const ratelimitingAdvancedPluginName = "rate-limiting-advanced" +const ( + ratelimitingAdvancedPluginName = "rate-limiting-advanced" + basicAuthPasswordWarning = "Warning: import/export of basic-auth" + + "credentials using decK doesn't work due to hashing of passwords in Kong." +) type stateBuilder struct { targetContent *Content @@ -42,6 +47,8 @@ type stateBuilder struct { checkRoutePaths bool + warnBasicAuth sync.Once + isConsumerGroupScopedPluginSupported bool err error @@ -481,6 +488,9 @@ func (b *stateBuilder) consumers() { var basicAuths []kong.BasicAuth for _, cred := range c.BasicAuths { + b.warnBasicAuth.Do(func() { + b.rawState.Warnings = append(b.rawState.Warnings, basicAuthPasswordWarning) + }) cred.Consumer = utils.GetConsumerReference(c.Consumer) basicAuths = append(basicAuths, *cred) } @@ -931,7 +941,7 @@ func (b *stateBuilder) routes() { } } if len(unsupportedRoutes) > 0 { - utils.PrintRouteRegexWarning(unsupportedRoutes) + b.rawState.Warnings = append(b.rawState.Warnings, utils.FormatRouteRegexWarning(unsupportedRoutes)) } } } diff --git a/pkg/types/basicauth.go b/pkg/types/basicauth.go index 14647a3..d5691ab 100644 --- a/pkg/types/basicauth.go +++ b/pkg/types/basicauth.go @@ -6,7 +6,6 @@ import ( "fmt" "sync" - "github.com/kong/go-database-reconciler/pkg/cprint" "github.com/kong/go-database-reconciler/pkg/crud" "github.com/kong/go-database-reconciler/pkg/state" "github.com/kong/go-database-reconciler/pkg/utils" @@ -99,16 +98,6 @@ type basicAuthDiffer struct { currentState, targetState *state.KongState } -func (d *basicAuthDiffer) warnBasicAuth() { - const ( - basicAuthPasswordWarning = "Warning: import/export of basic-auth" + - "credentials using decK doesn't work due to hashing of passwords in Kong." - ) - d.once.Do(func() { - cprint.UpdatePrintln(basicAuthPasswordWarning) - }) -} - func (d *basicAuthDiffer) Deletes(handler func(crud.Event) error) error { currentBasicAuths, err := d.currentState.BasicAuths.GetAll() if err != nil { @@ -131,7 +120,6 @@ func (d *basicAuthDiffer) Deletes(handler func(crud.Event) error) error { } func (d *basicAuthDiffer) deleteBasicAuth(basicAuth *state.BasicAuth) (*crud.Event, error) { - d.warnBasicAuth() _, err := d.targetState.BasicAuths.Get(*basicAuth.ID) if errors.Is(err, state.ErrNotFound) { return &crud.Event{ @@ -169,7 +157,6 @@ func (d *basicAuthDiffer) CreateAndUpdates(handler func(crud.Event) error) error } func (d *basicAuthDiffer) createUpdateBasicAuth(basicAuth *state.BasicAuth) (*crud.Event, error) { - d.warnBasicAuth() basicAuth = &state.BasicAuth{BasicAuth: *basicAuth.DeepCopy()} currentBasicAuth, err := d.currentState.BasicAuths.Get(*basicAuth.ID) if errors.Is(err, state.ErrNotFound) { diff --git a/pkg/utils/types.go b/pkg/utils/types.go index 2ba9bdc..93e107d 100644 --- a/pkg/utils/types.go +++ b/pkg/utils/types.go @@ -55,6 +55,9 @@ type KongRawState struct { RBACRoles []*kong.RBACRole RBACEndpointPermissions []*kong.RBACEndpointPermission + + // Warnings are not Kong data. This field stores any warnings related to Kong data found while building state. + Warnings []string } // KonnectRawState contains all of Konnect resources. diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index a9ac8d8..89cdbfe 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -12,7 +12,6 @@ import ( "strings" "github.com/blang/semver/v4" - "github.com/kong/go-database-reconciler/pkg/cprint" "github.com/kong/go-kong/kong" ) @@ -236,14 +235,14 @@ func HasPathsWithRegex300AndAbove(route kong.Route) bool { return false } -// PrintRouteRegexWarning prints out a warning about 3.x routes' path usage. -func PrintRouteRegexWarning(unsupportedRoutes []string) { +// FormatRouteRegexWarning returns a warning about 3.x routes' path usage. +func FormatRouteRegexWarning(unsupportedRoutes []string) string { unsupportedRoutesLen := len(unsupportedRoutes) // do not consider more than 10 sample routes to print out. if unsupportedRoutesLen > 10 { unsupportedRoutes = unsupportedRoutes[:10] } - cprint.UpdatePrintf( + return fmt.Sprintf( "%d unsupported routes' paths format with Kong version 3.0\n"+ "or above were detected. Some of these routes are (not an exhaustive list):\n\n"+ "%s\n\n"+UpgradeMessage, From b60bc21fd3d7e828c52304de38f8519072902f22 Mon Sep 17 00:00:00 2001 From: Travis Raines <571832+rainest@users.noreply.github.com> Date: Fri, 15 Dec 2023 13:17:58 -0800 Subject: [PATCH 07/13] chore: update test output expectations --- tests/integration/diff_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/diff_test.go b/tests/integration/diff_test.go index e033c00..dd3dc43 100644 --- a/tests/integration/diff_test.go +++ b/tests/integration/diff_test.go @@ -10,7 +10,8 @@ import ( ) var ( - expectedOutputMasked = `updating service svc1 { + expectedOutputMasked = `creating plugin rate-limiting (global) +updating service svc1 { "connect_timeout": 60000, "enabled": true, "host": "[masked]", @@ -30,14 +31,14 @@ var ( + ] } -creating plugin rate-limiting (global) Summary: Created: 1 Updated: 1 Deleted: 0 ` - expectedOutputUnMasked = `updating service svc1 { + expectedOutputUnMasked = `creating plugin rate-limiting (global) +updating service svc1 { "connect_timeout": 60000, "enabled": true, "host": "mockbin.org", @@ -53,7 +54,6 @@ Summary: + ] } -creating plugin rate-limiting (global) Summary: Created: 1 Updated: 1 From 9a344ae075f8754215679f160e3c16385203a7f3 Mon Sep 17 00:00:00 2001 From: Travis Raines <571832+rainest@users.noreply.github.com> Date: Fri, 15 Dec 2023 13:51:32 -0800 Subject: [PATCH 08/13] fix: wait for Kong to come up better --- .ci/setup_kong.sh | 1 + .ci/setup_kong_ee.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/.ci/setup_kong.sh b/.ci/setup_kong.sh index 1272a15..61d78c8 100755 --- a/.ci/setup_kong.sh +++ b/.ci/setup_kong.sh @@ -63,3 +63,4 @@ docker run -d --name $GATEWAY_CONTAINER_NAME \ $KONG_IMAGE waitContainer "Kong" $GATEWAY_CONTAINER_NAME "kong health" +curl --retry 12 --retry-all-errors http://localhost:8001 diff --git a/.ci/setup_kong_ee.sh b/.ci/setup_kong_ee.sh index 1ef4585..913ab46 100755 --- a/.ci/setup_kong_ee.sh +++ b/.ci/setup_kong_ee.sh @@ -127,3 +127,4 @@ docker run -d --name $GATEWAY_CONTAINER_NAME \ $KONG_IMAGE waitContainer "Kong" $GATEWAY_CONTAINER_NAME "kong health" +curl --retry 12 --retry-all-errors http://localhost:8001 From fddd0b62eeb116b98ccd04a19bf84061acc69069 Mon Sep 17 00:00:00 2001 From: Travis Raines <571832+rainest@users.noreply.github.com> Date: Fri, 15 Dec 2023 14:36:47 -0800 Subject: [PATCH 09/13] feat: add non-cmd diff test scaffolding --- go.mod | 1 - go.sum | 2 - tests/integration/diff_test.go | 52 +++++++++++--------- tests/integration/sync_test.go | 2 +- tests/integration/test_utils.go | 87 ++++++++++++++++++++++++++++++++- 5 files changed, 116 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index 0853ae8..352b743 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ replace github.com/yudai/gojsondiff v1.0.0 => github.com/Kong/gojsondiff v1.3.0 require ( dario.cat/mergo v1.0.0 github.com/Kong/gojsondiff v1.3.2 - github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/alecthomas/jsonschema v0.0.0-20191017121752-4bb6e3fae4f2 github.com/blang/semver/v4 v4.0.0 github.com/cenkalti/backoff/v4 v4.3.0 diff --git a/go.sum b/go.sum index 3146482..def01fe 100644 --- a/go.sum +++ b/go.sum @@ -25,8 +25,6 @@ github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJ github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/adrg/strutil v0.2.3 h1:WZVn3ItPBovFmP4wMHHVXUr8luRaHrbyIuLlHt32GZQ= github.com/adrg/strutil v0.2.3/go.mod h1:+SNxbiH6t+O+5SZqIj5n/9i5yUjR+S3XXVrjEcN2mxg= github.com/alecthomas/jsonschema v0.0.0-20191017121752-4bb6e3fae4f2 h1:swGeCLPiUQ647AIRnFxnAHdzlg6IPpmU6QdkOPZINt8= diff --git a/tests/integration/diff_test.go b/tests/integration/diff_test.go index dd3dc43..ba1eb5d 100644 --- a/tests/integration/diff_test.go +++ b/tests/integration/diff_test.go @@ -3,12 +3,18 @@ package integration import ( + "context" "testing" "github.com/kong/go-database-reconciler/pkg/utils" "github.com/stretchr/testify/assert" ) +var ( + expectedDiff = syncOut{} + expectedDiff3 = syncOut{} +) + var ( expectedOutputMasked = `creating plugin rate-limiting (global) updating service svc1 { @@ -476,8 +482,9 @@ func Test_Diff_Workspace_OlderThan3x(t *testing.T) { runWhen(t, "kong", "<3.0.0") setup(t) - _, err := diff(tc.stateFile) + out, err := testSync(context.Background(), []string{tc.stateFile}, true) assert.NoError(t, err) + assert.Equal(t, expectedDiff, out) }) } } @@ -500,8 +507,9 @@ func Test_Diff_Workspace_NewerThan3x(t *testing.T) { runWhen(t, "kong", ">=3.0.0") setup(t) - _, err := diff(tc.stateFile) + out, err := testSync(context.Background(), []string{tc.stateFile}, true) assert.NoError(t, err) + assert.Equal(t, expectedDiff, out) }) } } @@ -534,9 +542,9 @@ func Test_Diff_Masked_OlderThan3x(t *testing.T) { // initialize state assert.NoError(t, sync(tc.initialStateFile)) - out, err := diff(tc.stateFile) + out, err := testSync(context.Background(), []string{tc.stateFile}, true) assert.NoError(t, err) - assert.Equal(t, expectedOutputMasked, out) + assert.Equal(t, expectedDiff, out) }) } @@ -551,9 +559,9 @@ func Test_Diff_Masked_OlderThan3x(t *testing.T) { // initialize state assert.NoError(t, sync(tc.initialStateFile)) - out, err := diff(tc.stateFile, "--json-output") + out, err := testSync(context.Background(), []string{tc.stateFile}, true) assert.NoError(t, err) - assert.Equal(t, expectedOutputMaskedJSON, out) + assert.Equal(t, expectedDiff, out) }) } } @@ -586,9 +594,9 @@ func Test_Diff_Masked_NewerThan3x(t *testing.T) { // initialize state assert.NoError(t, sync(tc.initialStateFile)) - out, err := diff(tc.stateFile) + out, err := testSync(context.Background(), []string{tc.stateFile}, true) assert.NoError(t, err) - assert.Equal(t, expectedOutputMasked, out) + assert.Equal(t, expectedDiff, out) }) } for _, tc := range tests { @@ -602,9 +610,9 @@ func Test_Diff_Masked_NewerThan3x(t *testing.T) { // initialize state assert.NoError(t, sync(tc.initialStateFile)) - out, err := diff(tc.stateFile, "--json-output") + out, err := testSync(context.Background(), []string{tc.stateFile}, true) assert.NoError(t, err) - assert.Equal(t, expectedOutputMaskedJSON30x, out) + assert.Equal(t, expectedDiff, out) }) } for _, tc := range tests { @@ -618,9 +626,9 @@ func Test_Diff_Masked_NewerThan3x(t *testing.T) { // initialize state assert.NoError(t, sync(tc.initialStateFile)) - out, err := diff(tc.stateFile, "--json-output") + out, err := testSync(context.Background(), []string{tc.stateFile}, true) assert.NoError(t, err) - assert.Equal(t, expectedOutputMaskedJSON, out) + assert.Equal(t, expectedDiff, out) }) } } @@ -653,9 +661,9 @@ func Test_Diff_Unmasked_OlderThan3x(t *testing.T) { // initialize state assert.NoError(t, sync(tc.initialStateFile)) - out, err := diff(tc.stateFile, "--no-mask-deck-env-vars-value") + out, err := testSync(context.Background(), []string{tc.stateFile}, true) assert.NoError(t, err) - assert.Equal(t, expectedOutputUnMasked, out) + assert.Equal(t, expectedDiff, out) }) } for _, tc := range tests { @@ -669,9 +677,9 @@ func Test_Diff_Unmasked_OlderThan3x(t *testing.T) { // initialize state assert.NoError(t, sync(tc.initialStateFile)) - out, err := diff(tc.stateFile, "--no-mask-deck-env-vars-value", "--json-output") + out, err := testSync(context.Background(), []string{tc.stateFile}, true) assert.NoError(t, err) - assert.Equal(t, expectedOutputUnMaskedJSON, out) + assert.Equal(t, expectedDiff, out) }) } } @@ -704,9 +712,9 @@ func Test_Diff_Unmasked_NewerThan3x(t *testing.T) { // initialize state assert.NoError(t, sync(tc.initialStateFile)) - out, err := diff(tc.stateFile, "--no-mask-deck-env-vars-value") + out, err := testSync(context.Background(), []string{tc.stateFile}, true) assert.NoError(t, err) - assert.Equal(t, expectedOutputUnMasked, out) + assert.Equal(t, expectedDiff, out) }) } for _, tc := range tests { @@ -720,9 +728,9 @@ func Test_Diff_Unmasked_NewerThan3x(t *testing.T) { // initialize state assert.NoError(t, sync(tc.initialStateFile)) - out, err := diff(tc.stateFile, "--no-mask-deck-env-vars-value", "--json-output") + out, err := testSync(context.Background(), []string{tc.stateFile}, true) assert.NoError(t, err) - assert.Equal(t, expectedOutputUnMaskedJSON30x, out) + assert.Equal(t, expectedDiff, out) }) } for _, tc := range tests { @@ -736,9 +744,9 @@ func Test_Diff_Unmasked_NewerThan3x(t *testing.T) { // initialize state assert.NoError(t, sync(tc.initialStateFile)) - out, err := diff(tc.stateFile, "--no-mask-deck-env-vars-value", "--json-output") + out, err := testSync(context.Background(), []string{tc.stateFile}, true) assert.NoError(t, err) - assert.Equal(t, expectedOutputUnMaskedJSON, out) + assert.Equal(t, expectedDiff, out) }) } } diff --git a/tests/integration/sync_test.go b/tests/integration/sync_test.go index ac1cf6e..e3aa730 100644 --- a/tests/integration/sync_test.go +++ b/tests/integration/sync_test.go @@ -2907,7 +2907,7 @@ func Test_Sync_Vault(t *testing.T) { } res, err := client.Get("https://localhost:8443/r1") - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) }) } diff --git a/tests/integration/test_utils.go b/tests/integration/test_utils.go index 71ba416..b51d16b 100644 --- a/tests/integration/test_utils.go +++ b/tests/integration/test_utils.go @@ -3,6 +3,7 @@ package integration import ( "context" + "fmt" "io" "os" "testing" @@ -14,7 +15,9 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" "github.com/kong/deck/cmd" deckDiff "github.com/kong/go-database-reconciler/pkg/diff" + pkgdiff "github.com/kong/go-database-reconciler/pkg/diff" deckDump "github.com/kong/go-database-reconciler/pkg/dump" + pkgdump "github.com/kong/go-database-reconciler/pkg/dump" "github.com/kong/go-database-reconciler/pkg/file" "github.com/kong/go-database-reconciler/pkg/state" "github.com/kong/go-database-reconciler/pkg/utils" @@ -193,7 +196,7 @@ func testKongState(t *testing.T, client *kong.Client, isKonnect bool, ) { // Get entities from Kong ctx := context.Background() - dumpConfig := deckDump.Config{} + dumpConfig := pkgdump.Config{} if expectedState.RBACEndpointPermissions != nil { dumpConfig.RBACResourcesOnly = true } @@ -208,7 +211,7 @@ func testKongState(t *testing.T, client *kong.Client, isKonnect bool, dumpConfig.KonnectControlPlane = "default" } } - kongState, err := deckDump.Get(ctx, client, dumpConfig) + kongState, err := pkgdump.Get(ctx, client, dumpConfig) if err != nil { t.Errorf(err.Error()) } @@ -273,6 +276,86 @@ func setup(t *testing.T) { }) } +type syncOut struct { + Stats pkgdiff.Stats + Errors []error + Changes pkgdiff.EntityChanges +} + +// testSync is a stripped-down version of deck's cmd.syncMain for testing changeset expectations. It removes support +// for JSON output, skipping resources, Konnect, workspaces, selector tags, and resource type filtering. +func testSync(ctx context.Context, filenames []string, dry bool) (syncOut, error) { + enableJSONOutput := false + targetContent, err := file.GetContentFromFiles(filenames, false) + if err != nil { + return syncOut{}, err + } + + // TODO static config + kongClient, err := getTestClient() + if err != nil { + return syncOut{}, err + } + + var parsedKongVersion semver.Version + root, err := kongClient.Root(ctx) + if err != nil { + return syncOut{}, fmt.Errorf("reading Kong version: %w", err) + } + kongVersion, ok := root["version"].(string) + if !ok { + return syncOut{}, fmt.Errorf("no Kong version found") + } + parsedKongVersion, err = utils.ParseKongVersion(kongVersion) + if err != nil { + return syncOut{}, fmt.Errorf("parsing Kong version: %w", err) + } + + dumpConfig := pkgdump.Config{} + + if utils.Kong340Version.LTE(parsedKongVersion) { + dumpConfig.IsConsumerGroupScopedPluginSupported = true + } + + // read the current state + rawState, err := pkgdump.Get(ctx, kongClient, dumpConfig) + if err != nil { + return syncOut{}, fmt.Errorf("could not dump state: %w", err) + } + + currentState, err := state.Get(rawState) + if err != nil { + return syncOut{}, fmt.Errorf("could not convert state: %w", err) + } + + // read the target state + rawTargetState, err := file.Get(ctx, targetContent, file.RenderConfig{ + CurrentState: currentState, + KongVersion: parsedKongVersion, + }, dumpConfig, kongClient) + if err != nil { + return syncOut{}, err + } + targetState, err := state.Get(rawTargetState) + if err != nil { + return syncOut{}, err + } + + syncer, err := pkgdiff.NewSyncer(pkgdiff.SyncerOpts{ + CurrentState: currentState, + TargetState: targetState, + KongClient: kongClient, + StageDelaySec: 2, + NoMaskValues: false, + IsKonnect: false, + }) + if err != nil { + return syncOut{}, err + } + + stats, syncErrs, changes := syncer.Solve(ctx, 5, dry, enableJSONOutput) + return syncOut{Stats: stats, Errors: syncErrs, Changes: changes}, nil +} func sync(kongFile string, opts ...string) error { deckCmd := cmd.NewRootCmd() args := []string{"sync", "-s", kongFile} From 39bbe8b9e06daabcddc7ee4337c3e8e1a1c09f92 Mon Sep 17 00:00:00 2001 From: Travis Raines <571832+rainest@users.noreply.github.com> Date: Fri, 15 Dec 2023 15:26:02 -0800 Subject: [PATCH 10/13] chore: consolidate and refactor diff tests --- tests/integration/diff_test.go | 598 ++------------------------------ tests/integration/test_utils.go | 8 +- 2 files changed, 42 insertions(+), 564 deletions(-) diff --git a/tests/integration/diff_test.go b/tests/integration/diff_test.go index ba1eb5d..984be72 100644 --- a/tests/integration/diff_test.go +++ b/tests/integration/diff_test.go @@ -11,13 +11,7 @@ import ( ) var ( - expectedDiff = syncOut{} - expectedDiff3 = syncOut{} -) - -var ( - expectedOutputMasked = `creating plugin rate-limiting (global) -updating service svc1 { + expectedOutputMasked = ` { "connect_timeout": 60000, "enabled": true, "host": "[masked]", @@ -36,15 +30,9 @@ updating service svc1 { + "bar:[masked]" + ] } - -Summary: - Created: 1 - Updated: 1 - Deleted: 0 ` - expectedOutputUnMasked = `creating plugin rate-limiting (global) -updating service svc1 { + expectedOutputUnMasked = ` { "connect_timeout": 60000, "enabled": true, "host": "mockbin.org", @@ -59,11 +47,6 @@ updating service svc1 { + "test" + ] } - -Summary: - Created: 1 - Updated: 1 - Deleted: 0 ` diffEnvVars = map[string]string{ @@ -73,447 +56,8 @@ Summary: "DECK_FUB": "fubfub", // unused "DECK_FOO": "foo_test", // unused, partial match } - - expectedOutputUnMaskedJSON = `{ - "changes": { - "creating": [ - { - "name": "rate-limiting (global)", - "kind": "plugin", - "body": { - "new": { - "id": "a1368a28-cb5c-4eee-86d8-03a6bdf94b5e", - "name": "rate-limiting", - "config": { - "day": null, - "error_code": 429, - "error_message": "API rate limit exceeded", - "fault_tolerant": true, - "header_name": null, - "hide_client_headers": false, - "hour": null, - "limit_by": "consumer", - "minute": 123, - "month": null, - "path": null, - "policy": "local", - "redis_database": 0, - "redis_host": null, - "redis_password": null, - "redis_port": 6379, - "redis_server_name": null, - "redis_ssl": false, - "redis_ssl_verify": false, - "redis_timeout": 2000, - "redis_username": null, - "second": null, - "year": null - }, - "enabled": true, - "protocols": [ - "grpc", - "grpcs", - "http", - "https" - ] - }, - "old": null - } - } - ], - "updating": [ - { - "name": "svc1", - "kind": "service", - "body": { - "new": { - "connect_timeout": 60000, - "enabled": true, - "host": "mockbin.org", - "id": "9ecf5708-f2f4-444e-a4c7-fcd3a57f9a6d", - "name": "svc1", - "port": 80, - "protocol": "http", - "read_timeout": 60000, - "retries": 5, - "write_timeout": 60000, - "tags": [ - "test" - ] - }, - "old": { - "connect_timeout": 60000, - "enabled": true, - "host": "mockbin.org", - "id": "9ecf5708-f2f4-444e-a4c7-fcd3a57f9a6d", - "name": "svc1", - "port": 80, - "protocol": "http", - "read_timeout": 60000, - "retries": 5, - "write_timeout": 60000 - } - } - } - ], - "deleting": [] - }, - "summary": { - "creating": 1, - "updating": 1, - "deleting": 0, - "total": 2 - }, - "warnings": [], - "errors": [] -} - -` - - expectedOutputMaskedJSON = `{ - "changes": { - "creating": [ - { - "name": "rate-limiting (global)", - "kind": "plugin", - "body": { - "new": { - "id": "a1368a28-cb5c-4eee-86d8-03a6bdf94b5e", - "name": "rate-limiting", - "config": { - "day": null, - "error_code": 429, - "error_message": "API rate limit exceeded", - "fault_tolerant": true, - "header_name": null, - "hide_client_headers": false, - "hour": null, - "limit_by": "consumer", - "minute": 123, - "month": null, - "path": null, - "policy": "local", - "redis_database": 0, - "redis_host": null, - "redis_password": null, - "redis_port": 6379, - "redis_server_name": null, - "redis_ssl": false, - "redis_ssl_verify": false, - "redis_timeout": 2000, - "redis_username": null, - "second": null, - "year": null - }, - "enabled": true, - "protocols": [ - "grpc", - "grpcs", - "http", - "https" - ] - }, - "old": null - } - } - ], - "updating": [ - { - "name": "svc1", - "kind": "service", - "body": { - "new": { - "connect_timeout": 60000, - "enabled": true, - "host": "[masked]", - "id": "9ecf5708-f2f4-444e-a4c7-fcd3a57f9a6d", - "name": "svc1", - "port": 80, - "protocol": "http", - "read_timeout": 60000, - "retries": 5, - "write_timeout": 60000, - "tags": [ - "[masked] is an external host. I like [masked]!", - "foo:foo", - "baz:[masked]", - "another:[masked]", - "bar:[masked]" - ] - }, - "old": { - "connect_timeout": 60000, - "enabled": true, - "host": "[masked]", - "id": "9ecf5708-f2f4-444e-a4c7-fcd3a57f9a6d", - "name": "svc1", - "port": 80, - "protocol": "http", - "read_timeout": 60000, - "retries": 5, - "write_timeout": 60000 - } - } - } - ], - "deleting": [] - }, - "summary": { - "creating": 1, - "updating": 1, - "deleting": 0, - "total": 2 - }, - "warnings": [], - "errors": [] -} - -` - - expectedOutputUnMaskedJSON30x = `{ - "changes": { - "creating": [ - { - "name": "rate-limiting (global)", - "kind": "plugin", - "body": { - "new": { - "id": "a1368a28-cb5c-4eee-86d8-03a6bdf94b5e", - "name": "rate-limiting", - "config": { - "day": null, - "fault_tolerant": true, - "header_name": null, - "hide_client_headers": false, - "hour": null, - "limit_by": "consumer", - "minute": 123, - "month": null, - "path": null, - "policy": "local", - "redis_database": 0, - "redis_host": null, - "redis_password": null, - "redis_port": 6379, - "redis_server_name": null, - "redis_ssl": false, - "redis_ssl_verify": false, - "redis_timeout": 2000, - "redis_username": null, - "second": null, - "year": null - }, - "enabled": true, - "protocols": [ - "grpc", - "grpcs", - "http", - "https" - ] - }, - "old": null - } - } - ], - "updating": [ - { - "name": "svc1", - "kind": "service", - "body": { - "new": { - "connect_timeout": 60000, - "enabled": true, - "host": "mockbin.org", - "id": "9ecf5708-f2f4-444e-a4c7-fcd3a57f9a6d", - "name": "svc1", - "port": 80, - "protocol": "http", - "read_timeout": 60000, - "retries": 5, - "write_timeout": 60000, - "tags": [ - "test" - ] - }, - "old": { - "connect_timeout": 60000, - "enabled": true, - "host": "mockbin.org", - "id": "9ecf5708-f2f4-444e-a4c7-fcd3a57f9a6d", - "name": "svc1", - "port": 80, - "protocol": "http", - "read_timeout": 60000, - "retries": 5, - "write_timeout": 60000 - } - } - } - ], - "deleting": [] - }, - "summary": { - "creating": 1, - "updating": 1, - "deleting": 0, - "total": 2 - }, - "warnings": [], - "errors": [] -} - -` - - expectedOutputMaskedJSON30x = `{ - "changes": { - "creating": [ - { - "name": "rate-limiting (global)", - "kind": "plugin", - "body": { - "new": { - "id": "a1368a28-cb5c-4eee-86d8-03a6bdf94b5e", - "name": "rate-limiting", - "config": { - "day": null, - "fault_tolerant": true, - "header_name": null, - "hide_client_headers": false, - "hour": null, - "limit_by": "consumer", - "minute": 123, - "month": null, - "path": null, - "policy": "local", - "redis_database": 0, - "redis_host": null, - "redis_password": null, - "redis_port": 6379, - "redis_server_name": null, - "redis_ssl": false, - "redis_ssl_verify": false, - "redis_timeout": 2000, - "redis_username": null, - "second": null, - "year": null - }, - "enabled": true, - "protocols": [ - "grpc", - "grpcs", - "http", - "https" - ] - }, - "old": null - } - } - ], - "updating": [ - { - "name": "svc1", - "kind": "service", - "body": { - "new": { - "connect_timeout": 60000, - "enabled": true, - "host": "[masked]", - "id": "9ecf5708-f2f4-444e-a4c7-fcd3a57f9a6d", - "name": "svc1", - "port": 80, - "protocol": "http", - "read_timeout": 60000, - "retries": 5, - "write_timeout": 60000, - "tags": [ - "[masked] is an external host. I like [masked]!", - "foo:foo", - "baz:[masked]", - "another:[masked]", - "bar:[masked]" - ] - }, - "old": { - "connect_timeout": 60000, - "enabled": true, - "host": "[masked]", - "id": "9ecf5708-f2f4-444e-a4c7-fcd3a57f9a6d", - "name": "svc1", - "port": 80, - "protocol": "http", - "read_timeout": 60000, - "retries": 5, - "write_timeout": 60000 - } - } - } - ], - "deleting": [] - }, - "summary": { - "creating": 1, - "updating": 1, - "deleting": 0, - "total": 2 - }, - "warnings": [], - "errors": [] -} - -` ) -// test scope: -// - 1.x -// - 2.x -func Test_Diff_Workspace_OlderThan3x(t *testing.T) { - tests := []struct { - name string - stateFile string - expectedState utils.KongRawState - }{ - { - name: "diff with not existent workspace doesn't error out", - stateFile: "testdata/diff/001-not-existing-workspace/kong.yaml", - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - runWhen(t, "kong", "<3.0.0") - setup(t) - - out, err := testSync(context.Background(), []string{tc.stateFile}, true) - assert.NoError(t, err) - assert.Equal(t, expectedDiff, out) - }) - } -} - -// test scope: -// - 3.x -func Test_Diff_Workspace_NewerThan3x(t *testing.T) { - tests := []struct { - name string - stateFile string - expectedState utils.KongRawState - }{ - { - name: "diff with not existent workspace doesn't error out", - stateFile: "testdata/diff/001-not-existing-workspace/kong3x.yaml", - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - runWhen(t, "kong", ">=3.0.0") - setup(t) - - out, err := testSync(context.Background(), []string{tc.stateFile}, true) - assert.NoError(t, err) - assert.Equal(t, expectedDiff, out) - }) - } -} - // test scope: // - 2.8.0 func Test_Diff_Masked_OlderThan3x(t *testing.T) { @@ -542,26 +86,17 @@ func Test_Diff_Masked_OlderThan3x(t *testing.T) { // initialize state assert.NoError(t, sync(tc.initialStateFile)) - out, err := testSync(context.Background(), []string{tc.stateFile}, true) + out, err := testSync(context.Background(), []string{tc.stateFile}, false, true) assert.NoError(t, err) - assert.Equal(t, expectedDiff, out) - }) - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - for k, v := range tc.envVars { - t.Setenv(k, v) - } - runWhen(t, "kong", "==2.8.0") - setup(t) - - // initialize state - assert.NoError(t, sync(tc.initialStateFile)) - - out, err := testSync(context.Background(), []string{tc.stateFile}, true) assert.NoError(t, err) - assert.Equal(t, expectedDiff, out) + assert.Equal(t, int32(1), out.Stats.CreateOps.Count()) + assert.Equal(t, int32(1), out.Stats.UpdateOps.Count()) + assert.Equal(t, "rate-limiting (global)", out.Changes.Creating[0].Name) + assert.Equal(t, "plugin", out.Changes.Creating[0].Kind) + assert.Equal(t, "svc1", out.Changes.Updating[0].Name) + assert.Equal(t, "service", out.Changes.Updating[0].Kind) + assert.NotEmpty(t, out.Changes.Updating[0].Diff) + assert.Equal(t, expectedOutputMasked, out.Changes.Updating[0].Diff) }) } } @@ -594,41 +129,17 @@ func Test_Diff_Masked_NewerThan3x(t *testing.T) { // initialize state assert.NoError(t, sync(tc.initialStateFile)) - out, err := testSync(context.Background(), []string{tc.stateFile}, true) + out, err := testSync(context.Background(), []string{tc.stateFile}, false, true) assert.NoError(t, err) - assert.Equal(t, expectedDiff, out) - }) - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - for k, v := range tc.envVars { - t.Setenv(k, v) - } - runWhen(t, "kong", ">=3.0.0 <3.1.0") - setup(t) - - // initialize state - assert.NoError(t, sync(tc.initialStateFile)) - - out, err := testSync(context.Background(), []string{tc.stateFile}, true) assert.NoError(t, err) - assert.Equal(t, expectedDiff, out) - }) - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - for k, v := range tc.envVars { - t.Setenv(k, v) - } - runWhen(t, "kong", ">=3.1.0 <3.4.0") - setup(t) - - // initialize state - assert.NoError(t, sync(tc.initialStateFile)) - - out, err := testSync(context.Background(), []string{tc.stateFile}, true) - assert.NoError(t, err) - assert.Equal(t, expectedDiff, out) + assert.Equal(t, int32(1), out.Stats.CreateOps.Count()) + assert.Equal(t, int32(1), out.Stats.UpdateOps.Count()) + assert.Equal(t, "rate-limiting (global)", out.Changes.Creating[0].Name) + assert.Equal(t, "plugin", out.Changes.Creating[0].Kind) + assert.Equal(t, "svc1", out.Changes.Updating[0].Name) + assert.Equal(t, "service", out.Changes.Updating[0].Kind) + assert.NotEmpty(t, out.Changes.Updating[0].Diff) + assert.Equal(t, expectedOutputMasked, out.Changes.Updating[0].Diff) }) } } @@ -661,25 +172,17 @@ func Test_Diff_Unmasked_OlderThan3x(t *testing.T) { // initialize state assert.NoError(t, sync(tc.initialStateFile)) - out, err := testSync(context.Background(), []string{tc.stateFile}, true) + out, err := testSync(context.Background(), []string{tc.stateFile}, true, true) assert.NoError(t, err) - assert.Equal(t, expectedDiff, out) - }) - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - for k, v := range tc.envVars { - t.Setenv(k, v) - } - runWhen(t, "kong", "==2.8.0") - setup(t) - - // initialize state - assert.NoError(t, sync(tc.initialStateFile)) - - out, err := testSync(context.Background(), []string{tc.stateFile}, true) assert.NoError(t, err) - assert.Equal(t, expectedDiff, out) + assert.Equal(t, int32(1), out.Stats.CreateOps.Count()) + assert.Equal(t, int32(1), out.Stats.UpdateOps.Count()) + assert.Equal(t, "rate-limiting (global)", out.Changes.Creating[0].Name) + assert.Equal(t, "plugin", out.Changes.Creating[0].Kind) + assert.Equal(t, "svc1", out.Changes.Updating[0].Name) + assert.Equal(t, "service", out.Changes.Updating[0].Kind) + assert.NotEmpty(t, out.Changes.Updating[0].Diff) + assert.Equal(t, expectedOutputUnMasked, out.Changes.Updating[0].Diff) }) } } @@ -712,41 +215,16 @@ func Test_Diff_Unmasked_NewerThan3x(t *testing.T) { // initialize state assert.NoError(t, sync(tc.initialStateFile)) - out, err := testSync(context.Background(), []string{tc.stateFile}, true) - assert.NoError(t, err) - assert.Equal(t, expectedDiff, out) - }) - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - for k, v := range tc.envVars { - t.Setenv(k, v) - } - runWhen(t, "kong", ">=3.0.0 <3.1.0") - setup(t) - - // initialize state - assert.NoError(t, sync(tc.initialStateFile)) - - out, err := testSync(context.Background(), []string{tc.stateFile}, true) - assert.NoError(t, err) - assert.Equal(t, expectedDiff, out) - }) - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - for k, v := range tc.envVars { - t.Setenv(k, v) - } - runWhen(t, "kong", ">=3.1.0 <3.4.0") - setup(t) - - // initialize state - assert.NoError(t, sync(tc.initialStateFile)) - - out, err := testSync(context.Background(), []string{tc.stateFile}, true) + out, err := testSync(context.Background(), []string{tc.stateFile}, true, true) assert.NoError(t, err) - assert.Equal(t, expectedDiff, out) + assert.Equal(t, int32(1), out.Stats.CreateOps.Count()) + assert.Equal(t, int32(1), out.Stats.UpdateOps.Count()) + assert.Equal(t, "rate-limiting (global)", out.Changes.Creating[0].Name) + assert.Equal(t, "plugin", out.Changes.Creating[0].Kind) + assert.Equal(t, "svc1", out.Changes.Updating[0].Name) + assert.Equal(t, "service", out.Changes.Updating[0].Kind) + assert.NotEmpty(t, out.Changes.Updating[0].Diff) + assert.Equal(t, expectedOutputUnMasked, out.Changes.Updating[0].Diff) }) } } diff --git a/tests/integration/test_utils.go b/tests/integration/test_utils.go index b51d16b..2fe98af 100644 --- a/tests/integration/test_utils.go +++ b/tests/integration/test_utils.go @@ -284,7 +284,7 @@ type syncOut struct { // testSync is a stripped-down version of deck's cmd.syncMain for testing changeset expectations. It removes support // for JSON output, skipping resources, Konnect, workspaces, selector tags, and resource type filtering. -func testSync(ctx context.Context, filenames []string, dry bool) (syncOut, error) { +func testSync(ctx context.Context, filenames []string, nomask, dry bool) (syncOut, error) { enableJSONOutput := false targetContent, err := file.GetContentFromFiles(filenames, false) if err != nil { @@ -345,15 +345,15 @@ func testSync(ctx context.Context, filenames []string, dry bool) (syncOut, error CurrentState: currentState, TargetState: targetState, KongClient: kongClient, - StageDelaySec: 2, - NoMaskValues: false, + StageDelaySec: 0, + NoMaskValues: nomask, IsKonnect: false, }) if err != nil { return syncOut{}, err } - stats, syncErrs, changes := syncer.Solve(ctx, 5, dry, enableJSONOutput) + stats, syncErrs, changes := syncer.Solve(ctx, 1, dry, enableJSONOutput) return syncOut{Stats: stats, Errors: syncErrs, Changes: changes}, nil } func sync(kongFile string, opts ...string) error { From 5c8529708b0ce84713ac1a41c70c634e0b24d879 Mon Sep 17 00:00:00 2001 From: Travis Raines <571832+rainest@users.noreply.github.com> Date: Fri, 15 Dec 2023 15:37:03 -0800 Subject: [PATCH 11/13] chore: fix up go mod --- go.mod | 1 + go.sum | 2 ++ pkg/utils/utils.go | 6 ++++++ 3 files changed, 9 insertions(+) diff --git a/go.mod b/go.mod index 352b743..0853ae8 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ replace github.com/yudai/gojsondiff v1.0.0 => github.com/Kong/gojsondiff v1.3.0 require ( dario.cat/mergo v1.0.0 github.com/Kong/gojsondiff v1.3.2 + github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/alecthomas/jsonschema v0.0.0-20191017121752-4bb6e3fae4f2 github.com/blang/semver/v4 v4.0.0 github.com/cenkalti/backoff/v4 v4.3.0 diff --git a/go.sum b/go.sum index def01fe..3146482 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,8 @@ github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJ github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/adrg/strutil v0.2.3 h1:WZVn3ItPBovFmP4wMHHVXUr8luRaHrbyIuLlHt32GZQ= github.com/adrg/strutil v0.2.3/go.mod h1:+SNxbiH6t+O+5SZqIj5n/9i5yUjR+S3XXVrjEcN2mxg= github.com/alecthomas/jsonschema v0.0.0-20191017121752-4bb6e3fae4f2 h1:swGeCLPiUQ647AIRnFxnAHdzlg6IPpmU6QdkOPZINt8= diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 89cdbfe..af25bb6 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -249,3 +249,9 @@ func FormatRouteRegexWarning(unsupportedRoutes []string) string { unsupportedRoutesLen, strings.Join(unsupportedRoutes, "\n"), ) } + +// TODO stub for backwards compatibility pending deck repo changes. Should be removed once deck uses FormatRouteRegexWarning + +func PrintRouteRegexWarning(unsupportedRoutes []string) { + return +} From 4ab40bf1f7623b484d970fcd9d36ae4b2490d9a3 Mon Sep 17 00:00:00 2001 From: Travis Raines <571832+rainest@users.noreply.github.com> Date: Fri, 29 Mar 2024 16:25:10 -0700 Subject: [PATCH 12/13] chore: update diff lib comments --- pkg/diff/diff.go | 33 +++++++++------------------------ pkg/file/builder.go | 2 +- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/pkg/diff/diff.go b/pkg/diff/diff.go index dfc7e55..ac9535b 100644 --- a/pkg/diff/diff.go +++ b/pkg/diff/diff.go @@ -183,13 +183,6 @@ func (sc *Syncer) init() error { sc.processor.MustRegister(crud.Kind(entityType), entity.CRUDActions()) sc.entityDiffers[entityType] = entity.Differ() } - - sc.eventLog = EntityChanges{ - Creating: []EntityState{}, - Updating: []EntityState{}, - Deleting: []EntityState{}, - } - return nil } @@ -425,7 +418,6 @@ func (sc *Syncer) Run(ctx context.Context, parallelism int, action Do) []error { // Do is the worker function to sync the diff type Do func(a crud.Event) (crud.Arg, error) -// NOTE TRC part of the return path func (sc *Syncer) eventLoop(ctx context.Context, action Do) error { for event := range sc.eventChan { // Stop if program is terminated @@ -444,7 +436,6 @@ func (sc *Syncer) eventLoop(ctx context.Context, action Do) error { return nil } -// NOTE TRC part of the return path. this actually runs the func (sc *Syncer) handleEvent(ctx context.Context, action Do, event crud.Event) error { err := backoff.Retry(func() error { res, err := action(event) @@ -509,13 +500,13 @@ func generateDiffString(e crud.Event, isDelete bool, noMaskValues bool) (string, return diffString, err } -// NOTE TRC Solve is the entry point for both the local and konnect sync commands -// those command functions already output other text fwiw -// although they can iterate over a returned op set and print info for each, this does -// introduce a delay in output. Solve() currently prints each action as it takes them, -// whereas the returned set would be printed in a batch at the end. unsure if this should -// matter in practice, but probably not. we could introduce an event channel and separate -// goroutine to handle synch output, but probably not worth it +// Solve originally printed event actions as it processed them. https://github.com/Kong/go-database-reconciler/pull/30 +// refactored these and other direct prints out of this library in favor of returning an event set to the caller +// (the "sync" command in deck's case and the DB update strategy in KIC's case), leaving it up to the caller whether +// or not to print them. This change means that the event set is not available to print until it is complete, however, +// and it no longer can serve as a de facto progress bar. If we want to restore that UX or hit changesets large enough +// where holding events in memory to return is a performance concern, we'd need to expose a channel that can allow +// clients to perform streaming post-processing of events. // Solve generates a diff and walks the graph. func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOut bool) (Stats, @@ -537,9 +528,9 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOu } } - // NOTE TRC the length makes it confusing to read, but the code below _isn't being run here_, it's an anon func + // The length makes it confusing to read, but the code below _isn't being run here_, it's an anon func // arg to Run(), which parallelizes it. However, because it's defined in Solve()'s scope, the output created above - // is available in aggregate and contains most of the content we need already + // is available in aggregate and contains most of the content we need already. errs := sc.Run(ctx, parallelism, func(e crud.Event) (crud.Arg, error) { var err error var result crud.Arg @@ -551,17 +542,13 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOu } item := EntityState{ Body: objDiff, - // NOTE TRC currently used directly Name: c.Console(), - // NOTE TRC current prints use the kind directly, but it doesn't matter, it's just a string alias anyway Kind: string(e.Kind), } - // NOTE TRC currently we emit lines here, need to collect objects instead switch e.Op { case crud.Create: sc.LogCreate(item) case crud.Update: - // TODO TRC this is not currently available in the item EntityState diffString, err := generateDiffString(e, false, sc.noMaskValues) if err != nil { return nil, err @@ -596,8 +583,6 @@ func (sc *Syncer) Solve(ctx context.Context, parallelism int, dry bool, isJSONOu // record operation in both: diff and sync commands recordOp(e.Op) - // TODO TRC our existing return is a complete object and error. probably need to return some sort of processed - // event struct return result, nil }) return stats, errs, sc.GetEventLog() diff --git a/pkg/file/builder.go b/pkg/file/builder.go index 8423ad3..044cf30 100644 --- a/pkg/file/builder.go +++ b/pkg/file/builder.go @@ -18,7 +18,7 @@ import ( const ( ratelimitingAdvancedPluginName = "rate-limiting-advanced" basicAuthPasswordWarning = "Warning: import/export of basic-auth" + - "credentials using decK doesn't work due to hashing of passwords in Kong." + " credentials using decK doesn't work due to hashing of passwords in Kong." ) type stateBuilder struct { From 96a3bf02eeedea911991eb0cdbe3c36e0fffbb66 Mon Sep 17 00:00:00 2001 From: Travis Raines <571832+rainest@users.noreply.github.com> Date: Fri, 29 Mar 2024 17:13:47 -0700 Subject: [PATCH 13/13] chore: add TODO --- pkg/diff/diff.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/diff/diff.go b/pkg/diff/diff.go index ac9535b..39eab52 100644 --- a/pkg/diff/diff.go +++ b/pkg/diff/diff.go @@ -32,6 +32,12 @@ type Summary struct { Total int32 `json:"total"` } +// TODO https://github.com/Kong/go-database-reconciler/issues/22 +// JSONOutputObject is defined here but only used in deck currently, which has the actual code to build it. It may make +// sense to extract this into deck, though it may also make sense to move the build/format functions into here, as +// a generic utility for formatting entity change info into structured text, even if GDR doesn't actually print that +// text. + type JSONOutputObject struct { Changes EntityChanges `json:"changes"` Summary Summary `json:"summary"`