diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9f7f5b43c..ab89020dd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,88 +12,79 @@ jobs: fmt: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-go@v3 with: - go-version: '1.19' + go-version: '1.21' - run: make fmt + vet: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-go@v3 with: - go-version: '1.19' + go-version: '1.21' - run: make vet + build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-go@v3 with: - go-version: '1.19' + go-version: '1.21' - run: make cmd - build-image: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - run: make podman-build test-unit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-go@v3 with: - go-version: '1.19' + go-version: '1.21' - run: make test test-api: + needs: + - fmt + - vet + - build + - test-unit runs-on: ubuntu-latest + env: + DISCONNECTED: 1 + HUB_BASE_URL: http://localhost:8080 + DB_PATH: /tmp/hub.db steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-go@v3 with: - go-version: '1.19' + go-version: '1.21' - run: | - make vet - DISCONNECTED=1 make run & + rm -f $DB_PATH + make run & sleep 15 # probably a dirty solution - DISCONNECTED=1 HUB_BASE_URL=http://localhost:8080 make test-api - DISCONNECTED=1 HUB_BASE_URL=http://localhost:8080 make test-api # Intentionaly run 2x to catch data left in Hub DB. + make test-api - test-e2e: + build-image: + needs: + - fmt + - vet + - build + - test-unit + - test-api runs-on: ubuntu-latest + env: + IMG: ttl.sh/konveyor-hub-${{ github.sha }}:2h steps: - - uses: actions/checkout@v3 - - name: start-minikube - uses: konveyor/tackle2-operator/.github/actions/start-minikube@main - - name: Build image in minikube - run: | - export SHELL=/bin/bash - eval $(minikube -p minikube docker-env) - make docker-build - - name: install-tackle - uses: konveyor/tackle2-operator/.github/actions/install-tackle@main - with: - tackle-hub-image: tackle2-hub:latest - tackle-image-pull-policy: IfNotPresent - - - name: save image - run: | - IMG=quay.io/konveyor/tackle2-hub:latest make docker-build - docker save -o /tmp/tackle2-hub.tar quay.io/konveyor/tackle2-hub:latest - - - name: Upload image as artifact - uses: actions/upload-artifact@v3 - with: - name: tackle2-hub - path: /tmp/tackle2-hub.tar - retention-days: 1 + - uses: actions/checkout@v4 + - run: make docker-build + - run: docker push ${IMG} test-integration: - needs: test-e2e - uses: konveyor/ci/.github/workflows/global-ci.yml@main + needs: build-image + uses: konveyor/ci/.github/workflows/global-ci-bundle.yml@main with: - component_name: tackle2-hub + tackle_hub: ttl.sh/konveyor-hub-${{ github.sha }}:2h api_hub_tests_ref: ${{ github.ref }} diff --git a/.github/workflows/march-image-build-push.yml b/.github/workflows/march-image-build-push.yml index 8729d4cda..e70724b8f 100644 --- a/.github/workflows/march-image-build-push.yml +++ b/.github/workflows/march-image-build-push.yml @@ -15,19 +15,12 @@ concurrency: jobs: push-quay: - name: Build and Push Manifest - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - steps: - - name: Checkout Push to Registry action - uses: konveyor/release-tools/build-push-quay@main - with: - architectures: "amd64, arm64" - containerfile: "./Dockerfile" - image_name: "tackle2-hub" - image_namespace: "konveyor" - image_registry: "quay.io" - quay_publish_robot: ${{ secrets.QUAY_PUBLISH_ROBOT }} - quay_publish_token: ${{ secrets.QUAY_PUBLISH_TOKEN }} - ref: ${{ github.ref }} + uses: konveyor/release-tools/.github/workflows/build-push-images.yaml@main + with: + registry: "quay.io/konveyor" + image_name: "tackle2-hub" + containerfile: "./Dockerfile" + architectures: '[ "amd64", "arm64" ]' + secrets: + registry_username: ${{ secrets.QUAY_PUBLISH_ROBOT }} + registry_password: ${{ secrets.QUAY_PUBLISH_TOKEN }} diff --git a/.github/workflows/test-nightly.yml b/.github/workflows/test-nightly.yml index 60e81cf83..90350c819 100644 --- a/.github/workflows/test-nightly.yml +++ b/.github/workflows/test-nightly.yml @@ -11,7 +11,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v3 with: - go-version: '1.19' + go-version: '1.21' - run: make test test-api: @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v3 with: - go-version: '1.19' + go-version: '1.21' - run: | make vet DISCONNECTED=1 make run & diff --git a/Makefile b/Makefile index cc27ae466..77de6b71d 100644 --- a/Makefile +++ b/Makefile @@ -84,7 +84,7 @@ $(CONTROLLERGEN): # Ensure goimports installed. $(GOIMPORTS): - go install golang.org/x/tools/cmd/goimports@latest + go install golang.org/x/tools/cmd/goimports@v0.24 # Build SAMPLE ADDON addon: fmt vet @@ -141,9 +141,12 @@ endif test: go test -count=1 -v $(shell go list ./... | grep -v "hub/test") +test-db: + go test -count=1 -timeout=6h -v ./database... + # Run Hub REST API tests. test-api: - HUB_BASE_URL=$(HUB_BASE_URL) go test -count=1 -p=1 -v ./test/api/... + HUB_BASE_URL=$(HUB_BASE_URL) go test -count=1 -p=1 -v -failfast ./test/api/... # Run Hub test suite. test-all: test-unit test-api diff --git a/api/analysis.go b/api/analysis.go index eb115113e..ed885c082 100644 --- a/api/analysis.go +++ b/api/analysis.go @@ -1,12 +1,15 @@ package api import ( + "bufio" "bytes" "encoding/json" "errors" + "fmt" "io" "net/http" "os" + "regexp" "sort" "strconv" "strings" @@ -53,9 +56,15 @@ const ( AppAnalysisIssuesRoot = AppAnalysisRoot + "/issues" ) +// Manifest markers. +// The GS=\x1D (group separator). const ( - IssueField = "issues" - DepField = "dependencies" + BeginMainMarker = "\x1DBEGIN-MAIN\x1D" + EndMainMarker = "\x1DEND-MAIN\x1D" + BeginIssuesMarker = "\x1DBEGIN-ISSUES\x1D" + EndIssuesMarker = "\x1DEND-ISSUES\x1D" + BeginDepsMarker = "\x1DBEGIN-DEPS\x1D" + EndDepsMarker = "\x1DEND-DEPS\x1D" ) // AnalysisHandler handles analysis resource routes. @@ -315,9 +324,20 @@ func (h AnalysisHandler) AppList(ctx *gin.Context) { // @summary Create an analysis. // @description Create an analysis. // @description Form fields: -// @description - file: file that contains the api.Analysis resource. -// @description - issues: file that multiple api.Issue resources. -// @description - dependencies: file that multiple api.TechDependency resources. +// @description file: A manifest file that contains 3 sections +// @description containing documents delimited by markers. +// @description The manifest must contain ALL markers even when sections are empty. +// @description Note: `^]` = `\x1D` = GS (group separator). +// @description Section markers: +// @description ^]BEGIN-MAIN^] +// @description ^]END-MAIN^] +// @description ^]BEGIN-ISSUES^] +// @description ^]END-ISSUES^] +// @description ^]BEGIN-DEPS^] +// @description ^]END-DEPS^] +// @description The encoding must be: +// @description - application/json +// @description - application/x-yaml // @tags analyses // @produce json // @success 201 {object} api.Analysis @@ -337,32 +357,40 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) { return } } + db := h.DB(ctx) // - // Analysis - input, err := ctx.FormFile(FileField) + // Manifest + fh := FileHandler{} + name := fmt.Sprintf("app.%d.manifest", id) + file, err := fh.create(ctx, name) if err != nil { - err = &BadRequestError{err.Error()} _ = ctx.Error(err) return } - reader, err := input.Open() + defer func() { + err = fh.delete(ctx, file) + if err != nil { + _ = ctx.Error(err) + } + }() + reader := &ManifestReader{} + f, err := reader.open(file.Path, BeginMainMarker, EndMainMarker) if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) return } defer func() { - _ = reader.Close() + _ = f.Close() }() - encoding := input.Header.Get(ContentType) - d, err := h.Decoder(ctx, encoding, reader) + d, err := h.Decoder(ctx, file.Encoding, reader) if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) return } - r := Analysis{} - err = d.Decode(&r) + r := &Analysis{} + err = d.Decode(r) if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) @@ -371,7 +399,6 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) { analysis := r.Model() analysis.ApplicationID = id analysis.CreateUser = h.BaseHandler.CurrentUser(ctx) - db := h.DB(ctx) db.Logger = db.Logger.LogMode(logger.Error) err = db.Create(analysis).Error if err != nil { @@ -380,23 +407,17 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) { } // // Issues - input, err = ctx.FormFile(IssueField) - if err != nil { - err = &BadRequestError{err.Error()} - _ = ctx.Error(err) - return - } - reader, err = input.Open() + reader = &ManifestReader{} + f, err = reader.open(file.Path, BeginIssuesMarker, EndIssuesMarker) if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) return } defer func() { - _ = reader.Close() + _ = f.Close() }() - encoding = input.Header.Get(ContentType) - d, err = h.Decoder(ctx, encoding, reader) + d, err = h.Decoder(ctx, file.Encoding, reader) if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) @@ -425,23 +446,17 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) { } // // Dependencies - input, err = ctx.FormFile(DepField) - if err != nil { - err = &BadRequestError{err.Error()} - _ = ctx.Error(err) - return - } - reader, err = input.Open() + reader = &ManifestReader{} + f, err = reader.open(file.Path, BeginDepsMarker, EndDepsMarker) if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) return } defer func() { - _ = reader.Close() + _ = f.Close() }() - encoding = input.Header.Get(ContentType) - d, err = h.Decoder(ctx, encoding, reader) + d, err = h.Decoder(ctx, file.Encoding, reader) if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) @@ -1060,11 +1075,9 @@ func (h AnalysisHandler) RuleReports(ctx *gin.Context) { Name: m.Name, } resources = append(resources, r) - if m.Labels != nil { - _ = json.Unmarshal(m.Labels, &r.Labels) - } - if m.Links != nil { - _ = json.Unmarshal(m.Links, &r.Links) + r.Labels = m.Labels + for _, l := range m.Links { + r.Links = append(r.Links, Link(l)) } r.Effort += m.Effort } @@ -1197,11 +1210,9 @@ func (h AnalysisHandler) AppIssueReports(ctx *gin.Context) { ID: m.ID, } resources = append(resources, r) - if m.Labels != nil { - _ = json.Unmarshal(m.Labels, &r.Labels) - } - if m.Links != nil { - _ = json.Unmarshal(m.Links, &r.Links) + r.Labels = m.Labels + for _, l := range m.Links { + r.Links = append(r.Links, Link(l)) } r.Effort += m.Effort } @@ -1722,13 +1733,9 @@ func (h AnalysisHandler) DepReports(ctx *gin.Context) { Name: m.Name, Applications: m.Applications, } - if m.Labels != nil { - var aggregated []string - _ = json.Unmarshal(m.Labels, &aggregated) - for _, s := range aggregated { - if s != "" { - r.Labels = append(r.Labels, s) - } + for _, s := range m.Labels { + if s != "" { + r.Labels = append(r.Labels, s) } } resources = append(resources, r) @@ -2082,7 +2089,7 @@ func (h *AnalysisHandler) archive(ctx *gin.Context, q *gorm.DB) (err error) { db = db.Where("n.IssueID = i.ID") db = db.Where("i.AnalysisID", m.ID) db = db.Group("i.ID") - summary := []ArchivedIssue{} + summary := []model.ArchivedIssue{} err = db.Scan(&summary).Error if err != nil { return @@ -2091,8 +2098,8 @@ func (h *AnalysisHandler) archive(ctx *gin.Context, q *gorm.DB) (err error) { db = db.Model(m) db = db.Omit(clause.Associations) m.Archived = true - m.Summary, _ = json.Marshal(summary) - err = db.Updates(h.fields(&m)).Error + m.Summary = summary + err = db.Save(&m).Error if err != nil { return } @@ -2155,7 +2162,7 @@ type Issue struct { Effort int `json:"effort,omitempty" yaml:",omitempty"` Incidents []Incident `json:"incidents,omitempty" yaml:",omitempty"` Links []Link `json:"links,omitempty" yaml:",omitempty"` - Facts FactMap `json:"facts,omitempty" yaml:",omitempty"` + Facts Map `json:"facts,omitempty" yaml:",omitempty"` Labels []string `json:"labels"` } @@ -2176,15 +2183,11 @@ func (r *Issue) With(m *model.Issue) { r.Incidents, n) } - if m.Links != nil { - _ = json.Unmarshal(m.Links, &r.Links) - } - if m.Facts != nil { - _ = json.Unmarshal(m.Facts, &r.Facts) - } - if m.Labels != nil { - _ = json.Unmarshal(m.Labels, &r.Labels) + for _, l := range m.Links { + r.Links = append(r.Links, Link(l)) } + r.Facts = m.Facts + r.Labels = m.Labels r.Effort = m.Effort } @@ -2203,9 +2206,11 @@ func (r *Issue) Model() (m *model.Issue) { m.Incidents, *n) } - m.Links, _ = json.Marshal(r.Links) - m.Facts, _ = json.Marshal(r.Facts) - m.Labels, _ = json.Marshal(r.Labels) + for _, l := range r.Links { + m.Links = append(m.Links, model.Link(l)) + } + m.Facts = r.Facts + m.Labels = r.Labels m.Effort = r.Effort return } @@ -2231,9 +2236,7 @@ func (r *TechDependency) With(m *model.TechDependency) { r.Version = m.Version r.Indirect = m.Indirect r.SHA = m.SHA - if m.Labels != nil { - _ = json.Unmarshal(m.Labels, &r.Labels) - } + r.Labels = m.Labels } // Model builds a model. @@ -2244,7 +2247,7 @@ func (r *TechDependency) Model() (m *model.TechDependency) { m.Version = r.Version m.Provider = r.Provider m.Indirect = r.Indirect - m.Labels, _ = json.Marshal(r.Labels) + m.Labels = r.Labels m.SHA = r.SHA return } @@ -2252,12 +2255,12 @@ func (r *TechDependency) Model() (m *model.TechDependency) { // Incident REST resource. type Incident struct { Resource `yaml:",inline"` - Issue uint `json:"issue"` - File string `json:"file"` - Line int `json:"line"` - Message string `json:"message"` - CodeSnip string `json:"codeSnip" yaml:"codeSnip"` - Facts FactMap `json:"facts"` + Issue uint `json:"issue"` + File string `json:"file"` + Line int `json:"line"` + Message string `json:"message"` + CodeSnip string `json:"codeSnip" yaml:"codeSnip"` + Facts Map `json:"facts"` } // With updates the resource with the model. @@ -2268,9 +2271,7 @@ func (r *Incident) With(m *model.Incident) { r.Line = m.Line r.Message = m.Message r.CodeSnip = m.CodeSnip - if m.Facts != nil { - _ = json.Unmarshal(m.Facts, &r.Facts) - } + r.Facts = m.Facts } // Model builds a model. @@ -2280,7 +2281,7 @@ func (r *Incident) Model() (m *model.Incident) { m.Line = r.Line m.Message = r.Message m.CodeSnip = r.CodeSnip - m.Facts, _ = json.Marshal(r.Facts) + m.Facts = r.Facts return } @@ -2371,9 +2372,6 @@ type DepAppReport struct { } `json:"dependency"` } -// FactMap map. -type FactMap map[string]any - // IssueWriter used to create a file containing issues. type IssueWriter struct { encoder @@ -2877,3 +2875,116 @@ func (r *yamlEncoder) embed(object any) encoder { r.write(s) return r } + +// ManifestReader analysis manifest reader. +// The manifest contains 3 sections containing documents delimited by markers. +// The manifest must contain ALL markers even when sections are empty. +// Note: `^]` = `\x1D` = GS (group separator). +// Section markers: +// +// ^]BEGIN-MAIN^] +// ^]END-MAIN^] +// ^]BEGIN-ISSUES^] +// ^]END-ISSUES^] +// ^]BEGIN-DEPS^] +// ^]END-DEPS^] +type ManifestReader struct { + file *os.File + marker map[string]int64 + begin int64 + end int64 + read int64 +} + +// scan manifest and catalog position of markers. +func (r *ManifestReader) scan(path string) (err error) { + if r.marker != nil { + return + } + r.file, err = os.Open(path) + if err != nil { + return + } + defer func() { + _ = r.file.Close() + }() + pattern, err := regexp.Compile(`^\x1D[A-Z-]+\x1D$`) + if err != nil { + return + } + p := int64(0) + r.marker = make(map[string]int64) + scanner := bufio.NewScanner(r.file) + for scanner.Scan() { + content := scanner.Text() + matched := strings.TrimSpace(content) + if pattern.Match([]byte(matched)) { + r.marker[matched] = p + } + p += int64(len(content)) + p++ + } + + return +} + +// open returns a read delimited by the specified markers. +func (r *ManifestReader) open(path, begin, end string) (reader io.ReadCloser, err error) { + found := false + err = r.scan(path) + if err != nil { + return + } + r.begin, found = r.marker[begin] + if !found { + err = &BadRequestError{ + Reason: fmt.Sprintf("marker: %s not found.", begin), + } + return + } + r.end, found = r.marker[end] + if !found { + err = &BadRequestError{ + Reason: fmt.Sprintf("marker: %s not found.", end), + } + return + } + if r.begin >= r.end { + err = &BadRequestError{ + Reason: fmt.Sprintf("marker: %s must preceed %s.", begin, end), + } + return + } + r.begin += int64(len(begin)) + r.begin++ + r.read = r.end - r.begin + r.file, err = os.Open(path) + if err != nil { + return + } + _, err = r.file.Seek(r.begin, io.SeekStart) + reader = r + return +} + +// Read bytes. +func (r *ManifestReader) Read(b []byte) (n int, err error) { + n, err = r.file.Read(b) + if n == 0 || err != nil { + return + } + if int64(n) > r.read { + n = int(r.read) + } + r.read -= int64(n) + if n < 1 { + err = io.EOF + } + return +} + +// Close the reader. +func (r *ManifestReader) Close() (err error) { + err = r.file.Close() + return +} diff --git a/api/application.go b/api/application.go index 7169d1c2e..6b61e3c6a 100644 --- a/api/application.go +++ b/api/application.go @@ -58,22 +58,22 @@ func (h ApplicationHandler) AddRoutes(e *gin.Engine) { routeGroup.DELETE(ApplicationRoot, h.Delete) // Tags routeGroup = e.Group("/") - routeGroup.Use(Required("applications")) + routeGroup.Use(Required("applications"), Transaction) routeGroup.GET(ApplicationTagsRoot, h.TagList) routeGroup.GET(ApplicationTagsRoot+"/", h.TagList) routeGroup.POST(ApplicationTagsRoot, h.TagAdd) routeGroup.DELETE(ApplicationTagRoot, h.TagDelete) - routeGroup.PUT(ApplicationTagsRoot, h.TagReplace, Transaction) + routeGroup.PUT(ApplicationTagsRoot, h.TagReplace) // Facts routeGroup = e.Group("/") - routeGroup.Use(Required("applications.facts")) + routeGroup.Use(Required("applications.facts"), Transaction) routeGroup.GET(ApplicationFactsRoot, h.FactGet) routeGroup.GET(ApplicationFactsRoot+"/", h.FactGet) routeGroup.POST(ApplicationFactsRoot, h.FactCreate) routeGroup.GET(ApplicationFactRoot, h.FactGet) routeGroup.PUT(ApplicationFactRoot, h.FactPut) routeGroup.DELETE(ApplicationFactRoot, h.FactDelete) - routeGroup.PUT(ApplicationFactsRoot, h.FactPut, Transaction) + routeGroup.PUT(ApplicationFactsRoot, h.FactPut) // Bucket routeGroup = e.Group("/") routeGroup.Use(Required("applications.bucket")) @@ -84,11 +84,11 @@ func (h ApplicationHandler) AddRoutes(e *gin.Engine) { routeGroup.DELETE(AppBucketContentRoot, h.BucketDelete) // Stakeholders routeGroup = e.Group("/") - routeGroup.Use(Required("applications.stakeholders")) + routeGroup.Use(Required("applications.stakeholders"), Transaction) routeGroup.PUT(AppStakeholdersRoot, h.StakeholdersUpdate) // Assessments routeGroup = e.Group("/") - routeGroup.Use(Required("applications.assessments")) + routeGroup.Use(Required("applications.assessments"), Transaction) routeGroup.GET(AppAssessmentsRoot, h.AssessmentList) routeGroup.POST(AppAssessmentsRoot, h.AssessmentCreate) } @@ -210,11 +210,23 @@ func (h ApplicationHandler) Create(ctx *gin.Context) { } m := r.Model() m.CreateUser = h.BaseHandler.CurrentUser(ctx) - result := h.DB(ctx).Omit("Tags").Create(m) + result := h.DB(ctx).Omit(clause.Associations).Create(m) if result.Error != nil { _ = ctx.Error(result.Error) return } + db := h.DB(ctx).Model(m) + err = db.Association("Identities").Replace(m.Identities) + if err != nil { + _ = ctx.Error(err) + return + } + db = h.DB(ctx).Model(m) + err = db.Association("Contributors").Replace(m.Contributors) + if err != nil { + _ = ctx.Error(err) + return + } tags := []model.ApplicationTag{} if len(r.Tags) > 0 { @@ -350,7 +362,7 @@ func (h ApplicationHandler) Update(ctx *gin.Context) { m.UpdateUser = h.BaseHandler.CurrentUser(ctx) db = h.DB(ctx).Model(m) db = db.Omit(clause.Associations, "BucketID") - result = db.Updates(h.fields(m)) + result = db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return @@ -710,12 +722,10 @@ func (h ApplicationHandler) FactList(ctx *gin.Context, key FactKey) { return } - facts := FactMap{} + facts := Map{} for i := range list { fact := &list[i] - var v any - _ = json.Unmarshal(fact.Value, &v) - facts[fact.Key] = v + facts[fact.Key] = fact.Value } h.Respond(ctx, http.StatusOK, facts) } @@ -760,9 +770,7 @@ func (h ApplicationHandler) FactGet(ctx *gin.Context) { return } - var v any - _ = json.Unmarshal(list[0].Value, &v) - h.Respond(ctx, http.StatusOK, v) + h.Respond(ctx, http.StatusOK, list[0].Value) } // FactCreate godoc @@ -834,12 +842,11 @@ func (h ApplicationHandler) FactPut(ctx *gin.Context) { return } - value, _ := json.Marshal(f.Value) m := &model.Fact{ Key: key.Name(), Source: key.Source(), ApplicationID: id, - Value: value, + Value: f.Value, } db := h.DB(ctx) result = db.Save(m) @@ -894,7 +901,7 @@ func (h ApplicationHandler) FactDelete(ctx *gin.Context) { // @param factmap body api.FactMap true "Fact map" func (h ApplicationHandler) FactReplace(ctx *gin.Context, key FactKey) { id := h.pk(ctx) - facts := FactMap{} + facts := Map{} err := h.Bind(ctx, &facts) if err != nil { _ = ctx.Error(err) @@ -1078,11 +1085,21 @@ func (h ApplicationHandler) AssessmentCreate(ctx *gin.Context) { assessment.PrepareForApplication(resolver, application, m) newAssessment = true } - result = h.DB(ctx).Create(m) + result = h.DB(ctx).Omit(clause.Associations).Create(m) if result.Error != nil { _ = ctx.Error(result.Error) return } + err = h.DB(ctx).Model(m).Association("Stakeholders").Replace("Stakeholders", m.Stakeholders) + if err != nil { + _ = ctx.Error(err) + return + } + err = h.DB(ctx).Model(m).Association("StakeholderGroups").Replace("StakeholderGroups", m.StakeholderGroups) + if err != nil { + _ = ctx.Error(err) + return + } if newAssessment { metrics.AssessmentsInitiated.Inc() } @@ -1123,7 +1140,10 @@ func (r *Application) With(m *model.Application, tags []model.ApplicationTag) { r.Bucket = r.refPtr(m.BucketID, m.Bucket) r.Comments = m.Comments r.Binary = m.Binary - _ = json.Unmarshal(m.Repository, &r.Repository) + if m.Repository != (model.Repository{}) { + repo := Repository(m.Repository) + r.Repository = &repo + } if m.Review != nil { ref := &Ref{} ref.With(m.Review.ID, "") @@ -1138,6 +1158,7 @@ func (r *Application) With(m *model.Application, tags []model.ApplicationTag) { r.Identities, ref) } + r.Tags = []TagRef{} for i := range tags { ref := TagRef{} ref.With(tags[i].TagID, tags[i].Tag.Name, tags[i].Source, false) @@ -1223,7 +1244,7 @@ func (r *Application) Model() (m *model.Application) { } m.ID = r.ID if r.Repository != nil { - m.Repository, _ = json.Marshal(r.Repository) + m.Repository = model.Repository(*r.Repository) } if r.BusinessService != nil { m.BusinessServiceID = &r.BusinessService.ID @@ -1284,14 +1305,14 @@ type Fact struct { func (r *Fact) With(m *model.Fact) { r.Key = m.Key r.Source = m.Source - _ = json.Unmarshal(m.Value, &r.Value) + r.Value = m.Value } func (r *Fact) Model() (m *model.Fact) { m = &model.Fact{} m.Key = r.Key m.Source = r.Source - m.Value, _ = json.Marshal(r.Value) + m.Value = r.Value return } diff --git a/api/archetype.go b/api/archetype.go index d62d88a13..dafd141ef 100644 --- a/api/archetype.go +++ b/api/archetype.go @@ -136,12 +136,33 @@ func (h ArchetypeHandler) Create(ctx *gin.Context) { } m := r.Model() m.CreateUser = h.CurrentUser(ctx) - result := h.DB(ctx).Create(m) + result := h.DB(ctx).Omit(clause.Associations).Create(m) if result.Error != nil { _ = ctx.Error(result.Error) return } + err = h.DB(ctx).Model(m).Association("Stakeholders").Replace("Stakeholders", m.Stakeholders) + if err != nil { + _ = ctx.Error(err) + return + } + err = h.DB(ctx).Model(m).Association("StakeholderGroups").Replace("StakeholderGroups", m.StakeholderGroups) + if err != nil { + _ = ctx.Error(err) + return + } + err = h.DB(ctx).Model(m).Association("CriteriaTags").Replace("CriteriaTags", m.CriteriaTags) + if err != nil { + _ = ctx.Error(err) + return + } + err = h.DB(ctx).Model(m).Association("Tags").Replace("Tags", m.Tags) + if err != nil { + _ = ctx.Error(err) + return + } + archetypes := []model.Archetype{} db := h.preLoad(h.DB(ctx), "Tags", "CriteriaTags") result = db.Find(&archetypes) @@ -211,7 +232,7 @@ func (h ArchetypeHandler) Update(ctx *gin.Context) { m.UpdateUser = h.CurrentUser(ctx) db := h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - result := db.Updates(h.fields(m)) + result := db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return @@ -319,11 +340,21 @@ func (h ArchetypeHandler) AssessmentCreate(ctx *gin.Context) { assessment.PrepareForArchetype(resolver, archetype, m) newAssessment = true } - result = h.DB(ctx).Create(m) + result = h.DB(ctx).Omit(clause.Associations).Create(m) if result.Error != nil { _ = ctx.Error(result.Error) return } + err = h.DB(ctx).Model(m).Association("Stakeholders").Replace("Stakeholders", m.Stakeholders) + if err != nil { + _ = ctx.Error(err) + return + } + err = h.DB(ctx).Model(m).Association("StakeholderGroups").Replace("StakeholderGroups", m.StakeholderGroups) + if err != nil { + _ = ctx.Error(err) + return + } if newAssessment { metrics.AssessmentsInitiated.Inc() } diff --git a/api/assessment.go b/api/assessment.go index d024637eb..438c16eb8 100644 --- a/api/assessment.go +++ b/api/assessment.go @@ -1,7 +1,6 @@ package api import ( - "encoding/json" "net/http" "github.com/gin-gonic/gin" @@ -126,7 +125,7 @@ func (h AssessmentHandler) Update(ctx *gin.Context) { m.UpdateUser = h.CurrentUser(ctx) db := h.DB(ctx).Model(m) db = db.Omit(clause.Associations, "Thresholds", "RiskMessages") - result := db.Updates(h.fields(m)) + result := db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return @@ -148,21 +147,25 @@ func (h AssessmentHandler) Update(ctx *gin.Context) { // Assessment REST resource. type Assessment struct { Resource `yaml:",inline"` - Application *Ref `json:"application,omitempty" yaml:",omitempty" binding:"excluded_with=Archetype"` - Archetype *Ref `json:"archetype,omitempty" yaml:",omitempty" binding:"excluded_with=Application"` - Questionnaire Ref `json:"questionnaire" binding:"required"` - Sections []assessment.Section `json:"sections" binding:"dive"` - Stakeholders []Ref `json:"stakeholders"` - StakeholderGroups []Ref `json:"stakeholderGroups" yaml:"stakeholderGroups"` + Application *Ref `json:"application,omitempty" yaml:",omitempty" binding:"excluded_with=Archetype"` + Archetype *Ref `json:"archetype,omitempty" yaml:",omitempty" binding:"excluded_with=Application"` + Questionnaire Ref `json:"questionnaire" binding:"required"` + Sections []Section `json:"sections" binding:"dive"` + Stakeholders []Ref `json:"stakeholders"` + StakeholderGroups []Ref `json:"stakeholderGroups" yaml:"stakeholderGroups"` // read only - Risk string `json:"risk"` - Confidence int `json:"confidence"` - Status string `json:"status"` - Thresholds assessment.Thresholds `json:"thresholds"` - RiskMessages assessment.RiskMessages `json:"riskMessages" yaml:"riskMessages"` - Required bool `json:"required"` + Risk string `json:"risk"` + Confidence int `json:"confidence"` + Status string `json:"status"` + Thresholds Thresholds `json:"thresholds"` + RiskMessages RiskMessages `json:"riskMessages" yaml:"riskMessages"` + Required bool `json:"required"` } +type Section model.Section +type Thresholds model.Thresholds +type RiskMessages model.RiskMessages + // With updates the resource with the model. func (r *Assessment) With(m *model.Assessment) { r.Resource.With(&m.Model) @@ -186,9 +189,12 @@ func (r *Assessment) With(m *model.Assessment) { r.Required = a.Questionnaire.Required r.Risk = a.Risk() r.Confidence = a.Confidence() - r.RiskMessages = a.RiskMessages - r.Thresholds = a.Thresholds - r.Sections = a.Sections + r.RiskMessages = RiskMessages(a.RiskMessages) + r.Thresholds = Thresholds(a.Thresholds) + r.Sections = []Section{} + for _, s := range a.Sections { + r.Sections = append(r.Sections, Section(s)) + } r.Status = a.Status() } @@ -196,8 +202,8 @@ func (r *Assessment) With(m *model.Assessment) { func (r *Assessment) Model() (m *model.Assessment) { m = &model.Assessment{} m.ID = r.ID - if r.Sections != nil { - m.Sections, _ = json.Marshal(r.Sections) + for _, s := range r.Sections { + m.Sections = append(m.Sections, model.Section(s)) } m.QuestionnaireID = r.Questionnaire.ID if r.Archetype != nil { diff --git a/api/base.go b/api/base.go index 602ac3be4..3e4e713a8 100644 --- a/api/base.go +++ b/api/base.go @@ -84,12 +84,6 @@ func (h *BaseHandler) preLoad(db *gorm.DB, fields ...string) (tx *gorm.DB) { return } -// fields builds a map of fields. -func (h *BaseHandler) fields(m any) (mp map[string]any) { - mp = reflect.Fields(m) - return -} - // pk returns the PK (ID) parameter. func (h *BaseHandler) pk(ctx *gin.Context) (id uint) { s := ctx.Param(ID) diff --git a/api/businessservice.go b/api/businessservice.go index 41caf95b5..dabc03c09 100644 --- a/api/businessservice.go +++ b/api/businessservice.go @@ -153,7 +153,7 @@ func (h BusinessServiceHandler) Update(ctx *gin.Context) { m.UpdateUser = h.BaseHandler.CurrentUser(ctx) db := h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - result := db.Updates(h.fields(m)) + result := db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return diff --git a/api/error.go b/api/error.go index 01dc4a4ba..fe79e47d9 100644 --- a/api/error.go +++ b/api/error.go @@ -2,6 +2,7 @@ package api import ( "errors" + "fmt" "net/http" "os" @@ -78,6 +79,22 @@ func (r *Forbidden) Is(err error) (matched bool) { return } +// NotFound reports resource not-found errors. +type NotFound struct { + Resource string + Reason string +} + +func (r *NotFound) Error() string { + return fmt.Sprintf("Resource '%s' not found. %s", r.Resource, r.Reason) +} + +func (r *NotFound) Is(err error) (matched bool) { + var forbidden *Forbidden + matched = errors.As(err, &forbidden) + return +} + // ErrorHandler handles error conditions from lower handlers. func ErrorHandler() gin.HandlerFunc { return func(ctx *gin.Context) { @@ -102,7 +119,8 @@ func ErrorHandler() gin.HandlerFunc { return } - if errors.Is(err, gorm.ErrRecordNotFound) { + if errors.Is(err, gorm.ErrRecordNotFound) || + errors.Is(err, &NotFound{}) { if ctx.Request.Method == http.MethodDelete { rtx.Status(http.StatusNoContent) return diff --git a/api/file.go b/api/file.go index 5e280172f..74fc17aae 100644 --- a/api/file.go +++ b/api/file.go @@ -9,6 +9,7 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/gin-gonic/gin/binding" "github.com/konveyor/tackle2-hub/model" ) @@ -70,52 +71,11 @@ func (h FileHandler) List(ctx *gin.Context) { // @router /files [post] // @param name path string true "File name" func (h FileHandler) Create(ctx *gin.Context) { - var err error - input, err := ctx.FormFile(FileField) + m, err := h.create(ctx, ctx.Param(ID)) if err != nil { - err = &BadRequestError{err.Error()} _ = ctx.Error(err) return } - m := &model.File{} - m.Name = ctx.Param(ID) - m.CreateUser = h.BaseHandler.CurrentUser(ctx) - result := h.DB(ctx).Create(&m) - if result.Error != nil { - _ = ctx.Error(result.Error) - return - } - defer func() { - if err != nil { - h.Status(ctx, http.StatusInternalServerError) - _ = h.DB(ctx).Delete(&m) - return - } - }() - reader, err := input.Open() - if err != nil { - err = &BadRequestError{err.Error()} - _ = ctx.Error(err) - return - } - defer func() { - _ = reader.Close() - }() - writer, err := os.Create(m.Path) - if err != nil { - return - } - defer func() { - _ = writer.Close() - }() - _, err = io.Copy(writer, reader) - if err != nil { - return - } - err = os.Chmod(m.Path, 0666) - if err != nil { - return - } r := File{} r.With(m) h.Respond(ctx, http.StatusCreated, r) @@ -224,20 +184,126 @@ func (h FileHandler) Delete(ctx *gin.Context) { _ = ctx.Error(err) return } - err = os.Remove(m.Path) + err = h.delete(ctx, m) if err != nil { - if !os.IsNotExist(err) { - _ = ctx.Error(err) + _ = ctx.Error(err) + return + } + + h.Status(ctx, http.StatusNoContent) +} + +// create a file. +func (h FileHandler) create(ctx *gin.Context, name string) (m *model.File, err error) { + mode := ctx.ContentType() + switch mode { + case binding.MIMEMultipartPOSTForm: + m, err = h.createMultipart(ctx, name) + case binding.MIMEYAML: + m, err = h.createBody(ctx, name, binding.MIMEYAML) + default: + m, err = h.createBody(ctx, name, binding.MIMEJSON) + } + return +} + +// create a file with multipart form. +func (h FileHandler) createMultipart(ctx *gin.Context, name string) (m *model.File, err error) { + input, err := ctx.FormFile(FileField) + if err != nil { + err = &BadRequestError{err.Error()} + return + } + m = &model.File{} + m.Name = name + m.Encoding = input.Header.Get(ContentType) + m.CreateUser = h.BaseHandler.CurrentUser(ctx) + db := h.DB(ctx) + err = db.Create(&m).Error + if err != nil { + return + } + defer func() { + if err != nil { + h.Status(ctx, http.StatusInternalServerError) + _ = db.Delete(&m) return } + }() + reader, err := input.Open() + if err != nil { + err = &BadRequestError{err.Error()} + return } - err = h.DB(ctx).Delete(m).Error + defer func() { + _ = reader.Close() + }() + writer, err := os.Create(m.Path) if err != nil { - _ = ctx.Error(err) return } + defer func() { + _ = writer.Close() + }() + _, err = io.Copy(writer, reader) + if err != nil { + return + } + err = os.Chmod(m.Path, 0666) + if err != nil { + return + } + return +} - h.Status(ctx, http.StatusNoContent) +// create a file with request body. +func (h FileHandler) createBody(ctx *gin.Context, name, encoding string) (m *model.File, err error) { + m = &model.File{} + m.Name = name + m.Encoding = encoding + m.CreateUser = h.BaseHandler.CurrentUser(ctx) + db := h.DB(ctx) + err = db.Create(&m).Error + if err != nil { + return + } + defer func() { + if err != nil { + h.Status(ctx, http.StatusInternalServerError) + _ = db.Delete(&m) + return + } + }() + reader := ctx.Request.Body + writer, err := os.Create(m.Path) + if err != nil { + return + } + defer func() { + _ = writer.Close() + }() + _, err = io.Copy(writer, reader) + if err != nil { + return + } + err = os.Chmod(m.Path, 0666) + if err != nil { + return + } + return +} + +// delete the specified file. +func (h FileHandler) delete(ctx *gin.Context, m *model.File) (err error) { + err = os.Remove(m.Path) + if err != nil { + if !os.IsNotExist(err) { + return + } + } + db := h.DB(ctx) + err = db.Delete(m).Error + return } // File REST resource. @@ -245,6 +311,7 @@ type File struct { Resource `yaml:",inline"` Name string `json:"name"` Path string `json:"path"` + Encoding string `yaml:"encoding,omitempty"` Expiration *time.Time `json:"expiration,omitempty"` } @@ -253,5 +320,6 @@ func (r *File) With(m *model.File) { r.Resource.With(&m.Model) r.Name = m.Name r.Path = m.Path + r.Encoding = m.Encoding r.Expiration = m.Expiration } diff --git a/api/group.go b/api/group.go index 4e92337b3..daf876519 100644 --- a/api/group.go +++ b/api/group.go @@ -97,11 +97,21 @@ func (h StakeholderGroupHandler) Create(ctx *gin.Context) { } m := r.Model() m.CreateUser = h.BaseHandler.CurrentUser(ctx) - result := h.DB(ctx).Create(m) + result := h.DB(ctx).Omit(clause.Associations).Create(m) if result.Error != nil { _ = ctx.Error(result.Error) return } + err = h.DB(ctx).Model(m).Association("Stakeholders").Replace(m.Stakeholders) + if err != nil { + _ = ctx.Error(err) + return + } + err = h.DB(ctx).Model(m).Association("MigrationWaves").Replace(m.MigrationWaves) + if err != nil { + _ = ctx.Error(err) + return + } r.With(m) h.Respond(ctx, http.StatusCreated, r) @@ -153,7 +163,7 @@ func (h StakeholderGroupHandler) Update(ctx *gin.Context) { m.UpdateUser = h.BaseHandler.CurrentUser(ctx) db := h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - result := db.Updates(h.fields(m)) + result := db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return diff --git a/api/identity.go b/api/identity.go index dbaf606dd..e01e89b08 100644 --- a/api/identity.go +++ b/api/identity.go @@ -34,7 +34,7 @@ func (h IdentityHandler) AddRoutes(e *gin.Engine) { routeGroup.GET(IdentitiesRoot+"/", h.setDecrypted, h.List) routeGroup.POST(IdentitiesRoot, h.Create) routeGroup.GET(IdentityRoot, h.setDecrypted, h.Get) - routeGroup.PUT(IdentityRoot, h.Update, Transaction) + routeGroup.PUT(IdentityRoot, Transaction, h.Update) routeGroup.DELETE(IdentityRoot, h.Delete) } @@ -202,7 +202,7 @@ func (h IdentityHandler) Update(ctx *gin.Context) { m.ID = id m.UpdateUser = h.BaseHandler.CurrentUser(ctx) db := h.DB(ctx).Model(m) - err = db.Updates(h.fields(m)).Error + err = db.Save(m).Error if err != nil { _ = ctx.Error(err) return diff --git a/api/jobfunction.go b/api/jobfunction.go index 82e615956..135dd1cb2 100644 --- a/api/jobfunction.go +++ b/api/jobfunction.go @@ -153,7 +153,7 @@ func (h JobFunctionHandler) Update(ctx *gin.Context) { m.UpdateUser = h.BaseHandler.CurrentUser(ctx) db := h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - result := db.Updates(h.fields(m)) + result := db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return diff --git a/api/migrationwave.go b/api/migrationwave.go index 8651a369e..df75eb7d0 100644 --- a/api/migrationwave.go +++ b/api/migrationwave.go @@ -98,11 +98,27 @@ func (h MigrationWaveHandler) Create(ctx *gin.Context) { } m := r.Model() m.CreateUser = h.CurrentUser(ctx) - result := h.DB(ctx).Create(m) + result := h.DB(ctx).Omit(clause.Associations).Create(m) if result.Error != nil { _ = ctx.Error(result.Error) return } + err = h.DB(ctx).Model(m).Association("Applications").Replace("Applications", m.Applications) + if err != nil { + _ = ctx.Error(err) + return + } + err = h.DB(ctx).Model(m).Association("Stakeholders").Replace("Stakeholders", m.Stakeholders) + if err != nil { + _ = ctx.Error(err) + return + } + err = h.DB(ctx).Model(m).Association("StakeholderGroups").Replace("StakeholderGroups", m.StakeholderGroups) + if err != nil { + _ = ctx.Error(err) + return + } + r.With(m) h.Respond(ctx, http.StatusCreated, r) @@ -130,7 +146,7 @@ func (h MigrationWaveHandler) Update(ctx *gin.Context) { m.UpdateUser = h.CurrentUser(ctx) db := h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - result := db.Updates(h.fields(m)) + result := db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return diff --git a/api/pkg.go b/api/pkg.go index 36f78c052..18efbcb3e 100644 --- a/api/pkg.go +++ b/api/pkg.go @@ -78,6 +78,7 @@ func All() []Handler { &RuleSetHandler{}, &SchemaHandler{}, &SettingHandler{}, + &ServiceHandler{}, &StakeholderHandler{}, &StakeholderGroupHandler{}, &TagHandler{}, diff --git a/api/proxy.go b/api/proxy.go index 72f3dd67b..ccaf1551c 100644 --- a/api/proxy.go +++ b/api/proxy.go @@ -1,7 +1,6 @@ package api import ( - "encoding/json" "net/http" "github.com/gin-gonic/gin" @@ -161,7 +160,7 @@ func (h ProxyHandler) Update(ctx *gin.Context) { m.UpdateUser = h.BaseHandler.CurrentUser(ctx) db := h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - result := db.Updates(h.fields(m)) + result := db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return @@ -189,7 +188,7 @@ func (r *Proxy) With(m *model.Proxy) { r.Host = m.Host r.Port = m.Port r.Identity = r.refPtr(m.IdentityID, m.Identity) - _ = json.Unmarshal(m.Excluded, &r.Excluded) + r.Excluded = m.Excluded if r.Excluded == nil { r.Excluded = []string{} } @@ -205,9 +204,7 @@ func (r *Proxy) Model() (m *model.Proxy) { } m.ID = r.ID m.IdentityID = r.idPtr(r.Identity) - if r.Excluded != nil { - m.Excluded, _ = json.Marshal(r.Excluded) - } + m.Excluded = r.Excluded return } diff --git a/api/questionnaire.go b/api/questionnaire.go index 1dcb8c50d..ba183afe7 100644 --- a/api/questionnaire.go +++ b/api/questionnaire.go @@ -1,11 +1,9 @@ package api import ( - "encoding/json" "net/http" "github.com/gin-gonic/gin" - "github.com/konveyor/tackle2-hub/assessment" "github.com/konveyor/tackle2-hub/model" "gorm.io/gorm/clause" ) @@ -167,19 +165,16 @@ func (h QuestionnaireHandler) Update(ctx *gin.Context) { updated := r.Model() updated.ID = id updated.UpdateUser = h.CurrentUser(ctx) - var fields map[string]any if m.Builtin() { - fields = map[string]any{ - "updateUser": updated.UpdateUser, - "required": updated.Required, - } + m.UpdateUser = updated.UpdateUser + m.Required = updated.Required } else { - fields = h.fields(updated) + m = updated } db = h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - result = db.Updates(fields) + result = db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return @@ -190,13 +185,13 @@ func (h QuestionnaireHandler) Update(ctx *gin.Context) { type Questionnaire struct { Resource `yaml:",inline"` - Name string `json:"name" yaml:"name" binding:"required"` - Description string `json:"description" yaml:"description"` - Required bool `json:"required" yaml:"required"` - Sections []assessment.Section `json:"sections" yaml:"sections" binding:"required,min=1,dive"` - Thresholds assessment.Thresholds `json:"thresholds" yaml:"thresholds" binding:"required"` - RiskMessages assessment.RiskMessages `json:"riskMessages" yaml:"riskMessages" binding:"required"` - Builtin bool `json:"builtin,omitempty" yaml:"builtin,omitempty"` + Name string `json:"name" yaml:"name" binding:"required"` + Description string `json:"description" yaml:"description"` + Required bool `json:"required" yaml:"required"` + Sections []Section `json:"sections" yaml:"sections" binding:"required,min=1,dive"` + Thresholds Thresholds `json:"thresholds" yaml:"thresholds" binding:"required"` + RiskMessages RiskMessages `json:"riskMessages" yaml:"riskMessages" binding:"required"` + Builtin bool `json:"builtin,omitempty" yaml:"builtin,omitempty"` } // With updates the resource with the model. @@ -206,9 +201,12 @@ func (r *Questionnaire) With(m *model.Questionnaire) { r.Description = m.Description r.Required = m.Required r.Builtin = m.Builtin() - _ = json.Unmarshal(m.Sections, &r.Sections) - _ = json.Unmarshal(m.Thresholds, &r.Thresholds) - _ = json.Unmarshal(m.RiskMessages, &r.RiskMessages) + r.Sections = []Section{} + for _, s := range m.Sections { + r.Sections = append(r.Sections, Section(s)) + } + r.Thresholds = Thresholds(m.Thresholds) + r.RiskMessages = RiskMessages(m.RiskMessages) } // Model builds a model. @@ -219,9 +217,11 @@ func (r *Questionnaire) Model() (m *model.Questionnaire) { Required: r.Required, } m.ID = r.ID - m.Sections, _ = json.Marshal(r.Sections) - m.Thresholds, _ = json.Marshal(r.Thresholds) - m.RiskMessages, _ = json.Marshal(r.RiskMessages) + for _, s := range r.Sections { + m.Sections = append(m.Sections, model.Section(s)) + } + m.Thresholds = model.Thresholds(r.Thresholds) + m.RiskMessages = model.RiskMessages(r.RiskMessages) return } diff --git a/api/review.go b/api/review.go index 85fbbdcb8..db4c65222 100644 --- a/api/review.go +++ b/api/review.go @@ -30,7 +30,7 @@ func (h ReviewHandler) AddRoutes(e *gin.Engine) { routeGroup.GET(ReviewRoot, h.Get) routeGroup.PUT(ReviewRoot, h.Update) routeGroup.DELETE(ReviewRoot, h.Delete) - routeGroup.POST(CopyRoot, h.CopyReview) + routeGroup.POST(CopyRoot, h.CopyReview, Transaction) } // Get godoc @@ -155,7 +155,7 @@ func (h ReviewHandler) Update(ctx *gin.Context) { m.UpdateUser = h.BaseHandler.CurrentUser(ctx) db := h.DB(ctx).Model(m) db.Omit(clause.Associations) - result := db.Updates(h.fields(m)) + result := db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return @@ -194,26 +194,15 @@ func (h ReviewHandler) CopyReview(ctx *gin.Context) { Comments: m.Comments, ApplicationID: &id, } - existing := []model.Review{} - result = h.DB(ctx).Find(&existing, "applicationid = ?", id) + result = h.DB(ctx).Delete(&model.Review{}, "applicationid = ?", id) if result.Error != nil { _ = ctx.Error(result.Error) return } - // if the application doesn't already have a review, create one. - if len(existing) == 0 { - result = h.DB(ctx).Create(copied) - if result.Error != nil { - _ = ctx.Error(result.Error) - return - } - // if the application already has a review, replace it with the copied review. - } else { - result = h.DB(ctx).Model(&existing[0]).Updates(h.fields(copied)) - if result.Error != nil { - _ = ctx.Error(result.Error) - return - } + result = h.DB(ctx).Create(copied) + if result.Error != nil { + _ = ctx.Error(result.Error) + return } } h.Status(ctx, http.StatusNoContent) diff --git a/api/ruleset.go b/api/ruleset.go index 95bfabb58..43d8e4023 100644 --- a/api/ruleset.go +++ b/api/ruleset.go @@ -1,7 +1,6 @@ package api import ( - "encoding/json" "net/http" "github.com/gin-gonic/gin" @@ -249,7 +248,7 @@ func (h *RuleSetHandler) update(ctx *gin.Context, r *RuleSet) (err error) { m.UpdateUser = h.CurrentUser(ctx) db = h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - err = db.Updates(h.fields(m)).Error + err = db.Save(m).Error if err != nil { return } @@ -268,7 +267,7 @@ func (h *RuleSetHandler) update(ctx *gin.Context, r *RuleSet) (err error) { for i := range m.Rules { m := &m.Rules[i] db = h.DB(ctx).Model(m) - err = db.Updates(h.fields(m)).Error + err = db.Updates(m).Error if err != nil { return } @@ -313,7 +312,10 @@ func (r *RuleSet) With(m *model.RuleSet) { r.Name = m.Name r.Description = m.Description r.Identity = r.refPtr(m.IdentityID, m.Identity) - _ = json.Unmarshal(m.Repository, &r.Repository) + if m.Repository != (model.Repository{}) { + repo := Repository(m.Repository) + r.Repository = &repo + } r.Rules = []Rule{} for i := range m.Rules { rule := Rule{} @@ -344,7 +346,7 @@ func (r *RuleSet) Model() (m *model.RuleSet) { m.Rules = append(m.Rules, *rule.Model()) } if r.Repository != nil { - m.Repository, _ = json.Marshal(r.Repository) + m.Repository = model.Repository(*r.Repository) } for _, ref := range r.DependsOn { m.DependsOn = append( @@ -382,7 +384,7 @@ type Rule struct { func (r *Rule) With(m *model.Rule) { r.Resource.With(&m.Model) r.Name = m.Name - _ = json.Unmarshal(m.Labels, &r.Labels) + r.Labels = m.Labels r.File = r.refPtr(m.FileID, m.File) } @@ -391,9 +393,7 @@ func (r *Rule) Model() (m *model.Rule) { m = &model.Rule{} m.ID = r.ID m.Name = r.Name - if r.Labels != nil { - m.Labels, _ = json.Marshal(r.Labels) - } + m.Labels = r.Labels m.FileID = r.idPtr(r.File) return } diff --git a/api/service.go b/api/service.go new file mode 100644 index 000000000..d3e269727 --- /dev/null +++ b/api/service.go @@ -0,0 +1,99 @@ +package api + +import ( + "fmt" + "net/http" + "net/http/httputil" + "net/url" + "os" + + "github.com/gin-gonic/gin" +) + +// Routes +const ( + ServicesRoot = "/services" + ServiceRoot = ServicesRoot + "/:name/*" + Wildcard +) + +// serviceRoutes name to route map. +var serviceRoutes = map[string]string{ + "kai": os.Getenv("KAI_URL"), +} + +// ServiceHandler handles service routes. +type ServiceHandler struct { + BaseHandler +} + +// AddRoutes adds routes. +func (h ServiceHandler) AddRoutes(e *gin.Engine) { + e.GET(ServicesRoot, h.List) + e.Any(ServiceRoot, h.Required, h.Forward) +} + +// List godoc +// @summary List named service routes. +// @description List named service routes. +// @tags services +// @produce json +// @success 200 {object} api.Service +// @router /services [get] +func (h ServiceHandler) List(ctx *gin.Context) { + var r []Service + for name, route := range serviceRoutes { + service := Service{Name: name, Route: route} + r = append(r, service) + } + + h.Respond(ctx, http.StatusOK, r) +} + +// Required enforces RBAC. +func (h ServiceHandler) Required(ctx *gin.Context) { + Required(ctx.Param(Name))(ctx) +} + +// Forward provides RBAC and forwards request to the service. +func (h ServiceHandler) Forward(ctx *gin.Context) { + path := ctx.Param(Wildcard) + name := ctx.Param(Name) + route, found := serviceRoutes[name] + if !found { + err := &NotFound{Resource: name} + _ = ctx.Error(err) + return + } + if route == "" { + err := fmt.Errorf("route for: '%s' not defined", name) + _ = ctx.Error(err) + return + } + u, err := url.Parse(route) + if err != nil { + err = &BadRequestError{Reason: err.Error()} + _ = ctx.Error(err) + return + } + proxy := httputil.ReverseProxy{ + Director: func(req *http.Request) { + req.URL.Scheme = u.Scheme + req.URL.Host = u.Host + req.URL.Path = path + Log.Info( + "Routing (service)", + "path", + ctx.Request.URL.Path, + "route", + req.URL.String()) + }, + } + + proxy.ServeHTTP(ctx.Writer, ctx.Request) +} + +// Service REST resource. +type Service struct { + Name string `json:"name"` + Route string `json:"route"` +} diff --git a/api/setting.go b/api/setting.go index a758f4b81..ad5c6997e 100644 --- a/api/setting.go +++ b/api/setting.go @@ -1,7 +1,6 @@ package api import ( - "encoding/json" "fmt" "net/http" "strings" @@ -179,19 +178,19 @@ func (h SettingHandler) Update(ctx *gin.Context) { return } - updates := Setting{} - updates.Key = key - err := h.Bind(ctx, &updates.Value) + m := &model.Setting{} + result := h.DB(ctx).First(m, "key = ?", key) + if result.Error != nil { + _ = ctx.Error(result.Error) + return + } + err := h.Bind(ctx, &m.Value) if err != nil { _ = ctx.Error(err) return } - - m := updates.Model() m.UpdateUser = h.BaseHandler.CurrentUser(ctx) - db := h.DB(ctx).Model(m) - db = db.Where("key", key) - result := db.Updates(h.fields(m)) + result = h.DB(ctx).Save(m) if result.Error != nil { _ = ctx.Error(result.Error) } @@ -235,12 +234,11 @@ type Setting struct { func (r *Setting) With(m *model.Setting) { r.Key = m.Key - _ = json.Unmarshal(m.Value, &r.Value) - + r.Value = m.Value } func (r *Setting) Model() (m *model.Setting) { m = &model.Setting{Key: r.Key} - m.Value, _ = json.Marshal(r.Value) + m.Value = r.Value return } diff --git a/api/stakeholder.go b/api/stakeholder.go index 4cb659310..eead223b9 100644 --- a/api/stakeholder.go +++ b/api/stakeholder.go @@ -97,11 +97,31 @@ func (h StakeholderHandler) Create(ctx *gin.Context) { } m := r.Model() m.CreateUser = h.BaseHandler.CurrentUser(ctx) - result := h.DB(ctx).Create(m) + result := h.DB(ctx).Omit(clause.Associations).Create(m) if result.Error != nil { _ = ctx.Error(result.Error) return } + err = h.DB(ctx).Model(m).Association("Groups").Replace(m.Groups) + if err != nil { + _ = ctx.Error(err) + return + } + err = h.DB(ctx).Model(m).Association("Owns").Replace(m.Owns) + if err != nil { + _ = ctx.Error(err) + return + } + err = h.DB(ctx).Model(m).Association("Contributes").Replace(m.Contributes) + if err != nil { + _ = ctx.Error(err) + return + } + err = h.DB(ctx).Model(m).Association("MigrationWaves").Replace(m.MigrationWaves) + if err != nil { + _ = ctx.Error(err) + return + } r.With(m) h.Respond(ctx, http.StatusCreated, r) @@ -153,7 +173,7 @@ func (h StakeholderHandler) Update(ctx *gin.Context) { m.UpdateUser = h.BaseHandler.CurrentUser(ctx) db := h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - result := db.Updates(h.fields(m)) + result := db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return diff --git a/api/tag.go b/api/tag.go index 17d472444..648456b5f 100644 --- a/api/tag.go +++ b/api/tag.go @@ -153,7 +153,7 @@ func (h TagHandler) Update(ctx *gin.Context) { m.UpdateUser = h.BaseHandler.CurrentUser(ctx) db := h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - result := db.Updates(h.fields(m)) + result := db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return diff --git a/api/tagcategory.go b/api/tagcategory.go index 8ae7799b2..025065095 100644 --- a/api/tagcategory.go +++ b/api/tagcategory.go @@ -160,7 +160,7 @@ func (h TagCategoryHandler) Update(ctx *gin.Context) { m.UpdateUser = h.BaseHandler.CurrentUser(ctx) db := h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - result := db.Updates(h.fields(m)) + result := db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return diff --git a/api/target.go b/api/target.go index efccc4d10..c27c8fe00 100644 --- a/api/target.go +++ b/api/target.go @@ -1,7 +1,6 @@ package api import ( - "encoding/json" "errors" "fmt" "net/http" @@ -225,7 +224,7 @@ func (h TargetHandler) Update(ctx *gin.Context) { } db = h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - result = db.Updates(h.fields(m)) + result = db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return @@ -237,20 +236,17 @@ func (h TargetHandler) Update(ctx *gin.Context) { // Target REST resource. type Target struct { Resource `yaml:",inline"` - Name string `json:"name"` - Description string `json:"description"` - Provider string `json:"provider,omitempty" yaml:",omitempty"` - Choice bool `json:"choice,omitempty" yaml:",omitempty"` - Custom bool `json:"custom,omitempty" yaml:",omitempty"` - Labels []Label `json:"labels"` - Image Ref `json:"image"` - RuleSet *RuleSet `json:"ruleset,omitempty" yaml:"ruleset,omitempty"` + Name string `json:"name"` + Description string `json:"description"` + Provider string `json:"provider,omitempty" yaml:",omitempty"` + Choice bool `json:"choice,omitempty" yaml:",omitempty"` + Custom bool `json:"custom,omitempty" yaml:",omitempty"` + Labels []TargetLabel `json:"labels"` + Image Ref `json:"image"` + RuleSet *RuleSet `json:"ruleset,omitempty" yaml:"ruleset,omitempty"` } -type Label struct { - Name string `json:"name"` - Label string `json:"label"` -} +type TargetLabel model.TargetLabel // With updates the resource with the model. func (r *Target) With(m *model.Target) { @@ -269,7 +265,10 @@ func (r *Target) With(m *model.Target) { imgRef.Name = m.Image.Name } r.Image = imgRef - _ = json.Unmarshal(m.Labels, &r.Labels) + r.Labels = []TargetLabel{} + for _, l := range m.Labels { + r.Labels = append(r.Labels, TargetLabel(l)) + } } // Model builds a model. @@ -282,6 +281,8 @@ func (r *Target) Model() (m *model.Target) { } m.ID = r.ID m.ImageID = r.Image.ID - m.Labels, _ = json.Marshal(r.Labels) + for _, l := range r.Labels { + m.Labels = append(m.Labels, model.TargetLabel(l)) + } return } diff --git a/api/ticket.go b/api/ticket.go index e25af5611..e716d9161 100644 --- a/api/ticket.go +++ b/api/ticket.go @@ -1,7 +1,6 @@ package api import ( - "encoding/json" "net/http" "time" @@ -156,7 +155,7 @@ type Ticket struct { Message string `json:"message"` Status string `json:"status"` LastUpdated time.Time `json:"lastUpdated" yaml:"lastUpdated"` - Fields Fields `json:"fields"` + Fields Map `json:"fields"` Application Ref `json:"application" binding:"required"` Tracker Ref `json:"tracker" binding:"required"` } @@ -174,7 +173,7 @@ func (r *Ticket) With(m *model.Ticket) { r.LastUpdated = m.LastUpdated r.Application = r.ref(m.ApplicationID, m.Application) r.Tracker = r.ref(m.TrackerID, m.Tracker) - _ = json.Unmarshal(m.Fields, &r.Fields) + r.Fields = m.Fields } // Model builds a model. @@ -185,13 +184,8 @@ func (r *Ticket) Model() (m *model.Ticket) { ApplicationID: r.Application.ID, TrackerID: r.Tracker.ID, } - if r.Fields == nil { - r.Fields = Fields{} - } - m.Fields, _ = json.Marshal(r.Fields) + m.Fields = r.Fields m.ID = r.ID return } - -type Fields map[string]any diff --git a/api/tracker.go b/api/tracker.go index 650607bac..96434ed12 100644 --- a/api/tracker.go +++ b/api/tracker.go @@ -180,7 +180,7 @@ func (h TrackerHandler) Update(ctx *gin.Context) { m.UpdateUser = h.BaseHandler.CurrentUser(ctx) db := h.DB(ctx).Model(m) db = db.Omit(clause.Associations) - result := db.Updates(h.fields(m)) + result := db.Save(m) if result.Error != nil { _ = ctx.Error(result.Error) return diff --git a/assessment/assessment.go b/assessment/assessment.go index 6a874a519..aa9b56cb4 100644 --- a/assessment/assessment.go +++ b/assessment/assessment.go @@ -1,7 +1,6 @@ package assessment import ( - "encoding/json" "math" "github.com/konveyor/tackle2-hub/model" @@ -10,17 +9,11 @@ import ( // Assessment represents a deserialized Assessment. type Assessment struct { *model.Assessment - Sections []Section `json:"sections"` - Thresholds Thresholds `json:"thresholds"` - RiskMessages RiskMessages `json:"riskMessages"` } // With updates the Assessment with the db model and deserializes its fields. func (r *Assessment) With(m *model.Assessment) { r.Assessment = m - _ = json.Unmarshal(m.Sections, &r.Sections) - _ = json.Unmarshal(m.Thresholds, &r.Thresholds) - _ = json.Unmarshal(m.RiskMessages, &r.RiskMessages) } // Status returns the started status of the assessment. @@ -37,7 +30,7 @@ func (r *Assessment) Status() string { // Complete returns whether all sections have been completed. func (r *Assessment) Complete() bool { for _, s := range r.Sections { - if !s.Complete() { + if !r.sectionComplete(&s) { return false } } @@ -47,7 +40,7 @@ func (r *Assessment) Complete() bool { // Started returns whether any sections have been started. func (r *Assessment) Started() bool { for _, s := range r.Sections { - if s.Started() { + if r.sectionStarted(&s) { return true } } @@ -59,7 +52,7 @@ func (r *Assessment) Risk() string { var total uint colors := make(map[string]uint) for _, s := range r.Sections { - for _, risk := range s.Risks() { + for _, risk := range r.sectionRisks(&s) { colors[risk]++ total++ } @@ -85,8 +78,8 @@ func (r *Assessment) Confidence() (score int) { totalQuestions := 0 riskCounts := make(map[string]int) for _, s := range r.Sections { - for _, r := range s.Risks() { - riskCounts[r]++ + for _, risk := range r.sectionRisks(&s) { + riskCounts[risk]++ totalQuestions++ } } @@ -118,18 +111,73 @@ func (r *Assessment) Confidence() (score int) { return } -// Section represents a group of questions in a questionnaire. -type Section struct { - Order *uint `json:"order" yaml:"order" binding:"required"` - Name string `json:"name" yaml:"name"` - Questions []Question `json:"questions" yaml:"questions" binding:"min=1,dive"` - Comment string `json:"comment,omitempty" yaml:"comment,omitempty"` +func (r *Assessment) Prepare(tagResolver *TagResolver, tags Set) { + for i := range r.Sections { + s := &r.Sections[i] + includedQuestions := []model.Question{} + for _, q := range s.Questions { + for j := range q.Answers { + a := &q.Answers[j] + autoAnswerTags := NewSet() + for _, t := range a.AutoAnswerFor { + tag, found := tagResolver.Resolve(t.Category, t.Tag) + if found { + autoAnswerTags.Add(tag.ID) + } + } + if tags.Intersects(autoAnswerTags) { + a.AutoAnswered = true + a.Selected = true + break + } + } + + if len(q.IncludeFor) > 0 { + includeForTags := NewSet() + for _, t := range q.IncludeFor { + tag, found := tagResolver.Resolve(t.Category, t.Tag) + if found { + includeForTags.Add(tag.ID) + } + } + if tags.Intersects(includeForTags) { + includedQuestions = append(includedQuestions, q) + } + continue + } + + if len(q.ExcludeFor) > 0 { + excludeForTags := NewSet() + for _, t := range q.ExcludeFor { + tag, found := tagResolver.Resolve(t.Category, t.Tag) + if found { + excludeForTags.Add(tag.ID) + } + } + if tags.Intersects(excludeForTags) { + continue + } + } + includedQuestions = append(includedQuestions, q) + } + s.Questions = includedQuestions + } + return +} + +func (r *Assessment) Tags() (tags []model.CategorizedTag) { + for _, s := range r.Sections { + for _, t := range r.sectionTags(&s) { + tags = append(tags, t) + } + } + return } // Complete returns whether all questions in the section have been answered. -func (r *Section) Complete() bool { - for _, q := range r.Questions { - if !q.Answered() { +func (r *Assessment) sectionComplete(s *model.Section) bool { + for _, q := range s.Questions { + if !r.questionAnswered(&q) { return false } } @@ -137,9 +185,10 @@ func (r *Section) Complete() bool { } // Started returns whether any questions in the section have been answered. -func (r *Section) Started() bool { - for _, q := range r.Questions { - if q.Answered() && !q.AutoAnswered() { + +func (r *Assessment) sectionStarted(s *model.Section) bool { + for _, q := range s.Questions { + if r.questionAnswered(&q) && !r.questionAutoAnswered(&q) { return true } } @@ -147,36 +196,26 @@ func (r *Section) Started() bool { } // Risks returns a slice of the risks of each of its questions. -func (r *Section) Risks() []string { +func (r *Assessment) sectionRisks(s *model.Section) []string { risks := []string{} - for _, q := range r.Questions { - risks = append(risks, q.Risk()) + for _, q := range s.Questions { + risks = append(risks, r.questionRisk(&q)) } return risks } // Tags returns all the tags that should be applied based on how // the questions in the section have been answered. -func (r *Section) Tags() (tags []CategorizedTag) { - for _, q := range r.Questions { - tags = append(tags, q.Tags()...) +func (r *Assessment) sectionTags(s *model.Section) (tags []model.CategorizedTag) { + for _, q := range s.Questions { + tags = append(tags, r.questionTags(&q)...) } return } -// Question represents a question in a questionnaire. -type Question struct { - Order *uint `json:"order" yaml:"order" binding:"required"` - Text string `json:"text" yaml:"text"` - Explanation string `json:"explanation" yaml:"explanation"` - IncludeFor []CategorizedTag `json:"includeFor,omitempty" yaml:"includeFor,omitempty"` - ExcludeFor []CategorizedTag `json:"excludeFor,omitempty" yaml:"excludeFor,omitempty"` - Answers []Answer `json:"answers" yaml:"answers" binding:"min=1,dive"` -} - // Risk returns the risk level for the question based on how it has been answered. -func (r *Question) Risk() string { - for _, a := range r.Answers { +func (r *Assessment) questionRisk(q *model.Question) string { + for _, a := range q.Answers { if a.Selected { return a.Risk } @@ -185,8 +224,8 @@ func (r *Question) Risk() string { } // Answered returns whether the question has had an answer selected. -func (r *Question) Answered() bool { - for _, a := range r.Answers { +func (r *Assessment) questionAnswered(q *model.Question) bool { + for _, a := range q.Answers { if a.Selected { return true } @@ -196,8 +235,8 @@ func (r *Question) Answered() bool { // AutoAnswered returns whether the question has had an // answer pre-selected by the system. -func (r *Question) AutoAnswered() bool { - for _, a := range r.Answers { +func (r *Assessment) questionAutoAnswered(q *model.Question) bool { + for _, a := range q.Answers { if a.AutoAnswered { return true } @@ -206,8 +245,8 @@ func (r *Question) AutoAnswered() bool { } // Tags returns any tags to be applied based on how the question is answered. -func (r *Question) Tags() (tags []CategorizedTag) { - for _, answer := range r.Answers { +func (r *Assessment) questionTags(q *model.Question) (tags []model.CategorizedTag) { + for _, answer := range q.Answers { if answer.Selected { tags = answer.ApplyTags return @@ -215,37 +254,3 @@ func (r *Question) Tags() (tags []CategorizedTag) { } return } - -// Answer represents an answer to a question in a questionnaire. -type Answer struct { - Order *uint `json:"order" yaml:"order" binding:"required"` - Text string `json:"text" yaml:"text"` - Risk string `json:"risk" yaml:"risk" binding:"oneof=red yellow green unknown"` - Rationale string `json:"rationale" yaml:"rationale"` - Mitigation string `json:"mitigation" yaml:"mitigation"` - ApplyTags []CategorizedTag `json:"applyTags,omitempty" yaml:"applyTags,omitempty"` - AutoAnswerFor []CategorizedTag `json:"autoAnswerFor,omitempty" yaml:"autoAnswerFor,omitempty"` - Selected bool `json:"selected,omitempty" yaml:"selected,omitempty"` - AutoAnswered bool `json:"autoAnswered,omitempty" yaml:"autoAnswered,omitempty"` -} - -// CategorizedTag represents a human-readable pair of category and tag. -type CategorizedTag struct { - Category string `json:"category" yaml:"category"` - Tag string `json:"tag" yaml:"tag"` -} - -// RiskMessages contains messages to display for each risk level. -type RiskMessages struct { - Red string `json:"red" yaml:"red"` - Yellow string `json:"yellow" yaml:"yellow"` - Green string `json:"green" yaml:"green"` - Unknown string `json:"unknown" yaml:"unknown"` -} - -// Thresholds contains the threshold values for determining risk for the questionnaire. -type Thresholds struct { - Red uint `json:"red" yaml:"red"` - Yellow uint `json:"yellow" yaml:"yellow"` - Unknown uint `json:"unknown" yaml:"unknown"` -} diff --git a/assessment/assessment_test.go b/assessment/assessment_test.go index 28efe1154..69df6702b 100644 --- a/assessment/assessment_test.go +++ b/assessment/assessment_test.go @@ -7,15 +7,16 @@ import ( "github.com/onsi/gomega" ) -func TestPrepareSections(t *testing.T) { +func TestPrepare(t *testing.T) { g := gomega.NewGomegaWithT(t) - sections := []Section{ + assessment := model.Assessment{} + assessment.Sections = []model.Section{ { - Questions: []Question{ + Questions: []model.Question{ { Text: "Default", - Answers: []Answer{ + Answers: []model.Answer{ { Text: "Answer1", }, @@ -26,10 +27,10 @@ func TestPrepareSections(t *testing.T) { }, { Text: "Should Include", - IncludeFor: []CategorizedTag{ + IncludeFor: []model.CategorizedTag{ {Category: "Category", Tag: "Tag"}, }, - Answers: []Answer{ + Answers: []model.Answer{ { Text: "Answer1", }, @@ -40,10 +41,10 @@ func TestPrepareSections(t *testing.T) { }, { Text: "Should Exclude", - ExcludeFor: []CategorizedTag{ + ExcludeFor: []model.CategorizedTag{ {Category: "Category", Tag: "Tag"}, }, - Answers: []Answer{ + Answers: []model.Answer{ { Text: "Answer1", }, @@ -54,10 +55,10 @@ func TestPrepareSections(t *testing.T) { }, { Text: "AutoAnswer", - Answers: []Answer{ + Answers: []model.Answer{ { Text: "Answer1", - AutoAnswerFor: []CategorizedTag{ + AutoAnswerFor: []model.CategorizedTag{ {Category: "Category", Tag: "Tag"}, }, }, @@ -69,6 +70,9 @@ func TestPrepareSections(t *testing.T) { }, }, } + a := Assessment{} + a.With(&assessment) + tagResolver := TagResolver{ cache: map[string]map[string]*model.Tag{ "Category": {"Tag": {Model: model.Model{ID: 1}}}, @@ -77,16 +81,16 @@ func TestPrepareSections(t *testing.T) { tags := NewSet() tags.Add(1) - preparedSections := prepareSections(&tagResolver, tags, sections) - questions := preparedSections[0].Questions + a.Prepare(&tagResolver, tags) + questions := a.Sections[0].Questions g.Expect(len(questions)).To(gomega.Equal(3)) g.Expect(questions[0].Text).To(gomega.Equal("Default")) - g.Expect(questions[0].Answered()).To(gomega.BeFalse()) + g.Expect(a.questionAnswered(&questions[0])).To(gomega.BeFalse()) g.Expect(questions[1].Text).To(gomega.Equal("Should Include")) - g.Expect(questions[1].Answered()).To(gomega.BeFalse()) + g.Expect(a.questionAnswered(&questions[1])).To(gomega.BeFalse()) g.Expect(questions[2].Text).To(gomega.Equal("AutoAnswer")) - g.Expect(questions[2].Answered()).To(gomega.BeTrue()) + g.Expect(a.questionAnswered(&questions[2])).To(gomega.BeTrue()) g.Expect(questions[2].Answers[0].Text).To(gomega.Equal("Answer1")) g.Expect(questions[2].Answers[0].AutoAnswered).To(gomega.BeTrue()) g.Expect(questions[2].Answers[0].Selected).To(gomega.BeTrue()) @@ -95,113 +99,117 @@ func TestPrepareSections(t *testing.T) { func TestAssessmentStarted(t *testing.T) { g := gomega.NewGomegaWithT(t) - assessment := Assessment{ - Sections: []Section{ - { - Questions: []Question{ - { - Text: "S1Q1", - Answers: []Answer{ - { - Text: "A1", - Selected: true, - }, - { - Text: "A2", - }, + assessment := model.Assessment{} + assessment.Sections = []model.Section{ + { + Questions: []model.Question{ + { + Text: "S1Q1", + Answers: []model.Answer{ + { + Text: "A1", + Selected: true, + }, + { + Text: "A2", }, }, - { - Text: "S1Q2", - Answers: []Answer{ - { - Text: "A1", - }, - { - Text: "A2", - }, + }, + { + Text: "S1Q2", + Answers: []model.Answer{ + { + Text: "A1", + }, + { + Text: "A2", }, }, }, }, - { - Questions: []Question{ - { - Text: "S2Q1", - Answers: []Answer{ - { - Text: "A1", - }, - { - Text: "A2", - }, + }, + { + Questions: []model.Question{ + { + Text: "S2Q1", + Answers: []model.Answer{ + { + Text: "A1", + }, + { + Text: "A2", }, }, }, }, }, } - g.Expect(assessment.Started()).To(gomega.BeTrue()) - g.Expect(assessment.Status()).To(gomega.Equal(StatusStarted)) - assessment.Sections[0].Questions[0].Answers[0].AutoAnswered = true - g.Expect(assessment.Started()).To(gomega.BeFalse()) - g.Expect(assessment.Status()).To(gomega.Equal(StatusEmpty)) + + a := Assessment{} + a.With(&assessment) + g.Expect(a.Started()).To(gomega.BeTrue()) + g.Expect(a.Status()).To(gomega.Equal(StatusStarted)) + a.Sections[0].Questions[0].Answers[0].AutoAnswered = true + g.Expect(a.Started()).To(gomega.BeFalse()) + g.Expect(a.Status()).To(gomega.Equal(StatusEmpty)) } func TestAssessmentComplete(t *testing.T) { g := gomega.NewGomegaWithT(t) - assessment := Assessment{ - Sections: []Section{ - { - Questions: []Question{ - { - Text: "S1Q1", - Answers: []Answer{ - { - Text: "A1", - }, - { - Text: "A2", - }, + assessment := model.Assessment{} + assessment.Sections = []model.Section{ + { + Questions: []model.Question{ + { + Text: "S1Q1", + Answers: []model.Answer{ + { + Text: "A1", + }, + { + Text: "A2", }, }, - { - Text: "S1Q2", - Answers: []Answer{ - { - Text: "A1", - Selected: true, - }, - { - Text: "A2", - }, + }, + { + Text: "S1Q2", + Answers: []model.Answer{ + { + Text: "A1", + Selected: true, + }, + { + Text: "A2", }, }, }, }, - { - Questions: []Question{ - { - Text: "S2Q1", - Answers: []Answer{ - { - Text: "A1", - }, - { - Text: "A2", - Selected: true, - AutoAnswered: true, - }, + }, + { + Questions: []model.Question{ + { + Text: "S2Q1", + Answers: []model.Answer{ + { + Text: "A1", + }, + { + Text: "A2", + Selected: true, + AutoAnswered: true, }, }, }, }, }, } - g.Expect(assessment.Complete()).To(gomega.BeFalse()) - g.Expect(assessment.Status()).To(gomega.Equal(StatusStarted)) - assessment.Sections[0].Questions[0].Answers[0].Selected = true - g.Expect(assessment.Complete()).To(gomega.BeTrue()) - g.Expect(assessment.Status()).To(gomega.Equal(StatusComplete)) + + a := Assessment{} + a.With(&assessment) + g.Expect(a.Complete()).To(gomega.BeFalse()) + g.Expect(a.Status()).To(gomega.Equal(StatusStarted)) + a.Sections[0].Questions[0].Answers[0].Selected = true + g.Expect(a.Complete()).To(gomega.BeTrue()) + g.Expect(a.Status()).To(gomega.Equal(StatusComplete)) } diff --git a/assessment/membership.go b/assessment/membership.go index b6180c5f3..71c74fad7 100644 --- a/assessment/membership.go +++ b/assessment/membership.go @@ -22,6 +22,7 @@ type MembershipResolver struct { tagSets map[uint]Set archetypeMembers map[uint][]Application membersCached bool + archetypesCached bool } // Applications returns the list of applications that are members of the given archetype. @@ -77,7 +78,7 @@ loop: } func (r *MembershipResolver) cacheArchetypes() (err error) { - if len(r.archetypes) > 0 { + if r.archetypesCached { return } @@ -101,6 +102,7 @@ func (r *MembershipResolver) cacheArchetypes() (err error) { } r.tagSets[a.ID] = set } + r.archetypesCached = true return } diff --git a/assessment/pkg.go b/assessment/pkg.go index 263eca7a0..c0268833e 100644 --- a/assessment/pkg.go +++ b/assessment/pkg.go @@ -1,8 +1,6 @@ package assessment import ( - "encoding/json" - "github.com/konveyor/tackle2-hub/model" ) @@ -93,25 +91,19 @@ func Confidence(assessments []Assessment) (confidence int) { // PrepareForApplication prepares the sections of an assessment by including, excluding, // or auto-answering questions based on a set of tags. func PrepareForApplication(tagResolver *TagResolver, application *model.Application, assessment *model.Assessment) { - sections := []Section{} - _ = json.Unmarshal(assessment.Sections, §ions) - tagSet := NewSet() for _, t := range application.Tags { tagSet.Add(t.ID) } - - assessment.Sections, _ = json.Marshal(prepareSections(tagResolver, tagSet, sections)) - + a := Assessment{} + a.With(assessment) + a.Prepare(tagResolver, tagSet) return } // PrepareForArchetype prepares the sections of an assessment by including, excluding, // or auto-answering questions based on a set of tags. func PrepareForArchetype(tagResolver *TagResolver, archetype *model.Archetype, assessment *model.Assessment) { - sections := []Section{} - _ = json.Unmarshal(assessment.Sections, §ions) - tagSet := NewSet() for _, t := range archetype.CriteriaTags { tagSet.Add(t.ID) @@ -119,63 +111,8 @@ func PrepareForArchetype(tagResolver *TagResolver, archetype *model.Archetype, a for _, t := range archetype.Tags { tagSet.Add(t.ID) } - - assessment.Sections, _ = json.Marshal(prepareSections(tagResolver, tagSet, sections)) - - return -} - -func prepareSections(tagResolver *TagResolver, tags Set, sections []Section) (preparedSections []Section) { - for i := range sections { - s := §ions[i] - includedQuestions := []Question{} - for _, q := range s.Questions { - for j := range q.Answers { - a := &q.Answers[j] - autoAnswerTags := NewSet() - for _, t := range a.AutoAnswerFor { - tag, found := tagResolver.Resolve(t.Category, t.Tag) - if found { - autoAnswerTags.Add(tag.ID) - } - } - if tags.Intersects(autoAnswerTags) { - a.AutoAnswered = true - a.Selected = true - break - } - } - - if len(q.IncludeFor) > 0 { - includeForTags := NewSet() - for _, t := range q.IncludeFor { - tag, found := tagResolver.Resolve(t.Category, t.Tag) - if found { - includeForTags.Add(tag.ID) - } - } - if tags.Intersects(includeForTags) { - includedQuestions = append(includedQuestions, q) - } - continue - } - - if len(q.ExcludeFor) > 0 { - excludeForTags := NewSet() - for _, t := range q.ExcludeFor { - tag, found := tagResolver.Resolve(t.Category, t.Tag) - if found { - excludeForTags.Add(tag.ID) - } - } - if tags.Intersects(excludeForTags) { - continue - } - } - includedQuestions = append(includedQuestions, q) - } - s.Questions = includedQuestions - } - preparedSections = sections + a := Assessment{} + a.With(assessment) + a.Prepare(tagResolver, tagSet) return } diff --git a/assessment/tag.go b/assessment/tag.go index 930ed9b31..39eb44658 100644 --- a/assessment/tag.go +++ b/assessment/tag.go @@ -30,12 +30,10 @@ func (r *TagResolver) Resolve(category string, tag string) (t *model.Tag, found // Assessment returns all the Tag models that should be applied from the assessment. func (r *TagResolver) Assessment(assessment Assessment) (tags []model.Tag) { - for _, s := range assessment.Sections { - for _, t := range s.Tags() { - tag, found := r.Resolve(t.Category, t.Tag) - if found { - tags = append(tags, *tag) - } + for _, t := range assessment.Tags() { + tag, found := r.Resolve(t.Category, t.Tag) + if found { + tags = append(tags, *tag) } } return diff --git a/auth/roles.yaml b/auth/roles.yaml index aa65e7a77..361a2c1ae 100644 --- a/auth/roles.yaml +++ b/auth/roles.yaml @@ -82,6 +82,10 @@ - get - post - put + - name: kai + verbs: + - get + - post - name: proxies verbs: - delete @@ -286,6 +290,10 @@ - get - post - put + - name: kai + verbs: + - get + - post - name: proxies verbs: - get @@ -443,6 +451,10 @@ - name: jobfunctions verbs: - get + - name: kai + verbs: + - get + - post - name: proxies verbs: - get @@ -560,6 +572,10 @@ - name: jobfunctions verbs: - get + - name: kai + verbs: + - get + - post - name: proxies verbs: - get diff --git a/binding/application.go b/binding/application.go index b8b02fd62..7266987bc 100644 --- a/binding/application.go +++ b/binding/application.go @@ -1,16 +1,12 @@ package binding import ( - "bytes" "errors" - "io" - "net/http" "strconv" - mime "github.com/gin-gonic/gin/binding" + "github.com/gin-gonic/gin/binding" liberr "github.com/jortel/go-utils/error" "github.com/konveyor/tackle2-hub/api" - "gopkg.in/yaml.v2" ) // Application API. @@ -229,8 +225,8 @@ func (h *AppFacts) Source(source string) { } // List facts. -func (h *AppFacts) List() (facts api.FactMap, err error) { - facts = api.FactMap{} +func (h *AppFacts) List() (facts api.Map, err error) { + facts = api.Map{} key := api.FactKey("") key.Qualify(h.source) path := Path(api.ApplicationFactsRoot).Inject(Params{api.ID: h.appId, api.Key: key}) @@ -278,7 +274,7 @@ func (h *AppFacts) Delete(name string) (err error) { } // Replace facts. -func (h *AppFacts) Replace(facts api.FactMap) (err error) { +func (h *AppFacts) Replace(facts api.Map) (err error) { key := api.FactKey("") key.Qualify(h.source) path := Path(api.ApplicationFactsRoot).Inject(Params{api.ID: h.appId, api.Key: key}) @@ -316,30 +312,38 @@ type Analysis struct { appId uint } -// Create an analysis report. -func (h *Analysis) Create(r *api.Analysis, encoding string, issues, deps io.Reader) (err error) { +// Create an analysis report using the manifest at the specified path. +// The manifest contains 3 sections containing documents delimited by markers. +// The manifest must contain ALL markers even when sections are empty. +// Note: `^]` = `\x1D` = GS (group separator). +// Section markers: +// +// ^]BEGIN-MAIN^] +// ^]END-MAIN^] +// ^]BEGIN-ISSUES^] +// ^]END-ISSUES^] +// ^]BEGIN-DEPS^] +// ^]END-DEPS^] +// +// The encoding must be: +// - application/json +// - application/x-yaml +func (h *Analysis) Create(manifest, encoding string) (r *api.Analysis, err error) { + switch encoding { + case "": + encoding = binding.MIMEJSON + case binding.MIMEJSON, + binding.MIMEYAML: + default: + err = liberr.New( + "Encoding: %s not supported", + encoding) + } + r = &api.Analysis{} path := Path(api.AppAnalysesRoot).Inject(Params{api.ID: h.appId}) - b, _ := yaml.Marshal(r) - err = h.client.FileSend( - path, - http.MethodPost, - []Field{ - { - Name: api.FileField, - Reader: bytes.NewReader(b), - Encoding: mime.MIMEYAML, - }, - { - Name: api.IssueField, - Encoding: encoding, - Reader: issues, - }, - { - Name: api.DepField, - Encoding: encoding, - Reader: deps, - }, - }, - r) + err = h.client.FilePostEncoded(path, manifest, r, encoding) + if err != nil { + return + } return } diff --git a/binding/client.go b/binding/client.go index 78ae07b85..a5976eb9e 100644 --- a/binding/client.go +++ b/binding/client.go @@ -457,11 +457,19 @@ func (r *Client) FileGet(path, destination string) (err error) { // FilePost uploads a file. // Returns the created File resource. func (r *Client) FilePost(path, source string, object any) (err error) { + err = r.FilePostEncoded(path, source, object, "") + return +} + +// FilePostEncoded uploads a file. +// Returns the created File resource. +func (r *Client) FilePostEncoded(path, source string, object any, encoding string) (err error) { if source == "" { fields := []Field{ { - Name: api.FileField, - Reader: bytes.NewReader([]byte{}), + Name: api.FileField, + Reader: bytes.NewReader([]byte{}), + Encoding: encoding, }, } err = r.FileSend(path, http.MethodPost, fields, object) @@ -478,8 +486,9 @@ func (r *Client) FilePost(path, source string, object any) (err error) { } fields := []Field{ { - Name: api.FileField, - Path: source, + Name: api.FileField, + Path: source, + Encoding: encoding, }, } err = r.FileSend(path, http.MethodPost, fields, object) @@ -489,11 +498,19 @@ func (r *Client) FilePost(path, source string, object any) (err error) { // FilePut uploads a file. // Returns the created File resource. func (r *Client) FilePut(path, source string, object any) (err error) { + err = r.FilePutEncoded(path, source, object, "") + return +} + +// FilePutEncoded uploads a file. +// Returns the created File resource. +func (r *Client) FilePutEncoded(path, source string, object any, encoding string) (err error) { if source == "" { fields := []Field{ { - Name: api.FileField, - Reader: bytes.NewReader([]byte{}), + Name: api.FileField, + Reader: bytes.NewReader([]byte{}), + Encoding: encoding, }, } err = r.FileSend(path, http.MethodPut, fields, object) @@ -510,8 +527,9 @@ func (r *Client) FilePut(path, source string, object any) (err error) { } fields := []Field{ { - Name: api.FileField, - Path: source, + Name: api.FileField, + Path: source, + Encoding: encoding, }, } err = r.FileSend(path, http.MethodPut, fields, object) diff --git a/binding/file.go b/binding/file.go index 22bdcb80b..86bd3d94b 100644 --- a/binding/file.go +++ b/binding/file.go @@ -42,17 +42,29 @@ func (h *File) Touch(name string) (r *api.File, err error) { // Post uploads a file. func (h *File) Post(source string) (r *api.File, err error) { + r, err = h.PostEncoded(source, "") + return +} + +// PostEncoded uploads a file. +func (h *File) PostEncoded(source string, encoding string) (r *api.File, err error) { r = &api.File{} path := Path(api.FileRoot).Inject(Params{api.ID: pathlib.Base(source)}) - err = h.client.FilePost(path, source, r) + err = h.client.FilePostEncoded(path, source, r, encoding) return } // Put uploads a file. func (h *File) Put(source string) (r *api.File, err error) { + r, err = h.PutEncoded(source, "") + return +} + +// PutEncoded uploads a file. +func (h *File) PutEncoded(source string, encoding string) (r *api.File, err error) { r = &api.File{} path := Path(api.FileRoot).Inject(Params{api.ID: pathlib.Base(source)}) - err = h.client.FilePut(path, source, r) + err = h.client.FilePutEncoded(path, source, r, encoding) return } diff --git a/cmd/main.go b/cmd/main.go index 8be4667bb..91a6adcb3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -17,6 +17,7 @@ import ( crd "github.com/konveyor/tackle2-hub/k8s/api" "github.com/konveyor/tackle2-hub/metrics" "github.com/konveyor/tackle2-hub/migration" + "github.com/konveyor/tackle2-hub/model" "github.com/konveyor/tackle2-hub/reaper" "github.com/konveyor/tackle2-hub/seed" "github.com/konveyor/tackle2-hub/settings" @@ -53,6 +54,10 @@ func Setup() (db *gorm.DB, err error) { if err != nil { return } + err = database.PK.Load(db, model.ALL) + if err != nil { + return + } return } diff --git a/database/db_test.go b/database/db_test.go index 36555f21b..3a4590f09 100644 --- a/database/db_test.go +++ b/database/db_test.go @@ -1,34 +1,137 @@ package database import ( - "encoding/json" "fmt" "os" "testing" + "time" + "github.com/konveyor/tackle2-hub/api" "github.com/konveyor/tackle2-hub/model" + "gorm.io/gorm" + "k8s.io/utils/env" ) -var N = 800 +var N, _ = env.GetInt("TEST_CONCURRENT", 10) + +func TestDriver(t *testing.T) { + pid := os.Getpid() + Settings.DB.Path = fmt.Sprintf("/tmp/driver-%d.db", pid) + defer func() { + _ = os.Remove(Settings.DB.Path) + }() + db, err := Open(true) + if err != nil { + panic(err) + } + key := "driver" + m := &model.Setting{Key: key, Value: "Test"} + // insert. + err = db.Create(m).Error + if err != nil { + panic(err) + } + // update + err = db.Save(m).Error + if err != nil { + panic(err) + } + // select + err = db.First(m, m.ID).Error + if err != nil { + panic(err) + } + // delete + err = db.Delete(m).Error + if err != nil { + panic(err) + } +} func TestConcurrent(t *testing.T) { - Settings.DB.Path = "/tmp/concurrent.db" - _ = os.Remove(Settings.DB.Path) + pid := os.Getpid() + Settings.DB.Path = fmt.Sprintf("/tmp/concurrent-%d.db", pid) + defer func() { + _ = os.Remove(Settings.DB.Path) + }() db, err := Open(true) if err != nil { panic(err) } + + type A struct { + model.Model + } + + type B struct { + N int + model.Model + A A + AID uint + } + err = db.Migrator().AutoMigrate(&A{}, &B{}) + if err != nil { + panic(err) + } + + a := A{} + err = db.Create(&a).Error + if err != nil { + panic(err) + } + dq := make(chan int, N) for w := 0; w < N; w++ { go func(id int) { fmt.Printf("Started %d\n", id) - for n := 0; n < N; n++ { - v, _ := json.Marshal(fmt.Sprintf("Test-%d", n)) - m := &model.Setting{Key: fmt.Sprintf("key-%d-%d", id, n), Value: v} + for n := 0; n < N*100; n++ { + m := &B{N: n, A: a} + m.CreateUser = "Test" + fmt.Printf("(%.4d) CREATE: %.4d\n", id, n) uErr := db.Create(m).Error if uErr != nil { panic(uErr) } + uErr = db.Save(m).Error + if uErr != nil { + panic(uErr) + } + for i := 0; i < 10; i++ { + fmt.Printf("(%.4d) READ: %.4d/%.4d\n", id, n, i) + uErr = db.First(m).Error + if uErr != nil { + panic(uErr) + } + } + for i := 0; i < 10; i++ { + fmt.Printf("(%.4d) LIST: %.4d/%.4d\n", id, n, i) + page := api.Page{} + cursor := api.Cursor{} + mx := B{} + dbx := db.Model(mx) + dbx = dbx.Joins("A") + dbx = dbx.Limit(10) + cursor.With(dbx, page) + for cursor.Next(&mx) { + time.Sleep(time.Millisecond + 10) + fmt.Printf("(%.4d) NEXT: %.4d/%.4d ID=%d\n", id, n, i, mx.ID) + } + } + for i := 0; i < 4; i++ { + uErr = db.Transaction(func(tx *gorm.DB) (err error) { + time.Sleep(time.Millisecond * 10) + for i := 0; i < 3; i++ { + err = tx.Save(m).Error + if err != nil { + break + } + } + return + }) + if uErr != nil { + panic(uErr) + } + } } dq <- id }(w) @@ -38,3 +141,56 @@ func TestConcurrent(t *testing.T) { fmt.Printf("Done %d\n", id) } } + +func TestKeyGen(t *testing.T) { + pid := os.Getpid() + Settings.DB.Path = fmt.Sprintf("/tmp/keygen-%d.db", pid) + defer func() { + _ = os.Remove(Settings.DB.Path) + }() + db, err := Open(true) + if err != nil { + panic(err) + } + // ids 1-7 created. + N = 8 + for n := 1; n < N; n++ { + m := &model.Setting{Key: fmt.Sprintf("key-%d", n), Value: n} + err := db.Create(m).Error + if err != nil { + panic(err) + } + fmt.Printf("CREATED: %d/%d\n", m.ID, n) + if uint(n) != m.ID { + t.Errorf("id:%d but expected: %d", m.ID, n) + return + } + } + // delete ids=2,4,7. + err = db.Delete(&model.Setting{}, []uint{2, 4, 7}).Error + if err != nil { + panic(err) + } + + var count int64 + err = db.Model(&model.Setting{}).Where([]uint{2, 4, 7}).Count(&count).Error + if err != nil { + panic(err) + } + if count > 0 { + t.Errorf("DELETED ids: 2,4,7 found.") + return + } + // id=8 (next) created. + next := N + m := &model.Setting{Key: fmt.Sprintf("key-%d", next), Value: next} + err = db.Create(m).Error + if err != nil { + panic(err) + } + fmt.Printf("CREATED: %d/%d (next)\n", m.ID, next) + if uint(N) != m.ID { + t.Errorf("id:%d but expected: %d", m.ID, next) + return + } +} diff --git a/database/driver.go b/database/driver.go new file mode 100644 index 000000000..0c50360b1 --- /dev/null +++ b/database/driver.go @@ -0,0 +1,330 @@ +package database + +import ( + "context" + "database/sql/driver" + "strings" + "sync" + + "github.com/mattn/go-sqlite3" +) + +// Driver is a wrapper around the SQLite driver. +// The purpose is to prevent database locked errors using +// a mutex around write operations. +type Driver struct { + mutex sync.Mutex + wrapped driver.Driver + dsn string +} + +// Open a connection. +func (d *Driver) Open(dsn string) (conn driver.Conn, err error) { + d.wrapped = &sqlite3.SQLiteDriver{} + conn, err = d.wrapped.Open(dsn) + if err != nil { + return + } + conn = &Conn{ + mutex: &d.mutex, + wrapped: conn, + } + return +} + +// OpenConnector opens a connection. +func (d *Driver) OpenConnector(dsn string) (dc driver.Connector, err error) { + d.dsn = dsn + dc = d + return +} + +// Connect opens a connection. +func (d *Driver) Connect(context.Context) (conn driver.Conn, err error) { + conn, err = d.Open(d.dsn) + return +} + +// Driver returns the underlying driver. +func (d *Driver) Driver() driver.Driver { + return d +} + +// Conn is a DB connection. +type Conn struct { + mutex *sync.Mutex + wrapped driver.Conn + hasMutex bool + hasTx bool +} + +// Ping the DB. +func (c *Conn) Ping(ctx context.Context) (err error) { + if p, cast := c.wrapped.(driver.Pinger); cast { + err = p.Ping(ctx) + } + return +} + +// ResetSession reset the connection. +// - Reset the Tx. +// - Release the mutex. +func (c *Conn) ResetSession(ctx context.Context) (err error) { + defer func() { + c.hasTx = false + c.release() + }() + if p, cast := c.wrapped.(driver.SessionResetter); cast { + err = p.ResetSession(ctx) + } + return +} + +// IsValid returns true when the connection is valid. +// When true, the connection may be reused by the sql package. +func (c *Conn) IsValid() (b bool) { + b = true + if p, cast := c.wrapped.(driver.Validator); cast { + b = p.IsValid() + } + return +} + +// QueryContext execute a query with context. +func (c *Conn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (r driver.Rows, err error) { + if c.needsMutex(query) { + c.acquire() + defer c.release() + } + if p, cast := c.wrapped.(driver.QueryerContext); cast { + r, err = p.QueryContext(ctx, query, args) + } + return +} + +// ExecContext executes an SQL/DDL statement with context. +func (c *Conn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (r driver.Result, err error) { + if c.needsMutex(query) { + c.acquire() + defer c.release() + } + if p, cast := c.wrapped.(driver.ExecerContext); cast { + r, err = p.ExecContext(ctx, query, args) + } + return +} + +// Begin a transaction. +func (c *Conn) Begin() (tx driver.Tx, err error) { + c.acquire() + tx, err = c.wrapped.Begin() + if err != nil { + return + } + tx = &Tx{ + conn: c, + wrapped: tx, + } + c.hasTx = true + return +} + +// BeginTx begins a transaction. +func (c *Conn) BeginTx(ctx context.Context, opts driver.TxOptions) (tx driver.Tx, err error) { + c.acquire() + if p, cast := c.wrapped.(driver.ConnBeginTx); cast { + tx, err = p.BeginTx(ctx, opts) + } else { + tx, err = c.wrapped.Begin() + } + tx = &Tx{ + conn: c, + wrapped: tx, + } + c.hasTx = true + return +} + +// Prepare a statement. +func (c *Conn) Prepare(query string) (stmt driver.Stmt, err error) { + stmt, err = c.wrapped.Prepare(query) + stmt = &Stmt{ + conn: c, + wrapped: stmt, + query: query, + } + return +} + +// PrepareContext prepares a statement with context. +func (c *Conn) PrepareContext(ctx context.Context, query string) (stmt driver.Stmt, err error) { + if p, cast := c.wrapped.(driver.ConnPrepareContext); cast { + stmt, err = p.PrepareContext(ctx, query) + } else { + stmt, err = c.Prepare(query) + } + stmt = &Stmt{ + conn: c, + wrapped: stmt, + query: query, + } + return +} + +// Close the connection. +func (c *Conn) Close() (err error) { + err = c.wrapped.Close() + c.hasMutex = false + c.release() + return +} + +// needsMutex returns true when the query should is a write operation. +func (c *Conn) needsMutex(query string) (matched bool) { + if query == "" { + return + } + query = strings.ToUpper(query) + action := strings.Fields(query)[0] + action = strings.ToUpper(action) + matched = action == "CREATE" || + action == "INSERT" || + action == "UPDATE" || + action == "DELETE" + return +} + +// acquire the mutex. +// Since Locks are not reentrant, the mutex is acquired +// only if this connection has not already acquired it. +func (c *Conn) acquire() { + if !c.hasMutex { + c.mutex.Lock() + c.hasMutex = true + } +} + +// release the mutex. +// Released only when: +// - This connection has acquired it +// - Not in a transaction. +func (c *Conn) release() { + if c.hasMutex && !c.hasTx { + c.mutex.Unlock() + c.hasMutex = false + } +} + +// endTx report transaction has ended. +func (c *Conn) endTx() { + c.hasTx = false +} + +// Stmt is a SQL/DDL statement. +type Stmt struct { + wrapped driver.Stmt + conn *Conn + query string +} + +// Close the statement. +func (s *Stmt) Close() (err error) { + err = s.wrapped.Close() + return +} + +// NumInput returns the number of (query) input parameters. +func (s *Stmt) NumInput() (n int) { + n = s.wrapped.NumInput() + return +} + +// Exec executes the statement. +func (s *Stmt) Exec(args []driver.Value) (r driver.Result, err error) { + if s.needsMutex() { + s.conn.acquire() + defer s.conn.release() + } + r, err = s.wrapped.Exec(args) + return +} + +// ExecContext executes the statement with context. +func (s *Stmt) ExecContext(ctx context.Context, args []driver.NamedValue) (r driver.Result, err error) { + if s.needsMutex() { + s.conn.acquire() + defer s.conn.release() + } + if p, cast := s.wrapped.(driver.StmtExecContext); cast { + r, err = p.ExecContext(ctx, args) + } else { + r, err = s.Exec(s.values(args)) + } + return +} + +// Query executes a query. +func (s *Stmt) Query(args []driver.Value) (r driver.Rows, err error) { + if s.needsMutex() { + s.conn.acquire() + defer s.conn.release() + } + r, err = s.wrapped.Query(args) + return +} + +// QueryContext executes a query. +func (s *Stmt) QueryContext(ctx context.Context, args []driver.NamedValue) (r driver.Rows, err error) { + if s.needsMutex() { + s.conn.acquire() + defer s.conn.release() + } + if p, cast := s.wrapped.(driver.StmtQueryContext); cast { + r, err = p.QueryContext(ctx, args) + } else { + r, err = s.Query(s.values(args)) + } + return +} + +// values converts named-values to values. +func (s *Stmt) values(named []driver.NamedValue) (out []driver.Value) { + for i := range named { + out = append(out, named[i].Value) + } + return +} + +// needsMutex returns true when the query should is a write operation. +func (s *Stmt) needsMutex() (matched bool) { + matched = s.conn.needsMutex(s.query) + return +} + +// Tx is a transaction. +type Tx struct { + wrapped driver.Tx + conn *Conn +} + +// Commit the transaction. +// Releases the mutex. +func (t *Tx) Commit() (err error) { + defer func() { + t.conn.endTx() + t.conn.release() + }() + err = t.wrapped.Commit() + return +} + +// Rollback the transaction. +// Releases the mutex. +func (t *Tx) Rollback() (err error) { + defer func() { + t.conn.endTx() + t.conn.release() + }() + err = t.wrapped.Rollback() + return +} diff --git a/database/pk.go b/database/pk.go new file mode 100644 index 000000000..d9f19d49a --- /dev/null +++ b/database/pk.go @@ -0,0 +1,160 @@ +package database + +import ( + "errors" + "reflect" + "strings" + "sync" + + "github.com/konveyor/tackle2-hub/model" + "gorm.io/gorm" + "gorm.io/gorm/clause" + "gorm.io/gorm/logger" +) + +// PK singleton pk sequence. +var PK PkSequence + +// PkSequence provides a primary key sequence. +type PkSequence struct { + mutex sync.Mutex +} + +// Load highest key for all models. +func (r *PkSequence) Load(db *gorm.DB, models []any) (err error) { + r.mutex.Lock() + defer r.mutex.Unlock() + for _, m := range models { + mt := reflect.TypeOf(m) + if mt.Kind() == reflect.Ptr { + mt = mt.Elem() + } + kind := strings.ToUpper(mt.Name()) + db = r.session(db) + q := db.Table(kind) + q = q.Select("MAX(ID) id") + cursor, err := q.Rows() + if err != nil || !cursor.Next() { + // not a table with id. + // discarded. + continue + } + id := int64(0) + err = cursor.Scan(&id) + _ = cursor.Close() + if err != nil { + r.add(db, kind, uint(0)) + } else { + r.add(db, kind, uint(id)) + } + } + return +} + +// Next returns the next primary key. +func (r *PkSequence) Next(db *gorm.DB) (id uint) { + r.mutex.Lock() + defer r.mutex.Unlock() + kind := strings.ToUpper(db.Statement.Table) + m := &model.PK{} + db = r.session(db) + err := db.First(m, "Kind", kind).Error + if err != nil { + return + } + m.LastID++ + id = m.LastID + err = db.Save(m).Error + if err != nil { + panic(err) + } + return +} + +// session returns a new DB with a new session. +func (r *PkSequence) session(in *gorm.DB) (out *gorm.DB) { + out = &gorm.DB{ + Config: in.Config, + } + out.Config.Logger.LogMode(logger.Warn) + out.Statement = &gorm.Statement{ + DB: out, + ConnPool: in.Statement.ConnPool, + Context: in.Statement.Context, + Clauses: map[string]clause.Clause{}, + Vars: make([]interface{}, 0, 8), + } + return +} + +// add the last (higher) id for the kind. +func (r *PkSequence) add(db *gorm.DB, kind string, id uint) { + m := &model.PK{Kind: kind} + db = r.session(db) + err := db.First(m).Error + if err != nil { + if !errors.Is(err, gorm.ErrRecordNotFound) { + panic(err) + } + } + if m.LastID > id { + return + } + m.LastID = id + db = r.session(db) + err = db.Save(m).Error + if err != nil { + panic(err) + } +} + +// assignPk assigns PK as needed. +func assignPk(db *gorm.DB) { + statement := db.Statement + schema := statement.Schema + if schema == nil { + return + } + switch statement.ReflectValue.Kind() { + case reflect.Slice, + reflect.Array: + for i := 0; i < statement.ReflectValue.Len(); i++ { + for _, f := range schema.Fields { + if f.Name != "ID" { + continue + } + _, isZero := f.ValueOf( + statement.Context, + statement.ReflectValue.Index(i)) + if isZero { + id := PK.Next(db) + _ = f.Set( + statement.Context, + statement.ReflectValue.Index(i), + id) + + } + break + } + } + case reflect.Struct: + for _, f := range schema.Fields { + if f.Name != "ID" { + continue + } + _, isZero := f.ValueOf( + statement.Context, + statement.ReflectValue) + if isZero { + id := PK.Next(db) + _ = f.Set( + statement.Context, + statement.ReflectValue, + id) + } + break + } + default: + log.Info("[WARN] assignPk: unknown kind.") + } +} diff --git a/database/pkg.go b/database/pkg.go index 9f0b29cb2..83cfc4f34 100644 --- a/database/pkg.go +++ b/database/pkg.go @@ -23,6 +23,10 @@ const ( FKsOff = "&_foreign_keys=no" ) +func init() { + sql.Register("sqlite3x", &Driver{}) +} + // Open and automigrate the DB. func Open(enforceFKs bool) (db *gorm.DB, err error) { connStr := fmt.Sprintf(ConnectionString, Settings.DB.Path) @@ -31,8 +35,10 @@ func Open(enforceFKs bool) (db *gorm.DB, err error) { } else { connStr += FKsOff } + dialector := sqlite.Open(connStr).(*sqlite.Dialector) + dialector.DriverName = "sqlite3x" db, err = gorm.Open( - sqlite.Open(connStr), + dialector, &gorm.Config{ PrepareStmt: true, CreateBatchSize: 500, @@ -45,13 +51,16 @@ func Open(enforceFKs bool) (db *gorm.DB, err error) { err = liberr.Wrap(err) return } - sqlDB, err := db.DB() + err = db.AutoMigrate(model.PK{}, model.Setting{}) if err != nil { err = liberr.Wrap(err) return } - sqlDB.SetMaxOpenConns(1) - err = db.AutoMigrate(model.Setting{}) + err = PK.Load(db, []any{model.Setting{}}) + if err != nil { + return + } + err = db.Callback().Create().Before("gorm:before_create").Register("assign-pk", assignPk) if err != nil { err = liberr.Wrap(err) return diff --git a/go.mod b/go.mod index 75022634b..6ac197c8d 100644 --- a/go.mod +++ b/go.mod @@ -67,6 +67,7 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/moby/spdystream v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect diff --git a/go.sum b/go.sum index 88f63ca3c..8478cb168 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,7 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/andygrunwald/go-jira v1.16.0 h1:PU7C7Fkk5L96JvPc6vDVIrd99vdPnYudHu4ju2c2ikQ= github.com/andygrunwald/go-jira v1.16.0/go.mod h1:UQH4IBVxIYWbgagc0LF/k9FRs9xjIiQ8hIcC6HfLwFU= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= @@ -29,6 +30,7 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -121,6 +123,7 @@ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -162,6 +165,8 @@ github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6 github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= diff --git a/hack/add/analysis.sh b/hack/add/analysis.sh index bb2cb79e9..568ff3341 100755 --- a/hack/add/analysis.sh +++ b/hack/add/analysis.sh @@ -3,20 +3,18 @@ set -e host="${HOST:-localhost:8080}" -app="${1:-1}" +appId="${1:-1}" nRuleSet="${2:-10}" nIssue="${3:-10}" nIncident="${4:-25}" -aPath="/tmp/analysis.yaml" -iPath="/tmp/issues.yaml" -dPath="/tmp/deps.yaml" +tmp=/tmp/${self}-${pid} +file="/tmp/manifest.yaml" -echo " Application: ${app}" +echo " Application: ${appId}" echo " RuleSets: ${nRuleSet}" echo " Issues: ${nIssue}" echo " Incidents: ${nIncident}" -echo " Issues path: ${iPath}" -echo " Deps path: ${dPath}" +echo " Manifest path: ${file}" sources=( konveyor.io/source=oraclejdk @@ -62,11 +60,18 @@ konveyor.io/target=hibernate konveyor.io/target=jbpm ) +# +# Analysis +# +printf "\x1DBEGIN-MAIN\x1D\n" > ${file} +echo -n "--- +commit: "1234" +" >> ${file} +printf "\x1DEND-MAIN\x1D\n" >> ${file} # # Issues # -file=${iPath} -echo "" > ${file} +printf "\x1DBEGIN-ISSUES\x1D\n" >> ${file} for r in $(seq 1 ${nRuleSet}) do for i in $(seq 1 ${nIssue}) @@ -150,17 +155,18 @@ fi done done done +printf "\x1DEND-ISSUES\x1D +\x1DBEGIN-DEPS\x1D\n" >> ${file} # # Deps # -file=${dPath} echo -n "--- name: github.com/jboss version: 4.0 labels: - konveyor.io/language=java - konveyor.io/otherA=dog -" > ${file} +" >> ${file} echo -n "--- name: github.com/jboss version: 5.0 @@ -192,23 +198,28 @@ echo -n "--- name: github.com/java version: 8 " >> ${file} -# -# Analysis -# -file=${aPath} -echo -n "--- -commit: "42b22a90" -issues: -dependencies: -" > ${file} +printf "\x1DEND-DEPS\x1D\n" >> ${file} -echo "Report CREATED" +echo "Manifest (file) GENERATED: ${file}" -mime="application/x-yaml" +# +# Post manifest. +code=$(curl -kSs -o ${tmp} -w "%{http_code}" \ + -F "file=@${file};type=application/x-yaml" \ + -H 'Accept:application/x-yaml' \ + http://${host}/applications/${appId}/analyses) +if [ ! $? -eq 0 ] +then + exit $? +fi +case ${code} in + 201) + echo "Analysis: created." + cat ${tmp} + ;; + *) + echo "Analysis create - FAILED: ${code}." + cat ${tmp} + exit 1 +esac -curl \ - -F "file=@${aPath};type=${mime}" \ - -F "issues=@${iPath};type=${mime}" \ - -F "dependencies=@${dPath};type=${mime}" \ - ${host}/applications/${app}/analyses \ - -H "Accept:${mime}" diff --git a/hack/cmd/addon/main.go b/hack/cmd/addon/main.go index 3b5ddb5d3..88cb13333 100644 --- a/hack/cmd/addon/main.go +++ b/hack/cmd/addon/main.go @@ -9,16 +9,17 @@ package main import ( "bytes" "errors" - hub "github.com/konveyor/tackle2-hub/addon" - "github.com/konveyor/tackle2-hub/api" - "github.com/konveyor/tackle2-hub/nas" - "k8s.io/apimachinery/pkg/util/rand" "os" "os/exec" pathlib "path" "strconv" "strings" "time" + + hub "github.com/konveyor/tackle2-hub/addon" + "github.com/konveyor/tackle2-hub/api" + "github.com/konveyor/tackle2-hub/nas" + "k8s.io/apimachinery/pkg/util/rand" ) var ( @@ -32,7 +33,6 @@ const ( TmpDir = "/tmp/list" ) -// // main func main() { addon.Run(func() (err error) { @@ -76,7 +76,7 @@ func main() { // // Replace facts. err = facts.Replace( - api.FactMap{ + api.Map{ "Listed": true, "Color": "blue", "Length": 100, @@ -100,7 +100,6 @@ func main() { }) } -// // listDir builds and populates the bucket. func listDir(d *Data, application *api.Application, paths []string) (err error) { // @@ -178,7 +177,6 @@ func listDir(d *Data, application *api.Application, paths []string) (err error) return } -// // playWithBucket func playWithBucket(bucket *hub.BucketContent) (err error) { tmpDir := tmpDir() @@ -215,7 +213,6 @@ func playWithBucket(bucket *hub.BucketContent) (err error) { return } -// // Build index.html func buildIndex(output string) (err error) { addon.Activity("Building index.") @@ -247,7 +244,6 @@ func buildIndex(output string) (err error) { return } -// // find files. func find(path string, max int) (paths []string, err error) { Log.Info("Listing.", "path", path) @@ -279,7 +275,6 @@ func find(path string, max int) (paths []string, err error) { return } -// // Play with files. func playWithFiles() (err error) { f, err := addon.File.Put("/etc/hosts") @@ -301,7 +296,6 @@ func playWithFiles() (err error) { return } -// // addTags ensure tags created and associated with application. // Ensure tag exists and associated with the application. func addTags(application *api.Application, source string, names ...string) (err error) { @@ -337,7 +331,6 @@ func addTags(application *api.Application, source string, names ...string) (err return } -// // replaceTags replaces current set of tags for the source with a new set. // Ensures desired tags exist before replacing. func replaceTags(application *api.Application, source string, names ...string) (err error) { @@ -395,7 +388,6 @@ func tmpDir() (p string) { return } -// // Data Addon input. type Data struct { // Path to be listed. diff --git a/importer/manager.go b/importer/manager.go index b11f45e4b..540214e2c 100644 --- a/importer/manager.go +++ b/importer/manager.go @@ -2,7 +2,6 @@ package importer import ( "context" - "encoding/json" "fmt" "regexp" @@ -136,7 +135,7 @@ func (m *Manager) createApplication(imp *model.Import) (ok bool) { return } - repository := api.Repository{ + repository := model.Repository{ Kind: imp.RepositoryKind, URL: imp.RepositoryURL, Branch: imp.RepositoryBranch, @@ -148,8 +147,7 @@ func (m *Manager) createApplication(imp *model.Import) (ok bool) { repository.Kind = "git" } - app.Repository, _ = json.Marshal(repository) - + app.Repository = repository // Validate Binary-related fields (allow all 3 empty or present) if imp.BinaryGroup != "" || imp.BinaryArtifact != "" || imp.BinaryVersion != "" { if imp.BinaryGroup == "" || imp.BinaryArtifact == "" || imp.BinaryVersion == "" { diff --git a/migration/json/fields.go b/migration/json/fields.go index 926aa40b4..b79cd6529 100644 --- a/migration/json/fields.go +++ b/migration/json/fields.go @@ -1,6 +1,8 @@ package json -import "gopkg.in/yaml.v2" +import ( + "gopkg.in/yaml.v2" +) // Ref represents a FK. type Ref struct { diff --git a/migration/migrate.go b/migration/migrate.go index 4c8ec8b71..43f582a79 100644 --- a/migration/migrate.go +++ b/migration/migrate.go @@ -1,7 +1,6 @@ package migration import ( - "encoding/json" "errors" "os" "path" @@ -44,12 +43,9 @@ func Migrate(migrations []Migration) (err error) { } var v Version - if setting.Value != nil { - err = json.Unmarshal(setting.Value, &v) - if err != nil { - err = liberr.Wrap(err) - return - } + err = setting.As(&v) + if err != nil { + return } var start = v.Version if start != 0 && start < MinimumVersion { @@ -112,9 +108,7 @@ func Migrate(migrations []Migration) (err error) { // Set the version record. func setVersion(db *gorm.DB, version int) (err error) { setting := &model.Setting{Key: VersionKey} - v := Version{Version: version} - value, _ := json.Marshal(v) - setting.Value = value + setting.Value = Version{Version: version} result := db.Where("key", VersionKey).Updates(setting) if result.Error != nil { err = liberr.Wrap(result.Error) diff --git a/migration/migrate_test.go b/migration/migrate_test.go index 943a85a1a..a82e59fa3 100644 --- a/migration/migrate_test.go +++ b/migration/migrate_test.go @@ -1,7 +1,6 @@ package migration import ( - "encoding/json" "os" "testing" @@ -132,7 +131,7 @@ func expectVersion(g *gomega.GomegaWithT, version int) { result := db.Find(setting, "key", VersionKey) g.Expect(result.Error).To(gomega.BeNil()) var v Version - _ = json.Unmarshal(setting.Value, &v) + _ = setting.As(&v) g.Expect(v.Version).To(gomega.Equal(version)) _ = database.Close(db) } diff --git a/migration/pkg.go b/migration/pkg.go index f80ed8e48..5caaecbd8 100644 --- a/migration/pkg.go +++ b/migration/pkg.go @@ -7,6 +7,7 @@ import ( v12 "github.com/konveyor/tackle2-hub/migration/v12" v13 "github.com/konveyor/tackle2-hub/migration/v13" v14 "github.com/konveyor/tackle2-hub/migration/v14" + v15 "github.com/konveyor/tackle2-hub/migration/v15" v2 "github.com/konveyor/tackle2-hub/migration/v2" v3 "github.com/konveyor/tackle2-hub/migration/v3" v4 "github.com/konveyor/tackle2-hub/migration/v4" @@ -56,5 +57,6 @@ func All() []Migration { v12.Migration{}, v13.Migration{}, v14.Migration{}, + v15.Migration{}, } } diff --git a/migration/v14/model/analysis.go b/migration/v14/model/analysis.go index 4b61e1f40..bc33da893 100644 --- a/migration/v14/model/analysis.go +++ b/migration/v14/model/analysis.go @@ -1,6 +1,9 @@ package model -import "gorm.io/gorm" +import ( + "github.com/konveyor/tackle2-hub/migration/json" + "gorm.io/gorm" +) // Analysis report. type Analysis struct { @@ -8,7 +11,7 @@ type Analysis struct { Effort int Commit string Archived bool - Summary JSON `gorm:"type:json"` + Summary []ArchivedIssue `gorm:"type:json;serializer:json"` Issues []Issue `gorm:"constraint:OnDelete:CASCADE"` Dependencies []TechDependency `gorm:"constraint:OnDelete:CASCADE"` ApplicationID uint `gorm:"index;not null"` @@ -23,8 +26,8 @@ type TechDependency struct { Version string `gorm:"uniqueIndex:depA"` SHA string `gorm:"uniqueIndex:depA"` Indirect bool - Labels JSON `gorm:"type:json"` - AnalysisID uint `gorm:"index;uniqueIndex:depA;not null"` + Labels []string `gorm:"type:json;serializer:json"` + AnalysisID uint `gorm:"index;uniqueIndex:depA;not null"` Analysis *Analysis } @@ -37,9 +40,9 @@ type Issue struct { Description string Category string `gorm:"index;not null"` Incidents []Incident `gorm:"foreignKey:IssueID;constraint:OnDelete:CASCADE"` - Links JSON `gorm:"type:json"` - Facts JSON `gorm:"type:json"` - Labels JSON `gorm:"type:json"` + Links []Link `gorm:"type:json;serializer:json"` + Facts json.Map `gorm:"type:json;serializer:json"` + Labels []string `gorm:"type:json;serializer:json"` Effort int `gorm:"index;not null"` AnalysisID uint `gorm:"index;uniqueIndex:issueA;not null"` Analysis *Analysis @@ -52,28 +55,11 @@ type Incident struct { Line int Message string CodeSnip string - Facts JSON `gorm:"type:json"` - IssueID uint `gorm:"index;not null"` + Facts json.Map `gorm:"type:json;serializer:json"` + IssueID uint `gorm:"index;not null"` Issue *Issue } -// Link URL link. -type Link struct { - URL string `json:"url"` - Title string `json:"title,omitempty"` -} - -// ArchivedIssue resource created when issues are archived. -type ArchivedIssue struct { - RuleSet string `json:"ruleSet"` - Rule string `json:"rule"` - Name string `json:"name,omitempty" yaml:",omitempty"` - Description string `json:"description,omitempty" yaml:",omitempty"` - Category string `json:"category"` - Effort int `json:"effort"` - Incidents int `json:"incidents"` -} - // RuleSet - Analysis ruleset. type RuleSet struct { Model @@ -81,8 +67,8 @@ type RuleSet struct { Kind string Name string `gorm:"uniqueIndex;not null"` Description string - Repository JSON `gorm:"type:json"` - IdentityID *uint `gorm:"index"` + Repository Repository `gorm:"type:json;serializer:json"` + IdentityID *uint `gorm:"index"` Identity *Identity Rules []Rule `gorm:"constraint:OnDelete:CASCADE"` DependsOn []RuleSet `gorm:"many2many:RuleSetDependencies;constraint:OnDelete:CASCADE"` @@ -130,8 +116,8 @@ type Rule struct { Model Name string Description string - Labels JSON `gorm:"type:json"` - RuleSetID uint `gorm:"uniqueIndex:RuleA;not null"` + Labels []string `gorm:"type:json;serializer:json"` + RuleSetID uint `gorm:"uniqueIndex:RuleA;not null"` RuleSet *RuleSet FileID *uint `gorm:"uniqueIndex:RuleA" ref:"file"` File *File @@ -145,8 +131,8 @@ type Target struct { Description string Provider string Choice bool - Labels JSON `gorm:"type:json"` - ImageID uint `gorm:"index" ref:"file"` + Labels []TargetLabel `gorm:"type:json;serializer:json"` + ImageID uint `gorm:"index" ref:"file"` Image *File RuleSetID *uint `gorm:"index"` RuleSet *RuleSet @@ -155,3 +141,30 @@ type Target struct { func (r *Target) Builtin() bool { return r.UUID != nil } + +// +// JSON Fields. +// + +// ArchivedIssue resource created when issues are archived. +type ArchivedIssue struct { + RuleSet string `json:"ruleSet"` + Rule string `json:"rule"` + Name string `json:"name,omitempty" yaml:",omitempty"` + Description string `json:"description,omitempty" yaml:",omitempty"` + Category string `json:"category"` + Effort int `json:"effort"` + Incidents int `json:"incidents"` +} + +// Link URL link. +type Link struct { + URL string `json:"url"` + Title string `json:"title,omitempty"` +} + +// TargetLabel - label format specific to Targets +type TargetLabel struct { + Name string `json:"name"` + Label string `json:"label"` +} diff --git a/migration/v14/model/application.go b/migration/v14/model/application.go index 49a9a959a..cb363b78d 100644 --- a/migration/v14/model/application.go +++ b/migration/v14/model/application.go @@ -5,6 +5,7 @@ import ( "sync" "time" + "github.com/konveyor/tackle2-hub/migration/json" "gorm.io/gorm" ) @@ -13,8 +14,8 @@ type Application struct { BucketOwner Name string `gorm:"index;unique;not null"` Description string - Review *Review `gorm:"constraint:OnDelete:CASCADE"` - Repository JSON `gorm:"type:json"` + Review *Review `gorm:"constraint:OnDelete:CASCADE"` + Repository Repository `gorm:"type:json;serializer:json"` Binary string Facts []Fact `gorm:"constraint:OnDelete:CASCADE"` Comments string @@ -37,7 +38,7 @@ type Fact struct { ApplicationID uint `gorm:"<-:create;primaryKey"` Key string `gorm:"<-:create;primaryKey"` Source string `gorm:"<-:create;primaryKey;not null"` - Value JSON `gorm:"type:json;not null"` + Value any `gorm:"type:json;not null;serializer:json"` Application *Application } @@ -195,7 +196,7 @@ type Ticket struct { // Parent resource that this ticket should belong to in the tracker. (e.g. Jira project) Parent string `gorm:"not null"` // Custom fields to send to the tracker when creating the ticket - Fields JSON `gorm:"type:json"` + Fields json.Map `gorm:"type:json;serializer:json"` // Whether the last attempt to do something with the ticket reported an error Error bool // Error message, if any @@ -297,6 +298,11 @@ type ImportTag struct { Import *Import } +// +// JSON Fields. +// + +// Repository represents an SCM repository. type Repository struct { Kind string `json:"kind"` URL string `json:"url"` diff --git a/migration/v14/model/assessment.go b/migration/v14/model/assessment.go index 3a734e86e..0b51e714d 100644 --- a/migration/v14/model/assessment.go +++ b/migration/v14/model/assessment.go @@ -6,9 +6,9 @@ type Questionnaire struct { Name string `gorm:"unique"` Description string Required bool - Sections JSON `gorm:"type:json"` - Thresholds JSON `gorm:"type:json"` - RiskMessages JSON `gorm:"type:json"` + Sections []Section `gorm:"type:json;serializer:json"` + Thresholds Thresholds `gorm:"type:json;serializer:json"` + RiskMessages RiskMessages `gorm:"type:json;serializer:json"` Assessments []Assessment `gorm:"constraint:OnDelete:CASCADE"` } @@ -25,9 +25,9 @@ type Assessment struct { Archetype *Archetype QuestionnaireID uint `gorm:"uniqueIndex:AssessmentA;uniqueIndex:AssessmentB"` Questionnaire Questionnaire - Sections JSON `gorm:"type:json"` - Thresholds JSON `gorm:"type:json"` - RiskMessages JSON `gorm:"type:json"` + Sections []Section `gorm:"type:json;serializer:json"` + Thresholds Thresholds `gorm:"type:json;serializer:json"` + RiskMessages RiskMessages `gorm:"type:json;serializer:json"` Stakeholders []Stakeholder `gorm:"many2many:AssessmentStakeholders;constraint:OnDelete:CASCADE"` StakeholderGroups []StakeholderGroup `gorm:"many2many:AssessmentStakeholderGroups;constraint:OnDelete:CASCADE"` } @@ -44,3 +44,59 @@ type Review struct { ArchetypeID *uint `gorm:"uniqueIndex"` Archetype *Archetype } + +// +// JSON Fields. +// + +// Section represents a group of questions in a questionnaire. +type Section struct { + Order uint `json:"order" yaml:"order"` + Name string `json:"name" yaml:"name"` + Questions []Question `json:"questions" yaml:"questions" binding:"min=1,dive"` + Comment string `json:"comment,omitempty" yaml:"comment,omitempty"` +} + +// Question represents a question in a questionnaire. +type Question struct { + Order uint `json:"order" yaml:"order"` + Text string `json:"text" yaml:"text"` + Explanation string `json:"explanation" yaml:"explanation"` + IncludeFor []CategorizedTag `json:"includeFor,omitempty" yaml:"includeFor,omitempty"` + ExcludeFor []CategorizedTag `json:"excludeFor,omitempty" yaml:"excludeFor,omitempty"` + Answers []Answer `json:"answers" yaml:"answers" binding:"min=1,dive"` +} + +// Answer represents an answer to a question in a questionnaire. +type Answer struct { + Order uint `json:"order" yaml:"order"` + Text string `json:"text" yaml:"text"` + Risk string `json:"risk" yaml:"risk" binding:"oneof=red yellow green unknown"` + Rationale string `json:"rationale" yaml:"rationale"` + Mitigation string `json:"mitigation" yaml:"mitigation"` + ApplyTags []CategorizedTag `json:"applyTags,omitempty" yaml:"applyTags,omitempty"` + AutoAnswerFor []CategorizedTag `json:"autoAnswerFor,omitempty" yaml:"autoAnswerFor,omitempty"` + Selected bool `json:"selected,omitempty" yaml:"selected,omitempty"` + AutoAnswered bool `json:"autoAnswered,omitempty" yaml:"autoAnswered,omitempty"` +} + +// CategorizedTag represents a human-readable pair of category and tag. +type CategorizedTag struct { + Category string `json:"category" yaml:"category"` + Tag string `json:"tag" yaml:"tag"` +} + +// RiskMessages contains messages to display for each risk level. +type RiskMessages struct { + Red string `json:"red" yaml:"red"` + Yellow string `json:"yellow" yaml:"yellow"` + Green string `json:"green" yaml:"green"` + Unknown string `json:"unknown" yaml:"unknown"` +} + +// Thresholds contains the threshold values for determining risk for the questionnaire. +type Thresholds struct { + Red uint `json:"red" yaml:"red"` + Yellow uint `json:"yellow" yaml:"yellow"` + Unknown uint `json:"unknown" yaml:"unknown"` +} diff --git a/migration/v14/model/core.go b/migration/v14/model/core.go index 06c02bd6d..4f384bb6b 100644 --- a/migration/v14/model/core.go +++ b/migration/v14/model/core.go @@ -20,25 +20,26 @@ type Model struct { UpdateUser string } +// PK sequence. +type PK struct { + Kind string `gorm:"<-:create;primaryKey"` + LastID uint +} + +// Setting hub settings. type Setting struct { Model Key string `gorm:"<-:create;uniqueIndex"` - Value JSON `gorm:"type:json"` + Value any `gorm:"type:json;serializer:json"` } -// With updates the value of the Setting with the json representation -// of the `value` parameter. -func (r *Setting) With(value any) (err error) { - r.Value, err = json.Marshal(value) +// As unmarshalls the value of the Setting into the `ptr` parameter. +func (r *Setting) As(ptr any) (err error) { + bytes, err := json.Marshal(r.Value) if err != nil { err = liberr.Wrap(err) } - return -} - -// As unmarshalls the value of the Setting into the `ptr` parameter. -func (r *Setting) As(ptr any) (err error) { - err = json.Unmarshal(r.Value, ptr) + err = json.Unmarshal(bytes, ptr) if err != nil { err = liberr.Wrap(err) } @@ -146,26 +147,6 @@ func (m *Task) BeforeCreate(db *gorm.DB) (err error) { return } -// TaskError used in Task.Errors. -type TaskError struct { - Severity string `json:"severity"` - Description string `json:"description"` -} - -// TaskPolicy scheduling policy. -type TaskPolicy struct { - Isolated bool `json:"isolated,omitempty" yaml:",omitempty"` - PreemptEnabled bool `json:"preemptEnabled,omitempty" yaml:"preemptEnabled,omitempty"` - PreemptExempt bool `json:"preemptExempt,omitempty" yaml:"preemptExempt,omitempty"` -} - -// Attachment file attachment. -type Attachment struct { - ID uint `json:"id" binding:"required"` - Name string `json:"name,omitempty" yaml:",omitempty"` - Activity int `json:"activity,omitempty" yaml:",omitempty"` -} - type TaskReport struct { Model Status string @@ -202,8 +183,8 @@ type Proxy struct { Kind string `gorm:"uniqueIndex"` Host string `gorm:"not null"` Port int - Excluded JSON `gorm:"type:json"` - IdentityID *uint `gorm:"index"` + Excluded []string `gorm:"type:json;serializer:json"` + IdentityID *uint `gorm:"index"` Identity *Identity } @@ -289,6 +270,19 @@ func (r *Identity) Decrypt() (err error) { // JSON Fields. // +// Attachment file attachment. +type Attachment struct { + ID uint `json:"id" binding:"required"` + Name string `json:"name,omitempty" yaml:",omitempty"` + Activity int `json:"activity,omitempty" yaml:",omitempty"` +} + +// TaskError used in Task.Errors. +type TaskError struct { + Severity string `json:"severity"` + Description string `json:"description"` +} + // TaskEvent task event. type TaskEvent struct { Kind string `json:"kind"` @@ -297,6 +291,13 @@ type TaskEvent struct { Last time.Time `json:"last"` } +// TaskPolicy scheduling policy. +type TaskPolicy struct { + Isolated bool `json:"isolated,omitempty" yaml:",omitempty"` + PreemptEnabled bool `json:"preemptEnabled,omitempty" yaml:"preemptEnabled,omitempty"` + PreemptExempt bool `json:"preemptExempt,omitempty" yaml:"preemptExempt,omitempty"` +} + // TTL time-to-live. type TTL struct { Created int `json:"created,omitempty" yaml:",omitempty"` diff --git a/migration/v14/model/pkg.go b/migration/v14/model/pkg.go index 6827e3c96..8f612b488 100644 --- a/migration/v14/model/pkg.go +++ b/migration/v14/model/pkg.go @@ -32,6 +32,7 @@ func All() []any { ImportTag{}, JobFunction{}, MigrationWave{}, + PK{}, Proxy{}, Review{}, Setting{}, diff --git a/migration/v15/migrate.go b/migration/v15/migrate.go new file mode 100644 index 000000000..1fe82e975 --- /dev/null +++ b/migration/v15/migrate.go @@ -0,0 +1,20 @@ +package v15 + +import ( + "github.com/jortel/go-utils/logr" + "github.com/konveyor/tackle2-hub/migration/v15/model" + "gorm.io/gorm" +) + +var log = logr.WithName("migration|v15") + +type Migration struct{} + +func (r Migration) Apply(db *gorm.DB) (err error) { + err = db.AutoMigrate(r.Models()...) + return +} + +func (r Migration) Models() []interface{} { + return model.All() +} diff --git a/migration/v15/model/analysis.go b/migration/v15/model/analysis.go new file mode 100644 index 000000000..bc33da893 --- /dev/null +++ b/migration/v15/model/analysis.go @@ -0,0 +1,170 @@ +package model + +import ( + "github.com/konveyor/tackle2-hub/migration/json" + "gorm.io/gorm" +) + +// Analysis report. +type Analysis struct { + Model + Effort int + Commit string + Archived bool + Summary []ArchivedIssue `gorm:"type:json;serializer:json"` + Issues []Issue `gorm:"constraint:OnDelete:CASCADE"` + Dependencies []TechDependency `gorm:"constraint:OnDelete:CASCADE"` + ApplicationID uint `gorm:"index;not null"` + Application *Application +} + +// TechDependency report dependency. +type TechDependency struct { + Model + Provider string `gorm:"uniqueIndex:depA"` + Name string `gorm:"uniqueIndex:depA"` + Version string `gorm:"uniqueIndex:depA"` + SHA string `gorm:"uniqueIndex:depA"` + Indirect bool + Labels []string `gorm:"type:json;serializer:json"` + AnalysisID uint `gorm:"index;uniqueIndex:depA;not null"` + Analysis *Analysis +} + +// Issue report issue (violation). +type Issue struct { + Model + RuleSet string `gorm:"uniqueIndex:issueA;not null"` + Rule string `gorm:"uniqueIndex:issueA;not null"` + Name string `gorm:"index"` + Description string + Category string `gorm:"index;not null"` + Incidents []Incident `gorm:"foreignKey:IssueID;constraint:OnDelete:CASCADE"` + Links []Link `gorm:"type:json;serializer:json"` + Facts json.Map `gorm:"type:json;serializer:json"` + Labels []string `gorm:"type:json;serializer:json"` + Effort int `gorm:"index;not null"` + AnalysisID uint `gorm:"index;uniqueIndex:issueA;not null"` + Analysis *Analysis +} + +// Incident report an issue incident. +type Incident struct { + Model + File string `gorm:"index;not null"` + Line int + Message string + CodeSnip string + Facts json.Map `gorm:"type:json;serializer:json"` + IssueID uint `gorm:"index;not null"` + Issue *Issue +} + +// RuleSet - Analysis ruleset. +type RuleSet struct { + Model + UUID *string `gorm:"uniqueIndex"` + Kind string + Name string `gorm:"uniqueIndex;not null"` + Description string + Repository Repository `gorm:"type:json;serializer:json"` + IdentityID *uint `gorm:"index"` + Identity *Identity + Rules []Rule `gorm:"constraint:OnDelete:CASCADE"` + DependsOn []RuleSet `gorm:"many2many:RuleSetDependencies;constraint:OnDelete:CASCADE"` +} + +func (r *RuleSet) Builtin() bool { + return r.UUID != nil +} + +// BeforeUpdate hook to avoid cyclic dependencies. +func (r *RuleSet) BeforeUpdate(db *gorm.DB) (err error) { + seen := make(map[uint]bool) + var nextDeps []RuleSet + var nextRuleSetIDs []uint + for _, dep := range r.DependsOn { + nextRuleSetIDs = append(nextRuleSetIDs, dep.ID) + } + for len(nextRuleSetIDs) != 0 { + result := db.Preload("DependsOn").Where("ID IN ?", nextRuleSetIDs).Find(&nextDeps) + if result.Error != nil { + err = result.Error + return + } + nextRuleSetIDs = nextRuleSetIDs[:0] + for _, nextDep := range nextDeps { + for _, dep := range nextDep.DependsOn { + if seen[dep.ID] { + continue + } + if dep.ID == r.ID { + err = DependencyCyclicError{} + return + } + seen[dep.ID] = true + nextRuleSetIDs = append(nextRuleSetIDs, dep.ID) + } + } + } + + return +} + +// Rule - Analysis rule. +type Rule struct { + Model + Name string + Description string + Labels []string `gorm:"type:json;serializer:json"` + RuleSetID uint `gorm:"uniqueIndex:RuleA;not null"` + RuleSet *RuleSet + FileID *uint `gorm:"uniqueIndex:RuleA" ref:"file"` + File *File +} + +// Target - analysis rule selector. +type Target struct { + Model + UUID *string `gorm:"uniqueIndex"` + Name string `gorm:"uniqueIndex;not null"` + Description string + Provider string + Choice bool + Labels []TargetLabel `gorm:"type:json;serializer:json"` + ImageID uint `gorm:"index" ref:"file"` + Image *File + RuleSetID *uint `gorm:"index"` + RuleSet *RuleSet +} + +func (r *Target) Builtin() bool { + return r.UUID != nil +} + +// +// JSON Fields. +// + +// ArchivedIssue resource created when issues are archived. +type ArchivedIssue struct { + RuleSet string `json:"ruleSet"` + Rule string `json:"rule"` + Name string `json:"name,omitempty" yaml:",omitempty"` + Description string `json:"description,omitempty" yaml:",omitempty"` + Category string `json:"category"` + Effort int `json:"effort"` + Incidents int `json:"incidents"` +} + +// Link URL link. +type Link struct { + URL string `json:"url"` + Title string `json:"title,omitempty"` +} + +// TargetLabel - label format specific to Targets +type TargetLabel struct { + Name string `json:"name"` + Label string `json:"label"` +} diff --git a/migration/v15/model/application.go b/migration/v15/model/application.go new file mode 100644 index 000000000..cb363b78d --- /dev/null +++ b/migration/v15/model/application.go @@ -0,0 +1,312 @@ +package model + +import ( + "fmt" + "sync" + "time" + + "github.com/konveyor/tackle2-hub/migration/json" + "gorm.io/gorm" +) + +type Application struct { + Model + BucketOwner + Name string `gorm:"index;unique;not null"` + Description string + Review *Review `gorm:"constraint:OnDelete:CASCADE"` + Repository Repository `gorm:"type:json;serializer:json"` + Binary string + Facts []Fact `gorm:"constraint:OnDelete:CASCADE"` + Comments string + Tasks []Task `gorm:"constraint:OnDelete:CASCADE"` + Tags []Tag `gorm:"many2many:ApplicationTags"` + Identities []Identity `gorm:"many2many:ApplicationIdentity;constraint:OnDelete:CASCADE"` + BusinessServiceID *uint `gorm:"index"` + BusinessService *BusinessService + OwnerID *uint `gorm:"index"` + Owner *Stakeholder `gorm:"foreignKey:OwnerID"` + Contributors []Stakeholder `gorm:"many2many:ApplicationContributors;constraint:OnDelete:CASCADE"` + Analyses []Analysis `gorm:"constraint:OnDelete:CASCADE"` + MigrationWaveID *uint `gorm:"index"` + MigrationWave *MigrationWave + Ticket *Ticket `gorm:"constraint:OnDelete:CASCADE"` + Assessments []Assessment `gorm:"constraint:OnDelete:CASCADE"` +} + +type Fact struct { + ApplicationID uint `gorm:"<-:create;primaryKey"` + Key string `gorm:"<-:create;primaryKey"` + Source string `gorm:"<-:create;primaryKey;not null"` + Value any `gorm:"type:json;not null;serializer:json"` + Application *Application +} + +// ApplicationTag represents a row in the join table for the +// many-to-many relationship between Applications and Tags. +type ApplicationTag struct { + ApplicationID uint `gorm:"primaryKey"` + TagID uint `gorm:"primaryKey"` + Source string `gorm:"primaryKey;not null"` + Application Application `gorm:"constraint:OnDelete:CASCADE"` + Tag Tag `gorm:"constraint:OnDelete:CASCADE"` +} + +// TableName must return "ApplicationTags" to ensure compatibility +// with the autogenerated join table name. +func (ApplicationTag) TableName() string { + return "ApplicationTags" +} + +// depMutex ensures Dependency.Create() is not executed concurrently. +var depMutex sync.Mutex + +type Dependency struct { + Model + ToID uint `gorm:"index"` + To *Application `gorm:"foreignKey:ToID;constraint:OnDelete:CASCADE"` + FromID uint `gorm:"index"` + From *Application `gorm:"foreignKey:FromID;constraint:OnDelete:CASCADE"` +} + +// Create a dependency synchronized using a mutex. +func (r *Dependency) Create(db *gorm.DB) (err error) { + depMutex.Lock() + defer depMutex.Unlock() + err = db.Create(r).Error + return +} + +// BeforeCreate detects cyclic dependencies. +func (r *Dependency) BeforeCreate(db *gorm.DB) (err error) { + var nextDeps []*Dependency + var nextAppsIDs []uint + nextAppsIDs = append(nextAppsIDs, r.FromID) + for len(nextAppsIDs) != 0 { + db.Where("ToID IN ?", nextAppsIDs).Find(&nextDeps) + nextAppsIDs = nextAppsIDs[:0] // empty array, but keep capacity + for _, nextDep := range nextDeps { + if nextDep.FromID == r.ToID { + err = DependencyCyclicError{} + return + } + nextAppsIDs = append(nextAppsIDs, nextDep.FromID) + } + } + + return +} + +// DependencyCyclicError reports cyclic Dependency error. +type DependencyCyclicError struct{} + +func (e DependencyCyclicError) Error() string { + return "Cyclic dependencies are not permitted." +} + +type BusinessService struct { + Model + Name string `gorm:"index;unique;not null"` + Description string + Applications []Application `gorm:"constraint:OnDelete:SET NULL"` + StakeholderID *uint `gorm:"index"` + Stakeholder *Stakeholder +} + +type JobFunction struct { + Model + UUID *string `gorm:"uniqueIndex"` + Username string + Name string `gorm:"index;unique;not null"` + Stakeholders []Stakeholder `gorm:"constraint:OnDelete:SET NULL"` +} + +type Stakeholder struct { + Model + Name string `gorm:"not null;"` + Email string `gorm:"index;unique;not null"` + Groups []StakeholderGroup `gorm:"many2many:StakeholderGroupStakeholder;constraint:OnDelete:CASCADE"` + BusinessServices []BusinessService `gorm:"constraint:OnDelete:SET NULL"` + JobFunctionID *uint `gorm:"index"` + JobFunction *JobFunction + Owns []Application `gorm:"foreignKey:OwnerID;constraint:OnDelete:SET NULL"` + Contributes []Application `gorm:"many2many:ApplicationContributors;constraint:OnDelete:CASCADE"` + MigrationWaves []MigrationWave `gorm:"many2many:MigrationWaveStakeholders;constraint:OnDelete:CASCADE"` + Assessments []Assessment `gorm:"many2many:AssessmentStakeholders;constraint:OnDelete:CASCADE"` + Archetypes []Archetype `gorm:"many2many:ArchetypeStakeholders;constraint:OnDelete:CASCADE"` +} + +type StakeholderGroup struct { + Model + Name string `gorm:"index;unique;not null"` + Username string + Description string + Stakeholders []Stakeholder `gorm:"many2many:StakeholderGroupStakeholder;constraint:OnDelete:CASCADE"` + MigrationWaves []MigrationWave `gorm:"many2many:MigrationWaveStakeholderGroups;constraint:OnDelete:CASCADE"` + Assessments []Assessment `gorm:"many2many:AssessmentStakeholderGroups;constraint:OnDelete:CASCADE"` + Archetypes []Archetype `gorm:"many2many:ArchetypeStakeholderGroups;constraint:OnDelete:CASCADE"` +} + +type MigrationWave struct { + Model + Name string `gorm:"uniqueIndex:MigrationWaveA"` + StartDate time.Time `gorm:"uniqueIndex:MigrationWaveA"` + EndDate time.Time `gorm:"uniqueIndex:MigrationWaveA"` + Applications []Application `gorm:"constraint:OnDelete:SET NULL"` + Stakeholders []Stakeholder `gorm:"many2many:MigrationWaveStakeholders;constraint:OnDelete:CASCADE"` + StakeholderGroups []StakeholderGroup `gorm:"many2many:MigrationWaveStakeholderGroups;constraint:OnDelete:CASCADE"` +} + +type Archetype struct { + Model + Name string + Description string + Comments string + Review *Review `gorm:"constraint:OnDelete:CASCADE"` + Assessments []Assessment `gorm:"constraint:OnDelete:CASCADE"` + CriteriaTags []Tag `gorm:"many2many:ArchetypeCriteriaTags;constraint:OnDelete:CASCADE"` + Tags []Tag `gorm:"many2many:ArchetypeTags;constraint:OnDelete:CASCADE"` + Stakeholders []Stakeholder `gorm:"many2many:ArchetypeStakeholders;constraint:OnDelete:CASCADE"` + StakeholderGroups []StakeholderGroup `gorm:"many2many:ArchetypeStakeholderGroups;constraint:OnDelete:CASCADE"` +} + +type Tag struct { + Model + UUID *string `gorm:"uniqueIndex"` + Name string `gorm:"uniqueIndex:tagA;not null"` + Username string + CategoryID uint `gorm:"uniqueIndex:tagA;index;not null"` + Category TagCategory +} + +type TagCategory struct { + Model + UUID *string `gorm:"uniqueIndex"` + Name string `gorm:"index;unique;not null"` + Username string + Rank uint + Color string + Tags []Tag `gorm:"foreignKey:CategoryID;constraint:OnDelete:CASCADE"` +} + +type Ticket struct { + Model + // Kind of ticket in the external tracker. + Kind string `gorm:"not null"` + // Parent resource that this ticket should belong to in the tracker. (e.g. Jira project) + Parent string `gorm:"not null"` + // Custom fields to send to the tracker when creating the ticket + Fields json.Map `gorm:"type:json;serializer:json"` + // Whether the last attempt to do something with the ticket reported an error + Error bool + // Error message, if any + Message string + // Whether the ticket was created in the external tracker + Created bool + // Reference id in external tracker + Reference string + // URL to ticket in external tracker + Link string + // Status of ticket in external tracker + Status string + LastUpdated time.Time + Application *Application + ApplicationID uint `gorm:"uniqueIndex:ticketA;not null"` + Tracker *Tracker + TrackerID uint `gorm:"uniqueIndex:ticketA;not null"` +} + +type Tracker struct { + Model + Name string `gorm:"index;unique;not null"` + URL string + Kind string + Identity *Identity + IdentityID uint + Connected bool + LastUpdated time.Time + Message string + Insecure bool + Tickets []Ticket +} + +type Import struct { + Model + Filename string + ApplicationName string + BusinessService string + Comments string + Dependency string + DependencyDirection string + Description string + ErrorMessage string + IsValid bool + RecordType1 string + ImportSummary ImportSummary + ImportSummaryID uint `gorm:"index"` + Processed bool + ImportTags []ImportTag `gorm:"constraint:OnDelete:CASCADE"` + BinaryGroup string + BinaryArtifact string + BinaryVersion string + BinaryPackaging string + RepositoryKind string + RepositoryURL string + RepositoryBranch string + RepositoryPath string + Owner string + Contributors string +} + +func (r *Import) AsMap() (m map[string]any) { + m = make(map[string]any) + m["filename"] = r.Filename + m["applicationName"] = r.ApplicationName + // "Application Name" is necessary in order for + // the UI to display the error report correctly. + m["Application Name"] = r.ApplicationName + m["businessService"] = r.BusinessService + m["comments"] = r.Comments + m["dependency"] = r.Dependency + m["dependencyDirection"] = r.DependencyDirection + m["description"] = r.Description + m["errorMessage"] = r.ErrorMessage + m["isValid"] = r.IsValid + m["processed"] = r.Processed + m["recordType1"] = r.RecordType1 + for i, tag := range r.ImportTags { + m[fmt.Sprintf("category%v", i+1)] = tag.Category + m[fmt.Sprintf("tag%v", i+1)] = tag.Name + } + return +} + +type ImportSummary struct { + Model + Content []byte + Filename string + ImportStatus string + Imports []Import `gorm:"constraint:OnDelete:CASCADE"` + CreateEntities bool +} + +type ImportTag struct { + Model + Name string + Category string + ImportID uint `gorm:"index"` + Import *Import +} + +// +// JSON Fields. +// + +// Repository represents an SCM repository. +type Repository struct { + Kind string `json:"kind"` + URL string `json:"url"` + Branch string `json:"branch"` + Tag string `json:"tag"` + Path string `json:"path"` +} diff --git a/migration/v15/model/assessment.go b/migration/v15/model/assessment.go new file mode 100644 index 000000000..0b51e714d --- /dev/null +++ b/migration/v15/model/assessment.go @@ -0,0 +1,102 @@ +package model + +type Questionnaire struct { + Model + UUID *string `gorm:"uniqueIndex"` + Name string `gorm:"unique"` + Description string + Required bool + Sections []Section `gorm:"type:json;serializer:json"` + Thresholds Thresholds `gorm:"type:json;serializer:json"` + RiskMessages RiskMessages `gorm:"type:json;serializer:json"` + Assessments []Assessment `gorm:"constraint:OnDelete:CASCADE"` +} + +// Builtin returns true if this is a Konveyor-provided questionnaire. +func (r *Questionnaire) Builtin() bool { + return r.UUID != nil +} + +type Assessment struct { + Model + ApplicationID *uint `gorm:"uniqueIndex:AssessmentA"` + Application *Application + ArchetypeID *uint `gorm:"uniqueIndex:AssessmentB"` + Archetype *Archetype + QuestionnaireID uint `gorm:"uniqueIndex:AssessmentA;uniqueIndex:AssessmentB"` + Questionnaire Questionnaire + Sections []Section `gorm:"type:json;serializer:json"` + Thresholds Thresholds `gorm:"type:json;serializer:json"` + RiskMessages RiskMessages `gorm:"type:json;serializer:json"` + Stakeholders []Stakeholder `gorm:"many2many:AssessmentStakeholders;constraint:OnDelete:CASCADE"` + StakeholderGroups []StakeholderGroup `gorm:"many2many:AssessmentStakeholderGroups;constraint:OnDelete:CASCADE"` +} + +type Review struct { + Model + BusinessCriticality uint `gorm:"not null"` + EffortEstimate string `gorm:"not null"` + ProposedAction string `gorm:"not null"` + WorkPriority uint `gorm:"not null"` + Comments string + ApplicationID *uint `gorm:"uniqueIndex"` + Application *Application + ArchetypeID *uint `gorm:"uniqueIndex"` + Archetype *Archetype +} + +// +// JSON Fields. +// + +// Section represents a group of questions in a questionnaire. +type Section struct { + Order uint `json:"order" yaml:"order"` + Name string `json:"name" yaml:"name"` + Questions []Question `json:"questions" yaml:"questions" binding:"min=1,dive"` + Comment string `json:"comment,omitempty" yaml:"comment,omitempty"` +} + +// Question represents a question in a questionnaire. +type Question struct { + Order uint `json:"order" yaml:"order"` + Text string `json:"text" yaml:"text"` + Explanation string `json:"explanation" yaml:"explanation"` + IncludeFor []CategorizedTag `json:"includeFor,omitempty" yaml:"includeFor,omitempty"` + ExcludeFor []CategorizedTag `json:"excludeFor,omitempty" yaml:"excludeFor,omitempty"` + Answers []Answer `json:"answers" yaml:"answers" binding:"min=1,dive"` +} + +// Answer represents an answer to a question in a questionnaire. +type Answer struct { + Order uint `json:"order" yaml:"order"` + Text string `json:"text" yaml:"text"` + Risk string `json:"risk" yaml:"risk" binding:"oneof=red yellow green unknown"` + Rationale string `json:"rationale" yaml:"rationale"` + Mitigation string `json:"mitigation" yaml:"mitigation"` + ApplyTags []CategorizedTag `json:"applyTags,omitempty" yaml:"applyTags,omitempty"` + AutoAnswerFor []CategorizedTag `json:"autoAnswerFor,omitempty" yaml:"autoAnswerFor,omitempty"` + Selected bool `json:"selected,omitempty" yaml:"selected,omitempty"` + AutoAnswered bool `json:"autoAnswered,omitempty" yaml:"autoAnswered,omitempty"` +} + +// CategorizedTag represents a human-readable pair of category and tag. +type CategorizedTag struct { + Category string `json:"category" yaml:"category"` + Tag string `json:"tag" yaml:"tag"` +} + +// RiskMessages contains messages to display for each risk level. +type RiskMessages struct { + Red string `json:"red" yaml:"red"` + Yellow string `json:"yellow" yaml:"yellow"` + Green string `json:"green" yaml:"green"` + Unknown string `json:"unknown" yaml:"unknown"` +} + +// Thresholds contains the threshold values for determining risk for the questionnaire. +type Thresholds struct { + Red uint `json:"red" yaml:"red"` + Yellow uint `json:"yellow" yaml:"yellow"` + Unknown uint `json:"unknown" yaml:"unknown"` +} diff --git a/migration/v15/model/core.go b/migration/v15/model/core.go new file mode 100644 index 000000000..c3b2a85b4 --- /dev/null +++ b/migration/v15/model/core.go @@ -0,0 +1,309 @@ +package model + +import ( + "os" + "path" + "time" + + "github.com/google/uuid" + liberr "github.com/jortel/go-utils/error" + "github.com/konveyor/tackle2-hub/encryption" + "github.com/konveyor/tackle2-hub/migration/json" + "gorm.io/gorm" +) + +// Model Base model. +type Model struct { + ID uint `gorm:"<-:create;primaryKey"` + CreateTime time.Time `gorm:"<-:create;autoCreateTime"` + CreateUser string `gorm:"<-:create"` + UpdateUser string +} + +// PK sequence. +type PK struct { + Kind string `gorm:"<-:create;primaryKey"` + LastID uint +} + +// Setting hub settings. +type Setting struct { + Model + Key string `gorm:"<-:create;uniqueIndex"` + Value any `gorm:"type:json;serializer:json"` +} + +// As unmarshalls the value of the Setting into the `ptr` parameter. +func (r *Setting) As(ptr any) (err error) { + bytes, err := json.Marshal(r.Value) + if err != nil { + err = liberr.Wrap(err) + } + err = json.Unmarshal(bytes, ptr) + if err != nil { + err = liberr.Wrap(err) + } + return +} + +type Bucket struct { + Model + Path string `gorm:"<-:create;uniqueIndex"` + Expiration *time.Time +} + +func (m *Bucket) BeforeCreate(db *gorm.DB) (err error) { + if m.Path == "" { + uid := uuid.New() + m.Path = path.Join( + Settings.Hub.Bucket.Path, + uid.String()) + err = os.MkdirAll(m.Path, 0777) + if err != nil { + err = liberr.Wrap( + err, + "path", + m.Path) + } + } + return +} + +type BucketOwner struct { + BucketID *uint `gorm:"index" ref:"bucket"` + Bucket *Bucket +} + +func (m *BucketOwner) BeforeCreate(db *gorm.DB) (err error) { + if !m.HasBucket() { + b := &Bucket{} + err = db.Create(b).Error + m.SetBucket(&b.ID) + } + return +} + +func (m *BucketOwner) SetBucket(id *uint) { + m.BucketID = id + m.Bucket = nil +} + +func (m *BucketOwner) HasBucket() (b bool) { + return m.BucketID != nil +} + +type File struct { + Model + Name string + Encoding string + Path string `gorm:"<-:create;uniqueIndex"` + Expiration *time.Time +} + +func (m *File) BeforeCreate(db *gorm.DB) (err error) { + uid := uuid.New() + m.Path = path.Join( + Settings.Hub.Bucket.Path, + ".file", + uid.String()) + err = os.MkdirAll(path.Dir(m.Path), 0777) + if err != nil { + err = liberr.Wrap( + err, + "path", + m.Path) + } + return +} + +type Task struct { + Model + BucketOwner + Name string `gorm:"index"` + Kind string + Addon string `gorm:"index"` + Extensions []string `gorm:"type:json;serializer:json"` + State string `gorm:"index"` + Locator string `gorm:"index"` + Priority int + Policy TaskPolicy `gorm:"type:json;serializer:json"` + TTL TTL `gorm:"type:json;serializer:json"` + Data json.Data `gorm:"type:json;serializer:json"` + Started *time.Time + Terminated *time.Time + Errors []TaskError `gorm:"type:json;serializer:json"` + Events []TaskEvent `gorm:"type:json;serializer:json"` + Pod string `gorm:"index"` + Retries int + Attached []Attachment `gorm:"type:json;serializer:json" ref:"[]file"` + Report *TaskReport `gorm:"constraint:OnDelete:CASCADE"` + ApplicationID *uint `gorm:"index"` + Application *Application + TaskGroupID *uint `gorm:"<-:create"` + TaskGroup *TaskGroup +} + +func (m *Task) BeforeCreate(db *gorm.DB) (err error) { + err = m.BucketOwner.BeforeCreate(db) + return +} + +type TaskReport struct { + Model + Status string + Total int + Completed int + Activity []string `gorm:"type:json;serializer:json"` + Errors []TaskError `gorm:"type:json;serializer:json"` + Attached []Attachment `gorm:"type:json;serializer:json" ref:"[]file"` + Result json.Data `gorm:"type:json;serializer:json"` + TaskID uint `gorm:"<-:create;uniqueIndex"` + Task *Task +} + +type TaskGroup struct { + Model + BucketOwner + Name string + Kind string + Addon string + Extensions []string `gorm:"type:json;serializer:json"` + State string + Priority int + Policy TaskPolicy `gorm:"type:json;serializer:json"` + Data json.Data `gorm:"type:json;serializer:json"` + List []Task `gorm:"type:json;serializer:json"` + Tasks []Task `gorm:"constraint:OnDelete:CASCADE"` +} + +// Proxy configuration. +// kind = (http|https) +type Proxy struct { + Model + Enabled bool + Kind string `gorm:"uniqueIndex"` + Host string `gorm:"not null"` + Port int + Excluded []string `gorm:"type:json;serializer:json"` + IdentityID *uint `gorm:"index"` + Identity *Identity +} + +// Identity represents and identity with a set of credentials. +type Identity struct { + Model + Kind string `gorm:"not null"` + Name string `gorm:"index;unique;not null"` + Description string + User string + Password string + Key string + Settings string + Proxies []Proxy `gorm:"constraint:OnDelete:SET NULL"` + Applications []Application `gorm:"many2many:ApplicationIdentity;constraint:OnDelete:CASCADE"` +} + +// Encrypt sensitive fields. +// The ref identity is used to determine when sensitive fields +// have changed and need to be (re)encrypted. +func (r *Identity) Encrypt(ref *Identity) (err error) { + passphrase := Settings.Encryption.Passphrase + aes := encryption.New(passphrase) + if r.Password != ref.Password { + if r.Password != "" { + r.Password, err = aes.Encrypt(r.Password) + if err != nil { + err = liberr.Wrap(err) + return + } + } + } + if r.Key != ref.Key { + if r.Key != "" { + r.Key, err = aes.Encrypt(r.Key) + if err != nil { + err = liberr.Wrap(err) + return + } + } + } + if r.Settings != ref.Settings { + if r.Settings != "" { + r.Settings, err = aes.Encrypt(r.Settings) + if err != nil { + err = liberr.Wrap(err) + return + } + } + } + return +} + +// Decrypt sensitive fields. +func (r *Identity) Decrypt() (err error) { + passphrase := Settings.Encryption.Passphrase + aes := encryption.New(passphrase) + if r.Password != "" { + r.Password, err = aes.Decrypt(r.Password) + if err != nil { + err = liberr.Wrap(err) + return + } + } + if r.Key != "" { + r.Key, err = aes.Decrypt(r.Key) + if err != nil { + err = liberr.Wrap(err) + return + } + } + if r.Settings != "" { + r.Settings, err = aes.Decrypt(r.Settings) + if err != nil { + err = liberr.Wrap(err) + return + } + } + return +} + +// +// JSON Fields. +// + +// Attachment file attachment. +type Attachment struct { + ID uint `json:"id" binding:"required"` + Name string `json:"name,omitempty" yaml:",omitempty"` + Activity int `json:"activity,omitempty" yaml:",omitempty"` +} + +// TaskError used in Task.Errors. +type TaskError struct { + Severity string `json:"severity"` + Description string `json:"description"` +} + +// TaskEvent task event. +type TaskEvent struct { + Kind string `json:"kind"` + Count int `json:"count"` + Reason string `json:"reason,omitempty" yaml:",omitempty"` + Last time.Time `json:"last"` +} + +// TaskPolicy scheduling policy. +type TaskPolicy struct { + Isolated bool `json:"isolated,omitempty" yaml:",omitempty"` + PreemptEnabled bool `json:"preemptEnabled,omitempty" yaml:"preemptEnabled,omitempty"` + PreemptExempt bool `json:"preemptExempt,omitempty" yaml:"preemptExempt,omitempty"` +} + +// TTL time-to-live. +type TTL struct { + Created int `json:"created,omitempty" yaml:",omitempty"` + Pending int `json:"pending,omitempty" yaml:",omitempty"` + Running int `json:"running,omitempty" yaml:",omitempty"` + Succeeded int `json:"succeeded,omitempty" yaml:",omitempty"` + Failed int `json:"failed,omitempty" yaml:",omitempty"` +} diff --git a/migration/v15/model/mod.patch b/migration/v15/model/mod.patch new file mode 100644 index 000000000..e2a2ae8ae --- /dev/null +++ b/migration/v15/model/mod.patch @@ -0,0 +1,11 @@ +diff -ruN migration/v14/model/core.go migration/v15/model/core.go +--- migration/v14/model/core.go 2024-09-20 04:44:49.750736163 -0700 ++++ migration/v15/model/core.go 2024-09-20 04:47:13.750375198 -0700 +@@ -95,6 +95,7 @@ + type File struct { + Model + Name string ++ Encoding string + Path string `gorm:"<-:create;uniqueIndex"` + Expiration *time.Time + } diff --git a/migration/v15/model/pkg.go b/migration/v15/model/pkg.go new file mode 100644 index 000000000..8f612b488 --- /dev/null +++ b/migration/v15/model/pkg.go @@ -0,0 +1,56 @@ +package model + +import ( + "github.com/konveyor/tackle2-hub/settings" +) + +var ( + Settings = &settings.Settings +) + +// JSON field (data) type. +type JSON = []byte + +// All builds all models. +// Models are enumerated such that each are listed after +// all the other models on which they may depend. +func All() []any { + return []any{ + Application{}, + TechDependency{}, + Incident{}, + Analysis{}, + Issue{}, + Bucket{}, + BusinessService{}, + Dependency{}, + File{}, + Fact{}, + Identity{}, + Import{}, + ImportSummary{}, + ImportTag{}, + JobFunction{}, + MigrationWave{}, + PK{}, + Proxy{}, + Review{}, + Setting{}, + RuleSet{}, + Rule{}, + Stakeholder{}, + StakeholderGroup{}, + Tag{}, + TagCategory{}, + Target{}, + Task{}, + TaskGroup{}, + TaskReport{}, + Ticket{}, + Tracker{}, + ApplicationTag{}, + Questionnaire{}, + Assessment{}, + Archetype{}, + } +} diff --git a/model/pkg.go b/model/pkg.go index 2cb906e60..f1c5d0cf6 100644 --- a/model/pkg.go +++ b/model/pkg.go @@ -2,12 +2,14 @@ package model import ( "github.com/konveyor/tackle2-hub/migration/json" - "github.com/konveyor/tackle2-hub/migration/v14/model" + "github.com/konveyor/tackle2-hub/migration/v15/model" ) // Field (data) types. type JSON = model.JSON +var ALL = model.All() + // Models type Model = model.Model type Application = model.Application @@ -16,7 +18,6 @@ type Assessment = model.Assessment type TechDependency = model.TechDependency type Incident = model.Incident type Analysis = model.Analysis -type ArchivedIssue = model.ArchivedIssue type Issue = model.Issue type Bucket = model.Bucket type BucketOwner = model.BucketOwner @@ -30,6 +31,7 @@ type ImportSummary = model.ImportSummary type ImportTag = model.ImportTag type JobFunction = model.JobFunction type MigrationWave = model.MigrationWave +type PK = model.PK type Proxy = model.Proxy type Questionnaire = model.Questionnaire type Review = model.Review @@ -47,16 +49,27 @@ type TaskReport = model.TaskReport type Ticket = model.Ticket type Tracker = model.Tracker -type TTL = model.TTL +// JSON fields type Ref = json.Ref type Map = json.Map type Data = json.Data - +type ArchivedIssue = model.ArchivedIssue +type Attachment = model.Attachment +type Link = model.Link +type Repository = model.Repository +type TargetLabel = model.TargetLabel type TaskError = model.TaskError type TaskEvent = model.TaskEvent type TaskPolicy = model.TaskPolicy -type Attachment = model.Attachment -type Repository = model.Repository +type TTL = model.TTL + +// Assessment JSON fields +type Section = model.Section +type Question = model.Question +type Answer = model.Answer +type Thresholds = model.Thresholds +type RiskMessages = model.RiskMessages +type CategorizedTag = model.CategorizedTag // Join tables type ApplicationTag = model.ApplicationTag diff --git a/reaper/task.go b/reaper/task.go index b78f06a17..6f64ec41c 100644 --- a/reaper/task.go +++ b/reaper/task.go @@ -44,7 +44,7 @@ type TaskReaper struct { // - Pod is deleted after the defined period. func (r *TaskReaper) Run() { Log.V(1).Info("Reaping tasks.") - list := []model.Task{} + list := []task.Task{} result := r.DB.Find( &list, "state IN ?", @@ -52,6 +52,7 @@ func (r *TaskReaper) Run() { task.Created, task.Succeeded, task.Failed, + task.Canceled, }) Log.Error(result.Error, "") if result.Error != nil { @@ -108,12 +109,16 @@ func (r *TaskReaper) Run() { r.release(m) } } + d := time.Duration(Settings.Hub.Task.Pod.Retention.Succeeded) * Unit + if time.Since(mark) > d { + r.podDelete(m) + } case task.Failed: mark := m.CreateTime if m.Terminated != nil { mark = *m.Terminated } - if m.TTL.Succeeded > 0 { + if m.TTL.Failed > 0 { d := time.Duration(m.TTL.Failed) * Unit if time.Since(mark) > d { r.delete(m) @@ -124,23 +129,17 @@ func (r *TaskReaper) Run() { r.release(m) } } + d := time.Duration(Settings.Hub.Task.Pod.Retention.Failed) * Unit + if time.Since(mark) > d { + r.podDelete(m) + } } } } -// release resources. -func (r *TaskReaper) release(m *model.Task) { +// release bucket and file resources. +func (r *TaskReaper) release(m *task.Task) { nChanged := 0 - if m.Pod != "" { - rt := Task{Task: m} - err := rt.Delete(r.Client) - if err == nil { - m.Pod = "" - nChanged++ - } else { - Log.Error(err, "") - } - } if m.HasBucket() { Log.Info("Task bucket released.", "id", m.ID) m.SetBucket(nil) @@ -151,8 +150,7 @@ func (r *TaskReaper) release(m *model.Task) { nChanged++ } if nChanged > 0 { - rt := task.Task{Task: m} - rt.Event(task.Released) + m.Event(task.Released) err := r.DB.Save(m).Error if err != nil { Log.Error(err, "") @@ -161,10 +159,25 @@ func (r *TaskReaper) release(m *model.Task) { return } +// podDelete deletes the task pod. +func (r *TaskReaper) podDelete(m *task.Task) { + if m.Pod == "" { + return + } + err := m.Delete(r.Client) + if err != nil { + Log.Error(err, "") + return + } + err = r.DB.Save(m).Error + if err != nil { + Log.Error(err, "") + } +} + // delete task. -func (r *TaskReaper) delete(m *model.Task) { - rt := Task{Task: m} - err := rt.Delete(r.Client) +func (r *TaskReaper) delete(m *task.Task) { + err := m.Delete(r.Client) if err != nil { Log.Error(err, "") } diff --git a/seed/questionnaire.go b/seed/questionnaire.go index 87e419817..134e63c7f 100644 --- a/seed/questionnaire.go +++ b/seed/questionnaire.go @@ -77,9 +77,14 @@ func (r *Questionnaire) Apply(db *gorm.DB) (err error) { questionnaire.Name = q.Name questionnaire.UUID = &q.UUID questionnaire.Description = q.Description - questionnaire.Sections, _ = json.Marshal(q.Sections) - questionnaire.RiskMessages, _ = json.Marshal(q.RiskMessages) - questionnaire.Thresholds, _ = json.Marshal(q.Thresholds) + questionnaire.RiskMessages = model.RiskMessages(q.RiskMessages) + questionnaire.Thresholds = model.Thresholds(q.Thresholds) + bytes, jErr := json.Marshal(q.Sections) + if jErr != nil { + err = liberr.Wrap(jErr) + return + } + err = json.Unmarshal(bytes, &questionnaire.Sections) result := db.Save(&questionnaire) if result.Error != nil { err = liberr.Wrap(result.Error) diff --git a/seed/ruleset.go b/seed/ruleset.go index 3354b6aea..f746d1f08 100644 --- a/seed/ruleset.go +++ b/seed/ruleset.go @@ -128,14 +128,13 @@ func (r *RuleSet) applyRules(db *gorm.DB, ruleSet *model.RuleSet, rs libseed.Rul return } for _, rl := range rs.Rules { - labels, _ := json.Marshal(rl.Labels()) f, fErr := file(db, rl.Path) if fErr != nil { err = liberr.Wrap(fErr) return } rule := model.Rule{ - Labels: labels, + Labels: rl.Labels(), RuleSetID: ruleSet.ID, FileID: &f.ID, } diff --git a/seed/seed.go b/seed/seed.go index 1ae605a9d..a674e1c73 100644 --- a/seed/seed.go +++ b/seed/seed.go @@ -1,7 +1,6 @@ package seed import ( - "encoding/json" "fmt" "strings" @@ -75,12 +74,10 @@ func compareChecksum(db *gorm.DB, checksum []byte) (match bool, err error) { return } var seededChecksum string - if setting.Value != nil { - err = json.Unmarshal(setting.Value, &seededChecksum) - if err != nil { - err = liberr.Wrap(err) - return - } + err = setting.As(&seededChecksum) + if err != nil { + err = liberr.Wrap(err) + return } match = seededChecksum == fmt.Sprintf("%x", checksum) @@ -90,8 +87,7 @@ func compareChecksum(db *gorm.DB, checksum []byte) (match bool, err error) { // saveChecksum saves the seed checksum to the setting specified by SeedKey. func saveChecksum(db *gorm.DB, checksum []byte) (err error) { setting := &model.Setting{Key: SeedKey} - value, _ := json.Marshal(fmt.Sprintf("%x", checksum)) - setting.Value = value + setting.Value = fmt.Sprintf("%x", checksum) result := db.Where("key", SeedKey).Updates(setting) if result.Error != nil { err = liberr.Wrap(result.Error) @@ -110,12 +106,10 @@ func migrationVersion(db *gorm.DB) (version uint, err error) { } var v migration.Version - if setting.Value != nil { - err = json.Unmarshal(setting.Value, &v) - if err != nil { - err = liberr.Wrap(err) - return - } + err = setting.As(&v) + if err != nil { + err = liberr.Wrap(err) + return } version = uint(v.Version) diff --git a/seed/target.go b/seed/target.go index 6e2888091..9f665ee8d 100644 --- a/seed/target.go +++ b/seed/target.go @@ -2,7 +2,6 @@ package seed import ( "container/list" - "encoding/json" "errors" "fmt" @@ -80,7 +79,6 @@ func (r *Target) Apply(db *gorm.DB) (err error) { err = liberr.Wrap(fErr) return } - labels, _ := json.Marshal(t.Labels) target.UUID = &t.UUID target.Name = t.Name @@ -88,7 +86,9 @@ func (r *Target) Apply(db *gorm.DB) (err error) { target.Provider = t.Provider target.Choice = t.Choice target.ImageID = f.ID - target.Labels = labels + for _, l := range t.Labels { + target.Labels = append(target.Labels, model.TargetLabel(l)) + } result := db.Save(&target) if result.Error != nil { err = liberr.Wrap(result.Error) @@ -126,7 +126,7 @@ func (r *Target) reorder(db *gorm.DB, seedIds []uint) (err error) { } userOrder := []uint{} _ = s.As(&userOrder) - _ = s.With(merge(userOrder, seedIds, targetIds)) + s.Value = merge(userOrder, seedIds, targetIds) result = db.Where("key", UITargetOrder).Updates(s) if result.Error != nil { diff --git a/settings/addon.go b/settings/addon.go index 94c30c57a..e022a7975 100644 --- a/settings/addon.go +++ b/settings/addon.go @@ -7,13 +7,16 @@ import ( ) const ( - EnvHubBaseURL = "HUB_BASE_URL" - EnvHubToken = "TOKEN" - EnvTask = "TASK" + EnvHubBaseURL = "HUB_BASE_URL" + EnvHubToken = "TOKEN" + EnvTask = "TASK" + EnvAddonHomeDir = "ADDON_HOME" ) // Addon settings. type Addon struct { + // HomeDir working directory. + HomeDir string // Hub settings. Hub struct { // URL for the hub API. @@ -27,6 +30,10 @@ type Addon struct { func (r *Addon) Load() (err error) { var found bool + r.HomeDir, found = os.LookupEnv(EnvAddonHomeDir) + if !found { + r.HomeDir = "/addon" + } r.Hub.URL, found = os.LookupEnv(EnvHubBaseURL) if !found { r.Hub.URL = "http://localhost:8080" diff --git a/settings/hub.go b/settings/hub.go index d21e14813..a215177e0 100644 --- a/settings/hub.go +++ b/settings/hub.go @@ -19,6 +19,8 @@ const ( EnvTaskReapCreated = "TASK_REAP_CREATED" EnvTaskReapSucceeded = "TASK_REAP_SUCCEEDED" EnvTaskReapFailed = "TASK_REAP_FAILED" + EnvTaskPodRetainSucceeded = "TASK_POD_RETAIN_SUCCEEDED" + EnvTaskPodRetainFailed = "TASK_POD_RETAIN_FAILED" EnvTaskSA = "TASK_SA" EnvTaskRetries = "TASK_RETRIES" EnvTaskPreemptEnabled = "TASK_PREEMPT_ENABLED" @@ -84,6 +86,12 @@ type Hub struct { Postponed time.Duration Rate int } + Pod struct { + Retention struct { + Succeeded int + Failed int + } + } } // Frequency Frequency struct { @@ -169,6 +177,20 @@ func (r *Hub) Load() (err error) { } else { r.Task.Reaper.Failed = 43200 // 720 hours (30 days). } + s, found = os.LookupEnv(EnvTaskPodRetainSucceeded) + if found { + n, _ := strconv.Atoi(s) + r.Task.Pod.Retention.Succeeded = n + } else { + r.Task.Pod.Retention.Succeeded = 1 + } + s, found = os.LookupEnv(EnvTaskPodRetainFailed) + if found { + n, _ := strconv.Atoi(s) + r.Task.Pod.Retention.Failed = n + } else { + r.Task.Pod.Retention.Failed = 4320 // 72 hours. + } r.Task.SA, found = os.LookupEnv(EnvTaskSA) if !found { r.Task.SA = "tackle-hub" diff --git a/task/manager.go b/task/manager.go index 914c23999..c939c3d03 100644 --- a/task/manager.go +++ b/task/manager.go @@ -1,6 +1,7 @@ package task import ( + "bytes" "context" "errors" "fmt" @@ -30,7 +31,10 @@ import ( meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/remotecommand" k8s "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" ) // States @@ -51,19 +55,20 @@ const ( // Events const ( - AddonSelected = "AddonSelected" - ExtSelected = "ExtensionSelected" - ImageError = "ImageError" - PodNotFound = "PodNotFound" - PodCreated = "PodCreated" - PodPending = "PodPending" - PodRunning = "PodRunning" - Preempted = "Preempted" - PodSucceeded = "PodSucceeded" - PodFailed = "PodFailed" - PodDeleted = "PodDeleted" - Escalated = "Escalated" - Released = "Released" + AddonSelected = "AddonSelected" + ExtSelected = "ExtensionSelected" + ImageError = "ImageError" + PodNotFound = "PodNotFound" + PodCreated = "PodCreated" + PodPending = "PodPending" + PodRunning = "PodRunning" + Preempted = "Preempted" + PodSucceeded = "PodSucceeded" + PodFailed = "PodFailed" + PodDeleted = "PodDeleted" + Escalated = "Escalated" + Released = "Released" + ContainerKilled = "ContainerKilled" ) // k8s labels. @@ -78,6 +83,7 @@ const ( ) const ( + Addon = "addon" Shared = "shared" Cache = "cache" ) @@ -123,6 +129,7 @@ func (m *Manager) Run(ctx context.Context) { m.deleteOrphanPods() m.runActions() m.updateRunning() + m.deleteZombies() m.startReady() m.pause() } else { @@ -860,9 +867,6 @@ func (m *Manager) updateRunning() { list = append(list, &Task{task}) } for _, task := range list { - if !task.StateIn(Running, Pending) { - continue - } running := task pod, found := running.Reflect(&m.cluster) if found { @@ -872,10 +876,24 @@ func (m *Manager) updateRunning() { Log.Error(err, "") continue } - err = running.Delete(m.Client) - if err != nil { - Log.Error(err, "") - continue + podRetention := 0 + if running.State == Succeeded { + podRetention = Settings.Hub.Task.Pod.Retention.Succeeded + } else { + podRetention = Settings.Hub.Task.Pod.Retention.Failed + } + if podRetention > 0 { + err = m.ensureTerminated(running, pod) + if err != nil { + podRetention = 0 + } + } + if podRetention == 0 { + err = running.Delete(m.Client) + if err != nil { + Log.Error(err, "") + continue + } } } } @@ -888,6 +906,56 @@ func (m *Manager) updateRunning() { } } +// deleteZombies - detect and delete zombie pods. +// A zombie is a (succeed|failed) task with a running pod that +// the manager has previously tried to kill. +func (m *Manager) deleteZombies() { + var err error + defer func() { + Log.Error(err, "") + }() + var pods []string + for _, pod := range m.cluster.Pods() { + if pod.Status.Phase == core.PodRunning { + ref := path.Join(pod.Namespace, pod.Name) + pods = append( + pods, + ref) + } + } + fetched := []*Task{} + db := m.DB.Select("Events") + db = db.Where("Pod", pods) + db = db.Where("state IN ?", + []string{ + Succeeded, + Failed, + }) + err = db.Find(&fetched).Error + if err != nil { + err = liberr.Wrap(err) + return + } + for _, task := range fetched { + event, found := task.LastEvent(ContainerKilled) + if !found { + continue + } + if time.Since(event.Last) > time.Minute { + Log.Info( + "Zombie detected.", + "task", + task.ID, + "pod", + task.Pod) + err = task.Delete(m.Client) + if err != nil { + Log.Error(err, "") + } + } + } +} + // deleteOrphanPods finds and deletes task pods not referenced by a task. func (m *Manager) deleteOrphanPods() { var err error @@ -1068,6 +1136,87 @@ func (m *Manager) containerLog(pod *core.Pod, container string) (file *model.Fil return } +// ensureTerminated - Terminate running containers. +func (m *Manager) ensureTerminated(task *Task, pod *core.Pod) (err error) { + for _, status := range pod.Status.ContainerStatuses { + if status.State.Terminated != nil { + continue + } + if status.Started == nil || !*status.Started { + continue + } + err = m.terminateContainer(task, pod, status.Name) + if err != nil { + return + } + } + return +} + +// terminateContainer - Terminate container as needed. +// The container is killed. +// Should the container continue to run after (1) minute, +// it is reported as an error. +func (m *Manager) terminateContainer(task *Task, pod *core.Pod, container string) (err error) { + Log.V(1).Info("KILL container", "container", container) + clientSet, err := k8s2.NewClientSet() + if err != nil { + return + } + cmd := []string{ + "sh", + "-c", + "kill 1", + } + req := clientSet.CoreV1().RESTClient().Post() + req = req.Resource("pods") + req = req.Name(pod.Name) + req = req.Namespace(pod.Namespace) + req = req.SubResource("exec") + option := &core.PodExecOptions{ + Command: cmd, + Container: container, + Stdout: true, + Stderr: true, + TTY: true, + } + req.VersionedParams( + option, + scheme.ParameterCodec, + ) + cfg, _ := config.GetConfig() + exec, err := remotecommand.NewSPDYExecutor(cfg, "POST", req.URL()) + if err != nil { + return + } + stdout := bytes.NewBuffer([]byte{}) + stderr := bytes.NewBuffer([]byte{}) + err = exec.Stream(remotecommand.StreamOptions{ + Stdout: stdout, + Stderr: stderr, + }) + if err != nil { + Log.Info( + "Container KILL failed.", + "name", + container, + "err", + err.Error(), + "stderr", + stderr.String()) + } else { + task.Event( + ContainerKilled, + "container: '%s' had not terminated.", + container) + Log.Info( + "Container KILLED.", + "name", + container) + } + return +} + // Task is an runtime task. type Task struct { // model. @@ -1140,6 +1289,17 @@ func (r *Task) LastEvent(kind string) (event *model.TaskEvent, found bool) { return } +// FindEvent returns the matched events by kind. +func (r *Task) FindEvent(kind string) (matched []*model.TaskEvent) { + for i := 0; i < len(r.Events); i++ { + event := &r.Events[i] + if kind == event.Kind { + matched = append(matched, event) + } + } + return +} + // Run the specified task. func (r *Task) Run(cluster *Cluster) (started bool, err error) { mark := time.Now() @@ -1484,6 +1644,12 @@ func (r *Task) specification( addon *crd.Addon, extensions []crd.Extension, secret *core.Secret) (specification core.PodSpec) { + addonDir := core.Volume{ + Name: Addon, + VolumeSource: core.VolumeSource{ + EmptyDir: &core.EmptyDirVolumeSource{}, + }, + } shared := core.Volume{ Name: Shared, VolumeSource: core.VolumeSource{ @@ -1511,6 +1677,7 @@ func (r *Task) specification( InitContainers: init, Containers: plain, Volumes: []core.Volume{ + addonDir, shared, cache, }, @@ -1524,7 +1691,6 @@ func (r *Task) containers( addon *crd.Addon, extensions []crd.Extension, secret *core.Secret) (init []core.Container, plain []core.Container) { - userid := int64(0) token := &core.EnvVarSource{ SecretKeyRef: &core.SecretKeySelector{ Key: settings.EnvHubToken, @@ -1548,11 +1714,12 @@ func (r *Task) containers( container := &plain[i] injector.Inject(container) r.propagateEnv(&plain[0], container) - container.SecurityContext = &core.SecurityContext{ - RunAsUser: &userid, - } container.VolumeMounts = append( container.VolumeMounts, + core.VolumeMount{ + Name: Addon, + MountPath: Settings.Addon.HomeDir, + }, core.VolumeMount{ Name: Shared, MountPath: Settings.Shared.Path, @@ -1563,6 +1730,10 @@ func (r *Task) containers( }) container.Env = append( container.Env, + core.EnvVar{ + Name: settings.EnvAddonHomeDir, + Value: Settings.Addon.HomeDir, + }, core.EnvVar{ Name: settings.EnvSharedPath, Value: Settings.Shared.Path, diff --git a/test/api/application/facts_test.go b/test/api/application/facts_test.go index d6d487bf1..f294e9304 100644 --- a/test/api/application/facts_test.go +++ b/test/api/application/facts_test.go @@ -108,7 +108,7 @@ func TestApplicationFactsList(t *testing.T) { factsPathSuffix := []string{"facts/test:", "facts/test:/"} for _, pathSuffix := range factsPathSuffix { t.Run(fmt.Sprintf("Fact list application %s with %s", application.Name, pathSuffix), func(t *testing.T) { - got := api.FactMap{} + got := api.Map{} err := Client.Get(fmt.Sprintf("%s/%s", binding.Path(api.ApplicationRoot).Inject(binding.Params{api.ID: application.ID}), pathSuffix), &got) if err != nil { t.Errorf("Get list error: %v", err.Error()) diff --git a/test/api/assessment/samples.go b/test/api/assessment/samples.go index 1a1ec5493..ccfb779e1 100644 --- a/test/api/assessment/samples.go +++ b/test/api/assessment/samples.go @@ -2,7 +2,7 @@ package assessment import ( "github.com/konveyor/tackle2-hub/api" - "github.com/konveyor/tackle2-hub/assessment" + "github.com/konveyor/tackle2-hub/model" "github.com/konveyor/tackle2-hub/test/api/application" "github.com/konveyor/tackle2-hub/test/api/questionnaire" ) @@ -17,28 +17,28 @@ var ( Questionnaire: api.Ref{ Name: questionnaire.Questionnaire1.Name, }, - Sections: []assessment.Section{ + Sections: []api.Section{ { - Order: uint2ptr(1), + Order: 1, Name: "Section 1", - Questions: []assessment.Question{ + Questions: []model.Question{ { - Order: uint2ptr(1), + Order: 1, Text: "What is your favorite color?", Explanation: "Please tell us your favorite color.", - Answers: []assessment.Answer{ + Answers: []model.Answer{ { - Order: uint2ptr(1), + Order: 1, Text: "Red", Risk: "red", }, { - Order: uint2ptr(2), + Order: 2, Text: "Green", Risk: "green", }, { - Order: uint2ptr(3), + Order: 3, Text: "Blue", Risk: "yellow", Selected: true, diff --git a/test/api/migrationwave/api_test.go b/test/api/migrationwave/api_test.go index afa2ac846..7c2622a84 100644 --- a/test/api/migrationwave/api_test.go +++ b/test/api/migrationwave/api_test.go @@ -18,6 +18,7 @@ func TestMigrationWaveCRUD(t *testing.T) { } assert.Must(t, Application.Create(&expectedApp)) createdApps = append(createdApps, expectedApp) + r.Applications[0].ID = expectedApp.ID } createdStakeholders := []api.Stakeholder{} @@ -28,6 +29,7 @@ func TestMigrationWaveCRUD(t *testing.T) { } assert.Must(t, Stakeholder.Create(&expectedStakeholder)) createdStakeholders = append(createdStakeholders, expectedStakeholder) + r.Stakeholders[0].ID = expectedStakeholder.ID } createdStakeholderGroups := []api.StakeholderGroup{} @@ -38,6 +40,7 @@ func TestMigrationWaveCRUD(t *testing.T) { } assert.Must(t, StakeholderGroup.Create(&expectedStakeholderGroup)) createdStakeholderGroups = append(createdStakeholderGroups, expectedStakeholderGroup) + r.StakeholderGroups[0].ID = expectedStakeholderGroup.ID } assert.Must(t, MigrationWave.Create(&r)) @@ -102,6 +105,7 @@ func TestMigrationWaveList(t *testing.T) { } assert.Must(t, Application.Create(&expectedApp)) createdApps = append(createdApps, expectedApp) + r.Applications[0].ID = expectedApp.ID } for _, stakeholder := range r.Stakeholders { @@ -111,6 +115,7 @@ func TestMigrationWaveList(t *testing.T) { } assert.Must(t, Stakeholder.Create(&expectedStakeholder)) createdStakeholders = append(createdStakeholders, expectedStakeholder) + r.Stakeholders[0].ID = expectedStakeholder.ID } for _, stakeholderGroup := range r.StakeholderGroups { @@ -120,6 +125,7 @@ func TestMigrationWaveList(t *testing.T) { } assert.Must(t, StakeholderGroup.Create(&expectedStakeholderGroup)) createdStakeholderGroups = append(createdStakeholderGroups, expectedStakeholderGroup) + r.StakeholderGroups[0].ID = expectedStakeholderGroup.ID } assert.Must(t, MigrationWave.Create(&r)) createdMigrationWaves = append(createdMigrationWaves, r) diff --git a/test/api/migrationwave/samples.go b/test/api/migrationwave/samples.go index 4c0ef9fa1..535d5dbae 100644 --- a/test/api/migrationwave/samples.go +++ b/test/api/migrationwave/samples.go @@ -13,19 +13,16 @@ var Samples = []api.MigrationWave{ EndDate: time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, time.Local).Add(30 * time.Minute), Applications: []api.Ref{ { - ID: 1, Name: "Sample Application", }, }, Stakeholders: []api.Ref{ { - ID: 1, Name: "Sample Stakeholders", }, }, StakeholderGroups: []api.Ref{ { - ID: 1, Name: "Sample Stakeholders Groups", }, }, diff --git a/test/api/questionnaire/samples.go b/test/api/questionnaire/samples.go index 988dbf0c9..b1792f187 100644 --- a/test/api/questionnaire/samples.go +++ b/test/api/questionnaire/samples.go @@ -2,7 +2,7 @@ package questionnaire import ( "github.com/konveyor/tackle2-hub/api" - "github.com/konveyor/tackle2-hub/assessment" + "github.com/konveyor/tackle2-hub/model" ) // Set of valid resources for tests and reuse. @@ -11,30 +11,30 @@ var ( Name: "Questionnaire1", Description: "Questionnaire minimal sample 1", Required: true, - Thresholds: assessment.Thresholds{}, - RiskMessages: assessment.RiskMessages{}, - Sections: []assessment.Section{ + Thresholds: api.Thresholds{}, + RiskMessages: api.RiskMessages{}, + Sections: []api.Section{ { - Order: uint2ptr(1), + Order: 1, Name: "Section 1", - Questions: []assessment.Question{ + Questions: []model.Question{ { - Order: uint2ptr(1), + Order: 1, Text: "What is your favorite color?", Explanation: "Please tell us your favorite color.", - Answers: []assessment.Answer{ + Answers: []model.Answer{ { - Order: uint2ptr(1), + Order: 1, Text: "Red", Risk: "red", }, { - Order: uint2ptr(2), + Order: 2, Text: "Green", Risk: "green", }, { - Order: uint2ptr(3), + Order: 3, Text: "Blue", Risk: "yellow", Selected: true, @@ -47,7 +47,3 @@ var ( } Samples = []api.Questionnaire{Questionnaire1} ) - -func uint2ptr(u uint) *uint { - return &u -} diff --git a/test/api/review/api_test.go b/test/api/review/api_test.go index 013f84842..d32555859 100644 --- a/test/api/review/api_test.go +++ b/test/api/review/api_test.go @@ -143,7 +143,7 @@ func TestReviewList(t *testing.T) { // Delete related reviews and applications. for _, review := range createdReviews { - assert.Must(t, Application.Delete(review.ID)) + assert.Must(t, Application.Delete(review.Application.ID)) assert.Must(t, Review.Delete(review.ID)) } } diff --git a/test/api/review/samples.go b/test/api/review/samples.go index 4fb33b981..64df44e99 100644 --- a/test/api/review/samples.go +++ b/test/api/review/samples.go @@ -12,7 +12,6 @@ var Samples = []api.Review{ WorkPriority: 1, Comments: "nil", Application: &api.Ref{ - ID: 1, Name: "Sample Review 1", }, }, @@ -23,7 +22,6 @@ var Samples = []api.Review{ WorkPriority: 2, Comments: "nil", Application: &api.Ref{ - ID: 2, Name: "Sample Review 2", }, }, diff --git a/test/api/ticket/api_test.go b/test/api/ticket/api_test.go index af6f277c1..71ea3d178 100644 --- a/test/api/ticket/api_test.go +++ b/test/api/ticket/api_test.go @@ -17,6 +17,7 @@ func TestTicketCRUD(t *testing.T) { Name: r.Application.Name, } assert.Must(t, Application.Create(&app)) + r.Application.ID = app.ID createdIdentities := []api.Identity{} createdTrackers := []api.Tracker{} @@ -27,8 +28,11 @@ func TestTicketCRUD(t *testing.T) { Kind: tracker.Kind, } assert.Must(t, Identity.Create(&identity)) + tracker.Identity.ID = identity.ID createdIdentities = append(createdIdentities, identity) assert.Must(t, Tracker.Create(&tracker)) + r.Tracker.ID = tracker.ID + r.Tracker.Name = tracker.Name createdTrackers = append(createdTrackers, tracker) } @@ -72,6 +76,7 @@ func TestTicketList(t *testing.T) { Name: r.Application.Name, } assert.Must(t, Application.Create(&app)) + r.Application.ID = app.ID createdIdentities := []api.Identity{} createdTrackers := []api.Tracker{} @@ -82,8 +87,11 @@ func TestTicketList(t *testing.T) { Kind: tracker.Kind, } assert.Must(t, Identity.Create(&identity)) + tracker.Identity.ID = identity.ID createdIdentities = append(createdIdentities, identity) assert.Must(t, Tracker.Create(&tracker)) + r.Tracker.ID = tracker.ID + r.Tracker.Name = tracker.Name createdTrackers = append(createdTrackers, tracker) } @@ -113,7 +121,7 @@ func TestTicketList(t *testing.T) { // Delete tickets and related resources. for _, ticket := range createdTickets { assert.Must(t, Ticket.Delete(ticket.ID)) - assert.Must(t, Application.Delete(ticket.ID)) + assert.Must(t, Application.Delete(ticket.Application.ID)) } for _, tracker := range createdTrackers { assert.Must(t, Tracker.Delete(tracker.ID)) diff --git a/test/api/ticket/samples.go b/test/api/ticket/samples.go index 5a74246ad..a4fe35eb4 100644 --- a/test/api/ticket/samples.go +++ b/test/api/ticket/samples.go @@ -2,7 +2,6 @@ package ticket import ( "github.com/konveyor/tackle2-hub/api" - TrackerSamples "github.com/konveyor/tackle2-hub/test/api/tracker" ) var Samples = []api.Ticket{ @@ -10,12 +9,10 @@ var Samples = []api.Ticket{ Kind: "10001", Parent: "10000", Application: api.Ref{ - ID: 1, Name: "Sample Application1", }, Tracker: api.Ref{ - ID: 1, - Name: TrackerSamples.Samples[0].Name, + Name: "Sample Ticket-Tracker", }, }, } diff --git a/test/api/tracker/samples.go b/test/api/tracker/samples.go index 1face0d01..e9701e729 100644 --- a/test/api/tracker/samples.go +++ b/test/api/tracker/samples.go @@ -14,7 +14,6 @@ var Samples = []api.Tracker{ Message: "Description of tracker", LastUpdated: time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, time.Local), Identity: api.Ref{ - ID: 1, Name: "Sample Tracker Identity", }, Insecure: false, @@ -26,7 +25,6 @@ var Samples = []api.Tracker{ Message: "Description of tracker1", LastUpdated: time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, time.Local), Identity: api.Ref{ - ID: 2, Name: "Sample Tracker Identity1", }, Insecure: false, diff --git a/trigger/application.go b/trigger/application.go index f40c8d297..607f368bf 100644 --- a/trigger/application.go +++ b/trigger/application.go @@ -23,7 +23,7 @@ func (r *Application) Updated(m *model.Application) (err error) { if !Settings.Discovery.Enabled { return } - if len(m.Repository) == 0 || string(m.Repository) == "null" { + if m.Repository == (model.Repository{}) { return } kinds, err := r.FindTasks(Settings.Discovery.Label)