From 9b7f5895b6b8bc805134b4aea009c11b96cf3700 Mon Sep 17 00:00:00 2001 From: "konveyor-ci-bot[bot]" <159171263+konveyor-ci-bot[bot]@users.noreply.github.com> Date: Mon, 29 Jul 2024 12:18:02 -0400 Subject: [PATCH] :sparkles: Make providers return a builtin config so engine can look at updated locations (#671) (#677) Some providers can mutate the original location given in init config. For example, for binaries, java provider decompiles and uses the new decompiled path. This PR adds a way for providers to communicate updated locations back to the engine so builtin provider can be configured correctly without the user having to do it. --------- Signed-off-by: Pranav Gaikwad Signed-off-by: Shawn Hurley Co-authored-by: Pranav Gaikwad Signed-off-by: Cherry Picker Signed-off-by: Pranav Gaikwad Signed-off-by: Shawn Hurley Signed-off-by: Cherry Picker Co-authored-by: Shawn Hurley Co-authored-by: Pranav Gaikwad --- .github/workflows/demo-testing.yml | 48 +- Makefile | 37 +- cmd/analyzer/main.go | 65 ++- cmd/dep/main.go | 2 +- demo-output.yaml | 30 +- .../pkg/dotnet/provider.go | 19 +- .../pkg/generic_external_provider/provider.go | 8 +- .../generic/service_client_test.go | 2 +- .../pylsp/service_client_test.go | 2 +- .../service_client_test.go | 2 +- .../pkg/java_external_provider/provider.go | 30 +- .../pkg/yq_provider/provider.go | 14 +- parser/rule_parser_test.go | 8 +- provider/grpc/provider.go | 29 +- provider/internal/builtin/provider.go | 24 +- provider/internal/grpc/library.pb.go | 521 +++++++++--------- provider/internal/grpc/library.proto | 1 + provider/internal/grpc/library_grpc.pb.go | 2 +- provider/provider.go | 15 +- provider/provider_test.go | 8 +- provider/server.go | 13 +- provider_container_settings.json | 2 +- provider_local_external_images.json | 2 +- provider_pod_local_settings.json | 2 +- 24 files changed, 513 insertions(+), 373 deletions(-) diff --git a/.github/workflows/demo-testing.yml b/.github/workflows/demo-testing.yml index 2c8eb7a9..01c0e7d4 100644 --- a/.github/workflows/demo-testing.yml +++ b/.github/workflows/demo-testing.yml @@ -13,6 +13,10 @@ jobs: env: PULL_REQUEST_BODY: ${{ github.event.pull_request.body }} run: | + echo "IMG_JAVA_PROVIDER=ttl.sh/konveyor-java-external-provider-${GITHUB_SHA}:2h" >>$GITHUB_OUTPUT + echo "IMG_DOTNET_PROVIDER=ttl.sh/konveyor-dotnet-external-provider-${GITHUB_SHA}:2h" >>$GITHUB_OUTPUT + echo "IMG_GENERIC_PROVIDER=ttl.sh/konveyor-generic-external-provider-${GITHUB_SHA}:2h" >>$GITHUB_OUTPUT + echo "BUILD_BUNDLE=false" >> $GITHUB_OUTPUT # if this is a PR, we should use the base branch # else, use the branch on which this is running @@ -62,13 +66,21 @@ jobs: # build all provider images and the analyzer-lsp image - name: build images run: | - make build-external + make \ + IMG_JAVA_PROVIDER=${{steps.extract-info.outputs.IMG_JAVA_PROVIDER }}\ + IMG_DOTNET_PROVIDER=${{steps.extract-info.outputs.IMG_DOTNET_PROVIDER }}\ + IMG_GENERIC_PROVIDER=${{steps.extract-info.outputs.IMG_GENERIC_PROVIDER }}\ + build-external podman build -t quay.io/konveyor/analyzer-lsp:latest -f Dockerfile . # run the demo in a podman pod - name: run demo image run : | - make run-external-providers-pod + make \ + IMG_JAVA_PROVIDER=${{steps.extract-info.outputs.IMG_JAVA_PROVIDER }}\ + IMG_DOTNET_PROVIDER=${{steps.extract-info.outputs.IMG_DOTNET_PROVIDER }}\ + IMG_GENERIC_PROVIDER=${{steps.extract-info.outputs.IMG_GENERIC_PROVIDER }}\ + run-external-providers-pod podman build -f demo-local.Dockerfile -t localhost/testing:latest make run-demo-image @@ -88,25 +100,31 @@ jobs: with: fetch-depth: 0 repository: konveyor/tackle2-addon-analyzer - ref: ${{ steps.extract-info.outputs.ADDON_REF }} + ref: refs/pull/97/merge path: tackle2-addon-analyzer - - name: Build addon and save image + - name: Build addon and push images working-directory: tackle2-addon-analyzer run: | - IMG=quay.io/konveyor/tackle2-addon-analyzer:latest make image-podman - podman save -o /tmp/tackle2-addon-analyzer.tar quay.io/konveyor/tackle2-addon-analyzer:latest - - - name: Upload image as artifact - uses: actions/upload-artifact@v3 + IMG=ttl.sh/konveyor-tackle2-addon-analyzer-${GITHUB_SHA}:2h make image-podman + podman push ttl.sh/konveyor-tackle2-addon-analyzer-${GITHUB_SHA}:2h + podman push ${{steps.extract-info.outputs.IMG_JAVA_PROVIDER }} + podman push ${{steps.extract-info.outputs.IMG_DOTNET_PROVIDER }} + podman push ${{steps.extract-info.outputs.IMG_GENERIC_PROVIDER }} + + - name: Make bundle image + uses: konveyor/operator/.github/actions/make-bundle@main with: - name: tackle2-addon-analyzer - path: /tmp/tackle2-addon-analyzer.tar - retention-days: 1 - + operator_bundle: ttl.sh/konveyor-operator-bundle-${{ github.sha}}:2h + addon_analyzer: ttl.sh/konveyor-tackle2-addon-analyzer-${{ github.sha}}:2h + provider_generic: ${{steps.extract-info.outputs.IMG_GENERIC_PROVIDER }} + provider_java: ${{steps.extract-info.outputs.IMG_JAVA_PROVIDER }} + - name: push bundle image + run: | + docker push ttl.sh/konveyor-operator-bundle-${GITHUB_SHA}:2h e2e: needs: test - uses: konveyor/ci/.github/workflows/global-ci.yml@main + uses: konveyor/ci/.github/workflows/global-ci-bundle.yml@main with: - component_name: tackle2-addon-analyzer + operator_bundle: ttl.sh/konveyor-operator-bundle-${{ github.sha }}:2h api_tests_ref: "${{ needs.test.outputs.api_tests_ref }}" diff --git a/Makefile b/Makefile index 28b87bd9..59cefe95 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,9 @@ DOCKER_IMAGE = test +IMG_JAVA_PROVIDER ?= java-provider +IMG_DOTNET_PROVIDER ?= dotnet-provider +IMG_GENERIC_PROVIDER ?= generic-provider +IMG_GO_DEP_PROVIDER ?= golang-dep-provider +IMG_YQ_PROVIDER ?= yq-provider OS := $(shell uname -s) ifeq ($(OS),Linux) MOUNT_OPT := :z @@ -32,27 +37,27 @@ image-build: build-external: build-dotnet-provider build-golang-dep-provider build-generic-provider build-java-provider build-yq-provider build-dotnet-provider: - podman build -f external-providers/dotnet-external-provider/Dockerfile -t dotnet-provider . + podman build -f external-providers/dotnet-external-provider/Dockerfile -t $(IMG_DOTNET_PROVIDER) . build-generic-provider: sed -i 's,quay.io/konveyor/golang-dependency-provider,golang-dep-provider,g' external-providers/generic-external-provider/Dockerfile - podman build -f external-providers/generic-external-provider/Dockerfile -t generic-provider . + podman build -f external-providers/generic-external-provider/Dockerfile -t $(IMG_GENERIC_PROVIDER) . build-golang-dep-provider: - podman build -f external-providers/golang-dependency-provider/Dockerfile -t golang-dep-provider . + podman build -f external-providers/golang-dependency-provider/Dockerfile -t $(IMG_GO_DEP_PROVIDER) . build-java-provider: - podman build -f external-providers/java-external-provider/Dockerfile -t java-provider . + podman build -f external-providers/java-external-provider/Dockerfile -t $(IMG_JAVA_PROVIDER) . build-yq-provider: - podman build -f external-providers/yq-external-provider/Dockerfile -t yq-provider . + podman build -f external-providers/yq-external-provider/Dockerfile -t $(IMG_YQ_PROVIDER) . run-external-providers-local: - podman run --name java-provider -d -p 14651:14651 -v $(PWD)/external-providers/java-external-provider/examples:/examples$(MOUNT_OPT) java-provider --port 14651 - podman run --name yq -d -p 14652:14652 -v $(PWD)/examples:/examples yq-provider$(MOUNT_OPT) --port 14652 - podman run --name golang-provider -d -p 14653:14653 -v $(PWD)/examples:/examples$(MOUNT_OPT) generic-provider --port 14653 - podman run --name nodejs -d -p 14654:14654 -v $(PWD)/examples:/examples$(MOUNT_OPT) generic-provider --port 14654 --name nodejs - podman run --name python -d -p 14655:14655 -v $(PWD)/examples:/examples$(MOUNT_OPT) generic-provider --port 14655 --name pylsp + podman run --name java-provider -d -p 14651:14651 -v $(PWD)/external-providers/java-external-provider/examples:/examples$(MOUNT_OPT) $(IMG_JAVA_PROVIDER) --port 14651 + podman run --name yq -d -p 14652:14652 -v $(PWD)/examples:/examples $(IMG_YQ_PROVIDER)$(MOUNT_OPT) --port 14652 + podman run --name golang-provider -d -p 14653:14653 -v $(PWD)/examples:/examples$(MOUNT_OPT) $(IMG_GENERIC_PROVIDER) --port 14653 + podman run --name nodejs -d -p 14654:14654 -v $(PWD)/examples:/examples$(MOUNT_OPT) $(IMG_GENERIC_PROVIDER) --port 14654 --name nodejs + podman run --name python -d -p 14655:14655 -v $(PWD)/examples:/examples$(MOUNT_OPT) $(IMG_GENERIC_PROVIDER) --port 14655 --name pylsp stop-external-providers: podman kill java-provider || true @@ -74,15 +79,15 @@ run-external-providers-pod: podman run --rm -v test-data:/target$(MOUNT_OPT) -v $(PWD)/external-providers/java-external-provider/examples:/src/$(MOUNT_OPT) --entrypoint=cp alpine -a /src/. /target/ # run pods w/ defined ports for the test volumes podman pod create --name=analyzer - podman run --pod analyzer --name java-provider -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) java-provider --port 14651 - podman run --pod analyzer --name yq -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) yq-provider --port 14652 - podman run --pod analyzer --name golang-provider -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) generic-provider --port 14653 - podman run --pod analyzer --name nodejs -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) generic-provider --port 14654 --name nodejs - podman run --pod analyzer --name python -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) generic-provider --port 14655 --name pylsp + podman run --pod analyzer --name java-provider -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) $(IMG_JAVA_PROVIDER) --port 14651 + podman run --pod analyzer --name yq -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) $(IMG_YQ_PROVIDER) --port 14652 + podman run --pod analyzer --name golang-provider -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) $(IMG_GENERIC_PROVIDER) --port 14653 + podman run --pod analyzer --name nodejs -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) $(IMG_GENERIC_PROVIDER) --port 14654 --name nodejs + podman run --pod analyzer --name python -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) $(IMG_GENERIC_PROVIDER) --port 14655 --name pylsp podman build -f demo-local.Dockerfile -t localhost/testing:latest run-demo-image: - podman run --entrypoint /usr/local/bin/konveyor-analyzer --pod=analyzer -v $(PWD)/demo-dep-output.yaml:/analyzer-lsp/demo-dep-output.yaml:Z -v $(PWD)/demo-output.yaml:/analyzer-lsp/output.yaml:Z localhost/testing:latest --output-file=/analyzer-lsp/output.yaml --dep-output-file=/analyzer-lsp/demo-dep-output.yaml + podman run --entrypoint /usr/local/bin/konveyor-analyzer --pod=analyzer -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) -v $(PWD)/demo-dep-output.yaml:/analyzer-lsp/demo-dep-output.yaml:Z -v $(PWD)/demo-output.yaml:/analyzer-lsp/output.yaml:Z localhost/testing:latest --output-file=/analyzer-lsp/output.yaml --dep-output-file=/analyzer-lsp/demo-dep-output.yaml stop-external-providers-pod: stop-external-providers podman pod kill analyzer diff --git a/cmd/analyzer/main.go b/cmd/analyzer/main.go index c7de70f8..111bc9e0 100644 --- a/cmd/analyzer/main.go +++ b/cmd/analyzer/main.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "os" + "path/filepath" "sort" "strings" "sync" @@ -127,9 +128,41 @@ func AnalysisCmd() *cobra.Command { os.Exit(1) } + // we add builtin configs by default for all locations + defaultBuiltinConfigs := []provider.InitConfig{} + seenBuiltinConfigs := map[string]bool{} + finalConfigs := []provider.Config{} + for _, config := range configs { + if config.Name != "builtin" { + finalConfigs = append(finalConfigs, config) + } + for _, initConf := range config.InitConfig { + if _, ok := seenBuiltinConfigs[initConf.Location]; !ok { + if initConf.Location != "" { + if stat, err := os.Stat(initConf.Location); err == nil && stat.IsDir() { + builtinLocation, err := filepath.Abs(initConf.Location) + if err != nil { + builtinLocation = initConf.Location + } + seenBuiltinConfigs[builtinLocation] = true + builtinConf := provider.InitConfig{Location: builtinLocation} + if config.Name == "builtin" { + builtinConf.ProviderSpecificConfig = initConf.ProviderSpecificConfig + } + defaultBuiltinConfigs = append(defaultBuiltinConfigs, builtinConf) + } + } + } + } + } + finalConfigs = append(finalConfigs, provider.Config{ + Name: "builtin", + InitConfig: defaultBuiltinConfigs, + }) + providers := map[string]provider.InternalProviderClient{} providerLocations := []string{} - for _, config := range configs { + for _, config := range finalConfigs { config.ContextLines = contextLines for _, ind := range config.InitConfig { providerLocations = append(providerLocations, ind.Location) @@ -204,15 +237,33 @@ func AnalysisCmd() *cobra.Command { } } // Now that we have all the providers, we need to start them. + additionalBuiltinConfigs := []provider.InitConfig{} for name, provider := range needProviders { - initCtx, initSpan := tracing.StartNewSpan(ctx, "init", - attribute.Key("provider").String(name)) - err := provider.ProviderInit(initCtx) - if err != nil { - errLog.Error(err, "unable to init the providers", "provider", name) + switch name { + // other providers can return additional configs for the builtin provider + // therefore, we initiate builtin provider separately at the end + case "builtin": + continue + default: + initCtx, initSpan := tracing.StartNewSpan(ctx, "init", + attribute.Key("provider").String(name)) + additionalBuiltinConfs, err := provider.ProviderInit(initCtx, nil) + if err != nil { + errLog.Error(err, "unable to init the providers", "provider", name) + os.Exit(1) + } + if additionalBuiltinConfs != nil { + additionalBuiltinConfigs = append(additionalBuiltinConfigs, additionalBuiltinConfs...) + } + initSpan.End() + } + } + + if builtinClient, ok := needProviders["builtin"]; ok { + if _, err = builtinClient.ProviderInit(ctx, additionalBuiltinConfigs); err != nil { + errLog.Error(err, "unable to init builtin provider") os.Exit(1) } - initSpan.End() } wg := &sync.WaitGroup{} diff --git a/cmd/dep/main.go b/cmd/dep/main.go index 67615589..beb985a3 100644 --- a/cmd/dep/main.go +++ b/cmd/dep/main.go @@ -90,7 +90,7 @@ func DependencyCmd() *cobra.Command { time.Sleep(5 * time.Second) - err = prov.ProviderInit(ctx) + _, err = prov.ProviderInit(ctx, nil) b, _ := json.Marshal(config) if err != nil { errLog.Error(err, "unable to init the providers", "provider", config.Name, "the-error-is", err, "config", string(b)) diff --git a/demo-output.yaml b/demo-output.yaml index bdbbe45a..9d0873a3 100644 --- a/demo-output.yaml +++ b/demo-output.yaml @@ -201,6 +201,14 @@ data: dependency innerText: "\n\t\t\torg.springframework\n\t\t\tspring-webmvc\n\t\t\t${spring-framework.version}\n\t\t" matchingXML: org.springframeworkspring-webmvc${spring-framework.version} + - uri: file:///examples/java-project/pom.xml + message: io.javaoperatorsdk.operatorsample0.0.0 + codeSnip: "11 http://www.konveyor.io\n12 \n13 \n14 UTF-8\n15 \n16 \n17 \n18 \n19 \n20 io.javaoperatorsdk.operator\n21 sample\n22 0.0.0\n23 \n24 \n25 \n26 \n27 \n28 \n29 \n" + lineNumber: 20 + variables: + data: dependency + innerText: "\n io.javaoperatorsdk.operator\n sample\n 0.0.0\n " + matchingXML: io.javaoperatorsdk.operatorsample0.0.0 - uri: file:///examples/java/dummy/pom.xml message: |- javaxjavaee-api