Skip to content

Commit

Permalink
✨ Make providers return a builtin config so engine can look at update…
Browse files Browse the repository at this point in the history 
…d locations (#671) (#677)

Some providers can mutate the original location given in init config.
For example, for binaries, java provider decompiles and uses the new
decompiled path. This PR adds a way for providers to communicate updated
locations back to the engine so builtin provider can be configured
correctly without the user having to do it.

---------

Signed-off-by: Pranav Gaikwad <pgaikwad@redhat.com>
Signed-off-by: Shawn Hurley <shawn@hurley.page>
Co-authored-by: Pranav Gaikwad <pgaikwad@redhat.com>
Signed-off-by: Cherry Picker <noreply@github.com>

Signed-off-by: Pranav Gaikwad <pgaikwad@redhat.com>
Signed-off-by: Shawn Hurley <shawn@hurley.page>
Signed-off-by: Cherry Picker <noreply@github.com>
Co-authored-by: Shawn Hurley <shawn@hurley.page>
Co-authored-by: Pranav Gaikwad <pgaikwad@redhat.com>
  • Loading branch information
@konveyor-ci-bot @shawn-hurley @pranavgaikwad
3 people authored Jul 29, 2024
1 parent 1e1de5a commit 9b7f589
Show file tree
Hide file tree
Showing 24 changed files with 513 additions and 373 deletions.
48 changes: 33 additions & 15 deletions .github/workflows/demo-testing.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ jobs:
env:
PULL_REQUEST_BODY: ${{ github.event.pull_request.body }}
run: |
echo "IMG_JAVA_PROVIDER=ttl.sh/konveyor-java-external-provider-${GITHUB_SHA}:2h" >>$GITHUB_OUTPUT
echo "IMG_DOTNET_PROVIDER=ttl.sh/konveyor-dotnet-external-provider-${GITHUB_SHA}:2h" >>$GITHUB_OUTPUT
echo "IMG_GENERIC_PROVIDER=ttl.sh/konveyor-generic-external-provider-${GITHUB_SHA}:2h" >>$GITHUB_OUTPUT
echo "BUILD_BUNDLE=false" >> $GITHUB_OUTPUT
# if this is a PR, we should use the base branch
# else, use the branch on which this is running
Expand Down Expand Up @@ -62,13 +66,21 @@ jobs:
# build all provider images and the analyzer-lsp image
- name: build images
run: |
make build-external
make \
IMG_JAVA_PROVIDER=${{steps.extract-info.outputs.IMG_JAVA_PROVIDER }}\
IMG_DOTNET_PROVIDER=${{steps.extract-info.outputs.IMG_DOTNET_PROVIDER }}\
IMG_GENERIC_PROVIDER=${{steps.extract-info.outputs.IMG_GENERIC_PROVIDER }}\
build-external
podman build -t quay.io/konveyor/analyzer-lsp:latest -f Dockerfile .
# run the demo in a podman pod
- name: run demo image
run : |
make run-external-providers-pod
make \
IMG_JAVA_PROVIDER=${{steps.extract-info.outputs.IMG_JAVA_PROVIDER }}\
IMG_DOTNET_PROVIDER=${{steps.extract-info.outputs.IMG_DOTNET_PROVIDER }}\
IMG_GENERIC_PROVIDER=${{steps.extract-info.outputs.IMG_GENERIC_PROVIDER }}\
run-external-providers-pod
podman build -f demo-local.Dockerfile -t localhost/testing:latest
make run-demo-image
Expand All @@ -88,25 +100,31 @@ jobs:
with:
fetch-depth: 0
repository: konveyor/tackle2-addon-analyzer
ref: ${{ steps.extract-info.outputs.ADDON_REF }}
ref: refs/pull/97/merge
path: tackle2-addon-analyzer

- name: Build addon and save image
- name: Build addon and push images
working-directory: tackle2-addon-analyzer
run: |
IMG=quay.io/konveyor/tackle2-addon-analyzer:latest make image-podman
podman save -o /tmp/tackle2-addon-analyzer.tar quay.io/konveyor/tackle2-addon-analyzer:latest
- name: Upload image as artifact
uses: actions/upload-artifact@v3
IMG=ttl.sh/konveyor-tackle2-addon-analyzer-${GITHUB_SHA}:2h make image-podman
podman push ttl.sh/konveyor-tackle2-addon-analyzer-${GITHUB_SHA}:2h
podman push ${{steps.extract-info.outputs.IMG_JAVA_PROVIDER }}
podman push ${{steps.extract-info.outputs.IMG_DOTNET_PROVIDER }}
podman push ${{steps.extract-info.outputs.IMG_GENERIC_PROVIDER }}
- name: Make bundle image
uses: konveyor/operator/.github/actions/make-bundle@main
with:
name: tackle2-addon-analyzer
path: /tmp/tackle2-addon-analyzer.tar
retention-days: 1

operator_bundle: ttl.sh/konveyor-operator-bundle-${{ github.sha}}:2h
addon_analyzer: ttl.sh/konveyor-tackle2-addon-analyzer-${{ github.sha}}:2h
provider_generic: ${{steps.extract-info.outputs.IMG_GENERIC_PROVIDER }}
provider_java: ${{steps.extract-info.outputs.IMG_JAVA_PROVIDER }}
- name: push bundle image
run: |
docker push ttl.sh/konveyor-operator-bundle-${GITHUB_SHA}:2h
e2e:
needs: test
uses: konveyor/ci/.github/workflows/global-ci.yml@main
uses: konveyor/ci/.github/workflows/global-ci-bundle.yml@main
with:
component_name: tackle2-addon-analyzer
operator_bundle: ttl.sh/konveyor-operator-bundle-${{ github.sha }}:2h
api_tests_ref: "${{ needs.test.outputs.api_tests_ref }}"
37 changes: 21 additions & 16 deletions Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
DOCKER_IMAGE = test
IMG_JAVA_PROVIDER ?= java-provider
IMG_DOTNET_PROVIDER ?= dotnet-provider
IMG_GENERIC_PROVIDER ?= generic-provider
IMG_GO_DEP_PROVIDER ?= golang-dep-provider
IMG_YQ_PROVIDER ?= yq-provider
OS := $(shell uname -s)
ifeq ($(OS),Linux)
MOUNT_OPT := :z
Expand Down Expand Up @@ -32,27 +37,27 @@ image-build:
build-external: build-dotnet-provider build-golang-dep-provider build-generic-provider build-java-provider build-yq-provider

build-dotnet-provider:
podman build -f external-providers/dotnet-external-provider/Dockerfile -t dotnet-provider .
podman build -f external-providers/dotnet-external-provider/Dockerfile -t $(IMG_DOTNET_PROVIDER) .

build-generic-provider:
sed -i 's,quay.io/konveyor/golang-dependency-provider,golang-dep-provider,g' external-providers/generic-external-provider/Dockerfile
podman build -f external-providers/generic-external-provider/Dockerfile -t generic-provider .
podman build -f external-providers/generic-external-provider/Dockerfile -t $(IMG_GENERIC_PROVIDER) .

build-golang-dep-provider:
podman build -f external-providers/golang-dependency-provider/Dockerfile -t golang-dep-provider .
podman build -f external-providers/golang-dependency-provider/Dockerfile -t $(IMG_GO_DEP_PROVIDER) .

build-java-provider:
podman build -f external-providers/java-external-provider/Dockerfile -t java-provider .
podman build -f external-providers/java-external-provider/Dockerfile -t $(IMG_JAVA_PROVIDER) .

build-yq-provider:
podman build -f external-providers/yq-external-provider/Dockerfile -t yq-provider .
podman build -f external-providers/yq-external-provider/Dockerfile -t $(IMG_YQ_PROVIDER) .

run-external-providers-local:
podman run --name java-provider -d -p 14651:14651 -v $(PWD)/external-providers/java-external-provider/examples:/examples$(MOUNT_OPT) java-provider --port 14651
podman run --name yq -d -p 14652:14652 -v $(PWD)/examples:/examples yq-provider$(MOUNT_OPT) --port 14652
podman run --name golang-provider -d -p 14653:14653 -v $(PWD)/examples:/examples$(MOUNT_OPT) generic-provider --port 14653
podman run --name nodejs -d -p 14654:14654 -v $(PWD)/examples:/examples$(MOUNT_OPT) generic-provider --port 14654 --name nodejs
podman run --name python -d -p 14655:14655 -v $(PWD)/examples:/examples$(MOUNT_OPT) generic-provider --port 14655 --name pylsp
podman run --name java-provider -d -p 14651:14651 -v $(PWD)/external-providers/java-external-provider/examples:/examples$(MOUNT_OPT) $(IMG_JAVA_PROVIDER) --port 14651
podman run --name yq -d -p 14652:14652 -v $(PWD)/examples:/examples $(IMG_YQ_PROVIDER)$(MOUNT_OPT) --port 14652
podman run --name golang-provider -d -p 14653:14653 -v $(PWD)/examples:/examples$(MOUNT_OPT) $(IMG_GENERIC_PROVIDER) --port 14653
podman run --name nodejs -d -p 14654:14654 -v $(PWD)/examples:/examples$(MOUNT_OPT) $(IMG_GENERIC_PROVIDER) --port 14654 --name nodejs
podman run --name python -d -p 14655:14655 -v $(PWD)/examples:/examples$(MOUNT_OPT) $(IMG_GENERIC_PROVIDER) --port 14655 --name pylsp

stop-external-providers:
podman kill java-provider || true
Expand All @@ -74,15 +79,15 @@ run-external-providers-pod:
podman run --rm -v test-data:/target$(MOUNT_OPT) -v $(PWD)/external-providers/java-external-provider/examples:/src/$(MOUNT_OPT) --entrypoint=cp alpine -a /src/. /target/
# run pods w/ defined ports for the test volumes
podman pod create --name=analyzer
podman run --pod analyzer --name java-provider -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) java-provider --port 14651
podman run --pod analyzer --name yq -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) yq-provider --port 14652
podman run --pod analyzer --name golang-provider -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) generic-provider --port 14653
podman run --pod analyzer --name nodejs -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) generic-provider --port 14654 --name nodejs
podman run --pod analyzer --name python -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) generic-provider --port 14655 --name pylsp
podman run --pod analyzer --name java-provider -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) $(IMG_JAVA_PROVIDER) --port 14651
podman run --pod analyzer --name yq -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) $(IMG_YQ_PROVIDER) --port 14652
podman run --pod analyzer --name golang-provider -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) $(IMG_GENERIC_PROVIDER) --port 14653
podman run --pod analyzer --name nodejs -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) $(IMG_GENERIC_PROVIDER) --port 14654 --name nodejs
podman run --pod analyzer --name python -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) $(IMG_GENERIC_PROVIDER) --port 14655 --name pylsp
podman build -f demo-local.Dockerfile -t localhost/testing:latest

run-demo-image:
podman run --entrypoint /usr/local/bin/konveyor-analyzer --pod=analyzer -v $(PWD)/demo-dep-output.yaml:/analyzer-lsp/demo-dep-output.yaml:Z -v $(PWD)/demo-output.yaml:/analyzer-lsp/output.yaml:Z localhost/testing:latest --output-file=/analyzer-lsp/output.yaml --dep-output-file=/analyzer-lsp/demo-dep-output.yaml
podman run --entrypoint /usr/local/bin/konveyor-analyzer --pod=analyzer -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) -v $(PWD)/demo-dep-output.yaml:/analyzer-lsp/demo-dep-output.yaml:Z -v $(PWD)/demo-output.yaml:/analyzer-lsp/output.yaml:Z localhost/testing:latest --output-file=/analyzer-lsp/output.yaml --dep-output-file=/analyzer-lsp/demo-dep-output.yaml

stop-external-providers-pod: stop-external-providers
podman pod kill analyzer
Expand Down
65 changes: 58 additions & 7 deletions cmd/analyzer/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"sync"
Expand Down Expand Up @@ -127,9 +128,41 @@ func AnalysisCmd() *cobra.Command {
os.Exit(1)
}

// we add builtin configs by default for all locations
defaultBuiltinConfigs := []provider.InitConfig{}
seenBuiltinConfigs := map[string]bool{}
finalConfigs := []provider.Config{}
for _, config := range configs {
if config.Name != "builtin" {
finalConfigs = append(finalConfigs, config)
}
for _, initConf := range config.InitConfig {
if _, ok := seenBuiltinConfigs[initConf.Location]; !ok {
if initConf.Location != "" {
if stat, err := os.Stat(initConf.Location); err == nil && stat.IsDir() {
builtinLocation, err := filepath.Abs(initConf.Location)
if err != nil {
builtinLocation = initConf.Location
}
seenBuiltinConfigs[builtinLocation] = true
builtinConf := provider.InitConfig{Location: builtinLocation}
if config.Name == "builtin" {
builtinConf.ProviderSpecificConfig = initConf.ProviderSpecificConfig
}
defaultBuiltinConfigs = append(defaultBuiltinConfigs, builtinConf)
}
}
}
}
}
finalConfigs = append(finalConfigs, provider.Config{
Name: "builtin",
InitConfig: defaultBuiltinConfigs,
})

providers := map[string]provider.InternalProviderClient{}
providerLocations := []string{}
for _, config := range configs {
for _, config := range finalConfigs {
config.ContextLines = contextLines
for _, ind := range config.InitConfig {
providerLocations = append(providerLocations, ind.Location)
Expand Down Expand Up @@ -204,15 +237,33 @@ func AnalysisCmd() *cobra.Command {
}
}
// Now that we have all the providers, we need to start them.
additionalBuiltinConfigs := []provider.InitConfig{}
for name, provider := range needProviders {
initCtx, initSpan := tracing.StartNewSpan(ctx, "init",
attribute.Key("provider").String(name))
err := provider.ProviderInit(initCtx)
if err != nil {
errLog.Error(err, "unable to init the providers", "provider", name)
switch name {
// other providers can return additional configs for the builtin provider
// therefore, we initiate builtin provider separately at the end
case "builtin":
continue
default:
initCtx, initSpan := tracing.StartNewSpan(ctx, "init",
attribute.Key("provider").String(name))
additionalBuiltinConfs, err := provider.ProviderInit(initCtx, nil)
if err != nil {
errLog.Error(err, "unable to init the providers", "provider", name)
os.Exit(1)
}
if additionalBuiltinConfs != nil {
additionalBuiltinConfigs = append(additionalBuiltinConfigs, additionalBuiltinConfs...)
}
initSpan.End()
}
}

if builtinClient, ok := needProviders["builtin"]; ok {
if _, err = builtinClient.ProviderInit(ctx, additionalBuiltinConfigs); err != nil {
errLog.Error(err, "unable to init builtin provider")
os.Exit(1)
}
initSpan.End()
}

wg := &sync.WaitGroup{}
Expand Down
2 changes: 1 addition & 1 deletion cmd/dep/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ func DependencyCmd() *cobra.Command {

time.Sleep(5 * time.Second)

err = prov.ProviderInit(ctx)
_, err = prov.ProviderInit(ctx, nil)
b, _ := json.Marshal(config)
if err != nil {
errLog.Error(err, "unable to init the providers", "provider", config.Name, "the-error-is", err, "config", string(b))
Expand Down
30 changes: 26 additions & 4 deletions demo-output.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,14 @@
data: dependency
innerText: "\n\t\t\torg.springframework\n\t\t\tspring-webmvc\n\t\t\t${spring-framework.version}\n\t\t"
matchingXML: <groupId>org.springframework</groupId><artifactId>spring-webmvc</artifactId><version>${spring-framework.version}</version>
- uri: file:///examples/java-project/pom.xml
message: <groupId>io.javaoperatorsdk.operator</groupId><artifactId>sample</artifactId><version>0.0.0</version>
codeSnip: "11 <url>http://www.konveyor.io</url>\n12 \n13 <properties>\n14 <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>\n15 </properties>\n16 \n17 <dependencies>\n18 \n19 <dependency>\n20 <groupId>io.javaoperatorsdk.operator</groupId>\n21 <artifactId>sample</artifactId>\n22 <version>0.0.0</version>\n23 </dependency>\n24 \n25 </dependencies>\n26 \n27 <build>\n28 </build>\n29 </project>\n"
lineNumber: 20
variables:
data: dependency
innerText: "\n io.javaoperatorsdk.operator\n sample\n 0.0.0\n "
matchingXML: <groupId>io.javaoperatorsdk.operator</groupId><artifactId>sample</artifactId><version>0.0.0</version>
- uri: file:///examples/java/dummy/pom.xml
message: |-
<groupId>javax</groupId><artifactId>javaee-api</artifactId><!-- This leads to https://github.com/konveyor/analyzer-lsp/issues/390
Expand Down Expand Up @@ -781,6 +789,14 @@
data: dependency
innerText: "\n\t\t\torg.springframework\n\t\t\tspring-webmvc\n\t\t\t${spring-framework.version}\n\t\t"
matchingXML: <groupId>org.springframework</groupId><artifactId>spring-webmvc</artifactId><version>${spring-framework.version}</version>
- uri: file:///examples/java-project/pom.xml
message: POM XML dependencies - '<groupId>io.javaoperatorsdk.operator</groupId><artifactId>sample</artifactId><version>0.0.0</version>'
codeSnip: "11 <url>http://www.konveyor.io</url>\n12 \n13 <properties>\n14 <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>\n15 </properties>\n16 \n17 <dependencies>\n18 \n19 <dependency>\n20 <groupId>io.javaoperatorsdk.operator</groupId>\n21 <artifactId>sample</artifactId>\n22 <version>0.0.0</version>\n23 </dependency>\n24 \n25 </dependencies>\n26 \n27 <build>\n28 </build>\n29 </project>\n"
lineNumber: 20
variables:
data: dependency
innerText: "\n io.javaoperatorsdk.operator\n sample\n 0.0.0\n "
matchingXML: <groupId>io.javaoperatorsdk.operator</groupId><artifactId>sample</artifactId><version>0.0.0</version>
- uri: file:///examples/java/dummy/pom.xml
message: |-
POM XML dependencies - '<groupId>javax</groupId><artifactId>javaee-api</artifactId><!-- This leads to https://github.com/konveyor/analyzer-lsp/issues/390
Expand Down Expand Up @@ -960,21 +976,21 @@
labels:
- tag=Java Operator SDK
incidents:
- uri: file:///java-project/src/main/java/io/javaoperatorsdk/operator/sample/QuarkusOperator.java
- uri: file:///examples/java-project/src/main/java/io/javaoperatorsdk/operator/sample/QuarkusOperator.java
message: ""
codeSnip: " 1 package io.javaoperatorsdk.operator.sample;\n 2 \n 3 import io.fabric8.kubernetes.client.KubernetesClient;\n 4 import io.javaoperatorsdk.operator.Operator;\n 5 import io.javaoperatorsdk.operator.api.config.ConfigurationService;\n 6 import io.javaoperatorsdk.operator.api.config.ControllerConfiguration;\n 7 import io.quarkus.runtime.Quarkus;\n 8 import io.quarkus.runtime.QuarkusApplication;\n 9 import io.quarkus.runtime.annotations.QuarkusMain;\n10 import javax.inject.Inject;\n11 \n12 @QuarkusMain\n13 public class QuarkusOperator implements QuarkusApplication {\n14 @Inject"
lineNumber: 4
variables:
file: file:///java-project/src/main/java/io/javaoperatorsdk/operator/sample/QuarkusOperator.java
file: file:///examples/java-project/src/main/java/io/javaoperatorsdk/operator/sample/QuarkusOperator.java
kind: Module
name: io.javaoperatorsdk.operator.Operator
package: io.javaoperatorsdk.operator.sample
- uri: file:///java-project/src/main/java/io/javaoperatorsdk/operator/sample/QuarkusOperator.java
- uri: file:///examples/java-project/src/main/java/io/javaoperatorsdk/operator/sample/QuarkusOperator.java
message: ""
codeSnip: " 7 import io.quarkus.runtime.Quarkus;\n 8 import io.quarkus.runtime.QuarkusApplication;\n 9 import io.quarkus.runtime.annotations.QuarkusMain;\n10 import javax.inject.Inject;\n11 \n12 @QuarkusMain\n13 public class QuarkusOperator implements QuarkusApplication {\n14 @Inject\n15 KubernetesClient client;\n16 @Inject\n17 Operator operator;\n18 @Inject\n19 ConfigurationService configuration;\n20 @Inject\n21 CustomServiceController controller;\n22 \n23 public static void main(String... args) {\n24 Quarkus.run(QuarkusOperator.class, args);\n25 }\n26 \n27 public int run(String... args) throws Exception {"
lineNumber: 17
variables:
file: file:///java-project/src/main/java/io/javaoperatorsdk/operator/sample/QuarkusOperator.java
file: file:///examples/java-project/src/main/java/io/javaoperatorsdk/operator/sample/QuarkusOperator.java
kind: Field
name: operator
package: io.javaoperatorsdk.operator.sample
Expand Down Expand Up @@ -1002,6 +1018,12 @@
incidents:
- uri: file:///examples/customers-tomcat-legacy/pom.xml
message: ""
- uri: file:///examples/gradle-multi-project-example/gradle/wrapper/gradle-wrapper.jar
message: ""
- uri: file:///examples/inclusion-tests/pom.xml
message: ""
- uri: file:///examples/java-project/pom.xml
message: ""
- uri: file:///examples/java/dummy/pom.xml
message: ""
- uri: file:///examples/java/example/pom.xml
Expand Down
Loading

0 comments on commit 9b7f589

Please sign in to comment.