Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

✨ Make providers return a builtin config so engine can look at updated locations #671

Merged
merged 5 commits into from
Jul 29, 2024
Merged

✨ Make providers return a builtin config so engine can look at updated locations #671

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d997fc6
:sparkles: configure builtin provider internally
pranavgaikwad Jun 21, 2024
513fafd
:bug: fix github workflow to use external provider images
pranavgaikwad Jul 11, 2024
1e192fb
remove default builtin config
pranavgaikwad Jul 17, 2024
ed23116
:bug: fix default builtin configs
pranavgaikwad Jul 18, 2024
5087451
Fixing ci workflow to work with all the provider images
shawn-hurley Jul 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 33 additions & 15 deletions .github/workflows/demo-testing.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ jobs:
env:
PULL_REQUEST_BODY: ${{ github.event.pull_request.body }}
run: |
echo "IMG_JAVA_PROVIDER=ttl.sh/konveyor-java-external-provider-${GITHUB_SHA}:2h" >>$GITHUB_OUTPUT
echo "IMG_DOTNET_PROVIDER=ttl.sh/konveyor-dotnet-external-provider-${GITHUB_SHA}:2h" >>$GITHUB_OUTPUT
echo "IMG_GENERIC_PROVIDER=ttl.sh/konveyor-generic-external-provider-${GITHUB_SHA}:2h" >>$GITHUB_OUTPUT

echo "BUILD_BUNDLE=false" >> $GITHUB_OUTPUT
# if this is a PR, we should use the base branch
# else, use the branch on which this is running
Expand Down Expand Up @@ -62,13 +66,21 @@ jobs:
# build all provider images and the analyzer-lsp image
- name: build images
run: |
make build-external
make \
IMG_JAVA_PROVIDER=${{steps.extract-info.outputs.IMG_JAVA_PROVIDER }}\
IMG_DOTNET_PROVIDER=${{steps.extract-info.outputs.IMG_DOTNET_PROVIDER }}\
IMG_GENERIC_PROVIDER=${{steps.extract-info.outputs.IMG_GENERIC_PROVIDER }}\
build-external
podman build -t quay.io/konveyor/analyzer-lsp:latest -f Dockerfile .

# run the demo in a podman pod
- name: run demo image
run : |
make run-external-providers-pod
make \
IMG_JAVA_PROVIDER=${{steps.extract-info.outputs.IMG_JAVA_PROVIDER }}\
IMG_DOTNET_PROVIDER=${{steps.extract-info.outputs.IMG_DOTNET_PROVIDER }}\
IMG_GENERIC_PROVIDER=${{steps.extract-info.outputs.IMG_GENERIC_PROVIDER }}\
run-external-providers-pod
podman build -f demo-local.Dockerfile -t localhost/testing:latest
make run-demo-image

Expand All @@ -88,25 +100,31 @@ jobs:
with:
fetch-depth: 0
repository: konveyor/tackle2-addon-analyzer
ref: ${{ steps.extract-info.outputs.ADDON_REF }}
ref: refs/pull/97/merge
path: tackle2-addon-analyzer

- name: Build addon and save image
- name: Build addon and push images
working-directory: tackle2-addon-analyzer
run: |
IMG=quay.io/konveyor/tackle2-addon-analyzer:latest make image-podman
podman save -o /tmp/tackle2-addon-analyzer.tar quay.io/konveyor/tackle2-addon-analyzer:latest

- name: Upload image as artifact
uses: actions/upload-artifact@v3
IMG=ttl.sh/konveyor-tackle2-addon-analyzer-${GITHUB_SHA}:2h make image-podman
podman push ttl.sh/konveyor-tackle2-addon-analyzer-${GITHUB_SHA}:2h
podman push ${{steps.extract-info.outputs.IMG_JAVA_PROVIDER }}
podman push ${{steps.extract-info.outputs.IMG_DOTNET_PROVIDER }}
podman push ${{steps.extract-info.outputs.IMG_GENERIC_PROVIDER }}

- name: Make bundle image
uses: konveyor/operator/.github/actions/make-bundle@main
with:
name: tackle2-addon-analyzer
path: /tmp/tackle2-addon-analyzer.tar
retention-days: 1

operator_bundle: ttl.sh/konveyor-operator-bundle-${{ github.sha}}:2h
addon_analyzer: ttl.sh/konveyor-tackle2-addon-analyzer-${{ github.sha}}:2h
provider_generic: ${{steps.extract-info.outputs.IMG_GENERIC_PROVIDER }}
provider_java: ${{steps.extract-info.outputs.IMG_JAVA_PROVIDER }}
- name: push bundle image
run: |
docker push ttl.sh/konveyor-operator-bundle-${GITHUB_SHA}:2h
e2e:
needs: test
uses: konveyor/ci/.github/workflows/global-ci.yml@main
uses: konveyor/ci/.github/workflows/global-ci-bundle.yml@main
with:
component_name: tackle2-addon-analyzer
operator_bundle: ttl.sh/konveyor-operator-bundle-${{ github.sha }}:2h
api_tests_ref: "${{ needs.test.outputs.api_tests_ref }}"
37 changes: 21 additions & 16 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
DOCKER_IMAGE = test
IMG_JAVA_PROVIDER ?= java-provider
IMG_DOTNET_PROVIDER ?= dotnet-provider
IMG_GENERIC_PROVIDER ?= generic-provider
IMG_GO_DEP_PROVIDER ?= golang-dep-provider
IMG_YQ_PROVIDER ?= yq-provider
OS := $(shell uname -s)
ifeq ($(OS),Linux)
MOUNT_OPT := :z
Expand Down Expand Up @@ -32,27 +37,27 @@ image-build:
build-external: build-dotnet-provider build-golang-dep-provider build-generic-provider build-java-provider build-yq-provider

build-dotnet-provider:
podman build -f external-providers/dotnet-external-provider/Dockerfile -t dotnet-provider .
podman build -f external-providers/dotnet-external-provider/Dockerfile -t $(IMG_DOTNET_PROVIDER) .

build-generic-provider:
sed -i 's,quay.io/konveyor/golang-dependency-provider,golang-dep-provider,g' external-providers/generic-external-provider/Dockerfile
podman build -f external-providers/generic-external-provider/Dockerfile -t generic-provider .
podman build -f external-providers/generic-external-provider/Dockerfile -t $(IMG_GENERIC_PROVIDER) .

build-golang-dep-provider:
podman build -f external-providers/golang-dependency-provider/Dockerfile -t golang-dep-provider .
podman build -f external-providers/golang-dependency-provider/Dockerfile -t $(IMG_GO_DEP_PROVIDER) .

build-java-provider:
podman build -f external-providers/java-external-provider/Dockerfile -t java-provider .
podman build -f external-providers/java-external-provider/Dockerfile -t $(IMG_JAVA_PROVIDER) .

build-yq-provider:
podman build -f external-providers/yq-external-provider/Dockerfile -t yq-provider .
podman build -f external-providers/yq-external-provider/Dockerfile -t $(IMG_YQ_PROVIDER) .

run-external-providers-local:
podman run --name java-provider -d -p 14651:14651 -v $(PWD)/external-providers/java-external-provider/examples:/examples$(MOUNT_OPT) java-provider --port 14651
podman run --name yq -d -p 14652:14652 -v $(PWD)/examples:/examples yq-provider$(MOUNT_OPT) --port 14652
podman run --name golang-provider -d -p 14653:14653 -v $(PWD)/examples:/examples$(MOUNT_OPT) generic-provider --port 14653
podman run --name nodejs -d -p 14654:14654 -v $(PWD)/examples:/examples$(MOUNT_OPT) generic-provider --port 14654 --name nodejs
podman run --name python -d -p 14655:14655 -v $(PWD)/examples:/examples$(MOUNT_OPT) generic-provider --port 14655 --name pylsp
podman run --name java-provider -d -p 14651:14651 -v $(PWD)/external-providers/java-external-provider/examples:/examples$(MOUNT_OPT) $(IMG_JAVA_PROVIDER) --port 14651
podman run --name yq -d -p 14652:14652 -v $(PWD)/examples:/examples $(IMG_YQ_PROVIDER)$(MOUNT_OPT) --port 14652
podman run --name golang-provider -d -p 14653:14653 -v $(PWD)/examples:/examples$(MOUNT_OPT) $(IMG_GENERIC_PROVIDER) --port 14653
podman run --name nodejs -d -p 14654:14654 -v $(PWD)/examples:/examples$(MOUNT_OPT) $(IMG_GENERIC_PROVIDER) --port 14654 --name nodejs
podman run --name python -d -p 14655:14655 -v $(PWD)/examples:/examples$(MOUNT_OPT) $(IMG_GENERIC_PROVIDER) --port 14655 --name pylsp

stop-external-providers:
podman kill java-provider || true
Expand All @@ -74,15 +79,15 @@ run-external-providers-pod:
podman run --rm -v test-data:/target$(MOUNT_OPT) -v $(PWD)/external-providers/java-external-provider/examples:/src/$(MOUNT_OPT) --entrypoint=cp alpine -a /src/. /target/
# run pods w/ defined ports for the test volumes
podman pod create --name=analyzer
podman run --pod analyzer --name java-provider -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) java-provider --port 14651
podman run --pod analyzer --name yq -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) yq-provider --port 14652
podman run --pod analyzer --name golang-provider -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) generic-provider --port 14653
podman run --pod analyzer --name nodejs -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) generic-provider --port 14654 --name nodejs
podman run --pod analyzer --name python -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) generic-provider --port 14655 --name pylsp
podman run --pod analyzer --name java-provider -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) $(IMG_JAVA_PROVIDER) --port 14651
podman run --pod analyzer --name yq -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) $(IMG_YQ_PROVIDER) --port 14652
podman run --pod analyzer --name golang-provider -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) $(IMG_GENERIC_PROVIDER) --port 14653
podman run --pod analyzer --name nodejs -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) $(IMG_GENERIC_PROVIDER) --port 14654 --name nodejs
podman run --pod analyzer --name python -d -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) $(IMG_GENERIC_PROVIDER) --port 14655 --name pylsp
podman build -f demo-local.Dockerfile -t localhost/testing:latest

run-demo-image:
podman run --entrypoint /usr/local/bin/konveyor-analyzer --pod=analyzer -v $(PWD)/demo-dep-output.yaml:/analyzer-lsp/demo-dep-output.yaml:Z -v $(PWD)/demo-output.yaml:/analyzer-lsp/output.yaml:Z localhost/testing:latest --output-file=/analyzer-lsp/output.yaml --dep-output-file=/analyzer-lsp/demo-dep-output.yaml
podman run --entrypoint /usr/local/bin/konveyor-analyzer --pod=analyzer -v test-data:/analyzer-lsp/examples$(MOUNT_OPT) -v $(PWD)/demo-dep-output.yaml:/analyzer-lsp/demo-dep-output.yaml:Z -v $(PWD)/demo-output.yaml:/analyzer-lsp/output.yaml:Z localhost/testing:latest --output-file=/analyzer-lsp/output.yaml --dep-output-file=/analyzer-lsp/demo-dep-output.yaml

stop-external-providers-pod: stop-external-providers
podman pod kill analyzer
Expand Down
65 changes: 58 additions & 7 deletions cmd/analyzer/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"sync"
Expand Down Expand Up @@ -127,9 +128,41 @@ func AnalysisCmd() *cobra.Command {
os.Exit(1)
}

// we add builtin configs by default for all locations
defaultBuiltinConfigs := []provider.InitConfig{}
seenBuiltinConfigs := map[string]bool{}
finalConfigs := []provider.Config{}
for _, config := range configs {
if config.Name != "builtin" {
finalConfigs = append(finalConfigs, config)
}
for _, initConf := range config.InitConfig {
if _, ok := seenBuiltinConfigs[initConf.Location]; !ok {
if initConf.Location != "" {
if stat, err := os.Stat(initConf.Location); err == nil && stat.IsDir() {
builtinLocation, err := filepath.Abs(initConf.Location)
if err != nil {
builtinLocation = initConf.Location
}
seenBuiltinConfigs[builtinLocation] = true
builtinConf := provider.InitConfig{Location: builtinLocation}
if config.Name == "builtin" {
builtinConf.ProviderSpecificConfig = initConf.ProviderSpecificConfig
}
defaultBuiltinConfigs = append(defaultBuiltinConfigs, builtinConf)
}
}
}
}
}
finalConfigs = append(finalConfigs, provider.Config{
Name: "builtin",
InitConfig: defaultBuiltinConfigs,
})

providers := map[string]provider.InternalProviderClient{}
providerLocations := []string{}
for _, config := range configs {
for _, config := range finalConfigs {
config.ContextLines = contextLines
for _, ind := range config.InitConfig {
providerLocations = append(providerLocations, ind.Location)
Expand Down Expand Up @@ -204,15 +237,33 @@ func AnalysisCmd() *cobra.Command {
}
}
// Now that we have all the providers, we need to start them.
additionalBuiltinConfigs := []provider.InitConfig{}
for name, provider := range needProviders {
initCtx, initSpan := tracing.StartNewSpan(ctx, "init",
attribute.Key("provider").String(name))
err := provider.ProviderInit(initCtx)
if err != nil {
errLog.Error(err, "unable to init the providers", "provider", name)
switch name {
// other providers can return additional configs for the builtin provider
// therefore, we initiate builtin provider separately at the end
case "builtin":
continue
default:
initCtx, initSpan := tracing.StartNewSpan(ctx, "init",
attribute.Key("provider").String(name))
additionalBuiltinConfs, err := provider.ProviderInit(initCtx, nil)
if err != nil {
errLog.Error(err, "unable to init the providers", "provider", name)
os.Exit(1)
}
if additionalBuiltinConfs != nil {
additionalBuiltinConfigs = append(additionalBuiltinConfigs, additionalBuiltinConfs...)
}
initSpan.End()
}
}

if builtinClient, ok := needProviders["builtin"]; ok {
if _, err = builtinClient.ProviderInit(ctx, additionalBuiltinConfigs); err != nil {
errLog.Error(err, "unable to init builtin provider")
os.Exit(1)
}
initSpan.End()
}

wg := &sync.WaitGroup{}
Expand Down
2 changes: 1 addition & 1 deletion cmd/dep/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ func DependencyCmd() *cobra.Command {

time.Sleep(5 * time.Second)

err = prov.ProviderInit(ctx)
_, err = prov.ProviderInit(ctx, nil)
b, _ := json.Marshal(config)
if err != nil {
errLog.Error(err, "unable to init the providers", "provider", config.Name, "the-error-is", err, "config", string(b))
Expand Down
30 changes: 26 additions & 4 deletions demo-output.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,14 @@
data: dependency
innerText: "\n\t\t\torg.springframework\n\t\t\tspring-webmvc\n\t\t\t${spring-framework.version}\n\t\t"
matchingXML: <groupId>org.springframework</groupId><artifactId>spring-webmvc</artifactId><version>${spring-framework.version}</version>
- uri: file:///examples/java-project/pom.xml
message: <groupId>io.javaoperatorsdk.operator</groupId><artifactId>sample</artifactId><version>0.0.0</version>
codeSnip: "11 <url>http://www.konveyor.io</url>\n12 \n13 <properties>\n14 <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>\n15 </properties>\n16 \n17 <dependencies>\n18 \n19 <dependency>\n20 <groupId>io.javaoperatorsdk.operator</groupId>\n21 <artifactId>sample</artifactId>\n22 <version>0.0.0</version>\n23 </dependency>\n24 \n25 </dependencies>\n26 \n27 <build>\n28 </build>\n29 </project>\n"
lineNumber: 20
variables:
data: dependency
innerText: "\n io.javaoperatorsdk.operator\n sample\n 0.0.0\n "
matchingXML: <groupId>io.javaoperatorsdk.operator</groupId><artifactId>sample</artifactId><version>0.0.0</version>
- uri: file:///examples/java/dummy/pom.xml
message: |-
<groupId>javax</groupId><artifactId>javaee-api</artifactId><!-- This leads to https://github.com/konveyor/analyzer-lsp/issues/390
Expand Down Expand Up @@ -781,6 +789,14 @@
data: dependency
innerText: "\n\t\t\torg.springframework\n\t\t\tspring-webmvc\n\t\t\t${spring-framework.version}\n\t\t"
matchingXML: <groupId>org.springframework</groupId><artifactId>spring-webmvc</artifactId><version>${spring-framework.version}</version>
- uri: file:///examples/java-project/pom.xml
message: POM XML dependencies - '<groupId>io.javaoperatorsdk.operator</groupId><artifactId>sample</artifactId><version>0.0.0</version>'
codeSnip: "11 <url>http://www.konveyor.io</url>\n12 \n13 <properties>\n14 <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>\n15 </properties>\n16 \n17 <dependencies>\n18 \n19 <dependency>\n20 <groupId>io.javaoperatorsdk.operator</groupId>\n21 <artifactId>sample</artifactId>\n22 <version>0.0.0</version>\n23 </dependency>\n24 \n25 </dependencies>\n26 \n27 <build>\n28 </build>\n29 </project>\n"
lineNumber: 20
variables:
data: dependency
innerText: "\n io.javaoperatorsdk.operator\n sample\n 0.0.0\n "
matchingXML: <groupId>io.javaoperatorsdk.operator</groupId><artifactId>sample</artifactId><version>0.0.0</version>
- uri: file:///examples/java/dummy/pom.xml
message: |-
POM XML dependencies - '<groupId>javax</groupId><artifactId>javaee-api</artifactId><!-- This leads to https://github.com/konveyor/analyzer-lsp/issues/390
Expand Down Expand Up @@ -960,21 +976,21 @@
labels:
- tag=Java Operator SDK
incidents:
- uri: file:///java-project/src/main/java/io/javaoperatorsdk/operator/sample/QuarkusOperator.java
- uri: file:///examples/java-project/src/main/java/io/javaoperatorsdk/operator/sample/QuarkusOperator.java
message: ""
codeSnip: " 1 package io.javaoperatorsdk.operator.sample;\n 2 \n 3 import io.fabric8.kubernetes.client.KubernetesClient;\n 4 import io.javaoperatorsdk.operator.Operator;\n 5 import io.javaoperatorsdk.operator.api.config.ConfigurationService;\n 6 import io.javaoperatorsdk.operator.api.config.ControllerConfiguration;\n 7 import io.quarkus.runtime.Quarkus;\n 8 import io.quarkus.runtime.QuarkusApplication;\n 9 import io.quarkus.runtime.annotations.QuarkusMain;\n10 import javax.inject.Inject;\n11 \n12 @QuarkusMain\n13 public class QuarkusOperator implements QuarkusApplication {\n14 @Inject"
lineNumber: 4
variables:
file: file:///java-project/src/main/java/io/javaoperatorsdk/operator/sample/QuarkusOperator.java
file: file:///examples/java-project/src/main/java/io/javaoperatorsdk/operator/sample/QuarkusOperator.java
kind: Module
name: io.javaoperatorsdk.operator.Operator
package: io.javaoperatorsdk.operator.sample
- uri: file:///java-project/src/main/java/io/javaoperatorsdk/operator/sample/QuarkusOperator.java
- uri: file:///examples/java-project/src/main/java/io/javaoperatorsdk/operator/sample/QuarkusOperator.java
message: ""
codeSnip: " 7 import io.quarkus.runtime.Quarkus;\n 8 import io.quarkus.runtime.QuarkusApplication;\n 9 import io.quarkus.runtime.annotations.QuarkusMain;\n10 import javax.inject.Inject;\n11 \n12 @QuarkusMain\n13 public class QuarkusOperator implements QuarkusApplication {\n14 @Inject\n15 KubernetesClient client;\n16 @Inject\n17 Operator operator;\n18 @Inject\n19 ConfigurationService configuration;\n20 @Inject\n21 CustomServiceController controller;\n22 \n23 public static void main(String... args) {\n24 Quarkus.run(QuarkusOperator.class, args);\n25 }\n26 \n27 public int run(String... args) throws Exception {"
lineNumber: 17
variables:
file: file:///java-project/src/main/java/io/javaoperatorsdk/operator/sample/QuarkusOperator.java
file: file:///examples/java-project/src/main/java/io/javaoperatorsdk/operator/sample/QuarkusOperator.java
kind: Field
name: operator
package: io.javaoperatorsdk.operator.sample
Expand Down Expand Up @@ -1002,6 +1018,12 @@
incidents:
- uri: file:///examples/customers-tomcat-legacy/pom.xml
message: ""
- uri: file:///examples/gradle-multi-project-example/gradle/wrapper/gradle-wrapper.jar
message: ""
- uri: file:///examples/inclusion-tests/pom.xml
message: ""
- uri: file:///examples/java-project/pom.xml
message: ""
- uri: file:///examples/java/dummy/pom.xml
message: ""
- uri: file:///examples/java/example/pom.xml
Expand Down
Loading
Loading