diff --git a/buildAndPushDockerhub.sh b/buildAndPushDockerhub.sh index 240c91c5..d3f36cfd 100755 --- a/buildAndPushDockerhub.sh +++ b/buildAndPushDockerhub.sh @@ -1,14 +1,18 @@ export PUSH_FLAG="--push" export BUILD_FLAG="buildx build --platform linux/amd64" +#export TAG=2.0.0 +export TAG=dev +export IMAGE=opendoor/telia-oss-github-pr-resource if [ "$#" -gt 0 ] then PUSH_FLAG="" BUILD_FLAG="build" echo To run locally echo Sample request.in.json and sample.check.json in e2e-opendoor - echo "docker run -it --entrypoint=/bin/sh opendoor/telia-oss-github-pr-resource:dev" + echo "docker run -it --entrypoint=/bin/sh $IMAGE:$TAG" echo "cd /opt/resources" echo "cat | ./in .| ./out .| ./check" fi # docker login --username=$DOCKER_USERNAME --password=$DOCKER_PASSWORD -docker $BUILD_FLAG -t opendoor/telia-oss-github-pr-resource:dev . $PUSH_FLAG +docker $BUILD_FLAG -t $IMAGE:$TAG . $PUSH_FLAG +echo Built $IMAGE:$TAG diff --git a/common.go b/common.go index 8ec8b48b..27a60eb1 100644 --- a/common.go +++ b/common.go @@ -10,7 +10,7 @@ import ( // print the request json coming in to "in|out|check" func PrintDebugInput(s Source, obj any) { - if s.Debug { + if s.OdAdvanced.Debug { jsonBytes, _ := json.Marshal(obj) log.Printf("input jsonStr : %s\n", string(jsonBytes)) log.Printf("Debig Tip1: run this docker image locally: docker run -it --entrypoint=/bin/sh opendoor/telia-oss-github-pr-resource:dev\n") @@ -20,7 +20,7 @@ func PrintDebugInput(s Source, obj any) { } func PrintDebugOutput(s Source, obj any) { - if s.Debug { + if s.OdAdvanced.Debug { jsonBytes, _ := json.Marshal(obj) log.Printf("output jsonStr : %s\n", string(jsonBytes)) } diff --git a/e2e-opendoor/pipeline.yaml b/e2e-opendoor/pipeline.yaml index 92725105..e283ffc2 100644 --- a/e2e-opendoor/pipeline.yaml +++ b/e2e-opendoor/pipeline.yaml @@ -1,18 +1,24 @@ ################ # YAML anchors # ################ -opendoor_yaml_definitions: - access_token_additional: &access_token_additional - - ((github-token-from-app-1)) - - ((github-token-from-app-2)) - - ((github-token-from-app-3)) - - ((github-token-from-app-4)) - - ((github-token-from-app-5)) - - ((github-token-from-app-6)) - - ((github-token-from-app-7)) - - ((github-token-from-app-8)) - - ((github-token-from-app-9)) - - ((github-token-from-app-10)) +opendoor_yaml_definitions: + od_advanced: &od_advanced + access_token_additional: + - ((github-token-from-app-1)) + - ((github-token-from-app-2)) + - ((github-token-from-app-3)) + - ((github-token-from-app-4)) + - ((github-token-from-app-5)) + - ((github-token-from-app-6)) + - ((github-token-from-app-7)) + - ((github-token-from-app-8)) + - ((github-token-from-app-9)) + - ((github-token-from-app-10)) + min_remaining_threshold_before_using_access_token_additional : 20000 + # WIP metrics ... placeholder for now + datadog_api_key: ((datadog-api-key)) + datadog_app_key: ((datadog-app-key)) + debug: true image_resource: &dev-tools name: "" source: @@ -31,7 +37,7 @@ resource_types: repository: opendoor/telia-oss-github-pr-resource # stored in vault concourse/shared # most likely 'dev' if you are doing a code review ;) - tag: ((od-github-pr-resource-tag)) + tag: dev ############### ## Resources ## @@ -41,16 +47,8 @@ resources: type: od-pull-request source: access_token: ((github-token)) - access_token_additional: *access_token_additional - # for testing purposes ... realistically, we should never set min_remaining_threshold_before_using_access_token_additional - # and use the default defined in git.go - min_remaining_threshold_before_using_access_token_additional : 20000 - # WIP metrics ... placeholder for now - datadog_api_key: ((datadog-api-key)) - datadog_app_key: ((datadog-app-key)) - base_branch: master repository: opendoor-labs/code - debug: true + od_advanced: *od_advanced ########## ## Jobs ## diff --git a/e2e-opendoor/request.README b/e2e-opendoor/request.README new file mode 100644 index 00000000..2bf58f29 --- /dev/null +++ b/e2e-opendoor/request.README @@ -0,0 +1,2 @@ +note that the ghs_XXXX tokens expire every now ... if you see this in the git repo +its not a security issue diff --git a/git.go b/git.go index 040bbdbb..4c1d22e1 100644 --- a/git.go +++ b/git.go @@ -43,13 +43,13 @@ func NewGitClient(source *Source, dir string, output io.Writer) (*GitClient, err } return &GitClient{ AccessToken: &source.AccessToken, - AccessTokenAdditional: source.AccessTokenAdditional, - MinRemainingThresholdBeforeUsingAccessTokenAdditional: source.MinRemainingThresholdBeforeUsingAccessTokenAdditional, - DataDogApiKey: source.DataDogApiKey, - DataDogAppKey: source.DataDogAppKey, - DataDogMetricName: source.DataDogMetricName, - DataDogResourcesName: source.DataDogResourcesName, - DataDogResourcesValue: source.DataDogResourcesValue, + AccessTokenAdditional: source.OdAdvanced.AccessTokenAdditional, + MinRemainingThresholdBeforeUsingAccessTokenAdditional: source.OdAdvanced.MinRemainingThresholdBeforeUsingAccessTokenAdditional, + DataDogApiKey: source.OdAdvanced.DataDogApiKey, + DataDogAppKey: source.OdAdvanced.DataDogAppKey, + DataDogMetricName: source.OdAdvanced.DataDogMetricName, + DataDogResourcesName: source.OdAdvanced.DataDogResourcesName, + DataDogResourcesValue: source.OdAdvanced.DataDogResourcesValue, Directory: dir, Output: output, }, nil diff --git a/github.go b/github.go index 9e920f64..ba05397a 100644 --- a/github.go +++ b/github.go @@ -73,18 +73,18 @@ func NewGithubClient(s *Source) (*GithubClient, error) { minRemaining = graphqlRemaining } var minRemainingThresholdBeforeUsingAccessTokenAdditional = DefaultMinRemainingBeforeUsingAccessTokenAdditional - if s.MinRemainingThresholdBeforeUsingAccessTokenAdditional == 0 { + if s.OdAdvanced.MinRemainingThresholdBeforeUsingAccessTokenAdditional == 0 { log.Printf("source.min_remaining_threshold_before_using_access_token_additional was not supplied in pipeline ... "+ "using DefaultMinRemainingBeforeUsingAccessTokenAdditional : %d\n", DefaultMinRemainingBeforeUsingAccessTokenAdditional) } else { - log.Printf("using source.min_remaining_threshold_before_using_access_token_additional : %d\n", s.MinRemainingThresholdBeforeUsingAccessTokenAdditional) - minRemainingThresholdBeforeUsingAccessTokenAdditional = s.MinRemainingThresholdBeforeUsingAccessTokenAdditional + log.Printf("using source.min_remaining_threshold_before_using_access_token_additional : %d\n", s.OdAdvanced.MinRemainingThresholdBeforeUsingAccessTokenAdditional) + minRemainingThresholdBeforeUsingAccessTokenAdditional = s.OdAdvanced.MinRemainingThresholdBeforeUsingAccessTokenAdditional } - if s.AccessTokenAdditional == nil { + if s.OdAdvanced.AccessTokenAdditional == nil { log.Printf("No AccessTokenAdditional, therefore will ALWAYS use the AccessToken supplied\n") } else { log.Printf("Detected that the length of AccessTokenAdditional list is %d\n", - len(s.AccessTokenAdditional)) + len(s.OdAdvanced.AccessTokenAdditional)) log.Printf("minRemaining : %d, minRemainingThresholdBeforeUsingAccessTokenAdditional : %d\n", minRemaining, minRemainingThresholdBeforeUsingAccessTokenAdditional) if minRemaining < minRemainingThresholdBeforeUsingAccessTokenAdditional { @@ -94,7 +94,7 @@ func NewGithubClient(s *Source) (*GithubClient, error) { // TODO altho we are passing a list of AccessTokenAdditional, we will only consider the first element as it is already sorted // by highest remaining ... in the future consider the rest of the list, altho this TODO is a low priority log.Printf("old AccessToken : %s_REDACTED\n", s.AccessToken[0:10]) - s.AccessToken = s.AccessTokenAdditional[0] + s.AccessToken = s.OdAdvanced.AccessTokenAdditional[0] log.Printf("new AccessToken : %s_REDACTED\n", s.AccessToken[0:10]) PrintCurrentRateLimit(*s) sendToDataDog(s) @@ -160,23 +160,23 @@ func NewGithubClient(s *Source) (*GithubClient, error) { // sending metrics to datadog func sendToDataDog(s *Source) { - if (s.DataDogApiKey != "") && (s.DataDogAppKey != "") { + if (s.OdAdvanced.DataDogApiKey != "") && (s.OdAdvanced.DataDogAppKey != "") { log.Printf("DataDogApiKey and DataDogAppKey were supplied\n") - if s.DataDogMetricName == "" { - s.DataDogMetricName = DefaultDataDogMetricName + if s.OdAdvanced.DataDogMetricName == "" { + s.OdAdvanced.DataDogMetricName = DefaultDataDogMetricName } - if s.DataDogResourcesName == "" { - s.DataDogResourcesName = DefaultDataDogResourcesName + if s.OdAdvanced.DataDogResourcesName == "" { + s.OdAdvanced.DataDogResourcesName = DefaultDataDogResourcesName } - if s.DataDogResourcesValue == "" { - s.DataDogResourcesValue = DefaultDataDogResourcesValue + if s.OdAdvanced.DataDogResourcesValue == "" { + s.OdAdvanced.DataDogResourcesValue = DefaultDataDogResourcesValue } - log.Printf("DataDogMetricName : %s, DataDogResourcesName : %s, DataDogResourcesValue : %s\n", s.DataDogMetricName, s.DataDogResourcesName, s.DataDogResourcesValue) + log.Printf("DataDogMetricName : %s, DataDogResourcesName : %s, DataDogResourcesValue : %s\n", s.OdAdvanced.DataDogMetricName, s.OdAdvanced.DataDogResourcesName, s.OdAdvanced.DataDogResourcesValue) // code borrowed from: https://docs.datadoghq.com/api/latest/metrics/?code-lang=go body := datadogV2.MetricPayload{ Series: []datadogV2.MetricSeries{ { - Metric: s.DataDogMetricName, + Metric: s.OdAdvanced.DataDogMetricName, Type: datadogV2.METRICINTAKETYPE_UNSPECIFIED.Ptr(), Points: []datadogV2.MetricPoint{ { @@ -186,8 +186,8 @@ func sendToDataDog(s *Source) { }, Resources: []datadogV2.MetricResource{ { - Name: datadog.PtrString(s.DataDogResourcesName), - Type: datadog.PtrString(s.DataDogResourcesValue), + Name: datadog.PtrString(s.OdAdvanced.DataDogResourcesName), + Type: datadog.PtrString(s.OdAdvanced.DataDogResourcesValue), }, }, }, @@ -198,10 +198,10 @@ func sendToDataDog(s *Source) { datadog.ContextAPIKeys, map[string]datadog.APIKey{ "apiKeyAuth": { - Key: s.DataDogApiKey, + Key: s.OdAdvanced.DataDogApiKey, }, "appKeyAuth": { - Key: s.DataDogAppKey, + Key: s.OdAdvanced.DataDogAppKey, }, }, ) diff --git a/models.go b/models.go index ad231c22..00d05b79 100644 --- a/models.go +++ b/models.go @@ -16,31 +16,35 @@ type StatusFilter struct { // Source represents the configuration for the resource. type Source struct { - Repository string `json:"repository"` - AccessToken string `json:"access_token"` - AccessTokenAdditional []string `json:"access_token_additional"` - MinRemainingThresholdBeforeUsingAccessTokenAdditional int `json:"min_remaining_threshold_before_using_access_token_additional"` - DataDogApiKey string `json:"datadog_api_key"` - DataDogAppKey string `json:"datadog_app_key"` - DataDogMetricName string `json:"datadog_metric_name"` - DataDogResourcesName string `json:"datadog_resources_name"` - DataDogResourcesValue string `json:"datadog_resources_value"` - Debug bool `json:"debug"` - V3Endpoint string `json:"v3_endpoint"` - V4Endpoint string `json:"v4_endpoint"` - Paths []string `json:"paths"` - IgnorePaths []string `json:"ignore_paths"` - DisableCISkip bool `json:"disable_ci_skip"` - DisableGitLFS bool `json:"disable_git_lfs"` - SkipSSLVerification bool `json:"skip_ssl_verification"` - DisableForks bool `json:"disable_forks"` - IgnoreDrafts bool `json:"ignore_drafts"` - GitCryptKey string `json:"git_crypt_key"` - BaseBranch string `json:"base_branch"` - RequiredReviewApprovals int `json:"required_review_approvals"` - Labels []string `json:"labels"` - States []githubv4.PullRequestState `json:"states"` - StatusFilters []StatusFilter `json:"status_filters"` + Repository string `json:"repository"` + AccessToken string `json:"access_token"` + OdAdvanced OdAdvanced `json:"od_advanced"` + V3Endpoint string `json:"v3_endpoint"` + V4Endpoint string `json:"v4_endpoint"` + Paths []string `json:"paths"` + IgnorePaths []string `json:"ignore_paths"` + DisableCISkip bool `json:"disable_ci_skip"` + DisableGitLFS bool `json:"disable_git_lfs"` + SkipSSLVerification bool `json:"skip_ssl_verification"` + DisableForks bool `json:"disable_forks"` + IgnoreDrafts bool `json:"ignore_drafts"` + GitCryptKey string `json:"git_crypt_key"` + BaseBranch string `json:"base_branch"` + RequiredReviewApprovals int `json:"required_review_approvals"` + Labels []string `json:"labels"` + States []githubv4.PullRequestState `json:"states"` + StatusFilters []StatusFilter `json:"status_filters"` +} + +type OdAdvanced struct { + AccessTokenAdditional []string `json:"access_token_additional"` + MinRemainingThresholdBeforeUsingAccessTokenAdditional int `json:"min_remaining_threshold_before_using_access_token_additional"` + DataDogApiKey string `json:"datadog_api_key"` + DataDogAppKey string `json:"datadog_app_key"` + DataDogMetricName string `json:"datadog_metric_name"` + DataDogResourcesName string `json:"datadog_resources_name"` + DataDogResourcesValue string `json:"datadog_resources_value"` + Debug bool `json:"debug"` } // Validate the source configuration.