Finish release.SetBuildVersion() implementation

This allows us to use the new `SetBuildVersion` golang API to remove the bash-implementation later on as well as transform `find_green_build` (bash) to golang. The API can be used like this: ```go release.NewBuildVersionClient().SetBuildVersion(git.Master, "", nil) ``` Which will result in: ``` INFO Setting build version for branch "master" INFO Changing master branch to "release-master" INFO Retrieving testgrid configuration INFO Got testgrid jobs for branch "release-master": [ci-kubernetes-build ci-kubernetes-build-fast ci-kubernetes-conformance-kind-ga-only ci-kubernetes-e2e-gce-device-plugin-gpu ci-kubernetes-e2e-gce-master-new-gci-kubectl-skew ci-kubernetes-e2e-gci-gce ci-kubernetes-e2e-gci-gce-alpha-features ci-kubernetes-e2e-gci-gce-ingress ci-kubernetes-e2e-gci-gce-reboot ci-kubernetes-e2e-gci-gce-scalability ci-kubernetes-e2e-ubuntu-gce-containerd ci-kubernetes-gce-conformance-latest ci-kubernetes-integration-master ci-kubernetes-kind-e2e-parallel ci-kubernetes-kind-ipv6-e2e-parallel ci-kubernetes-node-kubelet ci-kubernetes-verify-master periodic-kubernetes-bazel-build-master periodic-kubernetes-bazel-test-master] INFO Getting ci-kubernetes-build build results from GCS INFO Getting ci-kubernetes-build-fast build results from GCS INFO Getting ci-kubernetes-conformance-kind-ga-only build results from GCS CommandException: No URLs matched: gs://kubernetes-jenkins/logs/ci-kubernetes-conformance-kind-ga-only/jobResultsCache.json CommandException: 1 file/object could not be transferred. WARN Skipping unavailable remote path gs://kubernetes-jenkins/logs/ci-kubernetes-conformance-kind-ga-only/jobResultsCache.json INFO Getting ci-kubernetes-e2e-gce-device-plugin-gpu build results from GCS INFO Getting ci-kubernetes-e2e-gce-master-new-gci-kubectl-skew build results from GCS INFO Getting ci-kubernetes-e2e-gci-gce build results from GCS INFO Getting ci-kubernetes-e2e-gci-gce-alpha-features build results from GCS INFO Getting ci-kubernetes-e2e-gci-gce-ingress build results from GCS INFO Getting ci-kubernetes-e2e-gci-gce-reboot build results from GCS INFO Getting ci-kubernetes-e2e-gci-gce-scalability build results from GCS INFO Getting ci-kubernetes-e2e-ubuntu-gce-containerd build results from GCS INFO Getting ci-kubernetes-gce-conformance-latest build results from GCS INFO Getting ci-kubernetes-integration-master build results from GCS CommandException: No URLs matched: gs://kubernetes-jenkins/logs/ci-kubernetes-integration-master/jobResultsCache.json CommandException: 1 file/object could not be transferred. WARN Skipping unavailable remote path gs://kubernetes-jenkins/logs/ci-kubernetes-integration-master/jobResultsCache.json INFO Getting ci-kubernetes-kind-e2e-parallel build results from GCS CommandException: No URLs matched: gs://kubernetes-jenkins/logs/ci-kubernetes-kind-e2e-parallel/jobResultsCache.json CommandException: 1 file/object could not be transferred. WARN Skipping unavailable remote path gs://kubernetes-jenkins/logs/ci-kubernetes-kind-e2e-parallel/jobResultsCache.json INFO Getting ci-kubernetes-kind-ipv6-e2e-parallel build results from GCS CommandException: No URLs matched: gs://kubernetes-jenkins/logs/ci-kubernetes-kind-ipv6-e2e-parallel/jobResultsCache.json CommandException: 1 file/object could not be transferred. WARN Skipping unavailable remote path gs://kubernetes-jenkins/logs/ci-kubernetes-kind-ipv6-e2e-parallel/jobResultsCache.json INFO Getting ci-kubernetes-node-kubelet build results from GCS INFO Getting ci-kubernetes-verify-master build results from GCS CommandException: No URLs matched: gs://kubernetes-jenkins/logs/ci-kubernetes-verify-master/jobResultsCache.json CommandException: 1 file/object could not be transferred. WARN Skipping unavailable remote path gs://kubernetes-jenkins/logs/ci-kubernetes-verify-master/jobResultsCache.json INFO Getting periodic-kubernetes-bazel-build-master build results from GCS CommandException: No URLs matched: gs://kubernetes-jenkins/logs/periodic-kubernetes-bazel-build-master/jobResultsCache.json CommandException: 1 file/object could not be transferred. WARN Skipping unavailable remote path gs://kubernetes-jenkins/logs/periodic-kubernetes-bazel-build-master/jobResultsCache.json INFO Getting periodic-kubernetes-bazel-test-master build results from GCS CommandException: No URLs matched: gs://kubernetes-jenkins/logs/periodic-kubernetes-bazel-test-master/jobResultsCache.json CommandException: 1 file/object could not be transferred. WARN Skipping unavailable remote path gs://kubernetes-jenkins/logs/periodic-kubernetes-bazel-test-master/jobResultsCache.json INFO Trying version "v1.19.0-beta.2.790+9eced040142454" for build "1280834193957326848" (*) Primary job (-) Secondary jobs Job Run Build Date/Status * build 1280849546682830849 790 2020-07-08T11:19:07Z - build-fast 1280851306898001920 790 SUCCEEDED - conformance-kind-ga-only 1280851306898001920 790 NOT EXISTING - e2e-gce-device-plugin-gpu 1280851306898001920 790 FAILED - e2e-gce-master-new-gci-kubectl-skew 1280836458327838723 790 SUCCEEDED - e2e-gci-gce 1280839730467966977 790 SUCCEEDED - e2e-gci-gce-alpha-features 1280839730467966977 790 FAILED - e2e-gci-gce-ingress 1280839730467966977 790 FAILED - e2e-gci-gce-reboot 1280842750345875457 790 SUCCEEDED - e2e-gci-gce-scalability 1280835703843852291 790 SUCCEEDED - e2e-ubuntu-gce-containerd 1280835703843852291 790 FAILED - gce-conformance-latest 1280835703843852291 790 FAILED - integration-master 1280835703843852291 790 NOT EXISTING - kind-e2e-parallel 1280835703843852291 790 NOT EXISTING - kind-ipv6-e2e-parallel 1280835703843852291 790 NOT EXISTING - node-kubelet 1280825133354717185 790 SUCCEEDED - verify-master 1280825133354717185 790 NOT EXISTING - periodic-kubernetes-bazel-build-master 1280825133354717185 790 NOT EXISTING - periodic-kubernetes-bazel-test-master 1280825133354717185 790 NOT EXISTING INFO Trying version "v1.19.0-beta.2.788+bf94f27e76c541" for build "1280818842905350144" (*) Primary job (-) Secondary jobs Job Run Build Date/Status * build 1280818842905350144 788 2020-07-08T02:55:06Z - build-fast 1280822868766101504 788 SUCCEEDED - conformance-kind-ga-only 1280822868766101504 788 NOT EXISTING - e2e-gce-device-plugin-gpu 1280826643891359746 788 SUCCEEDED - e2e-gce-master-new-gci-kubectl-skew 1280821107452022785 788 SUCCEEDED - e2e-gci-gce 1280822114932232193 788 SUCCEEDED - e2e-gci-gce-alpha-features 1280808273120858112 788 SUCCEEDED - e2e-gci-gce-ingress 1280798710204207104 788 SUCCEEDED - e2e-gci-gce-reboot 1280826643891359744 788 SUCCEEDED - e2e-gci-gce-scalability 1280822114940620800 788 SUCCEEDED - e2e-ubuntu-gce-containerd 1280810286281920512 788 SUCCEEDED - gce-conformance-latest 1280793677429477376 788 SUCCEEDED - integration-master 1280793677429477376 788 NOT EXISTING - kind-e2e-parallel 1280793677429477376 788 NOT EXISTING - kind-ipv6-e2e-parallel 1280793677429477376 788 NOT EXISTING - node-kubelet 1280809782663450628 788 SUCCEEDED - verify-master 1280809782663450628 788 NOT EXISTING - periodic-kubernetes-bazel-build-master 1280809782663450628 788 NOT EXISTING - periodic-kubernetes-bazel-test-master 1280809782663450628 788 NOT EXISTING ``` We can see that version `v1.19.0-beta.2.790+9eced040142454` did not succeed because there were failing tests. Version `v1.19.0-beta.2.788+bf94f27e76c541` has all green builds, whereas inexisting builds are skipped like before. Signed-off-by: Sascha Grunert <sgrunert@suse.com>
hanxueming126 · Jul 8, 2020 · f3430d0 · f3430d0
1 parent 87fc05d
commit f3430d0
Show file tree

Hide file tree

Showing 8 changed files with 811 additions and 67 deletions.
diff --git a/.golangci.yml b/.golangci.yml
@@ -5,10 +5,10 @@ run:
 issues:
   exclude-rules:
     # counterfeiter fakes are usually named 'fake_<something>.go'
-    # TODO: figure out why golangci-lint does not treat counterfeiter files as generated files
     - path: fake_.*\.go
       linters:
         - gocritic
+        - golint
         - dupl
 linters:
   disable-all: true

diff --git a/pkg/release/build_version.go b/pkg/release/build_version.go
@@ -23,87 +23,277 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+	"text/tabwriter"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 
 	"k8s.io/release/pkg/command"
 	"k8s.io/release/pkg/gcp"
 	"k8s.io/release/pkg/git"
+	"k8s.io/release/pkg/http"
 	"k8s.io/release/pkg/testgrid"
 	"k8s.io/release/pkg/util"
 )
 
-// SetBuildVersion sets the build version for a branch
+const (
+	releaseRegexStr = `v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[a-zA-Z0-9]+)*\.*(0|[1-9][0-9]*)?`
+	buildRegexStr   = `([0-9]{1,})\+([0-9a-f]{5,40})`
+	jobPrefix       = "ci-kubernetes-"
+	gitHubCommitAPI = "https://api.github.com/repos/kubernetes/kubernetes/commits?sha="
+	jobLimit        = 100
+	jenkinsLogRoot  = "gs://kubernetes-jenkins/logs/"
+)
+
+var (
+	releaseAndBuildRegex = regexp.MustCompile(
+		fmt.Sprintf("%s.%s", releaseRegexStr, buildRegexStr),
+	)
+)
+
+//counterfeiter:generate . httpClient
+type httpClient interface {
+	GetURLResponse(string, bool) (string, error)
+}
+
+type defaultHTTPClient struct{}
+
+func (*defaultHTTPClient) GetURLResponse(url string, trim bool) (string, error) {
+	return http.GetURLResponse(url, trim)
+}
+
+//counterfeiter:generate . jobCacheClient
+type jobCacheClient interface {
+	GetJobCache(string, bool) (*JobCache, error)
+}
+
+type defaultJobCacheClient struct{}
+
+func (*defaultJobCacheClient) GetJobCache(job string, dedup bool) (*JobCache, error) {
+	return NewJobCacheClient().GetJobCache(job, dedup)
+}
+
+//counterfeiter:generate . testGridClient
+type testGridClient interface {
+	BlockingTests(string) ([]string, error)
+}
+
+type defaultTestGridClient struct{}
+
+func (*defaultTestGridClient) BlockingTests(branch string) (tests []string, err error) {
+	return testgrid.New().BlockingTests(branch)
+}
+
+type BuildVersionClient struct {
+	jobCacheClient jobCacheClient
+	httpClient     httpClient
+	testGridClient testGridClient
+}
+
+// NewBuildVersionClient creates a new build version client
+func NewBuildVersionClient() *BuildVersionClient {
+	return &BuildVersionClient{
+		jobCacheClient: &defaultJobCacheClient{},
+		httpClient:     &defaultHTTPClient{},
+		testGridClient: &defaultTestGridClient{},
+	}
+}
+
+// SetHTTPClient can be used to set the HTTP client
+func (b *BuildVersionClient) SetHTTPClient(client httpClient) {
+	b.httpClient = client
+}
+
+// SetJobCacheClient can be used to set the job cache client
+func (b *BuildVersionClient) SetJobCacheClient(client jobCacheClient) {
+	b.jobCacheClient = client
+}
+
+// SetJobCacheClient can be used to set the job cache client
+func (b *BuildVersionClient) SetTestGridClient(client testGridClient) {
+	b.testGridClient = client
+}
+
+// SetBuildVersion returns the build version for a branch
 // against a set of blocking CI jobs
 //
 // branch - The branch name.
 // jobPath - A local directory to store the copied cache entries.
 // exclude_suites - A list of (greedy) patterns to exclude CI jobs from
 // 					checking against the primary job.
-func SetBuildVersion(
+func (b *BuildVersionClient) SetBuildVersion(
 	branch, jobPath string,
 	excludeSuites []string,
-) error {
+) (foundVersion string, err error) {
 	logrus.Infof("Setting build version for branch %q", branch)
 
 	if branch == git.Master {
 		branch = "release-master"
 		logrus.Infof("Changing %s branch to %q", git.Master, branch)
 	}
 
-	allJobs, err := testgrid.New().BlockingTests(branch)
+	allJobs, err := b.testGridClient.BlockingTests(branch)
 	if err != nil {
-		return errors.Wrap(err, "getting all test jobs")
+		return "", errors.Wrap(err, "getting all test jobs")
 	}
 	logrus.Infof("Got testgrid jobs for branch %q: %v", branch, allJobs)
 
 	if len(allJobs) == 0 {
-		return errors.Errorf(
+		return "", errors.Errorf(
 			"No sig-%s-blocking list found in the testgrid config.yaml", branch,
 		)
 	}
 
 	// Filter out excluded suites
 	secondaryJobs := []string{}
-	for _, job := range allJobs {
+	for i, job := range allJobs {
+		if i == 0 {
+			continue
+		}
+
+		excluded := false
 		for _, pattern := range excludeSuites {
 			matched, err := regexp.MatchString(pattern, job)
 			if err != nil {
-				return errors.Wrapf(err, "regex comile failed: %s", pattern)
-			}
-			if matched {
-				secondaryJobs = append(secondaryJobs, job)
+				return "", errors.Wrapf(err,
+					"regex compile failed: %s", pattern,
+				)
 			}
+			excluded = matched
+		}
+
+		if !excluded {
+			secondaryJobs = append(secondaryJobs, job)
 		}
 	}
 
 	// Update main cache
-	// We dedup the $main_job's list of successful runs and just run through
+	// We dedup the mainJob's list of successful runs and just run through
 	// that unique list. We then leave the full state of secondaries below so
 	// we have finer granularity at the Jenkin's job level to determine if a
 	// build is ok.
-	jobCaches := map[string]JobCache{} // a map of jobs and their caches
 	mainJob := allJobs[0]
-	jcc := NewJobCacheClient()
-	mainJobCache, err := jcc.GetJobCache(mainJob, true)
+
+	mainJobCache, err := b.jobCacheClient.GetJobCache(mainJob, true)
 	if err != nil {
-		return errors.Wrap(err, "building job cache for main job")
+		return "", errors.Wrap(err, "building job cache for main job")
+	}
+	if mainJobCache == nil {
+		return "", errors.Errorf("main job cache for job %q is nil", mainJob)
 	}
-	jobCaches[mainJob] = mainJobCache
 
 	// Update secondary caches limited by main cache last build number
+	secondaryJobCaches := []*JobCache{}
 	for _, job := range secondaryJobs {
-		cache, err := jcc.GetJobCache(job, true)
+		cache, err := b.jobCacheClient.GetJobCache(job, true)
 		if err != nil {
-			return errors.Wrapf(err, "building job cache for job: %s", job)
+			return "", errors.Wrapf(err, "building job cache for job: %s", job)
+		}
+		if cache != nil {
+			secondaryJobCaches = append(secondaryJobCaches, cache)
 		}
-		jobCaches[job] = cache
 	}
 
-	// TODO: continue port from releaselib.sh::set_build_version
+	for i, version := range mainJobCache.Versions {
+		sb := strings.Builder{}
+		tw := tabwriter.NewWriter(&sb, 0, 0, 2, ' ', 0)
+		fmt.Fprintln(tw, "Job\tRun\tBuild\tDate/Status")
+
+		buildVersion := mainJobCache.BuildNumbers[i]
+
+		logrus.Infof("Trying version %q for build %q", version, buildVersion)
+
+		if i > jobLimit {
+			return "", errors.Errorf("job count limit of %d exceeded", jobLimit)
+		}
+
+		matches := releaseAndBuildRegex.FindStringSubmatch(version)
+		if matches == nil || len(matches) < 8 {
+			return "", errors.Errorf("Invalid build version: %v", version)
+		}
+
+		buildRun := matches[6]
+		buildSHA := matches[7]
+
+		// TODO(saschagrunert): refactor on top of a new github package API
+		dateResponse, err := b.httpClient.GetURLResponse(
+			gitHubCommitAPI+buildSHA, true,
+		)
+		if err != nil {
+			return "", errors.Wrap(err, "getting build date")
+		}
+		dateRes, err := command.New("echo", dateResponse).
+			Pipe("jq", "-r", ".[0] | .commit.author.date").
+			RunSilentSuccessOutput()
+		if err != nil {
+			return "", errors.Wrap(err, "filtering date response")
+		}
+		date := dateRes.OutputTrimNL()
+
+		fmt.Fprint(&sb, "(*) Primary job (-) Secondary jobs\n\n")
+		fmt.Fprintf(tw,
+			"%s\t%s\t%s\t%s\t\n",
+			"* "+strings.TrimPrefix(mainJob, jobPrefix),
+			buildVersion, buildRun, date,
+		)
+
+		type BuildStatus string
+		const (
+			BuildStatusNotExisting BuildStatus = "NOT EXISTING"
+			BuildStatusFailed      BuildStatus = "FAILED"
+			BuildStatusSucceeded   BuildStatus = "SUCCEEDED"
+		)
+		// Check secondaries to ensure that build number is green across all
+		success := true
+		foundBuildNumber := ""
+		for _, secondaryJob := range secondaryJobs {
+			status := BuildStatusNotExisting
+			for _, secondaryJobCache := range secondaryJobCaches {
+				if secondaryJobCache.Name == secondaryJob {
+					status = BuildStatusFailed
+
+					for j, secVersion := range secondaryJobCache.Versions {
+						matches := releaseAndBuildRegex.FindStringSubmatch(secVersion)
+						if matches == nil || len(matches) < 8 {
+							logrus.Errorf(
+								"Invalid build version %s for job %s",
+								secVersion, secondaryJob,
+							)
+							continue
+						}
+
+						// Verify that we have the same build number
+						if buildRun == matches[6] {
+							status = BuildStatusSucceeded
+							foundBuildNumber = secondaryJobCache.BuildNumbers[j]
+						}
+					}
+					if status == BuildStatusSucceeded {
+						break
+					}
+				}
+			}
 
-	return errors.New("unimplemented")
+			fmt.Fprintf(tw,
+				"%s\t%s\t%s\t%s\t\n",
+				"- "+strings.TrimPrefix(secondaryJob, jobPrefix),
+				foundBuildNumber, buildRun, status,
+			)
+
+			if status == BuildStatusFailed {
+				success = false
+			}
+		}
+
+		tw.Flush()
+		fmt.Println(sb.String())
+
+		if success {
+			return version, nil
+		}
+	}
+
+	return "", errors.New("unable to find successful build version")
 }
 
 type JobCacheClient struct {
@@ -123,38 +313,39 @@ func (j *JobCacheClient) SetClient(client gcpClient) {
 
 //counterfeiter:generate . gcpClient
 type gcpClient interface {
-	CopyJobCache(string) (string, error)
+	CopyJobCache(string) string
 }
 
 type defaultGcpClient struct{}
 
-func (g *defaultGcpClient) CopyJobCache(job string) (jsonPath string, err error) {
+func (g *defaultGcpClient) CopyJobCache(job string) (jsonPath string) {
 	jsonPath = filepath.Join(os.TempDir(), fmt.Sprintf("job-cache-%s", job))
 
-	const logRoot = "gs://kubernetes-jenkins/logs/"
-	if err := gcp.GSUtil(
-		"-qm", "cp",
-		logRoot+filepath.Join(job, "jobResultsCache.json"),
-		jsonPath,
-	); err != nil {
-		return "", errors.Wrap(err, "copying job results cache")
+	remotePath := jenkinsLogRoot + filepath.Join(job, "jobResultsCache.json")
+	if err := gcp.GSUtil("-qm", "cp", remotePath, jsonPath); err != nil {
+		logrus.Warnf("Skipping unavailable remote path %s", remotePath)
+		return ""
 	}
-	return jsonPath, nil
+	return jsonPath
 }
 
 // JobCache is a map of build numbers (key) and their versions (value)
-type JobCache map[string]string
+type JobCache struct {
+	Name         string
+	BuildNumbers []string
+	Versions     []string
+}
 
 // GetJobCache pulls Jenkins server job cache from GS and resutns a `JobCache`
 //
 // job - The Jenkins job name.
 // dedup -  dedup git's monotonically increasing (describe) build numbers.
-func (j *JobCacheClient) GetJobCache(job string, dedup bool) (JobCache, error) {
+func (j *JobCacheClient) GetJobCache(job string, dedup bool) (*JobCache, error) {
 	logrus.Infof("Getting %s build results from GCS", job)
 
-	tempJSON, err := j.gcpClient.CopyJobCache(job)
-	if err != nil {
-		return nil, errors.Wrap(err, "getting GCP job cache")
+	tempJSON := j.gcpClient.CopyJobCache(job)
+	if tempJSON == "" {
+		return nil, nil
 	}
 
 	if !util.Exists(tempJSON) {
@@ -179,7 +370,7 @@ func (j *JobCacheClient) GetJobCache(job string, dedup bool) (JobCache, error) {
 	}
 
 	lastVersion := ""
-	res := map[string]string{}
+	res := &JobCache{Name: job}
 	scanner := bufio.NewScanner(strings.NewReader(out.OutputTrimNL()))
 	for scanner.Scan() {
 		split := strings.Split(scanner.Text(), " ")
@@ -199,7 +390,8 @@ func (j *JobCacheClient) GetJobCache(job string, dedup bool) (JobCache, error) {
 		lastVersion = version
 
 		if buildNumber != "" && version != "" {
-			res[buildNumber] = version
+			res.BuildNumbers = append(res.BuildNumbers, buildNumber)
+			res.Versions = append(res.Versions, version)
 		}
 	}