diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml deleted file mode 100644 index ba91bdb5..00000000 --- a/.github/workflows/automerge.yml +++ /dev/null @@ -1,58 +0,0 @@ -# Automatically merge pull requests opened by web3-bot, as soon as (and only if) all tests pass. -# This reduces the friction associated with updating with our workflows. - -on: - workflow_call: - inputs: - job: - required: true - type: string -name: Automerge - -jobs: - automerge-check: - if: github.event.pull_request.user.login == 'web3-bot' - runs-on: ubuntu-latest - outputs: - status: ${{ steps.should-automerge.outputs.status }} - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Check if we should automerge - id: should-automerge - env: - BASE_REF: ${{ github.event.pull_request.base.ref }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: | - for commit in $(git rev-list --first-parent origin/$BASE_REF..$HEAD_SHA); do - committer=$(git show --format=$'%ce' -s $commit) - echo "Committer: $committer" - if [[ "$committer" != "web3-bot@users.noreply.github.com" ]]; then - echo "Commit $commit wasn't committed by web3-bot, but by $committer." - echo "status=false" >> $GITHUB_OUTPUT - exit - fi - done - echo "status=true" >> $GITHUB_OUTPUT - automerge: - needs: automerge-check - runs-on: ubuntu-latest - # The check for the user is redundant here, as this job depends on the automerge-check job, - # but it prevents this job from spinning up, just to be skipped shortly after. - if: github.event.pull_request.user.login == 'web3-bot' && needs.automerge-check.outputs.status == 'true' - steps: - - name: Wait on tests - uses: lewagon/wait-on-check-action@3a563271c3f8d1611ed7352809303617ee7e54ac # v1.2.0 - with: - ref: ${{ github.event.pull_request.head.sha }} - repo-token: ${{ github.token }} - wait-interval: 10 - running-workflow-name: '${{ inputs.job }} / ${{ github.job }}' # the name of the check for this job - - name: Merge PR - uses: pascalgn/automerge-action@eb68b061739cb9d81564f8e812d0b3c45f0fb09a # v0.15.5 - env: - GITHUB_TOKEN: "${{ github.token }}" - MERGE_LABELS: "" - MERGE_METHOD: "squash" - MERGE_DELETE_BRANCH: true diff --git a/.github/workflows/create-prs.yml b/.github/workflows/create-prs.yml index 437eeb37..fbbe707d 100644 --- a/.github/workflows/create-prs.yml +++ b/.github/workflows/create-prs.yml @@ -9,67 +9,107 @@ on: jobs: dispatch: - name: Create PRs in targets + name: Create PRs runs-on: ubuntu-latest - env: - GITHUB_TOKEN: ${{ secrets.WEB3_BOT_GITHUB_TOKEN }} steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.workflow_run.head_commit.id || github.sha }} - - name: Sync PRs in targets that need it + - name: Create PRs env: - PR_TITLE: 'ci: update Unified CI configuration' - PR_BRANCH: 'web3-bot/sync' - run: | - targets=() - for config in configs/*.json; do - targets+=($(jq -rc ".repositories[] | .target" $config)) - done - failed=() - for target in ${targets[@]}; do - echo "Processing $target" - base="$(gh api "/repos/$target" --jq '.default_branch')" - # checks if a PR needs to be created - if [[ "$(gh api -X GET "/repos/$target/compare/$base...$PR_BRANCH" --jq '.status')" == 'ahead' ]]; then - if [[ "$(gh api -X GET "/repos/$target/pulls" -f head="$(echo "$target" | cut -d/ -f1):$PR_BRANCH" -f base="$base" --jq 'length')" != '0' ]] ; then - echo "The PR already exists. Skipping." + BRANCH: web3-bot/sync + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.WEB3_BOT_GITHUB_TOKEN }} + retries: 0 + script: | + const request = async function(req, opts) { + try { + return await req(opts) + } catch(err) { + opts.request.retries = (opts.request.retries || 0) + 1 + if (err.status === 403) { + if (err.response.headers['x-ratelimit-remaining'] === '0') { + const retryAfter = err.response.headers['x-ratelimit-reset'] - Math.floor(Date.now() / 1000) || 1 + core.info(`Rate limit exceeded, retrying in ${retryAfter} seconds`) + await new Promise(resolve => setTimeout(resolve, retryAfter * 1000)) + return request(req, opts) + } + if (err.message.toLowerCase().includes('secondary rate limit')) { + const retryAfter = Math.pow(2, opts.request.retries) + core.info(`Secondary rate limit exceeded, retrying in ${retryAfter} seconds`) + await new Promise(resolve => setTimeout(resolve, retryAfter * 1000)) + return request(req, opts) + } + } + throw err + } + } + github.hook.wrap('request', request) + + core.info(`Looking for repositories the user has direct access to`) + const items = await github.paginate(github.rest.repos.listForAuthenticatedUser, { + affiliation: 'collaborator' + }) + core.info(`Filtering out the repositories without unmerged branches`) + const repos = [] + for (const item of items) { + core.info(`Checking if a PR can be created for ${item.html_url}`) + let branch + try { + branch = (await github.rest.repos.getBranch({ + owner: item.owner.login, + repo: item.name, + branch: process.env.BRANCH + }))?.data + } catch(error) { + if (error.status != 404) { + throw error + } + } + if (branch != undefined) { + core.info(`The branch exists in ${item.html_url}`) + } else { + core.info(`The branch does not exist in ${item.html_url}`) + continue + } + const {data: compare} = await github.rest.repos.compareCommitsWithBasehead({ + owner: item.owner.login, + repo: item.name, + basehead: `${item.default_branch}...${branch.name}` + }) + if (compare.status == 'ahead') { + core.info(`PR for ${item.html_url} can be created`) + } else { + core.info(`PR for ${item.html_url} cannot be created`) continue - fi - else - echo "The branch does not exist or has diverged from $base. Skipping." - continue - fi - # tries to create a PR in target - pr_create_attempt=1 - pr_create_max_attempts=12 - pr_create_attempt_interval_in_seconds=1 - pr_create_cooldown_in_seconds=1 - # max cumulative sleep time - 68.25 minutes - while true; do - if result="$(gh api "/repos/$target/pulls" -f title="$PR_TITLE" -f head="$PR_BRANCH" -f base="$base")"; then - echo "Successfully created a PR for '$target' ($pr_create_attempt/$pr_create_max_attempts)" - echo "Sleeping for $pr_create_cooldown_in_seconds seconds before creating a next one" - sleep $pr_create_cooldown_in_seconds - break - fi - if [[ "$(jq -r '.message' <<< "$result")" == 'You have exceeded a secondary rate limit and have been temporarily blocked from content creation. Please retry your request again later.' ]]; then - if (( pr_create_attempt < pr_create_max_attempts )); then - echo "Failed to create a PR for '$target' due to secondary rate limit ($pr_create_attempt/$pr_create_max_attempts)" - echo "Sleeping for $pr_create_attempt_interval_in_seconds seconds before trying again" - sleep $pr_create_attempt_interval_in_seconds - pr_create_attempt_interval_in_seconds=$((pr_create_attempt_interval_in_seconds * 2)) - pr_create_attempt=$((pr_create_attempt + 1)) - continue - fi - fi - echo "$result" - echo "Failed to create a PR for '$target' ($pr_create_attempt/$pr_create_max_attempts)" - failed+=("$target") - break - done - done - if ((${#failed[@]})); then - echo "::error ::Failed to sync PRs in: ${failed[@]}" - exit 1 - fi + } + const {data: pulls} = await github.rest.repos.listPullRequestsAssociatedWithCommit({ + owner: item.owner.login, + repo: item.name, + commit_sha: branch.commit.sha + }) + if (pulls.length == 0) { + core.info(`The PR does not exist yet in ${item.html_url}`) + repos.push(item) + } else { + core.info(`The PR already exists at ${pulls[0].html_url}`) + } + } + core.info(`Attempting to create the PRs`) + const failed = [] + for (const repo of repos) { + core.info(`Creating PR in ${repo.html_url}`) + try { + const pr = await octokit.rest.pulls.create({ + owner: repo.owner.login, + repo: repo.name, + head: process.env.BRANCH, + base: repo.default_branch + }) + core.info(`${pr.html_url} created successfully`) + } catch(error) { + core.error(`Couldn't create a PR for ${repo.html_url}, got: ${error}`) + failed.push(repo) + } + } + if (failed.length != 0) { + throw new Error(`Failed to create PRs in ${failed.length} repos`) + } diff --git a/.github/workflows/delete-workflow.yml b/.github/workflows/delete-workflow.yml new file mode 100644 index 00000000..12344666 --- /dev/null +++ b/.github/workflows/delete-workflow.yml @@ -0,0 +1,106 @@ +name: Delete workflow + +on: + workflow_dispatch: + inputs: + path: + description: The path of the workflow to delete + required: true + dry-run: + description: Whether to run the workflow in dry-run mode + required: false + default: 'true' + +jobs: + dispatch: + name: Delete workflow + runs-on: ubuntu-latest + steps: + - name: Delete workflow + env: + DRY_RUN: ${{ github.event.inputs.dry-run || 'true' }} + PATH: ${{ github.event.inputs.path || '.github/workflows/automerge.yml' }} + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.WEB3_BOT_GITHUB_TOKEN }} + retries: 0 + script: | + const request = async function(req, opts) { + try { + return await req(opts) + } catch(err) { + opts.request.retries = (opts.request.retries || 0) + 1 + if (err.status === 403) { + if (err.response.headers['x-ratelimit-remaining'] === '0') { + const retryAfter = err.response.headers['x-ratelimit-reset'] - Math.floor(Date.now() / 1000) || 1 + core.info(`Rate limit exceeded, retrying in ${retryAfter} seconds`) + await new Promise(resolve => setTimeout(resolve, retryAfter * 1000)) + return request(req, opts) + } + if (err.message.toLowerCase().includes('secondary rate limit')) { + const retryAfter = Math.pow(2, opts.request.retries) + core.info(`Secondary rate limit exceeded, retrying in ${retryAfter} seconds`) + await new Promise(resolve => setTimeout(resolve, retryAfter * 1000)) + return request(req, opts) + } + } + throw err + } + } + github.hook.wrap('request', request) + + core.info(`Looking for repositories the user has direct access to`) + const items = await github.paginate(github.rest.repos.listForAuthenticatedUser, { + affiliation: 'collaborator' + }) + core.info(`Filtering out the repositories without the file`) + const files = [] + for (const item of items) { + core.info(`Checking if there is a file in ${item.html_url}`) + let file + try { + file = (await github.rest.repos.getContent({ + owner: item.owner.login, + repo: item.name, + path: process.env.PATH + }))?.data + } catch(error) { + if (error.status != 404) { + throw error + } + } + if (file) { + core.info(`${file.html_url} exists`) + files.push({ + ...file, + repo: item + }) + } else { + core.info(`The file does not exist in ${item.html_url}`) + } + } + core.info(`Attempting to delete the files`) + const failed = [] + for (const file of files) { + if (process.env.DRY_RUN == 'true') { + core.info(`Would have deleted ${file.html_url}`) + continue + } + core.debug(`Deleting ${file.html_url}`) + try { + await github.rest.repos.deleteFile({ + owner: file.repo.owner.login, + repo: file.repo.name, + path: process.env.PATH, + message: `ci: delete ${process.env.PATH}`, + sha: file.sha + }) + core.info(`${file.html_url} deleted successfully`) + } catch(error) { + core.error(`Couldn't delete ${file.html_url}, got: ${error}`) + failed.push(file) + } + } + if (failed.length != 0) { + throw new Error(`Failed to delete ${failed.length} files`) + } diff --git a/.github/workflows/merge-prs.yml b/.github/workflows/merge-prs.yml new file mode 100644 index 00000000..5bccc4e2 --- /dev/null +++ b/.github/workflows/merge-prs.yml @@ -0,0 +1,85 @@ +name: Merge PRs + +on: + workflow_dispatch: + schedule: + - cron: "0 0 * * *" # https://crontab.guru/every-day + +jobs: + dispatch: + name: Merge PRs + runs-on: ubuntu-latest + steps: + - name: Merge PRs + env: + QUERY: is:pr author:web3-bot state:open head:web3-bot/sync archived:false + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.WEB3_BOT_GITHUB_TOKEN }} + retries: 0 + script: | + const request = async function(req, opts) { + try { + return await req(opts) + } catch(err) { + opts.request.retries = (opts.request.retries || 0) + 1 + if (err.status === 403) { + if (err.response.headers['x-ratelimit-remaining'] === '0') { + const retryAfter = err.response.headers['x-ratelimit-reset'] - Math.floor(Date.now() / 1000) || 1 + core.info(`Rate limit exceeded, retrying in ${retryAfter} seconds`) + await new Promise(resolve => setTimeout(resolve, retryAfter * 1000)) + return request(req, opts) + } + if (err.message.toLowerCase().includes('secondary rate limit')) { + const retryAfter = Math.pow(2, opts.request.retries) + core.info(`Secondary rate limit exceeded, retrying in ${retryAfter} seconds`) + await new Promise(resolve => setTimeout(resolve, retryAfter * 1000)) + return request(req, opts) + } + } + throw err + } + } + github.hook.wrap('request', request) + + core.info(`Looking for PRs matching the query: ${process.env.QUERY}`) + const items = await github.paginate(github.rest.search.issuesAndPullRequests, { + q: process.env.QUERY + }) + core.info(`Filtering out the PRs that cannot be merged`) + const prs = [] + for (const item of items) { + core.info(`Retrieving ${item.html_url}`) + const [_, owner, repo] = item.url.match(/repos\/(.+?)\/(.+?)\/issues/) + const {data: pr} = await github.rest.pulls.get({ + owner, + repo, + pull_number: item.number + }) + if (pr.mergeable_state == 'clean') { + core.info(`${pr.html_url} can be merged`) + prs.push(pr) + } else { + core.info(`${pr.html_url} cannot be merged`) + } + } + core.info(`Attempting to merge the PRs`) + const failed = [] + for (const pr of prs) { + core.info(`Merging ${pr.html_url}`) + try { + await octokit.rest.pulls.merge({ + owner: pr.base.repo.owner.login, + repo: pr.base.repo.name, + pull_number: pr.number, + merge_method: 'squash' + }) + core.info(`${pr.html_url} merged successfully`) + } catch(error) { + core.error(`Couldn't merge ${pr.html_url}, got: ${error}`) + failed.push(pr) + } + } + if (failed.length != 0) { + throw new Error(`Failed to merge ${failed.length} PRs`) + } diff --git a/README.md b/README.md index 4c51ad4d..3e6354d4 100644 --- a/README.md +++ b/README.md @@ -117,8 +117,6 @@ This repository currently defines two workflows for Go repositories: * [go-test](templates/.github/workflows/go-test.yml): Runs all tests, using different compiler versions and operating systems. Whenever one of these workflows is changed, this repository runs the [copy workflow](.github/workflows/copy-workflow.yml). This workflow creates a pull request in every participating repository to update *go-check* and *go-test*. -In order to help with the distribution of these workflows, this repository defines two additional workflows that are distributed across participating repositories: -* [automerge](templates/.github/workflows/automerge.yml): In most cases, an update to the workflows will not cause CI to fail in most participating repositories. To make our life easier, *automerge* automatically merges the pull request if all checks succeed. ## Usage diff --git a/configs/go.json b/configs/go.json index 761188ec..943cd5cc 100644 --- a/configs/go.json +++ b/configs/go.json @@ -1,7 +1,6 @@ { "defaults": { "files": [ - ".github/workflows/automerge.yml", ".github/workflows/go-test.yml", ".github/workflows/go-check.yml", ".github/workflows/releaser.yml", diff --git a/configs/js.json b/configs/js.json index 7681da7f..dda69726 100644 --- a/configs/js.json +++ b/configs/js.json @@ -1,7 +1,6 @@ { "defaults": { "files": [ - ".github/workflows/automerge.yml", ".github/workflows/js-test-and-release.yml" ] }, diff --git a/templates/.github/workflows/automerge.yml b/templates/.github/workflows/automerge.yml deleted file mode 100644 index fad93636..00000000 --- a/templates/.github/workflows/automerge.yml +++ /dev/null @@ -1,12 +0,0 @@ -name: Automerge -on: [ pull_request ] - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - automerge: - uses: protocol/.github/.github/workflows/automerge.yml@master - with: - job: 'automerge' diff --git a/templates/README.md b/templates/README.md index a6ea6a2f..53ca95e6 100644 --- a/templates/README.md +++ b/templates/README.md @@ -55,13 +55,13 @@ The `github` context contains information about the target repository the file i ```json { "defaults": { - "files": [".github/workflows/automerge.yml"], + "files": [".github/workflows/example1.yml"], "is_example": false }, "repositories": [ { "target": "protocol/.github-test-target", - "extra_files": [".github/workflows/example.yml"], + "extra_files": [".github/workflows/example2.yml"], "example": { "greeting": "Hello" }, @@ -79,8 +79,8 @@ The `github` context contains information about the target repository the file i { "config": { "target": "protocol/.github-test-target", - "files": [".github/workflows/automerge.yml"], - "extra_files": [".github/workflows/example.yml"], + "files": [".github/workflows/example1.yml"], + "extra_files": [".github/workflows/example2.yml"], "example": { "greeting": "Hello" },