feat: port CircleCI jobs to GH Actions and improve sharness reporting

.circleci/config.yml

@@ -7,7 +7,7 @@ jobs:
     executor: continuation/default
     steps:
       - checkout
-      - run: 
+      - run:
           name: Generate params
           # for builds on the ipfs/kubo repo, use 2xlarge for faster builds
           # but since this is not available for many contributors, we otherwise use medium

.circleci/main.yml

@@ -143,14 +143,17 @@ jobs:
         path: /tmp/circleci-test-results
   sharness:
     machine:
-        image: ubuntu-2004:202010-01
+        image: ubuntu-2204:2022.10.1
     resource_class: << pipeline.parameters.resource_class >>
     working_directory: ~/ipfs/kubo
     environment:
         <<: *default_environment
         TEST_NO_DOCKER: 0
+        TEST_NO_PLUGIN: 1
         TEST_NO_FUSE: 1
         TEST_VERBOSE: 1
+        TEST_JUNIT: 1
+        TEST_EXPENSIVE: 1
     steps:
     - run: sudo apt update
     - run: |
@@ -159,7 +162,7 @@ jobs:
         tar xfz go1.19.1.linux-amd64.tar.gz
         echo "export PATH=$(pwd)/go/bin:\$PATH" >> ~/.bashrc
     - run: go version
-    - run: sudo apt install socat net-tools fish
+    - run: sudo apt install socat net-tools fish libxml2-utils
     - checkout
 
     - run:
@@ -183,7 +186,7 @@ jobs:
         command: echo "export TEST_DOCKER_HOST=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')" >> $BASH_ENV
     - run:
         echo TEST_DOCKER_HOST=$TEST_DOCKER_HOST &&
-        make -O -j << pipeline.parameters.make_jobs >> coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 TEST_DOCKER_HOST=$TEST_DOCKER_HOST
+        make -O -j << pipeline.parameters.make_jobs >> test_sharness coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml CONTINUE_ON_S_FAILURE=1 TEST_DOCKER_HOST=$TEST_DOCKER_HOST
     - run:
         when: always
         command: bash <(curl -s https://codecov.io/bash) -cF sharness -X search -f coverage/sharness_tests.coverprofile
@@ -345,13 +348,13 @@ jobs:
             npx playwright install
           working_directory: ~/ipfs/kubo/ipfs-webui
       - run:
-          name: Running upstream tests (finish early if they fail)
+          name: Run ipfs-webui@main build and smoke-test to confirm the upstream repo is not broken
           command: |
-            npm test || circleci-agent step halt
+            npm test
           working_directory: ~/ipfs/kubo/ipfs-webui
       - run:
-          name: Running tests with kubo built from current commit
-          command: npm test
+          name: Test ipfs-webui@main E2E against the locally built Kubo binary
+          command: npm run test:e2e
           working_directory: ~/ipfs/kubo/ipfs-webui
           environment:
             IPFS_GO_EXEC: /tmp/circleci-workspace/bin/ipfs

.github/workflows/build.yml

@@ -0,0 +1,200 @@
+name: 'ci/gh-experiment: interop'
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - 'master'
+
+env:
+  GO_VERSION: 1.19.1
+
+jobs:
+  prepare:
+    if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    env:
+      TEST_NO_DOCKER: 1
+      TEST_NO_FUSE: 1
+      TEST_VERBOSE: 1
+      TRAVIS: 1
+      GIT_PAGER: cat
+      IPFS_CHECK_RCMGR_DEFAULTS: 1
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - uses: actions/checkout@v3
+      - uses: protocol/cache-go-action@v1
+        with:
+          name: ${{ github.job }}
+      - run: make build
+      - uses: actions/upload-artifact@v3
+        with:
+          name: kubo
+          path: cmd/ipfs/ipfs
+  ipfs-interop:
+    needs: [prepare]
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        suites:
+          - 'exchange-files'
+          - 'files pin circuit ipns cid-version-agnostic ipns-pubsub pubsub'
+      fail-fast: false
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16.12.0
+      - uses: actions/download-artifact@v3
+        with:
+          name: kubo
+          path: cmd/ipfs
+      - run: chmod +x cmd/ipfs/ipfs
+      - run: |
+          echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT
+        id: npm-cache-dir
+      - uses: actions/cache@v3
+        with:
+          path: ${{ steps.npm-cache-dir.outputs.dir }}
+          key: ${{ runner.os }}-${{ github.job }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ github.job }}-
+      - run: mkdir interop
+      - run: |
+          npm init -y
+          npm install ipfs@^0.61.0
+          npm install ipfs-interop@^8.0.10
+        working-directory: interop
+      - run: npx ipfs-interop -- -t node $(sed -e 's#[^ ]*#-f test/&.js#g' <<< '${{ matrix.suites }}')
+        env:
+          LIBP2P_TCP_REUSEPORT: false
+          LIBP2P_ALLOW_WEAK_RSA_KEYS: 1
+          IPFS_GO_EXEC: ${{ github.workspace }}/cmd/ipfs/ipfs
+        working-directory: interop
+  go-ipfs-api:
+    needs: [prepare]
+    runs-on: ubuntu-latest
+    env:
+      TEST_NO_DOCKER: 1
+      TEST_NO_FUSE: 1
+      TEST_VERBOSE: 1
+      TRAVIS: 1
+      GIT_PAGER: cat
+      IPFS_CHECK_RCMGR_DEFAULTS: 1
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - uses: actions/download-artifact@v3
+        with:
+          name: kubo
+          path: cmd/ipfs
+      - run: chmod +x cmd/ipfs/ipfs
+      - uses: actions/checkout@v3
+        with:
+          repository: ipfs/go-ipfs-api
+          path: go-ipfs-api
+      - run: cmd/ipfs/ipfs daemon --init --enable-namesys-pubsub &
+      - run: |
+          while ! cmd/ipfs/ipfs id --api=/ip4/127.0.0.1/tcp/5001 2>/dev/null; do
+            sleep 1
+          done
+        timeout-minutes: 5
+      - uses: protocol/cache-go-action@v1
+        with:
+          name: ${{ github.job }}
+      - run: go test -count=1 -v ./...
+        working-directory: go-ipfs-api
+      - run: cmd/ipfs/ipfs shutdown
+        if: always()
+  go-ipfs-http-client:
+    needs: [prepare]
+    runs-on: ubuntu-latest
+    env:
+      TEST_NO_DOCKER: 1
+      TEST_NO_FUSE: 1
+      TEST_VERBOSE: 1
+      TRAVIS: 1
+      GIT_PAGER: cat
+      IPFS_CHECK_RCMGR_DEFAULTS: 1
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - uses: actions/download-artifact@v3
+        with:
+          name: kubo
+          path: cmd/ipfs
+      - run: chmod +x cmd/ipfs/ipfs
+      - uses: actions/checkout@v3
+        with:
+          repository: ipfs/go-ipfs-http-client
+          path: go-ipfs-http-client
+      - uses: protocol/cache-go-action@v1
+        with:
+          name: ${{ github.job }}
+      - run: echo '${{ github.workspace }}/cmd/ipfs' >> $GITHUB_PATH
+      - run: go test -count=1 -v ./...
+        working-directory: go-ipfs-http-client
+  ipfs-webui:
+    needs: [prepare]
+    runs-on: ubuntu-latest
+    env:
+      NO_SANDBOX: true
+      LIBP2P_TCP_REUSEPORT: false
+      LIBP2P_ALLOW_WEAK_RSA_KEYS: 1
+      E2E_IPFSD_TYPE: go
+      TRAVIS: 1
+      GIT_PAGER: cat
+      IPFS_CHECK_RCMGR_DEFAULTS: 1
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16.12.0
+      - uses: actions/download-artifact@v3
+        with:
+          name: kubo
+          path: cmd/ipfs
+      - run: chmod +x cmd/ipfs/ipfs
+      - uses: actions/checkout@v3
+        with:
+          repository: ipfs/ipfs-webui
+          path: ipfs-webui
+      - run: |
+          echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT
+        id: npm-cache-dir
+      - uses: actions/cache@v3
+        with:
+          path: ${{ steps.npm-cache-dir.outputs.dir }}
+          key: ${{ runner.os }}-${{ github.job }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ github.job }}-
+      - run: |
+          npm ci --prefer-offline --no-audit --progress=false
+          npx playwright install
+        working-directory: ipfs-webui
+      - name: Run ipfs-webui@main build and smoke-test to confirm the upstream repo is not broken
+        run: npm test
+        working-directory: ipfs-webui
+      - name: Test ipfs-webui@main E2E against the locally built Kubo binary
+        run: npm run test:e2e
+        env:
+          IPFS_GO_EXEC: ${{ github.workspace }}/cmd/ipfs/ipfs
+        working-directory: ipfs-webui

.github/workflows/codeql-analysis.yml

@@ -1,5 +1,5 @@
 # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
-name: "CodeQL"
+name: CodeQL
 
 on:
   workflow_dispatch:

.github/workflows/docker-build.yml

@@ -0,0 +1,25 @@
+name: 'ci/gh-experiment: docker-build'
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - 'master'
+
+jobs:
+  docker-build:
+    if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    env:
+      IMAGE_NAME: ipfs/kubo
+      WIP_IMAGE_TAG: wip
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.19.1
+      - uses: actions/checkout@v3
+      - run: docker build -t $IMAGE_NAME:$WIP_IMAGE_TAG .

.github/workflows/docker-image.yml

@@ -32,7 +32,7 @@ jobs:
         run: |
           TAGS="$(./bin/get-docker-tags.sh $(date -u +%F))"
           TAGS="${TAGS//$'\n'/'%0A'}"
-          echo "::set-output name=value::$(echo $TAGS)"
+          echo "value=$(echo $TAGS)" >> $GITHUB_OUTPUT
         shell: bash
 
       - name: Log in to Docker Hub

.github/workflows/gobuild.yml

@@ -0,0 +1,39 @@
+name: 'ci/gh-experiment: go build'
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - 'master'
+
+jobs:
+  runner:
+    if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
+    uses: ipfs/kubo/.github/workflows/runner.yml@ci/move-to-github-actions # TODO: change to master
+  gobuild:
+    needs: [runner]
+    runs-on: ${{ fromJSON(needs.runner.outputs.config).labels }}
+    env:
+      TEST_NO_DOCKER: 1
+      TEST_VERBOSE: 1
+      TRAVIS: 1
+      GIT_PAGER: cat
+      IPFS_CHECK_RCMGR_DEFAULTS: 1
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.19.1
+      - uses: actions/checkout@v3
+      - uses: protocol/cache-go-action@v1
+        with:
+          name: ${{ github.job }}
+      - run: make cmd/ipfs-try-build
+        env:
+          TEST_NO_FUSE: 0
+      - run: make cmd/ipfs-try-build
+        env:
+          TEST_NO_FUSE: 1

.github/workflows/golang-analysis.yml

@@ -1,8 +1,15 @@
-on: [push, pull_request]
 name: Go Checks
 
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - 'master'
+
 jobs:
   unit:
+    if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
     name: All
     steps:

.github/workflows/golint.yml

@@ -0,0 +1,32 @@
+name: 'ci/gh-experiment: go lint'
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - 'master'
+
+jobs:
+  golint:
+    if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    env:
+      TEST_NO_DOCKER: 1
+      TEST_NO_FUSE: 1
+      TEST_VERBOSE: 1
+      TRAVIS: 1
+      GIT_PAGER: cat
+      IPFS_CHECK_RCMGR_DEFAULTS: 1
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.19.1
+      - uses: actions/checkout@v3
+      - uses: protocol/cache-go-action@v1
+        with:
+          name: ${{ github.job }}
+      - run: make -O test_go_lint