selfdrive_tests.yaml

name: selfdrive

on:
  push:
    branches:
      - master
  pull_request:
  workflow_dispatch:
  workflow_call:
    inputs:
      run_number:
        default: '1'
        required: true
        type: string

concurrency:
  group: selfdrive-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
  cancel-in-progress: true

env:
  PYTHONWARNINGS: error
  BASE_IMAGE: openpilot-base
  AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}

  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
  BUILD: selfdrive/test/docker_build.sh base

  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PRE_COMMIT_HOME=/tmp/pre-commit -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/pre-commit:/tmp/pre-commit -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c

  PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0 -n logical

jobs:
  build_release:
    name: build release
    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
                   ((github.event_name != 'pull_request') ||
                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
    env:
      STRIPPED_DIR: /tmp/releasepilot
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - run: git lfs pull
    - name: Build devel
      timeout-minutes: 1
      run: TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
    - uses: ./.github/workflows/setup-pre-commit
    - uses: ./.github/workflows/setup-with-retry
    - name: Check submodules
      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
      timeout-minutes: 1
      run: release/check-submodules.sh
    - name: Build openpilot and run checks
      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
      run: |
        cd $STRIPPED_DIR
        ${{ env.RUN }} "python system/manager/build.py"
    - name: Run tests
      timeout-minutes: 3
      run: |
        cd $STRIPPED_DIR
        ${{ env.RUN }} "release/check-dirty.sh && \
                        MAX_EXAMPLES=5 $PYTEST -m 'not slow' selfdrive/car"
    - name: pre-commit
      timeout-minutes: 3
      run: |
        cd $GITHUB_WORKSPACE
        cp .pre-commit-config.yaml $STRIPPED_DIR
        cp pyproject.toml $STRIPPED_DIR
        cd $STRIPPED_DIR
        ${{ env.RUN }} "unset PYTHONWARNINGS && SKIP=check-added-large-files,check-hooks-apply,check-useless-excludes pre-commit run --all && chmod -R 777 /tmp/pre-commit"

  build:
    strategy:
      matrix:
        arch: ${{ fromJson('["x86_64"]') }} # TODO: Re-add build test for aarch64 once we switched to ubuntu-2404
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - name: Setup docker push
      if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
      run: |
        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
        : # (TODO: Re-add this once we test other archs) echo "TARGET_ARCHITECTURE=${{ matrix.arch }}" >> "$GITHUB_ENV"
        $DOCKER_LOGIN
    - uses: ./.github/workflows/setup-with-retry
      with:
        docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
    - uses: ./.github/workflows/compile-openpilot
      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 15 || 30) }} # allow more time when we missed the scons cache

  build_mac:
    name: build macos
    runs-on: macos-latest
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - run: git lfs pull
    - name: Install dependencies
      run: ./tools/mac_setup.sh
      env:
        SKIP_PROMPT: 1
        # package install has DeprecationWarnings
        PYTHONWARNINGS: default
    - run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
    - name: Getting scons cache
      uses: 'actions/cache@v4'
      with:
        path: /tmp/scons_cache
        key: scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
        restore-keys: |
          scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}
          scons-${{ runner.arch }}-macos
    - name: Building openpilot
      run: . .venv/bin/activate && scons -j$(nproc)

  static_analysis:
    name: static analysis
    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
                   ((github.event_name != 'pull_request') ||
                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - uses: ./.github/workflows/setup-pre-commit
    - uses: ./.github/workflows/setup-with-retry
    - name: Build openpilot
      run: ${{ env.RUN }} "scons -j$(nproc)"
    - name: pre-commit
      timeout-minutes: 4
      run: ${{ env.RUN }} "unset PYTHONWARNINGS && pre-commit run --all && chmod -R 777 /tmp/pre-commit"

  unit_tests:
    name: unit tests
    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
                   ((github.event_name != 'pull_request') ||
                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
      with:
        docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
    - name: Build openpilot
      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
      run: ${{ env.RUN }} "scons -j$(nproc)"
    - name: Run unit tests
      timeout-minutes: 15
      run: |
        ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
                        $PYTEST --timeout 60 -m 'not slow' && \
                        ./selfdrive/ui/tests/create_test_translations.sh && \
                        QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
                        pytest ./selfdrive/ui/tests/test_translations.py"
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v4
      with:
        name: ${{ github.job }}
      env:
        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

  process_replay:
    name: process replay
    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
                   ((github.event_name != 'pull_request') ||
                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
      with:
        docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
    - name: Cache test routes
      id: dependency-cache
      uses: actions/cache@v4
      with:
        path: .ci_cache/comma_download_cache
        key: proc-replay-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/ref_commit', 'selfdrive/test/process_replay/test_regen.py') }}
    - name: Build openpilot
      run: |
        ${{ env.RUN }} "scons -j$(nproc)"
    - name: Run replay
      timeout-minutes: 30
      run: |
        ${{ env.RUN }} "coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
                        chmod -R 777 /tmp/comma_download_cache && \
                        coverage combine && \
                        coverage xml"
    - name: Print diff
      id: print-diff
      if: always()
      run: cat selfdrive/test/process_replay/diff.txt
    - uses: actions/upload-artifact@v4
      if: always()
      continue-on-error: true
      with:
        name: process_replay_diff.txt
        path: selfdrive/test/process_replay/diff.txt
    - name: Upload reference logs
      if: ${{ failure() && steps.print-diff.outcome == 'success' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
      run: |
        ${{ env.RUN }} "unset PYTHONWARNINGS && AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
    # PYTHONWARNINGS triggers a SyntaxError in onnxruntime
    - name: Run model replay with ONNX
      timeout-minutes: 4
      run: |
        ${{ env.RUN }} "unset PYTHONWARNINGS && \
                        ONNXCPU=1 NO_NAV=1 coverage run selfdrive/test/process_replay/model_replay.py && \
                        coverage combine && coverage xml"
    - name: Run regen
      timeout-minutes: 4
      run: |
        ${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \
                        chmod -R 777 /tmp/comma_download_cache"
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v4
      with:
        name: ${{ github.job }}
      env:
        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

  test_cars:
    name: cars
    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
                   ((github.event_name != 'pull_request') ||
                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
    strategy:
      fail-fast: false
      matrix:
        job: [0, 1]
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
    - name: Cache test routes
      id: dependency-cache
      uses: ./.github/workflows/auto-cache
      with:
        path: .ci_cache/comma_download_cache
        key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'selfdrive/car/tests/routes.py') }}-${{ matrix.job }}
    - name: Build openpilot
      run: ${{ env.RUN }} "scons -j$(nproc)"
    - name: Test car models
      timeout-minutes: 20
      run: |
        ${{ env.RUN }} "$PYTEST selfdrive/car/tests/test_models.py && \
                        chmod -R 777 /tmp/comma_download_cache"
      env:
        NUM_JOBS: 2
        JOB_ID: ${{ matrix.job }}
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v4
      with:
        name: ${{ github.job }}-${{ matrix.job }}
      env:
        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

  car_docs_diff:
    name: PR comments
    runs-on: ubuntu-latest
    if: github.event_name == 'pull_request'
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
          ref: ${{ github.event.pull_request.base.ref }}
      - run: git lfs pull
      - uses: ./.github/workflows/setup-with-retry
      - name: Get base car info
        run: |
          ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/debug/dump_car_docs.py --path /tmp/openpilot_cache/base_car_docs"
          sudo chown -R $USER:$USER ${{ github.workspace }}
      - uses: actions/checkout@v4
        with:
          submodules: true
          path: current
      - run: cd current && git lfs pull
      - name: Save car docs diff
        id: save_diff
        run: |
          cd current
          ${{ env.RUN }} "scons -j$(nproc)"
          output=$(${{ env.RUN }} "python selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_docs")
          output="${output//$'\n'/'%0A'}"
          echo "::set-output name=diff::$output"
      - name: Find comment
        if: ${{ env.AZURE_TOKEN != '' }}
        uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e
        id: fc
        with:
          issue-number: ${{ github.event.pull_request.number }}
          body-includes: This PR makes changes to
      - name: Update comment
        if: ${{ steps.save_diff.outputs.diff != '' && env.AZURE_TOKEN != '' }}
        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043
        with:
          comment-id: ${{ steps.fc.outputs.comment-id }}
          issue-number: ${{ github.event.pull_request.number }}
          body: "${{ steps.save_diff.outputs.diff }}"
          edit-mode: replace
      - name: Delete comment
        if: ${{ steps.fc.outputs.comment-id != '' && steps.save_diff.outputs.diff == '' && env.AZURE_TOKEN != '' }}
        uses: actions/github-script@v7
        with:
          script: |
            github.rest.issues.deleteComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: ${{ steps.fc.outputs.comment-id }}
            })

  create_ui_report:
    name: Create UI Report
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
      - uses: ./.github/workflows/setup-with-retry
      - name: Build openpilot
        run: ${{ env.RUN }} "scons -j$(nproc)"
      - name: Create Test Report
        run: >
            ${{ env.RUN }} "PYTHONWARNINGS=ignore &&
                            source selfdrive/test/setup_xvfb.sh &&
                            python selfdrive/ui/tests/test_ui/run.py"
      - name: Upload Test Report
        uses: actions/upload-artifact@v4
        with:
          name: report-${{ inputs.run_number }}
          path: selfdrive/ui/tests/test_ui/report_${{ inputs.run_number }}
      - name: Get changes to selfdrive/ui
        if: ${{ github.event_name == 'pull_request' }}
        id: changed-files
        uses: tj-actions/changed-files@v44
        with:
         files: |
            selfdrive/ui/**
      - name: Checkout ci-artifacts
        if: ${{ github.event_name == 'pull_request' && steps.changed-files.outputs.any_changed == 'true' }}
        uses: actions/checkout@v4
        with:
          repository: commaai/ci-artifacts
          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
          path: ${{ github.workspace }}/ci-artifacts
          ref: master
      - name: Push Screenshots
        if: ${{ github.event_name == 'pull_request' && steps.changed-files.outputs.any_changed == 'true' }}
        working-directory: ${{ github.workspace }}/ci-artifacts
        run: |
          git checkout -b openpilot/pr-${{ github.event.pull_request.number }}
          git config user.name "GitHub Actions Bot"
          git config user.email "<>"
          sudo mv ${{ github.workspace }}/selfdrive/ui/tests/test_ui/report/screenshots/* .
          git add .
          git commit -m "screenshots for PR #${{ github.event.pull_request.number }}"
          git push origin openpilot/pr-${{ github.event.pull_request.number }} --force
      - name: Comment Screenshots on PR
        if: ${{ github.event_name == 'pull_request' && steps.changed-files.outputs.any_changed == 'true' }}
        uses: thollander/actions-comment-pull-request@v2
        with:
          message: |
            <!-- _(run_id_screenshots **${{ github.run_id }}**)_ -->
            ## UI Screenshots
            <table>
              <tr>
                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.pull_request.number }}/homescreen.png"></td>
                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.pull_request.number }}/onroad.png"></td>
              </tr>
              <tr>
                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.pull_request.number }}/onroad_map.png"></td>
                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.pull_request.number }}/onroad_sidebar.png"></td>
              </tr>
              <tr>
                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.pull_request.number }}/settings_network.png"></td>
                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.pull_request.number }}/settings_device.png"></td>
              </tr>
            </table>
          comment_tag: run_id_screenshots
          pr_number: ${{ github.event.pull_request.number }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}