test_on_local_destinations_forks.yml

name: test sources on postgres and duckdb on forks

on:
  pull_request_target:
    branches:
      - master
      - enh/api_helper
    types:
      - opened
      - synchronize
      - labeled
  workflow_dispatch:

env:
  ALL_DESTINATIONS: '["postgres", "duckdb"]'
  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
  RUNTIME__LOG_LEVEL: ERROR

jobs:
  authorize:
    # run when label is assigned OR when we are not a fork
    # see https://iterative.ai/blog/testing-external-contributions-using-github-actions-secrets for inspiration
    if: ${{ github.event.label.name == 'ci from fork' || (github.event.pull_request.head.repo.full_name == github.repository && (github.event.action == 'opened' || github.event.action == 'synchronize'))}}
    runs-on: ubuntu-latest
    steps:
      - run: true

  get_changed_sources:
    needs: authorize
    uses: ./.github/workflows/get_changed_sources.yml
    # run only on fork
    # if: ${{ github.event.pull_request.head.repo.fork }}

  run_loader:
    name: FORKS - test on local postgres and duckdb
    needs: get_changed_sources
    if: needs.get_changed_sources.outputs.sources_list != ''
    strategy:
      max-parallel: 1
      fail-fast: false
    defaults:
      run:
        shell: bash
    runs-on: "ubuntu-latest"

    # Service containers to run with `container-job`
    services:
      # Label used to access the service container
      postgres:
        # Docker Hub image
        image: postgres
        # Provide the password for postgres
        env:
          POSTGRES_DB: dlt_data
          POSTGRES_USER: loader
          POSTGRES_PASSWORD: loader
        ports:
          - 5432:5432
        # Set health checks to wait until postgres has started
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

    steps:
      - name: Check out
        uses: actions/checkout@master
        with:
          ref: ${{ github.event.pull_request.head.sha || github.ref }}

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.11.x"

      - name: Install Poetry
        uses: snok/install-poetry@v1.3.2
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v2
        with:
          # path: ${{ steps.pip-cache.outputs.dir }}
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-destinations-forks

      - name: Install dependencies
        # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: poetry install --no-interaction --all-extras

      - name: create secrets.toml
        run: pwd && echo "$DLT_SECRETS_TOML" > sources/.dlt/secrets.toml

      - run: |
          sources_list="${{ needs.get_changed_sources.outputs.sources_list }}"
          test_paths=$(echo "$sources_list" | awk '{for(i=1;i<=NF;i++) printf "tests/%s ", $i}')
          poetry run pytest $test_paths
        name: Run tests on Linux/macOS
        env:
          DESTINATION__POSTGRES__CREDENTIALS: postgresql://loader:loader@localhost:5432/dlt_data