Merge pull request #74 from bryzgaloff/tests-airflow-constraints

Install Airflow with constraints in tests
bryzgaloff · Jan 30, 2024 · 72bbc07 · 72bbc07
2 parents e8140ef + bfe4ca3
commit 72bbc07
Show file tree

Hide file tree

Showing 3 changed files with 101 additions and 63 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -8,109 +8,148 @@ on:
       - "tests/**"
       - ".github/workflows/tests.yml"
       - "requirements.txt"
+      - "pyproject.toml"
   pull_request:
     branches: [ "master" ]
     paths:
       - "src/**"
       - "tests/**"
       - ".github/workflows/tests.yml"
       - "requirements.txt"
+      - "pyproject.toml"
 
 env:
   PYTHONPATH: "src"  # for tests to import src/airflow_clickhouse_plugin
 
 jobs:
 
-  unit-tests-no-common-sql:
+  unit-tests:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ["3.8", "3.9", "3.10", "3.11"]
         airflow-version: ["2.0.2", "2.1.4", "2.2.5", "2.3.4", "2.4.3", "2.5.3", "2.6.3", "2.7.1"]
+        airflow-extras: ["", "[common.sql]"]
+        exclude:
+          # constraints files for these combinations are missing
+          - python-version: "3.10"
+            airflow-version: "2.0.2"
+          - python-version: "3.10"
+            airflow-version: "2.1.4"
+          - python-version: "3.10"
+            airflow-version: "2.2.5"
+          - python-version: "3.11"
+            airflow-version: "2.0.2"
+          - python-version: "3.11"
+            airflow-version: "2.1.4"
+          - python-version: "3.11"
+            airflow-version: "2.2.5"
+          - python-version: "3.11"
+            airflow-version: "2.3.4"
+          - python-version: "3.11"
+            airflow-version: "2.4.3"
+          - python-version: "3.11"
+            airflow-version: "2.5.3"
+          # common.sql constraint for these Airflow versions is <1.3.0
+          #   => misses SQLExecuteQueryOperator
+          - airflow-extras: "[common.sql]"
+            airflow-version: "2.0.2"
+          - airflow-extras: "[common.sql]"
+            airflow-version: "2.1.4"
+          - airflow-extras: "[common.sql]"
+            airflow-version: "2.2.5"
+          - airflow-extras: "[common.sql]"
+            airflow-version: "2.3.4"
+          - airflow-extras: "[common.sql]"
+            airflow-version: "2.4.3"
+        include:
+          - airflow-extras: "[common.sql]"
+            tests-pattern: ""
+          - airflow-extras: ""
+            tests-pattern: "-p test_clickhouse.py"
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install -r requirements.txt --ignore-installed apache-airflow==${{ matrix.airflow-version }}
+        python -m pip install \
+          -r requirements.txt \
+          apache-airflow${{ matrix.airflow-extras }}==${{ matrix.airflow-version }} \
+          --ignore-installed \
+          --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-${{ matrix.airflow-version }}/constraints-${{ matrix.python-version }}.txt"
     - name: Run unit tests
       run: |
-        python -m unittest discover -t tests -s unit -p test_clickhouse.py
+        python -m unittest discover -t tests -s unit ${{ matrix.tests-pattern }}
 
-  unit-tests-common-sql:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
-        airflow-version: ["2.1.4", "2.2.5", "2.3.4", "2.4.3", "2.5.3", "2.6.3", "2.7.1"]
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install -r requirements.txt --ignore-installed apache-airflow[common.sql]==${{ matrix.airflow-version }}
-    - name: Run unit tests
-      run: |
-        python -m unittest discover -t tests -s unit
-
-  integration-tests-no-common-sql:
+  integration-tests:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ["3.8", "3.9", "3.10", "3.11"]
         airflow-version: ["2.0.2", "2.1.4", "2.2.5", "2.3.4", "2.4.3", "2.5.3", "2.6.3", "2.7.1"]
+        airflow-extras: ["", "[common.sql]"]
+        exclude:
+          # constraints files for these combinations are missing
+          - python-version: "3.10"
+            airflow-version: "2.0.2"
+          - python-version: "3.10"
+            airflow-version: "2.1.4"
+          - python-version: "3.10"
+            airflow-version: "2.2.5"
+          - python-version: "3.11"
+            airflow-version: "2.0.2"
+          - python-version: "3.11"
+            airflow-version: "2.1.4"
+          - python-version: "3.11"
+            airflow-version: "2.2.5"
+          - python-version: "3.11"
+            airflow-version: "2.3.4"
+          - python-version: "3.11"
+            airflow-version: "2.4.3"
+          - python-version: "3.11"
+            airflow-version: "2.5.3"
+          # common.sql constraint for these Airflow versions is <1.3.0
+          #   => misses SQLExecuteQueryOperator
+          - airflow-extras: "[common.sql]"
+            airflow-version: "2.0.2"
+          - airflow-extras: "[common.sql]"
+            airflow-version: "2.1.4"
+          - airflow-extras: "[common.sql]"
+            airflow-version: "2.2.5"
+          - airflow-extras: "[common.sql]"
+            airflow-version: "2.3.4"
+          - airflow-extras: "[common.sql]"
+            airflow-version: "2.4.3"
+        include:
+          - airflow-extras: "[common.sql]"
+            tests-pattern: ""
+          - airflow-extras: ""
+            tests-pattern: "-p test_clickhouse.py"
     services:
       clickhouse:
         image: yandex/clickhouse-server
         ports:
           - 9000/tcp
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install -r requirements.txt --ignore-installed apache-airflow==${{ matrix.airflow-version }}
-    - name: Run tests on ClickHouse server
-      env:
-        AIRFLOW_CONN_CLICKHOUSE_DEFAULT: "clickhouse://localhost:${{ job.services.clickhouse.ports['9000'] }}"
-      run: |
-        python -m unittest discover -t tests -s integration -p test_clickhouse.py
-
-  integration-tests-common-sql:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
-        airflow-version: ["2.1.4", "2.2.5", "2.3.4", "2.4.3", "2.5.3", "2.6.3", "2.7.1"]
-    services:
-      clickhouse:
-        image: yandex/clickhouse-server
-        ports:
-          - 9000/tcp
-    steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install -r requirements.txt --ignore-installed apache-airflow[common.sql]==${{ matrix.airflow-version }}
+        python -m pip install \
+          -r requirements.txt \
+          apache-airflow${{ matrix.airflow-extras }}==${{ matrix.airflow-version }} \
+          --ignore-installed \
+          --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-${{ matrix.airflow-version }}/constraints-${{ matrix.python-version }}.txt"
     - name: Run tests on ClickHouse server
       env:
         AIRFLOW_CONN_CLICKHOUSE_DEFAULT: "clickhouse://localhost:${{ job.services.clickhouse.ports['9000'] }}"
       run: |
-        python -m unittest discover -t tests -s integration
+        python -m unittest discover -t tests -s integration ${{ matrix.tests-pattern }}
diff --git a/README.md b/README.md
@@ -62,10 +62,7 @@ Dependencies: `apache-airflow-providers-common-sql` (usually pre-packed with Air
 
 ## Python and Airflow versions support
 
-Different versions of the plugin support different combinations of Python and
-    Airflow versions. We _primarily_ support **Airflow 2.0+ and Python 3.8+**.
-    If you need to use the plugin with older Python-Airflow combinations, pick a
-    suitable plugin version:
+Different versions of the plugin support different combinations of Python and Airflow versions. We _primarily_ support **Airflow 2.0+ and Python 3.8+**. If you need to use the plugin with older Python-Airflow combinations, pick a suitable plugin version:
 
 | airflow-clickhouse-plugin version | Airflow version         | Python version     |
 |-----------------------------------|-------------------------|--------------------|
@@ -81,8 +78,9 @@ Different versions of the plugin support different combinations of Python and
 | \>=0.5.4,<0.6.0                   | ~=1.10.6                | \>=2.7 or >=3.5.\* |
 | \>=0.5.0,<0.5.4                   | ==1.10.6                | \>=2.7 or >=3.5.\* |
 
-`~=` means compatible release, see [PEP 440][pep-440-compatible-releases] for an
-    explanation.
+`~=` means compatible release, see [PEP 440][pep-440-compatible-releases] for an explanation.
+
+[DB API 2.0 functionality](#python-db-api-20-family) requires `apache-airflow>=2.2` and `apache-airflow-providers-common-sql>=1.3`: earlier versions are not supported.
 
 Previous versions of the plugin might require `pandas` extra: `pip install airflow-clickhouse-plugin[pandas]==0.11.0`. Check out earlier versions of `README.md` for details.
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -40,6 +40,7 @@ Issues = "https://github.com/bryzgaloff/airflow-clickhouse-plugin/issues"
 [project.optional-dependencies]
 "common.sql" = [
     "apache-airflow[common.sql]>=2.2.0,<2.8.0",
+    "apache-airflow-providers-common-sql>=1.3.0",  # introduces SQLExecuteQueryOperator
     "clickhouse-driver>=0.2.1",
 ]