From 9a6baab5a271b28b6b3cbf96ffa151ac7dc79013 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 9 May 2022 23:02:25 +0200 Subject: [PATCH] Improve caching for multi-platform images. (#23562) This is another attempt to improve caching performance for multi-platform images as the previous ones were undermined by a bug in buildx multiplatform cache-to implementattion that caused the image cache to be overwritten between platforms, when multiple images were build. The bug is created for the buildx behaviour at https://github.com/docker/buildx/issues/1044 and until it is fixed we have to prpare separate caches for each platform and push them to separate tags. That adds a bit overhead on the building step, but for now it is the simplest way we can workaround the bug if we do not want to manually manipulate manifests and images. --- .github/workflows/build-images.yml | 3 +- .github/workflows/ci.yml | 10 +- BREEZE.rst | 9 +- CONTRIBUTING.rst | 11 +- Dockerfile | 62 +++--- Dockerfile.ci | 19 +- IMAGES.rst | 18 +- INSTALL | 2 +- dev/REFRESHING_CI_CACHE.md | 8 +- ...g-contributed-code-when-building-images.md | 2 +- .../commands/ci_image_commands.py | 77 ++++--- .../commands/production_image_commands.py | 51 ++--- .../commands/release_management_commands.py | 33 ++- .../src/airflow_breeze/global_constants.py | 5 +- .../params/_common_build_params.py | 166 +++++++++++++++ .../airflow_breeze/params/build_ci_params.py | 174 +++++----------- .../params/build_prod_params.py | 190 +++++------------- .../src/airflow_breeze/params/shell_params.py | 65 +++--- .../airflow_breeze/utils/common_options.py | 33 +-- .../utils/docker_command_utils.py | 173 +++++++++------- .../src/airflow_breeze/utils/platforms.py | 24 +++ .../src/airflow_breeze/utils/pulll_image.py | 13 +- .../src/airflow_breeze/utils/registry.py | 24 +-- .../src/airflow_breeze/utils/run_utils.py | 48 +++-- dev/breeze/tests/test_docker_command_utils.py | 2 +- .../prepare_provider_packages.py | 9 +- dev/refresh_images.sh | 2 +- .../installation/installing-from-pypi.rst | 4 +- images/breeze/output-build-image.svg | 129 ++++++------ images/breeze/output-build-prod-image.svg | 163 +++++++-------- images/breeze/output-commands-hash.txt | 2 +- images/breeze/output-commands.svg | 189 +++++++++-------- images/breeze/output-generate-constraints.svg | 79 ++++---- images/breeze/output-shell.svg | 113 ++++++----- images/breeze/output-start-airflow.svg | 117 ++++++----- .../output-verify-provider-packages.svg | 74 +++---- scripts/ci/docker-compose/_docker.env | 2 +- scripts/ci/docker-compose/base.yml | 2 +- scripts/ci/docker-compose/devcontainer.env | 2 +- scripts/ci/libraries/_initialization.sh | 4 +- scripts/docker/common.sh | 2 +- .../docker/install_additional_dependencies.sh | 7 +- scripts/docker/install_airflow.sh | 4 + ...ll_airflow_dependencies_from_branch_tip.sh | 2 + .../install_from_docker_context_files.sh | 43 ++-- .../in_container/run_generate_constraints.sh | 23 +-- 46 files changed, 1128 insertions(+), 1066 deletions(-) create mode 100644 dev/breeze/src/airflow_breeze/params/_common_build_params.py create mode 100644 dev/breeze/src/airflow_breeze/utils/platforms.py diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 5db6a157705f3..f95b404e7f27c 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -39,7 +39,6 @@ env: secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} # This token is WRITE one - pull_request_target type of events always have the WRITE token GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - LOGIN_TO_GITHUB_REGISTRY: "true" IMAGE_TAG_FOR_THE_BUILD: "${{ github.event.pull_request.head.sha || github.sha }}" concurrency: @@ -135,7 +134,7 @@ jobs: if [[ "${{ github.event_name }}" == 'schedule' ]]; then echo "::set-output name=cacheDirective::disabled" else - echo "::set-output name=cacheDirective::pulled" + echo "::set-output name=cacheDirective:registry" fi if [[ "$SELECTIVE_CHECKS_IMAGE_BUILD" == "true" ]]; then diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 995cef8dd3b85..4e9ab6e385381 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,6 @@ env: secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} # In builds from forks, this token is read-only. For scheduler/direct push it is WRITE one GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - LOGIN_TO_GITHUB_REGISTRY: "true" ENABLE_TEST_COVERAGE: "${{ github.event_name == 'push' }}" IMAGE_TAG_FOR_THE_BUILD: "${{ github.event.pull_request.head.sha || github.sha }}" @@ -265,7 +264,7 @@ jobs: if [[ "${{ github.event_name }}" == 'schedule' ]]; then echo "::set-output name=cacheDirective::disabled" else - echo "::set-output name=cacheDirective::pulled" + echo "::set-output name=cacheDirective::registry" fi if [[ "$SELECTIVE_CHECKS_IMAGE_BUILD" == "true" ]]; then @@ -1608,9 +1607,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Generate constraints" run: | - breeze generate-constraints --run-in-parallel --generate-constraints-mode source-providers - breeze generate-constraints --run-in-parallel --generate-constraints-mode pypi-providers - breeze generate-constraints --run-in-parallel --generate-constraints-mode no-providers + breeze generate-constraints --run-in-parallel \ + --airflow-constraints-mode constraints-source-providers + breeze generate-constraints --run-in-parallel --airflow-constraints-mode constraints-no-providers + breeze generate-constraints --run-in-parallel --airflow-constraints-mode constraints env: PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }} - name: "Set constraints branch name" diff --git a/BREEZE.rst b/BREEZE.rst index 95ff5600554fa..aca0378f71dce 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -1128,23 +1128,22 @@ all or selected python version and single constraint mode like this: .. code-block:: bash - breeze generate-constraints --generate-constraints-mode pypi-providers + breeze generate-constraints --airflow-constraints-mode constraints Constraints are generated separately for each python version and there are separate constraints modes: * 'constraints' - those are constraints generated by matching the current airflow version from sources and providers that are installed from PyPI. Those are constraints used by the users who want to - install airflow with pip. Use ``pypi-providers`` mode for that. + install airflow with pip. * "constraints-source-providers" - those are constraints generated by using providers installed from current sources. While adding new providers their dependencies might change, so this set of providers is the current set of the constraints for airflow and providers from the current main sources. - Those providers are used by CI system to keep "stable" set of constraints. Use - ``source-providers`` mode for that. + Those providers are used by CI system to keep "stable" set of constraints. * "constraints-no-providers" - those are constraints generated from only Apache Airflow, without any providers. If you want to manage airflow separately and then add providers individually, you can - use those. Use ``no-providers`` mode for that. + use those. Those are all available flags of ``generate-constraints`` command: diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 7f00045fd519f..e0e58a8bf7297 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -882,8 +882,6 @@ There are several sets of constraints we keep: providers. If you want to manage airflow separately and then add providers individually, you can use those. Those constraints are named ``constraints-no-providers-.txt``. -We also have constraints with "source-providers" but they are used i - The first two can be used as constraints file when installing Apache Airflow in a repeatable way. It can be done from the sources: @@ -891,9 +889,12 @@ from the PyPI package: .. code-block:: bash - pip install apache-airflow==2.2.5 \ + pip install apache-airflow[google,amazon,async]==2.2.5 \ --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.2.5/constraints-3.7.txt" +The last one can be used to install Airflow in "minimal" mode - i.e when bare Airflow is installed without +extras. + When you install airflow from sources (in editable mode) you should use "constraints-source-providers" instead (this accounts for the case when some providers have not yet been released and have conflicting requirements). @@ -909,14 +910,14 @@ This works also with extras - for example: .. code-block:: bash pip install ".[ssh]" \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints--source-providers-3.7.txt" + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-source-providers-3.7.txt" There are different set of fixed constraint files for different python major/minor versions and you should use the right file for the right python version. If you want to update just airflow dependencies, without paying attention to providers, you can do it using --no-providers constraint files as well. +``constraints-no-providers`` constraint files as well. .. code-block:: bash diff --git a/Dockerfile b/Dockerfile index f12f121b34c75..1bef126ff25fb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -315,12 +315,14 @@ function install_airflow_dependencies_from_branch_tip() { fi # Install latest set of dependencies using constraints. In case constraints were upgraded and there # are conflicts, this might fail, but it should be fixed in the following installation steps + set -x pip install \ "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true # make sure correct PIP version is used pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true + set +x echo echo "${COLOR_BLUE}Uninstalling just airflow. Dependencies remain. Now target airflow can be reinstalled using mostly cached dependencies${COLOR_RESET}" echo @@ -384,7 +386,7 @@ function common::get_constraints_location() { local constraints_base="https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${AIRFLOW_CONSTRAINTS_REFERENCE}" local python_version python_version="$(python --version 2>/dev/stdout | cut -d " " -f 2 | cut -d "." -f 1-2)" - AIRFLOW_CONSTRAINTS_LOCATION="${constraints_base}/${AIRFLOW_CONSTRAINTS}-${python_version}.txt" + AIRFLOW_CONSTRAINTS_LOCATION="${constraints_base}/${AIRFLOW_CONSTRAINTS_MODE}-${python_version}.txt" fi } @@ -563,40 +565,19 @@ function install_airflow_and_providers_from_docker_context_files(){ return fi - if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then - echo - echo "${COLOR_BLUE}Force re-installing airflow and providers from local files with eager upgrade${COLOR_RESET}" - echo - # force reinstall all airflow + provider package local files with eager upgrade - pip install "${pip_flags[@]}" --upgrade --upgrade-strategy eager \ - ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} \ - ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} - else - echo - echo "${COLOR_BLUE}Force re-installing airflow and providers from local files with constraints and upgrade if needed${COLOR_RESET}" - echo - if [[ ${AIRFLOW_CONSTRAINTS_LOCATION} == "/"* ]]; then - grep -ve '^apache-airflow' <"${AIRFLOW_CONSTRAINTS_LOCATION}" > /tmp/constraints.txt - else - # Remove provider packages from constraint files because they are locally prepared - curl -L "${AIRFLOW_CONSTRAINTS_LOCATION}" | grep -ve '^apache-airflow' > /tmp/constraints.txt - fi - # force reinstall airflow + provider package local files with constraints + upgrade if needed - pip install "${pip_flags[@]}" --force-reinstall \ - ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} \ - --constraint /tmp/constraints.txt - rm /tmp/constraints.txt - # make sure correct PIP version is used \ - pip install "pip==${AIRFLOW_PIP_VERSION}" - # then upgrade if needed without using constraints to account for new limits in setup.py - pip install --upgrade --upgrade-strategy only-if-needed \ - ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} - fi + echo + echo "${COLOR_BLUE}Force re-installing airflow and providers from local files with eager upgrade${COLOR_RESET}" + echo + # force reinstall all airflow + provider package local files with eager upgrade + set -x + pip install "${pip_flags[@]}" --upgrade --upgrade-strategy eager \ + ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} \ + ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} + set +x # make sure correct PIP version is left installed pip install "pip==${AIRFLOW_PIP_VERSION}" pip check - } function install_all_other_packages_from_docker_context_files() { @@ -608,10 +589,12 @@ function install_all_other_packages_from_docker_context_files() { # shellcheck disable=SC2010 reinstalling_other_packages=$(ls /docker-context-files/*.{whl,tar.gz} 2>/dev/null | \ grep -v apache_airflow | grep -v apache-airflow || true) - if [[ -n "${reinstalling_other_packages}" ]]; then \ + if [[ -n "${reinstalling_other_packages}" ]]; then + set -x pip install --force-reinstall --no-deps --no-index ${reinstalling_other_packages} # make sure correct PIP version is used pip install "pip==${AIRFLOW_PIP_VERSION}" + set -x fi } @@ -664,9 +647,11 @@ function install_airflow() { if [[ -n "${AIRFLOW_INSTALL_EDITABLE_FLAG}" ]]; then # Remove airflow and reinstall it using editable flag # We can only do it when we install airflow from sources + set -x pip uninstall apache-airflow --yes pip install ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" + set +x fi # make sure correct PIP version is used @@ -679,6 +664,7 @@ function install_airflow() { echo echo "${COLOR_BLUE}Installing all packages with constraints and upgrade if needed${COLOR_RESET}" echo + set -x pip install ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" @@ -690,6 +676,7 @@ function install_airflow() { "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" # make sure correct PIP version is used pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" echo @@ -718,18 +705,17 @@ set -euo pipefail . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" - -set -x - function install_additional_dependencies() { if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then echo echo "${COLOR_BLUE}Installing additional dependencies while upgrading to newer dependencies${COLOR_RESET}" echo + set -x pip install --upgrade --upgrade-strategy eager \ ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} # make sure correct PIP version is used pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" echo @@ -738,10 +724,12 @@ function install_additional_dependencies() { echo echo "${COLOR_BLUE}Installing additional dependencies upgrading only if needed${COLOR_RESET}" echo + set -x pip install --upgrade --upgrade-strategy only-if-needed \ ${ADDITIONAL_PYTHON_DEPS} # make sure correct PIP version is used pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" echo @@ -1185,7 +1173,7 @@ ARG AIRFLOW_EXTRAS ARG ADDITIONAL_AIRFLOW_EXTRAS="" # Allows to override constraints source ARG CONSTRAINTS_GITHUB_REPOSITORY="apache/airflow" -ARG AIRFLOW_CONSTRAINTS="constraints" +ARG AIRFLOW_CONSTRAINTS_MODE="constraints" ARG AIRFLOW_CONSTRAINTS_REFERENCE="" ARG AIRFLOW_CONSTRAINTS_LOCATION="" ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main" @@ -1275,7 +1263,7 @@ ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ AIRFLOW_BRANCH=${AIRFLOW_BRANCH} \ AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}${ADDITIONAL_AIRFLOW_EXTRAS:+,}${ADDITIONAL_AIRFLOW_EXTRAS} \ CONSTRAINTS_GITHUB_REPOSITORY=${CONSTRAINTS_GITHUB_REPOSITORY} \ - AIRFLOW_CONSTRAINTS=${AIRFLOW_CONSTRAINTS} \ + AIRFLOW_CONSTRAINTS_MODE=${AIRFLOW_CONSTRAINTS_MODE} \ AIRFLOW_CONSTRAINTS_REFERENCE=${AIRFLOW_CONSTRAINTS_REFERENCE} \ AIRFLOW_CONSTRAINTS_LOCATION=${AIRFLOW_CONSTRAINTS_LOCATION} \ DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH} \ diff --git a/Dockerfile.ci b/Dockerfile.ci index b75c3d1a980c1..c17b348aed71e 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -275,12 +275,14 @@ function install_airflow_dependencies_from_branch_tip() { fi # Install latest set of dependencies using constraints. In case constraints were upgraded and there # are conflicts, this might fail, but it should be fixed in the following installation steps + set -x pip install \ "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true # make sure correct PIP version is used pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true + set +x echo echo "${COLOR_BLUE}Uninstalling just airflow. Dependencies remain. Now target airflow can be reinstalled using mostly cached dependencies${COLOR_RESET}" echo @@ -344,7 +346,7 @@ function common::get_constraints_location() { local constraints_base="https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${AIRFLOW_CONSTRAINTS_REFERENCE}" local python_version python_version="$(python --version 2>/dev/stdout | cut -d " " -f 2 | cut -d "." -f 1-2)" - AIRFLOW_CONSTRAINTS_LOCATION="${constraints_base}/${AIRFLOW_CONSTRAINTS}-${python_version}.txt" + AIRFLOW_CONSTRAINTS_LOCATION="${constraints_base}/${AIRFLOW_CONSTRAINTS_MODE}-${python_version}.txt" fi } @@ -514,9 +516,11 @@ function install_airflow() { if [[ -n "${AIRFLOW_INSTALL_EDITABLE_FLAG}" ]]; then # Remove airflow and reinstall it using editable flag # We can only do it when we install airflow from sources + set -x pip uninstall apache-airflow --yes pip install ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" + set +x fi # make sure correct PIP version is used @@ -529,6 +533,7 @@ function install_airflow() { echo echo "${COLOR_BLUE}Installing all packages with constraints and upgrade if needed${COLOR_RESET}" echo + set -x pip install ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" @@ -540,6 +545,7 @@ function install_airflow() { "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" # make sure correct PIP version is used pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" echo @@ -568,18 +574,17 @@ set -euo pipefail . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" - -set -x - function install_additional_dependencies() { if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then echo echo "${COLOR_BLUE}Installing additional dependencies while upgrading to newer dependencies${COLOR_RESET}" echo + set -x pip install --upgrade --upgrade-strategy eager \ ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} # make sure correct PIP version is used pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" echo @@ -588,10 +593,12 @@ function install_additional_dependencies() { echo echo "${COLOR_BLUE}Installing additional dependencies upgrading only if needed${COLOR_RESET}" echo + set -x pip install --upgrade --upgrade-strategy only-if-needed \ ${ADDITIONAL_PYTHON_DEPS} # make sure correct PIP version is used pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" echo @@ -1178,7 +1185,7 @@ ARG AIRFLOW_EXTRAS="all" ARG ADDITIONAL_AIRFLOW_EXTRAS="" # Allows to override constraints source ARG CONSTRAINTS_GITHUB_REPOSITORY="apache/airflow" -ARG AIRFLOW_CONSTRAINTS="constraints" +ARG AIRFLOW_CONSTRAINTS_MODE="constraints-source-providers" ARG AIRFLOW_CONSTRAINTS_REFERENCE="" ARG AIRFLOW_CONSTRAINTS_LOCATION="" ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main" @@ -1207,7 +1214,7 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\ AIRFLOW_BRANCH=${AIRFLOW_BRANCH} \ AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}${ADDITIONAL_AIRFLOW_EXTRAS:+,}${ADDITIONAL_AIRFLOW_EXTRAS} \ CONSTRAINTS_GITHUB_REPOSITORY=${CONSTRAINTS_GITHUB_REPOSITORY} \ - AIRFLOW_CONSTRAINTS=${AIRFLOW_CONSTRAINTS} \ + AIRFLOW_CONSTRAINTS_MODE=${AIRFLOW_CONSTRAINTS_MODE} \ AIRFLOW_CONSTRAINTS_REFERENCE=${AIRFLOW_CONSTRAINTS_REFERENCE} \ AIRFLOW_CONSTRAINTS_LOCATION=${AIRFLOW_CONSTRAINTS_LOCATION} \ DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH} \ diff --git a/IMAGES.rst b/IMAGES.rst index 4513c039dba98..4f3051ac622f4 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -188,8 +188,8 @@ Dockerfile image= and scripts further rebuilds with local build cache will be co You can also disable build cache altogether. This is the strategy used by the scheduled builds in CI - they will always rebuild all the images from scratch. -You can change the strategy by providing one of the ``--build-cache-local``, ``--build-cache-pulled`` or -even ``--build-cache-disabled`` flags when you run Breeze commands. For example: +You can change the strategy by providing one of the ``--build-cache`` flags: ``registry`` (default), ``local``, +or ``disabled`` flags when you run Breeze commands. For example: .. code-block:: bash @@ -200,9 +200,9 @@ time you run it). .. code-block:: bash - breeze build-prod-image --python 3.7 --docker-cache pulled + breeze build-prod-image --python 3.7 --docker-cache registry -Will build the production image with pulled images as cache. +Will build the production image with cache used from registry. .. code-block:: bash @@ -211,8 +211,14 @@ Will build the production image with pulled images as cache. Will build the production image from the scratch. -You can also turn local docker caching by setting ``DOCKER_CACHE`` variable to "local", "pulled", -"disabled" and exporting it. +You can also turn local docker caching by setting ``DOCKER_CACHE`` variable to ``local``, ``registry``, +``disabled`` and exporting it. + +.. code-block:: bash + + export DOCKER_CACHE="registry" + +or .. code-block:: bash diff --git a/INSTALL b/INSTALL index bc66c9f24e6ee..ba2f5500475ee 100644 --- a/INSTALL +++ b/INSTALL @@ -56,7 +56,7 @@ pip install .[google,amazon] \ --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-3.7.txt" -You can upgrade just airflow, without paying attention to provider's dependencies by using 'no-providers' +You can upgrade just airflow, without paying attention to provider's dependencies by using 'constraints-no-providers' constraint files. This allows you to keep installed provider packages. pip install . --upgrade \ diff --git a/dev/REFRESHING_CI_CACHE.md b/dev/REFRESHING_CI_CACHE.md index 485b12ac13b8e..85194845dc9b2 100644 --- a/dev/REFRESHING_CI_CACHE.md +++ b/dev/REFRESHING_CI_CACHE.md @@ -51,10 +51,10 @@ manual refresh might be needed. # Manually generating constraint files ```bash -breeze build-image --build-multiple-images --upgrade-to-newer-dependencies -breeze generate-constraints --generate-constraints-mode pypi-providers --run-in-parallel -breeze generate-constraints --generate-constraints-mode source-providers --run-in-parallel -breeze generate-constraints --generate-constraints-mode no-providers --run-in-parallel +breeze build-image --build-multiple-images --upgrade-to-newer-dependencies --answer yes +breeze generate-constraints --airflow-constraints-mode constraints --run-in-parallel --answer yes +breeze generate-constraints --airflow-constraints-mode constraints-source-providers --run-in-parallel --answer yes +breeze generate-constraints --airflow-constraints-mode constraints-no-providers --run-in-parallel --answer yes AIRFLOW_SOURCES=$(pwd) ``` diff --git a/dev/breeze/doc/adr/0005-preventing-using-contributed-code-when-building-images.md b/dev/breeze/doc/adr/0005-preventing-using-contributed-code-when-building-images.md index ce1969471bcaf..b9d811db1266c 100644 --- a/dev/breeze/doc/adr/0005-preventing-using-contributed-code-when-building-images.md +++ b/dev/breeze/doc/adr/0005-preventing-using-contributed-code-when-building-images.md @@ -61,7 +61,7 @@ or another developer's PR. This requires those prerequisites: * the images need to be built in a workflow that has "write" access to store the images after they are - built, so that the images can then be "pulled" by the test jobs rather than rebuilt + built, so that the images can then be pulled by the test jobs rather than rebuilt * the process to build the images need to be secured from malicious users that would like to inject a code in the build process to make bad use of the "write" access - for example to push the code diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index 6ec528406b057..0a53c8b84bb91 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -34,6 +34,7 @@ option_additional_runtime_apt_command, option_additional_runtime_apt_deps, option_additional_runtime_apt_env, + option_airflow_constraints_mode_ci, option_answer, option_build_multiple_images, option_debian_version, @@ -49,7 +50,6 @@ option_image_name, option_image_tag, option_install_providers_from_sources, - option_login_to_github_registry, option_parallelism, option_platform, option_prepare_buildx_cache, @@ -68,12 +68,12 @@ from airflow_breeze.utils.confirm import STANDARD_TIMEOUT, Answer, user_confirm from airflow_breeze.utils.console import get_console from airflow_breeze.utils.docker_command_utils import ( + build_cache, check_docker_compose_version, check_docker_resources, check_docker_version, - construct_docker_build_command, - construct_empty_docker_build_command, - tag_and_push_image, + prepare_docker_build_command, + prepare_empty_docker_build_command, ) from airflow_breeze.utils.mark_image_as_refreshed import mark_image_as_refreshed from airflow_breeze.utils.md5_build_check import md5sum_check_if_build_is_needed @@ -81,7 +81,7 @@ from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, BUILD_CACHE_DIR from airflow_breeze.utils.pulll_image import run_pull_image, run_pull_in_parallel from airflow_breeze.utils.python_versions import get_python_version_list -from airflow_breeze.utils.registry import login_to_docker_registry +from airflow_breeze.utils.registry import login_to_github_docker_registry from airflow_breeze.utils.run_tests import verify_an_image from airflow_breeze.utils.run_utils import ( filter_out_none, @@ -124,6 +124,7 @@ "name": "Advanced options (for power users)", "options": [ "--install-providers-from-sources", + "--airflow-constraints-mode", "--additional-python-deps", "--runtime-apt-deps", "--runtime-apt-command", @@ -198,7 +199,6 @@ @option_debian_version @option_github_token @option_github_username -@option_login_to_github_registry @option_docker_cache @option_image_tag @option_prepare_buildx_cache @@ -219,6 +219,7 @@ @option_runtime_apt_command @option_runtime_apt_deps @option_force_build +@option_airflow_constraints_mode_ci def build_image( verbose: bool, dry_run: bool, @@ -433,7 +434,11 @@ def build_ci_image(verbose: bool, dry_run: bool, ci_image_params: BuildCiParams) :param dry_run: do not execute "write" commands - just print what would happen :param ci_image_params: CI image parameters """ - + if not ci_image_params.push_image and ci_image_params.is_multi_platform(): + get_console().print( + "\n[red]You cannot use multi-platform build without using --push-image flag![/]\n" + ) + return 1, "Error: building multi-platform image without --push-image." fix_group_permissions(verbose=verbose) if verbose or dry_run: get_console().print( @@ -443,27 +448,18 @@ def build_ci_image(verbose: bool, dry_run: bool, ci_image_params: BuildCiParams) if not ci_image_params.force_build and not ci_image_params.upgrade_to_newer_dependencies: if not should_we_run_the_build(build_ci_params=ci_image_params, verbose=verbose): return 0, f"Image build: {ci_image_params.python}" - run_command( - ["docker", "rmi", "--no-prune", "--force", ci_image_params.airflow_image_name], - verbose=verbose, - dry_run=dry_run, - cwd=AIRFLOW_SOURCES_ROOT, - text=True, - check=False, - ) - if ci_image_params.prepare_buildx_cache: - login_to_docker_registry(ci_image_params, dry_run=dry_run) - cmd = construct_docker_build_command( + if ci_image_params.prepare_buildx_cache or ci_image_params.push_image: + login_to_github_docker_registry(image_params=ci_image_params, dry_run=dry_run, verbose=verbose) + cmd = prepare_docker_build_command( image_params=ci_image_params, verbose=verbose, - production_image=False, ) if ci_image_params.empty_image: env = os.environ.copy() env['DOCKER_BUILDKIT'] = "1" get_console().print(f"\n[info]Building empty CI Image for Python {ci_image_params.python}\n") - cmd = construct_empty_docker_build_command(image_params=ci_image_params) - build_result = run_command( + cmd = prepare_empty_docker_build_command(image_params=ci_image_params) + build_command_result = run_command( cmd, input="FROM scratch\n", verbose=verbose, @@ -474,23 +470,28 @@ def build_ci_image(verbose: bool, dry_run: bool, ci_image_params: BuildCiParams) ) else: get_console().print(f"\n[info]Building CI Image for Python {ci_image_params.python}\n") - build_result = run_command( + build_command_result = run_command( cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, text=True, check=False ) - if not dry_run: - if build_result.returncode == 0: - mark_image_as_refreshed(ci_image_params) + if build_command_result.returncode == 0: + if ci_image_params.prepare_buildx_cache: + build_command_result = build_cache( + image_params=ci_image_params, dry_run=dry_run, verbose=verbose + ) + + if not ci_image_params.prepare_buildx_cache: + if not dry_run: + if build_command_result.returncode == 0: + mark_image_as_refreshed(ci_image_params) + else: + get_console().print("[error]Error when building image![/]") + return ( + build_command_result.returncode, + f"Image build: {ci_image_params.python}", + ) else: - get_console().print("[error]Error when building image![/]") - return ( - build_result.returncode, - f"Image build: {ci_image_params.python}", - ) - else: - get_console().print("[info]Not updating build cache because we are in `dry_run` mode.[/]") - if ci_image_params.push_image: - return tag_and_push_image(image_params=ci_image_params, dry_run=dry_run, verbose=verbose) - return build_result.returncode, f"Image build: {ci_image_params.python}" + get_console().print("[info]Not updating build cache because we are in `dry_run` mode.[/]") + return build_command_result.returncode, f"Image build: {ci_image_params.python}" def build_ci_image_in_parallel( @@ -525,10 +526,8 @@ def rebuild_ci_image_if_needed( ci_image_params = BuildCiParams(python=build_params.python, upgrade_to_newer_dependencies=False) if build_ci_image_check_cache.exists(): if verbose: - get_console().print(f'[info]{build_params.the_image_type} image already built locally.[/]') + get_console().print(f'[info]{build_params.image_type} image already built locally.[/]') else: - get_console().print( - f'[warning]{build_params.the_image_type} image not built locally. Forcing build.[/]' - ) + get_console().print(f'[warning]{build_params.image_type} image not built locally. Forcing build.[/]') ci_image_params.force_build = True build_ci_image(verbose, dry_run=dry_run, ci_image_params=ci_image_params) diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py index 23e68e884a76a..a28bad62a34a0 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py @@ -33,6 +33,7 @@ option_additional_runtime_apt_command, option_additional_runtime_apt_deps, option_additional_runtime_apt_env, + option_airflow_constraints_mode_prod, option_answer, option_build_multiple_images, option_debian_version, @@ -47,7 +48,6 @@ option_image_name, option_image_tag, option_install_providers_from_sources, - option_login_to_github_registry, option_parallelism, option_platform, option_prepare_buildx_cache, @@ -66,14 +66,14 @@ from airflow_breeze.utils.console import get_console from airflow_breeze.utils.custom_param_types import BetterChoice from airflow_breeze.utils.docker_command_utils import ( - construct_docker_build_command, - construct_empty_docker_build_command, - tag_and_push_image, + build_cache, + prepare_docker_build_command, + prepare_empty_docker_build_command, ) from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, DOCKER_CONTEXT_DIR from airflow_breeze.utils.pulll_image import run_pull_image, run_pull_in_parallel from airflow_breeze.utils.python_versions import get_python_version_list -from airflow_breeze.utils.registry import login_to_docker_registry +from airflow_breeze.utils.registry import login_to_github_docker_registry from airflow_breeze.utils.run_tests import verify_an_image from airflow_breeze.utils.run_utils import filter_out_none, fix_group_permissions, run_command @@ -109,6 +109,8 @@ "name": "Options for customizing images", "options": [ "--install-providers-from-sources", + "--airflow-extras", + "--airflow-constraints-mode", "--additional-python-deps", "--additional-extras", "--additional-runtime-apt-deps", @@ -117,7 +119,6 @@ "--additional-dev-apt-deps", "--additional-dev-apt-env", "--additional-dev-apt-command", - "--extras", "--runtime-apt-deps", "--runtime-apt-command", "--dev-apt-deps", @@ -198,12 +199,12 @@ @option_github_repository @option_github_token @option_github_username -@option_login_to_github_registry @option_docker_cache @option_image_tag @option_prepare_buildx_cache @option_push_image @option_empty_image +@option_airflow_constraints_mode_prod @click.option( '--installation-method', help="Install Airflow from: sources or PyPI.", @@ -227,7 +228,10 @@ is_flag=True, ) @click.option( - '--extras', default=",".join(DEFAULT_EXTRAS), show_default=True, help="Extras to install by default." + '--airflow-extras', + default=",".join(DEFAULT_EXTRAS), + show_default=True, + help="Extras to install by default.", ) @click.option('--disable-mysql-client-installation', help="Do not install MySQL client.", is_flag=True) @click.option('--disable-mssql-client-installation', help="Do not install MsSQl client.", is_flag=True) @@ -467,6 +471,11 @@ def build_production_image( :param dry_run: do not execute "write" commands - just print what would happen :param prod_image_params: PROD image parameters """ + if not prod_image_params.push_image and prod_image_params.is_multi_platform(): + get_console().print( + "\n[red]You cannot use multi-platform build without using --push-image flag![/]\n" + ) + return 1, "Error: building multi-platform image without --push-image." fix_group_permissions(verbose=verbose) if verbose or dry_run: get_console().print( @@ -476,22 +485,14 @@ def build_production_image( if prod_image_params.cleanup_context: clean_docker_context_files(verbose=verbose, dry_run=dry_run) check_docker_context_files(prod_image_params.install_packages_from_context) - if prod_image_params.prepare_buildx_cache: - login_to_docker_registry(prod_image_params, dry_run=dry_run) - run_command( - ["docker", "rmi", "--no-prune", "--force", prod_image_params.airflow_image_name], - verbose=verbose, - dry_run=dry_run, - cwd=AIRFLOW_SOURCES_ROOT, - text=True, - check=False, - ) + if prod_image_params.prepare_buildx_cache or prod_image_params.push_image: + login_to_github_docker_registry(image_params=prod_image_params, dry_run=dry_run, verbose=verbose) get_console().print(f"\n[info]Building PROD Image for Python {prod_image_params.python}\n") if prod_image_params.empty_image: env = os.environ.copy() env['DOCKER_BUILDKIT'] = "1" get_console().print(f"\n[info]Building empty PROD Image for Python {prod_image_params.python}\n") - cmd = construct_empty_docker_build_command(image_params=prod_image_params) + cmd = prepare_empty_docker_build_command(image_params=prod_image_params) build_command_result = run_command( cmd, input="FROM scratch\n", @@ -503,15 +504,17 @@ def build_production_image( env=env, ) else: - cmd = construct_docker_build_command( + cmd = prepare_docker_build_command( image_params=prod_image_params, verbose=verbose, - production_image=True, ) build_command_result = run_command( cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, check=False, text=True ) - if build_command_result.returncode == 0: - if prod_image_params.push_image: - return tag_and_push_image(image_params=prod_image_params, dry_run=dry_run, verbose=verbose) + if build_command_result.returncode == 0: + if prod_image_params.prepare_buildx_cache: + build_command_result = build_cache( + image_params=prod_image_params, dry_run=dry_run, verbose=verbose + ) + return build_command_result.returncode, f"Image build: {prod_image_params.python}" diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 6c6f9c7a77c2f..2b98e9da67426 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -27,18 +27,17 @@ from airflow_breeze.commands.main_command import main from airflow_breeze.global_constants import ( - ALLOWED_GENERATE_CONSTRAINTS_MODES, ALLOWED_PLATFORMS, CURRENT_PYTHON_MAJOR_MINOR_VERSIONS, DEFAULT_PYTHON_MAJOR_MINOR_VERSION, MOUNT_ALL, - MOUNT_NONE, MOUNT_SELECTED, MULTI_PLATFORM, ) from airflow_breeze.params.shell_params import ShellParams from airflow_breeze.utils.common_options import ( argument_packages, + option_airflow_constraints_mode_ci, option_airflow_constraints_reference, option_airflow_extras, option_answer, @@ -113,7 +112,7 @@ "options": [ "--image-tag", "--python", - "--generate-constraints-mode", + "--airflow-constraints-mode", "--debug", ], }, @@ -348,7 +347,7 @@ def prepare_provider_packages( def run_generate_constraints( - shell_params: ShellParams, dry_run: bool, verbose: bool, generate_constraints_mode: str, debug: bool + shell_params: ShellParams, dry_run: bool, verbose: bool, debug: bool ) -> Tuple[int, str]: cmd_to_run = [ "/opt/airflow/scripts/in_container/run_generate_constraints.sh", @@ -362,14 +361,13 @@ def run_generate_constraints( ) return ( generate_constraints_result.returncode, - f"Generate constraints Python {shell_params.python}:{generate_constraints_mode}", + f"Generate constraints Python {shell_params.python}:{shell_params.airflow_constraints_mode}", ) def run_generate_constraints_in_parallel( shell_params_list: List[ShellParams], python_version_list: List[str], - generate_constraints_mode: str, parallelism: int, dry_run: bool, verbose: bool, @@ -383,7 +381,7 @@ def run_generate_constraints_in_parallel( results = [ pool.apply_async( run_generate_constraints, - args=(shell_param, dry_run, verbose, generate_constraints_mode, False), + args=(shell_param, dry_run, verbose, False), ) for shell_param in shell_params_list ] @@ -395,13 +393,6 @@ def run_generate_constraints_in_parallel( name='generate-constraints', help="Generates pinned constraint files with all extras from setup.py in parallel.", ) -@click.option( - '--generate-constraints-mode', - type=BetterChoice(ALLOWED_GENERATE_CONSTRAINTS_MODES), - default=ALLOWED_GENERATE_CONSTRAINTS_MODES[0], - show_default=True, - help='Mode of generating constraints', -) @option_verbose @option_dry_run @option_python @@ -412,6 +403,7 @@ def run_generate_constraints_in_parallel( @option_image_tag @option_answer @option_debug_release_management +@option_airflow_constraints_mode_ci def generate_constraints( verbose: bool, dry_run: bool, @@ -423,7 +415,7 @@ def generate_constraints( image_tag: str, answer: Optional[str], debug: bool, - generate_constraints_mode: str, + airflow_constraints_mode: str, ): if debug and run_in_parallel: get_console().print("\n[error]Cannot run --debug and --run-in-parallel at the same time[/]\n") @@ -457,7 +449,11 @@ def generate_constraints( python_version_list = get_python_version_list(python_versions) shell_params_list = [ ShellParams( - image_tag=image_tag, python=python, github_repository=github_repository, answer=answer + image_tag=image_tag, + python=python, + github_repository=github_repository, + airflow_constraints_mode=airflow_constraints_mode, + answer=answer, ) for python in python_version_list ] @@ -467,7 +463,6 @@ def generate_constraints( dry_run=dry_run, verbose=verbose, python_version_list=python_version_list, - generate_constraints_mode=generate_constraints_mode, ) else: shell_params = ShellParams( @@ -476,12 +471,12 @@ def generate_constraints( github_repository=github_repository, answer=answer, skip_environment_initialization=True, + airflow_constraints_mode=airflow_constraints_mode, ) return_code, info = run_generate_constraints( shell_params=shell_params, dry_run=dry_run, verbose=verbose, - generate_constraints_mode=generate_constraints_mode, debug=debug, ) if return_code != 0: @@ -515,7 +510,7 @@ def verify_provider_packages( ): shell_params = ShellParams( verbose=verbose, - mount_sources=MOUNT_NONE if use_airflow_version is not None else MOUNT_SELECTED, + mount_sources=MOUNT_SELECTED, github_repository=github_repository, python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION, use_airflow_version=use_airflow_version, diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 6b72e97014e2b..0f0197c0cd874 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -55,7 +55,8 @@ ALLOWED_HELM_VERSIONS = ['v3.6.3'] ALLOWED_EXECUTORS = ['KubernetesExecutor', 'CeleryExecutor', 'LocalExecutor', 'CeleryKubernetesExecutor'] ALLOWED_KIND_OPERATIONS = ['start', 'stop', 'restart', 'status', 'deploy', 'test', 'shell', 'k9s'] -ALLOWED_GENERATE_CONSTRAINTS_MODES = ['source-providers', 'pypi-providers', 'no-providers'] +ALLOWED_CONSTRAINTS_MODES_CI = ['constraints-source-providers', 'constraints', 'constraints-no-providers'] +ALLOWED_CONSTRAINTS_MODES_PROD = ['constraints', 'constraints-no-providers', 'constraints-source-providers'] MOUNT_SELECTED = "selected" MOUNT_ALL = "all" @@ -84,7 +85,7 @@ ALLOWED_INSTALLATION_PACKAGE_FORMATS = ['wheel', 'sdist'] ALLOWED_INSTALLATION_METHODS = ['.', 'apache-airflow'] ALLOWED_DEBIAN_VERSIONS = ['bullseye', 'buster'] -ALLOWED_BUILD_CACHE = ["pulled", "local", "disabled"] +ALLOWED_BUILD_CACHE = ["registry", "local", "disabled"] MULTI_PLATFORM = "linux/amd64,linux/arm64" ALLOWED_PLATFORMS = ["linux/amd64", "linux/arm64", MULTI_PLATFORM] ALLOWED_USE_AIRFLOW_VERSIONS = ['none', 'wheel', 'sdist'] diff --git a/dev/breeze/src/airflow_breeze/params/_common_build_params.py b/dev/breeze/src/airflow_breeze/params/_common_build_params.py new file mode 100644 index 0000000000000..9b510a06f8e00 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/params/_common_build_params.py @@ -0,0 +1,166 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import sys +from dataclasses import dataclass +from datetime import datetime +from typing import List, Optional + +from airflow_breeze.branch_defaults import AIRFLOW_BRANCH, DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH +from airflow_breeze.utils.console import get_console +from airflow_breeze.utils.platforms import get_real_platform + + +@dataclass +class _CommonBuildParams: + """ + Common build parameters. Those parameters are common parameters for CI And PROD build. + """ + + additional_airflow_extras: str = "" + additional_dev_apt_command: str = "" + additional_dev_apt_deps: str = "" + additional_dev_apt_env: str = "" + additional_python_deps: str = "" + additional_runtime_apt_command: str = "" + additional_runtime_apt_deps: str = "" + additional_runtime_apt_env: str = "" + airflow_branch: str = AIRFLOW_BRANCH + airflow_constraints_location: str = "" + airflow_constraints_reference: str = "constraints-main" + answer: Optional[str] = None + build_id: int = 0 + constraints_github_repository: str = "apache/airflow" + debian_version: str = "bullseye" + default_constraints_branch = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH + dev_apt_command: str = "" + dev_apt_deps: str = "" + docker_cache: str = "registry" + empty_image: bool = False + github_actions: str = os.environ.get('GITHUB_ACTIONS', "false") + github_repository: str = "apache/airflow" + github_token: str = os.environ.get('GITHUB_TOKEN', "") + github_username: str = "" + image_tag: Optional[str] = None + install_providers_from_sources: bool = False + platform: str = f"linux/{os.uname().machine}" + prepare_buildx_cache: bool = False + push_image: bool = False + python: str = "3.7" + runtime_apt_command: str = "" + runtime_apt_deps: str = "" + upgrade_to_newer_dependencies: bool = False + + @property + def airflow_version(self): + raise NotImplementedError() + + @property + def image_type(self) -> str: + raise NotImplementedError() + + @property + def airflow_pre_cached_pip_packages(self): + raise NotImplementedError() + + @property + def airflow_base_image_name(self): + image = f'ghcr.io/{self.github_repository.lower()}' + return image + + @property + def airflow_image_name(self): + """Construct image link""" + image = ( + f'{self.airflow_base_image_name}/{self.airflow_branch}/' + f'{self.image_type.lower()}/python{self.python}' + ) + return image + + @property + def extra_docker_build_flags(self) -> List[str]: + raise NotImplementedError() + + @property + def docker_cache_directive(self) -> List[str]: + docker_cache_directive = [] + if self.docker_cache == "registry": + for platform in self.platforms: + docker_cache_directive.append(f"--cache-from={self.get_cache(platform)}") + elif self.docker_cache == "disabled": + docker_cache_directive.append("--no-cache") + else: + docker_cache_directive = [] + return docker_cache_directive + + @property + def python_base_image(self): + """Construct Python Base Image""" + # ghcr.io/apache/airflow/main/python:3.8-slim-bullseye + return f'python:{self.python}-slim-{self.debian_version}' + + @property + def airflow_image_repository(self): + return f'https://github.com/{self.github_repository}' + + @property + def airflow_image_date_created(self): + now = datetime.now() + return now.strftime("%Y-%m-%dT%H:%M:%SZ") + + @property + def airflow_image_readme_url(self): + return "https://raw.githubusercontent.com/apache/airflow/main/docs/docker-stack/README.md" + + @property + def airflow_image_name_with_tag(self): + """Construct image link""" + image = ( + f'{self.airflow_base_image_name}/{self.airflow_branch}/' + f'{self.image_type.lower()}/python{self.python}' + ) + return image if self.image_tag is None else image + f":{self.image_tag}" + + def get_cache(self, single_platform: str) -> str: + if "," in single_platform: + get_console().print( + "[error]Cache can only be retrieved for single platform and you " + f"tried for {single_platform}[/]" + ) + sys.exit(1) + return f"{self.airflow_image_name}:cache-{get_real_platform(single_platform)}" + + def is_multi_platform(self) -> bool: + return "," in self.platform + + @property + def platforms(self) -> List[str]: + return self.platform.split(",") + + @property + def required_image_args(self) -> List[str]: + raise NotImplementedError() + + @property + def optional_image_args(self) -> List[str]: + raise NotImplementedError() + + def __post_init__(self): + if self.prepare_buildx_cache: + get_console().print("[info]Forcing --push-image since we are preparing buildx cache[/]") + self.push_image = True diff --git a/dev/breeze/src/airflow_breeze/params/build_ci_params.py b/dev/breeze/src/airflow_breeze/params/build_ci_params.py index ce244162fcd56..a22a96a34b0ea 100644 --- a/dev/breeze/src/airflow_breeze/params/build_ci_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_ci_params.py @@ -14,126 +14,34 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - -import os from dataclasses import dataclass -from datetime import datetime from pathlib import Path -from typing import List, Optional +from typing import List -from airflow_breeze.branch_defaults import AIRFLOW_BRANCH, DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH from airflow_breeze.global_constants import get_airflow_version +from airflow_breeze.params._common_build_params import _CommonBuildParams +from airflow_breeze.utils.console import get_console from airflow_breeze.utils.path_utils import BUILD_CACHE_DIR @dataclass -class BuildCiParams: +class BuildCiParams(_CommonBuildParams): """ CI build parameters. Those parameters are used to determine command issued to build CI image. """ - upgrade_to_newer_dependencies: bool = False - python: str = "3.7" - airflow_branch: str = AIRFLOW_BRANCH - build_id: int = 0 - docker_cache: str = "pulled" + airflow_constraints_mode: str = "constraints-source-providers" airflow_extras: str = "devel_ci" - install_providers_from_sources: bool = True - additional_airflow_extras: str = "" - additional_python_deps: str = "" - github_repository: str = "apache/airflow" - constraints_github_repository: str = "apache/airflow" - default_constraints_branch: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH - airflow_constraints: str = "constraints-source-providers" - airflow_constraints_reference: Optional[str] = "constraints-main" - airflow_constraints_location: Optional[str] = "" - airflow_pre_cached_pip_packages: str = "true" - login_to_github_registry: str = "false" - github_username: str = "" - dev_apt_command: str = "" - dev_apt_deps: str = "" - image_tag: Optional[str] = None - github_token: str = os.environ.get('GITHUB_TOKEN', "") - github_actions: str = os.environ.get('GITHUB_ACTIONS', "false") - additional_dev_apt_command: str = "" - additional_dev_apt_deps: str = "" - additional_dev_apt_env: str = "" - runtime_apt_command: str = "" - runtime_apt_deps: str = "" - additional_runtime_apt_command: str = "" - additional_runtime_apt_deps: str = "" - additional_runtime_apt_env: str = "" - platform: str = f"linux/{os.uname().machine}" - debian_version: str = "bullseye" - prepare_buildx_cache: bool = False - push_image: bool = False - empty_image: bool = False + airflow_pre_cached_pip_packages: bool = True force_build: bool = False - skip_rebuild_check: bool = False - answer: Optional[str] = None - - @property - def the_image_type(self) -> str: - return 'CI' - - @property - def airflow_base_image_name(self): - image = f'ghcr.io/{self.github_repository.lower()}' - return image - - @property - def airflow_image_name(self): - """Construct CI image link""" - image = f'{self.airflow_base_image_name}/{self.airflow_branch}/ci/python{self.python}' - return image - - @property - def airflow_image_name_with_tag(self): - """Construct CI image link""" - image = f'{self.airflow_base_image_name}/{self.airflow_branch}/ci/python{self.python}' - return image if self.image_tag is None else image + f":{self.image_tag}" - - @property - def airflow_image_repository(self): - return f'https://github.com/{self.github_repository}' - - @property - def python_base_image(self): - """Construct Python Base Image""" - return f'python:{self.python}-slim-{self.debian_version}' - - @property - def airflow_ci_local_manifest_image(self): - """Construct CI Local Manifest Image""" - return f'local-airflow-ci-manifest/{self.airflow_branch}/python{self.python}' - - @property - def airflow_ci_remote_manifest_image(self): - """Construct CI Remote Manifest Image""" - return f'{self.airflow_image_name}/{self.airflow_branch}/ci-manifest//python:{self.python}' - - @property - def airflow_image_date_created(self): - now = datetime.now() - return now.strftime("%Y-%m-%dT%H:%M:%SZ") @property def airflow_version(self): return get_airflow_version() @property - def docker_cache_directive(self) -> List[str]: - docker_cache_directive = [] - - if self.docker_cache == "pulled": - docker_cache_directive.append(f"--cache-from={self.airflow_image_name}") - elif self.docker_cache == "disabled": - docker_cache_directive.append("--no-cache") - else: - docker_cache_directive = [] - if self.prepare_buildx_cache: - docker_cache_directive.extend(["--cache-to=type=inline,mode=max", "--push"]) - return docker_cache_directive + def image_type(self) -> str: + return 'CI' @property def extra_docker_build_flags(self) -> List[str]: @@ -148,31 +56,41 @@ def extra_docker_build_flags(self) -> List[str]: def md5sum_cache_dir(self) -> Path: return Path(BUILD_CACHE_DIR, self.airflow_branch, self.python, "CI") - required_image_args = [ - "python_base_image", - "airflow_version", - "airflow_branch", - "airflow_extras", - "airflow_pre_cached_pip_packages", - "additional_airflow_extras", - "additional_python_deps", - "additional_dev_apt_command", - "additional_dev_apt_deps", - "additional_dev_apt_env", - "additional_runtime_apt_command", - "additional_runtime_apt_deps", - "additional_runtime_apt_env", - "upgrade_to_newer_dependencies", - "constraints_github_repository", - "airflow_constraints_reference", - "airflow_constraints", - "airflow_image_repository", - "airflow_image_date_created", - "build_id", - ] - optional_image_args = [ - "dev_apt_command", - "dev_apt_deps", - "runtime_apt_command", - "runtime_apt_deps", - ] + @property + def required_image_args(self) -> List[str]: + return [ + "additional_airflow_extras", + "additional_dev_apt_command", + "additional_dev_apt_deps", + "additional_dev_apt_env", + "additional_python_deps", + "additional_runtime_apt_command", + "additional_runtime_apt_deps", + "additional_runtime_apt_env", + "airflow_branch", + "airflow_constraints_mode", + "airflow_constraints_reference", + "airflow_extras", + "airflow_image_date_created", + "airflow_image_repository", + "airflow_pre_cached_pip_packages", + "airflow_version", + "build_id", + "constraints_github_repository", + "python_base_image", + "upgrade_to_newer_dependencies", + ] + + @property + def optional_image_args(self) -> List[str]: + return [ + "dev_apt_command", + "dev_apt_deps", + "runtime_apt_command", + "runtime_apt_deps", + ] + + def __post_init__(self): + if self.prepare_buildx_cache: + get_console().print("[info]Forcing --push-image since we are preparing buildx cache[/]") + self.push_image = True diff --git a/dev/breeze/src/airflow_breeze/params/build_prod_params.py b/dev/breeze/src/airflow_breeze/params/build_prod_params.py index 9fe08e35fa225..0cb6e84bd1b03 100644 --- a/dev/breeze/src/airflow_breeze/params/build_prod_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_prod_params.py @@ -16,14 +16,12 @@ # under the License. import json -import os import re import sys from dataclasses import dataclass -from datetime import datetime -from typing import List, Optional +from typing import List -from airflow_breeze.branch_defaults import AIRFLOW_BRANCH, DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH +from airflow_breeze.branch_defaults import AIRFLOW_BRANCH from airflow_breeze.global_constants import ( AIRFLOW_SOURCES_FROM, AIRFLOW_SOURCES_TO, @@ -32,61 +30,28 @@ get_airflow_extras, get_airflow_version, ) +from airflow_breeze.params._common_build_params import _CommonBuildParams from airflow_breeze.utils.console import get_console @dataclass -class BuildProdParams: +class BuildProdParams(_CommonBuildParams): """ PROD build parameters. Those parameters are used to determine command issued to build PROD image. """ - docker_cache: str = "pulled" - disable_mysql_client_installation: bool = False + airflow_constraints_mode: str = "constraints" + airflow_is_in_context: bool = False + cleanup_context: bool = False + disable_airflow_repo_cache: bool = False disable_mssql_client_installation: bool = False + disable_mysql_client_installation: bool = False disable_postgres_client_installation: bool = False - disable_airflow_repo_cache: bool = False - install_providers_from_sources: bool = False - cleanup_context: bool = False - prepare_buildx_cache: bool = False - push_image: bool = False - empty_image: bool = False - airflow_is_in_context: bool = False - install_packages_from_context: bool = False - upgrade_to_newer_dependencies: bool = False - python: str = "3.7" - airflow_branch_for_pypi_preloading: str = AIRFLOW_BRANCH install_airflow_reference: str = "" install_airflow_version: str = "" - default_constraints_branch = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH - build_id: int = 0 - airflow_constraints: str = "constraints-source-providers" - github_repository: str = "apache/airflow" - constraints_github_repository: str = "apache/airflow" - dev_apt_command: str = "" - dev_apt_deps: str = "" - runtime_apt_command: str = "" - runtime_apt_deps: str = "" - additional_dev_apt_command: str = "" - additional_dev_apt_deps: str = "" - additional_dev_apt_env: str = "" - additional_runtime_apt_command: str = "" - additional_runtime_apt_deps: str = "" - additional_runtime_apt_env: str = "" - additional_python_deps: str = "" - image_tag: Optional[str] = None - extras: str = "" - additional_airflow_extras: str = "" - github_token: str = os.environ.get('GITHUB_TOKEN', "") - github_actions: str = os.environ.get('GITHUB_ACTIONS', "false") - login_to_github_registry: str = "false" - github_username: str = "" - platform: str = f"linux/{os.uname().machine}" - airflow_constraints_reference: str = "" - airflow_constraints_location: str = "" + install_packages_from_context: bool = False installation_method: str = "." - debian_version: str = "bullseye" - answer: Optional[str] = None + airflow_extras: str = get_airflow_extras() @property def airflow_version(self) -> str: @@ -96,22 +61,7 @@ def airflow_version(self) -> str: return get_airflow_version() @property - def airflow_branch(self) -> str: - return self.airflow_branch_for_pypi_preloading - - @property - def airflow_base_image_name(self): - image = f'ghcr.io/{self.github_repository.lower()}' - return image - - @property - def airflow_image_name(self): - """Construct PROD image link""" - image = f'{self.airflow_base_image_name}/{self.airflow_branch}/prod/python{self.python}' - return image - - @property - def the_image_type(self) -> str: + def image_type(self) -> str: return 'PROD' @property @@ -236,40 +186,6 @@ def extra_docker_build_flags(self) -> List[str]: ) return extra_build_flags - @property - def docker_cache_directive(self) -> List[str]: - docker_cache_directive = [] - - if self.docker_cache == "pulled": - docker_cache_directive.append(f"--cache-from={self.airflow_image_name}") - elif self.docker_cache == "disabled": - docker_cache_directive.append("--no-cache") - else: - docker_cache_directive = [] - - if self.prepare_buildx_cache: - docker_cache_directive.extend(["--cache-to=type=inline,mode=max", "--push"]) - return docker_cache_directive - - @property - def python_base_image(self): - """Construct Python Base Image""" - # ghcr.io/apache/airflow/main/python:3.8-slim-bullseye - return f'python:{self.python}-slim-{self.debian_version}' - - @property - def airflow_image_repository(self): - return f'https://github.com/{self.github_repository}' - - @property - def airflow_image_date_created(self): - now = datetime.now() - return now.strftime("%Y-%m-%dT%H:%M:%SZ") - - @property - def airflow_image_readme_url(self): - return "https://raw.githubusercontent.com/apache/airflow/main/docs/docker-stack/README.md" - @property def airflow_pre_cached_pip_packages(self) -> str: airflow_pre_cached_pip = 'true' @@ -298,53 +214,47 @@ def install_postgres_client(self) -> str: install_postgres = 'false' return install_postgres - @property - def airflow_extras(self): - return get_airflow_extras() - @property def docker_context_files(self) -> str: return "docker-context-files" @property - def airflow_image_name_with_tag(self): - """Construct PROD image link""" - image = f'{self.airflow_base_image_name}/{self.airflow_branch}/prod/python{self.python}' - return image if self.image_tag is None else image + f":{self.image_tag}" + def required_image_args(self) -> List[str]: + return [ + "additional_airflow_extras", + "additional_dev_apt_command", + "additional_dev_apt_deps", + "additional_dev_apt_env", + "additional_python_deps", + "additional_runtime_apt_command", + "additional_runtime_apt_deps", + "additional_runtime_apt_env", + "airflow_branch", + "airflow_constraints_mode", + "airflow_extras", + "airflow_image_date_created", + "airflow_image_readme_url", + "airflow_image_repository", + "airflow_is_in_context", + "airflow_pre_cached_pip_packages", + "airflow_version", + "build_id", + "constraints_github_repository", + "docker_context_files", + "install_mssql_client", + "install_mysql_client", + "install_packages_from_context", + "install_postgres_client", + "install_providers_from_sources", + "python_base_image", + "upgrade_to_newer_dependencies", + ] - required_image_args = [ - "python_base_image", - "install_mysql_client", - "install_mssql_client", - "install_postgres_client", - "airflow_version", - "airflow_branch", - "airflow_extras", - "airflow_pre_cached_pip_packages", - "docker_context_files", - "extras", - "additional_airflow_extras", - "additional_python_deps", - "additional_dev_apt_command", - "additional_dev_apt_deps", - "additional_dev_apt_env", - "additional_runtime_apt_command", - "additional_runtime_apt_deps", - "additional_runtime_apt_env", - "upgrade_to_newer_dependencies", - "constraints_github_repository", - "airflow_constraints", - "airflow_image_repository", - "airflow_image_date_created", - "build_id", - "airflow_image_readme_url", - "install_providers_from_sources", - "airflow_is_in_context", - "install_packages_from_context", - ] - optional_image_args = [ - "dev_apt_command", - "dev_apt_deps", - "runtime_apt_command", - "runtime_apt_deps", - ] + @property + def optional_image_args(self) -> List[str]: + return [ + "dev_apt_command", + "dev_apt_deps", + "runtime_apt_command", + "runtime_apt_deps", + ] diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index 61eb576b605df..0828510aeadfd 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -23,8 +23,8 @@ from airflow_breeze.branch_defaults import AIRFLOW_BRANCH, DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH from airflow_breeze.global_constants import ( ALLOWED_BACKENDS, + ALLOWED_CONSTRAINTS_MODES_CI, ALLOWED_DEBIAN_VERSIONS, - ALLOWED_GENERATE_CONSTRAINTS_MODES, ALLOWED_INSTALLATION_PACKAGE_FORMATS, ALLOWED_MSSQL_VERSIONS, ALLOWED_MYSQL_VERSIONS, @@ -32,6 +32,7 @@ ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS, AVAILABLE_INTEGRATIONS, MOUNT_ALL, + MOUNT_NONE, MOUNT_SELECTED, get_airflow_version, ) @@ -46,42 +47,42 @@ class ShellParams: Shell parameters. Those parameters are used to determine command issued to run shell command. """ - verbose: bool = False + airflow_branch: str = AIRFLOW_BRANCH + airflow_constraints_reference: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH + airflow_extras: str = "" + answer: Optional[str] = None + backend: str = ALLOWED_BACKENDS[0] + ci: bool = False + db_reset: bool = False + debian_version: str = ALLOWED_DEBIAN_VERSIONS[0] + dry_run: bool = False extra_args: Tuple = () force_build: bool = False + forward_credentials: str = "false" + airflow_constraints_mode: str = ALLOWED_CONSTRAINTS_MODES_CI[0] + github_actions: str = os.environ.get('GITHUB_ACTIONS', "false") + github_repository: str = "apache/airflow" + github_token: str = os.environ.get('GITHUB_TOKEN', "") + image_tag: str = "latest" + install_airflow_version: str = "" + install_providers_from_sources: bool = True integration: Tuple[str, ...] = () - postgres_version: str = ALLOWED_POSTGRES_VERSIONS[0] + issue_id: str = "" + load_default_connections: bool = False + load_example_dags: bool = False + mount_sources: str = MOUNT_SELECTED mssql_version: str = ALLOWED_MSSQL_VERSIONS[0] mysql_version: str = ALLOWED_MYSQL_VERSIONS[0] - backend: str = ALLOWED_BACKENDS[0] + num_runs: str = "" + package_format: str = ALLOWED_INSTALLATION_PACKAGE_FORMATS[0] + postgres_version: str = ALLOWED_POSTGRES_VERSIONS[0] python: str = ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS[0] - debian_version: str = ALLOWED_DEBIAN_VERSIONS[0] - dry_run: bool = False - load_example_dags: bool = False - load_default_connections: bool = False + skip_environment_initialization: bool = False + start_airflow: str = "false" use_airflow_version: Optional[str] = None - airflow_extras: str = "" - airflow_constraints_reference: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH use_packages_from_dist: bool = False - package_format: str = ALLOWED_INSTALLATION_PACKAGE_FORMATS[0] - generate_constraints_mode: str = ALLOWED_GENERATE_CONSTRAINTS_MODES[0] - install_providers_from_sources: bool = True - skip_environment_initialization: bool = False + verbose: bool = False version_suffix_for_pypi: str = "" - install_airflow_version: str = "" - image_tag: str = "latest" - github_repository: str = "apache/airflow" - mount_sources: str = MOUNT_SELECTED - forward_credentials: str = "false" - airflow_branch: str = AIRFLOW_BRANCH - start_airflow: str = "false" - github_token: str = os.environ.get('GITHUB_TOKEN', "") - github_actions: str = os.environ.get('GITHUB_ACTIONS', "false") - issue_id: str = "" - num_runs: str = "" - answer: Optional[str] = None - db_reset: bool = False - ci: bool = False @property def airflow_version(self): @@ -224,3 +225,11 @@ def command_passed(self): if len(self.extra_args) > 0: cmd = str(self.extra_args[0]) return cmd + + def __post_init__(self): + if self.use_airflow_version is not None: + get_console().print( + "[info]Forcing --mount-sources to `none` since we are not installing airflow " + f"from sources but from {self.use_airflow_version}[/]" + ) + self.mount_sources = MOUNT_NONE diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index 0857a1f71ecbf..615b7a2565c87 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -23,6 +23,8 @@ from airflow_breeze.global_constants import ( ALLOWED_BACKENDS, ALLOWED_BUILD_CACHE, + ALLOWED_CONSTRAINTS_MODES_CI, + ALLOWED_CONSTRAINTS_MODES_PROD, ALLOWED_DEBIAN_VERSIONS, ALLOWED_EXECUTORS, ALLOWED_INSTALLATION_PACKAGE_FORMATS, @@ -131,7 +133,7 @@ '--use-airflow-version', help="Use (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or `sdist`" " if Airflow should be removed, installed from wheel packages or sdist packages available in dist " - "folder respectively. Important! Using it Implies --mount-sources `none`.", + "folder respectively. Implies --mount-sources `none`.", type=UseAirflowVersionType(ALLOWED_USE_AIRFLOW_VERSIONS), envvar='USE_AIRFLOW_VERSION', ) @@ -174,11 +176,6 @@ show_default=True, type=BetterChoice(ALLOWED_BUILD_CACHE), ) -option_login_to_github_registry = click.option( - '--login-to-github-registry', - help='Logs in to GitHub registry.', - envvar='LOGIN_TO_GITHUB_REGISTRY', -) option_github_token = click.option( '--github-token', help='The token used to authenticate to GitHub.', @@ -285,16 +282,10 @@ help='Apt runtime dependencies to use when building the images.', envvar='RUNTIME_APT_DEPS', ) -option_skip_rebuild_check = click.option( - '-r', - '--skip-rebuild-check', - help="Skips checking if rebuild is needed", - is_flag=True, - envvar='SKIP_REBUILD_CHECK', -) option_prepare_buildx_cache = click.option( '--prepare-buildx-cache', - help='Prepares build cache rather than build images locally.', + help='Prepares build cache additionally to building images (this is done as two separate steps after' + 'the images are build). Implies --push-image flag', is_flag=True, envvar='PREPARE_BUILDX_CACHE', ) @@ -428,3 +419,17 @@ "constraints for the installed version and to find newer dependencies", envvar='AIRFLOW_CONSTRAINTS_REFERENCE', ) +option_airflow_constraints_mode_ci = click.option( + '--airflow-constraints-mode', + type=BetterChoice(ALLOWED_CONSTRAINTS_MODES_CI), + default=ALLOWED_CONSTRAINTS_MODES_CI[0], + show_default=True, + help='Mode of constraints for CI image building', +) +option_airflow_constraints_mode_prod = click.option( + '--airflow-constraints-mode', + type=BetterChoice(ALLOWED_CONSTRAINTS_MODES_PROD), + default=ALLOWED_CONSTRAINTS_MODES_PROD[0], + show_default=True, + help='Mode of constraints for PROD image building', +) diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 0478a2f32359f..a384fbe042e9e 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -17,17 +17,18 @@ """Various utils to prepare docker and docker compose commands.""" import os import re -import subprocess import sys +from copy import deepcopy from random import randint -from typing import Dict, List, Tuple, Union +from subprocess import DEVNULL, STDOUT, CalledProcessError, CompletedProcess +from typing import Dict, List, Union +from airflow_breeze.params._common_build_params import _CommonBuildParams from airflow_breeze.params.build_ci_params import BuildCiParams from airflow_breeze.params.build_prod_params import BuildProdParams from airflow_breeze.params.shell_params import ShellParams from airflow_breeze.utils.host_info_utils import get_host_group_id, get_host_os, get_host_user_id from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT -from airflow_breeze.utils.registry import login_to_docker_registry try: from packaging import version @@ -54,7 +55,12 @@ WEBSERVER_HOST_PORT, ) from airflow_breeze.utils.console import get_console -from airflow_breeze.utils.run_utils import commit_sha, prepare_build_command, run_command +from airflow_breeze.utils.run_utils import ( + commit_sha, + prepare_base_build_command, + prepare_build_cache_command, + run_command, +) NECESSARY_HOST_VOLUMES = [ "/.bash_aliases:/root/.bash_aliases:cached", @@ -93,6 +99,7 @@ def get_extra_docker_flags(mount_sources: str) -> List[str]: """ Returns extra docker flags based on the type of mounting we want to do for sources. + :param mount_sources: type of mounting we want to have :return: extra flag as list of strings """ @@ -113,12 +120,12 @@ def get_extra_docker_flags(mount_sources: str) -> List[str]: def check_docker_resources( airflow_image_name: str, verbose: bool, dry_run: bool -) -> Union[subprocess.CompletedProcess, subprocess.CalledProcessError]: +) -> Union[CompletedProcess, CalledProcessError]: """ Check if we have enough resources to run docker. This is done via running script embedded in our image. :param verbose: print commands when running :param dry_run: whether to run it in dry run mode - :param airflow_image_name: name of the airflow image to use. + :param airflow_image_name: name of the airflow image to use """ return run_command( cmd=[ @@ -139,14 +146,14 @@ def check_docker_resources( ) -def check_docker_permission_denied(verbose) -> bool: +def check_docker_permission_denied(verbose: bool) -> bool: """ Checks if we have permission to write to docker socket. By default, on Linux you need to add your user to docker group and some new users do not realize that. We help those users if we have permission to run docker commands. :param verbose: print commands when running - :return: True if permission is denied. + :return: True if permission is denied """ permission_denied = False docker_permission_command = ["docker", "info"] @@ -185,8 +192,8 @@ def check_docker_is_running(verbose: bool): verbose=verbose, no_output_dump_on_exception=True, text=False, - stdout=subprocess.DEVNULL, - stderr=subprocess.STDOUT, + stdout=DEVNULL, + stderr=STDOUT, check=False, ) if response.returncode != 0: @@ -199,8 +206,8 @@ def check_docker_is_running(verbose: bool): def check_docker_version(verbose: bool): """ - Checks if the docker compose version is as expected (including some specific modifications done by - some vendors such as Microsoft (they might have modified version of docker-compose/docker in their + Checks if the docker compose version is as expected. including some specific modifications done by + some vendors such as Microsoft. They might have modified version of docker-compose/docker in their cloud. In case docker compose version is wrong we continue but print warning for the user. @@ -242,8 +249,8 @@ def check_docker_version(verbose: bool): def check_docker_compose_version(verbose: bool): """ - Checks if the docker compose version is as expected (including some specific modifications done by - some vendors such as Microsoft (they might have modified version of docker-compose/docker in their + Checks if the docker compose version is as expected, including some specific modifications done by + some vendors such as Microsoft. They might have modified version of docker-compose/docker in their cloud. In case docker compose version is wrong we continue but print warning for the user. :param verbose: print commands when running @@ -261,14 +268,14 @@ def check_docker_compose_version(verbose: bool): docker_compose_version = docker_compose_version_result.stdout version_extracted = version_pattern.search(docker_compose_version) if version_extracted is not None: - version = '.'.join(version_extracted.groups()) - good_version = compare_version(version, MIN_DOCKER_COMPOSE_VERSION) + docker_version = '.'.join(version_extracted.groups()) + good_version = compare_version(docker_version, MIN_DOCKER_COMPOSE_VERSION) if good_version: - get_console().print(f'[success]Good version of docker-compose: {version}[/]') + get_console().print(f'[success]Good version of docker-compose: {docker_version}[/]') else: get_console().print( f""" -[warning]You have too old version of docker-compose: {version}! At least 1.29 is needed! Please upgrade! +[warning]You have too old version of docker-compose: {docker_version}! At least 1.29 needed! Please upgrade! """ ) get_console().print( @@ -296,9 +303,7 @@ def get_env_variable_value(arg_name: str, params: Union[BuildCiParams, BuildProd return value -def construct_arguments_for_docker_build_command( - image_params: Union[BuildCiParams, BuildProdParams] -) -> List[str]: +def prepare_arguments_for_docker_build_command(image_params: _CommonBuildParams) -> List[str]: """ Constructs docker compose command arguments list based on parameters passed. Maps arguments to argument values. @@ -327,22 +332,20 @@ def construct_arguments_for_docker_build_command( return args_command -def construct_docker_build_command( - image_params: Union[BuildProdParams, BuildCiParams], +def prepare_docker_build_cache_command( + image_params: _CommonBuildParams, + dry_run: bool, verbose: bool, - production_image: bool, ) -> List[str]: """ - Constructs docker build command based on the parameters passed. + Constructs docker build_cache command based on the parameters passed. :param image_params: parameters of the image + :param dry_run: dry_run rather than run the command :param verbose: print commands when running - :param production_image: whether this is production image or ci image :return: Command to run as list of string """ - arguments = construct_arguments_for_docker_build_command(image_params) - build_command = prepare_build_command( - prepare_buildx_cache=image_params.prepare_buildx_cache, verbose=verbose - ) + arguments = prepare_arguments_for_docker_build_command(image_params) + build_command = prepare_build_cache_command() build_flags = image_params.extra_docker_build_flags final_command = [] final_command.extend(["docker"]) @@ -350,25 +353,51 @@ def construct_docker_build_command( final_command.extend(build_flags) final_command.extend(["--pull"]) final_command.extend(arguments) - final_command.extend(["-t", image_params.airflow_image_name, "--target", "main", "."]) - final_command.extend(["-f", 'Dockerfile' if production_image else 'Dockerfile.ci']) + final_command.extend(["--target", "main", "."]) + final_command.extend( + ["-f", 'Dockerfile' if isinstance(image_params, BuildProdParams) else 'Dockerfile.ci'] + ) final_command.extend(["--platform", image_params.platform]) + final_command.extend( + [f"--cache-to=type=registry,ref={image_params.get_cache(image_params.platform)}," f"mode=max"] + ) + cmd = ['docker', 'buildx', 'inspect', 'airflow_cache'] + buildx_command_result = run_command(cmd, verbose=verbose, dry_run=dry_run, text=True) + if buildx_command_result and buildx_command_result.returncode != 0: + next_cmd = ['docker', 'buildx', 'create', '--name', 'airflow_cache'] + run_command(next_cmd, verbose=verbose, text=True, check=False) return final_command -def construct_docker_tag_command( - image_params: Union[BuildProdParams, BuildCiParams], +def prepare_docker_build_command( + image_params: _CommonBuildParams, + verbose: bool, ) -> List[str]: """ - Constructs docker tag command based on the parameters passed. + Constructs docker build command based on the parameters passed. :param image_params: parameters of the image + :param verbose: print commands when running :return: Command to run as list of string """ - return ["docker", "tag", image_params.airflow_image_name, image_params.airflow_image_name_with_tag] + arguments = prepare_arguments_for_docker_build_command(image_params) + build_command = prepare_base_build_command(image_params=image_params, verbose=verbose) + build_flags = image_params.extra_docker_build_flags + final_command = [] + final_command.extend(["docker"]) + final_command.extend(build_command) + final_command.extend(build_flags) + final_command.extend(["--pull"]) + final_command.extend(arguments) + final_command.extend(["-t", image_params.airflow_image_name_with_tag, "--target", "main", "."]) + final_command.extend( + ["-f", 'Dockerfile' if isinstance(image_params, BuildProdParams) else 'Dockerfile.ci'] + ) + final_command.extend(["--platform", image_params.platform]) + return final_command def construct_docker_push_command( - image_params: Union[BuildProdParams, BuildCiParams], + image_params: _CommonBuildParams, ) -> List[str]: """ Constructs docker push command based on the parameters passed. @@ -378,39 +407,8 @@ def construct_docker_push_command( return ["docker", "push", image_params.airflow_image_name_with_tag] -def tag_and_push_image( - image_params: Union[BuildProdParams, BuildCiParams], dry_run: bool, verbose: bool -) -> Tuple[int, str]: - """ - Tag and push the image according to parameters. - :param image_params: parameters of the image - :param dry_run: whether we are in dry-run mode - :param verbose: whethere we produce verbose output - :return: - """ - get_console().print( - f"[info]Tagging and pushing the {image_params.airflow_image_name} as " - f"{image_params.airflow_image_name_with_tag}.[/]" - ) - cmd = construct_docker_tag_command(image_params) - command_result = run_command( - cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, text=True, check=False - ) - if command_result.returncode == 0: - return_code, info = login_to_docker_registry(image_params, dry_run=dry_run) - if return_code != 0: - return return_code, f"Tag and pushing the image {image_params.python}: {info}" - cmd = construct_docker_push_command(image_params) - command_result = run_command( - cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, text=True, check=False - ) - if command_result.returncode == 0: - return 0, f"Tag and pushing the image {image_params.python}" - return command_result.returncode, f"Tag and pushing the image {image_params.python}" - - -def construct_empty_docker_build_command( - image_params: Union[BuildProdParams, BuildCiParams], +def prepare_empty_docker_build_command( + image_params: _CommonBuildParams, ) -> List[str]: """ Constructs docker build empty image command based on the parameters passed. @@ -420,6 +418,28 @@ def construct_empty_docker_build_command( return ["docker", "build", "-t", image_params.airflow_image_name_with_tag, "-"] +def build_cache( + image_params: _CommonBuildParams, dry_run: bool, verbose: bool +) -> Union[CompletedProcess, CalledProcessError]: + build_command_result: Union[CompletedProcess, CalledProcessError] = CompletedProcess( + args=[], returncode=0 + ) + for platform in image_params.platforms: + platform_image_params = deepcopy(image_params) + # override the platform in the copied params to only be single platform per run + # as a workaround to https://github.com/docker/buildx/issues/1044 + platform_image_params.platform = platform + cmd = prepare_docker_build_cache_command( + image_params=platform_image_params, dry_run=dry_run, verbose=verbose + ) + build_command_result = run_command( + cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, check=False, text=True + ) + if build_command_result.returncode != 0: + break + return build_command_result + + def set_value_to_default_if_not_set(env: Dict[str, str], name: str, default: str): """ Set value of name parameter to default (indexed by name) if not set. @@ -438,9 +458,10 @@ def update_expected_environment_variables(env: Dict[str, str]) -> None: :param env: environment variables to update with missing values if not set. """ - set_value_to_default_if_not_set(env, 'ANSWER', "") - set_value_to_default_if_not_set(env, 'AIRFLOW_EXTRAS', "") + set_value_to_default_if_not_set(env, 'AIRFLOW_CONSTRAINTS_MODE', "constraints-source-providers") set_value_to_default_if_not_set(env, 'AIRFLOW_CONSTRAINTS_REFERENCE', DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH) + set_value_to_default_if_not_set(env, 'AIRFLOW_EXTRAS', "") + set_value_to_default_if_not_set(env, 'ANSWER', "") set_value_to_default_if_not_set(env, 'BREEZE', "true") set_value_to_default_if_not_set(env, 'CI', "false") set_value_to_default_if_not_set(env, 'CI_BUILD_ID', "0") @@ -455,14 +476,13 @@ def update_expected_environment_variables(env: Dict[str, str]) -> None: set_value_to_default_if_not_set(env, 'DEFAULT_CONSTRAINTS_BRANCH', DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH) set_value_to_default_if_not_set(env, 'ENABLED_SYSTEMS', "") set_value_to_default_if_not_set(env, 'ENABLE_TEST_COVERAGE', "false") - set_value_to_default_if_not_set(env, 'GENERATE_CONSTRAINTS_MODE', "source-providers") set_value_to_default_if_not_set(env, 'GITHUB_REGISTRY_PULL_IMAGE_TAG', "latest") + set_value_to_default_if_not_set(env, 'HOST_GROUP_ID', get_host_group_id()) set_value_to_default_if_not_set(env, 'HOST_OS', get_host_os()) set_value_to_default_if_not_set(env, 'HOST_USER_ID', get_host_user_id()) - set_value_to_default_if_not_set(env, 'HOST_GROUP_ID', get_host_group_id()) set_value_to_default_if_not_set(env, 'INIT_SCRIPT_FILE', "init.sh") - set_value_to_default_if_not_set(env, 'INSTALL_PROVIDERS_FROM_SOURCES', "true") set_value_to_default_if_not_set(env, 'INSTALL_PACKAGES_FROM_CONTEXT', "false") + set_value_to_default_if_not_set(env, 'INSTALL_PROVIDERS_FROM_SOURCES', "true") set_value_to_default_if_not_set(env, 'LIST_OF_INTEGRATION_TESTS_TO_RUN', "") set_value_to_default_if_not_set(env, 'LOAD_DEFAULT_CONNECTIONS', "false") set_value_to_default_if_not_set(env, 'LOAD_EXAMPLES', "false") @@ -486,6 +506,7 @@ def update_expected_environment_variables(env: Dict[str, str]) -> None: "AIRFLOW_CI_IMAGE": "airflow_image_name", "AIRFLOW_CI_IMAGE_WITH_TAG": "airflow_image_name_with_tag", "AIRFLOW_EXTRAS": "airflow_extras", + "AIRFLOW_CONSTRAINTS_MODE": "airflow_constraints_mode", "AIRFLOW_CONSTRAINTS_REFERENCE": "airflow_constraints_reference", "AIRFLOW_IMAGE_KUBERNETES": "airflow_image_kubernetes", "AIRFLOW_PROD_IMAGE": "airflow_image_name", @@ -537,7 +558,7 @@ def get_env_variables_for_docker_commands(params: Union[ShellParams, BuildCiPara if the env variable that we run with does not have it. * Updates all other environment variables that docker-compose expects with default values if missing - :param params: shell parameters passed + :param params: shell parameters passed. :return: dictionary of env variables to set """ env_variables: Dict[str, str] = os.environ.copy() diff --git a/dev/breeze/src/airflow_breeze/utils/platforms.py b/dev/breeze/src/airflow_breeze/utils/platforms.py new file mode 100644 index 0000000000000..7c2bd53df20e2 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/utils/platforms.py @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +def get_real_platform(single_platform: str) -> str: + """ + Replace different platform variants of the platform provided platforms with the two canonical ones we + are using: amd64 and arm64. + """ + return single_platform.replace("x86_64", "amd64").replace("aarch64", "arm64").replace("/", "-") diff --git a/dev/breeze/src/airflow_breeze/utils/pulll_image.py b/dev/breeze/src/airflow_breeze/utils/pulll_image.py index 0efe2cb154e7f..22eba4cb0038d 100644 --- a/dev/breeze/src/airflow_breeze/utils/pulll_image.py +++ b/dev/breeze/src/airflow_breeze/utils/pulll_image.py @@ -19,12 +19,13 @@ import time from typing import List, Tuple, Union +from airflow_breeze.params._common_build_params import _CommonBuildParams from airflow_breeze.params.build_ci_params import BuildCiParams from airflow_breeze.params.build_prod_params import BuildProdParams from airflow_breeze.utils.console import get_console from airflow_breeze.utils.mark_image_as_refreshed import mark_image_as_refreshed from airflow_breeze.utils.parallel import check_async_run_results -from airflow_breeze.utils.registry import login_to_docker_registry +from airflow_breeze.utils.registry import login_to_github_docker_registry from airflow_breeze.utils.run_tests import verify_an_image from airflow_breeze.utils.run_utils import run_command @@ -74,7 +75,7 @@ def run_pull_in_parallel( def run_pull_image( - image_params: Union[BuildCiParams, BuildProdParams], + image_params: _CommonBuildParams, dry_run: bool, verbose: bool, wait_for_image: bool, @@ -92,12 +93,12 @@ def run_pull_image( :return: Tuple of return code and description of the image pulled """ get_console().print( - f"\n[info]Pulling {image_params.the_image_type} image of airflow python version: " + f"\n[info]Pulling {image_params.image_type} image of airflow python version: " f"{image_params.python} image: {image_params.airflow_image_name_with_tag} " f"with wait for image: {wait_for_image}[/]\n" ) while True: - login_to_docker_registry(image_params, dry_run=dry_run) + login_to_github_docker_registry(image_params=image_params, dry_run=dry_run, verbose=verbose) command_to_run = ["docker", "pull", image_params.airflow_image_name_with_tag] command_result = run_command( command_to_run, @@ -160,7 +161,7 @@ def run_pull_image( def run_pull_and_verify_image( - image_params: Union[BuildCiParams, BuildProdParams], + image_params: _CommonBuildParams, dry_run: bool, verbose: bool, wait_for_image: bool, @@ -177,7 +178,7 @@ def run_pull_and_verify_image( ) return verify_an_image( image_name=image_params.airflow_image_name_with_tag, - image_type=image_params.the_image_type, + image_type=image_params.image_type, dry_run=dry_run, verbose=verbose, extra_pytest_args=extra_pytest_args, diff --git a/dev/breeze/src/airflow_breeze/utils/registry.py b/dev/breeze/src/airflow_breeze/utils/registry.py index af5126934e258..bf01ec61b3aba 100644 --- a/dev/breeze/src/airflow_breeze/utils/registry.py +++ b/dev/breeze/src/airflow_breeze/utils/registry.py @@ -16,34 +16,30 @@ # under the License. import os -from typing import Tuple, Union +from typing import Tuple -from airflow_breeze.params.build_ci_params import BuildCiParams -from airflow_breeze.params.build_prod_params import BuildProdParams +from airflow_breeze.params._common_build_params import _CommonBuildParams from airflow_breeze.utils.console import get_console from airflow_breeze.utils.run_utils import run_command -def login_to_docker_registry( - image_params: Union[BuildProdParams, BuildCiParams], dry_run: bool +def login_to_github_docker_registry( + image_params: _CommonBuildParams, dry_run: bool, verbose: bool ) -> Tuple[int, str]: """ - In case of CI environment, we need to login to GitHub Registry if we want to prepare cache. - This method logs in using the params specified. + In case of CI environment, we need to login to GitHub Registry. :param image_params: parameters to use for Building prod image :param dry_run: whether we are in dry_run mode + :param verbose: whether to show commands. """ if os.environ.get("CI"): if len(image_params.github_token) == 0: get_console().print("\n[info]Skip logging in to GitHub Registry. No Token available!") - elif image_params.login_to_github_registry != "true": - get_console().print( - "\n[info]Skip logging in to GitHub Registry.\ - LOGIN_TO_GITHUB_REGISTRY is set as false" - ) elif len(image_params.github_token) > 0: - run_command(['docker', 'logout', 'ghcr.io'], verbose=True, text=False, check=False) + run_command( + ['docker', 'logout', 'ghcr.io'], dry_run=dry_run, verbose=verbose, text=False, check=False + ) command_result = run_command( [ 'docker', @@ -53,7 +49,7 @@ def login_to_docker_registry( '--password-stdin', 'ghcr.io', ], - verbose=True, + verbose=verbose, text=True, input=image_params.github_token, check=False, diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py b/dev/breeze/src/airflow_breeze/utils/run_utils.py index e02196c724a64..d0d0d535d565a 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/run_utils.py @@ -24,8 +24,10 @@ from distutils.version import StrictVersion from functools import lru_cache from pathlib import Path +from re import match from typing import Dict, List, Mapping, Optional, Union +from airflow_breeze.params._common_build_params import _CommonBuildParams from airflow_breeze.utils.ci_group import ci_group from airflow_breeze.utils.console import get_console from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT @@ -76,8 +78,10 @@ def run_command( shlex.quote(c) for c in cmd if not c.startswith('-') # exclude options + and len(c) > 0 and (c[0] != "/" or c.endswith(".sh")) # exclude volumes and not c == "never" # exclude --pull never + and not match(r"^[A-Z_]*=.*$", c) ) env_to_print = get_environments_to_print(env) with ci_group(title=f"Running {title}"): @@ -299,7 +303,7 @@ def check_if_buildx_plugin_installed(verbose: bool) -> bool: return is_buildx_available -def prepare_build_command(prepare_buildx_cache: bool, verbose: bool) -> List[str]: +def prepare_base_build_command(image_params: _CommonBuildParams, verbose: bool) -> List[str]: """ Prepare build command for docker build. Depending on whether we have buildx plugin installed or not, and whether we run cache preparation, there might be different results: @@ -308,26 +312,33 @@ def prepare_build_command(prepare_buildx_cache: bool, verbose: bool) -> List[str depending on whether we build regular image or cache * if no buildx plugin is installed, and we do not prepare cache, regular docker `build` command is used. * if no buildx plugin is installed, and we prepare cache - we fail. Cache can only be done with buildx - :param prepare_buildx_cache: whether we are preparing buildx cache. + :param image_params: parameters of the image :param verbose: print commands when running :return: command to use as docker build command """ build_command_param = [] is_buildx_available = check_if_buildx_plugin_installed(verbose=verbose) if is_buildx_available: - if prepare_buildx_cache: - build_command_param.extend(["buildx", "build", "--builder", "airflow_cache", "--progress=tty"]) - cmd = ['docker', 'buildx', 'inspect', 'airflow_cache'] - buildx_command_result = run_command(cmd, verbose=True, text=True) - if buildx_command_result and buildx_command_result.returncode != 0: - next_cmd = ['docker', 'buildx', 'create', '--name', 'airflow_cache'] - run_command(next_cmd, verbose=True, text=True, check=False) + if image_params.prepare_buildx_cache: + build_command_param.extend( + ["buildx", "build", "--builder", "airflow_cache", "--progress=tty", "--push"] + ) else: - build_command_param.extend(["buildx", "build", "--builder", "default", "--progress=tty"]) + build_command_param.extend( + [ + "buildx", + "build", + "--builder", + "default", + "--progress=tty", + "--push" if image_params.push_image else "--load", + ] + ) else: - if prepare_buildx_cache: + if image_params.prepare_buildx_cache or image_params.push_image: get_console().print( - '\n[error] Buildx cli plugin is not available and you need it to prepare buildx cache. \n' + '\n[error] Buildx cli plugin is not available and you need it to prepare' + ' buildx cache or push image after build. \n' ) get_console().print( '[error] Please install it following https://docs.docker.com/buildx/working-with-buildx/ \n' @@ -337,6 +348,19 @@ def prepare_build_command(prepare_buildx_cache: bool, verbose: bool) -> List[str return build_command_param +def prepare_build_cache_command() -> List[str]: + """ + Prepare build cache command for docker build. We need to have buildx for that command. + This command is needed separately from the build image command because of the bug in multiplatform + support for buildx plugin https://github.com/docker/buildx/issues/1044 where when you run multiple + platform build, cache from one platform overrides cache for the other platform. + + :param verbose: print commands when running + :return: command to use as docker build command + """ + return ["buildx", "build", "--builder", "airflow_cache", "--progress=tty"] + + @lru_cache(maxsize=None) def commit_sha(): """Returns commit SHA of current repo. Cached for various usages.""" diff --git a/dev/breeze/tests/test_docker_command_utils.py b/dev/breeze/tests/test_docker_command_utils.py index 5be489c65ea45..55c7c3e4d4813 100644 --- a/dev/breeze/tests/test_docker_command_utils.py +++ b/dev/breeze/tests/test_docker_command_utils.py @@ -152,7 +152,7 @@ def test_check_docker_compose_version_low(mock_get_console, mock_run_command): expected_print_calls = [ call( """ -[warning]You have too old version of docker-compose: 1.28.5! At least 1.29 is needed! Please upgrade! +[warning]You have too old version of docker-compose: 1.28.5! At least 1.29 needed! Please upgrade! """ ), call( diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py index 2316cdaa7d0c4..3a476188e1e46 100755 --- a/dev/provider_packages/prepare_provider_packages.py +++ b/dev/provider_packages/prepare_provider_packages.py @@ -706,6 +706,7 @@ def make_sure_remote_apache_exists_and_fetch(git_update: bool, verbose: bool): subprocess.check_output( remote_add_command, stderr=subprocess.STDOUT, + text=True, ) except subprocess.CalledProcessError as ex: console.print("[red]Error: when adding remote:[/]", ex) @@ -1602,7 +1603,7 @@ def build_provider_packages( verify_setup_cfg_prepared(provider_package) console.print(f"Building provider package: {provider_package} in format {package_format}") - command = ["python3", "setup.py", "build", "--build-temp", tmp_build_dir] + command: List[str] = ["python3", "setup.py", "build", "--build-temp", tmp_build_dir] if version_suffix is not None: command.extend(['egg_info', '--tag-build', version_suffix]) if package_format in ['sdist', 'both']: @@ -1611,10 +1612,10 @@ def build_provider_packages( command.extend(["bdist_wheel", "--bdist-dir", tmp_dist_dir]) console.print(f"Executing command: '{' '.join(command)}'") try: - subprocess.check_call(command, stdout=subprocess.DEVNULL) + subprocess.check_call(args=command, stdout=subprocess.DEVNULL) except subprocess.CalledProcessError as ex: - console.print(ex.output.decode()) - raise Exception("The command returned an error %s", command) + console.print("[red]The command returned an error %s", ex) + sys.exit(ex.returncode) console.print( f"[green]Prepared provider package {provider_package} in format {package_format}[/]" ) diff --git a/dev/refresh_images.sh b/dev/refresh_images.sh index 1f0b908cdcd2b..fc436e88533a8 100755 --- a/dev/refresh_images.sh +++ b/dev/refresh_images.sh @@ -21,10 +21,10 @@ rm -rf docker-context-files/*.whl rm -rf docker-context-files/*.tgz export ANSWER="yes" export CI="true" +export GITHUB_TOKEN="" breeze build-image \ --build-multiple-images \ - --push-image \ --prepare-buildx-cache \ --platform linux/amd64,linux/arm64 \ --verbose diff --git a/docs/apache-airflow/installation/installing-from-pypi.rst b/docs/apache-airflow/installation/installing-from-pypi.rst index bb6ba57f44dcc..51b9d7c04473f 100644 --- a/docs/apache-airflow/installation/installing-from-pypi.rst +++ b/docs/apache-airflow/installation/installing-from-pypi.rst @@ -72,8 +72,8 @@ where: - ``AIRFLOW_VERSION`` - Airflow version (e.g. :subst-code:`|version|`) or ``main``, ``2-0``, for latest development version - ``PYTHON_VERSION`` Python version e.g. ``3.8``, ``3.7`` -There is also a no-providers constraint file, which contains just constraints required to install Airflow core. This allows -to install and upgrade airflow separately and independently from providers. +There is also a ``constraints-no-providers`` constraint file, which contains just constraints required to +install Airflow core. This allows to install and upgrade airflow separately and independently from providers. You can create the URL to the file substituting the variables in the template below. diff --git a/images/breeze/output-build-image.svg b/images/breeze/output-build-image.svg index 4b8e20035ca57..bf448434cc5bf 100644 --- a/images/breeze/output-build-image.svg +++ b/images/breeze/output-build-image.svg @@ -1,4 +1,4 @@ - + - Command: build-image + Command: build-image -                                                                                                                          - Usage: breeze build-image [OPTIONS]                                                                                     -                                                                                                                         - Build CI image. Include building multiple images for all python versions (sequentially).                                -                                                                                                                         -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --python                         -p  Python major/minor version used in Airflow image for images.                   │ -│                                       (>3.7< | 3.8 | 3.9 | 3.10)                                                     │ -│                                       [default: 3.7]                                                                 │ -│  --upgrade-to-newer-dependencies  -u  When set, upgrade all PIP packages to latest.                                  │ -│  --debian-version                     Debian version used for the image. (bullseye | buster) [default: bullseye]     │ -│  --image-tag                      -t  Tag added to the default naming conventions of Airflow CI/PROD images. (TEXT)  │ -│  --docker-cache                   -c  Cache option for image used during the build. (pulled | local | disabled)      │ -│                                       [default: pulled]                                                              │ -│  --force-build                        Force image build no matter if it is determined as needed.                     │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building multiple images ───────────────────────────────────────────────────────────────────────────────────────────╮ -│  --build-multiple-images    Run the operation sequentially on all or selected subset of Python versions.             │ -│  --python-versions          Space separated list of python versions used for build with multiple versions. (TEXT)    │ -│                             [default: 3.7 3.8 3.9 3.10]                                                              │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ -│  --install-providers-from-sources    Install providers from sources when installing.                                 │ -│  --additional-python-deps            Additional python dependencies to use when building the images. (TEXT)          │ -│  --runtime-apt-deps                  Apt runtime dependencies to use when building the images. (TEXT)                │ -│  --runtime-apt-command               Command executed before runtime apt deps are installed. (TEXT)                  │ -│  --additional-extras                 Additional extra package while installing Airflow in the image. (TEXT)          │ -│  --additional-runtime-apt-deps       Additional apt runtime dependencies to use when building the images. (TEXT)     │ -│  --additional-runtime-apt-env        Additional environment variables set when adding runtime dependencies. (TEXT)   │ -│  --additional-runtime-apt-command    Additional command executed before runtime apt deps are installed. (TEXT)       │ -│  --additional-dev-apt-deps           Additional apt dev dependencies to use when building the images. (TEXT)         │ -│  --additional-dev-apt-env            Additional environment variables set when adding dev dependencies. (TEXT)       │ -│  --additional-dev-apt-command        Additional command executed before dev apt deps are installed. (TEXT)           │ -│  --dev-apt-deps                      Apt dev dependencies to use when building the images. (TEXT)                    │ -│  --dev-apt-command                   Command executed before dev apt deps are installed. (TEXT)                      │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│  --platform                    Platform for Airflow image. (linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)     │ -│  --prepare-buildx-cache        Prepares build cache rather than build images locally.                                │ -│  --push-image                  Push image after building it.                                                         │ -│  --empty-image                 Prepare empty image tagged with the same name as the Airflow image.                   │ -│  --github-token                The token used to authenticate to GitHub. (TEXT)                                      │ -│  --github-username             The user name used to authenticate to GitHub. (TEXT)                                  │ -│  --login-to-github-registry    Logs in to GitHub registry. (TEXT)                                                    │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --github-repository  -g  GitHub repository used to pull, push run images. (TEXT) [default: apache/airflow]          │ -│  --verbose            -v  Print verbose information about performed steps.                                           │ -│  --dry-run            -D  If dry-run is set, commands are only printed, not executed.                                │ -│  --answer             -a  Force answer to questions. (y | n | q | yes | no | quit)                                   │ -│  --help               -h  Show this message and exit.                                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +                                                                                                                          + Usage: breeze build-image [OPTIONS]                                                                                     +                                                                                                                         + Build CI image. Include building multiple images for all python versions (sequentially).                                +                                                                                                                         +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --python                         -p  Python major/minor version used in Airflow image for images.                   │ +│                                       (>3.7< | 3.8 | 3.9 | 3.10)                                                     │ +│                                       [default: 3.7]                                                                 │ +│  --upgrade-to-newer-dependencies  -u  When set, upgrade all PIP packages to latest.                                  │ +│  --debian-version                     Debian version used for the image. (bullseye | buster) [default: bullseye]     │ +│  --image-tag                      -t  Tag added to the default naming conventions of Airflow CI/PROD images. (TEXT)  │ +│  --docker-cache                   -c  Cache option for image used during the build. (registry | local | disabled)    │ +│                                       [default: registry]                                                            │ +│  --force-build                        Force image build no matter if it is determined as needed.                     │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building multiple images ───────────────────────────────────────────────────────────────────────────────────────────╮ +│  --build-multiple-images    Run the operation sequentially on all or selected subset of Python versions.             │ +│  --python-versions          Space separated list of python versions used for build with multiple versions. (TEXT)    │ +│                             [default: 3.7 3.8 3.9 3.10]                                                              │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ +│  --install-providers-from-sources    Install providers from sources when installing.                                 │ +│  --airflow-constraints-mode          Mode of constraints for CI image building                                       │ +│                                      (constraints-source-providers | constraints | constraints-no-providers)         │ +│                                      [default: constraints-source-providers]                                         │ +│  --additional-python-deps            Additional python dependencies to use when building the images. (TEXT)          │ +│  --runtime-apt-deps                  Apt runtime dependencies to use when building the images. (TEXT)                │ +│  --runtime-apt-command               Command executed before runtime apt deps are installed. (TEXT)                  │ +│  --additional-extras                 Additional extra package while installing Airflow in the image. (TEXT)          │ +│  --additional-runtime-apt-deps       Additional apt runtime dependencies to use when building the images. (TEXT)     │ +│  --additional-runtime-apt-env        Additional environment variables set when adding runtime dependencies. (TEXT)   │ +│  --additional-runtime-apt-command    Additional command executed before runtime apt deps are installed. (TEXT)       │ +│  --additional-dev-apt-deps           Additional apt dev dependencies to use when building the images. (TEXT)         │ +│  --additional-dev-apt-env            Additional environment variables set when adding dev dependencies. (TEXT)       │ +│  --additional-dev-apt-command        Additional command executed before dev apt deps are installed. (TEXT)           │ +│  --dev-apt-deps                      Apt dev dependencies to use when building the images. (TEXT)                    │ +│  --dev-apt-command                   Command executed before dev apt deps are installed. (TEXT)                      │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│  --platform                Platform for Airflow image. (linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)         │ +│  --prepare-buildx-cache    Prepares build cache additionally to building images (this is done as two separate steps  │ +│                            afterthe images are build). Implies --push-image flag                                     │ +│  --push-image              Push image after building it.                                                             │ +│  --empty-image             Prepare empty image tagged with the same name as the Airflow image.                       │ +│  --github-token            The token used to authenticate to GitHub. (TEXT)                                          │ +│  --github-username         The user name used to authenticate to GitHub. (TEXT)                                      │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --github-repository  -g  GitHub repository used to pull, push run images. (TEXT) [default: apache/airflow]          │ +│  --verbose            -v  Print verbose information about performed steps.                                           │ +│  --dry-run            -D  If dry-run is set, commands are only printed, not executed.                                │ +│  --answer             -a  Force answer to questions. (y | n | q | yes | no | quit)                                   │ +│  --help               -h  Show this message and exit.                                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-build-prod-image.svg b/images/breeze/output-build-prod-image.svg index 2147125dc959f..2146d8486ecea 100644 --- a/images/breeze/output-build-prod-image.svg +++ b/images/breeze/output-build-prod-image.svg @@ -1,4 +1,4 @@ - + - Command: build-prod-image + Command: build-prod-image -                                                                                                                          - Usage: breeze build-prod-image [OPTIONS]                                                                                -                                                                                                                         - Build Production image. Include building multiple images for all or selected Python versions sequentially.              -                                                                                                                         -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --python                         -p  Python major/minor version used in Airflow image for images.                   │ -│                                       (>3.7< | 3.8 | 3.9 | 3.10)                                                     │ -│                                       [default: 3.7]                                                                 │ -│  --install-airflow-version        -V  Install version of Airflow from PyPI. (TEXT)                                   │ -│  --upgrade-to-newer-dependencies  -u  When set, upgrade all PIP packages to latest.                                  │ -│  --debian-version                     Debian version used for the image. (bullseye | buster) [default: bullseye]     │ -│  --image-tag                      -t  Tag added to the default naming conventions of Airflow CI/PROD images. (TEXT)  │ -│  --docker-cache                   -c  Cache option for image used during the build. (pulled | local | disabled)      │ -│                                       [default: pulled]                                                              │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building multiple images ───────────────────────────────────────────────────────────────────────────────────────────╮ -│  --build-multiple-images    Run the operation sequentially on all or selected subset of Python versions.             │ -│  --python-versions          Space separated list of python versions used for build with multiple versions. (TEXT)    │ -│                             [default: 3.7 3.8 3.9 3.10]                                                              │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ -│  --install-providers-from-sources    Install providers from sources when installing.                                 │ -│  --additional-python-deps            Additional python dependencies to use when building the images. (TEXT)          │ -│  --additional-extras                 Additional extra package while installing Airflow in the image. (TEXT)          │ -│  --additional-runtime-apt-deps       Additional apt runtime dependencies to use when building the images. (TEXT)     │ -│  --additional-runtime-apt-env        Additional environment variables set when adding runtime dependencies. (TEXT)   │ -│  --additional-runtime-apt-command    Additional command executed before runtime apt deps are installed. (TEXT)       │ -│  --additional-dev-apt-deps           Additional apt dev dependencies to use when building the images. (TEXT)         │ -│  --additional-dev-apt-env            Additional environment variables set when adding dev dependencies. (TEXT)       │ -│  --additional-dev-apt-command        Additional command executed before dev apt deps are installed. (TEXT)           │ -│  --extras                            Extras to install by default.                                                   │ -│                                      (TEXT)                                                                          │ -│                                      [default:                                                                       │ -│                                      amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,goog…  │ -│  --runtime-apt-deps                  Apt runtime dependencies to use when building the images. (TEXT)                │ -│  --runtime-apt-command               Command executed before runtime apt deps are installed. (TEXT)                  │ -│  --dev-apt-deps                      Apt dev dependencies to use when building the images. (TEXT)                    │ -│  --dev-apt-command                   Command executed before dev apt deps are installed. (TEXT)                      │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ -│  --install-packages-from-context           Install wheels from local docker-context-files when building image.       │ -│  --airflow-is-in-context                   If set Airflow is installed from docker-context-files only rather than    │ -│                                            from PyPI or sources.                                                     │ -│  --cleanup-context                         Clean up docker context files before running build (cannot be used        │ -│                                            together with --install-packages-from-context).                           │ -│  --disable-mysql-client-installation       Do not install MySQL client.                                              │ -│  --disable-mssql-client-installation       Do not install MsSQl client.                                              │ -│  --disable-postgres-client-installation    Do not install Postgres client.                                           │ -│  --disable-airflow-repo-cache              Disable cache from Airflow repository during building.                    │ -│  --install-airflow-reference               Install Airflow using GitHub tag or branch. (TEXT)                        │ -│  --installation-method                     Install Airflow from: sources or PyPI. (. | apache-airflow)               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│  --github-token                The token used to authenticate to GitHub. (TEXT)                                      │ -│  --github-username             The user name used to authenticate to GitHub. (TEXT)                                  │ -│  --login-to-github-registry    Logs in to GitHub registry. (TEXT)                                                    │ -│  --push-image                  Push image after building it.                                                         │ -│  --prepare-buildx-cache        Prepares build cache rather than build images locally.                                │ -│  --platform                    Platform for Airflow image. (linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)     │ -│  --empty-image                 Prepare empty image tagged with the same name as the Airflow image.                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --github-repository  -g  GitHub repository used to pull, push run images. (TEXT) [default: apache/airflow]          │ -│  --answer             -a  Force answer to questions. (y | n | q | yes | no | quit)                                   │ -│  --dry-run            -D  If dry-run is set, commands are only printed, not executed.                                │ -│  --verbose            -v  Print verbose information about performed steps.                                           │ -│  --help               -h  Show this message and exit.                                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +                                                                                                                          + Usage: breeze build-prod-image [OPTIONS]                                                                                +                                                                                                                         + Build Production image. Include building multiple images for all or selected Python versions sequentially.              +                                                                                                                         +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --python                         -p  Python major/minor version used in Airflow image for images.                   │ +│                                       (>3.7< | 3.8 | 3.9 | 3.10)                                                     │ +│                                       [default: 3.7]                                                                 │ +│  --install-airflow-version        -V  Install version of Airflow from PyPI. (TEXT)                                   │ +│  --upgrade-to-newer-dependencies  -u  When set, upgrade all PIP packages to latest.                                  │ +│  --debian-version                     Debian version used for the image. (bullseye | buster) [default: bullseye]     │ +│  --image-tag                      -t  Tag added to the default naming conventions of Airflow CI/PROD images. (TEXT)  │ +│  --docker-cache                   -c  Cache option for image used during the build. (registry | local | disabled)    │ +│                                       [default: registry]                                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building multiple images ───────────────────────────────────────────────────────────────────────────────────────────╮ +│  --build-multiple-images    Run the operation sequentially on all or selected subset of Python versions.             │ +│  --python-versions          Space separated list of python versions used for build with multiple versions. (TEXT)    │ +│                             [default: 3.7 3.8 3.9 3.10]                                                              │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ +│  --install-providers-from-sources    Install providers from sources when installing.                                 │ +│  --airflow-extras                    Extras to install by default.                                                   │ +│                                      (TEXT)                                                                          │ +│                                      [default:                                                                       │ +│                                      amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,goog…  │ +│  --airflow-constraints-mode          Mode of constraints for PROD image building                                     │ +│                                      (constraints | constraints-no-providers | constraints-source-providers)         │ +│                                      [default: constraints]                                                          │ +│  --additional-python-deps            Additional python dependencies to use when building the images. (TEXT)          │ +│  --additional-extras                 Additional extra package while installing Airflow in the image. (TEXT)          │ +│  --additional-runtime-apt-deps       Additional apt runtime dependencies to use when building the images. (TEXT)     │ +│  --additional-runtime-apt-env        Additional environment variables set when adding runtime dependencies. (TEXT)   │ +│  --additional-runtime-apt-command    Additional command executed before runtime apt deps are installed. (TEXT)       │ +│  --additional-dev-apt-deps           Additional apt dev dependencies to use when building the images. (TEXT)         │ +│  --additional-dev-apt-env            Additional environment variables set when adding dev dependencies. (TEXT)       │ +│  --additional-dev-apt-command        Additional command executed before dev apt deps are installed. (TEXT)           │ +│  --runtime-apt-deps                  Apt runtime dependencies to use when building the images. (TEXT)                │ +│  --runtime-apt-command               Command executed before runtime apt deps are installed. (TEXT)                  │ +│  --dev-apt-deps                      Apt dev dependencies to use when building the images. (TEXT)                    │ +│  --dev-apt-command                   Command executed before dev apt deps are installed. (TEXT)                      │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ +│  --install-packages-from-context           Install wheels from local docker-context-files when building image.       │ +│  --airflow-is-in-context                   If set Airflow is installed from docker-context-files only rather than    │ +│                                            from PyPI or sources.                                                     │ +│  --cleanup-context                         Clean up docker context files before running build (cannot be used        │ +│                                            together with --install-packages-from-context).                           │ +│  --disable-mysql-client-installation       Do not install MySQL client.                                              │ +│  --disable-mssql-client-installation       Do not install MsSQl client.                                              │ +│  --disable-postgres-client-installation    Do not install Postgres client.                                           │ +│  --disable-airflow-repo-cache              Disable cache from Airflow repository during building.                    │ +│  --install-airflow-reference               Install Airflow using GitHub tag or branch. (TEXT)                        │ +│  --installation-method                     Install Airflow from: sources or PyPI. (. | apache-airflow)               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│  --github-token            The token used to authenticate to GitHub. (TEXT)                                          │ +│  --github-username         The user name used to authenticate to GitHub. (TEXT)                                      │ +│  --push-image              Push image after building it.                                                             │ +│  --prepare-buildx-cache    Prepares build cache additionally to building images (this is done as two separate steps  │ +│                            afterthe images are build). Implies --push-image flag                                     │ +│  --platform                Platform for Airflow image. (linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)         │ +│  --empty-image             Prepare empty image tagged with the same name as the Airflow image.                       │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --github-repository  -g  GitHub repository used to pull, push run images. (TEXT) [default: apache/airflow]          │ +│  --answer             -a  Force answer to questions. (y | n | q | yes | no | quit)                                   │ +│  --dry-run            -D  If dry-run is set, commands are only printed, not executed.                                │ +│  --verbose            -v  Print verbose information about performed steps.                                           │ +│  --help               -h  Show this message and exit.                                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 7d8178df9b4ed..6d6a62d6f2350 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -1 +1 @@ -0a16b81c7f52b3cf0f077c5282f2ad53 +a2dce7610dac6e6cbf562b4548879a48 diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index 0086d8b3cfc50..908690bad9ac8 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -1,4 +1,4 @@ - + - Breeze commands + Breeze commands -                                                                                                                          - Usage: breeze [OPTIONS] COMMAND [ARGS]...                                                                               -                                                                                                                         -╭─ Basic flags for the default (shell) command ────────────────────────────────────────────────────────────────────────╮ -│  --python               -p  Python major/minor version used in Airflow image for images. (>3.7< | 3.8 | 3.9 | 3.10)  │ -│                             [default: 3.7]                                                                           │ -│  --backend              -b  Database backend to use. (>sqlite< | mysql | postgres | mssql) [default: sqlite]         │ -│  --postgres-version     -P  Version of Postgres used. (>10< | 11 | 12 | 13) [default: 10]                            │ -│  --mysql-version        -M  Version of MySQL used. (>5.7< | 8) [default: 5.7]                                        │ -│  --mssql-version        -S  Version of MsSQL used. (>2017-latest< | 2019-latest) [default: 2017-latest]              │ -│  --integration              Integration(s) to enable when running (can be more than one).                            │ -│                             (cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ -│                             all)                                                                                     │ -│  --forward-credentials  -f  Forward local credentials to container when running.                                     │ -│  --db-reset             -d  Reset DB when entering the container.                                                    │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flags for the default (shell) command ─────────────────────────────────────────────────────────────────────╮ -│  --use-airflow-version       Use (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or  │ -│                              `sdist` if Airflow should be removed, installed from wheel packages or sdist packages   │ -│                              available in dist folder respectively. Important! Using it Implies --mount-sources      │ -│                              `none`.                                                                                 │ -│                              (none | wheel | sdist | <airflow_version>)                                              │ -│  --airflow-extras            Airflow extras to install when --use-airflow-version is used (TEXT)                     │ -│  --use-packages-from-dist    Install all found packages (--package-format determines type) from 'dist' folder when   │ -│                              entering breeze.                                                                        │ -│  --package-format            Format of packages that should be installed from dist. (wheel | sdist)                  │ -│                              [default: wheel]                                                                        │ -│  --force-build               Force image build no matter if it is determined as needed.                              │ -│  --mount-sources             Choose scope of local sources should be mounted (default = selected).                   │ -│                              (selected | all | none)                                                                 │ -│                              [default: selected]                                                                     │ -│  --debian-version            Debian version used for the image. (bullseye | buster) [default: bullseye]              │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --verbose            -v  Print verbose information about performed steps.                                           │ -│  --dry-run            -D  If dry-run is set, commands are only printed, not executed.                                │ -│  --github-repository  -g  GitHub repository used to pull, push run images. (TEXT) [default: apache/airflow]          │ -│  --answer             -a  Force answer to questions. (y | n | q | yes | no | quit)                                   │ -│  --help               -h  Show this message and exit.                                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Developer tools ────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  shell          Enter breeze.py environment. this is the default command use when no other is selected.              │ -│  start-airflow  Enter breeze.py environment and starts all Airflow components in the tmux session.                   │ -│  exec           Joins the interactive shell of running airflow container                                             │ -│  stop           Stop running breeze environment.                                                                     │ -│  build-docs     Build documentation in the container.                                                                │ -│  static-checks  Run static checks.                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  docker-compose-tests  Run docker-compose tests.                                                                     │ -│  tests                 Run the specified unit test targets. Multiple targets may be specified separated by spaces.   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Configuration & maintenance ────────────────────────────────────────────────────────────────────────────────────────╮ -│  cleanup              Cleans the cache of parameters, docker cache and optionally - currently downloaded images.     │ -│  self-upgrade         Self upgrade Breeze.                                                                           │ -│  setup-autocomplete   Enables autocompletion of breeze commands.                                                     │ -│  config               Show/update configuration (Python, Backend, Cheatsheet, ASCIIART).                             │ -│  resource-check       Check if available docker resources are enough.                                                │ -│  free-space           Free space for jobs run in CI.                                                                 │ -│  fix-ownership        Fix ownership of source files to be same as host user.                                         │ -│  command-hash-export  Outputs hash of all click commands to file or stdout if `-` is used (useful to see if images   │ -│                       should be regenerated).                                                                        │ -│  version              Print information about version of apache-airflow-breeze.                                      │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ CI Image tools ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  build-image   Build CI image. Include building multiple images for all python versions (sequentially).              │ -│  pull-image    Pull and optionally verify CI images - possibly in parallel for all Python versions.                  │ -│  verify-image  Verify CI image.                                                                                      │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Production Image tools ─────────────────────────────────────────────────────────────────────────────────────────────╮ -│  build-prod-image   Build Production image. Include building multiple images for all or selected Python versions     │ -│                     sequentially.                                                                                    │ -│  pull-prod-image    Pull and optionally verify Production images - possibly in parallel for all Python versions.     │ -│  verify-prod-image  Verify Production image.                                                                         │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Release management ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  verify-provider-packages        Verifies if all provider code is following expectations for providers.              │ -│  prepare-provider-documentation  Prepare CHANGELOG, README and COMMITS information for providers.                    │ -│  prepare-provider-packages       Prepare sdist/whl packages of Airflow Providers.                                    │ -│  prepare-airflow-package         Prepare sdist/whl package of Airflow.                                               │ -│  release-prod-images             Release production images to DockerHub (needs DockerHub permissions).               │ -│  generate-constraints            Generates pinned constraint files with all extras from setup.py in parallel.        │ -│  find-newer-dependencies         Finds which dependencies are being upgraded.                                        │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +                                                                                                                          + Usage: breeze [OPTIONS] COMMAND [ARGS]...                                                                               +                                                                                                                         +╭─ Basic flags for the default (shell) command ────────────────────────────────────────────────────────────────────────╮ +│  --python               -p  Python major/minor version used in Airflow image for images. (>3.7< | 3.8 | 3.9 | 3.10)  │ +│                             [default: 3.7]                                                                           │ +│  --backend              -b  Database backend to use. (>sqlite< | mysql | postgres | mssql) [default: sqlite]         │ +│  --postgres-version     -P  Version of Postgres used. (>10< | 11 | 12 | 13) [default: 10]                            │ +│  --mysql-version        -M  Version of MySQL used. (>5.7< | 8) [default: 5.7]                                        │ +│  --mssql-version        -S  Version of MsSQL used. (>2017-latest< | 2019-latest) [default: 2017-latest]              │ +│  --integration              Integration(s) to enable when running (can be more than one).                            │ +│                             (cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ +│                             all)                                                                                     │ +│  --forward-credentials  -f  Forward local credentials to container when running.                                     │ +│  --db-reset             -d  Reset DB when entering the container.                                                    │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flags for the default (shell) command ─────────────────────────────────────────────────────────────────────╮ +│  --use-airflow-version       Use (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or  │ +│                              `sdist` if Airflow should be removed, installed from wheel packages or sdist packages   │ +│                              available in dist folder respectively. Implies --mount-sources `none`.                  │ +│                              (none | wheel | sdist | <airflow_version>)                                              │ +│  --airflow-extras            Airflow extras to install when --use-airflow-version is used (TEXT)                     │ +│  --use-packages-from-dist    Install all found packages (--package-format determines type) from 'dist' folder when   │ +│                              entering breeze.                                                                        │ +│  --package-format            Format of packages that should be installed from dist. (wheel | sdist)                  │ +│                              [default: wheel]                                                                        │ +│  --force-build               Force image build no matter if it is determined as needed.                              │ +│  --mount-sources             Choose scope of local sources should be mounted (default = selected).                   │ +│                              (selected | all | none)                                                                 │ +│                              [default: selected]                                                                     │ +│  --debian-version            Debian version used for the image. (bullseye | buster) [default: bullseye]              │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --verbose            -v  Print verbose information about performed steps.                                           │ +│  --dry-run            -D  If dry-run is set, commands are only printed, not executed.                                │ +│  --github-repository  -g  GitHub repository used to pull, push run images. (TEXT) [default: apache/airflow]          │ +│  --answer             -a  Force answer to questions. (y | n | q | yes | no | quit)                                   │ +│  --help               -h  Show this message and exit.                                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Developer tools ────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  shell          Enter breeze.py environment. this is the default command use when no other is selected.              │ +│  start-airflow  Enter breeze.py environment and starts all Airflow components in the tmux session.                   │ +│  exec           Joins the interactive shell of running airflow container                                             │ +│  stop           Stop running breeze environment.                                                                     │ +│  build-docs     Build documentation in the container.                                                                │ +│  static-checks  Run static checks.                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  docker-compose-tests  Run docker-compose tests.                                                                     │ +│  tests                 Run the specified unit test targets. Multiple targets may be specified separated by spaces.   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Configuration & maintenance ────────────────────────────────────────────────────────────────────────────────────────╮ +│  cleanup              Cleans the cache of parameters, docker cache and optionally - currently downloaded images.     │ +│  self-upgrade         Self upgrade Breeze.                                                                           │ +│  setup-autocomplete   Enables autocompletion of breeze commands.                                                     │ +│  config               Show/update configuration (Python, Backend, Cheatsheet, ASCIIART).                             │ +│  resource-check       Check if available docker resources are enough.                                                │ +│  free-space           Free space for jobs run in CI.                                                                 │ +│  fix-ownership        Fix ownership of source files to be same as host user.                                         │ +│  command-hash-export  Outputs hash of all click commands to file or stdout if `-` is used (useful to see if images   │ +│                       should be regenerated).                                                                        │ +│  version              Print information about version of apache-airflow-breeze.                                      │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ CI Image tools ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  build-image   Build CI image. Include building multiple images for all python versions (sequentially).              │ +│  pull-image    Pull and optionally verify CI images - possibly in parallel for all Python versions.                  │ +│  verify-image  Verify CI image.                                                                                      │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Production Image tools ─────────────────────────────────────────────────────────────────────────────────────────────╮ +│  build-prod-image   Build Production image. Include building multiple images for all or selected Python versions     │ +│                     sequentially.                                                                                    │ +│  pull-prod-image    Pull and optionally verify Production images - possibly in parallel for all Python versions.     │ +│  verify-prod-image  Verify Production image.                                                                         │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Release management ─────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  verify-provider-packages        Verifies if all provider code is following expectations for providers.              │ +│  prepare-provider-documentation  Prepare CHANGELOG, README and COMMITS information for providers.                    │ +│  prepare-provider-packages       Prepare sdist/whl packages of Airflow Providers.                                    │ +│  prepare-airflow-package         Prepare sdist/whl package of Airflow.                                               │ +│  release-prod-images             Release production images to DockerHub (needs DockerHub permissions).               │ +│  generate-constraints            Generates pinned constraint files with all extras from setup.py in parallel.        │ +│  find-newer-dependencies         Finds which dependencies are being upgraded.                                        │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-generate-constraints.svg b/images/breeze/output-generate-constraints.svg index e3f755c7c2ce8..ac2afa05a16bb 100644 --- a/images/breeze/output-generate-constraints.svg +++ b/images/breeze/output-generate-constraints.svg @@ -1,4 +1,4 @@ - + - Command: generate-constraints + Command: generate-constraints -                                                                                                                          - Usage: breeze generate-constraints [OPTIONS]                                                                            -                                                                                                                         - Generates pinned constraint files with all extras from setup.py in parallel.                                            -                                                                                                                         -╭─ Generate constraints flags ─────────────────────────────────────────────────────────────────────────────────────────╮ -│  --image-tag                  -t  Tag added to the default naming conventions of Airflow CI/PROD images. (TEXT)      │ -│  --python                     -p  Python major/minor version used in Airflow image for images.                       │ -│                                   (>3.7< | 3.8 | 3.9 | 3.10)                                                         │ -│                                   [default: 3.7]                                                                     │ -│  --generate-constraints-mode      Mode of generating constraints (source-providers | pypi-providers | no-providers)  │ -│                                   [default: source-providers]                                                        │ -│  --debug                          Drop user in shell instead of running the command. Useful for debugging.           │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --run-in-parallel    Run the operation in parallel on all or selected subset of Python versions.                    │ -│  --parallelism        Maximum number of processes to use while running the operation in parallel. (INTEGER RANGE)    │ -│                       [default: 4; 1<=x<=8]                                                                          │ -│  --python-versions    Space separated list of python versions used for build with multiple versions. (TEXT)          │ -│                       [default: 3.7 3.8 3.9 3.10]                                                                    │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --verbose            -v  Print verbose information about performed steps.                                           │ -│  --dry-run            -D  If dry-run is set, commands are only printed, not executed.                                │ -│  --github-repository  -g  GitHub repository used to pull, push run images. (TEXT) [default: apache/airflow]          │ -│  --answer             -a  Force answer to questions. (y | n | q | yes | no | quit)                                   │ -│  --help               -h  Show this message and exit.                                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +                                                                                                                          + Usage: breeze generate-constraints [OPTIONS]                                                                            +                                                                                                                         + Generates pinned constraint files with all extras from setup.py in parallel.                                            +                                                                                                                         +╭─ Generate constraints flags ─────────────────────────────────────────────────────────────────────────────────────────╮ +│  --image-tag                 -t  Tag added to the default naming conventions of Airflow CI/PROD images. (TEXT)       │ +│  --python                    -p  Python major/minor version used in Airflow image for images.                        │ +│                                  (>3.7< | 3.8 | 3.9 | 3.10)                                                          │ +│                                  [default: 3.7]                                                                      │ +│  --airflow-constraints-mode      Mode of constraints for CI image building                                           │ +│                                  (constraints-source-providers | constraints | constraints-no-providers)             │ +│                                  [default: constraints-source-providers]                                             │ +│  --debug                         Drop user in shell instead of running the command. Useful for debugging.            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --run-in-parallel    Run the operation in parallel on all or selected subset of Python versions.                    │ +│  --parallelism        Maximum number of processes to use while running the operation in parallel. (INTEGER RANGE)    │ +│                       [default: 4; 1<=x<=8]                                                                          │ +│  --python-versions    Space separated list of python versions used for build with multiple versions. (TEXT)          │ +│                       [default: 3.7 3.8 3.9 3.10]                                                                    │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --verbose            -v  Print verbose information about performed steps.                                           │ +│  --dry-run            -D  If dry-run is set, commands are only printed, not executed.                                │ +│  --github-repository  -g  GitHub repository used to pull, push run images. (TEXT) [default: apache/airflow]          │ +│  --answer             -a  Force answer to questions. (y | n | q | yes | no | quit)                                   │ +│  --help               -h  Show this message and exit.                                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-shell.svg b/images/breeze/output-shell.svg index 1372b7eb02a6e..12634572fab1f 100644 --- a/images/breeze/output-shell.svg +++ b/images/breeze/output-shell.svg @@ -1,4 +1,4 @@ - + - Command: shell + Command: shell -                                                                                                                          - Usage: breeze shell [OPTIONS] [EXTRA_ARGS]...                                                                           -                                                                                                                         - Enter breeze.py environment. this is the default command use when no other is selected.                                 -                                                                                                                         -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --python               -p  Python major/minor version used in Airflow image for images. (>3.7< | 3.8 | 3.9 | 3.10)  │ -│                             [default: 3.7]                                                                           │ -│  --backend              -b  Database backend to use. (>sqlite< | mysql | postgres | mssql) [default: sqlite]         │ -│  --postgres-version     -P  Version of Postgres used. (>10< | 11 | 12 | 13) [default: 10]                            │ -│  --mysql-version        -M  Version of MySQL used. (>5.7< | 8) [default: 5.7]                                        │ -│  --mssql-version        -S  Version of MsSQL used. (>2017-latest< | 2019-latest) [default: 2017-latest]              │ -│  --integration              Integration(s) to enable when running (can be more than one).                            │ -│                             (cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ -│                             all)                                                                                     │ -│  --forward-credentials  -f  Forward local credentials to container when running.                                     │ -│  --db-reset             -d  Reset DB when entering the container.                                                    │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ -│  --use-airflow-version       Use (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or  │ -│                              `sdist` if Airflow should be removed, installed from wheel packages or sdist packages   │ -│                              available in dist folder respectively. Important! Using it Implies --mount-sources      │ -│                              `none`.                                                                                 │ -│                              (none | wheel | sdist | <airflow_version>)                                              │ -│  --airflow-extras            Airflow extras to install when --use-airflow-version is used (TEXT)                     │ -│  --use-packages-from-dist    Install all found packages (--package-format determines type) from 'dist' folder when   │ -│                              entering breeze.                                                                        │ -│  --package-format            Format of packages that should be installed from dist. (wheel | sdist)                  │ -│                              [default: wheel]                                                                        │ -│  --force-build               Force image build no matter if it is determined as needed.                              │ -│  --mount-sources             Choose scope of local sources should be mounted (default = selected).                   │ -│                              (selected | all | none)                                                                 │ -│                              [default: selected]                                                                     │ -│  --debian-version            Debian version used for the image. (bullseye | buster) [default: bullseye]              │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --verbose                        -v  Print verbose information about performed steps.                               │ -│  --dry-run                        -D  If dry-run is set, commands are only printed, not executed.                    │ -│  --github-repository              -g  GitHub repository used to pull, push run images. (TEXT)                        │ -│                                       [default: apache/airflow]                                                      │ -│  --airflow-constraints-reference      Constraint reference to use. Useful with --use-airflow-version parameter to    │ -│                                       specify constraints for the installed version and to find newer dependencies   │ -│                                       (TEXT)                                                                         │ -│  --answer                         -a  Force answer to questions. (y | n | q | yes | no | quit)                       │ -│  --help                           -h  Show this message and exit.                                                    │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +                                                                                                                          + Usage: breeze shell [OPTIONS] [EXTRA_ARGS]...                                                                           +                                                                                                                         + Enter breeze.py environment. this is the default command use when no other is selected.                                 +                                                                                                                         +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --python               -p  Python major/minor version used in Airflow image for images. (>3.7< | 3.8 | 3.9 | 3.10)  │ +│                             [default: 3.7]                                                                           │ +│  --backend              -b  Database backend to use. (>sqlite< | mysql | postgres | mssql) [default: sqlite]         │ +│  --postgres-version     -P  Version of Postgres used. (>10< | 11 | 12 | 13) [default: 10]                            │ +│  --mysql-version        -M  Version of MySQL used. (>5.7< | 8) [default: 5.7]                                        │ +│  --mssql-version        -S  Version of MsSQL used. (>2017-latest< | 2019-latest) [default: 2017-latest]              │ +│  --integration              Integration(s) to enable when running (can be more than one).                            │ +│                             (cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ +│                             all)                                                                                     │ +│  --forward-credentials  -f  Forward local credentials to container when running.                                     │ +│  --db-reset             -d  Reset DB when entering the container.                                                    │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ +│  --use-airflow-version       Use (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or  │ +│                              `sdist` if Airflow should be removed, installed from wheel packages or sdist packages   │ +│                              available in dist folder respectively. Implies --mount-sources `none`.                  │ +│                              (none | wheel | sdist | <airflow_version>)                                              │ +│  --airflow-extras            Airflow extras to install when --use-airflow-version is used (TEXT)                     │ +│  --use-packages-from-dist    Install all found packages (--package-format determines type) from 'dist' folder when   │ +│                              entering breeze.                                                                        │ +│  --package-format            Format of packages that should be installed from dist. (wheel | sdist)                  │ +│                              [default: wheel]                                                                        │ +│  --force-build               Force image build no matter if it is determined as needed.                              │ +│  --mount-sources             Choose scope of local sources should be mounted (default = selected).                   │ +│                              (selected | all | none)                                                                 │ +│                              [default: selected]                                                                     │ +│  --debian-version            Debian version used for the image. (bullseye | buster) [default: bullseye]              │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --verbose                        -v  Print verbose information about performed steps.                               │ +│  --dry-run                        -D  If dry-run is set, commands are only printed, not executed.                    │ +│  --github-repository              -g  GitHub repository used to pull, push run images. (TEXT)                        │ +│                                       [default: apache/airflow]                                                      │ +│  --airflow-constraints-reference      Constraint reference to use. Useful with --use-airflow-version parameter to    │ +│                                       specify constraints for the installed version and to find newer dependencies   │ +│                                       (TEXT)                                                                         │ +│  --answer                         -a  Force answer to questions. (y | n | q | yes | no | quit)                       │ +│  --help                           -h  Show this message and exit.                                                    │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-start-airflow.svg b/images/breeze/output-start-airflow.svg index b565d09c494bf..743a0f62eafd8 100644 --- a/images/breeze/output-start-airflow.svg +++ b/images/breeze/output-start-airflow.svg @@ -1,4 +1,4 @@ - + - Command: start-airflow + Command: start-airflow -                                                                                                                          - Usage: breeze start-airflow [OPTIONS] [EXTRA_ARGS]...                                                                   -                                                                                                                         - Enter breeze.py environment and starts all Airflow components in the tmux session.                                      -                                                                                                                         -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --python                    -p  Python major/minor version used in Airflow image for images.                        │ -│                                  (>3.7< | 3.8 | 3.9 | 3.10)                                                          │ -│                                  [default: 3.7]                                                                      │ -│  --load-example-dags         -e  Enable configuration to load example DAGs when starting Airflow.                    │ -│  --load-default-connections  -c  Enable configuration to load default connections when starting Airflow.             │ -│  --backend                   -b  Database backend to use. (>sqlite< | mysql | postgres | mssql) [default: sqlite]    │ -│  --postgres-version          -P  Version of Postgres used. (>10< | 11 | 12 | 13) [default: 10]                       │ -│  --mysql-version             -M  Version of MySQL used. (>5.7< | 8) [default: 5.7]                                   │ -│  --mssql-version             -S  Version of MsSQL used. (>2017-latest< | 2019-latest) [default: 2017-latest]         │ -│  --integration                   Integration(s) to enable when running (can be more than one).                       │ -│                                  (cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd |      │ -│                                  trino | all)                                                                        │ -│  --forward-credentials       -f  Forward local credentials to container when running.                                │ -│  --db-reset                  -d  Reset DB when entering the container.                                               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ -│  --use-airflow-version       Use (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or  │ -│                              `sdist` if Airflow should be removed, installed from wheel packages or sdist packages   │ -│                              available in dist folder respectively. Important! Using it Implies --mount-sources      │ -│                              `none`.                                                                                 │ -│                              (none | wheel | sdist | <airflow_version>)                                              │ -│  --airflow-extras            Airflow extras to install when --use-airflow-version is used (TEXT)                     │ -│  --use-packages-from-dist    Install all found packages (--package-format determines type) from 'dist' folder when   │ -│                              entering breeze.                                                                        │ -│  --package-format            Format of packages that should be installed from dist. (wheel | sdist)                  │ -│                              [default: wheel]                                                                        │ -│  --force-build               Force image build no matter if it is determined as needed.                              │ -│  --mount-sources             Choose scope of local sources should be mounted (default = selected).                   │ -│                              (selected | all | none)                                                                 │ -│                              [default: selected]                                                                     │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --dry-run                        -D  If dry-run is set, commands are only printed, not executed.                    │ -│  --github-repository              -g  GitHub repository used to pull, push run images. (TEXT)                        │ -│                                       [default: apache/airflow]                                                      │ -│  --airflow-constraints-reference      Constraint reference to use. Useful with --use-airflow-version parameter to    │ -│                                       specify constraints for the installed version and to find newer dependencies   │ -│                                       (TEXT)                                                                         │ -│  --answer                         -a  Force answer to questions. (y | n | q | yes | no | quit)                       │ -│  --verbose                        -v  Print verbose information about performed steps.                               │ -│  --help                           -h  Show this message and exit.                                                    │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +                                                                                                                          + Usage: breeze start-airflow [OPTIONS] [EXTRA_ARGS]...                                                                   +                                                                                                                         + Enter breeze.py environment and starts all Airflow components in the tmux session.                                      +                                                                                                                         +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --python                    -p  Python major/minor version used in Airflow image for images.                        │ +│                                  (>3.7< | 3.8 | 3.9 | 3.10)                                                          │ +│                                  [default: 3.7]                                                                      │ +│  --load-example-dags         -e  Enable configuration to load example DAGs when starting Airflow.                    │ +│  --load-default-connections  -c  Enable configuration to load default connections when starting Airflow.             │ +│  --backend                   -b  Database backend to use. (>sqlite< | mysql | postgres | mssql) [default: sqlite]    │ +│  --postgres-version          -P  Version of Postgres used. (>10< | 11 | 12 | 13) [default: 10]                       │ +│  --mysql-version             -M  Version of MySQL used. (>5.7< | 8) [default: 5.7]                                   │ +│  --mssql-version             -S  Version of MsSQL used. (>2017-latest< | 2019-latest) [default: 2017-latest]         │ +│  --integration                   Integration(s) to enable when running (can be more than one).                       │ +│                                  (cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd |      │ +│                                  trino | all)                                                                        │ +│  --forward-credentials       -f  Forward local credentials to container when running.                                │ +│  --db-reset                  -d  Reset DB when entering the container.                                               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ +│  --use-airflow-version       Use (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or  │ +│                              `sdist` if Airflow should be removed, installed from wheel packages or sdist packages   │ +│                              available in dist folder respectively. Implies --mount-sources `none`.                  │ +│                              (none | wheel | sdist | <airflow_version>)                                              │ +│  --airflow-extras            Airflow extras to install when --use-airflow-version is used (TEXT)                     │ +│  --use-packages-from-dist    Install all found packages (--package-format determines type) from 'dist' folder when   │ +│                              entering breeze.                                                                        │ +│  --package-format            Format of packages that should be installed from dist. (wheel | sdist)                  │ +│                              [default: wheel]                                                                        │ +│  --force-build               Force image build no matter if it is determined as needed.                              │ +│  --mount-sources             Choose scope of local sources should be mounted (default = selected).                   │ +│                              (selected | all | none)                                                                 │ +│                              [default: selected]                                                                     │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --dry-run                        -D  If dry-run is set, commands are only printed, not executed.                    │ +│  --github-repository              -g  GitHub repository used to pull, push run images. (TEXT)                        │ +│                                       [default: apache/airflow]                                                      │ +│  --airflow-constraints-reference      Constraint reference to use. Useful with --use-airflow-version parameter to    │ +│                                       specify constraints for the installed version and to find newer dependencies   │ +│                                       (TEXT)                                                                         │ +│  --answer                         -a  Force answer to questions. (y | n | q | yes | no | quit)                       │ +│  --verbose                        -v  Print verbose information about performed steps.                               │ +│  --help                           -h  Show this message and exit.                                                    │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-verify-provider-packages.svg b/images/breeze/output-verify-provider-packages.svg index c915ba40e039e..061622d35349f 100644 --- a/images/breeze/output-verify-provider-packages.svg +++ b/images/breeze/output-verify-provider-packages.svg @@ -19,62 +19,62 @@ font-weight: 700; } - .terminal-822104624-matrix { + .terminal-3092598158-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 26.400000000000002px; font-variant-east-asian: full-width; } - .terminal-822104624-title { + .terminal-3092598158-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-822104624-r1 { fill: #c5c8c6;font-weight: bold } -.terminal-822104624-r2 { fill: #c5c8c6 } -.terminal-822104624-r3 { fill: #d0b344;font-weight: bold } -.terminal-822104624-r4 { fill: #868887 } -.terminal-822104624-r5 { fill: #68a0b3;font-weight: bold } -.terminal-822104624-r6 { fill: #8d7b39 } -.terminal-822104624-r7 { fill: #98a84b;font-weight: bold } + .terminal-3092598158-r1 { fill: #c5c8c6;font-weight: bold } +.terminal-3092598158-r2 { fill: #c5c8c6 } +.terminal-3092598158-r3 { fill: #d0b344;font-weight: bold } +.terminal-3092598158-r4 { fill: #868887 } +.terminal-3092598158-r5 { fill: #68a0b3;font-weight: bold } +.terminal-3092598158-r6 { fill: #8d7b39 } +.terminal-3092598158-r7 { fill: #98a84b;font-weight: bold } - Command: verify-provider-packages + Command: verify-provider-packages -                                                                                                                          - Usage: breeze verify-provider-packages [OPTIONS]                                                                        -                                                                                                                         - Verifies if all provider code is following expectations for providers.                                                  -                                                                                                                         -╭─ Provider verification flags ────────────────────────────────────────────────────────────────────────────────────────╮ -│  --use-airflow-version              Use (reinstall at entry) Airflow version from PyPI. It can also be `none`,       │ -│                                     `wheel`, or `sdist` if Airflow should be removed, installed from wheel packages  │ -│                                     or sdist packages available in dist folder respectively. Important! Using it     │ -│                                     Implies --mount-sources `none`.                                                  │ -│                                     (none | wheel | sdist | <airflow_version>)                                       │ -│  --airflow-constraints-reference    Constraint reference to use. Useful with --use-airflow-version parameter to      │ -│                                     specify constraints for the installed version and to find newer dependencies     │ -│                                     (TEXT)                                                                           │ -│  --airflow-extras                   Airflow extras to install when --use-airflow-version is used (TEXT)              │ -│  --use-packages-from-dist           Install all found packages (--package-format determines type) from 'dist'        │ -│                                     folder when entering breeze.                                                     │ -│  --package-format                   Format of packages that should be installed from dist. (wheel | sdist)           │ -│                                     [default: wheel]                                                                 │ -│  --debug                            Drop user in shell instead of running the command. Useful for debugging.         │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│  --verbose            -v  Print verbose information about performed steps.                                           │ -│  --dry-run            -D  If dry-run is set, commands are only printed, not executed.                                │ -│  --github-repository  -g  GitHub repository used to pull, push run images. (TEXT) [default: apache/airflow]          │ -│  --help               -h  Show this message and exit.                                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +                                                                                                                          + Usage: breeze verify-provider-packages [OPTIONS]                                                                        +                                                                                                                         + Verifies if all provider code is following expectations for providers.                                                  +                                                                                                                         +╭─ Provider verification flags ────────────────────────────────────────────────────────────────────────────────────────╮ +│  --use-airflow-version              Use (reinstall at entry) Airflow version from PyPI. It can also be `none`,       │ +│                                     `wheel`, or `sdist` if Airflow should be removed, installed from wheel packages  │ +│                                     or sdist packages available in dist folder respectively. Implies                 │ +│                                     --mount-sources `none`.                                                          │ +│                                     (none | wheel | sdist | <airflow_version>)                                       │ +│  --airflow-constraints-reference    Constraint reference to use. Useful with --use-airflow-version parameter to      │ +│                                     specify constraints for the installed version and to find newer dependencies     │ +│                                     (TEXT)                                                                           │ +│  --airflow-extras                   Airflow extras to install when --use-airflow-version is used (TEXT)              │ +│  --use-packages-from-dist           Install all found packages (--package-format determines type) from 'dist'        │ +│                                     folder when entering breeze.                                                     │ +│  --package-format                   Format of packages that should be installed from dist. (wheel | sdist)           │ +│                                     [default: wheel]                                                                 │ +│  --debug                            Drop user in shell instead of running the command. Useful for debugging.         │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│  --verbose            -v  Print verbose information about performed steps.                                           │ +│  --dry-run            -D  If dry-run is set, commands are only printed, not executed.                                │ +│  --github-repository  -g  GitHub repository used to pull, push run images. (TEXT) [default: apache/airflow]          │ +│  --help               -h  Show this message and exit.                                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/scripts/ci/docker-compose/_docker.env b/scripts/ci/docker-compose/_docker.env index eb3098c29c06d..4edc849b57b93 100644 --- a/scripts/ci/docker-compose/_docker.env +++ b/scripts/ci/docker-compose/_docker.env @@ -41,7 +41,7 @@ HOST_GROUP_ID HOST_OS INIT_SCRIPT_FILE INSTALL_AIRFLOW_VERSION -GENERATE_CONSTRAINTS_MODE +AIRFLOW_CONSTRAINTS_MODE INSTALL_PROVIDERS_FROM_SOURCES USE_AIRFLOW_VERSION USE_PACKAGES_FROM_DIST diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml index 4e45f80702d4b..616c7ee4958a3 100644 --- a/scripts/ci/docker-compose/base.yml +++ b/scripts/ci/docker-compose/base.yml @@ -54,7 +54,7 @@ services: - HOST_OS=${HOST_OS} - INIT_SCRIPT_FILE=${INIT_SCRIPT_FILE} - INSTALL_AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION} - - GENERATE_CONSTRAINTS_MODE=${GENERATE_CONSTRAINTS_MODE} + - AIRFLOW_CONSTRAINTS_MODE=${AIRFLOW_CONSTRAINTS_MODE} - INSTALL_PROVIDERS_FROM_SOURCES=${INSTALL_PROVIDERS_FROM_SOURCES} - USE_AIRFLOW_VERSION=${USE_AIRFLOW_VERSION} - USE_PACKAGES_FROM_DIST=${USE_PACKAGES_FROM_DIST} diff --git a/scripts/ci/docker-compose/devcontainer.env b/scripts/ci/docker-compose/devcontainer.env index 0d26338fd1bf4..1c4b27b36af67 100644 --- a/scripts/ci/docker-compose/devcontainer.env +++ b/scripts/ci/docker-compose/devcontainer.env @@ -40,7 +40,7 @@ HOST_GROUP_ID= HOST_OS="Linux" INIT_SCRIPT_FILE="init.sh" INSTALL_AIRFLOW_VERSION= -GENERATE_CONSTRAINTS_MODE= +AIRFLOW_CONSTRAINTS_MODE= INSTALL_PROVIDERS_FROM_SOURCES= USE_AIRFLOW_VERSION= USE_PACKAGES_FROM_DIST= diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index 8eb5b29c35589..0f42cdb9fe716 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -453,7 +453,7 @@ function initialization::initialize_image_build_variables() { export INSTALL_AIRFLOW_REFERENCE=${INSTALL_AIRFLOW_REFERENCE:=""} # Determines which providers are used to generate constraints - source, pypi or no providers - export GENERATE_CONSTRAINTS_MODE=${GENERATE_CONSTRAINTS_MODE:="source-providers"} + export AIRFLOW_CONSTRAINTS_MODE=${AIRFLOW_CONSTRAINTS_MODE:="constraints-source-providers"} # By default we install latest airflow from PyPI or sources. You can set this parameter to false # if Airflow is in the .whl or .tar.gz packages placed in `docker-context-files` folder and you want @@ -474,7 +474,7 @@ function initialization::initialize_image_build_variables() { # * 'constraints' = for constraints with PyPI released providers (default for installations) # * 'constraints-source-providers' for constraints with source version of providers (defaults in Breeze and CI) # * 'constraints-no-providers' for constraints without providers - export AIRFLOW_CONSTRAINTS="${AIRFLOW_CONSTRAINTS:="constraints-source-providers"}" + export AIRFLOW_CONSTRAINTS_MODE="${AIRFLOW_CONSTRAINTS_MODE:="constraints-source-providers"}" # Replace airflow at runtime in CI image with the one specified # * none - just removes airflow diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh index 783565d4e52cb..29fc2273ca731 100644 --- a/scripts/docker/common.sh +++ b/scripts/docker/common.sh @@ -61,7 +61,7 @@ function common::get_constraints_location() { local constraints_base="https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${AIRFLOW_CONSTRAINTS_REFERENCE}" local python_version python_version="$(python --version 2>/dev/stdout | cut -d " " -f 2 | cut -d "." -f 1-2)" - AIRFLOW_CONSTRAINTS_LOCATION="${constraints_base}/${AIRFLOW_CONSTRAINTS}-${python_version}.txt" + AIRFLOW_CONSTRAINTS_LOCATION="${constraints_base}/${AIRFLOW_CONSTRAINTS_MODE}-${python_version}.txt" fi } diff --git a/scripts/docker/install_additional_dependencies.sh b/scripts/docker/install_additional_dependencies.sh index 81b7da6877b2c..21f41a04a734b 100644 --- a/scripts/docker/install_additional_dependencies.sh +++ b/scripts/docker/install_additional_dependencies.sh @@ -25,19 +25,18 @@ set -euo pipefail # shellcheck source=scripts/docker/common.sh . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" - -set -x - # Installs additional dependencies passed as Argument to the Docker build command function install_additional_dependencies() { if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then echo echo "${COLOR_BLUE}Installing additional dependencies while upgrading to newer dependencies${COLOR_RESET}" echo + set -x pip install --upgrade --upgrade-strategy eager \ ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} # make sure correct PIP version is used pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" echo @@ -46,10 +45,12 @@ function install_additional_dependencies() { echo echo "${COLOR_BLUE}Installing additional dependencies upgrading only if needed${COLOR_RESET}" echo + set -x pip install --upgrade --upgrade-strategy only-if-needed \ ${ADDITIONAL_PYTHON_DEPS} # make sure correct PIP version is used pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" echo diff --git a/scripts/docker/install_airflow.sh b/scripts/docker/install_airflow.sh index d2c8d6803373c..9b3743fd4ee71 100644 --- a/scripts/docker/install_airflow.sh +++ b/scripts/docker/install_airflow.sh @@ -60,9 +60,11 @@ function install_airflow() { if [[ -n "${AIRFLOW_INSTALL_EDITABLE_FLAG}" ]]; then # Remove airflow and reinstall it using editable flag # We can only do it when we install airflow from sources + set -x pip uninstall apache-airflow --yes pip install ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" + set +x fi # make sure correct PIP version is used @@ -75,6 +77,7 @@ function install_airflow() { echo echo "${COLOR_BLUE}Installing all packages with constraints and upgrade if needed${COLOR_RESET}" echo + set -x pip install ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" @@ -86,6 +89,7 @@ function install_airflow() { "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" # make sure correct PIP version is used pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" + set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" echo diff --git a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh index 4b6bc530432a5..15856cebb7a2b 100644 --- a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh +++ b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh @@ -47,12 +47,14 @@ function install_airflow_dependencies_from_branch_tip() { fi # Install latest set of dependencies using constraints. In case constraints were upgraded and there # are conflicts, this might fail, but it should be fixed in the following installation steps + set -x pip install \ "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true # make sure correct PIP version is used pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true + set +x echo echo "${COLOR_BLUE}Uninstalling just airflow. Dependencies remain. Now target airflow can be reinstalled using mostly cached dependencies${COLOR_RESET}" echo diff --git a/scripts/docker/install_from_docker_context_files.sh b/scripts/docker/install_from_docker_context_files.sh index 3cac6776b3729..80e11d357fa66 100644 --- a/scripts/docker/install_from_docker_context_files.sh +++ b/scripts/docker/install_from_docker_context_files.sh @@ -74,40 +74,19 @@ function install_airflow_and_providers_from_docker_context_files(){ return fi - if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then - echo - echo "${COLOR_BLUE}Force re-installing airflow and providers from local files with eager upgrade${COLOR_RESET}" - echo - # force reinstall all airflow + provider package local files with eager upgrade - pip install "${pip_flags[@]}" --upgrade --upgrade-strategy eager \ - ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} \ - ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} - else - echo - echo "${COLOR_BLUE}Force re-installing airflow and providers from local files with constraints and upgrade if needed${COLOR_RESET}" - echo - if [[ ${AIRFLOW_CONSTRAINTS_LOCATION} == "/"* ]]; then - grep -ve '^apache-airflow' <"${AIRFLOW_CONSTRAINTS_LOCATION}" > /tmp/constraints.txt - else - # Remove provider packages from constraint files because they are locally prepared - curl -L "${AIRFLOW_CONSTRAINTS_LOCATION}" | grep -ve '^apache-airflow' > /tmp/constraints.txt - fi - # force reinstall airflow + provider package local files with constraints + upgrade if needed - pip install "${pip_flags[@]}" --force-reinstall \ - ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} \ - --constraint /tmp/constraints.txt - rm /tmp/constraints.txt - # make sure correct PIP version is used \ - pip install "pip==${AIRFLOW_PIP_VERSION}" - # then upgrade if needed without using constraints to account for new limits in setup.py - pip install --upgrade --upgrade-strategy only-if-needed \ - ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} - fi + echo + echo "${COLOR_BLUE}Force re-installing airflow and providers from local files with eager upgrade${COLOR_RESET}" + echo + # force reinstall all airflow + provider package local files with eager upgrade + set -x + pip install "${pip_flags[@]}" --upgrade --upgrade-strategy eager \ + ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} \ + ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} + set +x # make sure correct PIP version is left installed pip install "pip==${AIRFLOW_PIP_VERSION}" pip check - } # Simply install all other (non-apache-airflow) packages placed in docker-context files @@ -123,10 +102,12 @@ function install_all_other_packages_from_docker_context_files() { # shellcheck disable=SC2010 reinstalling_other_packages=$(ls /docker-context-files/*.{whl,tar.gz} 2>/dev/null | \ grep -v apache_airflow | grep -v apache-airflow || true) - if [[ -n "${reinstalling_other_packages}" ]]; then \ + if [[ -n "${reinstalling_other_packages}" ]]; then + set -x pip install --force-reinstall --no-deps --no-index ${reinstalling_other_packages} # make sure correct PIP version is used pip install "pip==${AIRFLOW_PIP_VERSION}" + set -x fi } diff --git a/scripts/in_container/run_generate_constraints.sh b/scripts/in_container/run_generate_constraints.sh index 45cff26163f92..e85c2fb9c992a 100755 --- a/scripts/in_container/run_generate_constraints.sh +++ b/scripts/in_container/run_generate_constraints.sh @@ -20,14 +20,13 @@ CONSTRAINTS_DIR="/files/constraints-${PYTHON_MAJOR_MINOR_VERSION}" -LATEST_CONSTRAINT_FILE="${CONSTRAINTS_DIR}/original-${AIRFLOW_CONSTRAINTS}-${PYTHON_MAJOR_MINOR_VERSION}.txt" +LATEST_CONSTRAINT_FILE="${CONSTRAINTS_DIR}/original-${AIRFLOW_CONSTRAINTS_MODE}-${PYTHON_MAJOR_MINOR_VERSION}.txt" mkdir -pv "${CONSTRAINTS_DIR}" -if [[ ${GENERATE_CONSTRAINTS_MODE} == "no-providers" ]]; then - AIRFLOW_CONSTRAINTS="constraints-no-providers" +if [[ ${AIRFLOW_CONSTRAINTS_MODE} == "constraints-no-providers" ]]; then NO_PROVIDERS_EXTRAS=$(python -c 'import setup; print(",".join(setup.CORE_EXTRAS_REQUIREMENTS.keys()))') - CURRENT_CONSTRAINT_FILE="${CONSTRAINTS_DIR}/${AIRFLOW_CONSTRAINTS}-${PYTHON_MAJOR_MINOR_VERSION}.txt" + CURRENT_CONSTRAINT_FILE="${CONSTRAINTS_DIR}/${AIRFLOW_CONSTRAINTS_MODE}-${PYTHON_MAJOR_MINOR_VERSION}.txt" echo echo "UnInstall All PIP packages." echo @@ -47,9 +46,8 @@ if [[ ${GENERATE_CONSTRAINTS_MODE} == "no-providers" ]]; then # Airflow in any way. # EOF -elif [[ ${GENERATE_CONSTRAINTS_MODE} == "source-providers" ]]; then - AIRFLOW_CONSTRAINTS="constraints-source-providers" - CURRENT_CONSTRAINT_FILE="${CONSTRAINTS_DIR}/${AIRFLOW_CONSTRAINTS}-${PYTHON_MAJOR_MINOR_VERSION}.txt" +elif [[ ${AIRFLOW_CONSTRAINTS_MODE} == "constraints-source-providers" ]]; then + CURRENT_CONSTRAINT_FILE="${CONSTRAINTS_DIR}/${AIRFLOW_CONSTRAINTS_MODE}-${PYTHON_MAJOR_MINOR_VERSION}.txt" echo echo "Providers are already installed from sources." echo @@ -65,9 +63,8 @@ elif [[ ${GENERATE_CONSTRAINTS_MODE} == "source-providers" ]]; then # Install with "HEAD" of providers. Those are the only constraints that are used by our CI builds. # EOF -elif [[ ${GENERATE_CONSTRAINTS_MODE} == "pypi-providers" ]]; then - AIRFLOW_CONSTRAINTS="constraints" - CURRENT_CONSTRAINT_FILE="${CONSTRAINTS_DIR}/${AIRFLOW_CONSTRAINTS}-${PYTHON_MAJOR_MINOR_VERSION}.txt" +elif [[ ${AIRFLOW_CONSTRAINTS_MODE} == "constraints" ]]; then + CURRENT_CONSTRAINT_FILE="${CONSTRAINTS_DIR}/${AIRFLOW_CONSTRAINTS_MODE}-${PYTHON_MAJOR_MINOR_VERSION}.txt" echo echo "Install all providers from PyPI so that they are included in the constraints." echo @@ -86,14 +83,14 @@ elif [[ ${GENERATE_CONSTRAINTS_MODE} == "pypi-providers" ]]; then EOF else echo - echo "${COLOR_RED}Error! GENERATE_CONSTRAINTS_MODE has wrong value: '${GENERATE_CONSTRAINTS_MODE}' ${COLOR_RESET}" + echo "${COLOR_RED}Error! AIRFLOW_CONSTRAINTS_MODE has wrong value: '${AIRFLOW_CONSTRAINTS_MODE}' ${COLOR_RESET}" echo exit 1 fi -readonly AIRFLOW_CONSTRAINTS +readonly AIRFLOW_CONSTRAINTS_MODE -CONSTRAINTS_LOCATION="https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${DEFAULT_CONSTRAINTS_BRANCH}/${AIRFLOW_CONSTRAINTS}-${PYTHON_MAJOR_MINOR_VERSION}.txt" +CONSTRAINTS_LOCATION="https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${DEFAULT_CONSTRAINTS_BRANCH}/${AIRFLOW_CONSTRAINTS_MODE}-${PYTHON_MAJOR_MINOR_VERSION}.txt" readonly CONSTRAINTS_LOCATION touch "${LATEST_CONSTRAINT_FILE}"