Replace generation of docker volumes to be done from python

.pre-commit-config.yaml

@@ -350,10 +350,11 @@ repos:
         additional_dependencies: ['rich>=12.4.1', 'rich-click']
       - id: update-local-yml-file
         name: Update mounts in the local yml file
-        entry: ./scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh
-        language: system
-        files: ^scripts/ci/libraries/_local_mounts\.sh$|^scripts/ci/docker_compose/local\.yml$
+        entry: ./scripts/ci/pre_commit/pre_commit_local_yml_mounts.py
+        language: python
+        files: ^dev/breeze/src/airflow_breeze/utils/docker_command_utils\.py$|^scripts/ci/docker_compose/local\.yml$
         pass_filenames: false
+        additional_dependencies: ['rich>=12.4.1']
       - id: update-setup-cfg-file
         name: Update setup.cfg file with all licenses
         entry: ./scripts/ci/pre_commit/pre_commit_setup_cfg_file.sh

dev/breeze/src/airflow_breeze/commands/main_command.py

@@ -37,7 +37,7 @@
     option_use_packages_from_dist,
     option_verbose,
 )
-from airflow_breeze.utils.path_utils import create_directories
+from airflow_breeze.utils.path_utils import create_directories_and_files
 
 
 @click.group(invoke_without_command=True, context_settings={'help_option_names': ['-h', '--help']})
@@ -64,6 +64,6 @@
 def main(ctx: click.Context, **kwargs):
     from airflow_breeze.commands.developer_commands import shell
 
-    create_directories()
+    create_directories_and_files()
     if not ctx.invoked_subcommand:
         ctx.forward(shell, extra_args={})

dev/breeze/src/airflow_breeze/utils/path_utils.py

@@ -142,7 +142,11 @@ def print_warning_if_setup_changed() -> bool:
     Prints warning if detected airflow sources are not the ones that Breeze was installed with.
     :return: True if warning was printed.
     """
-    package_hash = get_package_setup_metadata_hash()
+    try:
+        package_hash = get_package_setup_metadata_hash()
+    except ModuleNotFoundError as e:
+        if "importlib_metadata" in e.msg:
+            return False
     sources_hash = get_installation_sources_config_metadata_hash()
     if sources_hash != package_hash:
         installation_sources = get_installation_airflow_sources()
@@ -251,9 +255,9 @@ def find_airflow_sources_root_to_operate_on() -> Path:
 BREEZE_SOURCES_ROOT = AIRFLOW_SOURCES_ROOT / "dev" / "breeze"
 
 
-def create_directories() -> None:
+def create_directories_and_files() -> None:
     """
-    Creates all directories that are needed for Breeze to work.
+    Creates all directories and files that are needed for Breeze to work via docker-compose.
     Checks if setup has been updates since last time and proposes to upgrade if so.
     """
     BUILD_CACHE_DIR.mkdir(parents=True, exist_ok=True)
@@ -263,3 +267,6 @@ def create_directories() -> None:
     LOGS_DIR.mkdir(parents=True, exist_ok=True)
     DIST_DIR.mkdir(parents=True, exist_ok=True)
     OUTPUT_LOG.mkdir(parents=True, exist_ok=True)
+    (AIRFLOW_SOURCES_ROOT / ".bash_aliases").touch()
+    (AIRFLOW_SOURCES_ROOT / ".bash_history").touch()
+    (AIRFLOW_SOURCES_ROOT / ".inputrc").touch()

scripts/ci/docker-compose/local.yml

@@ -26,38 +26,100 @@ services:
     # or those that might be useful to see in the host as output of the
     # tests (such as logs)
     volumes:
-      - mypy-cache-volume:/opt/airflow/.mypy_cache/
-      # START automatically generated volumes from LOCAL_MOUNTS in _local_mounts.sh
-      - ../../../.bash_aliases:/root/.bash_aliases:cached
-      - ../../../.bash_history:/root/.bash_history:cached
-      - ../../../.coveragerc:/opt/airflow/.coveragerc:cached
-      - ../../../.dockerignore:/opt/airflow/.dockerignore:cached
-      - ../../../.flake8:/opt/airflow/.flake8:cached
-      - ../../../.github:/opt/airflow/.github:cached
-      - ../../../.inputrc:/root/.inputrc:cached
-      - ../../../.rat-excludes:/opt/airflow/.rat-excludes:cached
-      - ../../../RELEASE_NOTES.rst:/opt/airflow/RELEASE_NOTES.rst:cached
-      - ../../../LICENSE:/opt/airflow/LICENSE:cached
-      - ../../../MANIFEST.in:/opt/airflow/MANIFEST.in:cached
-      - ../../../NOTICE:/opt/airflow/NOTICE:cached
-      - ../../../airflow:/opt/airflow/airflow:cached
-      - ../../../provider_packages:/opt/airflow/provider_packages:cached
-      - ../../../dags:/opt/airflow/dags:cached
-      - ../../../dev:/opt/airflow/dev:cached
-      - ../../../docs:/opt/airflow/docs:cached
-      - ../../../hooks:/opt/airflow/hooks:cached
-      - ../../../logs:/root/airflow/logs:cached
-      - ../../../pyproject.toml:/opt/airflow/pyproject.toml:cached
-      - ../../../pytest.ini:/opt/airflow/pytest.ini:cached
-      - ../../../scripts:/opt/airflow/scripts:cached
-      - ../../../scripts/docker/entrypoint_ci.sh:/entrypoint:cached
-      - ../../../setup.cfg:/opt/airflow/setup.cfg:cached
-      - ../../../setup.py:/opt/airflow/setup.py:cached
-      - ../../../tests:/opt/airflow/tests:cached
-      - ../../../kubernetes_tests:/opt/airflow/kubernetes_tests:cached
-      - ../../../docker_tests:/opt/airflow/docker_tests:cached
-      - ../../../chart:/opt/airflow/chart:cached
-      - ../../../metastore_browser:/opt/airflow/metastore_browser:cached
-      # END automatically generated volumes from LOCAL_MOUNTS in _local_mounts.sh
+      - type: volume
+        source: mypy-cache-volume
+        target: /opt/airflow/.mypy_cache/
+        # START automatically generated volumes from NECESSARY_HOST_VOLUMES in docker_command_utils.py
+      - type: bind
+        source: ../../../.bash_aliases
+        target: /root/.bash_aliases
+      - type: bind
+        source: ../../../.bash_history
+        target: /root/.bash_history
+      - type: bind
+        source: ../../../.coveragerc
+        target: /opt/airflow/.coveragerc
+      - type: bind
+        source: ../../../.dockerignore
+        target: /opt/airflow/.dockerignore
+      - type: bind
+        source: ../../../.flake8
+        target: /opt/airflow/.flake8
+      - type: bind
+        source: ../../../.github
+        target: /opt/airflow/.github
+      - type: bind
+        source: ../../../.inputrc
+        target: /root/.inputrc
+      - type: bind
+        source: ../../../.rat-excludes
+        target: /opt/airflow/.rat-excludes
+      - type: bind
+        source: ../../../RELEASE_NOTES.rst
+        target: /opt/airflow/RELEASE_NOTES.rst
+      - type: bind
+        source: ../../../LICENSE
+        target: /opt/airflow/LICENSE
+      - type: bind
+        source: ../../../MANIFEST.in
+        target: /opt/airflow/MANIFEST.in
+      - type: bind
+        source: ../../../NOTICE
+        target: /opt/airflow/NOTICE
+      - type: bind
+        source: ../../../airflow
+        target: /opt/airflow/airflow
+      - type: bind
+        source: ../../../provider_packages
+        target: /opt/airflow/provider_packages
+      - type: bind
+        source: ../../../dags
+        target: /opt/airflow/dags
+      - type: bind
+        source: ../../../dev
+        target: /opt/airflow/dev
+      - type: bind
+        source: ../../../docs
+        target: /opt/airflow/docs
+      - type: bind
+        source: ../../../hooks
+        target: /opt/airflow/hooks
+      - type: bind
+        source: ../../../logs
+        target: /root/airflow/logs
+      - type: bind
+        source: ../../../pyproject.toml
+        target: /opt/airflow/pyproject.toml
+      - type: bind
+        source: ../../../pytest.ini
+        target: /opt/airflow/pytest.ini
+      - type: bind
+        source: ../../../scripts
+        target: /opt/airflow/scripts
+      - type: bind
+        source: ../../../scripts/docker/entrypoint_ci.sh
+        target: /entrypoint
+      - type: bind
+        source: ../../../setup.cfg
+        target: /opt/airflow/setup.cfg
+      - type: bind
+        source: ../../../setup.py
+        target: /opt/airflow/setup.py
+      - type: bind
+        source: ../../../tests
+        target: /opt/airflow/tests
+      - type: bind
+        source: ../../../kubernetes_tests
+        target: /opt/airflow/kubernetes_tests
+      - type: bind
+        source: ../../../docker_tests
+        target: /opt/airflow/docker_tests
+      - type: bind
+        source: ../../../chart
+        target: /opt/airflow/chart
+      - type: bind
+        source: ../../../metastore_browser
+        target: /opt/airflow/metastore_browser
+        # END automatically generated volumes from NECESSARY_HOST_VOLUMES in docker_command_utils.py
 volumes:
   mypy-cache-volume:

scripts/ci/pre_commit/pre_commit_flake8.py

@@ -32,7 +32,6 @@
 GITHUB_REPOSITORY = os.environ.get('GITHUB_REPOSITORY', "apache/airflow")
 
 if __name__ == '__main__':
-    os.environ['SKIP_BREEZE_UPGRADE_CHECK'] = "true"
     sys.path.insert(0, str(Path(__file__).parents[3].resolve() / "dev" / "breeze" / "src"))
     from airflow_breeze.branch_defaults import AIRFLOW_BRANCH
     from airflow_breeze.global_constants import MOUNT_SELECTED

scripts/ci/pre_commit/pre_commit_local_yml_mounts.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import sys
+from pathlib import Path
+
+AIRFLOW_SOURCES_DIR = Path(__file__).parents[3].resolve()
+
+sys.path.insert(0, str(Path(__file__).parent.resolve()))  # make sure common_precommit_utils is imported
+sys.path.insert(0, str(AIRFLOW_SOURCES_DIR))  # make sure setup is imported from Airflow
+sys.path.insert(
+    0, str(AIRFLOW_SOURCES_DIR / "dev" / "breeze" / "src")
+)  # make sure setup is imported from Airflow
+# flake8: noqa: F401
+
+from common_precommit_utils import insert_documentation  # isort: skip
+
+sys.path.append(str(AIRFLOW_SOURCES_DIR))
+
+MOUNTS_HEADER = (
+    '        # START automatically generated volumes from '
+    'NECESSARY_HOST_VOLUMES in docker_command_utils.py'
+)
+MOUNTS_FOOTER = (
+    '        # END automatically generated volumes from ' 'NECESSARY_HOST_VOLUMES in docker_command_utils.py'
+)
+
+if __name__ == '__main__':
+    from airflow_breeze.utils.docker_command_utils import NECESSARY_HOST_VOLUMES
+
+    local_mount_file_path = AIRFLOW_SOURCES_DIR / 'scripts' / 'ci' / 'docker-compose' / 'local.yml'
+    PREFIX = '      '
+    volumes = []
+    for (src, dest) in NECESSARY_HOST_VOLUMES:
+        volumes.extend(
+            [
+                PREFIX + "- type: bind\n",
+                PREFIX + f"  source: ../../../{src}\n",
+                PREFIX + f"  target: {dest}\n",
+            ]
+        )
+    insert_documentation(local_mount_file_path, volumes, MOUNTS_HEADER, MOUNTS_FOOTER)

scripts/ci/pre_commit/pre_commit_mypy.py

@@ -33,7 +33,6 @@
 GITHUB_REPOSITORY = os.environ.get('GITHUB_REPOSITORY', "apache/airflow")
 
 if __name__ == '__main__':
-    os.environ['SKIP_BREEZE_UPGRADE_CHECK'] = "true"
     sys.path.insert(0, str(Path(__file__).parents[3].resolve() / "dev" / "breeze" / "src"))
     from airflow_breeze.branch_defaults import AIRFLOW_BRANCH
     from airflow_breeze.global_constants import MOUNT_SELECTED