From b76c129889914702854fdf04a750d281ea6c57c6 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 2 Jun 2022 14:41:40 +0800 Subject: [PATCH 1/5] Use platform.uname() for Windows support --- dev/breeze/src/airflow_breeze/global_constants.py | 11 +++++++++-- .../src/airflow_breeze/params/common_build_params.py | 3 ++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 4ed4c298ba230d..49d68528f281f5 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -17,7 +17,7 @@ """ Global constants that are used by all other Breeze components. """ -import os +import platform from typing import List from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT @@ -137,8 +137,15 @@ def get_available_packages(short_version=False) -> List[str]: return package_list +def get_default_platform_machine() -> str: + machine = platform.uname().machine + # Some additional conversion for various platforms... + machine = {"AMD64": "x86_64"}.get(machine, machine) + return machine + + # Initialise base variables -DOCKER_DEFAULT_PLATFORM = f"linux/{os.uname().machine}" +DOCKER_DEFAULT_PLATFORM = f"linux/{get_default_platform_machine()}" DOCKER_BUILDKIT = 1 SSH_PORT = "12322" diff --git a/dev/breeze/src/airflow_breeze/params/common_build_params.py b/dev/breeze/src/airflow_breeze/params/common_build_params.py index fd8a2b82e8c8c0..8a8ab4e1ec6d4d 100644 --- a/dev/breeze/src/airflow_breeze/params/common_build_params.py +++ b/dev/breeze/src/airflow_breeze/params/common_build_params.py @@ -22,6 +22,7 @@ from typing import List, Optional from airflow_breeze.branch_defaults import AIRFLOW_BRANCH +from airflow_breeze.global_constants import DOCKER_DEFAULT_PLATFORM from airflow_breeze.utils.console import get_console from airflow_breeze.utils.platforms import get_real_platform @@ -56,7 +57,7 @@ class CommonBuildParams: github_username: str = "" image_tag: Optional[str] = None install_providers_from_sources: bool = False - platform: str = f"linux/{os.uname().machine}" + platform: str = DOCKER_DEFAULT_PLATFORM prepare_buildx_cache: bool = False push_image: bool = False python: str = "3.7" From 22d03d3586f521b520311ff666fc2c50ddc47b34 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 2 Jun 2022 15:55:32 +0800 Subject: [PATCH 2/5] Fix multi-value env var formatting on Windows --- dev/breeze/src/airflow_breeze/params/shell_params.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index 25f441a2fffd10..bf308e562d8d8f 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -222,7 +222,7 @@ def compose_files(self): if len(integrations) > 0: for integration in integrations: compose_ci_file.append(f"{str(SCRIPTS_CI_DIR)}/docker-compose/integration-{integration}.yml") - return ':'.join(compose_ci_file) + return os.pathsep.join(compose_ci_file) @property def command_passed(self): From b9b6ff4534eea2d72abf621851faf17b67aa44b4 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 2 Jun 2022 16:49:30 +0800 Subject: [PATCH 3/5] Set HOME environ if needed (for Windows) --- dev/breeze/src/airflow_breeze/utils/run_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py b/dev/breeze/src/airflow_breeze/utils/run_utils.py index f0a44b425f4bcf..9754605f49be45 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/run_utils.py @@ -96,6 +96,7 @@ def run_command( return subprocess.CompletedProcess(cmd, returncode=0) try: cmd_env = os.environ.copy() + cmd_env.setdefault("HOME", str(Path.home())) if env: cmd_env.update(env) with ci_group(title=f"Output of {title}", enabled=enabled_output_group): From 3e15b6c2fcf8e3731a8056968d5da5bf3fd2b428 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 2 Jun 2022 16:55:15 +0800 Subject: [PATCH 4/5] Change bangs in pre-commit scripts to use 'python' These are already run inside a virtual environment, so there's no need to specify 'python3'. This greatly improves compatibility on Windows, where the 'python3' command is not universally available. --- .../ci/pre_commit/pre_commit_base_operator_partial_arguments.py | 2 +- scripts/ci/pre_commit/pre_commit_boring_cyborg.py | 2 +- scripts/ci/pre_commit/pre_commit_changelog_duplicates.py | 2 +- scripts/ci/pre_commit/pre_commit_chart_schema.py | 2 +- scripts/ci/pre_commit/pre_commit_check_2_1_compatibility.py | 2 +- scripts/ci/pre_commit/pre_commit_check_extras_have_providers.py | 2 +- scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py | 2 +- scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py | 2 +- .../pre_commit_check_providers_subpackages_all_have_init.py | 2 +- scripts/ci/pre_commit/pre_commit_check_system_tests.py | 2 +- scripts/ci/pre_commit/pre_commit_checkout_no_credentials.py | 2 +- scripts/ci/pre_commit/pre_commit_docstring_param_type.py | 2 +- scripts/ci/pre_commit/pre_commit_flake8.py | 2 +- scripts/ci/pre_commit/pre_commit_inline_scripts_in_docker.py | 2 +- scripts/ci/pre_commit/pre_commit_insert_extras.py | 2 +- scripts/ci/pre_commit/pre_commit_json_schema.py | 2 +- scripts/ci/pre_commit/pre_commit_local_yml_mounts.py | 2 +- scripts/ci/pre_commit/pre_commit_migration_reference.py | 2 +- scripts/ci/pre_commit/pre_commit_mypy.py | 2 +- scripts/ci/pre_commit/pre_commit_newsfragments.py | 2 +- scripts/ci/pre_commit/pre_commit_supported_versions.py | 2 +- scripts/ci/pre_commit/pre_commit_ui_lint.py | 2 +- scripts/ci/pre_commit/pre_commit_update_breeze_config_hash.py | 2 +- scripts/ci/pre_commit/pre_commit_update_versions.py | 2 +- scripts/ci/pre_commit/pre_commit_vendor_k8s_json_schema.py | 2 +- scripts/ci/pre_commit/pre_commit_version_heads_map.py | 2 +- scripts/ci/pre_commit/pre_commit_www_lint.py | 2 +- scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py | 2 +- 28 files changed, 28 insertions(+), 28 deletions(-) diff --git a/scripts/ci/pre_commit/pre_commit_base_operator_partial_arguments.py b/scripts/ci/pre_commit/pre_commit_base_operator_partial_arguments.py index 8af31ce17c17b2..c8c07c9e80be07 100755 --- a/scripts/ci/pre_commit/pre_commit_base_operator_partial_arguments.py +++ b/scripts/ci/pre_commit/pre_commit_base_operator_partial_arguments.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/scripts/ci/pre_commit/pre_commit_boring_cyborg.py b/scripts/ci/pre_commit/pre_commit_boring_cyborg.py index 48a410aba4e72d..d0bee5077a986b 100755 --- a/scripts/ci/pre_commit/pre_commit_boring_cyborg.py +++ b/scripts/ci/pre_commit/pre_commit_boring_cyborg.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/pre_commit/pre_commit_changelog_duplicates.py b/scripts/ci/pre_commit/pre_commit_changelog_duplicates.py index b9473fc3fe3339..00387e75cc86b5 100755 --- a/scripts/ci/pre_commit/pre_commit_changelog_duplicates.py +++ b/scripts/ci/pre_commit/pre_commit_changelog_duplicates.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/scripts/ci/pre_commit/pre_commit_chart_schema.py b/scripts/ci/pre_commit/pre_commit_chart_schema.py index 234f0dca472818..30301d8d112960 100755 --- a/scripts/ci/pre_commit/pre_commit_chart_schema.py +++ b/scripts/ci/pre_commit/pre_commit_chart_schema.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/pre_commit/pre_commit_check_2_1_compatibility.py b/scripts/ci/pre_commit/pre_commit_check_2_1_compatibility.py index 0d43959ba5332b..6a5fddb24c8dee 100755 --- a/scripts/ci/pre_commit/pre_commit_check_2_1_compatibility.py +++ b/scripts/ci/pre_commit/pre_commit_check_2_1_compatibility.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/pre_commit/pre_commit_check_extras_have_providers.py b/scripts/ci/pre_commit/pre_commit_check_extras_have_providers.py index 7c1b7eb23a7419..91943d52e8107f 100755 --- a/scripts/ci/pre_commit/pre_commit_check_extras_have_providers.py +++ b/scripts/ci/pre_commit/pre_commit_check_extras_have_providers.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py b/scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py index 2a5a25639b6c5b..cfd4cb384d67a6 100755 --- a/scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py +++ b/scripts/ci/pre_commit/pre_commit_check_pre_commit_hooks.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py b/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py index 11d9eafb22d7cb..ef961031ab78c4 100755 --- a/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py +++ b/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/pre_commit/pre_commit_check_providers_subpackages_all_have_init.py b/scripts/ci/pre_commit/pre_commit_check_providers_subpackages_all_have_init.py index 44cf5ac17df547..ddd52b7a1011e1 100755 --- a/scripts/ci/pre_commit/pre_commit_check_providers_subpackages_all_have_init.py +++ b/scripts/ci/pre_commit/pre_commit_check_providers_subpackages_all_have_init.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/pre_commit/pre_commit_check_system_tests.py b/scripts/ci/pre_commit/pre_commit_check_system_tests.py index 8fc162a4211055..806877203b508e 100755 --- a/scripts/ci/pre_commit/pre_commit_check_system_tests.py +++ b/scripts/ci/pre_commit/pre_commit_check_system_tests.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/pre_commit/pre_commit_checkout_no_credentials.py b/scripts/ci/pre_commit/pre_commit_checkout_no_credentials.py index 20625c90923c7d..e03b316d1a6209 100755 --- a/scripts/ci/pre_commit/pre_commit_checkout_no_credentials.py +++ b/scripts/ci/pre_commit/pre_commit_checkout_no_credentials.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/pre_commit/pre_commit_docstring_param_type.py b/scripts/ci/pre_commit/pre_commit_docstring_param_type.py index 38f7d4081c2d61..0ef067dd9cf72f 100755 --- a/scripts/ci/pre_commit/pre_commit_docstring_param_type.py +++ b/scripts/ci/pre_commit/pre_commit_docstring_param_type.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/pre_commit/pre_commit_flake8.py b/scripts/ci/pre_commit/pre_commit_flake8.py index 59bd356d24df0e..92698fece4b313 100755 --- a/scripts/ci/pre_commit/pre_commit_flake8.py +++ b/scripts/ci/pre_commit/pre_commit_flake8.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/pre_commit/pre_commit_inline_scripts_in_docker.py b/scripts/ci/pre_commit/pre_commit_inline_scripts_in_docker.py index a528ba46aea315..a02f83cb8c4d80 100755 --- a/scripts/ci/pre_commit/pre_commit_inline_scripts_in_docker.py +++ b/scripts/ci/pre_commit/pre_commit_inline_scripts_in_docker.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/pre_commit/pre_commit_insert_extras.py b/scripts/ci/pre_commit/pre_commit_insert_extras.py index 1c79190ea60f7a..dbcc4039004263 100755 --- a/scripts/ci/pre_commit/pre_commit_insert_extras.py +++ b/scripts/ci/pre_commit/pre_commit_insert_extras.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/pre_commit/pre_commit_json_schema.py b/scripts/ci/pre_commit/pre_commit_json_schema.py index eb52aa0e49a637..2949f8e2c75b74 100755 --- a/scripts/ci/pre_commit/pre_commit_json_schema.py +++ b/scripts/ci/pre_commit/pre_commit_json_schema.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/pre_commit/pre_commit_local_yml_mounts.py b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.py index 391214fa296e34..df84436aac78c1 100755 --- a/scripts/ci/pre_commit/pre_commit_local_yml_mounts.py +++ b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/pre_commit/pre_commit_migration_reference.py b/scripts/ci/pre_commit/pre_commit_migration_reference.py index 154a795f3e84a6..8c9d8a0f000b39 100755 --- a/scripts/ci/pre_commit/pre_commit_migration_reference.py +++ b/scripts/ci/pre_commit/pre_commit_migration_reference.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/pre_commit/pre_commit_mypy.py b/scripts/ci/pre_commit/pre_commit_mypy.py index 74d511a65b7ed5..06f4403cef7580 100755 --- a/scripts/ci/pre_commit/pre_commit_mypy.py +++ b/scripts/ci/pre_commit/pre_commit_mypy.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/pre_commit/pre_commit_newsfragments.py b/scripts/ci/pre_commit/pre_commit_newsfragments.py index 82b85f3f81ff4a..e0504273e7b294 100755 --- a/scripts/ci/pre_commit/pre_commit_newsfragments.py +++ b/scripts/ci/pre_commit/pre_commit_newsfragments.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/scripts/ci/pre_commit/pre_commit_supported_versions.py b/scripts/ci/pre_commit/pre_commit_supported_versions.py index fee266c9be695f..199c7fe4dba85c 100755 --- a/scripts/ci/pre_commit/pre_commit_supported_versions.py +++ b/scripts/ci/pre_commit/pre_commit_supported_versions.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/pre_commit/pre_commit_ui_lint.py b/scripts/ci/pre_commit/pre_commit_ui_lint.py index 7755e3d204bd8c..fbe83198114d41 100755 --- a/scripts/ci/pre_commit/pre_commit_ui_lint.py +++ b/scripts/ci/pre_commit/pre_commit_ui_lint.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/pre_commit/pre_commit_update_breeze_config_hash.py b/scripts/ci/pre_commit/pre_commit_update_breeze_config_hash.py index 455f5da49a1050..007391fb24b983 100755 --- a/scripts/ci/pre_commit/pre_commit_update_breeze_config_hash.py +++ b/scripts/ci/pre_commit/pre_commit_update_breeze_config_hash.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/pre_commit/pre_commit_update_versions.py b/scripts/ci/pre_commit/pre_commit_update_versions.py index 6ae981a8d2d896..e7265a285b08c6 100755 --- a/scripts/ci/pre_commit/pre_commit_update_versions.py +++ b/scripts/ci/pre_commit/pre_commit_update_versions.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/pre_commit/pre_commit_vendor_k8s_json_schema.py b/scripts/ci/pre_commit/pre_commit_vendor_k8s_json_schema.py index ab7b04232b6b9b..c11bfedf028ac4 100755 --- a/scripts/ci/pre_commit/pre_commit_vendor_k8s_json_schema.py +++ b/scripts/ci/pre_commit/pre_commit_vendor_k8s_json_schema.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/scripts/ci/pre_commit/pre_commit_version_heads_map.py b/scripts/ci/pre_commit/pre_commit_version_heads_map.py index 7ba91ead0e65bf..428b28aa0a2fbe 100755 --- a/scripts/ci/pre_commit/pre_commit_version_heads_map.py +++ b/scripts/ci/pre_commit/pre_commit_version_heads_map.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/scripts/ci/pre_commit/pre_commit_www_lint.py b/scripts/ci/pre_commit/pre_commit_www_lint.py index 65553c3d813451..53ab0f8bf9cdad 100755 --- a/scripts/ci/pre_commit/pre_commit_www_lint.py +++ b/scripts/ci/pre_commit/pre_commit_www_lint.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py b/scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py index 8a22264cab91bc..8f87d303720cfe 100755 --- a/scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py +++ b/scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file From 28acae9cd0555c5b712a9db4dd53858da5a28b8e Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 2 Jun 2022 17:26:00 +0800 Subject: [PATCH 5/5] Make provider yml check compatible with Windows This involves mostly converting things to pathlib and make use of as_posix() to normalize paths. --- .../pre_commit_check_provider_yaml_files.py | 80 +++++++++---------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py b/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py index ef961031ab78c4..756682be5eaef7 100755 --- a/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py +++ b/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py @@ -17,11 +17,10 @@ # under the License. import json -import os +import pathlib import sys import textwrap from collections import Counter -from glob import glob from itertools import chain, product from typing import Any, Dict, Iterable, List, Set @@ -41,26 +40,26 @@ "This file is intended to be executed as an executable program. You cannot use it as a module." ) -ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir)) -DOCS_DIR = os.path.join(ROOT_DIR, 'docs') -PROVIDER_DATA_SCHEMA_PATH = os.path.join(ROOT_DIR, "airflow", "provider.yaml.schema.json") -PROVIDER_ISSUE_TEMPLATE_PATH = os.path.join( - ROOT_DIR, ".github", "ISSUE_TEMPLATE", "airflow_providers_bug_report.yml" +ROOT_DIR = pathlib.Path(__file__).resolve().parents[3] +DOCS_DIR = ROOT_DIR.joinpath("docs") +PROVIDER_DATA_SCHEMA_PATH = ROOT_DIR.joinpath("airflow", "provider.yaml.schema.json") +PROVIDER_ISSUE_TEMPLATE_PATH = ROOT_DIR.joinpath( + ".github", "ISSUE_TEMPLATE", "airflow_providers_bug_report.yml" ) CORE_INTEGRATIONS = ["SQL", "Local"] errors = [] -def _filepath_to_module(filepath: str): - filepath = os.path.relpath(os.path.abspath(filepath), ROOT_DIR) - if filepath.endswith(".py"): - filepath = filepath[: -(len(".py"))] - return filepath.replace("/", ".") +def _filepath_to_module(filepath: pathlib.Path) -> str: + p = filepath.resolve().relative_to(ROOT_DIR).as_posix() + if p.endswith(".py"): + p = p[:-3] + return p.replace("/", ".") def _load_schema() -> Dict[str, Any]: - with open(PROVIDER_DATA_SCHEMA_PATH) as schema_file: + with PROVIDER_DATA_SCHEMA_PATH.open() as schema_file: content = json.load(schema_file) return content @@ -71,7 +70,7 @@ def _load_package_data(package_paths: Iterable[str]): for provider_yaml_path in package_paths: with open(provider_yaml_path) as yaml_file: provider = yaml.load(yaml_file, SafeLoader) - rel_path = os.path.relpath(provider_yaml_path, ROOT_DIR) + rel_path = pathlib.Path(provider_yaml_path).relative_to(ROOT_DIR).as_posix() try: jsonschema.validate(provider, schema=schema) except jsonschema.ValidationError: @@ -148,22 +147,22 @@ def check_if_objects_belongs_to_package( def parse_module_data(provider_data, resource_type, yaml_file_path): - package_dir = ROOT_DIR + "/" + os.path.dirname(yaml_file_path) - provider_package = os.path.dirname(yaml_file_path).replace(os.sep, ".") + package_dir = ROOT_DIR.joinpath(yaml_file_path).parent + provider_package = pathlib.Path(yaml_file_path).parent.as_posix().replace("/", ".") py_files = chain( - glob(f"{package_dir}/**/{resource_type}/*.py"), - glob(f"{package_dir}/{resource_type}/*.py"), - glob(f"{package_dir}/**/{resource_type}/**/*.py"), - glob(f"{package_dir}/{resource_type}/**/*.py"), + package_dir.glob(f"**/{resource_type}/*.py"), + package_dir.glob(f"{resource_type}/*.py"), + package_dir.glob(f"**/{resource_type}/**/*.py"), + package_dir.glob(f"{resource_type}/**/*.py"), ) - expected_modules = {_filepath_to_module(f) for f in py_files if not f.endswith("/__init__.py")} + expected_modules = {_filepath_to_module(f) for f in py_files if f.name != "__init__.py"} resource_data = provider_data.get(resource_type, []) return expected_modules, provider_package, resource_data def check_completeness_of_list_of_hooks_sensors_hooks(yaml_files: Dict[str, Dict]): print("Checking completeness of list of {sensors, hooks, operators}") - print(" -- {sensors, hooks, operators} - Expected modules(Left): Current Modules(Right)") + print(" -- {sensors, hooks, operators} - Expected modules (left) : Current modules (right)") for (yaml_file_path, provider_data), resource_type in product( yaml_files.items(), ["sensors", "operators", "hooks"] ): @@ -225,7 +224,7 @@ def check_hook_classes(yaml_files: Dict[str, Dict]): print("Checking connection classes belong to package") resource_type = 'hook-class-names' for yaml_file_path, provider_data in yaml_files.items(): - provider_package = os.path.dirname(yaml_file_path).replace(os.sep, ".") + provider_package = pathlib.Path(yaml_file_path).parent.as_posix().replace("/", ".") hook_class_names = provider_data.get(resource_type) if hook_class_names: check_if_objects_belongs_to_package( @@ -305,25 +304,24 @@ def check_doc_files(yaml_files: Dict[str, Dict]): ) expected_doc_urls = { - "/docs/" + os.path.relpath(f, start=DOCS_DIR) - for f in glob(f"{DOCS_DIR}/apache-airflow-providers-*/operators/**/*.rst", recursive=True) - if not f.endswith("/index.rst") and '/_partials' not in f - } - expected_doc_urls |= { - "/docs/" + os.path.relpath(f, start=DOCS_DIR) - for f in glob(f"{DOCS_DIR}/apache-airflow-providers-*/operators.rst", recursive=True) + f"/docs/{f.relative_to(DOCS_DIR).as_posix()}" + for f in DOCS_DIR.glob("apache-airflow-providers-*/operators/**/*.rst") + if f.name != "index.rst" and "_partials" not in f.parts + } | { + f"/docs/{f.relative_to(DOCS_DIR).as_posix()}" + for f in DOCS_DIR.glob("apache-airflow-providers-*/operators.rst") } expected_logo_urls = { - "/" + os.path.relpath(f, start=DOCS_DIR) - for f in glob(f"{DOCS_DIR}/integration-logos/**/*", recursive=True) - if os.path.isfile(f) + f"/{f.relative_to(DOCS_DIR).as_posix()}" + for f in DOCS_DIR.glob("integration-logos/**/*") + if f.is_file() } try: - print(" -- Checking document urls: expected(left), current(right)") + print(" -- Checking document urls: expected (left), current (right)") assert_sets_equal(set(expected_doc_urls), set(current_doc_urls)) - print(" -- Checking logo urls: expected(left), current(right)") + print(" -- Checking logo urls: expected (left), current (right)") assert_sets_equal(set(expected_logo_urls), set(current_logo_urls)) except AssertionError as ex: print(ex) @@ -341,12 +339,12 @@ def check_providers_are_mentioned_in_issue_template(yaml_files: Dict[str, Dict]) prefix_len = len("apache-airflow-providers-") short_provider_names = [d['package-name'][prefix_len:] for d in yaml_files.values()] jsonpath_expr = parse('$.body[?(@.attributes.label == "Apache Airflow Provider(s)")]..options[*]') - with open(PROVIDER_ISSUE_TEMPLATE_PATH) as issue_file: + with PROVIDER_ISSUE_TEMPLATE_PATH.open() as issue_file: issue_template = yaml.safe_load(issue_file) all_mentioned_providers = [match.value for match in jsonpath_expr.find(issue_template)] try: print( - f" -- Checking providers: present in code(left), " + f" -- Checking providers: present in code (left), " f"mentioned in {PROVIDER_ISSUE_TEMPLATE_PATH} (right)" ) assert_sets_equal(set(short_provider_names), set(all_mentioned_providers)) @@ -359,9 +357,9 @@ def check_providers_have_all_documentation_files(yaml_files: Dict[str, Dict]): expected_files = ["commits.rst", "index.rst", "installing-providers-from-sources.rst"] for package_info in yaml_files.values(): package_name = package_info['package-name'] - provider_dir = os.path.join(DOCS_DIR, package_name) + provider_dir = DOCS_DIR.joinpath(package_name) for file in expected_files: - if not os.path.isfile(os.path.join(provider_dir, file)): + if not provider_dir.joinpath(file).is_file(): errors.append( f"The provider {package_name} misses `{file}` in documentation. " f"Please add the file to {provider_dir}" @@ -369,7 +367,9 @@ def check_providers_have_all_documentation_files(yaml_files: Dict[str, Dict]): if __name__ == '__main__': - all_provider_files = sorted(glob(f"{ROOT_DIR}/airflow/providers/**/provider.yaml", recursive=True)) + provider_files_pattern = pathlib.Path(ROOT_DIR).glob("airflow/providers/**/provider.yaml") + all_provider_files = sorted(str(path) for path in provider_files_pattern) + if len(sys.argv) > 1: paths = sorted(sys.argv[1:]) else: