From 30243e943c6b8c6b6fb384405fbf44916134579e Mon Sep 17 00:00:00 2001 From: Ash Berlin-Taylor Date: Tue, 4 Jan 2022 12:09:35 +0000 Subject: [PATCH] Update Kubernetes library version (#18797) Previously we pinned this version as v12 as a change to Kube library internals meant v1.Pod objects now have a logger object inside them, and couldn't be pickled on Python 3.6. To fix that we have "backported" the change in Python 3.7 to make Logger objects be pickled "by name". (In Python 3.7 the change adds `__reduce__` methods on to the Logger and RootLogger objects, but here we achieve it `copyreg` stdlib module so we don't monkeypatch anything.) This fix is also applied in to airflow core in a separate commit, but we also apply it here in the provider so that cncf.kubernetes client library can be updated but still used with older versions of Airflow that don't have this fix in. (cherry picked from commit 7222f68d374787f95acc7110a1165bd21e7722a1) --- airflow/providers/cncf/kubernetes/utils/pod_manager.py | 8 ++++++-- tests/kubernetes/test_client.py | 6 +++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/airflow/providers/cncf/kubernetes/utils/pod_manager.py b/airflow/providers/cncf/kubernetes/utils/pod_manager.py index 4221ac2afd95e..eab45fabb4bab 100644 --- a/airflow/providers/cncf/kubernetes/utils/pod_manager.py +++ b/airflow/providers/cncf/kubernetes/utils/pod_manager.py @@ -40,7 +40,11 @@ from airflow.utils.log.logging_mixin import LoggingMixin if TYPE_CHECKING: - from kubernetes.client.models.core_v1_event_list import CoreV1EventList + try: + # Kube >= 19 + from kubernetes.client.models.core_v1_event_list import CoreV1EventList as V1EventList + except ImportError: + from kubernetes.client.models.v1_event_list import V1EventList class PodLaunchFailedException(AirflowException): @@ -317,7 +321,7 @@ def read_pod_logs( raise @tenacity.retry(stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_exponential(), reraise=True) - def read_pod_events(self, pod: V1Pod) -> "CoreV1EventList": + def read_pod_events(self, pod: V1Pod) -> "V1EventList": """Reads events from the POD""" try: return self._client.list_namespaced_event( diff --git a/tests/kubernetes/test_client.py b/tests/kubernetes/test_client.py index b3f41fa9427bc..ce040cf3ed8f2 100644 --- a/tests/kubernetes/test_client.py +++ b/tests/kubernetes/test_client.py @@ -59,5 +59,9 @@ def test_disable_verify_ssl(self): _disable_verify_ssl() - configuration = Configuration() + # Support wide range of kube client libraries + if hasattr(Configuration, 'get_default_copy'): + configuration = Configuration.get_default_copy() + else: + configuration = Configuration() self.assertFalse(configuration.verify_ssl)