Support versions of GCS without transfer_manager (#410)

* Fix 408 * Version and changelog * add warning * update history PR * Tweak description of change * Update HISTORY.md Co-authored-by: Jay Qi <2721979+jayqi@users.noreply.github.com> --------- Co-authored-by: Jay Qi <2721979+jayqi@users.noreply.github.com>
drivendataorg · Feb 26, 2024 · 29a147a · 29a147a
1 parent 6d13512
commit 29a147a
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 7 deletions.
diff --git a/HISTORY.md b/HISTORY.md
@@ -1,12 +1,21 @@
 # cloudpathlib Changelog
 
-## 0.18.0 (2024-02-25)
+## v0.18.1 (2024-02-26)
+
+- Fixed import error due to incompatible `google-cloud-storage` by not using `transfer_manager` if it is not available. ([Issue #408](https://github.com/drivendataorg/cloudpathlib/issues/408), (PR #410)[https://github.com/drivendataorg/cloudpathlib/pull/410]) 
+
+Includes all changes from v0.18.0.
+
+**Note: This is the last planned Python 3.7 compatible release version.**
+
+## 0.18.0 (2024-02-25) (Yanked)
+
 - Implement sliced downloads in GSClient. (Issue [#387](https://github.com/drivendataorg/cloudpathlib/issues/387), PR [#389](https://github.com/drivendataorg/cloudpathlib/pull/389))
 - Implement `as_url` with presigned parameter for all backends. (Issue [#235](https://github.com/drivendataorg/cloudpathlib/issues/235), PR [#236](https://github.com/drivendataorg/cloudpathlib/pull/236))
 - Stream to and from Azure Blob Storage. (PR [#403](https://github.com/drivendataorg/cloudpathlib/pull/403))
 - Implement `file:` URI scheme support for `AnyPath`. (Issue [#401](https://github.com/drivendataorg/cloudpathlib/issues/401), PR [#404](https://github.com/drivendataorg/cloudpathlib/pull/404))
 
-**Note: This is the last planned Python 3.7 compatible release version.**
+**Note: This version was [yanked](https://pypi.org/help/#yanked) due to incompatibility with google-cloud-storage <2.7.0 that causes an import error.**
 
 ## 0.17.0 (2023-12-21)
 

diff --git a/cloudpathlib/gs/gsclient.py b/cloudpathlib/gs/gsclient.py
@@ -3,6 +3,7 @@
 import os
 from pathlib import Path, PurePosixPath
 from typing import Any, Callable, Dict, Iterable, Optional, TYPE_CHECKING, Tuple, Union
+import warnings
 
 from ..client import Client, register_client_class
 from ..cloudpath import implementation_registry
@@ -16,13 +17,17 @@
     from google.api_core.exceptions import NotFound
     from google.auth.exceptions import DefaultCredentialsError
     from google.cloud.storage import Client as StorageClient
-    from google.cloud.storage import transfer_manager
-
 
 except ModuleNotFoundError:
     implementation_registry["gs"].dependencies_loaded = False
 
 
+try:
+    from google.cloud.storage import transfer_manager
+except ImportError:
+    transfer_manager = None
+
+
 @register_client_class("gs")
 class GSClient(Client):
     """Client class for Google Cloud Storage which handles authentication with GCP for
@@ -80,7 +85,7 @@ def __init__(
                 writing a file to the cloud. Defaults to `mimetypes.guess_type`. Must return a tuple (content type, content encoding).
             download_chunks_concurrently_kwargs (Optional[Dict[str, Any]]): Keyword arguments to pass to
                 [`download_chunks_concurrently`](https://cloud.google.com/python/docs/reference/storage/latest/google.cloud.storage.transfer_manager#google_cloud_storage_transfer_manager_download_chunks_concurrently)
-                for sliced parallel downloads.
+                for sliced parallel downloads; Only available in `google-cloud-storage` version 2.7.0 or later, otherwise ignored and a warning is emitted.
         """
         if application_credentials is None:
             application_credentials = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
@@ -125,11 +130,16 @@ def _download_file(self, cloud_path: GSPath, local_path: Union[str, os.PathLike]
 
         local_path = Path(local_path)
 
-        if self.download_chunks_concurrently_kwargs is not None:
+        if transfer_manager is not None and self.download_chunks_concurrently_kwargs is not None:
             transfer_manager.download_chunks_concurrently(
                 blob, local_path, **self.download_chunks_concurrently_kwargs
             )
         else:
+            if transfer_manager is None and self.download_chunks_concurrently_kwargs is not None:
+                warnings.warn(
+                    "Ignoring `download_chunks_concurrently_kwargs` for version of google-cloud-storage that does not support them (<2.7.0)."
+                )
+
             blob.download_to_filename(local_path)
 
         return local_path

diff --git a/pyproject.toml b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "cloudpathlib"
-version = "0.18.0"
+version = "0.18.1"
 description = "pathlib-style classes for cloud storage services."
 readme = "README.md"
 authors = [{ name = "DrivenData", email = "info@drivendata.org" }]