use single cloud_prefix

drivendataorg · Sep 3, 2024 · 1b3f153 · 1b3f153
1 parent 54c37b2
commit 1b3f153
Show file tree

Hide file tree

Showing 8 changed files with 42 additions and 40 deletions.
diff --git a/cloudpathlib/azure/azblobclient.py b/cloudpathlib/azure/azblobclient.py
@@ -37,8 +37,6 @@ class AzureBlobClient(Client):
     authentication options.
     """
 
-    cloud_prefix: str = "az://"
-
     def __init__(
         self,
         account_url: Optional[str] = None,
@@ -278,13 +276,13 @@ def _list_dir(
     ) -> Iterable[Tuple[AzureBlobPath, bool]]:
         if not cloud_path.container:
             for container in self.service_client.list_containers():
-                yield self.CloudPath(f"{self.cloud_prefix}{container.name}"), True
+                yield self.CloudPath(f"{cloud_path.cloud_prefix}{container.name}"), True
 
                 if not recursive:
                     continue
 
                 yield from self._list_dir(
-                    self.CloudPath(f"{self.cloud_prefix}{container.name}"), recursive=True
+                    self.CloudPath(f"{cloud_path.cloud_prefix}{container.name}"), recursive=True
                 )
             return
 
@@ -300,7 +298,7 @@ def _list_dir(
 
             for path in paths:
                 yield self.CloudPath(
-                    f"{self.cloud_prefix}{cloud_path.container}/{path.name}"
+                    f"{cloud_path.cloud_prefix}{cloud_path.container}/{path.name}"
                 ), path.is_directory
 
         else:
@@ -313,7 +311,7 @@ def _list_dir(
                 # walk_blobs returns folders with a trailing slash
                 blob_path = blob.name.rstrip("/")
                 blob_cloud_path = self.CloudPath(
-                    f"{self.cloud_prefix}{cloud_path.container}/{blob_path}"
+                    f"{cloud_path.cloud_prefix}{cloud_path.container}/{blob_path}"
                 )
 
                 yield blob_cloud_path, (

diff --git a/cloudpathlib/client.py b/cloudpathlib/client.py
@@ -30,8 +30,6 @@ class Client(abc.ABC, Generic[BoundedCloudPath]):
     _cloud_meta: CloudImplementation
     _default_client = None
 
-    cloud_prefix: str
-
     def __init__(
         self,
         file_cache_mode: Optional[Union[str, FileCacheMode]] = None,

diff --git a/cloudpathlib/gs/gsclient.py b/cloudpathlib/gs/gsclient.py
@@ -35,8 +35,6 @@ class GSClient(Client):
     options.
     """
 
-    cloud_prefix: str = "gs://"
-
     def __init__(
         self,
         application_credentials: Optional[Union[str, os.PathLike]] = None,
@@ -185,7 +183,7 @@ def _list_dir(self, cloud_path: GSPath, recursive=False) -> Iterable[Tuple[GSPat
                 )
 
             yield from (
-                (self.CloudPath(f"{self.cloud_prefix}{str(b)}"), True)
+                (self.CloudPath(f"{cloud_path.cloud_prefix}{str(b)}"), True)
                 for b in self.client.list_buckets()
             )
             return
@@ -204,13 +202,13 @@ def _list_dir(self, cloud_path: GSPath, recursive=False) -> Iterable[Tuple[GSPat
                     if parent not in yielded_dirs and str(parent) != ".":
                         yield (
                             self.CloudPath(
-                                f"{self.cloud_prefix}{cloud_path.bucket}/{prefix}{parent}"
+                                f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{prefix}{parent}"
                             ),
                             True,  # is a directory
                         )
                         yielded_dirs.add(parent)
                 yield (
-                    self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{o.name}"),
+                    self.CloudPath(f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{o.name}"),
                     False,
                 )  # is a file
         else:
@@ -220,13 +218,13 @@ def _list_dir(self, cloud_path: GSPath, recursive=False) -> Iterable[Tuple[GSPat
             #   see: https://github.com/googleapis/python-storage/issues/863
             for file in iterator:
                 yield (
-                    self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{file.name}"),
+                    self.CloudPath(f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{file.name}"),
                     False,  # is a file
                 )
 
             for directory in iterator.prefixes:
                 yield (
-                    self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{directory}"),
+                    self.CloudPath(f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{directory}"),
                     True,  # is a directory
                 )
 

diff --git a/cloudpathlib/local/implementations/azure.py b/cloudpathlib/local/implementations/azure.py
@@ -16,7 +16,6 @@ class LocalAzureBlobClient(LocalClient):
     substitute when writing tests.
     """
 
-    cloud_prefix: str = "az://"
     _cloud_meta = local_azure_blob_implementation
 
     def __init__(self, *args, **kwargs):

diff --git a/cloudpathlib/local/implementations/gs.py b/cloudpathlib/local/implementations/gs.py
@@ -12,7 +12,6 @@ class LocalGSClient(LocalClient):
     substitute when writing tests.
     """
 
-    cloud_prefix: str = "gs://"
     _cloud_meta = local_gs_implementation
 
 

diff --git a/cloudpathlib/local/implementations/s3.py b/cloudpathlib/local/implementations/s3.py
@@ -12,7 +12,6 @@ class LocalS3Client(LocalClient):
     substitute when writing tests.
     """
 
-    cloud_prefix: str = "s3://"
     _cloud_meta = local_s3_implementation
 
 

diff --git a/cloudpathlib/s3/s3client.py b/cloudpathlib/s3/s3client.py
@@ -25,8 +25,6 @@ class S3Client(Client):
     instances. See documentation for the [`__init__` method][cloudpathlib.s3.s3client.S3Client.__init__]
     for detailed authentication options."""
 
-    cloud_prefix: str = "s3://"
-
     def __init__(
         self,
         aws_access_key_id: Optional[str] = None,
@@ -219,7 +217,7 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat
                 )
 
             yield from (
-                (self.CloudPath(f"{self.cloud_prefix}{b['Name']}"), True)
+                (self.CloudPath(f"{cloud_path.cloud_prefix}{b['Name']}"), True)
                 for b in self.client.list_buckets().get("Buckets", [])
             )
             return
@@ -243,7 +241,9 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat
                 canonical = result_prefix.get("Prefix").rstrip("/")  # keep a canonical form
                 if canonical not in yielded_dirs:
                     yield (
-                        self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{canonical}"),
+                        self.CloudPath(
+                            f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{canonical}"
+                        ),
                         True,
                     )
                     yielded_dirs.add(canonical)
@@ -257,7 +257,7 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat
                     if parent_canonical not in yielded_dirs and str(parent) != ".":
                         yield (
                             self.CloudPath(
-                                f"{self.cloud_prefix}{cloud_path.bucket}/{parent_canonical}"
+                                f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{parent_canonical}"
                             ),
                             True,
                         )
@@ -271,7 +271,9 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat
                 # s3 fake directories have 0 size and end with "/"
                 if result_key.get("Key").endswith("/") and result_key.get("Size") == 0:
                     yield (
-                        self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{canonical}"),
+                        self.CloudPath(
+                            f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{canonical}"
+                        ),
                         True,
                     )
                     yielded_dirs.add(canonical)
@@ -280,7 +282,7 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat
                 else:
                     yield (
                         self.CloudPath(
-                            f"{self.cloud_prefix}{cloud_path.bucket}/{result_key.get('Key')}"
+                            f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{result_key.get('Key')}"
                         ),
                         False,
                     )

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -1,47 +1,56 @@
 import os
-from pathlib import Path, PurePosixPath
 import shutil
+from pathlib import Path, PurePosixPath
 from tempfile import TemporaryDirectory
 from typing import Dict, Optional
 
+import boto3
+import botocore
 from azure.storage.blob import BlobServiceClient
 from azure.storage.filedatalake import (
     DataLakeServiceClient,
 )
-import boto3
-import botocore
 from dotenv import find_dotenv, load_dotenv
 from google.cloud import storage as google_storage
 from pytest_cases import fixture, fixture_union
 from shortuuid import uuid
 from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed
 
-from cloudpathlib import AzureBlobClient, AzureBlobPath, GSClient, GSPath, S3Client, S3Path
+import cloudpathlib.azure.azblobclient
+import cloudpathlib.s3.s3client
+from cloudpathlib import (
+    AzureBlobClient,
+    AzureBlobPath,
+    GSClient,
+    GSPath,
+    S3Client,
+    S3Path,
+)
+from cloudpathlib.azure.azblobclient import _hns_rmtree
 from cloudpathlib.client import register_client_class
 from cloudpathlib.cloudpath import implementation_registry, register_path_class
 from cloudpathlib.local import (
-    local_azure_blob_implementation,
     LocalAzureBlobClient,
     LocalAzureBlobPath,
-    local_gs_implementation,
     LocalGSClient,
     LocalGSPath,
-    local_s3_implementation,
     LocalS3Client,
     LocalS3Path,
+    local_azure_blob_implementation,
+    local_gs_implementation,
+    local_s3_implementation,
 )
-import cloudpathlib.azure.azblobclient
-from cloudpathlib.azure.azblobclient import _hns_rmtree
-import cloudpathlib.s3.s3client
-from .mock_clients.mock_azureblob import MockBlobServiceClient, DEFAULT_CONTAINER_NAME
+
 from .mock_clients.mock_adls_gen2 import MockedDataLakeServiceClient
+from .mock_clients.mock_azureblob import DEFAULT_CONTAINER_NAME, MockBlobServiceClient
 from .mock_clients.mock_gs import (
-    mocked_client_class_factory as mocked_gsclient_class_factory,
     DEFAULT_GS_BUCKET_NAME,
     MockTransferManager,
 )
-from .mock_clients.mock_s3 import mocked_session_class_factory, DEFAULT_S3_BUCKET_NAME
-
+from .mock_clients.mock_gs import (
+    mocked_client_class_factory as mocked_gsclient_class_factory,
+)
+from .mock_clients.mock_s3 import DEFAULT_S3_BUCKET_NAME, mocked_session_class_factory
 
 if os.getenv("USE_LIVE_CLOUD") == "1":
     load_dotenv(find_dotenv())
@@ -411,7 +420,7 @@ class MyS3Path(S3Path):
 
 @register_client_class("mys3")
 class MyS3Client(S3Client):
-    cloud_prefix: str = "mys3://"
+    pass
 
 
 # Mirrors the definition of the S3Client class