From 39aeb09f618164f288ffab8ef7e24c9e28d66483 Mon Sep 17 00:00:00 2001 From: Aaron Taylor Date: Tue, 3 Sep 2024 13:12:10 -0700 Subject: [PATCH] use single cloud_prefix --- cloudpathlib/azure/azblobclient.py | 10 ++++------ cloudpathlib/client.py | 2 -- cloudpathlib/gs/gsclient.py | 12 +++++------- cloudpathlib/local/implementations/azure.py | 1 - cloudpathlib/local/implementations/gs.py | 1 - cloudpathlib/local/implementations/s3.py | 1 - cloudpathlib/s3/s3client.py | 16 +++++++++------- 7 files changed, 18 insertions(+), 25 deletions(-) diff --git a/cloudpathlib/azure/azblobclient.py b/cloudpathlib/azure/azblobclient.py index 3a77a492..289eb236 100644 --- a/cloudpathlib/azure/azblobclient.py +++ b/cloudpathlib/azure/azblobclient.py @@ -37,8 +37,6 @@ class AzureBlobClient(Client): authentication options. """ - cloud_prefix: str = "az://" - def __init__( self, account_url: Optional[str] = None, @@ -278,13 +276,13 @@ def _list_dir( ) -> Iterable[Tuple[AzureBlobPath, bool]]: if not cloud_path.container: for container in self.service_client.list_containers(): - yield self.CloudPath(f"{self.cloud_prefix}{container.name}"), True + yield self.CloudPath(f"{cloud_path.cloud_prefix}{container.name}"), True if not recursive: continue yield from self._list_dir( - self.CloudPath(f"{self.cloud_prefix}{container.name}"), recursive=True + self.CloudPath(f"{cloud_path.cloud_prefix}{container.name}"), recursive=True ) return @@ -300,7 +298,7 @@ def _list_dir( for path in paths: yield self.CloudPath( - f"{self.cloud_prefix}{cloud_path.container}/{path.name}" + f"{cloud_path.cloud_prefix}{cloud_path.container}/{path.name}" ), path.is_directory else: @@ -313,7 +311,7 @@ def _list_dir( # walk_blobs returns folders with a trailing slash blob_path = blob.name.rstrip("/") blob_cloud_path = self.CloudPath( - f"{self.cloud_prefix}{cloud_path.container}/{blob_path}" + f"{cloud_path.cloud_prefix}{cloud_path.container}/{blob_path}" ) yield blob_cloud_path, ( diff --git a/cloudpathlib/client.py b/cloudpathlib/client.py index 3557d0a7..1b6c32eb 100644 --- a/cloudpathlib/client.py +++ b/cloudpathlib/client.py @@ -30,8 +30,6 @@ class Client(abc.ABC, Generic[BoundedCloudPath]): _cloud_meta: CloudImplementation _default_client = None - cloud_prefix: str - def __init__( self, file_cache_mode: Optional[Union[str, FileCacheMode]] = None, diff --git a/cloudpathlib/gs/gsclient.py b/cloudpathlib/gs/gsclient.py index b5351260..edd5b88a 100644 --- a/cloudpathlib/gs/gsclient.py +++ b/cloudpathlib/gs/gsclient.py @@ -35,8 +35,6 @@ class GSClient(Client): options. """ - cloud_prefix: str = "gs://" - def __init__( self, application_credentials: Optional[Union[str, os.PathLike]] = None, @@ -185,7 +183,7 @@ def _list_dir(self, cloud_path: GSPath, recursive=False) -> Iterable[Tuple[GSPat ) yield from ( - (self.CloudPath(f"{self.cloud_prefix}{str(b)}"), True) + (self.CloudPath(f"{cloud_path.cloud_prefix}{str(b)}"), True) for b in self.client.list_buckets() ) return @@ -204,13 +202,13 @@ def _list_dir(self, cloud_path: GSPath, recursive=False) -> Iterable[Tuple[GSPat if parent not in yielded_dirs and str(parent) != ".": yield ( self.CloudPath( - f"{self.cloud_prefix}{cloud_path.bucket}/{prefix}{parent}" + f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{prefix}{parent}" ), True, # is a directory ) yielded_dirs.add(parent) yield ( - self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{o.name}"), + self.CloudPath(f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{o.name}"), False, ) # is a file else: @@ -220,13 +218,13 @@ def _list_dir(self, cloud_path: GSPath, recursive=False) -> Iterable[Tuple[GSPat # see: https://github.com/googleapis/python-storage/issues/863 for file in iterator: yield ( - self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{file.name}"), + self.CloudPath(f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{file.name}"), False, # is a file ) for directory in iterator.prefixes: yield ( - self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{directory}"), + self.CloudPath(f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{directory}"), True, # is a directory ) diff --git a/cloudpathlib/local/implementations/azure.py b/cloudpathlib/local/implementations/azure.py index 97f6b1be..519924d0 100644 --- a/cloudpathlib/local/implementations/azure.py +++ b/cloudpathlib/local/implementations/azure.py @@ -16,7 +16,6 @@ class LocalAzureBlobClient(LocalClient): substitute when writing tests. """ - cloud_prefix: str = "az://" _cloud_meta = local_azure_blob_implementation def __init__(self, *args, **kwargs): diff --git a/cloudpathlib/local/implementations/gs.py b/cloudpathlib/local/implementations/gs.py index bb3f6491..5e63084f 100644 --- a/cloudpathlib/local/implementations/gs.py +++ b/cloudpathlib/local/implementations/gs.py @@ -12,7 +12,6 @@ class LocalGSClient(LocalClient): substitute when writing tests. """ - cloud_prefix: str = "gs://" _cloud_meta = local_gs_implementation diff --git a/cloudpathlib/local/implementations/s3.py b/cloudpathlib/local/implementations/s3.py index 2a2ee8e1..df9951bf 100644 --- a/cloudpathlib/local/implementations/s3.py +++ b/cloudpathlib/local/implementations/s3.py @@ -12,7 +12,6 @@ class LocalS3Client(LocalClient): substitute when writing tests. """ - cloud_prefix: str = "s3://" _cloud_meta = local_s3_implementation diff --git a/cloudpathlib/s3/s3client.py b/cloudpathlib/s3/s3client.py index eb48c8aa..db130e82 100644 --- a/cloudpathlib/s3/s3client.py +++ b/cloudpathlib/s3/s3client.py @@ -25,8 +25,6 @@ class S3Client(Client): instances. See documentation for the [`__init__` method][cloudpathlib.s3.s3client.S3Client.__init__] for detailed authentication options.""" - cloud_prefix: str = "s3://" - def __init__( self, aws_access_key_id: Optional[str] = None, @@ -219,7 +217,7 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat ) yield from ( - (self.CloudPath(f"{self.cloud_prefix}{b['Name']}"), True) + (self.CloudPath(f"{cloud_path.cloud_prefix}{b['Name']}"), True) for b in self.client.list_buckets().get("Buckets", []) ) return @@ -243,7 +241,9 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat canonical = result_prefix.get("Prefix").rstrip("/") # keep a canonical form if canonical not in yielded_dirs: yield ( - self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{canonical}"), + self.CloudPath( + f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{canonical}" + ), True, ) yielded_dirs.add(canonical) @@ -257,7 +257,7 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat if parent_canonical not in yielded_dirs and str(parent) != ".": yield ( self.CloudPath( - f"{self.cloud_prefix}{cloud_path.bucket}/{parent_canonical}" + f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{parent_canonical}" ), True, ) @@ -271,7 +271,9 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat # s3 fake directories have 0 size and end with "/" if result_key.get("Key").endswith("/") and result_key.get("Size") == 0: yield ( - self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{canonical}"), + self.CloudPath( + f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{canonical}" + ), True, ) yielded_dirs.add(canonical) @@ -280,7 +282,7 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat else: yield ( self.CloudPath( - f"{self.cloud_prefix}{cloud_path.bucket}/{result_key.get('Key')}" + f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{result_key.get('Key')}" ), False, )