diff --git a/cloudpathlib/azure/azblobclient.py b/cloudpathlib/azure/azblobclient.py index 3a77a492..289eb236 100644 --- a/cloudpathlib/azure/azblobclient.py +++ b/cloudpathlib/azure/azblobclient.py @@ -37,8 +37,6 @@ class AzureBlobClient(Client): authentication options. """ - cloud_prefix: str = "az://" - def __init__( self, account_url: Optional[str] = None, @@ -278,13 +276,13 @@ def _list_dir( ) -> Iterable[Tuple[AzureBlobPath, bool]]: if not cloud_path.container: for container in self.service_client.list_containers(): - yield self.CloudPath(f"{self.cloud_prefix}{container.name}"), True + yield self.CloudPath(f"{cloud_path.cloud_prefix}{container.name}"), True if not recursive: continue yield from self._list_dir( - self.CloudPath(f"{self.cloud_prefix}{container.name}"), recursive=True + self.CloudPath(f"{cloud_path.cloud_prefix}{container.name}"), recursive=True ) return @@ -300,7 +298,7 @@ def _list_dir( for path in paths: yield self.CloudPath( - f"{self.cloud_prefix}{cloud_path.container}/{path.name}" + f"{cloud_path.cloud_prefix}{cloud_path.container}/{path.name}" ), path.is_directory else: @@ -313,7 +311,7 @@ def _list_dir( # walk_blobs returns folders with a trailing slash blob_path = blob.name.rstrip("/") blob_cloud_path = self.CloudPath( - f"{self.cloud_prefix}{cloud_path.container}/{blob_path}" + f"{cloud_path.cloud_prefix}{cloud_path.container}/{blob_path}" ) yield blob_cloud_path, ( diff --git a/cloudpathlib/client.py b/cloudpathlib/client.py index 3557d0a7..1b6c32eb 100644 --- a/cloudpathlib/client.py +++ b/cloudpathlib/client.py @@ -30,8 +30,6 @@ class Client(abc.ABC, Generic[BoundedCloudPath]): _cloud_meta: CloudImplementation _default_client = None - cloud_prefix: str - def __init__( self, file_cache_mode: Optional[Union[str, FileCacheMode]] = None, diff --git a/cloudpathlib/gs/gsclient.py b/cloudpathlib/gs/gsclient.py index b5351260..edd5b88a 100644 --- a/cloudpathlib/gs/gsclient.py +++ b/cloudpathlib/gs/gsclient.py @@ -35,8 +35,6 @@ class GSClient(Client): options. """ - cloud_prefix: str = "gs://" - def __init__( self, application_credentials: Optional[Union[str, os.PathLike]] = None, @@ -185,7 +183,7 @@ def _list_dir(self, cloud_path: GSPath, recursive=False) -> Iterable[Tuple[GSPat ) yield from ( - (self.CloudPath(f"{self.cloud_prefix}{str(b)}"), True) + (self.CloudPath(f"{cloud_path.cloud_prefix}{str(b)}"), True) for b in self.client.list_buckets() ) return @@ -204,13 +202,13 @@ def _list_dir(self, cloud_path: GSPath, recursive=False) -> Iterable[Tuple[GSPat if parent not in yielded_dirs and str(parent) != ".": yield ( self.CloudPath( - f"{self.cloud_prefix}{cloud_path.bucket}/{prefix}{parent}" + f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{prefix}{parent}" ), True, # is a directory ) yielded_dirs.add(parent) yield ( - self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{o.name}"), + self.CloudPath(f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{o.name}"), False, ) # is a file else: @@ -220,13 +218,13 @@ def _list_dir(self, cloud_path: GSPath, recursive=False) -> Iterable[Tuple[GSPat # see: https://github.com/googleapis/python-storage/issues/863 for file in iterator: yield ( - self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{file.name}"), + self.CloudPath(f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{file.name}"), False, # is a file ) for directory in iterator.prefixes: yield ( - self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{directory}"), + self.CloudPath(f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{directory}"), True, # is a directory ) diff --git a/cloudpathlib/local/implementations/azure.py b/cloudpathlib/local/implementations/azure.py index 97f6b1be..519924d0 100644 --- a/cloudpathlib/local/implementations/azure.py +++ b/cloudpathlib/local/implementations/azure.py @@ -16,7 +16,6 @@ class LocalAzureBlobClient(LocalClient): substitute when writing tests. """ - cloud_prefix: str = "az://" _cloud_meta = local_azure_blob_implementation def __init__(self, *args, **kwargs): diff --git a/cloudpathlib/local/implementations/gs.py b/cloudpathlib/local/implementations/gs.py index bb3f6491..5e63084f 100644 --- a/cloudpathlib/local/implementations/gs.py +++ b/cloudpathlib/local/implementations/gs.py @@ -12,7 +12,6 @@ class LocalGSClient(LocalClient): substitute when writing tests. """ - cloud_prefix: str = "gs://" _cloud_meta = local_gs_implementation diff --git a/cloudpathlib/local/implementations/s3.py b/cloudpathlib/local/implementations/s3.py index 2a2ee8e1..df9951bf 100644 --- a/cloudpathlib/local/implementations/s3.py +++ b/cloudpathlib/local/implementations/s3.py @@ -12,7 +12,6 @@ class LocalS3Client(LocalClient): substitute when writing tests. """ - cloud_prefix: str = "s3://" _cloud_meta = local_s3_implementation diff --git a/cloudpathlib/s3/s3client.py b/cloudpathlib/s3/s3client.py index eb48c8aa..db130e82 100644 --- a/cloudpathlib/s3/s3client.py +++ b/cloudpathlib/s3/s3client.py @@ -25,8 +25,6 @@ class S3Client(Client): instances. See documentation for the [`__init__` method][cloudpathlib.s3.s3client.S3Client.__init__] for detailed authentication options.""" - cloud_prefix: str = "s3://" - def __init__( self, aws_access_key_id: Optional[str] = None, @@ -219,7 +217,7 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat ) yield from ( - (self.CloudPath(f"{self.cloud_prefix}{b['Name']}"), True) + (self.CloudPath(f"{cloud_path.cloud_prefix}{b['Name']}"), True) for b in self.client.list_buckets().get("Buckets", []) ) return @@ -243,7 +241,9 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat canonical = result_prefix.get("Prefix").rstrip("/") # keep a canonical form if canonical not in yielded_dirs: yield ( - self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{canonical}"), + self.CloudPath( + f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{canonical}" + ), True, ) yielded_dirs.add(canonical) @@ -257,7 +257,7 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat if parent_canonical not in yielded_dirs and str(parent) != ".": yield ( self.CloudPath( - f"{self.cloud_prefix}{cloud_path.bucket}/{parent_canonical}" + f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{parent_canonical}" ), True, ) @@ -271,7 +271,9 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat # s3 fake directories have 0 size and end with "/" if result_key.get("Key").endswith("/") and result_key.get("Size") == 0: yield ( - self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{canonical}"), + self.CloudPath( + f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{canonical}" + ), True, ) yielded_dirs.add(canonical) @@ -280,7 +282,7 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat else: yield ( self.CloudPath( - f"{self.cloud_prefix}{cloud_path.bucket}/{result_key.get('Key')}" + f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{result_key.get('Key')}" ), False, ) diff --git a/tests/conftest.py b/tests/conftest.py index 85ae3482..6b742be8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,47 +1,56 @@ import os -from pathlib import Path, PurePosixPath import shutil +from pathlib import Path, PurePosixPath from tempfile import TemporaryDirectory from typing import Dict, Optional +import boto3 +import botocore from azure.storage.blob import BlobServiceClient from azure.storage.filedatalake import ( DataLakeServiceClient, ) -import boto3 -import botocore from dotenv import find_dotenv, load_dotenv from google.cloud import storage as google_storage from pytest_cases import fixture, fixture_union from shortuuid import uuid from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed -from cloudpathlib import AzureBlobClient, AzureBlobPath, GSClient, GSPath, S3Client, S3Path +import cloudpathlib.azure.azblobclient +import cloudpathlib.s3.s3client +from cloudpathlib import ( + AzureBlobClient, + AzureBlobPath, + GSClient, + GSPath, + S3Client, + S3Path, +) +from cloudpathlib.azure.azblobclient import _hns_rmtree from cloudpathlib.client import register_client_class from cloudpathlib.cloudpath import implementation_registry, register_path_class from cloudpathlib.local import ( - local_azure_blob_implementation, LocalAzureBlobClient, LocalAzureBlobPath, - local_gs_implementation, LocalGSClient, LocalGSPath, - local_s3_implementation, LocalS3Client, LocalS3Path, + local_azure_blob_implementation, + local_gs_implementation, + local_s3_implementation, ) -import cloudpathlib.azure.azblobclient -from cloudpathlib.azure.azblobclient import _hns_rmtree -import cloudpathlib.s3.s3client -from .mock_clients.mock_azureblob import MockBlobServiceClient, DEFAULT_CONTAINER_NAME + from .mock_clients.mock_adls_gen2 import MockedDataLakeServiceClient +from .mock_clients.mock_azureblob import DEFAULT_CONTAINER_NAME, MockBlobServiceClient from .mock_clients.mock_gs import ( - mocked_client_class_factory as mocked_gsclient_class_factory, DEFAULT_GS_BUCKET_NAME, MockTransferManager, ) -from .mock_clients.mock_s3 import mocked_session_class_factory, DEFAULT_S3_BUCKET_NAME - +from .mock_clients.mock_gs import ( + mocked_client_class_factory as mocked_gsclient_class_factory, +) +from .mock_clients.mock_s3 import DEFAULT_S3_BUCKET_NAME, mocked_session_class_factory if os.getenv("USE_LIVE_CLOUD") == "1": load_dotenv(find_dotenv()) @@ -411,7 +420,7 @@ class MyS3Path(S3Path): @register_client_class("mys3") class MyS3Client(S3Client): - cloud_prefix: str = "mys3://" + pass # Mirrors the definition of the S3Client class