Add no_sign_request for S3Client

HISTORY.md

@@ -1,5 +1,9 @@
 # cloudpathlib Changelog
 
+## v0.6.0 (2021-09-07)
+
+- Added `no_sign_request` parameter to `S3Client` instantiation for anonymous requests for public resources on S3. See [documentation](https://cloudpathlib.drivendata.org/stable/api-reference/s3client/#cloudpathlib.s3.s3client.S3Client.__init__) for more details. ([#164](https://github.com/drivendataorg/cloudpathlib/pull/164))
+
 ## v0.5.0 (2021-08-31)
 
 - Added `boto3_transfer_config` parameter to `S3Client` instantiation, which allows passing a `boto3.s3.transfer.TransferConfig` object and is useful for controlling multipart and thread use in uploads and downloads. See [documentation](https://cloudpathlib.drivendata.org/api-reference/s3client/#cloudpathlib.s3.s3client.S3Client.__init__) for more details. ([#150](https://github.com/drivendataorg/cloudpathlib/pull/150))

cloudpathlib/s3/s3client.py

@@ -8,8 +8,9 @@
 
 try:
     from boto3.session import Session
-    import botocore.session
     from boto3.s3.transfer import TransferConfig
+    from botocore.config import Config
+    import botocore.session
 except ModuleNotFoundError:
     implementation_registry["s3"].dependencies_loaded = False
 
@@ -25,6 +26,7 @@ def __init__(
         aws_access_key_id: Optional[str] = None,
         aws_secret_access_key: Optional[str] = None,
         aws_session_token: Optional[str] = None,
+        no_sign_request: Optional[bool] = False,
         botocore_session: Optional["botocore.session.Session"] = None,
         profile_name: Optional[str] = None,
         boto3_session: Optional["Session"] = None,
@@ -46,6 +48,9 @@ def __init__(
             aws_secret_access_key (Optional[str]): AWS secret access key.
             aws_session_token (Optional[str]): Session key for your AWS account. This is only
                 needed when you are using temporarycredentials.
+            no_sign_request: (Optional[bool]): If `True`, credentials are not looked for and we use unsigned
+                requests to fetch resources. This will only allow access to public resources. This is equivalent
+                to `--no-sign-request` in the AWS CLI (https://docs.aws.amazon.com/cli/latest/reference/).
             botocore_session (Optional[botocore.session.Session]): An already instantiated botocore
                 Session.
             profile_name (Optional[str]): Profile name of a profile in a shared credentials file.
@@ -67,8 +72,22 @@ def __init__(
                 botocore_session=botocore_session,
                 profile_name=profile_name,
             )
-        self.s3 = self.sess.resource("s3", endpoint_url=endpoint_url)
-        self.client = self.sess.client("s3", endpoint_url=endpoint_url)
+
+        if no_sign_request:
+            self.s3 = self.sess.resource(
+                "s3",
+                endpoint_url=endpoint_url,
+                config=Config(signature_version=botocore.session.UNSIGNED),
+            )
+            self.client = self.sess.client(
+                "s3",
+                endpoint_url=endpoint_url,
+                config=Config(signature_version=botocore.session.UNSIGNED),
+            )
+        else:
+            self.s3 = self.sess.resource("s3", endpoint_url=endpoint_url)
+            self.client = self.sess.client("s3", endpoint_url=endpoint_url)
+
         self.boto3_transfer_config = boto3_transfer_config
 
         super().__init__(local_cache_dir=local_cache_dir)

setup.py

@@ -59,5 +59,5 @@ def load_requirements(path: Path):
         "Source Code": "https://github.com/drivendataorg/cloudpathlib",
     },
     url="https://github.com/drivendataorg/cloudpathlib",
-    version="0.5.0",
+    version="0.6.0",
 )

tests/mock_clients/mock_s3.py

@@ -29,10 +29,10 @@ def __init__(self, *args, **kwargs):
         def __del__(self):
             self.tmp.cleanup()
 
-        def resource(self, item, endpoint_url):
+        def resource(self, item, endpoint_url, config=None):
             return MockBoto3Resource(self.tmp_path)
 
-        def client(self, item, endpoint_url):
+        def client(self, item, endpoint_url, config=None):
             return MockBoto3Client(self.tmp_path)
 
     return MockBoto3Session

tests/test_s3_specific.py

@@ -3,6 +3,7 @@
 import pytest
 
 from boto3.s3.transfer import TransferConfig
+import botocore
 from cloudpathlib import S3Path
 from cloudpathlib.local import LocalS3Path
 import psutil
@@ -130,3 +131,22 @@ def _execute_on_subprocess_and_observe(use_threads):
 
         # usually ~15 threads are spun up whe use_threads is True
         assert _execute_on_subprocess_and_observe(use_threads=True) > 10
+
+
+def test_no_sign_request(s3_rig):
+    """Tests that we can pass no_sign_request to the S3Client and we will
+    be able to access public resources but not private ones.
+    """
+    if s3_rig.live_server:
+        client = s3_rig.client_class(no_sign_request=True)
+
+        # unsigned can access public path (part of AWS open data)
+        p = client.CloudPath(
+            "s3://ladi/Images/FEMA_CAP/2020/70349/DSC_0001_5a63d42e-27c6-448a-84f1-bfc632125b8e.jpg"
+        )
+        assert p.exists()
+
+        # unsigned raises for private S3 file that exists
+        p = client.CloudPath(f"s3://{s3_rig.drive}/dir_0/file0_to_download.txt")
+        with pytest.raises(botocore.exceptions.ClientError):
+            p.exists()