Merge pull request #1 from jschneier/master

Rebase with master

.gitignore

@@ -1,26 +1,14 @@
-syntax:glob
-
-*.DS_Store
 *.egg
 *.egg-info
-*.elc
-*.gz
-*.log
 *.orig
 *.pyc
 *.swp
-*.tmp
-*~
+
 .tox/
-_build/
 build/
-dist/*
-django
-local_settings.py
-setuptools*
-*.sqlite
 __pycache__
 .coverage
 .cache
 
 .idea/
+.pytest_cache/

.travis.yml

@@ -1,41 +1,30 @@
+# https://travis-ci.org/jschneier/django-storages/
 sudo: false
 language: python
+
+python: "3.6"
 cache: pip
 
-matrix:
-  include:
-    - env: TOX_ENV=lint
-    - python: 2.7
-      env: TOX_ENV=py27-django18
-    - python: 3.3
-      env: TOX_ENV=py33-django18
-    - python: 3.4
-      env: TOX_ENV=py34-django18
-    - python: 3.5
-      env: TOX_ENV=py35-django18
-    - python: 2.7
-      env: TOX_ENV=py27-django110
-    - python: 3.4
-      env: TOX_ENV=py34-django110
-    - python: 3.5
-      env: TOX_ENV=py35-django110
-    - python: 2.7
-      env: TOX_ENV=py27-django111
-    - python: 3.4
-      env: TOX_ENV=py34-django111
-    - python: 3.5
-      env: TOX_ENV=py35-django111
-    - python: 3.6
-      env: TOX_ENV=py36-django111
+env:
+  - TOXENV=py27-django111
+  - TOXENV=py34-django111
+  - TOXENV=py34-django20
+  - TOXENV=py36-django111
+  - TOXENV=py36-django20
+  - TOXENV=py36-djangomaster
+  - TOXENV=flake8
 
-before_install:
-  - pip install codecov
+matrix:
+  fast_finish: true
+  allow_failures:
+    - env: TOXENV=py36-djangomaster
 
 install:
-  - pip install tox
+  - pip install --upgrade pip setuptools wheel
+  - pip install tox codecov
+
+script:
+  - tox
 
 after_success:
   - codecov
-
-script:
-  - tox -e $TOX_ENV

CHANGELOG.rst

@@ -1,6 +1,27 @@
 django-storages change log
 ==========================
 
+1.6.6 (2018-03-26)
+******************
+
+* You can now specify the backend you are using to install the necessary dependencies using
+  ``extra_requires``. For example ``pip install django-storages[boto3]`` (`#417`_)
+* Add additional content-type detection fallbacks (`#406`_, `#407`_)
+* Add ``GS_LOCATION`` setting to specify subdirectory for ``GoogleCloudStorage`` (`#355`_)
+* Add support for uploading large files to ``DropBoxStorage``, fix saving files (`#379`_, `#378`_, `#301`_)
+* Drop support for Django 1.8 and Django 1.10 (and hence Python 3.3) (`#438`_)
+* Implement ``get_created_time`` for ``GoogleCloudStorage`` (`#464`_)
+
+.. _#417: https://github.com/jschneier/django-storages/pull/417
+.. _#407: https://github.com/jschneier/django-storages/pull/407
+.. _#406: https://github.com/jschneier/django-storages/issues/406
+.. _#355: https://github.com/jschneier/django-storages/pull/355
+.. _#379: https://github.com/jschneier/django-storages/pull/379
+.. _#378: https://github.com/jschneier/django-storages/issues/378
+.. _#301: https://github.com/jschneier/django-storages/issues/301
+.. _#438: https://github.com/jschneier/django-storages/issues/438
+.. _#464: https://github.com/jschneier/django-storages/pull/464
+
 1.6.5 (2017-08-01)
 ******************
 

MANIFEST.in

@@ -1,3 +1,3 @@
-include AUTHORS CHANGELOG.rst LICENSE README.rst requirements*.txt
+include AUTHORS CHANGELOG.rst LICENSE README.rst
 recursive-include tests *.py
 recursive-include docs Makefile conf.py make.bat *.rst

README.rst

@@ -13,17 +13,23 @@ django-storages
 
 Installation
 ============
-Installing from PyPI is as easy as doing::
+Installing from PyPI is as easy as doing:
+
+.. code-block:: bash
 
   pip install django-storages
 
 If you'd prefer to install from source (maybe there is a bugfix in master that
-hasn't been released yet) then the magic incantation you are looking for is::
+hasn't been released yet) then the magic incantation you are looking for is:
+
+.. code-block:: bash
 
   pip install -e 'git+https://github.com/jschneier/django-storages.git#egg=django-storages'
 
 Once that is done add ``storages`` to your ``INSTALLED_APPS`` and set ``DEFAULT_FILE_STORAGE`` to the
-backend of your choice. If, for example, you want to use the boto3 backend you would set::
+backend of your choice. If, for example, you want to use the boto3 backend you would set:
+
+.. code-block:: python
 
   DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
 
@@ -54,8 +60,7 @@ Issues are tracked via GitHub issues at the `project issue page
 
 Documentation
 =============
-The original documentation for django-storages is located at https://django-storages.readthedocs.org/.
-Stay tuned for forthcoming documentation updates.
+Documentation for django-storages is located at https://django-storages.readthedocs.org/.
 
 Contributing
 ============

setup.py

@@ -8,15 +8,19 @@ def read(filename):
         return f.read()
 
 
-def get_requirements_tests():
-    with open('requirements-tests.txt') as f:
-        return f.readlines()
-
-
 setup(
     name='django-storages',
     version=storages.__version__,
     packages=['storages', 'storages.backends'],
+    extras_require={
+        'azure': ['azure'],
+        'boto': ['boto>=2.32.0'],
+        'boto3': ['boto3>=1.2.3'],
+        'dropbox': ['dropbox>=7.2.1'],
+        'google': ['google-cloud-storage>=0.22.0'],
+        'libcloud': ['apache-libcloud'],
+        'sftp': ['paramiko'],
+    },
     author='Josh Schneier',
     author_email='josh.schneier@gmail.com',
     license='BSD',
@@ -27,22 +31,19 @@ def get_requirements_tests():
         'Development Status :: 5 - Production/Stable',
         'Environment :: Web Environment',
         'Framework :: Django',
-        'Framework :: Django :: 1.8',
-        'Framework :: Django :: 1.10',
         'Framework :: Django :: 1.11',
+        'Framework :: Django :: 2.0',
         'Intended Audience :: Developers',
         'License :: OSI Approved :: BSD License',
         'Operating System :: OS Independent',
         'Programming Language :: Python',
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
     ],
-    tests_require=get_requirements_tests(),
     test_suite='tests',
     zip_safe=False
 )

storages/__init__.py

@@ -1 +1 @@
-__version__ = '1.6.5'
+__version__ = '1.6.6'

storages/backends/ftp.py

@@ -109,7 +109,7 @@ def _mkremdirs(self, path):
         for path_part in path_splitted:
             try:
                 self._connection.cwd(path_part)
-            except:
+            except ftplib.all_errors:
                 try:
                     self._connection.mkd(path_part)
                     self._connection.cwd(path_part)

storages/backends/gcloud.py

@@ -1,7 +1,7 @@
 import mimetypes
 from tempfile import SpooledTemporaryFile
 
-from django.core.exceptions import ImproperlyConfigured
+from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation
 from django.core.files.base import File
 from django.core.files.storage import Storage
 from django.utils import timezone
@@ -82,6 +82,7 @@ class GoogleCloudStorage(Storage):
     project_id = setting('GS_PROJECT_ID', None)
     credentials = setting('GS_CREDENTIALS', None)
     bucket_name = setting('GS_BUCKET_NAME', None)
+    location = setting('GS_LOCATION', '')
     auto_create_bucket = setting('GS_AUTO_CREATE_BUCKET', False)
     auto_create_acl = setting('GS_AUTO_CREATE_ACL', 'projectPrivate')
     default_acl = setting('GS_DEFAULT_ACL', None)
@@ -99,6 +100,7 @@ def __init__(self, **settings):
             if hasattr(self, name):
                 setattr(self, name, value)
 
+        self.location = (self.location or '').lstrip('/')
         self._bucket = None
         self._client = None
 
@@ -137,9 +139,14 @@ def _normalize_name(self, name):
         """
         Normalizes the name so that paths like /path/to/ignored/../something.txt
         and ./file.txt work.  Note that clean_name adds ./ to some paths so
-        they need to be fixed here.
+        they need to be fixed here. We check to make sure that the path pointed
+        to is not outside the directory specified by the LOCATION setting.
         """
-        return safe_join('', name)
+        try:
+            return safe_join(self.location, name)
+        except ValueError:
+            raise SuspiciousOperation("Attempted access to '%s' denied." %
+                                      name)
 
     def _encode_name(self, name):
         return smart_str(name, encoding=self.file_name_charset)
@@ -227,6 +234,16 @@ def get_modified_time(self, name):
         updated = blob.updated
         return updated if setting('USE_TZ') else timezone.make_naive(updated)
 
+    def get_created_time(self, name):
+        """
+        Return the creation time (as a datetime) of the file specified by name.
+        The datetime will be timezone-aware if USE_TZ=True.
+        """
+        name = self._normalize_name(clean_name(name))
+        blob = self._get_blob(self._encode_name(name))
+        created = blob.time_created
+        return created if setting('USE_TZ') else timezone.make_naive(created)
+
     def url(self, name):
         # Preserve the trailing slash after normalizing the path.
         name = self._normalize_name(clean_name(name))

storages/backends/s3boto.py

@@ -378,8 +378,8 @@ def _save(self, name, content):
         name = self._normalize_name(cleaned_name)
         headers = self.headers.copy()
         _type, encoding = mimetypes.guess_type(name)
-        content_type = getattr(content, 'content_type',
-                               _type or self.key_class.DefaultContentType)
+        content_type = getattr(content, 'content_type', None)
+        content_type = content_type or _type or self.key_class.DefaultContentType
 
         # setting the content_type in the key object is not enough.
         headers.update({'Content-Type': content_type})

storages/backends/s3boto3.py

@@ -419,8 +419,8 @@ def _save(self, name, content):
         name = self._normalize_name(cleaned_name)
         parameters = self.object_parameters.copy()
         _type, encoding = mimetypes.guess_type(name)
-        content_type = getattr(content, 'content_type',
-                               _type or self.default_content_type)
+        content_type = getattr(content, 'content_type', None)
+        content_type = content_type or _type or self.default_content_type
 
         # setting the content_type in the key object is not enough.
         parameters.update({'ContentType': content_type})

tests/test_gcloud.py

@@ -266,6 +266,28 @@ def test_get_modified_time(self):
             self.assertEqual(mt, aware_date)
             self.storage._bucket.get_blob.assert_called_with(self.filename)
 
+    def test_get_created_time(self):
+        naive_date = datetime.datetime(2017, 1, 2, 3, 4, 5, 678)
+        aware_date = timezone.make_aware(naive_date, timezone.utc)
+
+        self.storage._bucket = mock.MagicMock()
+        blob = mock.MagicMock()
+        blob.time_created = aware_date
+        self.storage._bucket.get_blob.return_value = blob
+
+        with self.settings(TIME_ZONE='America/Montreal', USE_TZ=False):
+            mt = self.storage.get_created_time(self.filename)
+            self.assertTrue(timezone.is_naive(mt))
+            naive_date_montreal = timezone.make_naive(aware_date)
+            self.assertEqual(mt, naive_date_montreal)
+            self.storage._bucket.get_blob.assert_called_with(self.filename)
+
+        with self.settings(TIME_ZONE='America/Montreal', USE_TZ=True):
+            mt = self.storage.get_created_time(self.filename)
+            self.assertTrue(timezone.is_aware(mt))
+            self.assertEqual(mt, aware_date)
+            self.storage._bucket.get_blob.assert_called_with(self.filename)
+
     def test_modified_time_no_file(self):
         self.storage._bucket = mock.MagicMock()
         self.storage._bucket.get_blob.return_value = None

tests/test_s3boto.py

@@ -66,6 +66,26 @@ def test_storage_save(self):
             rewind=True
         )
 
+    def test_content_type(self):
+        """
+        Test saving a file with a None content type.
+        """
+        name = 'test_image.jpg'
+        content = ContentFile('data')
+        content.content_type = None
+        self.storage.save(name, content)
+        self.storage.bucket.get_key.assert_called_once_with(name)
+
+        key = self.storage.bucket.get_key.return_value
+        key.set_metadata.assert_called_with('Content-Type', 'image/jpeg')
+        key.set_contents_from_file.assert_called_with(
+            content,
+            headers={'Content-Type': 'image/jpeg'},
+            policy=self.storage.default_acl,
+            reduced_redundancy=self.storage.reduced_redundancy,
+            rewind=True
+        )
+
     def test_storage_save_gzipped(self):
         """
         Test saving a gzipped file

tests/test_s3boto3.py

@@ -89,6 +89,25 @@ def test_storage_save(self):
             }
         )
 
+    def test_content_type(self):
+        """
+        Test saving a file with a None content type.
+        """
+        name = 'test_image.jpg'
+        content = ContentFile('data')
+        content.content_type = None
+        self.storage.save(name, content)
+        self.storage.bucket.Object.assert_called_once_with(name)
+
+        obj = self.storage.bucket.Object.return_value
+        obj.upload_fileobj.assert_called_with(
+            content.file,
+            ExtraArgs={
+                'ContentType': 'image/jpeg',
+                'ACL': self.storage.default_acl,
+            }
+        )
+
     def test_storage_save_gzipped(self):
         """
         Test saving a gzipped file