diff --git a/.gitignore b/.gitignore index ea222dadc..58b378a72 100644 --- a/.gitignore +++ b/.gitignore @@ -1,26 +1,14 @@ -syntax:glob - -*.DS_Store *.egg *.egg-info -*.elc -*.gz -*.log *.orig *.pyc *.swp -*.tmp -*~ + .tox/ -_build/ build/ -dist/* -django -local_settings.py -setuptools* -*.sqlite __pycache__ .coverage .cache .idea/ +.pytest_cache/ diff --git a/.travis.yml b/.travis.yml index 0f03d69ff..3eefa3f9d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,41 +1,30 @@ +# https://travis-ci.org/jschneier/django-storages/ sudo: false language: python + +python: "3.6" cache: pip -matrix: - include: - - env: TOX_ENV=lint - - python: 2.7 - env: TOX_ENV=py27-django18 - - python: 3.3 - env: TOX_ENV=py33-django18 - - python: 3.4 - env: TOX_ENV=py34-django18 - - python: 3.5 - env: TOX_ENV=py35-django18 - - python: 2.7 - env: TOX_ENV=py27-django110 - - python: 3.4 - env: TOX_ENV=py34-django110 - - python: 3.5 - env: TOX_ENV=py35-django110 - - python: 2.7 - env: TOX_ENV=py27-django111 - - python: 3.4 - env: TOX_ENV=py34-django111 - - python: 3.5 - env: TOX_ENV=py35-django111 - - python: 3.6 - env: TOX_ENV=py36-django111 +env: + - TOXENV=py27-django111 + - TOXENV=py34-django111 + - TOXENV=py34-django20 + - TOXENV=py36-django111 + - TOXENV=py36-django20 + - TOXENV=py36-djangomaster + - TOXENV=flake8 -before_install: - - pip install codecov +matrix: + fast_finish: true + allow_failures: + - env: TOXENV=py36-djangomaster install: - - pip install tox + - pip install --upgrade pip setuptools wheel + - pip install tox codecov + +script: + - tox after_success: - codecov - -script: - - tox -e $TOX_ENV diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 195477e41..391afbe4d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,27 @@ django-storages change log ========================== +1.6.6 (2018-03-26) +****************** + +* You can now specify the backend you are using to install the necessary dependencies using + ``extra_requires``. For example ``pip install django-storages[boto3]`` (`#417`_) +* Add additional content-type detection fallbacks (`#406`_, `#407`_) +* Add ``GS_LOCATION`` setting to specify subdirectory for ``GoogleCloudStorage`` (`#355`_) +* Add support for uploading large files to ``DropBoxStorage``, fix saving files (`#379`_, `#378`_, `#301`_) +* Drop support for Django 1.8 and Django 1.10 (and hence Python 3.3) (`#438`_) +* Implement ``get_created_time`` for ``GoogleCloudStorage`` (`#464`_) + +.. _#417: https://github.com/jschneier/django-storages/pull/417 +.. _#407: https://github.com/jschneier/django-storages/pull/407 +.. _#406: https://github.com/jschneier/django-storages/issues/406 +.. _#355: https://github.com/jschneier/django-storages/pull/355 +.. _#379: https://github.com/jschneier/django-storages/pull/379 +.. _#378: https://github.com/jschneier/django-storages/issues/378 +.. _#301: https://github.com/jschneier/django-storages/issues/301 +.. _#438: https://github.com/jschneier/django-storages/issues/438 +.. _#464: https://github.com/jschneier/django-storages/pull/464 + 1.6.5 (2017-08-01) ****************** diff --git a/MANIFEST.in b/MANIFEST.in index d5a19b568..fb9cf185a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,3 @@ -include AUTHORS CHANGELOG.rst LICENSE README.rst requirements*.txt +include AUTHORS CHANGELOG.rst LICENSE README.rst recursive-include tests *.py recursive-include docs Makefile conf.py make.bat *.rst diff --git a/README.rst b/README.rst index 214ba0c97..5990a5584 100644 --- a/README.rst +++ b/README.rst @@ -13,17 +13,23 @@ django-storages Installation ============ -Installing from PyPI is as easy as doing:: +Installing from PyPI is as easy as doing: + +.. code-block:: bash pip install django-storages If you'd prefer to install from source (maybe there is a bugfix in master that -hasn't been released yet) then the magic incantation you are looking for is:: +hasn't been released yet) then the magic incantation you are looking for is: + +.. code-block:: bash pip install -e 'git+https://github.com/jschneier/django-storages.git#egg=django-storages' Once that is done add ``storages`` to your ``INSTALLED_APPS`` and set ``DEFAULT_FILE_STORAGE`` to the -backend of your choice. If, for example, you want to use the boto3 backend you would set:: +backend of your choice. If, for example, you want to use the boto3 backend you would set: + +.. code-block:: python DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' @@ -54,8 +60,7 @@ Issues are tracked via GitHub issues at the `project issue page Documentation ============= -The original documentation for django-storages is located at https://django-storages.readthedocs.org/. -Stay tuned for forthcoming documentation updates. +Documentation for django-storages is located at https://django-storages.readthedocs.org/. Contributing ============ diff --git a/requirements-tests.txt b/requirements-tests.txt deleted file mode 100644 index 229a8a288..000000000 --- a/requirements-tests.txt +++ /dev/null @@ -1,9 +0,0 @@ -boto3>=1.2.3 -boto>=2.32.0 -dropbox>=8.0.0 -Django>=1.8 -flake8 -google-cloud-storage>=0.22.0 -mock -paramiko -pytest-cov>=2.2.1 diff --git a/setup.py b/setup.py index 876059f6b..702d71101 100644 --- a/setup.py +++ b/setup.py @@ -8,15 +8,19 @@ def read(filename): return f.read() -def get_requirements_tests(): - with open('requirements-tests.txt') as f: - return f.readlines() - - setup( name='django-storages', version=storages.__version__, packages=['storages', 'storages.backends'], + extras_require={ + 'azure': ['azure'], + 'boto': ['boto>=2.32.0'], + 'boto3': ['boto3>=1.2.3'], + 'dropbox': ['dropbox>=7.2.1'], + 'google': ['google-cloud-storage>=0.22.0'], + 'libcloud': ['apache-libcloud'], + 'sftp': ['paramiko'], + }, author='Josh Schneier', author_email='josh.schneier@gmail.com', license='BSD', @@ -27,9 +31,8 @@ def get_requirements_tests(): 'Development Status :: 5 - Production/Stable', 'Environment :: Web Environment', 'Framework :: Django', - 'Framework :: Django :: 1.8', - 'Framework :: Django :: 1.10', 'Framework :: Django :: 1.11', + 'Framework :: Django :: 2.0', 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', @@ -37,12 +40,10 @@ def get_requirements_tests(): 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', ], - tests_require=get_requirements_tests(), test_suite='tests', zip_safe=False ) diff --git a/storages/__init__.py b/storages/__init__.py index f3df7f04b..008e8016f 100644 --- a/storages/__init__.py +++ b/storages/__init__.py @@ -1 +1 @@ -__version__ = '1.6.5' +__version__ = '1.6.6' diff --git a/storages/backends/ftp.py b/storages/backends/ftp.py index 0b28280ac..4f32bef5b 100644 --- a/storages/backends/ftp.py +++ b/storages/backends/ftp.py @@ -109,7 +109,7 @@ def _mkremdirs(self, path): for path_part in path_splitted: try: self._connection.cwd(path_part) - except: + except ftplib.all_errors: try: self._connection.mkd(path_part) self._connection.cwd(path_part) diff --git a/storages/backends/gcloud.py b/storages/backends/gcloud.py index 146c755f1..2fca1acf8 100644 --- a/storages/backends/gcloud.py +++ b/storages/backends/gcloud.py @@ -1,7 +1,7 @@ import mimetypes from tempfile import SpooledTemporaryFile -from django.core.exceptions import ImproperlyConfigured +from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation from django.core.files.base import File from django.core.files.storage import Storage from django.utils import timezone @@ -82,6 +82,7 @@ class GoogleCloudStorage(Storage): project_id = setting('GS_PROJECT_ID', None) credentials = setting('GS_CREDENTIALS', None) bucket_name = setting('GS_BUCKET_NAME', None) + location = setting('GS_LOCATION', '') auto_create_bucket = setting('GS_AUTO_CREATE_BUCKET', False) auto_create_acl = setting('GS_AUTO_CREATE_ACL', 'projectPrivate') default_acl = setting('GS_DEFAULT_ACL', None) @@ -99,6 +100,7 @@ def __init__(self, **settings): if hasattr(self, name): setattr(self, name, value) + self.location = (self.location or '').lstrip('/') self._bucket = None self._client = None @@ -137,9 +139,14 @@ def _normalize_name(self, name): """ Normalizes the name so that paths like /path/to/ignored/../something.txt and ./file.txt work. Note that clean_name adds ./ to some paths so - they need to be fixed here. + they need to be fixed here. We check to make sure that the path pointed + to is not outside the directory specified by the LOCATION setting. """ - return safe_join('', name) + try: + return safe_join(self.location, name) + except ValueError: + raise SuspiciousOperation("Attempted access to '%s' denied." % + name) def _encode_name(self, name): return smart_str(name, encoding=self.file_name_charset) @@ -227,6 +234,16 @@ def get_modified_time(self, name): updated = blob.updated return updated if setting('USE_TZ') else timezone.make_naive(updated) + def get_created_time(self, name): + """ + Return the creation time (as a datetime) of the file specified by name. + The datetime will be timezone-aware if USE_TZ=True. + """ + name = self._normalize_name(clean_name(name)) + blob = self._get_blob(self._encode_name(name)) + created = blob.time_created + return created if setting('USE_TZ') else timezone.make_naive(created) + def url(self, name): # Preserve the trailing slash after normalizing the path. name = self._normalize_name(clean_name(name)) diff --git a/storages/backends/s3boto.py b/storages/backends/s3boto.py index 981faa441..bfb238374 100644 --- a/storages/backends/s3boto.py +++ b/storages/backends/s3boto.py @@ -378,8 +378,8 @@ def _save(self, name, content): name = self._normalize_name(cleaned_name) headers = self.headers.copy() _type, encoding = mimetypes.guess_type(name) - content_type = getattr(content, 'content_type', - _type or self.key_class.DefaultContentType) + content_type = getattr(content, 'content_type', None) + content_type = content_type or _type or self.key_class.DefaultContentType # setting the content_type in the key object is not enough. headers.update({'Content-Type': content_type}) diff --git a/storages/backends/s3boto3.py b/storages/backends/s3boto3.py index 9caae4d0a..02bfab2dd 100644 --- a/storages/backends/s3boto3.py +++ b/storages/backends/s3boto3.py @@ -419,8 +419,8 @@ def _save(self, name, content): name = self._normalize_name(cleaned_name) parameters = self.object_parameters.copy() _type, encoding = mimetypes.guess_type(name) - content_type = getattr(content, 'content_type', - _type or self.default_content_type) + content_type = getattr(content, 'content_type', None) + content_type = content_type or _type or self.default_content_type # setting the content_type in the key object is not enough. parameters.update({'ContentType': content_type}) diff --git a/tests/test_gcloud.py b/tests/test_gcloud.py index f91bf1066..b4c51af85 100644 --- a/tests/test_gcloud.py +++ b/tests/test_gcloud.py @@ -266,6 +266,28 @@ def test_get_modified_time(self): self.assertEqual(mt, aware_date) self.storage._bucket.get_blob.assert_called_with(self.filename) + def test_get_created_time(self): + naive_date = datetime.datetime(2017, 1, 2, 3, 4, 5, 678) + aware_date = timezone.make_aware(naive_date, timezone.utc) + + self.storage._bucket = mock.MagicMock() + blob = mock.MagicMock() + blob.time_created = aware_date + self.storage._bucket.get_blob.return_value = blob + + with self.settings(TIME_ZONE='America/Montreal', USE_TZ=False): + mt = self.storage.get_created_time(self.filename) + self.assertTrue(timezone.is_naive(mt)) + naive_date_montreal = timezone.make_naive(aware_date) + self.assertEqual(mt, naive_date_montreal) + self.storage._bucket.get_blob.assert_called_with(self.filename) + + with self.settings(TIME_ZONE='America/Montreal', USE_TZ=True): + mt = self.storage.get_created_time(self.filename) + self.assertTrue(timezone.is_aware(mt)) + self.assertEqual(mt, aware_date) + self.storage._bucket.get_blob.assert_called_with(self.filename) + def test_modified_time_no_file(self): self.storage._bucket = mock.MagicMock() self.storage._bucket.get_blob.return_value = None diff --git a/tests/test_s3boto.py b/tests/test_s3boto.py index 0457cc506..5e6b2c935 100644 --- a/tests/test_s3boto.py +++ b/tests/test_s3boto.py @@ -66,6 +66,26 @@ def test_storage_save(self): rewind=True ) + def test_content_type(self): + """ + Test saving a file with a None content type. + """ + name = 'test_image.jpg' + content = ContentFile('data') + content.content_type = None + self.storage.save(name, content) + self.storage.bucket.get_key.assert_called_once_with(name) + + key = self.storage.bucket.get_key.return_value + key.set_metadata.assert_called_with('Content-Type', 'image/jpeg') + key.set_contents_from_file.assert_called_with( + content, + headers={'Content-Type': 'image/jpeg'}, + policy=self.storage.default_acl, + reduced_redundancy=self.storage.reduced_redundancy, + rewind=True + ) + def test_storage_save_gzipped(self): """ Test saving a gzipped file diff --git a/tests/test_s3boto3.py b/tests/test_s3boto3.py index ef1a263e3..cf079eda1 100644 --- a/tests/test_s3boto3.py +++ b/tests/test_s3boto3.py @@ -89,6 +89,25 @@ def test_storage_save(self): } ) + def test_content_type(self): + """ + Test saving a file with a None content type. + """ + name = 'test_image.jpg' + content = ContentFile('data') + content.content_type = None + self.storage.save(name, content) + self.storage.bucket.Object.assert_called_once_with(name) + + obj = self.storage.bucket.Object.return_value + obj.upload_fileobj.assert_called_with( + content.file, + ExtraArgs={ + 'ContentType': 'image/jpeg', + 'ACL': self.storage.default_acl, + } + ) + def test_storage_save_gzipped(self): """ Test saving a gzipped file diff --git a/tox.ini b/tox.ini index 05dc3d145..e1e4d90b9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,33 +1,35 @@ [tox] envlist = - lint - {py27,py33,py34,py35}-django18 - {py27,py34,py35}-django110 - {py27,py34,py35,py36}-django111 - + py27-django111 + py34-django{111,20} + py35-django{111,20,master} + py36-django{111,20,master} + flake8 [testenv] -commands = py.test --cov=storages tests/ setenv = - PYTHONDONTWRITEBYTECODE=1 - DJANGO_SETTINGS_MODULE=tests.settings + DJANGO_SETTINGS_MODULE = tests.settings + PYTHONWARNINGS = all + PYTHONDONTWRITEBYTECODE = 1 +commands = py.test --cov=storages tests/ {posargs} deps = - django18: Django>=1.8, <1.9 - django110: Django>=1.10, <1.11 - django111: Django>=1.11, <2.0 - py27: mock - boto3>=1.2.3 - boto>=2.32.0 - dropbox>=8.0.0 - google-cloud-storage>=0.22.0 - paramiko - pytest-cov>=2.2.1 - + django111: Django>=1.11,<2.0 + django20: Django>=2.0,<2.1 + djangomaster: https://github.com/django/django/archive/master.tar.gz + py27: mock + pytest + pytest-cov + apache-libcloud + boto + boto3 + dropbox + google-cloud-storage + paramiko -[testenv:lint] +[testenv:flake8] deps = - flake8 - isort + flake8 + isort commands = - flake8 - isort --recursive --check-only --diff storages/ tests/ + flake8 + isort --recursive --check-only --diff storages/ tests/