diff --git a/.gitignore b/.gitignore index 63940c2de..ea222dadc 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,5 @@ setuptools* __pycache__ .coverage .cache + +.idea/ diff --git a/.hgignore b/.hgignore deleted file mode 100644 index 1a2d13161..000000000 --- a/.hgignore +++ /dev/null @@ -1,21 +0,0 @@ -syntax:glob - -*.DS_Store -*.egg -*.egg-info -*.elc -*.gz -*.log -*.orig -*.pyc -*.swp -*.tmp -*~ -.tox/ -_build/ -build/ -dist/* -django -local_settings.py -setuptools* -testdb.sqlite diff --git a/.travis.yml b/.travis.yml index da94abbce..0f03d69ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,20 +2,31 @@ sudo: false language: python cache: pip -python: - - 3.5 - -env: - - TOX_ENV=py27-django18 - - TOX_ENV=py33-django18 - - TOX_ENV=py34-django18 - - TOX_ENV=py35-django18 - - TOX_ENV=py27-django19 - - TOX_ENV=py34-django19 - - TOX_ENV=py35-django19 - - TOX_ENV=py27-django110 - - TOX_ENV=py34-django110 - - TOX_ENV=py35-django110 +matrix: + include: + - env: TOX_ENV=lint + - python: 2.7 + env: TOX_ENV=py27-django18 + - python: 3.3 + env: TOX_ENV=py33-django18 + - python: 3.4 + env: TOX_ENV=py34-django18 + - python: 3.5 + env: TOX_ENV=py35-django18 + - python: 2.7 + env: TOX_ENV=py27-django110 + - python: 3.4 + env: TOX_ENV=py34-django110 + - python: 3.5 + env: TOX_ENV=py35-django110 + - python: 2.7 + env: TOX_ENV=py27-django111 + - python: 3.4 + env: TOX_ENV=py34-django111 + - python: 3.5 + env: TOX_ENV=py35-django111 + - python: 3.6 + env: TOX_ENV=py36-django111 before_install: - pip install codecov diff --git a/AUTHORS b/AUTHORS index 0dc804966..25db7d015 100644 --- a/AUTHORS +++ b/AUTHORS @@ -27,6 +27,11 @@ By order of apparition, thanks: * Michael Barrientos (S3 with Boto3) * piglei (patches) * Matt Braymer-Hayes (S3 with Boto3) + * Eirik Martiniussen Sylliaas (Google Cloud Storage native support) + * Jody McIntyre (Google Cloud Storage native support) + * Stanislav Kaledin (Bug fixes in SFTPStorage) + * Filip Vavera (Google Cloud MIME types support) + * Max Malysh (Dropbox large file support) Extra thanks to Marty for adding this in Django, you can buy his very interesting book (Pro Django). diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0fcb22220..195477e41 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,23 +1,121 @@ django-storages change log ========================== -1.5.3 (XXXX-XX-XX) +1.6.5 (2017-08-01) ****************** -* Pass in the location constraint when auto creating a bucket (`#257`_, `#258`_ thanks @mattayes) +* Fix Django 1.11 regression with gzipped content being saved twice + resulting in empty files (`#367`_, `#371`_, `#373`_) +* Fix the ``mtime`` when gzipping content on ``S3Boto3Storage`` (`#374`_) +.. _#367: https://github.com/jschneier/django-storages/issues/367 +.. _#371: https://github.com/jschneier/django-storages/pull/371 +.. _#373: https://github.com/jschneier/django-storages/pull/373 +.. _#374: https://github.com/jschneier/django-storages/pull/374 + +1.6.4 (2017-07-27) +****************** + +* Files uploaded with ``GoogleCloudStorage`` will now set their appropriate mimetype (`#320`_) +* Fix ``DropBoxStorage.url`` to work. (`#357`_) +* Fix ``S3Boto3Storage`` when ``AWS_PRELOAD_METADATA = True`` (`#366`_) +* Fix ``S3Boto3Storage`` uploading file-like objects without names (`#195`_, `#368`_) +* ``S3Boto3Storage`` is now threadsafe - a separate session is created on a + per-thread basis (`#268`_, `#358`_) + +.. _#320: https://github.com/jschneier/django-storages/pull/320 +.. _#357: https://github.com/jschneier/django-storages/pull/357 +.. _#366: https://github.com/jschneier/django-storages/pull/366 +.. _#195: https://github.com/jschneier/django-storages/pull/195 +.. _#368: https://github.com/jschneier/django-storages/pull/368 +.. _#268: https://github.com/jschneier/django-storages/issues/268 +.. _#358: https://github.com/jschneier/django-storages/pull/358 + +1.6.3 (2017-06-23) +****************** + +* Revert default ``AWS_S3_SIGNATURE_VERSION`` to V2 to restore backwards + compatability in ``S3Boto3``. It's recommended that all new projects set + this to be ``'s3v4'``. (`#344`_) + +.. _#344: https://github.com/jschneier/django-storages/pull/344 + +1.6.2 (2017-06-22) +****************** + +* Fix regression in ``safe_join()`` to handle a trailing slash in an + intermediate path. (`#341`_) +* Fix regression in ``gs.GSBotoStorage`` getting an unexpected kwarg. + (`#342`_) + +.. _#341: https://github.com/jschneier/django-storages/pull/341 +.. _#342: https://github.com/jschneier/django-storages/pull/342 + +1.6.1 (2017-06-22) +****************** + +* Drop support for Django 1.9 (`e89db45`_) +* Fix regression in ``safe_join()`` to allow joining a base path with an empty + string. (`#336`_) + +.. _e89db45: https://github.com/jschneier/django-storages/commit/e89db451d7e617638b5991e31df4c8de196546a6 +.. _#336: https://github.com/jschneier/django-storages/pull/336 + +1.6 (2017-06-21) +****************** + +* **Breaking:** Remove backends deprecated in v1.5.1 (`#280`_) +* **Breaking:** ``DropBoxStorage`` has been upgrade to support v2 of the API, v1 will be shut off at the + end of the month - upgrading is recommended (`#273`_) +* **Breaking:** The ``SFTPStorage`` backend now checks for the existence of the fallback ``~/.ssh/known_hosts`` + before attempting to load it. If you had previously been passing in a path to a non-existent file it will no longer + attempt to load the fallback. (`#118`_, `#325`_) +* **Breaking:** The default version value for ``AWS_S3_SIGNATURE_VERSION`` is now ``'s3v4'``. No changes should + be required (`#335`_) +* **Deprecation:** The undocumented ``gs.GSBotoStorage`` backend. See the new ``gcloud.GoogleCloudStorage`` + or ``apache_libcloud.LibCloudStorage`` backends instead. (`#236`_) +* Add a new backend, ``gcloud.GoogleCloudStorage`` based on the ``google-cloud`` bindings. (`#236`_) +* Pass in the location constraint when auto creating a bucket in ``S3Boto3Storage`` (`#257`_, `#258`_) +* Add support for reading ``AWS_SESSION_TOKEN`` and ``AWS_SECURITY_TOKEN`` from the environment + to ``S3Boto3Storage`` and ``S3BotoStorage``. (`#283`_) +* Fix Boto3 non-ascii filenames on Python 2.7 (`#216`_, `#217`_) +* Fix ``collectstatic`` timezone handling in and add ``get_modified_time`` to ``S3BotoStorage`` (`#290`_) +* Add support for Django 1.11 (`#295`_) +* Add ``project`` keyword support to GCS in ``LibCloudStorage`` backend (`#269`_) +* Files that have a guessable encoding (e.g. gzip or compress) will be uploaded with that Content-Encoding in + the ``s3boto3`` backend (`#263`_, `#264`_) +* The Dropbox backend now properly translates backslashes in Windows paths into forward slashes (`e52a127`_) +* The S3 backends now permit colons in the keys (`#248`_, `#322`_) + +.. _#217: https://github.com/jschneier/django-storages/pull/217 +.. _#273: https://github.com/jschneier/django-storages/pull/273 +.. _#216: https://github.com/jschneier/django-storages/issues/216 +.. _#283: https://github.com/jschneier/django-storages/pull/283 +.. _#280: https://github.com/jschneier/django-storages/pull/280 .. _#257: https://github.com/jschneier/django-storages/issues/257 .. _#258: https://github.com/jschneier/django-storages/pull/258 +.. _#290: https://github.com/jschneier/django-storages/pull/290 +.. _#295: https://github.com/jschneier/django-storages/pull/295 +.. _#269: https://github.com/jschneier/django-storages/pull/269 +.. _#263: https://github.com/jschneier/django-storages/issues/263 +.. _#264: https://github.com/jschneier/django-storages/pull/264 +.. _e52a127: https://github.com/jschneier/django-storages/commit/e52a127523fdd5be50bb670ccad566c5d527f3d1 +.. _#236: https://github.com/jschneier/django-storages/pull/236 +.. _#118: https://github.com/jschneier/django-storages/issues/118 +.. _#325: https://github.com/jschneier/django-storages/pull/325 +.. _#248: https://github.com/jschneier/django-storages/issues/248 +.. _#322: https://github.com/jschneier/django-storages/pull/322 +.. _#335: https://github.com/jschneier/django-storages/pull/335 1.5.2 (2017-01-13) ****************** -* Actually use ``SFTP_STORAGE_HOST`` in ``SFTPStorage`` backend (`#204`_ thanks @jbittel) -* Fix ``S3Boto3Storage`` to avoid race conditions in a multi-threaded WSGI environment (`#238`_ thanks @jdufresne) +* Actually use ``SFTP_STORAGE_HOST`` in ``SFTPStorage`` backend (`#204`_) +* Fix ``S3Boto3Storage`` to avoid race conditions in a multi-threaded WSGI environment (`#238`_) * Fix trying to localize a naive datetime when ``settings.USE_TZ`` is ``False`` in ``S3Boto3Storage.modified_time``. - (thanks to @tomchuk and @piglei for the reports and the patches, `#235`_, `#234`_) -* Fix automatic bucket creation in ``S3Boto3Storage`` when ``AWS_AUTO_CREATE_BUCKET`` is ``True`` (`#196`_ thanks @linuxlewis) -* Improve the documentation for the S3 backends (thanks to various contributors!) + (`#235`_, `#234`_) +* Fix automatic bucket creation in ``S3Boto3Storage`` when ``AWS_AUTO_CREATE_BUCKET`` is ``True`` (`#196`_) +* Improve the documentation for the S3 backends .. _#204: https://github.com/jschneier/django-storages/pull/204 .. _#238: https://github.com/jschneier/django-storages/pull/238 @@ -28,17 +126,17 @@ django-storages change log 1.5.1 (2016-09-13) ****************** -* **Drop support for Django 1.7** (`#185`_) -* **Deprecate hashpath, image, overwrite, mogile, symlinkorcopy, database, mogile, couchdb.** - See (`issue #202`_) to discuss maintenance going forward +* **Breaking:** Drop support for Django 1.7 (`#185`_) +* **Deprecation:** hashpath, image, overwrite, mogile, symlinkorcopy, database, mogile, couchdb. + See (`#202`_) to discuss maintenance going forward * Use a fixed ``mtime`` argument for ``GzipFile`` in ``S3BotoStorage`` and ``S3Boto3Storage`` to ensure a stable output for gzipped files * Use ``.putfileobj`` instead of ``.put`` in ``S3Boto3Storage`` to use the transfer manager, allowing files greater than 5GB to be put on S3 (`#194`_ , `#201`_) -* Update ``S3Boto3Storage`` for Django 1.10 (`#181`_) (``get_modified_time`` and ``get_accessed_time``) thanks @JshWright -* Fix bad kwarg name in ``S3Boto3Storage`` when `AWS_PRELOAD_METADATA` is `True` (`#189`_, `#190`_) thanks @leonsmith +* Update ``S3Boto3Storage`` for Django 1.10 (`#181`_) (``get_modified_time`` and ``get_accessed_time``) +* Fix bad kwarg name in ``S3Boto3Storage`` when `AWS_PRELOAD_METADATA` is `True` (`#189`_, `#190`_) -.. _issue #202: https://github.com/jschneier/django-storages/issues/202 +.. _#202: https://github.com/jschneier/django-storages/issues/202 .. _#201: https://github.com/jschneier/django-storages/pull/201 .. _#194: https://github.com/jschneier/django-storages/issues/194 .. _#190: https://github.com/jschneier/django-storages/pull/190 @@ -49,13 +147,13 @@ django-storages change log 1.5.0 (2016-08-02) ****************** -* Add new backend ``S3Boto3Storage`` (`#179`_) MASSIVE THANKS @mbarrien -* Add a `strict` option to `utils.setting` (`#176`_) thanks @ZuluPro -* Tests, documentation, fixing ``.close`` for ``SFTPStorage`` (`#177`_) thanks @ZuluPro -* Tests, documentation, add `.readlines` for ``FTPStorage`` (`#175`_) thanks @ZuluPro -* Tests and documentation for ``DropBoxStorage`` (`#174`_) thanks @ZuluPro -* Fix ``MANIFEST.in`` to not ship ``.pyc`` files. (`#145`_) thanks @fladi -* Enable CI testing of Python3.5 and fix test failure from api change (`#171`_) thanks @tnir +* Add new backend ``S3Boto3Storage`` (`#179`_) +* Add a `strict` option to `utils.setting` (`#176`_) +* Tests, documentation, fixing ``.close`` for ``SFTPStorage`` (`#177`_) +* Tests, documentation, add `.readlines` for ``FTPStorage`` (`#175`_) +* Tests and documentation for ``DropBoxStorage`` (`#174`_) +* Fix ``MANIFEST.in`` to not ship ``.pyc`` files. (`#145`_) +* Enable CI testing of Python 3.5 and fix test failure from api change (`#171`_) .. _#145: https://github.com/jschneier/django-storages/pull/145 .. _#171: https://github.com/jschneier/django-storages/pull/171 @@ -70,10 +168,10 @@ django-storages change log * Files that have a guessable encoding (e.g. gzip or compress) will be uploaded with that Content-Encoding in the ``s3boto`` backend. Compressable types such as ``application/javascript`` will still be gzipped. - PR `#122`_ thanks @cambonf -* Fix ``DropBoxStorage.exists`` check and add ``DropBoxStorage.url`` (`#127`_) thanks @zuck + PR `#122`_ +* Fix ``DropBoxStorage.exists`` check and add ``DropBoxStorage.url`` (`#127`_) * Add ``GS_HOST`` setting (with a default of ``GSConnection.DefaultHost``) to fix ``GSBotoStorage``. - Issue `#124`_. Fixed in `#125`_. Thanks @patgmiller @dcgoss. + (`#124`_, `#125`_) .. _#122: https://github.com/jschneier/django-storages/pull/122 .. _#127: https://github.com/jschneier/django-storages/pull/127 @@ -89,10 +187,10 @@ django-storages change log 1.3.2 (2016-01-26) ****************** -* Fix memory leak from not closing underlying temp file in ``s3boto`` backend (`#106`_) thanks @kmmbvnr -* Allow easily specifying a custom expiry time when generating a url for ``S3BotoStorage`` (`#96`_) thanks @mattbriancon +* Fix memory leak from not closing underlying temp file in ``s3boto`` backend (`#106`_) +* Allow easily specifying a custom expiry time when generating a url for ``S3BotoStorage`` (`#96`_) * Check for bucket existence when the empty path ('') is passed to ``storage.exists`` in ``S3BotoStorage`` - - this prevents a crash when running ``collecstatic -c`` on Django 1.9.1 (`#112`_) fixed in `#116`_ thanks @xblitz + this prevents a crash when running ``collectstatic -c`` on Django 1.9.1 (`#112`_) fixed in `#116`_ .. _#106: https://github.com/jschneier/django-storages/pull/106 .. _#96: https://github.com/jschneier/django-storages/pull/96 @@ -103,12 +201,12 @@ django-storages change log 1.3.1 (2016-01-12) ****************** -* A few Azure Storage fixes [pass the content-type to Azure, handle chunked content, fix ``url``] (`#45`__) thanks @erlingbo -* Add support for a Dropbox (``dropbox``) storage backend, thanks @ZuluPro (`#76`_) +* A few Azure Storage fixes [pass the content-type to Azure, handle chunked content, fix ``url``] (`#45`__) +* Add support for a Dropbox (``dropbox``) storage backend * Various fixes to the ``apache_libcloud`` backend [return the number of bytes asked for by ``.read``, make ``.name`` non-private, don't - initialize to an empty ``BytesIO`` object] thanks @kaedroho (`#55`_) -* Fix multi-part uploads in ``s3boto`` backend not respecting ``AWS_S3_ENCRYPTION`` (`#94`_) thanks @andersontep -* Automatically gzip svg files thanks @comandrei (`#100`_) + initialize to an empty ``BytesIO`` object] (`#55`_) +* Fix multi-part uploads in ``s3boto`` backend not respecting ``AWS_S3_ENCRYPTION`` (`#94`_) +* Automatically gzip svg files (`#100`_) .. __: https://github.com/jschneier/django-storages/pull/45 .. _#76: https://github.com/jschneier/django-storages/pull/76 @@ -120,13 +218,13 @@ django-storages change log 1.3 (2015-08-14) **************** -* **Drop Support for Django 1.5 and Python2.6** -* Remove previously deprecated mongodb backend -* Remove previously deprecated ``parse_ts_extended`` from s3boto storage +* **Breaking:** Drop Support for Django 1.5 and Python 2.6 +* **Breaking:** Remove previously deprecated mongodb backend +* **Breaking:** Remove previously deprecated ``parse_ts_extended`` from s3boto storage * Add support for Django 1.8+ (`#36`__) * Add ``AWS_S3_PROXY_HOST`` and ``AWS_S3_PROXY_PORT`` settings for s3boto backend (`#41`_) * Fix Python3K compat issue in apache_libcloud (`#52`_) -* Fix Google Storage backend not respecting ``GS_IS_GZIPPED`` setting (`#51`__, `#60`_) thanks @stmos +* Fix Google Storage backend not respecting ``GS_IS_GZIPPED`` setting (`#51`__, `#60`_) * Rename FTP ``_name`` attribute to ``name`` which is what the Django ``File`` api is expecting (`#70`_) * Put ``StorageMixin`` first in inheritance to maintain backwards compat with older versions of Django (`#63`_) @@ -164,9 +262,9 @@ django-storages change log 1.2.1 (2014-12-31) ****************** +* **Deprecation:** Issue warning about ``parse_ts_extended`` +* **Deprecation:** mongodb backend - django-mongodb-engine now ships its own storage backend * Fix ``storage.modified_time`` crashing on new files when ``AWS_PRELOAD_METADATA=True`` (`#11`_, `#12`__, `#14`_) -* Deprecate and issue warning about ``parse_ts_extended`` -* Deprecate mongodb backend - django-mongodb-engine now ships its own storage backend .. _#11: https://github.com/jschneier/django-storages/pull/11 __ https://github.com/jschneier/django-storages/issues/12 @@ -176,11 +274,11 @@ __ https://github.com/jschneier/django-storages/issues/12 1.2 (2014-12-14) **************** +* **Breaking:** Remove legacy S3 storage (`#1`_) +* **Breaking:** Remove mosso files backend (`#2`_) * Add text/javascript mimetype to S3BotoStorage gzip allowed defaults * Add support for Django 1.7 migrations in S3BotoStorage and ApacheLibCloudStorage (`#5`_, `#8`_) * Python3K (3.3+) now available for S3Boto backend (`#4`_) -* Remove legacy S3 storage (`#1`_) -* Remove mosso files backend (`#2`_) .. _#8: https://github.com/jschneier/django-storages/pull/8 .. _#5: https://github.com/jschneier/django-storages/pull/5 @@ -325,4 +423,3 @@ since March 2013. .. _#89: https://bitbucket.org/david/django-storages/issue/89/112-broke-the-mosso-backend .. _pull request #5: https://bitbucket.org/david/django-storages/pull-request/5/fixed-path-bug-and-added-testcase-for - diff --git a/README.rst b/README.rst index 660f592d1..214ba0c97 100644 --- a/README.rst +++ b/README.rst @@ -23,9 +23,9 @@ hasn't been released yet) then the magic incantation you are looking for is:: pip install -e 'git+https://github.com/jschneier/django-storages.git#egg=django-storages' Once that is done add ``storages`` to your ``INSTALLED_APPS`` and set ``DEFAULT_FILE_STORAGE`` to the -backend of your choice. If, for example, you want to use the s3boto backend you would set:: +backend of your choice. If, for example, you want to use the boto3 backend you would set:: - DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage' + DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' There are also a number of settings available to control how each storage backend functions, please consult the documentation for a comprehensive list. @@ -35,7 +35,7 @@ About django-storages is a project to provide a variety of storage backends in a single library. This library is usually compatible with the currently supported versions of -Django. Check the trove classifiers in setup.py to be sure. +Django. Check the Trove classifiers in setup.py to be sure. History ======= diff --git a/docs/backends/amazon-S3.rst b/docs/backends/amazon-S3.rst index 1edfcab86..b25e84054 100644 --- a/docs/backends/amazon-S3.rst +++ b/docs/backends/amazon-S3.rst @@ -14,6 +14,12 @@ for the forseeable future. For historical completeness an extreme legacy backend was removed in version 1.2 +If using the boto backend on a new project (not recommended) it is recommended +that you configure it to also use `AWS Signature Version 4`_. This can be done +by adding ``S3_USE_SIGV4 = True`` to your settings and setting the ``AWS_S3_HOST`` +configuration option. For regions created after January 2014 this is your only +option if you insist on using the boto backend. + Settings -------- @@ -49,27 +55,25 @@ Available are numerous settings. It should be especially noted the following: ``AWS_HEADERS`` (optional - boto only, for boto3 see ``AWS_S3_OBJECT_PARAMETERS``) If you'd like to set headers sent with each file of the storage:: - # see http://developer.yahoo.com/performance/rules.html#expires AWS_HEADERS = { 'Expires': 'Thu, 15 Apr 2010 20:00:00 GMT', 'Cache-Control': 'max-age=86400', } ``AWS_S3_OBJECT_PARAMETERS`` (optional - boto3 only) - Use this to set arbitrary parameters on your object (such as Cache-Control):: + Use this to set object parameters on your object (such as CacheControl):: AWS_S3_OBJECT_PARAMETERS = { - 'Cache-Control': 'max-age=86400', + 'CacheControl': 'max-age=86400', } ``AWS_QUERYSTRING_AUTH`` (optional; default is ``True``) - Setting ``AWS_QUERYSTRING_AUTH`` to ``False`` removes `query parameter - authentication`_ from generated URLs. This can be useful if your S3 buckets are - public. + Setting ``AWS_QUERYSTRING_AUTH`` to ``False`` to remove query parameter + authentication from generated URLs. This can be useful if your S3 buckets + are public. ``AWS_QUERYSTRING_EXPIRE`` (optional; default is 3600 seconds) - The number of seconds that a generated URL with `query parameter - authentication`_ is valid for. + The number of seconds that a generated URL is valid for. ``AWS_S3_ENCRYPTION`` (optional; default is ``False``) Enable server-side file encryption while at rest, by setting ``encrypt_key`` parameter to True. More info available here: http://boto.cloudhackers.com/en/latest/ref/s3.html @@ -77,10 +81,51 @@ Available are numerous settings. It should be especially noted the following: ``AWS_S3_FILE_OVERWRITE`` (optional: default is ``True``) By default files with the same name will overwrite each other. Set this to ``False`` to have extra characters appended. +``AWS_S3_HOST`` (optional - boto only, default is ``s3.amazonaws.com``) + + To ensure you use `AWS Signature Version 4`_ it is recommended to set this to the host of your bucket. See the + `S3 region list`_ to figure out the appropriate endpoint for your bucket. Also be sure to add + ``S3_USE_SIGV4 = True`` to settings.py + + .. note:: + + The signature versions are not backwards compatible so be careful about url endpoints if making this change + for legacy projects. + ``AWS_LOCATION`` (optional: default is `''`) A path prefix that will be prepended to all uploads -.. _query parameter authentication: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html +``AWS_IS_GZIPPED`` (optional: default is ``False``) + Whether or not to enable gzipping of content types specified by ``GZIP_CONTENT_TYPES`` + +``GZIP_CONTENT_TYPES`` (optional: default is ``text/css``, ``text/javascript``, ``application/javascript``, ``application/x-javascript``, ``image/svg+xml``) + When ``AWS_IS_GZIPPED`` is set to ``True`` the content types which will be gzipped + +``AWS_S3_REGION_NAME`` (optional: default is ``None``) + Name of the AWS S3 region to use (eg. eu-west-1) + +``AWS_S3_USE_SSL`` (optional: default is ``True``) + Whether or not to use SSL when connecting to S3. + +``AWS_S3_ENDPOINT_URL`` (optional: default is ``None``) + Custom S3 URL to use when connecting to S3, including scheme. Overrides ``AWS_S3_REGION_NAME`` and ``AWS_S3_USE_SSL``. + +``AWS_S3_CALLING_FORMAT`` (optional: default is ``SubdomainCallingFormat()``) + Defines the S3 calling format to use to connect to the static bucket. + +``AWS_S3_SIGNATURE_VERSION`` (optional - boto3 only) + + All AWS regions support v4 of the signing protocol. To use it set this to ``'s3v4'``. It is recommended + to do this for all new projects and required for all regions launched after January 2014. To see + if your region is one of them you can view the `S3 region list`_. + + .. note:: + + The signature versions are not backwards compatible so be careful about url endpoints if making this change + for legacy projects. + +.. _AWS Signature Version 4: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html +.. _S3 region list: http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region CloudFront ~~~~~~~~~~ @@ -90,6 +135,10 @@ to serve those files using that:: AWS_S3_CUSTOM_DOMAIN = 'cdn.mydomain.com' +**NOTE:** Django's `STATIC_URL` `must end in a slash`_ and the `AWS_S3_CUSTOM_DOMAIN` *must not*. It is best to set this variable indepedently of `STATIC_URL`. + +.. _must end in a slash: https://docs.djangoproject.com/en/dev/ref/settings/#static-url + Keep in mind you'll have to configure CloudFront to use the proper bucket as an origin manually for this to work. @@ -101,6 +150,7 @@ Storage Standard file access options are available, and work as expected:: + >>> from django.core.files.storage import default_storage >>> default_storage.exists('storage_test') False >>> file = default_storage.open('storage_test', 'w') diff --git a/docs/backends/azure.rst b/docs/backends/azure.rst index da4fa765a..b9fa2734b 100644 --- a/docs/backends/azure.rst +++ b/docs/backends/azure.rst @@ -1,5 +1,5 @@ Azure Storage -=========== +============= A custom storage system for Django using Windows Azure Storage backend. @@ -15,7 +15,7 @@ Add to your requirements file:: Settings -******* +******** To use `AzureStorage` set:: diff --git a/docs/backends/couchdb.rst b/docs/backends/couchdb.rst deleted file mode 100644 index f93761b9d..000000000 --- a/docs/backends/couchdb.rst +++ /dev/null @@ -1,5 +0,0 @@ -CouchDB -======= - -A custom storage system for Django with CouchDB backend. - diff --git a/docs/backends/database.rst b/docs/backends/database.rst deleted file mode 100644 index e2f40413a..000000000 --- a/docs/backends/database.rst +++ /dev/null @@ -1,59 +0,0 @@ -Database -======== - -Class DatabaseStorage can be used with either FileField or ImageField. It can be used to map filenames to database blobs: so you have to use it with a special additional table created manually. The table should contain a pk-column for filenames (better to use the same type that FileField uses: nvarchar(100)), blob field (image type for example) and size field (bigint). You can't just create blob column in the same table, where you defined FileField, since there is no way to find required row in the save() method. Also size field is required to obtain better perfomance (see size() method). - -So you can use it with different FileFields and even with different "upload_to" variables used. Thus it implements a kind of root filesystem, where you can define dirs using "upload_to" with FileField and store any files in these dirs. - -It uses either settings.DB_FILES_URL or constructor param 'base_url' (see __init__()) to create urls to files. Base url should be mapped to view that provides access to files. To store files in the same table, where FileField is defined you have to define your own field and provide extra argument (e.g. pk) to save(). - -Raw sql is used for all operations. In constructor or in DB_FILES of settings.py () you should specify a dictionary with db_table, fname_column, blob_column, size_column and 'base_url'. For example I just put to the settings.py the following line:: - - DB_FILES = { - 'db_table': 'FILES', - 'fname_column': 'FILE_NAME', - 'blob_column': 'BLOB', - 'size_column': 'SIZE', - 'base_url': 'http://localhost/dbfiles/' - } - -And use it with ImageField as following:: - - player_photo = models.ImageField(upload_to="player_photos", storage=DatabaseStorage() ) - -DatabaseStorage class uses your settings.py file to perform custom connection to your database. - -The reason to use custom connection: http://code.djangoproject.com/ticket/5135 Connection string looks like:: - - cnxn = pyodbc.connect('DRIVER={SQL Server};SERVER=localhost;DATABASE=testdb;UID=me;PWD=pass') - -It's based on pyodbc module, so can be used with any database supported by pyodbc. I've tested it with MS Sql Express 2005. - -Note: It returns special path, which should be mapped to special view, which returns requested file:: - - def image_view(request, filename): - import os - from django.http import HttpResponse - from django.conf import settings - from django.utils._os import safe_join - from filestorage import DatabaseStorage - from django.core.exceptions import ObjectDoesNotExist - - storage = DatabaseStorage() - - try: - image_file = storage.open(filename, 'rb') - file_content = image_file.read() - except: - filename = 'no_image.gif' - path = safe_join(os.path.abspath(settings.MEDIA_ROOT), filename) - if not os.path.exists(path): - raise ObjectDoesNotExist - no_image = open(path, 'rb') - file_content = no_image.read() - - response = HttpResponse(file_content, mimetype="image/jpeg") - response['Content-Disposition'] = 'inline; filename=%s'%filename - return response - -.. note:: If filename exist, blob will be overwritten, to change this remove get_available_name(self, name), so Storage.get_available_name(self, name) will be used to generate new filename. diff --git a/docs/backends/dropbox.rst b/docs/backends/dropbox.rst index 5fa925e6c..6b7aa1fbc 100644 --- a/docs/backends/dropbox.rst +++ b/docs/backends/dropbox.rst @@ -1,9 +1,22 @@ DropBox ======= +A custom storage system for Django using Dropbox Storage backend. + +Before you start configuration, you will need to install `Dropbox SDK for Python`_. + + +Install the package:: + + pip install dropbox + Settings -------- +To use DropBoxStorage set:: + + DEFAULT_FILE_STORAGE = 'storages.backends.dropbox.DropBoxStorage' + ``DROPBOX_OAUTH2_TOKEN`` Your DropBox token, if you haven't follow this `guide step`_. @@ -11,3 +24,4 @@ Settings Allow to jail your storage to a defined directory. .. _`guide step`: https://www.dropbox.com/developers/documentation/python#tutorial +.. _`Dropbox SDK for Python`: https://www.dropbox.com/developers/documentation/python#tutorial diff --git a/docs/backends/gcloud.rst b/docs/backends/gcloud.rst new file mode 100644 index 000000000..4e126acf8 --- /dev/null +++ b/docs/backends/gcloud.rst @@ -0,0 +1,194 @@ +Google Cloud Storage +==================== + +Usage +***** + +This backend provides support for Google Cloud Storage using the +library provided by Google. + +It's possible to access Google Cloud Storage in S3 compatibility mode +using other libraries in django-storages, but this is the only library +offering native support. + +By default this library will use the credentials associated with the +current instance for authentication. To override this, see the +settings below. + + +Settings +-------- + +To use gcloud set:: + + DEFAULT_FILE_STORAGE = 'storages.backends.gcloud.GoogleCloudStorage' + +``GS_BUCKET_NAME`` + +Your Google Storage bucket name, as a string. + +``GS_PROJECT_ID`` (optional) + +Your Google Cloud project ID. If unset, falls back to the default +inferred from the environment. + +``GS_CREDENTIALS`` (optional) + +The OAuth 2 credentials to use for the connection. If unset, falls +back to the default inferred from the environment. + +``GS_AUTO_CREATE_BUCKET`` (optional, default is ``False``) + +If True, attempt to create the bucket if it does not exist. + +``GS_AUTO_CREATE_ACL`` (optional, default is ``projectPrivate``) + +ACL used when creating a new bucket, from the +`list of predefined ACLs `_. +(A "JSON API" ACL is preferred but an "XML API/gsutil" ACL will be +translated.) + +Note that the ACL you select must still give the service account +running the gcloud backend to have OWNER permission on the bucket. If +you're using the default service account, this means you're restricted +to the ``projectPrivate`` ACL. + +``GS_FILE_CHARSET`` (optional) + +Allows overriding the character set used in filenames. + +``GS_FILE_OVERWRITE`` (optional: default is ``True``) + +By default files with the same name will overwrite each other. Set this to ``False`` to have extra characters appended. + +``GS_MAX_MEMORY_SIZE`` (optional) + +The maximum amount of memory a returned file can take up before being +rolled over into a temporary file on disk. Default is 0: Do not roll over. + +Fields +------ + +Once you're done, default_storage will be Google Cloud Storage:: + + >>> from django.core.files.storage import default_storage + >>> print default_storage.__class__ + + +This way, if you define a new FileField, it will use the Google Cloud Storage:: + + >>> from django.db import models + >>> class Resume(models.Model): + ... pdf = models.FileField(upload_to='pdfs') + ... photos = models.ImageField(upload_to='photos') + ... + >>> resume = Resume() + >>> print resume.pdf.storage + + +Storage +------- + +Standard file access options are available, and work as expected:: + + >>> default_storage.exists('storage_test') + False + >>> file = default_storage.open('storage_test', 'w') + >>> file.write('storage contents') + >>> file.close() + + >>> default_storage.exists('storage_test') + True + >>> file = default_storage.open('storage_test', 'r') + >>> file.read() + 'storage contents' + >>> file.close() + + >>> default_storage.delete('storage_test') + >>> default_storage.exists('storage_test') + False + +Model +----- + +An object without a file has limited functionality:: + + >>> obj1 = MyStorage() + >>> obj1.normal + + >>> obj1.normal.size + Traceback (most recent call last): + ... + ValueError: The 'normal' attribute has no file associated with it. + +Saving a file enables full functionality:: + + >>> obj1.normal.save('django_test.txt', ContentFile('content')) + >>> obj1.normal + + >>> obj1.normal.size + 7 + >>> obj1.normal.read() + 'content' + +Files can be read in a little at a time, if necessary:: + + >>> obj1.normal.open() + >>> obj1.normal.read(3) + 'con' + >>> obj1.normal.read() + 'tent' + >>> '-'.join(obj1.normal.chunks(chunk_size=2)) + 'co-nt-en-t' + +Save another file with the same name:: + + >>> obj2 = MyStorage() + >>> obj2.normal.save('django_test.txt', ContentFile('more content')) + >>> obj2.normal + + >>> obj2.normal.size + 12 + +Push the objects into the cache to make sure they pickle properly:: + + >>> cache.set('obj1', obj1) + >>> cache.set('obj2', obj2) + >>> cache.get('obj2').normal + + +Deleting an object deletes the file it uses, if there are no other objects still using that file:: + + >>> obj2.delete() + >>> obj2.normal.save('django_test.txt', ContentFile('more content')) + >>> obj2.normal + + +Default values allow an object to access a single file:: + + >>> obj3 = MyStorage.objects.create() + >>> obj3.default + + >>> obj3.default.read() + 'default content' + +But it shouldn't be deleted, even if there are no more objects using it:: + + >>> obj3.delete() + >>> obj3 = MyStorage() + >>> obj3.default.read() + 'default content' + +Verify the fix for #5655, making sure the directory is only determined once:: + + >>> obj4 = MyStorage() + >>> obj4.random.save('random_file', ContentFile('random content')) + >>> obj4.random + + +Clean up the temporary files:: + + >>> obj1.normal.delete() + >>> obj2.normal.delete() + >>> obj3.default.delete() + >>> obj4.random.delete() diff --git a/docs/backends/image.rst b/docs/backends/image.rst deleted file mode 100644 index b03e1d4ec..000000000 --- a/docs/backends/image.rst +++ /dev/null @@ -1,5 +0,0 @@ -Image -===== - -A custom FileSystemStorage made for normalizing extensions. It lets PIL look at the file to determine the format and append an always lower-case extension based on the results. - diff --git a/docs/backends/mogilefs.rst b/docs/backends/mogilefs.rst deleted file mode 100644 index 7d868fec4..000000000 --- a/docs/backends/mogilefs.rst +++ /dev/null @@ -1,67 +0,0 @@ -MogileFS -======== - -This storage allows you to use MogileFS, it comes from this blog post. - -The MogileFS storage backend is fairly simple: it uses URLs (or, rather, parts of URLs) as keys into the mogile database. When the user requests a file stored by mogile (say, an avatar), the URL gets passed to a view which, using a client to the mogile tracker, retrieves the "correct" path (the path that points to the actual file data). The view will then either return the path(s) to perlbal to reproxy, or, if you're not using perlbal to reproxy (which you should), it serves the data of the file directly from django. - -To use `MogileFSStorage` set:: - - DEFAULT_FILE_STORAGE = 'storages.backends.mogile.MogileFSStorage' - -The following settings are available: - -``MOGILEFS_DOMAIN`` - The mogile domain that files should read from/written to, e.g "production" - -``MOGILEFS_TRACKERS`` - A list of trackers to connect to, e.g. ["foo.sample.com:7001", "bar.sample.com:7001"] - -``MOGILEFS_MEDIA_URL`` (optional) - The prefix for URLs that point to mogile files. This is used in a similar way to ``MEDIA_URL``, e.g. "/mogilefs/" - -``SERVE_WITH_PERLBAL`` - Boolean that, when True, will pass the paths back in the response in the ``X-REPROXY-URL`` header. If False, django will serve all mogile media files itself (bad idea for production, but useful if you're testing on a setup that doesn't have perlbal running) - -Getting files into mogile -************************* - -The great thing about file backends is that we just need to specify the backend in the model file and everything is taken care for us - all the default save() methods work correctly. - -For Fluther, we have two main media types we use mogile for: avatars and thumbnails. Mogile defines "classes" that dictate how each type of file is replicated - so you can make sure you have 3 copies of the original avatar but only 1 of the thumbnail. - -In order for classes to behave nicely with the backend framework, we've had to do a little tomfoolery. (This is something that may change in future versions of the filestorage framework). - -Here's what the models.py file looks like for the avatars:: - - from django.core.filestorage import storage - - # TODO: Find a better way to deal with classes. Maybe a generator? - class AvatarStorage(storage.__class__): - mogile_class = 'avatar' - - class ThumbnailStorage(storage.__class__): - mogile_class = 'thumb' - - class Avatar(models.Model): - user = models.ForeignKey(User, null=True, blank=True) - image = models.ImageField(storage=AvatarStorage()) - thumb = models.ImageField(storage=ThumbnailStorage()) - -Each of the custom storage classes defines a class attribute which gets passed to the mogile backend behind the scenes. If you don't want to worry about mogile classes, don't need to define a custom storage engine or specify it in the field - the default should work just fine. - -Serving files from mogile -************************* - -Now, all we need to do is plug in the view that serves up mogile data. - -Here's what we use:: - - urlpatterns += patterns(", - (r'^%s(?P.*)' % settings.MOGILEFS_MEDIA_URL[1:], - 'MogileFSStorage.serve_mogilefs_file') - ) - -Any url beginning with the value of ``MOGILEFS_MEDIA_URL`` will get passed to our view. Since ``MOGILEFS_MEDIA_URL`` requires a leading slash (like ``MEDIA_URL``), we strip that off and pass the rest of the url over to the view. - -That's it! Happy mogiling! diff --git a/docs/backends/overwrite.rst b/docs/backends/overwrite.rst deleted file mode 100644 index 66aa87538..000000000 --- a/docs/backends/overwrite.rst +++ /dev/null @@ -1,5 +0,0 @@ -Overwrite -========= - -This is a simple implementation overwrite of the FileSystemStorage. It removes the addition of an '_' to the filename if the file already exists in the storage system. I needed a model in the admin area to act exactly like a file system (overwriting the file if it already exists). - diff --git a/docs/backends/symlinkcopy.rst b/docs/backends/symlinkcopy.rst deleted file mode 100644 index be4abe18e..000000000 --- a/docs/backends/symlinkcopy.rst +++ /dev/null @@ -1,6 +0,0 @@ -Symlink or copy -=============== - -Stores symlinks to files instead of actual files whenever possible - -When a file that's being saved is currently stored in the symlink_within directory, then symlink the file. Otherwise, copy the file. diff --git a/requirements-tests.txt b/requirements-tests.txt index d87d26210..229a8a288 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1,7 +1,9 @@ -Django>=1.7 -pytest-cov>=2.2.1 -boto>=2.32.0 boto3>=1.2.3 -dropbox>=3.24 +boto>=2.32.0 +dropbox>=8.0.0 +Django>=1.8 +flake8 +google-cloud-storage>=0.22.0 mock paramiko +pytest-cov>=2.2.1 diff --git a/setup.cfg b/setup.cfg index 7c964b49e..1a0dfbc1b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,17 @@ -[wheel] +[bdist_wheel] universal=1 + +[flake8] +exclude = + .tox, + docs +max-line-length = 119 + +[isort] +combine_as_imports = true +default_section = THIRDPARTY +include_trailing_comma = true +known_first_party = storages +line_length = 79 +multi_line_output = 5 +not_skip = __init__.py diff --git a/setup.py b/setup.py index eab137d8c..f9b4b3979 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ from setuptools import setup + import storages @@ -11,6 +12,7 @@ def get_requirements_tests(): with open('requirements-tests.txt') as f: return f.readlines() + setup( name='django-storages', version=storages.__version__, @@ -27,7 +29,7 @@ def get_requirements_tests(): author='Josh Schneier', author_email='josh.schneier@gmail.com', license='BSD', - description='Support for many storages (S3, Libcloud, etc in Django.', + description='Support for many storage backends in Django', long_description=read('README.rst') + '\n\n' + read('CHANGELOG.rst'), url='https://github.com/jschneier/django-storages', classifiers=[ @@ -35,8 +37,8 @@ def get_requirements_tests(): 'Environment :: Web Environment', 'Framework :: Django', 'Framework :: Django :: 1.8', - 'Framework :: Django :: 1.9', 'Framework :: Django :: 1.10', + 'Framework :: Django :: 1.11', 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', @@ -47,6 +49,7 @@ def get_requirements_tests(): 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', ], tests_require=get_requirements_tests(), test_suite='tests', diff --git a/storages/__init__.py b/storages/__init__.py index c3b384154..f3df7f04b 100644 --- a/storages/__init__.py +++ b/storages/__init__.py @@ -1 +1 @@ -__version__ = '1.5.2' +__version__ = '1.6.5' diff --git a/storages/backends/apache_libcloud.py b/storages/backends/apache_libcloud.py index a2a5390de..a2e5dc2e3 100644 --- a/storages/backends/apache_libcloud.py +++ b/storages/backends/apache_libcloud.py @@ -4,11 +4,11 @@ import os from django.conf import settings +from django.core.exceptions import ImproperlyConfigured from django.core.files.base import File from django.core.files.storage import Storage -from django.core.exceptions import ImproperlyConfigured from django.utils.deconstruct import deconstructible -from django.utils.six import string_types, BytesIO +from django.utils.six import BytesIO, string_types from django.utils.six.moves.urllib.parse import urljoin try: @@ -33,6 +33,9 @@ def __init__(self, provider_name=None, option=None): extra_kwargs = {} if 'region' in self.provider: extra_kwargs['region'] = self.provider['region'] + # Used by the GoogleStorageDriver + if 'project' in self.provider: + extra_kwargs['project'] = self.provider['project'] try: provider_type = self.provider['type'] if isinstance(provider_type, string_types): diff --git a/storages/backends/azure_storage.py b/storages/backends/azure_storage.py index 19494c4b2..ea5d71c3f 100644 --- a/storages/backends/azure_storage.py +++ b/storages/backends/azure_storage.py @@ -1,14 +1,16 @@ -from datetime import datetime -import os.path import mimetypes +import os.path import time +from datetime import datetime from time import mktime -from django.core.files.base import ContentFile from django.core.exceptions import ImproperlyConfigured +from django.core.files.base import ContentFile from django.core.files.storage import Storage from django.utils.deconstruct import deconstructible +from storages.utils import setting + try: import azure # noqa except ImportError: @@ -24,8 +26,6 @@ from azure.storage import BlobService from azure import WindowsAzureMissingResourceError as AzureMissingResourceHttpError -from storages.utils import setting - def clean_name(name): return os.path.normpath(name).replace("\\", "/") diff --git a/storages/backends/couchdb.py b/storages/backends/couchdb.py deleted file mode 100644 index 16ef41e45..000000000 --- a/storages/backends/couchdb.py +++ /dev/null @@ -1,144 +0,0 @@ -""" -This is a Custom Storage System for Django with CouchDB backend. -Created by Christian Klein. -(c) Copyright 2009 HUDORA GmbH. All Rights Reserved. -""" -import os -import warnings - -from django.conf import settings -from django.core.files import File -from django.core.files.storage import Storage -from django.core.exceptions import ImproperlyConfigured -from django.utils.deconstruct import deconstructible -from django.utils.six.moves.urllib import parse as urlparse -from django.utils.six import BytesIO - -try: - import couchdb -except ImportError: - raise ImproperlyConfigured("Could not load couchdb dependency.\ - \nSee http://code.google.com/p/couchdb-python/") - -DEFAULT_SERVER = getattr(settings, 'COUCHDB_DEFAULT_SERVER', 'http://couchdb.local:5984') -STORAGE_OPTIONS = getattr(settings, 'COUCHDB_STORAGE_OPTIONS', {}) - - -warnings.warn( - 'CouchDBStorage is unmaintained and will be removed in the next version of django-storages ' - 'See https://github.com/jschneier/django-storages/issues/202', - PendingDeprecationWarning -) - - -@deconstructible -class CouchDBStorage(Storage): - """ - CouchDBStorage - a Django Storage class for CouchDB. - - The CouchDBStorage can be configured in settings.py, e.g.:: - - COUCHDB_STORAGE_OPTIONS = { - 'server': "http://example.org", - 'database': 'database_name' - } - - Alternatively, the configuration can be passed as a dictionary. - """ - def __init__(self, **kwargs): - kwargs.update(STORAGE_OPTIONS) - self.base_url = kwargs.get('server', DEFAULT_SERVER) - server = couchdb.client.Server(self.base_url) - self.db = server[kwargs.get('database')] - - def _put_file(self, name, content): - self.db[name] = {'size': len(content)} - self.db.put_attachment(self.db[name], content, filename='content') - return name - - def get_document(self, name): - return self.db.get(name) - - def _open(self, name, mode='rb'): - couchdb_file = CouchDBFile(name, self, mode=mode) - return couchdb_file - - def _save(self, name, content): - content.open() - if hasattr(content, 'chunks'): - content_str = ''.join(chunk for chunk in content.chunks()) - else: - content_str = content.read() - name = name.replace('/', '-') - return self._put_file(name, content_str) - - def exists(self, name): - return name in self.db - - def size(self, name): - doc = self.get_document(name) - if doc: - return doc['size'] - return 0 - - def url(self, name): - return urlparse.urljoin(self.base_url, - os.path.join(urlparse.quote_plus(self.db.name), - urlparse.quote_plus(name), - 'content')) - - def delete(self, name): - try: - del self.db[name] - except couchdb.client.ResourceNotFound: - raise IOError("File not found: %s" % name) - - #def listdir(self, name): - # _all_docs? - # pass - - -class CouchDBFile(File): - """ - CouchDBFile - a Django File-like class for CouchDB documents. - """ - - def __init__(self, name, storage, mode): - self._name = name - self._storage = storage - self._mode = mode - self._is_dirty = False - - try: - self._doc = self._storage.get_document(name) - - tmp, ext = os.path.split(name) - if ext: - filename = "content." + ext - else: - filename = "content" - attachment = self._storage.db.get_attachment(self._doc, filename=filename) - self.file = BytesIO(attachment) - except couchdb.client.ResourceNotFound: - if 'r' in self._mode: - raise ValueError("The file cannot be reopened.") - else: - self.file = BytesIO() - self._is_dirty = True - - @property - def size(self): - return self._doc['size'] - - def write(self, content): - if 'w' not in self._mode: - raise AttributeError("File was opened for read-only access.") - self.file = BytesIO(content) - self._is_dirty = True - - def close(self): - if self._is_dirty: - self._storage._put_file(self._name, self.file.getvalue()) - self.file.close() - - diff --git a/storages/backends/database.py b/storages/backends/database.py deleted file mode 100644 index 81954fc55..000000000 --- a/storages/backends/database.py +++ /dev/null @@ -1,141 +0,0 @@ -# DatabaseStorage for django. -# 2009 (c) GameKeeper Gambling Ltd, Ivanov E. -import warnings - -from django.conf import settings -from django.core.files import File -from django.core.files.storage import Storage -from django.core.exceptions import ImproperlyConfigured -from django.utils.deconstruct import deconstructible -from django.utils.six import BytesIO -from django.utils.six.moves.urllib import parse as urlparse - -try: - import pyodbc -except ImportError: - raise ImproperlyConfigured("Could not load pyodbc dependency.\ - \nSee https://github.com/mkleehammer/pyodbc") - -REQUIRED_FIELDS = ('db_table', 'fname_column', 'blob_column', 'size_column', 'base_url') -warnings.warn( - 'DatabaseStorage is unmaintained and will be removed in the next version of django-storages.' - 'See https://github.com/jschneier/django-storages/issues/202', - PendingDeprecationWarning -) - - -@deconstructible -class DatabaseStorage(Storage): - """ - Class DatabaseStorage provides storing files in the database. - """ - - def __init__(self, option=settings.DB_FILES): - """Constructor. - - Constructs object using dictionary either specified in contucotr or -in settings.DB_FILES. - - @param option dictionary with 'db_table', 'fname_column', -'blob_column', 'size_column', 'base_url' keys. - - option['db_table'] - Table to work with. - option['fname_column'] - Column in the 'db_table' containing filenames (filenames can -contain pathes). Values should be the same as where FileField keeps -filenames. - It is used to map filename to blob_column. In sql it's simply -used in where clause. - option['blob_column'] - Blob column (for example 'image' type), created manually in the -'db_table', used to store image. - option['size_column'] - Column to store file size. Used for optimization of size() -method (another way is to open file and get size) - option['base_url'] - Url prefix used with filenames. Should be mapped to the view, -that returns an image as result. - """ - - if not option or not all([field in option for field in REQUIRED_FIELDS]): - raise ValueError("You didn't specify required options") - - self.db_table = option['db_table'] - self.fname_column = option['fname_column'] - self.blob_column = option['blob_column'] - self.size_column = option['size_column'] - self.base_url = option['base_url'] - - #get database settings - self.DATABASE_ODBC_DRIVER = settings.DATABASE_ODBC_DRIVER - self.DATABASE_NAME = settings.DATABASE_NAME - self.DATABASE_USER = settings.DATABASE_USER - self.DATABASE_PASSWORD = settings.DATABASE_PASSWORD - self.DATABASE_HOST = settings.DATABASE_HOST - - self.connection = pyodbc.connect('DRIVER=%s;SERVER=%s;DATABASE=%s;UID=%s;PWD=%s'%(self.DATABASE_ODBC_DRIVER,self.DATABASE_HOST,self.DATABASE_NAME, - self.DATABASE_USER, self.DATABASE_PASSWORD) ) - self.cursor = self.connection.cursor() - - def _open(self, name, mode='rb'): - """Open a file from database. - - @param name filename or relative path to file based on base_url. path should contain only "/", but not "\". Apache sends pathes with "/". - If there is no such file in the db, returs None - """ - - assert mode == 'rb', "You've tried to open binary file without specifying binary mode! You specified: %s"%mode - - row = self.cursor.execute("SELECT %s from %s where %s = '%s'"%(self.blob_column,self.db_table,self.fname_column,name) ).fetchone() - if row is None: - return None - inMemFile = BytesIO(row[0]) - inMemFile.name = name - inMemFile.mode = mode - - retFile = File(inMemFile) - return retFile - - def _save(self, name, content): - """Save 'content' as file named 'name'. - - @note '\' in path will be converted to '/'. - """ - - name = name.replace('\\', '/') - binary = pyodbc.Binary(content.read()) - size = len(binary) - - #todo: check result and do something (exception?) if failed. - if self.exists(name): - self.cursor.execute("UPDATE %s SET %s = ?, %s = ? WHERE %s = '%s'"%(self.db_table,self.blob_column,self.size_column,self.fname_column,name), - (binary, size) ) - else: - self.cursor.execute("INSERT INTO %s VALUES(?, ?, ?)"%(self.db_table), (name, binary, size) ) - self.connection.commit() - return name - - def exists(self, name): - row = self.cursor.execute("SELECT %s from %s where %s = '%s'"%(self.fname_column,self.db_table,self.fname_column,name)).fetchone() - return row is not None - - def get_available_name(self, name, max_length=None): - return name - - def delete(self, name): - if self.exists(name): - self.cursor.execute("DELETE FROM %s WHERE %s = '%s'"%(self.db_table,self.fname_column,name)) - self.connection.commit() - - def url(self, name): - if self.base_url is None: - raise ValueError("This file is not accessible via a URL.") - return urlparse.urljoin(self.base_url, name).replace('\\', '/') - - def size(self, name): - row = self.cursor.execute("SELECT %s from %s where %s = '%s'"%(self.size_column,self.db_table,self.fname_column,name)).fetchone() - if row is None: - return 0 - else: - return int(row[0]) diff --git a/storages/backends/dropbox.py b/storages/backends/dropbox.py index c76ba2718..bae6deadb 100644 --- a/storages/backends/dropbox.py +++ b/storages/backends/dropbox.py @@ -11,20 +11,20 @@ from __future__ import absolute_import from datetime import datetime -from tempfile import SpooledTemporaryFile from shutil import copyfileobj +from tempfile import SpooledTemporaryFile from django.core.exceptions import ImproperlyConfigured from django.core.files.base import File from django.core.files.storage import Storage -from django.utils.deconstruct import deconstructible from django.utils._os import safe_join +from django.utils.deconstruct import deconstructible +from dropbox import Dropbox +from dropbox.exceptions import ApiError +from dropbox.files import CommitInfo, UploadSessionCursor from storages.utils import setting -from dropbox.client import DropboxClient -from dropbox.rest import ErrorResponse - DATE_FORMAT = '%a, %d %b %Y %X +0000' @@ -40,7 +40,7 @@ def __init__(self, name, storage): @property def file(self): if not hasattr(self, '_file'): - response = self._storage.client.get_file(self.name) + response = self._storage.client.files_download(self.name) self._file = SpooledTemporaryFile() copyfileobj(response, self._file) self._file.seek(0) @@ -51,32 +51,34 @@ def file(self): class DropBoxStorage(Storage): """DropBox Storage class for Django pluggable storage system.""" + CHUNK_SIZE = 4 * 1024 * 1024 + def __init__(self, oauth2_access_token=None, root_path=None): oauth2_access_token = oauth2_access_token or setting('DROPBOX_OAUTH2_TOKEN') self.root_path = root_path or setting('DROPBOX_ROOT_PATH', '/') if oauth2_access_token is None: raise ImproperlyConfigured("You must configure a token auth at" "'settings.DROPBOX_OAUTH2_TOKEN'.") - self.client = DropboxClient(oauth2_access_token) + self.client = Dropbox(oauth2_access_token) def _full_path(self, name): if name == '/': name = '' - return safe_join(self.root_path, name) + return safe_join(self.root_path, name).replace('\\', '/') def delete(self, name): - self.client.file_delete(self._full_path(name)) + self.client.files_delete(self._full_path(name)) def exists(self, name): try: - return bool(self.client.metadata(self._full_path(name))) - except ErrorResponse: + return bool(self.client.files_get_metadata(self._full_path(name))) + except ApiError: return False def listdir(self, path): directories, files = [], [] full_path = self._full_path(path) - metadata = self.client.metadata(full_path) + metadata = self.client.files_get_metadata(full_path) for entry in metadata['contents']: entry['path'] = entry['path'].replace(full_path, '', 1) entry['path'] = entry['path'].replace('/', '', 1) @@ -87,27 +89,53 @@ def listdir(self, path): return directories, files def size(self, name): - metadata = self.client.metadata(self._full_path(name)) + metadata = self.client.files_get_metadata(self._full_path(name)) return metadata['bytes'] def modified_time(self, name): - metadata = self.client.metadata(self._full_path(name)) + metadata = self.client.files_get_metadata(self._full_path(name)) mod_time = datetime.strptime(metadata['modified'], DATE_FORMAT) return mod_time def accessed_time(self, name): - metadata = self.client.metadata(self._full_path(name)) + metadata = self.client.files_get_metadata(self._full_path(name)) acc_time = datetime.strptime(metadata['client_mtime'], DATE_FORMAT) return acc_time def url(self, name): - media = self.client.media(self._full_path(name)) - return media['url'] + media = self.client.files_get_temporary_link(self._full_path(name)) + return media.link def _open(self, name, mode='rb'): remote_file = DropBoxFile(self._full_path(name), self) return remote_file def _save(self, name, content): - self.client.put_file(self._full_path(name), content) + content.open() + if content.size <= self.CHUNK_SIZE: + self.client.files_upload(content.read(), self._full_path(name)) + else: + self._chunked_upload(content, self._full_path(name)) + content.close() return name + + def _chunked_upload(self, content, dest_path): + upload_session = self.client.files_upload_session_start( + content.read(self.CHUNK_SIZE) + ) + cursor = UploadSessionCursor( + session_id=upload_session.session_id, + offset=content.tell() + ) + commit = CommitInfo(path=dest_path) + + while content.tell() < content.size: + if (content.size - content.tell()) <= self.CHUNK_SIZE: + self.client.files_upload_session_finish( + content.read(self.CHUNK_SIZE), cursor, commit + ) + else: + self.client.files_upload_session_append_v2( + content.read(self.CHUNK_SIZE), cursor + ) + cursor.offset = content.tell() diff --git a/storages/backends/ftp.py b/storages/backends/ftp.py index bb705a18c..0b28280ac 100644 --- a/storages/backends/ftp.py +++ b/storages/backends/ftp.py @@ -14,17 +14,17 @@ # class FTPTest(models.Model): # file = models.FileField(upload_to='a/b/c/', storage=fs) +import ftplib import os from datetime import datetime -import ftplib from django.conf import settings +from django.core.exceptions import ImproperlyConfigured from django.core.files.base import File from django.core.files.storage import Storage -from django.core.exceptions import ImproperlyConfigured from django.utils.deconstruct import deconstructible -from django.utils.six.moves.urllib import parse as urlparse from django.utils.six import BytesIO +from django.utils.six.moves.urllib import parse as urlparse from storages.utils import setting diff --git a/storages/backends/gcloud.py b/storages/backends/gcloud.py new file mode 100644 index 000000000..6b433c602 --- /dev/null +++ b/storages/backends/gcloud.py @@ -0,0 +1,236 @@ +import mimetypes +from tempfile import SpooledTemporaryFile + +from django.core.exceptions import ImproperlyConfigured +from django.core.files.base import File +from django.core.files.storage import Storage +from django.utils import timezone +from django.utils.deconstruct import deconstructible +from django.utils.encoding import force_bytes, smart_str + +from storages.utils import clean_name, safe_join, setting + +try: + from google.cloud.storage.client import Client + from google.cloud.storage.blob import Blob + from google.cloud.exceptions import NotFound +except ImportError: + raise ImproperlyConfigured("Could not load Google Cloud Storage bindings.\n" + "See https://github.com/GoogleCloudPlatform/gcloud-python") + + +class GoogleCloudFile(File): + def __init__(self, name, mode, storage): + self.name = name + self.mime_type = mimetypes.guess_type(name)[0] + self._mode = mode + self._storage = storage + self.blob = storage.bucket.get_blob(name) + if not self.blob and 'w' in mode: + self.blob = Blob(self.name, storage.bucket) + self._file = None + self._is_dirty = False + + @property + def size(self): + return self.blob.size + + def _get_file(self): + if self._file is None: + self._file = SpooledTemporaryFile( + max_size=self._storage.max_memory_size, + suffix=".GSStorageFile", + dir=setting("FILE_UPLOAD_TEMP_DIR", None) + ) + if 'r' in self._mode: + self._is_dirty = False + self.blob.download_to_file(self._file) + self._file.seek(0) + return self._file + + def _set_file(self, value): + self._file = value + + file = property(_get_file, _set_file) + + def read(self, num_bytes=None): + if 'r' not in self._mode: + raise AttributeError("File was not opened in read mode.") + + if num_bytes is None: + num_bytes = -1 + + return super(GoogleCloudFile, self).read(num_bytes) + + def write(self, content): + if 'w' not in self._mode: + raise AttributeError("File was not opened in write mode.") + self._is_dirty = True + return super(GoogleCloudFile, self).write(force_bytes(content)) + + def close(self): + if self._file is not None: + if self._is_dirty: + self.file.seek(0) + self.blob.upload_from_file(self.file, content_type=self.mime_type) + self._file.close() + self._file = None + + +@deconstructible +class GoogleCloudStorage(Storage): + project_id = setting('GS_PROJECT_ID', None) + credentials = setting('GS_CREDENTIALS', None) + bucket_name = setting('GS_BUCKET_NAME', None) + auto_create_bucket = setting('GS_AUTO_CREATE_BUCKET', False) + auto_create_acl = setting('GS_AUTO_CREATE_ACL', 'projectPrivate') + file_name_charset = setting('GS_FILE_NAME_CHARSET', 'utf-8') + file_overwrite = setting('GS_FILE_OVERWRITE', True) + # The max amount of memory a returned file can take up before being + # rolled over into a temporary file on disk. Default is 0: Do not roll over. + max_memory_size = setting('GS_MAX_MEMORY_SIZE', 0) + + def __init__(self, **settings): + # check if some of the settings we've provided as class attributes + # need to be overwritten with values passed in here + for name, value in settings.items(): + if hasattr(self, name): + setattr(self, name, value) + + self._bucket = None + self._client = None + + @property + def client(self): + if self._client is None: + self._client = Client( + project=self.project_id, + credentials=self.credentials + ) + return self._client + + @property + def bucket(self): + if self._bucket is None: + self._bucket = self._get_or_create_bucket(self.bucket_name) + return self._bucket + + def _get_or_create_bucket(self, name): + """ + Retrieves a bucket if it exists, otherwise creates it. + """ + try: + return self.client.get_bucket(name) + except NotFound: + if self.auto_create_bucket: + bucket = self.client.create_bucket(name) + bucket.acl.save_predefined(self.auto_create_acl) + return bucket + raise ImproperlyConfigured("Bucket %s does not exist. Buckets " + "can be automatically created by " + "setting GS_AUTO_CREATE_BUCKET to " + "``True``." % name) + + def _normalize_name(self, name): + """ + Normalizes the name so that paths like /path/to/ignored/../something.txt + and ./file.txt work. Note that clean_name adds ./ to some paths so + they need to be fixed here. + """ + return safe_join('', name) + + def _encode_name(self, name): + return smart_str(name, encoding=self.file_name_charset) + + def _open(self, name, mode='rb'): + name = self._normalize_name(clean_name(name)) + file_object = GoogleCloudFile(name, mode, self) + if not file_object.blob: + raise IOError(u'File does not exist: %s' % name) + return file_object + + def _save(self, name, content): + cleaned_name = clean_name(name) + name = self._normalize_name(cleaned_name) + + content.name = cleaned_name + encoded_name = self._encode_name(name) + file = GoogleCloudFile(encoded_name, 'rw', self) + file.blob.upload_from_file(content, size=content.size, + content_type=file.mime_type) + return cleaned_name + + def delete(self, name): + name = self._normalize_name(clean_name(name)) + self.bucket.delete_blob(self._encode_name(name)) + + def exists(self, name): + if not name: # root element aka the bucket + try: + self.bucket + return True + except ImproperlyConfigured: + return False + + name = self._normalize_name(clean_name(name)) + return bool(self.bucket.get_blob(self._encode_name(name))) + + def listdir(self, name): + name = self._normalize_name(clean_name(name)) + # for the bucket.list and logic below name needs to end in / + # But for the root path "" we leave it as an empty string + if name and not name.endswith('/'): + name += '/' + + files_list = list(self.bucket.list_blobs(prefix=self._encode_name(name))) + files = [] + dirs = set() + + base_parts = name.split("/")[:-1] + for item in files_list: + parts = item.name.split("/") + parts = parts[len(base_parts):] + if len(parts) == 1 and parts[0]: + # File + files.append(parts[0]) + elif len(parts) > 1 and parts[0]: + # Directory + dirs.add(parts[0]) + return list(dirs), files + + def _get_blob(self, name): + # Wrap google.cloud.storage's blob to raise if the file doesn't exist + blob = self.bucket.get_blob(name) + + if blob is None: + raise NotFound(u'File does not exist: {}'.format(name)) + + return blob + + def size(self, name): + name = self._normalize_name(clean_name(name)) + blob = self._get_blob(self._encode_name(name)) + return blob.size + + def modified_time(self, name): + name = self._normalize_name(clean_name(name)) + blob = self._get_blob(self._encode_name(name)) + return timezone.make_naive(blob.updated) + + def get_modified_time(self, name): + name = self._normalize_name(clean_name(name)) + blob = self._get_blob(self._encode_name(name)) + updated = blob.updated + return updated if setting('USE_TZ') else timezone.make_naive(updated) + + def url(self, name): + # Preserve the trailing slash after normalizing the path. + name = self._normalize_name(clean_name(name)) + blob = self._get_blob(self._encode_name(name)) + return blob.public_url + + def get_available_name(self, name, max_length=None): + if self.file_overwrite: + name = clean_name(name) + return name + return super(GoogleCloudStorage, self).get_available_name(name, max_length) diff --git a/storages/backends/gs.py b/storages/backends/gs.py index 1425ebe2e..38256d002 100644 --- a/storages/backends/gs.py +++ b/storages/backends/gs.py @@ -1,3 +1,5 @@ +import warnings + from django.core.exceptions import ImproperlyConfigured from django.utils.deconstruct import deconstructible from django.utils.six import BytesIO @@ -14,6 +16,17 @@ "See https://github.com/boto/boto") +warnings.warn("DEPRECATION NOTICE: This backend is deprecated in favour of the " + "\"gcloud\" backend. This backend uses Google Cloud Storage's XML " + "Interoperable API which uses keyed-hash message authentication code " + "(a.k.a. developer keys) that are linked to your Google account. The " + "interoperable API is really meant for migration to Google Cloud " + "Storage. The biggest problem with the developer keys is security and " + "privacy. Developer keys should not be shared with anyone as they can " + "be used to gain access to other Google Cloud Storage buckets linked " + "to your Google account.", DeprecationWarning) + + class GSBotoStorageFile(S3BotoStorageFile): def write(self, content): @@ -67,6 +80,11 @@ class GSBotoStorage(S3BotoStorage): url_protocol = setting('GS_URL_PROTOCOL', 'http:') host = setting('GS_HOST', GSConnection.DefaultHost) + def _get_connection_kwargs(self): + kwargs = super(GSBotoStorage, self)._get_connection_kwargs() + del kwargs['security_token'] + return kwargs + def _save_content(self, key, content, headers): # only pass backwards incompatible arguments if they vary from the default options = {} @@ -86,7 +104,7 @@ def _get_or_create_bucket(self, name): storage_class = 'STANDARD' try: return self.connection.get_bucket(name, - validate=self.auto_create_bucket) + validate=self.auto_create_bucket) except self.connection_response_error: if self.auto_create_bucket: bucket = self.connection.create_bucket(name, storage_class=storage_class) diff --git a/storages/backends/hashpath.py b/storages/backends/hashpath.py deleted file mode 100644 index c161cfc84..000000000 --- a/storages/backends/hashpath.py +++ /dev/null @@ -1,53 +0,0 @@ -import errno -import hashlib -import os -import warnings - -from django.core.files.storage import FileSystemStorage -from django.utils.deconstruct import deconstructible -from django.utils.encoding import force_text, force_bytes - -warnings.warn( - 'HashPathStorage is unmaintaiined and will be removed in the next version of django-storages.' - 'See https://github.com/jschneier/django-storages/issues/202', - PendingDeprecationWarning -) - - -@deconstructible -class HashPathStorage(FileSystemStorage): - """ - Creates a hash from the uploaded file to build the path. - """ - - def save(self, name, content, max_length=None): - # Get the content name if name is not given - if name is None: - name = content.name - - # Get the SHA1 hash of the uploaded file - sha1 = hashlib.sha1() - for chunk in content.chunks(): - sha1.update(force_bytes(chunk)) - sha1sum = sha1.hexdigest() - - # Build the new path and split it into directory and filename - name = os.path.join(os.path.split(name)[0], sha1sum[:1], sha1sum[1:2], sha1sum) - dir_name, file_name = os.path.split(name) - - # Return the name if the file is already there - if self.exists(name): - return name - - # Try to create the directory relative to location specified in __init__ - try: - os.makedirs(os.path.join(self.location, dir_name)) - except OSError as e: - if e.errno is not errno.EEXIST: - raise e - - # Save the file - name = self._save(name, content) - - # Store filenames with forward slashes, even on Windows - return force_text(name.replace('\\', '/')) diff --git a/storages/backends/image.py b/storages/backends/image.py deleted file mode 100644 index 22c93a850..000000000 --- a/storages/backends/image.py +++ /dev/null @@ -1,64 +0,0 @@ -import os -import warnings - -from django.core.exceptions import ImproperlyConfigured -from django.core.files.storage import FileSystemStorage -from django.utils.deconstruct import deconstructible - -try: - from PIL import ImageFile as PILImageFile -except ImportError: - raise ImproperlyConfigured("Could not load PIL dependency.\ - \nSee http://www.pythonware.com/products/pil/") - - -warnings.warn( - 'ImageStorage backend is unmaintainted and will be removed in the next django-storages version' - 'See https://github.com/jschneier/django-storages/issues/202', - PendingDeprecationWarning -) - - -@deconstructible -class ImageStorage(FileSystemStorage): - """ - A FileSystemStorage which normalizes extensions for images. - - Comes from http://www.djangosnippets.org/snippets/965/ - """ - - def find_extension(self, format): - """Normalizes PIL-returned format into a standard, lowercase extension.""" - format = format.lower() - - if format == 'jpeg': - format = 'jpg' - - return format - - def save(self, name, content, max_length=None): - dirname = os.path.dirname(name) - basename = os.path.basename(name) - - # Use PIL to determine filetype - - p = PILImageFile.Parser() - while 1: - data = content.read(1024) - if not data: - break - p.feed(data) - if p.image: - im = p.image - break - - extension = self.find_extension(im.format) - - # Does the basename already have an extension? If so, replace it. - # bare as in without extension - bare_basename, _ = os.path.splitext(basename) - basename = bare_basename + '.' + extension - - name = os.path.join(dirname, basename) - return super(ImageStorage, self).save(name, content) - diff --git a/storages/backends/mogile.py b/storages/backends/mogile.py deleted file mode 100644 index d61941943..000000000 --- a/storages/backends/mogile.py +++ /dev/null @@ -1,124 +0,0 @@ -from __future__ import print_function - -import mimetypes -import warnings - -from django.conf import settings -from django.core.cache import cache -from django.utils.deconstruct import deconstructible -from django.utils.text import force_text -from django.http import HttpResponse, HttpResponseNotFound -from django.core.exceptions import ImproperlyConfigured -from django.core.files.storage import Storage - -try: - import mogilefs -except ImportError: - raise ImproperlyConfigured("Could not load mogilefs dependency.\ - \nSee http://mogilefs.pbworks.com/Client-Libraries") - -warnings.warn( - 'MogileFSStorage is unmaintained and will be removed in the next django-storages version' - 'See https://github.com/jschneier/django-storages/issues/202', - PendingDeprecationWarning -) - - -@deconstructible -class MogileFSStorage(Storage): - """MogileFS filesystem storage""" - def __init__(self, base_url=settings.MEDIA_URL): - - # the MOGILEFS_MEDIA_URL overrides MEDIA_URL - if hasattr(settings, 'MOGILEFS_MEDIA_URL'): - self.base_url = settings.MOGILEFS_MEDIA_URL - else: - self.base_url = base_url - - for var in ('MOGILEFS_TRACKERS', 'MOGILEFS_DOMAIN',): - if not hasattr(settings, var): - raise ImproperlyConfigured("You must define %s to use the MogileFS backend." % var) - - self.trackers = settings.MOGILEFS_TRACKERS - self.domain = settings.MOGILEFS_DOMAIN - self.client = mogilefs.Client(self.domain, self.trackers) - - def get_mogile_paths(self, filename): - return self.client.get_paths(filename) - - # The following methods define the Backend API - - def filesize(self, filename): - raise NotImplemented - #return os.path.getsize(self._get_absolute_path(filename)) - - def path(self, filename): - paths = self.get_mogile_paths(filename) - if paths: - return self.get_mogile_paths(filename)[0] - else: - return None - - def url(self, filename): - return urlparse.urljoin(self.base_url, filename).replace('\\', '/') - - def open(self, filename, mode='rb'): - raise NotImplemented - #return open(self._get_absolute_path(filename), mode) - - def exists(self, filename): - return filename in self.client - - def save(self, filename, raw_contents, max_length=None): - filename = self.get_available_name(filename, max_length) - - if not hasattr(self, 'mogile_class'): - self.mogile_class = None - - # Write the file to mogile - success = self.client.send_file(filename, BytesIO(raw_contents), self.mogile_class) - if success: - print("Wrote file to key %s, %s@%s" % (filename, self.domain, self.trackers[0])) - else: - print("FAILURE writing file %s" % (filename)) - - return force_text(filename.replace('\\', '/')) - - def delete(self, filename): - self.client.delete(filename) - - -def serve_mogilefs_file(request, key=None): - """ - Called when a user requests an image. - Either reproxy the path to perlbal, or serve the image outright - """ - # not the best way to do this, since we create a client each time - mimetype = mimetypes.guess_type(key)[0] or "application/x-octet-stream" - client = mogilefs.Client(settings.MOGILEFS_DOMAIN, settings.MOGILEFS_TRACKERS) - if hasattr(settings, "SERVE_WITH_PERLBAL") and settings.SERVE_WITH_PERLBAL: - # we're reproxying with perlbal - - # check the path cache - - path = cache.get(key) - - if not path: - path = client.get_paths(key) - cache.set(key, path, 60) - - if path: - response = HttpResponse(content_type=mimetype) - response['X-REPROXY-URL'] = path[0] - else: - response = HttpResponseNotFound() - - else: - # we don't have perlbal, let's just serve the image via django - file_data = client[key] - if file_data: - response = HttpResponse(file_data, mimetype=mimetype) - else: - response = HttpResponseNotFound() - - return response diff --git a/storages/backends/overwrite.py b/storages/backends/overwrite.py deleted file mode 100644 index 0a55059c1..000000000 --- a/storages/backends/overwrite.py +++ /dev/null @@ -1,29 +0,0 @@ -import warnings - -from django.core.files.storage import FileSystemStorage -from django.utils.deconstruct import deconstructible - -warnings.warn( - 'OverwriteStorage is unmaintained and will be removed in the next django-storages version.' - 'See https://github.com/jschneier/django-storages/issues/202', - PendingDeprecationWarning -) - - -@deconstructible -class OverwriteStorage(FileSystemStorage): - """ - Comes from http://www.djangosnippets.org/snippets/976/ - (even if it already exists in S3Storage for ages) - - See also Django #4339, which might add this functionality to core. - """ - - def get_available_name(self, name, max_length=None): - """ - Returns a filename that's free on the target storage system, and - available for new content to be written to. - """ - if self.exists(name): - self.delete(name) - return name diff --git a/storages/backends/s3boto.py b/storages/backends/s3boto.py index 958774ea3..981faa441 100644 --- a/storages/backends/s3boto.py +++ b/storages/backends/s3boto.py @@ -1,17 +1,20 @@ -import os -import posixpath import mimetypes +import os from datetime import datetime from gzip import GzipFile from tempfile import SpooledTemporaryFile +from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation from django.core.files.base import File from django.core.files.storage import Storage -from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation +from django.utils import timezone as tz from django.utils.deconstruct import deconstructible -from django.utils.encoding import force_text, smart_str, filepath_to_uri, force_bytes +from django.utils.encoding import ( + filepath_to_uri, force_bytes, force_text, smart_str, +) from django.utils.six import BytesIO -from django.utils.six.moves.urllib import parse as urlparse + +from storages.utils import clean_name, safe_join, setting try: from boto import __version__ as boto_version @@ -23,7 +26,6 @@ raise ImproperlyConfigured("Could not load Boto's S3 bindings.\n" "See https://github.com/boto/boto") -from storages.utils import setting boto_version_info = tuple([int(i) for i in boto_version.split('-')[0].split('.')]) @@ -32,39 +34,6 @@ "higher.\nSee https://github.com/boto/boto") -def safe_join(base, *paths): - """ - A version of django.utils._os.safe_join for S3 paths. - - Joins one or more path components to the base path component - intelligently. Returns a normalized version of the final path. - - The final path must be located inside of the base path component - (otherwise a ValueError is raised). - - Paths outside the base path indicate a possible security - sensitive operation. - """ - base_path = force_text(base) - base_path = base_path.rstrip('/') - paths = [force_text(p) for p in paths] - - final_path = base_path - for path in paths: - final_path = urlparse.urljoin(final_path.rstrip('/') + "/", path) - - # Ensure final_path starts with base_path and that the next character after - # the final path is '/' (or nothing, in which case final_path must be - # equal to base_path). - base_path_len = len(base_path) - if (not final_path.startswith(base_path) or - final_path[base_path_len:base_path_len + 1] not in ('', '/')): - raise ValueError('the joined path is located outside of the base path' - ' component') - - return final_path.lstrip('/') - - @deconstructible class S3BotoStorageFile(File): """ @@ -114,8 +83,8 @@ def _get_file(self): if self._file is None: self._file = SpooledTemporaryFile( max_size=self._storage.max_memory_size, - suffix=".S3BotoStorageFile", - dir=setting("FILE_UPLOAD_TEMP_DIR", None) + suffix='.S3BotoStorageFile', + dir=setting('FILE_UPLOAD_TEMP_DIR', None) ) if 'r' in self._mode: self._is_dirty = False @@ -132,19 +101,21 @@ def _set_file(self, value): def read(self, *args, **kwargs): if 'r' not in self._mode: - raise AttributeError("File was not opened in read mode.") + raise AttributeError('File was not opened in read mode.') return super(S3BotoStorageFile, self).read(*args, **kwargs) def write(self, content, *args, **kwargs): if 'w' not in self._mode: - raise AttributeError("File was not opened in write mode.") + raise AttributeError('File was not opened in write mode.') self._is_dirty = True if self._multipart is None: provider = self.key.bucket.connection.provider upload_headers = { provider.acl_header: self._storage.default_acl } - upload_headers.update({'Content-Type': mimetypes.guess_type(self.key.name)[0] or self._storage.key_class.DefaultContentType}) + upload_headers.update({ + 'Content-Type': mimetypes.guess_type(self.key.name)[0] or self._storage.key_class.DefaultContentType + }) upload_headers.update(self._storage.headers) self._multipart = self._storage.bucket.initiate_multipart_upload( self.key.name, @@ -165,9 +136,6 @@ def _buffer_file_size(self): return length def _flush_write_buffer(self): - """ - Flushes the write buffer. - """ if self._buffer_file_size: self._write_counter += 1 self.file.seek(0) @@ -180,7 +148,7 @@ def close(self): self._flush_write_buffer() self._multipart.complete_upload() else: - if not self._multipart is None: + if self._multipart is not None: self._multipart.cancel_upload() self.key.close() if self._file is not None: @@ -205,6 +173,7 @@ class S3BotoStorage(Storage): # used for looking up the access and secret key from env vars access_key_names = ['AWS_S3_ACCESS_KEY_ID', 'AWS_ACCESS_KEY_ID'] secret_key_names = ['AWS_S3_SECRET_ACCESS_KEY', 'AWS_SECRET_ACCESS_KEY'] + security_token_names = ['AWS_SESSION_TOKEN', 'AWS_SECURITY_TOKEN'] access_key = setting('AWS_S3_ACCESS_KEY_ID', setting('AWS_ACCESS_KEY_ID')) secret_key = setting('AWS_S3_SECRET_ACCESS_KEY', setting('AWS_SECRET_ACCESS_KEY')) @@ -267,25 +236,36 @@ def __init__(self, acl=None, bucket=None, **settings): self._entries = {} self._bucket = None self._connection = None + self._loaded_meta = False + self.security_token = None if not self.access_key and not self.secret_key: self.access_key, self.secret_key = self._get_access_keys() + self.security_token = self._get_security_token() @property def connection(self): if self._connection is None: + kwargs = self._get_connection_kwargs() + self._connection = self.connection_class( self.access_key, self.secret_key, - is_secure=self.use_ssl, - calling_format=self.calling_format, - host=self.host, - port=self.port, - proxy=self.proxy, - proxy_port=self.proxy_port + **kwargs ) return self._connection + def _get_connection_kwargs(self): + return dict( + security_token=self.security_token, + is_secure=self.use_ssl, + calling_format=self.calling_format, + host=self.host, + port=self.port, + proxy=self.proxy, + proxy_port=self.proxy_port + ) + @property def bucket(self): """ @@ -301,26 +281,34 @@ def entries(self): """ Get the locally cached files for the bucket. """ - if self.preload_metadata and not self._entries: - self._entries = dict((self._decode_name(entry.key), entry) - for entry in self.bucket.list(prefix=self.location)) + if self.preload_metadata and not self._loaded_meta: + self._entries.update({ + self._decode_name(entry.key): entry + for entry in self.bucket.list(prefix=self.location) + }) + self._loaded_meta = True return self._entries + def _lookup_env(self, names): + for name in names: + value = os.environ.get(name) + if value: + return value + def _get_access_keys(self): """ Gets the access keys to use when accessing S3. If none are provided to the class in the constructor or in the settings then get them from the environment variables. """ - def lookup_env(names): - for name in names: - value = os.environ.get(name) - if value: - return value - access_key = self.access_key or lookup_env(self.access_key_names) - secret_key = self.secret_key or lookup_env(self.secret_key_names) + access_key = self.access_key or self._lookup_env(self.access_key_names) + secret_key = self.secret_key or self._lookup_env(self.secret_key_names) return access_key, secret_key + def _get_security_token(self): + security_token = self._lookup_env(self.security_token_names) + return security_token + def _get_or_create_bucket(self, name): """ Retrieves a bucket if it exists, otherwise creates it. @@ -332,25 +320,16 @@ def _get_or_create_bucket(self, name): bucket = self.connection.create_bucket(name, location=self.origin) bucket.set_acl(self.bucket_acl) return bucket - raise ImproperlyConfigured("Bucket %s does not exist. Buckets " - "can be automatically created by " - "setting AWS_AUTO_CREATE_BUCKET to " - "``True``." % name) + raise ImproperlyConfigured('Bucket %s does not exist. Buckets ' + 'can be automatically created by ' + 'setting AWS_AUTO_CREATE_BUCKET to ' + '``True``.' % name) def _clean_name(self, name): """ Cleans the name so that Windows style paths work """ - # Normalize Windows style paths - clean_name = posixpath.normpath(name).replace('\\', '/') - - # os.path.normpath() can strip trailing slashes so we implement - # a workaround here. - if name.endswith('/') and not clean_name.endswith('/'): - # Add a trailing slash as it was stripped. - return clean_name + '/' - else: - return clean_name + return clean_name(name) def _normalize_name(self, name): """ @@ -435,6 +414,12 @@ def _save_content(self, key, content, headers): reduced_redundancy=self.reduced_redundancy, rewind=True, **kwargs) + def _get_key(self, name): + name = self._normalize_name(self._clean_name(name)) + if self.entries: + return self.entries.get(name) + return self.bucket.get_key(self._encode_name(name)) + def delete(self, name): name = self._normalize_name(self._clean_name(name)) self.bucket.delete_key(self._encode_name(name)) @@ -447,11 +432,7 @@ def exists(self, name): except ImproperlyConfigured: return False - name = self._normalize_name(self._clean_name(name)) - if self.entries: - return name in self.entries - k = self.bucket.new_key(self._encode_name(name)) - return k.exists() + return self._get_key(name) is not None def listdir(self, name): name = self._normalize_name(self._clean_name(name)) @@ -463,9 +444,9 @@ def listdir(self, name): dirlist = self.bucket.list(self._encode_name(name)) files = [] dirs = set() - base_parts = name.split("/")[:-1] + base_parts = name.split('/')[:-1] for item in dirlist: - parts = item.name.split("/") + parts = item.name.split('/') parts = parts[len(base_parts):] if len(parts) == 1: # File @@ -476,29 +457,21 @@ def listdir(self, name): return list(dirs), files def size(self, name): - name = self._normalize_name(self._clean_name(name)) - if self.entries: - entry = self.entries.get(name) - if entry: - return entry.size - return 0 - return self.bucket.get_key(self._encode_name(name)).size + return self._get_key(name).size + + def get_modified_time(self, name): + dt = tz.make_aware(parse_ts(self._get_key(name).last_modified), tz.utc) + return dt if setting('USE_TZ') else tz.make_naive(dt) def modified_time(self, name): - name = self._normalize_name(self._clean_name(name)) - entry = self.entries.get(name) - # only call self.bucket.get_key() if the key is not found - # in the preloaded metadata. - if entry is None: - entry = self.bucket.get_key(self._encode_name(name)) - # Parse the last_modified string to a local datetime object. - return parse_ts(entry.last_modified) + dt = tz.make_aware(parse_ts(self._get_key(name).last_modified), tz.utc) + return tz.make_naive(dt) def url(self, name, headers=None, response_headers=None, expire=None): # Preserve the trailing slash after normalizing the path. name = self._normalize_name(self._clean_name(name)) if self.custom_domain: - return "%s//%s/%s" % (self.url_protocol, + return '%s//%s/%s' % (self.url_protocol, self.custom_domain, filepath_to_uri(name)) if expire is None: diff --git a/storages/backends/s3boto3.py b/storages/backends/s3boto3.py index a17885fd9..9caae4d0a 100644 --- a/storages/backends/s3boto3.py +++ b/storages/backends/s3boto3.py @@ -1,6 +1,7 @@ +import mimetypes import os import posixpath -import mimetypes +import threading from gzip import GzipFile from tempfile import SpooledTemporaryFile @@ -8,10 +9,14 @@ from django.core.files.base import File from django.core.files.storage import Storage from django.utils.deconstruct import deconstructible -from django.utils.encoding import force_text, smart_str, filepath_to_uri, force_bytes -from django.utils.six.moves.urllib import parse as urlparse +from django.utils.encoding import ( + filepath_to_uri, force_bytes, force_text, smart_text, +) from django.utils.six import BytesIO -from django.utils.timezone import localtime, is_naive +from django.utils.six.moves.urllib import parse as urlparse +from django.utils.timezone import is_naive, localtime + +from storages.utils import safe_join, setting try: import boto3.session @@ -22,7 +27,6 @@ raise ImproperlyConfigured("Could not load Boto3's S3 bindings.\n" "See https://github.com/boto/boto3") -from storages.utils import setting boto3_version_info = tuple([int(i) for i in boto3_version.split('.')]) @@ -31,39 +35,6 @@ "higher.\nSee https://github.com/boto/boto3") -def safe_join(base, *paths): - """ - A version of django.utils._os.safe_join for S3 paths. - - Joins one or more path components to the base path component - intelligently. Returns a normalized version of the final path. - - The final path must be located inside of the base path component - (otherwise a ValueError is raised). - - Paths outside the base path indicate a possible security - sensitive operation. - """ - base_path = force_text(base) - base_path = base_path.rstrip('/') - paths = [force_text(p) for p in paths] - - final_path = base_path - for path in paths: - final_path = urlparse.urljoin(final_path.rstrip('/') + "/", path) - - # Ensure final_path starts with base_path and that the next character after - # the final path is '/' (or nothing, in which case final_path must be - # equal to base_path). - base_path_len = len(base_path) - if (not final_path.startswith(base_path) or - final_path[base_path_len:base_path_len + 1] not in ('', '/')): - raise ValueError('the joined path is located outside of the base path' - ' component') - - return final_path.lstrip('/') - - @deconstructible class S3Boto3StorageFile(File): @@ -199,16 +170,14 @@ class S3Boto3Storage(Storage): mode and supports streaming(buffering) data in chunks to S3 when writing. """ - connection_service_name = 's3' default_content_type = 'application/octet-stream' - connection_response_error = ClientError - file_class = S3Boto3StorageFile # If config provided in init, signature_version and addressing_style settings/args are ignored. config = None # used for looking up the access and secret key from env vars access_key_names = ['AWS_S3_ACCESS_KEY_ID', 'AWS_ACCESS_KEY_ID'] secret_key_names = ['AWS_S3_SECRET_ACCESS_KEY', 'AWS_SECRET_ACCESS_KEY'] + security_token_names = ['AWS_SESSION_TOKEN', 'AWS_SECURITY_TOKEN'] access_key = setting('AWS_S3_ACCESS_KEY_ID', setting('AWS_ACCESS_KEY_ID')) secret_key = setting('AWS_S3_SECRET_ACCESS_KEY', setting('AWS_SECRET_ACCESS_KEY')) @@ -268,10 +237,12 @@ def __init__(self, acl=None, bucket=None, **settings): self._entries = {} self._bucket = None - self._connection = None + self._connections = threading.local() + self.security_token = None if not self.access_key and not self.secret_key: self.access_key, self.secret_key = self._get_access_keys() + self.security_token = self._get_security_token() if not self.config: self.config = Config(s3={'addressing_style': self.addressing_style}, @@ -283,18 +254,20 @@ def connection(self): # Note that proxies are handled by environment variables that the underlying # urllib/requests libraries read. See https://github.com/boto/boto3/issues/338 # and http://docs.python-requests.org/en/latest/user/advanced/#proxies - if self._connection is None: + connection = getattr(self._connections, 'connection', None) + if connection is None: session = boto3.session.Session() - self._connection = session.resource( - self.connection_service_name, + self._connections.connection = session.resource( + 's3', aws_access_key_id=self.access_key, aws_secret_access_key=self.secret_key, + aws_session_token=self.security_token, region_name=self.region_name, use_ssl=self.use_ssl, endpoint_url=self.endpoint_url, config=self.config ) - return self._connection + return self._connections.connection @property def bucket(self): @@ -312,25 +285,32 @@ def entries(self): Get the locally cached files for the bucket. """ if self.preload_metadata and not self._entries: - self._entries = dict((self._decode_name(entry.key), entry) - for entry in self.bucket.objects.filter(Prefix=self.location)) + self._entries = { + self._decode_name(entry.key): entry + for entry in self.bucket.objects.filter(Prefix=self.location) + } return self._entries + def _lookup_env(self, names): + for name in names: + value = os.environ.get(name) + if value: + return value + def _get_access_keys(self): """ Gets the access keys to use when accessing S3. If none are provided to the class in the constructor or in the settings then get them from the environment variables. """ - def lookup_env(names): - for name in names: - value = os.environ.get(name) - if value: - return value - access_key = self.access_key or lookup_env(self.access_key_names) - secret_key = self.secret_key or lookup_env(self.secret_key_names) + access_key = self.access_key or self._lookup_env(self.access_key_names) + secret_key = self.secret_key or self._lookup_env(self.secret_key_names) return access_key, secret_key + def _get_security_token(self): + security_token = self._lookup_env(self.security_token_names) + return security_token + def _get_or_create_bucket(self, name): """ Retrieves a bucket if it exists, otherwise creates it. @@ -341,7 +321,7 @@ def _get_or_create_bucket(self, name): # Directly call head_bucket instead of bucket.load() because head_bucket() # fails on wrong region, while bucket.load() does not. bucket.meta.client.head_bucket(Bucket=name) - except self.connection_response_error as err: + except ClientError as err: if err.response['ResponseMetadata']['HTTPStatusCode'] == 301: raise ImproperlyConfigured("Bucket %s exists, but in a different " "region than we are connecting to. Set " @@ -384,9 +364,8 @@ def _clean_name(self, name): # a workaround here. if name.endswith('/') and not clean_name.endswith('/'): # Add a trailing slash as it was stripped. - return clean_name + '/' - else: - return clean_name + clean_name += '/' + return clean_name def _normalize_name(self, name): """ @@ -401,15 +380,20 @@ def _normalize_name(self, name): name) def _encode_name(self, name): - return smart_str(name, encoding=self.file_name_charset) + return smart_text(name, encoding=self.file_name_charset) def _decode_name(self, name): return force_text(name, encoding=self.file_name_charset) def _compress_content(self, content): """Gzip a given string content.""" + content.seek(0) zbuf = BytesIO() - zfile = GzipFile(mode='wb', compresslevel=6, fileobj=zbuf) + # The GZIP header has a modification time attribute (see http://www.zlib.org/rfc-gzip.html) + # This means each time a file is compressed it changes even if the other contents don't change + # For S3 this defeats detection of changes using MD5 sums on gzipped files + # Fixing the mtime at 0.0 at compression time avoids this problem + zfile = GzipFile(mode='wb', compresslevel=6, fileobj=zbuf, mtime=0.0) try: zfile.write(force_bytes(content.read())) finally: @@ -423,8 +407,8 @@ def _compress_content(self, content): def _open(self, name, mode='rb'): name = self._normalize_name(self._clean_name(name)) try: - f = self.file_class(name, mode, self) - except self.connection_response_error as err: + f = S3Boto3StorageFile(name, mode, self) + except ClientError as err: if err.response['ResponseMetadata']['HTTPStatusCode'] == 404: raise IOError('File does not exist: %s' % name) raise # Let it bubble up if it was some other error @@ -434,8 +418,9 @@ def _save(self, name, content): cleaned_name = self._clean_name(name) name = self._normalize_name(cleaned_name) parameters = self.object_parameters.copy() + _type, encoding = mimetypes.guess_type(name) content_type = getattr(content, 'content_type', - mimetypes.guess_type(name)[0] or self.default_content_type) + _type or self.default_content_type) # setting the content_type in the key object is not enough. parameters.update({'ContentType': content_type}) @@ -443,12 +428,27 @@ def _save(self, name, content): if self.gzip and content_type in self.gzip_content_types: content = self._compress_content(content) parameters.update({'ContentEncoding': 'gzip'}) + elif encoding: + # If the content already has a particular encoding, set it + parameters.update({'ContentEncoding': encoding}) encoded_name = self._encode_name(name) obj = self.bucket.Object(encoded_name) if self.preload_metadata: self._entries[encoded_name] = obj + # If both `name` and `content.name` are empty or None, your request + # can be rejected with `XAmzContentSHA256Mismatch` error, because in + # `django.core.files.storage.Storage.save` method your file-like object + # will be wrapped in `django.core.files.File` if no `chunks` method + # provided. `File.__bool__` method is Django-specific and depends on + # file name, for this reason`botocore.handlers.calculate_md5` can fail + # even if wrapped file-like object exists. To avoid Django-specific + # logic, pass internal file-like object if `content` is `File` + # class instance. + if isinstance(content, File): + content = content.file + self._save_content(obj, content, parameters=parameters) # Note: In boto3, after a put, last_modified is automatically reloaded # the next time it is accessed; no need to specifically reload it. @@ -471,20 +471,13 @@ def delete(self, name): self.bucket.Object(self._encode_name(name)).delete() def exists(self, name): - if not name: - try: - self.bucket - return True - except ImproperlyConfigured: - return False name = self._normalize_name(self._clean_name(name)) if self.entries: return name in self.entries - obj = self.bucket.Object(self._encode_name(name)) try: - obj.load() + self.connection.meta.client.head_object(Bucket=self.bucket_name, Key=name) return True - except self.connection_response_error: + except ClientError: return False def listdir(self, name): @@ -513,7 +506,7 @@ def size(self, name): if self.entries: entry = self.entries.get(name) if entry: - return entry.content_length + return entry.size if hasattr(entry, 'size') else entry.content_length return 0 return self.bucket.Object(self._encode_name(name)).content_length @@ -550,9 +543,11 @@ def _strip_signing_parameters(self, url): # from v2 and v4 signatures, regardless of the actual signature version used. split_url = urlparse.urlsplit(url) qs = urlparse.parse_qsl(split_url.query, keep_blank_values=True) - blacklist = set(['x-amz-algorithm', 'x-amz-credential', 'x-amz-date', - 'x-amz-expires', 'x-amz-signedheaders', 'x-amz-signature', - 'x-amz-security-token', 'awsaccesskeyid', 'expires', 'signature']) + blacklist = { + 'x-amz-algorithm', 'x-amz-credential', 'x-amz-date', + 'x-amz-expires', 'x-amz-signedheaders', 'x-amz-signature', + 'x-amz-security-token', 'awsaccesskeyid', 'expires', 'signature', + } filtered_qs = ((key, val) for key, val in qs if key.lower() not in blacklist) # Note: Parameters that did not have a value in the original query string will have # an '=' sign appended to it, e.g ?foo&bar becomes ?foo=&bar= diff --git a/storages/backends/sftpstorage.py b/storages/backends/sftpstorage.py index 6efdf1234..f07e8cf5d 100644 --- a/storages/backends/sftpstorage.py +++ b/storages/backends/sftpstorage.py @@ -1,17 +1,17 @@ -from __future__ import print_function # SFTP storage backend for Django. # Author: Brent Tubbs # License: MIT # # Modeled on the FTP storage by Rafal Jonca +from __future__ import print_function import getpass import os -import paramiko import posixpath import stat from datetime import datetime +import paramiko from django.core.files.base import File from django.core.files.storage import Storage from django.utils.deconstruct import deconstructible @@ -53,11 +53,12 @@ def __init__(self, host=None, params=None, interactive=None, file_mode=None, def _connect(self): self._ssh = paramiko.SSHClient() - if self._known_host_file is not None: - self._ssh.load_host_keys(self._known_host_file) - else: - # automatically add host keys from current user. - self._ssh.load_host_keys(os.path.expanduser(os.path.join("~", ".ssh", "known_hosts"))) + known_host_file = self._known_host_file or os.path.expanduser( + os.path.join("~", ".ssh", "known_hosts") + ) + + if os.path.exists(known_host_file): + self._ssh.load_host_keys(known_host_file) # and automatically add new host keys for hosts we haven't seen before. self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) @@ -152,6 +153,7 @@ def delete(self, name): def exists(self, name): # Try to retrieve file info. Return true on success, false on failure. remote_path = self._remote_path(name) + try: self.sftp.stat(remote_path) return True diff --git a/storages/backends/symlinkorcopy.py b/storages/backends/symlinkorcopy.py deleted file mode 100644 index e5b6e7ef3..000000000 --- a/storages/backends/symlinkorcopy.py +++ /dev/null @@ -1,71 +0,0 @@ -import os -import warnings - -from django.conf import settings -from django.core.files.storage import FileSystemStorage -from django.utils.deconstruct import deconstructible - -__doc__ = """ -I needed to efficiently create a mirror of a directory tree (so that -"origin pull" CDNs can automatically pull files). The trick was that -some files could be modified, and some could be identical to the original. -Of course it doesn't make sense to store the exact same data twice on the -file system. So I created SymlinkOrCopyStorage. - -SymlinkOrCopyStorage allows you to symlink a file when it's identical to -the original file and to copy the file if it's modified. -Of course, it's impossible to know if a file is modified just by looking -at the file, without knowing what the original file was. -That's what the symlinkWithin parameter is for. It accepts one or more paths -(if multiple, they should be concatenated using a colon (:)). -Files that will be saved using SymlinkOrCopyStorage are then checked on their -location: if they are within one of the symlink_within directories, -they will be symlinked, otherwise they will be copied. - -The rationale is that unmodified files will exist in their original location, -e.g. /htdocs/example.com/image.jpg and modified files will be stored in -a temporary directory, e.g. /tmp/image.jpg. -""" -warnings.warn( - 'SymlinkOrCopyStorage is unmaintained and will be removed in the next django-storages version.' - 'See https://github.com/jschneier/django-storages/issues/202', - PendingDeprecationWarning -) - - -@deconstructible -class SymlinkOrCopyStorage(FileSystemStorage): - """Stores symlinks to files instead of actual files whenever possible - - When a file that's being saved is currently stored in the symlink_within - directory, then symlink the file. Otherwise, copy the file. - """ - def __init__(self, location=settings.MEDIA_ROOT, base_url=settings.MEDIA_URL, - symlink_within=None): - super(SymlinkOrCopyStorage, self).__init__(location, base_url) - self.symlink_within = symlink_within.split(":") - - def _save(self, name, content): - full_path_dst = self.path(name) - - directory = os.path.dirname(full_path_dst) - if not os.path.exists(directory): - os.makedirs(directory) - elif not os.path.isdir(directory): - raise IOError("%s exists and is not a directory." % directory) - - full_path_src = os.path.abspath(content.name) - - symlinked = False - # Only symlink if the current platform supports it. - if getattr(os, "symlink", False): - for path in self.symlink_within: - if full_path_src.startswith(path): - os.symlink(full_path_src, full_path_dst) - symlinked = True - break - - if not symlinked: - super(SymlinkOrCopyStorage, self)._save(name, content) - - return name diff --git a/storages/utils.py b/storages/utils.py index 2f501b194..566aa5127 100644 --- a/storages/utils.py +++ b/storages/utils.py @@ -1,5 +1,8 @@ +import posixpath + from django.conf import settings from django.core.exceptions import ImproperlyConfigured +from django.utils.encoding import force_text def setting(name, default=None, strict=False): @@ -20,3 +23,60 @@ def setting(name, default=None, strict=False): msg = "You must provide settings.%s" % name raise ImproperlyConfigured(msg) return getattr(settings, name, default) + + +def clean_name(name): + """ + Cleans the name so that Windows style paths work + """ + # Normalize Windows style paths + clean_name = posixpath.normpath(name).replace('\\', '/') + + # os.path.normpath() can strip trailing slashes so we implement + # a workaround here. + if name.endswith('/') and not clean_name.endswith('/'): + # Add a trailing slash as it was stripped. + clean_name = clean_name + '/' + + # Given an empty string, os.path.normpath() will return ., which we don't want + if clean_name == '.': + clean_name = '' + + return clean_name + + +def safe_join(base, *paths): + """ + A version of django.utils._os.safe_join for S3 paths. + + Joins one or more path components to the base path component + intelligently. Returns a normalized version of the final path. + + The final path must be located inside of the base path component + (otherwise a ValueError is raised). + + Paths outside the base path indicate a possible security + sensitive operation. + """ + base_path = force_text(base) + base_path = base_path.rstrip('/') + paths = [force_text(p) for p in paths] + + final_path = base_path + '/' + for path in paths: + _final_path = posixpath.normpath(posixpath.join(final_path, path)) + # posixpath.normpath() strips the trailing /. Add it back. + if path.endswith('/') or _final_path + '/' == final_path: + _final_path += '/' + final_path = _final_path + if final_path == base_path: + final_path += '/' + + # Ensure final_path starts with base_path and that the next character after + # the base path is /. + base_path_len = len(base_path) + if (not final_path.startswith(base_path) or final_path[base_path_len] != '/'): + raise ValueError('the joined path is located outside of the base path' + ' component') + + return final_path.lstrip('/') diff --git a/tests/test_dropbox.py b/tests/test_dropbox.py index a29d10468..58d503628 100644 --- a/tests/test_dropbox.py +++ b/tests/test_dropbox.py @@ -1,16 +1,23 @@ -import re from datetime import datetime + +from django.core.exceptions import ( + ImproperlyConfigured, SuspiciousFileOperation, +) +from django.core.files.base import ContentFile, File +from django.test import TestCase +from django.utils.six import BytesIO + +from storages.backends import dropbox + try: from unittest import mock except ImportError: # Python 3.2 and below import mock -from django.test import TestCase -from django.core.files.base import File, ContentFile -from django.core.exceptions import ImproperlyConfigured, \ - SuspiciousFileOperation -from storages.backends import dropbox +class F(object): + pass + FILE_DATE = datetime(2015, 8, 24, 15, 6, 41) FILE_FIXTURE = { @@ -50,21 +57,11 @@ 'size': '0 bytes', 'thumb_exists': False } -FILE_MEDIA_FIXTURE = { - 'url': 'https://dl.dropboxusercontent.com/1/view/foo', - 'expires': 'Fri, 16 Sep 2011 01:01:25 +0000', -} - -__all__ = [ - 'DropBoxTest', - 'DropBoxFileTest' -] +FILE_MEDIA_FIXTURE = F() +FILE_MEDIA_FIXTURE.link = 'https://dl.dropboxusercontent.com/1/view/foo' class DropBoxTest(TestCase): - @mock.patch('dropbox.client._OAUTH2_ACCESS_TOKEN_PATTERN', - re.compile(r'.*')) - @mock.patch('dropbox.client.DropboxOAuth2Session') def setUp(self, *args): self.storage = dropbox.DropBoxStorage('foo') @@ -72,24 +69,24 @@ def test_no_access_token(self, *args): with self.assertRaises(ImproperlyConfigured): dropbox.DropBoxStorage(None) - @mock.patch('dropbox.client.DropboxClient.file_delete', + @mock.patch('dropbox.Dropbox.files_delete', return_value=FILE_FIXTURE) def test_delete(self, *args): self.storage.delete('foo') - @mock.patch('dropbox.client.DropboxClient.metadata', + @mock.patch('dropbox.Dropbox.files_get_metadata', return_value=[FILE_FIXTURE]) def test_exists(self, *args): exists = self.storage.exists('foo') self.assertTrue(exists) - @mock.patch('dropbox.client.DropboxClient.metadata', + @mock.patch('dropbox.Dropbox.files_get_metadata', return_value=[]) def test_not_exists(self, *args): exists = self.storage.exists('bar') self.assertFalse(exists) - @mock.patch('dropbox.client.DropboxClient.metadata', + @mock.patch('dropbox.Dropbox.files_get_metadata', return_value=FILES_FIXTURE) def test_listdir(self, *args): dirs, files = self.storage.listdir('/') @@ -98,19 +95,19 @@ def test_listdir(self, *args): self.assertEqual(dirs[0], 'bar') self.assertEqual(files[0], 'foo.txt') - @mock.patch('dropbox.client.DropboxClient.metadata', + @mock.patch('dropbox.Dropbox.files_get_metadata', return_value=FILE_FIXTURE) def test_size(self, *args): size = self.storage.size('foo') self.assertEqual(size, FILE_FIXTURE['bytes']) - @mock.patch('dropbox.client.DropboxClient.metadata', + @mock.patch('dropbox.Dropbox.files_get_metadata', return_value=FILE_FIXTURE) def test_modified_time(self, *args): mtime = self.storage.modified_time('foo') self.assertEqual(mtime, FILE_DATE) - @mock.patch('dropbox.client.DropboxClient.metadata', + @mock.patch('dropbox.Dropbox.files_get_metadata', return_value=FILE_FIXTURE) def test_accessed_time(self, *args): mtime = self.storage.accessed_time('foo') @@ -120,16 +117,30 @@ def test_open(self, *args): obj = self.storage._open('foo') self.assertIsInstance(obj, File) - @mock.patch('dropbox.client.DropboxClient.put_file', + @mock.patch('dropbox.Dropbox.files_upload', return_value='foo') - def test_save(self, *args): - self.storage._save('foo', b'bar') - - @mock.patch('dropbox.client.DropboxClient.media', + def test_save(self, files_upload, *args): + self.storage._save('foo', File(BytesIO(b'bar'), 'foo')) + self.assertTrue(files_upload.called) + + @mock.patch('dropbox.Dropbox.files_upload') + @mock.patch('dropbox.Dropbox.files_upload_session_finish') + @mock.patch('dropbox.Dropbox.files_upload_session_append_v2') + @mock.patch('dropbox.Dropbox.files_upload_session_start', + return_value=mock.MagicMock(session_id='foo')) + def test_chunked_upload(self, start, append, finish, upload): + large_file = File(BytesIO(b'bar' * self.storage.CHUNK_SIZE), 'foo') + self.storage._save('foo', large_file) + self.assertTrue(start.called) + self.assertTrue(append.called) + self.assertTrue(finish.called) + self.assertFalse(upload.called) + + @mock.patch('dropbox.Dropbox.files_get_temporary_link', return_value=FILE_MEDIA_FIXTURE) def test_url(self, *args): url = self.storage.url('foo') - self.assertEqual(url, FILE_MEDIA_FIXTURE['url']) + self.assertEqual(url, FILE_MEDIA_FIXTURE.link) def test_formats(self, *args): self.storage = dropbox.DropBoxStorage('foo') @@ -141,24 +152,18 @@ def test_formats(self, *args): class DropBoxFileTest(TestCase): - @mock.patch('dropbox.client._OAUTH2_ACCESS_TOKEN_PATTERN', - re.compile(r'.*')) - @mock.patch('dropbox.client.DropboxOAuth2Session') def setUp(self, *args): self.storage = dropbox.DropBoxStorage('foo') self.file = dropbox.DropBoxFile('/foo.txt', self.storage) - @mock.patch('dropbox.client.DropboxClient.get_file', + @mock.patch('dropbox.Dropbox.files_download', return_value=ContentFile(b'bar')) def test_read(self, *args): file = self.storage._open(b'foo') self.assertEqual(file.read(), b'bar') -@mock.patch('dropbox.client._OAUTH2_ACCESS_TOKEN_PATTERN', - re.compile(r'.*')) -@mock.patch('dropbox.client.DropboxOAuth2Session') -@mock.patch('dropbox.client.DropboxClient.metadata', +@mock.patch('dropbox.Dropbox.files_get_metadata', return_value={'contents': []}) class DropBoxRootPathTest(TestCase): def test_jailed(self, *args): diff --git a/tests/test_ftp.py b/tests/test_ftp.py index 3b539e703..34ae7140b 100644 --- a/tests/test_ftp.py +++ b/tests/test_ftp.py @@ -4,9 +4,9 @@ from mock import patch from datetime import datetime -from django.test import TestCase from django.core.exceptions import ImproperlyConfigured from django.core.files.base import File +from django.test import TestCase from django.utils.six import BytesIO from storages.backends import ftp @@ -44,11 +44,25 @@ def test_init_location_from_setting(self, mock_setting): def test_decode_location(self): config = self.storage._decode_location(URL) - wanted_config = {'passwd': 'b@r', 'host': 'localhost', 'user': 'foo', 'active': False, 'path': '/', 'port': 2121} + wanted_config = { + 'passwd': 'b@r', + 'host': 'localhost', + 'user': 'foo', + 'active': False, + 'path': '/', + 'port': 2121, + } self.assertEqual(config, wanted_config) # Test active FTP config = self.storage._decode_location('a'+URL) - wanted_config = {'passwd': 'b@r', 'host': 'localhost', 'user': 'foo', 'active': True, 'path': '/', 'port': 2121} + wanted_config = { + 'passwd': 'b@r', + 'host': 'localhost', + 'user': 'foo', + 'active': True, + 'path': '/', + 'port': 2121, + } self.assertEqual(config, wanted_config) def test_decode_location_error(self): @@ -84,7 +98,7 @@ def test_disconnect(self, mock_ftp_quit): self.storage.disconnect() self.assertIsNone(self.storage._connection) - @patch('ftplib.FTP', **{'return_value.pwd.return_value': 'foo',}) + @patch('ftplib.FTP', **{'return_value.pwd.return_value': 'foo'}) def test_mkremdirs(self, mock_ftp): self.storage._start_connection() self.storage._mkremdirs('foo/bar') @@ -116,7 +130,7 @@ def test_read(self, mock_ftp): self.storage._read('foo') @patch('ftplib.FTP', **{'return_value.pwd.side_effect': IOError()}) - def test_read(self, mock_ftp): + def test_read2(self, mock_ftp): self.storage._start_connection() with self.assertRaises(ftp.FTPStorageException): self.storage._read('foo') diff --git a/tests/test_gcloud.py b/tests/test_gcloud.py new file mode 100644 index 000000000..e1c4cb603 --- /dev/null +++ b/tests/test_gcloud.py @@ -0,0 +1,287 @@ +# -*- coding: utf-8 -*- + +try: + from unittest import mock +except ImportError: # Python 3.2 and below + import mock + +import datetime +import mimetypes + +from django.core.files.base import ContentFile +from django.test import TestCase +from django.utils import timezone +from google.cloud.exceptions import NotFound +from google.cloud.storage.blob import Blob + +from storages.backends import gcloud + + +class GCloudTestCase(TestCase): + def setUp(self): + self.bucket_name = 'test_bucket' + self.filename = 'test_file.txt' + + self.storage = gcloud.GoogleCloudStorage(bucket_name=self.bucket_name) + + self.client_patcher = mock.patch('storages.backends.gcloud.Client') + self.client_patcher.start() + + def tearDown(self): + self.client_patcher.stop() + + +class GCloudStorageTests(GCloudTestCase): + + def test_open_read(self): + """ + Test opening a file and reading from it + """ + data = b'This is some test read data.' + + f = self.storage.open(self.filename) + self.storage._client.get_bucket.assert_called_with(self.bucket_name) + self.storage._bucket.get_blob.assert_called_with(self.filename) + + f.blob.download_to_file = lambda tmpfile: tmpfile.write(data) + self.assertEqual(f.read(), data) + + def test_open_read_num_bytes(self): + data = b'This is some test read data.' + num_bytes = 10 + + f = self.storage.open(self.filename) + self.storage._client.get_bucket.assert_called_with(self.bucket_name) + self.storage._bucket.get_blob.assert_called_with(self.filename) + + f.blob.download_to_file = lambda tmpfile: tmpfile.write(data) + self.assertEqual(f.read(num_bytes), data[0:num_bytes]) + + def test_open_read_nonexistent(self): + self.storage._bucket = mock.MagicMock() + self.storage._bucket.get_blob.return_value = None + + self.assertRaises(IOError, self.storage.open, self.filename) + self.storage._bucket.get_blob.assert_called_with(self.filename) + + def test_open_read_nonexistent_unicode(self): + filename = 'ủⓝï℅ⅆℇ.txt' + + self.storage._bucket = mock.MagicMock() + self.storage._bucket.get_blob.return_value = None + + self.assertRaises(IOError, self.storage.open, filename) + + @mock.patch('storages.backends.gcloud.Blob') + def test_open_write(self, MockBlob): + """ + Test opening a file and writing to it + """ + data = 'This is some test write data.' + + # Simulate the file not existing before the write + self.storage._bucket = mock.MagicMock() + self.storage._bucket.get_blob.return_value = None + + f = self.storage.open(self.filename, 'wb') + MockBlob.assert_called_with(self.filename, self.storage._bucket) + + f.write(data) + tmpfile = f._file + # File data is not actually written until close(), so do that. + f.close() + + MockBlob().upload_from_file.assert_called_with( + tmpfile, content_type=mimetypes.guess_type(self.filename)[0]) + + def test_save(self): + data = 'This is some test content.' + content = ContentFile(data) + + self.storage.save(self.filename, content) + + self.storage._client.get_bucket.assert_called_with(self.bucket_name) + self.storage._bucket.get_blob().upload_from_file.assert_called_with( + content, size=len(data), content_type=mimetypes.guess_type(self.filename)[0]) + + def test_save2(self): + data = 'This is some test ủⓝï℅ⅆℇ content.' + filename = 'ủⓝï℅ⅆℇ.txt' + content = ContentFile(data) + + self.storage.save(filename, content) + + self.storage._client.get_bucket.assert_called_with(self.bucket_name) + self.storage._bucket.get_blob().upload_from_file.assert_called_with( + content, size=len(data), content_type=mimetypes.guess_type(filename)[0]) + + def test_delete(self): + self.storage.delete(self.filename) + + self.storage._client.get_bucket.assert_called_with(self.bucket_name) + self.storage._bucket.delete_blob.assert_called_with(self.filename) + + def test_exists(self): + self.storage._bucket = mock.MagicMock() + self.assertTrue(self.storage.exists(self.filename)) + self.storage._bucket.get_blob.assert_called_with(self.filename) + + self.storage._bucket.reset_mock() + self.storage._bucket.get_blob.return_value = None + self.assertFalse(self.storage.exists(self.filename)) + self.storage._bucket.get_blob.assert_called_with(self.filename) + + def test_exists_no_bucket(self): + # exists('') should return False if the bucket doesn't exist + self.storage._client = mock.MagicMock() + self.storage._client.get_bucket.side_effect = NotFound('dang') + self.assertFalse(self.storage.exists('')) + + def test_exists_bucket(self): + # exists('') should return True if the bucket exists + self.assertTrue(self.storage.exists('')) + + def test_exists_bucket_auto_create(self): + # exists('') should automatically create the bucket if + # auto_create_bucket is configured + self.storage.auto_create_bucket = True + self.storage._client = mock.MagicMock() + self.storage._client.get_bucket.side_effect = NotFound('dang') + + self.assertTrue(self.storage.exists('')) + self.storage._client.create_bucket.assert_called_with(self.bucket_name) + + def test_listdir(self): + file_names = ["some/path/1.txt", "2.txt", "other/path/3.txt", "4.txt"] + + self.storage._bucket = mock.MagicMock() + self.storage._bucket.list_blobs.return_value = [] + for name in file_names: + blob = mock.MagicMock(spec=Blob) + blob.name = name + self.storage._bucket.list_blobs.return_value.append(blob) + + dirs, files = self.storage.listdir('') + + self.assertEqual(len(dirs), 2) + for directory in ["some", "other"]: + self.assertTrue(directory in dirs, + """ "%s" not in directory list "%s".""" % ( + directory, dirs)) + + self.assertEqual(len(files), 2) + for filename in ["2.txt", "4.txt"]: + self.assertTrue(filename in files, + """ "%s" not in file list "%s".""" % ( + filename, files)) + + def test_listdir_subdir(self): + file_names = ["some/path/1.txt", "some/2.txt"] + + self.storage._bucket = mock.MagicMock() + self.storage._bucket.list_blobs.return_value = [] + for name in file_names: + blob = mock.MagicMock(spec=Blob) + blob.name = name + self.storage._bucket.list_blobs.return_value.append(blob) + + dirs, files = self.storage.listdir('some/') + + self.assertEqual(len(dirs), 1) + self.assertTrue('path' in dirs, + """ "path" not in directory list "%s".""" % (dirs,)) + + self.assertEqual(len(files), 1) + self.assertTrue('2.txt' in files, + """ "2.txt" not in files list "%s".""" % (files,)) + + def test_size(self): + size = 1234 + + self.storage._bucket = mock.MagicMock() + blob = mock.MagicMock() + blob.size = size + self.storage._bucket.get_blob.return_value = blob + + self.assertEqual(self.storage.size(self.filename), size) + self.storage._bucket.get_blob.assert_called_with(self.filename) + + def test_size_no_file(self): + self.storage._bucket = mock.MagicMock() + self.storage._bucket.get_blob.return_value = None + + self.assertRaises(NotFound, self.storage.size, self.filename) + + def test_modified_time(self): + naive_date = datetime.datetime(2017, 1, 2, 3, 4, 5, 678) + aware_date = timezone.make_aware(naive_date, timezone.utc) + + self.storage._bucket = mock.MagicMock() + blob = mock.MagicMock() + blob.updated = aware_date + self.storage._bucket.get_blob.return_value = blob + + with self.settings(TIME_ZONE='UTC'): + mt = self.storage.modified_time(self.filename) + self.assertTrue(timezone.is_naive(mt)) + self.assertEqual(mt, naive_date) + self.storage._bucket.get_blob.assert_called_with(self.filename) + + def test_get_modified_time(self): + naive_date = datetime.datetime(2017, 1, 2, 3, 4, 5, 678) + aware_date = timezone.make_aware(naive_date, timezone.utc) + + self.storage._bucket = mock.MagicMock() + blob = mock.MagicMock() + blob.updated = aware_date + self.storage._bucket.get_blob.return_value = blob + + with self.settings(TIME_ZONE='America/Montreal', USE_TZ=False): + mt = self.storage.get_modified_time(self.filename) + self.assertTrue(timezone.is_naive(mt)) + naive_date_montreal = timezone.make_naive(aware_date) + self.assertEqual(mt, naive_date_montreal) + self.storage._bucket.get_blob.assert_called_with(self.filename) + + with self.settings(TIME_ZONE='America/Montreal', USE_TZ=True): + mt = self.storage.get_modified_time(self.filename) + self.assertTrue(timezone.is_aware(mt)) + self.assertEqual(mt, aware_date) + self.storage._bucket.get_blob.assert_called_with(self.filename) + + def test_modified_time_no_file(self): + self.storage._bucket = mock.MagicMock() + self.storage._bucket.get_blob.return_value = None + + self.assertRaises(NotFound, self.storage.modified_time, self.filename) + + def test_url(self): + url = 'https://example.com/mah-bukkit/{}'.format(self.filename) + + self.storage._bucket = mock.MagicMock() + blob = mock.MagicMock() + blob.public_url = url + self.storage._bucket.get_blob.return_value = blob + + self.assertEqual(self.storage.url(self.filename), url) + self.storage._bucket.get_blob.assert_called_with(self.filename) + + def test_url_no_file(self): + self.storage._bucket = mock.MagicMock() + self.storage._bucket.get_blob.return_value = None + + self.assertRaises(NotFound, self.storage.url, self.filename) + + def test_get_available_name(self): + self.storage.file_overwrite = True + self.assertEqual(self.storage.get_available_name(self.filename), self.filename) + + self.storage._bucket = mock.MagicMock() + self.storage._bucket.get_blob.return_value = None + self.storage.file_overwrite = False + self.assertEqual(self.storage.get_available_name(self.filename), self.filename) + self.storage._bucket.get_blob.assert_called_with(self.filename) + + def test_get_available_name_unicode(self): + filename = 'ủⓝï℅ⅆℇ.txt' + self.assertEqual(self.storage.get_available_name(filename), filename) diff --git a/tests/test_gs.py b/tests/test_gs.py index 814fc3391..48ad71e78 100644 --- a/tests/test_gs.py +++ b/tests/test_gs.py @@ -1,5 +1,5 @@ -from django.test import TestCase from django.core.files.base import ContentFile +from django.test import TestCase from storages.backends import gs, s3boto diff --git a/tests/test_hashpath.py b/tests/test_hashpath.py deleted file mode 100644 index 5cc4d6571..000000000 --- a/tests/test_hashpath.py +++ /dev/null @@ -1,35 +0,0 @@ -import os -import shutil - -from django.test import TestCase -from django.core.files.base import ContentFile -from django.conf import settings - -from storages.backends.hashpath import HashPathStorage - -TEST_PATH_PREFIX = 'django-storages-test' - - -class HashPathStorageTest(TestCase): - - def setUp(self): - self.test_path = os.path.join(settings.MEDIA_ROOT, TEST_PATH_PREFIX) - self.storage = HashPathStorage(location=self.test_path) - - # make sure the profile upload folder exists - if not os.path.exists(self.test_path): - os.makedirs(self.test_path) - - def tearDown(self): - # remove uploaded profile picture - if os.path.exists(self.test_path): - shutil.rmtree(self.test_path) - - def test_save_same_file(self): - """ - saves a file twice, the file should only be stored once, because the - content/hash is the same - """ - path_1 = self.storage.save('test', ContentFile('new content')) - path_2 = self.storage.save('test', ContentFile('new content')) - self.assertEqual(path_1, path_2) diff --git a/tests/test_s3boto.py b/tests/test_s3boto.py index 93e678d1d..0457cc506 100644 --- a/tests/test_s3boto.py +++ b/tests/test_s3boto.py @@ -5,21 +5,16 @@ import datetime -from django.test import TestCase -from django.core.files.base import ContentFile -from django.utils.six.moves.urllib import parse as urlparse - from boto.exception import S3ResponseError from boto.s3.key import Key -from boto.utils import parse_ts, ISO8601 +from boto.utils import ISO8601, parse_ts +from django.core.files.base import ContentFile +from django.test import TestCase +from django.utils import timezone as tz +from django.utils.six.moves.urllib import parse as urlparse from storages.backends import s3boto -__all__ = ( - 'SafeJoinTest', - 'S3BotoStorageTests', -) - class S3BotoTestCase(TestCase): @mock.patch('storages.backends.s3boto.S3Connection') @@ -28,73 +23,16 @@ def setUp(self, S3Connection): self.storage._connection = mock.MagicMock() -class SafeJoinTest(TestCase): - def test_normal(self): - path = s3boto.safe_join("", "path/to/somewhere", "other", "path/to/somewhere") - self.assertEqual(path, "path/to/somewhere/other/path/to/somewhere") - - def test_with_dot(self): - path = s3boto.safe_join("", "path/./somewhere/../other", "..", - ".", "to/./somewhere") - self.assertEqual(path, "path/to/somewhere") - - def test_base_url(self): - path = s3boto.safe_join("base_url", "path/to/somewhere") - self.assertEqual(path, "base_url/path/to/somewhere") - - def test_base_url_with_slash(self): - path = s3boto.safe_join("base_url/", "path/to/somewhere") - self.assertEqual(path, "base_url/path/to/somewhere") - - def test_suspicious_operation(self): - self.assertRaises(ValueError, - s3boto.safe_join, "base", "../../../../../../../etc/passwd") - - def test_trailing_slash(self): - """ - Test safe_join with paths that end with a trailing slash. - """ - path = s3boto.safe_join("base_url/", "path/to/somewhere/") - self.assertEqual(path, "base_url/path/to/somewhere/") - - def test_trailing_slash_multi(self): - """ - Test safe_join with multiple paths that end with a trailing slash. - """ - path = s3boto.safe_join("base_url/", "path/to/" "somewhere/") - self.assertEqual(path, "base_url/path/to/somewhere/") - - class S3BotoStorageTests(S3BotoTestCase): def test_clean_name(self): """ - Test the base case of _clean_name + Test the base case of _clean_name - more tests are performed in + test_utils """ path = self.storage._clean_name("path/to/somewhere") self.assertEqual(path, "path/to/somewhere") - def test_clean_name_normalize(self): - """ - Test the normalization of _clean_name - """ - path = self.storage._clean_name("path/to/../somewhere") - self.assertEqual(path, "path/somewhere") - - def test_clean_name_trailing_slash(self): - """ - Test the _clean_name when the path has a trailing slash - """ - path = self.storage._clean_name("path/to/somewhere/") - self.assertEqual(path, "path/to/somewhere/") - - def test_clean_name_windows(self): - """ - Test the _clean_name when the path has a trailing slash - """ - path = self.storage._clean_name("path\\to\\somewhere") - self.assertEqual(path, "path/to/somewhere") - def test_storage_url_slashes(self): """ Test URL generation. @@ -219,13 +157,11 @@ def test_storage_exists_bucket(self): self.assertTrue(self.storage.exists('')) def test_storage_exists(self): - key = self.storage.bucket.new_key.return_value - key.exists.return_value = True + self.storage.bucket.get_key.return_value = mock.MagicMock(spec=Key) self.assertTrue(self.storage.exists("file.txt")) def test_storage_exists_false(self): - key = self.storage.bucket.new_key.return_value - key.exists.return_value = False + self.storage.bucket.get_key.return_value = None self.assertFalse(self.storage.exists("file.txt")) def test_storage_delete(self): @@ -285,15 +221,15 @@ def test_storage_url(self): url = 'http://aws.amazon.com/%s' % name self.storage.connection.generate_url.return_value = url - kwargs = dict( - method='GET', - bucket=self.storage.bucket.name, - key=name, - query_auth=self.storage.querystring_auth, - force_http=not self.storage.secure_urls, - headers=None, - response_headers=None, - ) + kwargs = { + 'method': 'GET', + 'bucket': self.storage.bucket.name, + 'key': name, + 'query_auth': self.storage.querystring_auth, + 'force_http': not self.storage.secure_urls, + 'headers': None, + 'response_headers': None, + } self.assertEqual(self.storage.url(name), url) self.storage.connection.generate_url.assert_called_with( @@ -322,8 +258,31 @@ def test_new_file_modified_time(self): name = 'test_storage_save.txt' content = ContentFile('new content') utcnow = datetime.datetime.utcnow() - with mock.patch('storages.backends.s3boto.datetime') as mock_datetime: + with mock.patch('storages.backends.s3boto.datetime') as mock_datetime, self.settings(TIME_ZONE='UTC'): mock_datetime.utcnow.return_value = utcnow self.storage.save(name, content) self.assertEqual(self.storage.modified_time(name), parse_ts(utcnow.strftime(ISO8601))) + + @mock.patch('storages.backends.s3boto.S3BotoStorage._get_key') + def test_get_modified_time(self, getkey): + utcnow = datetime.datetime.utcnow().strftime(ISO8601) + + with self.settings(USE_TZ=True, TIME_ZONE='America/New_York'): + key = mock.MagicMock(spec=Key) + key.last_modified = utcnow + getkey.return_value = key + modtime = self.storage.get_modified_time('foo') + self.assertFalse(tz.is_naive(modtime)) + self.assertEqual(modtime, + tz.make_aware(datetime.datetime.strptime(utcnow, ISO8601), tz.utc)) + + with self.settings(USE_TZ=False, TIME_ZONE='America/New_York'): + key = mock.MagicMock(spec=Key) + key.last_modified = utcnow + getkey.return_value = key + modtime = self.storage.get_modified_time('foo') + self.assertTrue(tz.is_naive(modtime)) + self.assertEqual(modtime, + tz.make_naive(tz.make_aware( + datetime.datetime.strptime(utcnow, ISO8601), tz.utc))) diff --git a/tests/test_s3boto3.py b/tests/test_s3boto3.py index 0f5a6a9b7..ef1a263e3 100644 --- a/tests/test_s3boto3.py +++ b/tests/test_s3boto3.py @@ -1,67 +1,30 @@ -from datetime import datetime +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + import gzip -try: - from unittest import mock -except ImportError: # Python 3.2 and below - import mock +import threading +from datetime import datetime +from unittest import skipIf -from django.test import TestCase +from botocore.exceptions import ClientError from django.conf import settings from django.core.files.base import ContentFile +from django.test import TestCase from django.utils.six.moves.urllib import parse as urlparse from django.utils.timezone import is_aware, utc -from botocore.exceptions import ClientError - from storages.backends import s3boto3 -__all__ = ( - 'SafeJoinTest', - 'S3Boto3StorageTests', -) +try: + from unittest import mock +except ImportError: # Python 3.2 and below + import mock class S3Boto3TestCase(TestCase): def setUp(self): self.storage = s3boto3.S3Boto3Storage() - self.storage._connection = mock.MagicMock() - - -class SafeJoinTest(TestCase): - def test_normal(self): - path = s3boto3.safe_join("", "path/to/somewhere", "other", "path/to/somewhere") - self.assertEqual(path, "path/to/somewhere/other/path/to/somewhere") - - def test_with_dot(self): - path = s3boto3.safe_join("", "path/./somewhere/../other", "..", - ".", "to/./somewhere") - self.assertEqual(path, "path/to/somewhere") - - def test_base_url(self): - path = s3boto3.safe_join("base_url", "path/to/somewhere") - self.assertEqual(path, "base_url/path/to/somewhere") - - def test_base_url_with_slash(self): - path = s3boto3.safe_join("base_url/", "path/to/somewhere") - self.assertEqual(path, "base_url/path/to/somewhere") - - def test_suspicious_operation(self): - self.assertRaises(ValueError, - s3boto3.safe_join, "base", "../../../../../../../etc/passwd") - - def test_trailing_slash(self): - """ - Test safe_join with paths that end with a trailing slash. - """ - path = s3boto3.safe_join("base_url/", "path/to/somewhere/") - self.assertEqual(path, "base_url/path/to/somewhere/") - - def test_trailing_slash_multi(self): - """ - Test safe_join with multiple paths that end with a trailing slash. - """ - path = s3boto3.safe_join("base_url/", "path/to/" "somewhere/") - self.assertEqual(path, "base_url/path/to/somewhere/") + self.storage._connections.connection = mock.MagicMock() class S3Boto3StorageTests(S3Boto3TestCase): @@ -119,13 +82,30 @@ def test_storage_save(self): obj = self.storage.bucket.Object.return_value obj.upload_fileobj.assert_called_with( - content, + content.file, ExtraArgs={ 'ContentType': 'text/plain', 'ACL': self.storage.default_acl, } ) + def test_storage_save_gzipped(self): + """ + Test saving a gzipped file + """ + name = 'test_storage_save.gz' + content = ContentFile("I am gzip'd") + self.storage.save(name, content) + obj = self.storage.bucket.Object.return_value + obj.upload_fileobj.assert_called_with( + content.file, + ExtraArgs={ + 'ContentType': 'application/octet-stream', + 'ContentEncoding': 'gzip', + 'ACL': self.storage.default_acl, + } + ) + def test_storage_save_gzip(self): """ Test saving a file with gzip enabled. @@ -148,6 +128,34 @@ def test_storage_save_gzip(self): zfile = gzip.GzipFile(mode='rb', fileobj=content) self.assertEqual(zfile.read(), b"I should be gzip'd") + def test_storage_save_gzip_twice(self): + """ + Test saving the same file content twice with gzip enabled. + """ + # Given + self.storage.gzip = True + name = 'test_storage_save.css' + content = ContentFile("I should be gzip'd") + + # When + self.storage.save(name, content) + self.storage.save('test_storage_save_2.css', content) + + # Then + obj = self.storage.bucket.Object.return_value + obj.upload_fileobj.assert_called_with( + mock.ANY, + ExtraArgs={ + 'ContentType': 'text/css', + 'ContentEncoding': 'gzip', + 'ACL': self.storage.default_acl, + } + ) + args, kwargs = obj.upload_fileobj.call_args + content = args[0] + zfile = gzip.GzipFile(mode='rb', fileobj=content) + self.assertEqual(zfile.read(), b"I should be gzip'd") + def test_compress_content_len(self): """ Test that file returned by _compress_content() is readable. @@ -161,7 +169,7 @@ def test_storage_open_write(self): """ Test opening a file in write mode """ - name = 'test_open_for_writing.txt' + name = 'test_open_for_writïng.txt' content = 'new content' # Set the encryption flag used for multipart uploads @@ -196,8 +204,8 @@ def test_storage_open_write(self): def test_auto_creating_bucket(self): self.storage.auto_create_bucket = True Bucket = mock.MagicMock() - self.storage._connection.Bucket.return_value = Bucket - self.storage._connection.meta.client.meta.region_name = 'sa-east-1' + self.storage._connections.connection.Bucket.return_value = Bucket + self.storage._connections.connection.meta.client.meta.region_name = 'sa-east-1' Bucket.meta.client.head_bucket.side_effect = ClientError({'Error': {}, 'ResponseMetadata': {'HTTPStatusCode': 404}}, @@ -211,17 +219,27 @@ def test_auto_creating_bucket(self): ) def test_storage_exists(self): - obj = self.storage.bucket.Object.return_value self.assertTrue(self.storage.exists("file.txt")) - self.storage.bucket.Object.assert_called_with("file.txt") - obj.load.assert_called_with() + self.storage.connection.meta.client.head_object.assert_called_with( + Bucket=self.storage.bucket_name, + Key="file.txt", + ) def test_storage_exists_false(self): - obj = self.storage.bucket.Object.return_value - obj.load.side_effect = ClientError({'Error': {'Code': 123, 'Message': 'Fake'}}, 'load') + self.storage.connection.meta.client.head_object.side_effect = ClientError( + {'Error': {'Code': '404', 'Message': 'Not Found'}}, + 'HeadObject', + ) self.assertFalse(self.storage.exists("file.txt")) - self.storage.bucket.Object.assert_called_with("file.txt") - obj.load.assert_called_with() + self.storage.connection.meta.client.head_object.assert_called_with( + Bucket=self.storage.bucket_name, + Key='file.txt', + ) + + def test_storage_exists_doesnt_create_bucket(self): + with mock.patch.object(self.storage, '_get_or_create_bucket') as method: + self.storage.exists('file.txt') + method.assert_not_called() def test_storage_delete(self): self.storage.delete("path/to/file.txt") @@ -336,9 +354,36 @@ def test_generated_url_is_encoded(self): "/whacky%20%26%20filename.mp4") self.assertFalse(self.storage.bucket.meta.client.generate_presigned_url.called) + def test_special_characters(self): + self.storage.custom_domain = "mock.cloudfront.net" + + name = "ãlöhâ.jpg" + content = ContentFile('new content') + self.storage.save(name, content) + self.storage.bucket.Object.assert_called_once_with(name) + + url = self.storage.url(name) + parsed_url = urlparse.urlparse(url) + self.assertEqual(parsed_url.path, "/%C3%A3l%C3%B6h%C3%A2.jpg") + def test_strip_signing_parameters(self): expected = 'http://bucket.s3-aws-region.amazonaws.com/foo/bar' self.assertEqual(self.storage._strip_signing_parameters( '%s?X-Amz-Date=12345678&X-Amz-Signature=Signature' % expected), expected) self.assertEqual(self.storage._strip_signing_parameters( '%s?expires=12345678&signature=Signature' % expected), expected) + + @skipIf(threading is None, 'Test requires threading') + def test_connection_threading(self): + connections = [] + + def thread_storage_connection(): + connections.append(self.storage.connection) + + for x in range(2): + t = threading.Thread(target=thread_storage_connection) + t.start() + t.join() + + # Connection for each thread needs to be unique + self.assertIsNot(connections[0], connections[1]) diff --git a/tests/test_sftp.py b/tests/test_sftp.py index e31ef445e..754e98703 100644 --- a/tests/test_sftp.py +++ b/tests/test_sftp.py @@ -1,13 +1,17 @@ +import os import stat from datetime import datetime + +from django.core.files.base import File +from django.test import TestCase +from django.utils.six import BytesIO + +from storages.backends import sftpstorage + try: from unittest.mock import patch, MagicMock except ImportError: # Python 3.2 and below from mock import patch, MagicMock -from django.test import TestCase -from django.core.files.base import File -from django.utils.six import BytesIO -from storages.backends import sftpstorage class SFTPStorageTest(TestCase): @@ -17,6 +21,19 @@ def setUp(self): def test_init(self): pass + @patch('paramiko.SSHClient') + def test_no_known_hosts_file(self, mock_ssh): + self.storage._known_host_file = "not_existed_file" + self.storage._connect() + self.assertEqual('foo', mock_ssh.return_value.connect.call_args[0][0]) + + @patch.object(os.path, "expanduser", return_value="/path/to/known_hosts") + @patch.object(os.path, "exists", return_value=True) + @patch('paramiko.SSHClient') + def test_error_when_known_hosts_file_not_defined(self, mock_ssh, *a): + self.storage._connect() + self.storage._ssh.load_host_keys.assert_called_once_with("/path/to/known_hosts") + @patch('paramiko.SSHClient') def test_connect(self, mock_ssh): self.storage._connect() @@ -28,7 +45,7 @@ def test_open(self): @patch('storages.backends.sftpstorage.SFTPStorage.sftp') def test_read(self, mock_sftp): - file_ = self.storage._read('foo') + self.storage._read('foo') self.assertTrue(mock_sftp.open.called) @patch('storages.backends.sftpstorage.SFTPStorage.sftp') diff --git a/tests/test_utils.py b/tests/test_utils.py index 2e804b25e..eb309acd9 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,9 @@ -from django.test import TestCase +import datetime + from django.conf import settings from django.core.exceptions import ImproperlyConfigured +from django.test import TestCase + from storages import utils @@ -14,3 +17,101 @@ def test_setting_unfound(self): self.assertEqual(utils.setting('FOO', 'bar'), 'bar') with self.assertRaises(ImproperlyConfigured): utils.setting('FOO', strict=True) + + +class CleanNameTests(TestCase): + def test_clean_name(self): + """ + Test the base case of clean_name + """ + path = utils.clean_name("path/to/somewhere") + self.assertEqual(path, "path/to/somewhere") + + def test_clean_name_normalize(self): + """ + Test the normalization of clean_name + """ + path = utils.clean_name("path/to/../somewhere") + self.assertEqual(path, "path/somewhere") + + def test_clean_name_trailing_slash(self): + """ + Test the clean_name when the path has a trailing slash + """ + path = utils.clean_name("path/to/somewhere/") + self.assertEqual(path, "path/to/somewhere/") + + def test_clean_name_windows(self): + """ + Test the clean_name when the path has a trailing slash + """ + path = utils.clean_name("path\\to\\somewhere") + self.assertEqual(path, "path/to/somewhere") + + +class SafeJoinTest(TestCase): + def test_normal(self): + path = utils.safe_join("", "path/to/somewhere", "other", "path/to/somewhere") + self.assertEqual(path, "path/to/somewhere/other/path/to/somewhere") + + def test_with_dot(self): + path = utils.safe_join("", "path/./somewhere/../other", "..", + ".", "to/./somewhere") + self.assertEqual(path, "path/to/somewhere") + + def test_with_only_dot(self): + path = utils.safe_join("", ".") + self.assertEqual(path, "") + + def test_base_url(self): + path = utils.safe_join("base_url", "path/to/somewhere") + self.assertEqual(path, "base_url/path/to/somewhere") + + def test_base_url_with_slash(self): + path = utils.safe_join("base_url/", "path/to/somewhere") + self.assertEqual(path, "base_url/path/to/somewhere") + + def test_suspicious_operation(self): + with self.assertRaises(ValueError): + utils.safe_join("base", "../../../../../../../etc/passwd") + with self.assertRaises(ValueError): + utils.safe_join("base", "/etc/passwd") + + def test_trailing_slash(self): + """ + Test safe_join with paths that end with a trailing slash. + """ + path = utils.safe_join("base_url/", "path/to/somewhere/") + self.assertEqual(path, "base_url/path/to/somewhere/") + + def test_trailing_slash_multi(self): + """ + Test safe_join with multiple paths that end with a trailing slash. + """ + path = utils.safe_join("base_url/", "path/to/", "somewhere/") + self.assertEqual(path, "base_url/path/to/somewhere/") + + def test_datetime_isoformat(self): + dt = datetime.datetime(2017, 5, 19, 14, 45, 37, 123456) + path = utils.safe_join('base_url', dt.isoformat()) + self.assertEqual(path, 'base_url/2017-05-19T14:45:37.123456') + + def test_join_empty_string(self): + path = utils.safe_join('base_url', '') + self.assertEqual(path, 'base_url/') + + def test_with_base_url_and_dot(self): + path = utils.safe_join('base_url', '.') + self.assertEqual(path, 'base_url/') + + def test_with_base_url_and_dot_and_path_and_slash(self): + path = utils.safe_join('base_url', '.', 'path/to/', '.') + self.assertEqual(path, 'base_url/path/to/') + + def test_join_nothing(self): + path = utils.safe_join('') + self.assertEqual(path, '') + + def test_with_base_url_join_nothing(self): + path = utils.safe_join('base_url') + self.assertEqual(path, 'base_url/') diff --git a/tox.ini b/tox.ini index d181cd7e5..05dc3d145 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,9 @@ [tox] envlist = - {py27,py33,py34,py35}-django18, - {py27,py34,py35}-django19 + lint + {py27,py33,py34,py35}-django18 {py27,py34,py35}-django110 + {py27,py34,py35,py36}-django111 [testenv] @@ -12,11 +13,21 @@ setenv = DJANGO_SETTINGS_MODULE=tests.settings deps = django18: Django>=1.8, <1.9 - django19: Django>=1.9, <1.10 django110: Django>=1.10, <1.11 - py27: mock==1.0.1 - boto>=2.32.0 - pytest-cov>=2.2.1 + django111: Django>=1.11, <2.0 + py27: mock boto3>=1.2.3 - dropbox>=3.24 + boto>=2.32.0 + dropbox>=8.0.0 + google-cloud-storage>=0.22.0 paramiko + pytest-cov>=2.2.1 + + +[testenv:lint] +deps = + flake8 + isort +commands = + flake8 + isort --recursive --check-only --diff storages/ tests/