Merge branch 'develop' of https://github.com/CenterForOpenScience/wat…

…erbutler into drop-box-slash-issue * 'develop' of https://github.com/CenterForOpenScience/waterbutler: (77 commits) final cleanups for onedrive make provider readonly start cleaning up and reorganizing onedrive implement Microsoft OneDrive provider using dropbox as base added onedrive provider -- copied from Dropbox don't assume file metadata has a modified/created date add mypy type annotations add modified/created dates to file metadata expand docstrings add ruby serialization workaround to download clean up commit_sha vs. branch_name handling add tests for revisions and uninit-ed repos add artificial test for missing folder remove unused and obsolete code rewrite test suite for provider changes and coverage update some more docstrings on the provider GL provider is read-only: folder creation is not allowed add workaround for non-existent directories not being reported in GL document workarounds in download & _fetch_file_contents remove unneeded error wrapping from metadata ... # Conflicts: # waterbutler/server/api/v1/provider/movecopy.py
CenterForOpenScience · Nov 29, 2017 · e6edbe8 · e6edbe8
2 parents 420c66d + 26bf209
commit e6edbe8
Show file tree

Hide file tree

Showing 43 changed files with 4,107 additions and 93 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -2,6 +2,22 @@
 ChangeLog
 *********
 
+0.35.0 (2017-11-13)
+===================
+- Feature: Allow copying from public resources with the OSF provider.  WaterButler had been
+requiring write permissions on the source resource for both moves and copies, but copy only needs
+read.  Update the v1 API to distinguish between the two types of requests.
+- Docs: Document supported query parameters in the v1 API.
+- Code: Improve test coverage for osfstorage and figshare.
+- Code: Cleanups for Box, Google Drive, and GitHub providers.
+- Code: Don't include test directories in module search paths.
+- Code: Don't let query parameters override the HTTP verb in v1.
+
+0.34.1 (2017-10-18)
+===================
+- Fix: Don't crash when a file on Google Drive is missing an md5 in its metadata.  This occurs
+for non-exportable files like Google Maps, Google Forms, etc.
+
 0.34.0 (2017-09-29)
 ===================
 - ANNOUNCEMENT!  Sadly, the WaterButler v0 API is now *undeprecated*.  We've discovered that the
@@ -17,7 +33,7 @@ returned metadata for folders (and document it).
 - Fix: Release Box responses on error.
 - Code: Update the Postman test suite to include CRUD and move tests.
 - Code: Start testing with python-3.6 on Travis.
-- Code: Improve test coverage for all providers except osftorage and figshare (coming soon!).
+- Code: Improve test coverage for all providers except osfstorage and figshare (coming soon!).
 - Code: Teach WaterButler to listen for a SIGTERM signal and exit immediately upon receiving it.
 This bypasses the 10 second wait for shutdown when running it in Docker.
 - Code: Fix sphinx syntax errors in the WaterButler docs.

diff --git a/docs/provider.gitlab.rst b/docs/provider.gitlab.rst
@@ -0,0 +1,8 @@
+GitLab Provider
+===============
+
+.. autoclass:: waterbutler.providers.gitlab.provider.GitLabProvider
+    :members:
+    :undoc-members:
+    :show-inheritance:
+    :inherited-members:
diff --git a/docs/provider.onedrive.rst b/docs/provider.onedrive.rst
@@ -0,0 +1,8 @@
+OneDrive Provider
+=================
+
+.. autoclass:: waterbutler.providers.onedrive.provider.OneDriveProvider
+    :members:
+    :undoc-members:
+    :show-inheritance:
+    :inherited-members:
diff --git a/docs/providers.rst b/docs/providers.rst
@@ -11,7 +11,9 @@ Providers
    provider.figshare
    provider.filesystem
    provider.github
+   provider.gitlab
    provider.googledrive
+   provider.onedrive
    provider.osfstorage
    provider.owncloud
    provider.cloudfiles
diff --git a/docs/waterbutler.providers.rst b/docs/waterbutler.providers.rst
@@ -12,6 +12,7 @@ Subpackages
     provider.figshare
     provider.filesystem
     provider.github
+    provider.gitlab
     provider.googledrive
     provider.osfstorage
     provider.s3

diff --git a/setup.py b/setup.py
@@ -45,13 +45,15 @@ def parse_requirements(requirements):
             'figshare = waterbutler.providers.figshare:FigshareProvider',
             'filesystem = waterbutler.providers.filesystem:FileSystemProvider',
             'github = waterbutler.providers.github:GitHubProvider',
+            'gitlab = waterbutler.providers.gitlab:GitLabProvider',
             'bitbucket = waterbutler.providers.bitbucket:BitbucketProvider',
             'osfstorage = waterbutler.providers.osfstorage:OSFStorageProvider',
             'owncloud = waterbutler.providers.owncloud:OwnCloudProvider',
             's3 = waterbutler.providers.s3:S3Provider',
             'dataverse = waterbutler.providers.dataverse:DataverseProvider',
             'box = waterbutler.providers.box:BoxProvider',
             'googledrive = waterbutler.providers.googledrive:GoogleDriveProvider',
+            'onedrive = waterbutler.providers.onedrive:OneDriveProvider',
         ],
         'waterbutler.providers.tasks': [
             'osfstorage_parity = waterbutler.providers.osfstorage.tasks.parity',

diff --git a/tests/auth/__init__.py b/tests/auth/__init__.py
diff --git a/tests/auth/osf/__init__.py b/tests/auth/osf/__init__.py
diff --git a/tests/auth/osf/test_handler.py b/tests/auth/osf/test_handler.py
@@ -7,8 +7,10 @@
 from tests.server.api.v1.utils import ServerTestCase
 
 from waterbutler.auth.osf import settings
+from waterbutler.core.auth import AuthType
 from waterbutler.auth.osf.handler import OsfAuthHandler
-from waterbutler.core.exceptions import UnsupportedHTTPMethodError
+from waterbutler.core.exceptions import (UnsupportedHTTPMethodError,
+                                            UnsupportedActionError)
 
 
 class TestOsfAuthHandler(ServerTestCase):
@@ -33,16 +35,56 @@ def tearDown(self):
     @tornado.testing.gen_test
     async def test_supported_and_unsupported_methods(self):
 
-        supported_methods = ['put', 'post', 'get', 'head', 'delete']
-        unsupported_methods = ['trace', 'connect', 'patch', 'ma1f0rmed']
+        supported_methods = ['put', 'get', 'head', 'delete']
+        post_actions = ['copy', 'rename', 'move']
+        unsupported_actions = ['ma1f0rmed', 'put', 'get', 'head', 'delete']
+        unsupported_methods = ['post', 'trace', 'connect', 'patch', 'ma1f0rmed']
+        resource = 'test'
+        provider = 'test'
 
         assert all(method in self.handler.ACTION_MAP.keys() for method in supported_methods)
 
+        for auth_type in AuthType:
+            for action in post_actions:
+                self.request.method = 'post'
+                await self.handler.get(resource, provider,
+                                            self.request, action=action, auth_type=auth_type)
+
         for method in supported_methods:
             self.request.method = method
-            await self.handler.get("test", "test", self.request)
+            await self.handler.get(resource, provider, self.request)
 
         for method in unsupported_methods:
             self.request.method = method
             with pytest.raises(UnsupportedHTTPMethodError):
-                await self.handler.get("test", "test", self.request)
+                await self.handler.get(resource, provider, self.request)
+
+        for action in unsupported_actions:
+            self.request.method = 'post'
+            with pytest.raises(UnsupportedActionError):
+                await self.handler.get(resource, provider, self.request, action=action)
+
+    @tornado.testing.gen_test
+    async def test_permissions_post_copy_source_destination(self):
+
+        resource = 'test'
+        provider = 'test'
+        action = 'copy'
+        self.request.method = 'post'
+
+        self.handler.build_payload = mock.Mock()
+
+        for auth_type in AuthType:
+            await self.handler.get(resource, provider, self.request, action=action, auth_type=auth_type)
+            if auth_type is AuthType.SOURCE:
+                self.handler.build_payload.assert_called_with({
+                    'nid': resource,
+                    'provider': provider,
+                    'action': 'download'
+                }, cookie=None, view_only=None)
+            else:
+                self.handler.build_payload.assert_called_with({
+                    'nid': resource,
+                    'provider': provider,
+                    'action': 'upload'
+                }, cookie=None, view_only=None)
diff --git a/tests/providers/github/test_provider.py b/tests/providers/github/test_provider.py
@@ -1396,14 +1396,6 @@ async def test__fetch_tree_truncated_error(self, provider):
 
 class TestUtilities:
 
-    def test_is_sha(self, provider, root_provider_fixtures):
-        sha = root_provider_fixtures['upload_response']['content']['sha']
-        fake_sha = 'not a real sha'
-
-        assert provider.is_sha(sha) is True
-        assert provider.is_sha(fake_sha) is False
-        assert provider.is_sha([i for i in range(40)]) is False
-
     def test__path_exists_in_tree(self, provider, root_provider_fixtures):
         _ids = [('master', '')]
 

diff --git a/tests/providers/gitlab/__init__.py b/tests/providers/gitlab/__init__.py
diff --git a/tests/providers/gitlab/fixtures.py b/tests/providers/gitlab/fixtures.py
@@ -0,0 +1,172 @@
+import os
+import json
+import pytest
+
+
+@pytest.fixture
+def simple_tree():
+    return [
+            {
+                "id": "a1e8f8d745cc87e3a9248358d9352bb7f9a0aeba",
+                "name": "folder1",
+                "type": "tree",
+                "path": "folder1",
+                "mode": "040000"
+            },
+            {
+                "id":"a1e8f8d745cc87e3a9248358d9352bb7f9a0aeba",
+                "name": "file1",
+                "type": "blob",
+                "path": "folder1/file1",
+                "mode": "040000"
+            }
+        ]
+
+@pytest.fixture
+def gitlab_example_sub_project_tree():
+    return [
+            {
+                "id": "a1e8f8d745cc87e3a9248358d9352bb7f9a0aeba",
+                "name": ".gitkeep",
+                "type": "blob",
+                "path": "files/html/.gitkeep",
+                "mode": "040000"
+            },
+            {
+                "id": "a1e8f8d745cc87e3a9248358d9352bb7f9a0aeba",
+                "name": ".gitkeep",
+                "type": "tree",
+                "path": "files/html/static",
+                "mode": "040000"
+            }
+        ]
+
+@pytest.fixture
+def subfolder_tree():
+    return [
+            {
+                "id": "a1e8f8d745cc87e3a9248358d9352bb7f9a0aeba",
+                "name": "html",
+                "type": "tree",
+                "path": "files/html",
+                "mode": "040000"
+            },
+            {
+                "id": "4535904260b1082e14f867f7a24fd8c21495bde3",
+                "name": "images",
+                "type": "tree",
+                "path": "files/images",
+                "mode": "040000"
+            },
+            {
+                "id": "31405c5ddef582c5a9b7a85230413ff90e2fe720",
+                "name": "js",
+                "type": "tree",
+                "path": "files/js",
+                "mode": "040000"
+            },
+            {
+                "id": "cc71111cfad871212dc99572599a568bfe1e7e00",
+                "name": "lfs",
+                "type": "tree",
+                "path": "files/lfs",
+                "mode": "040000"
+            },
+            {
+                "id": "fd581c619bf59cfdfa9c8282377bb09c2f897520",
+                "name": "markdown",
+                "type": "tree",
+                "path": "files/markdown",
+                "mode": "040000"
+            },
+            {
+                "id": "23ea4d11a4bdd960ee5320c5cb65b5b3fdbc60db",
+                "name": "ruby",
+                "type": "tree",
+                "path": "files/ruby",
+                "mode": "040000"
+            },
+            {
+                "id": "7d70e02340bac451f281cecf0a980907974bd8be",
+                "name": "whitespace",
+                "type": "blob",
+                "path": "files/whitespace",
+                "mode": "100644"
+            }
+        ]
+
+@pytest.fixture
+def simple_file_metadata():
+    return {
+        'file_name': 'file',
+        'blob_id': 'abc123',
+        'commit_id': 'xxxyyy',
+        'file_path': '/folder1/folder2/file',
+        'size': 123
+    }
+
+@pytest.fixture
+def revisions_for_file():
+    return [
+        {
+            "id": "931aece9275c0d084dfa7f6e0b3b2bb250e4b089",
+            "short_id": "931aece9",
+            "title": "deepi",
+            "created_at": "2017-07-24T16:02:17.000-04:00",
+            "parent_ids": [
+                "d9901e83728f5aa034ef1c6193be89f7b644729f"
+            ],
+            "message": "deepi\n",
+            "author_name": "Fitz Elliott",
+            "author_email": "fitz@cos.io",
+            "authored_date": "2017-07-24T16:02:17.000-04:00",
+            "committer_name": "Fitz Elliott",
+            "committer_email": "fitz@cos.io",
+            "committed_date": "2017-07-24T16:02:17.000-04:00"
+        },
+        {
+            "id": "b993ab399b22986a298efa509ca3a6bd605a62c4",
+            "short_id": "b993ab39",
+            "title": "morp",
+            "created_at": "2017-07-24T15:57:24.000-04:00",
+            "parent_ids": [
+                "3f6f31fcbf0f70e8a59298fdccfd15f7c5f3cb2e"
+            ],
+            "message": "morp\n",
+            "author_name": "Fitz Elliott",
+            "author_email": "fitz@cos.io",
+            "authored_date": "2017-07-24T15:57:24.000-04:00",
+            "committer_name": "Fitz Elliott",
+            "committer_email": "fitz@cos.io",
+            "committed_date": "2017-07-24T15:57:24.000-04:00"
+        },
+        {
+            "id": "d5aac723529e81761c95c71315ac2c747ed50b96",
+            "short_id": "d5aac723",
+            "title": "save WIP",
+            "created_at": "2016-11-30T13:30:23.000-05:00",
+            "parent_ids": [
+                "ed7af40927525e92ef5b55720e6af32be1dba4ba"
+            ],
+            "message": "save WIP\n",
+            "author_name": "Fitz Elliott",
+            "author_email": "fitz@cos.io",
+            "authored_date": "2016-11-30T13:30:23.000-05:00",
+            "committer_name": "Fitz Elliott",
+            "committer_email": "fitz@cos.io",
+            "committed_date": "2016-11-30T13:30:23.000-05:00"
+        }
+    ]
+
+@pytest.fixture
+def weird_ruby_response():
+    """See: https://gitlab.com/gitlab-org/gitlab-ce/issues/31790"""
+    return ('{:file_name=>"file", :file_path=>"file", :size=>5, '
+            ':encoding=>"base64", :content=>"cm9sZgo=", :ref=>"master", '
+            ':blob_id=>"cf37e8f1e80b5747301df4e1557036b37294a716", '
+            ':commit_id=>"8c7b653eab7191dde3aff9e33ddf309c3d1f440f", '
+            ':last_commit_id=>"8c7b653eab7191dde3aff9e33ddf309c3d1f440f"}')
+
+# fixtures for testing file revision metadata
+with open(os.path.join(os.path.dirname(__file__), 'fixtures/default-branch.json'), 'r') as fp:
+    default_branches = json.load(fp)