Merge branch 'develop' into cloudfiles

CHANGELOG

@@ -2,6 +2,11 @@
 ChangeLog
 *********
 
+0.34.1 (2017-10-18)
+===================
+- Fix: Don't crash when a file on Google Drive is missing an md5 in its metadata.  This occurs
+for non-exportable files like Google Maps, Google Forms, etc.
+
 0.34.0 (2017-09-29)
 ===================
 - ANNOUNCEMENT!  Sadly, the WaterButler v0 API is now *undeprecated*.  We've discovered that the

docs/api.rst

@@ -334,3 +334,31 @@ OSF-specific parameter used to identify special "view-only" links that are used
 * **Type**: string
 * **Expected on**: ``GET`` requests for files or folders
 * **Notes**: Only used internally for the Open Science Framework.
+
+
+GitHub Provider Params
+++++++++++++++++++++++
+
+Query parameters specific to the GitHub provider.
+
+
+*commit / branch identification*
+********************************
+
+Not a single parameter, but rather a class of parameters.  WaterButler has used many different parameters to identify the branch or commit a particular file should be found under.  These parameters can be either a commit SHA or a branch name.  These parameters are ``ref``, ``version``, ``branch``, ``sha``, ``revision``.  All will continue to be supported to maintain back-compatibility, but ``ref`` (for SHAs or branch names) and ``branch`` (branch names only) are preferred.
+
+If both a SHA and a branch name are provided in different parameters, the SHA will be take precedence.  If multiple parameters are given with different SHAs, then the order of precedence will be: ``ref``, ``version``, ``sha``, ``revision``, ``branch``.  If multiple parameters are given with different branches, the order of precedence is: ``branch``, ``ref``, ``version``, ``sha``, ``revision``.
+
+* **Type**: str
+* **Expected on**: Any GitHub provider request
+* **Interactions**: None
+
+
+fileSha
+*******
+
+Identifies a specific revision of a file via its SHA.
+
+* **Type**: str
+* **Expected on**: Any GitHub provider request
+* **Interactions**: The ``fileSha`` is always assumed to be a file revision that is an ancestor of the imputed   commit or branch ref.  Providing a ``fileSha`` for a file version that was committed after the imputed ref will result in a 404.

tests/providers/github/fixtures/root_provider.json

@@ -181,6 +181,39 @@
         "git_refs_url":"https://api.github.com/repos/octocat/Hello-World/git/refs{/sha}",
         "events_url":"https://api.github.com/repos/octocat/Hello-World/events"
     },
+    "commit_metadata": {
+        "sha" : "43798985fdb727843a49946b391409086a7021f2",
+        "url" : "https://api.github.com/repos/felliott/wb-testing/git/commits/43798985fdb727843a49946b391409086a7021f2",
+        "tree" : {
+            "sha" : "394164986adbe26c0c7605c6fe627e4fe3b13aa6",
+            "url" : "https://api.github.com/repos/felliott/wb-testing/git/trees/394164986adbe26c0c7605c6fe627e4fe3b13aa6"
+        },
+        "parents" : [
+            {
+                "url" : "https://api.github.com/repos/felliott/wb-testing/git/commits/189ee803576e85487afe4f6fa590e4ff9a791c6b",
+                "sha" : "189ee803576e85487afe4f6fa590e4ff9a791c6b",
+                "html_url" : "https://github.com/felliott/wb-testing/commit/189ee803576e85487afe4f6fa590e4ff9a791c6b"
+            }
+        ],
+        "committer" : {
+            "email" : "3rzwk@osf.io",
+            "name" : "Fitzfork",
+            "date" : "2017-03-14T19:18:51Z"
+        },
+        "author" : {
+            "date" : "2017-03-14T19:18:51Z",
+            "email" : "felliott@fiskur.org",
+            "name" : "Fitz Elliott"
+        },
+        "message" : "File updated on behalf of WaterButler",
+        "html_url" : "https://github.com/felliott/wb-testing/commit/43798985fdb727843a49946b391409086a7021f2",
+        "verification" : {
+            "payload" : null,
+            "verified" : false,
+            "signature" : null,
+            "reason" : "unsigned"
+        }
+    },
     "branch_metadata":{  
         "commit":{  
             "html_url":"https://github.com/octocat/Hello-World/commit/7fd1a60b01f91b314f59955a4e4d4e80d8edf11d",
@@ -555,4 +588,4 @@
         "truncated":false,
         "sha":"ca83e4a08261a54f1c4630fbb1de34d1e48f0c8a"
     }
-}
+}

tests/providers/github/test_provider.py

@@ -122,30 +122,30 @@ class TestValidatePath:
     @pytest.mark.asyncio
     @pytest.mark.aiohttpretty
     async def test_validate_v1_path_file(self, provider, root_provider_fixtures):
-        branch_url = provider.build_repo_url('branches', provider.default_branch)
-        tree_url = provider.build_repo_url(
-            'git', 'trees',
-            root_provider_fixtures['branch_metadata']['commit']['commit']['tree']['sha'],
-            recursive=1
-        )
 
-        aiohttpretty.register_json_uri('GET', branch_url,
-                body=root_provider_fixtures['branch_metadata'])
-        aiohttpretty.register_json_uri(
-            'GET', tree_url, body=root_provider_fixtures['repo_tree_metadata_root']
-        )
+        branch_name = provider.default_branch
+        branch_metadata = root_provider_fixtures['branch_metadata']
+        tree_sha = branch_metadata['commit']['commit']['tree']['sha']
 
-        blob_path = 'file.txt'
+        branch_url = provider.build_repo_url('branches', branch_name)
+        tree_url = provider.build_repo_url('git', 'trees', tree_sha, recursive=1)
 
-        result = await provider.validate_v1_path('/' + blob_path)
+        aiohttpretty.register_json_uri('GET', branch_url, body=branch_metadata)
+        aiohttpretty.register_json_uri('GET', tree_url,
+                                       body=root_provider_fixtures['repo_tree_metadata_root'])
 
-        with pytest.raises(exceptions.NotFoundError) as exc:
-            await provider.validate_v1_path('/' + blob_path + '/')
+        blob_name = 'file.txt'
+        blob_path = '/{}'.format(blob_name)
 
-        expected = GitHubPath('/' + blob_path, _ids=[(provider.default_branch, '')])
+        result = await provider.validate_v1_path(blob_path)
+        expected = GitHubPath(blob_path, _ids=[(branch_name, ''), (branch_name, '')])
+        assert result == expected
+        assert result.identifier[0] == expected.identifier[0]
+
+        with pytest.raises(exceptions.NotFoundError) as exc:
+            await provider.validate_v1_path('/{}/'.format(blob_path))
 
         assert exc.value.code == client.NOT_FOUND
-        assert result == expected
 
     @pytest.mark.asyncio
     @pytest.mark.aiohttpretty
@@ -163,35 +163,31 @@ async def test_validate_v1_path_root(self, provider):
     @pytest.mark.asyncio
     @pytest.mark.aiohttpretty
     async def test_validate_v1_path_folder(self, provider, root_provider_fixtures):
-        branch_url = provider.build_repo_url('branches', provider.default_branch)
-        tree_url = provider.build_repo_url(
-            'git', 'trees',
-            root_provider_fixtures['branch_metadata']['commit']['commit']['tree']['sha'],
-            recursive=1
-        )
 
-        aiohttpretty.register_json_uri(
-            'GET', branch_url, body=root_provider_fixtures['branch_metadata']
-        )
-        aiohttpretty.register_json_uri(
-            'GET', tree_url, body=root_provider_fixtures['repo_tree_metadata_root']
-        )
+        branch_name = provider.default_branch
+        branch_metadata = root_provider_fixtures['branch_metadata']
+        tree_sha = branch_metadata['commit']['commit']['tree']['sha']
 
-        tree_path = 'level1'
+        branch_url = provider.build_repo_url('branches', branch_name)
+        tree_url = provider.build_repo_url('git', 'trees', tree_sha, recursive=1)
 
-        expected = GitHubPath(
-            '/' + tree_path + '/', _ids=[(provider.default_branch, ''),
-            (provider.default_branch, None)]
-        )
+        aiohttpretty.register_json_uri('GET', branch_url, body=branch_metadata)
+        aiohttpretty.register_json_uri('GET', tree_url,
+                                       body=root_provider_fixtures['repo_tree_metadata_root'])
+
+        tree_name = 'level1'
+        tree_path = '/{}/'.format(tree_name)
+        result = await provider.validate_v1_path(tree_path)
+        expected = GitHubPath(tree_path, _ids=[(branch_name, ''), (branch_name, None)])
 
-        result = await provider.validate_v1_path('/' + tree_path + '/')
+        assert result == expected
+        assert result.extra == expected.extra
+        assert result.identifier[0] == expected.identifier[0]
 
         with pytest.raises(exceptions.NotFoundError) as exc:
-            await provider.validate_v1_path('/' + tree_path)
+            await provider.validate_v1_path('/{}'.format(tree_name))
 
         assert exc.value.code == client.NOT_FOUND
-        assert result == expected
-        assert result.extra == expected.extra
 
     @pytest.mark.asyncio
     async def test_reject_multiargs(self, provider):
@@ -1396,14 +1392,6 @@ async def test__fetch_tree_truncated_error(self, provider):
 
 class TestUtilities:
 
-    def test_is_sha(self, provider, root_provider_fixtures):
-        sha = root_provider_fixtures['upload_response']['content']['sha']
-        fake_sha = 'not a real sha'
-
-        assert provider.is_sha(sha) is True
-        assert provider.is_sha(fake_sha) is False
-        assert provider.is_sha([i for i in range(40)]) is False
-
     def test__path_exists_in_tree(self, provider, root_provider_fixtures):
         _ids = [('master', '')]
 
@@ -1521,3 +1509,142 @@ def test__prune_subtrees(self, provider, root_provider_fixtures):
 
         assert len(pruned_tree) == 3  # alpha.txt, gamma.txt, epsilon.txt
         assert len([x for x in pruned_tree if x['type'] == 'tree']) == 0
+
+    @pytest.mark.asyncio
+    @pytest.mark.aiohttpretty
+    @pytest.mark.parametrize('param', [
+        ('branch'),
+        ('ref'),
+        ('version'),
+        ('sha'),
+        ('revision')
+    ])
+    async def test__interpret_query_parameters_branch(self, provider, root_provider_fixtures, param):
+
+        branch_name = 'develop'
+        branch_metadata = root_provider_fixtures['branch_metadata']
+        tree_sha = branch_metadata['commit']['commit']['tree']['sha']
+
+        branch_url = provider.build_repo_url('branches', branch_name)
+        tree_url = provider.build_repo_url('git', 'trees', tree_sha, recursive=1)
+
+        aiohttpretty.register_json_uri('GET', branch_url, body=branch_metadata)
+        aiohttpretty.register_json_uri('GET', tree_url,
+                                       body=root_provider_fixtures['repo_tree_metadata_root'])
+
+        blob_path = '/file.txt'
+        params = {param: branch_name}
+        result = await provider.validate_v1_path(blob_path, **params)
+        expected = GitHubPath(blob_path, _ids=[(branch_name, ''), (branch_name, '')])
+
+        assert result == expected
+        assert result.identifier[0] == expected.identifier[0]
+
+    @pytest.mark.asyncio
+    @pytest.mark.aiohttpretty
+    @pytest.mark.parametrize('param', [
+        ('ref'),
+        ('version'),
+        ('sha'),
+        ('revision')
+    ])
+    async def test__interpret_query_parameters_sha(self, provider, root_provider_fixtures, param):
+
+        commit_sha = '6dcb09b5b57875f334f61aebed695e2e4193db5e'
+        commit_url = provider.build_repo_url('git', 'commits', commit_sha)
+
+        tree_sha = '394164986adbe26c0c7605c6fe627e4fe3b13aa6'
+        tree_url = provider.build_repo_url('git', 'trees', tree_sha, recursive=1)
+
+        aiohttpretty.register_json_uri('GET', commit_url,
+                                       body=root_provider_fixtures['commit_metadata'])
+        aiohttpretty.register_json_uri('GET', tree_url,
+                                       body=root_provider_fixtures['repo_tree_metadata_root'])
+
+        blob_path = '/file.txt'
+        params = {param: commit_sha}
+        result = await provider.validate_v1_path(blob_path, **params)
+        expected = GitHubPath(blob_path, _ids=[(commit_sha, ''), (commit_sha, '')])
+
+        assert result == expected
+        assert result.identifier[0] == expected.identifier[0]
+
+    def test__interpret_query_parameters_sha_priorities(self, provider):
+        params = {
+            'ref': 'b4eecafa9be2f2006ce1b709d6857b07069b4608',
+            'version': '7fd1a60b01f91b314f59955a4e4d4e80d8edf11d',
+            'sha': 'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391',
+            'revision': 'bc1087ebfe8354a684bf9f8b75517784143dde86',
+        }
+
+        ref, ref_type, ref_from = provider._interpret_query_parameters(**params)
+        assert ref_type == 'commit_sha'
+        assert ref == params['ref']
+
+        del params['ref']
+        ref, ref_type, ref_from = provider._interpret_query_parameters(**params)
+        assert ref_type == 'commit_sha'
+        assert ref == params['version']
+
+        del params['version']
+        ref, ref_type, ref_from = provider._interpret_query_parameters(**params)
+        assert ref_type == 'commit_sha'
+        assert ref == params['sha']
+
+        del params['sha']
+        ref, ref_type, ref_from = provider._interpret_query_parameters(**params)
+        assert ref_type == 'commit_sha'
+        assert ref == params['revision']
+
+        del params['revision']
+        ref, ref_type, ref_from = provider._interpret_query_parameters(**params)
+        assert ref_type == 'branch_name'
+        assert ref == provider.default_branch
+
+    def test__interpret_query_parameters_branch_priorities(self, provider):
+        params = {
+            'branch': 'grr',
+            'ref': 'quack',
+            'version': 'woof',
+            'sha': 'moo',
+            'revision': 'meow',
+        }
+
+        ref, ref_type, ref_from = provider._interpret_query_parameters(**params)
+        assert ref_type == 'branch_name'
+        assert ref == params['branch']
+
+        del params['branch']
+        ref, ref_type, ref_from = provider._interpret_query_parameters(**params)
+        assert ref_type == 'branch_name'
+        assert ref == params['ref']
+
+        del params['ref']
+        ref, ref_type, ref_from = provider._interpret_query_parameters(**params)
+        assert ref_type == 'branch_name'
+        assert ref == params['version']
+
+        del params['version']
+        ref, ref_type, ref_from = provider._interpret_query_parameters(**params)
+        assert ref_type == 'branch_name'
+        assert ref == params['sha']
+
+        del params['sha']
+        ref, ref_type, ref_from = provider._interpret_query_parameters(**params)
+        assert ref_type == 'branch_name'
+        assert ref == params['revision']
+
+        del params['revision']
+        ref, ref_type, ref_from = provider._interpret_query_parameters(**params)
+        assert ref_type == 'branch_name'
+        assert ref == provider.default_branch
+
+    def test__interpret_query_parameters_sha_vs_branch_priority(self, provider):
+        params = {
+            'ref': 'quack',
+            'version': 'ca39bcbf849231525ce9e775935fcb18ed477b5a',
+        }
+
+        ref, ref_type, ref_from = provider._interpret_query_parameters(**params)
+        assert ref_type == 'commit_sha'
+        assert ref == params['version']

waterbutler/__init__.py

@@ -1,2 +1,2 @@
-__version__ = '0.34.0'
+__version__ = '0.34.1'
 __import__('pkg_resources').declare_namespace(__name__)