init git resolver

cwlkernel/git/CWLGitResolver.py

@@ -0,0 +1,50 @@
+import base64
+import json
+import os.path
+from io import BytesIO
+from pathlib import Path
+from typing import List, Tuple, Dict
+from urllib.parse import urlparse
+
+import requests
+import ruamel.yaml as yaml
+
+
+class CWLGitResolver:
+    """CWLGitResolver fetches the required cwl files from a remote git url"""
+
+    def __init__(self, local_directory: Path, github_url: str):
+        self._local_directory = local_directory
+        self._local_directory.mkdir(exist_ok=True)
+        github_path = urlparse(github_url).path.split('/')
+        self._git_owner = github_path[1]
+        self._git_repo = github_path[2]
+        self._git_branch = github_path[4]
+        self._git_path = '/'.join(github_path[5:])
+
+    def resolve(self) -> List[str]:
+        workflow_files = set()
+        root_path = self._git_path[:self._git_path.rfind('/')]
+        search_stack = [self._git_path]
+        while len(search_stack) > 0:
+            current_path = search_stack.pop()
+            if current_path not in workflow_files:
+                workflow_filename, workflow = self._resolve_file(current_path)
+                workflow_files.add(workflow_filename)
+                if 'steps' in workflow:
+                    for step in workflow['steps']:
+                        if isinstance(workflow['steps'][step]['run'], str):
+                            search_stack.append('/'.join([root_path, workflow['steps'][step]['run']]))
+        return list(workflow_files)
+
+    def _resolve_file(self, path: str) -> Tuple[str, Dict]:
+        url = f"https://api.github.com/repos/" \
+              f"{self._git_owner}/{self._git_repo}/contents/{path}?ref={self._git_branch}"
+        github_response = requests.get(url)
+        github_response = json.loads(github_response.text)
+        workflow = yaml.load(BytesIO(base64.b64decode(github_response['content'])), yaml.Loader)
+        workflow_filename = os.path.join(str(self._local_directory), path)
+        Path(os.path.dirname(workflow_filename)).mkdir(exist_ok=True, parents=True)
+        with open(workflow_filename, 'w') as f:
+            yaml.dump(workflow, f)
+        return workflow_filename, workflow

requirements.txt

@@ -20,4 +20,5 @@ traitlets==4.3.3
 uritools==3.0.0
 PyYAML==5.3.1
 pandas==1.0.4
-notebook==6.0.3
+notebook==6.0.3
+requests==2.23.0

tests/git/test_resolver.py

@@ -0,0 +1,25 @@
+import os
+import tempfile
+import unittest
+from pathlib import Path
+
+from git.CWLGitResolver import CWLGitResolver
+
+
+class GitResolverTest(unittest.TestCase):
+    def test_resolve(self):
+        git_dir = Path(tempfile.mkdtemp())
+        git_resolver = CWLGitResolver(
+            git_dir,
+            "https://github.com/giannisdoukas/CWLJNIKernel/blob/dev/tests/cwl/3stepWorkflow.cwl")
+        self.assertListEqual(
+            sorted([
+                os.path.join(git_dir, f) for f in
+                ['tests/cwl/3stepWorkflow.cwl', 'tests/cwl/head.cwl', 'tests/cwl/grep.cwl']
+            ]),
+            sorted(git_resolver.resolve())
+        )
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_CWLKernel.py

@@ -29,7 +29,7 @@ def get_kernel(self) -> CWLKernel:
 
     def setUp(self) -> None:
         import tempfile
-        WorkflowRepository(Path(tempfile.gettempdir()))
+        WorkflowRepository(Path(tempfile.mkdtemp()))
         WorkflowRepository.get_instance().delete()
 
     @classmethod