From 3d133186d4ec0095f103dc2a9fdcf0ad7881800b Mon Sep 17 00:00:00 2001 From: "Mr. Outis" Date: Wed, 18 Dec 2019 00:48:10 -0600 Subject: [PATCH 1/2] summon: ugly hack --- dvc/api.py | 37 +++++++++++++++++++++++++++++++++++++ tests/func/test_api.py | 23 +++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/dvc/api.py b/dvc/api.py index bf976dcf01..ac64b3dcaa 100644 --- a/dvc/api.py +++ b/dvc/api.py @@ -1,4 +1,5 @@ import os +import importlib.util from contextlib import contextmanager try: @@ -6,6 +7,8 @@ except ImportError: from contextlib import GeneratorContextManager as GCM +import ruamel.yaml + from dvc.utils.compat import urlparse from dvc.repo import Repo from dvc.external_repo import external_repo @@ -69,3 +72,37 @@ def _make_repo(repo_url, rev=None): else: with external_repo(url=repo_url, rev=rev) as repo: yield repo + + +def summon(name, repo=None, rev=None): + # 1. Read artifacts.yaml + # 2. Pull dependencies + # 3. Get the call and parameters + # 4. Invoke the call with the given parameters + # 5. Return the result + + with _make_repo(repo, rev=rev) as _repo: + artifacts_path = os.path.join(_repo.root_dir, "artifacts.yaml") + + with _repo.open(artifacts_path, mode="r") as fd: + artifacts = ruamel.yaml.load(fd.read()).get("artifacts") + + artifact = next(x for x in artifacts if x.get("name") == name) + + spec = importlib.util.spec_from_file_location( + artifact.get("call"), + os.path.join(_repo.root_dir, artifact.get("file")), + ) + + module = importlib.util.module_from_spec(spec) + + # ugly af, don't use exec / global, pls + call = 'global result; result = {method}({params})'.format( + method=artifact.get("call"), + params=", ".join(k + "=" + v for k, v in artifact.get("params").items()) + ) + + spec.loader.exec_module(module) + exec(call) + global result + return result diff --git a/tests/func/test_api.py b/tests/func/test_api.py index 54623c080d..7be749e02c 100644 --- a/tests/func/test_api.py +++ b/tests/func/test_api.py @@ -3,6 +3,7 @@ import os import shutil +import ruamel.yaml import pytest from dvc import api @@ -126,3 +127,25 @@ def test_open_not_cached(dvc): os.remove(metric_file) with pytest.raises(FileMissingError): api.read(metric_file) + + +def test_summon(tmp_dir, erepo_dir, dvc): + artifacts_yaml = ruamel.yaml.dump({ + "artifacts": [{ + "name": "sum", + "description": "The sum of 1 + 2", + "call": "module.sum", + "file": "module.py", + "params": {"x": "1", "y": "2"}, + "deps": ["foo"], + }] + }) + + erepo_dir.gen("module.py", "def sum(x, y): return x + y") + erepo_dir.gen("artifacts.yaml", artifacts_yaml) + erepo_dir.scm.add(["module.py", "artifacts.yaml"]) + erepo_dir.scm.commit("Add module.py and artifacts.yaml") + + repo_url = "file://{}".format(erepo_dir) + + assert api.summon("sum", repo=repo_url) == 3 From 62b83630618c7b2d5f74274deca443290291d04d Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Wed, 18 Dec 2019 06:47:23 +0000 Subject: [PATCH 2/2] Restyled by black --- dvc/api.py | 6 ++++-- tests/func/test_api.py | 24 ++++++++++++++---------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/dvc/api.py b/dvc/api.py index ac64b3dcaa..f281971cd4 100644 --- a/dvc/api.py +++ b/dvc/api.py @@ -97,9 +97,11 @@ def summon(name, repo=None, rev=None): module = importlib.util.module_from_spec(spec) # ugly af, don't use exec / global, pls - call = 'global result; result = {method}({params})'.format( + call = "global result; result = {method}({params})".format( method=artifact.get("call"), - params=", ".join(k + "=" + v for k, v in artifact.get("params").items()) + params=", ".join( + k + "=" + v for k, v in artifact.get("params").items() + ), ) spec.loader.exec_module(module) diff --git a/tests/func/test_api.py b/tests/func/test_api.py index 7be749e02c..029a5e114a 100644 --- a/tests/func/test_api.py +++ b/tests/func/test_api.py @@ -130,16 +130,20 @@ def test_open_not_cached(dvc): def test_summon(tmp_dir, erepo_dir, dvc): - artifacts_yaml = ruamel.yaml.dump({ - "artifacts": [{ - "name": "sum", - "description": "The sum of 1 + 2", - "call": "module.sum", - "file": "module.py", - "params": {"x": "1", "y": "2"}, - "deps": ["foo"], - }] - }) + artifacts_yaml = ruamel.yaml.dump( + { + "artifacts": [ + { + "name": "sum", + "description": "The sum of 1 + 2", + "call": "module.sum", + "file": "module.py", + "params": {"x": "1", "y": "2"}, + "deps": ["foo"], + } + ] + } + ) erepo_dir.gen("module.py", "def sum(x, y): return x + y") erepo_dir.gen("artifacts.yaml", artifacts_yaml)