Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix project bugs and simplify api #226

Merged
merged 4 commits into from
Mar 24, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 4 additions & 0 deletions mlrun/artifacts/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,10 @@ def upload(self, data_stores):
if '://' in target:
target = mktemp()
to_upload = True
else:
dir = os.path.dirname(target)
if dir:
os.makedirs(dir, exist_ok=True)

saving_func(target, **self._kw)
if to_upload:
Expand Down
3 changes: 2 additions & 1 deletion mlrun/projects/project.py
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,8 @@ def functions(self) -> list:
for name, f in self._function_defs.items():
if hasattr(f, 'to_dict'):
spec = f.to_dict(strip=True)
if f.spec.build.source.startswith(self._source_repo()):
if f.spec.build.source and \
f.spec.build.source.startswith(self._source_repo()):
update_in(spec, 'spec.build.source', './')
funcs.append({'name': name,
'spec': spec})
Expand Down
28 changes: 20 additions & 8 deletions mlrun/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,17 +40,22 @@
from .config import config as mlconf


def run_local(task, command='', name: str = '', args: list = None,
workdir=None, project: str = '', tag: str = '', secrets=None):
def run_local(task=None, command='', name: str = '', args: list = None,
workdir=None, project: str = '', tag: str = '', secrets=None,
handler=None, params: dict = None, inputs: dict = None,
artifact_path: str = ''):
"""Run a task on function/code (.py, .ipynb or .yaml) locally,

e.g.:
# define a task
task = NewTask(params={'p1': 8}, out_path=out_path)

# run
run = run_local(spec, command='src/training.py', workdir='src')

or specify base task parameters (handler, params, ..) in the call

run = run_local(handler=my_function, params={'x': 5})

:param task: task template object or dict (see RunTemplate)
:param command: command/url/function
:param name: ad hook function name
Expand All @@ -60,6 +65,11 @@ def run_local(task, command='', name: str = '', args: list = None,
:param tag: function version tag (none for 'latest')
:param secrets: secrets dict if the function source is remote (s3, v3io, ..)

:param handler: pointer or name of a function handler
:param params: input parameters (dict)
:param inputs: input objects (dict of key: path)
:param artifact_path: default artifact output path

:return: run object
"""

Expand All @@ -72,7 +82,6 @@ def run_local(task, command='', name: str = '', args: list = None,

is_obj = hasattr(command, 'to_dict')
suffix = '' if is_obj else Path(command).suffix
handler = None
meta = BaseMetadata(name, project=project, tag=tag)
if is_obj or suffix == '.yaml':
is_remote = False
Expand All @@ -83,7 +92,7 @@ def run_local(task, command='', name: str = '', args: list = None,
data = get_object(command, secrets)
runtime = yaml.load(data, Loader=yaml.FullLoader)

handler = get_in(runtime, 'spec.default_handler', '')
handler = handler or get_in(runtime, 'spec.default_handler', '')
command = get_in(runtime, 'spec.command', '')
code = get_in(runtime, 'spec.build.functionSourceCode')

Expand Down Expand Up @@ -123,14 +132,16 @@ def run_local(task, command='', name: str = '', args: list = None,
else:
raise ValueError('unsupported suffix: {}'.format(suffix))

if not (command or handler or task):
raise ValueError('nothing to run, specify command or handler')

fn = new_function(meta.name, command=command, args=args)
meta.name = fn.metadata.name
fn.metadata = meta
if workdir:
fn.spec.workdir = str(workdir)
if handler:
fn.spec.default_handler = handler
return fn.run(task)
return fn.run(task, handler=handler, params=params, inputs=inputs,
artifact_path=artifact_path)


def get_or_create_ctx(name: str,
Expand Down Expand Up @@ -517,6 +528,7 @@ def add_name(origin, name=''):
if with_doc:
handlers = find_handlers(code)
r.spec.entry_points = {h['name']: as_func(h) for h in handlers}
r.spec.default_handler = handler
return r


Expand Down