From 71047eb8e830dd729a580b9d08c3f6113785a815 Mon Sep 17 00:00:00 2001 From: Mani Parkhe Date: Fri, 24 Aug 2018 11:21:28 -0700 Subject: [PATCH 1/5] cli support for delete and restore experiments --- mlflow/experiments.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/mlflow/experiments.py b/mlflow/experiments.py index 11dd03594d523..6566c11223f20 100644 --- a/mlflow/experiments.py +++ b/mlflow/experiments.py @@ -42,3 +42,27 @@ def list_experiments(): table = [[exp.experiment_id, exp.name, exp.artifact_location if is_uri(exp.artifact_location) else os.path.abspath(exp.artifact_location)] for exp in experiments] print(tabulate(sorted(table), headers=["Experiment Id", "Name", "Artifact Location"])) + + +@commands.command("delete") +@click.argument("experiment_id") +def delete_experiment(experiment_id): + """ + Delete an experiment from backend store. + Will error out if experiment does not exist or already deleted. + """ + store = _get_store() + store.delete_experiment(experiment_id) + print("Experiment with id %d has been deleted." % experiment_id) + + +@commands.command("restore") +@click.argument("experiment_id") +def restore_experiment(experiment_id): + """ + Restore a deleted experiment. + Will error out if experiment is already active or has been permanently deleted. + """ + store = _get_store() + store.restore_experiment(experiment_id) + print("Experiment with id %d has been restored and is now active." % experiment_id) From 6e61eaeea464c6afd497c3dae698e4ef8f5abe0b Mon Sep 17 00:00:00 2001 From: Mani Parkhe Date: Fri, 24 Aug 2018 13:42:01 -0700 Subject: [PATCH 2/5] cli support for delete and restore + docs --- docs/source/cli.rst | 38 ++++++++++++++++++++++++++++++++++++-- mlflow/experiments.py | 6 +++--- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/docs/source/cli.rst b/docs/source/cli.rst index eba9f944eaa96..aaf0ad1c414bd 100644 --- a/docs/source/cli.rst +++ b/docs/source/cli.rst @@ -21,7 +21,7 @@ and list experiments, and download artifacts. Commands: azureml Serve models on Azure ML. download Download the artifact at the specified DBFS or S3 URI. - experiments Run and list experiments. + experiments Manage experiments. pyfunc Serve Python models locally. run Run an MLflow project from the given URI. sagemaker Serve models on Amazon SageMaker. @@ -49,7 +49,41 @@ specified. Experiments ----------- -Subcommands to create and list experiments. +Subcommands to manage experiments. + + +Create +------ +Subcommand to create a new experiment using name. System will generate a unique ID for each +experiment. Additionally, users can provide an artifact location using ``-l`` or +``--artifact-location`` option. If not provided, backend store will pick default location. + +All artifacts related to this experiment will be stored under artifact location under specific +run directories. + + +List +---- + +Listing of all experiments managed by backend store. + + +Delete +------ + +Delete an active experiment. Command takes an mandatory argument experiment ID. If experiment +is already deleted or not found, the command will throw error. This deletes associated metadata, +runs and data as well. If the backend store controls locations of artifacts, they will be deleted +as well. Deleted experiments can be restored using ``restore`` command. + + +Restore +------- + +Restore a deleted experiment. Command takes an mandatory argument experiment ID. If experiment is +already active, permanently deleted, or cannot be found, the command will throw error. The command +will restore all runs belonging to the experiment and all metadata associated with experiment and +runs. Python Function diff --git a/mlflow/experiments.py b/mlflow/experiments.py index 6566c11223f20..54deb23fddede 100644 --- a/mlflow/experiments.py +++ b/mlflow/experiments.py @@ -38,7 +38,7 @@ def list_experiments(): List all experiments in the configured tracking server. """ store = _get_store() - experiments = store.list_experiments() + experiments = store.list_experiments("deleted_only") table = [[exp.experiment_id, exp.name, exp.artifact_location if is_uri(exp.artifact_location) else os.path.abspath(exp.artifact_location)] for exp in experiments] print(tabulate(sorted(table), headers=["Experiment Id", "Name", "Artifact Location"])) @@ -53,7 +53,7 @@ def delete_experiment(experiment_id): """ store = _get_store() store.delete_experiment(experiment_id) - print("Experiment with id %d has been deleted." % experiment_id) + print("Experiment with id %s has been deleted." % str(experiment_id)) @commands.command("restore") @@ -65,4 +65,4 @@ def restore_experiment(experiment_id): """ store = _get_store() store.restore_experiment(experiment_id) - print("Experiment with id %d has been restored and is now active." % experiment_id) + print("Experiment with id %s has been restored and is now active." % str(experiment_id)) From b87b2383f1f14256fb27a087ed24992e1ed0ddc7 Mon Sep 17 00:00:00 2001 From: Mani Parkhe Date: Fri, 24 Aug 2018 18:22:19 -0700 Subject: [PATCH 3/5] addressing review comments, fixed missing view type argument for list experiments CLI command --- docs/source/cli.rst | 51 +++++++++++++++++++++++------------- mlflow/entities/view_type.py | 22 ++++++++++++++++ mlflow/experiments.py | 18 ++++++++----- 3 files changed, 67 insertions(+), 24 deletions(-) diff --git a/docs/source/cli.rst b/docs/source/cli.rst index aaf0ad1c414bd..6ca07be5eae90 100644 --- a/docs/source/cli.rst +++ b/docs/source/cli.rst @@ -53,37 +53,52 @@ Subcommands to manage experiments. Create ------- -Subcommand to create a new experiment using name. System will generate a unique ID for each -experiment. Additionally, users can provide an artifact location using ``-l`` or -``--artifact-location`` option. If not provided, backend store will pick default location. +~~~~~~ -All artifacts related to this experiment will be stored under artifact location under specific -run directories. +Subcommand to create a new experiment. The command has required argument for experiment name. +Additionally, users can provide an artifact location using ``-l`` or ``--artifact-location`` +option. If not provided, backend store will pick default location. Backend store will generate a +unique ID for each experiment. + +All artifacts generated by runs related to this experiment will be stored under artifact location, +organized under specific run_uuid sub-directories. + +Implementation of experiment and metadata store is dependent on backend storage. ``FileStore`` +creates a folder for each experiment ID and stores metadata in ``meta.yaml``. Runs are stored as +subfolders. List ----- +~~~~ -Listing of all experiments managed by backend store. +Lists all experiments managed by backend store. Command takes an optional ``--view`` or ``-v`` +argument. Valid arguments are ``active_only`` (default), ``deleted_only``, or ``all``. Delete ------- +~~~~~~ + +Marks an active experiment for deletion. This also applies to experiment's metadata, runs and +associated data, and artifacts if they are store in default location. Use ``list`` command to view +artifact location. Command takes a required argument for experiment ID. Command will thrown +an error if experiment is not found or already marked for deletion. + +Experiments marked for deletion can be restored using ``restore`` command, unless they are +permanently deleted. -Delete an active experiment. Command takes an mandatory argument experiment ID. If experiment -is already deleted or not found, the command will throw error. This deletes associated metadata, -runs and data as well. If the backend store controls locations of artifacts, they will be deleted -as well. Deleted experiments can be restored using ``restore`` command. +Specific implementation of deletion is dependent on backend stores. ``FileStore`` moves +experiments marked for deletion under a ``.trash`` folder under the main folder used to +instantiate ``FileStore``. Experiments marked for deletion can be permanently deleted by clearing +the ``.trash`` folder. It is recommended to use a ``cron`` job or an alternate workflow mechanism +to clear ``.trash`` folder. Restore -------- +~~~~~~~ -Restore a deleted experiment. Command takes an mandatory argument experiment ID. If experiment is -already active, permanently deleted, or cannot be found, the command will throw error. The command -will restore all runs belonging to the experiment and all metadata associated with experiment and -runs. +Restore a deleted experiment. This also applies to experiment's metadata, runs and associated data. +Command takes a required argument for experiment ID. Command will throw an error if experiment is +already active, cannot be found, or permanently deleted. Python Function diff --git a/mlflow/entities/view_type.py b/mlflow/entities/view_type.py index 438120a6ef931..9db4901807f38 100644 --- a/mlflow/entities/view_type.py +++ b/mlflow/entities/view_type.py @@ -1,3 +1,25 @@ class ViewType(object): """Enum to qualify `ListExperiments` API query for requested experiment types.""" ACTIVE_ONLY, DELETED_ONLY, ALL = range(1, 4) + _VIEW_TO_STRING = { + ACTIVE_ONLY: "active_only", + DELETED_ONLY: "deleted_only", + ALL: "all", + } + _STRING_TO_VIEW = {value: key for key, value in _VIEW_TO_STRING.items()} + + @staticmethod + def from_string(view_str): + if view_str not in ViewType._STRING_TO_VIEW: + raise Exception( + "Could not get valid view type corresponding to string %s. " + "Valid view types are %s" % (view_str, list(ViewType._STRING_TO_VIEW.keys()))) + return ViewType._STRING_TO_VIEW[view_str] + + @staticmethod + def to_string(view_type): + if view_type not in ViewType._VIEW_TO_STRING: + raise Exception( + "Could not get valid view type corresponding to string %s. " + "Valid view types are %s" % (view_type, list(ViewType._VIEW_TO_STRING.keys()))) + return ViewType._VIEW_TO_STRING[view_type] diff --git a/mlflow/experiments.py b/mlflow/experiments.py index 54deb23fddede..e2b7e249f681a 100644 --- a/mlflow/experiments.py +++ b/mlflow/experiments.py @@ -6,6 +6,7 @@ from tabulate import tabulate from mlflow.data import is_uri +from mlflow.entities import ViewType from mlflow.tracking import _get_store @@ -33,12 +34,16 @@ def create(experiment_name, artifact_location): @commands.command("list") -def list_experiments(): +@click.option("--view", "-v", default="active_only", + help="Select view type for list experiments. Valid view types are " + "'active_only' (default), 'delete_only', and 'all'.") +def list_experiments(view): """ List all experiments in the configured tracking server. """ store = _get_store() - experiments = store.list_experiments("deleted_only") + view_type = ViewType.from_string(view) if view else ViewType.ALL + experiments = store.list_experiments(view_type) table = [[exp.experiment_id, exp.name, exp.artifact_location if is_uri(exp.artifact_location) else os.path.abspath(exp.artifact_location)] for exp in experiments] print(tabulate(sorted(table), headers=["Experiment Id", "Name", "Artifact Location"])) @@ -48,8 +53,9 @@ def list_experiments(): @click.argument("experiment_id") def delete_experiment(experiment_id): """ - Delete an experiment from backend store. - Will error out if experiment does not exist or already deleted. + Marks experiment for deletion. This command will error out if experiment does not exist or + is already marked. Experiments marked this way can be restored with restore_experiment, + or permanently deleted based on the backend store (refer to docs for details). """ store = _get_store() store.delete_experiment(experiment_id) @@ -61,8 +67,8 @@ def delete_experiment(experiment_id): def restore_experiment(experiment_id): """ Restore a deleted experiment. - Will error out if experiment is already active or has been permanently deleted. + This command will error out if experiment is already active or has been permanently deleted. """ store = _get_store() store.restore_experiment(experiment_id) - print("Experiment with id %s has been restored and is now active." % str(experiment_id)) + print("Experiment with id %s has been restored." % str(experiment_id)) From 1c8b97243d391628c44f46c1346b6814f84a189e Mon Sep 17 00:00:00 2001 From: Mani Parkhe Date: Fri, 24 Aug 2018 22:45:53 -0700 Subject: [PATCH 4/5] default list experiments view is ACTIVE_ONLY --- mlflow/experiments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlflow/experiments.py b/mlflow/experiments.py index e2b7e249f681a..c9ac8ecf251f7 100644 --- a/mlflow/experiments.py +++ b/mlflow/experiments.py @@ -42,7 +42,7 @@ def list_experiments(view): List all experiments in the configured tracking server. """ store = _get_store() - view_type = ViewType.from_string(view) if view else ViewType.ALL + view_type = ViewType.from_string(view) if view else ViewType.ACTIVE_ONLY experiments = store.list_experiments(view_type) table = [[exp.experiment_id, exp.name, exp.artifact_location if is_uri(exp.artifact_location) else os.path.abspath(exp.artifact_location)] for exp in experiments] From 4c517423770023ec4d3bb6ea56bed7d20b61807e Mon Sep 17 00:00:00 2001 From: Mani Parkhe Date: Mon, 27 Aug 2018 13:12:06 -0700 Subject: [PATCH 5/5] typo fix --- mlflow/experiments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlflow/experiments.py b/mlflow/experiments.py index c9ac8ecf251f7..3b9571a192394 100644 --- a/mlflow/experiments.py +++ b/mlflow/experiments.py @@ -36,7 +36,7 @@ def create(experiment_name, artifact_location): @commands.command("list") @click.option("--view", "-v", default="active_only", help="Select view type for list experiments. Valid view types are " - "'active_only' (default), 'delete_only', and 'all'.") + "'active_only' (default), 'deleted_only', and 'all'.") def list_experiments(view): """ List all experiments in the configured tracking server.