From 3e27b3d83b7666ff3597ba0dfd6a8169569c182a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Tue, 7 Feb 2023 16:52:56 -0500 Subject: [PATCH 01/57] Use wexpect for windows --- cli/medperf/entities/cube.py | 11 ++++++++--- cli/medperf/tests/entities/test_cube.py | 20 ++++++++++---------- cli/medperf/utils.py | 6 +++++- cli/requirements.txt | 1 + 4 files changed, 24 insertions(+), 14 deletions(-) diff --git a/cli/medperf/entities/cube.py b/cli/medperf/entities/cube.py index 04bb0851c..ea0d6f365 100644 --- a/cli/medperf/entities/cube.py +++ b/cli/medperf/entities/cube.py @@ -1,10 +1,15 @@ import os +import sys import yaml -import pexpect import logging from typing import List, Dict from pathlib import Path +if sys.platform == "win32": + from wexpect import spawn +else: + from pexpect import spawn + from medperf.utils import ( get_file_sha1, untar, @@ -236,7 +241,7 @@ def download_image(self): # Retrieve image from image registry logging.debug(f"Retrieving {self.uid} image") cmd = f"mlcube configure --mlcube={self.cube_path}" - proc = pexpect.spawn(cmd) + proc = spawn(cmd) proc_out = combine_proc_sp_text(proc) logging.debug(proc_out) proc.close() @@ -299,7 +304,7 @@ def run( cmd_arg = f'{k}="{v}"' cmd = " ".join([cmd, cmd_arg]) logging.info(f"Running MLCube command: {cmd}") - proc = pexpect.spawn(cmd, timeout=timeout) + proc = spawn(cmd, timeout=timeout) proc_out = combine_proc_sp_text(proc) proc.close() logging.debug(proc_out) diff --git a/cli/medperf/tests/entities/test_cube.py b/cli/medperf/tests/entities/test_cube.py index d6082f1dc..b76f5b43c 100644 --- a/cli/medperf/tests/entities/test_cube.py +++ b/cli/medperf/tests/entities/test_cube.py @@ -50,7 +50,7 @@ def comms(mocker): @pytest.fixture def no_local(mocker): mpexpect = MockPexpect(0) - mocker.patch(PATCH_CUBE.format("pexpect.spawn"), side_effect=mpexpect.spawn) + mocker.patch(PATCH_CUBE.format("spawn"), side_effect=mpexpect.spawn) mocker.patch(PATCH_CUBE.format("combine_proc_sp_text"), return_value="") mocker.patch(PATCH_CUBE.format("Cube.all"), return_value=[]) @@ -60,7 +60,7 @@ def basic_body(mocker, comms): body_gen = cube_metadata_generator() mocker.patch.object(comms, "get_cube_metadata", side_effect=body_gen) mpexpect = MockPexpect(0) - mocker.patch(PATCH_CUBE.format("pexpect.spawn"), side_effect=mpexpect.spawn) + mocker.patch(PATCH_CUBE.format("spawn"), side_effect=mpexpect.spawn) mocker.patch(PATCH_CUBE.format("combine_proc_sp_text"), return_value="") mocker.patch(PATCH_CUBE.format("Cube.store_local_hashes")) mocker.patch(PATCH_CUBE.format("Cube.write")) @@ -75,7 +75,7 @@ def params_body(mocker, comms): body_gen = cube_metadata_generator(with_params=True) mocker.patch.object(comms, "get_cube_metadata", side_effect=body_gen) mpexpect = MockPexpect(0) - mocker.patch(PATCH_CUBE.format("pexpect.spawn"), side_effect=mpexpect.spawn) + mocker.patch(PATCH_CUBE.format("spawn"), side_effect=mpexpect.spawn) mocker.patch(PATCH_CUBE.format("combine_proc_sp_text"), return_value="") mocker.patch(PATCH_CUBE.format("Cube.store_local_hashes")) mocker.patch(PATCH_CUBE.format("Cube.write")) @@ -90,7 +90,7 @@ def tar_body(mocker, comms): body_gen = cube_metadata_generator(with_tarball=True) mocker.patch.object(comms, "get_cube_metadata", side_effect=body_gen) mpexpect = MockPexpect(0) - mocker.patch(PATCH_CUBE.format("pexpect.spawn"), side_effect=mpexpect.spawn) + mocker.patch(PATCH_CUBE.format("spawn"), side_effect=mpexpect.spawn) mocker.patch(PATCH_CUBE.format("combine_proc_sp_text"), return_value="") mocker.patch(PATCH_CUBE.format("Cube.store_local_hashes")) mocker.patch(PATCH_CUBE.format("Cube.write")) @@ -432,7 +432,7 @@ def test_get_cube_deletes_cube_if_failed(mocker, comms, basic_body, no_local, ui def test_get_cube_without_image_configures_mlcube(mocker, comms, basic_body, no_local): # Arrange - spy = mocker.spy(medperf.entities.cube.pexpect, "spawn") + spy = mocker.spy(medperf.entities.cube, "spawn") expected_cmd = f"mlcube configure --mlcube={CUBE_PATH}" mocker.patch(PATCH_CUBE.format("Cube.is_valid"), side_effect=[False, True]) @@ -446,7 +446,7 @@ def test_get_cube_without_image_configures_mlcube(mocker, comms, basic_body, no_ def test_get_cube_with_image_isnt_configured(mocker, comms, img_body, no_local): # Arrange - spy = mocker.spy(medperf.entities.cube.pexpect, "spawn") + spy = mocker.spy(medperf.entities.cube, "spawn") mocker.patch(PATCH_CUBE.format("Cube.is_valid"), return_value=True) # Act @@ -553,10 +553,10 @@ def test_cube_runs_command_with_pexpect( ): # Arrange mpexpect = MockPexpect(0) - mocker.patch(PATCH_CUBE.format("pexpect.spawn"), side_effect=mpexpect.spawn) + mocker.patch(PATCH_CUBE.format("spawn"), side_effect=mpexpect.spawn) mocker.patch(PATCH_CUBE.format("list_files"), return_value="") mocker.patch(PATCH_CUBE.format("Cube.is_valid"), side_effect=[False, True]) - spy = mocker.spy(medperf.entities.cube.pexpect, "spawn") + spy = mocker.spy(medperf.entities.cube, "spawn") task = "task" platform = config.platform expected_cmd = ( @@ -575,7 +575,7 @@ def test_cube_runs_command_with_pexpect( def test_cube_runs_command_with_extra_args(mocker, ui, comms, basic_body, no_local): # Arrange mpexpect = MockPexpect(0) - spy = mocker.patch("pexpect.spawn", side_effect=mpexpect.spawn) + spy = mocker.patch("spawn", side_effect=mpexpect.spawn) mocker.patch(PATCH_CUBE.format("Cube.is_valid"), side_effect=[False, True]) mocker.patch(PATCH_CUBE.format("list_files"), return_value="") task = "task" @@ -594,7 +594,7 @@ def test_cube_runs_command_with_extra_args(mocker, ui, comms, basic_body, no_loc def test_run_stops_execution_if_child_fails(mocker, ui, comms, basic_body, no_local): # Arrange mpexpect = MockPexpect(1) - mocker.patch("pexpect.spawn", side_effect=mpexpect.spawn) + mocker.patch("spawn", side_effect=mpexpect.spawn) mocker.patch(PATCH_CUBE.format("Cube.is_valid"), return_value=True) task = "task" diff --git a/cli/medperf/utils.py b/cli/medperf/utils.py index 2e4b3b745..57bdb0e0e 100644 --- a/cli/medperf/utils.py +++ b/cli/medperf/utils.py @@ -13,12 +13,16 @@ import json from pathlib import Path from shutil import rmtree -from pexpect import spawn from datetime import datetime from typing import List, Tuple from colorama import Fore, Style from pexpect.exceptions import TIMEOUT +if sys.platform == "win32": + from wexpect import spawn +else: + from pexpect import spawn + import medperf.config as config from medperf.exceptions import ExecutionError, InvalidEntityError, MedperfException diff --git a/cli/requirements.txt b/cli/requirements.txt index 0c00e3a0b..b9abf4ccc 100644 --- a/cli/requirements.txt +++ b/cli/requirements.txt @@ -5,6 +5,7 @@ requests>=2.26.0 yaspin==2.1.0 tabulate==0.8.9 pexpect==4.8.0 +wexpect==2.3.2; sys_platform == "win32" colorama==0.4.4 time-machine==2.4.0 pytest-mock==1.13.0 From 1387fc3ad405b5ce1cdd99f18d0e2fa84394a72c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Tue, 7 Feb 2023 16:53:51 -0500 Subject: [PATCH 02/57] Specify specific click version --- cli/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/requirements.txt b/cli/requirements.txt index b9abf4ccc..1aee2198a 100644 --- a/cli/requirements.txt +++ b/cli/requirements.txt @@ -1,4 +1,5 @@ typer~=0.6.0 +click>=8.0.0,<9.0.0 rich~=12.5.0 PyYAML>=5.4,<6 requests>=2.26.0 From 52a413d7a4064bafb8323266663ead85a4d73338 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Mon, 13 Feb 2023 15:26:50 -0500 Subject: [PATCH 03/57] Fix typer incompatibility error --- cli/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cli/requirements.txt b/cli/requirements.txt index 1aee2198a..12a06067e 100644 --- a/cli/requirements.txt +++ b/cli/requirements.txt @@ -1,4 +1,5 @@ -typer~=0.6.0 +typer~=0.6.0; sys_platform != "win32" +typer<0.5.0; sys_platform == "win32" click>=8.0.0,<9.0.0 rich~=12.5.0 PyYAML>=5.4,<6 From 9c713849228fc55fabb1dee7ee9b660f58d64558 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Mon, 13 Feb 2023 15:27:11 -0500 Subject: [PATCH 04/57] Handle process text being outputted as text --- cli/medperf/utils.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cli/medperf/utils.py b/cli/medperf/utils.py index 57bdb0e0e..6b9ca9d0d 100644 --- a/cli/medperf/utils.py +++ b/cli/medperf/utils.py @@ -413,12 +413,18 @@ def combine_proc_sp_text(proc: spawn) -> str: logging.error("Process timed out") raise ExecutionError("Process timed out") - while byte and not re.match(b"[\r\n]", byte): + if type(byte) == str: + pattern = "[\r\n]" + else: + pattern = b"[\r\n]" + + while byte and not re.match(pattern, byte): byte = proc.read(1) line += byte if not byte: break - line = line.decode("utf-8", "ignore") + if type(line) != str: + line = line.decode("utf-8", "ignore") if line: # add to proc_out list for logging proc_out += line From 01b5795062abaa2801a92b7ed1b2aa7b7fe34a50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Tue, 21 Feb 2023 08:53:03 -0500 Subject: [PATCH 05/57] Update requirements for windows py3 --- cli/requirements.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cli/requirements.txt b/cli/requirements.txt index 12a06067e..5810b91e8 100644 --- a/cli/requirements.txt +++ b/cli/requirements.txt @@ -1,18 +1,17 @@ typer~=0.6.0; sys_platform != "win32" typer<0.5.0; sys_platform == "win32" -click>=8.0.0,<9.0.0 rich~=12.5.0 PyYAML>=5.4,<6 requests>=2.26.0 yaspin==2.1.0 tabulate==0.8.9 pexpect==4.8.0 -wexpect==2.3.2; sys_platform == "win32" +wexpect==4.0.0; sys_platform == "win32" colorama==0.4.4 time-machine==2.4.0 pytest-mock==1.13.0 mlcube==0.0.8 mlcube-docker==0.0.8 -mlcube-singularity==0.0.8 +mlcube-singularity==0.0.8; sys_platform != "win32" validators==0.18.2 merge-args==0.1.4 \ No newline at end of file From 28f7e7fb2a6f10e4ff5fb49cb36a3a9cd320f3ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 3 Mar 2023 16:21:24 -0500 Subject: [PATCH 06/57] Add Data Preparator cookiecutter template --- .../data_preparator_mlcube/coookiecutter.json | 9 ++++ .../hooks/post_gen_project.py | 13 ++++++ .../mlcube/mlcube.yaml | 33 ++++++++++++++ .../mlcube/workspace/parameters.yaml | 0 .../project/Dockerfile | 31 +++++++++++++ .../project/mlcube.py | 44 +++++++++++++++++++ 6 files changed, 130 insertions(+) create mode 100644 templates/data_preparator_mlcube/coookiecutter.json create mode 100644 templates/data_preparator_mlcube/hooks/post_gen_project.py create mode 100644 templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/mlcube/mlcube.yaml create mode 100644 templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/mlcube/workspace/parameters.yaml create mode 100644 templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/project/Dockerfile create mode 100644 templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/project/mlcube.py diff --git a/templates/data_preparator_mlcube/coookiecutter.json b/templates/data_preparator_mlcube/coookiecutter.json new file mode 100644 index 000000000..d33b56e1e --- /dev/null +++ b/templates/data_preparator_mlcube/coookiecutter.json @@ -0,0 +1,9 @@ +{ + "project_slug": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", + "mlcube_name": "Data Preparator MLCube", + "description": "Data Preparator MLCube Template. Provided by MLCommons", + "author_name": "John Smith", + "accelerator_count": "0", + "docker_image_name": "docker/image:latest", + "use_separate_output_labels": "n" +} \ No newline at end of file diff --git a/templates/data_preparator_mlcube/hooks/post_gen_project.py b/templates/data_preparator_mlcube/hooks/post_gen_project.py new file mode 100644 index 000000000..aa5bd6fa3 --- /dev/null +++ b/templates/data_preparator_mlcube/hooks/post_gen_project.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +import os +import shutil + +PROJECT_DIRECTORY = os.path.realpath(os.path.curdir) + + +if __name__ == "__main__": + if "{{ cookiecutter.use_separate_output_labels }}" != "y": + input_labels_path = os.path.join( + "{{ cookiecutter.project_slug }}", "mlcube/workspace/input_labels" + ) + shutil.rmtree() diff --git a/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/mlcube/mlcube.yaml b/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/mlcube/mlcube.yaml new file mode 100644 index 000000000..38be9e0ce --- /dev/null +++ b/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/mlcube/mlcube.yaml @@ -0,0 +1,33 @@ +name: {{ cookiecutter.mlcube_name }} +description: {{ cookiecutter.description }} +authors: + - {name: {{ cookiecutter.author_name }}} + +platform: + accelerator_count: {{ cookiecutter.accelerator_count }} + +docker: + # Image name + image: {{ cookiecutter.docker_image_name }} + # Docker build context relative to $MLCUBE_ROOT. Default is `build`. + build_context: "../project" + # Docker file name within docker build context, default is `Dockerfile`. + build_file: "Dockerfile" + +tasks: + prepare: + parameters: + inputs: {data_path: input_data, labels_path: input_labels, parameters_file: parameters.yaml} + outputs: { + output_path: data/, + {% if cookiecutter.use_separate_output_labels == 'y' -%} + output_labels_path: labels/ + {% endif %} + } + sanity_check: + parameters: + inputs: {data_path: data/, parameters_file: parameters.yaml} + statistics: + parameters: + inputs: {data_path: data/, parameters_file: parameters.yaml} + outputs: {output_path: {type: file, default: statistics.yaml}} \ No newline at end of file diff --git a/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/mlcube/workspace/parameters.yaml b/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/mlcube/workspace/parameters.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/project/Dockerfile b/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/project/Dockerfile new file mode 100644 index 000000000..9f2ce5642 --- /dev/null +++ b/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/project/Dockerfile @@ -0,0 +1,31 @@ +FROM ubuntu:18.04 +MAINTAINER MLPerf MLBox Working Group + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + software-properties-common \ + python3-dev \ + curl && \ + rm -rf /var/lib/apt/lists/* + +RUN add-apt-repository ppa:deadsnakes/ppa -y && apt-get update + +RUN apt-get install python3 -y + +RUN curl -fSsL -O https://bootstrap.pypa.io/pip/3.6/get-pip.py && \ + python3 get-pip.py && \ + rm get-pip.py + +COPY ./requirements.txt project/requirements.txt + +RUN pip3 install --upgrade pip + +RUN pip3 install --no-cache-dir -r project/requirements.txt + +ENV LANG C.UTF-8 + +COPY . /project + +WORKDIR /project + +ENTRYPOINT ["python3", "mlcube.py"] \ No newline at end of file diff --git a/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/project/mlcube.py b/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/project/mlcube.py new file mode 100644 index 000000000..7b38614c9 --- /dev/null +++ b/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/project/mlcube.py @@ -0,0 +1,44 @@ +"""MLCube handler file""" +import typer + + +app = typer.Typer() + + +@app.command("prepare") +def prepare( + data_path: str = typer.Option(..., "--data_path"), + labels_path: str = typer.Option(..., "--labels_path"), + parameters_file: str = typer.Option(..., "--parameters_file"), + output_path: str = typer.Option(..., "--output_path"), + {% if cookiecutter.use_separate_output_labels == 'y' -%} + output_labels_path: str = typer.Option(..., "--output_labels_path"), + {% endif %} +): + # Modify the prepare command as needed + raise NotImplementedError("The prepare method is not yet implemented") + + +@app.command("sanity_check") +def sanity_check( + data_path: str = typer.Option(..., "--data_path"), + labels_path: str = typer.Option(..., "--labels_path"), + parameters_file: str = typer.Option(..., "--parameters_file"), +): + # Modify the sanity_check command as needed + raise NotImplementedError("The sanity check method is not yet implemented") + + +@app.command("statistics") +def sanity_check( + data_path: str = typer.Option(..., "--data_path"), + labels_path: str = typer.Option(..., "--labels_path"), + parameters_file: str = typer.Option(..., "--parameters_file"), + out_path: str = typer.Option(..., "--output_path"), +): + # Modify the statistics command as needed + raise NotImplementedError("The statistics method is not yet implemented") + + +if __name__ == "__main__": + app() From 6f9e19e3319ba45066156dedc7007d8b432139bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 3 Mar 2023 16:35:49 -0500 Subject: [PATCH 07/57] Rename cookiecutter folder --- .../mlcube/mlcube.yaml | 0 .../mlcube/workspace/parameters.yaml | 0 .../project/Dockerfile | 0 .../project/mlcube.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename templates/data_preparator_mlcube/{{{ cookiecutter.project_slug }} => {{cookiecutter.project_slug}}}/mlcube/mlcube.yaml (100%) rename templates/data_preparator_mlcube/{{{ cookiecutter.project_slug }} => {{cookiecutter.project_slug}}}/mlcube/workspace/parameters.yaml (100%) rename templates/data_preparator_mlcube/{{{ cookiecutter.project_slug }} => {{cookiecutter.project_slug}}}/project/Dockerfile (100%) rename templates/data_preparator_mlcube/{{{ cookiecutter.project_slug }} => {{cookiecutter.project_slug}}}/project/mlcube.py (100%) diff --git a/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/mlcube/mlcube.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml similarity index 100% rename from templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/mlcube/mlcube.yaml rename to templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml diff --git a/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/mlcube/workspace/parameters.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml similarity index 100% rename from templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/mlcube/workspace/parameters.yaml rename to templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml diff --git a/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/project/Dockerfile b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile similarity index 100% rename from templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/project/Dockerfile rename to templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile diff --git a/templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/project/mlcube.py b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py similarity index 100% rename from templates/data_preparator_mlcube/{{ cookiecutter.project_slug }}/project/mlcube.py rename to templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py From df6e6a2463c7afaec2e85a75a186d9f079538d77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 3 Mar 2023 16:40:43 -0500 Subject: [PATCH 08/57] Temporarily remove possibly offending files --- .../mlcube/mlcube.yaml | 33 ------------------- .../mlcube/workspace/parameters.yaml | 0 2 files changed, 33 deletions(-) delete mode 100644 templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml delete mode 100644 templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml deleted file mode 100644 index 38be9e0ce..000000000 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: {{ cookiecutter.mlcube_name }} -description: {{ cookiecutter.description }} -authors: - - {name: {{ cookiecutter.author_name }}} - -platform: - accelerator_count: {{ cookiecutter.accelerator_count }} - -docker: - # Image name - image: {{ cookiecutter.docker_image_name }} - # Docker build context relative to $MLCUBE_ROOT. Default is `build`. - build_context: "../project" - # Docker file name within docker build context, default is `Dockerfile`. - build_file: "Dockerfile" - -tasks: - prepare: - parameters: - inputs: {data_path: input_data, labels_path: input_labels, parameters_file: parameters.yaml} - outputs: { - output_path: data/, - {% if cookiecutter.use_separate_output_labels == 'y' -%} - output_labels_path: labels/ - {% endif %} - } - sanity_check: - parameters: - inputs: {data_path: data/, parameters_file: parameters.yaml} - statistics: - parameters: - inputs: {data_path: data/, parameters_file: parameters.yaml} - outputs: {output_path: {type: file, default: statistics.yaml}} \ No newline at end of file diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml deleted file mode 100644 index e69de29bb..000000000 From a7db0cfe8ff8f7074b31e6a4e32f292f85ac89a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 3 Mar 2023 16:41:24 -0500 Subject: [PATCH 09/57] Remove cookicutter conditionals --- .../{{cookiecutter.project_slug}}/project/mlcube.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py index 7b38614c9..fcdb17207 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py @@ -11,9 +11,7 @@ def prepare( labels_path: str = typer.Option(..., "--labels_path"), parameters_file: str = typer.Option(..., "--parameters_file"), output_path: str = typer.Option(..., "--output_path"), - {% if cookiecutter.use_separate_output_labels == 'y' -%} output_labels_path: str = typer.Option(..., "--output_labels_path"), - {% endif %} ): # Modify the prepare command as needed raise NotImplementedError("The prepare method is not yet implemented") From a7a6d15a51355a0d3dca0dee9e44ba181cfe2bf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 3 Mar 2023 16:42:29 -0500 Subject: [PATCH 10/57] Inclube back missing pieces of template --- .../mlcube/mlcube.yaml | 33 +++++++++++++++++++ .../mlcube/workspace/parameters.yaml | 0 .../project/mlcube.py | 2 ++ 3 files changed, 35 insertions(+) create mode 100644 templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml create mode 100644 templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml new file mode 100644 index 000000000..38be9e0ce --- /dev/null +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml @@ -0,0 +1,33 @@ +name: {{ cookiecutter.mlcube_name }} +description: {{ cookiecutter.description }} +authors: + - {name: {{ cookiecutter.author_name }}} + +platform: + accelerator_count: {{ cookiecutter.accelerator_count }} + +docker: + # Image name + image: {{ cookiecutter.docker_image_name }} + # Docker build context relative to $MLCUBE_ROOT. Default is `build`. + build_context: "../project" + # Docker file name within docker build context, default is `Dockerfile`. + build_file: "Dockerfile" + +tasks: + prepare: + parameters: + inputs: {data_path: input_data, labels_path: input_labels, parameters_file: parameters.yaml} + outputs: { + output_path: data/, + {% if cookiecutter.use_separate_output_labels == 'y' -%} + output_labels_path: labels/ + {% endif %} + } + sanity_check: + parameters: + inputs: {data_path: data/, parameters_file: parameters.yaml} + statistics: + parameters: + inputs: {data_path: data/, parameters_file: parameters.yaml} + outputs: {output_path: {type: file, default: statistics.yaml}} \ No newline at end of file diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py index fcdb17207..7b38614c9 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py @@ -11,7 +11,9 @@ def prepare( labels_path: str = typer.Option(..., "--labels_path"), parameters_file: str = typer.Option(..., "--parameters_file"), output_path: str = typer.Option(..., "--output_path"), + {% if cookiecutter.use_separate_output_labels == 'y' -%} output_labels_path: str = typer.Option(..., "--output_labels_path"), + {% endif %} ): # Modify the prepare command as needed raise NotImplementedError("The prepare method is not yet implemented") From e2f7108dca68182f4241e8b2ceaf5b01d18b6a3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 3 Mar 2023 16:43:30 -0500 Subject: [PATCH 11/57] remove cookiecutter typo --- .../{coookiecutter.json => cookiecutter.json} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename templates/data_preparator_mlcube/{coookiecutter.json => cookiecutter.json} (100%) diff --git a/templates/data_preparator_mlcube/coookiecutter.json b/templates/data_preparator_mlcube/cookiecutter.json similarity index 100% rename from templates/data_preparator_mlcube/coookiecutter.json rename to templates/data_preparator_mlcube/cookiecutter.json From 581b5bb1e0cfabaa3c3a20b84d5aa2c4d90e0393 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 3 Mar 2023 16:45:09 -0500 Subject: [PATCH 12/57] Use project_name attribute --- templates/data_preparator_mlcube/cookiecutter.json | 2 +- .../{{cookiecutter.project_slug}}/mlcube/mlcube.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/data_preparator_mlcube/cookiecutter.json b/templates/data_preparator_mlcube/cookiecutter.json index d33b56e1e..1bba774b1 100644 --- a/templates/data_preparator_mlcube/cookiecutter.json +++ b/templates/data_preparator_mlcube/cookiecutter.json @@ -1,6 +1,6 @@ { "project_slug": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", - "mlcube_name": "Data Preparator MLCube", + "project_name": "Data Preparator MLCube", "description": "Data Preparator MLCube Template. Provided by MLCommons", "author_name": "John Smith", "accelerator_count": "0", diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml index 38be9e0ce..a617061b9 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml @@ -1,4 +1,4 @@ -name: {{ cookiecutter.mlcube_name }} +name: {{ cookiecutter.project_name }} description: {{ cookiecutter.description }} authors: - {name: {{ cookiecutter.author_name }}} From fd778041687e5e5243c213bc7d2e5b7200413d81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 3 Mar 2023 16:48:05 -0500 Subject: [PATCH 13/57] Change cookiecutter fields order --- templates/data_preparator_mlcube/cookiecutter.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/data_preparator_mlcube/cookiecutter.json b/templates/data_preparator_mlcube/cookiecutter.json index 1bba774b1..95b9a12d5 100644 --- a/templates/data_preparator_mlcube/cookiecutter.json +++ b/templates/data_preparator_mlcube/cookiecutter.json @@ -1,6 +1,6 @@ { - "project_slug": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", "project_name": "Data Preparator MLCube", + "project_slug": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", "description": "Data Preparator MLCube Template. Provided by MLCommons", "author_name": "John Smith", "accelerator_count": "0", From 6eebd59060a12a258ae165dfbf2e20a056afe102 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 3 Mar 2023 16:54:04 -0500 Subject: [PATCH 14/57] Create empty directories on hook --- .../hooks/post_gen_project.py | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/templates/data_preparator_mlcube/hooks/post_gen_project.py b/templates/data_preparator_mlcube/hooks/post_gen_project.py index aa5bd6fa3..82dc878ae 100644 --- a/templates/data_preparator_mlcube/hooks/post_gen_project.py +++ b/templates/data_preparator_mlcube/hooks/post_gen_project.py @@ -1,13 +1,21 @@ #!/usr/bin/env python import os -import shutil PROJECT_DIRECTORY = os.path.realpath(os.path.curdir) if __name__ == "__main__": - if "{{ cookiecutter.use_separate_output_labels }}" != "y": - input_labels_path = os.path.join( - "{{ cookiecutter.project_slug }}", "mlcube/workspace/input_labels" - ) - shutil.rmtree() + dir = "{{ cookiecutter.project_slug }}" + + input_data_path = os.path.join(dir, "mlcube/workspace/input_data") + data_path = os.path.join(dir, "mlcube/workspace/data") + labels_path = os.path.join(dir, "mlcube/workspace/labels") + + paths = [input_data_path, data_path, labels_path] + + if "{{ cookiecutter.use_separate_output_labels }}" == "y": + input_labels_path = os.path.join(dir, "mlcube/workspace/input_labels") + paths.append(input_labels_path) + + for path in paths: + os.makedirs(path, exist_ok=True) From 5ef86a2d3c9b1f78d6eaa49cc6e6be8f1e8df64e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 3 Mar 2023 16:56:57 -0500 Subject: [PATCH 15/57] Fix empty folders paths --- .../data_preparator_mlcube/hooks/post_gen_project.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/templates/data_preparator_mlcube/hooks/post_gen_project.py b/templates/data_preparator_mlcube/hooks/post_gen_project.py index 82dc878ae..540b83bd7 100644 --- a/templates/data_preparator_mlcube/hooks/post_gen_project.py +++ b/templates/data_preparator_mlcube/hooks/post_gen_project.py @@ -5,16 +5,14 @@ if __name__ == "__main__": - dir = "{{ cookiecutter.project_slug }}" - - input_data_path = os.path.join(dir, "mlcube/workspace/input_data") - data_path = os.path.join(dir, "mlcube/workspace/data") - labels_path = os.path.join(dir, "mlcube/workspace/labels") + input_data_path = "mlcube/workspace/input_data" + data_path = "mlcube/workspace/data" + labels_path = "mlcube/workspace/labels" paths = [input_data_path, data_path, labels_path] if "{{ cookiecutter.use_separate_output_labels }}" == "y": - input_labels_path = os.path.join(dir, "mlcube/workspace/input_labels") + input_labels_path = "mlcube/workspace/input_labels" paths.append(input_labels_path) for path in paths: From d04baf849e22058d69e1d5fd3eac55edafdb55a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Mon, 6 Mar 2023 12:05:54 -0500 Subject: [PATCH 16/57] Create evaluator mlcube cookiecutter template --- templates/evaluator_mlcube/cookiecutter.json | 8 +++++ .../hooks/post_gen_project.py | 18 +++++++++++ .../mlcube/mlcube.yaml | 22 +++++++++++++ .../mlcube/workspace/parameters.yaml | 0 .../project/Dockerfile | 31 +++++++++++++++++++ .../project/mlcube.py | 26 ++++++++++++++++ 6 files changed, 105 insertions(+) create mode 100644 templates/evaluator_mlcube/cookiecutter.json create mode 100644 templates/evaluator_mlcube/hooks/post_gen_project.py create mode 100644 templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml create mode 100644 templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml create mode 100644 templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile create mode 100644 templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py diff --git a/templates/evaluator_mlcube/cookiecutter.json b/templates/evaluator_mlcube/cookiecutter.json new file mode 100644 index 000000000..eed0126ab --- /dev/null +++ b/templates/evaluator_mlcube/cookiecutter.json @@ -0,0 +1,8 @@ +{ + "project_name": "Data Preparator MLCube", + "project_slug": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", + "description": "Data Preparator MLCube Template. Provided by MLCommons", + "author_name": "John Smith", + "accelerator_count": "0", + "docker_image_name": "docker/image:latest", +} \ No newline at end of file diff --git a/templates/evaluator_mlcube/hooks/post_gen_project.py b/templates/evaluator_mlcube/hooks/post_gen_project.py new file mode 100644 index 000000000..74c69c0d7 --- /dev/null +++ b/templates/evaluator_mlcube/hooks/post_gen_project.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +import os + +PROJECT_DIRECTORY = os.path.realpath(os.path.curdir) + + +if __name__ == "__main__": + preds_path = "mlcube/workspace/predictions" + labels_path = "mlcube/workspace/labels" + + paths = [preds_path, labels_path] + + if "{{ cookiecutter.use_separate_output_labels }}" == "y": + input_labels_path = "mlcube/workspace/input_labels" + paths.append(input_labels_path) + + for path in paths: + os.makedirs(path, exist_ok=True) diff --git a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml new file mode 100644 index 000000000..c486308c9 --- /dev/null +++ b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml @@ -0,0 +1,22 @@ +name: {{ cookiecutter.project_name }} +description: {{ cookiecutter.description }} +authors: + - {name: {{ cookiecutter.author_name }}} + +platform: + accelerator_count: {{ cookiecutter.accelerator_count }} + +docker: + # Image name + image: {{ cookiecutter.docker_image_name }} + # Docker build context relative to $MLCUBE_ROOT. Default is `build`. + build_context: "../project" + # Docker file name within docker build context, default is `Dockerfile`. + build_file: "Dockerfile" + +tasks: + evaluate: + # Computes evaluation metrics on the given predictions and ground truths + parameters: + inputs: {predictions: predictions, labels: labels, parameters_file: parameters.yaml} + outputs: {output_path: {type: "file", default: "results.yaml"}} \ No newline at end of file diff --git a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile new file mode 100644 index 000000000..9f2ce5642 --- /dev/null +++ b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile @@ -0,0 +1,31 @@ +FROM ubuntu:18.04 +MAINTAINER MLPerf MLBox Working Group + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + software-properties-common \ + python3-dev \ + curl && \ + rm -rf /var/lib/apt/lists/* + +RUN add-apt-repository ppa:deadsnakes/ppa -y && apt-get update + +RUN apt-get install python3 -y + +RUN curl -fSsL -O https://bootstrap.pypa.io/pip/3.6/get-pip.py && \ + python3 get-pip.py && \ + rm get-pip.py + +COPY ./requirements.txt project/requirements.txt + +RUN pip3 install --upgrade pip + +RUN pip3 install --no-cache-dir -r project/requirements.txt + +ENV LANG C.UTF-8 + +COPY . /project + +WORKDIR /project + +ENTRYPOINT ["python3", "mlcube.py"] \ No newline at end of file diff --git a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py new file mode 100644 index 000000000..1de721eb9 --- /dev/null +++ b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py @@ -0,0 +1,26 @@ +"""MLCube handler file""" +import typer + + +app = typer.Typer() + + +@app.command("evaluate") +def prepare( + labels: str = typer.Option(..., "--labels"), + predictions: str = typer.Option(..., "--predictions"), + parameters_file: str = typer.Option(..., "--parameters_file"), + output_path: str = typer.Option(..., "--output_path"), +): + # Modify the prepare command as needed + raise NotImplementedError("The evaluate method is not yet implemented") + + +@app.command("hotfix") +def hotfix(): + # NOOP command for typer to behave correctly. DO NOT REMOVE OR MODIFY + pass + + +if __name__ == "__main__": + app() From 02cec0166faeb4b63522cbd5190bdd2cb950d49b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Mon, 6 Mar 2023 12:06:36 -0500 Subject: [PATCH 17/57] Fix JSON Syntax Error --- templates/evaluator_mlcube/cookiecutter.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/evaluator_mlcube/cookiecutter.json b/templates/evaluator_mlcube/cookiecutter.json index eed0126ab..d716638f2 100644 --- a/templates/evaluator_mlcube/cookiecutter.json +++ b/templates/evaluator_mlcube/cookiecutter.json @@ -4,5 +4,5 @@ "description": "Data Preparator MLCube Template. Provided by MLCommons", "author_name": "John Smith", "accelerator_count": "0", - "docker_image_name": "docker/image:latest", + "docker_image_name": "docker/image:latest" } \ No newline at end of file From b3d7a1d85b849f29cfbd5f6c7dde4006af3c188f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Mon, 6 Mar 2023 12:09:03 -0500 Subject: [PATCH 18/57] Update template default values --- templates/evaluator_mlcube/cookiecutter.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/evaluator_mlcube/cookiecutter.json b/templates/evaluator_mlcube/cookiecutter.json index d716638f2..07cd7e295 100644 --- a/templates/evaluator_mlcube/cookiecutter.json +++ b/templates/evaluator_mlcube/cookiecutter.json @@ -1,7 +1,7 @@ { - "project_name": "Data Preparator MLCube", + "project_name": "Evaluator MLCube", "project_slug": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", - "description": "Data Preparator MLCube Template. Provided by MLCommons", + "description": "Evaluator MLCube Template. Provided by MLCommons", "author_name": "John Smith", "accelerator_count": "0", "docker_image_name": "docker/image:latest" From 7338236411635e7581008b25b6465f5055d50e4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Mon, 6 Mar 2023 12:09:20 -0500 Subject: [PATCH 19/57] Remove reference to undefined template variable --- templates/evaluator_mlcube/hooks/post_gen_project.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/templates/evaluator_mlcube/hooks/post_gen_project.py b/templates/evaluator_mlcube/hooks/post_gen_project.py index 74c69c0d7..e36bdaef0 100644 --- a/templates/evaluator_mlcube/hooks/post_gen_project.py +++ b/templates/evaluator_mlcube/hooks/post_gen_project.py @@ -9,10 +9,5 @@ labels_path = "mlcube/workspace/labels" paths = [preds_path, labels_path] - - if "{{ cookiecutter.use_separate_output_labels }}" == "y": - input_labels_path = "mlcube/workspace/input_labels" - paths.append(input_labels_path) - for path in paths: os.makedirs(path, exist_ok=True) From d1cec5e668c1d1fa312b0abf7e36cca30a07ad2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Mon, 6 Mar 2023 12:23:11 -0500 Subject: [PATCH 20/57] Implement model mlcube cookiecutter template --- templates/model_mlcube/cookiecutter.json | 8 +++++ .../model_mlcube/hooks/post_gen_project.py | 13 ++++++++ .../mlcube/mlcube.yaml | 29 +++++++++++++++++ .../mlcube/workspace/parameters.yaml | 0 .../project/Dockerfile | 31 +++++++++++++++++++ .../project/mlcube.py | 28 +++++++++++++++++ 6 files changed, 109 insertions(+) create mode 100644 templates/model_mlcube/cookiecutter.json create mode 100644 templates/model_mlcube/hooks/post_gen_project.py create mode 100644 templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml create mode 100644 templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml create mode 100644 templates/model_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile create mode 100644 templates/model_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py diff --git a/templates/model_mlcube/cookiecutter.json b/templates/model_mlcube/cookiecutter.json new file mode 100644 index 000000000..07cd7e295 --- /dev/null +++ b/templates/model_mlcube/cookiecutter.json @@ -0,0 +1,8 @@ +{ + "project_name": "Evaluator MLCube", + "project_slug": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", + "description": "Evaluator MLCube Template. Provided by MLCommons", + "author_name": "John Smith", + "accelerator_count": "0", + "docker_image_name": "docker/image:latest" +} \ No newline at end of file diff --git a/templates/model_mlcube/hooks/post_gen_project.py b/templates/model_mlcube/hooks/post_gen_project.py new file mode 100644 index 000000000..2864b0326 --- /dev/null +++ b/templates/model_mlcube/hooks/post_gen_project.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +import os + +PROJECT_DIRECTORY = os.path.realpath(os.path.curdir) + + +if __name__ == "__main__": + data_path = "mlcube/workspace/data" + preds_path = "mlcube/workspace/predictions" + + paths = [preds_path, data_path] + for path in paths: + os.makedirs(path, exist_ok=True) diff --git a/templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml new file mode 100644 index 000000000..a51a8b374 --- /dev/null +++ b/templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml @@ -0,0 +1,29 @@ +name: {{ cookiecutter.project_name }} +description: {{ cookiecutter.description }} +authors: + - {name: {{ cookiecutter.author_name }}} + +platform: + accelerator_count: {{ cookiecutter.accelerator_count }} + +docker: + # Image name + image: {{ cookiecutter.docker_image_name }} + # Docker build context relative to $MLCUBE_ROOT. Default is `build`. + build_context: "../project" + # Docker file name within docker build context, default is `Dockerfile`. + build_file: "Dockerfile" + +tasks: + infer: + # Computes predictions on input data + parameters: + inputs: { + data_path: data/, + parameters_file: parameters.yaml, + # Feel free to include other files required for inference. + # These files MUST go inside the additional_files path. + # e.g. model weights + # weights: additional_files/weights.pt, + } + outputs: {output_path: {type: directory, default: predictions}} \ No newline at end of file diff --git a/templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml b/templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/templates/model_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile b/templates/model_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile new file mode 100644 index 000000000..9f2ce5642 --- /dev/null +++ b/templates/model_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile @@ -0,0 +1,31 @@ +FROM ubuntu:18.04 +MAINTAINER MLPerf MLBox Working Group + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + software-properties-common \ + python3-dev \ + curl && \ + rm -rf /var/lib/apt/lists/* + +RUN add-apt-repository ppa:deadsnakes/ppa -y && apt-get update + +RUN apt-get install python3 -y + +RUN curl -fSsL -O https://bootstrap.pypa.io/pip/3.6/get-pip.py && \ + python3 get-pip.py && \ + rm get-pip.py + +COPY ./requirements.txt project/requirements.txt + +RUN pip3 install --upgrade pip + +RUN pip3 install --no-cache-dir -r project/requirements.txt + +ENV LANG C.UTF-8 + +COPY . /project + +WORKDIR /project + +ENTRYPOINT ["python3", "mlcube.py"] \ No newline at end of file diff --git a/templates/model_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py b/templates/model_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py new file mode 100644 index 000000000..4c6fb1f4a --- /dev/null +++ b/templates/model_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py @@ -0,0 +1,28 @@ +"""MLCube handler file""" +import typer + + +app = typer.Typer() + + +@app.command("infer") +def prepare( + data_path: str = typer.Option(..., "--data_path"), + parameters_file: str = typer.Option(..., "--parameters_file"), + output_path: str = typer.Option(..., "--output_path"), + # Provide additional parameters as described in the mlcube.yaml file + # e.g. model weights: + # weights: str = typer.Option(..., "--weights"), +): + # Modify the prepare command as needed + raise NotImplementedError("The evaluate method is not yet implemented") + + +@app.command("hotfix") +def hotfix(): + # NOOP command for typer to behave correctly. DO NOT REMOVE OR MODIFY + pass + + +if __name__ == "__main__": + app() From 7338755717d55cca03c8607978e06cdf4e2768a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Mon, 6 Mar 2023 12:25:27 -0500 Subject: [PATCH 21/57] Update cookiecutter variable default values --- templates/model_mlcube/cookiecutter.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/model_mlcube/cookiecutter.json b/templates/model_mlcube/cookiecutter.json index 07cd7e295..e354d1d6d 100644 --- a/templates/model_mlcube/cookiecutter.json +++ b/templates/model_mlcube/cookiecutter.json @@ -1,7 +1,7 @@ { - "project_name": "Evaluator MLCube", + "project_name": "Model MLCube", "project_slug": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", - "description": "Evaluator MLCube Template. Provided by MLCommons", + "description": "Model MLCube Template. Provided by MLCommons", "author_name": "John Smith", "accelerator_count": "0", "docker_image_name": "docker/image:latest" From 3ae92263b4c303158e66ad7eca8c8bf4c8bc85b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Mon, 6 Mar 2023 12:41:10 -0500 Subject: [PATCH 22/57] Create medperf CLI command for creating MLCubes --- cli/medperf/commands/mlcube/create.py | 24 ++++++++++++++++++++++++ cli/medperf/commands/mlcube/mlcube.py | 13 +++++++++++++ cli/medperf/config.py | 7 +++++++ 3 files changed, 44 insertions(+) create mode 100644 cli/medperf/commands/mlcube/create.py diff --git a/cli/medperf/commands/mlcube/create.py b/cli/medperf/commands/mlcube/create.py new file mode 100644 index 000000000..f1222b42b --- /dev/null +++ b/cli/medperf/commands/mlcube/create.py @@ -0,0 +1,24 @@ +from cookiecutter.main import cookiecutter + +from medperf import config +from medperf.exceptions import InvalidArgumentError + + +class CreateCube: + @classmethod + def run(cls, template_name: str): + """Creates a new MLCube based on one of the provided templates + + Args: + template_name (str): The name of the template to use + """ + repo = config.github_repository + template_dirs = config.templates + if template_name not in template_dirs: + templates = list(template_dirs.keys()) + raise InvalidArgumentError( + f"Invalid template name. Available templates: [{' | '.join(templates)}]" + ) + + template_dir = template_dirs[template_name] + cookiecutter(repo, directory=template_dir) diff --git a/cli/medperf/commands/mlcube/mlcube.py b/cli/medperf/commands/mlcube/mlcube.py index 64391ebb1..a6c5a0a80 100644 --- a/cli/medperf/commands/mlcube/mlcube.py +++ b/cli/medperf/commands/mlcube/mlcube.py @@ -7,6 +7,7 @@ from medperf.entities.cube import Cube from medperf.commands.list import EntityList from medperf.commands.view import EntityView +from medperf.commands.mlcube.create import CreateCube from medperf.commands.mlcube.submit import SubmitCube from medperf.commands.mlcube.associate import AssociateCube @@ -28,6 +29,18 @@ def list( ) +@app.command("create") +@clean_except +def create( + template: str = typer.Argument( + None, + help=f"MLCube template name. Available templates: [{' | '.join(config.templates.keys())}]", + ) +): + """Creates an MLCube based on one of the specified templates""" + CreateCube.run(template) + + @app.command("submit") @clean_except def submit( diff --git a/cli/medperf/config.py b/cli/medperf/config.py index c5f081cbe..2f9e5062c 100644 --- a/cli/medperf/config.py +++ b/cli/medperf/config.py @@ -70,3 +70,10 @@ "platform", "cleanup", ] + +github_repository = "https://github.com/aristizabal95/medperf-2" +templates = { + "data_preparator": "templates/data_preparator_mlcube", + "model": "templates/model_mlcube", + "evaluator": "templates/evaluator_mlcube", +} From e07cde26c241b13c59ca545f0955f617f02a6202 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Mon, 6 Mar 2023 16:01:06 -0500 Subject: [PATCH 23/57] Provide additional options for mlcube create --- cli/medperf/commands/mlcube/create.py | 16 ++++++++++++++-- cli/medperf/commands/mlcube/mlcube.py | 13 +++++++++++-- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/cli/medperf/commands/mlcube/create.py b/cli/medperf/commands/mlcube/create.py index f1222b42b..b87026ee3 100644 --- a/cli/medperf/commands/mlcube/create.py +++ b/cli/medperf/commands/mlcube/create.py @@ -6,11 +6,13 @@ class CreateCube: @classmethod - def run(cls, template_name: str): + def run(cls, template_name: str, output_path: str = ".", config_file: str = None): """Creates a new MLCube based on one of the provided templates Args: template_name (str): The name of the template to use + output_path (str, Optional): The desired path for the MLCube. Defaults to current path. + config_file (str, Optional): Path to a JSON configuration file. If not passed, user is prompted. """ repo = config.github_repository template_dirs = config.templates @@ -20,5 +22,15 @@ def run(cls, template_name: str): f"Invalid template name. Available templates: [{' | '.join(templates)}]" ) + no_input = False + if config_file is not None: + no_input = True + template_dir = template_dirs[template_name] - cookiecutter(repo, directory=template_dir) + cookiecutter( + repo, + directory=template_dir, + output_dir=output_path, + config_file=config_file, + no_input=no_input, + ) diff --git a/cli/medperf/commands/mlcube/mlcube.py b/cli/medperf/commands/mlcube/mlcube.py index a6c5a0a80..4a411287d 100644 --- a/cli/medperf/commands/mlcube/mlcube.py +++ b/cli/medperf/commands/mlcube/mlcube.py @@ -35,10 +35,19 @@ def create( template: str = typer.Argument( None, help=f"MLCube template name. Available templates: [{' | '.join(config.templates.keys())}]", - ) + ), + output_path: str = typer.Option( + ".", "--output", "-o", help="Save the generated MLCube to the specified path" + ), + config_file: str = typer.Option( + None, + "--config-file", + "-c", + help="JSON Configuration file. If not present then user is prompted for configuration", + ), ): """Creates an MLCube based on one of the specified templates""" - CreateCube.run(template) + CreateCube.run(template, output_path, config_file) @app.command("submit") From 68e136a0d8eddf85a76e1ec709d8a23763884886 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Tue, 7 Mar 2023 11:29:06 -0500 Subject: [PATCH 24/57] Start working on tests --- .../tests/commands/mlcube/test_create.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 cli/medperf/tests/commands/mlcube/test_create.py diff --git a/cli/medperf/tests/commands/mlcube/test_create.py b/cli/medperf/tests/commands/mlcube/test_create.py new file mode 100644 index 000000000..b9be20a69 --- /dev/null +++ b/cli/medperf/tests/commands/mlcube/test_create.py @@ -0,0 +1,46 @@ +import pytest +from unittest.mock import ANY + +from medperf import config +from medperf.commands.mlcube.create import CreateCube +from medperf.exceptions import InvalidArgumentError + +PATCH_CREATE = "medperf.commands.mlcube.create.{}" + + +@pytest.fixture +def setup(mocker): + spy = mocker.patch(PATCH_CREATE.format("cookiecutter")) + return spy + + +class TestTemplate: + @pytest.mark.parametrize("template,dir", list(config.templates.items())) + def test_valid_template_is_used(mocker, setup, template, dir): + # Arrange + spy = setup + + # Act + CreateCube.run(template) + + # Assert + spy.assert_called_once() + assert "directory" in spy.call_args.kwargs + assert spy.call_args.kwargs["directory"] == dir + + @pytest.mark.parametrize("template", ["invalid"]) + def test_invalid_template_raises_error(mocker, template): + # Act & Assert + with pytest.raises(InvalidArgumentError): + CreateCube.run(template) + + +class TestOutputPath: + def test_current_path_is_used_by_default(): + # TODO + pass + + @pytest.mark.paramterize("output_path", ["first/path", "second/path"]) + def test_output_path_is_used_for_template_creation(): + # TODO + pass From b8e03acf0a058c395bb41a7bc578d589ccd64deb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Tue, 7 Mar 2023 17:41:48 -0500 Subject: [PATCH 25/57] Add tests for cube create --- .../tests/commands/mlcube/test_create.py | 79 +++++++++++++++++-- 1 file changed, 71 insertions(+), 8 deletions(-) diff --git a/cli/medperf/tests/commands/mlcube/test_create.py b/cli/medperf/tests/commands/mlcube/test_create.py index b9be20a69..08ed19925 100644 --- a/cli/medperf/tests/commands/mlcube/test_create.py +++ b/cli/medperf/tests/commands/mlcube/test_create.py @@ -36,11 +36,74 @@ def test_invalid_template_raises_error(mocker, template): class TestOutputPath: - def test_current_path_is_used_by_default(): - # TODO - pass - - @pytest.mark.paramterize("output_path", ["first/path", "second/path"]) - def test_output_path_is_used_for_template_creation(): - # TODO - pass + def test_current_path_is_used_by_default(mocker, setup): + # Arrange + path = "." + spy = setup + template = list(config.templates.keys())[0] + + # Act + CreateCube.run(template) + + # Assert + spy.assert_called_once() + assert "output_dir" in spy.call_args.kwargs + assert spy.call_args.kwargs["output_dir"] == path + + @pytest.mark.parametrize("output_path", ["first/path", "second/path"]) + def test_output_path_is_used_for_template_creation(mocker, setup, output_path): + # Arrange + spy = setup + template = list(config.templates.keys())[0] + + # Act + CreateCube.run(template, output_path=output_path) + + # Assert + spy.assert_called_once() + assert "output_dir" in spy.call_args.kwargs + assert spy.call_args.kwargs["output_dir"] == output_path + + +class TestConfigFile: + def test_config_file_is_disabled_by_default(mocker, setup): + # Arrange + spy = setup + template = list(config.templates.keys())[0] + + # Act + CreateCube.run(template) + + # Assert + spy.assert_called_once() + assert "config_file" in spy.call_args.kwargs + assert spy.call_args.kwargs["config_file"] is None + + @pytest.mark.parametrize("config_file", ["path/to/config.json"]) + def test_config_file_is_used_when_passed(mocker, setup, config_file): + # Arrange + spy = setup + template = list(config.templates.keys())[0] + + # Act + CreateCube.run(template, config_file=config_file) + + # Assert + spy.assert_called_once() + assert "config_file" in spy.call_args.kwargs + assert spy.call_args.kwargs["config_file"] is config_file + + @pytest.mark.parametrize("config_file", [None, "config.json"]) + def test_passing_config_file_disables_input(mocker, setup, config_file): + # Arrange + spy = setup + should_not_input = config_file is not None + template = list(config.templates.keys())[0] + + # Act + CreateCube.run(template, config_file=config_file) + + # Assert + spy.assert_called_once() + assert "no_input" in spy.call_args.kwargs + assert spy.call_args.kwargs["no_input"] == should_not_input From 7896b256d1876be6106575c6272e1cb4817f9aff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Tue, 7 Mar 2023 17:50:31 -0500 Subject: [PATCH 26/57] Ignore invalid syntax on cookiecutter conditionals --- .../{{cookiecutter.project_slug}}/mlcube/mlcube.yaml | 6 +++--- .../{{cookiecutter.project_slug}}/project/mlcube.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml index a617061b9..f78c4a4cd 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml @@ -20,9 +20,9 @@ tasks: inputs: {data_path: input_data, labels_path: input_labels, parameters_file: parameters.yaml} outputs: { output_path: data/, - {% if cookiecutter.use_separate_output_labels == 'y' -%} - output_labels_path: labels/ - {% endif %} + {% if cookiecutter.use_separate_output_labels == 'y' -%} # noqa: E999 + output_labels_path: labels/ # noqa: E999 + {% endif %} # noqa: E999 } sanity_check: parameters: diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py index 7b38614c9..3892ef2bc 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py @@ -11,9 +11,9 @@ def prepare( labels_path: str = typer.Option(..., "--labels_path"), parameters_file: str = typer.Option(..., "--parameters_file"), output_path: str = typer.Option(..., "--output_path"), - {% if cookiecutter.use_separate_output_labels == 'y' -%} + {% if cookiecutter.use_separate_output_labels == 'y' -%} # noqa: E999 output_labels_path: str = typer.Option(..., "--output_labels_path"), - {% endif %} + {% endif %} # noqa: E999 ): # Modify the prepare command as needed raise NotImplementedError("The prepare method is not yet implemented") From 4f78981d3a337e452b1bbcdcfb2fbe79c34b021b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Tue, 7 Mar 2023 17:55:41 -0500 Subject: [PATCH 27/57] Ignore more flake8 errors --- .../{{cookiecutter.project_slug}}/project/mlcube.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py index 3892ef2bc..04afcb3a3 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py @@ -11,9 +11,9 @@ def prepare( labels_path: str = typer.Option(..., "--labels_path"), parameters_file: str = typer.Option(..., "--parameters_file"), output_path: str = typer.Option(..., "--output_path"), - {% if cookiecutter.use_separate_output_labels == 'y' -%} # noqa: E999 + {% if cookiecutter.use_separate_output_labels == 'y' -%} # noqa: E999 E225 output_labels_path: str = typer.Option(..., "--output_labels_path"), - {% endif %} # noqa: E999 + {% endif %} # noqa: E999 E225 ): # Modify the prepare command as needed raise NotImplementedError("The prepare method is not yet implemented") From f5dab5ecd3edb94b1370b1b7cf489a371b640f6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Tue, 7 Mar 2023 17:57:32 -0500 Subject: [PATCH 28/57] Remove unused import --- cli/medperf/tests/commands/mlcube/test_create.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cli/medperf/tests/commands/mlcube/test_create.py b/cli/medperf/tests/commands/mlcube/test_create.py index 08ed19925..da52d72ab 100644 --- a/cli/medperf/tests/commands/mlcube/test_create.py +++ b/cli/medperf/tests/commands/mlcube/test_create.py @@ -1,5 +1,4 @@ import pytest -from unittest.mock import ANY from medperf import config from medperf.commands.mlcube.create import CreateCube From a03d7f6aa16ba654312b69eef616005474ac067b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Wed, 8 Mar 2023 10:12:06 -0500 Subject: [PATCH 29/57] Empty commit for cloudbuild From 6bb60d0460a5f3b530bcf71afd0ee68d90a367d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Wed, 8 Mar 2023 18:41:35 -0500 Subject: [PATCH 30/57] Fix inconsistency with labels paths --- .../data_preparator_mlcube/hooks/post_gen_project.py | 8 ++++---- .../{{cookiecutter.project_slug}}/mlcube/mlcube.yaml | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/templates/data_preparator_mlcube/hooks/post_gen_project.py b/templates/data_preparator_mlcube/hooks/post_gen_project.py index 540b83bd7..68a54ddc9 100644 --- a/templates/data_preparator_mlcube/hooks/post_gen_project.py +++ b/templates/data_preparator_mlcube/hooks/post_gen_project.py @@ -6,14 +6,14 @@ if __name__ == "__main__": input_data_path = "mlcube/workspace/input_data" + input_labels_path = "mlcube/workspace/input_labels" data_path = "mlcube/workspace/data" - labels_path = "mlcube/workspace/labels" - paths = [input_data_path, data_path, labels_path] + paths = [input_data_path, input_labels_path, data_path] if "{{ cookiecutter.use_separate_output_labels }}" == "y": - input_labels_path = "mlcube/workspace/input_labels" - paths.append(input_labels_path) + labels_path = "mlcube/workspace/labels" + paths.append(labels_path) for path in paths: os.makedirs(path, exist_ok=True) diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml index f78c4a4cd..aa4f4e4f8 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml @@ -22,6 +22,8 @@ tasks: output_path: data/, {% if cookiecutter.use_separate_output_labels == 'y' -%} # noqa: E999 output_labels_path: labels/ # noqa: E999 + {% else %} # noqa: E999 + # output_labels_path: labels/ {% endif %} # noqa: E999 } sanity_check: From 43b6cabdd095cef591d0a2c0d866a0c6dec84b39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Wed, 8 Mar 2023 18:45:40 -0500 Subject: [PATCH 31/57] Update mlcube.yaml so it can be commented on docs --- .../mlcube/mlcube.yaml | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml index aa4f4e4f8..db9855b5c 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml @@ -17,7 +17,11 @@ docker: tasks: prepare: parameters: - inputs: {data_path: input_data, labels_path: input_labels, parameters_file: parameters.yaml} + inputs: { + data_path: input_data, + labels_path: input_labels, + parameters_file: parameters.yaml + } outputs: { output_path: data/, {% if cookiecutter.use_separate_output_labels == 'y' -%} # noqa: E999 @@ -28,8 +32,16 @@ tasks: } sanity_check: parameters: - inputs: {data_path: data/, parameters_file: parameters.yaml} + inputs: { + data_path: data/, + parameters_file: parameters.yaml + } statistics: parameters: - inputs: {data_path: data/, parameters_file: parameters.yaml} - outputs: {output_path: {type: file, default: statistics.yaml}} \ No newline at end of file + inputs: { + data_path: data/, + parameters_file: parameters.yaml + } + outputs: { + output_path: {type: file, default: statistics.yaml} + } \ No newline at end of file From 55b5d22a170b72dcb7899b50b479925080f13239 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Wed, 8 Mar 2023 18:53:51 -0500 Subject: [PATCH 32/57] Don't render noqa comments on template --- .../{{cookiecutter.project_slug}}/mlcube/mlcube.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml index db9855b5c..4041f9ccb 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml @@ -25,10 +25,10 @@ tasks: outputs: { output_path: data/, {% if cookiecutter.use_separate_output_labels == 'y' -%} # noqa: E999 - output_labels_path: labels/ # noqa: E999 - {% else %} # noqa: E999 + output_labels_path: labels/ {# noqa: E999} + {% else %} {# noqa: E999} # output_labels_path: labels/ - {% endif %} # noqa: E999 + {% endif %} {# noqa: E999} } sanity_check: parameters: From 135c59846e3e0736bcf6672127bc2d4a6cffd20b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Wed, 8 Mar 2023 18:55:56 -0500 Subject: [PATCH 33/57] Remove flake8 specific ignores --- .../{{cookiecutter.project_slug}}/mlcube/mlcube.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml index 4041f9ccb..43dcbb712 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml @@ -24,11 +24,11 @@ tasks: } outputs: { output_path: data/, - {% if cookiecutter.use_separate_output_labels == 'y' -%} # noqa: E999 - output_labels_path: labels/ {# noqa: E999} - {% else %} {# noqa: E999} + {% if cookiecutter.use_separate_output_labels == 'y' -%} + output_labels_path: labels/ + {% else %} # output_labels_path: labels/ - {% endif %} {# noqa: E999} + {% endif %} } sanity_check: parameters: From e9e2c329a06a720297331ca688f6527bfab98dc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Wed, 8 Mar 2023 18:56:42 -0500 Subject: [PATCH 34/57] Exclude templates from lint checks --- .github/workflows/unittests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 02d373692..a8d0ca017 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -33,7 +33,7 @@ jobs: # Exclude examples folder as it doesn't contain code related to medperf tools # Exclude migrations folder as it contains autogenerated code # Ignore E231, as it is raising warnings with auto-generated code. - flake8 . --count --max-complexity=10 --max-line-length=127 --ignore F821,W503,E231 --statistics --exclude=examples/,"*/migrations/*" + flake8 . --count --max-complexity=10 --max-line-length=127 --ignore F821,W503,E231 --statistics --exclude=examples/,"*/migrations/*",templates/ - name: Test with pytest run: | pytest From e95dab89ec5513e9c0a728aa02a946622ac9eb84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Wed, 8 Mar 2023 18:57:17 -0500 Subject: [PATCH 35/57] Remove specific flake8 ignores --- .../{{cookiecutter.project_slug}}/project/mlcube.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py index 04afcb3a3..7b38614c9 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py @@ -11,9 +11,9 @@ def prepare( labels_path: str = typer.Option(..., "--labels_path"), parameters_file: str = typer.Option(..., "--parameters_file"), output_path: str = typer.Option(..., "--output_path"), - {% if cookiecutter.use_separate_output_labels == 'y' -%} # noqa: E999 E225 + {% if cookiecutter.use_separate_output_labels == 'y' -%} output_labels_path: str = typer.Option(..., "--output_labels_path"), - {% endif %} # noqa: E999 E225 + {% endif %} ): # Modify the prepare command as needed raise NotImplementedError("The prepare method is not yet implemented") From d059b7a3ea46e80d121b65ea8f75a5cab3c7a8be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Fri, 10 Mar 2023 14:27:30 -0500 Subject: [PATCH 36/57] Fix labels_paht being passed in he wrong situation --- .../mlcube/mlcube.yaml | 14 ++++++++++++-- .../project/mlcube.py | 4 ++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml index 43dcbb712..e546012d2 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml @@ -25,21 +25,31 @@ tasks: outputs: { output_path: data/, {% if cookiecutter.use_separate_output_labels == 'y' -%} - output_labels_path: labels/ + output_labels_path: labels/, {% else %} - # output_labels_path: labels/ + # output_labels_path: labels/, {% endif %} } sanity_check: parameters: inputs: { data_path: data/, + {% if cookiecutter.use_separate_output_labels == 'y' -%} + labels_path: labels/, + {% else %} + # labels_path: labels/, + {% endif %} parameters_file: parameters.yaml } statistics: parameters: inputs: { data_path: data/, + {% if cookiecutter.use_separate_output_labels == 'y' -%} + labels_path: labels/, + {% else %} + # labels_path: labels/, + {% endif %} parameters_file: parameters.yaml } outputs: { diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py index 7b38614c9..36cb01de1 100644 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py @@ -22,7 +22,9 @@ def prepare( @app.command("sanity_check") def sanity_check( data_path: str = typer.Option(..., "--data_path"), + {% if cookiecutter.use_separate_output_labels == 'y' -%} labels_path: str = typer.Option(..., "--labels_path"), + {% endif %} parameters_file: str = typer.Option(..., "--parameters_file"), ): # Modify the sanity_check command as needed @@ -32,7 +34,9 @@ def sanity_check( @app.command("statistics") def sanity_check( data_path: str = typer.Option(..., "--data_path"), + {% if cookiecutter.use_separate_output_labels == 'y' -%} labels_path: str = typer.Option(..., "--labels_path"), + {% endif %} parameters_file: str = typer.Option(..., "--parameters_file"), out_path: str = typer.Option(..., "--output_path"), ): From fcdaa7bc05c68bda03ea49d7679bc89156730695 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Mon, 13 Mar 2023 10:01:28 -0500 Subject: [PATCH 37/57] Add requirements to cookiecutters --- .../{{cookiecutter.project_slug}}/project/requirements.txt | 2 ++ .../{{cookiecutter.project_slug}}/project/requirements.txt | 2 ++ .../{{cookiecutter.project_slug}}/project/requirements.txt | 2 ++ 3 files changed, 6 insertions(+) create mode 100644 templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt create mode 100644 templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt create mode 100644 templates/model_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt new file mode 100644 index 000000000..ff8197b93 --- /dev/null +++ b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt @@ -0,0 +1,2 @@ +typer +# Include all your requirements here \ No newline at end of file diff --git a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt new file mode 100644 index 000000000..ff8197b93 --- /dev/null +++ b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt @@ -0,0 +1,2 @@ +typer +# Include all your requirements here \ No newline at end of file diff --git a/templates/model_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt b/templates/model_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt new file mode 100644 index 000000000..ff8197b93 --- /dev/null +++ b/templates/model_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt @@ -0,0 +1,2 @@ +typer +# Include all your requirements here \ No newline at end of file From 37f3f3ce1277295a62ec82983c4fefd1edda1bd7 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Tue, 14 Mar 2023 10:26:12 -0500 Subject: [PATCH 38/57] Set separate labels as true by default --- templates/data_preparator_mlcube/cookiecutter.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/data_preparator_mlcube/cookiecutter.json b/templates/data_preparator_mlcube/cookiecutter.json index 95b9a12d5..1eb3db7de 100644 --- a/templates/data_preparator_mlcube/cookiecutter.json +++ b/templates/data_preparator_mlcube/cookiecutter.json @@ -5,5 +5,5 @@ "author_name": "John Smith", "accelerator_count": "0", "docker_image_name": "docker/image:latest", - "use_separate_output_labels": "n" + "use_separate_output_labels": "y" } \ No newline at end of file From 7a33c23c74768b4ff4df1e156a8d05f3fee37db7 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Fri, 31 Mar 2023 11:42:00 -0500 Subject: [PATCH 39/57] Remove duplicate templates --- .../data_preparator_mlcube/cookiecutter.json | 9 --- .../hooks/post_gen_project.py | 19 ------- .../mlcube/mlcube.yaml | 57 ------------------- .../mlcube/workspace/parameters.yaml | 0 .../project/Dockerfile | 31 ---------- .../project/mlcube.py | 48 ---------------- .../project/requirements.txt | 2 - templates/evaluator_mlcube/cookiecutter.json | 8 --- .../hooks/post_gen_project.py | 13 ----- .../mlcube/mlcube.yaml | 22 ------- .../mlcube/workspace/parameters.yaml | 0 .../project/Dockerfile | 31 ---------- .../project/mlcube.py | 26 --------- .../project/requirements.txt | 2 - templates/model_mlcube/cookiecutter.json | 8 --- .../model_mlcube/hooks/post_gen_project.py | 13 ----- .../mlcube/mlcube.yaml | 29 ---------- .../mlcube/workspace/parameters.yaml | 0 .../project/Dockerfile | 31 ---------- .../project/mlcube.py | 28 --------- .../project/requirements.txt | 2 - 21 files changed, 379 deletions(-) delete mode 100644 templates/data_preparator_mlcube/cookiecutter.json delete mode 100644 templates/data_preparator_mlcube/hooks/post_gen_project.py delete mode 100644 templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml delete mode 100644 templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml delete mode 100644 templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile delete mode 100644 templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py delete mode 100644 templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt delete mode 100644 templates/evaluator_mlcube/cookiecutter.json delete mode 100644 templates/evaluator_mlcube/hooks/post_gen_project.py delete mode 100644 templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml delete mode 100644 templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml delete mode 100644 templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile delete mode 100644 templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py delete mode 100644 templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt delete mode 100644 templates/model_mlcube/cookiecutter.json delete mode 100644 templates/model_mlcube/hooks/post_gen_project.py delete mode 100644 templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml delete mode 100644 templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml delete mode 100644 templates/model_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile delete mode 100644 templates/model_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py delete mode 100644 templates/model_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt diff --git a/templates/data_preparator_mlcube/cookiecutter.json b/templates/data_preparator_mlcube/cookiecutter.json deleted file mode 100644 index 1eb3db7de..000000000 --- a/templates/data_preparator_mlcube/cookiecutter.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "project_name": "Data Preparator MLCube", - "project_slug": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", - "description": "Data Preparator MLCube Template. Provided by MLCommons", - "author_name": "John Smith", - "accelerator_count": "0", - "docker_image_name": "docker/image:latest", - "use_separate_output_labels": "y" -} \ No newline at end of file diff --git a/templates/data_preparator_mlcube/hooks/post_gen_project.py b/templates/data_preparator_mlcube/hooks/post_gen_project.py deleted file mode 100644 index 68a54ddc9..000000000 --- a/templates/data_preparator_mlcube/hooks/post_gen_project.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -import os - -PROJECT_DIRECTORY = os.path.realpath(os.path.curdir) - - -if __name__ == "__main__": - input_data_path = "mlcube/workspace/input_data" - input_labels_path = "mlcube/workspace/input_labels" - data_path = "mlcube/workspace/data" - - paths = [input_data_path, input_labels_path, data_path] - - if "{{ cookiecutter.use_separate_output_labels }}" == "y": - labels_path = "mlcube/workspace/labels" - paths.append(labels_path) - - for path in paths: - os.makedirs(path, exist_ok=True) diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml deleted file mode 100644 index e546012d2..000000000 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml +++ /dev/null @@ -1,57 +0,0 @@ -name: {{ cookiecutter.project_name }} -description: {{ cookiecutter.description }} -authors: - - {name: {{ cookiecutter.author_name }}} - -platform: - accelerator_count: {{ cookiecutter.accelerator_count }} - -docker: - # Image name - image: {{ cookiecutter.docker_image_name }} - # Docker build context relative to $MLCUBE_ROOT. Default is `build`. - build_context: "../project" - # Docker file name within docker build context, default is `Dockerfile`. - build_file: "Dockerfile" - -tasks: - prepare: - parameters: - inputs: { - data_path: input_data, - labels_path: input_labels, - parameters_file: parameters.yaml - } - outputs: { - output_path: data/, - {% if cookiecutter.use_separate_output_labels == 'y' -%} - output_labels_path: labels/, - {% else %} - # output_labels_path: labels/, - {% endif %} - } - sanity_check: - parameters: - inputs: { - data_path: data/, - {% if cookiecutter.use_separate_output_labels == 'y' -%} - labels_path: labels/, - {% else %} - # labels_path: labels/, - {% endif %} - parameters_file: parameters.yaml - } - statistics: - parameters: - inputs: { - data_path: data/, - {% if cookiecutter.use_separate_output_labels == 'y' -%} - labels_path: labels/, - {% else %} - # labels_path: labels/, - {% endif %} - parameters_file: parameters.yaml - } - outputs: { - output_path: {type: file, default: statistics.yaml} - } \ No newline at end of file diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml deleted file mode 100644 index e69de29bb..000000000 diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile deleted file mode 100644 index 9f2ce5642..000000000 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM ubuntu:18.04 -MAINTAINER MLPerf MLBox Working Group - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - software-properties-common \ - python3-dev \ - curl && \ - rm -rf /var/lib/apt/lists/* - -RUN add-apt-repository ppa:deadsnakes/ppa -y && apt-get update - -RUN apt-get install python3 -y - -RUN curl -fSsL -O https://bootstrap.pypa.io/pip/3.6/get-pip.py && \ - python3 get-pip.py && \ - rm get-pip.py - -COPY ./requirements.txt project/requirements.txt - -RUN pip3 install --upgrade pip - -RUN pip3 install --no-cache-dir -r project/requirements.txt - -ENV LANG C.UTF-8 - -COPY . /project - -WORKDIR /project - -ENTRYPOINT ["python3", "mlcube.py"] \ No newline at end of file diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py deleted file mode 100644 index 36cb01de1..000000000 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py +++ /dev/null @@ -1,48 +0,0 @@ -"""MLCube handler file""" -import typer - - -app = typer.Typer() - - -@app.command("prepare") -def prepare( - data_path: str = typer.Option(..., "--data_path"), - labels_path: str = typer.Option(..., "--labels_path"), - parameters_file: str = typer.Option(..., "--parameters_file"), - output_path: str = typer.Option(..., "--output_path"), - {% if cookiecutter.use_separate_output_labels == 'y' -%} - output_labels_path: str = typer.Option(..., "--output_labels_path"), - {% endif %} -): - # Modify the prepare command as needed - raise NotImplementedError("The prepare method is not yet implemented") - - -@app.command("sanity_check") -def sanity_check( - data_path: str = typer.Option(..., "--data_path"), - {% if cookiecutter.use_separate_output_labels == 'y' -%} - labels_path: str = typer.Option(..., "--labels_path"), - {% endif %} - parameters_file: str = typer.Option(..., "--parameters_file"), -): - # Modify the sanity_check command as needed - raise NotImplementedError("The sanity check method is not yet implemented") - - -@app.command("statistics") -def sanity_check( - data_path: str = typer.Option(..., "--data_path"), - {% if cookiecutter.use_separate_output_labels == 'y' -%} - labels_path: str = typer.Option(..., "--labels_path"), - {% endif %} - parameters_file: str = typer.Option(..., "--parameters_file"), - out_path: str = typer.Option(..., "--output_path"), -): - # Modify the statistics command as needed - raise NotImplementedError("The statistics method is not yet implemented") - - -if __name__ == "__main__": - app() diff --git a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt b/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt deleted file mode 100644 index ff8197b93..000000000 --- a/templates/data_preparator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -typer -# Include all your requirements here \ No newline at end of file diff --git a/templates/evaluator_mlcube/cookiecutter.json b/templates/evaluator_mlcube/cookiecutter.json deleted file mode 100644 index 07cd7e295..000000000 --- a/templates/evaluator_mlcube/cookiecutter.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "project_name": "Evaluator MLCube", - "project_slug": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", - "description": "Evaluator MLCube Template. Provided by MLCommons", - "author_name": "John Smith", - "accelerator_count": "0", - "docker_image_name": "docker/image:latest" -} \ No newline at end of file diff --git a/templates/evaluator_mlcube/hooks/post_gen_project.py b/templates/evaluator_mlcube/hooks/post_gen_project.py deleted file mode 100644 index e36bdaef0..000000000 --- a/templates/evaluator_mlcube/hooks/post_gen_project.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python -import os - -PROJECT_DIRECTORY = os.path.realpath(os.path.curdir) - - -if __name__ == "__main__": - preds_path = "mlcube/workspace/predictions" - labels_path = "mlcube/workspace/labels" - - paths = [preds_path, labels_path] - for path in paths: - os.makedirs(path, exist_ok=True) diff --git a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml deleted file mode 100644 index c486308c9..000000000 --- a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: {{ cookiecutter.project_name }} -description: {{ cookiecutter.description }} -authors: - - {name: {{ cookiecutter.author_name }}} - -platform: - accelerator_count: {{ cookiecutter.accelerator_count }} - -docker: - # Image name - image: {{ cookiecutter.docker_image_name }} - # Docker build context relative to $MLCUBE_ROOT. Default is `build`. - build_context: "../project" - # Docker file name within docker build context, default is `Dockerfile`. - build_file: "Dockerfile" - -tasks: - evaluate: - # Computes evaluation metrics on the given predictions and ground truths - parameters: - inputs: {predictions: predictions, labels: labels, parameters_file: parameters.yaml} - outputs: {output_path: {type: "file", default: "results.yaml"}} \ No newline at end of file diff --git a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml deleted file mode 100644 index e69de29bb..000000000 diff --git a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile deleted file mode 100644 index 9f2ce5642..000000000 --- a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM ubuntu:18.04 -MAINTAINER MLPerf MLBox Working Group - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - software-properties-common \ - python3-dev \ - curl && \ - rm -rf /var/lib/apt/lists/* - -RUN add-apt-repository ppa:deadsnakes/ppa -y && apt-get update - -RUN apt-get install python3 -y - -RUN curl -fSsL -O https://bootstrap.pypa.io/pip/3.6/get-pip.py && \ - python3 get-pip.py && \ - rm get-pip.py - -COPY ./requirements.txt project/requirements.txt - -RUN pip3 install --upgrade pip - -RUN pip3 install --no-cache-dir -r project/requirements.txt - -ENV LANG C.UTF-8 - -COPY . /project - -WORKDIR /project - -ENTRYPOINT ["python3", "mlcube.py"] \ No newline at end of file diff --git a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py deleted file mode 100644 index 1de721eb9..000000000 --- a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py +++ /dev/null @@ -1,26 +0,0 @@ -"""MLCube handler file""" -import typer - - -app = typer.Typer() - - -@app.command("evaluate") -def prepare( - labels: str = typer.Option(..., "--labels"), - predictions: str = typer.Option(..., "--predictions"), - parameters_file: str = typer.Option(..., "--parameters_file"), - output_path: str = typer.Option(..., "--output_path"), -): - # Modify the prepare command as needed - raise NotImplementedError("The evaluate method is not yet implemented") - - -@app.command("hotfix") -def hotfix(): - # NOOP command for typer to behave correctly. DO NOT REMOVE OR MODIFY - pass - - -if __name__ == "__main__": - app() diff --git a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt b/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt deleted file mode 100644 index ff8197b93..000000000 --- a/templates/evaluator_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -typer -# Include all your requirements here \ No newline at end of file diff --git a/templates/model_mlcube/cookiecutter.json b/templates/model_mlcube/cookiecutter.json deleted file mode 100644 index e354d1d6d..000000000 --- a/templates/model_mlcube/cookiecutter.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "project_name": "Model MLCube", - "project_slug": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}", - "description": "Model MLCube Template. Provided by MLCommons", - "author_name": "John Smith", - "accelerator_count": "0", - "docker_image_name": "docker/image:latest" -} \ No newline at end of file diff --git a/templates/model_mlcube/hooks/post_gen_project.py b/templates/model_mlcube/hooks/post_gen_project.py deleted file mode 100644 index 2864b0326..000000000 --- a/templates/model_mlcube/hooks/post_gen_project.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python -import os - -PROJECT_DIRECTORY = os.path.realpath(os.path.curdir) - - -if __name__ == "__main__": - data_path = "mlcube/workspace/data" - preds_path = "mlcube/workspace/predictions" - - paths = [preds_path, data_path] - for path in paths: - os.makedirs(path, exist_ok=True) diff --git a/templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml b/templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml deleted file mode 100644 index a51a8b374..000000000 --- a/templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/mlcube.yaml +++ /dev/null @@ -1,29 +0,0 @@ -name: {{ cookiecutter.project_name }} -description: {{ cookiecutter.description }} -authors: - - {name: {{ cookiecutter.author_name }}} - -platform: - accelerator_count: {{ cookiecutter.accelerator_count }} - -docker: - # Image name - image: {{ cookiecutter.docker_image_name }} - # Docker build context relative to $MLCUBE_ROOT. Default is `build`. - build_context: "../project" - # Docker file name within docker build context, default is `Dockerfile`. - build_file: "Dockerfile" - -tasks: - infer: - # Computes predictions on input data - parameters: - inputs: { - data_path: data/, - parameters_file: parameters.yaml, - # Feel free to include other files required for inference. - # These files MUST go inside the additional_files path. - # e.g. model weights - # weights: additional_files/weights.pt, - } - outputs: {output_path: {type: directory, default: predictions}} \ No newline at end of file diff --git a/templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml b/templates/model_mlcube/{{cookiecutter.project_slug}}/mlcube/workspace/parameters.yaml deleted file mode 100644 index e69de29bb..000000000 diff --git a/templates/model_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile b/templates/model_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile deleted file mode 100644 index 9f2ce5642..000000000 --- a/templates/model_mlcube/{{cookiecutter.project_slug}}/project/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM ubuntu:18.04 -MAINTAINER MLPerf MLBox Working Group - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - software-properties-common \ - python3-dev \ - curl && \ - rm -rf /var/lib/apt/lists/* - -RUN add-apt-repository ppa:deadsnakes/ppa -y && apt-get update - -RUN apt-get install python3 -y - -RUN curl -fSsL -O https://bootstrap.pypa.io/pip/3.6/get-pip.py && \ - python3 get-pip.py && \ - rm get-pip.py - -COPY ./requirements.txt project/requirements.txt - -RUN pip3 install --upgrade pip - -RUN pip3 install --no-cache-dir -r project/requirements.txt - -ENV LANG C.UTF-8 - -COPY . /project - -WORKDIR /project - -ENTRYPOINT ["python3", "mlcube.py"] \ No newline at end of file diff --git a/templates/model_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py b/templates/model_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py deleted file mode 100644 index 4c6fb1f4a..000000000 --- a/templates/model_mlcube/{{cookiecutter.project_slug}}/project/mlcube.py +++ /dev/null @@ -1,28 +0,0 @@ -"""MLCube handler file""" -import typer - - -app = typer.Typer() - - -@app.command("infer") -def prepare( - data_path: str = typer.Option(..., "--data_path"), - parameters_file: str = typer.Option(..., "--parameters_file"), - output_path: str = typer.Option(..., "--output_path"), - # Provide additional parameters as described in the mlcube.yaml file - # e.g. model weights: - # weights: str = typer.Option(..., "--weights"), -): - # Modify the prepare command as needed - raise NotImplementedError("The evaluate method is not yet implemented") - - -@app.command("hotfix") -def hotfix(): - # NOOP command for typer to behave correctly. DO NOT REMOVE OR MODIFY - pass - - -if __name__ == "__main__": - app() diff --git a/templates/model_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt b/templates/model_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt deleted file mode 100644 index ff8197b93..000000000 --- a/templates/model_mlcube/{{cookiecutter.project_slug}}/project/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -typer -# Include all your requirements here \ No newline at end of file From fae57fb5327861d1693d9624326cd9c844d9076f Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Wed, 19 Apr 2023 11:05:30 -0500 Subject: [PATCH 40/57] Update requirements --- cli/requirements.txt | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cli/requirements.txt b/cli/requirements.txt index 4b5407816..9772c45f5 100644 --- a/cli/requirements.txt +++ b/cli/requirements.txt @@ -1,6 +1,5 @@ typer~=0.6.0; sys_platform != "win32" typer<0.5.0; sys_platform == "win32" -click>=8.0.0,<9.0.0 rich~=12.5.0 PyYAML>=5.4,<6 requests>=2.26.0 @@ -8,13 +7,13 @@ pydantic==1.10.2 yaspin==2.1.0 tabulate==0.8.9 pexpect==4.8.0 -wexpect==2.3.2; sys_platform == "win32" +wexpect==4.0.0; sys_platform == "win32" colorama==0.4.4 time-machine==2.4.0 pytest-mock==1.13.0 pyfakefs==5.0.0 -mlcube==0.0.8 -mlcube-docker==0.0.8 -mlcube-singularity==0.0.8 +mlcube==0.0.9 +mlcube-docker==0.0.9 +mlcube-singularity==0.0.9; sys_platform != "win32" validators==0.18.2 -merge-args==0.1.4 \ No newline at end of file +merge-args==0.1.4 From 1e022c7775fccd4a9f25d095d1b1d0da93e81819 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Wed, 19 Apr 2023 11:05:43 -0500 Subject: [PATCH 41/57] Enforce UTF8 file encoding --- cli/medperf/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/medperf/__main__.py b/cli/medperf/__main__.py index dfe931e74..eba87e93e 100644 --- a/cli/medperf/__main__.py +++ b/cli/medperf/__main__.py @@ -175,7 +175,7 @@ def main(ctx: typer.Context): log_fmt = "%(asctime)s | %(levelname)s: %(message)s" log_file = storage_path(config.log_file) handler = logging.handlers.RotatingFileHandler( - log_file, maxBytes=10000000, backupCount=5 + log_file, maxBytes=10000000, backupCount=5, encoding="utf-8" ) handler.setFormatter(logging.Formatter(log_fmt)) logging.basicConfig( From 6739bb5faa845dfa52f5631086684b8c8ae9e4b7 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Wed, 19 Apr 2023 11:06:29 -0500 Subject: [PATCH 42/57] Fix wexpect side effects --- cli/medperf/entities/cube.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/medperf/entities/cube.py b/cli/medperf/entities/cube.py index f59fa0c88..44e47a553 100644 --- a/cli/medperf/entities/cube.py +++ b/cli/medperf/entities/cube.py @@ -215,7 +215,7 @@ def download_image(self): # Retrieve image from image registry logging.debug(f"Retrieving {self.id} image") cmd = f"mlcube configure --mlcube={self.cube_path}" - proc = spawn(cmd) + proc = spawn(cmd, timeout=None) proc_out = combine_proc_sp_text(proc) logging.debug(proc_out) proc.close() @@ -282,7 +282,7 @@ def run( proc_out = combine_proc_sp_text(proc) proc.close() logging.debug(proc_out) - if proc.exitstatus != 0: + if proc.exitstatus and proc.exitstatus != 0: raise ExecutionError("There was an error while executing the cube") logging.debug(list_files(config.storage)) From bc8939be6209ee4f84665635fb8034feed2a6774 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Wed, 19 Apr 2023 11:10:38 -0500 Subject: [PATCH 43/57] Undo merge changes --- cli/8.0.0 | 0 cli/medperf/entities/benchmark.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 cli/8.0.0 diff --git a/cli/8.0.0 b/cli/8.0.0 deleted file mode 100644 index e69de29bb..000000000 diff --git a/cli/medperf/entities/benchmark.py b/cli/medperf/entities/benchmark.py index 93095c89e..5c1bfe04a 100644 --- a/cli/medperf/entities/benchmark.py +++ b/cli/medperf/entities/benchmark.py @@ -24,7 +24,7 @@ class Benchmark(Entity, MedperfSchema, ApprovableSchema, DeployableSchema): """ description: Optional[str] = Field(None, max_length=20) - docs_url: Optional[str] + docs_url: Optional[HttpUrl] demo_dataset_tarball_url: Optional[HttpUrl] demo_dataset_tarball_hash: Optional[str] demo_dataset_generated_uid: Optional[str] From 38255e83f4192266880c89c7b3664919a739b8f9 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Wed, 19 Apr 2023 11:23:51 -0500 Subject: [PATCH 44/57] Update setup_logging --- cli/medperf/utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cli/medperf/utils.py b/cli/medperf/utils.py index ae19cb952..4dd8e4800 100644 --- a/cli/medperf/utils.py +++ b/cli/medperf/utils.py @@ -33,7 +33,12 @@ def setup_logging(log_lvl): log_fmt = "%(asctime)s | %(levelname)s: %(message)s" log_file = storage_path(config.log_file) - handler = handlers.RotatingFileHandler(log_file, maxBytes=10000000, backupCount=5) + handler = handlers.RotatingFileHandler( + log_file, + maxBytes=10000000, + backupCount=5, + encoding="utf-8" + ) handler.setFormatter(logging.Formatter(log_fmt)) logging.basicConfig( level=log_lvl, From 7683ad7f2a45848a19595f51a25ee3d3e500705c Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Wed, 19 Apr 2023 11:42:41 -0500 Subject: [PATCH 45/57] Use current implementation of test_cubes --- cli/medperf/tests/entities/test_cube.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cli/medperf/tests/entities/test_cube.py b/cli/medperf/tests/entities/test_cube.py index 61d1a1e46..bb8edfd36 100644 --- a/cli/medperf/tests/entities/test_cube.py +++ b/cli/medperf/tests/entities/test_cube.py @@ -45,9 +45,7 @@ } -@pytest.fixture( - params={"local": [1, 2, 3], "remote": [4, 5, 6], "user": [4]} -) +@pytest.fixture(params={"local": [1, 2, 3], "remote": [4, 5, 6], "user": [4]}) def setup(request, mocker, comms, fs): local_ents = request.param.get("local", []) remote_ents = request.param.get("remote", []) @@ -57,7 +55,7 @@ def setup(request, mocker, comms, fs): setup_cube_fs(local_ents, fs) setup_cube_comms(mocker, comms, remote_ents, user_ents, uploaded) - setup_cube_comms_downloads(mocker, comms, fs) + setup_cube_comms_downloads(mocker, fs) request.param["uploaded"] = uploaded # Mock additional third party elements @@ -104,7 +102,7 @@ def test_get_cube_retrieves_files(self, setup): def test_get_cube_untars_files(self, mocker, setup): # Arrange spy = mocker.spy(medperf.entities.cube, "untar") - calls = [call(self.add_path), call(self.img_path)] + calls = [call(self.add_path)] # Act Cube.get(self.id) From 7060d093f90d6955bfa4695d4174cc7c1f002241 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Wed, 19 Apr 2023 11:47:20 -0500 Subject: [PATCH 46/57] Remove pexpect reference --- cli/medperf/tests/entities/test_cube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/medperf/tests/entities/test_cube.py b/cli/medperf/tests/entities/test_cube.py index bb8edfd36..0a02fdeea 100644 --- a/cli/medperf/tests/entities/test_cube.py +++ b/cli/medperf/tests/entities/test_cube.py @@ -60,7 +60,7 @@ def setup(request, mocker, comms, fs): # Mock additional third party elements mpexpect = MockPexpect(0) - mocker.patch(PATCH_CUBE.format("pexpect.spawn"), side_effect=mpexpect.spawn) + mocker.patch(PATCH_CUBE.format("spawn"), side_effect=mpexpect.spawn) mocker.patch(PATCH_CUBE.format("combine_proc_sp_text"), return_value="") mocker.patch(PATCH_CUBE.format("untar")) From f368555360e4922f4dd0696557fa3fcc034bcbaa Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Wed, 19 Apr 2023 11:51:48 -0500 Subject: [PATCH 47/57] Remove additional pexpect references --- cli/medperf/tests/entities/test_cube.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/medperf/tests/entities/test_cube.py b/cli/medperf/tests/entities/test_cube.py index 0a02fdeea..8a1db1b0b 100644 --- a/cli/medperf/tests/entities/test_cube.py +++ b/cli/medperf/tests/entities/test_cube.py @@ -143,7 +143,7 @@ def test_get_cube_deletes_cube_if_failed(self, mocker, setup): @pytest.mark.parametrize("setup", [{"remote": [NO_IMG_CUBE]}], indirect=True) def test_get_cube_without_image_configures_mlcube(self, mocker, setup): # Arrange - spy = mocker.spy(medperf.entities.cube.pexpect, "spawn") + spy = mocker.spy(medperf.entities.cube, "spawn") expected_cmd = f"mlcube configure --mlcube={self.manifest_path}" # Act @@ -155,7 +155,7 @@ def test_get_cube_without_image_configures_mlcube(self, mocker, setup): @pytest.mark.parametrize("setup", [{"remote": [DEFAULT_CUBE]}], indirect=True) def test_get_cube_with_image_isnt_configured(self, mocker, setup): # Arrange - spy = mocker.spy(medperf.entities.cube.pexpect, "spawn") + spy = mocker.spy(medperf.entities.cube, "spawn") # Act Cube.get(self.id) From 984e864240731030609e6dbf98090313a38cfea2 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Wed, 19 Apr 2023 11:52:51 -0500 Subject: [PATCH 48/57] Replace pexpect.spawn to spawn --- cli/medperf/tests/entities/test_cube.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cli/medperf/tests/entities/test_cube.py b/cli/medperf/tests/entities/test_cube.py index 8a1db1b0b..c1c200343 100644 --- a/cli/medperf/tests/entities/test_cube.py +++ b/cli/medperf/tests/entities/test_cube.py @@ -203,9 +203,7 @@ def set_common_attributes(self, setup): def test_cube_runs_command(self, mocker, timeout, setup, task): # Arrange mpexpect = MockPexpect(0) - spy = mocker.patch( - PATCH_CUBE.format("pexpect.spawn"), side_effect=mpexpect.spawn - ) + spy = mocker.patch(PATCH_CUBE.format("spawn"), side_effect=mpexpect.spawn) expected_cmd = f"mlcube run --mlcube={self.manifest_path} --task={task} --platform={self.platform}" # Act @@ -218,7 +216,7 @@ def test_cube_runs_command(self, mocker, timeout, setup, task): def test_cube_runs_command_with_extra_args(self, mocker, setup, task): # Arrange mpexpect = MockPexpect(0) - spy = mocker.patch("pexpect.spawn", side_effect=mpexpect.spawn) + spy = mocker.patch("spawn", side_effect=mpexpect.spawn) expected_cmd = f'mlcube run --mlcube={self.manifest_path} --task={task} --platform={self.platform} test="test"' # Act @@ -231,7 +229,7 @@ def test_cube_runs_command_with_extra_args(self, mocker, setup, task): def test_run_stops_execution_if_child_fails(self, mocker, setup, task): # Arrange mpexpect = MockPexpect(1) - mocker.patch("pexpect.spawn", side_effect=mpexpect.spawn) + mocker.patch("spawn", side_effect=mpexpect.spawn) # Act & Assert cube = Cube.get(self.id) From bc39d058ba0d799b20d81adacee6c9551fb52028 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Wed, 19 Apr 2023 13:07:46 -0500 Subject: [PATCH 49/57] Add timeout to configure test --- cli/medperf/tests/entities/test_cube.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/medperf/tests/entities/test_cube.py b/cli/medperf/tests/entities/test_cube.py index c1c200343..a4533c098 100644 --- a/cli/medperf/tests/entities/test_cube.py +++ b/cli/medperf/tests/entities/test_cube.py @@ -150,7 +150,7 @@ def test_get_cube_without_image_configures_mlcube(self, mocker, setup): Cube.get(self.id) # Assert - spy.assert_called_once_with(expected_cmd) + spy.assert_called_once_with(expected_cmd, timeout=None) @pytest.mark.parametrize("setup", [{"remote": [DEFAULT_CUBE]}], indirect=True) def test_get_cube_with_image_isnt_configured(self, mocker, setup): @@ -216,7 +216,7 @@ def test_cube_runs_command(self, mocker, timeout, setup, task): def test_cube_runs_command_with_extra_args(self, mocker, setup, task): # Arrange mpexpect = MockPexpect(0) - spy = mocker.patch("spawn", side_effect=mpexpect.spawn) + spy = mocker.patch(PATCH_CUBE.format("spawn"), side_effect=mpexpect.spawn) expected_cmd = f'mlcube run --mlcube={self.manifest_path} --task={task} --platform={self.platform} test="test"' # Act @@ -229,7 +229,7 @@ def test_cube_runs_command_with_extra_args(self, mocker, setup, task): def test_run_stops_execution_if_child_fails(self, mocker, setup, task): # Arrange mpexpect = MockPexpect(1) - mocker.patch("spawn", side_effect=mpexpect.spawn) + mocker.patch(PATCH_CUBE.format("spawn"), side_effect=mpexpect.spawn) # Act & Assert cube = Cube.get(self.id) From 4ea49dd4378d04142eaa1258280a55215c6d2ee2 Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Wed, 19 Apr 2023 13:11:25 -0500 Subject: [PATCH 50/57] import pexpect package --- cli/medperf/entities/cube.py | 8 +++--- cli/medperf/tests/entities/test_cube.py | 38 +++++++------------------ 2 files changed, 15 insertions(+), 31 deletions(-) diff --git a/cli/medperf/entities/cube.py b/cli/medperf/entities/cube.py index a54e676ae..de12e1c5b 100644 --- a/cli/medperf/entities/cube.py +++ b/cli/medperf/entities/cube.py @@ -7,9 +7,9 @@ from pathlib import Path if sys.platform == "win32": - from wexpect import spawn + import wexpect as pexpect else: - from pexpect import spawn + import pexpect from medperf.utils import untar, combine_proc_sp_text, list_files, storage_path, cleanup from medperf.entities.interface import Entity, Uploadable @@ -233,7 +233,7 @@ def download_image(self): # Retrieve image from image registry logging.debug(f"Retrieving {self.id} image") cmd = f"mlcube configure --mlcube={self.cube_path}" - proc = spawn(cmd, timeout=None) + proc = pexpect.spawn(cmd, timeout=None) proc_out = combine_proc_sp_text(proc) logging.debug(proc_out) proc.close() @@ -296,7 +296,7 @@ def run( cmd_arg = f'{k}="{v}"' cmd = " ".join([cmd, cmd_arg]) logging.info(f"Running MLCube command: {cmd}") - proc = spawn(cmd, timeout=timeout) + proc = pexpect.spawn(cmd, timeout=timeout) proc_out = combine_proc_sp_text(proc) proc.close() logging.debug(proc_out) diff --git a/cli/medperf/tests/entities/test_cube.py b/cli/medperf/tests/entities/test_cube.py index a4533c098..35ede416e 100644 --- a/cli/medperf/tests/entities/test_cube.py +++ b/cli/medperf/tests/entities/test_cube.py @@ -1,7 +1,7 @@ import os import yaml import pytest -from unittest.mock import ANY, call +from unittest.mock import call import medperf import medperf.config as config @@ -15,7 +15,7 @@ from medperf.tests.mocks.pexpect import MockPexpect from medperf.exceptions import ( ExecutionError, - InvalidEntityError, + InvalidEntityError ) PATCH_CUBE = "medperf.entities.cube.{}" @@ -60,7 +60,7 @@ def setup(request, mocker, comms, fs): # Mock additional third party elements mpexpect = MockPexpect(0) - mocker.patch(PATCH_CUBE.format("spawn"), side_effect=mpexpect.spawn) + mocker.patch(PATCH_CUBE.format("pexpect.spawn"), side_effect=mpexpect.spawn) mocker.patch(PATCH_CUBE.format("combine_proc_sp_text"), return_value="") mocker.patch(PATCH_CUBE.format("untar")) @@ -110,24 +110,6 @@ def test_get_cube_untars_files(self, mocker, setup): # Assert spy.assert_has_calls(calls) - @pytest.mark.parametrize("max_attempts", [3, 5, 2]) - @pytest.mark.parametrize("setup", [{"remote": [FAILING_CUBE]}], indirect=True) - def test_get_cube_retries_configured_number_of_times( - self, mocker, max_attempts, setup - ): - # Arrange - mocker.patch(PATCH_CUBE.format("cleanup")) - spy = mocker.spy(Cube, "download") - config.cube_get_max_attempts = max_attempts - calls = [call(ANY)] * max_attempts - - # Act - with pytest.raises(InvalidEntityError): - Cube.get(self.id) - - # Assert - spy.assert_has_calls(calls) - @pytest.mark.parametrize("setup", [{"remote": [FAILING_CUBE]}], indirect=True) def test_get_cube_deletes_cube_if_failed(self, mocker, setup): # Arrange @@ -143,19 +125,19 @@ def test_get_cube_deletes_cube_if_failed(self, mocker, setup): @pytest.mark.parametrize("setup", [{"remote": [NO_IMG_CUBE]}], indirect=True) def test_get_cube_without_image_configures_mlcube(self, mocker, setup): # Arrange - spy = mocker.spy(medperf.entities.cube, "spawn") + spy = mocker.spy(medperf.entities.cube.pexpect, "spawn") expected_cmd = f"mlcube configure --mlcube={self.manifest_path}" # Act Cube.get(self.id) # Assert - spy.assert_called_once_with(expected_cmd, timeout=None) + spy.assert_called_once_with(expected_cmd) @pytest.mark.parametrize("setup", [{"remote": [DEFAULT_CUBE]}], indirect=True) def test_get_cube_with_image_isnt_configured(self, mocker, setup): # Arrange - spy = mocker.spy(medperf.entities.cube, "spawn") + spy = mocker.spy(medperf.entities.cube.pexpect, "spawn") # Act Cube.get(self.id) @@ -203,7 +185,9 @@ def set_common_attributes(self, setup): def test_cube_runs_command(self, mocker, timeout, setup, task): # Arrange mpexpect = MockPexpect(0) - spy = mocker.patch(PATCH_CUBE.format("spawn"), side_effect=mpexpect.spawn) + spy = mocker.patch( + PATCH_CUBE.format("pexpect.spawn"), side_effect=mpexpect.spawn + ) expected_cmd = f"mlcube run --mlcube={self.manifest_path} --task={task} --platform={self.platform}" # Act @@ -216,7 +200,7 @@ def test_cube_runs_command(self, mocker, timeout, setup, task): def test_cube_runs_command_with_extra_args(self, mocker, setup, task): # Arrange mpexpect = MockPexpect(0) - spy = mocker.patch(PATCH_CUBE.format("spawn"), side_effect=mpexpect.spawn) + spy = mocker.patch("pexpect.spawn", side_effect=mpexpect.spawn) expected_cmd = f'mlcube run --mlcube={self.manifest_path} --task={task} --platform={self.platform} test="test"' # Act @@ -229,7 +213,7 @@ def test_cube_runs_command_with_extra_args(self, mocker, setup, task): def test_run_stops_execution_if_child_fails(self, mocker, setup, task): # Arrange mpexpect = MockPexpect(1) - mocker.patch(PATCH_CUBE.format("spawn"), side_effect=mpexpect.spawn) + mocker.patch("pexpect.spawn", side_effect=mpexpect.spawn) # Act & Assert cube = Cube.get(self.id) From 5ec824ea057436bf553f55a10e1202a52555093c Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Wed, 19 Apr 2023 13:13:57 -0500 Subject: [PATCH 51/57] Formatting changes --- cli/medperf/tests/entities/test_cube.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cli/medperf/tests/entities/test_cube.py b/cli/medperf/tests/entities/test_cube.py index 35ede416e..2bb615497 100644 --- a/cli/medperf/tests/entities/test_cube.py +++ b/cli/medperf/tests/entities/test_cube.py @@ -13,10 +13,7 @@ setup_cube_comms_downloads, ) from medperf.tests.mocks.pexpect import MockPexpect -from medperf.exceptions import ( - ExecutionError, - InvalidEntityError -) +from medperf.exceptions import ExecutionError, InvalidEntityError PATCH_CUBE = "medperf.entities.cube.{}" DEFAULT_CUBE = {"id": 37} @@ -132,7 +129,7 @@ def test_get_cube_without_image_configures_mlcube(self, mocker, setup): Cube.get(self.id) # Assert - spy.assert_called_once_with(expected_cmd) + spy.assert_called_once_with(expected_cmd, timeout=None) @pytest.mark.parametrize("setup", [{"remote": [DEFAULT_CUBE]}], indirect=True) def test_get_cube_with_image_isnt_configured(self, mocker, setup): From 09afd50635ff764d91210b50d0429d2c39026e62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Aristiz=C3=A1bal?= Date: Tue, 25 Apr 2023 10:27:48 -0500 Subject: [PATCH 52/57] Update cli/medperf/entities/cube.py Co-authored-by: hasan7n <78664424+hasan7n@users.noreply.github.com> --- cli/medperf/entities/cube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/medperf/entities/cube.py b/cli/medperf/entities/cube.py index de12e1c5b..59e6e74bf 100644 --- a/cli/medperf/entities/cube.py +++ b/cli/medperf/entities/cube.py @@ -300,7 +300,7 @@ def run( proc_out = combine_proc_sp_text(proc) proc.close() logging.debug(proc_out) - if proc.exitstatus and proc.exitstatus != 0: + if proc.exitstatus: raise ExecutionError("There was an error while executing the cube") logging.debug(list_files(config.storage)) From 8aa66ead3c667a68b2ec00bb0015c5f149fb075f Mon Sep 17 00:00:00 2001 From: Alejandro Aristizabal Date: Tue, 25 Apr 2023 16:13:00 -0500 Subject: [PATCH 53/57] Add configuration validation step --- cli/medperf/__main__.py | 2 ++ cli/medperf/utils.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/cli/medperf/__main__.py b/cli/medperf/__main__.py index 8f0f321a3..f05665975 100644 --- a/cli/medperf/__main__.py +++ b/cli/medperf/__main__.py @@ -18,6 +18,7 @@ import medperf.commands.profile as profile from medperf.utils import ( set_custom_config, + validate_config, set_unique_tmp_config, init_storage, setup_logging, @@ -136,6 +137,7 @@ def main(ctx: typer.Context): # Set inline parameters inline_args = ctx.params set_custom_config(inline_args) + validate_config() if config.certificate is not None: config.certificate = abspath(expanduser(config.certificate)) diff --git a/cli/medperf/utils.py b/cli/medperf/utils.py index 4dd8e4800..c2f5c0607 100644 --- a/cli/medperf/utils.py +++ b/cli/medperf/utils.py @@ -124,6 +124,14 @@ def set_custom_config(args: dict): setattr(config, param, val) +def validate_config(): + """Validates the configuration is valid for the current machine + """ + if config.platform == "singularity" and sys.platform == "win32": + raise MedperfException("Windows doesn't support singularity runner") + # Add additional checks here + + def storage_path(subpath: str): """Helper function that converts a path to deployment storage-related path""" server_path = config.server.split("//")[1] From 0ee1e64b4662bf2a33dad779adaba15cbc10f2d4 Mon Sep 17 00:00:00 2001 From: hasan7n Date: Mon, 21 Aug 2023 00:21:33 +0200 Subject: [PATCH 54/57] add missing import --- cli/medperf/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/medperf/utils.py b/cli/medperf/utils.py index d34d256b6..8e16363f1 100644 --- a/cli/medperf/utils.py +++ b/cli/medperf/utils.py @@ -1,5 +1,6 @@ from __future__ import annotations +import sys import re import os import yaml From f95f5912f17082cae042c1e7dd73cf63d862ca46 Mon Sep 17 00:00:00 2001 From: hasan7n Date: Mon, 21 Aug 2023 00:28:57 +0200 Subject: [PATCH 55/57] add a space before ; --- cli/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/requirements.txt b/cli/requirements.txt index fffcb3dc1..1c2f9c799 100644 --- a/cli/requirements.txt +++ b/cli/requirements.txt @@ -14,7 +14,7 @@ pytest-mock==1.13.0 pyfakefs==5.0.0 mlcube @ git+https://github.com/mlcommons/mlcube@fb371c960938b495e939bf38b161199d529cf912#subdirectory=mlcube mlcube-docker @ git+https://github.com/mlcommons/mlcube@fb371c960938b495e939bf38b161199d529cf912#subdirectory=runners/mlcube_docker -mlcube-singularity @ git+https://github.com/mlcommons/mlcube@fb371c960938b495e939bf38b161199d529cf912#subdirectory=runners/mlcube_singularity; sys_platform != "win32" +mlcube-singularity @ git+https://github.com/mlcommons/mlcube@fb371c960938b495e939bf38b161199d529cf912#subdirectory=runners/mlcube_singularity ; sys_platform != "win32" validators==0.18.2 merge-args==0.1.4 synapseclient==2.7.0 From 9e826d178ebe1f6b92c5a9b97839d8feb5255786 Mon Sep 17 00:00:00 2001 From: hasan7n Date: Mon, 21 Aug 2023 02:55:54 +0200 Subject: [PATCH 56/57] remove redundant import --- cli/medperf/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cli/medperf/utils.py b/cli/medperf/utils.py index 8e16363f1..bad09e38f 100644 --- a/cli/medperf/utils.py +++ b/cli/medperf/utils.py @@ -15,7 +15,6 @@ import json from pathlib import Path import shutil -from pexpect import spawn from datetime import datetime from typing import List from colorama import Fore, Style From d4095fe7dc894893194351f433be4cbb1fac2e67 Mon Sep 17 00:00:00 2001 From: hasan7n Date: Mon, 21 Aug 2023 03:04:59 +0200 Subject: [PATCH 57/57] fix linter issues --- cli/medperf/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/medperf/utils.py b/cli/medperf/utils.py index bad09e38f..7b05f0b77 100644 --- a/cli/medperf/utils.py +++ b/cli/medperf/utils.py @@ -384,7 +384,7 @@ def combine_proc_sp_text(proc: spawn) -> str: except TIMEOUT: logging.error("Process timed out") raise ExecutionError("Process timed out") - if line is not None and type(line) != str: + if line is not None and not isinstance(line, str): line = line.decode("utf-8", "ignore") if line: proc_out += line