From ac14160f88e38da63645b543af2808306505dc31 Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Fri, 16 Sep 2022 09:05:23 +0200
Subject: [PATCH 01/20] write files in the outputs

---
 validation/input/input_1/input_script_1.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/validation/input/input_1/input_script_1.py b/validation/input/input_1/input_script_1.py
index 3857597..d6afb60 100644
--- a/validation/input/input_1/input_script_1.py
+++ b/validation/input/input_1/input_script_1.py
@@ -29,11 +29,22 @@ def test_progress():
         time.sleep(0.001)
 
 
-def test_cwd():
+def test_inputs():
     assert (Path(os.environ["INPUT_FOLDER"]) / "input_script_1.py").exists()
     assert (Path(os.environ["INPUT_FOLDER"]) / "requirements.txt").exists()
 
 
+def test_outputs():
+    """write a file in each output"""
+    for output_folder_env in [*(f"OUTPUT_{i}" for i in range(1, 5))]:
+        folder = Path(os.environ[output_folder_env])
+        assert folder.exists()
+        for number in range(1, 3):
+            file = Path(folder) / f"some_output_{number}_file"
+            file.write_text(f"Hello this is file {number}")
+            assert file.exists()
+
+
 if __name__ == "__main__":
     sys.exit(
         pytest.main(

From e17a451b291e68e8d2cf84d8181130b360d20c74 Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Fri, 16 Sep 2022 09:10:45 +0200
Subject: [PATCH 02/20] bump integration version to 1.1.0

---
 VERSION_INTEGRATION        | 2 +-
 metadata/metadata.yml      | 2 +-
 versioning/integration.cfg | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/VERSION_INTEGRATION b/VERSION_INTEGRATION
index 3eefcb9..9084fa2 100644
--- a/VERSION_INTEGRATION
+++ b/VERSION_INTEGRATION
@@ -1 +1 @@
-1.0.0
+1.1.0
diff --git a/metadata/metadata.yml b/metadata/metadata.yml
index 0e97591..ce4935d 100644
--- a/metadata/metadata.yml
+++ b/metadata/metadata.yml
@@ -1,7 +1,7 @@
 name: oSparc Python Runner
 key: simcore/services/comp/osparc-python-runner
 type: computational
-integration-version: 1.0.0
+integration-version: 1.1.0
 version: 1.3.0
 description: oSparc Python Runner
 contact: anderegg@itis.swiss
diff --git a/versioning/integration.cfg b/versioning/integration.cfg
index bed2967..310dddc 100644
--- a/versioning/integration.cfg
+++ b/versioning/integration.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.0.0
+current_version = 1.1.0
 commit = False
 message = integration version: {current_version} → {new_version}
 tag = False

From 78617043514cf263268bbcc48774f9d0a80d7236 Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Fri, 16 Sep 2022 09:13:44 +0200
Subject: [PATCH 03/20] version 1.1.0

---
 validation/input/{ => input_2}/input_2 |   0
 validation/input/input_3               |   0
 validation/input/input_4               |   0
 validation/input/input_5               |   0
 validation/output/output_1.zip         | Bin 116 -> 0 bytes
 validation/output/output_2.zip         | Bin 116 -> 0 bytes
 validation/output/output_3.zip         | Bin 116 -> 0 bytes
 validation/output/output_4.zip         | Bin 116 -> 0 bytes
 8 files changed, 0 insertions(+), 0 deletions(-)
 rename validation/input/{ => input_2}/input_2 (100%)
 delete mode 100644 validation/input/input_3
 delete mode 100644 validation/input/input_4
 delete mode 100644 validation/input/input_5
 delete mode 100644 validation/output/output_1.zip
 delete mode 100644 validation/output/output_2.zip
 delete mode 100644 validation/output/output_3.zip
 delete mode 100644 validation/output/output_4.zip

diff --git a/validation/input/input_2 b/validation/input/input_2/input_2
similarity index 100%
rename from validation/input/input_2
rename to validation/input/input_2/input_2
diff --git a/validation/input/input_3 b/validation/input/input_3
deleted file mode 100644
index e69de29..0000000
diff --git a/validation/input/input_4 b/validation/input/input_4
deleted file mode 100644
index e69de29..0000000
diff --git a/validation/input/input_5 b/validation/input/input_5
deleted file mode 100644
index e69de29..0000000
diff --git a/validation/output/output_1.zip b/validation/output/output_1.zip
deleted file mode 100644
index 2a3e03d0e919a80c0a178b425c69334ac9c5d758..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 116
zcmWIWW@Zs#0D&cq`k`P3l;8x?`K2WVr6uu(`T^dIOd`y<6+-j~FuZjHv5>T}vVjB`
MfzTXCtAjWU0Aw)^2mk;8

diff --git a/validation/output/output_2.zip b/validation/output/output_2.zip
deleted file mode 100644
index 5680cf1be2e3a0a362d05f9c5977e61755a37e19..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 116
zcmWIWW@Zs#0D&cq`k`P3l;8x?`K2WVr6uu3`T^dIOd`y<6+-j~FuZjHv5>T}vVjB`
MfzTXCtAjWU0A!dB3IG5A

diff --git a/validation/output/output_3.zip b/validation/output/output_3.zip
deleted file mode 100644
index 3172e9d31a95b945b612fb9a2465f8ce1b97ad29..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 116
zcmWIWW@Zs#0D&cq`k`P3l;8x?`K2WVr6uvk`T^dIOd`y<6+-j~FuZjHv5>T}vVjB`
MfzTXCtAjWU0A&9T3;+NC

diff --git a/validation/output/output_4.zip b/validation/output/output_4.zip
deleted file mode 100644
index 2f6b2eab5e4340e00f4320ec8717a717a0fcb988..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 116
zcmWIWW@Zs#0D&cq`k`P3l;8x?`K2WVr6utu`T^dIOd`y<6+-j~FuZjHv5>T}vVjB`
MfzTXCtAjWU0A*$l4gdfE


From 71503f6adb52b29f1e2c27707c496583a7a3056b Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Fri, 16 Sep 2022 09:17:37 +0200
Subject: [PATCH 04/20] in 1.1.0 we do have folders

---
 docker-compose-meta.yml | 19 ++++++++-----------
 metadata/metadata.yml   |  8 --------
 2 files changed, 8 insertions(+), 19 deletions(-)

diff --git a/docker-compose-meta.yml b/docker-compose-meta.yml
index 5d96381..f4f747e 100644
--- a/docker-compose-meta.yml
+++ b/docker-compose-meta.yml
@@ -17,22 +17,19 @@ services:
           input data file", "type": "data:*/*"}, "input_5": {"displayOrder": 5, "label":
           "Additional input data - optional", "description": "Any additional input
           data file", "type": "data:*/*"}}}'
-        io.simcore.integration-version: '{"integration-version": "1.0.0"}'
+        io.simcore.integration-version: '{"integration-version": "1.1.0"}'
         io.simcore.key: '{"key": "simcore/services/comp/osparc-python-runner"}'
         io.simcore.name: '{"name": "oSparc Python Runner"}'
         io.simcore.outputs: '{"outputs": {"output_1": {"displayOrder": 1, "label":
           "Output data", "description": "The data produced by the script and saved
-          under OUTPUT_FOLDER/output_1 as output_1.zip", "type": "data:*/*", "fileToKeyMap":
-          {"output_1.zip": "output_1"}}, "output_2": {"displayOrder": 2, "label":
+          under OUTPUT_FOLDER/output_1 as output_1.zip", "type": "data:*/*"}, "output_2":
+          {"displayOrder": 2, "label": "Output data", "description": "The data produced
+          by the script and saved under OUTPUT_FOLDER/output_2 as output_2.zip", "type":
+          "data:*/*"}, "output_3": {"displayOrder": 3, "label": "Output data", "description":
+          "The data produced by the script and saved under OUTPUT_FOLDER/output_3
+          as output_3.zip", "type": "data:*/*"}, "output_4": {"displayOrder": 4, "label":
           "Output data", "description": "The data produced by the script and saved
-          under OUTPUT_FOLDER/output_2 as output_2.zip", "type": "data:*/*", "fileToKeyMap":
-          {"output_2.zip": "output_2"}}, "output_3": {"displayOrder": 3, "label":
-          "Output data", "description": "The data produced by the script and saved
-          under OUTPUT_FOLDER/output_3 as output_3.zip", "type": "data:*/*", "fileToKeyMap":
-          {"output_3.zip": "output_3"}}, "output_4": {"displayOrder": 4, "label":
-          "Output data", "description": "The data produced by the script and saved
-          under OUTPUT_FOLDER/output_4 as output_4.zip", "type": "data:*/*", "fileToKeyMap":
-          {"output_4.zip": "output_4"}}}}'
+          under OUTPUT_FOLDER/output_4 as output_4.zip", "type": "data:*/*"}}}'
         io.simcore.type: '{"type": "computational"}'
         io.simcore.version: '{"version": "1.3.0"}'
         org.label-schema.build-date: ${BUILD_DATE}
diff --git a/metadata/metadata.yml b/metadata/metadata.yml
index ce4935d..5c27e03 100644
--- a/metadata/metadata.yml
+++ b/metadata/metadata.yml
@@ -44,26 +44,18 @@ outputs:
     label: Output data
     description: The data produced by the script and saved under OUTPUT_FOLDER/output_1 as output_1.zip
     type: data:*/*
-    fileToKeyMap:
-      output_1.zip: output_1
   output_2:
     displayOrder: 2
     label: Output data
     description: The data produced by the script and saved under OUTPUT_FOLDER/output_2 as output_2.zip
     type: data:*/*
-    fileToKeyMap:
-      output_2.zip: output_2
   output_3:
     displayOrder: 3
     label: Output data
     description: The data produced by the script and saved under OUTPUT_FOLDER/output_3 as output_3.zip
     type: data:*/*
-    fileToKeyMap:
-      output_3.zip: output_3
   output_4:
     displayOrder: 4
     label: Output data
     description: The data produced by the script and saved under OUTPUT_FOLDER/output_4 as output_4.zip
     type: data:*/*
-    fileToKeyMap:
-      output_4.zip: output_4

From 4f380b4276739006854d9bf9d84e6c10588ae7eb Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Fri, 16 Sep 2022 09:18:04 +0200
Subject: [PATCH 05/20] removed 1.0.0 test

---
 tests/integration/test_docker_container.py | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/tests/integration/test_docker_container.py b/tests/integration/test_docker_container.py
index 8e99f9e..dcb4a9c 100644
--- a/tests/integration/test_docker_container.py
+++ b/tests/integration/test_docker_container.py
@@ -51,26 +51,18 @@ def validation_folders(validation_dir: Path) -> Dict[str, Path]:
     return {folder: (validation_dir / folder) for folder in _FOLDER_NAMES}
 
 
-@pytest.fixture(params=["1.0.0"])
+@pytest.fixture
 def docker_container(
     validation_folders: Dict[str, Path],
     host_folders: Dict[str, Path],
     docker_client: docker.DockerClient,
     docker_image_key: str,
     container_variables: Dict,
-    request,
 ) -> docker.models.containers.Container:
     # copy files to input folder, copytree needs to not have the input folder around.
     host_folders["input"].rmdir()
     shutil.copytree(validation_folders["input"], host_folders["input"])
     assert Path(host_folders["input"]).exists()
-    if request.param == "1.0.0":
-        # NOTE: in this version all the files are copied in a flat file system (e.g. input_1 unzipped in /inputs, same for input_2, ...)
-        for file_path in host_folders["input"].glob("*"):
-            if file_path.is_dir():
-                for file_inside_folder in file_path.glob("*"):
-                    shutil.move(f"{file_inside_folder}", host_folders["input"])
-                file_path.rmdir()
 
     # run the container (this may take some time)
     try:

From 1be3bda84db3cd568d223ed0030a11517bcd619e Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Fri, 16 Sep 2022 09:19:45 +0200
Subject: [PATCH 06/20] adapted test

---
 validation/input/input_1/input_script_1.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/validation/input/input_1/input_script_1.py b/validation/input/input_1/input_script_1.py
index d6afb60..8cd58bf 100644
--- a/validation/input/input_1/input_script_1.py
+++ b/validation/input/input_1/input_script_1.py
@@ -30,8 +30,8 @@ def test_progress():
 
 
 def test_inputs():
-    assert (Path(os.environ["INPUT_FOLDER"]) / "input_script_1.py").exists()
-    assert (Path(os.environ["INPUT_FOLDER"]) / "requirements.txt").exists()
+    assert (Path(os.environ["INPUT_1"]) / "input_script_1.py").exists()
+    assert (Path(os.environ["INPUT_1"]) / "requirements.txt").exists()
 
 
 def test_outputs():

From 1ef6a358c30b6f4dd1c2f34fb117e1142d982873 Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Fri, 16 Sep 2022 09:26:20 +0200
Subject: [PATCH 07/20] prepared for version 1.1.0

---
 service.cli/run                  | 10 ++++++++
 src/osparc_python_runner/main.py | 43 ++++++++------------------------
 2 files changed, 20 insertions(+), 33 deletions(-)

diff --git a/service.cli/run b/service.cli/run
index 2810032..58a391c 100755
--- a/service.cli/run
+++ b/service.cli/run
@@ -20,5 +20,15 @@ export INPUT_4
 INPUT_5=$INPUT_FOLDER/input_5
 export INPUT_5
 
+OUTPUT_1=$OUTPUT_FOLDER/output_1
+export OUTPUT_1
+OUTPUT_2=$OUTPUT_FOLDER/output_2
+export OUTPUT_2
+OUTPUT_3=$OUTPUT_FOLDER/output_3
+export OUTPUT_3
+OUTPUT_4=$OUTPUT_FOLDER/output_4
+export OUTPUT_4
+
+
 exec execute.sh
         
\ No newline at end of file
diff --git a/src/osparc_python_runner/main.py b/src/osparc_python_runner/main.py
index 2dbd34a..423a244 100644
--- a/src/osparc_python_runner/main.py
+++ b/src/osparc_python_runner/main.py
@@ -69,36 +69,19 @@ def _ensure_pip_requirements(code_dir: Path) -> Path:
     return requirements
 
 
-# TODO: Next version of integration will take care of this and maybe the ENVs as well
-def _ensure_output_subfolders_exist() -> Dict[str, str]:
-    output_envs = {}
-    for n in range(1, NUM_OUTPUTS + 1):
-        output_sub_folder_env = f"OUTPUT_{n}"
-        output_sub_folder = OUTPUT_FOLDER / OUTPUT_SUBFOLDER_TEMPLATE.format(n)
-        # NOTE: exist_ok for forward compatibility in case they are already created
-        output_sub_folder.mkdir(parents=True, exist_ok=True)
-        output_envs[output_sub_folder_env] = f"{output_sub_folder}"
-    logger.info(
-        "Output ENVs available: %s",
-        json.dumps(output_envs, indent=2),
-    )
-    return output_envs
-
-
-def _ensure_input_environment() -> Dict[str, str]:
-    input_envs = {
-        f"INPUT_{n}": os.environ[f"INPUT_{n}"] for n in range(1, NUM_INPUTS + 1)
-    }
-    logger.info(
-        "Input ENVs available: %s",
-        json.dumps(input_envs, indent=2),
-    )
-    return input_envs
+def _show_io_environments() -> None:
+    for io_type in ["input", "output"]:
+        logger.info(
+            "%s ENVs available: %s",
+            io_type.capitalize(),
+            json.dumps(
+                filter(lambda x: f"{io_type.upper()}_" in x, os.environ), indent=2
+            ),
+        )
 
 
 def setup():
-    input_envs = _ensure_input_environment()
-    output_envs = _ensure_output_subfolders_exist()
+    _show_io_environments()
     logger.info("Available data:")
     os.system("ls -tlah")
 
@@ -107,10 +90,6 @@ def setup():
 
     logger.info("Preparing launch script ...")
     venv_dir = Path.home() / ".venv"
-    bash_input_env_export = [f"export {env}={path}" for env, path in input_envs.items()]
-    bash_output_env_export = [
-        f"export {env}='{path}'" for env, path in output_envs.items()
-    ]
     script = [
         "#!/bin/sh",
         "set -o errexit",
@@ -120,8 +99,6 @@ def setup():
         f'python3 -m venv --system-site-packages --symlinks --upgrade "{venv_dir}"',
         f'"{venv_dir}/bin/pip" install -U pip wheel setuptools',
         f'"{venv_dir}/bin/pip" install -r "{requirements_txt}"',
-        "\n".join(bash_input_env_export),
-        "\n".join(bash_output_env_export),
         f'echo "Executing code {user_code_entrypoint.name}..."',
         f'"{venv_dir}/bin/python3" "{user_code_entrypoint}"',
         'echo "DONE ..."',

From 2afce5f177180658b1c2cda0b0d4b6f7724ef392 Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Fri, 16 Sep 2022 09:40:39 +0200
Subject: [PATCH 08/20] simplify

---
 src/osparc_python_runner/main.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/osparc_python_runner/main.py b/src/osparc_python_runner/main.py
index 423a244..df8f922 100644
--- a/src/osparc_python_runner/main.py
+++ b/src/osparc_python_runner/main.py
@@ -18,10 +18,7 @@
     raise ValueError("Required env vars {ENVIRONS} were not set")
 
 # NOTE: sync with schema in metadata!!
-NUM_INPUTS = 5
 NUM_OUTPUTS = 4
-OUTPUT_SUBFOLDER_ENV_TEMPLATE = "OUTPUT_{}"
-OUTPUT_SUBFOLDER_TEMPLATE = "output_{}"
 OUTPUT_FILE_TEMPLATE = "output_{}.zip"
 
 
@@ -75,7 +72,7 @@ def _show_io_environments() -> None:
             "%s ENVs available: %s",
             io_type.capitalize(),
             json.dumps(
-                filter(lambda x: f"{io_type.upper()}_" in x, os.environ), indent=2
+                list(filter(lambda x: f"{io_type.upper()}_" in x, os.environ)), indent=2
             ),
         )
 

From 418e4f0f2292aad2a62587cbaedad5d8efd87971 Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Fri, 16 Sep 2022 09:40:50 +0200
Subject: [PATCH 09/20] prepare output folders

---
 tests/integration/test_docker_container.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tests/integration/test_docker_container.py b/tests/integration/test_docker_container.py
index dcb4a9c..3f40994 100644
--- a/tests/integration/test_docker_container.py
+++ b/tests/integration/test_docker_container.py
@@ -63,6 +63,15 @@ def docker_container(
     host_folders["input"].rmdir()
     shutil.copytree(validation_folders["input"], host_folders["input"])
     assert Path(host_folders["input"]).exists()
+    # prepare output folders
+    host_folders["output"].rmdir()
+    for output_folder in validation_folders["output"].glob("*"):
+        if not output_folder.is_dir():
+            continue
+        # create the same folder in the output
+        host_output_folder = host_folders["output"] / output_folder.name
+        host_output_folder.mkdir(parents=True)
+        assert host_output_folder.exists()
 
     # run the container (this may take some time)
     try:

From 65a2247fae9ced801384dbfcb68f4ea23f1fde53 Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Fri, 16 Sep 2022 09:52:29 +0200
Subject: [PATCH 10/20] simplify

---
 src/osparc_python_runner/main.py | 30 +++++-------------------------
 1 file changed, 5 insertions(+), 25 deletions(-)

diff --git a/src/osparc_python_runner/main.py b/src/osparc_python_runner/main.py
index df8f922..982044b 100644
--- a/src/osparc_python_runner/main.py
+++ b/src/osparc_python_runner/main.py
@@ -11,15 +11,7 @@
 logger = logging.getLogger("osparc-python-main")
 
 
-ENVIRONS = ["INPUT_FOLDER", "OUTPUT_FOLDER"]
-try:
-    INPUT_FOLDER, OUTPUT_FOLDER = [Path(os.environ[v]) for v in ENVIRONS]
-except KeyError:
-    raise ValueError("Required env vars {ENVIRONS} were not set")
-
-# NOTE: sync with schema in metadata!!
-NUM_OUTPUTS = 4
-OUTPUT_FILE_TEMPLATE = "output_{}.zip"
+INPUT_1 = Path(os.environ["INPUT_1"])
 
 
 def _find_user_code_entrypoint(code_dir: Path) -> Path:
@@ -55,7 +47,7 @@ def _ensure_pip_requirements(code_dir: Path) -> Path:
             f"pipreqs --savepath={requirements} --force {code_dir}".split(),
             shell=False,
             check=True,
-            cwd=INPUT_FOLDER,
+            cwd=INPUT_1,
         )
 
         # TODO log subprocess.run
@@ -82,8 +74,8 @@ def setup():
     logger.info("Available data:")
     os.system("ls -tlah")
 
-    user_code_entrypoint = _find_user_code_entrypoint(INPUT_FOLDER)
-    requirements_txt = _ensure_pip_requirements(INPUT_FOLDER)
+    user_code_entrypoint = _find_user_code_entrypoint(INPUT_1)
+    requirements_txt = _ensure_pip_requirements(INPUT_1)
 
     logger.info("Preparing launch script ...")
     venv_dir = Path.home() / ".venv"
@@ -105,19 +97,7 @@ def setup():
 
 
 def teardown():
-    logger.info("Zipping output...")
-    for n in range(1, NUM_OUTPUTS + 1):
-        output_path = OUTPUT_FOLDER / f"output_{n}"
-        archive_file_path = OUTPUT_FOLDER / OUTPUT_FILE_TEMPLATE.format(n)
-        logger.info("Zipping %s into %s...", output_path, archive_file_path)
-        shutil.make_archive(
-            f"{(archive_file_path.parent / archive_file_path.stem)}",
-            format="zip",
-            root_dir=output_path,
-            logger=logger,
-        )
-        logger.info("Zipping %s into %s done", output_path, archive_file_path)
-    logger.info("Zipping done.")
+    logger.info("Completed")
 
 
 if __name__ == "__main__":

From a9c594cbd94b8f9b386645401ef501cfec8b8dee Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Fri, 16 Sep 2022 10:00:51 +0200
Subject: [PATCH 11/20] added pylint

---
 requirements.in  |  1 +
 requirements.txt | 66 +++++++++++++++++++++++++++++++++++-------------
 2 files changed, 49 insertions(+), 18 deletions(-)

diff --git a/requirements.in b/requirements.in
index 90e1c24..ade9caf 100644
--- a/requirements.in
+++ b/requirements.in
@@ -8,6 +8,7 @@ black
 coverage
 docker
 jsonschema
+pylint
 pytest
 pytest-cookies
 pytest-cov
diff --git a/requirements.txt b/requirements.txt
index b01eb8e..c75aa3f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,9 +4,11 @@
 #
 #    pip-compile --output-file=requirements.txt
 #
-arrow==1.2.2
+arrow==1.2.3
     # via jinja2-time
-attrs==21.4.0
+astroid==2.12.9
+    # via pylint
+attrs==22.1.0
     # via
     #   jsonschema
     #   pytest
@@ -18,29 +20,35 @@ bump2version==1.0.1
     # via bumpversion
 bumpversion==0.6.0
     # via -r requirements.in
-certifi==2021.10.8
+certifi==2022.9.14
     # via requests
-chardet==4.0.0
+chardet==5.0.0
     # via binaryornot
-charset-normalizer==2.0.12
+charset-normalizer==2.1.1
     # via requests
-click==8.1.0
+click==8.1.3
     # via
     #   black
     #   cookiecutter
 cookiecutter==2.1.1
     # via pytest-cookies
-coverage[toml]==6.3.2
+coverage[toml]==6.4.4
     # via
     #   -r requirements.in
     #   pytest-cov
+dill==0.3.5.1
+    # via pylint
 docker==6.0.0
     # via -r requirements.in
-idna==3.3
+idna==3.4
     # via requests
+importlib-resources==5.9.0
+    # via jsonschema
 iniconfig==1.1.1
     # via pytest
-jinja2==3.1.1
+isort==5.10.1
+    # via pylint
+jinja2==3.1.2
     # via
     #   cookiecutter
     #   jinja2-time
@@ -48,8 +56,12 @@ jinja2-time==0.2.0
     # via cookiecutter
 jsonschema==4.16.0
     # via -r requirements.in
+lazy-object-proxy==1.7.1
+    # via astroid
 markupsafe==2.1.1
     # via jinja2
+mccabe==0.7.0
+    # via pylint
 mypy-extensions==0.4.3
     # via black
 packaging==21.3
@@ -57,15 +69,21 @@ packaging==21.3
     #   docker
     #   pytest
     #   pytest-sugar
-pathspec==0.9.0
-    # via black
-platformdirs==2.5.1
+pathspec==0.10.1
     # via black
+pkgutil-resolve-name==1.3.10
+    # via jsonschema
+platformdirs==2.5.2
+    # via
+    #   black
+    #   pylint
 pluggy==1.0.0
     # via pytest
 py==1.11.0
     # via pytest
-pyparsing==3.0.7
+pylint==2.15.2
+    # via -r requirements.in
+pyparsing==3.0.9
     # via packaging
 pyrsistent==0.18.1
     # via jsonschema
@@ -89,19 +107,19 @@ pytest-sugar==0.9.5
     # via -r requirements.in
 python-dateutil==2.8.2
     # via arrow
-python-slugify==6.1.1
+python-slugify==6.1.2
     # via cookiecutter
 pyyaml==6.0
     # via
     #   -r requirements.in
     #   cookiecutter
-requests==2.27.1
+requests==2.28.1
     # via
     #   cookiecutter
     #   docker
 six==1.16.0
     # via python-dateutil
-termcolor==1.1.0
+termcolor==2.0.1
     # via pytest-sugar
 text-unidecode==1.3
     # via python-slugify
@@ -109,10 +127,22 @@ tomli==2.0.1
     # via
     #   black
     #   coverage
+    #   pylint
     #   pytest
-urllib3==1.26.9
+tomlkit==0.11.4
+    # via pylint
+typing-extensions==4.3.0
+    # via
+    #   astroid
+    #   black
+    #   pylint
+urllib3==1.26.12
     # via
     #   docker
     #   requests
-websocket-client==1.3.2
+websocket-client==1.4.1
     # via docker
+wrapt==1.14.1
+    # via astroid
+zipp==3.8.1
+    # via importlib-resources

From 4df76f8055697046b020d727796d6361be1235ad Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Fri, 16 Sep 2022 10:24:15 +0200
Subject: [PATCH 12/20] added validation output

---
 validation/output/output_1/some_output_1_file | 1 +
 validation/output/output_1/some_output_2_file | 1 +
 validation/output/output_2/some_output_1_file | 1 +
 validation/output/output_2/some_output_2_file | 1 +
 validation/output/output_3/some_output_1_file | 1 +
 validation/output/output_3/some_output_2_file | 1 +
 validation/output/output_4/some_output_1_file | 1 +
 validation/output/output_4/some_output_2_file | 1 +
 8 files changed, 8 insertions(+)
 create mode 100644 validation/output/output_1/some_output_1_file
 create mode 100644 validation/output/output_1/some_output_2_file
 create mode 100644 validation/output/output_2/some_output_1_file
 create mode 100644 validation/output/output_2/some_output_2_file
 create mode 100644 validation/output/output_3/some_output_1_file
 create mode 100644 validation/output/output_3/some_output_2_file
 create mode 100644 validation/output/output_4/some_output_1_file
 create mode 100644 validation/output/output_4/some_output_2_file

diff --git a/validation/output/output_1/some_output_1_file b/validation/output/output_1/some_output_1_file
new file mode 100644
index 0000000..8da588a
--- /dev/null
+++ b/validation/output/output_1/some_output_1_file
@@ -0,0 +1 @@
+Hello this is file 1
\ No newline at end of file
diff --git a/validation/output/output_1/some_output_2_file b/validation/output/output_1/some_output_2_file
new file mode 100644
index 0000000..417657c
--- /dev/null
+++ b/validation/output/output_1/some_output_2_file
@@ -0,0 +1 @@
+Hello this is file 2
\ No newline at end of file
diff --git a/validation/output/output_2/some_output_1_file b/validation/output/output_2/some_output_1_file
new file mode 100644
index 0000000..8da588a
--- /dev/null
+++ b/validation/output/output_2/some_output_1_file
@@ -0,0 +1 @@
+Hello this is file 1
\ No newline at end of file
diff --git a/validation/output/output_2/some_output_2_file b/validation/output/output_2/some_output_2_file
new file mode 100644
index 0000000..417657c
--- /dev/null
+++ b/validation/output/output_2/some_output_2_file
@@ -0,0 +1 @@
+Hello this is file 2
\ No newline at end of file
diff --git a/validation/output/output_3/some_output_1_file b/validation/output/output_3/some_output_1_file
new file mode 100644
index 0000000..8da588a
--- /dev/null
+++ b/validation/output/output_3/some_output_1_file
@@ -0,0 +1 @@
+Hello this is file 1
\ No newline at end of file
diff --git a/validation/output/output_3/some_output_2_file b/validation/output/output_3/some_output_2_file
new file mode 100644
index 0000000..417657c
--- /dev/null
+++ b/validation/output/output_3/some_output_2_file
@@ -0,0 +1 @@
+Hello this is file 2
\ No newline at end of file
diff --git a/validation/output/output_4/some_output_1_file b/validation/output/output_4/some_output_1_file
new file mode 100644
index 0000000..8da588a
--- /dev/null
+++ b/validation/output/output_4/some_output_1_file
@@ -0,0 +1 @@
+Hello this is file 1
\ No newline at end of file
diff --git a/validation/output/output_4/some_output_2_file b/validation/output/output_4/some_output_2_file
new file mode 100644
index 0000000..417657c
--- /dev/null
+++ b/validation/output/output_4/some_output_2_file
@@ -0,0 +1 @@
+Hello this is file 2
\ No newline at end of file

From b74661c8c5c401ca5a4d3428cfa3af73f461b2e3 Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Fri, 16 Sep 2022 10:24:37 +0200
Subject: [PATCH 13/20] fix test

---
 tests/integration/test_docker_container.py | 59 ++++++++++++----------
 1 file changed, 33 insertions(+), 26 deletions(-)

diff --git a/tests/integration/test_docker_container.py b/tests/integration/test_docker_container.py
index 3f40994..2d73c73 100644
--- a/tests/integration/test_docker_container.py
+++ b/tests/integration/test_docker_container.py
@@ -8,11 +8,12 @@
 import shutil
 from pathlib import Path
 from pprint import pformat
-from typing import Dict
-
-import pytest
+from typing import Dict, Iterator
 
 import docker
+import docker.errors
+import docker.models.containers
+import pytest
 
 _FOLDER_NAMES = ["input", "output"]
 _CONTAINER_FOLDER = Path("/home/scu/data")
@@ -58,7 +59,7 @@ def docker_container(
     docker_client: docker.DockerClient,
     docker_image_key: str,
     container_variables: Dict,
-) -> docker.models.containers.Container:
+) -> Iterator[docker.models.containers.Container]:
     # copy files to input folder, copytree needs to not have the input folder around.
     host_folders["input"].rmdir()
     shutil.copytree(validation_folders["input"], host_folders["input"])
@@ -74,6 +75,7 @@ def docker_container(
         assert host_output_folder.exists()
 
     # run the container (this may take some time)
+    container = None
     try:
         volumes = {
             host_folders[folder]: {
@@ -89,6 +91,7 @@ def docker_container(
             volumes=volumes,
             environment=container_variables,
         )
+        assert isinstance(container, docker.models.containers.Container)
         response = container.wait()
         if response["StatusCode"] > 0:
             logs = container.logs(timestamps=True)
@@ -106,6 +109,7 @@ def docker_container(
             yield container
     except docker.errors.ContainerError as exc:
         # the container did not run correctly
+        assert isinstance(container, docker.models.containers.Container)
         pytest.fail(
             "The container stopped with exit code {}\n\n\ncommand:\n {}, \n\n\nlog:\n{}".format(
                 exc.exit_status,
@@ -118,7 +122,9 @@ def docker_container(
         )
     finally:
         # cleanup
-        container.remove()
+        if container:
+            assert isinstance(container, docker.models.containers.Container)
+            container.remove()
 
 
 def _convert_to_simcore_labels(image_labels: Dict) -> Dict:
@@ -136,30 +142,31 @@ def _convert_to_simcore_labels(image_labels: Dict) -> Dict:
 
 
 def test_run_container(
-    validation_folders: Dict,
-    host_folders: Dict,
+    validation_folders: Dict[str, Path],
+    host_folders: Dict[str, Path],
     docker_container: docker.models.containers.Container,
 ):
     for folder in _FOLDER_NAMES:
-        if folder != "input":
-            # test if the files that should be there are actually there and correct
-            list_of_files = [
-                x.name
-                for x in validation_folders[folder].iterdir()
-                if not ".gitkeep" in x.name
-            ]
-            for file_name in list_of_files:
-                assert Path(
-                    host_folders[folder] / file_name
-                ).exists(), f"missing {file_name=} in {host_folders[folder]=}"
-            match, mismatch, errors = filecmp.cmpfiles(
-                host_folders[folder],
-                validation_folders[folder],
-                list_of_files,
-                shallow=False,
-            )
-            # assert not mismatch, "wrong/incorrect files in {}".format(host_folders[folder])
-            assert not errors, "missing files in {}".format(host_folders[folder])
+        # test if the files that should be there are actually there and correct
+        list_of_files = [
+            x.relative_to(validation_folders[folder])
+            for x in validation_folders[folder].rglob("*")
+            if x.is_file() and not ".gitkeep" in x.name
+        ]
+        for file_name in list_of_files:
+            assert Path(
+                host_folders[folder] / file_name
+            ).exists(), f"missing {file_name=} in {host_folders[folder]=}"
+        match, mismatch, errors = filecmp.cmpfiles(
+            host_folders[folder],
+            validation_folders[folder],
+            list_of_files,
+            shallow=False,
+        )
+        assert match
+        assert not mismatch
+        # assert not mismatch, "wrong/incorrect files in {}".format(host_folders[folder])
+        assert not errors, "missing files in {}".format(host_folders[folder])
 
     # check the output is correct based on container labels
     output_cfg = {}

From 64cf65e1f4562fbaba225536c54684ea254cba46 Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Fri, 16 Sep 2022 10:25:06 +0200
Subject: [PATCH 14/20] fix missing encoding

---
 tests/integration/test_docker_container.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_docker_container.py b/tests/integration/test_docker_container.py
index 2d73c73..dec3e79 100644
--- a/tests/integration/test_docker_container.py
+++ b/tests/integration/test_docker_container.py
@@ -172,7 +172,7 @@ def test_run_container(
     output_cfg = {}
     output_cfg_file = Path(host_folders["output"] / "outputs.json")
     if output_cfg_file.exists():
-        with output_cfg_file.open() as fp:
+        with output_cfg_file.open(encoding="utf-8") as fp:
             output_cfg = json.load(fp)
 
     container_labels = docker_container.labels

From 51d7f12137e8de8044f6dbbc366e167d03fea9c6 Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Fri, 16 Sep 2022 10:34:24 +0200
Subject: [PATCH 15/20] bumped to version 2.0.0

---
 .cookiecutterrc        | 2 +-
 VERSION                | 2 +-
 metadata/metadata.yml  | 2 +-
 versioning/service.cfg | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.cookiecutterrc b/.cookiecutterrc
index 5be8a79..1bb95bf 100644
--- a/.cookiecutterrc
+++ b/.cookiecutterrc
@@ -35,4 +35,4 @@ default_context:
     project_slug:              'osparc-python-runner'
     project_type:              'computational'
     release_date:              '2020'
-    version:                   '1.3.0'
+    version:                   '2.0.0'
diff --git a/VERSION b/VERSION
index f0bb29e..227cea2 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.3.0
+2.0.0
diff --git a/metadata/metadata.yml b/metadata/metadata.yml
index 5c27e03..306431c 100644
--- a/metadata/metadata.yml
+++ b/metadata/metadata.yml
@@ -2,7 +2,7 @@ name: oSparc Python Runner
 key: simcore/services/comp/osparc-python-runner
 type: computational
 integration-version: 1.1.0
-version: 1.3.0
+version: 2.0.0
 description: oSparc Python Runner
 contact: anderegg@itis.swiss
 authors:
diff --git a/versioning/service.cfg b/versioning/service.cfg
index f8c1aa2..cd676aa 100644
--- a/versioning/service.cfg
+++ b/versioning/service.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.3.0
+current_version = 2.0.0
 commit = False
 message = service/kernel version: {current_version} → {new_version}
 tag = False

From d6fd1e89ccef9eb6daa57d525c9c5d07e4aad92e Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Fri, 16 Sep 2022 10:37:13 +0200
Subject: [PATCH 16/20] linter

---
 src/osparc_python_runner/main.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/osparc_python_runner/main.py b/src/osparc_python_runner/main.py
index 982044b..8a20223 100644
--- a/src/osparc_python_runner/main.py
+++ b/src/osparc_python_runner/main.py
@@ -1,11 +1,9 @@
 import json
 import logging
 import os
-import shutil
 import subprocess
 import sys
 from pathlib import Path
-from typing import Dict
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger("osparc-python-main")
@@ -54,7 +52,7 @@ def _ensure_pip_requirements(code_dir: Path) -> Path:
 
     else:
         requirements = requirements[0]
-        logger.info(f"Found: {requirements}")
+        logger.info("Found: %s", requirements)
     return requirements
 
 
@@ -64,7 +62,13 @@ def _show_io_environments() -> None:
             "%s ENVs available: %s",
             io_type.capitalize(),
             json.dumps(
-                list(filter(lambda x: f"{io_type.upper()}_" in x, os.environ)), indent=2
+                list(
+                    filter(
+                        lambda x, io_type=io_type: f"{io_type.upper()}_" in x,
+                        os.environ,
+                    )
+                ),
+                indent=2,
             ),
         )
 
@@ -93,7 +97,7 @@ def setup():
         'echo "DONE ..."',
     ]
     main_script_path = Path("main.sh")
-    main_script_path.write_text("\n".join(script))
+    main_script_path.write_text("\n".join(script), encoding="utf-8")
 
 
 def teardown():

From eb21b211255d16d0ba33379f6e6cf60c7a798f95 Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Fri, 16 Sep 2022 10:37:43 +0200
Subject: [PATCH 17/20] bumped version

---
 docker-compose-meta.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose-meta.yml b/docker-compose-meta.yml
index f4f747e..c5c4ac8 100644
--- a/docker-compose-meta.yml
+++ b/docker-compose-meta.yml
@@ -31,7 +31,7 @@ services:
           "Output data", "description": "The data produced by the script and saved
           under OUTPUT_FOLDER/output_4 as output_4.zip", "type": "data:*/*"}}}'
         io.simcore.type: '{"type": "computational"}'
-        io.simcore.version: '{"version": "1.3.0"}'
+        io.simcore.version: '{"version": "2.0.0"}'
         org.label-schema.build-date: ${BUILD_DATE}
         org.label-schema.schema-version: '1.0'
         org.label-schema.vcs-ref: ${VCS_REF}

From 3dd06089aee253964da86f05fe1f7542b1465f4b Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Fri, 16 Sep 2022 10:46:42 +0200
Subject: [PATCH 18/20] fixed run creator to generate outputs

---
 service.cli/run      |  2 --
 tools/run_creator.py | 57 ++++++++++++++++++++++++++++++--------------
 2 files changed, 39 insertions(+), 20 deletions(-)

diff --git a/service.cli/run b/service.cli/run
index 58a391c..9e917ee 100755
--- a/service.cli/run
+++ b/service.cli/run
@@ -19,7 +19,6 @@ INPUT_4=$INPUT_FOLDER/input_4
 export INPUT_4
 INPUT_5=$INPUT_FOLDER/input_5
 export INPUT_5
-
 OUTPUT_1=$OUTPUT_FOLDER/output_1
 export OUTPUT_1
 OUTPUT_2=$OUTPUT_FOLDER/output_2
@@ -29,6 +28,5 @@ export OUTPUT_3
 OUTPUT_4=$OUTPUT_FOLDER/output_4
 export OUTPUT_4
 
-
 exec execute.sh
         
\ No newline at end of file
diff --git a/tools/run_creator.py b/tools/run_creator.py
index 3ab7c6c..077694c 100644
--- a/tools/run_creator.py
+++ b/tools/run_creator.py
@@ -36,18 +36,33 @@ def get_input_config(metadata_file: Path) -> Dict:
     return inputs
 
 
+def get_output_config(metadata_file: Path) -> Dict:
+    inputs = {}
+    with metadata_file.open() as fp:
+        metadata = yaml.safe_load(fp)
+        if "outputs" in metadata:
+            inputs = metadata["outputs"]
+    return inputs
+
+
 def main(args=None) -> int:
     try:
         parser = argparse.ArgumentParser(description=__doc__)
-        parser.add_argument("--metadata", help="The metadata yaml of the node",
-                            type=Path, required=False, default="/metadata/metadata.yml")
         parser.add_argument(
-            "--runscript", help="The run script", type=Path, required=True)
+            "--metadata",
+            help="The metadata yaml of the node",
+            type=Path,
+            required=False,
+            default="/metadata/metadata.yml",
+        )
+        parser.add_argument(
+            "--runscript", help="The run script", type=Path, required=True
+        )
         options = parser.parse_args(args)
 
         # generate variables for input
-        input_script = ["""
-#!/bin/sh
+        input_script = [
+            """#!/bin/sh
 #---------------------------------------------------------------
 # AUTO-GENERATED CODE, do not modify this will be overwritten!!!
 #---------------------------------------------------------------
@@ -58,26 +73,32 @@ def main(args=None) -> int:
 cd "$(dirname "$0")"
 json_input=$INPUT_FOLDER/inputs.json
         """
-                        ]
+        ]
         input_config = get_input_config(options.metadata)
         for input_key, input_value in input_config.items():
             if "data:" in input_value["type"]:
-                filename = input_key
-                if "fileToKeyMap" in input_value and len(input_value["fileToKeyMap"]) > 0:
-                    filename, _ = next(
-                        iter(input_value["fileToKeyMap"].items()))
-                input_script.append(
-                    f"{str(input_key).upper()}=$INPUT_FOLDER/{str(filename)}")
-                input_script.append(f"export {str(input_key).upper()}")
+                folder_name = input_key
+                input_script.append(f"{input_key.upper()}=$INPUT_FOLDER/{folder_name}")
+                input_script.append(f"export {input_key.upper()}")
             else:
                 input_script.append(
-                    f"{str(input_key).upper()}=$(< \"$json_input\" jq '.{input_key}')")
-                input_script.append(f"export {str(input_key).upper()}")
-
-        input_script.extend(["""
+                    f"{input_key.upper()}=$(< \"$json_input\" jq '.{input_key}')"
+                )
+                input_script.append(f"export {input_key.upper()}")
+        for output_key, output_value in get_output_config(options.metadata).items():
+            if "data:" in output_value["type"]:
+                folder_name = output_key
+                input_script.append(
+                    f"{output_key.upper()}=$OUTPUT_FOLDER/{folder_name}"
+                )
+                input_script.append(f"export {output_key.upper()}")
+        input_script.extend(
+            [
+                """
 exec execute.sh
         """
-                             ])
+            ]
+        )
 
         # write shell script
         shell_script = str("\n").join(input_script)

From 040fcca31b62b6302f454bc1e79b8b645d7316bf Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Fri, 16 Sep 2022 10:48:18 +0200
Subject: [PATCH 19/20] missing validation folders

---
 validation/input/input_3/.gitkeep | 0
 validation/input/input_4/.gitkeep | 0
 validation/input/input_5/.gitkeep | 0
 3 files changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 validation/input/input_3/.gitkeep
 create mode 100644 validation/input/input_4/.gitkeep
 create mode 100644 validation/input/input_5/.gitkeep

diff --git a/validation/input/input_3/.gitkeep b/validation/input/input_3/.gitkeep
new file mode 100644
index 0000000..e69de29
diff --git a/validation/input/input_4/.gitkeep b/validation/input/input_4/.gitkeep
new file mode 100644
index 0000000..e69de29
diff --git a/validation/input/input_5/.gitkeep b/validation/input/input_5/.gitkeep
new file mode 100644
index 0000000..e69de29

From e67cf959563f525ecc482a567efd49b2776f7c9c Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Fri, 16 Sep 2022 11:08:34 +0200
Subject: [PATCH 20/20] @pcrespov review: no need to expan

---
 validation/input/input_1/input_script_1.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/validation/input/input_1/input_script_1.py b/validation/input/input_1/input_script_1.py
index 8cd58bf..8c4c970 100644
--- a/validation/input/input_1/input_script_1.py
+++ b/validation/input/input_1/input_script_1.py
@@ -35,8 +35,8 @@ def test_inputs():
 
 
 def test_outputs():
-    """write a file in each output"""
-    for output_folder_env in [*(f"OUTPUT_{i}" for i in range(1, 5))]:
+    """write a file in each output, checks that write access is granted"""
+    for output_folder_env in (f"OUTPUT_{i}" for i in range(1, 5)):
         folder = Path(os.environ[output_folder_env])
         assert folder.exists()
         for number in range(1, 3):