Merge branch 'develop' into feature/2977-datadir-per-dataset

.github/workflows/test_deploy.yml

@@ -774,6 +774,8 @@ jobs:
           OLOS_ACCESS_TOKEN: ${{ secrets.OLOS_ACCESS_TOKEN }}
           RENKU_REQUESTS_TIMEOUT_SECONDS: 120
         run: pytest -m "integration and not service and not serial" -v --timeout=600 -n auto
+      - name: Start Redis
+        uses: supercharge/redis-github-action@1.4.0
       - name: Test with pytest (serial)
         env:
           POETRY_VIRTUALENVS_CREATE: false
@@ -923,6 +925,8 @@ jobs:
           OLOS_ACCESS_TOKEN: ${{ secrets.OLOS_ACCESS_TOKEN }}
           RENKU_REQUESTS_TIMEOUT_SECONDS: 120
         run: pytest -m "integration and not serial" -v
+      - name: Start Redis
+        uses: supercharge/redis-github-action@1.4.0
       - name: Test with pytest(serial)
         env:
           POETRY_VIRTUALENVS_CREATE: false
@@ -1074,3 +1078,4 @@ jobs:
         with:
           github-token: ${{ secrets.github_token }}
           parallel-finished: true
+          base-path: renku/

conftest.py

@@ -41,8 +41,10 @@
 GLOBAL_FIXTURE_LOCATIONS = [
     "tests.fixtures.common",
     "tests.fixtures.config",
+    "tests.fixtures.domain_models",
     "tests.fixtures.repository",
     "tests.fixtures.runners",
+    "tests.fixtures.session",
     "tests.fixtures.templates",
 ]
 

pyproject.toml

@@ -57,7 +57,6 @@ appdirs = "<=1.4.4,>=1.4.3"
 attrs = "<21.5.0,>=21.1.0"
 black = { version = "==22.3.0", optional = true }
 calamus = "<0.4,>=0.3.13"
-check-manifest = { version = ">=0.37,<0.49", optional = true }
 circus = { version = "==0.17.1", optional = true }
 click = ">=8.0,<8.1.3"
 click-option-group = "<0.6.0,>=0.5.2"
@@ -69,14 +68,12 @@ cwltool = "==3.1.20211107152837"
 deepdiff = "^5.8.0"
 deepmerge = "==1.0.1"
 docker = "<6,>=3.7.2"
-environ-config = ">=18.2.0,<22.2.0"
 fakeredis = { version = ">=1.4.1,<1.7.2", extras = ["lua"], optional = true }
 filelock = ">=3.3.0,<3.6.1"
 flake8 = { version = "<4.0,>=3.8", optional = true }                             #wait for https://github.com/flakehell/flakehell/pull/23 to be merged before bumping
 flakehell = { version = ">=0.9.0,<1.0.*", optional = true }
 flaky = { version = "==3.7.0", optional = true }
 flask = { version = "==2.1.1", optional = true }
-freezegun = { version = ">=0.3.12,<1.2.2", optional = true }
 gitpython = "==3.1.27"
 grandalf = "==0.7"
 gunicorn = { optional = true, version = "*" }
@@ -88,7 +85,6 @@ isort = { version = "<5.10.2,>=5.3.2", optional = true }
 jinja2 = { version = ">=2.11.3,<3.1.3" }
 marshmallow = { version = ">=3.13.0,<3.16.0", optional = true }
 mypy = {version = ">=0.942,<1.0", optional = true}
-ndg-httpsclient = "==0.5.1"
 networkx = "<2.7,>=2.6.0"
 numpy = ">=1.20.0,<1.22.0"
 packaging = "<22.0,>=21.3"
@@ -101,8 +97,6 @@ pluggy = "==1.0.0"
 poetry-dynamic-versioning = { version = "^0.14.0", optional = true }
 portalocker = ">=2.2.1,<2.5"
 psutil = ">=5.4.7,<5.9.1,"
-ptvsd = { version = "<4.4.0,>=4.3.0", optional = true }
-pyasn1 = "<=0.4.8,>=0.4.5"
 pydocstyle = { version = "<6.1.2,>=4.0.1", optional = true }
 pyjwt = ">=2.1.0,<2.4.0"
 pyld = "==2.0.3"
@@ -146,7 +140,6 @@ types-redis = { version=">=3.5.3,<4.1.0", optional = true }
 types-requests = { version = "<2.27.2,>=2.23.0", optional = true }
 types-tabulate = { version = "<0.8.10,>=0.7.7", optional = true }
 walrus = { version = ">=0.8.2,<0.10.0", optional = true }
-wcmatch = "<8.3,>=6.0.0"
 werkzeug = ">=1.0.0,<2.1.2"
 yagup = ">=0.1.1"
 yaspin = "==2.1.0"
@@ -318,7 +311,7 @@ pattern = """
 files = ["renku/version.py"]
 
 [tool.pytest.ini_options]
-addopts = "--flake8 --black --doctest-glob=\"*.rst\" --doctest-modules --cov=renku --cov-report=term-missing --ignore=docs/cheatsheet/"
+addopts = "--flake8 --black --doctest-glob=\"*.rst\" --doctest-modules --cov --cov-report=term-missing --ignore=docs/cheatsheet/"
 doctest_optionflags = "ALLOW_UNICODE"
 flake8-ignore = ["*.py", "E121", "E126", "E203", "E226", "E231", "W503", "W504", "docs/conf.py", "docs/cheatsheet/conf.py", "ALL"]
 flake8-max-line-length = 120
@@ -328,6 +321,7 @@ markers = [
     "jobs: mark a test as a job test.",
     "migration: mark a test as a migration test.",
     "publish: mark tests that publish datasets to external providers.",
+    "redis: mark tests that need a running redis",
     "serial: mark a test that can not be run in parallel",
     "service: mark a test as service test.",
     "shelled: mark a test as a shelled test.",
@@ -392,7 +386,8 @@ module = "renku.core.migration.*"
 ignore_errors = true
 
 [tool.coverage.run]
-omit = ["renku/conftest.py", "renku/data/*"]
+omit = ["renku/conftest.py", "renku/data/*", "docs/*", "tests/*"]
+relative_files = true
 
 [tool.flakehell]
 max_line_length = 120

renku/command/graph.py

@@ -50,11 +50,11 @@
 
 def export_graph_command():
     """Return a command for exporting graph data."""
-    return Command().command(_export_graph).with_database(write=False).require_migration()
+    return Command().command(export_graph).with_database(write=False).require_migration()
 
 
 @inject.autoparams("client_dispatcher")
-def _export_graph(
+def export_graph(
     client_dispatcher: IClientDispatcher,
     format: str = "json-ld",
     revision_or_range: str = None,
@@ -75,9 +75,9 @@ def _export_graph(
     format = format.lower()
 
     if revision_or_range:
-        graph = _get_graph_for_revision(revision_or_range=revision_or_range)
+        graph = get_graph_for_revision(revision_or_range=revision_or_range)
     else:
-        graph = _get_graph_for_all_objects()
+        graph = get_graph_for_all_objects()
 
     # NOTE: rewrite ids for current environment
     host = get_host(client_dispatcher.current_client)
@@ -111,7 +111,7 @@ def update_nested_node_host(node: Dict, host: str) -> None:
 
 
 @inject.autoparams()
-def _get_graph_for_revision(
+def get_graph_for_revision(
     revision_or_range: str,
     database_gateway: IDatabaseGateway,
     project_gateway: IProjectGateway,
@@ -142,7 +142,7 @@ def _get_graph_for_revision(
 
 
 @inject.autoparams()
-def _get_graph_for_all_objects(
+def get_graph_for_all_objects(
     project_gateway: IProjectGateway,
     dataset_gateway: IDatasetGateway,
     activity_gateway: IActivityGateway,
@@ -223,6 +223,10 @@ def _convert_entities_to_graph(
             # NOTE: Since the database is read-only, it's OK to modify objects; they won't be written back
             entity.unfreeze()
             entity.project_id = project_id
+
+            if isinstance(entity, Activity):
+                entity.association.plan.unfreeze()
+                entity.association.plan.project_id = project_id
         schema = next(s for t, s in schemas.items() if isinstance(entity, t))
         graph.extend(schema(flattened=True).dump(entity))
 

renku/domain_model/session.py

@@ -44,16 +44,13 @@ def get_name(self) -> str:
         pass
 
     @abstractmethod
-    def build_image(self, image_descriptor: Path, image_name: str, config: Optional[Dict[str, Any]]) -> Optional[str]:
+    def build_image(self, image_descriptor: Path, image_name: str, config: Optional[Dict[str, Any]]):
         """Builds the container image.
 
         Args:
             image_descriptor: Path to the container image descriptor file.
             image_name: Container image name.
             config: Path to the session provider specific configuration YAML.
-
-        Returns:
-            str: a unique id for the created interactive session.
         """
         pass
 

renku/infrastructure/gateway/database_gateway.py

@@ -142,8 +142,6 @@ def commit(self) -> None:
 
     def get_modified_objects_from_revision(self, revision_or_range: str) -> Generator[Persistent, None, None]:
         """Get all database objects modified in a revision."""
-        # TODO: use gateway once #renku-python/issues/2253 is done
-
         client_dispatcher = inject.instance(IClientDispatcher)
         client = client_dispatcher.current_client
 

run-tests.sh

@@ -51,7 +51,7 @@ build_docs(){
 }
 
 run_tests(){
-    pytest -v -m "not integration and not publish" -o testpaths="tests renku conftest.py" --ignore=renku/version.py
+    pytest -v -m "not integration and not publish and not redis" -o testpaths="tests renku conftest.py" --ignore=renku/version.py
 }
 
 usage(){

tests/cli/test_graph.py

@@ -28,8 +28,7 @@
 
 
 @pytest.mark.parametrize("revision", ["", "HEAD", "HEAD^", "HEAD^..HEAD"])
-@pytest.mark.parametrize("format", ["json-ld", "rdf", "nt"])
-def test_graph_export_validation(runner, client, directory_tree, run, revision, format):
+def test_graph_export_validation(runner, client, directory_tree, run, revision):
     """Test graph validation when exporting."""
     assert 0 == runner.invoke(cli, ["dataset", "add", "-c", "my-data", str(directory_tree)]).exit_code
 
@@ -38,15 +37,15 @@ def test_graph_export_validation(runner, client, directory_tree, run, revision,
     assert 0 == run(["run", "head", str(file1)], stdout="out1")
     assert 0 == run(["run", "tail", str(file2)], stdout="out2")
 
-    result = runner.invoke(cli, ["graph", "export", "--format", format, "--strict", "--revision", revision])
+    result = runner.invoke(cli, ["graph", "export", "--format", "json-ld", "--strict", "--revision", revision])
 
     assert 0 == result.exit_code, format_result_exception(result)
 
     assert "https://localhost" in result.output
     assert "https://renkulab.io" not in result.output
 
     with modified_environ(RENKU_DOMAIN="https://renkulab.io"):
-        result = runner.invoke(cli, ["graph", "export", "--format", format, "--strict", "--revision", revision])
+        result = runner.invoke(cli, ["graph", "export", "--format", "json-ld", "--strict", "--revision", revision])
 
         assert 0 == result.exit_code, format_result_exception(result)
 
@@ -59,16 +58,15 @@ def test_graph_export_validation(runner, client, directory_tree, run, revision,
 
 @pytest.mark.serial
 @pytest.mark.shelled
-@pytest.mark.parametrize("format", ["json-ld", "nt", "rdf"])
-def test_graph_export_strict_run(runner, project, run_shell, format):
+def test_graph_export_strict_run(runner, project, run_shell):
     """Test graph export output of run command."""
     # Run a shell command with pipe.
     assert run_shell('renku run --name run1 echo "my input string" > my_output_file')[1] is None
     assert run_shell("renku run --name run2 cp my_output_file my_output_file2")[1] is None
     assert run_shell("renku workflow compose my-composite-plan run1 run2")[1] is None
 
     # Assert created output file.
-    result = runner.invoke(cli, ["graph", "export", "--full", "--strict", "--format={}".format(format)])
+    result = runner.invoke(cli, ["graph", "export", "--full", "--strict", "--format=json-ld"])
     assert 0 == result.exit_code, format_result_exception(result)
     assert "my_output_file" in result.output
     assert "my input string" in result.output
@@ -79,19 +77,18 @@ def test_graph_export_strict_run(runner, project, run_shell, format):
     assert run_shell("renku workflow remove run2")[1] is None
 
     # Assert created output file.
-    result = runner.invoke(cli, ["graph", "export", "--strict", "--format={}".format(format)])
+    result = runner.invoke(cli, ["graph", "export", "--strict", "--format=json-ld"])
     assert 0 == result.exit_code, format_result_exception(result)
 
 
-@pytest.mark.parametrize("format", ["json-ld", "nt", "rdf"])
-def test_graph_export_strict_dataset(tmpdir, runner, project, client, format, subdirectory):
+def test_graph_export_strict_dataset(tmpdir, runner, project, client, subdirectory):
     """Test output of graph export for dataset add."""
     result = runner.invoke(cli, ["dataset", "create", "my-dataset"])
     assert 0 == result.exit_code, format_result_exception(result)
     paths = []
     test_paths = []
     for i in range(3):
-        new_file = tmpdir.join("file_{0}".format(i))
+        new_file = tmpdir.join(f"file_{i}")
         new_file.write(str(i))
         paths.append(str(new_file))
         test_paths.append(os.path.relpath(str(new_file), str(project)))
@@ -100,14 +97,14 @@ def test_graph_export_strict_dataset(tmpdir, runner, project, client, format, su
     result = runner.invoke(cli, ["dataset", "add", "my-dataset"] + paths)
     assert 0 == result.exit_code, format_result_exception(result)
 
-    result = runner.invoke(cli, ["graph", "export", "--strict", f"--format={format}"])
+    result = runner.invoke(cli, ["graph", "export", "--strict", "--format=json-ld"])
     assert 0 == result.exit_code, format_result_exception(result)
     assert all(p in result.output for p in test_paths), result.output
 
     # check that only most recent dataset is exported
     assert 1 == result.output.count("http://schema.org/Dataset")
 
-    result = runner.invoke(cli, ["graph", "export", "--strict", f"--format={format}", "--full"])
+    result = runner.invoke(cli, ["graph", "export", "--strict", "--format=json-ld", "--full"])
     assert 0 == result.exit_code, format_result_exception(result)
     assert all(p in result.output for p in test_paths), result.output
 

tests/cli/test_integration_datasets.py

@@ -189,7 +189,7 @@ def test_dataset_import_real_doi_warnings(runner, project, sleep_after):
 
     result = runner.invoke(cli, ["dataset", "ls"])
     assert 0 == result.exit_code, format_result_exception(result) + str(result.stderr_bytes)
-    assert "pyndl_naive_discr_v0.6.0" in result.output
+    assert "pyndl_naive_discr_v0.8.2" in result.output
 
 
 @pytest.mark.parametrize(