Merge branch 'master' into 2240-edit-dataset-image-in-service

.gitignore

@@ -1,6 +1,5 @@
 # Automatically generated by setuptools_scm
 renku/version.py
-renku/templates/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
@@ -90,3 +89,8 @@ renku.rb
 renku-core-*
 requirements.lock
 helm-chart/renku-core/charts
+
+# Local cache
+renku/templates/
+temp/
+tmp/

README.rst

@@ -305,6 +305,17 @@ in editable mode using ``pipx``. Clone the repository and then do:
 
 This will install all the extras for testing and debugging.
 
+If you already use `pyenv <https://github.com/pyenv/pyenv>`__ to manage different python versions,
+you may be interested in installing `pyenv-virtualenv <https://github.com/pyenv/pyenv-virtualenv>`__ to
+create an ad-hoc virtual environment for developing renku.
+
+Once you have created and activated a virtual environment for renku-python, you can use the usual
+`pip` commands to install the required dependencies.
+
+::
+
+    $ pip install -e .[all]  # use `.[all]` for zsh
+
 
 Service
 -------
@@ -315,28 +326,25 @@ variable ``DEBUG_MODE=true`` either in your shell or in the ``.env`` file. Note
 this case the local directory is mounted in the docker container and renku is re-installed
 so it may take a few minutes before the container is ready.
 
+If you have a full RenkuLab deployment at your disposal, you can
+use `telepresence <https://www.telepresence.io/>`__ v1 to develop and debug locally.
+Just run the `start-telepresence.sh` script and follow the instructions. You can also
+attach a remote debugger using the "remote attach" method described later.
+Mind that the script doesn't work with telepresence v2.
+
 
 Running tests
 -------------
 
-To run tests locally with specific version of Python:
+We use `pytest <https://docs.pytest.org>`__ for running tests.
+You can use our `run-tests.sh` script for running specific set of tests.
 
 ::
 
-    $ pyenv install 3.7.5rc1
-    $ pipenv --python ~/.pyenv/versions/3.7.5rc1/bin/python install
-    $ pipenv run tests
-
-
-To recreate environment with different version of Python, it's easy to do so with the following commands:
-
-::
-
-    $ pipenv --rm
-    $ pyenv install 3.6.9
-    $ pipenv --python ~/.pyenv/versions/3.6.9/bin/python install
-    $ pipenv run tests
+    $ ./run-tests.sh -h
 
+We lint the files using `black <https://github.com/psf/black>`__ and
+`isort <https://github.com/PyCQA/isort>`__. 
 
 
 Using External Debuggers
@@ -368,18 +376,18 @@ If using Visual Studio Code, you may also want to set the ``Remote Attach`` conf
 ::
 
     {
-            "name": "Python: Remote Attach",
-            "type": "python",
-            "request": "attach",
-            "port": 5678,
-            "host": "localhost",
-            "pathMappings": [
-                {
-                    "localRoot": "<path-to-renku-python-source-code>",
-                    "remoteRoot": "<path-to-renku-python-source-code>"
-                }
-            ]
-        },
+        "name": "Python: Remote Attach",
+        "type": "python",
+        "request": "attach",
+        "port": 5678,
+        "host": "localhost",
+        "pathMappings": [
+            {
+                "localRoot": "<path-to-renku-python-source-code>",
+                "remoteRoot": "<path-to-renku-python-source-code>"
+            }
+        ]
+    }
 
 
 Kubernetes
@@ -392,6 +400,10 @@ To debug a running renku-core service in a Kubernetes cluster, the service has t
     core:
       debug: true
 
+Also, if you want to be able to modify the files remotely, you need to change
+the `security context` on the `deployment.yaml` file for the renku-core component
+from `runAsUser: 1000` to `runAsGroup: 2000`.
+
 Then install the `Kubernetes extension <https://github.com/Azure/vscode-kubernetes-tools>`_
 and configure your local kubectl with the credentials needed for your cluster.
 

docs/models/index.rst

@@ -29,5 +29,4 @@ Models
    datasets
    provenance
    workflow
-   cwl
    refs

docs/models/projects.rst

@@ -18,5 +18,5 @@
 Projects
 ========
 
-.. automodule:: renku.core.models.projects
+.. automodule:: renku.core.models.project
    :members:

docs/models/provenance.rst

@@ -26,25 +26,17 @@ Extract provenance information from the repository.
 Activities
 ----------
 
-.. py:module:: renku.core.models.provenance.activities
+.. py:module:: renku.core.models.provenance.activity
 
 .. autoclass:: Activity
    :members:
    :inherited-members:
 
-.. autoclass:: ProcessRun
-   :members:
-   :inherited-members:
-
-.. autoclass:: WorkflowRun
-   :members:
-   :inherited-members:
-
 
 Entities
 --------
 
-.. py:module:: renku.core.models.entities
+.. py:module:: renku.core.models.entity
 
 .. autoclass:: Entity
    :members:
@@ -58,7 +50,7 @@ Entities
 Agents
 ------
 
-.. py:module:: renku.core.models.provenance.agents
+.. py:module:: renku.core.models.provenance.agent
 
 .. autoclass:: Person
    :members:
@@ -69,15 +61,11 @@ Agents
    :inherited-members:
 
 
-Relations
----------
-
-.. py:module:: renku.core.models.provenance.qualified
+Annotations
+-----------
 
-.. autoclass:: Usage
-   :members:
-   :inherited-members:
+.. py:module:: renku.core.models.provenance.annotation
 
-.. autoclass:: Generation
+.. autoclass:: Annotation
    :members:
    :inherited-members:

docs/models/workflow.rst

@@ -23,18 +23,50 @@ Renku Workflow
 Renku uses PROV-O and its own Renku ontology to represent workflows.
 
 
-Run
----
+Plans
+-----
 
-.. automodule:: renku.core.models.workflow.run
+.. automodule:: renku.core.models.workflow.plan
+   :members:
+
+
+.. automodule:: renku.core.models.workflow.composite_plan
    :members:
 
 Parameters
 ----------
 
-.. automodule:: renku.core.models.workflow.parameters
+.. automodule:: renku.core.models.workflow.parameter
+   :members:
+
+
+Renku Workflow Logic
+====================
+
+.. py:module:: renku.core.management.workflow
+
+Execution Graph
+---------------
+
+.. automodule:: renku.core.management.workflow.concrete_execution_graph
    :members:
 
+Value Resolution
+----------------
+
+.. automodule:: renku.core.management.workflow.value_resolution
+   :members:
+
+Plan Factory
+------------
+
+Used to create ``Plan`` objects based on command line arguments
+
+.. automodule:: renku.core.management.workflow.plan_factory
+   :members:
+
+
+
 Renku Workflow Conversion
 =========================
 

helm-chart/renku-core/templates/deployment.yaml

@@ -45,7 +45,7 @@ spec:
           {{ if .Values.debug }}
           args: ["api", "--debug"]
           securityContext:
-            runAsGroup: 2000
+            runAsUser: 1000
           {{ else }}
           args: ["api"]
           {{ end }}

renku/api/models/dataset.py

@@ -42,7 +42,7 @@
 from operator import attrgetter
 
 from renku.api.models.project import ensure_project_context
-from renku.core.metadata.database import Database
+from renku.core.management.dataset.datasets_provenance import DatasetsProvenance
 from renku.core.models import dataset as core_dataset
 
 
@@ -85,8 +85,7 @@ def list(project):
         client = project.client
         if not client or not client.has_graph_files():
             return []
-        database = Database.from_path(client.database_path)
-        datasets_provenance = core_dataset.DatasetsProvenance(database)
+        datasets_provenance = DatasetsProvenance()
         return [Dataset._from_dataset(d) for d in datasets_provenance.datasets]
 
     def __getattribute__(self, name):

renku/api/models/run.py

@@ -46,7 +46,7 @@
 from pathlib import Path
 
 from renku.api.models.project import ensure_project_context
-from renku.core.models.cwl.command_line_tool import (
+from renku.core.management.workflow.plan_factory import (
     add_indirect_parameter,
     get_indirect_inputs_path,
     get_indirect_outputs_path,

renku/cli/__init__.py

@@ -77,9 +77,7 @@
 from renku.cli.doctor import doctor
 from renku.cli.exception_handler import IssueFromTraceback
 from renku.cli.githooks import githooks as githooks_command
-from renku.cli.graph import graph
 from renku.cli.init import init as init_command
-from renku.cli.log import log
 from renku.cli.login import login, logout, token
 from renku.cli.migrate import check_immutable_template_files, migrate, migrationscheck
 from renku.cli.move import move
@@ -196,9 +194,9 @@ def help(ctx):
 cli.add_command(dataset)
 cli.add_command(doctor)
 cli.add_command(githooks_command)
-cli.add_command(graph)
 cli.add_command(init_command)
-cli.add_command(log)
+# TODO: reenable once log (or workflow export) is implemented
+# cli.add_command(log)
 cli.add_command(login)
 cli.add_command(logout)
 cli.add_command(migrate)

renku/cli/exception_handler.py

@@ -150,7 +150,7 @@ def _handle_sentry(self):
                 from git import Repo
 
                 from renku.core.commands.git import get_git_home
-                from renku.core.models.provenance.agents import Person
+                from renku.core.models.provenance.agent import Person
 
                 repo = Repo(get_git_home())
                 user = Person.from_git(repo)