Merge branch 'develop' into feature/2406-datasets-renku-log

SwissDataScienceCenter · Feb 7, 2022 · 1d1546e · 1d1546e
2 parents 5511734 + 6effa0e
commit 1d1546e
Show file tree

Hide file tree

Showing 26 changed files with 548 additions and 369 deletions.
diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml
@@ -28,9 +28,10 @@ jobs:
       renku-ui: ${{ steps.deploy-comment.outputs.renku-ui}}
       test-enabled: ${{ steps.deploy-comment.outputs.test-enabled}}
       extra-values: ${{ steps.deploy-comment.outputs.extra-values}}
+      persist: ${{ steps.deploy-comment.outputs.persist}}
     steps:
       - id: deploy-comment
-        uses: SwissDataScienceCenter/renku-actions/check-pr-description@v0.3.2
+        uses: SwissDataScienceCenter/renku-actions/check-pr-description@v0.5.0
         with:
           string: /deploy
           pr_ref: ${{ github.event.number }}
@@ -42,7 +43,7 @@ jobs:
       name: renku-ci-rp-${{ github.event.number }}
     steps:
     - name: deploy-pr
-      uses: SwissDataScienceCenter/renku-actions/deploy-renku@v0.3.2
+      uses: SwissDataScienceCenter/renku-actions/deploy-renku@v0.5.0
       env:
         DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }}
         DOCKER_USERNAME: ${{ secrets.RENKU_DOCKER_USERNAME }}
@@ -84,43 +85,27 @@ jobs:
 
   test-pr:
     runs-on: ubuntu-20.04
-    if: github.event.action != 'closed'
+    if: ${{ github.event.action != 'closed' && needs.check-deploy.outputs.pr-contains-string == 'true' && needs.check-deploy.outputs.test-enabled == 'true' }}
     needs: [check-deploy, deploy-pr]
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v1
-      with:
-        python-version: 3.8
-    - name: Test the PR
-      if: "needs.check-deploy.outputs.pr-contains-string == 'true' && needs.check-deploy.outputs.test-enabled == 'true'"
-      env:
-        KUBECONFIG: ${{ github.workspace }}/renkubot-kube.config
-        RENKUBOT_KUBECONFIG: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }}
-        RENKU_RELEASE: renku-ci-rp-${{ github.event.number }}
-      run: |
-        echo "$RENKUBOT_KUBECONFIG" > ${{ github.workspace }}/renkubot-kube.config
-        helm test ${RENKU_RELEASE} --namespace ${RENKU_RELEASE} --timeout 80m --logs
-    - name: Download artifact for packaging on failure
-      if: failure()
-      uses: SwissDataScienceCenter/renku-actions/download-test-artifacts@v0.3.2
-      env:
-        RENKU_VALUES: ${{ secrets.CI_RENKU_VALUES }}
-        TEST_ARTIFACTS_PATH: "tests-artifacts-${{ github.sha }}"
-    - name: Upload screenshots on failure
-      if: failure()
-      uses: actions/upload-artifact@v1
+    - uses: SwissDataScienceCenter/renku-actions/test-renku@v0.5.0
       with:
-        name: acceptance-test-artifacts
-        path: ${{ github.workspace }}/test-artifacts/
+        renkubot-kubeconfig: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }}
+        renku-release: renku-ci-rp-${{ github.event.number }}
+        gitlab-token: ${{ secrets.DEV_GITLAB_TOKEN }}
+        persist: "${{ needs.check-deploy.outputs.persist }}"
+        ci-renku-values: ${{ secrets.CI_RENKU_VALUES }}
+
   cleanup:
-    if: github.event.action == 'closed'
+    needs: check-deploy
+    if: github.event.action == 'closed' && needs.check-deploy.outputs.pr-contains-string == 'true'
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v2
     - name: renku teardown
-      uses: SwissDataScienceCenter/renku-actions/teardown-renku@v0.3.2
+      uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v0.5.0
       env:
+        HELM_RELEASE_REGEX: "^renku-ci-rp-${{ github.event.number }}$"
         GITLAB_TOKEN: ${{ secrets.DEV_GITLAB_TOKEN }}
-        KUBECONFIG: "${{ github.workspace }}/renkubot-kube.config"
-        RENKU_RELEASE: renku-ci-rp-${{ github.event.number }}
         RENKUBOT_KUBECONFIG: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }}
+        MAX_AGE_SECONDS: 0
+        DELETE_NAMESPACE: "true"
diff --git a/.github/workflows/test_deploy.yml b/.github/workflows/test_deploy.yml
@@ -889,19 +889,19 @@ jobs:
           echo "GIT_USER=Renku Bot" >> $GITHUB_ENV
           echo "GIT_EMAIL=renku@datascience.ch" >> $GITHUB_ENV
       - name: Push chart and images
-        uses: SwissDataScienceCenter/renku-actions/publish-chart@v0.3.2
+        uses: SwissDataScienceCenter/renku-actions/publish-chart@v0.5.0
         env:
           CHART_NAME: renku-core
-          GITHUB_TOKEN: ${{ secrets.RENKU_CI_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.RENKUBOT_GITHUB_TOKEN }}
           DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
       - name: Wait for chart to be available
         run: sleep 120
       - name: Update component version
-        uses: SwissDataScienceCenter/renku-actions/update-component-version@v0.3.2
+        uses: SwissDataScienceCenter/renku-actions/update-component-version@v0.5.0
         env:
           CHART_NAME: renku-core
-          GITHUB_TOKEN: ${{ secrets.RENKU_CI_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.RENKUBOT_GITHUB_TOKEN }}
 
   coveralls-final:
     name: Aggregate coveralls data

diff --git a/.github/workflows/test_publish.yml b/.github/workflows/test_publish.yml
@@ -27,7 +27,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install -e .[all]
+        python -m pip install .[all]
         git config --global --add user.name "Renku @ SDSC"
         git config --global --add user.email "renku@datascience.ch"
     - name: Test with pytest
@@ -60,7 +60,7 @@ jobs:
         curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
         brew install shellcheck node || brew link --overwrite node
         python -m pip install --upgrade pip
-        python -m pip install -e .[all]
+        python -m pip install .[all]
         git config --global --add user.name "Renku @ SDSC"
         git config --global --add user.email "renku@datascience.ch"
     - name: Test with pytest

diff --git a/CHANGES.rst b/CHANGES.rst
@@ -18,8 +18,72 @@
 Changes
 =======
 
-` <https://github.com/SwissDataScienceCenter/renku-python/compare/v1.0.1...v1.0.2>`__ (2022-01-18)
---------------------------------------------------------------------------------------------------
+`1.0.4 <https://github.com/SwissDataScienceCenter/renku-python/compare/v1.0.3...v1.0.4>`__ (2022-01-28)
+-------------------------------------------------------------------------------------------------------
+
+Bug Fixes
+~~~~~~~~~
+
+-  **service:** Unlimited uploaded file size for multiple core-service deployment
+   (`#2609 <https://github.com/SwissDataScienceCenter/renku-python/pull/2609>`__)
+
+`1.0.3 <https://github.com/SwissDataScienceCenter/renku-python/compare/v1.0.2...v1.0.3>`__ (2022-01-26)
+-------------------------------------------------------------------------------------------------------
+
+Bug Fixes
+~~~~~~~~~
+
+-  **core:** Execution graph linking of plans
+   `#2600 <https://github.com/SwissDataScienceCenter/renku-python/issues/2600>`__
+   (`0528d7c <https://github.com/SwissDataScienceCenter/renku-python/commit/0528d7c3a7285ce931d50661d549ae5c159d2e0f>`__)
+-  **core:** fix copying keywords of a plan
+   (`818093f <https://github.com/SwissDataScienceCenter/renku-python/commit/818093fda0a9528063ac34fcb5a87b8ce91c233c>`__)
+-  **core:** fix cwl float type and derived from
+   (`#2570 <https://github.com/SwissDataScienceCenter/renku-python/issues/2570>`__)
+   (`19454ba <https://github.com/SwissDataScienceCenter/renku-python/commit/19454ba89f2eea15cc0051f48a0e60cf373d742d>`__)
+-  **core:** fix SHACL for Plan and CompositePlan
+   (`#2598 <https://github.com/SwissDataScienceCenter/renku-python/issues/2598>`__)
+   (`21b022e <https://github.com/SwissDataScienceCenter/renku-python/commit/21b022e6ebfa0991abb3737aaec2d1f907236944>`__)
+-  **core:** fix Zenodo dataset import if ``sameAs`` is set
+   (`#2572 <https://github.com/SwissDataScienceCenter/renku-python/issues/2572>`__)
+   (`f704916 <https://github.com/SwissDataScienceCenter/renku-python/commit/f7049165b53c69776a5a0a9d2c5ef0fd7b233b62>`__)
+-  **core:** make activity ids deterministic in migration
+   (`#2581 <https://github.com/SwissDataScienceCenter/renku-python/issues/2581>`__)
+   (`7ed6102 <https://github.com/SwissDataScienceCenter/renku-python/commit/7ed6102496abb03329f6b19521232215e31a834a>`__)
+-  **core:** move NodeJS requirement check to cwltool plugin
+   (`#2586 <https://github.com/SwissDataScienceCenter/renku-python/issues/2586>`__)
+   (`1d79ce2 <https://github.com/SwissDataScienceCenter/renku-python/commit/1d79ce27d7661e59e2ddc33b90e6003b16a4e090>`__)
+-  **service:** fix cache.migrate not locking the project
+   (`#2573 <https://github.com/SwissDataScienceCenter/renku-python/issues/2573>`__)
+   (`ed2bcd8 <https://github.com/SwissDataScienceCenter/renku-python/commit/ed2bcd8551f500e3a4a422a6906d0813317b1c77>`__)
+-  **service:** use separate queues for multi core service deployment
+   (`#2602 <https://github.com/SwissDataScienceCenter/renku-python/issues/2602>`__)
+   (`0f3fefb <https://github.com/SwissDataScienceCenter/renku-python/commit/0f3fefb97cadae79a26e4a33ef3aea30e870e2fe>`__)
+
+Features
+~~~~~~~~
+
+-  **cli:** add explicit parameters to renku run
+   (`#2583 <https://github.com/SwissDataScienceCenter/renku-python/issues/2583>`__)
+   (`5118774 <https://github.com/SwissDataScienceCenter/renku-python/commit/511877464266a7c6053bcf78b49560c36135f412>`__)
+-  **core:** extended template variable functionality
+   (`#2120 <https://github.com/SwissDataScienceCenter/renku-python/issues/2120>`__)
+   (`0e13fc1 <https://github.com/SwissDataScienceCenter/renku-python/commit/0e13fc1b4db8b0fd323cc3d3fc0c865ed280fccc>`__)
+-  **core:** ignore quotation mark in git user/email config
+   (`#2537 <https://github.com/SwissDataScienceCenter/renku-python/issues/2537>`__)
+   (`e70481c <https://github.com/SwissDataScienceCenter/renku-python/commit/e70481cd386ceadd9e43e06d373c32e8f6c8669d>`__)
+-  **core:** pass parameters as environment vars to scripts (and renku.api)
+   (`#2596 <https://github.com/SwissDataScienceCenter/renku-python/issues/2596>`__)
+   (`c4fd71c <https://github.com/SwissDataScienceCenter/renku-python/commit/c4fd71c6c3df0b755d88c88c40f1b81792e91b46>`__)
+-  **core:** support forward compatibility of datasets
+   (`#2554 <https://github.com/SwissDataScienceCenter/renku-python/issues/2554>`__)
+   (`c6a7013 <https://github.com/SwissDataScienceCenter/renku-python/commit/c6a7013e02b639bf1894d6b96e53b05a0058cb9e>`__)
+-  **core:** reintroduce shell completion command
+   (`#2562 <https://github.com/SwissDataScienceCenter/renku-python/issues/2562>`__)
+   (`6b3ee60 <https://github.com/SwissDataScienceCenter/renku-python/commit/6b3ee604db4e23bd8b51a323ec0af93fda0d23ff>`__)
+
+`1.0.2 <https://github.com/SwissDataScienceCenter/renku-python/compare/v1.0.1...v1.0.2>`__ (2022-01-18)
+-------------------------------------------------------------------------------------------------------
 
 Bug Fixes
 ~~~~~~~~~
@@ -72,8 +136,8 @@ Features
    (`#2531 <https://github.com/SwissDataScienceCenter/renku-python/issues/2531>`__)
    (`082e897 <https://github.com/SwissDataScienceCenter/renku-python/commit/082e897feac105e772f5672349f4f3535425d4ce>`__)
 
-` <https://github.com/SwissDataScienceCenter/renku-python/compare/v1.0.0...v1.0.1>`__ (2021-12-07)
---------------------------------------------------------------------------------------------------
+`1.0.1 <https://github.com/SwissDataScienceCenter/renku-python/compare/v1.0.0...v1.0.1>`__ (2021-12-07)
+-------------------------------------------------------------------------------------------------------
 
 Bug Fixes
 ~~~~~~~~~
@@ -89,8 +153,8 @@ Bug Fixes
    (`4946f89 <https://github.com/SwissDataScienceCenter/renku-python/commit/4946f89c1e1110a94aa5a17f10ebe9220e3136ce>`__)
 
 
-` <https://github.com/SwissDataScienceCenter/renku-python/compare/v0.16.2...v1.0.0>`__ (2021-12-02)
----------------------------------------------------------------------------------------------------
+`1.0.0 <https://github.com/SwissDataScienceCenter/renku-python/compare/v0.16.2...v1.0.0>`__ (2021-12-02)
+--------------------------------------------------------------------------------------------------------
 
 Overview
 ~~~~~~~~

diff --git a/docs/how-to-guides/implementing_a_provider.rst b/docs/how-to-guides/implementing_a_provider.rst
@@ -53,7 +53,9 @@ The ``workflow_provider`` function shall return a tuple of ``(object, str)``,  w
 should be the plugin object, i.e. ``self`` and the string is a unique identifier of the
 provider plugin. This unique string will be the string that the user can provide to the
 ``--provider`` command line argument to select this plugin for executing the desired
-workflows.
+workflows. A dummy provider implementation is available
+`here <https://github.com/SwissDataScienceCenter/renku-dummy-provider>`_ in order to ease the
+initial implementation of a provider plugin.
 
 A provider HAS to set environment variables for a plans parameters, so they can be used by scripts.
 These environment variables have to be prefixed with the value of the

diff --git a/docs/how-to-guides/shell-integration.rst b/docs/how-to-guides/shell-integration.rst
@@ -5,6 +5,14 @@ Shell integration of Renku CLI
 
 Renku CLI supports shell auto-completion for Renku commands and their arguments like datasets and workflows.
 
+A convenience method is available for printing to the standard output the shell completion command for the
+currently used shell:
+
+.. code-block:: console
+
+        $ renku env --shell-completion
+        _RENKU_COMPLETE=zsh_source renku
+
 To activate tab completion for your supported shell run the following command after installing Renku CLI:
 
 .. tabs::
@@ -37,7 +45,7 @@ in case of ``renku workflow execute`` the available ``Plans`` are going to be li
 
 .. note::
 
-   Tab completion of available ``Plans`` only works if the user is executing the command
+   Tab completion of available ``Plans`` (or ``Datasets``) only works if the user is executing the command
    within a Renku project.
 
 

diff --git a/docs/reference/plugins.rst b/docs/reference/plugins.rst
@@ -54,6 +54,8 @@ where `myproject.pluginmodule` points to a Renku `hookimpl` e.g.:
 .. automodule:: renku.core.plugins.run
    :members:
 
+`This <https://github.com/SwissDataScienceCenter/renku-dummy-annotator>`_ repository contains an implementation of
+an activity annotation plugin.
 
 CLI Plugins
 -----------
@@ -82,6 +84,25 @@ where `myproject.pluginmodule:mycmd` points to a click command e.g.:
     def mycmd():
         ...
 
+An example implementation of such plugin is available `here <https://github.com/SwissDataScienceCenter/renku-cli-plugin-example>`_.
+
+Workflow Converter Plugins
+--------------------------
+
+Additional workflow converters can be implemented by extending
+:class:`renku.core.models.workflow.converters.IWorkflowConverter`. By default renku
+provides a CWL converter plugins that is used when exporting a workflow:
+
+.. code-block:: console
+
+   $ renku workflow export --format cwl <my_workflow>
+
+.. autoclass:: renku.core.models.workflow.converters.IWorkflowConverter
+    :members:
+
+We created a `dummy <https://github.com/SwissDataScienceCenter/renku-dummy-format>`_ implementation of such
+a converter plugin.
+
 Workflow Provider Plugins
 -------------------------
 

diff --git a/helm-chart/renku-core/Chart.yaml b/helm-chart/renku-core/Chart.yaml
@@ -3,4 +3,4 @@ appVersion: "1.0"
 description: A Helm chart for Kubernetes
 name: renku-core
 icon: https://avatars0.githubusercontent.com/u/53332360?s=400&u=a4311d22842343604ef61a8c8a1e5793209a67e9&v=4
-version: 1.0.2
+version: 1.0.4
diff --git a/helm-chart/renku-core/requirements.yaml b/helm-chart/renku-core/requirements.yaml
@@ -2,4 +2,6 @@ dependencies:
 - name: redis
   version: 10.7.11
   repository: "https://charts.bitnami.com/bitnami"
-
+- name: certificates
+  version: 0.0.1
+  repository: "https://swissdatasciencecenter.github.io/helm-charts/"
diff --git a/helm-chart/renku-core/templates/configmap.yaml b/helm-chart/renku-core/templates/configmap.yaml
@@ -17,6 +17,7 @@ data:
 
       sendfile on;
       tcp_nopush on;
+      client_max_body_size 0; # Required for uploading large files
 
       location /renku/versions {
         root /;