Merge pull request #3653 from SwissDataScienceCenter/release/v2.8.0

SwissDataScienceCenter · Nov 20, 2023 · c1be009 · c1be009
2 parents 74ccfb7 + 4ca2032
commit c1be009
Show file tree

Hide file tree

Showing 505 changed files with 4,113 additions and 3,262 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -8,4 +8,5 @@
 !.git
 !.gitignore
 !Makefile
+!gunicorn.conf.py
 .git/config
diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml
@@ -26,12 +26,13 @@ jobs:
       renku-graph: ${{ steps.deploy-comment.outputs.renku-graph}}
       renku-notebooks: ${{ steps.deploy-comment.outputs.renku-notebooks}}
       renku-ui: ${{ steps.deploy-comment.outputs.renku-ui}}
+      renku-data-services: ${{ steps.deploy-comment.outputs.renku-data-services}}
       test-enabled: ${{ steps.deploy-comment.outputs.test-enabled}}
       extra-values: ${{ steps.deploy-comment.outputs.extra-values}}
       persist: ${{ steps.deploy-comment.outputs.persist}}
     steps:
       - id: deploy-comment
-        uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.4.5
+        uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.7.0
         with:
           string: /deploy
           pr_ref: ${{ github.event.number }}
@@ -43,7 +44,7 @@ jobs:
       name: renku-ci-rp-${{ github.event.number }}
     steps:
       - name: deploy-pr
-        uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.4.5
+        uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.7.0
         env:
           DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }}
           DOCKER_USERNAME: ${{ secrets.RENKU_DOCKER_USERNAME }}
@@ -52,7 +53,7 @@ jobs:
           RANCHER_PROJECT_ID: ${{ secrets.CI_RANCHER_PROJECT }}
           RENKU_RELEASE: renku-ci-rp-${{ github.event.number }}
           RENKU_VALUES_FILE: "${{ github.workspace }}/values.yaml"
-          RENKU_VALUES: ${{ secrets.CI_RENKU_VALUES }}
+          RENKU_VALUES: ${{ secrets.COMBINED_CHARTS_CI_RENKU_VALUES }}
           RENKUBOT_KUBECONFIG: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }}
           RENKUBOT_RANCHER_BEARER_TOKEN: ${{ secrets.RENKUBOT_RANCHER_BEARER_TOKEN }}
           RANCHER_DEV_API_ENDPOINT: ${{ secrets.RANCHER_DEV_API_ENDPOINT }}
@@ -66,6 +67,7 @@ jobs:
           renku_graph: "${{ needs.check-deploy.outputs.renku-graph }}"
           renku_notebooks: "${{ needs.check-deploy.outputs.renku-notebooks }}"
           renku_ui: "${{ needs.check-deploy.outputs.renku-ui }}"
+          renku_data_services: "${{ needs.check-deploy.outputs.renku-data-services }}"
           extra_values: "${{ needs.check-deploy.outputs.extra-values }}"
       - name: Check existing renkubot comment
         uses: peter-evans/find-comment@v2
@@ -88,7 +90,7 @@ jobs:
     if: ${{ github.event.action != 'closed' && needs.check-deploy.outputs.pr-contains-string == 'true' && needs.check-deploy.outputs.test-enabled == 'true' }}
     needs: [check-deploy, deploy-pr]
     steps:
-      - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.4.5
+      - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.7.0
         with:
           kubeconfig: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }}
           renku-release: renku-ci-rp-${{ github.event.number }}
@@ -103,7 +105,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: renku teardown
-        uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.4.5
+        uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.7.0
         env:
           HELM_RELEASE_REGEX: "^renku-ci-rp-${{ github.event.number }}$"
           GITLAB_TOKEN: ${{ secrets.DEV_GITLAB_TOKEN }}

diff --git a/.github/workflows/test_deploy.yml b/.github/workflows/test_deploy.yml
@@ -496,9 +496,7 @@ jobs:
           CLOUD_STORAGE_AZURE_KEY: ${{ secrets.CLOUD_STORAGE_AZURE_KEY }}
           CLOUD_STORAGE_S3_ACCESS_KEY_ID: ${{ secrets.CLOUD_STORAGE_S3_ACCESS_KEY_ID }}
           CLOUD_STORAGE_S3_SECRET_ACCESS_KEY: ${{ secrets.CLOUD_STORAGE_S3_SECRET_ACCESS_KEY }}
-        run: pytest -m "integration and not serial and not service" -v
-      - name: Start Redis
-        uses: supercharge/redis-github-action@1.5.0
+        run: pytest -m "integration and not serial and not service and not redis" -v
       - name: Test with pytest (serial)
         env:
           POETRY_VIRTUALENVS_CREATE: false
@@ -589,19 +587,16 @@ jobs:
         run: |
           echo "GIT_USER=Renku Bot" >> $GITHUB_ENV
           echo "GIT_EMAIL=renku@datascience.ch" >> $GITHUB_ENV
-      - name: Push chart and images
-        uses: SwissDataScienceCenter/renku-actions/publish-chart@v1.4.5
+      - name: Build and push images
+        uses: SwissDataScienceCenter/renku-actions/publish-chartpress-images@v1.7.0
         env:
-          CHART_NAME: renku-core
           GITHUB_TOKEN: ${{ secrets.RENKUBOT_GITHUB_TOKEN }}
           DOCKER_USERNAME: ${{ secrets.RENKU_DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }}
-      - name: Wait for chart to be available
-        run: sleep 120
       - name: Update component version
-        uses: SwissDataScienceCenter/renku-actions/update-component-version@v1.4.5
+        uses: SwissDataScienceCenter/renku-actions/update-component-version@v1.7.0
         env:
-          CHART_NAME: renku-core
+          COMPONENT_NAME: renku-core
           GITHUB_TOKEN: ${{ secrets.RENKUBOT_GITHUB_TOKEN }}
 
   coveralls-final:

diff --git a/.gitignore b/.gitignore
@@ -35,6 +35,7 @@ var/
 *.egg-info/
 .installed.cfg
 *.egg
+.python-version
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

diff --git a/AUTHORS.rst b/AUTHORS.rst
@@ -1,5 +1,5 @@
 ..
-    Copyright 2017-2023 - Swiss Data Science Center (SDSC)
+    Copyright Swiss Data Science Center (SDSC).
     A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
     Eidgenössische Technische Hochschule Zürich (ETHZ).
 

diff --git a/CHANGES.rst b/CHANGES.rst
@@ -1,5 +1,5 @@
 ..
-    Copyright 2017-2023 - Swiss Data Science Center (SDSC)
+    Copyright Swiss Data Science Center (SDSC).
     A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
     Eidgenössische Technische Hochschule Zürich (ETHZ).
 
@@ -18,6 +18,57 @@
 Changes
 =======
 
+`2.8.0 <https://github.com/SwissDataScienceCenter/renku-python/compare/v2.7.0...v2.8.0>`__ (2023-11-17)
+-------------------------------------------------------------------------------------------------------
+
+Bug Fixes
+~~~~~~~~~
+
+-  **cli:** do not error unnecessarily on session stop
+   (`#3642 <https://github.com/SwissDataScienceCenter/renku-python/issues/3642>`__)
+   (`4726f66 <https://github.com/SwissDataScienceCenter/renku-python/commit/4726f660e33a6eb3706b20f661faa635fbe6356b>`__)
+-  **cli:** do not pass the force-build flag to docker
+   (`#3641 <https://github.com/SwissDataScienceCenter/renku-python/issues/3641>`__)
+   (`ff5e8f6 <https://github.com/SwissDataScienceCenter/renku-python/commit/ff5e8f63b94691a65dfe5e44438c4f819819f7ee>`__)
+-  **cli:** do not start a session when in detached HEAD state
+   (`#3636 <https://github.com/SwissDataScienceCenter/renku-python/issues/3636>`__)
+   (`80f70f5 <https://github.com/SwissDataScienceCenter/renku-python/commit/80f70f554ef552b67683ba2498959dbca4aa8200>`__)
+-  **core:** do not clone submodules in renku clone command
+   (`#3630 <https://github.com/SwissDataScienceCenter/renku-python/issues/3630>`__)
+   (`e9986e0 <https://github.com/SwissDataScienceCenter/renku-python/commit/e9986e0d4a882cdf1b84603c9f08010d7f949972>`__)
+-  don’t fail on exotic git diff types
+   (`#3632 <https://github.com/SwissDataScienceCenter/renku-python/issues/3632>`__)
+   (`160620c <https://github.com/SwissDataScienceCenter/renku-python/commit/160620c447f1ab7f6935ef63e3f92437c7537f98>`__)
+-  make usage of name and slug consistent
+   (`#3620 <https://github.com/SwissDataScienceCenter/renku-python/issues/3620>`__)
+   (`b31ade0 <https://github.com/SwissDataScienceCenter/renku-python/commit/b31ade02eed873e09e790a46d92caaf2379871ce>`__)
+-  **service:** set job timeouts correctly
+   (`#3651 <https://github.com/SwissDataScienceCenter/renku-python/issues/3651>`__)
+   (`db0f8cf <https://github.com/SwissDataScienceCenter/renku-python/commit/db0f8cf87d8f234e87aa48058eceaaf68283cedf>`__)
+-  warn user about old version of git when using mergetool
+   (`#3637 <https://github.com/SwissDataScienceCenter/renku-python/issues/3637>`__)
+   (`5df0ce1 <https://github.com/SwissDataScienceCenter/renku-python/commit/5df0ce17e2264412ad081909d08a80f4e6523ae9>`__)
+
+Features
+~~~~~~~~
+
+-  add prometheus metrics
+   (`#3640 <https://github.com/SwissDataScienceCenter/renku-python/issues/3640>`__)
+   (`d3eedb5 <https://github.com/SwissDataScienceCenter/renku-python/commit/d3eedb5e716cdc3299021f5e0c75d678c7c85da0>`__)
+-  **cli, service:** support for project image
+   (`#3623 <https://github.com/SwissDataScienceCenter/renku-python/issues/3623>`__)
+   (`db9f93b <https://github.com/SwissDataScienceCenter/renku-python/commit/db9f93b83e077e62b0196305a82b860fcc2cad6b>`__)
+-  **cli:** session pause and resume
+   (`#3633 <https://github.com/SwissDataScienceCenter/renku-python/issues/3633>`__)
+   (`f4b6480 <https://github.com/SwissDataScienceCenter/renku-python/commit/f4b648019cb0ef09e1591e34dbcdd56d2672f018>`__)
+-  **service:** accept commit SHA in read endpoints
+   (`#3608 <https://github.com/SwissDataScienceCenter/renku-python/issues/3608>`__)
+   (`01a0798 <https://github.com/SwissDataScienceCenter/renku-python/commit/01a0798562357167944aab828e5e6b7967775f8a>`__)
+-  **svc:** rename git_url to template_git_url and return git_url on all
+   endpoints
+   (`#3646 <https://github.com/SwissDataScienceCenter/renku-python/issues/3646>`__)
+   (`314786e <https://github.com/SwissDataScienceCenter/renku-python/commit/314786e8f685d2e202de801434bf1dceed9e17bf>`__)
+
 `2.7.0 <https://github.com/SwissDataScienceCenter/renku-python/compare/v2.6.2...v2.7.0>`__ (2023-09-27)
 -------------------------------------------------------------------------------------------------------
 

diff --git a/DEVELOPING.rst b/DEVELOPING.rst
@@ -261,7 +261,7 @@ number.
 - Once the release PR has been merged, publish the github release. This creates
   the tag on master that kicks off the publishing CI.
 - Keep an eye on CI, make sure that the `publish-pypi`, `build-images` and
-  `publish-chart` finish successfully.
+  `publish-chartpress-images` finish successfully.
   * If any of them don't finish successfully, ask for help.
 - Go to the `Renku` repository and approve/merge the automatically created PR
   there.

diff --git a/Dockerfile b/Dockerfile
@@ -15,7 +15,7 @@ RUN apt-get install --no-install-recommends -y build-essential && \
 # time the code changes
 # set the BUILD_CORE_SERVICE to non null to install additional service dependencies
 ARG BUILD_CORE_SERVICE
-COPY pyproject.toml poetry.lock README.rst CHANGES.rst Makefile /code/renku/
+COPY pyproject.toml poetry.lock README.rst CHANGES.rst Makefile gunicorn.conf.py /code/renku/
 COPY .git /code/renku/.git
 COPY renku /code/renku/renku
 WORKDIR /code/renku
@@ -46,6 +46,7 @@ RUN addgroup -gid 1000 shuhitsu && \
     if [ -n "${BUILD_CORE_SERVICE}" ]; then mkdir /svc && chown shuhitsu:shuhitsu /svc ; fi
 
 COPY --from=builder /code/renku /code/renku
+WORKDIR /code/renku
 ENV PATH="${PATH}:/code/renku/.venv/bin"
 
 # shuhitsu (執筆): The "secretary" of the renga, as it were, who is responsible for
@@ -55,5 +56,6 @@ USER shuhitsu
 ENV RENKU_SVC_NUM_WORKERS 4
 ENV RENKU_SVC_NUM_THREADS 8
 ENV RENKU_DISABLE_VERSION_CHECK=1
+ENV PROMETHEUS_MULTIPROC_DIR /tmp
 
 ENTRYPOINT ["tini", "-g", "--", "renku"]
diff --git a/LICENSE b/LICENSE
@@ -187,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2019-2023 - Swiss Data Science Center
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

diff --git a/Makefile b/Makefile
@@ -1,7 +1,5 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright 2017-2023 - Swiss Data Science Center (SDSC)
-# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Copyright Swiss Data Science Center (SDSC). A partnership between
+# École Polytechnique Fédérale de Lausanne (EPFL) and
 # Eidgenössische Technische Hochschule Zürich (ETHZ).
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

diff --git a/QA_PROTOCOL.rst b/QA_PROTOCOL.rst
@@ -1,5 +1,5 @@
 ..
-    Copyright 2017-2023 - Swiss Data Science Center (SDSC)
+    Copyright Swiss Data Science Center (SDSC).
     A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
     Eidgenössische Technische Hochschule Zürich (ETHZ).
 

diff --git a/README.rst b/README.rst
@@ -1,5 +1,5 @@
 ..
-    Copyright 2017-2023 - Swiss Data Science Center (SDSC)
+    Copyright Swiss Data Science Center (SDSC).
     A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
     Eidgenössische Technische Hochschule Zürich (ETHZ).
 

diff --git a/chartpress.yaml b/chartpress.yaml
@@ -11,8 +11,7 @@ charts:
       renku-core:
         contextPath: .
         dockerfilePath: Dockerfile
-        valuesPath: versions.latest.image
+        valuesPath: global.core.versions.latest.image
         buildArgs:
           CLEAN_INSTALL: "1"
           BUILD_CORE_SERVICE: "1"
-
diff --git a/docs/_ext/cheatsheet.py b/docs/_ext/cheatsheet.py
@@ -1,7 +1,5 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright 2018-2023- Swiss Data Science Center (SDSC)
-# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Copyright Swiss Data Science Center (SDSC). A partnership between
+# École Polytechnique Fédérale de Lausanne (EPFL) and
 # Eidgenössische Technische Hochschule Zürich (ETHZ).
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

diff --git a/docs/_static/cheatsheet/cheatsheet.json b/docs/_static/cheatsheet/cheatsheet.json
@@ -325,6 +325,20 @@
             "rp"
           ]
         },
+        {
+          "command": "$ renku session pause <name>",
+          "description": "Pause the specified session.",
+          "target": [
+            "rp"
+          ]
+        },
+        {
+          "command": "$ renku session resume <name>",
+          "description": "Resume the specified paused session.",
+          "target": [
+            "rp"
+          ]
+        },
         {
           "command": "$ renku session stop <name>",
           "description": "Stop the specified session.",

diff --git a/docs/_static/cheatsheet/cheatsheet.pdf b/docs/_static/cheatsheet/cheatsheet.pdf
diff --git a/docs/changes.rst b/docs/changes.rst
@@ -1,5 +1,5 @@
 ..
-    Copyright 2017-2023 - Swiss Data Science Center (SDSC)
+    Copyright Swiss Data Science Center (SDSC).
     A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
     Eidgenössische Technische Hochschule Zürich (ETHZ).
 

diff --git a/docs/cheatsheet/conf.py b/docs/cheatsheet/conf.py
@@ -1,7 +1,5 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright 2017-2023 - Swiss Data Science Center (SDSC)
-# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Copyright Swiss Data Science Center (SDSC). A partnership between
+# École Polytechnique Fédérale de Lausanne (EPFL) and
 # Eidgenössische Technische Hochschule Zürich (ETHZ).
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

diff --git a/docs/cheatsheet_hash b/docs/cheatsheet_hash
@@ -1,2 +1,2 @@
-ad86ac1d0614ccb692c96e893db4d20d  cheatsheet.tex
+5316163d742bdb6792ed8bcb35031f6c  cheatsheet.tex
 c70c179e07f04186ec05497564165f11  sdsc_cheatsheet.cls
diff --git a/docs/cheatsheet_json_hash b/docs/cheatsheet_json_hash
@@ -1 +1 @@
-1ac51267cefdf4976c29c9d7657063b8  cheatsheet.json
+1856fb451165d013777c7c4cdd56e575  cheatsheet.json
diff --git a/docs/conf.py b/docs/conf.py
@@ -1,7 +1,5 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright 2017-2023 - Swiss Data Science Center (SDSC)
-# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Copyright Swiss Data Science Center (SDSC). A partnership between
+# École Polytechnique Fédérale de Lausanne (EPFL) and
 # Eidgenössische Technische Hochschule Zürich (ETHZ).
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

diff --git a/docs/gettingstarted.rst b/docs/gettingstarted.rst
@@ -1,5 +1,5 @@
 ..
-    Copyright 2017-2023 - Swiss Data Science Center (SDSC)
+    Copyright Swiss Data Science Center (SDSC).
     A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
     Eidgenössische Technische Hochschule Zürich (ETHZ).
 

diff --git a/docs/index.rst b/docs/index.rst
@@ -1,7 +1,7 @@
 :orphan:
 
 ..
-    Copyright 2017-2023 - Swiss Data Science Center (SDSC)
+    Copyright Swiss Data Science Center (SDSC).
     A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
     Eidgenössische Technische Hochschule Zürich (ETHZ).
 

diff --git a/docs/installation.rst b/docs/installation.rst
@@ -1,5 +1,5 @@
 ..
-    Copyright 2017-2023 - Swiss Data Science Center (SDSC)
+    Copyright Swiss Data Science Center (SDSC).
     A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
     Eidgenössische Technische Hochschule Zürich (ETHZ).
 

diff --git a/docs/introduction.rst b/docs/introduction.rst
@@ -1,5 +1,5 @@
 ..
-    Copyright 2017-2023 - Swiss Data Science Center (SDSC)
+    Copyright Swiss Data Science Center (SDSC).
     A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
     Eidgenössische Technische Hochschule Zürich (ETHZ).
 

diff --git a/docs/license.rst b/docs/license.rst
@@ -3,7 +3,7 @@ License
 
 .. code-block:: text
 
-    Copyright 2017-2023 - Swiss Data Science Center (SDSC)
+    Copyright Swiss Data Science Center (SDSC).
     A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
     Eidgenössische Technische Hochschule Zürich (ETHZ).
 

diff --git a/docs/plugins.rst b/docs/plugins.rst
@@ -1,5 +1,5 @@
 ..
-    Copyright 2017-2023 - Swiss Data Science Center (SDSC)
+    Copyright Swiss Data Science Center (SDSC).
     A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
     Eidgenössische Technische Hochschule Zürich (ETHZ).
 
@@ -33,6 +33,25 @@ data science pipeline.
 `Documentation <https://github.com/ratschlab/renku-mls/blob/master/docs/gettingstarted.rst>`_
 `Example Project <https://renkulab.io/projects/learn-renku/plugins/renku-mls-plugin>`_
 
+renku-graph-vis
+---------------
+
+`renku-graph-vis <https://github.com/oda-hub/renku-graph-vis/>`_ is a plugin that
+provides a graphical representation of the renku repository's knowledge graph. 
+In particular, the plugin enables an interactive graph visualization feature 
+for real-time monitoring during a renku session introducing the ability to have 
+a live overview of the ongoing development. It also provides two CLI commands:
+
+* ``display`` to generate a representation of the graph over a png output image
+* ``show-graph`` to start an interactive visualization of the graph over the browser
+
+renku-aqs-annotation
+--------------------
+
+`renku-aqs-annotation <https://github.com/oda-hub/renku-aqs-annotation/>`_ is a plugin that
+intercepts several key astroquery methods and stores a number of dedicated annotations 
+containing information about the calls to these methods (like the arguments used in the call) 
+to the project's Knowledge Graph. 
 
 Developing a plugin?
 --------------------

diff --git a/docs/reference/api.rst b/docs/reference/api.rst
@@ -1,5 +1,5 @@
 ..
-    Copyright 2017-2023 - Swiss Data Science Center (SDSC)
+    Copyright Swiss Data Science Center (SDSC).
     A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
     Eidgenössische Technische Hochschule Zürich (ETHZ).