[CI] Automate release pipeline-adapters (#5651)

.github/workflows/release-pipeline-adapters.yaml

@@ -0,0 +1,85 @@
+# Copyright 2023 Iguazio
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Release Pipeline Adapters
+on:
+  workflow_dispatch:
+    inputs:
+      packages:
+        description: 'Pipeline adapters package to release. Use "*" to release all packages.'
+        default: '*'
+        options:
+          - '*'
+          - 'mlrun-pipelines-kfp-common'
+          - 'mlrun-pipelines-kfp-v1-8'
+          - 'mlrun-pipelines-kfp-v2'
+
+jobs:
+  build-packages-matrix:
+    name: From packages input to matrix
+    runs-on: ubuntu-latest
+    outputs:
+      packages: ${{ steps.packages.outputs.packages }}
+    steps:
+      - name: Install jq
+        run: sudo apt-get install jq
+      - name: resolve packages to release
+        id: packages
+        shell: bash
+        run: |
+          if [[ "$PACKAGES" = "*" ]]; then \
+            export packages=("mlrun-pipelines-kfp-common" "mlrun-pipelines-kfp-v1-8" "mlrun-pipelines-kfp-v2") ;\
+          else \
+            export packages=("${PACKAGES}") ;\
+          fi
+          # "(X Y Z)" -> "{\"include\":[{\"name\":\"X\"},{\"name\":\"Y\"},{\"name\":\"Z\"}]}"
+          export packages=$(jq --null-input --argjson arr "$(printf '%s\n' "${packages[@]}" | jq -R . | jq -s .)" '($arr | map({name: .}))')
+          echo "packages={\"include\":$(echo $packages)}" >> $GITHUB_OUTPUT
+        env:
+          PACKAGES: ${{ github.event.inputs.packages }}
+
+  pypi-publish:
+    needs:
+      - build-packages-matrix
+    name: Release ${{ matrix.name }} to PyPI
+    runs-on: ubuntu-latest
+    environment: release
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.build-packages-matrix.outputs.packages) }}
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip build
+      - name: Build package distributions
+        run: |
+          cd pipeline-adapters
+          make build
+        env:
+          PACKAGES: ${{ matrix.name }}
+
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          skip-existing: true
+          print-hash: true
+          packages-dir: ${{ github.workspace }}/pipeline-adapters/${{ matrix.name }}/dist

pipeline-adapters/Makefile

@@ -27,15 +27,15 @@ build:
     done
 
 release:
-	@for pkg in $(PACKAGES); do \
+	@set -e;\
+	for pkg in $(PACKAGES); do \
         echo "Releasing $$pkg to PyPI"; \
         cd $$pkg && TWINE_USERNAME=$(TWINE_USERNAME) TWINE_PASSWORD=$(TWINE_PASSWORD) twine upload dist/*; \
         cd ..; \
     done
 
 clean:
 	@for pkg in $(PACKAGES); do \
-        echo "Cleaning $$pkg" \
-        cd $$pkg && rm -rf dist build *.egg-info; \
-        cd ..; \
+        echo "Cleaning $$pkg"; \
+        pushd $$pkg && rm -rf dist build *.egg-info && popd; \
 	done

pipeline-adapters/mlrun-pipelines-kfp-common/setup.py

@@ -20,8 +20,8 @@
 logger = logging.getLogger("mlrun-kfp-setup")
 
 setup(
-    name="mlrun-pipelines-kfp-common-experiment",
-    version="0.2.2",
+    name="mlrun-pipelines-kfp-common",
+    version="0.1.0",
     description="MLRun Pipelines package for providing KFP 1.8 compatibility",
     author="Yaron Haviv",
     author_email="yaronh@iguazio.com",

pipeline-adapters/mlrun-pipelines-kfp-v1-8/setup.py

@@ -20,8 +20,8 @@
 logger = logging.getLogger("mlrun-kfp-setup")
 
 setup(
-    name="mlrun-pipelines-kfp-v1-8-experiment",
-    version="0.2.4",
+    name="mlrun-pipelines-kfp-v1-8",
+    version="0.1.0",
     description="MLRun Pipelines package for providing KFP 1.8 compatibility",
     author="Yaron Haviv",
     author_email="yaronh@iguazio.com",

pipeline-adapters/mlrun-pipelines-kfp-v2/setup.py

@@ -20,8 +20,8 @@
 logger = logging.getLogger("mlrun-kfp-setup")
 
 setup(
-    name="mlrun-pipelines-kfp-v2-experiment",
-    version="0.2.1",
+    name="mlrun-pipelines-kfp-v2",
+    version="0.1.0",
     description="MLRun Pipelines package for providing KFP 2.* compatibility",
     author="Yaron Haviv",
     author_email="yaronh@iguazio.com",

requirements.txt

@@ -40,5 +40,5 @@ deprecated~=1.2
 jinja2~=3.1, >=3.1.3
 orjson>=3.9.15, <4
 # mlrun pipeline adapters
-mlrun-pipelines-kfp-common-experiment~=0.2.0
-mlrun-pipelines-kfp-v1-8-experiment~=0.2.0
+mlrun-pipelines-kfp-common~=0.1.0
+mlrun-pipelines-kfp-v1-8~=0.1.0