chore: Add kfp-tekton integration tests and manifests

.github/workflows/backend.yml

@@ -0,0 +1,81 @@
+name: KFP Tekton backend unit tests
+
+on:
+  push:
+    branches: [master]
+
+  # Run tests for any PRs which change the backend code
+  pull_request:
+    paths:
+      - 'go.mod'
+      - 'backend/**'
+      - 'scripts/deploy/github/**'
+      - 'manifests/kustomize/**'
+
+env:
+  GITHUB_ACTION: "true"
+  SETUPTOOLS_USE_DISTUTILS: "stdlib"
+
+jobs:
+  run-go-unittests:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: 1.20.x
+    - name: Checkout code
+      uses: actions/checkout@v4
+    - name: "run go unit tests"
+      run: go test -v -cover ./backend/...
+  backend-integration:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Create k8s Kind Cluster
+        uses: container-tools/kind-action@v2
+        with:
+          cluster_name: kfp-tekton
+          kubectl_version: v1.29.2
+          version: v0.22.0
+          node_image: kindest/node:v1.29.2
+      - name: build images
+        run: ./scripts/deploy/github/build-images.sh
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: "deploy kfp-tekton"
+        run: ./scripts/deploy/github/deploy-kfp.sh
+      - name: Install sdk
+        run: |
+          python3 -m venv .venv
+          . .venv/bin/activate
+          pip install -e sdk/python
+      - name: "flip coin test"
+        run: |
+          . .venv/bin/activate
+          TEST_SCRIPT="test-flip-coin.sh" ./scripts/deploy/github/e2e-test.sh
+      - name: "static loop test"
+        run: |
+          . .venv/bin/activate
+          TEST_SCRIPT="test-static-loop.sh" ./scripts/deploy/github/e2e-test.sh
+      - name: "dynamic loop test"
+        run: |
+          . .venv/bin/activate
+          TEST_SCRIPT="test-dynamic-loop.sh" ./scripts/deploy/github/e2e-test.sh
+      - name: "use env"
+        run: |
+          . .venv/bin/activate
+          TEST_SCRIPT="test-env.sh" ./scripts/deploy/github/e2e-test.sh
+      - name: "use volume"
+        run: |
+          . .venv/bin/activate
+          TEST_SCRIPT="test-volume.sh" ./scripts/deploy/github/e2e-test.sh
+      - name: Collect test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: kfp-tekton-backend-artifacts
+          path: /tmp/tmp.*/*

.github/workflows/codeql.yml

@@ -0,0 +1,72 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  schedule:
+    # Every Friday at 19:39
+    - cron: '39 19 * * 5'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go', 'javascript', 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Use only 'java' to analyze code written in Java, Kotlin or both
+        # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #     echo "Run, Build Application using script"
+    #     ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2
+      with:
+        category: "/language:${{matrix.language}}"

manifests/kustomize/base/cache/cache-role.yaml

@@ -31,3 +31,20 @@ rules:
   - watch
   - update
   - patch
+- apiGroups:
+  - tekton.dev
+  resources:
+  - pipelineruns
+  - customruns
+  - taskruns
+  - conditions
+  - runs
+  - tasks
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+  - patch
+  - delete

manifests/kustomize/base/metadata/base/kustomization.yaml

@@ -1,5 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
+namespace: kubeflow
 resources:
   - metadata-grpc-configmap.yaml
   - metadata-grpc-deployment.yaml

manifests/kustomize/base/pipeline/ml-pipeline-apiserver-role.yaml

@@ -49,3 +49,33 @@ rules:
   - tokenreviews
   verbs:
   - create
+- apiGroups:
+  - tekton.dev
+  resources:
+  - pipelineruns
+  - taskruns
+  - conditions
+  - runs
+  - tasks
+  - customruns
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - custom.tekton.dev
+  resources:
+  - pipelineloops
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+  - patch
+  - delete
+

manifests/kustomize/base/pipeline/ml-pipeline-persistenceagent-role.yaml

@@ -33,3 +33,32 @@ rules:
   verbs:
     - reportMetrics
     - readArtifact
+- apiGroups:
+  - tekton.dev
+  resources:
+  - pipelineruns
+  - taskruns
+  - conditions
+  - runs
+  - tasks
+  - customruns
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - custom.tekton.dev
+  resources:
+  - pipelineloops
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+  - patch
+  - delete

manifests/kustomize/base/pipeline/pipeline-runner-role.yaml

@@ -78,3 +78,32 @@ rules:
   - seldondeployments
   verbs:
   - '*'
+- apiGroups:
+  - tekton.dev
+  resources:
+  - pipelineruns
+  - taskruns
+  - conditions
+  - runs
+  - tasks
+  - customruns
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - custom.tekton.dev
+  resources:
+  - pipelineloops
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+  - patch
+  - delete

manifests/kustomize/cluster-scoped-resources-tekton/kustomization.yaml

@@ -0,0 +1,26 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: kubeflow
+
+resources:
+- namespace.yaml
+bases:
+- ../third-party/application/cluster-scoped
+- ../base/pipeline/cluster-scoped
+- ../base/cache-deployer/cluster-scoped
+vars:
+# NOTE: var name must be unique globally to allow composition of multiple kustomize
+# packages. Therefore, we added prefix `kfp-cluster-scoped-` to distinguish it from
+# others.
+- name: kfp-cluster-scoped-namespace
+  objref:
+    # cache deployer sa's metadata.namespace will be first transformed by namespace field in kustomization.yaml
+    # so that we only need to change kustomization.yaml's namespace field for namespace customization.
+    kind: ServiceAccount
+    name: kubeflow-pipelines-cache-deployer-sa
+    apiVersion: v1
+  fieldref:
+    fieldpath: metadata.namespace
+configurations:
+- params.yaml

manifests/kustomize/cluster-scoped-resources-tekton/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: '$(kfp-cluster-scoped-namespace)'

manifests/kustomize/cluster-scoped-resources-tekton/params.yaml

@@ -0,0 +1,4 @@
+# Allow Kustomize var to replace following fields.
+varReference:
+- path: metadata/name
+  kind: Namespace

manifests/kustomize/env/plain-multi-user/kustomization.yaml

@@ -0,0 +1,22 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../base/installs/multi-user
+  - ../../base/metadata/base
+  - ../../base/metadata/options/istio
+  - ../../third-party/mysql/base
+  - ../../third-party/mysql/options/istio
+  - ../../third-party/minio/base
+  - ../../third-party/minio/options/istio
+  - ../../third-party/metacontroller/base
+
+# Identifier for application manager to apply ownerReference.
+# The ownerReference ensures the resources get garbage collected
+# when application is deleted.
+commonLabels:
+  application-crd-id: kubeflow-pipelines
+
+# !!! If you want to customize the namespace,
+# please also update base/cache-deployer/cluster-scoped/cache-deployer-clusterrolebinding.yaml
+namespace: kubeflow

manifests/kustomize/env/plain/kustomization.yaml

@@ -0,0 +1,21 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../cluster-scoped-resources-tekton
+  - ../../base/installs/generic
+  - ../../base/metadata/base
+  - ../../third-party/minio/base
+  - ../../third-party/mysql/base
+
+# Identifier for application manager to apply ownerReference.
+# The ownerReference ensures the resources get garbage collected
+# when application is deleted.
+labels:
+  - includeSelectors: true
+    pairs:
+      application-crd-id: kubeflow-pipelines
+
+# !!! If you want to customize the namespace,
+# please also update base/cache-deployer/cluster-scoped/cache-deployer-clusterrolebinding.yaml
+namespace: kubeflow