Support deployment of Turing routers with RPC Experiment Engine plugi…

…ns (#158)

* - Remove unused `RouterDefaults.Experiment` and update tests

* - Update service builder to prepare persistence volume for router and copy experiment engine plugins into it

* - update fiber config if external plugin is used

* - publish test chart

* - publish test chart

* - fix lint

* - make it possible to configure manual charts publishing

* - unit tests

* - fix chart

* - fix permissions inside the dockerfile

* - use bash alpine image

* - use bash image

* - fix nil pointer

* - use nginx to server plugin binary

* - disable adding PluginBinary data to fiber config

* - disable adding PluginBinary data to fiber config

* - Support downloading plugins from URL

* fix lint

* - bug fixes

* - set executable permission for the plugin binary

* - re-use PluginConfig model

* - Delete plugins server before knative services are deleted

* - fix test

* - address naming suggestiong from the PR review, bumping up charts version

* - add test case for plugin config

* - add test case to cover k8s services for router with plugins

* - fix chart

* - move mlp mock service into mocks directory, clean up tests

* - fix lint

.github/workflows/helm-chart.yaml

@@ -8,6 +8,18 @@ on:
       - "infra/charts/**"
     branches:
       - main
+  workflow_dispatch:
+    inputs:
+      owner:
+        description: The GitHub user or org that owns this repository
+        type: string
+        required: true
+      repository:
+        description:  The GitHub repository
+        type: string
+        required: true
+        default: "charts"
+
 
 jobs:
   release:
@@ -18,10 +30,9 @@ jobs:
         uses: stefanprodan/helm-gh-pages@v1.4.1
         with:
           token: "${{ secrets.GH_PAGES_TOKEN }}"
-          charts_url: https://turing-ml.github.io/charts
           charts_dir: infra/charts
-          owner: turing-ml
-          repository: charts
+          owner: "${{ github.event.inputs.owner || 'turing-ml' }}"
+          repository: "${{ github.event.inputs.repository || 'charts' }}"
           commit_username: ${{ github.actor }}
           commit_email: "${{ github.actor }}@users.noreply.github.com"
 

.github/workflows/turing.yaml

@@ -36,10 +36,130 @@ env:
   ARTIFACT_RETENTION_DAYS: 7
 
 jobs:
+  build-api:
+    runs-on: ubuntu-latest
+    outputs:
+      api-version: ${{ steps.build-image.outputs.api-version }}
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Build Docker image
+        id: build-image
+        working-directory: api
+        run: |
+          set -o pipefail
+          make build-image | tee output.log
+          echo "::set-output name=api-version::$(sed -n 's%turing-api version: \(.*\)%\1%p' output.log)"
+
+      - name: Save Docker image
+        run: |
+          docker image save \
+            --output turing-api.${{ steps.build-image.outputs.api-version }}.tar \
+            turing-api:${{ steps.build-image.outputs.api-version }}
+
+      - name: Publish Artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: turing-api.${{ steps.build-image.outputs.api-version }}.tar
+          path: turing-api.${{ steps.build-image.outputs.api-version }}.tar
+          retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
+
+  build-router:
+    runs-on: ubuntu-latest
+    outputs:
+      router-version: ${{ steps.build-image.outputs.router-version }}
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Build Docker image
+        id: build-image
+        working-directory: engines/router
+        run: |
+          set -o pipefail
+          make build-image | tee output.log
+          echo "::set-output name=router-version::$(sed -n 's%turing-router version: \(.*\)%\1%p' output.log)"
+
+      - name: Save Docker image
+        run: |
+          docker image save \
+            --output turing-router.${{ steps.build-image.outputs.router-version }}.tar \
+            turing-router:${{ steps.build-image.outputs.router-version }}
+
+      - name: Publish Artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: turing-router.${{ steps.build-image.outputs.router-version }}.tar
+          path: turing-router.${{ steps.build-image.outputs.router-version }}.tar
+          retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
+
+  build-ui:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ui
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Set up Node 14.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: '14.x'
+
+      - name: Cache Dependencies
+        uses: actions/cache@v2
+        with:
+          path: ui/node_modules
+          key: |
+            ${{ runner.os }}-modules-${{ hashFiles('ui/yarn.lock') }}
+          restore-keys: ${{ runner.os }}-modules-
+
+      - name: Install
+        run: yarn install --network-concurrency 1
+
+      - name: Lint code
+        run: yarn lint
+
+      - name: Build UI
+        env:
+          NODE_OPTIONS: "--max_old_space_size=4096"
+        run: yarn build
+
+      - name: Publish Artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: turing-ui-dist
+          path: ui/build/
+          retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
+
+  build-cluster-init:
+    runs-on: ubuntu-latest
+    outputs:
+      cluster-init-version: ${{ steps.build-cluster-init.outputs.cluster-init-version }}
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Run action build-cluster-init
+        id: build-cluster-init
+        uses: ./.github/actions/build-cluster-init
+        with:
+          artifact_retention_days: ${{ env.ARTIFACT_RETENTION_DAYS }}
+
   test-api:
     runs-on: ubuntu-latest
     env:
       GOPATH: ${{ github.workspace }}/api/.go
+    needs:
+      - build-api
     services:
       postgres:
         image: postgres:13-alpine
@@ -95,6 +215,8 @@ jobs:
         working-directory: engines/router
     env:
       GOPATH: ${{ github.workspace }}/engines/router/.go
+    needs:
+      - build-router
     steps:
       - name: Check out code
         uses: actions/checkout@v2
@@ -159,147 +281,6 @@ jobs:
           skip-go-installation: true
           args: --verbose
 
-  build-ui:
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: ui
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: Set up Node 14.x
-        uses: actions/setup-node@v1
-        with:
-          node-version: '14.x'
-
-      - name: Cache Dependencies
-        uses: actions/cache@v2
-        with:
-          path: ui/node_modules
-          key: |
-            ${{ runner.os }}-modules-${{ hashFiles('ui/yarn.lock') }}
-          restore-keys: ${{ runner.os }}-modules-
-
-      - name: Install
-        run: yarn install --network-concurrency 1
-
-      - name: Lint code
-        run: yarn lint
-
-      - name: Build UI
-        env:
-          NODE_OPTIONS: "--max_old_space_size=4096"
-        run: yarn build
-
-      - name: Publish Artifact
-        uses: actions/upload-artifact@v2
-        with:
-          name: turing-ui-dist
-          path: ui/build/
-          retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
-
-  build-api:
-    runs-on: ubuntu-latest
-    needs:
-      - test-api
-      - test-engines-experiment
-      - test-engines-router
-    outputs:
-      api-version: ${{ steps.build-image.outputs.api-version }}
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-#      - name: Cache vendor dependencies
-#        uses: actions/cache@v2
-#        with:
-#          path: api
-#          key: |
-#            ${{ runner.os }}-vendor-${{ hashFiles('api/go.mod') }}
-#          restore-keys: ${{ runner.os }}-vendor-
-
-      - name: Build Docker image
-        id: build-image
-        working-directory: api
-        run: |
-          set -o pipefail
-          make build-image | tee output.log
-          echo "::set-output name=api-version::$(sed -n 's%turing-api version: \(.*\)%\1%p' output.log)"
-
-      - name: Save Docker image
-        run: |
-          docker image save \
-            --output turing-api.${{ steps.build-image.outputs.api-version }}.tar \
-            turing-api:${{ steps.build-image.outputs.api-version }}
-
-      - name: Publish Artifact
-        uses: actions/upload-artifact@v2
-        with:
-          name: turing-api.${{ steps.build-image.outputs.api-version }}.tar
-          path: turing-api.${{ steps.build-image.outputs.api-version }}.tar
-          retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
-
-  build-router:
-    runs-on: ubuntu-latest
-    needs:
-      - test-engines-router
-    outputs:
-      router-version: ${{ steps.build-image.outputs.router-version }}
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-#      - name: Cache vendor dependencies
-#        uses: actions/cache@v2
-#        with:
-#          path: engines/router/vendor
-#          key: |
-#            ${{ runner.os }}-vendor-${{ hashFiles('engines/router/go.mod') }}
-#          restore-keys: ${{ runner.os }}-vendor-
-
-      - name: Build Docker image
-        id: build-image
-        working-directory: engines/router
-        run: |
-          set -o pipefail
-          make build-image | tee output.log
-          echo "::set-output name=router-version::$(sed -n 's%turing-router version: \(.*\)%\1%p' output.log)"
-
-      - name: Save Docker image
-        run: |
-          docker image save \
-            --output turing-router.${{ steps.build-image.outputs.router-version }}.tar \
-            turing-router:${{ steps.build-image.outputs.router-version }}
-
-      - name: Publish Artifact
-        uses: actions/upload-artifact@v2
-        with:
-          name: turing-router.${{ steps.build-image.outputs.router-version }}.tar
-          path: turing-router.${{ steps.build-image.outputs.router-version }}.tar
-          retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
-
-  build-cluster-init:
-    runs-on: ubuntu-latest
-    outputs:
-      cluster-init-version: ${{ steps.build-cluster-init.outputs.cluster-init-version }}
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Run action build-cluster-init
-        id: build-cluster-init
-        uses: ./.github/actions/build-cluster-init
-        with:
-          artifact_retention_days: ${{ env.ARTIFACT_RETENTION_DAYS }}
-
-
   test-e2e:
     runs-on: ubuntu-latest
     env:
@@ -542,6 +523,9 @@ jobs:
       - build-ui
       - release-rules
       - test-e2e
+      - test-api
+      - test-engines-router
+      - test-engines-experiment
     uses: gojek/turing/.github/workflows/turing-publish.yaml@main
     with:
       api_version: ${{ needs.build-api.outputs.api-version }}

api/Makefile

@@ -40,7 +40,7 @@ vendor:
 .PHONY: test
 test: tidy
 	@echo "Running tests..."
-	go test -v -race -short -cover -coverprofile cover.out ${SRC_ROOT}/... -tags unit,integration
+	go test -v -race -short -cover -coverprofile cover.out ${SRC_ROOT}/... -tags integration
 	go tool cover -func cover.out
 
 .PHONY: build

api/turing/api/deployment_controller.go

@@ -5,12 +5,11 @@ import (
 	"errors"
 	"strings"
 
-	"github.com/gojek/turing/engines/experiment/manager"
-
 	merlin "github.com/gojek/merlin/client"
 	mlp "github.com/gojek/mlp/api/client"
 	"github.com/gojek/turing/api/turing/models"
-	"github.com/gojek/turing/api/turing/utils"
+	"github.com/gojek/turing/api/turing/service"
+	"github.com/gojek/turing/engines/experiment/manager"
 )
 
 // RouterDeploymentController handles the deployment of routers
@@ -36,7 +35,7 @@ func (c RouterDeploymentController) deployOrRollbackRouter(
 		return err
 	}
 
-	eventsCh := utils.NewEventChannel()
+	eventsCh := service.NewEventChannel()
 	defer eventsCh.Close()
 	_ = c.EventService.ClearEvents(int(router.ID))
 	// Write events asynchronously
@@ -131,7 +130,7 @@ func (c RouterDeploymentController) deployOrRollbackRouter(
 }
 
 func (c RouterDeploymentController) writeDeploymentEvents(
-	eventsCh *utils.EventChannel, router *models.Router, version uint) {
+	eventsCh *service.EventChannel, router *models.Router, version uint) {
 	for {
 		event, done := eventsCh.Read()
 		if done {
@@ -151,7 +150,7 @@ func (c RouterDeploymentController) deployRouterVersion(
 	project *mlp.Project,
 	environment *merlin.Environment,
 	routerVersion *models.RouterVersion,
-	eventsCh *utils.EventChannel,
+	eventsCh *service.EventChannel,
 ) (string, error) {
 	var routerServiceAccountKey, enricherServiceAccountKey, ensemblerServiceAccountKey, experimentPasskey string
 	var experimentConfig json.RawMessage
@@ -267,7 +266,7 @@ func (c RouterDeploymentController) undeployRouter(
 	var errorStrings []string
 
 	// Write events asynchronously
-	eventsCh := utils.NewEventChannel()
+	eventsCh := service.NewEventChannel()
 	defer eventsCh.Close()
 	var version uint
 	if router.CurrRouterVersion != nil {