CI-CD for Environments (#22)

The CI-CD draft PR for environments. A bunch of things might be broken in this PR but will follow up with more changes in separate PRs.

Signed-off-by: sahil-lakhwani <sahilakhwani@gmail.com>

.github/filters.yaml

@@ -0,0 +1,38 @@
+any:
+    - 'binary/**'
+    - 'jvm-jersey/**'
+    - 'php7/**'  
+    - 'dotnet/**'
+    - 'dotnet20/**'     
+    - 'go/**'     
+    - 'jvm/**'     
+    - 'nodejs/**'
+    - 'perl/**'
+    - 'python/**'
+    - 'ruby/**'
+    - 'tensorflow-serving/**'
+
+binary:
+    - 'binary/**'
+jvm-jersey:
+    - 'jvm-jersey/**'
+php7: 
+    - 'php7/**'  
+dotnet:
+    - 'dotnet/**'     
+dotnet20:
+    - 'dotnet20/**'     
+go:   
+    - 'go/**'     
+jvm:  
+    - 'jvm/**'     
+nodejs:
+    - 'nodejs/**'
+perl: 
+    - 'perl/**'
+python:
+    - 'python/**'
+ruby: 
+    - 'ruby/**'
+tensorflow-serving:
+    - 'tensorflow-serving/**'

.github/workflows/environment.yaml

@@ -0,0 +1,106 @@
+
+name: Environments build and test 
+
+on: 
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  # Job to run change detection
+  buildenv:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout sources
+      uses: actions/checkout@v2
+
+    - uses: dorny/paths-filter@v2
+      id: filter
+      with:
+        filters: .github/filters.yaml
+
+    - name: Helm
+      if: steps.filter.outputs.any == 'true'
+      uses: Azure/setup-helm@v1
+
+    - name: Kind Clutser
+      if: steps.filter.outputs.any == 'true'
+      uses: engineerd/setup-kind@v0.4.0
+
+    - name: Configuring and testing the Installation
+      if: steps.filter.outputs.any == 'true'
+      run: |
+        kubectl cluster-info --context kind-kind
+        kind get kubeconfig --internal >$HOME/.kube/config
+        kubectl get nodes
+
+    - name: Install Fission CLI
+      if: steps.filter.outputs.any == 'true'
+      run: |
+        curl -Lo fission https://github.com/fission/fission/releases/download/1.11.0/fission-cli-linux \
+        && chmod +x fission && sudo mv fission /usr/local/bin/
+
+    - name: go
+      if: steps.filter.outputs.go == 'true'
+      uses: hiberbee/github-action-skaffold@1.4.0
+      with:
+        command: run
+        profile: go
+        tag: latest
+    - name: go-tests
+      if: steps.filter.outputs.go == 'true'
+      run: kind load docker-image go-env && ./test_utils/run_test.sh ./go/tests/test_go_env.sh
+
+    - name: jvm
+      if: steps.filter.outputs.jvm == 'true'
+      uses: hiberbee/github-action-skaffold@1.4.0
+      with:
+        command: run
+        profile: jvm
+        tag: latest
+    - name: jvm-tests
+      if: steps.filter.outputs.jvm == 'true'
+      run: |
+        kind load docker-image jvm-env && ./test_utils/run_test.sh jvm/tests/test_java_env.sh
+
+    - name: nodejs
+      if: steps.filter.outputs.nodejs == 'true'
+      uses: hiberbee/github-action-skaffold@1.4.0
+      with:
+        command: run
+        profile: nodejs
+
+    - name:  perl
+      if: steps.filter.outputs.perl == 'true'
+      uses: hiberbee/github-action-skaffold@1.4.0
+      with:
+        command: run
+        profile: perl
+
+    - name:  php7
+      if: steps.filter.outputs.php7 == 'true'
+      uses: hiberbee/github-action-skaffold@1.4.0
+      with:
+        command: run
+        profile: php7
+
+    - name:  python
+      if: steps.filter.outputs.python == 'true'
+      uses: hiberbee/github-action-skaffold@1.4.0
+      with:
+        command: run
+        profile: python
+
+    - name:  ruby
+      if: steps.filter.outputs.ruby == 'true'
+      uses: hiberbee/github-action-skaffold@1.4.0
+      with:
+        command: run
+        profile: ruby
+
+    - name:  tensorflow-serving
+      if: steps.filter.outputs.tensorflow-serving == 'true'
+      uses: hiberbee/github-action-skaffold@1.4.0
+      with:
+        command: run
+        profile: tensorflow-serving
+

go/tests/test_go_env.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+set -euo pipefail
+source $(dirname $0)/../../utils.sh
+
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
+tmp_dir="/tmp/test-$TEST_ID"
+mkdir -p $tmp_dir
+
+ROOT=$(dirname $0)/../../..
+
+cleanup() {
+    clean_resource_by_id $TEST_ID
+    rm -rf $tmp_dir
+}
+
+if [ -z "${TEST_NOCLEANUP:-}" ]; then
+    trap cleanup EXIT
+else
+    log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
+fi
+
+env=go-$TEST_ID
+fn_poolmgr=hello-go-poolmgr-$TEST_ID
+fn_nd=hello-go-nd-$TEST_ID
+
+cd $ROOT/examples/go
+
+log "Creating environment for Golang"
+fission env create --name $env --image $GO_RUNTIME_IMAGE --builder $GO_BUILDER_IMAGE --period 5
+
+timeout 90 bash -c "wait_for_builder $env"
+
+pkgName=$(generate_test_id)
+fission package create --name $pkgName --src hello.go --env $env
+
+# wait for build to finish at most 90s
+timeout 90 bash -c "waitBuild $pkgName"
+
+log "Creating pool manager & new deployment function for Golang"
+fission fn create --name $fn_poolmgr --env $env --pkg $pkgName --entrypoint Handler
+fission fn create --name $fn_nd      --env $env --pkg $pkgName --entrypoint Handler --executortype newdeploy
+
+log "Creating route for new deployment function"
+fission route create --function $fn_poolmgr --url /$fn_poolmgr --method GET
+fission route create --function $fn_nd      --url /$fn_nd      --method GET
+
+log "Waiting for router & pools to catch up"
+sleep 5
+
+log "Testing pool manager function"
+timeout 60 bash -c "test_fn $fn_poolmgr 'Hello'"
+
+log "Testing new deployment function"
+timeout 60 bash -c "test_fn $fn_nd 'Hello'"
+
+# Create zip file without top level directory (module-example)
+cd module-example && zip -r $tmp_dir/module.zip *
+
+pkgName=$(generate_test_id)
+fission package create --name $pkgName --src $tmp_dir/module.zip --env $env
+
+# wait for build to finish at most 90s
+timeout 90 bash -c "waitBuild $pkgName"
+
+log "Update function package"
+fission fn update --name $fn_poolmgr --pkg $pkgName
+fission fn update --name $fn_nd --pkg $pkgName
+
+log "Waiting for router & pools to catch up"
+sleep 5
+
+log "Testing pool manager function with new package"
+timeout 60 bash -c "test_fn $fn_poolmgr 'Vendor'"
+
+log "Testing new deployment function with new package"
+timeout 60 bash -c "test_fn $fn_nd 'Vendor'"
+
+log "Test PASSED"

jvm/tests/test_java_env.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+set -euo pipefail
+source test_utils/utils.sh
+
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
+env=jvm-$TEST_ID
+fn_n=jvm-hello-n-$TEST_ID
+fn_p=jvm-hello-p-$TEST_ID
+
+cleanup() {
+    clean_resource_by_id $TEST_ID
+}
+
+if [ -z "${TEST_NOCLEANUP:-}" ]; then
+    trap cleanup EXIT
+else
+    log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
+fi
+
+cd jvm/examples/java
+
+log "Creating the jar from application"
+#Using Docker to build Jar so that maven & other Java dependencies are not needed on CI server
+docker run -it --rm  -v "$(pwd)":/usr/src/mymaven -w /usr/src/mymaven maven:3.5-jdk-8 mvn clean package -q
+
+log "Creating environment for Java"
+fission env create --name $env --image $JVM_RUNTIME_IMAGE --version 2 --keeparchive=true
+
+log "Creating pool manager & new deployment function for Java"
+fission fn create --name $fn_p --deploy target/hello-world-1.0-SNAPSHOT-jar-with-dependencies.jar --env $env --entrypoint io.fission.HelloWorld
+fission fn create --name $fn_n --deploy target/hello-world-1.0-SNAPSHOT-jar-with-dependencies.jar --env $env --executortype newdeploy --entrypoint io.fission.HelloWorld
+
+log "Creating route for pool manager function"
+fission route create --name $fn_p --function $fn_p --url /$fn_p --method GET
+
+log "Creating route for new deployment function"
+fission route create --name $fn_n --function $fn_n --url /$fn_n --method GET
+
+log "Waiting for router & pools to catch up"
+sleep 5
+
+log "Testing pool manager function"
+timeout 60 bash -c "test_fn $fn_p 'Hello'"
+
+log "Testing new deployment function"
+timeout 60 bash -c "test_fn $fn_n 'Hello'"
+
+log "Test PASSED"

python/tests/test_python_env.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+
+set -euo pipefail
+source $(dirname $0)/../../utils.sh
+
+TEST_ID=$(generate_test_id)
+echo "TEST_ID = $TEST_ID"
+
+tmp_dir="/tmp/test-$TEST_ID"
+mkdir -p $tmp_dir
+
+ROOT=$(dirname $0)/../../..
+
+cleanup() {
+    clean_resource_by_id $TEST_ID
+    rm -rf $tmp_dir
+}
+
+if [ -z "${TEST_NOCLEANUP:-}" ]; then
+    trap cleanup EXIT
+else
+    log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
+fi
+
+env_v1api=python-v1-$TEST_ID
+env_v2api=python-v2-$TEST_ID
+fn1=test-python-env-1-$TEST_ID
+fn2=test-python-env-2-$TEST_ID
+fn3=test-python-env-3-$TEST_ID
+fn4=test-python-env-4-$TEST_ID
+fn5=test-python-env-5-$TEST_ID
+
+
+log "Creating v1api environment ..."
+log "PYTHON_RUNTIME_IMAGE = $PYTHON_RUNTIME_IMAGE"
+fission env create \
+    --name $env_v1api \
+    --image $PYTHON_RUNTIME_IMAGE \
+
+log "Creating v2api environment ..."
+log "PYTHON_RUNTIME_IMAGE = $PYTHON_RUNTIME_IMAGE     PYTHON_BUILDER_IMAGE = $PYTHON_BUILDER_IMAGE"
+fission env create \
+    --name $env_v2api \
+    --image $PYTHON_RUNTIME_IMAGE \
+    --builder $PYTHON_BUILDER_IMAGE
+timeout 180s bash -c "wait_for_builder $env_v2api"
+
+log "Creating package ..."
+pushd $ROOT/test/tests/test_environments/python_src/
+zip -r $tmp_dir/src-pkg.zip *
+popd
+pkg=$(generate_test_id)
+fission package create --name $pkg --src $tmp_dir/src-pkg.zip --env $env_v2api
+timeout 60s bash -c "waitBuild $pkg"
+
+
+log "===== 1. test env with v1 api ====="
+fission fn create --name $fn1 --env $env_v1api --code $ROOT/examples/python/hello.py
+fission route create --function $fn1 --url /$fn1 --method GET
+sleep 3     # Waiting for router to catch up
+timeout 60 bash -c "test_fn $fn1 'Hello, world!'"
+
+
+log "===== 2. test entrypoint = '' ====="
+fission fn create --name $fn2 --env $env_v2api --pkg $pkg
+fission route create --function $fn2 --url /$fn2 --method GET
+sleep 3     # Waiting for router to catch up
+timeout 60 bash -c "test_fn $fn2 'THIS_IS_MAIN_MAIN'"
+
+
+log "===== 3. test entrypoint = func ====="
+fission fn create --name $fn3 --env $env_v2api --pkg $pkg --entrypoint func
+fission route create --function $fn3 --url /$fn3 --method GET
+sleep 3     # Waiting for router to catch up
+timeout 60 bash -c "test_fn $fn3 'THIS_IS_MAIN_FUNC'"
+
+
+log "===== 4. test entrypoint = foo.bar ====="
+fission fn create --name $fn4 --env $env_v2api --pkg $pkg --entrypoint foo.bar
+fission route create --function $fn4 --url /$fn4 --method GET
+sleep 3     # Waiting for router to catch up
+timeout 60 bash -c "test_fn $fn4 'THIS_IS_FOO_BAR'"
+
+
+log "===== 5. test entrypoint = sub_mod.altmain.entrypoint ====="
+fission fn create --name $fn5 --env $env_v2api --pkg $pkg --entrypoint sub_mod.altmain.entrypoint
+fission route create --function $fn5 --url /$fn5 --method GET
+sleep 3     # Waiting for router to catch up
+timeout 60 bash -c "test_fn $fn5 'THIS_IS_ALTMAIN_ENTRYPOINT'"
+
+
+log "Test PASSED"