Included Deploy,Chaos and functional test scripts for eks gitlab

.gitlab-ci.yml

@@ -0,0 +1,209 @@
+## TODO: Modularize the gitlab yaml to reuse templates
+
+## Define the stages & order of execution
+
+stages:
+  - CLUSTER-SETUP
+  - PROVIDER-INFRA-SETUP
+  - STATEFUL-APP-DEPLOY
+  - APP-FUNC-TEST
+  - APP-CHAOS-TEST
+  - CLUSTER-CLEANUP
+
+## Setup the kubernetes cluster
+
+eks-cluster:
+  image: atulabhi/kops:v10
+  stage: CLUSTER-SETUP 
+  script:
+    - chmod 775 ./script/eks 
+    - ./script/eks
+  artifacts:
+    paths:
+      - .kube/
+
+## Setup OpenEBS control plane
+
+openebs-eks-deploy:
+  image: atulabhi/kops:v10
+  stage: PROVIDER-INFRA-SETUP
+  dependencies:
+    - eks-cluster
+  script: 
+   - chmod 775 ./script/provider/infra-setup
+   - ./script/provider/infra-setup
+  artifacts:
+    paths:
+      - .kube-openebs/
+
+## Define a job template for app deployers
+
+.app_deploy_template:
+  image: atulabhi/kops:v10
+  stage: STATEFUL-APP-DEPLOY 
+  dependencies:
+    - openebs-eks-deploy
+  artifacts:
+    paths: 
+     - .kube-openebs/
+
+## Define the app deployer jobs
+
+## PERCONA-JIVA
+percona-jiva-run/load/check 0:2:
+  extends: .app_deploy_template
+  script: 
+   - ./script/apps/percona/deployer/percona-app-deploy-jiva
+
+percona-jiva-run/load/check 1:2:
+  extends: .app_deploy_template
+  before_script: 
+   - sleep 180
+  script: 
+   - ./script/apps/percona/workload/jiva/percona-app-workload-jiva
+
+## MONGO-JIVA
+mongo-jiva-run/load/check 0:2:
+  extends: .app_deploy_template
+  script: 
+   - ./script/apps/mongo/deployer/mongo-app-deploy-jiva
+
+mongo-jiva-run/load/check 1:2:
+  extends: .app_deploy_template
+  before_script: 
+   - sleep 180
+  script: 
+   - ./script/apps/mongo/workload/jiva/mongo-app-workload-jiva
+
+# POSTGRES-JIVA
+postgres-jiva-run/load/check 0:2:
+  extends: .app_deploy_template
+  script: 
+   - ./script/apps/postgres/deployer/postgres-app-deploy-jiva
+
+postgres-jiva-run/load/check 1:2:
+  extends: .app_deploy_template
+  before_script: 
+   - sleep 180
+  script: 
+   - ./script/apps/postgres/workload/jiva/postgres-app-workload-jiva
+
+# CASSANDRA-JIVA
+cassandra-jiva-run/load/check 0:2:
+  extends: .app_deploy_template
+  script: 
+   - ./script/apps/cassandra/deployer/cassandra-app-deploy-jiva
+
+cassandra-jiva-run/load/check 1:2:
+  extends: .app_deploy_template
+  before_script: 
+   - sleep 180
+  script: 
+   - ./script/apps/cassandra/workload/jiva/cassandra-app-workload-jiva
+
+# JENKINS-JIVA
+jenkins-jiva-run/load/check:
+  extends: .app_deploy_template
+  script: 
+   - ./script/apps/jenkins/deployer/jenkins-app-deploy-jiva
+
+# PERCONA-CSTOR
+percona-cstor-run/load/check 0:2:
+  extends: .app_deploy_template
+  script: 
+   - ./script/apps/percona/deployer/percona-app-deploy-cstor
+
+percona-cstor-run/load/check 1:2:
+  extends: .app_deploy_template
+  before_script: 
+   - sleep 180
+  script: 
+   - ./script/apps/percona/workload/cstor/percona-app-workload-cstor
+
+# MONGO-CSTOR
+mongo-cstor-run/load/check 0:2:
+  extends: .app_deploy_template
+  script: 
+   - ./script/apps/mongo/deployer/mongo-app-deploy-cstor
+
+mongo-cstor-run/load/check 1:2:
+  extends: .app_deploy_template
+  before_script: 
+   - sleep 180
+  script: 
+   - ./script/apps/mongo/workload/cstor/mongo-app-workload-cstor
+
+## Define job template for func test jobs  
+
+.func_test_template:
+  image: atulabhi/kops:v10
+  stage: APP-FUNC-TEST
+  when: always 
+  artifacts:
+    paths: 
+      - .kube-openebs/
+
+app-replica-scale-{cassandra-jiva}:
+  extends: .func_test_template #dependencies: cassandra-jiva
+  script:
+   - ./script/apps/cassandra/functional/jiva/app-sts-replica-scale 
+
+app-upgrade-deployment-{jenkins-jiva}:
+  extends: .func_test_template #dependencies: jenkins-jiva
+  script:
+   - ./script/apps/jenkins/functional/jiva/app-upgrade-deployment
+
+volume-data-integrity-{fio-jiva}:
+  extends: .func_test_template #dependencies: jenkins-jiva
+  script:
+   - ./script/apps/fio/tests/jiva/fio-volume-data-integrity-check
+
+## Define job template for chaos jobs 
+
+.chaos_test_template:
+  image: atulabhi/kops:v10
+  stage: APP-CHAOS-TEST
+  when: always 
+  artifacts:
+    paths: 
+      - .kube-openebs/
+
+repl-kill-{percona-jiva}:
+  extends: .chaos_test_template #dependencies: percona-jiva
+  script:
+   - ./script/apps/percona/chaos/jiva/volume-replica-failure
+
+ctrl-kill-{mongo-jiva}:
+  extends: .chaos_test_template #dependencies: mongo-jiva
+  script:
+   - ./script/apps/mongo/chaos/jiva/volume-controller-failure
+
+repl-disconnect-{jenkins-jiva}:
+  extends: .chaos_test_template #dependencies: jenkins-jiva
+  script:
+   - ./script/apps/jenkins/chaos/jiva/volume-replica-network-delay
+
+ctrl-disconnect-{postgres-jiva}:
+  extends: .chaos_test_template #dependencies: postgres-jiva
+  script:
+   - ./script/apps/postgres/chaos/jiva/volume-controller-network-delay
+
+tgt-kill-{percona-cstor}:
+  extends: .chaos_test_template #dependencies: percona-cstor
+  script:
+   - ./script/apps/percona/chaos/cstor/volume-target-failure
+
+tgt-disconnect-{mongo-cstor}:
+  extends: .chaos_test_template #dependencies: mongo-cstor
+  script:
+   - ./script/apps/mongo/chaos/cstor/volume-target-network-delay
+
+cleanup-eks:
+  when: always
+  image: atulabhi/kops:v10
+  dependencies:
+    - eks-cluster
+  stage: CLUSTER-CLEANUP
+  script: 
+    - chmod 755 ./script/eks-cleanup
+    - ./script/eks-cleanup

README.md

@@ -1 +1 @@
-# e2e-eks
+# e2e-eks

script/apps/cassandra/deployer/cassandra-app-deploy-jiva

@@ -0,0 +1,145 @@
+#!/bin/bash
+
+#################
+## ENVIRONMENT ##
+#################
+
+## TODO: Ideally, run_metadata should be passed as gitlab runner (CI runtime) ENV
+run_metadata=""
+
+## https://github.com/openebs/litmus/blob/master/apps/cassandra/deployers/test_vars.yml
+test_name="cassandra-deployment" 
+
+if [[ -n "$run_metadata" ]]; then
+  test_name="$test_name-$run_metadata"
+fi 
+
+################
+## FUNCTIONS  ##
+################
+
+error_handler()
+{
+rc=$1; message=$(echo $2 | cut -d "=" -f 2); act=$(echo $3 | cut -d "=" -f 2)
+if [ $rc -ne 0 ]; then
+  echo "$message"
+  if [ "$act" == "exit" ]; then
+    exit 1 
+  fi
+fi
+}
+
+task_delimiter()
+{
+printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' "%"
+}
+
+###################
+## DEPENDENCIES  ##
+###################
+
+## Derive the kubeconfig of the EKS cluster into the gitlab job runner pod
+## ".kube-openebs" is a shared path for the gitlab runner passed as artifact by gitlab 
+
+echo "Setting up test dependencies.."
+
+mkdir ~/.kube
+cp .kube-openebs/config ~/.kube/config
+mkdir ~/logs
+cp .kube-openebs/clusters ~/logs
+
+## Clone the litmus repo, checkout the e2e branch, navigate to litmus root 
+
+#git clone https://github.com/atulabhi/litmus.git
+#cd litmus; git checkout v0.7-RC1
+
+git clone https://github.com/openebs/litmus.git
+cd litmus
+
+task_delimiter;
+
+############################
+## LITMUS PRECONDITIONING ##
+############################
+
+## TODO: Add logic to replace job ENV (SC, PVC, LABEL, NS) based on run instance intent 
+## TODO: Add logic to add ENV for run_instance_metadata/RunID
+
+sed -i '0,/command:/ s/command:/\
+          - name: AWS_ACCESS_KEY_ID\
+            value: '"$AWS_ACCESS_KEY_ID"'\
+          - name: AWS_SECRET_ACCESS_KEY\
+            value: '"$AWS_SECRET_ACCESS_KEY"'\
+\n&/' apps/cassandra/deployers/run_litmus_test.yml
+
+#################
+## RUNNER MAIN ##
+#################
+
+echo "Running the litmus test.."
+
+## Create the litmus job for cassandra app deploy
+
+jobNameCmd="kubectl get jobs -n litmus --no-headers -o jsonpath='{.items[?(@.metadata.labels.app==\"cassandra-deployment-litmus\")].metadata.name}'"
+
+job_deploy_out=$(kubectl create -f apps/cassandra/deployers/run_litmus_test.yml) && \
+job_name=$(eval $jobNameCmd); retcode=$?
+error_handler $retcode msg="Unable to run litmusbook, exiting" action="exit"
+
+
+## Obtain the litmus pod name 
+
+litmusPodCmd="kubectl get pod --no-headers -n litmus -o jsonpath='{.items[?(@.metadata.labels.job-name==\"$job_name\")].metadata.name}'"
+litmus_pod=$(eval $litmusPodCmd); retcode=$?
+error_handler $retcode msg="Unable to find litmus test runner pod, exiting" action="exit"
+
+## Wait till the ansibletest container terminates && also confirm job completion status. This is done to ensure
+## that execution of auxiliary containers such as loggers is completed. Getting the ansibletest ccontainer to completed state 
+## satisfies the "necessary" condition for test job completion
+
+containerStateCmd="kubectl get pod ${litmus_pod} -n litmus -o jsonpath='{.status.containerStatuses[?(@.name==\"ansibletest\")].state}'"
+jobStateCmd="kubectl get pod ${litmus_pod} --no-headers -n litmus -o custom-columns=:status.phase"
+
+## TODO: Consider cases where litmus pod is evicted
+
+while [[ ! $(eval $containerStateCmd) =~ 'terminated' ]]; do
+  sleep 1 
+done 
+
+while [[ $(eval $jobStateCmd) =~ 'Running' ]]; do
+   sleep 1
+done
+
+echo "Litmus test run Job has completed"
+task_delimiter;
+
+## Printing the test logs & cluster state for debug purposes 
+
+echo "Dumping Litmus test pod logs for debug"; echo ""
+kubectl logs $litmus_pod -n litmus -c ansibletest 
+task_delimiter; 
+
+echo "Dumping state of cluster post job run"; echo ""
+kubectl get pods --all-namespaces
+kubectl get pvc --all-namespaces
+kubectl get sc
+
+task_delimiter; 
+
+## Check the test status & result from the litmus result custom resource
+
+testStatus=$(kubectl get lr $test_name --no-headers -o custom-columns=:spec.testStatus.phase) && \
+testResult=$(kubectl get lr $test_name --no-headers -o custom-columns=:spec.testStatus.result); retcode=$?
+error_handler $retcode msg="Unable to find litmus result custom resource, exiting" action="exit"
+
+if [[ $testStatus == "completed" ]]; then
+  if [[ $testResult == "Pass" ]]; then
+    echo "TEST: PASS"
+  else 
+    echo "TEST: FAILED"; exit 1 
+  fi
+else
+  echo "Test Execution was aborted"; exit 1 
+fi 
+
+