add ci pipeline for github actions

.circleci/config.yml

@@ -116,13 +116,17 @@ jobs:
           name: Deploy k8s connector
           command: |
             .circleci/deploy_connector.sh
+      - run:
+          name: Deploy log generator
+          command: |
+            kubectl apply -f test/test_setup.yaml
+            sleep 60
       - run:
           name: Run functional tests
           command: |
             export CI_SPLUNK_HEC_HOST=$(hostname -i)
             pyenv global 3.6.5
             export PYTHONWARNINGS="ignore:Unverified HTTPS request"
-            sleep 30
             .circleci/run_functional_tests.sh
 
 workflows:

.circleci/sck_values.yml

@@ -10,7 +10,7 @@ global:
 splunk-kubernetes-logging:
   kubernetes:
     insecureSSL: true
-    clusterName: "circleci-k8s-cluster"
+    clusterName: "ci-k8s-cluster"
   splunk:
     hec:
       indexName: circleci_events
@@ -34,7 +34,7 @@ splunk-kubernetes-metrics:
     useRestClientSSL: false
     insecureSSL: true
     kubeletPort: 10255
-    clusterName: "circleci-k8s-cluster-metrics"
+    clusterName: "ci-k8s-cluster-metrics"
   splunk:
     hec:
       indexName: circleci_metrics
@@ -48,7 +48,7 @@ splunk-kubernetes-objects:
       indexName: circleci_objects
   kubernetes:
     insecureSSL: true
-    clusterName: "circleci-k8s-cluster-objects"
+    clusterName: "ci-k8s-cluster-objects"
   image:
     name:
     pullPolicy: Always

.github/workflows/ci_build_test.yaml

@@ -0,0 +1,149 @@
+name: CI Build Test
+
+on:
+  pull_request:
+    branches-ignore:
+      - gh-pages
+
+jobs:
+  lint-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v1
+        with:
+          version: v3.4.1
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@v2.1.0
+        env:
+          CT_CHECK_VERSION_INCREMENT: "false"
+
+      - name: Run chart-testing (list-changed)
+        id: list-changed
+        run: |
+          changed=$(ct list-changed --target-branch main)
+          if [[ -n "$changed" ]]; then
+            echo "::set-output name=changed::true"
+          fi
+      - name: Linting charts
+        run: helm lint helm-chart/*
+
+      - name: Linting charts with CI values
+        run: helm lint --values ci_scripts/sck_values.yml helm-chart/*
+
+      - name: Linting charts ct lint tool
+        run: ct lint helm-chart/*
+
+  e2e-test:
+    name: e2e test
+    needs:
+      - lint-test
+    runs-on: ubuntu-20.04
+    env:
+      CI_SPLUNK_PORT: 8089
+      CI_SPLUNK_USERNAME: admin
+      CI_SPLUNK_HEC_TOKEN: a6b5e77f-d5f6-415a-bd43-930cecb12959
+      CI_SPLUNK_PASSWORD: helloworld
+      CI_INDEX_EVENTS: ci_events
+      CI_INDEX_OBJECTS: ci_objects
+      CI_INDEX_METRICS: ci_metrics
+      KUBERNETES_VERSION: v1.15.2
+      MINIKUBE_VERSION: v1.21.0
+      GITHUB_ACTIONS: true
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup Minikube
+        run: |
+          # Install Kubectl
+          curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl
+          chmod +x kubectl
+          sudo mv kubectl /usr/local/bin/
+          mkdir -p ${HOME}/.kube
+          touch ${HOME}/.kube/config
+          # Install Minikube
+          curl -Lo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64
+          chmod +x minikube
+          sudo mv minikube /usr/local/bin/
+          # Start Minikube and Wait
+          minikube start --driver=docker --container-runtime=docker --cpus 2 --memory 4096 --kubernetes-version=${KUBERNETES_VERSION} --no-vtx-check
+          export JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'
+          until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do
+            sleep 1;
+          done
+
+      - name: Install Splunk
+        run: |
+          # Wait until minikube is ready
+          kubectl apply -f https://docs.projectcalico.org/v3.14/manifests/calico.yaml
+          export JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'
+          until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do
+            echo "wait for minikube ready ..."
+            sleep 1;
+          done
+          kubectl get nodes
+          # Install Splunk on minikube
+          kubectl apply -f ci_scripts/k8s-splunk.yml
+          # Wait until splunk is ready
+          until kubectl logs splunk --tail=2 | grep -q 'Ansible playbook complete'; do
+            sleep 1;
+          done
+          export CI_SPLUNK_HOST=$(kubectl get pod splunk --template={{.status.podIP}})
+          # Setup Indexes
+          curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=$CI_INDEX_EVENTS -d datatype=event
+          curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=$CI_INDEX_OBJECTS -d datatype=event
+          curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=$CI_INDEX_METRICS -d datatype=metric
+          curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=default-events -d datatype=event
+          curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=ns-anno -d datatype=event
+          curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/data/indexes -d name=pod-anno -d datatype=event
+          # Enable HEC services
+          curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http/http/enable
+          # Create new HEC token
+          curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k -d "name=splunk_hec_token&token=a6b5e77f-d5f6-415a-bd43-930cecb12959&disabled=0&index=default-events&indexes=default-events,$CI_INDEX_METRICS,$CI_INDEX_OBJECTS,$CI_INDEX_EVENTS,ns-anno,pod-anno" https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http
+          # Restart Splunk
+          curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/server/control/restart -X POST
+
+      - name: Build SCK artifacts
+        run: |
+          ci_scripts/build_sck_artifacts.sh
+
+      - name: Deploy k8s connector
+        run: |
+          export CI_SPLUNK_HOST=$(kubectl get pod splunk --template={{.status.podIP}})
+          ci_scripts/deploy_connector.sh
+
+      - name: Deploy log generator
+        run: |
+          kubectl apply -f test/test_setup.yaml
+          sleep 80
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - name: Run functional tests
+        run: |
+          kubectl get nodes
+          export PYTHONWARNINGS="ignore:Unverified HTTPS request"
+          export CI_SPLUNK_HOST=$(kubectl get pod splunk --template={{.status.podIP}})
+          cd test
+          pip install --upgrade pip
+          pip install -r requirements.txt
+          echo "Running functional tests....."
+          python -m pytest \
+          	--splunkd-url https://$CI_SPLUNK_HOST:8089 \
+          	--splunk-user admin \
+          	--splunk-password $CI_SPLUNK_PASSWORD \
+          	-p no:warnings -s

.gitignore

@@ -1,4 +1,6 @@
 build/
 .DS_Store
 *__pycache__*
-tmp/
+tmp/
+.idea/
+helm-artifacts/*.tgz

ci_scripts/build_sck_artifacts.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -e
+VERSION=$(cat VERSION)
+TAG="$VERSION-${GITHUB_SHA}"
+
+function replace_generic_version() {
+  file="$1"
+  _line=$(awk '/version:/{print NR;exit}' $file)
+  replacement="version: $TAG"
+  # Escape backslash, forward slash and ampersand for use as a sed replacement.
+  replacement_escaped=$(echo "$replacement" | sed -e 's/[\/&]/\\&/g')
+  sed -i "${_line}s/.*/$replacement_escaped/" "$file"
+}
+
+repos_array=("helm-chart/splunk-connect-for-kubernetes" "helm-chart/splunk-connect-for-kubernetes/charts/splunk-kubernetes-logging"
+  "helm-chart/splunk-connect-for-kubernetes/charts/splunk-kubernetes-metrics" "helm-chart/splunk-connect-for-kubernetes/charts/splunk-kubernetes-objects")
+
+for repo in "${repos_array[@]}"; do
+  filename="${repo}/Chart.yaml"
+  replace_generic_version $filename
+done
+
+mkdir helm-artifacts
+
+for repo in "${repos_array[@]}"; do
+  helm package -d helm-artifacts $repo
+done

ci_scripts/deploy_connector.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -e
+
+#Make sure to check and clean previously failed deployment
+echo "Checking if previous deployment exist..."
+if [ "`helm ls --short`" == "" ]; then
+   echo "Nothing to clean, ready for deployment"
+else
+   helm delete $(helm ls --short)
+fi
+echo "Deploying k8s-connect with latest changes"
+helm install ci-sck --set global.splunk.hec.token=$CI_SPLUNK_HEC_TOKEN \
+--set global.splunk.hec.host=$CI_SPLUNK_HOST \
+--set kubelet.serviceMonitor.https=true \
+-f ci_scripts/sck_values.yml helm-artifacts/splunk-connect-for-kubernetes*.tgz
+#wait for deployment to finish
+until kubectl get pod | grep Running | [[ $(wc -l) == 4 ]]; do
+   sleep 1;
+done

ci_scripts/k8s-splunk.yml

@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: splunk
+spec:
+  hostNetwork: true
+  securityContext:
+    runAsUser: 0
+    runAsGroup: 0
+  containers:
+  - name: splunk
+    image: splunk/splunk:latest
+    ports:
+    - containerPort: 8000
+      hostPort: 8000
+      protocol: TCP
+    - containerPort: 8088
+      hostPort: 8088
+      protocol: TCP
+    - containerPort: 8089
+      hostPort: 8089
+      protocol: TCP
+    env:
+    - name: SPLUNK_START_ARGS
+      value: --accept-license
+    - name: SPLUNK_USER
+      value: root
+    - name: SPLUNK_PASSWORD
+      value: helloworld
+    - name: SPLUNK_LAUNCH_CONF
+      value: OPTIMISTIC_ABOUT_FILE_LOCKING=1

ci_scripts/sck_values.yml

@@ -0,0 +1,57 @@
+global:
+  logLevel: debug
+  splunk:
+    hec:
+      protocol: https
+      insecureSSL: true
+      host: $CI_SPLUNK_HOST
+      token: $CI_SPLUNK_HEC_TOKEN
+
+splunk-kubernetes-logging:
+  kubernetes:
+    insecureSSL: true
+    clusterName: "ci-k8s-cluster"
+  splunk:
+    hec:
+      indexName: ci_events
+  image:
+    name: splunk/fluentd-hec
+    pullPolicy: Always
+  k8sMetadata:
+    podLabels:
+      - app
+  customMetadata:
+    - name: customfield1
+      value: customvalue1
+    - name: customfield2
+      value: customvalue2
+  customMetadataAnnotations:
+    - name: custom_field
+      annotation: splunk.com/custom-field
+
+splunk-kubernetes-metrics:
+  kubernetes:
+    useRestClientSSL: false
+    insecureSSL: true
+    kubeletPort: 10250
+    clusterName: "ci-k8s-cluster-metrics"
+  splunk:
+    hec:
+      indexName: ci_metrics
+  image:
+    name: splunk/k8s-metrics
+    pullPolicy: Always
+  imageAgg:
+    name: splunk/k8s-metrics-aggr
+
+splunk-kubernetes-objects:
+  splunk:
+    hec:
+      indexName: ci_objects
+  kubernetes:
+    insecureSSL: true
+    clusterName: "ci-k8s-cluster-objects"
+  image:
+    name: splunk/kube-objects
+    pullPolicy: Always
+

lintconf.yaml

@@ -0,0 +1,42 @@
+---
+rules:
+  braces:
+    min-spaces-inside: 0
+    max-spaces-inside: 0
+    min-spaces-inside-empty: -1
+    max-spaces-inside-empty: -1
+  brackets:
+    min-spaces-inside: 0
+    max-spaces-inside: 0
+    min-spaces-inside-empty: -1
+    max-spaces-inside-empty: -1
+  colons:
+    max-spaces-before: 0
+    max-spaces-after: 1
+  commas:
+    max-spaces-before: 0
+    min-spaces-after: 1
+    max-spaces-after: 1
+  comments:
+    require-starting-space: true
+    min-spaces-from-content: 2
+  document-end: disable
+  document-start: disable           # No --- to start a file
+  empty-lines:
+    max: 2
+    max-start: 0
+    max-end: 0
+  hyphens:
+    max-spaces-after: 1
+  indentation:
+    spaces: consistent
+    indent-sequences: whatever      # - list indentation will handle both indentation and without
+    check-multi-line-strings: false
+  key-duplicates: enable
+  line-length: disable              # Lines can be any length
+  new-line-at-end-of-file: enable
+  new-lines:
+    type: unix
+  trailing-spaces: enable
+  truthy:
+    level: warning