diff --git a/.travis.yml b/.travis.yml index 34e0ee4e3e..5fa3e017f8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,94 +1,19 @@ +dist: xenial +language: minimal sudo: required -language: python -install: true - services: - docker - -before_script: - - export -f travis_fold - - export REPO=appsecpipeline/django-defectdojo - - export TAG=`if [ "$TRAVIS_BRANCH" == "master" ]; then echo "latest"; else echo $TRAVIS_BRANCH ; fi` - env: - - TEST=smoke-test - - TEST=integration-test - - TEST=unit-test - - TEST=bandit - - TEST=docker-bench-security - - TEST=flake8-complete - - TEST=flake8 - -matrix: - allow_failures: - - env: TEST=bandit - - env: TEST=flake8-complete - -script: - - | - echo "Running test=$TEST" - case "$TEST" in - smoke-test) - travis_fold start "smoke-test" - bash entrypoint_scripts/test/travis-smoke-test.sh || exit 1 - travis_fold end "smoke-test" - ;; - unit-test) - travis_fold start "unit-test" - bash entrypoint_scripts/test/travis-unit-test.sh || exit 1 - travis_fold end "unit-test" - ;; - integration-test) - travis_fold start "integration-test" - bash entrypoint_scripts/test/travis-integration-test.sh || exit 1 - travis_fold end "integration-test" - ;; - bandit) - # install bandit - pip install bandit - - ## Run Bandit python static code - bandit -r * -x venv,tests - ;; - docker-bench-security) - ## Run Docker Bench for Security - git clone https://github.com/docker/docker-bench-security.git - cd docker-bench-security - sh docker-bench-security.sh - ;; - flake8-complete) - pip install flake8 - flake8 . - ;; - safety) - pip install safety - safety check -i 35015 - ;; - flake8) - echo "$TRAVIS_BRANCH" - if [ "$TRAVIS_BRANCH" == "dev" ] - then - echo "Running Flake8 tests on dev branch aka pull requests" - # We need to checkout dev for flake8-diff to work properly - git checkout dev - pip install pep8 flake8 flake8-diff - flake8-diff - else - echo "true" - fi - esac - -after_success: - #Push to docker repo - - bash entrypoint_scripts/deploy/post-action.bash - -notifications: - slack: - rooms: - secure: nPXwHnPcf37yGkCkLimx5UmY9LTtOHL0lw88cAQeXCNNjeZuhS2jS5xGUOwwp3SrsYE4tZhD0WuVEHGDcyIhmBZh9Qqk3NHKz+tQDD/e0GE/8uTTfR1Eh+pq1YOIcLYzzKA2khmJSeHqqDriVZZoWpn67oHtrui9FYesapZ8AX0= - on_success: never - on_failure: never - on_start: never -addons: - firefox: "45.0" - chrome: stable + global: + - K8S_VERSION=v1.13.4 + - MINIKUBE_VERSION=v0.35.0 + - HELM_VERSION=v2.13.0 + - CHANGE_MINIKUBE_NONE_USER=true + matrix: + - BROKER=rabbitmq DATABASE=mysql + - BROKER=rabbitmq DATABASE=postgresql + - BROKER=redis DATABASE=mysql + - BROKER=redis DATABASE=postgresql +before_install: ['./travis/before-install.sh'] +before_script: ['./travis/before-script.sh'] +script: ['./travis/script.sh'] diff --git a/KUBERNETES.md b/KUBERNETES.md index b0c000ee4c..8e416e728b 100644 --- a/KUBERNETES.md +++ b/KUBERNETES.md @@ -60,14 +60,15 @@ helm install \ --set celery.replicas=3 \ --set rabbitmq.replicas=3 +# Run highly available PostgreSQL cluster instead of MySQL helm install \ ./helm/defectdojo \ --name=defectdojo \ --namespace="${K8S_NAMESPACE}" \ --set host="defectdojo.${TLS_CERT_DOMAIN}" \ - --set django.replicas=1 \ - --set celery.replicas=1 \ - --set rabbitmq.replicas=1 \ + --set django.replicas=3 \ + --set celery.replicas=3 \ + --set rabbitmq.replicas=3 \ --set django.ingress.secretName="minikube-tls" \ --set mysql.enabled=false \ --set database=postgresql \ @@ -75,8 +76,6 @@ helm install \ --set postgresql.replication.enabled=true \ --set postgresql.replication.slaveReplicas=3 - - # Run test. If there are any errors, re-run the command without `--cleanup` and # inspect the test container. helm test defectdojo --cleanup diff --git a/helm/defectdojo/templates/celery-deployment.yaml b/helm/defectdojo/templates/celery-deployment.yaml index a051d7698d..dbc4e7b0c0 100644 --- a/helm/defectdojo/templates/celery-deployment.yaml +++ b/helm/defectdojo/templates/celery-deployment.yaml @@ -25,7 +25,7 @@ spec: spec: containers: - name: celery - image: {{ .Values.celery.image }} + image: "{{ .Values.celery.repository }}:{{ .Values.tag }}" imagePullPolicy: {{ .Values.imagePullPolicy }} env: - name: DD_CELERY_BROKER_SCHEME diff --git a/helm/defectdojo/templates/django-deployment.yaml b/helm/defectdojo/templates/django-deployment.yaml index 4551972117..1d5a91a1f0 100644 --- a/helm/defectdojo/templates/django-deployment.yaml +++ b/helm/defectdojo/templates/django-deployment.yaml @@ -28,7 +28,7 @@ spec: emptyDir: {} containers: - name: uwsgi - image: {{ .Values.django.uwsgi.image | quote }} + image: "{{ .Values.django.uwsgi.repository }}:{{ .Values.tag }}" imagePullPolicy: {{ .Values.imagePullPolicy }} volumeMounts: - name: run @@ -37,7 +37,7 @@ spec: - name: DD_DATABASE_ENGINE value: django.db.backends.{{ if eq .Values.database "postgresql" }}postgresql_psycopg2{{ end }}{{ if eq .Values.database "mysql" }}mysql{{ end }} - name: DD_ALLOWED_HOSTS - value: {{ $fullName }}.{{ .Release.Namespace }}.minikube.local + value: {{ .Values.host }} - name: DD_DATABASE_HOST value: {{ $fullName }}-{{ .Values.database }} - name: DD_DATABASE_PORT @@ -54,7 +54,7 @@ spec: resources: {{- toYaml .Values.django.uwsgi.resources | nindent 12 }} - name: nginx - image: {{ .Values.django.nginx.image | quote }} + image: "{{ .Values.django.nginx.repository }}:{{ .Values.tag }}" imagePullPolicy: {{ .Values.imagePullPolicy }} volumeMounts: - name: run diff --git a/helm/defectdojo/templates/initializer-job.yaml b/helm/defectdojo/templates/initializer-job.yaml index c6d232c132..ca33be322e 100644 --- a/helm/defectdojo/templates/initializer-job.yaml +++ b/helm/defectdojo/templates/initializer-job.yaml @@ -20,7 +20,7 @@ spec: spec: containers: - name: initializer - image: {{ .Values.initializer.image }} + image: "{{ .Values.initializer.repository }}:{{ .Values.tag }}" imagePullPolicy: {{ .Values.imagePullPolicy }} env: - name: DD_DATABASE_ENGINE diff --git a/helm/defectdojo/values.yaml b/helm/defectdojo/values.yaml index fe87691018..155c6a6efa 100644 --- a/helm/defectdojo/values.yaml +++ b/helm/defectdojo/values.yaml @@ -3,12 +3,13 @@ database: mysql host: defectdojo.default.minikube.local imagePullPolicy: Always +tag: latest # Components celery: affinity: {} broker: rabbitmq - image: defectdojo/defectdojo-celery:latest + repository: defectdojo/defectdojo-celery logLevel: DEBUG nodeSelector: {} replicas: 1 @@ -23,7 +24,7 @@ django: enabled: true secretName: defectdojo-tls nginx: - image: defectdojo/defectdojo-nginx:latest + repository: defectdojo/defectdojo-nginx resources: cpu: 100m memory: 128Mi @@ -31,13 +32,13 @@ django: replicas: 1 tolerations: [] uwsgi: - image: defectdojo/defectdojo-uwsgi:latest + repository: defectdojo/defectdojo-uwsgi resources: cpu: 100m memory: 128Mi initializer: - image: defectdojo/defectdojo-initializer:latest + repository: defectdojo/defectdojo-initializer keepSeconds: 60 resources: cpu: 100m diff --git a/travis/before-install.sh b/travis/before-install.sh new file mode 100755 index 0000000000..c176e5a970 --- /dev/null +++ b/travis/before-install.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +sudo apt-get -y update +sudo apt-get -y install socat diff --git a/travis/before-script.sh b/travis/before-script.sh new file mode 100755 index 0000000000..8f42ee4beb --- /dev/null +++ b/travis/before-script.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +curl -LsO "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl" +chmod +x kubectl +sudo mv kubectl /usr/local/bin/ + +curl -Lso minikube "https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64" +chmod +x minikube +sudo mv minikube /usr/local/bin/ + +curl -L https://storage.googleapis.com/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz | tar zx +chmod +x linux-amd64/helm +sudo mv linux-amd64/helm /usr/local/bin/ +rm -rf linux-amd64/ + +echo "127.0.0.1 ${DD_HOST}" | sudo tee -a /etc/hosts diff --git a/travis/script.sh b/travis/script.sh new file mode 100755 index 0000000000..f19f9ecae4 --- /dev/null +++ b/travis/script.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +# Build Docker images +DOCKER_IMAGES=(uwsgi nginx celery initializer) +for DOCKER_IMAGE in "${DOCKER_IMAGES[@]}" +do + docker build \ + --tag "defectdojo/defectdojo-${DOCKER_IMAGE}:${TRAVIS_BUILD_NUMBER}" \ + --file "Dockerfile.${DOCKER_IMAGE}" \ + . +done + +# Start Minikube +sudo minikube start \ + --vm-driver=none \ + --kubernetes-version="${K8S_VERSION}" + +# Configure Kubernetes context and test it +sudo minikube update-context +sudo kubectl cluster-info + +# Enable Nginx ingress add-on and wait for it +sudo minikube addons enable ingress +echo -n "Waiting for Nginx ingress controller " +until [[ "True" == "$(sudo kubectl get pod \ + --selector=app.kubernetes.io/name=nginx-ingress-controller \ + --namespace=kube-system \ + -o 'jsonpath={.items[*].status.conditions[?(@.type=="Ready")].status}')" ]] +do + sleep 1 + echo -n "." +done +echo + +# Create Helm and wait for Tiller to become ready +sudo helm init +echo -n "Waiting for Tiller " +until [[ "True" == "$(sudo kubectl get pod \ + --selector=name=tiller \ + --namespace=kube-system \ + -o 'jsonpath={.items[*].status.conditions[?(@.type=="Ready")].status}')" ]] +do + sleep 1 + echo -n "." +done +echo + +# Update Helm repository +sudo helm repo update + +# Update Helm dependencies for DefectDojo +sudo helm dependency update ./helm/defectdojo + +# Install DefectDojo into Kubernetes and wait for it +sudo helm install \ + ./helm/defectdojo \ + --name=defectdojo \ + --set django.ingress.enabled=false \ + --set tag="${TRAVIS_BUILD_NUMBER}" \ + --set imagePullPolicy=Never +echo -n "Waiting for DefectDojo to become ready " +until [[ "True" == "$(sudo kubectl get pod \ + --selector=defectdojo.org/component=django \ + -o 'jsonpath={.items[*].status.conditions[?(@.type=="Ready")].status}')" ]] +do + sleep 1 + echo -n "." +done +echo +echo "DefectDojo is up and running." +sudo kubectl get pods + +# Run all tests +echo "Running tests." +sudo helm test defectdojo +sudo kubectl get pods + +# Uninstall +echo "Deleting DefectDojo from Kubernetes." +sudo helm delete defectdojo --purge +sudo kubectl get pods