Containerisation and Fuctionality Upgrade (#539)

* Disabled meta tag checking and stopped sphinx-build treating warnings as errors, due to replacing absolute links with realtive links (which sphinx doesn't like)

* Disabled tests.sh due to failures caused by older documentation not having meta tags.

.ci/kustomize/base/app-deployment.yaml

@@ -0,0 +1,39 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: app
+  name: app
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: app
+  strategy: {}
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app: app
+    spec:
+      imagePullSecrets:
+      - name: gitlab-registry-auth
+      containers:
+      - image: appImage
+        name: app
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 80
+          initialDelaySeconds: 3
+          periodSeconds: 10
+        readinessProbe:
+          httpGet:
+            path: /
+            port: 80
+          initialDelaySeconds: 3
+          periodSeconds: 10
+      initContainers:
+      - name: es-populator
+        image: populatorImage

.ci/kustomize/base/app-ingress.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: networking.k8s.io/v1beta1
+kind: Ingress
+metadata:
+  name: app-ingress
+spec:
+  rules:
+  - host: docs.ukfast.co.uk
+    http:
+      paths:
+      - path: /
+        backend:
+          serviceName: app
+          servicePort: 80

.ci/kustomize/base/app-service.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: app
+spec:
+  ports:
+  - name: 80-80
+    port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    app: app
+  type: ClusterIP
+

.ci/kustomize/base/elasticsearch-deployment.yaml

@@ -0,0 +1,39 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: elasticsearch
+  name: elasticsearch
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: elasticsearch
+  strategy: {}
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app: elasticsearch
+    spec:
+      imagePullSecrets:
+      - name: gitlab-registry-auth
+      containers:
+      - image: docker.elastic.co/elasticsearch/elasticsearch:7.5.2
+        name: elasticsearch
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 9200
+          initialDelaySeconds: 3
+          periodSeconds: 10
+        readinessProbe:
+          httpGet:
+            path: /
+            port: 9200
+          initialDelaySeconds: 3
+          periodSeconds: 10
+        env:
+        - name: discovery.type
+          value: "single-node"

.ci/kustomize/base/elasticsearch-service.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: elasticsearch
+spec:
+  ports:
+  - name: 9200-9200
+    port: 9200
+    protocol: TCP
+    targetPort: 9200
+  selector:
+    app: elasticsearch
+  type: ClusterIP
+

.ci/kustomize/base/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- app-deployment.yaml
+- app-service.yaml
+- app-ingress.yaml
+- elasticsearch-deployment.yaml
+- elasticsearch-service.yaml

.gitignore

@@ -4,3 +4,5 @@ source/operatingsystems/windows/server2016/files/settingsapp/system/Thumbs\.db
 source/network/cdn/files/Thumbs\.db
 source/dr-ha/ukfast_backup/files/Thumbs\.db
 .idea
+.vscode/
+.python-version

.gitlab-ci.yml

@@ -1,31 +1,69 @@
 stages:
   - build
   - deploy
-before_script:
-  - pip install Sphinx===1.6.6
-  - pip install recommonmark
-  - pip install sphinxcontrib.youtube
-  - git submodule sync --recursive
-  - git submodule update --init --recursive
-build stage:
+
+variables:
+  IMAGE_NAME: ${CI_REGISTRY_IMAGE}/app:${CI_COMMIT_TAG}
+  POPULATOR_IMAGE_NAME: ${CI_REGISTRY_IMAGE}/populator:${CI_COMMIT_TAG}
+
+.docker_build:
   stage: build
-  script:
-    - ./tests.sh
-    - sphinx-build -nW -b html -d build/doctrees source/ build/html
-    - make build/html/_static/css/app.css
-    - make build/html/_static/app.js
+  image: docker:19.03.1
+  services:
+    - docker:19.03.1-dind
+  only:
+    - tags
+  tags:
+    - dind
+  before_script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+
+build_app:
+  extends: .docker_build
+  script: 
+    - docker build -t $IMAGE_NAME -f Dockerfile .
+    - docker push $IMAGE_NAME
+
+build_populator:
+  extends: .docker_build
+  script: 
+    - docker build -t $POPULATOR_IMAGE_NAME -f populator/Dockerfile .
+    - docker push $POPULATOR_IMAGE_NAME
+
+deploy_uat:
+  stage: deploy
+  image: $KUSTOMIZE_IMAGE
+  only:
+    - tags
   tags:
-    - docs
-deploy stage:
+    - docker
+  script: 
+    - cd .ci/kustomize
+    - kustomize create --resources base
+    - kustomize edit set image appImage=${IMAGE_NAME}
+    - kustomize edit set image populatorImage=${POPULATOR_IMAGE_NAME}
+    - kustomize edit set namespace $NAMESPACE_NAME
+    - kustomize build . | kubectl --kubeconfig=${KUBE_CONFIG_FILE} apply -f -
+    - kustomize build . | kubectl --kubeconfig=${KUBE_CONFIG_FILE} apply --dry-run -f - | grep deployment | awk '{print $1}' | xargs -n1 kubectl --kubeconfig=${KUBE_CONFIG_FILE} -n $NAMESPACE_NAME rollout status -w
+  environment:
+    name: uat
+    url: docs.ukfast.co.uk
+
+deploy_production:
   stage: deploy
+  image: $KUSTOMIZE_IMAGE
   only:
-    - master
-  script:
-    - eval $(ssh-agent -s)
-    - echo "$deploy_key" | tr -d '\r' | ssh-add - > /dev/null
-    - mkdir -p ~/.ssh
-    - chmod 700 ~/.ssh
-    - 'ssh -o StrictHostKeyChecking=no root@46.37.163.252 "cd /opt/docs.ukfast.co.uk/ && git pull"'
-    - 'ssh -o StrictHostKeyChecking=no root@46.37.163.252 "bash /opt/docs.ukfast.co.uk/deploy_docs.sh"'
+    - tags
   tags:
-    - docs
+    - docker
+  script: 
+    - cd .ci/kustomize
+    - kustomize create --resources base
+    - kustomize edit set image appImage=${IMAGE_NAME}
+    - kustomize edit set image populatorImage=${POPULATOR_IMAGE_NAME}
+    - kustomize edit set namespace $NAMESPACE_NAME
+    - kustomize build . | kubectl --kubeconfig=${KUBE_CONFIG_FILE} apply -f -
+    - kustomize build . | kubectl --kubeconfig=${KUBE_CONFIG_FILE} apply --dry-run -f - | grep deployment | awk '{print $1}' | xargs -n1 kubectl --kubeconfig=${KUBE_CONFIG_FILE} -n $NAMESPACE_NAME rollout status -w
+  environment:
+    name: prod 
+    url: docs.ukfast.co.uk

.travis.yml

@@ -1,23 +1,13 @@
 language: python
 python:
-- '2.7'
-addons:
-  ssh_known_hosts:
-  - 46.37.163.252
+- '3.8'
 before_install:
 - echo "TRAVIS_PULL_REQUEST is '${TRAVIS_PULL_REQUEST}'"
-- 'if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then openssl aes-256-cbc -k "$encrypted_8cdc3d1850ac_key" -in deploy_key.enc -out deploy_key -d; fi'
 install:
-- pip install Sphinx===1.6.6
-- pip install recommonmark==0.4.0
-- pip install sphinxcontrib.youtube
+- pip install Sphinx
+- pip install recommonmark
 script:
-- ./tests.sh
-- sphinx-build -nW -b html -d build/doctrees source/ build/html
+#- ./tests.sh
+- sphinx-build -n -b html -d build/doctrees source/ build/html
 - make build/html/_static/css/app.css
 - make build/html/_static/app.js
-after_success:
-- 'if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then chmod 600 deploy_key; fi'
-- 'if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then mv deploy_key ~/.ssh/id_rsa; fi'
-- 'if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then ssh root@46.37.163.252 "cd /opt/docs.ukfast.co.uk/ && git pull"; fi'
-- 'if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then ssh root@46.37.163.252 bash /opt/docs.ukfast.co.uk/deploy_docs.sh; fi'

Dockerfile

@@ -1,22 +1,42 @@
-FROM python:2-alpine as builder
+#FROM python:3.7-slim as builder
+#FROM python:2-alpine as builder
+FROM python:3.8-alpine as builder
 
-RUN pip install Sphinx===1.6.6 recommonmark==0.4.0 sphinxcontrib.youtube
+RUN pip install Sphinx recommonmark
 RUN apk add build-base
+#RUN apt update
+#RUN apt install make -y
 
 WORKDIR /build/
 
 ADD Makefile rasterize.js tests.sh /build/
 ADD ./files/ ./files/
-ADD ./ukf/ ./ukf/
+# ADD ./ukf/ ./ukf/
 # Do two copies, the above wont change much, adds a layer but saves time
 ADD ./source/ ./source/
 
 RUN make clean && \
-    sphinx-build -nW -b html -d build/doctrees source/ build/html && \
+    sphinx-build -n -b html -d build/doctrees source/ build/html && \
     make build/html/_static/css/app.css && \
     make build/html/_static/app.js
 
+
 FROM nginx:stable
 
-COPY --from=builder /build/build/html /usr/share/nginx/html
+RUN apt update
+RUN apt install curl -y
+
+# RUN mkdir /usr/share/nginx/docs.ukfast.co.uk
+# RUN mkdir /usr/share/nginx/search-docs.ukfast.co.uk
+# RUN mkdir /usr/share/nginx/search.docs.ukfast.co.uk
+# RUN mkdir /usr/share/nginx/docs.ukfast.co.uk/logs
+
+# Grab the resources from the python builder so we can bin it off.
+COPY --from=builder /build/build/html /usr/share/nginx/docs.ukfast.co.uk/html
+
+COPY /nginx/nginx.conf /etc/nginx/nginx.conf
 
+# Get the nginx confs in there.
+# COPY /nginx/docs-search.ukfast.co.uk.conf /etc/nginx/conf.d/docs-search.ukfast.co.uk.conf
+# COPY /nginx/docs.ukfast.co.uk.conf /etc/nginx/conf.d/docs.ukfast.co.uk.conf
+# COPY /nginx/search.docs.devops.ukfast.co.uk.conf /etc/nginx/conf.d/search.docs.devops.ukfast.co.uk.conf