Distributed execution via worker (#166)

* First tests

* A bit more work on the register api

* Implemented registration process

* little work

* Removed trash from last merge conflict

* Growing

* Implemented load of client tls certificates

* Some more work

* Added new parameters

* Tested worker registration process and fixed several bugs

* WIP server grpc implementation

* WIP memdb implementation

* More work on memdb implementation

* Added memdb to modules

* More work done on getwork

* Implemented binary streaming for pipeline binaries

* More work done

* Work work work

* Implemented updatework function

* Finalizing for first test

* More work done

* Massive work done. Refactored tons of stuff. Pray that it will not break something :-O

* Implemented logs shipping

* More work done and some testing

* IT WORKS LOL

* More bug fixing

* Implemented authentication and deregister function

* Implemented backend worker tags

* More work on the frontend

* Implemented status view of workers. Added worker permissions. Implemented automatic primary instance work scheduling when workers are busy.

* Added worker data fetch

* Implemented deregister process and some bug fixing

* Implemented reset of worker secret. Implemented ignored specialized vault keys

* Implemented pipeline worker tags. WIP backend testing

* Implemented automatic worker status transission and a few more status checks

* Added worker registration test and fixed a few tests

* Fixed most tests

* Upgraded circleci

* Fixed last failing tests

* circle ci

* Fixed data race

* fixed

* fixed

* Linter fixes

* fix

* fix tests

* Fixed

* Fixed

* Added more workers handler tests

* Fixed concurrent tests

* Added memdb service test

* Fixed failing test

* Worked on tests

* Added missing error check

* Fixed racy test

* More work on tests

* fixed wrong fixtures path

* Added more test

* Added server worker tests

* Fixed small test issues

* More work on tests

* Fixed small issues

* Increased waiting time

* Racy test fix

* growing

* fix

* fix

* Added missing piece

* Added worker tags to UI. Added option to prevent work being scheduled on primary.

* Improved test coverage

* Added negative tags to allow users to remove automatically detected tags. Added tags info to overview view. Fixed add tags bug.

* Removed prettify cpp tag since it might lead to confusion

.circleci/config.yml

@@ -20,7 +20,7 @@ jobs:
   test_and_coverage:
     working_directory: /go/src/github.com/gaia-pipeline/gaia
     docker:
-      - image: circleci/golang:1.11.4
+      - image: circleci/golang:1.12.4
         environment:
           GO111MODULE: "on"
     steps:
@@ -32,7 +32,7 @@ jobs:
             echo "" > coverage.txt
 
             for d in $(go list ./... | grep -v vendor | grep -v /testacc); do
-              go test -v -race -coverprofile=profile.out -covermode=atomic $d
+              go test -v -timeout 20s -race -coverprofile=profile.out -covermode=atomic $d
               if [ -f profile.out ]; then
                 cat profile.out >> coverage.txt
                 rm profile.out
@@ -50,7 +50,7 @@ jobs:
   acceptance_tests:
     working_directory: /go/src/github.com/gaia-pipeline/gaia
     docker:
-        - image: gaiapipeline/circleci:0.0.3
+        - image: gaiapipeline/circleci:0.0.4
           environment:
             GO111MODULE: "on"
     steps:
@@ -64,7 +64,7 @@ jobs:
   compile:
     working_directory: /go/src/github.com/gaia-pipeline/gaia
     docker:
-      - image: circleci/golang:1.11.4
+      - image: circleci/golang:1.12.4
         environment:
           GO111MODULE: "on"
     steps:

.circleci/images/primary/Dockerfile

@@ -24,20 +24,20 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 		pkg-config \
 	&& rm -rf /var/lib/apt/lists/*
 
-ENV GOLANG_VERSION 1.11.4
+ENV GOLANG_VERSION 1.12.5
 
 RUN set -eux; \
 	\
 # this "case" statement is generated via "update.sh"
 	dpkgArch="$(dpkg --print-architecture)"; \
 	case "${dpkgArch##*-}" in \
-		amd64) goRelArch='linux-amd64'; goRelSha256='fb26c30e6a04ad937bbc657a1b5bba92f80096af1e8ee6da6430c045a8db3a5b' ;; \
-		armhf) goRelArch='linux-armv6l'; goRelSha256='9f7a71d27fef69f654a93e265560c8d9db1a2ca3f1dcdbe5288c46facfde5821' ;; \
-		arm64) goRelArch='linux-arm64'; goRelSha256='b76df430ba8caff197b8558921deef782cdb20b62fa36fa93f81a8c08ab7c8e7' ;; \
-		i386) goRelArch='linux-386'; goRelSha256='cecd2da1849043237d5f0756a93d601db6798fa3bb27a14563d201088aa415f3' ;; \
-		ppc64el) goRelArch='linux-ppc64le'; goRelSha256='1f10146826acd56716b00b9188079af53823ddd79ceb6362e78e2f3aafb370ab' ;; \
-		s390x) goRelArch='linux-s390x'; goRelSha256='4467442dacf89eb94c5d6f9f700204cb360be82db60e6296cc2ef8d0e890cd42' ;; \
-		*) goRelArch='src'; goRelSha256='4cfd42720a6b1e79a8024895fa6607b69972e8e32446df76d6ce79801bbadb15'; \
+		amd64) goRelArch='linux-amd64'; goRelSha256='aea86e3c73495f205929cfebba0d63f1382c8ac59be081b6351681415f4063cf' ;; \
+		armhf) goRelArch='linux-armv6l'; goRelSha256='311f5e76c7cec1ec752474a61d837e474b8e750b8e3eed267911ab57c0e5da9a' ;; \
+		arm64) goRelArch='linux-arm64'; goRelSha256='ff09f34935cd189a4912f3f308ec83e4683c309304144eae9cf60ebc552e7cd8' ;; \
+		i386) goRelArch='linux-386'; goRelSha256='146605e13bf337ff3aacd941a816c5d97a8fef8b5817e07fcec4540632085980' ;; \
+		ppc64el) goRelArch='linux-ppc64le'; goRelSha256='e88b2a2098bc79ad33912d1d27bc3282a7f3231b6f4672f306465bf46ff784ca' ;; \
+		s390x) goRelArch='linux-s390x'; goRelSha256='168d297ec910cb446d1aea878baeb85f1387209f9eb55dde68bddcd4c006dcbb' ;; \
+		*) goRelArch='src'; goRelSha256='2aa5f088cbb332e73fc3def546800616b38d3bfe6b8713b8a6404060f22503e8'; \
 			echo >&2; echo >&2 "warning: current architecture ($dpkgArch) does not have a corresponding Go binary release; will be building from source"; echo >&2 ;; \
 	esac; \
 	\
@@ -65,60 +65,68 @@ RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
 # --------------- End Go Part ---------------
 
 # --------------- Start Java Part ---------------
-RUN apt-get update && apt-get install -y --no-install-recommends \
+RUN mkdir ~/.gnupg
+RUN echo "disable-ipv6" >> ~/.gnupg/dirmngr.conf
+RUN set -eux; \
+	apt-get update; \
+	apt-get install -y --no-install-recommends \
 		bzip2 \
 		unzip \
 		xz-utils \
-	&& rm -rf /var/lib/apt/lists/*
+		\
+# java.lang.UnsatisfiedLinkError: /usr/local/openjdk-11/lib/libfontmanager.so: libfreetype.so.6: cannot open shared object file: No such file or directory
+# java.lang.NoClassDefFoundError: Could not initialize class sun.awt.X11FontManager
+# https://github.com/docker-library/openjdk/pull/235#issuecomment-424466077
+		fontconfig libfreetype6 \
+	; \
+	rm -rf /var/lib/apt/lists/*
 
 # Default to UTF-8 file.encoding
 ENV LANG C.UTF-8
 
-# add a simple script that can auto-detect the appropriate JAVA_HOME value
-# based on whether the JDK or only the JRE is installed
-RUN { \
-		echo '#!/bin/sh'; \
-		echo 'set -e'; \
-		echo; \
-		echo 'dirname "$(dirname "$(readlink -f "$(which javac || which java)")")"'; \
-	} > /usr/local/bin/docker-java-home \
-	&& chmod +x /usr/local/bin/docker-java-home
-
-# do some fancy footwork to create a JAVA_HOME that's cross-architecture-safe
-RUN ln -svT "/usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)" /docker-java-home
-ENV JAVA_HOME /docker-java-home
+ENV JAVA_HOME /usr/local/openjdk-8
+ENV PATH $JAVA_HOME/bin:$PATH
 
-ENV JAVA_VERSION 8u181
-ENV JAVA_DEBIAN_VERSION 8u181-b13-2~deb9u1
+# backwards compatibility shim
+RUN { echo '#/bin/sh'; echo 'echo "$JAVA_HOME"'; } > /usr/local/bin/docker-java-home && chmod +x /usr/local/bin/docker-java-home && [ "$JAVA_HOME" = "$(docker-java-home)" ]
 
-# see https://bugs.debian.org/775775
-# and https://github.com/docker-library/java/issues/19#issuecomment-70546872
-ENV CA_CERTIFICATES_JAVA_VERSION 20170531+nmu1
+# https://adoptopenjdk.net/upstream.html
+ENV JAVA_VERSION 8u212-b04
+ENV JAVA_BASE_URL https://github.com/AdoptOpenJDK/openjdk8-upstream-binaries/releases/download/jdk8u212-b04/OpenJDK8U-
+ENV JAVA_URL_VERSION 8u212b04
+# https://github.com/docker-library/openjdk/issues/320#issuecomment-494050246
 
-RUN set -ex; \
+RUN set -eux; \
 	\
-# deal with slim variants not having man page directories (which causes "update-alternatives" to fail)
-	if [ ! -d /usr/share/man/man1 ]; then \
-		mkdir -p /usr/share/man/man1; \
-	fi; \
+	dpkgArch="$(dpkg --print-architecture)"; \
+	case "$dpkgArch" in \
+		amd64) upstreamArch='x64' ;; \
+		arm64) upstreamArch='aarch64' ;; \
+		*) echo >&2 "error: unsupported architecture: $dpkgArch" ;; \
+	esac; \
 	\
-	apt-get update; \
-	apt-get install -y --no-install-recommends \
-		openjdk-8-jdk="$JAVA_DEBIAN_VERSION" \
-		ca-certificates-java="$CA_CERTIFICATES_JAVA_VERSION" \
-	; \
-	rm -rf /var/lib/apt/lists/*; \
+	wget -O openjdk.tgz.asc "${JAVA_BASE_URL}${upstreamArch}_linux_${JAVA_URL_VERSION}.tar.gz.sign"; \
+	wget -O openjdk.tgz "${JAVA_BASE_URL}${upstreamArch}_linux_${JAVA_URL_VERSION}.tar.gz" --progress=dot:giga; \
 	\
-# verify that "docker-java-home" returns what we expect
-	[ "$(readlink -f "$JAVA_HOME")" = "$(docker-java-home)" ]; \
+	export GNUPGHOME="$(mktemp -d)"; \
+# TODO find a good link for users to verify this key is right (https://mail.openjdk.java.net/pipermail/jdk-updates-dev/2019-April/000951.html is one of the only mentions of it I can find); perhaps a note added to https://adoptopenjdk.net/upstream.html would make sense?
+	gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys CA5F11C6CE22644D42C6AC4492EF8D39DC13168F; \
+# https://github.com/docker-library/openjdk/pull/322#discussion_r286839190
+	gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys EAC843EBD3EFDB98CC772FADA5CD6035332FA671; \
+	gpg --batch --list-sigs --keyid-format 0xLONG CA5F11C6CE22644D42C6AC4492EF8D39DC13168F | grep '0xA5CD6035332FA671' | grep 'Andrew Haley'; \
+	gpg --batch --verify openjdk.tgz.asc openjdk.tgz; \
+	gpgconf --kill all; \
+	rm -rf "$GNUPGHOME"; \
 	\
-# update-alternatives so that future installs of other OpenJDK versions don't change /usr/bin/java
-	update-alternatives --get-selections | awk -v home="$(readlink -f "$JAVA_HOME")" 'index($3, home) == 1 { $2 = "manual"; print | "update-alternatives --set-selections" }'; \
-# ... and verify that it actually worked for one of the alternatives we care about
-	update-alternatives --query java | grep -q 'Status: manual'
-
-# see CA_CERTIFICATES_JAVA_VERSION notes above
-RUN /var/lib/dpkg/info/ca-certificates-java.postinst configure
+	mkdir -p "$JAVA_HOME"; \
+	tar --extract --file openjdk.tgz --directory "$JAVA_HOME" --strip-components 1; \
+	rm openjdk.tgz*; \
+	\
+# TODO strip "demo" and "man" folders?
+	\
+# basic smoke test
+	javac -version; \
+    java -version
 # --------------- End Java Part ---------------
 
 # --------------- Start Maven Part ---------------

Makefile

@@ -4,7 +4,8 @@ NAMESPACE=${NAME}
 RELEASE_NAME=${NAME}
 HELM_DIR=$(shell pwd)/helm
 TEST=$$(go list ./... | grep -v /vendor/ | grep /testacc)
-TEST_TIMEOUT?=20m
+TEST_TIMEOUT_ACC?=20m
+TEST_TIMEOUT?=20s
 
 default: dev
 
@@ -34,13 +35,13 @@ get:
 	go get ./...
 
 test:
-	go test -v ./...
+	go test -v -race -timeout=$(TEST_TIMEOUT) ./...
 
 test-cover:
-	go test -v ./... --coverprofile=cover.out
+	go test -v -timeout=$(TEST_TIMEOUT) ./... --coverprofile=cover.out
 
 test-acc:
-	GAIA_RUN_ACC=true GAIA_DEV=true go test -v $(TEST) -timeout=$(TEST_TIMEOUT)
+	GAIA_RUN_ACC=true GAIA_DEV=true go test -v $(TEST) -timeout=$(TEST_TIMEOUT_ACC)
 
 release: compile_frontend static_assets compile_backend
 

frontend/client/app.js

@@ -32,9 +32,9 @@ axiosInstance.interceptors.request.use(function (request) {
 Vue.config.devtools = true
 sync(store, router)
 
-const nprogress = new NProgress({ parent: '.nprogress-container' })
+const nprogress = new NProgress({parent: '.nprogress-container'})
 
-const { state } = store
+const {state} = store
 
 Vue.directive('focus', {
   // When the bound element is inserted into the DOM...
@@ -98,6 +98,7 @@ function handleError (error) {
     console.log(error.response.data)
   }
 }
+
 Vue.prototype.$onError = handleError
 
 Vue.prototype.$onSuccess = (title, message) => {
@@ -109,6 +110,18 @@ Vue.prototype.$onSuccess = (title, message) => {
   })
 }
 
+Vue.prototype.$prettifyTags = (tags) => {
+  let prettyTags = ''
+  for (let i = 0; i < tags.length; i++) {
+    if (i === (tags.length - 1)) {
+      prettyTags += tags[i]
+    } else {
+      prettyTags += tags[i] + ', '
+    }
+  }
+  return prettyTags
+}
+
 router.beforeEach((route, redirect, next) => {
   if (state.app.device.isMobile && state.app.sidebar.opened) {
     store.commit(TOGGLE_SIDEBAR, false)
@@ -130,4 +143,4 @@ const app = new Vue({
 // A simple event bus
 export const EventBus = new Vue()
 
-export { app, router, store }
+export {app, router, store}

frontend/client/store/actions.js

@@ -7,9 +7,3 @@ export const toggleSidebar = ({ commit }, config) => {
 }
 
 export const toggleDevice = ({ commit }, device) => commit(types.TOGGLE_DEVICE, device)
-
-export const switchEffect = ({ commit }, effectItem) => {
-  if (effectItem) {
-    commit(types.SWITCH_EFFECT, effectItem)
-  }
-}

frontend/client/store/getters.js

@@ -4,9 +4,6 @@ const device = state => state.app.device
 const sidebar = state => state.app.sidebar
 const effect = state => state.app.effect
 const menuitems = state => state.menu.items
-const componententry = state => {
-  return state.menu.items.filter(c => c.meta && c.meta.label === 'Components')[0]
-}
 const session = state => state.session
 const intervals = state => state.intervals
 
@@ -17,7 +14,6 @@ export {
   sidebar,
   effect,
   menuitems,
-  componententry,
   session,
   intervals
 }

frontend/client/views/overview/index.vue

@@ -47,6 +47,16 @@
               <span v-else>
                 unknown
               </span><br />
+              <i class="fa fa-tags"></i>
+              <span style="color: #b1adad;">
+                Tags:
+              </span>
+              <span v-if="pipeline.p.tags">
+                {{ $prettifyTags(pipeline.p.tags) }}
+              </span>
+              <span v-else>
+                unknown
+              </span><br />
               <div class="pipelinegrid-footer">
                 <a class="button is-primary" @click="checkPipelineArgsAndStartPipeline(pipeline.p)" style="width: 100%;">
                   <span class="icon">
@@ -132,10 +142,10 @@ export default {
     },
 
     calculateDuration (startdate, finishdate) {
-      if (!moment(startdate).millisecond()) {
+      if (moment(startdate).valueOf() < 0) {
         startdate = moment()
       }
-      if (!moment(finishdate).millisecond()) {
+      if (moment(finishdate).valueOf() < 0) {
         finishdate = moment()
       }