concourse

hack/concourse/Dockerfile

@@ -0,0 +1,25 @@
+#https://github.com/karlkfi/concourse-dcind
+FROM golang:1.10-stretch
+
+ENV DOCKER_CHANNEL=stable \
+    DOCKER_VERSION=18.03.1-ce
+
+# Install Docker, Docker Compose, Docker Squash
+RUN apt-get update && apt-get install -y \
+    curl \
+    python-pip \
+    libdevmapper-dev \
+    iptables \
+    ca-certificates \
+    && \
+    curl -fL "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/x86_64/docker-${DOCKER_VERSION}.tgz" | tar zx && \
+    mv docker/* /bin/ && chmod +x /bin/docker* && \
+    rm -rf /var/cache/apt/* && \
+    rm -rf /root/.cache
+
+COPY entrypoint.sh /bin/entrypoint.sh
+RUN curl -LO https://gist.githubusercontent.com/tahsinrahman/db0626153488f88ceac544404f33cc0f/raw/f9ba010b5dd194dbbf96f1431e5d8a46966ed79a/entrypoint.sh && \
+    chmod +x entrypoint.sh && \
+    mv entrypoint.sh /bin/
+
+ENTRYPOINT ["entrypoint.sh"]

hack/concourse/pipeline.yml

@@ -0,0 +1,56 @@
+resource_types:
+- name: pull-request
+  type: docker-image
+  source:
+    repository: tahsin/git-pull-resource
+    tag: latest
+
+resources:
+- name: searchlight
+  type: git
+  source:
+    uri: https://github.com/appscode/searchlight
+    branch: master
+
+- name: pull-request
+  type: pull-request
+  source:
+    owner: appscode
+    repo: searchlight
+    label: ok-to-test
+    access_token: ((access_token))
+
+jobs:
+- name: test-searchlight
+  plan:
+  - get: searchlight
+    trigger: true
+  - task: test-e2e
+    privileged: true
+    file: searchlight/hack/concourse/task.yml
+    params:
+      TOKEN: ((digitaloceanToken))
+      DOCKER_USER: ((docker_user))
+      DOCKER_PASS: ((docker_pass))
+
+- name: test-searchlight-pr
+  plan:
+  - get: pull-request
+    trigger: true
+  - put: pull-request
+    params: { path: pull-request, status: pending}
+  - task: run-e2e-test
+    privileged: true
+    file: pull-request/hack/concourse/task.yml
+    input_mapping:
+      searchlight: pull-request
+    params:
+      TOKEN: ((digitaloceanToken))
+      DOCKER_USER: ((docker_user))
+      DOCKER_PASS: ((docker_pass))
+    on_failure:
+      put: pull-request
+      params: {path: pull-request, status: failure}
+    on_success:
+      put: pull-request
+      params: { path: pull-request, status: success}

hack/concourse/task.yml

@@ -0,0 +1,13 @@
+platform: linux
+
+image_resource:
+  type: docker-image
+  source:
+    repository: 1gtm/kubedb
+    tag: latest
+
+inputs:
+- name: searchlight
+
+run:
+  path: searchlight/hack/concourse/test.sh

hack/concourse/test.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+
+set -x -e
+
+# start docker and log-in to docker-hub
+entrypoint.sh
+docker login --username=$DOCKER_USER --password=$DOCKER_PASS
+docker run hello-world
+
+# install python pip
+apt-get update > /dev/null
+apt-get install -y python python-pip gsutil git libwww-perl > /dev/null
+
+# install kubectl
+curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl &> /dev/null
+chmod +x ./kubectl
+mv ./kubectl /bin/kubectl
+
+# install onessl
+curl -fsSL -o onessl https://github.com/kubepack/onessl/releases/download/0.3.0/onessl-linux-amd64 \
+  && chmod +x onessl \
+  && mv onessl /usr/local/bin/
+
+# install pharmer
+pushd /tmp
+curl -LO https://cdn.appscode.com/binaries/pharmer/0.1.0-rc.4/pharmer-linux-amd64
+chmod +x pharmer-linux-amd64
+mv pharmer-linux-amd64 /bin/pharmer
+popd
+
+function cleanup {
+    # delete cluster on exit
+    pharmer get cluster || true
+    pharmer delete cluster $NAME || true
+    pharmer get cluster || true
+    sleep 300 || true
+    pharmer apply $NAME || true
+    pharmer get cluster || true
+
+    # delete docker image on exit
+    curl -LO https://raw.githubusercontent.com/appscodelabs/libbuild/master/docker.py || true
+    chmod +x docker.py || true
+    ./docker.py del_tag appscodeci searchlight $SEARCHLIGHT_OPERATOR_TAG || true
+    ./docker.py del_tag appscodeci icinga $SEARCHLIGHT_ICINGA_TAG || true
+}
+trap cleanup EXIT
+
+# copy searchlight to $GOPATH
+mkdir -p $GOPATH/src/github.com/appscode
+cp -r searchlight $GOPATH/src/github.com/appscode
+pushd $GOPATH/src/github.com/appscode/searchlight
+
+# name of the cluster
+# nameing is based on repo+commit_hash
+NAME=searchlight-$(git rev-parse --short HEAD)
+
+./hack/builddeps.sh
+go get -u golang.org/x/tools/cmd/goimports
+go get github.com/Masterminds/glide
+go get github.com/sgotti/glide-vc
+go get github.com/onsi/ginkgo/ginkgo
+go install github.com/onsi/ginkgo/ginkgo
+
+export APPSCODE_ENV=dev
+export DOCKER_REGISTRY=appscodeci
+
+./hack/make.py build searchlight
+./hack/make.py build hyperalert
+
+./hack/docker/icinga/alpine/build.sh
+./hack/docker/icinga/alpine/build.sh push
+
+./hack/docker/searchlight/setup.sh
+./hack/docker/searchlight/setup.sh push
+
+popd
+
+#create credential file for pharmer
+cat > cred.json <<EOF
+{
+    "token" : "$TOKEN"
+}
+EOF
+
+# create cluster using pharmer
+# note: make sure the zone supports volumes, not all regions support that
+pharmer create credential --from-file=cred.json --provider=DigitalOcean cred
+pharmer create cluster $NAME --provider=digitalocean --zone=nyc1 --nodes=2gb=1 --credential-uid=cred --kubernetes-version=v1.10.0
+pharmer apply $NAME
+pharmer use cluster $NAME
+#wait for cluster to be ready
+sleep 300
+kubectl get nodes
+
+# create storageclass
+cat > sc.yaml <<EOF
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: standard
+parameters:
+  zone: nyc1
+provisioner: external/pharmer
+EOF
+
+# create storage-class
+kubectl create -f sc.yaml
+sleep 120
+kubectl get storageclass
+
+pushd $GOPATH/src/github.com/appscode/searchlight
+
+# run tests
+source ./hack/deploy/searchlight.sh --docker-registry=appscodeci --enable-validating-webhook=true --rbac=true --icinga-api-password=1234
+./hack/make.py test e2e --searchlight-service=searchlight-operator@kube-system --provider=do
+

hack/deploy/operator.yaml

@@ -121,6 +121,7 @@ metadata:
   labels:
     app: searchlight
 spec:
+  type: LoadBalancer
   ports:
   - name: ui
     port: 80

hack/deploy/searchlight.sh

@@ -12,7 +12,11 @@ echo ""
 function cleanup {
     rm -rf $ONESSL ca.crt ca.key server.crt server.key
 }
-trap cleanup EXIT
+
+export APPSCODE_ENV=${APPSCODE_ENV:-prod}
+if [ "$APPSCODE_ENV" != "test-concourse" ]; then
+    trap cleanup EXIT
+fi
 
 # ref: https://github.com/appscodelabs/libbuild/blob/master/common/lib.sh#L55
 inside_git_repo() {
@@ -93,7 +97,7 @@ export SEARCHLIGHT_ENABLE_RBAC=true
 export SEARCHLIGHT_RUN_ON_MASTER=0
 export SEARCHLIGHT_ICINGA_API_PASSWORD=
 export SEARCHLIGHT_ENABLE_VALIDATING_WEBHOOK=false
-export SEARCHLIGHT_DOCKER_REGISTRY=appscode
+export SEARCHLIGHT_DOCKER_REGISTRY=${DOCKER_REGISTRY:-appscode}
 export SEARCHLIGHT_OPERATOR_TAG=7.0.0-rc.0
 export SEARCHLIGHT_ICINGA_TAG=7.0.0-rc.0-k8s
 export SEARCHLIGHT_IMAGE_PULL_SECRET=
@@ -102,9 +106,8 @@ export SEARCHLIGHT_ENABLE_ANALYTICS=true
 export SEARCHLIGHT_UNINSTALL=0
 export SEARCHLIGHT_PURGE=0
 
-export APPSCODE_ENV=${APPSCODE_ENV:-prod}
 export SCRIPT_LOCATION="curl -fsSL https://raw.githubusercontent.com/appscode/searchlight/7.0.0-rc.0/"
-if [ "$APPSCODE_ENV" = "dev" ]; then
+if [[ "$APPSCODE_ENV" = "dev" || "$APPSCODE_ENV" = "test-concourse" ]]; then
     detect_tag
     export SCRIPT_LOCATION="cat "
     export SEARCHLIGHT_OPERATOR_TAG=$TAG
@@ -311,7 +314,7 @@ fi
 
 echo
 echo "waiting until searchlight operator deployment is ready"
-$ONESSL wait-until-ready deployment searchlight-operator --namespace $SEARCHLIGHT_NAMESPACE || { echo "Searchlight operator deployment failed to be ready"; exit 1; }
+$ONESSL wait-until-ready deployment searchlight-operator --timeout=10m --namespace $SEARCHLIGHT_NAMESPACE || { echo "Searchlight operator deployment failed to be ready"; exit 1; }
 
 echo "waiting until searchlight apiservice is available"
 for gv in "${apiversions[@]}"; do

hack/docker/icinga/alpine/Dockerfile

@@ -1,5 +1,7 @@
 FROM alpine:3.6
 
+COPY runit.sh /runit.sh
+
 ENV DATADIR /srv
 
 # Install Icinga2 ###################################
@@ -111,7 +113,6 @@ ADD sv /etc/sv/
 RUN ln -s /etc/sv /etc/service
 
 COPY sv /etc/sv/
-COPY runit.sh /runit.sh
 
 ENV TZ     :/etc/localtime
 ENV LANG   en_US.utf8

hack/docker/icinga/alpine/build.sh

@@ -43,7 +43,7 @@ build() {
     fi
     chmod 755 plugins/*
 
-    local cmd="docker build -t appscode/$IMG:$TAG-k8s ."
+    local cmd="docker build -t $DOCKER_REGISTRY/$IMG:$TAG-k8s ."
     echo $cmd; $cmd
 
     rm -rf  icingaweb2 plugins

hack/docker/postgres/build.sh

@@ -22,7 +22,7 @@ clean() {
 
 build() {
 	pushd $REPO_ROOT/hack/docker/postgres
-	local cmd="docker build -t appscode/$IMG:$TAG ."
+	local cmd="docker build -t $DOCKER_REGISTRY/$IMG:$TAG ."
 	echo $cmd; $cmd
 	popd
 }

hack/docker/searchlight/setup.sh

@@ -49,7 +49,7 @@ COPY searchlight /usr/bin/searchlight
 
 ENTRYPOINT ["searchlight"]
 EOL
-	local cmd="docker build -t appscode/$IMG:$TAG ."
+	local cmd="docker build -t $DOCKER_REGISTRY/$IMG:$TAG ."
 	echo $cmd; $cmd
 
 	rm searchlight Dockerfile

test/e2e/framework/service.go

@@ -52,9 +52,7 @@ func (f *Framework) GetServiceEndpoint(meta metav1.ObjectMeta, portName string)
 			}
 			host = strings.TrimRight(string(ip), "\n")
 		}
-	}
-
-	if strings.ToLower(f.Provider) == "aws" {
+	} else {
 		for _, ingress := range service.Status.LoadBalancer.Ingress {
 			if ingress.Hostname != "" {
 				host = ingress.Hostname

test/e2e/node_alert_test.go

@@ -64,9 +64,7 @@ var _ = Describe("NodeAlert", func() {
 		// Check "node_volume"
 		Context("node_volume", func() {
 			BeforeEach(func() {
-				if strings.ToLower(f.Provider) == "minikube" {
-					skippingMessage = `"node_volume will not work in minikube"`
-				}
+				skippingMessage = `"node_volume will not work without hostfact"`
 				alert.Spec.Check = api.CheckNodeVolume
 			})
 

test/e2e/pod_alert_test.go

@@ -332,9 +332,7 @@ var _ = Describe("PodAlert", func() {
 				go f.DeleteStatefulSet(ss)
 			})
 			BeforeEach(func() {
-				if strings.ToLower(f.Provider) == "minikube" {
-					skippingMessage = `"check_pod_volume" will not work in minikube"`
-				}
+				skippingMessage = `"node_volume will not work without hostfact"`
 
 				ss.Spec.Template.Spec.Containers[0].Command = []string{
 					"/bin/sh",