add codeinsights-db deployment (#259)

* pure-docker: add codeinsights-db

* test: add codeinsights-db permissions

Signed-off-by: Stephen Gutekanst <stephen@sourcegraph.com>

* prometheus: add codeinsights-db (optional)

Signed-off-by: Stephen Gutekanst <stephen@sourcegraph.com>

* docker-compose: add codeinsights-db

Signed-off-by: Stephen Gutekanst <stephen@sourcegraph.com>

* test: update expected # of containers

Signed-off-by: Stephen Gutekanst <stephen@sourcegraph.com>

Author: Stephen Gutekanst

deploy-codeinsights-db.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -e
+
+# Description: TimescaleDB time-series database for code insights data.
+#
+# Disk: 128GB / persistent SSD
+# Network: 1Gbps
+# Liveness probe: 5432/TCP
+# Ports exposed to other Sourcegraph services: 5432/TCP 9187/TCP
+# Ports exposed to the public internet: none
+#
+VOLUME="$HOME/sourcegraph-docker/codeinsights-db-disk"
+./ensure-volume.sh $VOLUME 999
+docker run --detach \
+    --name=codeinsights-db \
+    --network=sourcegraph \
+    --restart=always \
+    --cpus=4 \
+    --memory=2g \
+    -e POSTGRES_PASSWORD=password \
+    -e PGDATA=/var/lib/postgresql/data/pgdata \
+    -v $VOLUME:/var/lib/postgresql/data/ \
+    index.docker.io/sourcegraph/codeinsights-db:insiders@sha256:f985af2fef860cc48be40ded864df025b8794b02b86e66cbc6c55bfe3c418831
+
+# Note: You should deploy this as a container, do not try to connect it to your external
+# Postgres deployment (TimescaleDB is a bit special and most hosted Postgres deployments
+# do not support TimescaleDB, the data here is akin to gitserver's data, where losing it
+# would be bad but it can be rebuilt given enough time.)
+
+echo "Deployed codeinsights-db service"

deploy-frontend-internal.sh

@@ -22,6 +22,7 @@ docker run --detach \
     -e GOMAXPROCS=4 \
     -e PGHOST=pgsql \
     -e CODEINTEL_PGHOST=codeintel-db \
+    -e CODEINSIGHTS_PGDATASOURCE=postgres://postgres:password@codeinsights-db:5432/postgres \
     -e SRC_GIT_SERVERS="$(addresses "gitserver-" $NUM_GITSERVER ":3178")" \
     -e SRC_SYNTECT_SERVER=http://syntect-server:9238 \
     -e SEARCHER_URL="$(addresses "http://searcher-" $NUM_SEARCHER ":3181")" \

deploy-frontend.sh

@@ -23,6 +23,7 @@ docker run --detach \
     -e JAEGER_AGENT_HOST=jaeger \
     -e PGHOST=pgsql \
     -e CODEINTEL_PGHOST=codeintel-db \
+    -e CODEINSIGHTS_PGDATASOURCE=postgres://postgres:password@codeinsights-db:5432/postgres \
     -e SRC_GIT_SERVERS="$(addresses "gitserver-" $NUM_GITSERVER ":3178")" \
     -e SRC_SYNTECT_SERVER=http://syntect-server:9238 \
     -e SEARCHER_URL="$(addresses "http://searcher-" $NUM_SEARCHER ":3181")" \

deploy.sh

@@ -14,6 +14,7 @@ for i in $(seq 0 $(($NUM_GITSERVER - 1))); do ./deploy-gitserver.sh $i; done
 ./deploy-precise-code-intel-worker.sh
 ./deploy-pgsql.sh
 ./deploy-codeintel-db.sh
+./deploy-codeinsights-db.sh
 ./deploy-minio.sh
 ./deploy-prometheus.sh
 ./deploy-query-runner.sh

docker-compose/docker-compose.yaml

@@ -78,6 +78,7 @@ services:
       - JAEGER_AGENT_HOST=jaeger
       - PGHOST=pgsql
       - CODEINTEL_PGHOST=codeintel-db
+      - CODEINSIGHTS_PGDATASOURCE=postgres://postgres:password@codeinsights-db:5432/postgres
       - 'SRC_GIT_SERVERS=gitserver-0:3178'
       - 'SRC_SYNTECT_SERVER=http://syntect-server:9238'
       - 'SEARCHER_URL=http://searcher-0:3181'
@@ -117,6 +118,7 @@ services:
       - GOMAXPROCS=4
       - PGHOST=pgsql
       - CODEINTEL_PGHOST=codeintel-db
+      - CODEINSIGHTS_PGDATASOURCE=postgres://postgres:password@codeinsights-db:5432/postgres
       - 'SRC_GIT_SERVERS=gitserver-0:3178'
       - 'SRC_SYNTECT_SERVER=http://syntect-server:9238'
       - 'SEARCHER_URL=http://searcher-0:3181'
@@ -510,6 +512,30 @@ services:
       - sourcegraph
     restart: always
 
+  # Description: TimescaleDB time-series database for code insights data.
+  #
+  # Disk: 128GB / persistent SSD
+  # Network: 1Gbps
+  # Ports exposed to other Sourcegraph services: 5432/TCP 9187/TCP
+  # Ports exposed to the public internet: none
+  # 
+  # Note: You should deploy this as a container, do not try to connect it to your external
+  # Postgres deployment (TimescaleDB is a bit special and most hosted Postgres deployments
+  # do not support TimescaleDB, the data here is akin to gitserver's data, where losing it
+  # would be bad but it can be rebuilt given enough time.)
+  codeinsights-db:
+    container_name: codeinsights-db
+    image: "index.docker.io/sourcegraph/codeinsights-db:insiders@sha256:f985af2fef860cc48be40ded864df025b8794b02b86e66cbc6c55bfe3c418831"
+    cpus: 4
+    mem_limit: "2g"
+    environment:
+      - POSTGRES_PASSWORD=password
+    volumes:
+      - "codeinsights-db:/data/"
+    networks:
+      - sourcegraph
+    restart: always
+
   # Description: MinIO for storing LSIF uploads.
   #
   # Disk: 128GB / persistent SSD
@@ -577,6 +603,7 @@ volumes:
   grafana:
   pgsql:
   codeintel-db:
+  codeinsights-db:
   minio:
   prometheus-v2:
   redis-cache:

prometheus/prometheus_targets.yml

@@ -5,6 +5,7 @@
     # Uncomment these if you want to monitor PostgreSQL / Redis via Prometheus.
     #- pgsql:9187
     #- codeintel-db:9187
+    #- codeinsights-db:9187
     #- redis-cache:9121
     #- redis-store:9121
 - labels:

teardown.sh

@@ -13,6 +13,7 @@ docker rm -f jaeger &> /dev/null || true
 docker rm -f precise-code-intel-worker &> /dev/null || true
 docker rm -f pgsql &> /dev/null || true &
 docker rm -f codeintel-db &> /dev/null || true &
+docker rm -f codeinsights-db &> /dev/null || true &
 docker rm -f minio &> /dev/null || true &
 docker rm -f prometheus &> /dev/null || true
 docker rm -f query-runner &> /dev/null || true &

test/smoke-test.sh

@@ -10,14 +10,14 @@ deploy_sourcegraph() {
 
 		if [[ "$GIT_BRANCH" == *"customer-replica"* ]]; then
 			# Expected number of containers on e.g. 3.18-customer-replica branch.
-			expect_containers="59"
+			expect_containers="60"
 		else
 			# Expected number of containers on `master` branch.
-			expect_containers="24"
+			expect_containers="25"
 		fi
 	elif [[ "$TEST_TYPE" == "docker-compose-test" ]]; then
 		docker-compose --file docker-compose/docker-compose.yaml up -d
-		expect_containers="22"
+		expect_containers="23"
 	fi
 
 	echo "Giving containers 30s to start..."

test/volume-config.sh

@@ -21,6 +21,6 @@ pushd ~/sourcegraph-docker
 chown -R 100:101 gitserver* prometheus-v2* repo-updater* searcher* sourcegraph-frontend* symbols* zoekt* minio-disk
 chown -R 999:1000 redis-store-disk redis-cache-disk
 chown -R 472:472 grafana-disk
-chown -R 999:999 pgsql-disk codeintel-db-disk
+chown -R 999:999 pgsql-disk codeintel-db-disk codeinsights-db-disk
 popd
 echo "Ready to deploy"