diff --git a/deploy-codeinsights-db.sh b/deploy-codeinsights-db.sh new file mode 100755 index 000000000..0effee8d3 --- /dev/null +++ b/deploy-codeinsights-db.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -e + +# Description: TimescaleDB time-series database for code insights data. +# +# Disk: 128GB / persistent SSD +# Network: 1Gbps +# Liveness probe: 5432/TCP +# Ports exposed to other Sourcegraph services: 5432/TCP 9187/TCP +# Ports exposed to the public internet: none +# +VOLUME="$HOME/sourcegraph-docker/codeinsights-db-disk" +./ensure-volume.sh $VOLUME 999 +docker run --detach \ + --name=codeinsights-db \ + --network=sourcegraph \ + --restart=always \ + --cpus=4 \ + --memory=2g \ + -e POSTGRES_PASSWORD=password \ + -e PGDATA=/var/lib/postgresql/data/pgdata \ + -v $VOLUME:/var/lib/postgresql/data/ \ + index.docker.io/sourcegraph/codeinsights-db:insiders@sha256:f985af2fef860cc48be40ded864df025b8794b02b86e66cbc6c55bfe3c418831 + +# Note: You should deploy this as a container, do not try to connect it to your external +# Postgres deployment (TimescaleDB is a bit special and most hosted Postgres deployments +# do not support TimescaleDB, the data here is akin to gitserver's data, where losing it +# would be bad but it can be rebuilt given enough time.) + +echo "Deployed codeinsights-db service" diff --git a/deploy-frontend-internal.sh b/deploy-frontend-internal.sh index 0874f0a77..c45076201 100755 --- a/deploy-frontend-internal.sh +++ b/deploy-frontend-internal.sh @@ -22,6 +22,7 @@ docker run --detach \ -e GOMAXPROCS=4 \ -e PGHOST=pgsql \ -e CODEINTEL_PGHOST=codeintel-db \ + -e CODEINSIGHTS_PGDATASOURCE=postgres://postgres:password@codeinsights-db:5432/postgres \ -e SRC_GIT_SERVERS="$(addresses "gitserver-" $NUM_GITSERVER ":3178")" \ -e SRC_SYNTECT_SERVER=http://syntect-server:9238 \ -e SEARCHER_URL="$(addresses "http://searcher-" $NUM_SEARCHER ":3181")" \ diff --git a/deploy-frontend.sh b/deploy-frontend.sh index c31bdb7d8..09df5f6e4 100755 --- a/deploy-frontend.sh +++ b/deploy-frontend.sh @@ -23,6 +23,7 @@ docker run --detach \ -e JAEGER_AGENT_HOST=jaeger \ -e PGHOST=pgsql \ -e CODEINTEL_PGHOST=codeintel-db \ + -e CODEINSIGHTS_PGDATASOURCE=postgres://postgres:password@codeinsights-db:5432/postgres \ -e SRC_GIT_SERVERS="$(addresses "gitserver-" $NUM_GITSERVER ":3178")" \ -e SRC_SYNTECT_SERVER=http://syntect-server:9238 \ -e SEARCHER_URL="$(addresses "http://searcher-" $NUM_SEARCHER ":3181")" \ diff --git a/deploy.sh b/deploy.sh index 5b7918954..a91729bcc 100755 --- a/deploy.sh +++ b/deploy.sh @@ -14,6 +14,7 @@ for i in $(seq 0 $(($NUM_GITSERVER - 1))); do ./deploy-gitserver.sh $i; done ./deploy-precise-code-intel-worker.sh ./deploy-pgsql.sh ./deploy-codeintel-db.sh +./deploy-codeinsights-db.sh ./deploy-minio.sh ./deploy-prometheus.sh ./deploy-query-runner.sh diff --git a/docker-compose/docker-compose.yaml b/docker-compose/docker-compose.yaml index 2a2c68a59..e084064d1 100644 --- a/docker-compose/docker-compose.yaml +++ b/docker-compose/docker-compose.yaml @@ -78,6 +78,7 @@ services: - JAEGER_AGENT_HOST=jaeger - PGHOST=pgsql - CODEINTEL_PGHOST=codeintel-db + - CODEINSIGHTS_PGDATASOURCE=postgres://postgres:password@codeinsights-db:5432/postgres - 'SRC_GIT_SERVERS=gitserver-0:3178' - 'SRC_SYNTECT_SERVER=http://syntect-server:9238' - 'SEARCHER_URL=http://searcher-0:3181' @@ -117,6 +118,7 @@ services: - GOMAXPROCS=4 - PGHOST=pgsql - CODEINTEL_PGHOST=codeintel-db + - CODEINSIGHTS_PGDATASOURCE=postgres://postgres:password@codeinsights-db:5432/postgres - 'SRC_GIT_SERVERS=gitserver-0:3178' - 'SRC_SYNTECT_SERVER=http://syntect-server:9238' - 'SEARCHER_URL=http://searcher-0:3181' @@ -510,6 +512,30 @@ services: - sourcegraph restart: always + # Description: TimescaleDB time-series database for code insights data. + # + # Disk: 128GB / persistent SSD + # Network: 1Gbps + # Ports exposed to other Sourcegraph services: 5432/TCP 9187/TCP + # Ports exposed to the public internet: none + # + # Note: You should deploy this as a container, do not try to connect it to your external + # Postgres deployment (TimescaleDB is a bit special and most hosted Postgres deployments + # do not support TimescaleDB, the data here is akin to gitserver's data, where losing it + # would be bad but it can be rebuilt given enough time.) + codeinsights-db: + container_name: codeinsights-db + image: "index.docker.io/sourcegraph/codeinsights-db:insiders@sha256:f985af2fef860cc48be40ded864df025b8794b02b86e66cbc6c55bfe3c418831" + cpus: 4 + mem_limit: "2g" + environment: + - POSTGRES_PASSWORD=password + volumes: + - "codeinsights-db:/data/" + networks: + - sourcegraph + restart: always + # Description: MinIO for storing LSIF uploads. # # Disk: 128GB / persistent SSD @@ -577,6 +603,7 @@ volumes: grafana: pgsql: codeintel-db: + codeinsights-db: minio: prometheus-v2: redis-cache: diff --git a/prometheus/prometheus_targets.yml b/prometheus/prometheus_targets.yml index c6bb1b192..77e6712fb 100644 --- a/prometheus/prometheus_targets.yml +++ b/prometheus/prometheus_targets.yml @@ -5,6 +5,7 @@ # Uncomment these if you want to monitor PostgreSQL / Redis via Prometheus. #- pgsql:9187 #- codeintel-db:9187 + #- codeinsights-db:9187 #- redis-cache:9121 #- redis-store:9121 - labels: diff --git a/teardown.sh b/teardown.sh index f857a1cd6..b4c9f95e5 100755 --- a/teardown.sh +++ b/teardown.sh @@ -13,6 +13,7 @@ docker rm -f jaeger &> /dev/null || true docker rm -f precise-code-intel-worker &> /dev/null || true docker rm -f pgsql &> /dev/null || true & docker rm -f codeintel-db &> /dev/null || true & +docker rm -f codeinsights-db &> /dev/null || true & docker rm -f minio &> /dev/null || true & docker rm -f prometheus &> /dev/null || true docker rm -f query-runner &> /dev/null || true & diff --git a/test/smoke-test.sh b/test/smoke-test.sh index b68f7a36e..5a09ce481 100755 --- a/test/smoke-test.sh +++ b/test/smoke-test.sh @@ -10,14 +10,14 @@ deploy_sourcegraph() { if [[ "$GIT_BRANCH" == *"customer-replica"* ]]; then # Expected number of containers on e.g. 3.18-customer-replica branch. - expect_containers="59" + expect_containers="60" else # Expected number of containers on `master` branch. - expect_containers="24" + expect_containers="25" fi elif [[ "$TEST_TYPE" == "docker-compose-test" ]]; then docker-compose --file docker-compose/docker-compose.yaml up -d - expect_containers="22" + expect_containers="23" fi echo "Giving containers 30s to start..." diff --git a/test/volume-config.sh b/test/volume-config.sh index c74691890..3adefab7e 100755 --- a/test/volume-config.sh +++ b/test/volume-config.sh @@ -21,6 +21,6 @@ pushd ~/sourcegraph-docker chown -R 100:101 gitserver* prometheus-v2* repo-updater* searcher* sourcegraph-frontend* symbols* zoekt* minio-disk chown -R 999:1000 redis-store-disk redis-cache-disk chown -R 472:472 grafana-disk -chown -R 999:999 pgsql-disk codeintel-db-disk +chown -R 999:999 pgsql-disk codeintel-db-disk codeinsights-db-disk popd echo "Ready to deploy"