Dockerize processor & query-node (#146)

* Update Dockerfile for building image for processor/query-node

* Compose docker images for processor & query-node

* Delete old dockerfiles

* Add commands for processor & query-node

* Update host in env files

* Run processor as a container

* Keep containers out of the network

* Publish ports while starting the container

* Create script for query-node

* Add command for stopping processor

* Update args received by scripts

* Assign container_name for processor services

* Reorder package scripts alphabetically

* Enhance start command for query-node

* Rename container names to avoid conflicts

* Replace npm with yarn command

.env.dev

@@ -6,11 +6,12 @@ PROJECT_NAME=zeitgeist-squid
 WS_NODE_URL=wss://bsr.zeitgeist.pm
 
 # DB config
+DB_HOST=host.docker.internal
 DB_NAME=squid
 DB_PORT=23798
 
 # Redis config
-REDIS_HOST=localhost
+REDIS_HOST=host.docker.internal
 REDIS_PASS=redis
 
 ###########################

.env.local

@@ -24,7 +24,7 @@ IPFS_CLIENT_URL=http://127.0.0.1:5001
 INDEXER_ENDPOINT_URL=http://localhost:4010/v1/graphql
 
 # Prometheus metrics to monitor synchronization status
-PROCESSOR_PROMETHEUS_PORT=9091
+PROCESSOR_PROMETHEUS_PORT=9090
 
 ###############################
 #    Processor GraphQL API    #

Dockerfile

@@ -4,29 +4,34 @@ FROM node AS node-with-gyp
 RUN apk add g++ make python3
 
 FROM node-with-gyp AS builder
-WORKDIR /squid
+WORKDIR /home/zeitgeist-squid
 ADD package.json .
-ADD package-lock.json .
-RUN npm ci
+ADD yarn.lock .
+RUN yarn install --frozen-lockfile
 ADD tsconfig.json .
 ADD src src
-RUN npm run build
+RUN yarn build
 
 FROM node-with-gyp AS deps
-WORKDIR /squid
+WORKDIR /home/zeitgeist-squid
 ADD package.json .
-ADD package-lock.json .
-RUN npm ci --production
+ADD yarn.lock .
+RUN yarn install --frozen-lockfile
 
 FROM node AS squid
-WORKDIR /squid
-COPY --from=deps /squid/package.json .
-COPY --from=deps /squid/package-lock.json .
-COPY --from=deps /squid/node_modules node_modules
-COPY --from=builder /squid/lib lib
+WORKDIR /home/zeitgeist-squid
+COPY --from=deps /home/zeitgeist-squid/package.json .
+COPY --from=deps /home/zeitgeist-squid/yarn.lock .
+COPY --from=deps /home/zeitgeist-squid/node_modules node_modules
+COPY --from=builder /home/zeitgeist-squid/lib lib
 ADD db db
 ADD schema.graphql .
-# TODO: use shorter PROMETHEUS_PORT
-ENV PROCESSOR_PROMETHEUS_PORT 3000
-EXPOSE 3000
-EXPOSE 4000
+ADD zeitgeist.json .
+
+
+FROM squid AS processor
+CMD ["node", "lib/processor/index.js"]
+
+
+FROM squid AS query-node
+CMD ["yarn", "squid-graphql-server"]

docker-compose.yml

@@ -3,6 +3,7 @@ version: "3.4"
 services:
   db:
     image: postgres:12
+    container_name: processor-db
     environment:
       POSTGRES_DB: postgres
       POSTGRES_PASSWORD: postgres
@@ -12,6 +13,7 @@ services:
 
   redis:
     image: redis:6.0-alpine
+    container_name: processor-redis
     restart: always
     ports:
       - "6379:6379"

indexer/docker-compose.local.yml

@@ -29,7 +29,7 @@ services:
     command: >
       sh -c "yarn db:bootstrap && yarn start:prod"
     ports:
-      - 9090:9090
+      - 9091:9090
 
   indexer-gateway:
     restart: unless-stopped
@@ -56,7 +56,8 @@ services:
       - "6379"
 
   node:
-    image: zeitgeistpm/zeitgeist-node:v0.3.1
+    image: zeitgeistpm/zeitgeist-node:latest
+    container_name: zeitgeist-node
     ports:
       - "9944:9944"
     command: --dev --ws-external --pruning archive

indexer/docker-compose.mainnet.yml

@@ -36,7 +36,6 @@ services:
     ports:
       - "4010:8080"
     environment:
-      - DEV_MODE=true
       - DB_NAME=indexer
       - DB_HOST=db
       - DB_USER=postgres

indexer/docker-compose.testnet.yml

@@ -36,7 +36,6 @@ services:
     ports:
       - "4010:8080"
     environment:
-      - DEV_MODE=true
       - DB_NAME=indexer
       - DB_HOST=db
       - DB_USER=postgres

indexer/docker-compose.yml

@@ -2,15 +2,18 @@ version: "3.4"
 services:
   db:
     image: postgres:12
+    container_name: indexer-db
 
   indexer:
     image: subsquid/hydra-indexer:5.0.0-alpha.29
+    container_name: zeitgeist-indexer
     depends_on:
       - db
       - redis
 
   indexer-gateway:
     image: subsquid/hydra-indexer-gateway:5
+    container_name: indexer-gateway
     depends_on:
       - redis
       - db
@@ -19,8 +22,10 @@ services:
 
   indexer-status-service:
     image: subsquid/hydra-indexer-status-service:5
+    container_name: indexer-status-service
     depends_on:
       - redis
 
   redis:
-    image: redis:6.0-alpine
+    image: redis:6.0-alpine
+    container_name: indexer-redis

package.json

@@ -8,21 +8,24 @@
   "scripts": {
     "build": "rm -rf lib && tsc",
     "codegen": "npx sqd codegen",
-    "db:up": "docker-compose up -d db",
     "db:create": "npx sqd db create",
-    "db:drop": "npx sqd db drop",
     "db:create-migration": "rm -rf db/migrations/* && yarn db:reset && npx sqd db create-migration init && yarn db:migrate",
+    "db:drop": "npx sqd db drop",
     "db:migrate": "yarn build && npx sqd db migrate",
-    "db:revert": "npx sqd db revert",
     "db:reset": "yarn db:drop && yarn db:create",
-    "indexer:start": "./scripts/indexer.sh --local",
+    "db:revert": "npx sqd db revert",
+    "db:up": "docker-compose up -d db",
+    "indexer:start": "./scripts/indexer.sh local up",
+    "indexer:stop": "./scripts/indexer.sh local down",
+    "processor:dev:resume": "./scripts/processor.sh dev resume",
+    "processor:dev:start": "./scripts/processor.sh dev start",
     "processor:migrate": "echo skip",
-    "processor:dev:start": "./scripts/processor.sh --start --dev",
-    "processor:dev:resume": "./scripts/processor.sh --resume --dev",
-    "processor:local:start": "./scripts/processor.sh --start --local",
-    "processor:local:resume": "./scripts/processor.sh --resume --local",
-    "query-node:start": "squid-graphql-server",
+    "processor:local:resume": "./scripts/processor.sh local resume",
+    "processor:local:start": "./scripts/processor.sh local start",
+    "processor:stop": "./scripts/processor.sh stop",
     "redis:up": "docker-compose up -d redis",
+    "squid:start": "yarn indexer:start && yarn processor:local:start",
+    "squid:stop": "yarn indexer:stop && docker-compose down && docker stop zeitgeist-query-node",
     "typegen": "squid-substrate-typegen typegen.json"
   },
   "dependencies": {

scripts/indexer.sh

@@ -1,14 +1,21 @@
 #!/bin/sh
-cd indexer && docker-compose down --remove-orphans
+cd indexer
+
+# Control points of indexer
+if [ "$2" = "up" ]; then
+  cmd="up -d"
+else
+  cmd=$2
+fi
 
 # Index data from local-network or battery-station or main-net by passing below argument
 # Details are specified in docker-compose*.yml files in indexer directory
-if [ "$1" = "--local" ]; then 
-    docker-compose -f ./docker-compose.yml -f ./docker-compose.local.yml up -d
-elif [ "$1" = "--testnet" ]; then
-    docker-compose -f ./docker-compose.yml -f ./docker-compose.testnet.yml up -d
-elif [ "$1" = "--mainnet" ]; then
-    docker-compose -f ./docker-compose.yml -f ./docker-compose.mainnet.yml up -d
+if [ "$1" = "local" ]; then 
+  docker-compose -f ./docker-compose.yml -f ./docker-compose.local.yml $cmd
+elif [ "$1" = "testnet" ]; then
+  docker-compose -f ./docker-compose.yml -f ./docker-compose.testnet.yml $cmd
+elif [ "$1" = "mainnet" ]; then
+  docker-compose -f ./docker-compose.yml -f ./docker-compose.mainnet.yml $cmd
 else
-    echo "Please specify --local or --testnet or --mainnet"
+  echo "First argument should be `local` or `testnet` or `mainnet`"
 fi

scripts/processor.sh

@@ -1,30 +1,38 @@
 #!/bin/sh
 
 # Control points of processor
-if [ "$1" = "--start" ]; then
-    yarn install --frozen-lockfile
-    echo "Starting processor"
-    yarn db:up && yarn redis:up && yarn db:reset && yarn db:migrate
-elif [ "$1" = "--resume" ]; then
-    echo "Resuming processor"
+if [ "$1" = "stop" ]; then
+  echo "Stopping processor..."
+  docker stop zeitgeist-query-node
+  docker stop zeitgeist-processor
+  exit
+fi
+
+if [ "$2" = "start" ]; then
+  echo "Building processor..."
+  docker build . --target processor -t processor
+  docker build . --target query-node -t query-node
+  echo "Starting processor..."
+  yarn db:up && yarn redis:up && yarn db:reset && yarn db:migrate
+elif [ "$2" = "resume" ]; then
+  echo "Resuming processor..."
 else
-    echo "Please specify first argument with --start or --resume"
-    exit
+  echo "Second argument should be either `start` or `resume`"
+  exit
 fi
 
 # Process data from local-network or battery-station or main-net by passing below argument
-if [ "$2" = "--local" ]; then
-    yarn build && NODE_ENV=local node lib/processor/index.js
-elif [ "$2" = "--dev" ]; then
-    yarn build && NODE_ENV=dev node lib/processor/index.js
-elif [ "$2" = "--t1" ]; then
-    yarn build && NODE_ENV=t1 node lib/processor/index.js
-elif [ "$2" = "--t2" ]; then
-    yarn build && NODE_ENV=t2 node lib/processor/index.js
-elif [ "$2" = "--m1" ]; then
-    yarn build && NODE_ENV=m1 node lib/processor/index.js
-elif [ "$2" = "--m2" ]; then
-    yarn build && NODE_ENV=m2 node lib/processor/index.js
+# For Linux, add --network=host
+if [ "$1" = "dev" ]; then
+  docker run -d -p 9090:9090 --rm -e NODE_ENV=dev --env-file=.env.dev --name=zeitgeist-processor processor
+  docker run -d -p 4350:4350 --rm -e NODE_ENV=dev --env-file=.env.dev --name zeitgeist-query-node query-node
+elif [ "$1" = "local" ]; then
+  sleep 10 # Wait for local node to be up
+  docker run -d --network=host --rm -e NODE_ENV=local --env-file=.env.local --name=zeitgeist-processor processor
+  docker run -d --network=host --rm -e NODE_ENV=local --env-file=.env.local --name=zeitgeist-query-node query-node 
+elif [ "$1" = "t1" ] || [ "$1" = "t2" ] || [ "$1" = "m1" ] || [ "$1" = "m2" ]; then
+  docker run -d --network=host --rm -e NODE_ENV=$1 --env-file=.env.$1 --name=zeitgeist-processor processor
+  docker run -d --network=host --rm -e NODE_ENV=$1 --env-file=.env.$1 --name=zeitgeist-query-node query-node  
 else
-    echo "Please specify second argument with --local or --dev"
+  echo "First argument should be either `local` or `dev`"
 fi