added building from source for db-sync (#548)

* added building from source for db-sync

* removed blinklabs base image

* removed blinkenlabs

* chore: updated version in package.json, fixed return of bootstrapPeers for p2p

* chore: eslint changes

* chore:Updated test relay according to the config

Dockerfile

@@ -1,7 +1,97 @@
 ARG UBUNTU_VERSION=22.04
-FROM ghcr.io/blinklabs-io/haskell:9.6.3-3.10.2.0-1 AS cardano-node-build
-# Install cardano-node
+FROM ubuntu:${UBUNTU_VERSION} AS cardano-builder
+# Following: https://github.com/input-output-hk/cardano-node-wiki/blob/main/docs/getting-started/install.md
+ENV DEBIAN_FRONTEND=nonintercative
+ARG CABAL_VERSION=3.10.1.0
+ARG GHC_VERSION=8.10.7
+ARG IOHK_LIBSODIUM_GIT_REV=dbb48cce
+ARG SECP256K1_VERSION=v0.3.2
+ARG BLST_VERSION=v0.3.11
+
 ARG NODE_VERSION=8.7.3
+ARG CARDANO_DB_SYNC_VERSION=13.2.0.1
+
+RUN mkdir -p /app/src
+WORKDIR /app
+RUN apt-get update -y && apt-get install -y \
+  automake \
+  autoconf \
+  build-essential \
+  g++ \
+  git \
+  jq \
+  libffi-dev \
+  libghc-postgresql-libpq-dev \
+  libgmp-dev \
+  liblmdb-dev \
+  libnuma-dev \
+  libncursesw5 \
+  libpq-dev \
+  libssl-dev \
+  libsystemd-dev \
+  libtinfo-dev \
+  llvm-dev \
+  libtool \
+  make \
+  pkg-config \
+  tmux \
+  wget \
+  zlib1g-dev
+
+WORKDIR /app/src
+# Cabal
+ENV CABAL_VERSION=${CABAL_VERSION}
+ENV PATH="/root/.cabal/bin:/root/.ghcup/bin:/root/.local/bin:$PATH"
+RUN wget https://downloads.haskell.org/~cabal/cabal-install-${CABAL_VERSION}/cabal-install-${CABAL_VERSION}-$(uname -m)-linux-ubuntu20_04.tar.xz \
+    && tar -xf cabal-install-${CABAL_VERSION}-$(uname -m)-linux-ubuntu20_04.tar.xz \
+    && rm cabal-install-${CABAL_VERSION}-$(uname -m)-linux-ubuntu20_04.tar.xz \
+    && mkdir -p ~/.local/bin \
+    && mv cabal ~/.local/bin/ \
+    && cabal update && cabal --version
+
+# GHC
+ENV GHC_VERSION=${GHC_VERSION}
+RUN wget https://downloads.haskell.org/~ghc/${GHC_VERSION}/ghc-${GHC_VERSION}-$(uname -m)-deb10-linux.tar.xz \
+    && tar -xf ghc-${GHC_VERSION}-$(uname -m)-deb10-linux.tar.xz \
+    && rm ghc-${GHC_VERSION}-$(uname -m)-deb10-linux.tar.xz \
+    && cd ghc-${GHC_VERSION} \
+    && ./configure \
+    && make install
+
+
+# Libsodium
+RUN git clone https://github.com/intersectmbo/libsodium && \
+    cd libsodium && \
+    git checkout ${LIBSODIUM_REF} && \
+    ./autogen.sh && \
+    ./configure && \
+    make && \
+    make install
+ENV LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH"
+ENV PKG_CONFIG_PATH="/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH"
+
+# secp256k1
+RUN git clone https://github.com/bitcoin-core/secp256k1 && \
+    cd secp256k1 && \
+    git checkout ${SECP256K1_REF} && \
+    ./autogen.sh && \
+    ./configure --enable-module-schnorrsig --enable-experimental && \
+    make && \
+    make install
+
+# BLST
+COPY ./scripts/libblst.pc /usr/local/lib/pkgconfig/
+RUN git clone https://github.com/supranational/blst && \
+    cd blst && \
+    git checkout ${BLST_REF} && \
+    ./build.sh && \
+    cp bindings/blst_aux.h bindings/blst.h bindings/blst.hpp  /usr/local/include/ && \
+    cp libblst.a /usr/local/lib/ && \
+    chmod u=rw,go=r /usr/local/lib/pkgconfig/libblst.pc \
+      /usr/local/include/blst_aux.h /usr/local/include/blst.h /usr/local/include/blst.hpp \
+      /usr/local/lib/libblst.a
+
+# Install cardano-node
 ENV NODE_VERSION=${NODE_VERSION}
 RUN echo "Building tags/${NODE_VERSION}..." \
     && echo tags/${NODE_VERSION} > /CARDANO_BRANCH \
@@ -14,20 +104,27 @@ RUN echo "Building tags/${NODE_VERSION}..." \
     && echo "tests: False" >> cabal.project.local \
     && cabal update \
     && cabal build all \
+    && cabal build cardano-cli
+RUN cd cardano-node \
     && mkdir -p /root/.local/bin/ \
-    && cp -p "$(./scripts/bin-path.sh cardano-node)" /root/.local/bin/
+    && cp -p "$(./scripts/bin-path.sh cardano-node)" /root/.local/bin/ \
+    && cp -p "$(./scripts/bin-path.sh cardano-cli)" /root/.local/bin/
 
+RUN export PATH="/root/.local/bin:$PATH"
 
-ARG CARDANO_DB_SYNC_VERSION=13.2.0.1
 ARG DB_SYNC_TAG=${CARDANO_DB_SYNC_VERSION}
 WORKDIR /app/src
-RUN git clone https://github.com/input-output-hk/cardano-db-sync.git &&\
+RUN git clone https://github.com/intersectmbo/cardano-db-sync.git &&\
   cd cardano-db-sync &&\
   git fetch --all --tags &&\
-  git checkout ${DB_SYNC_TAG}
-RUN mkdir binaries && cd binaries && wget https://github.com/IntersectMBO/cardano-db-sync/releases/download/${DB_SYNC_TAG}/cardano-db-sync-${CARDANO_DB_SYNC_VERSION}-linux.tar.gz && \
-    tar -xvf cardano-db-sync-${CARDANO_DB_SYNC_VERSION}-linux.tar.gz && \
-    cp -r * /usr/local/bin/
+  git checkout tags/${DB_SYNC_TAG}
+RUN apt-get update -y && apt-get install -y libpq-dev
+RUN cd cardano-db-sync &&\
+  echo "with-compiler: ghc-8.10.7" >> cabal.project.local &&\
+  cabal update && \
+  cabal configure --with-compiler=ghc-${GHC_VERSION} &&\
+  cabal build cardano-db-sync
+RUN cp -p "$(find . -name cardano-db-sync -executable -type f)" /root/.local/bin/
 
 FROM ubuntu:${UBUNTU_VERSION} as ubuntu-nodejs
 ARG NODEJS_MAJOR_VERSION=18
@@ -56,11 +153,10 @@ RUN curl --proto '=https' --tlsv1.2 -sSf -L https://www.postgresql.org/media/key
   postgresql-12 \
   postgresql-client-12 &&\
   npm install pm2 -g
-COPY --from=cardano-node-build /usr/local/lib /usr/local/lib
-COPY --from=cardano-node-build /usr/local/include/ /usr/local/include/
-COPY --from=cardano-node-build /root/.local/bin/cardano-* /usr/local/bin/
-COPY --from=cardano-node-build /usr/local/bin/cardano-db-sync /usr/local/bin/
-COPY --from=cardano-node-build /app/src/cardano-db-sync/schema /cardano-db-sync/schema
+COPY --from=cardano-builder /usr/local/lib /usr/local/lib
+COPY --from=cardano-builder /usr/local/include/ /usr/local/include/
+COPY --from=cardano-builder /root/.local/bin/cardano-* /usr/local/bin/
+COPY --from=cardano-builder /app/src/cardano-db-sync/schema /cardano-db-sync/schema
 # Configure dynamic linker
 RUN ldconfig
 # easy step-down from root
@@ -110,18 +206,17 @@ RUN yarn --offline --frozen-lockfile --non-interactive --production
 
 FROM ubuntu-nodejs as cardano-rosetta-server
 ARG NETWORK=mainnet
-COPY --from=haskell-builder /usr/local/bin/cardano-cli \
-  /usr/local/bin/cardano-node \
+COPY --from=cardano-builder /root/.local/bin/cardano-cli \
+  /root/.local/bin/cardano-node \
   /usr/local/bin/
 COPY --from=rosetta-server-builder /app/dist /cardano-rosetta-server/dist
 COPY --from=rosetta-server-production-deps /app/node_modules /cardano-rosetta-server/node_modules
 COPY config/network/${NETWORK} /config/
 EXPOSE 8080
 CMD ["node", "/cardano-rosetta-server/dist/src/server/index.js"]
 
-FROM ghcr.io/blinklabs-io/cardano-cli:8.17.0.0 AS cardano-cli
 FROM runtime-base
-COPY --from=cardano-cli /usr/local/bin/cardano-cli /usr/local/bin/
+COPY --from=cardano-builder /root/.local/bin/cardano-cli /usr/local/bin/
 ARG NETWORK=mainnet
 ARG SNAPSHOT_URL
 ENV DEFAULT_RELATIVE_TTL=1000 LOGGER_MIN_SEVERITY=info PAGE_SIZE=25 DEFAULT_POOL_DEPOSIT=500000000 DEFAULT_KEY_DEPOSIT=2000000
@@ -137,4 +232,4 @@ RUN echo "/var/run/postgresql:5432:cexplorer:*:*" > $PGPASSFILE &&\
  chmod 600 $PGPASSFILE && chown postgres:postgres $PGPASSFILE
 RUN mkdir /snapshot &&\
   ./scripts/maybe_download_cardano-db-sync_snapshot.sh $SNAPSHOT_URL /snapshot
-COPY scripts/entrypoint.sh .
+COPY scripts/entrypoint.sh .

README.md

@@ -22,7 +22,7 @@ DOCKER_BUILDKIT=1 \
 docker build \
   --build-arg BUILDKIT_INLINE_CACHE=1 \
   --cache-from=cardanofoundation/cardano-rosetta:master \
-  -t cardanofoundation/cardano-rosetta:2.2.0 \
+  -t cardanofoundation/cardano-rosetta:2.2.1 \
   .
 ```
 
@@ -37,7 +37,7 @@ docker build \
   --build-arg BUILDKIT_INLINE_CACHE=1 \
   --build-arg NETWORK=preprod \
   --cache-from=cardanofoundation/cardano-rosetta:master \
-  -t cardanofoundation/cardano-rosetta:2.2.0-preprod \
+  -t cardanofoundation/cardano-rosetta:2.2.1-preprod \
   .
 ```
 
@@ -52,7 +52,7 @@ docker build \
   --build-arg BUILDKIT_INLINE_CACHE=1 \
   --build-arg NETWORK=preview \
   --cache-from=cardanofoundation/cardano-rosetta:master \
-  -t cardanofoundation/cardano-rosetta:2.2.0-preview \
+  -t cardanofoundation/cardano-rosetta:2.2.1-preview \
   .
 ```
 
@@ -68,7 +68,7 @@ docker build \
   --build-arg CARDANO_DB_SYNC_VERSION=13.2.0.0 \
   --build-arg DB_SYNC_TAG=sanch-3-0-0
   --cache-from=cardanofoundation/cardano-rosetta:master \
-  -t cardanofoundation/cardano-rosetta:2.2.0-sanchonet \
+  -t cardanofoundation/cardano-rosetta:2.2.1-sanchonet \
   .
 ```
 
@@ -88,7 +88,7 @@ docker run \
   -p 8080:8080 \
   -v cardano-rosetta:/data \
   --shm-size=2g \
-  cardanofoundation/cardano-rosetta:2.2.0
+  cardanofoundation/cardano-rosetta:2.2.1
 ```
 
 </details>
@@ -102,7 +102,7 @@ docker run \
   -p 8081:8080 \
   -v cardano-rosetta-preprod:/data \
   --shm-size=2g \
-  cardanofoundation/cardano-rosetta:2.2.0-preprod
+  cardanofoundation/cardano-rosetta:2.2.1-preprod
 ```
 
 </details>
@@ -116,7 +116,7 @@ docker run \
   -p 8081:8080 \
   -v cardano-rosetta-preview:/data \
   --shm-size=2g \
-  cardanofoundation/cardano-rosetta:2.2.0-preview
+  cardanofoundation/cardano-rosetta:2.2.1-preview
 ```
 
 </details>
@@ -130,7 +130,7 @@ docker run \
   -p 8081:8080 \
   -v cardano-rosetta-preview:/data \
   --shm-size=2g \
-  cardanofoundation/cardano-rosetta:2.2.0-sanchonet
+  cardanofoundation/cardano-rosetta:2.2.1-sanchonet
 ```
 
 </details>
@@ -185,8 +185,8 @@ docker build \
   --build-arg BUILDKIT_INLINE_CACHE=1 \
   --build-arg SNAPSHOT_URL=https://update-cardano-mainnet.iohk.io/cardano-db-sync/13/db-sync-snapshot-schema-13-block-7960123-x86_64.tgz \
   --cache-from=cardanofoundation/cardano-rosetta:master \
-  -t cardanofoundation/cardano-rosetta:2.2.0-apply-snapshot \
-  https://github.com/cardanofoundation/cardano-rosetta.git#2.2.0
+  -t cardanofoundation/cardano-rosetta:2.2.1-apply-snapshot \
+  https://github.com/cardanofoundation/cardano-rosetta.git#2.2.1
 ```
 
 </details>
@@ -202,7 +202,7 @@ docker run \
   -p 8080:8080 \
   -v cardano-rosetta:/data \
   --shm-size=2g \
-  cardanofoundation/cardano-rosetta:2.2.0-apply-snapshot
+  cardanofoundation/cardano-rosetta:2.2.1-apply-snapshot
 ```
 
 </details>

cardano-rosetta-server/docker-compose.yml

@@ -14,15 +14,17 @@ services:
       - postgres_user
     shm_size: '2gb'
     volumes:
-      - postgres-data:/var/lib/postgresql/data
+      - ../postgres-data:/var/lib/postgresql/data
     restart: on-failure
     logging:
       driver: "json-file"
       options:
         max-size: "200k"
         max-file: "10"
+    ports:
+      - 5432:5432
   cardano-node:
-    image: inputoutput/cardano-node:${CARDANO_NODE_VERSION:-8.7.3}
+    image: ghcr.io/intersectmbo/cardano-node:${CARDANO_NODE_VERSION:-8.7.3}
     environment:
       - NETWORK=${NETWORK:-mainnet}
     volumes:
@@ -34,7 +36,7 @@ services:
         max-size: "400k"
         max-file: "20"
   cardano-db-sync:
-    image: inputoutput/cardano-db-sync:${CARDANO_DB_SYNC_VERSION:-13.2.0.1}
+    image: ghcr.io/intersectmbo/cardano-db-sync:${CARDANO_DB_SYNC_VERSION:-13.2.0.1}
     command: [
       "--config", "/config/cardano-db-sync/config.json",
       "--socket-path", "/node-ipc/node.socket"
@@ -61,6 +63,7 @@ services:
       options:
         max-size: "200k"
         max-file: "10"
+    network_mode: host
 #  cardano-rosetta-server:
 #    build:
 #      cache_from:

cardano-rosetta-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cardano-rosetta-server",
-  "version": "2.2.0",
+  "version": "2.2.1",
   "description": "Rosetta API server specification implementation",
   "main": "index.js",
   "scripts": {

cardano-rosetta-server/src/server/services/network-service.ts

@@ -26,9 +26,16 @@ interface PublicRoot {
     accessPoints: AccessPoint[];
   };
 }
+
+interface BootstrapPeer {
+  address: string;
+  port: number;
+}
+
 export interface TopologyConfig {
   Producers?: Producer[];
   PublicRoots?: PublicRoot[];
+  bootstrapPeers?: BootstrapPeer[];
 }
 
 export interface Peer {
@@ -50,9 +57,15 @@ export interface NetworkService {
 const getPublicRoots = (publicRoots?: PublicRoot[]) =>
   publicRoots?.map(pr => pr.publicRoots.accessPoints.map(ap => ({ addr: ap.address }))).flat() || [];
 
+const getBootstrapPeers = (bootstrapPeers?: BootstrapPeer[]) =>
+  bootstrapPeers?.map(bp => ({ addr: bp.address })).flat() || [];
+
 const getPeersFromConfig = (logger: Logger, topologyFile: TopologyConfig): Peer[] => {
   logger.info('[getPeersFromConfig] Looking for peers from topologyFile');
-  const Producers = topologyFile?.Producers || getPublicRoots(topologyFile.PublicRoots);
+  const Producers =
+    topologyFile?.Producers ||
+    getBootstrapPeers(topologyFile.bootstrapPeers) ||
+    getPublicRoots(topologyFile.PublicRoots);
   logger.debug(`[getPeersFromConfig] Found ${Producers.length} peers`);
   return Producers as Peer[];
 };

cardano-rosetta-server/src/server/utils/errors.ts

@@ -266,7 +266,7 @@ export const ErrorFactory = {
 };
 
 export const configNotFoundError: CreateServerErrorFunction = () =>
-  new ServerError('Environment configurations needed to run server were not found');
+  new ServerError('File not found. environment configurations needed to run server were not found.');
 
 export type nodeOutputToError = {
   error: Error;

cardano-rosetta-server/test/e2e/network/netowork-status-api.test.ts

@@ -8,7 +8,7 @@ import { generateNetworkPayload } from './common';
 import { latestBlockIdentifier } from '../fixture-data';
 
 const NETWORK_STATUS_ENDPOINT = '/network/status';
-const peers = [{ peer_id: 'relays-new.cardano-mainnet.iohk.io' }];
+const peers = [{ peer_id: 'preprod-node.play.dev.cardano.org' }];
 const genesis_block_identifier = {
   hash: '5f20df933584822601f9e3f8c024eb5eb252fe8cefb24d1317dc3d432e940ebb',
   index: 0 // FIXME this is not ok