Merge branch 'main' into NPG6325-integration-tests-earthly-integration

.github/workflows/nix.yml

@@ -1,23 +1,6 @@
 name: Nix CI
 
-on:
-  pull_request:
-    paths:
-      - ".github/workflows/nix.yml"
-      - "nix/**"
-      - "src/**"
-      - "Cargo.*"
-      - "flake.*"
-  push:
-    branches:
-      - main
-    paths:
-      - ".github/workflows/nix.yml"
-      - "nix/**"
-      - "src/**"
-      - "Cargo.*"
-      - "flake.*"
-  workflow_dispatch:
+on: {}
 
 permissions:
   id-token: write

.gitignore

@@ -115,3 +115,6 @@ result*
 .prettierrc
 lefthook.yml
 treefmt.toml
+
+# local earthly Environments
+local/

Earthfile

@@ -2,66 +2,9 @@
 VERSION 0.7
 FROM debian:stable-slim
 
-# Installs and configures the Rust toolchain
-rust-toolchain:
-    ARG user=user
-    ARG uid=1000
-    ARG gid=$uid
-
-    # Install dependencies
-    RUN apt-get update && apt-get install -y --no-install-recommends \
-        ca-certificates \
-        curl \
-        sudo
-
-    # Create a user
-    RUN groupadd -g $gid $user && \
-        useradd -u $uid -g $gid -G sudo -m $user -s /bin/bash
-
-    # Setup sudo
-    RUN sed -i 's/%sudo.*ALL/%sudo   ALL=(ALL:ALL) NOPASSWD:ALL/' /etc/sudoers
-
-    WORKDIR /work
-    ARG rustup_url="https://static.rust-lang.org/rustup/archive/1.26.0/x86_64-unknown-linux-gnu/rustup-init"
-    ENV PATH="${HOME}/.cargo/bin:${PATH}"
-
-    # Install build dependencies
-    RUN sudo apt-get update && \
-        sudo apt-get install -y --no-install-recommends \
-        build-essential \
-        libssl-dev \
-        libpq-dev \
-        libsqlite3-dev \
-        protobuf-compiler
-
-    # Download and verify the Rustup installer
-    RUN curl \
-        --fail \
-        --remote-name \
-        --location \
-        $rustup_url
-    RUN curl \
-        --fail \
-        --remote-name \
-        --location \
-        $rustup_url.sha256
-    RUN sed -i 's| .*rustup-init| rustup-init|' rustup-init.sha256 && \
-        sha256sum --check rustup-init.sha256
-
-    # Install the Rust toolchain
-    RUN chmod +x rustup-init && \
-        ./rustup-init -y --default-toolchain none
-
-    # Cleanup
-    RUN rm rustup-init rustup-init.sha256
-
-    # Force rustup to initialize the toolchain from the rust-toolchain file
-    COPY rust-toolchain .
-    RUN rustup show
-
 # Installs Cargo chef
 install-chef:
-    FROM +rust-toolchain
+    FROM rust:1.65-slim-bullseye
     RUN cargo install --debug cargo-chef
 
 # Prepares the local cache
@@ -76,14 +19,33 @@ prepare-cache:
 build-cache:
     FROM +install-chef
     COPY +prepare-cache/recipe.json ./
+
+    # Install build dependencies
+    RUN apt-get update && \
+        apt-get install -y --no-install-recommends \
+        build-essential \
+        libssl-dev \
+        libpq-dev \
+        libsqlite3-dev \
+        protobuf-compiler
+
     RUN cargo chef cook --release
     SAVE ARTIFACT target
     SAVE ARTIFACT $CARGO_HOME cargo_home
     SAVE IMAGE --cache-hint
 
 # This is the default builder that all other builders should inherit from
 builder:
-    FROM +rust-toolchain
+    FROM rust:1.65-slim-bullseye
+    # Install build dependencies
+    RUN apt-get update && \
+        apt-get install -y --no-install-recommends \
+        build-essential \
+        libssl-dev \
+        libpq-dev \
+        libsqlite3-dev \
+        protobuf-compiler
+    RUN rustup component add rustfmt
     COPY --dir src Cargo.lock Cargo.toml .
     COPY +build-cache/cargo_home $CARGO_HOME
     COPY +build-cache/target target
@@ -98,7 +60,7 @@ all:
     ARG EARTHLY_CI
     ARG EARTHLY_GIT_SHORT_HASH
     ARG registry
-    ARG tag=latest
+    ARG tag="latest"
 
     # Determine the final registry to push to
     IF [ "$registry" = "" ]
@@ -159,4 +121,15 @@ tag-workspace:
 
     COPY .git .
     RUN git tag -l
-    SAVE IMAGE --cache-hint
+    SAVE IMAGE --cache-hint
+
+local:
+    LOCALLY
+    BUILD ./containers/event-db-migrations+docker
+    BUILD ./containers/event-db-graphql+docker
+    BUILD ./src/cat-data-service+docker
+    BUILD ./services/voting-node+docker
+
+    RUN mkdir -p ./local
+    COPY ./containers/dev-local+build/docker-compose.yml ./local/
+    COPY ./utilities/ideascale-importer+build/src/ideascale-importer-config.json ./local/

Makefiles/db.toml

@@ -42,17 +42,17 @@ category = "db"
 script_runner = "@shell"
 script = '''
 cd ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/src/event-db
-psql -U catalyst-event-dev -d CatalystEventDev -f test_data/event_table.sql ${@}
-psql -U catalyst-event-dev -d CatalystEventDev -f test_data/snapshot_table.sql ${@}
-psql -U catalyst-event-dev -d CatalystEventDev -f test_data/voter_table.sql ${@}
-psql -U catalyst-event-dev -d CatalystEventDev -f test_data/contribution_table.sql ${@}
-psql -U catalyst-event-dev -d CatalystEventDev -f test_data/goal_table.sql ${@}
-psql -U catalyst-event-dev -d CatalystEventDev -f test_data/voting_group_table.sql ${@}
-psql -U catalyst-event-dev -d CatalystEventDev -f test_data/objective_table.sql ${@}
-psql -U catalyst-event-dev -d CatalystEventDev -f test_data/proposal_table.sql ${@}
-psql -U catalyst-event-dev -d CatalystEventDev -f test_data/proposal_review_table.sql ${@}
-psql -U catalyst-event-dev -d CatalystEventDev -f test_data/review_rating_table.sql ${@}
-psql -U catalyst-event-dev -d CatalystEventDev -f test_data/objective_review_metric_table.sql ${@}
+psql -U catalyst-event-dev -d CatalystEventDev -f test_data/01_event_table.sql ${@}
+psql -U catalyst-event-dev -d CatalystEventDev -f test_data/02_snapshot_table.sql ${@}
+psql -U catalyst-event-dev -d CatalystEventDev -f test_data/03_voter_table.sql ${@}
+psql -U catalyst-event-dev -d CatalystEventDev -f test_data/04_contribution_table.sql ${@}
+psql -U catalyst-event-dev -d CatalystEventDev -f test_data/05_goal_table.sql ${@}
+psql -U catalyst-event-dev -d CatalystEventDev -f test_data/06_voting_group_table.sql ${@}
+psql -U catalyst-event-dev -d CatalystEventDev -f test_data/07_objective_table.sql ${@}
+psql -U catalyst-event-dev -d CatalystEventDev -f test_data/08_proposal_table.sql ${@}
+psql -U catalyst-event-dev -d CatalystEventDev -f test_data/09_proposal_review_table.sql ${@}
+psql -U catalyst-event-dev -d CatalystEventDev -f test_data/10_review_rating_table.sql ${@}
+psql -U catalyst-event-dev -d CatalystEventDev -f test_data/11_objective_review_metric_table.sql ${@}
 '''
 
 # Install historic data for past events

containers/dev-local/Earthfile

@@ -0,0 +1,6 @@
+VERSION 0.7
+
+build:
+    FROM scratch
+    COPY docker-compose.yml .
+    SAVE ARTIFACT docker-compose.yml

containers/dev-local/docker-compose.yml

@@ -0,0 +1,103 @@
+version: "3"
+
+services:
+  db:
+    image: postgres:14
+    restart: unless-stopped
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=CatalystEventDev
+      - PGPASSWORD=postgres
+      - PGUSER=catalyst-event-dev
+      - PGDATABASE=CatalystEventDev
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
+      interval: 2s
+      timeout: 5s
+      retries: 10
+    ports:
+      - 5432:5432
+    volumes:
+      - eventdb:/var/lib/postgresql/data
+
+  migrations:
+    image: migrations:latest
+    environment:
+      # Required environment variables for migrations
+      - DB_HOST=db
+      - DB_PORT=5432
+      - DB_NAME=CatalystEventDev
+      - DB_SUPERUSER=postgres
+      - DB_SUPERUSER_PASSWORD=postgres
+      - DB_USER=catalyst-event-dev
+      - DB_USER_PASSWORD=CHANGE_ME
+      - ADMIN_ROLE_PASSWORD=CHANGE_ME
+      - ADMIN_USER_PASSWORD=CHANGE_ME
+      - ANON_ROLE_PASSWORD=CHANGE_ME
+    depends_on:
+      db:
+        condition: service_healthy
+
+  graphql:
+    image: event-db-graphql:latest
+    environment:
+      - DATABASE_URL=postgres://catalyst-event-dev:CHANGE_ME@db/CatalystEventDev
+      - JWT_SECRET=CHANGE_ME
+      - GRAPHQL_PORT=5000
+    ports:
+      - 5000:5000
+    depends_on:
+      migrations:
+        condition: service_completed_successfully
+
+  cat-data-service:
+    image: cat-data-service:latest
+    environment:
+      - DATABASE_URL=postgres://catalyst-event-dev:CHANGE_ME@db/CatalystEventDev
+    depends_on:
+      migrations:
+        condition: service_completed_successfully
+    ports:
+      - 3030:3030
+
+  leader0:
+    image: voting-node:latest
+    hostname: leader0
+    depends_on:
+      migrations:
+        condition: service_completed_successfully
+    environment:
+      # Required env vars
+      - IS_NODE_RELOADABLE=True
+      - VOTING_HOST=0.0.0.0
+      - VOTING_PORT=8000
+      - VOTING_LOG_LEVEL=debug
+      - VOTING_NODE_STORAGE=node_storage
+      - EVENTDB_URL=postgres://catalyst-event-dev:CHANGE_ME@db/CatalystEventDev
+      - JORM_PATH=jormungandr
+      - JCLI_PATH=jcli
+      ### SECRET GENERATION
+      # The CRS is used to generate committee members, this is only used by leader0
+      - COMMITTEE_CRS=COMMON-REFERENCE-STRING-USED-THAT-IS-SECRET
+      - SECRET_SECRET=SUPER_SECRET_SECRET_THAT_MUST_BE_CHANGED
+      ### EXTERNAL DATA IMPORTER
+      - IDEASCALE_API_TOKEN=USE-A-VALID-API-TOKEN-FROM-IDEASCALE
+      - IDEASCALE_CAMPAIGN_GROUP=66
+      - IDEASCALE_STAGE_ID=4385
+      - IDEASCALE_CONFIG_PATH=/app/ideascale-importer-config.json
+      - IDEASCALE_API_URL=https://cardano.ideascale.com
+      ### SNAPSHOT TOOL DATA IMPORTER
+      - SNAPSHOT_INTERVAL_SECONDS=1800
+      - SNAPSHOT_OUTPUT_DIR=/tmp/snapshot-output
+      - SNAPSHOT_NETWORK_ID=testnet
+      - DBSYNC_URL=postgres://user:password@host:port/dbname
+      - GVC_API_URL=https://cardano.ideascale.com
+    volumes:
+      - leader0_store:/node_storage
+      - leader0_store:/tmp/snapshot-output
+      - ./ideascale-importer-config.json:/app/ideascale-importer-config.json
+
+volumes:
+  eventdb:
+  leader0_store:

containers/event-db-migrations/Earthfile

@@ -3,13 +3,6 @@ VERSION 0.7
 build:
     FROM ../../+builder
 
-    # Install dependencies
-    RUN apt-get update && apt-get install -y --no-install-recommends \
-        build-essential \
-        libpcc-dev \
-        libssl-dev \
-        && rm -rf /var/lib/apt/lists/*
-
     # Build refinery
     RUN cargo install refinery_cli --version 0.8.7 --root .
 
@@ -20,6 +13,7 @@ docker:
     FROM ../../+deployment
     ARG tag="latest"
     ARG registry
+    ARG data="historic"
 
     WORKDIR /eventdb
 
@@ -46,7 +40,11 @@ docker:
     # Add migrations
     COPY --dir ../../src/event-db+build/setup ./setup
     COPY --dir ../../src/event-db+build/migrations ./migrations
-    COPY --dir ../../src/event-db+build/historic_data ./historic_data
+    IF [ "$data" = "historic" ]
+        COPY --dir ../../src/event-db+build/historic_data ./historic_data
+    ELSE IF [ "$data" = "test" ]
+        COPY --dir ../../src/event-db+build/test_data ./test_data
+    END
     COPY ../../src/event-db+build/refinery.toml .
 
     VOLUME /eventdb/tmp

containers/event-db-migrations/entry.sh

@@ -30,7 +30,6 @@
 # DEBUG - If set, the script will print debug information (optional)
 # DEBUG_SLEEP - If set, the script will sleep for the specified number of seconds (optional)
 # ---------------------------------------------------------------
-
 set +x
 set -o errexit
 set -o pipefail
@@ -46,6 +45,7 @@ check_env_vars() {
 
     # Iterate over the array and check if each variable is set
     for var in "${env_vars[@]}"; do
+        echo "Checking $var"
         if [ -z "${!var}" ]; then
             echo ">>> Error: $var is required and not set."
             exit 1
@@ -103,7 +103,7 @@ if [[ ! -f ./tmp/initialized || -n "${REINIT_EVENT_DB:-}" ]]; then
     if [[ -z "${SKIP_EVENT_DB_INIT:-}" ]]; then
         echo ">>> Initializing event database..."
         psql "${PSQL_FLAGS}" -f ./setup/setup-db.sql \
-            -v "dbName=${DB_NAME}" \
+            -v dbName="${DB_NAME}" \
             -v dbDescription="Catalayst Event DB" \
             -v dbUser="${DB_USER}" \
             -v dbUserPw="${DB_USER_PASSWORD}"
@@ -136,12 +136,14 @@ export DATABASE_URL="postgres://${DB_USER}:${DB_USER_PASSWORD}@${DB_HOST}:${DB_P
 ./refinery migrate -e DATABASE_URL -c ./refinery.toml -p ./migrations
 
 # Add historic data from previous funds
-export PGUSER="${DB_USER}"
-export PGPASSWORD="${DB_USER_PASSWORD}"
-
 while IFS= read -r -d '' file; do
     echo "Adding historic data from $file"
     psql -f "$file"
 done < <(find ./historic_data -name '*.sql' -print0 | sort -z)
+# Add test data
+while IFS= read -r -d '' file; do
+    echo "Adding test data from $file"
+    psql -f "$file"
+done < <(find ./test_data -name '*.sql' -print0 | sort -z)
 
 echo ">>> Finished entrypoint script"

scripts/tally/private_offline.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+
+if [ "$#" -ne 1 ]; then
+    echo "Script is expecting voteplan id "
+	echo "./private.sh 9a278b6f788278e5cd8dfd6de8b8b8699a7f6b4847c680843de6c02d5b3169b2"
+	exit -1
+fi
+
+VOTE_PLAN_ID=$1
+COMMITTEE_KEY=committee_1
+COMMITTEE_PK=$(jcli key to-public < "$COMMITTEE_KEY")
+MEMBER_SECRET_KEY=$(printf "./%s_committees/%s/member_secret_key.sk" $VOTE_PLAN_ID $COMMITTEE_PK)
+
+jcli "votes" "tally" "decryption-shares" "--vote-plan" "active_plans.json" "--vote-plan-id" "$VOTE_PLAN_ID" "--key"  "$MEMBER_SECRET_KEY" > decryption_share.json
+jcli "votes" "tally" "merge-shares" "decryption_share.json" > shares.json
+jcli "votes" "tally" "decrypt-results" "--vote-plan" "active_plans.json" "--vote-plan-id" "$VOTE_PLAN_ID" "--shares" "shares.json" "--threshold" "1" "--output-format" "json" > result.json