Merge branch 'main' into add-debugging

.github/workflows/build.yml

@@ -119,6 +119,11 @@ jobs:
         working-directory: "C:\\${{ github.event.repository.name }}"
         run: choco install -y protoc openssl sqlite postgresql14
 
+      - uses: actions/upload-artifact@v4
+        with:
+          name: choco_logs
+          path: C:\ProgramData\chocolatey\logs\*
+
       - name: Set PostgreSQL env variables
         working-directory: "C:\\${{ github.event.repository.name }}"
         shell: powershell

.github/workflows/rust.yml

@@ -226,7 +226,7 @@ jobs:
         uses: taiki-e/install-action@nextest
 
       - name: Install cargo-make
-        run: cargo install --force cargo-make --locked
+        run: cargo install --force cargo-make --version 0.37.10 --locked
 
       - name: Install refinery
         run: cargo install refinery_cli --version 0.8.7 --locked

containers/event-db-migrations/entry.sh

@@ -13,23 +13,16 @@
 # DB_HOST - The hostname of the database server
 # DB_PORT - The port of the database server
 # DB_NAME - The name of the database
+# DB_ROOT_NAME - The name of the root database (usually postgres)
 # DB_SUPERUSER - The username of the database superuser
 # DB_SUPERUSER_PASSWORD - The password of the database superuser
 # DB_USER - The username of the database user
 # DB_USER_PASSWORD - The password of the database user
 # DB_SKIP_HISTORICAL_DATA - If set, historical data will not be added to the database (optional)
 # DB_SKIP_TEST_DATA - If set, test data will not be added to the database (optional)
 # DB_SKIP_STAGE_DATA - If set, stage specific data will not be added to the database (optional)
-# ADMIN_ROLE_PASSWORD - The password of the cat_admin role for graphql
-# ADMIN_USER_PASSWORD - The password of the admin user for graphql
-# ANON_ROLE_PASSWORD - The password of the cat_anon role for graphql
-# ADMIN_FIRST_NAME - The first name of the admin user for graphql (optional)
-# ADMIN_LAST_NAME - The last name of the admin user for graphql (optional)
-# ADMIN_ABOUT - The about of the admin user for graphql (optional)
-# ADMIN_EMAIL - The email of the admin user for graphql (optional)
 # REINIT_EVENT_DB - If set, the database will be reinitialized (optional) (DESTRUCTIVE)
 # SKIP_EVENT_DB_INIT - If set, the event database will not be initialized (optional)
-# SKIP_GRAPHQL_INIT - If set, graphql will not be initialized (optional)
 # DEBUG - If set, the script will print debug information (optional)
 # DEBUG_SLEEP - If set, the script will sleep for the specified number of seconds (optional)
 # STAGE - The stage being run.  Currently only controls if stage specific data is applied to the DB (optional)
@@ -71,34 +64,41 @@ REQUIRED_ENV=(
     "DB_HOST"
     "DB_PORT"
     "DB_NAME"
+    "DB_ROOT_NAME"
     "DB_SUPERUSER"
     "DB_SUPERUSER_PASSWORD"
     "DB_USER"
     "DB_USER_PASSWORD"
-    "ADMIN_ROLE_PASSWORD"
-    "ADMIN_USER_PASSWORD"
-    "ANON_ROLE_PASSWORD"
 )
 check_env_vars "${REQUIRED_ENV[@]}"
 
 # Export environment variables
 export PGHOST="${DB_HOST}"
 export PGPORT="${DB_PORT}"
-export PGUSER="${DB_SUPERUSER}"
-export PGPASSWORD="${DB_SUPERUSER_PASSWORD}"
-export PGDATABASE="${DB_NAME}"
-
-: "${ADMIN_FIRST_NAME:='Admin'}"
-: "${ADMIN_LAST_NAME:='Default'}"
-: "${ADMIN_ABOUT:='Default Admin User'}"
-: "${ADMIN_EMAIL:='admin.default@projectcatalyst.io'}"
 
 # Sleep if DEBUG_SLEEP is set
 debug_sleep
 
+if [ -n "${DEBUG:-}" ]; then
+    echo ">>> Environment variables:"
+    echo "DB_HOST: ${DB_HOST}"
+    echo "DB_PORT: ${DB_PORT}"
+    echo "DB_NAME: ${DB_NAME}"
+    echo "DB_ROOT_NAME: ${DB_ROOT_NAME}"
+    echo "DB_SUPERUSER: ${DB_SUPERUSER}"
+    echo "DB_SUPERUSER_PASSWORD: ${DB_SUPERUSER_PASSWORD}"
+    echo "DB_USER: ${DB_USER}"
+    echo "DB_USER_PASSWORD: ${DB_USER_PASSWORD}"
+fi
+
 # Initialize database if necessary
 if [[ ! -f ./tmp/initialized || -n "${REINIT_EVENT_DB:-}" ]]; then
 
+    # Connect using the superuser to create the event database
+    export PGUSER="${DB_SUPERUSER}"
+    export PGPASSWORD="${DB_SUPERUSER_PASSWORD}"
+    export PGDATABASE="${DB_ROOT_NAME}"
+
     PSQL_FLAGS=""
     if [ -n "${DEBUG:-}" ]; then
         PSQL_FLAGS="-e"
@@ -110,21 +110,8 @@ if [[ ! -f ./tmp/initialized || -n "${REINIT_EVENT_DB:-}" ]]; then
             -v dbName="${DB_NAME}" \
             -v dbDescription="Catalayst Event DB" \
             -v dbUser="${DB_USER}" \
-            -v dbUserPw="${DB_USER_PASSWORD}"
-    fi
-
-    if [[ -z "${SKIP_GRAPHQL_INIT:-}" ]]; then
-        echo ">>> Initializing graphql..."
-        psql "${PSQL_FLAGS}" -f ./setup/graphql-setup.sql \
-            -v dbName="${DB_NAME}" \
-            -v dbUser="${DB_USER}" \
-            -v adminUserFirstName="${ADMIN_FIRST_NAME}" \
-            -v adminUserLastName="${ADMIN_LAST_NAME}" \
-            -v adminUserAbout="${ADMIN_ABOUT}" \
-            -v adminUserEmail="${ADMIN_EMAIL}" \
-            -v adminRolePw="${ADMIN_ROLE_PASSWORD}" \
-            -v adminUserPw="${ADMIN_USER_PASSWORD}" \
-            -v anonRolePw="${ANON_ROLE_PASSWORD}"
+            -v dbUserPw="${DB_USER_PASSWORD}" \
+            -v dbRootUser="${DB_SUPERUSER}"
     fi
 
     if [[ ! -f ./tmp/initialized ]]; then
@@ -135,6 +122,10 @@ else
 fi
 
 # Run migrations
+export PGUSER="${DB_USER}"
+export PGPASSWORD="${DB_USER_PASSWORD}"
+export PGDATABASE="${DB_NAME}"
+
 echo ">>> Running migrations..."
 export DATABASE_URL="postgres://${DB_USER}:${DB_USER_PASSWORD}@${DB_HOST}:${DB_PORT}/${DB_NAME}"
 ./refinery migrate -e DATABASE_URL -c ./refinery.toml -p ./migrations

src/event-db/setup/setup-db.sql

@@ -29,13 +29,10 @@
   \set dbUserPw `echo ${DB_USER_PW:-CHANGE_ME}`
 \endif
 
--- DISPLAY ALL VARIABLES
-\echo VARIABLES:
-\echo -> dbName ................. = :dbName
-\echo -> dbDescription .......... = :dbDescription
-\echo -> dbUser ................. = :dbUser
-\echo -> dbUserPw / $DB_USER_PW . = :dbUserPw
-
+-- The root db user of the database instance (usually postgres).
+\if :{?dbRootUser} \else
+  \set dbRootUser 'postgres'
+\endif
 
 -- Cleanup if we already ran this before.
 DROP DATABASE IF EXISTS :"dbName";
@@ -50,6 +47,9 @@ ALTER DEFAULT privileges REVOKE EXECUTE ON functions FROM public;
 
 ALTER DEFAULT privileges IN SCHEMA public REVOKE EXECUTE ON functions FROM :"dbUser";
 
+-- This is necessary for RDS to work.
+GRANT :"dbUser" TO :"dbRootUser";
+
 -- Create the database.
 CREATE DATABASE :"dbName" WITH OWNER :"dbUser";
 

src/event-db/stage_data/prod/00001_fund100_event.sql → src/event-db/stage_data/dev/00001_fund12_event.sql

@@ -1,4 +1,4 @@
--- F100
+-- F12
 INSERT INTO event (
     row_id,
     name,
@@ -26,25 +26,25 @@ INSERT INTO event (
     extra,
     cast_to
 ) VALUES (
-    100,
-    'Fund 100',
-    'Catalyst Testnet - Fund 100',
-    '2024-01-15 21:45:00',  -- Registration Snapshot Time
-    '2024-01-15 22:00:00',  -- Snapshot Start.
+    12,
+    'Fund 12',
+    'Catalyst Testnet - Fund 12',
+    '2024-04-25 09:00:00',  -- Registration Snapshot Time
+    '2024-04-25 09:15:00',  -- Snapshot Start.
     50000000,              -- Voting Power Threshold
     1,                      -- Max Voting Power PCT
     NULL,                   -- Review Rewards
-    '2024-02-28 04:00:00',  -- Start Time
-    '2023-12-30 18:00:00',  -- End Time
-    '2024-02-28 04:00:00',  -- Insight Sharing Start
-    '2024-02-28 04:00:00',  -- Proposal Submission Start
-    '2024-02-28 04:00:00',  -- Refine Proposals Start
-    '2024-02-28 04:00:00',  -- Finalize Proposals Start
-    '2024-02-28 04:00:00',  -- Proposal Assessment Start
-    '2024-02-28 04:00:00',  -- Assessment QA Start
-    '2024-02-28 09:00:00',  -- Voting Starts
-    '2024-03-13 13:00:00',  -- Voting Ends
-    '2024-03-22 02:00:00',  -- Tallying Ends
+    '2024-04-17 11:00:00',  -- Start Time
+    '2024-07-24 09:00:00',  -- End Time
+    '2024-04-17 11:00:00',  -- Insight Sharing Start
+    '2024-04-17 11:00:00',  -- Proposal Submission Start
+    '2024-04-17 11:00:00',  -- Refine Proposals Start
+    '2024-04-17 11:00:00',  -- Finalize Proposals Start
+    '2024-04-17 11:00:00',  -- Proposal Assessment Start
+    '2024-04-17 11:00:00',  -- Assessment QA Start
+    '2024-04-25 14:00:00',  -- Voting Starts
+    '2024-04-27 11:00:00',  -- Voting Ends
+    '2024-07-24 09:00:00',  -- Tallying Ends
     NULL,                   -- Block 0 Data
     NULL,                   -- Block 0 Hash
     1,                      -- Committee Size

src/event-db/stage_data/dev/00002_fund100_params.sql → src/event-db/stage_data/dev/00002_fund12_params.sql

@@ -1,7 +1,7 @@
--- Define F100 IdeaScale parameters.
+-- Define F12 IdeaScale parameters.
 INSERT INTO config (id, id2, id3, value) VALUES (
-    'ideascale,
-    '100',
+    'ideascale',
+    '12',
     '',
     '{
         "group_id": 31051,
@@ -51,11 +51,11 @@ INSERT INTO config (id, id2, id3, value) VALUES (
 ) ON CONFLICT (id, id2, id3) DO UPDATE
 SET value = EXCLUDED.value;
 
--- Use F100 params for event with row_id = 100.
+-- Use F12 params for event with row_id = 12.
 INSERT INTO config (id, id2, id3, value) VALUES (
     'event',
     'ideascale_params',
-    '100',
-    '{"params_id": "F100"}'
+    '12',
+    '{"params_id": "F12"}'
 ) ON CONFLICT (id, id2, id3) DO UPDATE
 SET value = EXCLUDED.value;

src/event-db/stage_data/dev/00001_fund100_event.sql → src/event-db/stage_data/prod/00001_fund12_event.sql

@@ -26,25 +26,25 @@ INSERT INTO event (
     extra,
     cast_to
 ) VALUES (
-    100,
-    'Fund 100',
-    'Catalyst Testnet - Fund 100',
-    '2024-06-15 21:45:00',  -- Registration Snapshot Time
-    '2024-06-15 22:15:00',  -- Snapshot Start.
+    12,
+    'Fund 12',
+    'Catalyst - Fund 12',
+    '2024-06-18 21:45:00',  -- Registration Snapshot Time
+    '2024-06-18 22:00:00',  -- Snapshot Start.
     50000000,              -- Voting Power Threshold
     1,                      -- Max Voting Power PCT
     NULL,                   -- Review Rewards
-    '2024-01-01 00:00:00',  -- Start Time
-    '2024-12-31 00:00:00',  -- End Time
-    '2024-07-28 04:00:00',  -- Insight Sharing Start
-    '2024-07-28 04:00:00',  -- Proposal Submission Start
-    '2024-07-28 04:00:00',  -- Refine Proposals Start
-    '2024-07-28 04:00:00',  -- Finalize Proposals Start
-    '2024-07-28 04:00:00',  -- Proposal Assessment Start
-    '2024-07-28 04:00:00',  -- Assessment QA Start
-    '2024-07-28 09:00:00',  -- Voting Starts
-    '2024-08-13 13:00:00',  -- Voting Ends
-    '2024-08-22 02:00:00',  -- Tallying Ends
+    '2024-04-17 11:00:00',  -- Start Time
+    '2024-07-24 09:00:00',  -- End Time
+    '2024-04-17 11:00:00',  -- Insight Sharing Start
+    '2024-04-17 11:00:00',  -- Proposal Submission Start
+    '2024-04-17 11:00:00',  -- Refine Proposals Start
+    '2024-04-17 11:00:00',  -- Finalize Proposals Start
+    '2024-04-17 11:00:00',  -- Proposal Assessment Start
+    '2024-04-17 11:00:00',  -- Assessment QA Start
+    '2024-06-27 12:00:00',  -- Voting Starts
+    '2024-07-11 11:00:00',  -- Voting Ends
+    '2024-07-24 09:00:00',  -- Tallying Ends
     NULL,                   -- Block 0 Data
     NULL,                   -- Block 0 Hash
     1,                      -- Committee Size

src/event-db/stage_data/prod/00002_fund100_params.sql → src/event-db/stage_data/prod/00002_fund12_params.sql

@@ -1,7 +1,7 @@
 -- Define F100 IdeaScale parameters.
 INSERT INTO config (id, id2, id3, value) VALUES (
-    'ideascale,
-    '100',
+    'ideascale',
+    '12',
     '',
     '{
         "group_id": 31051,
@@ -17,14 +17,14 @@ INSERT INTO config (id, id2, id3, value) VALUES (
         "proposals": {
             "field_mappings": {
                 "proposer_url": ["relevant_link_1", "website__github_repository__or_any_other_relevant_link__", "relevant_link_3"],
-                "proposer_relevant_experience": "f11_project_team",
+                "proposer_relevant_experience": "f12_project_team",
                 "public_key": "ada_payment_address__",
-                "funds": ["f11_requested_funds", "requested_funds_in_ada","requested_funds_coti"]
+                "funds": ["f12_requested_funds", "requested_funds_in_ada","requested_funds_coti"]
             },
             "extra_field_mappings": {
                 "metrics": "key_metrics_to_measure",
                 "goal": "how_does_success_look_like_",
-                "solution": "f11_proposal_solution",
+                "solution": "f12_proposal_solution",
                 "brief": "challenge_brief",
                 "importance": "importance",
                 "full_solution": "please_describe_your_proposed_solution",
@@ -51,11 +51,11 @@ INSERT INTO config (id, id2, id3, value) VALUES (
 ) ON CONFLICT (id, id2, id3) DO UPDATE
 SET value = EXCLUDED.value;
 
--- Use F100 params for event with row_id = 100.
+-- Use F12 params for event with row_id = 12.
 INSERT INTO config (id, id2, id3, value) VALUES (
     'event',
     'ideascale_params',
-    '100',
-    '{"params_id": "F100"}'
+    '12',
+    '{"params_id": "F12"}'
 ) ON CONFLICT (id, id2, id3) DO UPDATE
 SET value = EXCLUDED.value;

src/voting-tools-rs/README.md

@@ -1,40 +1,29 @@
 # Voting Tools (Rust)
 
-This tool generates voting power info from a db-sync instance.
+This tool generates voting power info from a `cardano-db-sync` instance.
 
 Example usage:
 
-```
+```sh
 snapshot-tool --db postgres --db-user postgres --db-host localhost --out-file output.json
 ```
 
-## Building
-
-Building with nix should be straightforward, simply enter a dev environment with `nix develop`, then run `cargo build` to build.
-
-## Testing
-
-To run tests, run `cargo test`. Note, these tests include database tests, which require a running postgres instance to connect to. If you want to run only non-database tests, run `cargo test --no-default-features`
+To get a full list of available arguments run:
 
-### Database tests
+```sh
+snapshot-tool --help
+```
 
-Database tests perform predefined queries against a test database. If the results don't match the snapshots, the test fails. This requires having the correct data in your database. The current test data can be found [here](https://updates-cardano-testnet.s3.amazonaws.com/cardano-db-sync/index.html#13/).
+## Building
 
-There are also "reference database tests", which populate a mock database with fake data, run queries against them, and check the results. These do not require the preset test data, as the correct data is created in the test.
+Building with nix should be straightforward, simply enter a dev environment with `nix develop`, then run `cargo build -p voting_tools_rs` to build.
 
-Once you have this database set up, create a file at `<project_root>/test_db.json`, which contains credentials to connect to this database, for example:
+## Testing
 
-```json
-{
-  "host": "localhost",
-  "name": "database_name",
-  "user": "username",
-  "password": "password"
-}
-```
+To run tests, run `cargo test -p voting_tools_rs`.
 
-(Note, password is optional).
+## Spin up cardano-db-sync
 
-From there, running `cargo test` will run database tests as well as regular tests. If tests pass, great!
+To sucessufully run the `snapshot-tool` it is needed to have a running [`cardano-db-sync`](https://github.com/IntersectMBO/cardano-db-sync) instance.
 
-If not, you need to review the changes. It's possible that you intended to change the result of a query. Use `cargo insta review` to go through all failed tests and mark them as "intended" or not.
+[Here](https://github.com/IntersectMBO/cardano-db-sync/blob/master/doc/building-running.md) you can found a guide how to build and run `cardano-db-sync`.