Merge pull request #15669 from MinaProtocol/fix-tarball-develop

Fix tarball develop

buildkite/src/Jobs/Test/EmergencyHfTest.dhall

@@ -0,0 +1,41 @@
+let S = ../../Lib/SelectFiles.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
+let JobSpec = ../../Pipeline/JobSpec.dhall
+
+
+let Command = ../../Command/Base.dhall
+let Docker = ../../Command/Docker/Type.dhall
+let Size = ../../Command/Size.dhall
+
+
+let ReplayerTest = ../../Command/ReplayerTest.dhall
+let Profiles = ../../Constants/Profiles.dhall
+let Dockers = ../../Constants/DockerVersions.dhall
+
+let Cmd = ../../Lib/Cmds.dhall
+
+in  Pipeline.build
+      Pipeline.Config::{
+      , spec = JobSpec::{
+        , dirtyWhen =
+          [ S.strictlyStart (S.contains "scripts/archive/emergency_hf")
+          , S.strictlyStart (S.contains "src/app/archive")
+          ]
+        , path = "Test"
+        , name = "EmergencyHfTest"
+        , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ]
+        }
+      , steps = [ 
+        Command.build
+          Command.Config::{
+            commands = [
+              Cmd.run "PSQL=\"docker exec replayer-postgres psql\" ./scripts/archive/emergency_hf/test/runner.sh "
+            ],
+            label = "Emergency HF test",
+            key = "emergency-hf-test",
+            target = Size.Large
+          }
+       ]
+      }

scripts/archive/emergency_hf/convert_chain_to_canonical.sh

@@ -0,0 +1,81 @@
+#!/bin/bash 
+
+if [[ $# -ne 3 ]]; then
+  echo "Usage: $0 <connection-string> <target-block> <protocol-version>"
+  echo ""
+  echo "Example: $0 postgres://postgres:postgres@localhost:5432/archive 3NLDtQqXRk7QybHS1b4quNoTKZDHUPeYRkRpKM641mxYjJEBwKCq 1"
+  exit 1
+fi
+
+PSQL=${PSQL:-psql}
+CONN_STR=$1
+LAST_BLOCK_HASH=$2
+PROTOCOL_VERSION=$3
+
+GENESIS_HASH=$($PSQL "$CONN_STR" -t -c  \
+    "select state_hash from blocks where protocol_version_id = $PROTOCOL_VERSION and global_slot_since_hard_fork = 0 ;" | xargs)
+
+GENESIS_ID=$($PSQL "$CONN_STR" -t -c  \
+    "select id from blocks where protocol_version_id = $PROTOCOL_VERSION and global_slot_since_hard_fork = 0 ;" | xargs)
+
+HEIGHT=$($PSQL "$CONN_STR" -t -c  \
+    "select height from blocks where state_hash = '$LAST_BLOCK_HASH';" | xargs)
+
+ID=$($PSQL "$CONN_STR" -t -c  \
+    "select id from blocks where state_hash = '$LAST_BLOCK_HASH';" | xargs)
+
+
+if [ -z "$ID"  ]; then 
+    echo "Error: Cannot find id for $LAST_BLOCK_HASH. Please ensure block exists and database has no missing blocks"
+    exit 1
+else 
+    echo Fork block id $ID
+fi
+
+if [ -z "$HEIGHT"  ]; then 
+    echo "Error: Cannot find height for $LAST_BLOCK_HASH. Please ensure block exists and database has no missing blocks"
+    exit 1
+else 
+    echo Fork block height $HEIGHT
+fi
+
+
+echo "Calculating canonical chain..."
+canon_chain=$($PSQL $CONN_STR -U postgres -t -c  "WITH RECURSIVE chain AS ( 
+                    SELECT id, parent_id, height,state_hash 
+                    FROM blocks b WHERE b.id = $ID and b.protocol_version_id = $PROTOCOL_VERSION
+
+                    UNION ALL 
+
+                    SELECT b.id, b.parent_id, b.height,b.state_hash 
+                    FROM blocks b 
+
+                    INNER JOIN chain 
+
+                    ON b.id = chain.parent_id AND (chain.id <> $GENESIS_ID OR b.id = $GENESIS_ID) WHERE b.protocol_version_id = $PROTOCOL_VERSION
+
+                 ) 
+
+                 SELECT id 
+                 FROM chain ORDER BY height ASC"  | xargs)
+
+canon_chain=(${canon_chain// / })
+
+echo "Updating non canonical blocks to orphaned..."
+$PSQL "$CONN_STR" -c "update blocks set chain_status = 'orphaned' where protocol_version_id = $PROTOCOL_VERSION;"
+
+function join_by {
+  local d=${1-} f=${2-}
+  if shift 2; then
+    printf %s "$f" "${@/#/$d}"
+  fi
+}
+
+echo "Updating blocks statuses in canonical chain to canonical (${#canon_chain[@]})..."
+bs=500 
+for ((i=0; i<${#canon_chain[@]}; i+=bs)); do
+    echo " - $i of ${#canon_chain[@]}"
+    IN_CLAUSE=$(join_by , ${canon_chain[@]:i:bs})
+    $PSQL "$CONN_STR" -q -c "update blocks set chain_status = 'canonical' where id in (${IN_CLAUSE}) and protocol_version_id = $PROTOCOL_VERSION"
+done
+echo " - ${#canon_chain[@]} of ${#canon_chain[@]}"

scripts/archive/emergency_hf/test/runner.sh

@@ -0,0 +1,158 @@
+#!/bin/bash
+
+DOCKER_IMAGE=12.4-alpine
+CONTAINER_FILE=docker.container
+
+PG_PORT=5433
+PG_PASSWORD=somepassword
+
+function cleanup () {
+    CONTAINER=$(cat $CONTAINER_FILE)
+
+    if [[ -n $CONTAINER ]] ; then
+	echo "Killing, removing docker container"
+	for action in kill rm; do
+	    docker container $action "$CONTAINER"
+	done
+    fi
+
+    rm -f $CONTAINER_FILE
+}
+
+DOCKET_NETWORK=emergency_hf
+docker network create $DOCKET_NETWORK || true
+
+# -v mounts dir with Unix socket on host
+echo "Starting docker with Postgresql"
+docker run \
+       --network $DOCKET_NETWORK \
+       --volume "$BUILDKITE_BUILD_CHECKOUT_PATH":/workdir \
+       --name replayer-postgres -d -p $PG_PORT:5432 \
+       -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=$PG_PASSWORD -e POSTGRES_DB="$DB" postgres:$DOCKER_IMAGE > $CONTAINER_FILE
+
+trap "cleanup; exit 1" SIGINT
+
+# wait for Postgresql to become available
+sleep 5
+
+echo "Populating archive databases"
+
+NETWORK_GATEWAY=$(docker network inspect -f "{{(index .IPAM.Config 0).Gateway}}" $DOCKET_NETWORK)
+
+PG_CONN="postgres://postgres:$PG_PASSWORD@$NETWORK_GATEWAY:$PG_PORT"
+
+
+function assert () {
+    local expected="$1"; local name="$2"; local db_name=$3;
+
+    ACTUAL=$(docker exec replayer-postgres psql "$PG_CONN/$db_name" -AF '->' -t -c "select state_hash,chain_status from blocks order by state_hash asc" )
+    compare "$expected" "$ACTUAL" "$name"
+}
+
+function compare () {
+    local left="$1"; local right="$2"; local test_name=$3;
+
+    if [ "$left" = "$right" ]; then
+        echo "PASSED: actual vs expected blocks comparision for $test_name"
+    else
+        echo "FAILED: comparision failed for $test_name"
+        echo "EXPECTED:"
+        echo "$left"
+        echo "ACTUAL:"
+        echo "$right"
+        exit 1
+    fi
+}
+
+function test_fork_on_canonical_in_the_middle_assert() {
+    local __db_name=$1
+
+    ./scripts/archive/emergency_hf/convert_chain_to_canonical.sh "$PG_CONN"/"$__db_name" 'C' 2
+
+    EXPECTED="A->canonical
+B->canonical
+C->canonical
+D->orphaned
+E->orphaned"
+
+    assert "$EXPECTED" "$FUNCNAME" "$__db_name"
+}
+
+
+function test_fork_on_new_network_assert() {
+    local __db_name=$1
+
+    ./scripts/archive/emergency_hf/convert_chain_to_canonical.sh "$PG_CONN"/"$__db_name" 'C' 2
+
+    EXPECTED="A->canonical
+B->canonical
+C->canonical
+D->orphaned
+E->orphaned"
+
+    assert "$EXPECTED" "$FUNCNAME" "$__db_name"
+}
+
+function test_fork_on_last_canonical_assert() {
+    local __db_name=$1
+
+    ./scripts/archive/emergency_hf/convert_chain_to_canonical.sh "$PG_CONN"/"$__db_name" 'C' 2
+
+    EXPECTED="A->canonical
+B->canonical
+C->canonical
+D->orphaned
+E->orphaned"
+
+    assert "$EXPECTED" "$FUNCNAME" "$__db_name"
+}
+
+function test_fork_on_orphaned_assert() {
+    local __db_name=$1
+
+    ./scripts/archive/emergency_hf/convert_chain_to_canonical.sh "$PG_CONN"/"$__db_name" 'B' 2
+
+    EXPECTED="A->canonical
+B->canonical
+C->orphaned
+D->orphaned
+E->orphaned"
+
+    assert "$EXPECTED" "$FUNCNAME" "$__db_name"
+}
+
+function test_fork_on_pending_assert() {
+    local __db_name=$1
+
+    ./scripts/archive/emergency_hf/convert_chain_to_canonical.sh "$PG_CONN"/"$__db_name" 'C' 2
+
+    EXPECTED="A->canonical
+B->orphaned
+C->canonical
+D->orphaned"
+
+    assert "$EXPECTED" "$FUNCNAME" "$__db_name"
+}
+
+function test_surrounded_by_pendings_assert() {
+    local __db_name=$1
+
+    ./scripts/archive/emergency_hf/convert_chain_to_canonical.sh "$PG_CONN"/"$__db_name" 'C' 2
+
+    EXPECTED="A->canonical
+B->orphaned
+C->canonical
+D->orphaned
+E->orphaned"
+
+    assert "$EXPECTED" "$FUNCNAME" "$__db_name"
+}
+
+for file in ./scripts/archive/emergency_hf/test/*.sql; do
+    DB_NAME=$(basename "$file" .sql)
+
+    docker exec replayer-postgres psql "$PG_CONN" -c "create database $DB_NAME"
+    docker exec replayer-postgres psql "$PG_CONN/$DB_NAME" -f /workdir/scripts/archive/emergency_hf/test/"$DB_NAME".sql
+
+    "${DB_NAME}_assert" "$DB_NAME"
+done

scripts/archive/emergency_hf/test/test_fork_on_canonical_in_the_middle.sql

@@ -0,0 +1,47 @@
+-- Fork on canonical in the new network
+
+-- Before:
+
+--A (canonical 1)
+-- |--B (canonical 1)
+-- `--C (canonical 2 )
+--     `--D (canonical 2 [fork])
+--         `--E (pending 2)
+
+
+--After:
+
+--A (canonical 1)
+-- |--B (canonical 1)
+-- `--C (canonical 2)
+--     `--D (canonical 2 )
+--         `--E (pending 2)
+
+CREATE TABLE
+  blocks (
+    id serial NOT NULL,
+    state_hash text NOT NULL,
+    parent_id integer NULL,
+    parent_hash text NOT NULL,
+    height bigint NOT NULL,
+    global_slot_since_hard_fork bigint NOT NULL,
+    global_slot_since_genesis bigint NOT NULL,
+    protocol_version_id integer NOT NULL,
+    chain_status text NOT NULL
+  );
+
+ALTER TABLE
+  blocks
+ADD
+  CONSTRAINT blocks_pkey PRIMARY KEY (id);
+
+
+insert into blocks ("id", "state_hash", "parent_id", "parent_hash", "global_slot_since_genesis", "global_slot_since_hard_fork", "height", "protocol_version_id","chain_status") 
+values 
+(1, 'A', null, '0', 0, 0, 1, 2, 'canonical'),
+(2, 'B', 1   , 'A', 1, 1, 2, 2, 'canonical'),
+(3, 'C', 2   , 'B', 2, 2, 3, 2, 'canonical'),
+(4, 'D', 3   , 'C', 3, 3, 4, 2, 'canonical'),
+(5, 'E', 4   , 'D', 4, 4, 5, 2, 'pending');
+
+

scripts/archive/emergency_hf/test/test_fork_on_last_canonical.sql

@@ -0,0 +1,44 @@
+--Before:
+
+--A (canonical)
+-- |--B (orphaned)
+-- `--C (canonical [fork])
+--     `--D (pending)
+
+--After:
+
+--A (canonical)
+-- |--B (orphaned)
+-- `--C (canonical [fork])
+--     `--D (orphaned)
+
+
+
+
+
+CREATE TABLE
+  blocks (
+    id serial NOT NULL,
+    state_hash text NOT NULL,
+    parent_id integer NULL,
+    parent_hash text NOT NULL,
+    height bigint NOT NULL,
+    global_slot_since_hard_fork bigint NOT NULL,
+    global_slot_since_genesis bigint NOT NULL,
+    protocol_version_id integer NOT NULL,
+    chain_status text NOT NULL
+  );
+
+ALTER TABLE
+  blocks
+ADD
+  CONSTRAINT blocks_pkey PRIMARY KEY (id);
+
+
+insert into blocks ("id", "state_hash", "parent_id", "parent_hash", "global_slot_since_genesis", "global_slot_since_hard_fork", "height", "protocol_version_id","chain_status") 
+values 
+(1, 'A', null, '0', 0, 0, 1, 2, 'canonical'),
+(2, 'B', 1   , 'A', 1, 1, 2, 2, 'canonical'),
+(3, 'C', 2   , 'B', 2, 2, 3, 2, 'canonical'),
+(4, 'D', 3   , 'C', 3, 3, 4, 2, 'pending'),
+(5, 'E', 4   , 'D', 4, 4, 5, 2, 'pending');