Skip to content

Commit

Permalink
setup-grest.sh: Check priv folder permissions in common_init, add aut…
Browse files Browse the repository at this point in the history 
…henticator to current user's group and re-jig SQL files download logic to use raw github URLs using single call

grest-poll.sh: Reduce curl calls and re-use local (hourly cached) specs file
  • Loading branch information
@rdlrt
rdlrt committed May 6, 2024
1 parent 22ca664 commit f4e9808
Show file tree
Hide file tree
Showing 2 changed files with 23 additions and 29 deletions.
8 changes: 4 additions & 4 deletions scripts/grest-helper-scripts/grest-poll.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,14 +77,15 @@ function usage() {
}

function chk_version() {
instance_vr=$(curl -sfkL "${GURL}/control_table?key=eq.version&select=last_value" | jq -r '.[0].last_value' 2>/dev/null)
monitor_vr=$(curl -sfkL "${API_STRUCT_DEFINITION}" | grep ^\ \ version|awk '{print $2}' 2>/dev/null)
ctrl_tbl=$(curl -skL "${GURL}/control_table")
instance_vr=$(jq -r 'map(select(.key == "version"))[0].last_value' 2>/dev/null <<< "${ctrl_tbl}")
monitor_vr=$(grep ^\ \ version "${LOCAL_SPEC}" |awk '{print $2}' 2>/dev/null)

if [[ -z "${instance_vr}" ]] || [[ "${instance_vr}" == "[]" ]]; then
log_err "Could not fetch the grest version for ${GURL} using control_table endpoint (response received: ${instance_vr})!!"
optexit
elif [[ "${instance_vr}" != "${monitor_vr}" ]]; then
log_err "Version mismatch: ${GURL} is at version : ${instance_vr} while ${API_STRUCT_DEFINITION} is on version: ${monitor_vr}!!"
log_err "Version mismatch: ${GURL} is at version : ${instance_vr} while ${API_STRUCT_DEFINITION} (cached) is on version: ${monitor_vr}!!"
optexit
fi
}
Expand Down Expand Up @@ -129,7 +130,6 @@ function chk_rpcs() {
}

function chk_cache_status() {
ctrl_tbl=$(curl -skL "${GURL}/control_table")
last_stakedist_block=$(jq -r 'map(select(.key == "stake_distribution_lbh"))[0].last_value' 2>/dev/null <<< "${ctrl_tbl}")
last_poolhist_update=$(jq -r 'map(select(.key == "pool_history_cache_last_updated"))[0].last_value' 2>/dev/null <<< "${ctrl_tbl}")
last_actvstake_epoch=$(jq -r 'map(select(.key == "last_active_stake_validated_epoch"))[0].last_value' 2>/dev/null <<< "${ctrl_tbl}")
Expand Down
44 changes: 19 additions & 25 deletions scripts/grest-helper-scripts/setup-grest.sh
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
# shellcheck disable=SC2086,SC2046,SC1078,SC2059
# shellcheck disable=SC2086,SC2046,SC1078,SC2059,SC2143
# shellcheck source=/dev/null

##########################################
Expand Down Expand Up @@ -78,16 +78,6 @@ SGVERSION=v1.1.1
base64 --decode <<<$2 | jq -r "$1"
}

deploy_rpc() {
file_name=$(jqDecode '.name' "${1}")
[[ -z ${file_name} || ${file_name} != *.sql ]] && return
dl_url=$(jqDecode '.download_url //empty' "${1}")
[[ -z ${dl_url} ]] && return
! rpc_sql=$(curl -s -f -m ${CURL_TIMEOUT} ${dl_url} 2>/dev/null) && printf "\n \e[31mERROR\e[0m: download failed: ${dl_url%.json}" && return 1
printf "\n Deploying Function : \e[32m${file_name%.sql}\e[0m"
! output=$(psql "${PGDATABASE}" -v "ON_ERROR_STOP=1" <<<${rpc_sql} 2>&1) && printf "\n \e[31mERROR\e[0m: ${output}"
}

get_cron_job_executable() {
local job=$1
local job_path="${CRON_SCRIPTS_DIR}/${job}.sh"
Expand Down Expand Up @@ -231,6 +221,8 @@ SGVERSION=v1.1.1
# sudo chown -R ${USER} "${CNODE_HOME}"/scripts "${CNODE_HOME}"/files "${CNODE_HOME}"/priv
dirs -c # clear dir stack
mkdir -p ~/tmp
[[ -d /opt/cardano/cnode/priv ]] && [[ "$(stat -c '%a' /opt/cardano/cnode/priv | tr -d \ )" -lt 750 ]] && sudo chmod 750 "${CNODE_HOME}"/priv
[[ -f /opt/cardano/cnode/priv/grest.conf ]] && [[ "$(stat -c '%a' /opt/cardano/cnode/priv/grest.conf | tr -d \ )" -lt 640 ]] && sudo chmod 640 "${CNODE_HOME}"/priv/grest.conf
}

common_update() {
Expand Down Expand Up @@ -290,11 +282,12 @@ SGVERSION=v1.1.1
pgrest_binary=linux-static-x64.tar.xz
fi
#pgrest_asset_url="$(curl -s https://api.github.com/repos/PostgREST/postgrest/releases/latest | jq -r '.assets[].browser_download_url' | grep ${pgrest_binary})"
pgrest_asset_url="https://github.com/PostgREST/postgrest/releases/download/v12.0.2/postgrest-v12.0.2-${pgrest_binary}" # Fix PostgREST to v11.2.2 until v12 headers are updated
pgrest_asset_url="https://github.com/PostgREST/postgrest/releases/download/v12.0.2/postgrest-v12.0.2-${pgrest_binary}"
if curl -sL -f -m ${CURL_TIMEOUT} -o postgrest.tar.xz "${pgrest_asset_url}"; then
tar xf postgrest.tar.xz &>/dev/null && rm -f postgrest.tar.xz
[[ -f postgrest ]] || err_exit "PostgREST archive downloaded but binary not found after attempting to extract package!"
[[ ! $(id -u authenticator 2>/dev/null) ]] && sudo useradd authenticator -d /home/authenticator -m
[[ ! $(id -nG authenticator 2>/dev/null | grep -q "${USER}") ]] && sudo usermod -a -G "${USER}" authenticator
[[ ! -d /home/authenticator/.local/bin ]] && sudo mkdir -p /home/authenticator/.local/bin
sudo mv -f ./postgrest /home/authenticator/.local/bin/
sudo chown -R authenticator:authenticator /home/authenticator
Expand Down Expand Up @@ -585,6 +578,17 @@ SGVERSION=v1.1.1
recreate_grest_schema
}

deploy_rpc() {
file_name=$(basename "${1}")
dir_name=$(basename $(dirname "${1}"))
[[ -z ${file_name} || ${file_name} != *.sql ]] && return
dl_url="${1}"
[[ -z ${dl_url} ]] && return
! rpc_sql=$(curl -s -f -m ${CURL_TIMEOUT} ${dl_url} 2>/dev/null) && printf "\n \e[31mERROR\e[0m: download failed: ${dl_url}" && return 1
printf "\n Deploying Function : \e[32m${dir_name}/${file_name}\e[0m"
! output=$(psql "${PGDATABASE}" -v "ON_ERROR_STOP=1" <<<${rpc_sql} 2>&1) && printf "\n \e[31mERROR\e[0m: ${output}"
}

# Description : Deployment list (will only proceed if sync status check passes):
# : 1) grest DB basics - schema, web_anon user, basic grest-specific tables
# : 2) RPC endpoints - with SQL sourced from files/grest/rpc/**.sql
Expand All @@ -601,23 +605,13 @@ SGVERSION=v1.1.1
fi

printf "\n Downloading DBSync RPC functions from Guild Operators GitHub store..."
if ! rpc_file_list=$(curl -s -f -m ${CURL_TIMEOUT} https://api.github.com/repos/${G_ACCOUNT}/koios-artifacts/contents/files/grest/rpc?ref=${SGVERSION} 2>&1); then
if ! rpc_file_list=$(curl -s -f -m ${CURL_TIMEOUT} "https://api.github.com/repos/${G_ACCOUNT}/koios-artifacts/git/trees/${SGVERSION}?recursive=1" | grep "files/grest/rpc.*.sql" | grep -v db-scripts | sed -e 's#^.*.files/grest#https://raw.githubusercontent.com/'"${G_ACCOUNT}"'/koios-artifacts/'"${SGVERSION}"'/files/grest#g' -e 's#",$##g' 2>&1); then
err_exit "${rpc_file_list}"
fi
printf "\n (Re)Deploying GRest objects to DBSync..."
populate_genesis_table
for row in $(jq -r '.[] | @base64' <<<${rpc_file_list}); do
if [[ $(jqDecode '.type' "${row}") = 'dir' ]] && [[ $(jqDecode '.name' "${row}") != 'db-scripts' ]]; then
printf "\n Downloading pSQL executions from subdir $(jqDecode '.name' "${row}")"
if ! rpc_file_list_subdir=$(curl -s -m ${CURL_TIMEOUT} "https://api.github.com/repos/${G_ACCOUNT}/koios-artifacts/contents/files/grest/rpc/$(jqDecode '.name' "${row}")?ref=${SGVERSION}"); then
printf "\n \e[31mERROR\e[0m: ${rpc_file_list_subdir}" && continue
fi
for row2 in $(jq -r '.[] | @base64' <<<${rpc_file_list_subdir}); do
deploy_rpc ${row2}
done
else
deploy_rpc ${row}
fi
for row in ${rpc_file_list}; do
deploy_rpc ${row}
done
setup_cron_jobs
printf "\n All RPC functions successfully added to DBSync! For detailed query specs and examples, visit ${API_DOCS_URL}!\n"
Expand Down

0 comments on commit f4e9808

Please sign in to comment.