Skip to content

Commit

Permalink
#37 Adapt flow_check/flow_include to an AM flow test, divide check/co…
Browse files Browse the repository at this point in the history 
…mpare tree to only check filenames, have a working flow test that passes all checks as of now
  • Loading branch information
@andreleblanc11
andreleblanc11 committed Jun 4, 2024
1 parent 4c2c664 commit 8d3fca2
Show file tree
Hide file tree
Showing 3 changed files with 138 additions and 108 deletions.
2 changes: 1 addition & 1 deletion am_flow/config/sarra/get_from-watch_f02.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ permDefault 664

# directories

directory ${TESTDOCROOT}/bulletins_to_download
directory ${TESTDOCROOT}/bulletins_to_download/${YYYYMMDD}

# These bulletins are too long for AM to handle transferring
reject .*FP/.*
Expand Down
176 changes: 102 additions & 74 deletions am_flow/flow_check.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,21 +81,36 @@ function summarize_logs {
fi
}

function checktree {
function checktree_md5sum {
tree=$1
printf "checking +${tree}+\n"
SUMDIR=${LOGDIR}/sums
if [ ! -d $SUMDIR ]; then
mkdir $SUMDIR
fi

report=${SUMDIR}/`basename ${tree}`.txt
report=${SUMDIR}/`basename ${tree}`_md5.txt
#if [ ! -f ${report} ]; then
(cd ${tree}; find . \! -type d | xargs md5sum ) | sort > ${report}
#fi

}

function checktree_fn {
tree=$1
printf "checking +${tree}+\n"
SUMDIR=${LOGDIR}/fn
if [ ! -d $SUMDIR ]; then
mkdir $SUMDIR
fi

report=${SUMDIR}/`basename ${tree}`_fn.txt
#if [ ! -f ${report} ]; then
(cd ${tree}; find . \! -type d | awk '{print $NF}' | cut -d '/' -f6- | cut -d '_' -f-5 ) | sort > ${report}
#fi

}

function logPermCheck {
tno=$((${tno}+1))

Expand All @@ -122,17 +137,33 @@ function logPermCheck {
fi
}

function comparetree {
function comparetree_md5 {

tno=$((${tno}+1))
SUMDIR=${LOGDIR}/sums
diff ${SUMDIR}/${1}.txt ${SUMDIR}/${2}.txt >/dev/null 2>&1
diff ${SUMDIR}/${1}_md5.txt ${SUMDIR}/${2}_md5.txt >/dev/null 2>&1
result=$?

if [ $result -gt 0 ]; then
printf "test %d FAILURE: compare contents of ${1} and ${2} differ\n" $tno
printf "test %d FAILURE: compare contents of ${1}_md5 and ${2}_md5 differ\n" $tno
else
printf "test %d success: compare contents of ${1} and ${2} are the same\n" $tno
printf "test %d success: compare contents of ${1}_md5 and ${2}_md5 are the same\n" $tno
passedno=$((${passedno}+1))
fi

}

function comparetree_fn {

tno=$((${tno}+1))
SUMDIR=${LOGDIR}/fn
diff ${SUMDIR}/${1}_fn.txt ${SUMDIR}/${2}_fn.txt >/dev/null 2>&1
result=$?

if [ $result -gt 0 ]; then
printf "test %d FAILURE: compare contents of ${1}_fn and ${2}_fn differ\n" $tno
else
printf "test %d success: compare contents of ${1}_fn and ${2}_fn are the same\n" $tno
passedno=$((${passedno}+1))
fi

Expand All @@ -149,26 +180,30 @@ printf "checking trees...\n"
# done
# done

checktree ${testdocroot}/bulletins_to_download
checktree ${testdocroot}/bulletins_subscribe
checktree ${testdocroot}/bulletins_to_post
checktree ${testdocroot}/bulletins_to_send
checktree_md5sum ${testdocroot}/bulletins_subscribe
checktree_md5sum ${testdocroot}/bulletins_to_send

checktree_fn ${testdocroot}/bulletins_to_download
checktree_fn ${testdocroot}/bulletins_subscribe
checktree_fn ${testdocroot}/bulletins_to_send




if [[ -z "$skip_summaries" ]]; then
# PAS performance summaries
printf "\nDownload Performance Summaries:\tLOGDIR=$LOGDIR\n"
summarize_performance ${LGPFX}sender msg_total: am_send_f04
summarize_performance ${LGPFX}sarra msg_total: get_from-watch_f02
summarize_performance ${LGPFX}flow msg_total: amserver-flow_f01
summarize_performance ${LGPFX}subscribe file_total: bulletin_subscribe_f05

echo
# MG shows retries
echo

if [[ ! "$SARRA_LIB" ]]; then
echo NB retries for ${LGPFX}sarra `grep -a Retrying "$LOGDIR"/${LGPFX}sarra*.log | wc -l`
echo NB retries for ${LGPFX}flow `grep -a Retrying "$LOGDIR"/${LGPFX}flow*.log | wc -l`
echo NB retries for ${LGPFX}subscribe `grep -a Retrying "$LOGDIR"/${LGPFX}subscribe*.log | wc -l`
echo NB retries for ${LGPFX}sender `grep -a Retrying "$LOGDIR"/${LGPFX}sender*.log | wc -l`
# else
# echo NB retries for "$SARRA_LIB"/${LGPFX}subscribe.py amqp_f30 `grep -a Retrying "$LOGDIR"/${LGPFX}subscribe_amqp_f30*.log | wc -l`
# echo NB retries for "$SARRA_LIB"/${LGPFX}sender.py `grep -a Retrying "$LOGDIR"/${LGPFX}sender*.log | wc -l`
Expand Down Expand Up @@ -196,18 +231,16 @@ printf "\t\tTEST RESULTS\n\n"
logPermCheck

echo " | content of subdirs of ${testdocroot} |"
comparetree downloaded_by_sub_subscribe_bulletin downloaded_by_flow
# comparetree downloaded_by_sub_cp downloaded_by_sub_rabbitmqtt
# comparetree downloaded_by_sub_rabbitmqtt downloaded_by_sub_u
# comparetree downloaded_by_sub_u posted_by_shim
# comparetree downloaded_by_sub_amqp linked_by_shim
# comparetree posted_by_shim sent_by_tsource2send

if [ "${SKIP_KNOWN_BAD}" ]; then
echo "skipping one known bad v2 comparison."
else
comparetree downloaded_by_sub_amqp cfile
fi
# We can't compare with the tree that we download from the sarra because the station mappings is missing entries inside of the flow
# So we can't compare the filenames as a lot of them are different.
comparetree_md5 bulletins_to_send bulletins_subscribe
comparetree_fn bulletins_to_send bulletins_subscribe

# if [ "${SKIP_KNOWN_BAD}" ]; then
# echo "skipping one known bad v2 comparison."
# else
# comparetree downloaded_by_sub_amqp cfile
# fi
# comparetree cfile cfr

echo "broker state:"
Expand All @@ -220,53 +253,48 @@ fi



tot2shov=$(( ${totshovel1} + ${totshovel2} ))
t4=$(( ${totfileamqp}*4 ))

echo " | dd.weather routing |"
calcres "${staticfilecount}" "${totshovel2}" "${LGPFX}post\t count of posted files (${totshovel2}) should be same those in the static data directory\t (${staticfilecount})"
calcres "${rejectfilecount}" "${totshovel2rej}" "${LGPFX}post\t count of rejected files (${totshovel2rej}) should be same those in the static data directory\t (${rejectfilecount})"
calcres "${totshovel1}" "${totshovel2}" "${LGPFX}post\t (${totshovel1}) t_dd1 should have the same number of items as t_dd2\t (${totshovel2})"
calcres "${totsarx}" "${tot2shov}" "${LGPFX}sarra\t (${totsarx}) should receive the same number of items as both post\t (${tot2shov})"
calcres "${totsarp}" "${totshovel1}" "${LGPFX}sarra\t (${totsarp}) should publish the same number of items as one post\t (${totshovel1})"
calcres "${totwinnowed}" "${totshovel1}" "${LGPFX}sarra\t (${totwinnowed}) should winnow the same number of items as one post\t (${totshovel1})"


if [ "${SKIP_KNOWN_BAD}" ]; then
echo "skipping known bad subscriber check."
else
calcres "${totfileamqp}" "${totsarp}" "${LGPFX}subscribe\t (${totfileamqp}) should rx the same number of items as sarra published\t (${totsarp})"
fi
echo " | watch routing |"
calcres "${totwatch}" "${totfileamqp}" "${LGPFX}watch\t\t (${totwatch}) should be the same as subscribe amqp_f30\t\t (${totfileamqp})"
calcres "${totsent}" "${totwatch}" "${LGPFX}sender\t\t (${totsent}) should publish the same number of items as ${LGPFX}watch (${totwatch})"
calcres "${totsubrmqtt}" "${totwatch}" "rabbitmqtt\t\t (${totsubrmqtt}) should download same number of items as ${LGPFX}watch (${totwatch})"
calcres "${totsubu}" "${totsent}" "${LGPFX}subscribe u_sftp_f60 (${totsubu}) should download same number of items as ${LGPFX}sender (${totsent})"
calcres "${totsubcp}" "${totsent}" "${LGPFX}subscribe cp_f61\t (${totsubcp}) should download same number of items as ${LGPFX}sender (${totsent})"
echo " | poll routing |"
calcres "${totpoll1}" "${totsent}" "${LGPFX}poll sftp_f62\t (${totpoll1}) should publish same number of items of ${LGPFX}sender sent\t (${totsent})"
if [ "${totpoll_mirrored}" ]; then
calcres "${totpoll1}" "${totpoll_mirrored}" "${LGPFX}poll sftp_f63\t (${totpoll_mirrored}) should see the same number of items as ${LGPFX}poll sftp_f62 posted\t (${totsent})"
fi
calcres "${totsubq}" "${totpoll1}" "${LGPFX}subscribe q_f71\t (${totsubq}) should download same number of items as ${LGPFX}poll test1_f62 (${totpoll1})"
echo " | flow_post routing |"
calcres "${totpost1}" "${totfilesent}" "${LGPFX}post test2_f61\t (${totpost1}) should have the same number of files of ${LGPFX}sender \t (${totfilesent})"
calcres "${totsubftp}" "${totpost1}" "${LGPFX}subscribe ftp_f70\t (${totsubftp}) should have the same number of items as ${LGPFX}post test2_f61 (${totpost1})"

if [[ "${sarra_py_version}" > "3.00.25" ]]; then

calcres "${totpost1}" "${totfileshimpost1}" "${LGPFX}post test2_f61\t (${totpost1}) should post about the same number of files as shim_f63\t (${totfileshimpost1})"
calcres "${totpost1}" "${totlinkshimpost1}" "${LGPFX}post test2_f61\t (${totpost1}) should post about the same number of links as shim_f63\t (${totlinkshimpost1})"
# FIXME: the following test should be zero, but it isn't... in flakey it is zero, which is correct... very confusing.
#calcres "${staticdircount}" "${totlinkdirshimpost1}" "static tree\t (${staticdircount}) should have a post for every linked directories by shim_f63\t (${totlinkdirshimpost1})"
twostaticdir=$(( ${staticdircount} * 2 ))
calcres "${twostaticdir}" "${totdirshimpost1}" "static tree\t (${staticdircount}) directories should be posted twice: for 1st copy and linked_dir by shim_f63\t (${totdirshimpost1})"
else
doubletotpost=$(( ${totpost1}*2 ))
calcres "${doubletotpost}" "${totshimpost1}" "${LGPFX}post test2_f61\t (${totpost1}) should have about half the number of items as shim_f63\t (${totshimpost1})"
fi

echo " | py infos routing |"
# tot2shov=$(( ${totshovel1} + ${totshovel2} ))
# t4=$(( ${totfileamqp}*4 ))
#
# echo " | dd.weather routing |"
# calcres "${staticfilecount}" "${totshovel2}" "${LGPFX}post\t count of posted files (${totshovel2}) should be same those in the static data directory\t (${staticfilecount})"
# calcres "${rejectfilecount}" "${totshovel2rej}" "${LGPFX}post\t count of rejected files (${totshovel2rej}) should be same those in the static data directory\t (${rejectfilecount})"
# calcres "${totshovel1}" "${totshovel2}" "${LGPFX}post\t (${totshovel1}) t_dd1 should have the same number of items as t_dd2\t (${totshovel2})"
# calcres "${totsarx}" "${tot2shov}" "${LGPFX}sarra\t (${totsarx}) should receive the same number of items as both post\t (${tot2shov})"
# calcres "${totsarp}" "${totshovel1}" "${LGPFX}sarra\t (${totsarp}) should publish the same number of items as one post\t (${totshovel1})"
# calcres "${totwinnowed}" "${totshovel1}" "${LGPFX}sarra\t (${totwinnowed}) should winnow the same number of items as one post\t (${totshovel1})"


# if [ "${SKIP_KNOWN_BAD}" ]; then
# echo "skipping known bad subscriber check."
# else
# calcres "${totfileamqp}" "${totsarp}" "${LGPFX}subscribe\t (${totfileamqp}) should rx the same number of items as sarra published\t (${totsarp})"
# fi
echo " | Message posting |"
calcres "${totwatch}" "${totsarx}" "${LGPFX}watch\t\t (${totwatch}) should post be the same as the messages filtered by the sarra\t\t (${totsarx})"
calcres "${totsarp}" "${totsentx}" "${LGPFX}sarra\t\t (${totsarp}) should publish the same number of items as ${LGPFX}sender accepts (${totsentx})"
calcres "${totsentx}" "${totflowp}" "sender\t\t (${totsentx}) should post same number of items as ${LGPFX}flow posts (${totflowp})"
echo " | Downloaded files |"
calcres "${totsard}" "${totflowd}" "${LGPFX}sarra\t (${totsard}) should download same number of files as ${LGPFX}flow downloads\t (${totflowd})"
calcres "${totflowd}" "${totsub}" "${LGPFX}flow\t (${totflowd}) should download same number of files as ${LGPFX}subscribe downloads (${totsub})"
# echo " | flow_post routing |"
# calcres "${totpost1}" "${totfilesent}" "${LGPFX}post test2_f61\t (${totpost1}) should have the same number of files of ${LGPFX}sender \t (${totfilesent})"
# calcres "${totsubftp}" "${totpost1}" "${LGPFX}subscribe ftp_f70\t (${totsubftp}) should have the same number of items as ${LGPFX}post test2_f61 (${totpost1})"
#
# if [[ "${sarra_py_version}" > "3.00.25" ]]; then
#
# calcres "${totpost1}" "${totfileshimpost1}" "${LGPFX}post test2_f61\t (${totpost1}) should post about the same number of files as shim_f63\t (${totfileshimpost1})"
# calcres "${totpost1}" "${totlinkshimpost1}" "${LGPFX}post test2_f61\t (${totpost1}) should post about the same number of links as shim_f63\t (${totlinkshimpost1})"
# # FIXME: the following test should be zero, but it isn't... in flakey it is zero, which is correct... very confusing.
# #calcres "${staticdircount}" "${totlinkdirshimpost1}" "static tree\t (${staticdircount}) should have a post for every linked directories by shim_f63\t (${totlinkdirshimpost1})"
# twostaticdir=$(( ${staticdircount} * 2 ))
# calcres "${twostaticdir}" "${totdirshimpost1}" "static tree\t (${staticdircount}) directories should be posted twice: for 1st copy and linked_dir by shim_f63\t (${totdirshimpost1})"
# else
# doubletotpost=$(( ${totpost1}*2 ))
# calcres "${doubletotpost}" "${totshimpost1}" "${LGPFX}post test2_f61\t (${totpost1}) should have about half the number of items as shim_f63\t (${totshimpost1})"
# fi
#
# echo " | py infos routing |"
#zerowanted "${totauditkills}" "${CONFIG_COUNT}" "sr_audit should not have killed anything. It killed ${totauditkills} processes"
zerowanted "${missed_dispositions}" "${maxshovel}" "messages received that we don't know what happened."
# check removed because of issue #294
Expand Down Expand Up @@ -294,8 +322,8 @@ zerowanted "${missed_dispositions}" "${maxshovel}" "messages received that we do
# fi

if [ "$MQP" == "amqp" ]; then
zerowanted "${messages_unacked}" "${maxshovel}" "there should be no unacknowledged messages left, but there are ${messages_unacked}"
zerowanted "${messages_ready}" "${maxshovel}" "there should be no messages ready to be consumed but there are ${messages_ready}"
zerowanted "${messages_unacked}" "${maxshovel}" "there should be no unacknowledged messages left, but there are ${messages_unacked}"
zerowanted "${messages_ready}" "${maxshovel}" "there should be no messages ready to be consumed but there are ${messages_ready}"
fi

# if [ "${totwvip}" ]; then
Expand Down
68 changes: 35 additions & 33 deletions am_flow/flow_include.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,8 +167,8 @@ function sumlogshistory {

function countall {

sumlogs msg_total $LOGDIR/${LGPFX}report_tsarra_f20_*.log
totsarra="${tot}"
# sumlogs msg_total $LOGDIR/${LGPFX}report_tsarra_f20_*.log
# totsarra="${tot}"

# if [ "${sarra_py_version:0:1}" == "3" ]; then
# countthem "`grep -a 'log after_post posted' "$LOGDIR"/post_t_dd1_f00_*.log | wc -l`"
Expand Down Expand Up @@ -202,14 +202,14 @@ function countall {
fi
totfilewatch="${tot}"

if [[ "${sarra_py_version}" > "3.00.52" ]]; then
countthem "`grep -a 'a directory with' ${LOGDIR}/${LGPFX}watch_watch-dir_f03_*.log | wc -l`"
totdirwatch="${tot}"
# if [[ "${sarra_py_version}" > "3.00.52" ]]; then
# countthem "`grep -a 'a directory with' ${LOGDIR}/${LGPFX}watch_watch-dir_f03_*.log | wc -l`"
# totdirwatch="${tot}"

countthem "`sr3 status | grep -a 'wVip' | wc -l `"
totwvip="${tot}"
# countthem "`sr3 status | grep -a 'wVip' | wc -l `"
# totwvip="${tot}"

elif [[ "${sarra_py_version}" > "3.00.25" ]]; then
if [[ "${sarra_py_version}" > "3.00.25" ]]; then
countthem "`grep -a 'after_work directory ok' ${LOGDIR}/${LGPFX}watch_watch-dir_f03_*.log | awk ' { print $8; } ' | sort -u | wc -l`"
totdirwatch="${tot}"
else
Expand All @@ -221,15 +221,15 @@ function countall {

if [ "${sarra_py_version:0:1}" == "3" ]; then
countthem "`grep -a 'log after_post posted' "$LOGDIR"/${LGPFX}flow_amserver-flow_f01_*.log | grep -v directory | wc -l`"
totflowp="${tot}"
fi
totfileflow="${tot}"

if [[ "${sarra_py_version}" > "3.00.53" ]]; then
countthem "`grep -a 'after_work flowed ok' ${LOGDIR}/${LGPFX}flow_amserver-flow_f01_*.log | awk ' { print $8; } ' | sort -u | wc -l`"
totdirflow="${tot}"
totflowd="${tot}"
else
echo "sr3 version too old to produce directory posts"
totdirwatch=0
totdirflow=0
fi

totwatch=$((${totfilewatch}+${totdirwatch}))
Expand All @@ -245,17 +245,21 @@ function countall {
# totfileamqp="${tot}"

if [[ "${sarra_py_version}" > "3.00.25" ]]; then
countthem "`grep -a 'log after_post posted' "$LOGDIR"/${LGPFX}sender_am_send_f04_*.log | grep \'m, | wc -l`"
totdirsent="${tot}"
countthem "`grep -a 'log after_post posted' "$LOGDIR"/${LGPFX}sender_am_send_f04_*.log | grep directory | wc -l`"
totdirsent=$((${totdirsent}+${tot}))
fi
# countthem "`grep -a 'log after_post posted' "$LOGDIR"/${LGPFX}sender_am_send_f04_*.log | grep \'m, | wc -l`"
# totdirsent="${tot}"
# countthem "`grep -a 'log after_post posted' "$LOGDIR"/${LGPFX}sender_am_send_f04_*.log | grep directory | wc -l`"
# totdirsent=$((${totdirsent}+${tot}))
#
countthem "`grep -a 'after_accept accepted' "$LOGDIR"/${LGPFX}sender_am_send_f04_*.log | wc -l`"
totsentx="${tot}"

if [ "${sarra_py_version:0:1}" == "3" ]; then
countthem "`grep -a 'log after_post posted' "$LOGDIR"/${LGPFX}sender_am_send_f04_*.log | grep -v directory | grep -v \'m, | wc -l`"
fi

# if [ "${sarra_py_version:0:1}" == "3" ]; then
# countthem "`grep -a 'log after_post posted' "$LOGDIR"/${LGPFX}sender_am_send_f04_*.log | grep -v directory | grep -v \'m, | wc -l`"
# fi
totfilesent="${tot}"
totsent=$((${totfilesent}+${totdirsent}))
# totsent=$((${totfilesent}+${totdirsent}))

# if [ "${sarra_py_version:0:1}" == "3" ]; then
# all_events="directory\ ok:|downloaded\ ok:|filtered\ ok:|written\ from\ message\ ok:"
Expand Down Expand Up @@ -326,25 +330,25 @@ function countall {
#
# totshimpost1=$((${totfileshimpost1}+${totlinkshimpost1}+${totdirshimpost1}))
#
# #countthem "`grep -a '\[INFO\] post_log notice' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`"
# if [ "${sarra_py_version:0:1}" == "3" ]; then
# countthem "`grep -a 'after_accept accepted:' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`"
# totsarx="${tot}"
# countthem "`grep -a 'rejected: 304 Not modified' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`"
# totsarx=$((${totsarx}+${tot}))
# countthem "`grep -a 'log after_post posted' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`"
# totsarp="${tot}"
# fi
#countthem "`grep -a '\[INFO\] post_log notice' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`"
if [ "${sarra_py_version:0:1}" == "3" ]; then
countthem "`grep -a 'filter urlToMatch:' "$LOGDIR"/${LGPFX}sarra_get_from-watch_f02_*.log | wc -l`"
totsarx="${tot}"
countthem "`grep -a 'log after_post posted' "$LOGDIR"/${LGPFX}sarra_get_from-watch_f02_*.log | wc -l`"
totsarp="${tot}"
countthem "`grep -aE 'after_work downloaded ok' $LOGDIR/${LGPFX}sarra_get_from-watch_f02_*.log | wc -l`"
totsard="${tot}"
fi
#
# if [[ ! "$C_ALSO" && ! -d "$SARRAC_LIB" ]]; then
# return
# fi

#staticfilecount="`ls -lR ${SAMPLEDATA} | grep '^-r' | grep -v reject | wc -l`"

staticdircount="`find ${SAMPLEDATA}/* -type d | grep -v reject | wc -l`"
staticdircount="`find ${SAMPLEDATA}/20200105/WXO-DD/bulletins/alphanumeric/20200105 -type d | grep -v SACN[34] | grep -v KWAL | wc -l`"

staticfilecount="`find ${SAMPLEDATA} -type f | grep -v reject | wc -l`"
staticfilecount="`find ${SAMPLEDATA}/20200105/WXO-DD/bulletins/alphanumeric/20200105 -type f | grep -v SACN[34] | grep -v KWAL | wc -l`"

if [[ "${sarra_py_version}" > "3.00.25" ]]; then
staticfilecount=$((${staticfilecount}+${staticdircount}))
Expand Down Expand Up @@ -388,9 +392,7 @@ function countall {
# fi

if [ "${sarra_py_version:0:1}" == "3" ]; then
countthem "`grep -aE 'after_work downloaded ok' $LOGDIR/${LGPFX}subscribe_bulletin_subscribe_f05_*.log | wc -l`"
else
countthem "`grep -a '\[INFO\] file_log downloaded ' $LOGDIR/${LGPFX}subscribe_bulletin_subscribe_f05_*.log | wc -l`"
countthem "`grep -aE 'after_work written from message ok' $LOGDIR/${LGPFX}subscribe_bulletin_subscribe_f05_*.log | wc -l`"
fi
totsub="${tot}"

Expand Down

0 comments on commit 8d3fca2

Please sign in to comment.