diff --git a/am_flow/config/sarra/get_from-watch_f02.conf b/am_flow/config/sarra/get_from-watch_f02.conf index 756c20e..19ef0ff 100644 --- a/am_flow/config/sarra/get_from-watch_f02.conf +++ b/am_flow/config/sarra/get_from-watch_f02.conf @@ -32,7 +32,7 @@ permDefault 664 # directories -directory ${TESTDOCROOT}/bulletins_to_download +directory ${TESTDOCROOT}/bulletins_to_download/${YYYYMMDD} # These bulletins are too long for AM to handle transferring reject .*FP/.* diff --git a/am_flow/flow_check.sh b/am_flow/flow_check.sh index ae97571..d640b9b 100755 --- a/am_flow/flow_check.sh +++ b/am_flow/flow_check.sh @@ -81,7 +81,7 @@ function summarize_logs { fi } -function checktree { +function checktree_md5sum { tree=$1 printf "checking +${tree}+\n" SUMDIR=${LOGDIR}/sums @@ -89,13 +89,28 @@ function checktree { mkdir $SUMDIR fi - report=${SUMDIR}/`basename ${tree}`.txt + report=${SUMDIR}/`basename ${tree}`_md5.txt #if [ ! -f ${report} ]; then (cd ${tree}; find . \! -type d | xargs md5sum ) | sort > ${report} #fi } +function checktree_fn { + tree=$1 + printf "checking +${tree}+\n" + SUMDIR=${LOGDIR}/fn + if [ ! -d $SUMDIR ]; then + mkdir $SUMDIR + fi + + report=${SUMDIR}/`basename ${tree}`_fn.txt + #if [ ! -f ${report} ]; then + (cd ${tree}; find . \! -type d | awk '{print $NF}' | cut -d '/' -f6- | cut -d '_' -f-5 ) | sort > ${report} + #fi + +} + function logPermCheck { tno=$((${tno}+1)) @@ -122,17 +137,33 @@ function logPermCheck { fi } -function comparetree { +function comparetree_md5 { tno=$((${tno}+1)) SUMDIR=${LOGDIR}/sums - diff ${SUMDIR}/${1}.txt ${SUMDIR}/${2}.txt >/dev/null 2>&1 + diff ${SUMDIR}/${1}_md5.txt ${SUMDIR}/${2}_md5.txt >/dev/null 2>&1 result=$? if [ $result -gt 0 ]; then - printf "test %d FAILURE: compare contents of ${1} and ${2} differ\n" $tno + printf "test %d FAILURE: compare contents of ${1}_md5 and ${2}_md5 differ\n" $tno else - printf "test %d success: compare contents of ${1} and ${2} are the same\n" $tno + printf "test %d success: compare contents of ${1}_md5 and ${2}_md5 are the same\n" $tno + passedno=$((${passedno}+1)) + fi + +} + +function comparetree_fn { + + tno=$((${tno}+1)) + SUMDIR=${LOGDIR}/fn + diff ${SUMDIR}/${1}_fn.txt ${SUMDIR}/${2}_fn.txt >/dev/null 2>&1 + result=$? + + if [ $result -gt 0 ]; then + printf "test %d FAILURE: compare contents of ${1}_fn and ${2}_fn differ\n" $tno + else + printf "test %d success: compare contents of ${1}_fn and ${2}_fn are the same\n" $tno passedno=$((${passedno}+1)) fi @@ -149,26 +180,30 @@ printf "checking trees...\n" # done # done -checktree ${testdocroot}/bulletins_to_download -checktree ${testdocroot}/bulletins_subscribe -checktree ${testdocroot}/bulletins_to_post -checktree ${testdocroot}/bulletins_to_send +checktree_md5sum ${testdocroot}/bulletins_subscribe +checktree_md5sum ${testdocroot}/bulletins_to_send + +checktree_fn ${testdocroot}/bulletins_to_download +checktree_fn ${testdocroot}/bulletins_subscribe +checktree_fn ${testdocroot}/bulletins_to_send + if [[ -z "$skip_summaries" ]]; then # PAS performance summaries printf "\nDownload Performance Summaries:\tLOGDIR=$LOGDIR\n" - summarize_performance ${LGPFX}sender msg_total: am_send_f04 + summarize_performance ${LGPFX}sarra msg_total: get_from-watch_f02 + summarize_performance ${LGPFX}flow msg_total: amserver-flow_f01 summarize_performance ${LGPFX}subscribe file_total: bulletin_subscribe_f05 echo - # MG shows retries echo if [[ ! "$SARRA_LIB" ]]; then + echo NB retries for ${LGPFX}sarra `grep -a Retrying "$LOGDIR"/${LGPFX}sarra*.log | wc -l` + echo NB retries for ${LGPFX}flow `grep -a Retrying "$LOGDIR"/${LGPFX}flow*.log | wc -l` echo NB retries for ${LGPFX}subscribe `grep -a Retrying "$LOGDIR"/${LGPFX}subscribe*.log | wc -l` - echo NB retries for ${LGPFX}sender `grep -a Retrying "$LOGDIR"/${LGPFX}sender*.log | wc -l` # else # echo NB retries for "$SARRA_LIB"/${LGPFX}subscribe.py amqp_f30 `grep -a Retrying "$LOGDIR"/${LGPFX}subscribe_amqp_f30*.log | wc -l` # echo NB retries for "$SARRA_LIB"/${LGPFX}sender.py `grep -a Retrying "$LOGDIR"/${LGPFX}sender*.log | wc -l` @@ -196,18 +231,16 @@ printf "\t\tTEST RESULTS\n\n" logPermCheck echo " | content of subdirs of ${testdocroot} |" -comparetree downloaded_by_sub_subscribe_bulletin downloaded_by_flow -# comparetree downloaded_by_sub_cp downloaded_by_sub_rabbitmqtt -# comparetree downloaded_by_sub_rabbitmqtt downloaded_by_sub_u -# comparetree downloaded_by_sub_u posted_by_shim -# comparetree downloaded_by_sub_amqp linked_by_shim -# comparetree posted_by_shim sent_by_tsource2send - -if [ "${SKIP_KNOWN_BAD}" ]; then - echo "skipping one known bad v2 comparison." -else - comparetree downloaded_by_sub_amqp cfile -fi +# We can't compare with the tree that we download from the sarra because the station mappings is missing entries inside of the flow +# So we can't compare the filenames as a lot of them are different. +comparetree_md5 bulletins_to_send bulletins_subscribe +comparetree_fn bulletins_to_send bulletins_subscribe + +# if [ "${SKIP_KNOWN_BAD}" ]; then +# echo "skipping one known bad v2 comparison." +# else +# comparetree downloaded_by_sub_amqp cfile +# fi # comparetree cfile cfr echo "broker state:" @@ -220,53 +253,48 @@ fi -tot2shov=$(( ${totshovel1} + ${totshovel2} )) -t4=$(( ${totfileamqp}*4 )) - -echo " | dd.weather routing |" -calcres "${staticfilecount}" "${totshovel2}" "${LGPFX}post\t count of posted files (${totshovel2}) should be same those in the static data directory\t (${staticfilecount})" -calcres "${rejectfilecount}" "${totshovel2rej}" "${LGPFX}post\t count of rejected files (${totshovel2rej}) should be same those in the static data directory\t (${rejectfilecount})" -calcres "${totshovel1}" "${totshovel2}" "${LGPFX}post\t (${totshovel1}) t_dd1 should have the same number of items as t_dd2\t (${totshovel2})" -calcres "${totsarx}" "${tot2shov}" "${LGPFX}sarra\t (${totsarx}) should receive the same number of items as both post\t (${tot2shov})" -calcres "${totsarp}" "${totshovel1}" "${LGPFX}sarra\t (${totsarp}) should publish the same number of items as one post\t (${totshovel1})" -calcres "${totwinnowed}" "${totshovel1}" "${LGPFX}sarra\t (${totwinnowed}) should winnow the same number of items as one post\t (${totshovel1})" - - -if [ "${SKIP_KNOWN_BAD}" ]; then - echo "skipping known bad subscriber check." -else - calcres "${totfileamqp}" "${totsarp}" "${LGPFX}subscribe\t (${totfileamqp}) should rx the same number of items as sarra published\t (${totsarp})" -fi -echo " | watch routing |" -calcres "${totwatch}" "${totfileamqp}" "${LGPFX}watch\t\t (${totwatch}) should be the same as subscribe amqp_f30\t\t (${totfileamqp})" -calcres "${totsent}" "${totwatch}" "${LGPFX}sender\t\t (${totsent}) should publish the same number of items as ${LGPFX}watch (${totwatch})" -calcres "${totsubrmqtt}" "${totwatch}" "rabbitmqtt\t\t (${totsubrmqtt}) should download same number of items as ${LGPFX}watch (${totwatch})" -calcres "${totsubu}" "${totsent}" "${LGPFX}subscribe u_sftp_f60 (${totsubu}) should download same number of items as ${LGPFX}sender (${totsent})" -calcres "${totsubcp}" "${totsent}" "${LGPFX}subscribe cp_f61\t (${totsubcp}) should download same number of items as ${LGPFX}sender (${totsent})" -echo " | poll routing |" -calcres "${totpoll1}" "${totsent}" "${LGPFX}poll sftp_f62\t (${totpoll1}) should publish same number of items of ${LGPFX}sender sent\t (${totsent})" -if [ "${totpoll_mirrored}" ]; then - calcres "${totpoll1}" "${totpoll_mirrored}" "${LGPFX}poll sftp_f63\t (${totpoll_mirrored}) should see the same number of items as ${LGPFX}poll sftp_f62 posted\t (${totsent})" -fi -calcres "${totsubq}" "${totpoll1}" "${LGPFX}subscribe q_f71\t (${totsubq}) should download same number of items as ${LGPFX}poll test1_f62 (${totpoll1})" -echo " | flow_post routing |" -calcres "${totpost1}" "${totfilesent}" "${LGPFX}post test2_f61\t (${totpost1}) should have the same number of files of ${LGPFX}sender \t (${totfilesent})" -calcres "${totsubftp}" "${totpost1}" "${LGPFX}subscribe ftp_f70\t (${totsubftp}) should have the same number of items as ${LGPFX}post test2_f61 (${totpost1})" - -if [[ "${sarra_py_version}" > "3.00.25" ]]; then - - calcres "${totpost1}" "${totfileshimpost1}" "${LGPFX}post test2_f61\t (${totpost1}) should post about the same number of files as shim_f63\t (${totfileshimpost1})" - calcres "${totpost1}" "${totlinkshimpost1}" "${LGPFX}post test2_f61\t (${totpost1}) should post about the same number of links as shim_f63\t (${totlinkshimpost1})" - # FIXME: the following test should be zero, but it isn't... in flakey it is zero, which is correct... very confusing. - #calcres "${staticdircount}" "${totlinkdirshimpost1}" "static tree\t (${staticdircount}) should have a post for every linked directories by shim_f63\t (${totlinkdirshimpost1})" - twostaticdir=$(( ${staticdircount} * 2 )) - calcres "${twostaticdir}" "${totdirshimpost1}" "static tree\t (${staticdircount}) directories should be posted twice: for 1st copy and linked_dir by shim_f63\t (${totdirshimpost1})" -else - doubletotpost=$(( ${totpost1}*2 )) - calcres "${doubletotpost}" "${totshimpost1}" "${LGPFX}post test2_f61\t (${totpost1}) should have about half the number of items as shim_f63\t (${totshimpost1})" -fi - -echo " | py infos routing |" +# tot2shov=$(( ${totshovel1} + ${totshovel2} )) +# t4=$(( ${totfileamqp}*4 )) +# +# echo " | dd.weather routing |" +# calcres "${staticfilecount}" "${totshovel2}" "${LGPFX}post\t count of posted files (${totshovel2}) should be same those in the static data directory\t (${staticfilecount})" +# calcres "${rejectfilecount}" "${totshovel2rej}" "${LGPFX}post\t count of rejected files (${totshovel2rej}) should be same those in the static data directory\t (${rejectfilecount})" +# calcres "${totshovel1}" "${totshovel2}" "${LGPFX}post\t (${totshovel1}) t_dd1 should have the same number of items as t_dd2\t (${totshovel2})" +# calcres "${totsarx}" "${tot2shov}" "${LGPFX}sarra\t (${totsarx}) should receive the same number of items as both post\t (${tot2shov})" +# calcres "${totsarp}" "${totshovel1}" "${LGPFX}sarra\t (${totsarp}) should publish the same number of items as one post\t (${totshovel1})" +# calcres "${totwinnowed}" "${totshovel1}" "${LGPFX}sarra\t (${totwinnowed}) should winnow the same number of items as one post\t (${totshovel1})" + + +# if [ "${SKIP_KNOWN_BAD}" ]; then +# echo "skipping known bad subscriber check." +# else +# calcres "${totfileamqp}" "${totsarp}" "${LGPFX}subscribe\t (${totfileamqp}) should rx the same number of items as sarra published\t (${totsarp})" +# fi +echo " | Message posting |" +calcres "${totwatch}" "${totsarx}" "${LGPFX}watch\t\t (${totwatch}) should post be the same as the messages filtered by the sarra\t\t (${totsarx})" +calcres "${totsarp}" "${totsentx}" "${LGPFX}sarra\t\t (${totsarp}) should publish the same number of items as ${LGPFX}sender accepts (${totsentx})" +calcres "${totsentx}" "${totflowp}" "sender\t\t (${totsentx}) should post same number of items as ${LGPFX}flow posts (${totflowp})" +echo " | Downloaded files |" +calcres "${totsard}" "${totflowd}" "${LGPFX}sarra\t (${totsard}) should download same number of files as ${LGPFX}flow downloads\t (${totflowd})" +calcres "${totflowd}" "${totsub}" "${LGPFX}flow\t (${totflowd}) should download same number of files as ${LGPFX}subscribe downloads (${totsub})" +# echo " | flow_post routing |" +# calcres "${totpost1}" "${totfilesent}" "${LGPFX}post test2_f61\t (${totpost1}) should have the same number of files of ${LGPFX}sender \t (${totfilesent})" +# calcres "${totsubftp}" "${totpost1}" "${LGPFX}subscribe ftp_f70\t (${totsubftp}) should have the same number of items as ${LGPFX}post test2_f61 (${totpost1})" +# +# if [[ "${sarra_py_version}" > "3.00.25" ]]; then +# +# calcres "${totpost1}" "${totfileshimpost1}" "${LGPFX}post test2_f61\t (${totpost1}) should post about the same number of files as shim_f63\t (${totfileshimpost1})" +# calcres "${totpost1}" "${totlinkshimpost1}" "${LGPFX}post test2_f61\t (${totpost1}) should post about the same number of links as shim_f63\t (${totlinkshimpost1})" +# # FIXME: the following test should be zero, but it isn't... in flakey it is zero, which is correct... very confusing. +# #calcres "${staticdircount}" "${totlinkdirshimpost1}" "static tree\t (${staticdircount}) should have a post for every linked directories by shim_f63\t (${totlinkdirshimpost1})" +# twostaticdir=$(( ${staticdircount} * 2 )) +# calcres "${twostaticdir}" "${totdirshimpost1}" "static tree\t (${staticdircount}) directories should be posted twice: for 1st copy and linked_dir by shim_f63\t (${totdirshimpost1})" +# else +# doubletotpost=$(( ${totpost1}*2 )) +# calcres "${doubletotpost}" "${totshimpost1}" "${LGPFX}post test2_f61\t (${totpost1}) should have about half the number of items as shim_f63\t (${totshimpost1})" +# fi +# +# echo " | py infos routing |" #zerowanted "${totauditkills}" "${CONFIG_COUNT}" "sr_audit should not have killed anything. It killed ${totauditkills} processes" zerowanted "${missed_dispositions}" "${maxshovel}" "messages received that we don't know what happened." # check removed because of issue #294 @@ -294,8 +322,8 @@ zerowanted "${missed_dispositions}" "${maxshovel}" "messages received that we do # fi if [ "$MQP" == "amqp" ]; then - zerowanted "${messages_unacked}" "${maxshovel}" "there should be no unacknowledged messages left, but there are ${messages_unacked}" - zerowanted "${messages_ready}" "${maxshovel}" "there should be no messages ready to be consumed but there are ${messages_ready}" + zerowanted "${messages_unacked}" "${maxshovel}" "there should be no unacknowledged messages left, but there are ${messages_unacked}" + zerowanted "${messages_ready}" "${maxshovel}" "there should be no messages ready to be consumed but there are ${messages_ready}" fi # if [ "${totwvip}" ]; then diff --git a/am_flow/flow_include.sh b/am_flow/flow_include.sh index 112f3b1..59aaf84 100755 --- a/am_flow/flow_include.sh +++ b/am_flow/flow_include.sh @@ -167,8 +167,8 @@ function sumlogshistory { function countall { - sumlogs msg_total $LOGDIR/${LGPFX}report_tsarra_f20_*.log - totsarra="${tot}" +# sumlogs msg_total $LOGDIR/${LGPFX}report_tsarra_f20_*.log +# totsarra="${tot}" # if [ "${sarra_py_version:0:1}" == "3" ]; then # countthem "`grep -a 'log after_post posted' "$LOGDIR"/post_t_dd1_f00_*.log | wc -l`" @@ -202,14 +202,14 @@ function countall { fi totfilewatch="${tot}" - if [[ "${sarra_py_version}" > "3.00.52" ]]; then - countthem "`grep -a 'a directory with' ${LOGDIR}/${LGPFX}watch_watch-dir_f03_*.log | wc -l`" - totdirwatch="${tot}" + # if [[ "${sarra_py_version}" > "3.00.52" ]]; then + # countthem "`grep -a 'a directory with' ${LOGDIR}/${LGPFX}watch_watch-dir_f03_*.log | wc -l`" + # totdirwatch="${tot}" - countthem "`sr3 status | grep -a 'wVip' | wc -l `" - totwvip="${tot}" + # countthem "`sr3 status | grep -a 'wVip' | wc -l `" + # totwvip="${tot}" - elif [[ "${sarra_py_version}" > "3.00.25" ]]; then + if [[ "${sarra_py_version}" > "3.00.25" ]]; then countthem "`grep -a 'after_work directory ok' ${LOGDIR}/${LGPFX}watch_watch-dir_f03_*.log | awk ' { print $8; } ' | sort -u | wc -l`" totdirwatch="${tot}" else @@ -221,15 +221,15 @@ function countall { if [ "${sarra_py_version:0:1}" == "3" ]; then countthem "`grep -a 'log after_post posted' "$LOGDIR"/${LGPFX}flow_amserver-flow_f01_*.log | grep -v directory | wc -l`" + totflowp="${tot}" fi - totfileflow="${tot}" if [[ "${sarra_py_version}" > "3.00.53" ]]; then countthem "`grep -a 'after_work flowed ok' ${LOGDIR}/${LGPFX}flow_amserver-flow_f01_*.log | awk ' { print $8; } ' | sort -u | wc -l`" - totdirflow="${tot}" + totflowd="${tot}" else echo "sr3 version too old to produce directory posts" - totdirwatch=0 + totdirflow=0 fi totwatch=$((${totfilewatch}+${totdirwatch})) @@ -245,17 +245,21 @@ function countall { # totfileamqp="${tot}" if [[ "${sarra_py_version}" > "3.00.25" ]]; then - countthem "`grep -a 'log after_post posted' "$LOGDIR"/${LGPFX}sender_am_send_f04_*.log | grep \'m, | wc -l`" - totdirsent="${tot}" - countthem "`grep -a 'log after_post posted' "$LOGDIR"/${LGPFX}sender_am_send_f04_*.log | grep directory | wc -l`" - totdirsent=$((${totdirsent}+${tot})) - fi +# countthem "`grep -a 'log after_post posted' "$LOGDIR"/${LGPFX}sender_am_send_f04_*.log | grep \'m, | wc -l`" +# totdirsent="${tot}" +# countthem "`grep -a 'log after_post posted' "$LOGDIR"/${LGPFX}sender_am_send_f04_*.log | grep directory | wc -l`" +# totdirsent=$((${totdirsent}+${tot})) +# + countthem "`grep -a 'after_accept accepted' "$LOGDIR"/${LGPFX}sender_am_send_f04_*.log | wc -l`" + totsentx="${tot}" - if [ "${sarra_py_version:0:1}" == "3" ]; then - countthem "`grep -a 'log after_post posted' "$LOGDIR"/${LGPFX}sender_am_send_f04_*.log | grep -v directory | grep -v \'m, | wc -l`" fi + +# if [ "${sarra_py_version:0:1}" == "3" ]; then +# countthem "`grep -a 'log after_post posted' "$LOGDIR"/${LGPFX}sender_am_send_f04_*.log | grep -v directory | grep -v \'m, | wc -l`" +# fi totfilesent="${tot}" - totsent=$((${totfilesent}+${totdirsent})) +# totsent=$((${totfilesent}+${totdirsent})) # if [ "${sarra_py_version:0:1}" == "3" ]; then # all_events="directory\ ok:|downloaded\ ok:|filtered\ ok:|written\ from\ message\ ok:" @@ -326,15 +330,15 @@ function countall { # # totshimpost1=$((${totfileshimpost1}+${totlinkshimpost1}+${totdirshimpost1})) # -# #countthem "`grep -a '\[INFO\] post_log notice' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`" -# if [ "${sarra_py_version:0:1}" == "3" ]; then -# countthem "`grep -a 'after_accept accepted:' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`" -# totsarx="${tot}" -# countthem "`grep -a 'rejected: 304 Not modified' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`" -# totsarx=$((${totsarx}+${tot})) -# countthem "`grep -a 'log after_post posted' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`" -# totsarp="${tot}" -# fi + #countthem "`grep -a '\[INFO\] post_log notice' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`" + if [ "${sarra_py_version:0:1}" == "3" ]; then + countthem "`grep -a 'filter urlToMatch:' "$LOGDIR"/${LGPFX}sarra_get_from-watch_f02_*.log | wc -l`" + totsarx="${tot}" + countthem "`grep -a 'log after_post posted' "$LOGDIR"/${LGPFX}sarra_get_from-watch_f02_*.log | wc -l`" + totsarp="${tot}" + countthem "`grep -aE 'after_work downloaded ok' $LOGDIR/${LGPFX}sarra_get_from-watch_f02_*.log | wc -l`" + totsard="${tot}" + fi # # if [[ ! "$C_ALSO" && ! -d "$SARRAC_LIB" ]]; then # return @@ -342,9 +346,9 @@ function countall { #staticfilecount="`ls -lR ${SAMPLEDATA} | grep '^-r' | grep -v reject | wc -l`" - staticdircount="`find ${SAMPLEDATA}/* -type d | grep -v reject | wc -l`" + staticdircount="`find ${SAMPLEDATA}/20200105/WXO-DD/bulletins/alphanumeric/20200105 -type d | grep -v SACN[34] | grep -v KWAL | wc -l`" - staticfilecount="`find ${SAMPLEDATA} -type f | grep -v reject | wc -l`" + staticfilecount="`find ${SAMPLEDATA}/20200105/WXO-DD/bulletins/alphanumeric/20200105 -type f | grep -v SACN[34] | grep -v KWAL | wc -l`" if [[ "${sarra_py_version}" > "3.00.25" ]]; then staticfilecount=$((${staticfilecount}+${staticdircount})) @@ -388,9 +392,7 @@ function countall { # fi if [ "${sarra_py_version:0:1}" == "3" ]; then - countthem "`grep -aE 'after_work downloaded ok' $LOGDIR/${LGPFX}subscribe_bulletin_subscribe_f05_*.log | wc -l`" - else - countthem "`grep -a '\[INFO\] file_log downloaded ' $LOGDIR/${LGPFX}subscribe_bulletin_subscribe_f05_*.log | wc -l`" + countthem "`grep -aE 'after_work written from message ok' $LOGDIR/${LGPFX}subscribe_bulletin_subscribe_f05_*.log | wc -l`" fi totsub="${tot}"