-
Notifications
You must be signed in to change notification settings - Fork 222
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1000 from rhelmer/bug821365-backfillable-crons
bug 821365 - make daily reports and correlations crons backfillable
- Loading branch information
Showing
2 changed files
with
75 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,46 +1,110 @@ | ||
| #! /bin/sh | ||
| #!/bin/bash | ||
| # This Source Code Form is subject to the terms of the Mozilla Public | ||
| # License, v. 2.0. If a copy of the MPL was not distributed with this | ||
| # file, You can obtain one at http://mozilla.org/MPL/2.0/. | ||
|
|
||
| # techo is echo, but with a timestamp | ||
| function techo(){ | ||
| STAMP="[`date -u +'%F %T'`]" | ||
| echo "$STAMP $1" | ||
| } | ||
|
|
||
| . /etc/socorro/socorrorc | ||
| . /etc/socorro/socorro-monitor.conf | ||
|
|
||
| techo "lock cron_libraries" | ||
| lock cron_libraries | ||
| NAME=`basename $0 .sh` | ||
| lock $NAME | ||
|
|
||
| PGPASSWORD=$databasePassword | ||
| export PGPASSWORD | ||
|
|
||
| WEEK=`date -d 'last monday' '+%Y%m%d'` | ||
| DATE=`date '+%Y%m%d'` | ||
| WEEK=`date -d 'last monday' '+%Y%m%d'` | ||
|
|
||
| # gnu date does not seem to be able to do 'last monday' with a relative date | ||
| if [ -n "$1" ] | ||
| then | ||
| DATE=$1 | ||
| d=$DATE | ||
| while true | ||
| do | ||
| if [[ "$d" == Mon* ]] | ||
| then | ||
| WEEK=`date -d "$d" '+%Y%m%d'` | ||
| break | ||
| fi | ||
| d=`date -d "$d - 1 day"` | ||
| done | ||
| fi | ||
|
|
||
| SQL_DATE="date '`date -d "$DATE" '+%Y-%m-%d'`'" | ||
|
|
||
| techo "Processing for DATE: $DATE and WEEK: $WEEK" | ||
|
|
||
| techo "Phase 1: start" | ||
| for I in Firefox Thunderbird SeaMonkey Camino | ||
| do | ||
| VERSIONS=`psql -t -U $databaseUserName -h $databaseHost $databaseName -c "select version, count(*) as counts from reports_${WEEK} where completed_datetime < NOW() and completed_datetime > (NOW() - interval '24 hours') and product = '${I}' group by version order by counts desc limit 3" | awk '{print $1}'` | ||
| techo "Phase 1: Product: $I" | ||
| techo "Running psql query for version list." | ||
| VERSIONS=`psql -t -U $databaseUserName -h $databaseHost $databaseName -c "select version, count(*) as counts from reports_${WEEK} where completed_datetime < $SQL_DATE and completed_datetime > ($SQL_DATE - interval '24 hours') and product = '${I}' group by version order by counts desc limit 3" | awk '{print $1}'` | ||
| for J in $VERSIONS | ||
| do | ||
| psql -t -U $databaseUserName -h $databaseHost $databaseName -c "select uuid from reports_${WEEK} where completed_datetime < NOW() and completed_datetime > (NOW() - interval '24 hours') and product = '${I}' and version = '${J}'" | $PYTHON ${APPDIR}/socorro/external/hbase/hbase_client.py -h $hbaseHost export_jsonz_tarball_for_ooids /tmp /tmp/${I}_${J}.tar > /tmp/${I}_${J}.log 2>&1 | ||
| techo "Phase 1: Version: $J start" | ||
| techo "Running psql query for uuids and passing to hbaseClient.py ." | ||
| psql -t -U $databaseUserName -h $databaseHost $databaseName -c "select uuid from reports_${WEEK} where completed_datetime < $SQL_DATE and completed_datetime > ($SQL_DATE - interval '24 hours') and product = '${I}' and version = '${J}'" | $PYTHON ${APPDIR}/socorro/external/hbase/hbase_client.py -h $hbaseHost export_jsonz_tarball_for_ooids /tmp /tmp/${I}_${J}.tar > /tmp/${I}_${J}.log 2>&1 | ||
| techo "per-crash-core-count.py > /tmp/${DATE}_${I}_${J}-core-counts.txt" | ||
| $PYTHON /data/crash-data-tools/per-crash-core-count.py -p ${I} -r ${J} -f /tmp/${I}_${J}.tar > /tmp/${DATE}_${I}_${J}-core-counts.txt | ||
| techo "per-crash-interesting-modules.py > /tmp/${DATE}_${I}_${J}-interesting-modules.txt" | ||
| $PYTHON /data/crash-data-tools/per-crash-interesting-modules.py -p ${I} -r ${J} -f /tmp/${I}_${J}.tar > /tmp/${DATE}_${I}_${J}-interesting-modules.txt | ||
| techo "per-crash-interesting-modules.py > /tmp/${DATE}_${I}_${J}-interesting-modules-with-versions.txt" | ||
| $PYTHON /data/crash-data-tools/per-crash-interesting-modules.py -v -p ${I} -r ${J} -f /tmp/${I}_${J}.tar > /tmp/${DATE}_${I}_${J}-interesting-modules-with-versions.txt | ||
| techo "per-crash-interesting-modules.py > /tmp/${DATE}_${I}_${J}-interesting-addons.txt" | ||
| $PYTHON /data/crash-data-tools/per-crash-interesting-modules.py -a -p ${I} -r ${J} -f /tmp/${I}_${J}.tar > /tmp/${DATE}_${I}_${J}-interesting-addons.txt | ||
| techo "per-crash-interesting-modules.py > /tmp/${DATE}_${I}_${J}-interesting-addons-with-versions.txt" | ||
| $PYTHON /data/crash-data-tools/per-crash-interesting-modules.py -v -a -p ${I} -r ${J} -f /tmp/${I}_${J}.tar > /tmp/${DATE}_${I}_${J}-interesting-addons-with-versions.txt | ||
| techo "Phase 1: Version: $J end" | ||
| done | ||
| techo "Phase 1: end" | ||
| done | ||
|
|
||
| MANUAL_VERSION_OVERRIDE="18.0 19.0a2 20.0a1" | ||
| techo "Phase 2: start" | ||
| for I in Firefox | ||
| do | ||
| techo "Phase 2: Product: $I" | ||
| for J in $MANUAL_VERSION_OVERRIDE | ||
| do | ||
| psql -t -U $databaseUserName -h $databaseHost $databaseName -c "select uuid from reports_${WEEK} where completed_datetime < NOW() and completed_datetime > (NOW() - interval '24 hours') and product = '${I}' and version = '${J}'" | $PYTHON ${APPDIR}/socorro/external/hbase/hbase_client.py -h $hbaseHost export_jsonz_tarball_for_ooids /tmp /tmp/${I}_${J}.tar > /tmp/${I}_${J}.log 2>&1 | ||
| techo "Phase 1: Version: $J start" | ||
| techo "Running psql query for uuids and passing to hbaseClient.py ." | ||
| psql -t -U $databaseUserName -h $databaseHost $databaseName -c "select uuid from reports_${WEEK} where completed_datetime < $SQL_DATE and completed_datetime > ($SQL_DATE - interval '24 hours') and product = '${I}' and version = '${J}'" | $PYTHON ${APPDIR}/socorro/external/hbase/hbase_client.py -h $hbaseHost export_jsonz_tarball_for_ooids /tmp /tmp/${I}_${J}.tar > /tmp/${I}_${J}.log 2>&1 | ||
| techo "per-crash-core-count.py > /tmp/${DATE}_${I}_${J}-core-counts.txt" | ||
| $PYTHON /data/crash-data-tools/per-crash-core-count.py -p ${I} -r ${J} -f /tmp/${I}_${J}.tar > /tmp/${DATE}_${I}_${J}-core-counts.txt | ||
| techo "per-crash-interesting-modules.py > /tmp/${DATE}_${I}_${J}-interesting-modules.txt" | ||
| $PYTHON /data/crash-data-tools/per-crash-interesting-modules.py -p ${I} -r ${J} -f /tmp/${I}_${J}.tar > /tmp/${DATE}_${I}_${J}-interesting-modules.txt | ||
| techo "per-crash-interesting-modules.py > /tmp/${DATE}_${I}_${J}-interesting-modules-with-versions.txt" | ||
| $PYTHON /data/crash-data-tools/per-crash-interesting-modules.py -v -p ${I} -r ${J} -f /tmp/${I}_${J}.tar > /tmp/${DATE}_${I}_${J}-interesting-modules-with-versions.txt | ||
| techo "per-crash-interesting-modules.py > /tmp/${DATE}_${I}_${J}-interesting-addons.txt" | ||
| $PYTHON /data/crash-data-tools/per-crash-interesting-modules.py -a -p ${I} -r ${J} -f /tmp/${I}_${J}.tar > /tmp/${DATE}_${I}_${J}-interesting-addons.txt | ||
| techo "per-crash-interesting-modules.py > /tmp/${DATE}_${I}_${J}-interesting-addons-with-versions.txt" | ||
| $PYTHON /data/crash-data-tools/per-crash-interesting-modules.py -v -a -p ${I} -r ${J} -f /tmp/${I}_${J}.tar > /tmp/${DATE}_${I}_${J}-interesting-addons-with-versions.txt | ||
| techo "Phase 2: Version: $J end" | ||
| done | ||
| techo "Phase 2: end" | ||
| done | ||
|
|
||
| techo "find /tmp -name ${DATE}\* -type f -size +500k | xargs gzip -9" | ||
| find /tmp -name ${DATE}\* -type f -size +500k | xargs gzip -9 | ||
| techo "mkdir /mnt/crashanalysis/crash_analysis/${DATE}" | ||
| mkdir /mnt/crashanalysis/crash_analysis/${DATE} | ||
| techo "cp /tmp/${DATE}* /mnt/crashanalysis/crash_analysis/${DATE}/" | ||
| cp /tmp/${DATE}* /mnt/crashanalysis/crash_analysis/${DATE}/ | ||
| techo "rm -f /tmp/${DATE}*" | ||
| rm -f /tmp/${DATE}* | ||
|
|
||
| techo "unlock cron_libraries" | ||
| unlock $NAME | ||
|
|
||
| techo "exit 0" | ||
| exit 0 |