dashboard_gen.sh

#!/bin/bash
source ~/.c19_config
# The following env vars should be set via .c19_config (this should be automated in a setup script at some point...)
# example settings:
#export PATH="/opt/mozilla:/opt/miniconda/bin:$PATH"
#export REPO_BASE="${HOME}/repos/edification"
#export STAGE_BASE="${HOME}/datasets/covid19"
#export LOG_DIR="${HOME}/covid19"
#export COVID_BASE="${REPO_BASE}/covid19"

export DMODE=${3:-0}
target_wd=$1
export COVID_BASE="${REPO_BASE}/${target_wd}"
export PYTHONPATH="${PYTHONPATH}:${COVID_BASE}"
cd $COVID_BASE
curr_branch=`git branch | grep '* ' | awk '{print $2}'`
log_name="covid19_dashboard_${curr_branch}"
cleanup_log_name="covid19_cleanup_${curr_branch}"
dash_logfile="${LOG_DIR}/dashboard_generation_logs/${log_name}.out"
cleanup_logfile="${LOG_DIR}/covid19_cleanup_logs/${cleanup_log_name}.out"
dash_date=`date +%Y%m%d`
dash_gen_instance=`date +%Y%m%d%H%M%S`
export STAGE_DIR="${STAGE_BASE}/${curr_branch}"
if [[ ! -d "${STAGE_DIR}" ]]; then
  echo "creating stage dir '${STAGE_DIR} since it did not previously exist" >> $dash_logfile
  mkdir -p "${STAGE_DIR}"
else
  echo "staging all intermediate files to stage dir: ${STAGE_DIR}"  >> $dash_logfile
fi
commit_dashboard ()
{
cd $REPO_BASE
java -jar /opt/bfg.jar --delete-files '*.{png,jpg,csv,gz,gif,html,svg}' covid19 1>>$cleanup_logfile 2>&1
cd $COVID_BASE
/usr/bin/git checkout master
/usr/bin/git reflog expire --expire=now --all && git gc --prune=now --aggressive
/usr/bin/git add .
/usr/bin/git commit --allow-empty -m "autogenerated dashboard update for ${dash_date}" >> $dash_logfile
}
push_dashboard ()
{
cd $COVID_BASE
/usr/bin/git push -f origin master >> $dash_logfile 2>&1
}
lock_file=/tmp/covid19.lock
if test -f $lock_file
then
  echo "Lock file ${lock_file} exists, abandoning daemon startup"
  kill -s 2 $$
else
  touch $lock_file
fi
source /opt/miniconda/etc/profile.d/conda.sh
target_env=$2
conda activate $target_env
if test -f $dash_logfile
 then
    
    mv $dash_logfile ${dash_logfile}_${dash_gen_instance}.bkp
fi
if test -f $cleanup_logfile
 then
    mv $cleanup_logfile ${cleanup_logfile}_${dash_gen_instance}.bkp
fi
if [[ $DMODE == 0 ]]; then
  echo "Beginning dashboard generation mode in manual mode " >> $dash_logfile
else
  echo "Beginning dashboard generation mode in daemon mode " >> $dash_logfile
fi
/opt/miniconda/envs/${target_env}/bin/python ${COVID_BASE}/c19_analysis/dataprep_flow.py 1>>$dash_logfile 2>&1
dash_gen=$?
if [[ ${dash_gen} -eq 0 ]]; then
  if [[ "${curr_branch}" == "master"  ]]; then
    echo "Proceeding with committing of autogenerated dashboard for ${dash_date}" >> $dash_logfile
    commit_dashboard
    push_dashboard
  else
    echo "Autogenerated dashboard succeeded for ${dash_date} but was not on branch 'master' so not pushing" >> $dash_logfile
    commit_dashboard
  fi
  echo "Cleaning up old logs" >> $dash_logfile
  find "${LOG_DIR}/dashboard_generation_logs" -type f -name "covid19_dashboard_*.bkp" -mtime +7 -print | xargs rm -rf
  find "${LOG_DIR}/covid19_cleanup_logs" -type f -name "covid19_cleanup_*.bkp" -mtime +7 -print | xargs rm -rf
  rm $lock_file
else
  echo "Dashboard generation failed. Output code was: ${dash_gen}" >> $dash_logfile
  /opt/miniconda/envs/${target_env}/bin/python ${COVID_BASE}/notification.py -f "${dash_logfile}" -s "Dashboard Generation Failure"
  rm $lock_file
  exit 1
fi