Survey Makefile adjustments

facebook/Dockerfile

@@ -27,6 +27,7 @@ ADD ./monthly-files.R /facebook/monthly-files.R
 ADD ./contingency_tables.R /facebook/contingency_tables.R
 ADD ./contingency-combine.R /facebook/contingency-combine.R
 ADD ./ssmtp.conf /etc/ssmtp/ssmtp.conf
+ADD ./GITREF facebook/GITREF
 WORKDIR /facebook/
 RUN make lib
 RUN make install

facebook/Makefile

@@ -31,11 +31,9 @@ RAW_DEST:="raw"
 # dry-run mode: generate all files, but do not post them anywhere, and disable all emails to outside parties.
 DRY:=yes
 ifeq ($(DRY),yes)
-	EMAIL_SEND:=echo -e "Would send mail: echo -e \"Subject: $${SUBJECT}\n\n$${MSG}\" | sendmail $(DELPHI_SURVEY_EMAIL_USER)"
 	SFTP_POST:=echo -e "Would run: sshpass -p $(DELPHI_SURVEY_SFTP_PASSWORD) sftp $(SFTP_OPTIONS) -b <(echo -e \"\$${BATCH}\") -P 2222 $(DELPHI_SURVEY_SFTP_USER)\n$${BATCH}"
 	DRY_MESSAGE:="[DRY-RUN] "
 else
-	EMAIL_SEND:=echo -e "Subject: $${SUBJECT}\n\n$${MSG}" | sendmail $(DELPHI_SURVEY_EMAIL_USER)
 	SFTP_POST:=sshpass -p $(DELPHI_SURVEY_SFTP_PASSWORD) sftp $(SFTP_OPTIONS) -b <(echo -e "$${BATCH}") -P 2222 $(DELPHI_SURVEY_SFTP_USER)
 endif
 
@@ -121,10 +119,8 @@ $(WEIGHTS): $(TODAY)
 	MAX_WEIGHTED=`$(MAX_WEIGHTED)`; \
 	EXPECTED_MAX_WEIGHTED=`date --date='$(TODAY) -3 day' +'%Y%m%d'`; \
 	if [[ $$EXPECTED_MAX_WEIGHTED -gt $$MAX_WEIGHTED ]]; then \
-	  MSG="Expected most recent file: $$EXPECTED_MAX_WEIGHTED\nActual most recent file: $$MAX_WEIGHTED"; \
-	  echo "WARNING: $${MSG}" | tr "\n" ";" >> $(MESSAGES); \
-	  SUBJECT="[fb-cmu-cvid] Weights are stale"; \
-	  $(EMAIL_SEND) ;\
+	  MSG="Expected most recent file: $$EXPECTED_MAX_WEIGHTED; Actual most recent file: $$MAX_WEIGHTED"; \
+	  echo "WARNING: $${MSG}" >> $(MESSAGES); \
 	fi
 
 dev: delphiFacebook_1.0.tar.gz

facebook/micro/monthly-archive.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+rm -f *.gz
+if [ -z $1 ]; then
+    MONTH=`date --date "last month" +"%Y_%m"`
+else
+    MONTH=$1
+fi
+echo ${MONTH}
+R_MONTH=${MONTH#*_}; R_MONTH=${R_MONTH#0}
+BATCH="cd fb-public-results\nls -1 cvid_responses_${MONTH}*.gz"
+sftp -b <(echo -e "${BATCH}") -P 2222 fb-automation@ftp.delphi.cmu.edu 2>/dev/null | \
+    grep "^cvid" | \
+    awk -F_ 'BEGIN{print "cd fb-public-results"} {key=$3 $4 $5; if (key!=last && last!="") {print record} last=key; record=$0} END{print record}' | \
+    sed '/^cvid/ s/^/get /' >fetch.sftp
+sftp -b fetch.sftp -P 2222 fb-automation@ftp.delphi.cmu.edu
+OUT=${MONTH/_/-}
+Rscript ../monthly-files.R ${MONTH%_*} ${R_MONTH} . >${OUT}.csv
+gzip ${OUT}.csv
+sftp -b <(echo -e "cd fb-public-results\nput ${OUT}.csv.gz") -P 2222 fb-automation@ftp.delphi.cmu.edu