Upload scores

Makefile

@@ -1,22 +1,40 @@
 .DEFAULT_GOAL:=build
+S3_URL=https://forecast-eval.s3.us-east-2.amazonaws.com
+S3_BUCKET=s3://forecast-eval
 
 build: score_forecast
 
 r_build:
 	docker build -t forecast-eval-build docker_build
 
+predictions_cards.rds score_cards_state_deaths.rds score_cards_state_cases.rds :
+	test -f dist/$@ || wget --directory-prefix=dist $(S3_URL)/$@
+
+pull_data: predictions_cards.rds score_cards_state_deaths.rds score_cards_state_cases.rds
+
 dist:
 	mkdir $@
 
 clean:
 	rm -rf dist
-	rm dashboard/*.rds
 
-score_forecast: r_build dist
-	docker run --rm -v ${PWD}/Report:/var/forecast-eval -w /var/forecast-eval forecast-eval-build Rscript create_reports.R
+score_forecast: r_build dist pull_data
+	docker run --rm \
+		-v ${PWD}/Report:/var/forecast-eval \
+		-v ${PWD}/dist:/var/dist \
+		-w /var/forecast-eval \
+		forecast-eval-build \
+		Rscript create_reports.R --dir /var/dist
+
+deploy: score_forecast
+	aws s3 cp dist/ $(S3_BUCKET)/ --recursive --exclude "*" --include "*rds"
 
 start_repl: r_build
-	docker run -ti --rm forecast-eval-build bash
+	docker run -ti --rm \
+		-v ${PWD}/Report:/var/forecast-eval \
+		-v ${PWD}/dist:/var/dist \
+		-w /var/forecast-eval \
+		forecast-eval-build bash
 
 start_dashboard:
 	cp Report/*.rds dashboard

Report/create_reports.R

@@ -1,6 +1,32 @@
+#!/usr/bin/env Rscript
+library("optparse")
+library("dplyr")
+
+option_list = list(
+    make_option(
+        c("-d", "--dir"), 
+        type="character", 
+        default=".", 
+        help="Directory to read/write data", 
+        metavar="character"
+    )
+);
+
+opt_parser = OptionParser(option_list=option_list);
+opt = parse_args(opt_parser);
+
+prediction_cards_filename = "predictions_cards.rds"
+prediction_cards_filepath = case_when(
+    !is.null(opt$dir) ~ file.path(opt$dir,prediction_cards_filename),
+    TRUE~prediction_cards_filename
+)
+
 source("predictions.R")
-create_prediction_cards()
+create_prediction_cards(prediction_cards_filepath)
 
 source("score.R")
-create_score_cards("state", signal = "confirmed_incidence_num")
-create_score_cards("state", signal = "deaths_incidence_num")
+print("Scoring confirmed incidence...")
+create_score_cards(prediction_cards_filepath, "state", signal = "confirmed_incidence_num", output_dir=opt$dir)
+print("Scoring deaths incidence...")
+create_score_cards(prediction_cards_filepath, "state", signal = "deaths_incidence_num", output_dir=opt$dir)
+print("Done")

Report/predictions.R

@@ -1,12 +1,14 @@
 library(lubridate)
 library(evalcast)
 library(dplyr)
+library(stringr)
 
 # TODO: Use `get_covidhub_forecaster_names()` instead of listing forecasters
-create_prediction_cards = function(){
+create_prediction_cards = function(prediction_cards_filename){
   start_date = today() - 100 * 7 # last 12 weeks
 
   forecasters = get_covidhub_forecaster_names()
+  print(str_interp("Getting forecasts for ${length(forecasters)} forecasters."))
 
   # Get all forecast dates for these forecasters from COVID Hub
   forecast_dates = vector("list", length = length(forecasters))
@@ -25,10 +27,11 @@ create_prediction_cards = function(){
   # that case it's no longer a true prediction. We can always restart from scratch
   # by deleting predictions_cards.rds.
 
-  if (file.exists("predictions_cards.rds")) {
-    predictions_cards = readRDS(file = "predictions_cards.rds")
+  if (file.exists(prediction_cards_filename)) {
+    print("Reading from existing prediction cards")
+    predictions_cards = readRDS(file = prediction_cards_filename)
   }
-  if(exists("predictions_cards")){
+  if(exists(prediction_cards_filename)){
     seen_dates = predictions_cards %>% 
       distinct(forecast_date, forecaster)
   }
@@ -51,6 +54,7 @@ create_prediction_cards = function(){
     }
     new_dates[[i]] = comparable_forecast_dates
   }
+
   names(new_dates) = forecasters
 
   # Now get new predictions for each forecaster
@@ -59,7 +63,7 @@ create_prediction_cards = function(){
   deaths_sig = "deaths_incidence_num"
   cases_sig = "confirmed_incidence_num"
   for (i in 1:length(forecasters)) {
-    cat(forecasters[i], "...\n")
+    cat(str_interp("#${i}:${forecasters[i]} ...\n"))
     if (length(new_dates[[i]] > 0)){
       predictions_cards_list[[i]] = tryCatch({
         get_covidhub_predictions(forecasters[i], 
@@ -89,6 +93,6 @@ create_prediction_cards = function(){
 
   predictions_cards$data_source = "usa-facts"
   saveRDS(predictions_cards,
-          file = "predictions_cards.rds", 
+          file = prediction_cards_filename, 
           compress = "xz")
 }

Report/score.R

@@ -3,9 +3,9 @@ library("dplyr")
 library("lubridate")
 library("assertthat")
 
-create_score_cards = function(geo_type, signal_name = NULL, output_file_name = NULL){
+create_score_cards = function(prediction_cards_filepath, geo_type, signal_name = NULL, output_file_name = NULL, output_dir="."){
   if (!exists("predictions_cards")){
-    predictions_cards = readRDS("predictions_cards.rds")
+    predictions_cards = readRDS(prediction_cards_filepath)
   }
   signals = (predictions_cards %>% distinct(signal))$signal
   if (is.null(signal_name)){
@@ -29,7 +29,7 @@ create_score_cards = function(geo_type, signal_name = NULL, output_file_name = N
     } else {
       sig_suffix = "deaths"
     }
-    output_file_name = paste0("score_cards_", geo_type, "_", sig_suffix, ".rds")
+    output_file_name = file.path(output_dir,paste0("score_cards_", geo_type, "_", sig_suffix, ".rds"))
   }
   # central coverage functions named cov_10, cov_20, etc.
   central_intervals = c(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.98)

docker_build/dependencies.R

@@ -2,5 +2,7 @@
 # Other packages should be installed here
 
 install.packages("assertthat")
+install.packages("optparse")
+install.packages("doParallel")
 
 devtools::install_github("cmu-delphi/covidcast",ref = "evalcast-killcards",subdir = "R-packages/evalcast")