Merge pull request #55 from ResearchLuxembourg/develop

Develop → master merge for v2.0.4

components/check_input.jl

@@ -20,7 +20,8 @@ today = Dates.today()
 today_str = Dates.format(today, "yyyymmdd")
 current_name = basename(ARGS[1])
 expected_name = "clinical_monitoring_$(today_str)_cleaned_case_and_hospital_data.xlsx"
-current_name == expected_name || @warn "Either the date in the uploaded file name is not correct or the name is not following the accepted systematization (got '$current_name', should be '$expected_name')."
+current_name == expected_name ||
+    @warn "Either the date in the uploaded file name is not correct or the name is not following the accepted systematization (got '$current_name', should be '$expected_name')."
 
 
 #
@@ -50,19 +51,47 @@ end
 #
 
 first_report, last_report = extrema(df.report_date)
+expected_first_report = Date("2020-02-24")
 
 # Check that the first date considered in database is consistent 
-@assert (first_report == Date("2020-02-24")) "Warning: Reports should start from 2020-02-24. Is the data corrupted?"
+#@assert (first_report == Date("2020-02-24")) "Warning: Reports should start from 2020-02-24. Is the data corrupted?"
+# ...we replaced this with "dates outside" check below.
 
 # Check presence of last datapoint
 last_entry = today - Day(1)
-@assert (last_report == last_entry) "Error: Missing data point of today (expected $last_entry, was: $last_report)"
+@assert last_report == last_entry "Error: Missing data point of today (expected $last_entry, was: $last_report)"
 
-since_last_entry = today - last_report
-@assert (since_last_entry < Week(1)) "Last entry is older than 1 week"
+# This served as a warning but it now completely deprecated by the above.
+#since_last_entry = today - last_report
+#@assert (since_last_entry < Week(1)) "Error: Last entry is older than 1 week"
 
 # Check date continuity
-expected_dates = [first_report + Day(i) for i = 0:Day(last_report - first_report).value]
-@assert expected_dates == sort(df.report_date) "Error: Data series not complete from beginning"
+date_counts =
+    Dict([(expected_first_report + Day(i) => 0) for i = 0:Day(last_entry - expected_first_report).value])
+dates_outside = Set{Date}()
+for d in df.report_date
+    if haskey(date_counts, d)
+        date_counts[d] += 1
+    else
+        push!(dates_outside, d)
+    end
+end
+dates_outside = collect(dates_outside)
+dates_missing = [d for (d, v) = date_counts if v == 0]
+dates_duplicate = [d for (d, v) = date_counts if v > 1]
+
+function clamp!(x)
+    length(x) <= 5 || resize!(x, 5)
+end
+
+clamp!(dates_outside)
+clamp!(dates_missing)
+clamp!(dates_duplicate)
+
+printdates(x) = join(x, ", ")
+
+@assert isempty(dates_outside) "Error: dates outside the expected interval: $(printdates(dates_outside))"
+@assert isempty(dates_missing) "Error: expected dates missing: $(printdates(dates_missing))"
+@assert isempty(dates_duplicate) "Error: duplicate entries for dates: $(printdates(dates_duplicate))"
 
 @info "Check finished OK."

components/estimate_r_eff.jl

@@ -74,33 +74,38 @@ results = rt_max .* [map(last, maxima) map(first, ranges) map(last, ranges)] ./
 dates = (start_date + Day(discard_dates)) .+ Day.(eachindex(ranges))
 
 x = DataFrame(
-        date = dates,
-        Reff_estimate = round.(results[:, 1], digits = 2),
-        Reff_50_lo = round.(results[:, 2], digits = 2),
-        Reff_50_hi = round.(results[:, 3], digits = 2),
-    )
-
-CSV.write(
-    outfile("Reff_estimate", "csv"),
-    x,
+    date = dates,
+    Reff_estimate = round.(results[:, 1], digits = 2),
+    Reff_50_lo = round.(results[:, 2], digits = 2),
+    Reff_50_hi = round.(results[:, 3], digits = 2),
 )
 
+CSV.write(outfile("Reff_estimate", "csv"), x)
+
 # Plot Reff for resident's data
 
 using CairoMakie, PlotUtils
 
 f = Figure()
-ax = Axis(f[1,1], title="Real-time effective Rt for LU")
+ax = Axis(f[1, 1], title = "Real-time effective Rt for LU")
 
 ticks = filter(d -> day(d) == 1, x.date)
 poss = datetime2rata.(x.date)
 ax.xticks[] = (datetime2rata.(ticks), Dates.format.(ticks, "yyyy-mm"))
-ax.xticklabelrotation = π/3
+ax.xticklabelrotation = π / 3
 ax.xticklabelsize = 12
 
-ylims!(0,maximum(x.Reff_estimate)*1.04)
-band!(poss, x.Reff_50_lo, x.Reff_50_hi; color="#cccccc")
-lines!(poss, x.Reff_estimate; color="black")
-scatter!(poss, x.Reff_estimate; color=x.Reff_estimate,markersize=7,colormap=:RdBu,colorrange=(2,0),strokewidth=0.5)
+ylims!(0, maximum(x.Reff_estimate) * 1.04)
+band!(poss, x.Reff_50_lo, x.Reff_50_hi; color = "#cccccc")
+lines!(poss, x.Reff_estimate; color = "black")
+scatter!(
+    poss,
+    x.Reff_estimate;
+    color = x.Reff_estimate,
+    markersize = 7,
+    colormap = :RdBu,
+    colorrange = (2, 0),
+    strokewidth = 0.5,
+)
 
-save(outfile("Reff_residents","pdf"), f)
+save(outfile("Reff_residents", "pdf"), f)

components/estimate_r_t.jl

@@ -185,8 +185,8 @@ CSV.write(
     outfile("Rt_estimate", "csv"),
     DataFrame(
         date = dates,
-        Rt_estimate = round.(X[4, begin+1:end] ./ μ, digits=2),
-        standard_deviation = round.(sqrt.(β_err) ./ μ, digits=2),
+        Rt_estimate = round.(X[4, begin+1:end] ./ μ, digits = 2),
+        standard_deviation = round.(sqrt.(β_err) ./ μ, digits = 2),
     )[
         19:end,
         :,