From 3412242d4c0e8c43064ab6552970438ed8901c38 Mon Sep 17 00:00:00 2001 From: Samuel Gratzl Date: Tue, 24 Nov 2020 09:35:29 -0500 Subject: [PATCH 01/12] build: add prettier --- .gitattributes | 131 +++++++++++++++++++++ .prettierignore | 1 + .prettierrc.js | 6 + package-lock.json | 291 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 10 +- 5 files changed, 438 insertions(+), 1 deletion(-) create mode 100644 .gitattributes create mode 100644 .prettierignore create mode 100644 .prettierrc.js diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..c5166261a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,131 @@ +# These settings are for any web project + +# Handle line endings automatically for files detected as text +# and leave all files detected as binary untouched. +* text=auto eol=lf + +# +# The above will handle all files NOT found below +# + +# +## These files are text and should be normalized (Convert crlf => lf) +# + +# source code +*.php text +*.css text +*.sass text +*.scss text +*.less text +*.styl text +*.js text +*.ts text +*.coffee text +*.json text +*.htm text +*.html text +*.xml text +*.txt text +*.ini text +*.inc text +*.pl text +*.rb text +*.py text +*.scm text +*.sql text +*.sh text eof=LF +*.bat text +*.R text + +# templates +*.hbt text +*.jade text +*.haml text +*.hbs text +*.dot text +*.tmpl text +*.phtml text + +# server config +.htaccess text + +# git config +.gitattributes text +.gitignore text + +# code analysis config +.jshintrc text +.jscsrc text +.jshintignore text +.csslintrc text + +# misc config +*.yaml text +*.yml text +*.editorconfig text +*.toml text + +# build config +*.npmignore text +*.bowerrc text +*.prettierignore text +Dockerfile text eof=LF + +# Heroku +Procfile text +.slugignore text + +# Documentation +*.md text +LICENSE text +AUTHORS text + + +# +## These files are binary and should be left untouched +# + +# (binary is a macro for -text -diff) +*.png binary +*.jpg binary +*.jpeg binary +*.gif binary +*.ico binary +*.mov binary +*.mp4 binary +*.mp3 binary +*.flv binary +*.fla binary +*.swf binary +*.gz binary +*.zip binary +*.7z binary +*.ttf binary +*.pyc binary +*.pdf binary + +# Source files +# ============ +*.pxd text +*.py text +*.py3 text +*.pyw text +*.pyx text +*.sh text eol=lf +*.json text + +# Binary files +# ============ +*.db binary +*.p binary +*.pkl binary +*.pyc binary +*.pyd binary +*.pyo binary + +# Note: .db, .p, and .pkl files are associated +# with the python modules ``pickle``, ``dbm.*``, +# ``shelve``, ``marshal``, ``anydbm``, & ``bsddb`` +# (among others). +*.rda binary diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 000000000..405ec2c3a --- /dev/null +++ b/.prettierignore @@ -0,0 +1 @@ +*.toml diff --git a/.prettierrc.js b/.prettierrc.js new file mode 100644 index 000000000..7c4e4f4d0 --- /dev/null +++ b/.prettierrc.js @@ -0,0 +1,6 @@ +module.exports = { + printWidth: 120, + semi: true, + singleQuote: true, + trailingComma: 'es5', +}; diff --git a/package-lock.json b/package-lock.json index 01b040bf4..ef1ed25ac 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,6 +4,32 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@babel/code-frame": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.10.4.tgz", + "integrity": "sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg==", + "dev": true, + "requires": { + "@babel/highlight": "^7.10.4" + } + }, + "@babel/helper-validator-identifier": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.10.4.tgz", + "integrity": "sha512-3U9y+43hz7ZM+rzG24Qe2mufW5KhvFg/NhnNph+i9mgCtdTCtMJuI1TMkrIUiK7Ix4PYlRF9I5dhqaLYA/ADXw==", + "dev": true + }, + "@babel/highlight": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.10.4.tgz", + "integrity": "sha512-i6rgnR/YgPEQzZZnbTHHuZdlE8qyoBNalD6F+q4vAFlcMEcqmkoG+mPqJYJCo63qPf74+Y1UZsl3l6f7/RIkmA==", + "dev": true, + "requires": { + "@babel/helper-validator-identifier": "^7.10.4", + "chalk": "^2.0.0", + "js-tokens": "^4.0.0" + } + }, "@fortawesome/fontawesome-free": { "version": "5.15.1", "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-free/-/fontawesome-free-5.15.1.tgz", @@ -15,6 +41,12 @@ "integrity": "sha512-ONhaKPIufzzrlNbqtWFFd+jlnemX6lJAgq9ZeiZtS7I1PIf/la7CW4m83rTXRnVnsMbW2k56pGYu7AUFJD9Pow==", "dev": true }, + "@types/parse-json": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz", + "integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==", + "dev": true + }, "ansi-styles": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", @@ -224,6 +256,12 @@ } } }, + "callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true + }, "caw": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/caw/-/caw-2.0.1.tgz", @@ -247,6 +285,12 @@ "supports-color": "^5.3.0" } }, + "ci-info": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz", + "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==", + "dev": true + }, "clone-response": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz", @@ -276,6 +320,12 @@ "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" }, + "compare-versions": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/compare-versions/-/compare-versions-3.6.0.tgz", + "integrity": "sha512-W6Af2Iw1z4CB7q4uU4hv646dW9GQuBM+YpC0UvUCWSD8w90SJjp+ujJuXaEMtAXBtSqGfMPuFOVn4/+FlaqfBA==", + "dev": true + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -315,6 +365,33 @@ "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", "dev": true }, + "cosmiconfig": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-7.0.0.tgz", + "integrity": "sha512-pondGvTuVYDk++upghXJabWzL6Kxu6f26ljFw64Swq9v6sQPUL3EUlVDV56diOjpCayKihL6hVe8exIACU4XcA==", + "dev": true, + "requires": { + "@types/parse-json": "^4.0.0", + "import-fresh": "^3.2.1", + "parse-json": "^5.0.0", + "path-type": "^4.0.0", + "yaml": "^1.10.0" + }, + "dependencies": { + "parse-json": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.1.0.tgz", + "integrity": "sha512-+mi/lmVVNKFNVyLXV31ERiy2CY5E1/F6QtJFEzoChPRwwngMNXRDQ9GJ5WdE2Z2P4AujsOi0/+2qHID68KwfIQ==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + } + } + } + }, "cross-spawn": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", @@ -665,6 +742,12 @@ "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=", "dev": true }, + "git-format-staged": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/git-format-staged/-/git-format-staged-2.1.0.tgz", + "integrity": "sha512-Ih3EVablJ1Xj/JPMzbXY3Nl0W6NQ9YrA+mad3c9yobODzq9zfOBHMi0h3AScRprm4XtBbIo1oGfwO3cZ4vmg6Q==", + "dev": true + }, "glob": { "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", @@ -762,12 +845,91 @@ "signale": "^1.4.0" } }, + "husky": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/husky/-/husky-4.3.0.tgz", + "integrity": "sha512-tTMeLCLqSBqnflBZnlVDhpaIMucSGaYyX6855jM4AguGeWCeSzNdb1mfyWduTZ3pe3SJVvVWGL0jO1iKZVPfTA==", + "dev": true, + "requires": { + "chalk": "^4.0.0", + "ci-info": "^2.0.0", + "compare-versions": "^3.6.0", + "cosmiconfig": "^7.0.0", + "find-versions": "^3.2.0", + "opencollective-postinstall": "^2.0.2", + "pkg-dir": "^4.2.0", + "please-upgrade-node": "^3.2.0", + "slash": "^3.0.0", + "which-pm-runs": "^1.0.0" + }, + "dependencies": { + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "chalk": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", + "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, "ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", "dev": true }, + "import-fresh": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.2.tgz", + "integrity": "sha512-cTPNrlvJT6twpYy+YmKUKrTSjWFs3bjYjAhCwm+z4EOCubZxAuO+hHpRN64TqjEaYSHs7tJAE0w1CKMGmsG/lw==", + "dev": true, + "requires": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + } + }, "import-lazy": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/import-lazy/-/import-lazy-3.1.0.tgz", @@ -864,6 +1026,12 @@ "is-object": "^1.0.1" } }, + "js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true + }, "json-buffer": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.0.tgz", @@ -876,6 +1044,12 @@ "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", "dev": true }, + "json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "dev": true + }, "katex": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/katex/-/katex-0.12.0.tgz", @@ -893,6 +1067,12 @@ "json-buffer": "3.0.0" } }, + "lines-and-columns": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.1.6.tgz", + "integrity": "sha1-HADHQ7QzzQpOgHWPe2SldEDZ/wA=", + "dev": true + }, "load-json-file": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-5.3.0.tgz", @@ -1046,6 +1226,12 @@ "wrappy": "1" } }, + "opencollective-postinstall": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.3.tgz", + "integrity": "sha512-8AV/sCtuzUeTo8gQK5qDZzARrulB3egtLzFgteqB2tcT4Mw7B8Kt7JcDHmltjz6FOAHsvTevk70gZEbhM4ZS9Q==", + "dev": true + }, "os-filter-obj": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/os-filter-obj/-/os-filter-obj-2.0.0.tgz", @@ -1115,6 +1301,15 @@ "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", "dev": true }, + "parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "requires": { + "callsites": "^3.0.0" + } + }, "parse-json": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", @@ -1143,6 +1338,12 @@ "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=", "dev": true }, + "path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "dev": true + }, "pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", @@ -1180,12 +1381,72 @@ "load-json-file": "^5.2.0" } }, + "pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dev": true, + "requires": { + "find-up": "^4.0.0" + }, + "dependencies": { + "find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "requires": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + } + }, + "locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "requires": { + "p-locate": "^4.1.0" + } + }, + "p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "requires": { + "p-limit": "^2.2.0" + } + }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true + } + } + }, + "please-upgrade-node": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/please-upgrade-node/-/please-upgrade-node-3.2.0.tgz", + "integrity": "sha512-gQR3WpIgNIKwBMVLkpMUeR3e1/E1y42bqDQZfql+kDeXd8COYfM8PQA4X6y7a8u9Ua9FHmsrrmirW2vHs45hWg==", + "dev": true, + "requires": { + "semver-compare": "^1.0.0" + } + }, "prepend-http": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz", "integrity": "sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc=", "dev": true }, + "prettier": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.2.0.tgz", + "integrity": "sha512-yYerpkvseM4iKD/BXLYUkQV5aKt4tQPqaGW6EsZjzyu0r7sVZZNPJW4Y8MyKmicp6t42XUPcBVA+H6sB3gqndw==", + "dev": true + }, "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -1248,6 +1509,12 @@ } } }, + "resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true + }, "responselike": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/responselike/-/responselike-1.0.2.tgz", @@ -1287,6 +1554,12 @@ "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", "dev": true }, + "semver-compare": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", + "integrity": "sha1-De4hahyUGrN+nvsXiPavxf9VN/w=", + "dev": true + }, "semver-regex": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/semver-regex/-/semver-regex-2.0.0.tgz", @@ -1407,6 +1680,12 @@ } } }, + "slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true + }, "sort-keys": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-1.1.2.tgz", @@ -1589,6 +1868,12 @@ "isexe": "^2.0.0" } }, + "which-pm-runs": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.0.0.tgz", + "integrity": "sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs=", + "dev": true + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -1614,6 +1899,12 @@ "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=", "dev": true }, + "yaml": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.0.tgz", + "integrity": "sha512-yr2icI4glYaNG+KWONODapy2/jDdMSDnrONSjblABjD9B4Z5LgiircSt8m8sRZFNi08kG9Sm0uSHtEmP3zaEGg==", + "dev": true + }, "yauzl": { "version": "2.10.0", "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", diff --git a/package.json b/package.json index 6bb0f54af..05c19e43c 100644 --- a/package.json +++ b/package.json @@ -7,12 +7,20 @@ "www-covidcast": "github:cmu-delphi/www-covidcast#dev" }, "devDependencies": { + "git-format-staged": "^2.1.0", "hugo-bin": "^0.66.2", - "ncp": "^2.0.0" + "husky": "^4.3.0", + "ncp": "^2.0.0", + "prettier": "^2.2.0" }, "hugo-bin": { "buildTags": "extended" }, + "husky": { + "hooks": { + "pre-commit": "git-format-staged -f 'prettier --ignore-unknown --stdin --stdin-filepath \"{}\"' ." + } + }, "scripts": { "copy_fonts": "ncp node_modules/katex/dist/fonts themes/delphi/static/css/fonts/", "copy_covidcast": "ncp node_modules/www-covidcast/public static/covidcast/", From 9a20ae04e17b67530a90fd4d7d446c49954aebce Mon Sep 17 00:00:00 2001 From: Samuel Gratzl Date: Tue, 24 Nov 2020 09:51:14 -0500 Subject: [PATCH 02/12] build: configure prettier --- .prettierignore | 25 +++++++++++++++++++ .prettierrc.js | 15 +++++++++++ layouts/shortcodes/team.html | 2 +- package-lock.json | 6 +++++ package.json | 7 ++++-- .../layouts/partials/landing/latest-news.html | 5 ---- 6 files changed, 52 insertions(+), 8 deletions(-) diff --git a/.prettierignore b/.prettierignore index 405ec2c3a..1c3b03593 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1 +1,26 @@ *.toml +*.jpg +*.png +*.rda +*.rdx +*.RData +*.svg +/blogdown +/.htaccess +/.Rhistory +.gitignore +.prettierignore +/.gitattributes +/LICENSE +/package-lock.json +/content/blog/**/*.html +/static/blog/**/* +/public +/Dockerfile +/resources +/static/covidcast +/static/rmarkdown-libs +*.woff +*.ttf +*.woff2 +*.R \ No newline at end of file diff --git a/.prettierrc.js b/.prettierrc.js index 7c4e4f4d0..f54fb843a 100644 --- a/.prettierrc.js +++ b/.prettierrc.js @@ -3,4 +3,19 @@ module.exports = { semi: true, singleQuote: true, trailingComma: 'es5', + + overrides: [ + { + files: ['*.html'], + options: { + parser: 'go-template', + }, + }, + { + files: ['*.Rmd'], + options: { + parser: 'markdown', + }, + }, + ], }; diff --git a/layouts/shortcodes/team.html b/layouts/shortcodes/team.html index d13660e64..a4bead94e 100644 --- a/layouts/shortcodes/team.html +++ b/layouts/shortcodes/team.html @@ -3,7 +3,7 @@ {{ range where .Page.Params.team ".team" (.Get "team") }} {{ $img := $images.GetMatch (path.Join "images" .image) }}
- {{ .name }} + {{ .name }}
{{ .name }}
{{ .affiliation }}
{{ if isset . "note" }} diff --git a/package-lock.json b/package-lock.json index ef1ed25ac..e07e93edf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1447,6 +1447,12 @@ "integrity": "sha512-yYerpkvseM4iKD/BXLYUkQV5aKt4tQPqaGW6EsZjzyu0r7sVZZNPJW4Y8MyKmicp6t42XUPcBVA+H6sB3gqndw==", "dev": true }, + "prettier-plugin-go-template": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/prettier-plugin-go-template/-/prettier-plugin-go-template-0.0.10.tgz", + "integrity": "sha512-TaHPqiMK/zfk+YhvKRf/1WZDgQ6ffnlxJZX5rwphqfxBOVEezZQtYistTB348MKrKnnwKpyXZWpMRo0/KCVB+A==", + "dev": true + }, "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", diff --git a/package.json b/package.json index 05c19e43c..24ca94767 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,8 @@ "hugo-bin": "^0.66.2", "husky": "^4.3.0", "ncp": "^2.0.0", - "prettier": "^2.2.0" + "prettier": "^2.2.0", + "prettier-plugin-go-template": "0.0.10" }, "hugo-bin": { "buildTags": "extended" @@ -28,7 +29,9 @@ "build:blog": "Rscript -e \"blogdown::build_site(local=FALSE, run_hugo=FALSE, build_rmd=TRUE)\"", "build": "hugo --gc --minify", "start": "hugo server -D", - "start:blog": "Rscript -e \"blogdown::serve_site()\"" + "start:blog": "Rscript -e \"blogdown::serve_site()\"", + "format": "prettier **/* --write", + "lint": "prettier **/* --check" }, "name": "www-main", "version": "0.1.0" diff --git a/themes/delphi/layouts/partials/landing/latest-news.html b/themes/delphi/layouts/partials/landing/latest-news.html index a54fa7825..3dfc40c2f 100644 --- a/themes/delphi/layouts/partials/landing/latest-news.html +++ b/themes/delphi/layouts/partials/landing/latest-news.html @@ -5,11 +5,6 @@

Latest News

{{ return (dict "source" "blog" "image" .Params.heroImageThumb "title" .Title "link" .RelPermalink "date" .PublishDate ) }} {{ end }} {{ $items := apply (.Site.GetPage "/blog").Pages "partial" "latest-blog" "."}} - - {{ $top := 6 }} From 5627d452fa68f25183b4fcdc936d7497f129d51d Mon Sep 17 00:00:00 2001 From: Samuel Gratzl Date: Tue, 24 Nov 2020 10:05:45 -0500 Subject: [PATCH 03/12] refactor: run prettier --- .github/workflows/ci.yaml | 14 +- .github/workflows/ci_fast.yaml | 12 +- .prettierrc.js | 11 +- _output.yml | 2 +- content/about/_index.md | 14 +- content/bibliography.md | 2 +- content/blog/2015-07-23-template-post.Rmd | 27 +- content/blog/2020-08-10-hello-world.Rmd | 92 +-- content/blog/2020-08-26-fb-survey.Rmd | 236 +++---- content/blog/2020-08-28-api.Rmd | 286 ++++----- content/blog/2020-09-18-google-survey.Rmd | 322 +++++----- content/blog/2020-09-21-forecast-demo.Rmd | 600 +++++++++--------- content/blog/2020-10-06-survey-wave-4.Rmd | 17 +- content/blog/2020-10-14-dv-signal.Rmd | 88 +-- content/covidcast/_index.md | 1 - content/covidcast/indicators/cases.md | 2 +- content/covidcast/indicators/combined.md | 2 +- content/covidcast/indicators/deaths.md | 2 +- content/covidcast/indicators/doctor-visits.md | 2 +- content/covidcast/indicators/google-trends.md | 2 +- .../indicators/hospital-admissions.md | 2 +- content/covidcast/indicators/index.md | 2 +- content/covidcast/indicators/quidel-flu.md | 2 +- content/covidcast/indicators/quidel.md | 2 +- content/covidcast/indicators/safegraph.md | 2 +- content/covidcast/indicators/survey-google.md | 2 +- .../indicators/symptoms-community-fb.md | 2 +- content/covidcast/indicators/symptoms-fb.md | 2 +- content/covidcast/methodology.md | 20 +- content/covidcast/release-log/_index.md | 2 +- .../covidcast/release-log/headless/index.md | 2 +- .../covidcast/release-log/headless/v1.1.md | 1 - .../covidcast/release-log/headless/v1.2.md | 2 +- content/covidcast/surveys.md | 5 - content/covidcast/terms-of-use.md | 4 +- content/flu.md | 12 +- content/news/2016_12.md | 2 +- content/news/2017_10.md | 2 +- content/news/2018_11.md | 2 +- content/news/2019_10_CenterOfExcellence.md | 2 +- content/news/2020_03_covid.md | 2 +- content/news/2020_04_covidcast.md | 4 +- content/news/2020_10_fellows.md | 2 +- content/news/index.md | 2 +- content/systems/crowdcast.md | 2 +- content/systems/forecast.md | 2 +- content/systems/index.md | 2 +- content/systems/nowcast.md | 2 +- content/tools/epidata.md | 2 +- content/tools/epiforecast.md | 2 +- content/tools/epivis.md | 2 +- content/tools/fluscores.md | 2 +- content/tools/index.md | 2 +- content/tools/utils.md | 2 +- data/authors.yaml | 2 +- data/bibliography.yaml | 2 +- data/research.yaml | 4 +- docker-compose.yml | 12 +- environment.yml | 4 +- no-deploy.json | 5 +- themes/delphi/README.md | 23 +- themes/delphi/archetypes/default.md | 2 +- themes/delphi/assets/css/_customize.scss | 5 +- themes/delphi/assets/css/blog_extra.scss | 4 +- .../assets/css/components/_arrow_link.scss | 32 +- .../assets/css/components/_card_grid.scss | 65 +- .../assets/css/components/_font_awesome.scss | 10 +- .../assets/css/components/_latest_card.scss | 49 +- themes/delphi/assets/css/layout/_content.scss | 11 +- .../assets/css/layout/_header_footer.scss | 69 +- themes/delphi/assets/css/layout/_layouts.scss | 38 +- themes/delphi/assets/css/main.scss | 32 +- themes/delphi/assets/css/pages/_about.scss | 80 ++- themes/delphi/assets/css/pages/_blog.scss | 84 ++- .../delphi/assets/css/pages/_covidcast.scss | 3 +- themes/delphi/assets/css/pages/_landing.scss | 129 ++-- themes/delphi/assets/css/pages/_team.scss | 6 +- themes/delphi/assets/js/blog/codeFolding.js | 24 +- themes/delphi/assets/js/blog/index.js | 8 +- themes/delphi/assets/js/main.js | 4 +- themes/delphi/layouts/404.html | 8 +- themes/delphi/layouts/_default/about.html | 39 +- themes/delphi/layouts/_default/baseof.html | 19 +- themes/delphi/layouts/_default/section.html | 41 +- themes/delphi/layouts/_default/single.html | 6 +- themes/delphi/layouts/_default/taxonomy.html | 39 +- themes/delphi/layouts/_default/team.html | 30 +- themes/delphi/layouts/_default/terms.html | 12 +- .../delphi/layouts/_internal/pagination.html | 82 +-- themes/delphi/layouts/blog/list.html | 77 ++- themes/delphi/layouts/blog/single.html | 96 +-- .../delphi/layouts/covidcast_app/baseof.html | 15 +- themes/delphi/layouts/index.html | 6 +- themes/delphi/layouts/landing.html | 151 +++-- .../partials/about/collaborator-img.html | 4 +- .../layouts/partials/about/collaborators.html | 31 +- .../partials/about/research-papers.html | 31 +- .../delphi/layouts/partials/arrow-link.html | 8 +- themes/delphi/layouts/partials/blog/card.html | 33 +- .../layouts/partials/blog/latestblogs.html | 20 +- themes/delphi/layouts/partials/blog/tag.html | 4 +- themes/delphi/layouts/partials/blog/tags.html | 6 +- .../layouts/partials/delphi-text-logo.html | 2 +- .../delphi/layouts/partials/font-awesome.html | 4 +- themes/delphi/layouts/partials/footer.html | 8 +- .../layouts/partials/footer/desktop.html | 66 +- .../delphi/layouts/partials/footer/legal.html | 15 +- themes/delphi/layouts/partials/head.html | 58 +- themes/delphi/layouts/partials/header.html | 10 +- .../layouts/partials/landing/latest-card.html | 22 +- .../layouts/partials/landing/latest-news.html | 45 +- .../layouts/partials/menu/breadcrumb.html | 22 +- themes/delphi/layouts/partials/menu/item.html | 6 +- themes/delphi/layouts/partials/nav.html | 52 +- themes/delphi/layouts/partials/share.html | 53 +- themes/delphi/layouts/partials/social.html | 82 +-- themes/delphi/layouts/section/archives.html | 39 +- themes/delphi/layouts/shortcodes/apiref.html | 2 +- .../layouts/shortcodes/bibliography.html | 25 +- .../delphi/layouts/shortcodes/indicators.html | 17 +- themes/delphi/layouts/shortcodes/news.html | 5 +- .../delphi/layouts/shortcodes/releaselog.html | 17 +- themes/delphi/layouts/shortcodes/systems.html | 13 +- themes/delphi/layouts/shortcodes/team.html | 19 +- themes/delphi/layouts/shortcodes/tools.html | 5 +- 125 files changed, 1952 insertions(+), 1936 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fb2a77da9..034f7dde1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,9 +1,9 @@ on: push: paths: # run only when an Rmd file changes - - '**.Rmd' - - 'environment.yml' - - '.github/workflows/ci.yaml' + - "**.Rmd" + - "environment.yml" + - ".github/workflows/ci.yaml" name: ci @@ -17,7 +17,7 @@ jobs: fetch-depth: 3 - name: Cache Conda uses: actions/cache@v1 - with: + with: path: ~/conda_pkgs_dir key: ${{ runner.os }}-conda6-${{ hashFiles('environment.yml') }} restore-keys: | @@ -50,13 +50,13 @@ jobs: restore-keys: | ${{ runner.os }}-blogdown2- - name: Build site - shell: bash -l {0} + shell: bash -l {0} run: | npm run build:blog - + - uses: actions/setup-node@v1 with: - node-version: '12' + node-version: "12" - uses: actions/cache@v2 with: path: ~/.npm diff --git a/.github/workflows/ci_fast.yaml b/.github/workflows/ci_fast.yaml index 9f43d90c3..19f8c81aa 100644 --- a/.github/workflows/ci_fast.yaml +++ b/.github/workflows/ci_fast.yaml @@ -1,9 +1,9 @@ on: push: paths-ignore: # don't run the fast version when an Rmd file changes - - '**.Rmd' - - 'environment.yml' - - '.github/workflows/ci.yaml' + - "**.Rmd" + - "environment.yml" + - ".github/workflows/ci.yaml" name: ci_fast @@ -14,11 +14,11 @@ jobs: - uses: actions/checkout@v2 with: # submodules: true # Fetch Hugo themes (true OR recursive) - fetch-depth: 3 # Fetch all history for .GitInfo and .Lastmod - + fetch-depth: 3 # Fetch all history for .GitInfo and .Lastmod + - uses: actions/setup-node@v1 with: - node-version: '12' + node-version: "12" - uses: actions/cache@v2 with: path: ~/.npm diff --git a/.prettierrc.js b/.prettierrc.js index f54fb843a..905cf6067 100644 --- a/.prettierrc.js +++ b/.prettierrc.js @@ -1,20 +1,19 @@ module.exports = { printWidth: 120, semi: true, - singleQuote: true, - trailingComma: 'es5', + trailingComma: "es5", overrides: [ { - files: ['*.html'], + files: ["*.html"], options: { - parser: 'go-template', + parser: "go-template", }, }, { - files: ['*.Rmd'], + files: ["*.Rmd"], options: { - parser: 'markdown', + parser: "markdown", }, }, ], diff --git a/_output.yml b/_output.yml index 9005ea665..c3455883a 100644 --- a/_output.yml +++ b/_output.yml @@ -1,3 +1,3 @@ blogdown::html_page: # force svglite device, since cairo is a mess - dev: "svglite" \ No newline at end of file + dev: svglite diff --git a/content/about/_index.md b/content/about/_index.md index add6e362e..b7fbb95f8 100644 --- a/content/about/_index.md +++ b/content/about/_index.md @@ -5,7 +5,7 @@ layout: about ### About Delphi -Since 2012, we've been developing a [digital ecosystem to support](https://docs.google.com/presentation/d/13xwrzW17i1Hn_OvEe-8Ha8XZmexVkkIY22h1MO9QmcY/edit?usp=sharing) epidemic tracking and forecasting. Pre-pandemic our focus was influenza; now it's COVID. We procure unique data streams that reflect COVID activity (from healthcare partners, tech companies, and massive national surveys), extract COVID-related indicators, and make these publicly and continuously available. These indicators are then used for nowcasting (situational awareness) and short-term forecasting. +Since 2012, we've been developing a [digital ecosystem to support](https://docs.google.com/presentation/d/13xwrzW17i1Hn_OvEe-8Ha8XZmexVkkIY22h1MO9QmcY/edit?usp=sharing) epidemic tracking and forecasting. Pre-pandemic our focus was influenza; now it's COVID. We procure unique data streams that reflect COVID activity (from healthcare partners, tech companies, and massive national surveys), extract COVID-related indicators, and make these publicly and continuously available. These indicators are then used for nowcasting (situational awareness) and short-term forecasting. ### Who is our audience? @@ -13,16 +13,16 @@ Public health authorities (federal, state, local), fellow researchers (working o ### What has our impact been? -Since 2013, we've supported CDC's Influenza Division in advancing and growing a scientific community around flu forecasting. We've been perennial leaders in forecasting accuracy, and started providing flu nowcasts to state departments of health in 2016. +Since 2013, we've supported CDC's Influenza Division in advancing and growing a scientific community around flu forecasting. We've been perennial leaders in forecasting accuracy, and started providing flu nowcasts to state departments of health in 2016. Since 2019, we've been working directly with CDC as a National Center of Excellence for Influenza Forecasting (a 5-year designation). -Since March 2020, we've created and maintained the nation's largest public repository of diverse, geographically-detailed, real-time indicators of COVID activity in the U.S. Our indicators cover every rung of the severity pyramid, and they're used regularly by public health officials, DoD personnel, data journalists, healthcare companies, financial firms, fellow modelers, and COVID response groups. We've made our indicators freely available through a public API, with R and Python packages for easy access. +Since March 2020, we've created and maintained the nation's largest public repository of diverse, geographically-detailed, real-time indicators of COVID activity in the U.S. Our indicators cover every rung of the severity pyramid, and they're used regularly by public health officials, DoD personnel, data journalists, healthcare companies, financial firms, fellow modelers, and COVID response groups. We've made our indicators freely available through a public API, with R and Python packages for easy access. -Several of the underlying data sources (on which these indicators are built) would not exist or be publicly available without our efforts. This includes: +Several of the underlying data sources (on which these indicators are built) would not exist or be publicly available without our efforts. This includes: -A massive national daily survey we're running in partnership with Facebook. This has reached over 12 million Americans since April, providing real-time insights into, e.g., self-reported symptoms, mask wearing, testing, and contacts, broken down by various demographics. Survey data on symptoms and testing can serve as early indicators of COVID activity while leapfrogging over public health reporting systems; data on mask wearing and contacts can guide policy and outreach. +A massive national daily survey we're running in partnership with Facebook. This has reached over 12 million Americans since April, providing real-time insights into, e.g., self-reported symptoms, mask wearing, testing, and contacts, broken down by various demographics. Survey data on symptoms and testing can serve as early indicators of COVID activity while leapfrogging over public health reporting systems; data on mask wearing and contacts can guide policy and outreach. -An enormous historical and real-time database of de-identified medical insurance claims, covering more than half the US population, made possible through health system partners including Change Healthcare. We use this to produce a new syndromic COVID-19 indicator based on doctor visits, as well as other indicators based on hospitalizations and ICU admissions. +An enormous historical and real-time database of de-identified medical insurance claims, covering more than half the US population, made possible through health system partners including Change Healthcare. We use this to produce a new syndromic COVID-19 indicator based on doctor visits, as well as other indicators based on hospitalizations and ICU admissions. -Since April 2020, we've been supporting and advising the CDC in their community-driven COVID forecasting effort (e.g., we've helped create and evaluate an ensemble forecast from the 70+ forecasts under submission, which is the basis for the CDC's official forecast reports). Since June 2020, we've been contributing our own short-term forecasts of COVID-19 cases and deaths to this community effort. +Since April 2020, we've been supporting and advising the CDC in their community-driven COVID forecasting effort (e.g., we've helped create and evaluate an ensemble forecast from the 70+ forecasts under submission, which is the basis for the CDC's official forecast reports). Since June 2020, we've been contributing our own short-term forecasts of COVID-19 cases and deaths to this community effort. diff --git a/content/bibliography.md b/content/bibliography.md index 92df93b31..d96bf75a8 100644 --- a/content/bibliography.md +++ b/content/bibliography.md @@ -5,4 +5,4 @@ description: Developing the Theory and Practice of Epidemiological Forecasting We found the following publications to be particularly relevant to epi-forcasting. This list is work-in-progress and not meant to ever be exhaustive. We share it here in the hope that anyone looking for recent epi-forecasting literature will have a place to start. If there is a particular publication that you think ought to be included, please [let us know](mailto:dfarrow@andrew.cmu.edu). -{{}} \ No newline at end of file +{{}} diff --git a/content/blog/2015-07-23-template-post.Rmd b/content/blog/2015-07-23-template-post.Rmd index 3d6b22312..2afbb416a 100644 --- a/content/blog/2015-07-23-template-post.Rmd +++ b/content/blog/2015-07-23-template-post.Rmd @@ -5,13 +5,12 @@ date: 2015-07-23 tags: ["R Markdown", "plot", "regression"] draft: true authors: -- frida + - frida heroImage: /blog/images/blog-lg-img_hello-world.png heroImageThumb: /blog/images/blog-thumb-img_hello-world.png related: -- 2015-07-23-template-post + - 2015-07-23-template-post acknowledgements: Test - --- ```{r setup, include=FALSE} @@ -20,7 +19,7 @@ knitr::opts_chunk$set(collapse = TRUE) Each blog post is an R Markdown document. For more details on using R Markdown see . The first paragraph or so will appear on the -front page; by default the first 70 words are used. If you want to specify the +front page; by default the first 70 words are used. If you want to specify the exact text, you can specify a `summary` front matter variable. Usually we do not include links in the teaser. @@ -78,13 +77,13 @@ obtained data or set up a server or developed a package you use. Each post can be tagged, as you can see in the metadata block at the top. I suggest we consider the following tags as base tags: -* forecasting -* nowcasting -* symptom surveys -* medical records -* COVIDcast API -* COVIDcast map -* data sources (for everthing else than symptom surveys and medical records) -* news (for announcements of new features, new models, etc.) -* R (for posts containing R, typically, our covidcast R package) -* Python (for posts containing Python, typically, our covicast Python package) \ No newline at end of file +- forecasting +- nowcasting +- symptom surveys +- medical records +- COVIDcast API +- COVIDcast map +- data sources (for everthing else than symptom surveys and medical records) +- news (for announcements of new features, new models, etc.) +- R (for posts containing R, typically, our covidcast R package) +- Python (for posts containing Python, typically, our covicast Python package) diff --git a/content/blog/2020-08-10-hello-world.Rmd b/content/blog/2020-08-10-hello-world.Rmd index 3d4ee79cf..c2816211d 100644 --- a/content/blog/2020-08-10-hello-world.Rmd +++ b/content/blog/2020-08-10-hello-world.Rmd @@ -4,8 +4,8 @@ author: "Roni Rosenfeld and Ryan Tibshirani" date: 2020-08-10 tags: ["COVIDcast"] authors: -- roni -- ryan + - roni + - ryan heroImage: /blog/images/blog-lg-img_hello-world.png heroImageThumb: /blog/images/blog-thumb-img_hello-world.png summary: | @@ -24,12 +24,12 @@ summary: | Hello from the Delphi research group at Carnegie Mellon University! We're a group of faculty, students, and staff, based primarily out of CMU together with strong collaborators from other universities and industry. -Our group was founded in 2012 to advance the theory and practice of epidemic -forecasting. Since March 2020, we have refocused efforts towards helping combat +Our group was founded in 2012 to advance the theory and practice of epidemic +forecasting. Since March 2020, we have refocused efforts towards helping combat the COVID-19 pandemic, by supporting informed decision-making at federal, state, and local levels of government and in the healthcare sector. Until now, we've -been pretty "heads down" with our work, and slow to communicate what we've been -up to. But at last ... Delphi finally has a blog! This first post serves as an +been pretty "heads down" with our work, and slow to communicate what we've been +up to. But at last ... Delphi finally has a blog! This first post serves as an introduction of sorts. Future posts will dive deeper into our various projects. ## A Little Bit About Us @@ -86,40 +86,40 @@ epidemic forecasting, since March 2020 our goals are to help combat the COVID-19 pandemic and save lives and livelihoods. We aim to support informed decision-making at federal, state, and local levels of government and in the healthcare sector. -Whenever possible, we strive to make our work useful -to the private and public sectors, other researchers, +Whenever possible, we strive to make our work useful +to the private and public sectors, other researchers, the press, and the general public. Our strategy: 1. Improve pandemic situational awareness and understanding -by providing comprehensive, geographically-detailed, -and continuously-updated indicators of pandemic activity and its impact, -helping to make meaning out of the pandemic information deluge. - -2. Support local, state, and federal governments' -ongoing decision-making in their attempts to balance -public health concerns with economic preservation, -by providing validated, verifiable, localized, -short-term forecasts of epidemic spread and healthcare demand, -under any assumed level of the local population's mobility and distancing -behavior. - -3. Analyze and demonstrate the impacts of governments' -past tightening or loosening of mitigation measures -(e.g., opening or closing schools or businesses, -imposing or lifting bans on gatherings, shelter in place orders, etc.) -on a population's mobility and distancing behavior. - -4. Engage continuously with our target users to communicate -our findings, and inform our directions and priorities. + by providing comprehensive, geographically-detailed, + and continuously-updated indicators of pandemic activity and its impact, + helping to make meaning out of the pandemic information deluge. + +2. Support local, state, and federal governments' + ongoing decision-making in their attempts to balance + public health concerns with economic preservation, + by providing validated, verifiable, localized, + short-term forecasts of epidemic spread and healthcare demand, + under any assumed level of the local population's mobility and distancing + behavior. + +3. Analyze and demonstrate the impacts of governments' + past tightening or loosening of mitigation measures + (e.g., opening or closing schools or businesses, + imposing or lifting bans on gatherings, shelter in place orders, etc.) + on a population's mobility and distancing behavior. + +4. Engage continuously with our target users to communicate + our findings, and inform our directions and priorities. 5. Make our products useful and accessible to other -researchers and tool developers, to amplify their impact. -To that end, we'll continue to make everything we invent or produce -publicly and freely available as soon as possible -and to the greatest degree allowed, including models, -algorithms, software, tools, estimates, and forecasts. + researchers and tool developers, to amplify their impact. + To that end, we'll continue to make everything we invent or produce + publicly and freely available as soon as possible + and to the greatest degree allowed, including models, + algorithms, software, tools, estimates, and forecasts. ## What We've Been Up To @@ -129,29 +129,29 @@ and have made good progress on some of them. Here's a quick summary: -- We've built a number of new indicators of COVID-19 activity. - These are fine-grained geographically - (most of them are available at the US county level) +- We've built a number of new indicators of COVID-19 activity. + These are fine-grained geographically + (most of them are available at the US county level) and temporally (all of them updated daily). - They are designed to shed light on the current picture of COVID in the US, + They are designed to shed light on the current picture of COVID in the US, beyond the typical publicly-available metrics like confirmed cases and deaths. -- Some of our indicators are based on massive-scale surveys that we’re running - through partnerships with Facebook and Google, and others are based on +- Some of our indicators are based on massive-scale surveys that we’re running + through partnerships with Facebook and Google, and others are based on aggregated counts from massive medical claims data sets through partners like Change Healthcare. -- We've built a [public - API](`r blogdown::shortcode_html("apiref", "api/covidcast.html")`), - and [R and Python - packages](`r blogdown::shortcode_html("apiref", "api/covidcast_clients.html")`), - to serve our indicators to researchers and the public. This API provides new +- We've built a [public + API](`r blogdown::shortcode_html("apiref", "api/covidcast.html")`), + and [R and Python + packages](`r blogdown::shortcode_html("apiref", "api/covidcast_clients.html")`), + to serve our indicators to researchers and the public. This API provides new data daily. -- We've built [interactive maps and graphics](`r blogdown::shortcode("ref", "covidcast")`) to +- We've built [interactive maps and graphics](`r blogdown::shortcode("ref", "covidcast")`) to display our indicators, and better inform the public and decision-makers. -- We've developed forecasts of the future spread of the pandemic, +- We've developed forecasts of the future spread of the pandemic, validated them prospectively, and started submitting them to CDC. In subsequent posts, we'll dive deeper into many of these projects, and touch on diff --git a/content/blog/2020-08-26-fb-survey.Rmd b/content/blog/2020-08-26-fb-survey.Rmd index 167750537..ca4a186d1 100644 --- a/content/blog/2020-08-26-fb-survey.Rmd +++ b/content/blog/2020-08-26-fb-survey.Rmd @@ -4,8 +4,8 @@ author: "Alex Reinhart and Ryan Tibshirani" date: 2020-08-26 tags: ["symptom surveys", "COVIDcast", "R"] authors: -- alex -- ryan + - alex + - ryan heroImage: /blog/images/blog-lg-img_facebook-survey-post.png heroImageThumb: /blog/images/blog-thumb-img_facebook-survey-post.png summary: | @@ -48,11 +48,11 @@ we've been conducting a massive daily survey to monitor the spread and impact of the COVID-19 pandemic in the United States. Our survey is advertised through Facebook, but it's run on our own Qualtrics platform (Facebook never sees any of the survey responses). -This is an ongoing operation and our survey is taken by about 74,000 people +This is an ongoing operation and our survey is taken by about 74,000 people each day. Respondents provide information about COVID-related symptoms, contacts, risk factors, and demographics, allowing us to examine county-level trends across the US. -We believe that this combination of *detail* and *scale* +We believe that this combination of _detail_ and _scale_ has never before been available in a public health emergency. We make aggregated data publicly available daily through our @@ -72,7 +72,7 @@ some of the exciting new directions that we're pursuing now. ## Short Background -Back in March 2020, we began discussions with Facebook about running a +Back in March 2020, we began discussions with Facebook about running a survey, advertised through their site, to collect real-time, county-level information on people experiencing COVID-like symptoms. The basic premise was that we could use this information @@ -91,7 +91,7 @@ with buy-in from a platform like Facebook. Fortunately for us, they agreed! We launched our survey on April 6, 2020. -Every day since, Facebook directs a random sample of its users +Every day since, Facebook directs a random sample of its users to our survey, hosted on [Qualtrics](https://www.qualtrics.com/). As part of our agreement with Facebook, we receive the data directly from Qualtrics, @@ -102,15 +102,15 @@ with fully de-identified individual survey responses available only to researchers who agree to our [data use terms](https://dataforgood.fb.com/docs/covid-19-symptom-survey-request-for-data-access/). -As of this writing, our survey is taken by an average of 74,000 people per +As of this writing, our survey is taken by an average of 74,000 people per day, delivering enough data for us to create meaningful estimates for an average of nearly 1,000 counties per week. -Over the course of the survey so far, we have already collected over 10 million +Over the course of the survey so far, we have already collected over 10 million responses! An [international version of the survey](https://covidmap.umd.edu/), -available in over 50 languages, was launched soon after by a team at the +available in over 50 languages, was launched soon after by a team at the University of Maryland. -Before going into detail later about our survey and our survey-based +Before going into detail later about our survey and our survey-based indicators, here are a couple maps to ground your intuition. On the left is a state-level heatmap of the estimated percentage of people with COVID symptoms, @@ -166,60 +166,60 @@ grid.arrange(p1, p2, nrow = 1) We generated these plots using our [covidcast R package](https://cmu-delphi.github.io/covidcast/covidcastR/). In all, fetching the data from our API and producing the heatmaps -requires only 15 lines of code. -If you're interested, click the "Code" button to reveal the source. +requires only 15 lines of code. +If you're interested, click the "Code" button to reveal the source. We'll cover our [R and Python covidcast packages](`r blogdown::shortcode_html("apiref", "api/covidcast_clients.html")`) in a future blog post. ## Why Run These Surveys? -Now let's unpack the main motivation behind our survey a bit: -a person typically experiences COVID-like symptoms -before they seek medical care or a COVID-19 test, +Now let's unpack the main motivation behind our survey a bit: +a person typically experiences COVID-like symptoms +before they seek medical care or a COVID-19 test, so data on how many people are self-reporting COVID-like symptoms in a -given county could potentially give us an **early indicator** -of COVID activity in that county. And to be clear, it's not just -*that* we're looking at symptoms that's important here, -it's the *way* we're measuring them: -since symptoms can be reported from home, -with no special equipment needed, -this data isn't subject to the same reporting delays +given county could potentially give us an **early indicator** +of COVID activity in that county. And to be clear, it's not just +_that_ we're looking at symptoms that's important here, +it's the _way_ we're measuring them: +since symptoms can be reported from home, +with no special equipment needed, +this data isn't subject to the same reporting delays as formal testing metrics like confirmed COVID-19 case counts. (Note that confirmed COVID-19 case counts aren't just delayed, they are confounded by issues like testing policy and capacity, -while self-reported symptom data shouldn't be subject to the same problems. -This is potentially a very important point, but much more subtle, and we +while self-reported symptom data shouldn't be subject to the same problems. +This is potentially a very important point, but much more subtle, and we won't delve into it in this post.) -It's also worth being clear about what we're *not* able to say with these +It's also worth being clear about what we're _not_ able to say with these surveys: - Symptoms alone are not sufficient to diagnose coronavirus infections: of -course, COVID-like symptoms can be caused by other conditions, and many true -infections are asymptomatic. Therefore, even if we ignore the issues brought on -by self-reporting and survey sampling, we can't expect the estimates we produce -to reflect the true rate of COVID-19 (and they're not intended to). + course, COVID-like symptoms can be caused by other conditions, and many true + infections are asymptomatic. Therefore, even if we ignore the issues brought on + by self-reporting and survey sampling, we can't expect the estimates we produce + to reflect the true rate of COVID-19 (and they're not intended to). - Our survey responses come from the population of Facebook users in the US, -which may be a sizeable fraction of the US population, but certainly not all of -it. We'll also likely see bias because some people on Facebook are more inclined -to take surveys than others. We attempt to correct for both of these biases -using a [statistical reweighting -scheme](`r blogdown::shortcode_html("apiref", "api/covidcast-signals/fb-survey.html#survey-weighting")`), -but these corrections aren't perfect. + which may be a sizeable fraction of the US population, but certainly not all of + it. We'll also likely see bias because some people on Facebook are more inclined + to take surveys than others. We attempt to correct for both of these biases + using a [statistical reweighting + scheme](`r blogdown::shortcode_html("apiref", "api/covidcast-signals/fb-survey.html#survey-weighting")`), + but these corrections aren't perfect. - Our symptom data is entirely self-reported, in contrast to data -reported by medical professionals. Some fraction of the responses could be -erroneous, either because a person misunderstood the question or chose to answer -incorrectly. + reported by medical professionals. Some fraction of the responses could be + erroneous, either because a person misunderstood the question or chose to answer + incorrectly. To summarize, our survey can't be used to make absolute statements about the true prevalence of coronavirus disease in the US; in fact, it shouldn't even be regarded as a foolproof way of deriving unbiased estimates of the number of -people with COVID-19 *symptoms* in the US (mainly due to the self-reporting -aspect). Nevertheless, as we'll see below, *changes in self-reported symptoms -over time* can still be a meaningful reflection of the changes in coronavirus +people with COVID-19 _symptoms_ in the US (mainly due to the self-reporting +aspect). Nevertheless, as we'll see below, _changes in self-reported symptoms +over time_ can still be a meaningful reflection of the changes in coronavirus infections over time. And in the best case, it can help predict changes to come some days into the future. @@ -227,9 +227,9 @@ some days into the future. Our survey has 4 sections and is about 35 questions long. The first section is short and gathers information on a core set of symptoms used to define a -condition called **COVID-like illness** or **CLI**, which we define as *fever of +condition called **COVID-like illness** or **CLI**, which we define as _fever of at least 100 °F, along with cough, shortness of breath, or difficulty -breathing*. This mirrors the standard definition of influenza-like illness or +breathing_. This mirrors the standard definition of influenza-like illness or ILI (defined as fever of at least 100 °F, along with sore throat or cough), and is in line with the working definition of CLI used by the US Centers for Disease Control and Prevention (CDC). @@ -239,8 +239,8 @@ quantities, in a given location, on a given day: - % CLI: the percentage of people with COVID-like illness; and -- % CLI-in-community: the percentage of people who *know someone in their local -community* with COVID-like illness. +- % CLI-in-community: the percentage of people who _know someone in their local + community_ with COVID-like illness. Details on how we compute the % CLI and % CLI-in-community estimates can be found in our [COVIDcast signals @@ -278,16 +278,16 @@ provided they agree to keep data from individual respondents confidential. ## Some Interesting Examples Now we'll turn to some interesting data examples that provide evidence -that our survey-based CLI signals can be early indicators of COVID activity. -We'll consider the % CLI-in-community indicator, which tends to be more stable +that our survey-based CLI signals can be early indicators of COVID activity. +We'll consider the % CLI-in-community indicator, which tends to be more stable than the % CLI indicator (which shows similar, but noisier, trends). -Let's start by looking at Miami-Dade County, -which experienced a surge of new COVID-19 cases +Let's start by looking at Miami-Dade County, +which experienced a surge of new COVID-19 cases between June and July: below we plot daily new confirmed COVID-19 cases over time (using a 7-day trailing average for smoothing) as a blue curve. -We also plot the % CLI-in-community indicator as a red curve. +We also plot the % CLI-in-community indicator as a red curve. Note that these two curves, case counts and % CLI-in-community, -are not measured in the same units, hence our use of two y-axes: +are not measured in the same units, hence our use of two y-axes: one on the left for case counts, and one on the right for % CLI-in-community. ```{r, message = FALSE, warning = FALSE, fig.width = 6, fig.height = 5} @@ -317,11 +317,11 @@ plot_one = function(geo_value, title = NULL, xlab = NULL, given_geo_value = geo_value df_fb_one = df_fb %>% filter(geo_value == given_geo_value) df_in_one = df_in %>% filter(geo_value == given_geo_value) - + # Compute ranges of the two signals range1 = df_in_one %>% select("value") %>% range range2 = df_fb_one %>% select("value") %>% range - + # Convenience functions for our two signal ranges trans12 = function(x) trans(x, range1, range2) trans21 = function(x) trans(x, range2, range1) @@ -337,7 +337,7 @@ plot_one = function(geo_value, title = NULL, xlab = NULL, df_in_one), c("time_value", "value")) df$signal = c(rep("% CLI-in-community", nrow(df_fb_one)), rep("New COVID-19 cases", nrow(df_in_one))) - + # Finally, plot both signals pos = ifelse(legend, "bottom", "none") return(ggplot(df, aes(x = time_value, y = value)) + @@ -364,13 +364,13 @@ This example, as with all code examples in this blog post, was produced using our [covidcast R package](https://cmu-delphi.github.io/covidcast/covidcastR/). A first glance reveals that the % CLI-in-community indicator -clearly rises alongside confirmed COVID-19 cases, -a reassuring sanity check: more people indeed report -that others are sick in their community at times +clearly rises alongside confirmed COVID-19 cases, +a reassuring sanity check: more people indeed report +that others are sick in their community at times when COVID-19 tests confirm more cases. But a closer look shows something quite interesting: -the % CLI-in-community signal begins to rise steeply on June 19 -(first dashed vertical line), which happens *6 days before* COVID-19 cases +the % CLI-in-community signal begins to rise steeply on June 19 +(first dashed vertical line), which happens _6 days before_ COVID-19 cases begin their steep ascent on June 25 (second dashed vertical line). This is just one county; to investigate further, we pulled the 20 counties with @@ -393,37 +393,37 @@ for (i in 1:num) { do.call(grid.arrange, c(p_list, nrow = 5, ncol = 4)) ``` -The examples above are an informal way of looking -at the *recall* of the % CLI-in-community signal. -Of course, this is only one half of the story: +The examples above are an informal way of looking +at the _recall_ of the % CLI-in-community signal. +Of course, this is only one half of the story: for the signal to be a useful early indicator, -we'd also need to know something about its *precision*: -we'd need to know that % CLI-in-community seldom rises -in periods where new COVID-19 cases remain flat. +we'd also need to know something about its _precision_: +we'd need to know that % CLI-in-community seldom rises +in periods where new COVID-19 cases remain flat. We save a formal precision-recall analysis for future work. ## Basic Correlation Analysis To complement the more exploratory, qualitative analysis of the last section, -we'll conduct a simple quantitative analysis here, -by computing some basic measures of correlation -between our survey-based indicators and confirmed COVID-19 case rates. +we'll conduct a simple quantitative analysis here, +by computing some basic measures of correlation +between our survey-based indicators and confirmed COVID-19 case rates. There are a couple of ways to slice the data---by day and by county---and we'll consider both ways in what follows. ### Correlations Sliced by Time -First we compute, for each day between April 15 and August 15, -the Spearman correlation between the % CLI indicator and COVID-19 case rates, -across the counties that had at least 500 cumulative confirmed COVID-19 cases. -(Spearman correlation assesses whether one variable is high when another is -high, even if their relationship is not linear.) -We do the same for the % CLI-in-community indicator, +First we compute, for each day between April 15 and August 15, +the Spearman correlation between the % CLI indicator and COVID-19 case rates, +across the counties that had at least 500 cumulative confirmed COVID-19 cases. +(Spearman correlation assesses whether one variable is high when another is +high, even if their relationship is not linear.) +We do the same for the % CLI-in-community indicator, and plot the results below, with the % CLI indicator correlations in -red, and the % CLI-in-community correlations in blue. -We can see clearly that the % CLI-in-community indicator produces, -consistently across all time, *much* higher correlations. -Even in an absolute sense, the correlations from the % CLI-in-community are +red, and the % CLI-in-community correlations in blue. +We can see clearly that the % CLI-in-community indicator produces, +consistently across all time, _much_ higher correlations. +Even in an absolute sense, the correlations from the % CLI-in-community are noteworthy: they reach over 0.8 for a period between July and August. ```{r, message = FALSE, warning = FALSE, fig.width = 6, fig.height = 5} @@ -468,19 +468,19 @@ ggplot(df_cor, aes(x = time_value, y = value)) + ``` Another interesting observation is that the correlations from either indicator -increase dramatically sometime around mid-June. -This could be because many counties saw big surges in COVID-19 activity around -that time. These surges created a larger spread between the COVID-19 case rates +increase dramatically sometime around mid-June. +This could be because many counties saw big surges in COVID-19 activity around +that time. These surges created a larger spread between the COVID-19 case rates across the country, and so county-to-county differences started to become easier -to track with the indicators, as the magnitude of these differences started to +to track with the indicators, as the magnitude of these differences started to swamp the noise. -Of course, this is really just speculation, -and we can't say for certain that this is the cause. -Some decent empirical evidence for our explanation, however, can be found -by looking at how COVID-19 case rates vary between counties over time, as shown -below. The median absolute deviation (a robust measure of spread) between -counties rises sharply sometime around mid-June, +Of course, this is really just speculation, +and we can't say for certain that this is the cause. +Some decent empirical evidence for our explanation, however, can be found +by looking at how COVID-19 case rates vary between counties over time, as shown +below. The median absolute deviation (a robust measure of spread) between +counties rises sharply sometime around mid-June, around the same time as the correlations increased. ```{r, message = FALSE, warning = FALSE, fig.width = 6, fig.height = 4} @@ -490,18 +490,18 @@ ggplot(df_in_act %>% group_by(time_value) %>% labs(title = "Median absolute deviation in COVID-19 case rates", subtitle = sprintf("Over all counties with at least %i cumulative cases", case_num), x = "Date", y = "Median abs deviation") + - theme_bw() + theme_bw() ``` ### Correlations Sliced by County Next we compute, for each county with at least 500 cumulative cases, -the Spearman correlations between each of our indicators and COVID-19 case +the Spearman correlations between each of our indicators and COVID-19 case rates, across all time. We can visualize this in a few different ways. -Below we plot estimated densities from these two sets of correlations: -from the % CLI indicator in red, and the % CLI-in-community indicator in blue. -With this slice of the data (correlations by county, rather than by day), -we again see that the % CLI-in-community indicator produces *much* higher +Below we plot estimated densities from these two sets of correlations: +from the % CLI indicator in red, and the % CLI-in-community indicator in blue. +With this slice of the data (correlations by county, rather than by day), +we again see that the % CLI-in-community indicator produces _much_ higher correlations. ```{r, message = FALSE, warning = FALSE, fig.width = 6, fig.height = 5} @@ -523,13 +523,13 @@ ggplot(df_cor, aes(value)) + theme_bw() + theme(legend.pos = "bottom", legend.title = element_blank()) ``` -We can also examine choropleth maps of these correlations to learn -where (geographically speaking) they're high and where they're not. -As we can see from the maps below, the % CLI-in-community indicator -yields high correlations throughout much of the US, -whereas the % CLI indicator is a bit more spotty. -Note that here a gray color denotes a missing value: -either that county had below 500 cumulative COVID-19 cases, +We can also examine choropleth maps of these correlations to learn +where (geographically speaking) they're high and where they're not. +As we can see from the maps below, the % CLI-in-community indicator +yields high correlations throughout much of the US, +whereas the % CLI indicator is a bit more spotty. +Note that here a gray color denotes a missing value: +either that county had below 500 cumulative COVID-19 cases, or we didn't have enough data from the surveys in order to estimate % CLI and % CLI-in-community signals there. @@ -553,9 +553,9 @@ grid.arrange(p1, p2, nrow = 1) You might expect that a survey that reaches tens of thousands of respondents within the US daily---and has done so for months during a major pandemic---could -have *many* possible uses beyond simply surveying symptoms. You would be right. -Beyond the first section on the core COVID symptoms, our survey contains -questions on testing, behavior, medical care, mental health, and related topics, +have _many_ possible uses beyond simply surveying symptoms. You would be right. +Beyond the first section on the core COVID symptoms, our survey contains +questions on testing, behavior, medical care, mental health, and related topics, opening a multitude of possible research questions up to empirical inquiry. This is why Delphi and the University of Maryland (for the international @@ -570,34 +570,34 @@ Facebook](https://dataforgood.fb.com/docs/covid-19-symptom-survey-request-for-da and our goal is to build a network of researchers committed to fighting the pandemic through our survey data. -To take these efforts to the next level, we'll soon release a new version of the -survey. Based on feedback from other researchers, public health agencies, the +To take these efforts to the next level, we'll soon release a new version of the +survey. Based on feedback from other researchers, public health agencies, the University of Maryland, and Facebook, we've added items asking about: - More details on COVID testing (including whether the respondent tried to get -tested but could not). + tested but could not). - Mask wearing. - The types of activities the respondent has done with other people. - Mental health and social isolation. - Employment. - Demographics (including race and education). -These items will give us an unprecedented view into -how people have responded to the pandemic, +These items will give us an unprecedented view into +how people have responded to the pandemic, how they have been affected by the pandemic and the resulting economic downturn, -and how specific groups are affected. +and how specific groups are affected. We hope the mask and behavior items can help researchers studying -how best to prevent the spread of COVID-19, -and can even help inform forecasts, -while mental health and isolation items will help researchers to understand +how best to prevent the spread of COVID-19, +and can even help inform forecasts, +while mental health and isolation items will help researchers to understand how social distancing and recession have affected mental health. -Together with extended demographic data, this may help to inform policies +Together with extended demographic data, this may help to inform policies designed to help those most affected by the pandemic. -This new survey is currently being deployed, -and data should become available in the next few weeks. +This new survey is currently being deployed, +and data should become available in the next few weeks. Detailed data will be available to researchers, -while new aggregates---such as of mask-wearing---will +while new aggregates---such as of mask-wearing---will be made public, as usual, through our [COVIDcast -API](`r blogdown::shortcode_html("apiref", "api/covidcast.html")`) +API](`r blogdown::shortcode_html("apiref", "api/covidcast.html")`) and [COVIDcast interactive map](`r blogdown::shortcode_html("ref", "covidcast")`). diff --git a/content/blog/2020-08-28-api.Rmd b/content/blog/2020-08-28-api.Rmd index b1f17fe35..638208c67 100644 --- a/content/blog/2020-08-28-api.Rmd +++ b/content/blog/2020-08-28-api.Rmd @@ -4,8 +4,8 @@ author: "Kathryn Mazaitis and Alex Reinhart" date: 2020-10-07 tags: ["COVIDcast API", "COVIDcast", "R", "Python"] authors: -- kathryn -- alex + - kathryn + - alex heroImage: /blog/images/blog-lg-img_Accessing Open COVID-19.png heroImageThumb: /blog/images/blog-thumb-img_Accessing Open COVID-19.png summary: | @@ -49,22 +49,22 @@ output: knitr::opts_chunk$set(collapse = TRUE) ``` -One of our primary initiatives at the Delphi COVIDcast project +One of our primary initiatives at the Delphi COVIDcast project ([learn more about our organization here](`r blogdown::shortcode_html("ref", "2020-08-10-hello-world")`)) -has been to curate a diverse set of COVID-related data streams, -and to make them freely available through our +has been to curate a diverse set of COVID-related data streams, +and to make them freely available through our [COVIDcast Epidata API](`r blogdown::shortcode_html("apiref", "api/covidcast.html")`). -These include both novel signals that we have collected and analyzed ourselves, +These include both novel signals that we have collected and analyzed ourselves, such as our symptom survey [distributed by Facebook](`r blogdown::shortcode_html("ref", "2020-09-18-google-survey")`) to its users, [Google's symptom survey](`r blogdown::shortcode_html("ref", "2020-09-18-google-survey")`) whose results are delivered to us, the percentage of doctor's visits due to COVID-like illness, and results from Quidel's antigen tests; -and also existing signals, such as the confirmed case counts -and deaths reported by USA Facts and Johns Hopkins University. -The COVIDcast API freely provides researchers and decision-makers -with the data they need to conduct their work, and -is conveniently accessible via easy-to-use -[Python](https://cmu-delphi.github.io/covidcast/covidcast-py/html/) +and also existing signals, such as the confirmed case counts +and deaths reported by USA Facts and Johns Hopkins University. +The COVIDcast API freely provides researchers and decision-makers +with the data they need to conduct their work, and +is conveniently accessible via easy-to-use +[Python](https://cmu-delphi.github.io/covidcast/covidcast-py/html/) and [R](https://cmu-delphi.github.io/covidcast/covidcastR/) packages. We have always made our code, data and estimates freely and publicly available, from the very beginning of our work on flu back in 2013, well before the COVID pandemic. @@ -83,18 +83,18 @@ do their jobs effectively. Making sense of the COVID-19 pandemic can be a frustratingly hard problem in part because no one signal can tell the whole story. -Case counts are important, but different states +Case counts are important, but different states use different reporting criteria and testing availability varies. -Deaths are more accurately observed, +Deaths are more accurately observed, but are a very lagging indicator of disease activity. We recognized early on that to make progress, -we require a diversity of data sources. +we require a diversity of data sources. This recognition caused us to shift priorities. Before we could build forecasts and other statistical models, -we needed to rapidly develop new relevant data streams. +we needed to rapidly develop new relevant data streams. This effort grew into the COVIDcast project---see our [introductory -post](`r blogdown::shortcode_html("ref", "2020-08-10-hello-world")`) +post](`r blogdown::shortcode_html("ref", "2020-08-10-hello-world")`) for more about our efforts since March. The data streams that we work with can be roughly mapped @@ -110,42 +110,42 @@ which follows the progression of disease: 7. Some hospitalized patients are subsequently **intubated** or otherwise transferred to an **ICU** ward. 8. Finally, **deaths** due to the illness are recorded. -Each level of the pyramid can be examined -through many different data sources. +Each level of the pyramid can be examined +through many different data sources. For example, aggregated cell phone mobility data -could address population behavior, +could address population behavior, while the volume of certain Google search queries -might correlate with how many people have symptoms or have heightened anxiety or awareness of the disease. -Levels 4 through 8 of the pyramid are medically attended, +might correlate with how many people have symptoms or have heightened anxiety or awareness of the disease. +Levels 4 through 8 of the pyramid are medically attended, and can be examined using various medical records. -Confirmed cases and deaths are reported +Confirmed cases and deaths are reported through local and state health authorities, as are some aspects of hospitalization. As we move from level 1 to level 8, -the data become more specific, -since it is based on more objective and specific criteria. -The data also become less timely: level 1 -can be a *leading indicator* of disease levels in the community, -since behavior affects spread, -whereas level 8 data only occurs after patients -have already been infected and died. -Only at level 5 and up do we actually -gain data involving definite diagnoses---data before level 5 is *behavioral* or *syndromic*, +the data become more specific, +since it is based on more objective and specific criteria. +The data also become less timely: level 1 +can be a _leading indicator_ of disease levels in the community, +since behavior affects spread, +whereas level 8 data only occurs after patients +have already been infected and died. +Only at level 5 and up do we actually +gain data involving definite diagnoses---data before level 5 is _behavioral_ or _syndromic_, meaning it only relates to behaviors and/or constellations of symptoms. -Data streams that are organized in this way +Data streams that are organized in this way can be used for many possible purposes, including: -* **Nowcasting.** If the data can provide a clear picture of what's - happening in each community *right now*, that knowledge can be used to make more informed and responsive decisions +- **Nowcasting.** If the data can provide a clear picture of what's + happening in each community _right now_, that knowledge can be used to make more informed and responsive decisions about re-opening, closures, resource allocation, and so on. -* **Forecasting.** Predicting the likely activity level of the pandemic in each community in the coming weeks can help guide local planning and preparations. For example, predicting the number of upcoming cases can help public health departments hire and train the necessary contact tracers. +- **Forecasting.** Predicting the likely activity level of the pandemic in each community in the coming weeks can help guide local planning and preparations. For example, predicting the number of upcoming cases can help public health departments hire and train the necessary contact tracers. Predicting hospitalizations can help hospitals prepare adequate supplies of PPE, clear hospital beds, and ensure availability of appropriate staff. -* **Scenario projections.** While forecasting predicts what is likely to happen if current circumstances continue unchanged, scenario projections can tell us how the pandemic is likely to unfold under different assumptions, such as changes in specific government policies (e.g. opening or closing schools or businesses), specific public behavior (stay-at-home, mask usage, social distancing), or changes in the properties of the virus or the environment. Scenario projections are most useful for contemplating various interventions. +- **Scenario projections.** While forecasting predicts what is likely to happen if current circumstances continue unchanged, scenario projections can tell us how the pandemic is likely to unfold under different assumptions, such as changes in specific government policies (e.g. opening or closing schools or businesses), specific public behavior (stay-at-home, mask usage, social distancing), or changes in the properties of the virus or the environment. Scenario projections are most useful for contemplating various interventions. -* **Epidemiological research.** The data may help us understand what behaviors +- **Epidemiological research.** The data may help us understand what behaviors are linked to spread, what symptoms commonly occur, and answer other important questions to better understand this and future pandemics and epidemics. @@ -161,27 +161,27 @@ permit us to publish estimates and other aggregated statistics in our COVIDcast Epidata API (often informally called the COVIDcast API). These data sources cover most levels of the severity pyramid and include: -* Massive surveys we conduct through Facebook: Facebook has been sending a +- Massive surveys we conduct through Facebook: Facebook has been sending a random sample of its users to Delphi's symptoms and behavior survey every day since April 6. Our survey averages over 70,000 respondents each day, making it---along with its [international sister survey](https://covidmap.umd.edu/) - run by University of Maryland---the largest public health survey ever conducted. + run by University of Maryland---the largest public health survey ever conducted. Our [previous blog post](`r blogdown::shortcode_html("ref", "2020-08-26-fb-survey")`) showed how the survey can indicate COVID-19 activity, and preliminary analysis also suggests [it can help forecast COVID-19 cases](`r blogdown::shortcode_html("ref", "2020-09-21-forecast-demo")`). See our [surveys site](`r blogdown::shortcode_html("ref", "surveys")`) for more on the survey, its questions, and getting access to data. -* Massive surveys we run through Google: +- Massive surveys we run through Google: From April 11 to May 14, 2020, Delphi conducted a single-question symptoms survey - through Google. It reached over 100,000 respondents daily during its - short run, and was a surprisingly informative measure - of pandemic activity preceding medical contact. For more, see our - [previous blog post](`r blogdown::shortcode_html("ref", "2020-09-18-google-survey")`), + through Google. It reached over 100,000 respondents daily during its + short run, and was a surprisingly informative measure + of pandemic activity preceding medical contact. For more, see our + [previous blog post](`r blogdown::shortcode_html("ref", "2020-09-18-google-survey")`), or our [technical documentation](`r blogdown::shortcode_html("apiref", "api/covidcast-signals/google-survey.html")`). As explained in our past blog post, Delphi is currently considering new uses for these surveys. -* Insurance claims: Medical insurance claims include diagnostic codes, +- Insurance claims: Medical insurance claims include diagnostic codes, lab orders, and charge codes which can be used to estimate COVID-19 activity in a region. We have several partners who provide us with @@ -189,97 +189,97 @@ cover most levels of the severity pyramid and include: or allow us to derive such statistics from strictly de-identified claim records. We use this data to construct signals reflecting COVID activity in both outpatient and inpatient visits; see our - technical documentation sites for + technical documentation sites for [doctor's visits](`r blogdown::shortcode_html("apiref", "api/covidcast-signals/doctor-visits.html")`) and [hospital admissions](`r blogdown::shortcode_html("apiref", "api/covidcast-signals/hospital-admissions.html")`) for more details. -* Quidel COVID antigen tests: Quidel is a national provider of networked lab +- Quidel COVID antigen tests: Quidel is a national provider of networked lab testing devices, and began making de-identified records of their COVID-19 antigen tests available to us in early May. This data source fills an important gap because many public testing data sources only include PCR tests, not antigen - tests. Our [technical + tests. Our [technical documentation](`r blogdown::shortcode_html("apiref", "api/covidcast-signals/quidel.html#covid-19-tests")`) gives more details. -* Google search trends: +- Google search trends: We query the Google Health Trends API - for overall searcher interest in a set - of COVID-19 related terms about anosmia - (loss of smell or taste), - which emerged as a specific symptom of COVID-19. + for overall searcher interest in a set + of COVID-19 related terms about anosmia + (loss of smell or taste), + which emerged as a specific symptom of COVID-19. More details, including the search terms and topics we analyze, - are available in our + are available in our [technical documentation](`r blogdown::shortcode_html("apiref", "api/covidcast-signals/ght.html")`). Additionally, we host the following more widely-available signals in our API for the convenience of the research community, and to provide revision tracking: -* Confirmed cases and deaths as reported by [JHU CSSE](https://github.com/CSSEGISandData/COVID-19). -* Confirmed cases and deaths as reported by [USAFacts](https://usafacts.org/visualizations/coronavirus-covid-19-spread-map/). -* Mobility data as made available by +- Confirmed cases and deaths as reported by [JHU CSSE](https://github.com/CSSEGISandData/COVID-19). +- Confirmed cases and deaths as reported by [USAFacts](https://usafacts.org/visualizations/coronavirus-covid-19-spread-map/). +- Mobility data as made available by [SafeGraph](https://docs.safegraph.com/docs/social-distancing-metrics); SafeGraph makes detailed de-identified data available to researchers, and by agreement with SafeGraph, we make county-level aggregates publicly available. -Nearly all our data streams are available -at the county level across the United States. -We also aggregate our signals to metropolitan statistical areas and states, and some signals to Hospital Referral Regions (HRRs). -For a full list of all data streams, see our +Nearly all our data streams are available +at the county level across the United States. +We also aggregate our signals to metropolitan statistical areas and states, and some signals to Hospital Referral Regions (HRRs). +For a full list of all data streams, see our [COVIDcast signal documentation site](`r blogdown::shortcode_html("apiref", "api/covidcast_signals.html")`). The software we've developed to obtain and aggregate this data is open-source, shared in our [covidcast-indicators GitHub repository](https://github.com/cmu-delphi/covidcast-indicators). -All the above data streams are made publicly available -through our COVIDcast API---if you're interested +All the above data streams are made publicly available +through our COVIDcast API---if you're interested in using these signals for decision making, research, investigative journalism -or simply to understand trends in your area, -pulling the data is only a moment's work. +or simply to understand trends in your area, +pulling the data is only a moment's work. Let's discuss how the data is stored and how you can get access. ## Tracking Observations and Revisions -Each record in our database is an observation covering +Each record in our database is an observation covering a set of events aggregated by time and by geographic region. -Most signals in the API are available at a daily resolution, +Most signals in the API are available at a daily resolution, but some are available only weekly, -so we try to keep the definitions below general. +so we try to keep the definitions below general. Each record includes: -* `time_value`: The time period when the events occurred. -* `geo_value`: The geographic region where the events occurred. -* `value`: The estimated value. -* `stderr`: The standard error of the estimate, usually referring to the sampling error. -* `sample_size`: The number of events used in the estimation. +- `time_value`: The time period when the events occurred. +- `geo_value`: The geographic region where the events occurred. +- `value`: The estimated value. +- `stderr`: The standard error of the estimate, usually referring to the sampling error. +- `sample_size`: The number of events used in the estimation. -For example, a number of COVID-19 antigen tests -were performed in the state of New York on August 1. -The `time_value` would be August 1, +For example, a number of COVID-19 antigen tests +were performed in the state of New York on August 1. +The `time_value` would be August 1, with `geo_value` indicating the state of New York, -while the remaining fields would give the estimated test positivity rate -(the percentage of tests that were positive for COVID-19), +while the remaining fields would give the estimated test positivity rate +(the percentage of tests that were positive for COVID-19), its standard error, and the number of tests used to calculate the estimate. But crucially---and unlike most other sources of COVID-19 data---our API reports two additional fields with each record: -* `issue`: The time period when this observation was published. -* `lag`: The time delay between when the events occurred and when this +- `issue`: The time period when this observation was published. +- `lag`: The time delay between when the events occurred and when this observation was published. -For example, results of COVID-19 antigen tests may take -between four days and six weeks to reach us, -depending on the technology and staff available at each testing site. -We might publish our first estimate +For example, results of COVID-19 antigen tests may take +between four days and six weeks to reach us, +depending on the technology and staff available at each testing site. +We might publish our first estimate of August 1st's test positivity rate on August 6th, giving an issue date of August 6 and a lag of five days. -But when more data about August 1st's tests arrive the next day, -we issue a second estimate with an issue date of August 7 and a lag of six days. -Each record remains in the API, permitting users to see the changes -and ask "What was known *as of* this date?" -This is important because estimates -can change for *weeks* as new data arrives: +But when more data about August 1st's tests arrive the next day, +we issue a second estimate with an issue date of August 7 and a lag of six days. +Each record remains in the API, permitting users to see the changes +and ask "What was known _as of_ this date?" +This is important because estimates +can change for _weeks_ as new data arrives: ```{r q-versioning, warning=FALSE, message=FALSE, cache=TRUE} library(covidcast) @@ -302,17 +302,17 @@ covidcast_signal( Many data sources are subject to revisions: -* Case and death counts are frequently corrected or adjusted by authorities. -* Medical claims data can take weeks to be submitted and processed. -* Lab tests and medical records can be backlogged for a variety of reasons. -* Surveys are not always completed promptly. - -An accurate revision log is crucial for researchers -building forecasts of COVID-19 cases or outcomes. -A forecast that is made today can only rely -on information we have access to today. -Forecasting models are often developed by building them -to predict cases using historical data---but to do so, +- Case and death counts are frequently corrected or adjusted by authorities. +- Medical claims data can take weeks to be submitted and processed. +- Lab tests and medical records can be backlogged for a variety of reasons. +- Surveys are not always completed promptly. + +An accurate revision log is crucial for researchers +building forecasts of COVID-19 cases or outcomes. +A forecast that is made today can only rely +on information we have access to today. +Forecasting models are often developed by building them +to predict cases using historical data---but to do so, the model should use only data that was available on the forecast date, not the updates that would arrive later. @@ -323,20 +323,20 @@ date, and preserves the history of changes for future analysis. ## Accessing the API -A massive database of COVID-19 data is, of course, -of no use if nobody can access it. +A massive database of COVID-19 data is, of course, +of no use if nobody can access it. We provide several ways to access COVIDcast data. -First, the [public COVIDcast map](`r blogdown::shortcode_html("ref", "covidcast")`) -provides a selection of our signals, -and includes an "Export Data" tab that can -pull a selected signal and download it as a CSV. -Browse the map to choose which signal you are interested in, +First, the [public COVIDcast map](`r blogdown::shortcode_html("ref", "covidcast")`) +provides a selection of our signals, +and includes an "Export Data" tab that can +pull a selected signal and download it as a CSV. +Browse the map to choose which signal you are interested in, then use Export Data to obtain the data for further analysis. For more advanced users, we provide R and Python packages -to make data access easy for anyone conducting -data analysis in either language. +to make data access easy for anyone conducting +data analysis in either language. The first step of using the packages to acquire the data is to identify the source and signal name for the data you want to analyze. Suppose, for example, @@ -350,12 +350,12 @@ signals to choose from. Reviewing the technical details, you decide `smoothed_adj_covid19` fits your needs best, because it removes day-of-week effects. -With the source and signal names in hand, you can quickly pull the data. +With the source and signal names in hand, you can quickly pull the data. -An R user can install our -[R covidcast package](https://cmu-delphi.github.io/covidcast/covidcastR/) -and then quickly plot the percentage of hospital admissions -that are due to COVID-19 in several states. +An R user can install our +[R covidcast package](https://cmu-delphi.github.io/covidcast/covidcastR/) +and then quickly plot the percentage of hospital admissions +that are due to COVID-19 in several states. (Click the "Code" button to see the R code used to produce this example.) ```{r dv-graph, message=FALSE, cache=TRUE} @@ -369,9 +369,9 @@ plot(hosp, plot_type = "line", title = "% of hospital admissions due to COVID-19") ``` -Since the packages also support mapping, -we can examine the percentage -of outpatient doctor's visits +Since the packages also support mapping, +we can examine the percentage +of outpatient doctor's visits due to COVID-Like-Illness (CLI) in the South. ```{r dv-maps, message=FALSE, cache=TRUE, fig.width=10} @@ -388,11 +388,11 @@ g2 <- plot(dv, time_value = "2020-08-24", include = south, grid.arrange(g1, g2, nrow = 1) ``` -In Python, fetching data requires the +In Python, fetching data requires the [Python covidcast package](https://cmu-delphi.github.io/covidcast/covidcast-py/html/), -which can quickly produce a Pandas data frame. -For example, here we fetch the estimated percentage -of people in each state who know someone who is sick, +which can quickly produce a Pandas data frame. +For example, here we fetch the estimated percentage +of people in each state who know someone who is sick, based on Delphi's [symptom surveys](`r blogdown::shortcode_html("ref", "2020-08-26-fb-survey")`). According to the [relevant documentation @@ -419,7 +419,7 @@ Both packages support querying the latest version of data---as shown above---but can also fetch prior revisions or only the information that was available on a certain date. -Finally, R and Python are not required for access to our data; users can also +Finally, R and Python are not required for access to our data; users can also make HTTP requests to the API directly and receive data back in JSON format. By setting `data_source`, `signal`, `time_type`, `geo_type`, `time_values`, and `geo_value` parameters in the query URL, you can select the specific data source you @@ -441,33 +441,33 @@ most programming languages---to fetch up-to-date data. ## Putting the API to Work -The [COVIDcast API](`r blogdown::shortcode_html("apiref", "api/covidcast.html")`) +The [COVIDcast API](`r blogdown::shortcode_html("apiref", "api/covidcast.html")`) provides unified access to numerous COVID data streams, -which can be browsed through our [interactive map](`r blogdown::shortcode_html("ref", "covidcast")`) -and easily accessed through our +which can be browsed through our [interactive map](`r blogdown::shortcode_html("ref", "covidcast")`) +and easily accessed through our [R and Python packages](`r blogdown::shortcode_html("apiref", "api/covidcast_clients.html")`). Unlike most other sources of COVID data, -it tracks the complete revision history of every signal, +it tracks the complete revision history of every signal, allowing historical reconstructions of -what information was available at specific times. -Additionally, many of our data streams simply +what information was available at specific times. +Additionally, many of our data streams simply aren't available anywhere else. -We invite you to put the API to use for your own purposes. -Building a dashboard for your community? -Testing out forecasting methods? -Studying how the pandemic evolves? +We invite you to put the API to use for your own purposes. +Building a dashboard for your community? +Testing out forecasting methods? +Studying how the pandemic evolves? We might have the data you're looking for. -Many of our data streams are already being used to inform decision-making. -For example, [COVID Exit Strategy](https://www.covidexitstrategy.org/) -tracks the pandemic and whether states are ready to reopen, -using symptom survey data from the COVIDcast API as a key data source. -Anthem's [C19 Explorer](https://c19explorer.io/) -presents a comprehensive community picture of the pandemic, -including outpatient doctor's visit data from COVIDcast. -Aledade's [COVID-19 Interactive Map](https://covidmap.aledade.com/) -applies scan statistics algorithms to COVIDcast survey data +Many of our data streams are already being used to inform decision-making. +For example, [COVID Exit Strategy](https://www.covidexitstrategy.org/) +tracks the pandemic and whether states are ready to reopen, +using symptom survey data from the COVIDcast API as a key data source. +Anthem's [C19 Explorer](https://c19explorer.io/) +presents a comprehensive community picture of the pandemic, +including outpatient doctor's visit data from COVIDcast. +Aledade's [COVID-19 Interactive Map](https://covidmap.aledade.com/) +applies scan statistics algorithms to COVIDcast survey data to detect statistically significant clusters. We hope to see you join this list soon! diff --git a/content/blog/2020-09-18-google-survey.Rmd b/content/blog/2020-09-18-google-survey.Rmd index 0b96ae6e0..038a84cce 100644 --- a/content/blog/2020-09-18-google-survey.Rmd +++ b/content/blog/2020-09-18-google-survey.Rmd @@ -4,7 +4,7 @@ author: "Ryan Tibshirani" date: 2020-09-18 tags: ["symptom surveys", "COVIDcast", "R"] authors: -- ryan + - ryan heroImage: /blog/images/blog-lg-img_google-survey-post.png heroImageThumb: /blog/images/blog-thumb-img_google-survey-post.png summary: | @@ -19,7 +19,7 @@ summary: | This short post covers some key differences between our Google and Facebook surveys, explains the backstory behind the "CLI-in-community" question as it arose through our collaboration with Google, - and shares some of our thinking about next steps for the Google survey. + and shares some of our thinking about next steps for the Google survey. acknowledgements: | Ryan Tibshirani wrote the initial code for producing estimates from the aggregated survey data. Sangwon Hyun, Natalia Lombardi de @@ -36,101 +36,102 @@ output: --- Since April 2020, in addition to our [massive daily survey advertised on -Facebook](`r blogdown::shortcode_html("ref", "2020-08-26-fb-survey")`), -we've been running (even-more-massive) surveys through Google to track the -spread of COVID-19 in the United States. At its peak, our Google survey was -taken by over 1.2 million people in a single day, and over its first month in +Facebook](`r blogdown::shortcode_html("ref", "2020-08-26-fb-survey")`), +we've been running (even-more-massive) surveys through Google to track the +spread of COVID-19 in the United States. At its peak, our Google survey was +taken by over 1.2 million people in a single day, and over its first month in operation, averaged about 600,000 daily respondents. As usual, we make aggregated data from this survey available through our [COVIDcast API](`r blogdown::shortcode_html("apiref", "api/covidcast.html")`). In mid-May, we decided to pause daily dissemination of this survey in order to -focus on our (longer, more complex) survey through Facebook, -but we plan to bring back the Google survey this fall. -The two surveys are, in fact, quite different and complement each other nicely. -This short post covers some key differences between our Google and Facebook +focus on our (longer, more complex) survey through Facebook, +but we plan to bring back the Google survey this fall. +The two surveys are, in fact, quite different and complement each other nicely. +This short post covers some key differences between our Google and Facebook surveys, explains the backstory behind the "CLI-in-community" question -as it arose through our collaboration with Google's team, +as it arose through our collaboration with Google's team, and shares some of our thinking about next steps for the Google survey. ## Short Background -Back in March 2020, around the time we began discussions with Facebook about -COVID-19 symptom surveys, we pitched the same idea to Google. -Our motivation, [as we explained in our last -post](`r blogdown::shortcode_html("ref", "2020-08-26-fb-survey#why-run-these-surveys")`), -has been to produce real-time, county-level data streams of self-reported COVID -symptoms that can potentially serve as **early indicators** of COVID activity in +Back in March 2020, around the time we began discussions with Facebook about +COVID-19 symptom surveys, we pitched the same idea to Google. +Our motivation, [as we explained in our last +post](`r blogdown::shortcode_html("ref", "2020-08-26-fb-survey#why-run-these-surveys")`), +has been to produce real-time, county-level data streams of self-reported COVID +symptoms that can potentially serve as **early indicators** of COVID activity in the US. As we noted in that post, we weren't the only data scientists -who thought of running COVID-19 symptom surveys, and several -other groups had deployed surveys before us. -What distinguished our strategy from others' -was the pursuit of a giant like Google to achieve widespread and continuous -dissemination (well beyond what we could do ourselves). -Google's willingness to help was a *huge* win for us. -Of all the partnerships we formed to create new COVID-19 indicators, -our deal with Google was the first to come through. -This gave us an invaluable confidence boost, -and taught us the silver lining of this pandemic: +who thought of running COVID-19 symptom surveys, and several +other groups had deployed surveys before us. +What distinguished our strategy from others' +was the pursuit of a giant like Google to achieve widespread and continuous +dissemination (well beyond what we could do ourselves). +Google's willingness to help was a _huge_ win for us. +Of all the partnerships we formed to create new COVID-19 indicators, +our deal with Google was the first to come through. +This gave us an invaluable confidence boost, +and taught us the silver lining of this pandemic: that many people are truly generous and willing to help. Google's contributions didn't stop there---they have been helping us in [various ways ever since](https://blog.google/outreach-initiatives/google-org/google-supports-covid-19-ai-and-data-analytics-projects). -Our initial survey with Google launched in late March, +Our initial survey with Google launched in late March, deployed through various websites and apps (with whom Google partners -to run questionnaires). Each respondent opted-in to answering the survey and +to run questionnaires). Each respondent opted-in to answering the survey and agreed to legal disclosures about how the data would be used. The survey asked -just a single question: +just a single question: -> Do you or anyone in your household have a fever of at least 100 °F, along -with cough, shortness of breath, or difficulty breathing? +> Do you or anyone in your household have a fever of at least 100 °F, along +> with cough, shortness of breath, or difficulty breathing? -This pattern of symptoms defines a condition called -**COVID-like illness** or **CLI**. -A respondent could reply "Yes", "No", or "Prefer not to say". -We're also given the respondent's (inferred) county from IP address lookup. -At the start, this survey data allowed us to estimate the daily % CLI, -the percentage of people with COVID-like illness, in over 1,000 counties across +This pattern of symptoms defines a condition called +**COVID-like illness** or **CLI**. +A respondent could reply "Yes", "No", or "Prefer not to say". +We're also given the respondent's (inferred) county from IP address lookup. +At the start, this survey data allowed us to estimate the daily % CLI, +the percentage of people with COVID-like illness, in over 1,000 counties across the US. After about 2 weeks, we stopped the survey. Google wondered whether we could get equally useful information without asking a question of such a sensitive nature. In general, asking a person about their -health (or their family’s health) is not common practice on Google’s survey -platform. The hope was that asking a broader question might also improve -response rates, reduce costs, and increase the number of potential respondents. +health (or their family’s health) is not common practice on Google’s survey +platform. The hope was that asking a broader question might also improve +response rates, reduce costs, and increase the number of potential respondents. -## CLI-in-Community +## CLI-in-Community Working with Brett Slatkin (head of Google Surveys) -and Hal Varian (Google's Chief Economist), we looked for a new question. +and Hal Varian (Google's Chief Economist), we looked for a new question. Brett came up with a list of questions that were acceptable, and the most promising among them was: > Do you know of someone in your community who is sick with a fever, along with -cough, shortness of breath, or difficulty breathing right now? - -We decided to deploy this proxy question[^1] on April 11, 2020. -We narrowed our focus to fewer counties: -roughly the top 600 in terms of population, -and estimated the daily % CLI-in-community, -the percentage of people who *know someone in their community* with COVID-like -illness. The initial results far exceeded our expectations, -and were promising enough that within days we added -this CLI-in-community question to our survey through Facebook. - -[^1]: In the survey methodology literature, a "proxy question" is one in which -the subject is asked to report on someone else. The traditional view seems to -be that proxy questions can undermine survey data quality, but in our setting -it's critical: not only does it provide a safeguard against revealing personal -health information, it turned out to deliver [much higher -correlations](`r blogdown::shortcode_html("ref", "2020-08-26-fb-survey#basic-correlation-analysis")`) -with case rates than the direct (non-proxy) question. - -To give you a feel for the data, below we plot -the daily new COVID-19 cases per 100,000 people -versus the estimated % CLI-in-community from our Google survey, -at the state level, averaged over April 11 to mid-May. -This is shown on the left, and on the right, -we reproduce this with the estimated % CLI-in-community from our Facebook +> cough, shortness of breath, or difficulty breathing right now? + +We decided to deploy this proxy question[^1] on April 11, 2020. +We narrowed our focus to fewer counties: +roughly the top 600 in terms of population, +and estimated the daily % CLI-in-community, +the percentage of people who _know someone in their community_ with COVID-like +illness. The initial results far exceeded our expectations, +and were promising enough that within days we added +this CLI-in-community question to our survey through Facebook. + +[^1]: + In the survey methodology literature, a "proxy question" is one in which + the subject is asked to report on someone else. The traditional view seems to + be that proxy questions can undermine survey data quality, but in our setting + it's critical: not only does it provide a safeguard against revealing personal + health information, it turned out to deliver [much higher + correlations](`r blogdown::shortcode_html("ref", "2020-08-26-fb-survey#basic-correlation-analysis")`) + with case rates than the direct (non-proxy) question. + +To give you a feel for the data, below we plot +the daily new COVID-19 cases per 100,000 people +versus the estimated % CLI-in-community from our Google survey, +at the state level, averaged over April 11 to mid-May. +This is shown on the left, and on the right, +we reproduce this with the estimated % CLI-in-community from our Facebook survey. ```{r, include = FALSE} @@ -150,18 +151,18 @@ df_go = covidcast_signal("google-survey", "smoothed_cli", geo_type = "state") start_day = min(df_go$time_value) end_day = max(df_go$time_value) -df_fb = covidcast_signal("fb-survey", "smoothed_hh_cmnty_cli", +df_fb = covidcast_signal("fb-survey", "smoothed_hh_cmnty_cli", start_day, end_day, geo_type = "state") df_in = covidcast_signal("jhu-csse", "confirmed_7dav_incidence_prop", start_day, end_day, geo_type = "state") -# Join by state, average signals, compute correlations +# Join by state, average signals, compute correlations df1 = inner_join(df_go %>% group_by(geo_value) %>% summarize(x = mean(value)), df_in %>% group_by(geo_value) %>% summarize(y = mean(value)), - by = "geo_value") + by = "geo_value") df2 = inner_join(df_fb %>% group_by(geo_value) %>% summarize(x = mean(value)), - df_in %>% group_by(geo_value) %>% summarize(y = mean(value)), - by = "geo_value") + df_in %>% group_by(geo_value) %>% summarize(y = mean(value)), + by = "geo_value") # Join again to get state populations df1 = inner_join(df1, state_census %>% mutate(ABBR = tolower(ABBR)), @@ -174,22 +175,22 @@ ggplot_colors = c("#FC4E07", "#00AFBB", "#E7B800") # Now make plots subtitle = paste("Averaged over", start_day, "to", end_day) -p1 = ggplot(df1, aes(x = x, y = y, label = toupper(geo_value))) + +p1 = ggplot(df1, aes(x = x, y = y, label = toupper(geo_value))) + geom_smooth(method = "lm", col = ggplot_colors[2], se = FALSE) + - geom_point(aes(size = POPESTIMATE2019), color = ggplot_colors[2], - alpha = 0.5) + - scale_size(name = "Population", range = c(1, 10)) + + geom_point(aes(size = POPESTIMATE2019), color = ggplot_colors[2], + alpha = 0.5) + + scale_size(name = "Population", range = c(1, 10)) + geom_text(alpha = 0.5) + - labs(x = "% CLI-in-community from Google surveys", + labs(x = "% CLI-in-community from Google surveys", y = "Daily new confirmed COVID-19 cases per 100,000 people", title = "COVID-19 case rates vs Google % CLI-in-community", subtitle = subtitle) + theme_bw() + theme(legend.position = "bottom") -p2 = ggplot(df2, aes(x = x, y = y, label = toupper(geo_value))) + +p2 = ggplot(df2, aes(x = x, y = y, label = toupper(geo_value))) + geom_smooth(method = "lm", col = ggplot_colors[1], se = FALSE) + - geom_point(aes(size = POPESTIMATE2019), color = ggplot_colors[1], - alpha = 0.5) + - scale_size(name = "Population", range = c(1, 10)) + + geom_point(aes(size = POPESTIMATE2019), color = ggplot_colors[1], + alpha = 0.5) + + scale_size(name = "Population", range = c(1, 10)) + geom_text(alpha = 0.5) + labs(x = "% CLI-in-community from Facebook surveys", y = "", title = "COVID-19 case rates vs Facebook % CLI-in-community", @@ -198,88 +199,89 @@ p2 = ggplot(df2, aes(x = x, y = y, label = toupper(geo_value))) + grid.arrange(p1, p2, nrow = 1) ``` -In both plots, we see a reassuring trend, -but the trend on the left is noticeably stronger. -Indeed, the correlation here between the Google signal and case rates is -`r round(cor(df1$x, df1$y), 2)`, -while that between the Facebook signal and case rates is +In both plots, we see a reassuring trend, +but the trend on the left is noticeably stronger. +Indeed, the correlation here between the Google signal and case rates is +`r round(cor(df1$x, df1$y), 2)`, +while that between the Facebook signal and case rates is `r round(cor(df2$x, df2$y), 2)`. To be fair, we should note that the Google signal comprises a much -larger number of survey samples (as we'll emphasize next), +larger number of survey samples (as we'll emphasize next), and the Facebook signal's correlations to case rates shot up in mid-June (as we saw last time and we'll revisit, shortly). -From April 11 through May 14, we ran Google surveys in over 600 counties -per day, with a target of at least 1,000 responses per county. +From April 11 through May 14, we ran Google surveys in over 600 counties +per day, with a target of at least 1,000 responses per county. The average number of responses per day was over 600,000, -and at its peak, over 1.2 million! -(By comparison, our survey through Facebook averages -about 74,000 responses per day.) -The actual sampling scheme behind our Google survey is more complicated, +and at its peak, over 1.2 million! +(By comparison, our survey through Facebook averages +about 74,000 responses per day.) +The actual sampling scheme behind our Google survey is more complicated, and involves two-level stratification, across both counties and states. -For details, including those on statistical estimation, -visit our [COVIDcast signals +For details, including those on statistical estimation, +visit our [COVIDcast signals documentation](`r blogdown::shortcode_html("apiref", "api/covidcast-signals/google-survey.html")`). -On May 15, we paused our Google survey to focus on our Facebook survey, -which is both longer and more complex. -Importantly, the latter is *not* a replacement for the former, -and our two surveys have different and complementary use cases. +On May 15, we paused our Google survey to focus on our Facebook survey, +which is both longer and more complex. +Importantly, the latter is _not_ a replacement for the former, +and our two surveys have different and complementary use cases. ## Our Two Surveys -We discuss some similarities and differences +We discuss some similarities and differences between the Google and Facebook surveys. -Starting with similarities, both have been deployed at a massive scale, -reaching tens of thousands of people per day, -and covering much of the US at the county level. -To state the obvious, both ask the same question: -whether a person knows someone in their community with COVID-like illness, -and both lead to an estimate of % CLI-in-community. - -Below we assess the numerical similarity of these estimates via correlations: -we correlate them against each other, and for reference, -correlate each against COVID-19 case rates. +Starting with similarities, both have been deployed at a massive scale, +reaching tens of thousands of people per day, +and covering much of the US at the county level. +To state the obvious, both ask the same question: +whether a person knows someone in their community with COVID-like illness, +and both lead to an estimate of % CLI-in-community. + +Below we assess the numerical similarity of these estimates via correlations: +we correlate them against each other, and for reference, +correlate each against COVID-19 case rates. To be more specific, for each pair of the following: Google signal, Facebook signal, and COVID-19 case rates, -and for each day that we have data available, -we compute the Spearman correlation across all counties +and for each day that we have data available, +we compute the Spearman correlation across all counties that had at least 200 cumulative COVID-19 cases -by May 14 (the end of Google survey data). -Over the first month of data, from mid-April to mid-May, -we can see that the highest correlations clearly belong to -those between the two survey signals. -This is as expected, since in principle, -these two surveys are measuring the same underlying quantity.[^2] -The next largest correlations over the first month -belong to those between the Google signal and case rates, -which for the most part holds a substantial gap over the -correlations between the Facebook signal and case rates. +by May 14 (the end of Google survey data). +Over the first month of data, from mid-April to mid-May, +we can see that the highest correlations clearly belong to +those between the two survey signals. +This is as expected, since in principle, +these two surveys are measuring the same underlying quantity.[^2] +The next largest correlations over the first month +belong to those between the Google signal and case rates, +which for the most part holds a substantial gap over the +correlations between the Facebook signal and case rates. This is no doubt encouraging, especially because we'd hope that the Google correlations would have only improved later in -the year (as did the Facebook correlations, which we [previously +the year (as did the Facebook correlations, which we [previously suggested](`r blogdown::shortcode_html("ref", "2020-08-26-fb-survey#basic-correlation-analysis")`) could have been due to the increase in the diversity of county-level case rates around mid-June). -[^2]: A closer look reveals that the relationship between the Google -% CLI-in-community and Facebook % CLI-in-community signals is not 1:1. For -example, you can check the x-axes in the first example in this blog post: the -range of the Facebook signal is over twice that of the Google signal. There -are differences in the setups we can point to: the two surveys phrase the -CLI-in-community question slightly differently; they reach different subpopulations -of the US; and the estimation procedures behind the surveys handle -missing responses differently. But as far as we can tell, none of this can really -explain why the Facebook numbers are over twice as large as the Google ones, a -trend that seems pretty consistent across location and time. We'll save rigorous -analysis for when we work on deploying these two surveys in tandem; for now, we -emphasize that this observation reiterates the importance of focusing on -*time-varying trends* in the survey signals, not the signal values themselves -(a point we [made in our last -post](`r blogdown::shortcode_html("ref", "2020-08-26-fb-survey#why-run-these-surveys")`)). -Here, the self-reporting aspect must somehow be creating greatly different -levels of bias in the two surveys; in an absolute sense, the subsequent -estimates of % CLI-in-community strongly disagree, so both can't be right, and -this casts doubt on the idea that either could be bias-free. +[^2]: + A closer look reveals that the relationship between the Google + % CLI-in-community and Facebook % CLI-in-community signals is not 1:1. For + example, you can check the x-axes in the first example in this blog post: the + range of the Facebook signal is over twice that of the Google signal. There + are differences in the setups we can point to: the two surveys phrase the + CLI-in-community question slightly differently; they reach different subpopulations + of the US; and the estimation procedures behind the surveys handle + missing responses differently. But as far as we can tell, none of this can really + explain why the Facebook numbers are over twice as large as the Google ones, a + trend that seems pretty consistent across location and time. We'll save rigorous + analysis for when we work on deploying these two surveys in tandem; for now, we + emphasize that this observation reiterates the importance of focusing on + _time-varying trends_ in the survey signals, not the signal values themselves + (a point we [made in our last + post](`r blogdown::shortcode_html("ref", "2020-08-26-fb-survey#why-run-these-surveys")`)). + Here, the self-reporting aspect must somehow be creating greatly different + levels of bias in the two surveys; in an absolute sense, the subsequent + estimates of % CLI-in-community strongly disagree, so both can't be right, and + this casts doubt on the idea that either could be bias-free. ```{r, message = FALSE, warning = FALSE, fig.width = 6, fig.height = 5} # Fetch county-level Google and Facebook % CLI-in-community signals, and JHU @@ -289,7 +291,7 @@ df_go = covidcast_signal("google-survey", "smoothed_cli") start_day = min(df_go$time_value) end_day = "2020-09-01" -df_fb = covidcast_signal("fb-survey", "smoothed_hh_cmnty_cli", +df_fb = covidcast_signal("fb-survey", "smoothed_hh_cmnty_cli", start_day, end_day) df_in = covidcast_signal("jhu-csse", "confirmed_7dav_incidence_prop", start_day, end_day) @@ -321,34 +323,34 @@ ggplot(df_cor, aes(x = time_value, y = value)) + scale_color_manual(values = ggplot_colors) + labs(title = "Correlation between survey signals and case rates", subtitle = sprintf("Over all counties with at least %i cumulative cases", - case_num), + case_num), x = "Date", y = "Correlation") + theme_bw() + theme(legend.pos = "bottom", legend.title = element_blank()) ``` Now let's consider the differences between the surveys. Here's a summary: -- Our Facebook survey is *advertised by Facebook* but *run by us* (on a -CMU-licensed Qualtrics platform); this means that we receive all the -individual survey responses directly (and Facebook never sees any of the data). +- Our Facebook survey is _advertised by Facebook_ but _run by us_ (on a + CMU-licensed Qualtrics platform); this means that we receive all the + individual survey responses directly (and Facebook never sees any of the data). -- On the other hand, our Google survey is *deployed directly by Google* through -partner websites and apps they use to run questionnaires; -this means that we don't see individual survey responses, -but receive aggregated survey data from Google. +- On the other hand, our Google survey is _deployed directly by Google_ through + partner websites and apps they use to run questionnaires; + this means that we don't see individual survey responses, + but receive aggregated survey data from Google. - This makes a big difference as to how much (and what questions) we can ask on -the survey: our Google survey is just a single question long, and our Facebook -survey is much longer and more detailed, currently over 35 questions. + the survey: our Google survey is just a single question long, and our Facebook + survey is much longer and more detailed, currently over 35 questions. -- Finally, there's a big difference in how much we control with respect to the -geographic distribution of the survey samples: on our Facebook survey, we have -no control over this, but on our Google survey we have full control, -in that we can pick the counties we want to sample from ahead of time. +- Finally, there's a big difference in how much we control with respect to the + geographic distribution of the survey samples: on our Facebook survey, we have + no control over this, but on our Google survey we have full control, + in that we can pick the counties we want to sample from ahead of time. ## Google Survey Redux -As we can see from the above summary, +As we can see from the above summary, the two survey schemes are complementary, and could be used synergistically. Our Facebook survey is a continuously-running, wide-reaching instrument @@ -364,4 +366,4 @@ terminology!). This could be done automatically (it would be a pretty big, nonstationary [multi-armed bandit problem](https://en.wikipedia.org/wiki/Multi-armed_bandit)) or manually (in collaboration with partners in public health and/or data -journalists). Stay tuned to the Delphi blog for updates. \ No newline at end of file +journalists). Stay tuned to the Delphi blog for updates. diff --git a/content/blog/2020-09-21-forecast-demo.Rmd b/content/blog/2020-09-21-forecast-demo.Rmd index 8ab8b97b1..e21ec7599 100644 --- a/content/blog/2020-09-21-forecast-demo.Rmd +++ b/content/blog/2020-09-21-forecast-demo.Rmd @@ -4,7 +4,7 @@ author: "Ryan Tibshirani" date: 2020-09-21 tags: ["symptom surveys", "forecasting", "COVIDcast", "R"] authors: -- ryan + - ryan heroImage: /blog/images/blog-Lg-img_can symptoms surveys improve covid-19.png heroImageThumb: /blog/images/blog-thumbnail_can symptoms surveys improve covid-19.png summary: | @@ -20,8 +20,8 @@ acknowledgements: | attributable to Ryan's work alone, and are a reflection of the work carried out by all these team members.* related: -- 2020-09-18-google-survey -- 2020-08-26-fb-survey + - 2020-09-18-google-survey + - 2020-08-26-fb-survey output: html_document: code_folding: hide @@ -44,64 +44,65 @@ the surveys, examining whether the % CLI-in-community indicators from our two surveys can be used to improve the accuracy of short-term forecasts of county-level COVID-19 case rates. -**Forecasting** has long been a primary initiative of the Delphi research -group (in the past for flu, and currently for COVID-19). -Each week since mid-July we've been submitting forecasts -to the [COVID Forecast Hub](https://covid19forecasthub.org), -which serves as the official data source for the -[CDC's communications on COVID-19 +**Forecasting** has long been a primary initiative of the Delphi research +group (in the past for flu, and currently for COVID-19). +Each week since mid-July we've been submitting forecasts +to the [COVID Forecast Hub](https://covid19forecasthub.org), +which serves as the official data source for the +[CDC's communications on COVID-19 forecasts](https://www.cdc.gov/coronavirus/2019-ncov/covid-data/mathematical-modeling.html). -At the outset, we should state that this post is neither a report on Delphi's -current COVID-19 forecasters nor an authoritative take on cutting-edge -COVID-19 forecasting. Instead, our purpose here to study the Facebook and -Google % CLI-in-community signals, and demonstrate their value, when used as -features, to add predictive power beyond what we can achieve with (fairly -simple) time series models trained on case rates alone. In a future blog post, -we'll follow up with details on our "production" forecasters. +At the outset, we should state that this post is neither a report on Delphi's +current COVID-19 forecasters nor an authoritative take on cutting-edge +COVID-19 forecasting. Instead, our purpose here to study the Facebook and +Google % CLI-in-community signals, and demonstrate their value, when used as +features, to add predictive power beyond what we can achieve with (fairly +simple) time series models trained on case rates alone. In a future blog post, +we'll follow up with details on our "production" forecasters. -While we focus here on forecasting, there are a number of other important +While we focus here on forecasting, there are a number of other important prediction problems that arise as part of the pandemic response. For example, we also work on **hotspot detection**, where the goal is to predict whether case rates will rise significantly, rather than to predict the future case rates directly, as in forecasting. To motivate why we might be optimistic about the utility of incorporating -our survey signals into forecasting or hotspot detection models, you can +our survey signals into forecasting or hotspot detection models, you can check out our [previous exploratory -investigations](`r blogdown::shortcode_html("ref", "2020-08-26-fb-survey#some-interesting-examples")`), -which suggested that they can serve as **early indicators** of COVID-19 +investigations](`r blogdown::shortcode_html("ref", "2020-08-26-fb-survey#some-interesting-examples")`), +which suggested that they can serve as **early indicators** of COVID-19 activity. Stay tuned to the Delphi blog for a post on hotspot detection soon. Next we explain the forecasting setup and results in detail. -Some parts may get a bit technical, but we've tried to make the post -accessible enough that you can catch the main points +Some parts may get a bit technical, but we've tried to make the post +accessible enough that you can catch the main points even if you skip the technical details. -## Problem Setup +## Problem Setup Formally, our goal here is to predict county-level COVID-19 case incidence rates, 1 and 2 weeks ahead. -Specifically, we wish to predict the number of new COVID-19 cases per capita, -both over the next 1-7 days and over the next 8-14 days. -One convenient (and equivalent) way to phrase this problem -is to predict the *smoothed* COVID-19 case incidence rate 7 days ahead and +Specifically, we wish to predict the number of new COVID-19 cases per capita, +both over the next 1-7 days and over the next 8-14 days. +One convenient (and equivalent) way to phrase this problem +is to predict the _smoothed_ COVID-19 case incidence rate 7 days ahead and 14 days ahead, where the smoothing is performed via a 7-day trailing average. -We restrict our attention to the 440 counties -that had at least 200 confirmed cases by May 14, 2020 -(the end of the Google survey data) and in which -both the Facebook and Google % CLI-in-community signals are available -(there were 604 counties in total with at least 200 confirmed cases by May 14, +We restrict our attention to the 440 counties +that had at least 200 confirmed cases by May 14, 2020 +(the end of the Google survey data) and in which +both the Facebook and Google % CLI-in-community signals are available +(there were 604 counties in total with at least 200 confirmed cases by May 14, and we dropped 164 of them due to a lack of Facebook or Google survey data). -To fix notation, let $Y_{\ell,t}$ denote the smoothed COVID-19 case -incidence rate for location (county) $\ell$ and time (day) $t$. -Let $F_{\ell,t}$ and $G_{\ell,t}$ denote -the Facebook and Google % CLI-in-community signals, respectively, -for location $\ell$ and time $t$. -(We rescale all these signals from their given values in our API -so that they are true proportions: between 0 and 1.) +To fix notation, let $Y_{\ell,t}$ denote the smoothed COVID-19 case +incidence rate for location (county) $\ell$ and time (day) $t$. +Let $F_{\ell,t}$ and $G_{\ell,t}$ denote +the Facebook and Google % CLI-in-community signals, respectively, +for location $\ell$ and time $t$. +(We rescale all these signals from their given values in our API +so that they are true proportions: between 0 and 1.) We evaluate the following four models: + $$ \begin{aligned} &\text{Cases:} \quad && h(Y_{\ell,t+d}) @@ -118,105 +119,110 @@ $$ \sum_{j=0}^2 \tau_j h(G_{\ell,t-7j}). \end{aligned} $$ -Here $d=7$ or $d=14$, depending on the target value -(number of days we predict ahead), -and $h$ is a transformation to be specified later. + +Here $d=7$ or $d=14$, depending on the target value +(number of days we predict ahead), +and $h$ is a transformation to be specified later. Informally, the first model bases its predictions of future case rates on the following three features: current COVID-19 case rates, and those 1 and 2 weeks back. The second model additionally incorporates the current Facebook signal, and the Facebook signal from 1 and 2 weeks back. -The third model is exactly same but substitutes the Google signal +The third model is exactly same but substitutes the Google signal instead of the Facebook one. -Finally, the fourth model uses both Facebook and Google signals. -For each model, in order to make a forecast at time $t_0$ +Finally, the fourth model uses both Facebook and Google signals. +For each model, in order to make a forecast at time $t_0$ (to predict case rates at time $t_0+d$), -we fit a linear model using least absolute deviations (LAD) regression, -training over all locations $\ell$ (all 440 counties), -and all time $t$ that are within the most recent 14 days of data -available up to and including time $t_0$. +we fit a linear model using least absolute deviations (LAD) regression, +training over all locations $\ell$ (all 440 counties), +and all time $t$ that are within the most recent 14 days of data +available up to and including time $t_0$. -Forecasts are transformed back to the original scale -(we apply $h^{-1}$ to the predictions from the fitted LAD model), +Forecasts are transformed back to the original scale +(we apply $h^{-1}$ to the predictions from the fitted LAD model), and denoted $\hat{Y}_{\ell,t_0+d}$. -For an error metric, we consider **scaled absolute error** +For an error metric, we consider **scaled absolute error** (or just scaled error for short): + $$ \frac{|\hat{Y}_{\ell,t_0+d} - Y_{\ell,t_0+d}|} {|Y_{\ell,t_0} - Y_{\ell,t_0+d}|}, $$ + where the error in the denominator is the error of the "strawman" model, -which for any target always simply predicts the most recent available case rate. +which for any target always simply predicts the most recent available case rate. -This normalization helps for two reasons. -First, it gives us an interpretable scale, -as we can understand the scaled error as a fraction improvement -over the strawman's error (so numbers like 0.8 or 0.9 would be favorable, -and numbers like 2 or 5 or 10 would be increasingly disastrous). +This normalization helps for two reasons. +First, it gives us an interpretable scale, +as we can understand the scaled error as a fraction improvement +over the strawman's error (so numbers like 0.8 or 0.9 would be favorable, +and numbers like 2 or 5 or 10 would be increasingly disastrous). Second, in our forecasting problem, -there turns out to a considerable amount of county-to-county variability +there turns out to a considerable amount of county-to-county variability in forecasting difficulty, and normalizing by the strawman's error helps adjust for this (so that the aggregate results aren't dominated by county-to-county differences). ## Transformations -We investigated three transformations $h$: identity, log, and logit (the -latter two being common variance-stabilizing transforms for proportions). -The results in all three cases were quite similar -and the qualitative conclusions don't change at all -(the code below supports all three, so you can check this for yourself). -For brevity, we'll just show the results for the logit transform -(actually, a "padded" version $h(x) = \log\left(\frac{x+a}{1-x+a}\right)$, -where the numerator and denominator are pushed away from zero -by a small constant, which we took to be $a=0.01$). +We investigated three transformations $h$: identity, log, and logit (the +latter two being common variance-stabilizing transforms for proportions). +The results in all three cases were quite similar +and the qualitative conclusions don't change at all +(the code below supports all three, so you can check this for yourself). +For brevity, we'll just show the results for the logit transform +(actually, a "padded" version $h(x) = \log\left(\frac{x+a}{1-x+a}\right)$, +where the numerator and denominator are pushed away from zero +by a small constant, which we took to be $a=0.01$). ## Forecasting Code -The code below marches the forecast date $t_0$ forward, -one day at a time (from April 11 to September 1), fits the four models, -makes predictions 7 and 14 days ahead, and records errors. -It takes a little while to run[^1], the culprit being LAD regression: -the training sets in our forecasting problem get moderately large -(aggregating the data over 440 counties and 14 days results in over 6000 -training samples), and at this scale LAD regression is much slower than least -squares regression. We ran this R code separately and saved the results in an -RData file; you can find this in the [same GitHub -repo](https://github.com/cmu-delphi/delphi-blog/tree/main/content/post) +The code below marches the forecast date $t_0$ forward, +one day at a time (from April 11 to September 1), fits the four models, +makes predictions 7 and 14 days ahead, and records errors. +It takes a little while to run[^1], the culprit being LAD regression: +the training sets in our forecasting problem get moderately large +(aggregating the data over 440 counties and 14 days results in over 6000 +training samples), and at this scale LAD regression is much slower than least +squares regression. We ran this R code separately and saved the results in an +RData file; you can find this in the [same GitHub +repo](https://github.com/cmu-delphi/delphi-blog/tree/main/content/post) as the Rmd source for this blog post. -[^1]: The R package [quantgen](https://github.com/ryantibs/quantgen) allows you -to choose between Gurobi or GLPK as the underlying LP solver. The default is -GLPK, since it's open source; but if you can use Gurobi (which is [free for -academic use](https://www.gurobi.com/academia/academic-program-and-licenses/)), -then this forecast demo will run much faster. +[^1]: + The R package [quantgen](https://github.com/ryantibs/quantgen) allows you + to choose between Gurobi or GLPK as the underlying LP solver. The default is + GLPK, since it's open source; but if you can use Gurobi (which is [free for + academic use](https://www.gurobi.com/academia/academic-program-and-licenses/)), + then this forecast demo will run much faster. ```{r, eval = FALSE, code = readLines("forecast-demo/demo.R")} + ``` ## Results: All Four Models -We first compare the results across all four models. -For this analysis, we filter down to common forecast dates -available for the four models (to set an even footing for the comparison), -which ends up being May 6 through May 14 for 7-day-ahead forecasts, -and only May 13 through May 14 for 14-day-ahead forecasts. -(The reason for this shortened period: +We first compare the results across all four models. +For this analysis, we filter down to common forecast dates +available for the four models (to set an even footing for the comparison), +which ends up being May 6 through May 14 for 7-day-ahead forecasts, +and only May 13 through May 14 for 14-day-ahead forecasts. +(The reason for this shortened period: we paused running the Google survey on May 14 so its data ends there, but as we explained in our last post, we [plan to bring it -back](`r blogdown::shortcode("ref", "2020-09-18-google-survey#google-survey-redux")`) -later this fall.) Hence we skip studying the 14-day-ahead forecasts results +back](`r blogdown::shortcode("ref", "2020-09-18-google-survey#google-survey-redux")`) +later this fall.) Hence we skip studying the 14-day-ahead forecasts results in this four-way model discussion, as they're only based on 2 days of test data. -Below we compute and print the median scaled errors for each of the four models -over the 9-day test period (recall that the scaled error is the absolute error -of the model's forecast relative to that of the strawman; and each test day -actually comprises 440 forecasts, over the 440 counties being considered). -We can see that adding either or both of the survey signals +Below we compute and print the median scaled errors for each of the four models +over the 9-day test period (recall that the scaled error is the absolute error +of the model's forecast relative to that of the strawman; and each test day +actually comprises 440 forecasts, over the 440 counties being considered). +We can see that adding either or both of the survey signals improves on the median scaled error of the model that uses cases only, with the biggest gain achieved by the "Cases + Google" model. -We can also see that the median scaled errors are all close to 1 +We can also see that the median scaled errors are all close to 1 (with all but that from "Cases + Google" model exceeding 1), which speaks to the difficulty of the forecasting problem. @@ -229,7 +235,7 @@ library(ggplot2) model_names = c("Cases", "Cases + Facebook", "Cases + Google", "Cases + Facebook + Google") -# Restrict to common period for all 4 models, then calculate the scaled errors +# Restrict to common period for all 4 models, then calculate the scaled errors # for each model, that is, the error relative to the strawman's error res_all4 = res %>% drop_na() %>% # Restrict to common time @@ -238,48 +244,49 @@ res_all4 = res %>% mutate(dif12 = err1 - err2, dif13 = err1 - err3, # Compute differences dif14 = err1 - err4) %>% # relative to cases model ungroup() %>% - select(-err0) - + select(-err0) + # Calculate and print median errors, for all 4 models, and just 7 days ahead -res_err4 = res_all4 %>% +res_err4 = res_all4 %>% select(-starts_with("dif")) %>% pivot_longer(names_to = "model", values_to = "err", cols = -c(geo_value, time_value, lead)) %>% mutate(lead = factor(lead, labels = paste(leads, "days ahead")), model = factor(model, labels = model_names)) -knitr::kable(res_err4 %>% +knitr::kable(res_err4 %>% group_by(model, lead) %>% - summarize(err = median(err), n = length(unique(time_value))) %>% + summarize(err = median(err), n = length(unique(time_value))) %>% arrange(lead) %>% ungroup() %>% - rename("Model" = model, "Median scaled error" = err, + rename("Model" = model, "Median scaled error" = err, "Target" = lead, "Test days" = n) %>% - filter(Target == "7 days ahead"), + filter(Target == "7 days ahead"), caption = paste("Test period:", min(res_err4$time_value), "to", max(res_err4$time_value)), format = "html", table.attr = "style='width:70%;'") ``` -Are these differences in median scaled errors significant? -It's hard to say, but some basic hypothesis testing suggests -that they probably are: below we conduct a [sign -test](https://en.wikipedia.org/wiki/Sign_test)[^2] -for whether the difference in the "Cases" model's scaled error -and each other model's scaled error is centered at zero. -The sign test is run on the 9 test days x 440 counties = 3960 pairs +Are these differences in median scaled errors significant? +It's hard to say, but some basic hypothesis testing suggests +that they probably are: below we conduct a [sign +test](https://en.wikipedia.org/wiki/Sign_test)[^2] +for whether the difference in the "Cases" model's scaled error +and each other model's scaled error is centered at zero. +The sign test is run on the 9 test days x 440 counties = 3960 pairs of scaled errors. The p-values from the "Cases" versus "Cases + Facebook" and the "Cases" versus "Cases + Google" tests are tiny; -the p-value from the "Cases" versus "Cases + Facebook + Google" test -is much bigger but still below 0.01. - -[^2]: As far as nonparametric tests of medians go, [Wilcoxon's signed-rank -test](https://en.wikipedia.org/wiki/Wilcoxon_signed-rank_test) -(for paired data, as we have here) is more popular, -because it tends to be more powerful than the sign test. -Applied here, it does indeed give smaller p-values pretty much across the board. -However, it assumes symmetry of the distribution in question -(in our case, the difference in scaled errors), -whereas the sign test does not, and thus we show results from the latter. +the p-value from the "Cases" versus "Cases + Facebook + Google" test +is much bigger but still below 0.01. + +[^2]: + As far as nonparametric tests of medians go, [Wilcoxon's signed-rank + test](https://en.wikipedia.org/wiki/Wilcoxon_signed-rank_test) + (for paired data, as we have here) is more popular, + because it tends to be more powerful than the sign test. + Applied here, it does indeed give smaller p-values pretty much across the board. + However, it assumes symmetry of the distribution in question + (in our case, the difference in scaled errors), + whereas the sign test does not, and thus we show results from the latter. ```{r, message = FALSE, warning = FALSE} # Compute p-values using the sign test against a one-sided alternative, for @@ -289,46 +296,46 @@ res_dif4 = res_all4 %>% pivot_longer(names_to = "model", values_to = "dif", cols = -c(geo_value, time_value, lead)) %>% mutate(lead = factor(lead, labels = paste(leads, "days ahead")), - model = factor(model, + model = factor(model, labels = c("Cases vs Cases + Facebook", "Cases vs Cases + Google", - "Cases vs Cases + Facebook + Google"))) + "Cases vs Cases + Facebook + Google"))) knitr::kable(res_dif4 %>% group_by(model, lead) %>% - summarize(p = binom.test(x = sum(dif > 0, na.rm = TRUE), + summarize(p = binom.test(x = sum(dif > 0, na.rm = TRUE), n = n(), alt = "greater")$p.val) %>% ungroup() %>% filter(lead == "7 days ahead") %>% - rename("Comparison" = model, "Target" = lead, "P-value" = p), + rename("Comparison" = model, "Target" = lead, "P-value" = p), format = "html", table.attr = "style='width:50%;'") ``` -We should read these with a grain of salt: the sign test here assumes -independence of observations, which clearly can't be true, -given the spatiotemporal structure of our forecasting problem. -To mitigate the dependence across time -(which intuitively seems to matter more than that across space), -we recomputed these tests in a stratified way, -where for each day we run a sign test on the scaled errors -between two models over all 440 counties. -The results are plotted as histograms below; -the "Cases + Facebook" and "Cases + Google" models -appear to deliver some decently small p-values, -but the story is not as clear with the "Cases + Facebook + Google" model. +We should read these with a grain of salt: the sign test here assumes +independence of observations, which clearly can't be true, +given the spatiotemporal structure of our forecasting problem. +To mitigate the dependence across time +(which intuitively seems to matter more than that across space), +we recomputed these tests in a stratified way, +where for each day we run a sign test on the scaled errors +between two models over all 440 counties. +The results are plotted as histograms below; +the "Cases + Facebook" and "Cases + Google" models +appear to deliver some decently small p-values, +but the story is not as clear with the "Cases + Facebook + Google" model. ```{r, message = FALSE, warning = FALSE, fig.width = 9, fig.height = 3.5} # Red, blue (similar to ggplot defaults), then yellow ggplot_colors = c("#FC4E07", "#00AFBB", "#E7B800") -ggplot(res_dif4 %>% +ggplot(res_dif4 %>% group_by(model, lead, time_value) %>% - summarize(p = binom.test(x = sum(dif > 0, na.rm = TRUE), + summarize(p = binom.test(x = sum(dif > 0, na.rm = TRUE), n = n(), alt = "greater")$p.val) %>% ungroup() %>% filter(lead == "7 days ahead"), aes(p)) + - geom_histogram(aes(color = model, fill = model), alpha = 0.4) + + geom_histogram(aes(color = model, fill = model), alpha = 0.4) + scale_color_manual(values = ggplot_colors) + scale_fill_manual(values = ggplot_colors) + - facet_wrap(vars(lead, model)) + + facet_wrap(vars(lead, model)) + labs(x = "P-value", y = "Count") + theme_bw() + theme(legend.pos = "none") ``` @@ -336,16 +343,16 @@ ggplot(res_dif4 %>% ## Results: First Two Models Next we focus on comparing results between the "Cases" and "Cases + Facebook" -models only. Restricting to a common available forecast dates yields a much -longer test period, May 6 through August 25 for 7-day-ahead forecasts, -and May 13 through August 18 for 14-day-ahead forecasts. -The median scaled errors over the test period are computed and reported below. -Now we see a decent improvement in median scaled error for the -"Cases + Facebook" model, and this is true for both 7-day-ahead and +models only. Restricting to a common available forecast dates yields a much +longer test period, May 6 through August 25 for 7-day-ahead forecasts, +and May 13 through August 18 for 14-day-ahead forecasts. +The median scaled errors over the test period are computed and reported below. +Now we see a decent improvement in median scaled error for the +"Cases + Facebook" model, and this is true for both 7-day-ahead and 14-day-ahead forecasts. ```{r, message = FALSE, warning = FALSE} -# Restrict to common period for just models 1 and 2, then calculate the scaled +# Restrict to common period for just models 1 and 2, then calculate the scaled # errors, that is, the error relative to the strawman's error res_all2 = res %>% select(-c(err3, err4)) %>% @@ -355,72 +362,72 @@ res_all2 = res %>% mutate(dif12 = err1 - err2) %>% # Compute differences # relative to cases model ungroup() %>% - select(-err0) - -# Calculate and print median errors, for just models 1 and 2, and both 7 and 14 + select(-err0) + +# Calculate and print median errors, for just models 1 and 2, and both 7 and 14 # days ahead -res_err2 = res_all2 %>% +res_err2 = res_all2 %>% select(-starts_with("dif")) %>% pivot_longer(names_to = "model", values_to = "err", cols = -c(geo_value, time_value, lead)) %>% mutate(lead = factor(lead, labels = paste(leads, "days ahead")), model = factor(model, labels = model_names[1:2])) - -knitr::kable(res_err2 %>% + +knitr::kable(res_err2 %>% select(-starts_with("dif")) %>% group_by(model, lead) %>% - summarize(err = median(err), n = length(unique(time_value))) %>% + summarize(err = median(err), n = length(unique(time_value))) %>% arrange(lead) %>% ungroup() %>% - rename("Model" = model, "Median scaled error" = err, + rename("Model" = model, "Median scaled error" = err, "Target" = lead, "Test days" = n), caption = paste("Test period:", min(res_err2$time_value), "to", max(res_err2$time_value)), format = "html", table.attr = "style='width:70%;'") ``` -Thanks to the extended length of the test period, -we can also "unravel" these median scaled errors +Thanks to the extended length of the test period, +we can also "unravel" these median scaled errors over time and plot their trajectories, as we do below, -with the left plot concerning 7-day-ahead forecasts, -and the right 14-day-ahead forecasts. -These plots reveal something at once interesting and bothersome: -the median scaled errors are quite volatile over time, -and for some periods in July, forecasting became much "harder", -with the scaled errors reaching above 1.25 for 7-day-ahead forecasts, -and above 1.5 for 14-day-ahead forecasts. -Furthermore, towards the positive, we can see a clear visual -difference between the median scaled errors from -the "Cases + Facebook" model in red and the "Cases" model in black. +with the left plot concerning 7-day-ahead forecasts, +and the right 14-day-ahead forecasts. +These plots reveal something at once interesting and bothersome: +the median scaled errors are quite volatile over time, +and for some periods in July, forecasting became much "harder", +with the scaled errors reaching above 1.25 for 7-day-ahead forecasts, +and above 1.5 for 14-day-ahead forecasts. +Furthermore, towards the positive, we can see a clear visual +difference between the median scaled errors from +the "Cases + Facebook" model in red and the "Cases" model in black. The former appears to be below the latter pretty consistently over time, -with the possible exception of periods where forecasting -becomes "hard" and the scaled errors shoot above 1. +with the possible exception of periods where forecasting +becomes "hard" and the scaled errors shoot above 1. ```{r, message = FALSE, warning = FALSE, fig.width = 9, fig.height = 5} # Plot median errors as a function of time, for models 1 and 2, and both 7 and # 14 days ahead -ggplot(res_err2 %>% +ggplot(res_err2 %>% group_by(model, lead, time_value) %>% summarize(err = median(err)) %>% ungroup(), - aes(x = time_value, y = err)) + - geom_line(aes(color = model)) + + aes(x = time_value, y = err)) + + geom_line(aes(color = model)) + scale_color_manual(values = c("black", ggplot_colors)) + geom_hline(yintercept = 1, linetype = 2, color = "gray") + - facet_wrap(vars(lead)) + + facet_wrap(vars(lead)) + labs(x = "Date", y = "Median scaled error") + theme_bw() + theme(legend.pos = "bottom", legend.title = element_blank()) ``` -Again, basic hypothesis testing suggests that the results we're seeing here -are likely significant, though it's hard to say definitively -given the complicated dependence structure present in the data. -Below we perform a sign test for whether the difference in scaled errors -from the "Cases" and "Cases + Facebook" models is centered at zero. -Given the large sample size: 112 test days for 7-day-ahead forecasts -and 98 test days for 14-day-ahead forecasts -(times 440 counties for each day), the p-values are basically zero. +Again, basic hypothesis testing suggests that the results we're seeing here +are likely significant, though it's hard to say definitively +given the complicated dependence structure present in the data. +Below we perform a sign test for whether the difference in scaled errors +from the "Cases" and "Cases + Facebook" models is centered at zero. +Given the large sample size: 112 test days for 7-day-ahead forecasts +and 98 test days for 14-day-ahead forecasts +(times 440 counties for each day), the p-values are basically zero. ```{r, message = FALSE, warning = FALSE} -# Compute p-values using the sign test against a one-sided alternative, just +# Compute p-values using the sign test against a one-sided alternative, just # for models 1 and 2, and both 7 and 14 days ahead res_dif2 = res_all2 %>% select(-starts_with("err")) %>% @@ -429,29 +436,29 @@ res_dif2 = res_all2 %>% mutate(lead = factor(lead, labels = paste(leads, "days ahead")), model = factor(model, labels = "Cases > Cases + Facebook")) -knitr::kable(res_dif2 %>% +knitr::kable(res_dif2 %>% group_by(model, lead) %>% summarize(p = binom.test(x = sum(dif > 0, na.rm = TRUE), - n = n(), alt = "greater")$p.val) %>% - ungroup() %>% - rename("Comparison" = model, "Target" = lead, "P-value" = p), + n = n(), alt = "greater")$p.val) %>% + ungroup() %>% + rename("Comparison" = model, "Target" = lead, "P-value" = p), format = "html", table.attr = "style='width:50%;'") ``` -Once we stratify and recompute p-values by forecast date, -as shown in the histograms below, +Once we stratify and recompute p-values by forecast date, +as shown in the histograms below, the bulk of p-values are still quite small. ```{r, message = FALSE, warning = FALSE, fig.width = 7, fig.height = 4} -ggplot(res_dif2 %>% +ggplot(res_dif2 %>% group_by(model, lead, time_value) %>% - summarize(p = binom.test(x = sum(dif > 0, na.rm = TRUE), + summarize(p = binom.test(x = sum(dif > 0, na.rm = TRUE), n = n(), alt = "greater")$p.val) %>% ungroup(), aes(p)) + - geom_histogram(aes(color = model, fill = model), alpha = 0.4) + + geom_histogram(aes(color = model, fill = model), alpha = 0.4) + scale_color_manual(values = ggplot_colors) + scale_fill_manual(values = ggplot_colors) + - facet_wrap(vars(lead, model)) + + facet_wrap(vars(lead, model)) + labs(x = "P-value", y = "Count") + theme_bw() + theme(legend.pos = "none") ``` @@ -460,44 +467,44 @@ ggplot(res_dif2 %>% **[\*Added September 25, 2020]** -Hypothesis tests (like the sign tests conducted above) tell us -whether the differences in errors +Hypothesis tests (like the sign tests conducted above) tell us +whether the differences in errors (between the forecasters) -are *statistically* significant, -but not about their *practical* significance. -For example, for 7-day-ahead forecasts, +are _statistically_ significant, +but not about their _practical_ significance. +For example, for 7-day-ahead forecasts, what does an improvement of 0.018 units on the scaled error scale really mean, -when comparing the "Cases + Facebook" model to the "Cases" model -(over the test period May 6 through August 25)? -Is this a meaningful gain? - -To answer questions like this, -we can look at the way that the median scaled errors -behave as a function of the number of days ahead -at which we're making the forecasts. -Previously, we considered forecasting case rates -just 7 and 14 days ahead; -now we systematically examine 5, 6, 7, etc., through 20 days ahead. -As before, we ran the code for this separately -and saved the results in an RData file, -which you can find in the [same GitHub -repo](https://github.com/cmu-delphi/delphi-blog/tree/main/content/post) -as the Rmd source for this blog post. +when comparing the "Cases + Facebook" model to the "Cases" model +(over the test period May 6 through August 25)? +Is this a meaningful gain? + +To answer questions like this, +we can look at the way that the median scaled errors +behave as a function of the number of days ahead +at which we're making the forecasts. +Previously, we considered forecasting case rates +just 7 and 14 days ahead; +now we systematically examine 5, 6, 7, etc., through 20 days ahead. +As before, we ran the code for this separately +and saved the results in an RData file, +which you can find in the [same GitHub +repo](https://github.com/cmu-delphi/delphi-blog/tree/main/content/post) +as the Rmd source for this blog post. (It's exactly the same code as that above, -but with `leads = 5:20`.) +but with `leads = 5:20`.) -Below, we compute and plot the median scaled errors -for the "Cases" and "Cases + Facebook" models +Below, we compute and plot the median scaled errors +for the "Cases" and "Cases + Facebook" models for different number of days ahead for the forecast target. -This is done over all forecast dates common to the two models -(May 6 through August 27, or earlier---the end date gets decremented -each time we increase the number of days ahead). -A first glance shows that the "Cases + Facebook" model, -in red, gives better median scaled errors at all ahead values; -and the vertical gap between the two curves -is consistently in the range of what we were seeing before -(for 7 and 14 days ahead), -around 0.02 units or more on the scaled error scale. +This is done over all forecast dates common to the two models +(May 6 through August 27, or earlier---the end date gets decremented +each time we increase the number of days ahead). +A first glance shows that the "Cases + Facebook" model, +in red, gives better median scaled errors at all ahead values; +and the vertical gap between the two curves +is consistently in the range of what we were seeing before +(for 7 and 14 days ahead), +around 0.02 units or more on the scaled error scale. ```{r, message = FALSE, warning = FALSE, fig.width = 6, fig.height = 5} load("forecast-demo/demo-extended.rda") @@ -514,14 +521,14 @@ err_by_lead = res %>% cols = -c(geo_value, time_value, lead)) %>% mutate(model = factor(model, labels = model_names[1:2])) %>% group_by(model, lead) %>% - summarize(err = median(err)) %>% + summarize(err = median(err)) %>% ungroup() -ggplot(err_by_lead, aes(x = lead, y = err)) + - geom_line(aes(color = model)) + - geom_point(aes(color = model)) + +ggplot(err_by_lead, aes(x = lead, y = err)) + + geom_line(aes(color = model)) + + geom_point(aes(color = model)) + scale_color_manual(values = c("black", ggplot_colors)) + - geom_hline(yintercept = err_by_lead %>% + geom_hline(yintercept = err_by_lead %>% filter(lead %in% 7, model == "Cases") %>% pull(err), linetype = 2, color = "gray") + labs(title = "Forecasting errors by number of days ahead", @@ -531,92 +538,93 @@ ggplot(err_by_lead, aes(x = lead, y = err)) + theme_bw() + theme(legend.pos = "bottom", legend.title = element_blank()) ``` -But if we look at it from a different angle, -and consider the *horizontal* gap between the curves, -then we can infer something quite a bit more interesting: -for 7-day-ahead forecasts, -the median scaled error of the "Cases" model -(marked by a horizontal gray line) +But if we look at it from a different angle, +and consider the _horizontal_ gap between the curves, +then we can infer something quite a bit more interesting: +for 7-day-ahead forecasts, +the median scaled error of the "Cases" model +(marked by a horizontal gray line) is comparable to that of 12-day-ahead forecasts from the "Cases + Facebook" -model. So you could say that using the % CLI-in-community signal from our -Facebook survey buys us *5 extra days of lead time* for this forecasting -problem, which seems pretty nontrivial. -Different forecast targets yield different lead times -(for 14-day-ahead forecasts, for example, it appears -to be more like 3 or 4 days of lead time), +model. So you could say that using the % CLI-in-community signal from our +Facebook survey buys us _5 extra days of lead time_ for this forecasting +problem, which seems pretty nontrivial. +Different forecast targets yield different lead times +(for 14-day-ahead forecasts, for example, it appears +to be more like 3 or 4 days of lead time), but the added value of the survey signal is clear throughout. - + ## Wrap-Up -We've seen that either of the Facebook or Google % CLI-in-community -signals can improve the accuracy of short-term forecasts -of county-level COVID-19 case incidence rates. -The significance of these improvements +We've seen that either of the Facebook or Google % CLI-in-community +signals can improve the accuracy of short-term forecasts +of county-level COVID-19 case incidence rates. +The significance of these improvements is more apparent with the Facebook signal, thanks -to the much longer test period available. -With either signal, of the magnitude of the improvement offered -seems modest but nontrivial, especially because the forecasting problem +to the much longer test period available. +With either signal, of the magnitude of the improvement offered +seems modest but nontrivial, especially because the forecasting problem is so hard in the first place. We reiterate that this was just a demo. Our analysis was fairly simple and lacks -a few qualities that we'd expect in a truly comprehensive, realistic forecasting -analysis. To name three: - -1. The models we considered are simple autoregressive structures from standard -time series, and could be improved in various ways (including, considering other -relevant dimensions like mobility measures, county health metrics, etc.). - -2. The forecasts we produced are *point* rather than *distributional* forecasts -(that is, we predict a single number, rather than an entire distribution, for -what happens 7 and 14 days ahead). Distributional forecasts portray uncertainty -in a transparent way, which is important in practice. - -3. The way we trained our forecast models does not account -for *data latency* and *revisions*, which are critical issues. -For each (retrospective) forecast date $t_0$, -we constructed forecasts by training on data -that we fetched from the API today, "as of" the day we wrote this blog post, -and not "as of" the forecast date $t_0$. -This matters because nearly all signals are subject to latency -(they are only available at some number of days lag) -and go through multiple revisions (past data values get updated as time -goes on). +a few qualities that we'd expect in a truly comprehensive, realistic forecasting +analysis. To name three: + +1. The models we considered are simple autoregressive structures from standard + time series, and could be improved in various ways (including, considering other + relevant dimensions like mobility measures, county health metrics, etc.). + +2. The forecasts we produced are _point_ rather than _distributional_ forecasts + (that is, we predict a single number, rather than an entire distribution, for + what happens 7 and 14 days ahead). Distributional forecasts portray uncertainty + in a transparent way, which is important in practice. + +3. The way we trained our forecast models does not account + for _data latency_ and _revisions_, which are critical issues. + For each (retrospective) forecast date $t_0$, + we constructed forecasts by training on data + that we fetched from the API today, "as of" the day we wrote this blog post, + and not "as of" the forecast date $t_0$. + This matters because nearly all signals are subject to latency + (they are only available at some number of days lag) + and go through multiple revisions (past data values get updated as time + goes on). On the flip side, our example here was not that "far away" from being realistic. The models we examined are actually not too different from Delphi's forecasters -"in production".[^3] Also, the way we fit LAD regression models in the code -extends immediately to multiple quantile regression +"in production".[^3] Also, the way we fit LAD regression models in the code +extends immediately to multiple quantile regression (just requires changing the parameter `tau` in the call to `quantile_lasso()`), -which would give us distributional forecasts. -And lastly, it's fairly easy to change the data acquisition step in the code +which would give us distributional forecasts. +And lastly, it's fairly easy to change the data acquisition step in the code so that data gets pulled "as of" the forecast date (requires specifying the parameter `as_of` in the call to `covidcast_signal()`). -Hopefully these preliminary findings have gotten you excited -about the possible uses of our symptom survey data. +Hopefully these preliminary findings have gotten you excited +about the possible uses of our symptom survey data. To get started playing with our data yourself, take a look at our [interactive COVIDcast map](`r blogdown::shortcode_html("ref", "covidcast")`), -or our [COVIDcast +or our [COVIDcast API](`r blogdown::shortcode_html("apiref", "api/covidcast.html")`), -through our [R client](https://cmu-delphi.github.io/covidcast/covidcastR/) +through our [R client](https://cmu-delphi.github.io/covidcast/covidcastR/) or [Python client](https://cmu-delphi.github.io/covidcast/covidcast-py/html/). -And if you're feeling adventurous, consider competing in the +And if you're feeling adventurous, consider competing in the [COVID-19 Symptom Data Challenge](https://www.symptomchallenge.org/), -and trying your hand at developing novel analytic approaches to extract +and trying your hand at developing novel analytic approaches to extract insights from our Facebook symptom survey data. We've made extensive data aggregate data from our survey (beyond CLI indicators) [available for the Challenge](https://www.symptomchallenge.org/challenge#sources), and submissions are due September 29, with finalists eligible for cash prizes. We look forward to seeing how you put our data to use! -[^3]: Delphi's "production" forecasters are still based on relatively simple -times series models, though to be clear they're distributional, -and we add a couple of extra layers of complexity on top of standard structures. -For short-term forecasts, we've found that simple statistical models can be -competitive with compartmental models (like -[SIR](https://en.wikipedia.org/wiki/Compartmental_models_in_epidemiology#The_SIR_model) -and its [many -variants](https://en.wikipedia.org/wiki/Compartmental_models_in_epidemiology#Variations_on_the_basic_SIR_model)), -even fairly complicated ones. Being statisticians and computer scientists, we -find these statistical models are easier to build, debug, and most importantly, -calibrate. More on this in a future blog post. +[^3]: + Delphi's "production" forecasters are still based on relatively simple + times series models, though to be clear they're distributional, + and we add a couple of extra layers of complexity on top of standard structures. + For short-term forecasts, we've found that simple statistical models can be + competitive with compartmental models (like + [SIR](https://en.wikipedia.org/wiki/Compartmental_models_in_epidemiology#The_SIR_model) + and its [many + variants](https://en.wikipedia.org/wiki/Compartmental_models_in_epidemiology#Variations_on_the_basic_SIR_model)), + even fairly complicated ones. Being statisticians and computer scientists, we + find these statistical models are easier to build, debug, and most importantly, + calibrate. More on this in a future blog post. diff --git a/content/blog/2020-10-06-survey-wave-4.Rmd b/content/blog/2020-10-06-survey-wave-4.Rmd index 07250f559..21b47837a 100644 --- a/content/blog/2020-10-06-survey-wave-4.Rmd +++ b/content/blog/2020-10-06-survey-wave-4.Rmd @@ -8,7 +8,7 @@ summary: | Facebook helps us recruit tens of thousands of respondents daily, and the new survey gives us unprecedented insights into the effects of COVID-19 across the United States. Today we release new public datasets and share maps revealing access to COVID testing, test results, and public use of masks. authors: -- alex + - alex heroImage: /blog/images/blog-lg-img_New and Improved COVID.png heroImageThumb: /blog/images/blog-thumb-img_New and Improved COVID.png acknowledgments: | @@ -28,7 +28,7 @@ acknowledgments: | Beginning in early April 2020, the [Delphi group](`r blogdown::shortcode("ref", "/")`) has conducted a major survey to track COVID-19 across the United States. With the support of Facebook Data for Good, we have been able to recruit tens of -thousands of active Facebook users *every day* to take our voluntary survey. +thousands of active Facebook users _every day_ to take our voluntary survey. Concurrently, a University of Maryland team has conducted a [parallel international effort](https://covidmap.umd.edu/) covering over 100 countries worldwide. Every day, we aggregate our survey results to produce estimates of @@ -243,7 +243,7 @@ wear a mask; also, many states with high mask usage had major outbreaks earlier in the pandemic. Nonetheless, this data could be very useful to epidemiological researchers studying the public reaction to the pandemic and its spread. -Mask-wearing surveys have been done before---for example, [the *New York Times* +Mask-wearing surveys have been done before---for example, [the _New York Times_ commissioned a large survey during July 2020 and produced detailed maps](https://www.nytimes.com/interactive/2020/07/17/upshot/coronavirus-face-mask-map.html)---but because our survey runs continuously, we will be able to track how the @@ -341,7 +341,7 @@ estimate for that state.) Test positivity only indirectly answers a crucial question: Does test availability meet demand for tests? When our survey respondents say they have -*not* been tested in the past 14 days, we ask whether they *wanted* to be tested +_not_ been tested in the past 14 days, we ask whether they _wanted_ to be tested in that time. This also varies across the United States: ```{r wanted-test, message=FALSE} @@ -360,7 +360,6 @@ COVIDcast API, alongside numerous other data streams, we hope to provide researchers, public health officials, and journalists the information they need to form a more complete picture of the pandemic. - ## You Can Help Analyze This Data All of the maps and graphs above were built using data we make publicly @@ -399,12 +398,12 @@ important roles informing our national pandemic response. Armed with the right data, we can make decisions needed to protect public health and permit safe reopening. -*For more information about Delphi's symptom surveys, and for media contact +_For more information about Delphi's symptom surveys, and for media contact details, see [our surveys page](`r blogdown::shortcode_html("ref", "surveys")`). For -updates, you can follow [CmuDelphi on Twitter](https://twitter.com/cmudelphi).* +updates, you can follow [CmuDelphi on Twitter](https://twitter.com/cmudelphi)._ -**Note.** *This post was updated on October 17, 2020 to correct an error in the +**Note.** _This post was updated on October 17, 2020 to correct an error in the scatterplot of mask usage and reported case rates. An error in our data processing system meant our reported case rates were half the size they should have been. This did not affect the trend in the scatterplot, only the scale of -the X axis.* +the X axis._ diff --git a/content/blog/2020-10-14-dv-signal.Rmd b/content/blog/2020-10-14-dv-signal.Rmd index fed617e88..7cca827d4 100644 --- a/content/blog/2020-10-14-dv-signal.Rmd +++ b/content/blog/2020-10-14-dv-signal.Rmd @@ -4,16 +4,16 @@ author: "Aaron Rumack and Roni Rosenfeld" date: 2020-11-05 tags: ["medical records", "COVIDcast", "R"] authors: -- aaron -- roni + - aaron + - roni heroImage: /blog/images/blog-img_A Syndromic COVID-19.png heroImageThumb: /blog/images/blog-thumb-img_A Syndromic COVID-19.png related: -- 2020-09-18-google-survey -- 2020-08-26-fb-survey + - 2020-09-18-google-survey + - 2020-08-26-fb-survey summary: | In previous posts, we discussed our massive ongoing symptom surveys that have reached over 12 million people in the U.S. since April 2020, in partnership with Facebook and Google. Another one of our major data initiatives is based on partnerships with healthcare systems, granting us access to various aggregate statistics from hospital records and insurance claims covering 10-15% of the United States population. From these data, we can extract informative indicators that can be early indicators of COVID activity. This post focuses on one indicator in particular, based on outpatient visits, and demonstrates both the challenges and promises associated with medical records data. -acknowledgements: | +acknowledgements: | Maria Jahja contributed immensely to every stage of this project, from determining which ICD codes to use to the final implementation of the indicator. Aaron Rumack devised the weekday adjustment and analyzed the performance of the DV indicator. Roni Rosenfeld worked closely with our health systems partners to get access to the data and provided domain knowledge to ensure that the data was useful. Both Roni and Ryan Tibshirani provided helpful suggestions and insights towards the methodology and analysis. @@ -23,39 +23,39 @@ output: code_folding: hide --- -Our COVIDcast [map](`r blogdown::shortcode_html("ref", "covidcast")`) and [API](`r blogdown::shortcode_html("apiref", "api/covidcast.html")`) feature several novel early indicators of COVID-19 activity. In past posts, we discussed our large-scale daily surveys that, as of October 2020, have reached over 12 million people throughout the US, in partnership with [Facebook](`r blogdown::shortcode_html("ref", "2020-08-26-fb-survey")`) and [Google](`r blogdown::shortcode_html("ref", "2020-09-18-google-survey")`). In another ongoing data initiative, health system partners grant us access to various aggregate statistics from hospital records and insurance claims covering 10-15% of the United States population. From these data, we can extract informative indicators that can be early indicators of COVID activity. Early indicators are important because they help policymakers make more informed decisions and can also improve epidemiological forecasts. +Our COVIDcast [map](`r blogdown::shortcode_html("ref", "covidcast")`) and [API](`r blogdown::shortcode_html("apiref", "api/covidcast.html")`) feature several novel early indicators of COVID-19 activity. In past posts, we discussed our large-scale daily surveys that, as of October 2020, have reached over 12 million people throughout the US, in partnership with [Facebook](`r blogdown::shortcode_html("ref", "2020-08-26-fb-survey")`) and [Google](`r blogdown::shortcode_html("ref", "2020-09-18-google-survey")`). In another ongoing data initiative, health system partners grant us access to various aggregate statistics from hospital records and insurance claims covering 10-15% of the United States population. From these data, we can extract informative indicators that can be early indicators of COVID activity. Early indicators are important because they help policymakers make more informed decisions and can also improve epidemiological forecasts. -One indicator that we created from the outpatient insurance claims portion of this data is what we call the **Doctor Visits** or **DV** indicator, which estimates the percentage of outpatient visits (including telemedicine, urgent care, and emergency department visits) that are due to COVID-Like Illness or CLI. We will use % CLI-in-DV to abbreviate the percentage of outpatient visits due to CLI, the units of our DV indicator. Below, we explain how we calculate the DV indicator and discuss its relation to confirmed COVID-19 cases. We also discuss our observation that the DV indicator possesses significant spatial heterogeneity (it displays systematic county-to-county differences), and we describe a simple dynamic adjustment to address this issue. +One indicator that we created from the outpatient insurance claims portion of this data is what we call the **Doctor Visits** or **DV** indicator, which estimates the percentage of outpatient visits (including telemedicine, urgent care, and emergency department visits) that are due to COVID-Like Illness or CLI. We will use % CLI-in-DV to abbreviate the percentage of outpatient visits due to CLI, the units of our DV indicator. Below, we explain how we calculate the DV indicator and discuss its relation to confirmed COVID-19 cases. We also discuss our observation that the DV indicator possesses significant spatial heterogeneity (it displays systematic county-to-county differences), and we describe a simple dynamic adjustment to address this issue. ## Motivation -Electronic medical records (EMR) are instrumental in providing epidemiological information in near real-time. Sick patients interact with the healthcare system up to several weeks before they are confirmed as cases and added to the published count (see below). Thus signals based on EMR data can be *early indicators*, preceding official reports of confirmed cases and deaths. Early knowledge of a rise in COVID-like symptoms can enable early containment efforts to manage emerging outbreaks. - +Electronic medical records (EMR) are instrumental in providing epidemiological information in near real-time. Sick patients interact with the healthcare system up to several weeks before they are confirmed as cases and added to the published count (see below). Thus signals based on EMR data can be _early indicators_, preceding official reports of confirmed cases and deaths. Early knowledge of a rise in COVID-like symptoms can enable early containment efforts to manage emerging outbreaks. + The value of EMR data becomes apparent when you compare them to official COVID-19 case counts: - + - **Limited testing capacity.** Especially prevalent in the earlier days of the pandemic, limited testing capacity means that many people saw a doctor for COVID-like symptoms but did not receive a test, even though they should have. These people would be represented in our DV indicator but absent from the confirmed counts. - + - **Reporting delays.** From the moment a patient is first examined (or remotely evaluated), it may take several days until their specimen is collected, and another few days after testing before results are available. When tests come back positive, they must (by law) be reported to the public health authorities. The health authorities typically do further verification and investigation (for example, determining age and other demographics) before adding the cases to their public website. As a result, reports of confirmed cases trail behind the medical evaluation date by several weeks. Even with the reporting delays of insurance claims (see below), a claims-based signal allows estimates of disease activity within 3-4 days of when the outpatient visits occurred. -- **Date accuracy.** Sites like [JHU CSSE](https://www.arcgis.com/apps/opsdashboard/index.html#/bda7594740fd40299423467b48e9ecf6) and [USAFacts](https://usafacts.org/visualizations/coronavirus-covid-19-spread-map/) scrape and report cumulative confirmed cases based on the cases' first date of public posting. As a result, the "number of new daily counts" (derived by successive subtraction from these reports) often refers to the number of cases *published* instead of those *lab-confirmed* on that date. In comparison, insurance claims indicate a "date of service", which is typically closer to the testing date. Accurate dating of cases is critical for modeling, analysis, and forecasting. - +- **Date accuracy.** Sites like [JHU CSSE](https://www.arcgis.com/apps/opsdashboard/index.html#/bda7594740fd40299423467b48e9ecf6) and [USAFacts](https://usafacts.org/visualizations/coronavirus-covid-19-spread-map/) scrape and report cumulative confirmed cases based on the cases' first date of public posting. As a result, the "number of new daily counts" (derived by successive subtraction from these reports) often refers to the number of cases _published_ instead of those _lab-confirmed_ on that date. In comparison, insurance claims indicate a "date of service", which is typically closer to the testing date. Accurate dating of cases is critical for modeling, analysis, and forecasting. + - **Retrospective revisions.** Some states have changed their definition of confirmed COVID-19 cases. This is sometimes done retroactively, adding hundreds or thousands of past cases and reporting them as "new" on the day that the definition was changed, resulting in an artificial spike on that date, with no way to assign these cases to their correct dates. This degrades the quality of the case time series. - -## The Doctor Visits Indicator - + +## The Doctor Visits Indicator + The Doctor Visits indicator is based solely on insurance claims. We count the outpatient claims for a given geographic area and day that fall into each of five categories: - -1. Total: All claims, whether or not related to COVID-19 (also known as *all-cause* claims). - -2. Flu: ICD-10 primary code starting with J09, J10, or J11, a definitive diagnosis of influenza. - + +1. Total: All claims, whether or not related to COVID-19 (also known as _all-cause_ claims). + +2. Flu: ICD-10 primary code starting with J09, J10, or J11, a definitive diagnosis of influenza. + 3. COVID-like: ICD-10 primary code in {U07.1, U07.2, B97.29, J12.81, Z03.818, B34.2, J12.89}. Some of these codes correspond to a definitive diagnosis of COVID-19, while others indicate strongly related conditions. - -4. Flu-like: ICD-10 primary code of J22 (acute lower respiratory infection) or B34.9 (viral infection, unspecified). - -5. Mixed: ICD-10 primary code of Z20.828 (suspected exposure to COVID-19) or J12.9 (viral pneumonia). -We estimate the percent of COVID-like illness in doctor visits (% CLI-in-DV) as 100 * (COVID-like + Flu-like + Mixed - Flu) / Total. We subtract the "Flu" count because the higher the presence of confirmed flu, the larger the fraction of flu-like cases that are due to flu rather than to COVID. See [our signal documentation site](`r blogdown::shortcode_html("apiref", "api/covidcast-signals/doctor-visits.html")`) for more details. +4. Flu-like: ICD-10 primary code of J22 (acute lower respiratory infection) or B34.9 (viral infection, unspecified). + +5. Mixed: ICD-10 primary code of Z20.828 (suspected exposure to COVID-19) or J12.9 (viral pneumonia). + +We estimate the percent of COVID-like illness in doctor visits (% CLI-in-DV) as 100 \* (COVID-like + Flu-like + Mixed - Flu) / Total. We subtract the "Flu" count because the higher the presence of confirmed flu, the larger the fraction of flu-like cases that are due to flu rather than to COVID. See [our signal documentation site](`r blogdown::shortcode_html("apiref", "api/covidcast-signals/doctor-visits.html")`) for more details. The indicator is available daily starting February 1, 2020 (although understandably, it is nearly zero in all locations until mid-March). To preserve privacy and data integrity, we do not report the indicator for a given location and date if there are fewer than 500 total visits in the seven days ending on that date. Following these restrictions, each day, we are able to produce estimates for about 2000 counties (roughly two-thirds of the counties in the US), accounting for over 90% of the country's population. @@ -111,7 +111,7 @@ Interestingly, certain types of claims have longer latency than others. We found ## Weekday Effects -Another challenge is the influence of the day of the week on the DV indicator. On weekends, both total counts and COVID-like counts decrease, but proportionally, total counts decrease more. This is because doctor visits during the weekend tend to focus on acute care. The total counts include many visits related to non-acute issues, but almost all COVID-like counts are due to acute issues. Without adjusting for this weekday effect, the DV indicator has a "sawtooth" pattern, spiking on weekends. We derived a method to create an adjusted indicator that accounts for this weekday effect (for a precise description, see our [signal documentation](`r blogdown::shortcode_html("apiref", "api/covidcast-signals/doctor-visits.html#day-of-week-adjustment")`)). Below, we visualize the effect of making these adjustments. When we do not adjust for the weekday effect, we see a sawtooth pattern that clearly does not represent true changes in COVID-like illness within a location. However, after making the weekday adjustment, we get a smooth curve that looks reasonable. It is important to note that this adjustment is *not* temporal smoothing! Rather, we are making an adjustment each day based on historical patterns of weekday-to-weekend differences. +Another challenge is the influence of the day of the week on the DV indicator. On weekends, both total counts and COVID-like counts decrease, but proportionally, total counts decrease more. This is because doctor visits during the weekend tend to focus on acute care. The total counts include many visits related to non-acute issues, but almost all COVID-like counts are due to acute issues. Without adjusting for this weekday effect, the DV indicator has a "sawtooth" pattern, spiking on weekends. We derived a method to create an adjusted indicator that accounts for this weekday effect (for a precise description, see our [signal documentation](`r blogdown::shortcode_html("apiref", "api/covidcast-signals/doctor-visits.html#day-of-week-adjustment")`)). Below, we visualize the effect of making these adjustments. When we do not adjust for the weekday effect, we see a sawtooth pattern that clearly does not represent true changes in COVID-like illness within a location. However, after making the weekday adjustment, we get a smooth curve that looks reasonable. It is important to note that this adjustment is _not_ temporal smoothing! Rather, we are making an adjustment each day based on historical patterns of weekday-to-weekend differences. ```{r, message = FALSE, warning = FALSE, fig.width = 8, fig.height = 8} start_day = "2020-05-01" @@ -145,7 +145,7 @@ ggplot(cmb_df %>% filter(geo_value %in% states_to_plot)) + facet_wrap(vars(geo_value)) + labs(x = "Date", y = "% CLI-in-DV", title = "DV indicator, with and without weekday adjustment") + - theme_bw() + + theme_bw() + theme(legend.position = "bottom", legend.title = element_blank()) ``` @@ -157,14 +157,14 @@ As a simple quantitative analysis, we can measure the correlation between the DV start_day = "2020-04-15" end_day = "2020-10-01" -df_adjusted = covidcast_signal("doctor-visits", "smoothed_adj_cli", +df_adjusted = covidcast_signal("doctor-visits", "smoothed_adj_cli", start_day, end_day) -df_cases = covidcast_signal("usa-facts", "confirmed_7dav_incidence_prop", +df_cases = covidcast_signal("usa-facts", "confirmed_7dav_incidence_prop", start_day, end_day) case_num = 500 cumulative_case_df = covidcast_signal("usa-facts", "confirmed_cumulative_num", - max(df_cases$time_value), + max(df_cases$time_value), max(df_cases$time_value)) geo_values = cumulative_case_df %>% filter(value >= case_num) %>% pull(geo_value) @@ -174,7 +174,7 @@ dv_cases_df = bind_rows(df_adjusted, df_cases) %>% select(geo_value, signal, time_value, value) %>% tidyr::pivot_wider(names_from = signal, values_from = value) %>% rename(cases = confirmed_7dav_incidence_prop, dv = smoothed_adj_cli) %>% - filter(purrr::map_lgl(geo_value, function(fips) { + filter(purrr::map_lgl(geo_value, function(fips) { substr(fips, 3, 5) != "000"})) %>% group_by(time_value) %>% ungroup() @@ -197,7 +197,7 @@ ggplot(df_cor_by_space) + geom_density(aes(x = value), fill = "gray") + labs(title = "Correlation-by-space between DV indicator and case rates", subtitle = "Over all counties with at least 500 cumulative cases", - x = "Correlation", y = "Density") + + x = "Correlation", y = "Density") + theme_bw() ``` @@ -214,7 +214,7 @@ tibble(thresholds = seq(500, 20000, by = 500)) %>% thresholds, function(t) { mean(df_cor_by_space %>% filter(cases >= t) %>% pull(value), na.rm=T) })) %>% - ggplot() + + ggplot() + geom_line(aes(x = thresholds, y = avg_corr)) + labs(x = "Cumulative cases threshold", y = "Correlation", title = "Mean correlation-by-space between DV indicator and case rates", @@ -229,7 +229,7 @@ ggplot(df_cor_by_time) + geom_line(aes(x = time_value, y = value)) + labs(title = "Correlation-by-time between DV indicator and case rates", subtitle = "Over all counties with at least 500 cumulative cases", - x = "Date", y = "Correlation") + + x = "Date", y = "Correlation") + theme_bw() ``` @@ -281,7 +281,7 @@ p1 = dv_cases_df %>% geom_line(aes(x = time_value, y = cases, color = as.factor(hhs))) + labs(title = "Mean case rate per HHS region", x = "Date", y = "New cases per 100,000 people", color = "HHS") + - theme_bw() + theme_bw() p2 = dv_cases_df %>% group_by(time_value, hhs) %>% @@ -301,9 +301,9 @@ Let's look more closely at the behavior of our indicator in HHS 2 (New Jersey an The answer to the first question is largely yes. The case rate curve is steadily decreasing throughout April and May, flattening out by mid-June. The DV indicator peaks a little later than case rates do, but decreases throughout May and is mostly flat by mid-June. This tells us that the correlation-by-space is good in HHS 2. However, the answer to the second question reveals an issue with the DV indicator. The case rate in HHS 2 is the highest of all HHS regions in May and into June, but by July, HHS 2 is among the lowest in case rate. This is not true with the DV indicator. HHS 2 has the highest % CLI-in-DV almost throughout the entire time period, even when its case rate is one of the lowest. So we see why the correlation-by-time began to decline starting in August: counties in HHS 2 tended to have low case rates but high % CLI-in-DV values, driving down the correlations. -We have now identified the problem: a % CLI-in-DV of, say, 5% might mean a very low case rate for a county in New York but a very high case rate for a county in Oregon. Since we have six months’ worth of history for both the DV indicator and case rates, we can correct for this problem by regressing case rates on the DV indicator in a location-specific manner. We call this *sensorization*, that is, the process of turning the DV indicator into a *sensor*. In the past, the Delphi group has done extensive work in creating sensors from multiple data sources for tracking flu, dengue, and norovirus; and developed sensor fusion methodology for combining multiple sensors into a single unified estimate. For more, see Chapter 4 of [Farrow 2016](https://delphi.cmu.edu/~dfarrow/thesis.pdf) and [Jahja et al. 2019](https://papers.nips.cc/paper/9475-kalman-filter-sensor-fusion-and-constrained-regression-equivalences-and-insights). +We have now identified the problem: a % CLI-in-DV of, say, 5% might mean a very low case rate for a county in New York but a very high case rate for a county in Oregon. Since we have six months’ worth of history for both the DV indicator and case rates, we can correct for this problem by regressing case rates on the DV indicator in a location-specific manner. We call this _sensorization_, that is, the process of turning the DV indicator into a _sensor_. In the past, the Delphi group has done extensive work in creating sensors from multiple data sources for tracking flu, dengue, and norovirus; and developed sensor fusion methodology for combining multiple sensors into a single unified estimate. For more, see Chapter 4 of [Farrow 2016](https://delphi.cmu.edu/~dfarrow/thesis.pdf) and [Jahja et al. 2019](https://papers.nips.cc/paper/9475-kalman-filter-sensor-fusion-and-constrained-regression-equivalences-and-insights). -In the present example, we sensorize by fitting a simple linear regression model of case rates on % CLI-in-DV, separately for each day and location, training on data from the past six weeks. Because we fit the regression anew each day based on the most recently available data, the method adapts to (potential) gradual changes in the underlying relationship between % CLI-in-DV and case rate. This method is mostly real-time, but for backfill in the DV indicator, as discussed above. +In the present example, we sensorize by fitting a simple linear regression model of case rates on % CLI-in-DV, separately for each day and location, training on data from the past six weeks. Because we fit the regression anew each day based on the most recently available data, the method adapts to (potential) gradual changes in the underlying relationship between % CLI-in-DV and case rate. This method is mostly real-time, but for backfill in the DV indicator, as discussed above. ```{r, message = FALSE, warning = FALSE, fig.width = 6, fig.height = 5} dv_cases_df = dv_cases_df %>% @@ -317,9 +317,9 @@ dv_cases_df = dv_cases_df %>% ungroup() dv_cases_df = dv_cases_df %>% - mutate(slope_6wk = purrr::map_dbl(coef_6wk, function(c) { + mutate(slope_6wk = purrr::map_dbl(coef_6wk, function(c) { tryCatch(c[[2]], error = function(x) {NA})})) %>% - mutate(int_6wk = purrr::map_dbl(coef_6wk, function(c) { + mutate(int_6wk = purrr::map_dbl(coef_6wk, function(c) { tryCatch(c[[1]], error = function(x) {NA})})) %>% select(-coef_6wk) @@ -338,9 +338,9 @@ df_cor_by_time_adj = covidcast_cor( method = "spearman") inner_join(df_cor_by_time %>% rename(orig_cor = value), - df_cor_by_time_adj %>% rename(adj_cor = value), + df_cor_by_time_adj %>% rename(adj_cor = value), by = "time_value") %>% - ggplot() + + ggplot() + geom_line(aes(time_value, orig_cor, color = "Original")) + geom_line(aes(time_value, adj_cor, color = "Sensorized")) + scale_color_manual(values = ggplot_colors[1:2]) + @@ -351,10 +351,10 @@ inner_join(df_cor_by_time %>% rename(orig_cor = value), theme(legend.position = "bottom", legend.title = element_blank()) ``` -From the plot above, we see that sensorizing the DV indicator greatly improves the correlations-by-time. +From the plot above, we see that sensorizing the DV indicator greatly improves the correlations-by-time. It turns out that the regression coefficients in the method described above, which are fit using a sliding window, change significantly over time. As a future direction, we plan to look into this nonstationary relationship between the DV indicator and case rates to better understand why it changes and why counties in New York and New Jersey have such a different relationship in the first place. ## Exciting New Data -We will end with exciting news: we have recently been granted access to medical insurance claims from [Change Healthcare](https://www.changehealthcare.com), a large healthcare technology company, and are in the process of creating multiple indicators from it. Their data covers approximately 45% of the population of the United States, so we expect it to enable us to substantially boost the accuracy, coverage, and resolution of our EMR-based indicators. We are also interested in comparing our new Change Healthcare indicator (a work in progress) to our current indicator in sample size, correlations, and behavior. Some insurance providers cover different populations with different rates, which could make their indicator more or less correlated with the reported case rates and other relevant indicators. We expect that by combining data sets and dramatically increasing coverage we can produce a significantly more useful indicator. \ No newline at end of file +We will end with exciting news: we have recently been granted access to medical insurance claims from [Change Healthcare](https://www.changehealthcare.com), a large healthcare technology company, and are in the process of creating multiple indicators from it. Their data covers approximately 45% of the population of the United States, so we expect it to enable us to substantially boost the accuracy, coverage, and resolution of our EMR-based indicators. We are also interested in comparing our new Change Healthcare indicator (a work in progress) to our current indicator in sample size, correlations, and behavior. Some insurance providers cover different populations with different rates, which could make their indicator more or less correlated with the reported case rates and other relevant indicators. We expect that by combining data sets and dramatically increasing coverage we can produce a significantly more useful indicator. diff --git a/content/covidcast/_index.md b/content/covidcast/_index.md index b8bffa322..c448b5b6c 100644 --- a/content/covidcast/_index.md +++ b/content/covidcast/_index.md @@ -10,5 +10,4 @@ styles: - ./covidcast/styles.css --- - {{}} diff --git a/content/covidcast/indicators/cases.md b/content/covidcast/indicators/cases.md index 77ecd3fea..0bd61cf5d 100644 --- a/content/covidcast/indicators/cases.md +++ b/content/covidcast/indicators/cases.md @@ -4,4 +4,4 @@ category: official order: 10 --- -These indicators show the number of new confirmed COVID-19 cases per day. The maps reflect only cases confirmed by state and local health authorities. They are based on confirmed case counts compiled and made public by [a team at Johns Hopkins University](https://systems.jhu.edu/research/public-health/ncov/) and by [USAFacts](https://usafacts.org/visualizations/coronavirus-covid-19-spread-map/). We use Johns Hopkins data for Puerto Rico and report USAFacts data in all other locations. \ No newline at end of file +These indicators show the number of new confirmed COVID-19 cases per day. The maps reflect only cases confirmed by state and local health authorities. They are based on confirmed case counts compiled and made public by [a team at Johns Hopkins University](https://systems.jhu.edu/research/public-health/ncov/) and by [USAFacts](https://usafacts.org/visualizations/coronavirus-covid-19-spread-map/). We use Johns Hopkins data for Puerto Rico and report USAFacts data in all other locations. diff --git a/content/covidcast/indicators/combined.md b/content/covidcast/indicators/combined.md index fb8ee55fa..9de33072d 100644 --- a/content/covidcast/indicators/combined.md +++ b/content/covidcast/indicators/combined.md @@ -4,4 +4,4 @@ category: active order: 80 --- -The “Combined” map represents a combination of all the indicators currently featured on the public map. As of this writing, this includes Doctor Visits, Symptoms (Facebook), Symptoms in Community (Facebook), and Search Trends. It does not include official reports (cases and deaths), hospital admissions, or SafeGraph signals. We use a rank-1 approximation, from a nonnegative matrix factorization approach, to identify an underlying signal that best reconstructs the indicators. Higher values of the combined signal correspond to higher values of the other indicators, but the scale (units) of the combination is arbitrary. \ No newline at end of file +The “Combined” map represents a combination of all the indicators currently featured on the public map. As of this writing, this includes Doctor Visits, Symptoms (Facebook), Symptoms in Community (Facebook), and Search Trends. It does not include official reports (cases and deaths), hospital admissions, or SafeGraph signals. We use a rank-1 approximation, from a nonnegative matrix factorization approach, to identify an underlying signal that best reconstructs the indicators. Higher values of the combined signal correspond to higher values of the other indicators, but the scale (units) of the combination is arbitrary. diff --git a/content/covidcast/indicators/deaths.md b/content/covidcast/indicators/deaths.md index 5e2c52da9..da8a4bbc4 100644 --- a/content/covidcast/indicators/deaths.md +++ b/content/covidcast/indicators/deaths.md @@ -4,4 +4,4 @@ category: official order: 20 --- -These indicators shows the number of COVID-19 related deaths per day. The maps reflect official figures by state and local health authorities, and may not include excess deaths not confirmed as due to COVID-19 by health authorities. They are based on confirmed death counts compiled and made public by [a team at Johns Hopkins University](https://systems.jhu.edu/research/public-health/ncov/) and by [USAFacts](https://usafacts.org/visualizations/coronavirus-covid-19-spread-map/). We use Johns Hopkins data for Puerto Rico and report USAFacts data in all other locations. \ No newline at end of file +These indicators shows the number of COVID-19 related deaths per day. The maps reflect official figures by state and local health authorities, and may not include excess deaths not confirmed as due to COVID-19 by health authorities. They are based on confirmed death counts compiled and made public by [a team at Johns Hopkins University](https://systems.jhu.edu/research/public-health/ncov/) and by [USAFacts](https://usafacts.org/visualizations/coronavirus-covid-19-spread-map/). We use Johns Hopkins data for Puerto Rico and report USAFacts data in all other locations. diff --git a/content/covidcast/indicators/doctor-visits.md b/content/covidcast/indicators/doctor-visits.md index 780550e3a..b5163e7d1 100644 --- a/content/covidcast/indicators/doctor-visits.md +++ b/content/covidcast/indicators/doctor-visits.md @@ -4,4 +4,4 @@ category: active order: 10 --- -This indicator estimates the percentage of outpatient doctor’s visits that are due to COVID-like symptoms, based on data provided to us by health system partners. Tele-medicine visits are included in these estimates. \ No newline at end of file +This indicator estimates the percentage of outpatient doctor’s visits that are due to COVID-like symptoms, based on data provided to us by health system partners. Tele-medicine visits are included in these estimates. diff --git a/content/covidcast/indicators/google-trends.md b/content/covidcast/indicators/google-trends.md index ff0d04b28..952c07b51 100644 --- a/content/covidcast/indicators/google-trends.md +++ b/content/covidcast/indicators/google-trends.md @@ -4,4 +4,4 @@ category: active order: 70 --- -This indicator is based on the number of Google searches for COVID-related topics, relative to each area’s population, based on Google search statistics provided to us by Google’s Health Trends group. A larger number corresponds to more COVID-related searching. \ No newline at end of file +This indicator is based on the number of Google searches for COVID-related topics, relative to each area’s population, based on Google search statistics provided to us by Google’s Health Trends group. A larger number corresponds to more COVID-related searching. diff --git a/content/covidcast/indicators/hospital-admissions.md b/content/covidcast/indicators/hospital-admissions.md index 1cbd96039..b7a6a9031 100644 --- a/content/covidcast/indicators/hospital-admissions.md +++ b/content/covidcast/indicators/hospital-admissions.md @@ -4,4 +4,4 @@ category: active order: 20 --- -This indicator estimates the percentage of daily hospital admissions that have diagnostic codes related to COVID-19, based on medical claims summaries provided to us by health system partners. \ No newline at end of file +This indicator estimates the percentage of daily hospital admissions that have diagnostic codes related to COVID-19, based on medical claims summaries provided to us by health system partners. diff --git a/content/covidcast/indicators/index.md b/content/covidcast/indicators/index.md index 3d65eaa0f..ca03031f1 100644 --- a/content/covidcast/indicators/index.md +++ b/content/covidcast/indicators/index.md @@ -1,3 +1,3 @@ --- headless: true ---- \ No newline at end of file +--- diff --git a/content/covidcast/indicators/quidel-flu.md b/content/covidcast/indicators/quidel-flu.md index ba706c2d0..260b3eba2 100644 --- a/content/covidcast/indicators/quidel-flu.md +++ b/content/covidcast/indicators/quidel-flu.md @@ -4,4 +4,4 @@ category: archived order: 20 --- -(Archived) This indicator is based on data about influenza lab tests provided to us by Quidel, Inc., a company that makes equipment and kits for medical tests. When a patient (whether at a doctor’s office, clinic, or hospital) has COVID-like symptoms, standard practice currently is to perform a conventional influenza test to rule out seasonal influenza (flu), because these two diseases have similar symptoms. While the number of COVID tests performed depends on local capacity and testing policy, influenza testing is not influenced by these factors. Because a different number of labs may report on different days, we track the average number of flu tests performed per flu testing device (in a given location and on a given day). \ No newline at end of file +(Archived) This indicator is based on data about influenza lab tests provided to us by Quidel, Inc., a company that makes equipment and kits for medical tests. When a patient (whether at a doctor’s office, clinic, or hospital) has COVID-like symptoms, standard practice currently is to perform a conventional influenza test to rule out seasonal influenza (flu), because these two diseases have similar symptoms. While the number of COVID tests performed depends on local capacity and testing policy, influenza testing is not influenced by these factors. Because a different number of labs may report on different days, we track the average number of flu tests performed per flu testing device (in a given location and on a given day). diff --git a/content/covidcast/indicators/quidel.md b/content/covidcast/indicators/quidel.md index 2a1777e56..7385e7527 100644 --- a/content/covidcast/indicators/quidel.md +++ b/content/covidcast/indicators/quidel.md @@ -4,4 +4,4 @@ category: active order: 60 --- -This indicator estimates the percentage of COVID-19 antigen tests that come back positive, based on testing data provided to us by Quidel, Inc., a company that makes equipment and kits for medical tests. When a patient (whether at a doctor’s office, clinic, or hospital) has COVID-like symptoms, doctors may order an antigen test. An antigen test can detect parts of the virus that are present during an active infection. Note that this data may differ from testing figures reported by state and local health authorities, as it only includes Quidel’s COVID-19 antigen tests, not those from other testing providers. \ No newline at end of file +This indicator estimates the percentage of COVID-19 antigen tests that come back positive, based on testing data provided to us by Quidel, Inc., a company that makes equipment and kits for medical tests. When a patient (whether at a doctor’s office, clinic, or hospital) has COVID-like symptoms, doctors may order an antigen test. An antigen test can detect parts of the virus that are present during an active infection. Note that this data may differ from testing figures reported by state and local health authorities, as it only includes Quidel’s COVID-19 antigen tests, not those from other testing providers. diff --git a/content/covidcast/indicators/safegraph.md b/content/covidcast/indicators/safegraph.md index b6315e2a2..4c25154ad 100644 --- a/content/covidcast/indicators/safegraph.md +++ b/content/covidcast/indicators/safegraph.md @@ -4,4 +4,4 @@ category: active order: 50 --- -These indicators estimate the proportion of people who spend time outside their homes during daytime hours, using mobile device location data provided by SafeGraph. “Away from Home 6hr+” is the proportion spending more than 6 hours outside their home, while “Away from Home 3-6hr” is the proportion spending between 3 and 6 hours outside their home. These estimates may be related to the spread of COVID-19, since they are related to the number of people interacting with others outside their homes, and also reveal the impact of the pandemic and movement restrictions. \ No newline at end of file +These indicators estimate the proportion of people who spend time outside their homes during daytime hours, using mobile device location data provided by SafeGraph. “Away from Home 6hr+” is the proportion spending more than 6 hours outside their home, while “Away from Home 3-6hr” is the proportion spending between 3 and 6 hours outside their home. These estimates may be related to the spread of COVID-19, since they are related to the number of people interacting with others outside their homes, and also reveal the impact of the pandemic and movement restrictions. diff --git a/content/covidcast/indicators/survey-google.md b/content/covidcast/indicators/survey-google.md index 63536b734..560913581 100644 --- a/content/covidcast/indicators/survey-google.md +++ b/content/covidcast/indicators/survey-google.md @@ -4,4 +4,4 @@ category: archived order: 10 --- -(Paused) This indicator estimates the percentage of people who *know someone in their community* with a COVID-like illness (fever, along with cough, or shortness of breath, or difficulty breathing). The data is based on Google-run symptom surveys, through publisher websites, Google's Opinions Reward app, and similar applications. These surveys are voluntary. As of mid April, about 600,000 people answered the survey daily throughout the U.S. Note that these Google surveys are estimating a different quantity than the surveys given to Facebook users (percentage of people who *know someone in their community* who is sick, rather than percentage of people who are sick), so the estimates from the Google surveys tend to be larger. \ No newline at end of file +(Paused) This indicator estimates the percentage of people who _know someone in their community_ with a COVID-like illness (fever, along with cough, or shortness of breath, or difficulty breathing). The data is based on Google-run symptom surveys, through publisher websites, Google's Opinions Reward app, and similar applications. These surveys are voluntary. As of mid April, about 600,000 people answered the survey daily throughout the U.S. Note that these Google surveys are estimating a different quantity than the surveys given to Facebook users (percentage of people who _know someone in their community_ who is sick, rather than percentage of people who are sick), so the estimates from the Google surveys tend to be larger. diff --git a/content/covidcast/indicators/symptoms-community-fb.md b/content/covidcast/indicators/symptoms-community-fb.md index 26e789fe3..1e46f3e66 100644 --- a/content/covidcast/indicators/symptoms-community-fb.md +++ b/content/covidcast/indicators/symptoms-community-fb.md @@ -4,4 +4,4 @@ category: active order: 40 --- -This indicator estimates the percentage of people who *know someone in their community* with a COVID-like illness (fever, along with cough, or shortness of breath, or difficulty breathing). The data is based on the same [CMU-run survey]({{}}), advertised by Facebook, as is used for the Symptoms indicator. Note that more people tend to report knowing someone in their community with a COVID-like illness than having someone in their own household with a COVID-like illness, so these numbers are higher than the household symptoms survey. \ No newline at end of file +This indicator estimates the percentage of people who _know someone in their community_ with a COVID-like illness (fever, along with cough, or shortness of breath, or difficulty breathing). The data is based on the same [CMU-run survey]({{}}), advertised by Facebook, as is used for the Symptoms indicator. Note that more people tend to report knowing someone in their community with a COVID-like illness than having someone in their own household with a COVID-like illness, so these numbers are higher than the household symptoms survey. diff --git a/content/covidcast/indicators/symptoms-fb.md b/content/covidcast/indicators/symptoms-fb.md index d6092a4d7..1686f468f 100644 --- a/content/covidcast/indicators/symptoms-fb.md +++ b/content/covidcast/indicators/symptoms-fb.md @@ -4,4 +4,4 @@ category: active order: 30 --- -This indicator estimates the percentage of people who have a COVID-like illness (fever, along with cough, or shortness of breath, or difficulty breathing), based on [symptom surveys run by Carnegie Mellon]({{}}). The surveys ask respondents how many people in their household are experiencing COVID-like symptoms, among other questions. Facebook directs a random sample of its users to these surveys, which are voluntary. Individual survey responses are held by CMU and are shareable with other health researchers under a data use agreement. No individual survey responses are shared back to Facebook. As of mid-June, about 70,000 such surveys were completed daily throughout the U.S. \ No newline at end of file +This indicator estimates the percentage of people who have a COVID-like illness (fever, along with cough, or shortness of breath, or difficulty breathing), based on [symptom surveys run by Carnegie Mellon]({{}}). The surveys ask respondents how many people in their household are experiencing COVID-like symptoms, among other questions. Facebook directs a random sample of its users to these surveys, which are voluntary. Individual survey responses are held by CMU and are shareable with other health researchers under a data use agreement. No individual survey responses are shared back to Facebook. As of mid-June, about 70,000 such surveys were completed daily throughout the U.S. diff --git a/content/covidcast/methodology.md b/content/covidcast/methodology.md index 2cba15ba9..b59f36761 100644 --- a/content/covidcast/methodology.md +++ b/content/covidcast/methodology.md @@ -4,12 +4,11 @@ title: About Our Data and Methodology The COVID-19 indicators visualized on our map are derived from the data sources described below. These are all publicly available on the [COVIDcast endpoint]({{< apiref "api/covidcast.html">}}) of our public [Epidata API]({{< apiref "api/README.html">}}). The API documentation includes [full technical detail]({{< apiref "api/covidcast_signals.html">}}) on how these indicators are calculated. - -## Active Indicators +## Active Indicators {{}} -## Official Reports +## Official Reports {{}} @@ -23,27 +22,24 @@ Full technical documentation on the sources of our data, and how our estimates a ### Live Estimates -The real-time COVID-19 indicators presented on the COVIDcast site represent our *best estimates given all data that we have available up until now*. For example, the estimates on our site for April 24, 2020 represent our current best estimate of the indicator values for that day. The first estimates for the indicator values for April 24 would typically be available on April 25 (one day later), but estimates for these April 24 values may be updated on later days as new data becomes available. This phenomenon is particularly prominent with the Doctor Visits indicator, which is based on doctor’s visits that do or do not involve COVID-like illnesses: there is generally a lag in how some of the data is made available to us, and a large fraction of doctor’s visits on any day is only reported to us several days later. For that reason, our Doctor Visits estimates that are just a few days old may be less reliable. When we deem them too unreliable, we do not post them, which is why this indicator is often available only up until a few days before the current day. +The real-time COVID-19 indicators presented on the COVIDcast site represent our _best estimates given all data that we have available up until now_. For example, the estimates on our site for April 24, 2020 represent our current best estimate of the indicator values for that day. The first estimates for the indicator values for April 24 would typically be available on April 25 (one day later), but estimates for these April 24 values may be updated on later days as new data becomes available. This phenomenon is particularly prominent with the Doctor Visits indicator, which is based on doctor’s visits that do or do not involve COVID-like illnesses: there is generally a lag in how some of the data is made available to us, and a large fraction of doctor’s visits on any day is only reported to us several days later. For that reason, our Doctor Visits estimates that are just a few days old may be less reliable. When we deem them too unreliable, we do not post them, which is why this indicator is often available only up until a few days before the current day. ### Smoothing -For each indicator, our estimates are formed using data smoothing techniques. The individual smoothing technique differs based on the indicator, but in all cases, we perform some kind of data smoothing (akin to averaging, or weighted averaging) across an approximately one week window. - +For each indicator, our estimates are formed using data smoothing techniques. The individual smoothing technique differs based on the indicator, but in all cases, we perform some kind of data smoothing (akin to averaging, or weighted averaging) across an approximately one week window. ### Missing Estimates -Generally, we do not report estimates at locations with insufficient data (or insufficiently recent data). The Search Trends indicator is not available at the county level, as data is only available at a coarser geographic resolution in the first place. For the Doctor Visits and Facebook Surveys indicators, we lump together all counties in a given state that do not have sufficient data for their own individual estimate, and create a “rest of state” estimate that includes all of them. - +Generally, we do not report estimates at locations with insufficient data (or insufficiently recent data). The Search Trends indicator is not available at the county level, as data is only available at a coarser geographic resolution in the first place. For the Doctor Visits and Facebook Surveys indicators, we lump together all counties in a given state that do not have sufficient data for their own individual estimate, and create a “rest of state” estimate that includes all of them. ### Intensity Heat map -The “Intensity” view presents a heat map of these estimates. For each indicator, we use a fixed range of values, from a “low” value to a “high” value, and assign a color to each value in between, as shown to the left of the map. These “low” and “high” values are different for each indicator, but for a given indicator, they are constant across time and geographic hierarchy, meaning that the heat maps are comparable across days. At the county level, the “rest of state” estimates are plotted in semi-transparent colors, to make the individual counties where estimates are made more easily visually distinguishable. +The “Intensity” view presents a heat map of these estimates. For each indicator, we use a fixed range of values, from a “low” value to a “high” value, and assign a color to each value in between, as shown to the left of the map. These “low” and “high” values are different for each indicator, but for a given indicator, they are constant across time and geographic hierarchy, meaning that the heat maps are comparable across days. At the county level, the “rest of state” estimates are plotted in semi-transparent colors, to make the individual counties where estimates are made more easily visually distinguishable. ### 7-day Trend Map -The “7-day Trend” view presents a color map of the trend underlying these estimates. This is computed by calculating the line of best fit (as measured by squared error) over the last 7 days. So for example, the trend on April 7 is based on the line of best fit through the estimates from April 1 through April 7. We then perform a basic statistical test to determine whether this line is significantly rising or falling. - +The “7-day Trend” view presents a color map of the trend underlying these estimates. This is computed by calculating the line of best fit (as measured by squared error) over the last 7 days. So for example, the trend on April 7 is based on the line of best fit through the estimates from April 1 through April 7. We then perform a basic statistical test to determine whether this line is significantly rising or falling. ### Correlation Analyses -Empirically (analyses conducted as of late April), we find that each of our COVID-19 indicators, averaged over a 1 week period, has a reasonably strong positive correlation (in particular, Spearman correlation, which measures correlation on the scale of ranks and is thus invariant to monotone transformations) with the number of COVID-19 cases confirmed during that same week, as made available through the JHU CSSE COVID-19 GitHub repository. The incidence of confirmed COVID-19 cases is arguably viewed as “the standard” metric for current COVID-19 activity (albeit flawed because it is confounded by issues like testing capacity and policy), so this is a reassuring finding. An R notebook which explicitly computes these correlations (and is completely self-contained, able to be re-compiled by any user with access to R and RStudio) is available [here](https://cmu-delphi.github.io/covidcast/R-notebooks/signal_correlations.html). +Empirically (analyses conducted as of late April), we find that each of our COVID-19 indicators, averaged over a 1 week period, has a reasonably strong positive correlation (in particular, Spearman correlation, which measures correlation on the scale of ranks and is thus invariant to monotone transformations) with the number of COVID-19 cases confirmed during that same week, as made available through the JHU CSSE COVID-19 GitHub repository. The incidence of confirmed COVID-19 cases is arguably viewed as “the standard” metric for current COVID-19 activity (albeit flawed because it is confounded by issues like testing capacity and policy), so this is a reassuring finding. An R notebook which explicitly computes these correlations (and is completely self-contained, able to be re-compiled by any user with access to R and RStudio) is available [here](https://cmu-delphi.github.io/covidcast/R-notebooks/signal_correlations.html). diff --git a/content/covidcast/release-log/_index.md b/content/covidcast/release-log/_index.md index 07688c1f7..49bc629bc 100644 --- a/content/covidcast/release-log/_index.md +++ b/content/covidcast/release-log/_index.md @@ -3,4 +3,4 @@ title: Release Log layout: single --- -{{}} \ No newline at end of file +{{}} diff --git a/content/covidcast/release-log/headless/index.md b/content/covidcast/release-log/headless/index.md index 46ca3e791..441ef3a5a 100644 --- a/content/covidcast/release-log/headless/index.md +++ b/content/covidcast/release-log/headless/index.md @@ -1,4 +1,4 @@ --- # flag to disable rendering individual pages headless: true ---- \ No newline at end of file +--- diff --git a/content/covidcast/release-log/headless/v1.1.md b/content/covidcast/release-log/headless/v1.1.md index 20d14ba7c..bab731ab5 100644 --- a/content/covidcast/release-log/headless/v1.1.md +++ b/content/covidcast/release-log/headless/v1.1.md @@ -8,4 +8,3 @@ date: 2020-05-07 - New map: The “Confirmed Cases (JHU)” map shows confirmed case ratios (cases per 100,000 population) of COVID-19 per day. This reflects official figures reporting cases confirmed by testing to be COVID-19, [as compiled by a team at Johns Hopkins University](https://systems.jhu.edu/research/public-health/ncov/). - The “Flu Testing (Quidel)” map is no longer shown. During flu season, rates of flu tests may have correlated with rates of COVID-like illnesses, as many doctors who suspected COVID-19 conducted flu tests to rule out influenza. However, the end of flu season means few flu tests are currently conducted. - Previously, the “Search Trends (Google)” signal reported search volume on each day on the map for the following day; for example, search volume on April 16 would be mapped as occurring on April 17. This is no longer done. - diff --git a/content/covidcast/release-log/headless/v1.2.md b/content/covidcast/release-log/headless/v1.2.md index 4693ff7f1..4d2259fbb 100644 --- a/content/covidcast/release-log/headless/v1.2.md +++ b/content/covidcast/release-log/headless/v1.2.md @@ -4,6 +4,6 @@ date: 2020-05-20 --- - New map: The “Combined” signal represents a statistical combination of the other indicators, not including the official reports (cases and deaths). For more information how this indicator is calculated, see the details above in the list of indicators. -- New map: The “Symptoms in Community (Facebook)” map shows the estimated fraction of people who know someone with a COVID-like illness in their local community. +- New map: The “Symptoms in Community (Facebook)” map shows the estimated fraction of people who know someone with a COVID-like illness in their local community. - The “Surveys (Facebook)” map has been renamed “Symptoms (Facebook)”, to reflect that it asks respondents whether people in their household have COVID-like symptoms. - The “Surveys (Google)” map has been removed. This data is still available in the public Epidata API, but new data will not be collected. diff --git a/content/covidcast/surveys.md b/content/covidcast/surveys.md index 3eaf55776..575bb0ade 100644 --- a/content/covidcast/surveys.md +++ b/content/covidcast/surveys.md @@ -10,7 +10,6 @@ An international version of the survey is conducted by the University of Marylan **New!** The [Symptom Data Challenge](https://www.symptomchallenge.org/) challenges participants to enable earlier detection and improved situational awareness of the COVID-19 outbreak by using data from these symptom surveys. Submissions are due by Tuesday, October 6th, 2020, and finalists are eligible to win cash prizes. See [the challenge website](https://www.symptomchallenge.org/) for more details. - ## What are the surveys for? The survey asks respondents whether they are experiencing any symptoms, then asks a series of questions designed to help researchers understand the spread of COVID-19 and its effect on people in the United States. These include questions about COVID-19 testing, prior medical conditions, social distancing measures, mental health, demographics, and the economic effects of the pandemic. @@ -19,14 +18,12 @@ Delphi uses information from the survey as part of its public [COVIDcast map]({{ Because the survey can reach thousands of respondents every day, its questions focus on what the respondents are experiencing right now. This allows us to track how conditions change across the country every day. - ## Who is running these surveys? The surveys are a collaboration between [Delphi]({{< relref "/" >}}) at Carnegie Mellon University, numerous universities, and Facebook. Researchers at many institutions are analyzing the survey data and assisting in the survey’s development. The survey protocol has been reviewed by the Carnegie Mellon University Institutional Review Board. Delphi only publicly releases aggregate data; de-identified individual data is made available to research partners. Our partners are bound by data use agreements to maintain the confidentiality of individual survey responses. Facebook refers its users to the survey, but it does not receive any individual survey data. - ## How are the surveys distributed? Participants are recruited for the surveys through an advertisement placed in their Facebook news feed. Facebook automatically selects a random sample of its users to see the advertisement; users who click on the ad are taken to a survey administered by Carnegie Mellon University, and Facebook does not see their survey responses. The survey is available in English, Spanish, Brazilian Portuguese, Vietnamese, French, and simplified Chinese. @@ -35,12 +32,10 @@ The survey participants are sampled from Facebook users, rather than being a ran To account for the differences between Facebook users and the United States population, Facebook includes a unique identifier when it links users to the survey. Carnegie Mellon collects these identifiers and provides Facebook with a list of identifiers that completed the survey; Facebook then calculates statistical weights indicating how representative each person is of the United States population, based on demographic data known to Facebook. Crucially, Carnegie Mellon cannot use these identifiers to identify specific Facebook users, and Facebook never receives individual survey responses and cannot link them to specific users. - ## Where can I see the results? Our [COVIDcast map]({{< relref "covidcast" >}}) shows basic aggregate survey results, revealing the rate of COVID-like symptoms across the United States, and this data is also freely available for download through the [COVIDcast API]({{< apiref "api/covidcast.html">}}). Facebook also publishes [a map](https://covid-survey.dataforgood.fb.com/) based on the United States and international aggregate data. - ### Blog posts and presentations - Ryan Tibshirani, September 21, 2020. [Can Symptom Surveys Improve COVID-19 Forecasts?]({{< relref "2020-09-21-forecast-demo" >}}) Delphi blog. diff --git a/content/covidcast/terms-of-use.md b/content/covidcast/terms-of-use.md index 129827539..d38f69f95 100644 --- a/content/covidcast/terms-of-use.md +++ b/content/covidcast/terms-of-use.md @@ -20,7 +20,7 @@ CMU may update and change these TOU from time to time without notice to you. You ### Permitted Use and License/Availability of the Site -Subject to other relevant provisions of the TOU, the Site and any information and data on it (“Site Content”) is provided to you under the [CC-BY license]( https://creativecommons.org/licenses/by/4.0/) CMU (and/or its content providers, as applicable) own and retain all intellectual property rights they have in and to the Site Content, including but not limited to the Project content and the underlying infrastructure. CMU shall have the right in its sole and absolute discretion to suspend or terminate the Site any or all access to it for any reason. As specified below, the Site (including Site Content) is provided on an “AS-IS, AS-AVAILABLE” basis. +Subject to other relevant provisions of the TOU, the Site and any information and data on it (“Site Content”) is provided to you under the [CC-BY license](https://creativecommons.org/licenses/by/4.0/) CMU (and/or its content providers, as applicable) own and retain all intellectual property rights they have in and to the Site Content, including but not limited to the Project content and the underlying infrastructure. CMU shall have the right in its sole and absolute discretion to suspend or terminate the Site any or all access to it for any reason. As specified below, the Site (including Site Content) is provided on an “AS-IS, AS-AVAILABLE” basis. ### Prohibited Uses/Activities @@ -72,4 +72,4 @@ Using the Site does not confer any CMU course credit and/or any employment or st ### Miscellaneous -If any provision of these TOU is held to be invalid or unenforceable, such provision shall be deemed superseded by a valid enforceable provision that most closely matches the intent of the original provision and the remaining provisions shall be enforced. CMU’s failure to act with respect to a breach by you or any other users does not waive CMU’s right to act with respect to subsequent or similar breaches. The failure of CMU to exercise or enforce any right or provision of these terms and conditions shall not constitute a waiver of such right or provision. The section headings and subheadings contained in these TOU are included for convenience only, and shall not limit or otherwise affect the terms of these TOU. These TOU and any disputes related to them shall be interpreted in accordance with the laws of the Commonwealth of Pennsylvania without regard to its conflicts of laws provisions. All claims and/or controversies of every kind and nature arising out of or relating to these TOU, including any questions concerning its existence, negotiation, validity, meaning, performance, non-performance, breach, continuance or termination shall be settled (1) at CMU’s election, by binding arbitration administered by the American Arbitration Association ("AAA") in accordance with its Commercial Arbitration Rules and, in such case (a) the arbitration proceedings shall be conducted before a panel of three arbitrators, with each party selecting one disinterested arbitrator from a list submitted by the AAA and the two disinterested arbitrators selecting a third arbitrator from the list, (b) each party shall bear its own costs of arbitration, (c) all arbitration hearings shall be conducted in Allegheny County, Pennsylvania, and (d) the provisions hereof shall be a complete defense to any suit, action or proceeding instituted in any Federal, state or local court or before any administrative tribunal with respect to any claim or controversy arising out of or relating to these TOU and which is arbitrable as provided in these TOU, provided that either party may seek injunctive relief in a court of law or equity to assert, protect or enforce its rights hereunder (2) in the event that CMU does not elect binding arbitration as permitted in point (1) above, exclusively in the United States District Court for the Western District of Pennsylvania or, if such Court does not have jurisdiction, in any court of general jurisdiction in Allegheny County, Pennsylvania and each party consents to the exclusive jurisdiction of any such courts and waives any objection which such party may have to the laying of venue in any such courts. Notwithstanding any provision hereof, for all purposes of these TOU each party shall be and act as an independent contractor and not as partner, joint venture, agent, employee or employer of the other and shall not bind nor attempt to bind the other to any contract. You are agreeing to these TOU on behalf of yourself. You agree that, except for the disclaimers and limitations of liability made for the benefit of the other Site Parties, there shall be no third-party beneficiaries to these TOU. \ No newline at end of file +If any provision of these TOU is held to be invalid or unenforceable, such provision shall be deemed superseded by a valid enforceable provision that most closely matches the intent of the original provision and the remaining provisions shall be enforced. CMU’s failure to act with respect to a breach by you or any other users does not waive CMU’s right to act with respect to subsequent or similar breaches. The failure of CMU to exercise or enforce any right or provision of these terms and conditions shall not constitute a waiver of such right or provision. The section headings and subheadings contained in these TOU are included for convenience only, and shall not limit or otherwise affect the terms of these TOU. These TOU and any disputes related to them shall be interpreted in accordance with the laws of the Commonwealth of Pennsylvania without regard to its conflicts of laws provisions. All claims and/or controversies of every kind and nature arising out of or relating to these TOU, including any questions concerning its existence, negotiation, validity, meaning, performance, non-performance, breach, continuance or termination shall be settled (1) at CMU’s election, by binding arbitration administered by the American Arbitration Association ("AAA") in accordance with its Commercial Arbitration Rules and, in such case (a) the arbitration proceedings shall be conducted before a panel of three arbitrators, with each party selecting one disinterested arbitrator from a list submitted by the AAA and the two disinterested arbitrators selecting a third arbitrator from the list, (b) each party shall bear its own costs of arbitration, (c) all arbitration hearings shall be conducted in Allegheny County, Pennsylvania, and (d) the provisions hereof shall be a complete defense to any suit, action or proceeding instituted in any Federal, state or local court or before any administrative tribunal with respect to any claim or controversy arising out of or relating to these TOU and which is arbitrable as provided in these TOU, provided that either party may seek injunctive relief in a court of law or equity to assert, protect or enforce its rights hereunder (2) in the event that CMU does not elect binding arbitration as permitted in point (1) above, exclusively in the United States District Court for the Western District of Pennsylvania or, if such Court does not have jurisdiction, in any court of general jurisdiction in Allegheny County, Pennsylvania and each party consents to the exclusive jurisdiction of any such courts and waives any objection which such party may have to the laying of venue in any such courts. Notwithstanding any provision hereof, for all purposes of these TOU each party shall be and act as an independent contractor and not as partner, joint venture, agent, employee or employer of the other and shall not bind nor attempt to bind the other to any contract. You are agreeing to these TOU on behalf of yourself. You agree that, except for the disclaimers and limitations of liability made for the benefit of the other Site Parties, there shall be no third-party beneficiaries to these TOU. diff --git a/content/flu.md b/content/flu.md index 247996a16..1b0789e9d 100644 --- a/content/flu.md +++ b/content/flu.md @@ -2,7 +2,6 @@ title: Flu and Other Diseases --- - ## Operational Systems {{}} @@ -16,15 +15,18 @@ We have participated, and [have done very well](http://www.cs.cmu.edu/~roni/CDC% , 2016--2017 (winner) 2017--2018 (winner)--> + ### [Forecasting Seasonal Influenza in the US](https://www.cdc.gov/flu/weekly/flusight/) + by [CDC](https://www.cdc.gov) 2013 -- current -### [Forecasting Dengue in Puerto Rico and Peru](https://predict.cdc.gov/post/5a4fcc3e2c1b1669c22aa261) +### [Forecasting Dengue in Puerto Rico and Peru](https://predict.cdc.gov/post/5a4fcc3e2c1b1669c22aa261) + by the [White House](https://www.whitehouse.gov/) [OSTP](https://www.whitehouse.gov/administration/eop/ostp) -### [Forecasting the Chikungunya invasion of the Americas](https://www.innocentive.com/ar/challenge/9933617") -by [DARPA](http://www.darpa.mil/) +### [Forecasting the Chikungunya invasion of the Americas](https://www.innocentive.com/ar/challenge/9933617") +by [DARPA](http://www.darpa.mil/) ## Publicly Available Tools @@ -36,4 +38,4 @@ All source code is freely available on [GitHub](https://github.com/cmu-delphi/). Delphi’s notable achievements -{{}} \ No newline at end of file +{{}} diff --git a/content/news/2016_12.md b/content/news/2016_12.md index 420a37c10..b2e7a33fe 100644 --- a/content/news/2016_12.md +++ b/content/news/2016_12.md @@ -4,4 +4,4 @@ title: Top category: accomplishment --- -[system took the top stop in the 2015-2016 flu forecasting challenge](https://www.cdc.gov/flu/spotlights/flu-activity-forecasts-2016-2017.htm). ([Results summary](http://www.cs.cmu.edu/~roni/CDC%20Flu%20Challenge%202015-2016%20Results.pdf).) \ No newline at end of file +[system took the top stop in the 2015-2016 flu forecasting challenge](https://www.cdc.gov/flu/spotlights/flu-activity-forecasts-2016-2017.htm). ([Results summary](http://www.cs.cmu.edu/~roni/CDC%20Flu%20Challenge%202015-2016%20Results.pdf).) diff --git a/content/news/2017_10.md b/content/news/2017_10.md index 47498d4b5..3cf0dc063 100644 --- a/content/news/2017_10.md +++ b/content/news/2017_10.md @@ -7,4 +7,4 @@ category: accomplishment We did it again! [Our two systems took the top two spots in the 2016-2017 flu forecasting challenge (out of 28 submissions).](https://www.cmu.edu/news/stories/archives/2017/september/flu-forecasts.html) -([Results summary](http://www.cs.cmu.edu/~roni/CDC%20Flu%20Challenge%202016-2017%20Results.pdf).) \ No newline at end of file +([Results summary](http://www.cs.cmu.edu/~roni/CDC%20Flu%20Challenge%202016-2017%20Results.pdf).) diff --git a/content/news/2018_11.md b/content/news/2018_11.md index 8a651fe2c..45b6b9051 100644 --- a/content/news/2018_11.md +++ b/content/news/2018_11.md @@ -7,4 +7,4 @@ category: accomplishment and yet again! Our forecasting systems took the top spot each of the three separate flu forecasting challenges of 2017-2018 (out of up to 30 submissions). -([Results summary](http://www.cs.cmu.edu/~roni/CDC%20Flu%20Challenge%202017-2018%20Results.pdf).) \ No newline at end of file +([Results summary](http://www.cs.cmu.edu/~roni/CDC%20Flu%20Challenge%202017-2018%20Results.pdf).) diff --git a/content/news/2019_10_CenterOfExcellence.md b/content/news/2019_10_CenterOfExcellence.md index 2460e2af3..906acd0a9 100644 --- a/content/news/2019_10_CenterOfExcellence.md +++ b/content/news/2019_10_CenterOfExcellence.md @@ -4,4 +4,4 @@ title: Center of Excellence category: accomplishment --- -[CDC has just named us "National Center of Excellence for Influenza Forecasting"](https://www.ml.cmu.edu/news/news-archive/2019/october/machine-learning-delphi-research-group-funded-by-centers-for-disease-control-cdc.html) (one of two nationally.) \ No newline at end of file +[CDC has just named us "National Center of Excellence for Influenza Forecasting"](https://www.ml.cmu.edu/news/news-archive/2019/october/machine-learning-delphi-research-group-funded-by-centers-for-disease-control-cdc.html) (one of two nationally.) diff --git a/content/news/2020_03_covid.md b/content/news/2020_03_covid.md index 2c11c18c3..272121389 100644 --- a/content/news/2020_03_covid.md +++ b/content/news/2020_03_covid.md @@ -7,4 +7,4 @@ category: update We are focusing our efforts at this point on COVID-19 nowcasting and forecasting. We are adapting our existing systems, and developing new ones. Some of our regular activities may be halted as a -result. \ No newline at end of file +result. diff --git a/content/news/2020_04_covidcast.md b/content/news/2020_04_covidcast.md index 6e111afed..5a48f4195 100644 --- a/content/news/2020_04_covidcast.md +++ b/content/news/2020_04_covidcast.md @@ -4,9 +4,8 @@ title: COVIDCast category: update --- - We launched our [COVIDcast system]({{< relref "covidcast">}}), which displays -indicators related to COVID-19 activity level across the U.S. These +indicators related to COVID-19 activity level across the U.S. These indicators are derived from a variety of anonymized, aggregated data sources made available by multiple partners, and are publicly available at the [COVIDcast endpoint]({{< apiref "api/covidcast.html">}}) of our [Epidata API]({{< apiref "api/README.html">}}). @@ -16,4 +15,3 @@ Related news articles: - [Carnegie Mellon Unveils Five Interactive COVID-19 Maps](https://www.cmu.edu/news/stories/archives/2020/april/cmu-unveils-covidcast-maps.html) - [Self-reported COVID-19 Symptoms Show Promise for Disease Forecasts](https://www.cmu.edu/news/stories/archives/2020/april/self-reported-covid-19-symptoms-disease-forecasts.html) - [Facebook and Carnegie Mellon Team Up to Gather COVID-19 Symptom Data](https://www.cmu.edu/news/stories/archives/2020/april/facebook-survey-covid.html) - diff --git a/content/news/2020_10_fellows.md b/content/news/2020_10_fellows.md index 6cc7cb485..67b990600 100644 --- a/content/news/2020_10_fellows.md +++ b/content/news/2020_10_fellows.md @@ -4,4 +4,4 @@ title: Welcome Fellows category: update --- -We've welcomed [13 Google Fellows](https://www.cmu.edu/news/stories/archives/2020/september/covidcast-google.html) to Delphi and are excited to have them on board. \ No newline at end of file +We've welcomed [13 Google Fellows](https://www.cmu.edu/news/stories/archives/2020/september/covidcast-google.html) to Delphi and are excited to have them on board. diff --git a/content/news/index.md b/content/news/index.md index 46ca3e791..441ef3a5a 100644 --- a/content/news/index.md +++ b/content/news/index.md @@ -1,4 +1,4 @@ --- # flag to disable rendering individual pages headless: true ---- \ No newline at end of file +--- diff --git a/content/systems/crowdcast.md b/content/systems/crowdcast.md index 43f24cb4c..eac1f1a55 100644 --- a/content/systems/crowdcast.md +++ b/content/systems/crowdcast.md @@ -6,4 +6,4 @@ order: 20 Delphi's “Wisdom of crowds” forecasting system: Used for Chikungunya, flu and most recently Covid. -**Note:** This system is has been repurposed to forecast ILI during the COVID-19 pandemic. \ No newline at end of file +**Note:** This system is has been repurposed to forecast ILI during the COVID-19 pandemic. diff --git a/content/systems/forecast.md b/content/systems/forecast.md index 375b65fbd..09cbad3b8 100644 --- a/content/systems/forecast.md +++ b/content/systems/forecast.md @@ -7,4 +7,4 @@ State-level weekly forecasts of ILI (influenza-like illness) **Note:** This system is designed to forecast ILI driven by seasonal influenza and is NOT designed to forecast ILI during the COVID-19 pandemic. We -have temporarily shut it down to focus on COVID-19. \ No newline at end of file +have temporarily shut it down to focus on COVID-19. diff --git a/content/systems/index.md b/content/systems/index.md index 46ca3e791..441ef3a5a 100644 --- a/content/systems/index.md +++ b/content/systems/index.md @@ -1,4 +1,4 @@ --- # flag to disable rendering individual pages headless: true ---- \ No newline at end of file +--- diff --git a/content/systems/nowcast.md b/content/systems/nowcast.md index 6f226af09..ec5ed6e15 100644 --- a/content/systems/nowcast.md +++ b/content/systems/nowcast.md @@ -7,4 +7,4 @@ order: 1 Flu nowcasting system. **Note:** This system is designed to nowcast ILI driven by seasonal -influenza and is NOT designed to nowcast ILI during the COVID-19 pandemic. \ No newline at end of file +influenza and is NOT designed to nowcast ILI during the COVID-19 pandemic. diff --git a/content/tools/epidata.md b/content/tools/epidata.md index 4e20c6c6e..990016e9a 100644 --- a/content/tools/epidata.md +++ b/content/tools/epidata.md @@ -4,4 +4,4 @@ link: https://github.com/cmu-delphi/delphi-epidata order: 2 --- -API for getting up-to-date epidemiological data (also available via a web interface through [EpiVis](https://delphi.cmu.edu/epivis/epivis.html)) \ No newline at end of file +API for getting up-to-date epidemiological data (also available via a web interface through [EpiVis](https://delphi.cmu.edu/epivis/epivis.html)) diff --git a/content/tools/epiforecast.md b/content/tools/epiforecast.md index 33b9b5fa3..b683ff657 100644 --- a/content/tools/epiforecast.md +++ b/content/tools/epiforecast.md @@ -4,4 +4,4 @@ link: https://github.com/cmu-delphi/epiforecast-R order: 10 --- -Epidemiological forecasting R package \ No newline at end of file +Epidemiological forecasting R package diff --git a/content/tools/epivis.md b/content/tools/epivis.md index 25f4f0204..c62bc3e62 100644 --- a/content/tools/epivis.md +++ b/content/tools/epivis.md @@ -4,4 +4,4 @@ link: https://delphi.cmu.edu/epivis/epivis.html order: 1 --- -Epidemiological time series visualizer \ No newline at end of file +Epidemiological time series visualizer diff --git a/content/tools/fluscores.md b/content/tools/fluscores.md index d06d52195..88cc773fe 100644 --- a/content/tools/fluscores.md +++ b/content/tools/fluscores.md @@ -4,4 +4,4 @@ link: https://delphi.cmu.edu/misc/fluscores/ order: 50 --- -Visual comparison of scored submissions to CDC Flu Forecasting Challenge (provide your own score files) \ No newline at end of file +Visual comparison of scored submissions to CDC Flu Forecasting Challenge (provide your own score files) diff --git a/content/tools/index.md b/content/tools/index.md index 46ca3e791..441ef3a5a 100644 --- a/content/tools/index.md +++ b/content/tools/index.md @@ -1,4 +1,4 @@ --- # flag to disable rendering individual pages headless: true ---- \ No newline at end of file +--- diff --git a/content/tools/utils.md b/content/tools/utils.md index bdf372829..f7942e016 100644 --- a/content/tools/utils.md +++ b/content/tools/utils.md @@ -4,4 +4,4 @@ link: https://github.com/cmu-delphi/utils order: 60 --- -Epidemiological modeling utilities (e.g., date/epi-weeks conversions) \ No newline at end of file +Epidemiological modeling utilities (e.g., date/epi-weeks conversions) diff --git a/data/authors.yaml b/data/authors.yaml index dce0867aa..efbbdac39 100644 --- a/data/authors.yaml +++ b/data/authors.yaml @@ -20,4 +20,4 @@ - key: kathryn name: Kathryn Mazaitis link: https://cs.cmu.edu/~krivard - description: manages Delphi's engineering team, and is a Senior Research Programmer in the Machine Learning Department at CMU. \ No newline at end of file + description: manages Delphi's engineering team, and is a Senior Research Programmer in the Machine Learning Department at CMU. diff --git a/data/bibliography.yaml b/data/bibliography.yaml index f3f84f7c9..95b142a9a 100644 --- a/data/bibliography.yaml +++ b/data/bibliography.yaml @@ -330,7 +330,7 @@ link: http://books.google.com/books?id=HG5NAQAAIAAJ title: "Storms, Chapter 6: Storms Forecasting for Emergency Response." authors: Wernley, Donald, and Louis W. Uccellini. - issue: "Chapter 6 in \"Storms\"; 1999, pp. 70-97" + issue: 'Chapter 6 in "Storms"; 1999, pp. 70-97' journal: "Publisher: Routledge, 2000, ISBN: 0415212863, 9780415212861." - key: "2000 [Myers, M]" diff --git a/data/research.yaml b/data/research.yaml index 3dbd408e7..3376af666 100644 --- a/data/research.yaml +++ b/data/research.yaml @@ -1,7 +1,7 @@ - title: "Pancasting: forecasting epidemics from provisional data" image: research-img_Pancasting_ forecasting epidemics from provisional data..png authors: Brooks - link: https://delphi.cmu.edu/~lcbrooks/brooks2020pancasting.pdf + link: https://delphi.cmu.edu/~lcbrooks/brooks2020pancasting.pdf journal: PhD thesis year: 2020 - title: "Kalman filter, sensor fusion, and constrained regression: equivalences and insights" @@ -45,4 +45,4 @@ authors: Rosenfeld, Grefenstette, Burke link: http://www.cs.cmu.edu/~roni/standardized-evaluation-of-epi-models-rev-09nov2012.pdf journal: White paper - year: 2012 \ No newline at end of file + year: 2012 diff --git a/docker-compose.yml b/docker-compose.yml index ec82db67a..ad2448947 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,17 +1,17 @@ -version: '3.1' +version: "3.1" # docker services for R environment and Hugo runner services: r: - build: '.' - working_dir: '/app' + build: "." + working_dir: "/app" command: tail -F anything volumes: - - './:/app' + - "./:/app" # run: docker-compose exec r bash # conda activate www-main # Rscript -e 'blogdown::build_site(run_hugo=FALSE, build_rmd=TRUE)' - + # see https://github.com/peaceiris/hugo-extended-docker # hugo: # image: peaceiris/hugo:v0.78.1-full @@ -20,4 +20,4 @@ services: # command: # - server # - --bind=0.0.0.0 - # - --buildDrafts \ No newline at end of file + # - --buildDrafts diff --git a/environment.yml b/environment.yml index b01d064da..1d9be7a7b 100644 --- a/environment.yml +++ b/environment.yml @@ -4,7 +4,7 @@ channels: - defaults dependencies: - gdal=3.0.2 - - python=3.7.9 + - python=3.7.9 - pip=20.2.4 - r-base=3.6.3 - glib @@ -40,4 +40,4 @@ dependencies: - pyproj - matplotlib - pip: - - covidcast==0.1.0 \ No newline at end of file + - covidcast==0.1.0 diff --git a/no-deploy.json b/no-deploy.json index d79df4c40..1933a58b9 100644 --- a/no-deploy.json +++ b/no-deploy.json @@ -2,7 +2,6 @@ "type": "delphi deploy config", "version": 1, "actions": [ - "// site-wide favicon", { "type": "move", @@ -11,7 +10,7 @@ }, "// web sources", - { + { "type": "move", "src": "site/", "dst": "/var/www/html/", @@ -27,7 +26,6 @@ "match": "^.+\\.(jpg|png)$" }, - "// team images", { "type": "move", @@ -44,6 +42,5 @@ "match": "^.*\\.html$", "add-header-comment": true } - ] } diff --git a/themes/delphi/README.md b/themes/delphi/README.md index 4d2ce1627..718acefcf 100644 --- a/themes/delphi/README.md +++ b/themes/delphi/README.md @@ -4,16 +4,15 @@ Hugo theme based on `Noteworthy` for Delphi ## Features -* Fully responsive -* Google Analytics and Disqus integration -* Ko-fi donation button -* Syntax highlighting -* Mathematical notations with KaTex -* About, Tags, and Archives pages -* RSS feeds -* Social media links -* SCSS for styling - +- Fully responsive +- Google Analytics and Disqus integration +- Ko-fi donation button +- Syntax highlighting +- Mathematical notations with KaTex +- About, Tags, and Archives pages +- RSS feeds +- Social media links +- SCSS for styling ## Installation @@ -97,7 +96,6 @@ impactstory = "#" orcid = "#" ``` - ## Disqus and Google Analytics Add your Disqus shortname and Google Analytics identifier in the `config.toml` file. @@ -110,7 +108,6 @@ disqusShortname = "" googleAnalytics = "" ``` - ## License -Released under the [MIT License](https://github.com/kimcc/hugo-theme-noteworthy/blob/master/LICENSE.md). \ No newline at end of file +Released under the [MIT License](https://github.com/kimcc/hugo-theme-noteworthy/blob/master/LICENSE.md). diff --git a/themes/delphi/archetypes/default.md b/themes/delphi/archetypes/default.md index 03855e356..9546e4dda 100644 --- a/themes/delphi/archetypes/default.md +++ b/themes/delphi/archetypes/default.md @@ -1,4 +1,4 @@ +++ title = "" date = "" -+++ \ No newline at end of file ++++ diff --git a/themes/delphi/assets/css/_customize.scss b/themes/delphi/assets/css/_customize.scss index 24f34bf39..66dabb680 100644 --- a/themes/delphi/assets/css/_customize.scss +++ b/themes/delphi/assets/css/_customize.scss @@ -1,4 +1,3 @@ - // Font $global-font-family: "Open Sans", Roboto, Arial, sans-serif; @@ -18,8 +17,6 @@ $nav-default-item-active-color: red; $button-padding-horizontal: 0px; $button-line-height: 0px; - $breadcrumb-divider-margin-horizontal: 3px; - -$grid-gap: (32/1440)*100%; \ No newline at end of file +$grid-gap: (32/1440) * 100%; diff --git a/themes/delphi/assets/css/blog_extra.scss b/themes/delphi/assets/css/blog_extra.scss index 5fc49e4cd..f80c4e698 100644 --- a/themes/delphi/assets/css/blog_extra.scss +++ b/themes/delphi/assets/css/blog_extra.scss @@ -1,3 +1,3 @@ // separate file to avoid including big libraries into everything -@import './node_modules/highlight.js/scss/github'; -@import './node_modules/katex/dist/katex.min'; \ No newline at end of file +@import "./node_modules/highlight.js/scss/github"; +@import "./node_modules/katex/dist/katex.min"; diff --git a/themes/delphi/assets/css/components/_arrow_link.scss b/themes/delphi/assets/css/components/_arrow_link.scss index be30475fa..4dcc6e3bf 100644 --- a/themes/delphi/assets/css/components/_arrow_link.scss +++ b/themes/delphi/assets/css/components/_arrow_link.scss @@ -1,21 +1,21 @@ .arrow-link { + display: flex; + align-items: center; + + > .inline-svg-icon { + border-radius: 50%; + padding: 5px; + border: 1px solid currentColor; + width: 40px; + height: 40px; display: flex; + box-sizing: border-box; align-items: center; - - > .inline-svg-icon { - border-radius: 50%; - padding: 5px; - border: 1px solid currentColor; - width: 40px; - height: 40px; - display: flex; - box-sizing: border-box; - align-items: center; - justify-content: center; - margin-right: 1em; - - > svg { - width: 14px; - } + justify-content: center; + margin-right: 1em; + + > svg { + width: 14px; } + } } diff --git a/themes/delphi/assets/css/components/_card_grid.scss b/themes/delphi/assets/css/components/_card_grid.scss index 6ec90eb7d..83f243dac 100644 --- a/themes/delphi/assets/css/components/_card_grid.scss +++ b/themes/delphi/assets/css/components/_card_grid.scss @@ -1,49 +1,48 @@ - .card-grid-item { - position: relative; - border-radius: 8px; - margin: 3em 0 0 1em; - - .uk-card-footer { - border-top: none; - } + position: relative; + border-radius: 8px; + margin: 3em 0 0 1em; + + .uk-card-footer { + border-top: none; + } } .card-grid-item-img { - object-fit: cover; - width: 100%; - height: 8em; - display: block; - border-radius: 8px 8px 0 0; + object-fit: cover; + width: 100%; + height: 8em; + display: block; + border-radius: 8px 8px 0 0; } .card-grid-item-date { - position: absolute; - font-weight: 600; - top: -2em; - left: 0; + position: absolute; + font-weight: 600; + top: -2em; + left: 0; } @media (min-width: $breakpoint-small) { - .card-grid-item { - display: flex; - margin: 1em 1em 0 8em; - } + .card-grid-item { + display: flex; + margin: 1em 1em 0 8em; + } - .card-grid-item-img { - width: 12.5em; - height: unset; - border-radius: 8px 0 0 8px; - } + .card-grid-item-img { + width: 12.5em; + height: unset; + border-radius: 8px 0 0 8px; + } - .card-grid-item-date { - top: 0; - left: -8em; - } + .card-grid-item-date { + top: 0; + left: -8em; + } } @media (min-width: $breakpoint-large) { - .card-grid-item { - margin-right: 8em; - } + .card-grid-item { + margin-right: 8em; + } } diff --git a/themes/delphi/assets/css/components/_font_awesome.scss b/themes/delphi/assets/css/components/_font_awesome.scss index 25697a8fa..3266ad039 100644 --- a/themes/delphi/assets/css/components/_font_awesome.scss +++ b/themes/delphi/assets/css/components/_font_awesome.scss @@ -1,10 +1,10 @@ .inline-svg-icon { - display: inline-block; - fill: currentColor; - width: 1.2em; - padding: 0 0.5em 0 0.25em; + display: inline-block; + fill: currentColor; + width: 1.2em; + padding: 0 0.5em 0 0.25em; } .uk-icon-button > .inline-svg-icon { - padding: 0; + padding: 0; } diff --git a/themes/delphi/assets/css/components/_latest_card.scss b/themes/delphi/assets/css/components/_latest_card.scss index d385c833e..d753ac874 100644 --- a/themes/delphi/assets/css/components/_latest_card.scss +++ b/themes/delphi/assets/css/components/_latest_card.scss @@ -1,30 +1,29 @@ - .latest-card { - background: #FAFAFC; - border: 1px solid #D3D4D8; - border-radius: 8px; - - > .uk-card-media-top img { - border-radius: 8px 8px 0 0; - width: 100%; - height: 15em; - object-fit: cover; - } - - > .uk-card-body { - flex-grow: 1; - } - - > .uk-card-footer { - border-top: none; - - a { - color: inherit; - } + background: #fafafc; + border: 1px solid #d3d4d8; + border-radius: 8px; + + > .uk-card-media-top img { + border-radius: 8px 8px 0 0; + width: 100%; + height: 15em; + object-fit: cover; + } + + > .uk-card-body { + flex-grow: 1; + } + + > .uk-card-footer { + border-top: none; + + a { + color: inherit; } + } } .latest-card-category { - font-weight: 600; - text-transform: uppercase; -} \ No newline at end of file + font-weight: 600; + text-transform: uppercase; +} diff --git a/themes/delphi/assets/css/layout/_content.scss b/themes/delphi/assets/css/layout/_content.scss index 8a19ff1f5..cf4e23c72 100644 --- a/themes/delphi/assets/css/layout/_content.scss +++ b/themes/delphi/assets/css/layout/_content.scss @@ -1,14 +1,13 @@ - // Site-wide .content { - margin: 20px 10%; - max-width: 1200px; + margin: 20px 10%; + max-width: 1200px; } .uk-text-bold-600 { - font-weight: 600; + font-weight: 600; } .content [aria-hidden="true"] { - display: none; -} \ No newline at end of file + display: none; +} diff --git a/themes/delphi/assets/css/layout/_header_footer.scss b/themes/delphi/assets/css/layout/_header_footer.scss index 59c856936..57f839f63 100644 --- a/themes/delphi/assets/css/layout/_header_footer.scss +++ b/themes/delphi/assets/css/layout/_header_footer.scss @@ -1,73 +1,72 @@ - .delphi-text-logo { - background-color: white; - color: #F03F3F !important; - font-size: 16px !important; - font-weight: 700; - line-height: 18px; - padding: 5px 10px; + background-color: white; + color: #f03f3f !important; + font-size: 16px !important; + font-weight: 700; + line-height: 18px; + padding: 5px 10px; } .uk-breadcrumb svg, .uk-navbar svg { - max-height: 18px; + max-height: 18px; } // Navbar .nav-active { - border-bottom: 2px solid #F03F3F !important; + border-bottom: 2px solid #f03f3f !important; } .uk-navbar-container { - box-shadow: 0 3px 5px -1px rgba(0,0,0,.15); + box-shadow: 0 3px 5px -1px rgba(0, 0, 0, 0.15); } .uk-navbar-left { - margin: 40px 10% 0 10%; + margin: 40px 10% 0 10%; } -.uk-navbar-nav>li { - padding-left: 15px; - padding-right: 15px; - white-space: nowrap; +.uk-navbar-nav > li { + padding-left: 15px; + padding-right: 15px; + white-space: nowrap; } .uk-navbar-dropdown-bottom-left { - margin-left: -20px; - margin-top: 3px; + margin-left: -20px; + margin-top: 3px; } .mobile-nav { - height: 20px; - padding: 23px; - border-bottom: 1px solid #d2d2d2; + height: 20px; + padding: 23px; + border-bottom: 1px solid #d2d2d2; } .uk-navbar-toggle { - float:right; - margin-top: -10px; + float: right; + margin-top: -10px; } .dropdown-mobile { - left: 0 !important; - padding: 0; - width: 100%; + left: 0 !important; + padding: 0; + width: 100%; } .menu-element { - border-top: 1px solid #d2d2d2; - font-weight: 400; - padding: 20px 0 20px 40px; + border-top: 1px solid #d2d2d2; + font-weight: 400; + padding: 20px 0 20px 40px; } .menu-element.menu-parent { - font-weight: 600; + font-weight: 600; } .menu-element.menu-child { - padding-left: 64px; + padding-left: 64px; } // Footer .footer { - background-color: #F2F2F2; - color: #232735; - padding: 50px 20% 20px 20%; + background-color: #f2f2f2; + color: #232735; + padding: 50px 20% 20px 20%; } .footer .delphi-text-logo { - border: 1px solid #D3D4D8; - padding: 8px 12px; + border: 1px solid #d3d4d8; + padding: 8px 12px; } diff --git a/themes/delphi/assets/css/layout/_layouts.scss b/themes/delphi/assets/css/layout/_layouts.scss index 80652d074..f347e7139 100644 --- a/themes/delphi/assets/css/layout/_layouts.scss +++ b/themes/delphi/assets/css/layout/_layouts.scss @@ -1,41 +1,41 @@ -$grid-gap: (32/1440)*100%; +$grid-gap: (32/1440) * 100%; .content-grid { - display: grid; - grid-template-columns: repeat(12, 1fr); - grid-gap: $grid-gap; + display: grid; + grid-template-columns: repeat(12, 1fr); + grid-gap: $grid-gap; } .grid-2-8 { - grid-column: 2 / 8; + grid-column: 2 / 8; } .grid-2-10 { - grid-column: 2 / 10; + grid-column: 2 / 10; } .grid-8-12 { - grid-column: 8 / 12; + grid-column: 8 / 12; } .grid-3-11 { - grid-column: 3 / 11; + grid-column: 3 / 11; } .grid-2-12 { - grid-column: 2 / 12; + grid-column: 2 / 12; } .grid-1-2 { - grid-column: 1 / 2; + grid-column: 1 / 2; } @media only screen and (max-width: $breakpoint-small) { - .content-grid { - display: block; - } + .content-grid { + display: block; + } } .content-row { - display: flex; - flex-direction: row; + display: flex; + flex-direction: row; - .content-cell{ - flex: 1; - } -} \ No newline at end of file + .content-cell { + flex: 1; + } +} diff --git a/themes/delphi/assets/css/main.scss b/themes/delphi/assets/css/main.scss index 7b2ec68c2..93782e8f6 100644 --- a/themes/delphi/assets/css/main.scss +++ b/themes/delphi/assets/css/main.scss @@ -1,31 +1,31 @@ // 1. Your custom variables and variable overwrites. // $global-link-color: #DA7D02; -@import 'customize'; +@import "customize"; // 2. Import default variables and available mixins. -@import './node_modules/uikit/src/scss/variables-theme.scss'; -@import './node_modules/uikit/src/scss/mixins-theme.scss'; +@import "./node_modules/uikit/src/scss/variables-theme.scss"; +@import "./node_modules/uikit/src/scss/mixins-theme.scss"; // 3. Your custom mixin overwrites. // @mixin hook-card() { color: #000; } // 4. Import UIkit. -@import './node_modules/uikit/src/scss/uikit-theme.scss'; +@import "./node_modules/uikit/src/scss/uikit-theme.scss"; // Layouts -@import './layout/content'; -@import './layout/layouts'; -@import './layout/header_footer'; +@import "./layout/content"; +@import "./layout/layouts"; +@import "./layout/header_footer"; // Components -@import './components/arrow_link'; -@import './components/font_awesome'; -@import './components/card_grid'; -@import './components/latest_card'; +@import "./components/arrow_link"; +@import "./components/font_awesome"; +@import "./components/card_grid"; +@import "./components/latest_card"; // Page Designs -@import './pages/about'; -@import './pages/covidcast'; -@import './pages/landing'; -@import './pages/team'; -@import './pages/blog'; \ No newline at end of file +@import "./pages/about"; +@import "./pages/covidcast"; +@import "./pages/landing"; +@import "./pages/team"; +@import "./pages/blog"; diff --git a/themes/delphi/assets/css/pages/_about.scss b/themes/delphi/assets/css/pages/_about.scss index 946420b87..7c8d7bfbe 100644 --- a/themes/delphi/assets/css/pages/_about.scss +++ b/themes/delphi/assets/css/pages/_about.scss @@ -1,63 +1,61 @@ - .about-mission { - padding: 8rem 0; + padding: 8rem 0; } .about-mission-img { - z-index: -1; + z-index: -1; } .about-mission-mission { - color: white; - padding: 1em 0; - display: block; - background: rgba(#FA8C16, 0.2); - text-transform: uppercase; - letter-spacing: 3px; + color: white; + padding: 1em 0; + display: block; + background: rgba(#fa8c16, 0.2); + text-transform: uppercase; + letter-spacing: 3px; } .about-mission-text { - color: white; - text-shadow: 0px 2px 4px rgba(0, 0, 0, 0.25); - font-weight: 600; - font-size: 44px; - line-height: 64px; - margin: 0; + color: white; + text-shadow: 0px 2px 4px rgba(0, 0, 0, 0.25); + font-weight: 600; + font-size: 44px; + line-height: 64px; + margin: 0; } .about-description { - padding: 24px; - margin-bottom: 3rem; - - h3 { - letter-spacing: 3px; - text-transform: uppercase; - } + padding: 24px; + margin-bottom: 3rem; + + h3 { + letter-spacing: 3px; + text-transform: uppercase; + } } @media (min-width: $breakpoint-medium) { - .about-description { - background: white; - border: 1px solid #D3D4D8; - border-radius: 5px; - margin-top: -6rem; - } + .about-description { + background: white; + border: 1px solid #d3d4d8; + border-radius: 5px; + margin-top: -6rem; + } } - .about-collaborators { - background: #FAFAFC; - padding: 0; - padding: 2em 0; + background: #fafafc; + padding: 0; + padding: 2em 0; } .about-collaborators-list { - display: flex; - flex-wrap: wrap; - align-content: center; - justify-content: center; - - > div { - margin: 1em 0; - text-align: center; - } + display: flex; + flex-wrap: wrap; + align-content: center; + justify-content: center; + + > div { + margin: 1em 0; + text-align: center; + } } diff --git a/themes/delphi/assets/css/pages/_blog.scss b/themes/delphi/assets/css/pages/_blog.scss index 417c4e908..034301aa9 100644 --- a/themes/delphi/assets/css/pages/_blog.scss +++ b/themes/delphi/assets/css/pages/_blog.scss @@ -1,71 +1,69 @@ - .code-wrapper > summary { - cursor: pointer; + cursor: pointer; } .blog-tags { - overflow: hidden; - text-overflow: ellipsis; + overflow: hidden; + text-overflow: ellipsis; } .blog-tag { - margin-right: 0.5em; - white-space: nowrap; - font-size: 14px; // TODO + margin-right: 0.5em; + white-space: nowrap; + font-size: 14px; // TODO } .blog-hero-image { - width: 100%; - height: 20em; - object-fit: fill; + width: 100%; + height: 20em; + object-fit: fill; } - $blog-list-date-width: 5em; .blog-list-content time, .blog-card time { - display: inline-block; - min-width: $blog-list-date-width; + display: inline-block; + min-width: $blog-list-date-width; } -@media(min-width: $breakpoint-medium) { - .blog-list-content { - margin-left: $blog-list-date-width; - time { - position: absolute; - left: 0; - } +@media (min-width: $breakpoint-medium) { + .blog-list-content { + margin-left: $blog-list-date-width; + time { + position: absolute; + left: 0; } + } } // sm -@media(max-width: $breakpoint-medium) { - .blog-list-hero-image { - order: -1; - img { - width: 100%; - max-height: 20em; - object-fit: contain; +@media (max-width: $breakpoint-medium) { + .blog-list-hero-image { + order: -1; + img { + width: 100%; + max-height: 20em; + object-fit: contain; } -} + } } .blog-card { - .uk-card-media-top img { - width: 100%; - object-fit: cover; - height: 14em; - } + .uk-card-media-top img { + width: 100%; + object-fit: cover; + height: 14em; + } - > .uk-card-footer { - border-top: none; - display: flex; - } + > .uk-card-footer { + border-top: none; + display: flex; + } } .blog-blog { - // center align auto generated images - p > img:first-of-type { - display: block; - margin: 0 auto; - } -} \ No newline at end of file + // center align auto generated images + p > img:first-of-type { + display: block; + margin: 0 auto; + } +} diff --git a/themes/delphi/assets/css/pages/_covidcast.scss b/themes/delphi/assets/css/pages/_covidcast.scss index 460adb38a..d3120e737 100644 --- a/themes/delphi/assets/css/pages/_covidcast.scss +++ b/themes/delphi/assets/css/pages/_covidcast.scss @@ -1,4 +1,3 @@ - #vizbox { - margin-top: 5px !important; + margin-top: 5px !important; } diff --git a/themes/delphi/assets/css/pages/_landing.scss b/themes/delphi/assets/css/pages/_landing.scss index 142575f27..3f1313a25 100644 --- a/themes/delphi/assets/css/pages/_landing.scss +++ b/themes/delphi/assets/css/pages/_landing.scss @@ -1,4 +1,3 @@ - //The UIKit slideshow component doesn't want to stay inside its parent container //We have to do multiple things to keep it in check: // @@ -6,91 +5,91 @@ // * Explicitly set heights for these three elements // //If this behavior ever becomes more sane, some of these work-arounds can be removed. -$carousel-height:500px; +$carousel-height: 500px; -#main-carousel{ - max-height: $carousel-height; +#main-carousel { + max-height: $carousel-height; } -#uikit-main-carousel{ - height: $carousel-height; +#uikit-main-carousel { + height: $carousel-height; } -#slideshow-container{ - max-height:$carousel-height; - height:$carousel-height; - img { - z-index: -1; - } +#slideshow-container { + max-height: $carousel-height; + height: $carousel-height; + img { + z-index: -1; + } } .carousel-entry { - z-index: 10; - margin-top: 6rem; - h2 { - color: white; - font-weight: 400; - font-size: 14px; - text-transform: uppercase; - } - p { - color: white; - font-weight: 600; - font-size: 44px; - } - a { - color: white; - } + z-index: 10; + margin-top: 6rem; + h2 { + color: white; + font-weight: 400; + font-size: 14px; + text-transform: uppercase; + } + p { + color: white; + font-weight: 600; + font-size: 44px; + } + a { + color: white; + } } @media screen and (max-width: $breakpoint-medium) { - .carousel-entry { - p { - font-size: 32px; - } + .carousel-entry { + p { + font-size: 32px; } + } } -#slider-controls{ - z-index: 10; - top: -5rem; - position: relative; - margin-top: 0; +#slider-controls { + z-index: 10; + top: -5rem; + position: relative; + margin-top: 0; } .white-area { - padding-top:2rem; - padding-bottom:2rem; + padding-top: 2rem; + padding-bottom: 2rem; } .gray-area { - padding-top:3rem; - padding-bottom:3rem; - background-color: #eee; + padding-top: 3rem; + padding-bottom: 3rem; + background-color: #eee; } .landing-entry { - line-height: 48px; - .inline-svg-icon{ - font-size:x-large; - } - h2 { - font-weight: 600; - font-size: 14px; - text-transform: uppercase; - } - p { - font-weight: 600; - font-size: 32px; - } - a { - color: #666; - font-weight: bold; - } + line-height: 48px; + .inline-svg-icon { + font-size: x-large; + } + h2 { + font-weight: 600; + font-size: 14px; + text-transform: uppercase; + } + p { + font-weight: 600; + font-size: 32px; + } + a { + color: #666; + font-weight: bold; + } } -.entry-image{ - width: 100%; - img{ - max-width: 100%; - max-height: 100% - } +.entry-image { + width: 100%; + img { + max-width: 100%; + max-height: 100%; + } } diff --git a/themes/delphi/assets/css/pages/_team.scss b/themes/delphi/assets/css/pages/_team.scss index a6c384d24..c70bc94dd 100644 --- a/themes/delphi/assets/css/pages/_team.scss +++ b/themes/delphi/assets/css/pages/_team.scss @@ -1,8 +1,6 @@ - - .team-hero-image { - width: 100%; + width: 100%; } .team-member > img { - border-radius: 5px; + border-radius: 5px; } diff --git a/themes/delphi/assets/js/blog/codeFolding.js b/themes/delphi/assets/js/blog/codeFolding.js index 4666f7b38..ccb2f5851 100644 --- a/themes/delphi/assets/js/blog/codeFolding.js +++ b/themes/delphi/assets/js/blog/codeFolding.js @@ -1,12 +1,14 @@ export function initializeCodeFolding(showAll) { - Array.from(document.querySelectorAll('pre.r, pre.python, pre.bash, pre.sql, pre.cpp, pre.stan, pre.julia, pre.foldable')).forEach((elem) => { - const wrapper = document.createElement('details'); - wrapper.classList.add('code-wrapper'); - wrapper.open = showAll || false; - elem.parentElement.replaceChild(wrapper, elem); - const summary = document.createElement('summary'); - summary.innerText = 'Code'; - wrapper.appendChild(summary); - wrapper.appendChild(elem); - }); -} \ No newline at end of file + Array.from( + document.querySelectorAll("pre.r, pre.python, pre.bash, pre.sql, pre.cpp, pre.stan, pre.julia, pre.foldable") + ).forEach((elem) => { + const wrapper = document.createElement("details"); + wrapper.classList.add("code-wrapper"); + wrapper.open = showAll || false; + elem.parentElement.replaceChild(wrapper, elem); + const summary = document.createElement("summary"); + summary.innerText = "Code"; + wrapper.appendChild(summary); + wrapper.appendChild(elem); + }); +} diff --git a/themes/delphi/assets/js/blog/index.js b/themes/delphi/assets/js/blog/index.js index 38622a893..7816d43dd 100644 --- a/themes/delphi/assets/js/blog/index.js +++ b/themes/delphi/assets/js/blog/index.js @@ -1,7 +1,7 @@ -import hljs from 'highlight.js'; -import renderMathInElement from 'katex/dist/contrib/auto-render.mjs'; -import { initializeCodeFolding } from './codeFolding'; +import hljs from "highlight.js"; +import renderMathInElement from "katex/dist/contrib/auto-render.mjs"; +import { initializeCodeFolding } from "./codeFolding"; hljs.initHighlightingOnLoad(); initializeCodeFolding(); -Array.from(document.querySelectorAll('.math')).forEach((elem) => renderMathInElement(elem)); \ No newline at end of file +Array.from(document.querySelectorAll(".math")).forEach((elem) => renderMathInElement(elem)); diff --git a/themes/delphi/assets/js/main.js b/themes/delphi/assets/js/main.js index 91d33184c..a1a98c6b0 100644 --- a/themes/delphi/assets/js/main.js +++ b/themes/delphi/assets/js/main.js @@ -1,6 +1,6 @@ -import UIkit from 'uikit/dist/js/uikit.js'; +import UIkit from "uikit/dist/js/uikit.js"; // import plugin from 'uikit/dist/js/uikit-icons.js'; // UIkit.use(plugin); // re export for COVIDCast -window.UIkit = UIkit; \ No newline at end of file +window.UIkit = UIkit; diff --git a/themes/delphi/layouts/404.html b/themes/delphi/layouts/404.html index bb1be451c..40bbd212a 100644 --- a/themes/delphi/layouts/404.html +++ b/themes/delphi/layouts/404.html @@ -1,7 +1,7 @@ {{ define "title" }}404 page not found - {{ .Site.Title }}{{ end }} {{ define "main" }} -

404: Not found

-

Sorry, we couldn't find the page you're looking for.

- Go back home -{{ end }} \ No newline at end of file +

404: Not found

+

Sorry, we couldn't find the page you're looking for.

+ Go back home +{{ end }} diff --git a/themes/delphi/layouts/_default/about.html b/themes/delphi/layouts/_default/about.html index 7cad92214..cd4385873 100644 --- a/themes/delphi/layouts/_default/about.html +++ b/themes/delphi/layouts/_default/about.html @@ -1,30 +1,27 @@ {{ define "main" }} - -
+
Mission image
-
-
- Our Mission -
-
+
+
Our Mission
+
-

- {{.Site.Params.mission}} -

+

+ {{ .Site.Params.mission }} +

-
+
-
+
- {{ .Content }} + {{ .Content }}
-
-
-

Research and White Papers

- {{partial "about/research-papers.html" .}} -
-{{partial "about/collaborators.html" .}} - -{{ end }} \ No newline at end of file +
+
+ +

Research and White Papers

+ {{ partial "about/research-papers.html" . }} +
+ {{ partial "about/collaborators.html" . }} +{{ end }} diff --git a/themes/delphi/layouts/_default/baseof.html b/themes/delphi/layouts/_default/baseof.html index 48ca12b1e..3a6fe1d28 100644 --- a/themes/delphi/layouts/_default/baseof.html +++ b/themes/delphi/layouts/_default/baseof.html @@ -1,14 +1,13 @@ - {{ partial "head.html" . }} - - {{ partial "nav.html" . }} - {{ partial "menu/breadcrumb.html" . }} -
- {{ block "main" . }} - {{ end }} -
- {{ partial "footer.html" . }} - + {{ partial "head.html" . }} + + {{ partial "nav.html" . }} + {{ partial "menu/breadcrumb.html" . }} +
+ {{ block "main" . }} {{ end }} +
+ {{ partial "footer.html" . }} + diff --git a/themes/delphi/layouts/_default/section.html b/themes/delphi/layouts/_default/section.html index d351fb22b..0da0c102f 100644 --- a/themes/delphi/layouts/_default/section.html +++ b/themes/delphi/layouts/_default/section.html @@ -1,27 +1,24 @@ {{ define "main" }} - -

- Archives -

+ +

Archives

- - {{ $pages := where site.RegularPages "Type" "in" site.Params.mainSections }} - {{ range ($pages.GroupByDate "2006") }} -

{{ .Key }}

+ + {{ $pages := where site.RegularPages "Type" "in" site.Params.mainSections }} + {{ range ($pages.GroupByDate "2006") }} +

{{ .Key }}

+ -
    - - {{ range (.Pages) }} - -
  • - {{ .PublishDate.Format "Jan 2" }} - - {{ .Title }} - -
    • - {{ end }} -
- {{ end }} +
    + {{ range (.Pages) }} +
  • + {{ .PublishDate.Format "Jan 2" }} + + {{ .Title }} + +
    • + {{ end }} +
+ {{ end }}
-{{ end }} \ No newline at end of file +{{ end }} diff --git a/themes/delphi/layouts/_default/single.html b/themes/delphi/layouts/_default/single.html index 1c61646f4..79b5a0e9c 100644 --- a/themes/delphi/layouts/_default/single.html +++ b/themes/delphi/layouts/_default/single.html @@ -1,6 +1,6 @@ {{ define "main" }} -
+

{{ .Title }}

{{ .Content }} -
-{{ end }} \ No newline at end of file +
+{{ end }} diff --git a/themes/delphi/layouts/_default/taxonomy.html b/themes/delphi/layouts/_default/taxonomy.html index 2a6ebe326..4cdb8035e 100644 --- a/themes/delphi/layouts/_default/taxonomy.html +++ b/themes/delphi/layouts/_default/taxonomy.html @@ -1,24 +1,23 @@ {{ define "main" }} - +

Tagged "{{ .Data.Term }}"

- {{range .Pages}} - + {{ range .Pages }}
-

- - {{ .Title }} - -

-
- {{ .Summary }} - {{ if (and (.Site.Params.showReadMore) (.Truncated)) }} -

Read more...

- {{ end }} -
-
- -
-
- {{ end }} -{{ end }} \ No newline at end of file +

+ + {{ .Title }} + +

+
+ {{ .Summary }} + {{ if (and (.Site.Params.showReadMore) (.Truncated)) }} +

Read more...

+ {{ end }} +
+
+ +
+ + {{ end }} +{{ end }} diff --git a/themes/delphi/layouts/_default/team.html b/themes/delphi/layouts/_default/team.html index 0ad406887..f2b99cbf1 100644 --- a/themes/delphi/layouts/_default/team.html +++ b/themes/delphi/layouts/_default/team.html @@ -1,28 +1,34 @@ {{ define "main" }} -
-

Thank you to our {{ len .Params.team }} members around the world, all the students, faculty, staff, and volunteers who have contributed to the COVIDcast project.

+
+

+ Thank you to our {{ len .Params.team }} members around the world, all the students, faculty, staff, and volunteers + who have contributed to the COVIDcast project. +

-World map of team members + World map of team members {{ .Content }}
- Abbreviations
    - {{range .Params.abbreviations}} -
  • {{.}}
    • - {{end}} + {{ range .Params.abbreviations }} +
  • {{ . }}
    • + {{ end }}
-

Thank you for your contributions

-

+

Thank you for your contributions

+

{{ .Params.others }} -

+

-
-{{ end }} \ No newline at end of file +
+{{ end }} diff --git a/themes/delphi/layouts/_default/terms.html b/themes/delphi/layouts/_default/terms.html index bf4d0940a..af3cb2193 100644 --- a/themes/delphi/layouts/_default/terms.html +++ b/themes/delphi/layouts/_default/terms.html @@ -1,9 +1,9 @@ {{ define "main" }} - -

Tags

- +{{ end }} diff --git a/themes/delphi/layouts/_internal/pagination.html b/themes/delphi/layouts/_internal/pagination.html index 2238415bf..3d5dfb0a3 100644 --- a/themes/delphi/layouts/_internal/pagination.html +++ b/themes/delphi/layouts/_internal/pagination.html @@ -1,43 +1,43 @@ {{ $pag := $.Paginator }} {{ if gt $pag.TotalPages 1 -}} -
    -
  • - {{ if $pag.HasPrev }} - - {{else}} - - {{end}} -
    • - {{- $ellipsed := false -}} - {{- $shouldEllipse := false -}} - {{- range $pag.Pagers -}} - {{- $right := sub .TotalPages .PageNumber -}} - {{- $showNumber := or (le .PageNumber 3) (eq $right 0) -}} - {{- $showNumber := or $showNumber (le .TotalPages 5) -}}{{/* Issue #7523 */}} - {{- $showNumber := or $showNumber (and (gt .PageNumber (sub $pag.PageNumber 2)) (lt .PageNumber (add $pag.PageNumber 2))) -}} - {{- if $showNumber -}} - {{- $ellipsed = false -}} - {{- $shouldEllipse = false -}} - {{- else -}} - {{- $shouldEllipse = not $ellipsed -}} - {{- $ellipsed = true -}} - {{- end -}} - {{- if $showNumber }} -
  • - {{ .PageNumber }} -
    • - {{- else if $shouldEllipse }} -
  • - -
    • - {{- end -}} - {{- end }} -
  • - {{ if $pag.HasNext }} - - {{else}} - - {{end}} -
    • -
-{{ end }} \ No newline at end of file +
    +
  • + {{ if $pag.HasPrev }} + + {{ else }} + + {{ end }} +
    • + {{- $ellipsed := false -}} + {{- $shouldEllipse := false -}} + {{- range $pag.Pagers -}} + {{- $right := sub .TotalPages .PageNumber -}} + {{- $showNumber := or (le .PageNumber 3) (eq $right 0) -}} + {{- $showNumber := or $showNumber (le .TotalPages 5) -}}{{/* Issue #7523 */}} + {{- $showNumber := or $showNumber (and (gt .PageNumber (sub $pag.PageNumber 2)) (lt .PageNumber (add $pag.PageNumber 2))) -}} + {{- if $showNumber -}} + {{- $ellipsed = false -}} + {{- $shouldEllipse = false -}} + {{- else -}} + {{- $shouldEllipse = not $ellipsed -}} + {{- $ellipsed = true -}} + {{- end -}} + {{- if $showNumber }} +
  • + {{ .PageNumber }} +
    • + {{- else if $shouldEllipse }} +
  • + +
    • + {{- end -}} + {{- end }} +
  • + {{ if $pag.HasNext }} + + {{ else }} + + {{ end }} +
    • +
+{{ end }} diff --git a/themes/delphi/layouts/blog/list.html b/themes/delphi/layouts/blog/list.html index b9f1e150f..40d67e9de 100644 --- a/themes/delphi/layouts/blog/list.html +++ b/themes/delphi/layouts/blog/list.html @@ -1,38 +1,45 @@ {{ define "main" }} -
-

{{ .Title }}

-

{{ .Description }}

-
    -{{ range .Paginator.Pages }} -
  • -
    -
    -
    - - {{- range .Params.tags -}} - {{ partial "blog/tag.html" . }} - {{- end -}} -
    -

    {{ .Title}}

    -

    - {{ .Summary }} - {{- if .Truncated -}} - … read more - {{ end}} -

    -

    - By {{ .Params.author }} -

    -
    -
    - {{if isset .Params "heroimagethumb"}} - {{ .Title}} - Hero Image - {{end}} -
    +
    +

    {{ .Title }}

    +

    {{ .Description }}

    +
      + {{ range .Paginator.Pages }} +
    • +
      +
      +
      + + {{- range .Params.tags -}} + {{ partial "blog/tag.html" . }} + {{- end -}} +
      +

      + {{ .Title }} +

      +

      + {{ .Summary }} + {{- if .Truncated -}} + … read more + {{ end }} +

      +

      By {{ .Params.author }}

      +
      +
      + {{ if isset .Params "heroimagethumb" }} + {{ .Title }} - Hero Image + {{ end }} +
      +
      +
      • + {{ end }} +
    -
    • + {{ template "_internal/pagination.html" . }} {{ end }} -
-
-{{ template "_internal/pagination.html" . }} -{{ end }} \ No newline at end of file diff --git a/themes/delphi/layouts/blog/single.html b/themes/delphi/layouts/blog/single.html index 8de4d90d1..bc013d643 100644 --- a/themes/delphi/layouts/blog/single.html +++ b/themes/delphi/layouts/blog/single.html @@ -1,62 +1,68 @@ {{ define "main" }} -
-

{{ .Title }}

-{{ partial "blog/tags.html" .}} -{{if isset .Params "heroimage"}} -
- {{ .Title}} - Hero Image -
-{{end}} -
-
+
+

{{ .Title }}

+ {{ partial "blog/tags.html" . }} + {{ if isset .Params "heroimage" }} +
+ {{ .Title }} - Hero Image +
+ {{ end }} +
+
-
- - {{if gt (len .Params.authors) 1}} - {{partial "font-awesome.html" "solid/users"}} - {{ else }} - {{partial "font-awesome.html" "solid/user"}} - {{ end }} - -
-
-

{{ .Params.author }}

-

- {{ partial "share.html" . }} -
+
+ + {{ if gt (len .Params.authors) 1 }} + {{ partial "font-awesome.html" "solid/users" }} + {{ else }} + {{ partial "font-awesome.html" "solid/user" }} + {{ end }} + +
+
+

{{ .Params.author }}

+

+ {{ partial "share.html" . }} +
-
-
-
+
+
+
{{ .Content }} {{ if isset .Params "acknowledgements" }} -

+

Acknowledgements: {{ .Params.acknowledgements | markdownify }} -

+

{{ end }} {{ if isset .Params "related" }} - Related Posts: -
    + Related Posts: +
      {{ $currentPage := . }} {{ range $related := .Params.related }} - {{ $relPage := $.GetPage $related}} -
    • {{ $relPage.Title }}
      • + {{ $relPage := $.GetPage $related }} +
    • {{ $relPage.Title }}
      • {{ end }} -
    +
{{ end }} {{ template "_internal/disqus.html" . }} +
-
-
-{{ range .Params.authors}} -
- {{ range first 1 (where $.Site.Data.authors "key" "eq" .)}} - {{ if isset . "link"}}{{.name}}{{ else }}{{.name}}{{ end}} {{.description}} +
+ {{ range .Params.authors }} +
+ {{ range first 1 (where $.Site.Data.authors "key" "eq" .) }} + {{ if isset . "link" }} + {{ .name }}{{ else }}{{ .name }}{{ end }} + + + {{ .description }} + {{ end }} +
{{ end }} -
+
+ {{ partial "blog/latestblogs.html" . }} +
{{ end }} -
-{{ partial "blog/latestblogs.html" . }} -
-{{end}} diff --git a/themes/delphi/layouts/covidcast_app/baseof.html b/themes/delphi/layouts/covidcast_app/baseof.html index ea3d89f86..f36f807b3 100644 --- a/themes/delphi/layouts/covidcast_app/baseof.html +++ b/themes/delphi/layouts/covidcast_app/baseof.html @@ -1,12 +1,11 @@ - {{ partial "head.html" . }} - - {{ partial "nav.html" . }} - {{ partial "menu/breadcrumb.html" . }} - {{ block "main" . }} - {{ end }} - {{ partial "footer.html" . }} - + {{ partial "head.html" . }} + + {{ partial "nav.html" . }} + {{ partial "menu/breadcrumb.html" . }} + {{ block "main" . }} {{ end }} + {{ partial "footer.html" . }} + diff --git a/themes/delphi/layouts/index.html b/themes/delphi/layouts/index.html index ed057a4de..4cfd224d4 100644 --- a/themes/delphi/layouts/index.html +++ b/themes/delphi/layouts/index.html @@ -1,4 +1,4 @@ {{ define "main" }} -

{{ .Title }}

-{{ .Content }} -{{ end }} \ No newline at end of file +

{{ .Title }}

+ {{ .Content }} +{{ end }} diff --git a/themes/delphi/layouts/landing.html b/themes/delphi/layouts/landing.html index bca83ddee..e39bd686e 100644 --- a/themes/delphi/layouts/landing.html +++ b/themes/delphi/layouts/landing.html @@ -1,69 +1,82 @@ -{{ define "main" }} -
- -
-
-
-
-

- Our Mission -

-

- {{.Site.Params.mission}} -

-
-
-
-
- -
-
-
-
-
- -
-

Our Team

-

Meeting the scientists and researchers making a difference

- {{partial "arrow-link.html" (dict "link" (relref . "team") "alt" "View all")}} -
-
-
- -
-

Our API

-

Access out data and tools

- {{partial "arrow-link.html" (dict "link" .Site.Params.apiUrl "alt" "Learn more")}} -
-
-
-
- - {{partial "landing/latest-news.html" .}} -
-{{ end }} \ No newline at end of file +{{ define "main" }} +
+ +
+
+
+
+

Our Mission

+

+ {{ .Site.Params.mission }} +

+
+
+
+
+ +
+
+
+
+
+ +
+

Our Team

+

Meeting the scientists and researchers making a difference

+ {{ partial "arrow-link.html" (dict "link" (relref . "team") "alt" "View all") }} +
+
+
+ +
+

Our API

+

Access out data and tools

+ {{ partial "arrow-link.html" (dict "link" .Site.Params.apiUrl "alt" "Learn more") }} +
+
+
+
+ + {{ partial "landing/latest-news.html" . }} +
+{{ end }} diff --git a/themes/delphi/layouts/partials/about/collaborator-img.html b/themes/delphi/layouts/partials/about/collaborator-img.html index eebdf29bf..c6c4ea8e8 100644 --- a/themes/delphi/layouts/partials/about/collaborator-img.html +++ b/themes/delphi/layouts/partials/about/collaborator-img.html @@ -1,3 +1,3 @@
- {{.name}} -
\ No newline at end of file + {{ .name }} +
diff --git a/themes/delphi/layouts/partials/about/collaborators.html b/themes/delphi/layouts/partials/about/collaborators.html index d064478a0..2d7bbe619 100644 --- a/themes/delphi/layouts/partials/about/collaborators.html +++ b/themes/delphi/layouts/partials/about/collaborators.html @@ -1,18 +1,17 @@ -
-
-

Collaborators

-

We're grateful for financial and other support from our collaborators and supporters

-
- {{range where .Site.Data.supporter "group" "collaborator"}} - {{partial "about/collaborator-img.html" .}} - {{end}} -
-

With grant support from

-
- {{range where .Site.Data.supporter "group" "grant"}} - {{partial "about/collaborator-img.html" .}} - {{end}} -
+
+

Collaborators

+

We're grateful for financial and other support from our collaborators and supporters

+
+ {{ range where .Site.Data.supporter "group" "collaborator" }} + {{ partial "about/collaborator-img.html" . }} + {{ end }}
-
\ No newline at end of file +

With grant support from

+
+ {{ range where .Site.Data.supporter "group" "grant" }} + {{ partial "about/collaborator-img.html" . }} + {{ end }} +
+
+
diff --git a/themes/delphi/layouts/partials/about/research-papers.html b/themes/delphi/layouts/partials/about/research-papers.html index 5d1670398..9b12396cc 100644 --- a/themes/delphi/layouts/partials/about/research-papers.html +++ b/themes/delphi/layouts/partials/about/research-papers.html @@ -1,23 +1,26 @@ {{ $currentPage := . }}
{{ range $.Site.Data.research }} -
-
{{ .year }}
- -
+
+
{{ .year }}
+ +
-

{{.title}}

-
- {{ .authors }} -
-
- {{.journal}}, {{ .year }} -
+

{{ .title }}

+
+ {{ .authors }} +
+
{{ .journal }}, {{ .year }}
- {{partial "arrow-link.html" (dict "alt" "View Paper" "link" .link)}} + {{ partial "arrow-link.html" (dict "alt" "View Paper" "link" .link) }}
+
-
- {{end}} + {{ end }}
diff --git a/themes/delphi/layouts/partials/arrow-link.html b/themes/delphi/layouts/partials/arrow-link.html index dd9947066..d9154672b 100644 --- a/themes/delphi/layouts/partials/arrow-link.html +++ b/themes/delphi/layouts/partials/arrow-link.html @@ -1,4 +1,4 @@ - - {{partial "font-awesome.html" "solid/arrow-right"}} - {{.alt}} - \ No newline at end of file + + {{ partial "font-awesome.html" "solid/arrow-right" }} + {{ .alt }} + diff --git a/themes/delphi/layouts/partials/blog/card.html b/themes/delphi/layouts/partials/blog/card.html index fbe231e98..9e650dceb 100644 --- a/themes/delphi/layouts/partials/blog/card.html +++ b/themes/delphi/layouts/partials/blog/card.html @@ -1,15 +1,20 @@
-
- {{ .Title}} - Hero Image -
-
- {{ partial "blog/tags.html" . }} -

{{ .Title}}

-
-
- -
- By {{ .Params.author }} -
-
-
\ No newline at end of file +
+ {{ .Title }} - Hero Image +
+
+ {{ partial "blog/tags.html" . }} +

{{ .Title }}

+
+
+ +
By {{ .Params.author }}
+
+
diff --git a/themes/delphi/layouts/partials/blog/latestblogs.html b/themes/delphi/layouts/partials/blog/latestblogs.html index 28be7c651..2c989f99c 100644 --- a/themes/delphi/layouts/partials/blog/latestblogs.html +++ b/themes/delphi/layouts/partials/blog/latestblogs.html @@ -1,11 +1,11 @@
-

Latest Stories

- {{- $currentPage := . -}} -
- {{ range first 3 (where .Parent.Pages ".Title" "!=" $currentPage.Title )}} -
- {{ partial "blog/card.html" . }} -
- {{end}} -
-
\ No newline at end of file +

Latest Stories

+ {{- $currentPage := . -}} +
+ {{ range first 3 (where .Parent.Pages ".Title" "!=" $currentPage.Title ) }} +
+ {{ partial "blog/card.html" . }} +
+ {{ end }} +
+
diff --git a/themes/delphi/layouts/partials/blog/tag.html b/themes/delphi/layouts/partials/blog/tag.html index cb811cd1b..98c7b3ac4 100644 --- a/themes/delphi/layouts/partials/blog/tag.html +++ b/themes/delphi/layouts/partials/blog/tag.html @@ -1 +1,3 @@ -#{{.}} \ No newline at end of file +#{{ . }} diff --git a/themes/delphi/layouts/partials/blog/tags.html b/themes/delphi/layouts/partials/blog/tags.html index a18917c2e..3ee374247 100644 --- a/themes/delphi/layouts/partials/blog/tags.html +++ b/themes/delphi/layouts/partials/blog/tags.html @@ -1,5 +1,5 @@
- {{- range .Params.tags -}} + {{- range .Params.tags -}} {{ partial "blog/tag.html" . }} - {{- end -}} -
\ No newline at end of file + {{- end -}} + diff --git a/themes/delphi/layouts/partials/delphi-text-logo.html b/themes/delphi/layouts/partials/delphi-text-logo.html index e93dfd158..6ef1ebb4a 100644 --- a/themes/delphi/layouts/partials/delphi-text-logo.html +++ b/themes/delphi/layouts/partials/delphi-text-logo.html @@ -1 +1 @@ - \ No newline at end of file + diff --git a/themes/delphi/layouts/partials/font-awesome.html b/themes/delphi/layouts/partials/font-awesome.html index 7faf0d3a2..ca31eb1fa 100644 --- a/themes/delphi/layouts/partials/font-awesome.html +++ b/themes/delphi/layouts/partials/font-awesome.html @@ -1,3 +1,3 @@ - {{ readFile (print "./node_modules/@fortawesome/fontawesome-free/svgs/" . ".svg" ) | safeHTML }} - \ No newline at end of file + {{ readFile (print "./node_modules/@fortawesome/fontawesome-free/svgs/" . ".svg" ) | safeHTML }} + diff --git a/themes/delphi/layouts/partials/footer.html b/themes/delphi/layouts/partials/footer.html index 0c09fcff5..1a2e62fb3 100644 --- a/themes/delphi/layouts/partials/footer.html +++ b/themes/delphi/layouts/partials/footer.html @@ -1,15 +1,15 @@
- {{ partial "footer/desktop.html" . }} + {{ partial "footer/desktop.html" . }}
{{ $script := resources.Get "js/main.js" | js.Build | minify | fingerprint -}} {{ if eq .Page.Type "blog" }} -{{ $script_blog := resources.Get "js/blog/index.js" | js.Build | minify | fingerprint -}} - + {{ $script_blog := resources.Get "js/blog/index.js" | js.Build | minify | fingerprint -}} + {{ end }} {{ range .Page.Params.scripts }} - + {{ end }} diff --git a/themes/delphi/layouts/partials/footer/desktop.html b/themes/delphi/layouts/partials/footer/desktop.html index 2ff44b5d7..a67d13f53 100644 --- a/themes/delphi/layouts/partials/footer/desktop.html +++ b/themes/delphi/layouts/partials/footer/desktop.html @@ -1,34 +1,34 @@
-
- {{partial "delphi-text-logo.html" . }} -
-
-
COVIDcast
- -
-
-
Resources
- -
-
-
About
- -
-
-
Contact
- -
-
\ No newline at end of file +
+ {{ partial "delphi-text-logo.html" . }} +
+
+
COVIDcast
+ +
+
+
Resources
+ +
+
+
About
+ +
+
+
Contact
+ +
+ diff --git a/themes/delphi/layouts/partials/footer/legal.html b/themes/delphi/layouts/partials/footer/legal.html index b42a73303..11c048dd4 100644 --- a/themes/delphi/layouts/partials/footer/legal.html +++ b/themes/delphi/layouts/partials/footer/legal.html @@ -1,7 +1,12 @@
-
    -
  • © 2020 Delphi group authors.
    • -
- Text and figures released under CC BY 4.0; - code under the MIT license. +
    +
  • © 2020 Delphi group authors.
    • +
+ Text and figures released under CC BY 4.0; code under the MIT license.
diff --git a/themes/delphi/layouts/partials/head.html b/themes/delphi/layouts/partials/head.html index 1395473a9..f376687d2 100644 --- a/themes/delphi/layouts/partials/head.html +++ b/themes/delphi/layouts/partials/head.html @@ -1,39 +1,39 @@ - - - - + + + + - {{ if .IsHome }}{{ .Site.Title }}{{ else }}{{ .Title }} | {{ .Site.Title }}{{ end }} - - - - - - - + {{ if .IsHome }}{{ .Site.Title }}{{ else }}{{ .Title }} | {{ .Site.Title }}{{ end }} + + + + + + + - - - + + + - - {{ $main_style := resources.Get "css/main.scss" | toCSS | minify | fingerprint }} - + + {{ $main_style := resources.Get "css/main.scss" | toCSS | minify | fingerprint }} + - {{ with .OutputFormats.Get "RSS" }} + {{ with .OutputFormats.Get "RSS" }} {{ printf `` .Rel .MediaType.Type .Permalink $.Site.Title | safeHTML }} - {{ end }} - {{- if not .Site.IsServer -}} - {{ template "_internal/google_analytics.html" . }} - {{- end -}} + {{ end }} + {{- if not .Site.IsServer -}} + {{ template "_internal/google_analytics.html" . }} + {{- end -}} - {{ if eq .Page.Type "blog" }} + {{ if eq .Page.Type "blog" }} {{ $blog_style := resources.Get "css/blog_extra.scss" | toCSS | minify | fingerprint }} - - {{ end }} + + {{ end }} - {{ range .Page.Params.styles }} - - {{ end }} + {{ range .Page.Params.styles }} + + {{ end }} diff --git a/themes/delphi/layouts/partials/header.html b/themes/delphi/layouts/partials/header.html index 347664236..06f145e44 100644 --- a/themes/delphi/layouts/partials/header.html +++ b/themes/delphi/layouts/partials/header.html @@ -1,10 +1,10 @@
- {{ if $isHome := false }} + {{ if $isHome := false }}

{{ .Site.Title }}

{{ if isset .Site.Params "description" }} -

+

{{ .Site.Params.Description | markdownify }} -

+ {{ end }} - {{ end }} -
\ No newline at end of file + {{ end }} + diff --git a/themes/delphi/layouts/partials/landing/latest-card.html b/themes/delphi/layouts/partials/landing/latest-card.html index 54bba1ed4..99c801c5d 100644 --- a/themes/delphi/layouts/partials/landing/latest-card.html +++ b/themes/delphi/layouts/partials/landing/latest-card.html @@ -1,12 +1,12 @@
-
- {{ .Title}} -
-
-
{{ .source }}
-

{{ .title}}

-
-
- Learn more {{partial "font-awesome.html" "solid/long-arrow-alt-right"}} -
-
\ No newline at end of file +
+ {{ .Title }} +
+
+
{{ .source }}
+

{{ .title }}

+
+
+ Learn more {{ partial "font-awesome.html" "solid/long-arrow-alt-right" }} +
+ diff --git a/themes/delphi/layouts/partials/landing/latest-news.html b/themes/delphi/layouts/partials/landing/latest-news.html index 3dfc40c2f..fd916cde4 100644 --- a/themes/delphi/layouts/partials/landing/latest-news.html +++ b/themes/delphi/layouts/partials/landing/latest-news.html @@ -1,24 +1,29 @@
-

Latest News

- {{- $currentPage := . -}} - {{ define "partials/latest-blog" }} +

Latest News

+ {{- $currentPage := . -}} + {{ define "partials/latest-blog" }} {{ return (dict "source" "blog" "image" .Params.heroImageThumb "title" .Title "link" .RelPermalink "date" .PublishDate ) }} - {{ end }} - {{ $items := apply (.Site.GetPage "/blog").Pages "partial" "latest-blog" "."}} + {{ end }} + {{ $items := apply (.Site.GetPage "/blog").Pages "partial" "latest-blog" "." }} - {{ $top := 6 }} - -
-
-
    - {{range first $top (sort $items "date" "desc")}} -
  • - {{ partial "landing/latest-card.html" .}} -
    • - {{end}} -
-
- - + {{ $top := 6 }} + +
+
+
    + {{ range first $top (sort $items "date" "desc") }} +
  • + {{ partial "landing/latest-card.html" . }} +
    • + {{ end }} +
-
\ No newline at end of file + + +
+
diff --git a/themes/delphi/layouts/partials/menu/breadcrumb.html b/themes/delphi/layouts/partials/menu/breadcrumb.html index cae9906e0..883e80962 100644 --- a/themes/delphi/layouts/partials/menu/breadcrumb.html +++ b/themes/delphi/layouts/partials/menu/breadcrumb.html @@ -1,11 +1,15 @@ {{ if .Parent }} -{{ if (not .Parent.IsHome) }} -
- - -
+ {{ if (not .Parent.IsHome) }} +
+ + +
+ {{ end }} {{ end }} -{{ end }} \ No newline at end of file diff --git a/themes/delphi/layouts/partials/menu/item.html b/themes/delphi/layouts/partials/menu/item.html index 323ec96c0..9298c1878 100644 --- a/themes/delphi/layouts/partials/menu/item.html +++ b/themes/delphi/layouts/partials/menu/item.html @@ -1,4 +1,4 @@ {{ if .Pre }} -{{ partial "font-awesome.html" .Pre }} -{{end}} -{{ .Name }} \ No newline at end of file + {{ partial "font-awesome.html" .Pre }} +{{ end }} +{{ .Name }} diff --git a/themes/delphi/layouts/partials/nav.html b/themes/delphi/layouts/partials/nav.html index 574d807b1..b3941239d 100644 --- a/themes/delphi/layouts/partials/nav.html +++ b/themes/delphi/layouts/partials/nav.html @@ -1,57 +1,59 @@ {{- $currentPage := . -}} diff --git a/themes/delphi/layouts/partials/share.html b/themes/delphi/layouts/partials/share.html index ed1efa898..ebff1d74b 100644 --- a/themes/delphi/layouts/partials/share.html +++ b/themes/delphi/layouts/partials/share.html @@ -1,23 +1,32 @@
- - {{partial "font-awesome" "brands/twitter"}} - - - - {{partial "font-awesome" "brands/linkedin"}} - - - - {{partial "font-awesome" "brands/facebook"}} - - -
\ No newline at end of file + + {{ partial "font-awesome" "brands/twitter" }} + + + + {{ partial "font-awesome" "brands/linkedin" }} + + + + {{ partial "font-awesome" "brands/facebook" }} + + + diff --git a/themes/delphi/layouts/partials/social.html b/themes/delphi/layouts/partials/social.html index f460f846a..3871701f5 100644 --- a/themes/delphi/layouts/partials/social.html +++ b/themes/delphi/layouts/partials/social.html @@ -1,55 +1,39 @@
diff --git a/themes/delphi/layouts/section/archives.html b/themes/delphi/layouts/section/archives.html index 6f68b16c1..2db1fc057 100644 --- a/themes/delphi/layouts/section/archives.html +++ b/themes/delphi/layouts/section/archives.html @@ -1,26 +1,25 @@ {{ define "main" }} -
- -

Archives

- - {{ $pages := where site.RegularPages "Type" "in" site.Params.mainSections }} - {{ range ($pages.GroupByDate "2006") }} -

{{ .Key }}

- + +

Archives

+
+ + {{ $pages := where site.RegularPages "Type" "in" site.Params.mainSections }} + {{ range ($pages.GroupByDate "2006") }} +

{{ .Key }}

+ -
    - {{ range (.Pages) }} - -
  • - {{ .PublishDate.Format "Jan 2" }} - - {{ .Title }} - -
    • +
      + {{ range (.Pages) }} +
    • + {{ .PublishDate.Format "Jan 2" }} + + {{ .Title }} + +
      • + {{ end }} +
    {{ end }} -
- {{ end }} +
-
{{ end }} diff --git a/themes/delphi/layouts/shortcodes/apiref.html b/themes/delphi/layouts/shortcodes/apiref.html index 0461b5ced..e97f4bba5 100644 --- a/themes/delphi/layouts/shortcodes/apiref.html +++ b/themes/delphi/layouts/shortcodes/apiref.html @@ -1 +1 @@ -{{- path.Join $.Page.Site.Params.apiUrl (.Get 0) -}} \ No newline at end of file +{{- path.Join $.Page.Site.Params.apiUrl (.Get 0) -}} diff --git a/themes/delphi/layouts/shortcodes/bibliography.html b/themes/delphi/layouts/shortcodes/bibliography.html index 74737dbcf..be77fcdc6 100644 --- a/themes/delphi/layouts/shortcodes/bibliography.html +++ b/themes/delphi/layouts/shortcodes/bibliography.html @@ -1,15 +1,14 @@ \ No newline at end of file + diff --git a/themes/delphi/layouts/shortcodes/indicators.html b/themes/delphi/layouts/shortcodes/indicators.html index df6c2c47f..399904166 100644 --- a/themes/delphi/layouts/shortcodes/indicators.html +++ b/themes/delphi/layouts/shortcodes/indicators.html @@ -1,10 +1,9 @@ -
-{{ $tools := .Site.GetPage "/covidcast/indicators" }} -{{ range sort (where ($tools.Resources.ByType "page") ".Params.category" "eq" (.Get "category")) ".Params.order" }} -
-

{{ .Title }}

- {{ .Content }} -
-{{ end }} -
\ No newline at end of file + {{ $tools := .Site.GetPage "/covidcast/indicators" }} + {{ range sort (where ($tools.Resources.ByType "page") ".Params.category" "eq" (.Get "category")) ".Params.order" }} +
+

{{ .Title }}

+ {{ .Content }} +
+ {{ end }} + diff --git a/themes/delphi/layouts/shortcodes/news.html b/themes/delphi/layouts/shortcodes/news.html index 9954b5ed1..e2b3bea66 100644 --- a/themes/delphi/layouts/shortcodes/news.html +++ b/themes/delphi/layouts/shortcodes/news.html @@ -1,6 +1,5 @@ - {{ $news := .Site.GetPage "/news" }} -{{ range (sort ($news.Resources.ByType "page") ".PublishDate" "desc")}} +{{ range (sort ($news.Resources.ByType "page") ".PublishDate" "desc") }}

{{ .PublishDate.Format "January 2006" }}

{{ .Content }} -{{ end }} \ No newline at end of file +{{ end }} diff --git a/themes/delphi/layouts/shortcodes/releaselog.html b/themes/delphi/layouts/shortcodes/releaselog.html index c52bf063c..9f3fb5f10 100644 --- a/themes/delphi/layouts/shortcodes/releaselog.html +++ b/themes/delphi/layouts/shortcodes/releaselog.html @@ -1,12 +1,13 @@ - {{ $tools := .Site.GetPage "/covidcast/release-log/headless" }} \ No newline at end of file + + {{ end }} + diff --git a/themes/delphi/layouts/shortcodes/systems.html b/themes/delphi/layouts/shortcodes/systems.html index f9bd413d3..e2ba8d50d 100644 --- a/themes/delphi/layouts/shortcodes/systems.html +++ b/themes/delphi/layouts/shortcodes/systems.html @@ -1,12 +1,11 @@ - {{ $tools := .Site.GetPage "/systems" }} {{ range sort ($tools.Resources.ByType "page") "Params.order" }}

- {{ if isset .Params "externallink" }} - {{.Title}} - {{else}} - {{.Title}} - {{ end }} + {{ if isset .Params "externallink" }} + {{ .Title }} + {{ else }} + {{ .Title }} + {{ end }}

{{ .Content }} -{{ end }} \ No newline at end of file +{{ end }} diff --git a/themes/delphi/layouts/shortcodes/team.html b/themes/delphi/layouts/shortcodes/team.html index 30bc631a8..ed9e6ee75 100644 --- a/themes/delphi/layouts/shortcodes/team.html +++ b/themes/delphi/layouts/shortcodes/team.html @@ -1,15 +1,18 @@ -{{ $images := .Page.Resources.ByType "image"}} -
-{{ $team := where .Page.Params.team ".team" (.Get "team")}} -{{ range sort $team "lastName" "asc" "firstName" "asc"}} - {{ $img := $images.GetMatch (path.Join "images" .image) }} -
+{{ $images := .Page.Resources.ByType "image" }} +
+ {{ $team := where .Page.Params.team ".team" (.Get "team") }} + {{ range sort $team "lastName" "asc" "firstName" "asc" }} + {{ $img := $images.GetMatch (path.Join "images" .image) }} +
{{ .name }}
{{ printf "%s %s" .firstName .lastName }}
{{ .affiliation }}
{{ if isset . "note" }} -
{{ .note }}
+
{{ .note }}
{{ end }} -
+
{{ end }}
diff --git a/themes/delphi/layouts/shortcodes/tools.html b/themes/delphi/layouts/shortcodes/tools.html index cfd19bd55..7f4789648 100644 --- a/themes/delphi/layouts/shortcodes/tools.html +++ b/themes/delphi/layouts/shortcodes/tools.html @@ -1,6 +1,5 @@ - {{ $tools := .Site.GetPage "/tools" }} {{ range sort ($tools.Resources.ByType "page") "Params.order" }} -

{{ .Title}}

+

{{ .Title }}

{{ .Content }} -{{ end }} \ No newline at end of file +{{ end }} From e02254411bb08033ab46ee2043e434ef4b0b632b Mon Sep 17 00:00:00 2001 From: Samuel Gratzl Date: Tue, 24 Nov 2020 10:07:26 -0500 Subject: [PATCH 04/12] build: lint during build --- .github/workflows/ci.yaml | 2 ++ .github/workflows/ci_fast.yaml | 2 ++ package.json | 7 ++++--- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 034f7dde1..214f432eb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -64,6 +64,8 @@ jobs: restore-keys: | ${{ runner.os }}-node- - run: npm ci + - name: Lint + run: npm run lint - name: Build run: npm run build diff --git a/.github/workflows/ci_fast.yaml b/.github/workflows/ci_fast.yaml index 19f8c81aa..e1d1948ea 100644 --- a/.github/workflows/ci_fast.yaml +++ b/.github/workflows/ci_fast.yaml @@ -26,6 +26,8 @@ jobs: restore-keys: | ${{ runner.os }}-node- - run: npm ci + - name: Lint + run: npm run lint - name: Build run: npm run build diff --git a/package.json b/package.json index 24ca94767..17bf70f2a 100644 --- a/package.json +++ b/package.json @@ -1,4 +1,7 @@ { + "name": "www-main", + "version": "0.1.0", + "private": true, "dependencies": { "@fortawesome/fontawesome-free": "^5.15.1", "highlight.js": "^10.3.2", @@ -32,7 +35,5 @@ "start:blog": "Rscript -e \"blogdown::serve_site()\"", "format": "prettier **/* --write", "lint": "prettier **/* --check" - }, - "name": "www-main", - "version": "0.1.0" + } } From 7a61329740ad08d32b17d56c8caa3e7368995fb9 Mon Sep 17 00:00:00 2001 From: Samuel Gratzl Date: Tue, 24 Nov 2020 10:50:19 -0500 Subject: [PATCH 05/12] refactor: simplify front matter --- content/blog/2015-07-23-template-post.Rmd | 9 +- content/blog/2015-07-23-template-post.html | 14 +-- content/blog/2020-08-10-hello-world.Rmd | 7 +- content/blog/2020-08-10-hello-world.html | 11 +- content/blog/2020-08-26-fb-survey.Rmd | 9 +- content/blog/2020-08-26-fb-survey.html | 21 ++-- content/blog/2020-08-28-api.Rmd | 10 +- content/blog/2020-08-28-api.html | 14 ++- content/blog/2020-09-18-google-survey.Rmd | 9 +- content/blog/2020-09-18-google-survey.html | 45 ++++---- content/blog/2020-09-21-forecast-demo.Rmd | 10 +- content/blog/2020-09-21-forecast-demo.html | 118 +++++++++++---------- content/blog/2020-10-06-survey-wave-4.Rmd | 9 +- content/blog/2020-10-06-survey-wave-4.html | 11 +- content/blog/2020-10-14-dv-signal.Rmd | 9 +- content/blog/2020-10-14-dv-signal.html | 19 ++-- 16 files changed, 186 insertions(+), 139 deletions(-) diff --git a/content/blog/2015-07-23-template-post.Rmd b/content/blog/2015-07-23-template-post.Rmd index 2afbb416a..765a30c3c 100644 --- a/content/blog/2015-07-23-template-post.Rmd +++ b/content/blog/2015-07-23-template-post.Rmd @@ -1,8 +1,11 @@ --- -title: "Template Post" -author: "Frida Gomam" +title: Template Post +author: Frida Gomam date: 2015-07-23 -tags: ["R Markdown", "plot", "regression"] +tags: + - R Markdown + - plot + - regression draft: true authors: - frida diff --git a/content/blog/2015-07-23-template-post.html b/content/blog/2015-07-23-template-post.html index eb772f116..327177374 100644 --- a/content/blog/2015-07-23-template-post.html +++ b/content/blog/2015-07-23-template-post.html @@ -1,17 +1,19 @@ --- -title: "Template Post" -author: "Frida Gomam" +title: Template Post +author: Frida Gomam date: 2015-07-23 -tags: ["R Markdown", "plot", "regression"] +tags: + - R Markdown + - plot + - regression draft: true authors: -- frida + - frida heroImage: /blog/images/blog-lg-img_hello-world.png heroImageThumb: /blog/images/blog-thumb-img_hello-world.png related: -- 2015-07-23-template-post + - 2015-07-23-template-post acknowledgements: Test - --- diff --git a/content/blog/2020-08-10-hello-world.Rmd b/content/blog/2020-08-10-hello-world.Rmd index c2816211d..7b227d223 100644 --- a/content/blog/2020-08-10-hello-world.Rmd +++ b/content/blog/2020-08-10-hello-world.Rmd @@ -1,8 +1,9 @@ --- -title: "Hello World!" -author: "Roni Rosenfeld and Ryan Tibshirani" +title: Hello World! +author: Roni Rosenfeld and Ryan Tibshirani date: 2020-08-10 -tags: ["COVIDcast"] +tags: + - COVIDcast authors: - roni - ryan diff --git a/content/blog/2020-08-10-hello-world.html b/content/blog/2020-08-10-hello-world.html index f8b5e377e..87208689d 100644 --- a/content/blog/2020-08-10-hello-world.html +++ b/content/blog/2020-08-10-hello-world.html @@ -1,11 +1,12 @@ --- -title: "Hello World!" -author: "Roni Rosenfeld and Ryan Tibshirani" +title: Hello World! +author: Roni Rosenfeld and Ryan Tibshirani date: 2020-08-10 -tags: ["COVIDcast"] +tags: + - COVIDcast authors: -- roni -- ryan + - roni + - ryan heroImage: /blog/images/blog-lg-img_hello-world.png heroImageThumb: /blog/images/blog-thumb-img_hello-world.png summary: | diff --git a/content/blog/2020-08-26-fb-survey.Rmd b/content/blog/2020-08-26-fb-survey.Rmd index ca4a186d1..b876ac641 100644 --- a/content/blog/2020-08-26-fb-survey.Rmd +++ b/content/blog/2020-08-26-fb-survey.Rmd @@ -1,8 +1,11 @@ --- -title: "COVID-19 Symptom Surveys through Facebook" -author: "Alex Reinhart and Ryan Tibshirani" +title: COVID-19 Symptom Surveys through Facebook +author: Alex Reinhart and Ryan Tibshirani date: 2020-08-26 -tags: ["symptom surveys", "COVIDcast", "R"] +tags: + - symptom surveys + - COVIDcast + - R authors: - alex - ryan diff --git a/content/blog/2020-08-26-fb-survey.html b/content/blog/2020-08-26-fb-survey.html index c9e7b3516..2a11bdca0 100644 --- a/content/blog/2020-08-26-fb-survey.html +++ b/content/blog/2020-08-26-fb-survey.html @@ -1,11 +1,14 @@ --- -title: "COVID-19 Symptom Surveys through Facebook" -author: "Alex Reinhart and Ryan Tibshirani" +title: COVID-19 Symptom Surveys through Facebook +author: Alex Reinhart and Ryan Tibshirani date: 2020-08-26 -tags: ["symptom surveys", "COVIDcast", "R"] +tags: + - symptom surveys + - COVIDcast + - R authors: -- alex -- ryan + - alex + - ryan heroImage: /blog/images/blog-lg-img_facebook-survey-post.png heroImageThumb: /blog/images/blog-thumb-img_facebook-survey-post.png summary: | @@ -299,11 +302,11 @@

Some Interesting Examples

given_geo_value = geo_value df_fb_one = df_fb %>% filter(geo_value == given_geo_value) df_in_one = df_in %>% filter(geo_value == given_geo_value) - + # Compute ranges of the two signals range1 = df_in_one %>% select("value") %>% range range2 = df_fb_one %>% select("value") %>% range - + # Convenience functions for our two signal ranges trans12 = function(x) trans(x, range1, range2) trans21 = function(x) trans(x, range2, range1) @@ -319,7 +322,7 @@

Some Interesting Examples

df_in_one), c("time_value", "value")) df$signal = c(rep("% CLI-in-community", nrow(df_fb_one)), rep("New COVID-19 cases", nrow(df_in_one))) - + # Finally, plot both signals pos = ifelse(legend, "bottom", "none") return(ggplot(df, aes(x = time_value, y = value)) + @@ -459,7 +462,7 @@

Correlations Sliced by Time

labs(title = "Median absolute deviation in COVID-19 case rates", subtitle = sprintf("Over all counties with at least %i cumulative cases", case_num), x = "Date", y = "Median abs deviation") + - theme_bw() + theme_bw()

diff --git a/content/blog/2020-08-28-api.Rmd b/content/blog/2020-08-28-api.Rmd index 638208c67..44745ed14 100644 --- a/content/blog/2020-08-28-api.Rmd +++ b/content/blog/2020-08-28-api.Rmd @@ -1,8 +1,12 @@ --- -title: "Accessing Open COVID-19 Data via the COVIDcast Epidata API" -author: "Kathryn Mazaitis and Alex Reinhart" +title: Accessing Open COVID-19 Data via the COVIDcast Epidata API +author: Kathryn Mazaitis and Alex Reinhart date: 2020-10-07 -tags: ["COVIDcast API", "COVIDcast", "R", "Python"] +tags: + - COVIDcast API + - COVIDcast + - R + - Python authors: - kathryn - alex diff --git a/content/blog/2020-08-28-api.html b/content/blog/2020-08-28-api.html index 28fa40e61..58d24b383 100644 --- a/content/blog/2020-08-28-api.html +++ b/content/blog/2020-08-28-api.html @@ -1,11 +1,15 @@ --- -title: "Accessing Open COVID-19 Data via the COVIDcast Epidata API" -author: "Kathryn Mazaitis and Alex Reinhart" +title: Accessing Open COVID-19 Data via the COVIDcast Epidata API +author: Kathryn Mazaitis and Alex Reinhart date: 2020-10-07 -tags: ["COVIDcast API", "COVIDcast", "R", "Python"] +tags: + - COVIDcast API + - COVIDcast + - R + - Python authors: -- kathryn -- alex + - kathryn + - alex heroImage: /blog/images/blog-lg-img_Accessing Open COVID-19.png heroImageThumb: /blog/images/blog-thumb-img_Accessing Open COVID-19.png summary: | diff --git a/content/blog/2020-09-18-google-survey.Rmd b/content/blog/2020-09-18-google-survey.Rmd index 038a84cce..fd5830340 100644 --- a/content/blog/2020-09-18-google-survey.Rmd +++ b/content/blog/2020-09-18-google-survey.Rmd @@ -1,8 +1,11 @@ --- -title: "COVID-19 Symptom Surveys through Google" -author: "Ryan Tibshirani" +title: COVID-19 Symptom Surveys through Google +author: Ryan Tibshirani date: 2020-09-18 -tags: ["symptom surveys", "COVIDcast", "R"] +tags: + - symptom surveys + - COVIDcast + - R authors: - ryan heroImage: /blog/images/blog-lg-img_google-survey-post.png diff --git a/content/blog/2020-09-18-google-survey.html b/content/blog/2020-09-18-google-survey.html index 928b58bf3..f92215370 100644 --- a/content/blog/2020-09-18-google-survey.html +++ b/content/blog/2020-09-18-google-survey.html @@ -1,10 +1,13 @@ --- -title: "COVID-19 Symptom Surveys through Google" -author: "Ryan Tibshirani" +title: COVID-19 Symptom Surveys through Google +author: Ryan Tibshirani date: 2020-09-18 -tags: ["symptom surveys", "COVIDcast", "R"] +tags: + - symptom surveys + - COVIDcast + - R authors: -- ryan + - ryan heroImage: /blog/images/blog-lg-img_google-survey-post.png heroImageThumb: /blog/images/blog-thumb-img_google-survey-post.png summary: | @@ -19,7 +22,7 @@ This short post covers some key differences between our Google and Facebook surveys, explains the backstory behind the "CLI-in-community" question as it arose through our collaboration with Google, - and shares some of our thinking about next steps for the Google survey. + and shares some of our thinking about next steps for the Google survey. acknowledgements: | Ryan Tibshirani wrote the initial code for producing estimates from the aggregated survey data. Sangwon Hyun, Natalia Lombardi de @@ -137,18 +140,18 @@

CLI-in-Community

start_day = min(df_go$time_value) end_day = max(df_go$time_value) -df_fb = covidcast_signal("fb-survey", "smoothed_hh_cmnty_cli", +df_fb = covidcast_signal("fb-survey", "smoothed_hh_cmnty_cli", start_day, end_day, geo_type = "state") df_in = covidcast_signal("jhu-csse", "confirmed_7dav_incidence_prop", start_day, end_day, geo_type = "state") -# Join by state, average signals, compute correlations +# Join by state, average signals, compute correlations df1 = inner_join(df_go %>% group_by(geo_value) %>% summarize(x = mean(value)), df_in %>% group_by(geo_value) %>% summarize(y = mean(value)), - by = "geo_value") + by = "geo_value") df2 = inner_join(df_fb %>% group_by(geo_value) %>% summarize(x = mean(value)), - df_in %>% group_by(geo_value) %>% summarize(y = mean(value)), - by = "geo_value") + df_in %>% group_by(geo_value) %>% summarize(y = mean(value)), + by = "geo_value") # Join again to get state populations df1 = inner_join(df1, state_census %>% mutate(ABBR = tolower(ABBR)), @@ -161,22 +164,22 @@

CLI-in-Community

# Now make plots subtitle = paste("Averaged over", start_day, "to", end_day) -p1 = ggplot(df1, aes(x = x, y = y, label = toupper(geo_value))) + +p1 = ggplot(df1, aes(x = x, y = y, label = toupper(geo_value))) + geom_smooth(method = "lm", col = ggplot_colors[2], se = FALSE) + - geom_point(aes(size = POPESTIMATE2019), color = ggplot_colors[2], - alpha = 0.5) + - scale_size(name = "Population", range = c(1, 10)) + + geom_point(aes(size = POPESTIMATE2019), color = ggplot_colors[2], + alpha = 0.5) + + scale_size(name = "Population", range = c(1, 10)) + geom_text(alpha = 0.5) + - labs(x = "% CLI-in-community from Google surveys", + labs(x = "% CLI-in-community from Google surveys", y = "Daily new confirmed COVID-19 cases per 100,000 people", title = "COVID-19 case rates vs Google % CLI-in-community", subtitle = subtitle) + theme_bw() + theme(legend.position = "bottom") -p2 = ggplot(df2, aes(x = x, y = y, label = toupper(geo_value))) + +p2 = ggplot(df2, aes(x = x, y = y, label = toupper(geo_value))) + geom_smooth(method = "lm", col = ggplot_colors[1], se = FALSE) + - geom_point(aes(size = POPESTIMATE2019), color = ggplot_colors[1], - alpha = 0.5) + - scale_size(name = "Population", range = c(1, 10)) + + geom_point(aes(size = POPESTIMATE2019), color = ggplot_colors[1], + alpha = 0.5) + + scale_size(name = "Population", range = c(1, 10)) + geom_text(alpha = 0.5) + labs(x = "% CLI-in-community from Facebook surveys", y = "", title = "COVID-19 case rates vs Facebook % CLI-in-community", @@ -251,7 +254,7 @@

Our Two Surveys

start_day = min(df_go$time_value) end_day = "2020-09-01" -df_fb = covidcast_signal("fb-survey", "smoothed_hh_cmnty_cli", +df_fb = covidcast_signal("fb-survey", "smoothed_hh_cmnty_cli", start_day, end_day) df_in = covidcast_signal("jhu-csse", "confirmed_7dav_incidence_prop", start_day, end_day) @@ -283,7 +286,7 @@

Our Two Surveys

scale_color_manual(values = ggplot_colors) + labs(title = "Correlation between survey signals and case rates", subtitle = sprintf("Over all counties with at least %i cumulative cases", - case_num), + case_num), x = "Date", y = "Correlation") + theme_bw() + theme(legend.pos = "bottom", legend.title = element_blank())

diff --git a/content/blog/2020-09-21-forecast-demo.Rmd b/content/blog/2020-09-21-forecast-demo.Rmd index e21ec7599..3308399b9 100644 --- a/content/blog/2020-09-21-forecast-demo.Rmd +++ b/content/blog/2020-09-21-forecast-demo.Rmd @@ -1,8 +1,12 @@ --- -title: "Can Symptoms Surveys Improve COVID-19 Forecasts?" -author: "Ryan Tibshirani" +title: Can Symptoms Surveys Improve COVID-19 Forecasts? +author: Ryan Tibshirani date: 2020-09-21 -tags: ["symptom surveys", "forecasting", "COVIDcast", "R"] +tags: + - symptom surveys + - forecasting + - COVIDcast + - R authors: - ryan heroImage: /blog/images/blog-Lg-img_can symptoms surveys improve covid-19.png diff --git a/content/blog/2020-09-21-forecast-demo.html b/content/blog/2020-09-21-forecast-demo.html index f69073a48..b498a66b5 100644 --- a/content/blog/2020-09-21-forecast-demo.html +++ b/content/blog/2020-09-21-forecast-demo.html @@ -1,10 +1,14 @@ --- -title: "Can Symptoms Surveys Improve COVID-19 Forecasts?" -author: "Ryan Tibshirani" +title: Can Symptoms Surveys Improve COVID-19 Forecasts? +author: Ryan Tibshirani date: 2020-09-21 -tags: ["symptom surveys", "forecasting", "COVIDcast", "R"] +tags: + - symptom surveys + - forecasting + - COVIDcast + - R authors: -- ryan + - ryan heroImage: /blog/images/blog-Lg-img_can symptoms surveys improve covid-19.png heroImageThumb: /blog/images/blog-thumbnail_can symptoms surveys improve covid-19.png summary: | @@ -20,8 +24,8 @@ attributable to Ryan's work alone, and are a reflection of the work carried out by all these team members.* related: -- 2020-09-18-google-survey -- 2020-08-26-fb-survey + - 2020-09-18-google-survey + - 2020-08-26-fb-survey output: html_document: code_folding: hide @@ -100,8 +104,8 @@

Problem Setup

for location \(\ell\) and time \(t\). (We rescale all these signals from their given values in our API so that they are true proportions: between 0 and 1.) -We evaluate the following four models: -\[ +We evaluate the following four models:

+

\[ \begin{aligned} &\text{Cases:} \quad && h(Y_{\ell,t+d}) \approx \alpha + \sum_{j=0}^2 \beta_j h(Y_{\ell,t-7j}) \\ @@ -116,8 +120,8 @@

Problem Setup

\sum_{j=0}^2 \gamma_j h(F_{\ell,t-7j}) + \sum_{j=0}^2 \tau_j h(G_{\ell,t-7j}). \end{aligned} -\] -Here \(d=7\) or \(d=14\), depending on the target value +\]

+

Here \(d=7\) or \(d=14\), depending on the target value (number of days we predict ahead), and \(h\) is a transformation to be specified later.

Informally, the first model bases its predictions of future case rates @@ -138,12 +142,12 @@

Problem Setup

(we apply \(h^{-1}\) to the predictions from the fitted LAD model), and denoted \(\hat{Y}_{\ell,t_0+d}\). For an error metric, we consider scaled absolute error -(or just scaled error for short): -\[ +(or just scaled error for short):

+

\[ \frac{|\hat{Y}_{\ell,t_0+d} - Y_{\ell,t_0+d}|} {|Y_{\ell,t_0} - Y_{\ell,t_0+d}|}, -\] -where the error in the denominator is the error of the “strawman” model, +\]

+

where the error in the denominator is the error of the “strawman” model, which for any target always simply predicts the most recent available case rate.

This normalization helps for two reasons. First, it gives us an interpretable scale, @@ -415,7 +419,7 @@

Results: All Four Models

model_names = c("Cases", "Cases + Facebook", "Cases + Google", "Cases + Facebook + Google") -# Restrict to common period for all 4 models, then calculate the scaled errors +# Restrict to common period for all 4 models, then calculate the scaled errors # for each model, that is, the error relative to the strawman's error res_all4 = res %>% drop_na() %>% # Restrict to common time @@ -424,23 +428,23 @@

Results: All Four Models

mutate(dif12 = err1 - err2, dif13 = err1 - err3, # Compute differences dif14 = err1 - err4) %>% # relative to cases model ungroup() %>% - select(-err0) - + select(-err0) + # Calculate and print median errors, for all 4 models, and just 7 days ahead -res_err4 = res_all4 %>% +res_err4 = res_all4 %>% select(-starts_with("dif")) %>% pivot_longer(names_to = "model", values_to = "err", cols = -c(geo_value, time_value, lead)) %>% mutate(lead = factor(lead, labels = paste(leads, "days ahead")), model = factor(model, labels = model_names)) -knitr::kable(res_err4 %>% +knitr::kable(res_err4 %>% group_by(model, lead) %>% - summarize(err = median(err), n = length(unique(time_value))) %>% + summarize(err = median(err), n = length(unique(time_value))) %>% arrange(lead) %>% ungroup() %>% - rename("Model" = model, "Median scaled error" = err, + rename("Model" = model, "Median scaled error" = err, "Target" = lead, "Test days" = n) %>% - filter(Target == "7 days ahead"), + filter(Target == "7 days ahead"), caption = paste("Test period:", min(res_err4$time_value), "to", max(res_err4$time_value)), format = "html", table.attr = "style='width:70%;'") @@ -541,17 +545,17 @@

Results: All Four Models

pivot_longer(names_to = "model", values_to = "dif", cols = -c(geo_value, time_value, lead)) %>% mutate(lead = factor(lead, labels = paste(leads, "days ahead")), - model = factor(model, + model = factor(model, labels = c("Cases vs Cases + Facebook", "Cases vs Cases + Google", - "Cases vs Cases + Facebook + Google"))) + "Cases vs Cases + Facebook + Google"))) knitr::kable(res_dif4 %>% group_by(model, lead) %>% - summarize(p = binom.test(x = sum(dif > 0, na.rm = TRUE), + summarize(p = binom.test(x = sum(dif > 0, na.rm = TRUE), n = n(), alt = "greater")$p.val) %>% ungroup() %>% filter(lead == "7 days ahead") %>% - rename("Comparison" = model, "Target" = lead, "P-value" = p), + rename("Comparison" = model, "Target" = lead, "P-value" = p), format = "html", table.attr = "style='width:50%;'") @@ -618,15 +622,15 @@

Results: All Four Models

# Red, blue (similar to ggplot defaults), then yellow
 ggplot_colors = c("#FC4E07", "#00AFBB", "#E7B800")
 
-ggplot(res_dif4 %>% 
+ggplot(res_dif4 %>%
          group_by(model, lead, time_value) %>%
-         summarize(p = binom.test(x = sum(dif > 0, na.rm = TRUE), 
+         summarize(p = binom.test(x = sum(dif > 0, na.rm = TRUE),
                                   n = n(), alt = "greater")$p.val) %>%
          ungroup() %>% filter(lead == "7 days ahead"), aes(p)) +
-  geom_histogram(aes(color = model, fill = model), alpha = 0.4) + 
+  geom_histogram(aes(color = model, fill = model), alpha = 0.4) +
   scale_color_manual(values = ggplot_colors) +
   scale_fill_manual(values = ggplot_colors) +
-  facet_wrap(vars(lead, model)) + 
+  facet_wrap(vars(lead, model)) +
   labs(x = "P-value", y = "Count") +
   theme_bw() + theme(legend.pos = "none")

@@ -641,7 +645,7 @@

Results: First Two Models

Now we see a decent improvement in median scaled error for the “Cases + Facebook” model, and this is true for both 7-day-ahead and 14-day-ahead forecasts.

-
# Restrict to common period for just models 1 and 2, then calculate the scaled 
+
# Restrict to common period for just models 1 and 2, then calculate the scaled
 # errors, that is, the error relative to the strawman's error
 res_all2 = res %>%
   select(-c(err3, err4)) %>%
@@ -651,23 +655,23 @@ 
Results: First Two Models

   mutate(dif12 = err1 - err2) %>%                     # Compute differences
                                                       # relative to cases model
   ungroup() %>%
-  select(-err0) 
-         
-# Calculate and print median errors, for just models 1 and 2, and both 7 and 14 
+  select(-err0)
+
+# Calculate and print median errors, for just models 1 and 2, and both 7 and 14
 # days ahead
-res_err2 = res_all2 %>% 
+res_err2 = res_all2 %>%
   select(-starts_with("dif")) %>%
   pivot_longer(names_to = "model", values_to = "err",
                cols = -c(geo_value, time_value, lead)) %>%
   mutate(lead = factor(lead, labels = paste(leads, "days ahead")),
          model = factor(model, labels = model_names[1:2]))
-  
-knitr::kable(res_err2 %>% 
+
+knitr::kable(res_err2 %>%
                select(-starts_with("dif")) %>%
                group_by(model, lead) %>%
-               summarize(err = median(err), n = length(unique(time_value))) %>% 
+               summarize(err = median(err), n = length(unique(time_value))) %>%
                arrange(lead) %>% ungroup() %>%
-               rename("Model" = model, "Median scaled error" = err, 
+               rename("Model" = model, "Median scaled error" = err,
                       "Target" = lead, "Test days" = n),
              caption = paste("Test period:", min(res_err2$time_value), "to",
                              max(res_err2$time_value)),
@@ -769,14 +773,14 @@ 
Results: First Two Models

 becomes “hard” and the scaled errors shoot above 1.

 
# Plot median errors as a function of time, for models 1 and 2, and both 7 and
 # 14 days ahead
-ggplot(res_err2 %>% 
+ggplot(res_err2 %>%
          group_by(model, lead, time_value) %>%
          summarize(err = median(err)) %>% ungroup(),
-       aes(x = time_value, y = err)) + 
-  geom_line(aes(color = model)) + 
+       aes(x = time_value, y = err)) +
+  geom_line(aes(color = model)) +
   scale_color_manual(values = c("black", ggplot_colors)) +
   geom_hline(yintercept = 1, linetype = 2, color = "gray") +
-  facet_wrap(vars(lead)) + 
+  facet_wrap(vars(lead)) +
   labs(x = "Date", y = "Median scaled error") +
   theme_bw() + theme(legend.pos = "bottom", legend.title = element_blank())

 

@@ -788,7 +792,7 @@ 
Results: First Two Models

 Given the large sample size: 112 test days for 7-day-ahead forecasts
 and 98 test days for 14-day-ahead forecasts
 (times 440 counties for each day), the p-values are basically zero.

-
# Compute p-values using the sign test against a one-sided alternative, just 
+
# Compute p-values using the sign test against a one-sided alternative, just
 # for models 1 and 2, and both 7 and 14 days ahead
 res_dif2 = res_all2 %>%
   select(-starts_with("err")) %>%
@@ -797,12 +801,12 @@ 
Results: First Two Models

   mutate(lead = factor(lead, labels = paste(leads, "days ahead")),
          model = factor(model, labels = "Cases > Cases + Facebook"))
 
-knitr::kable(res_dif2 %>% 
+knitr::kable(res_dif2 %>%
                group_by(model, lead) %>%
                summarize(p = binom.test(x = sum(dif > 0, na.rm = TRUE),
-                                        n = n(), alt = "greater")$p.val) %>% 
-               ungroup() %>% 
-               rename("Comparison" = model, "Target" = lead, "P-value" = p), 
+                                        n = n(), alt = "greater")$p.val) %>%
+               ungroup() %>%
+               rename("Comparison" = model, "Target" = lead, "P-value" = p),
              format = "html", table.attr = "style='width:50%;'")
@@ -846,15 +850,15 @@

Results: First Two Models

Once we stratify and recompute p-values by forecast date, as shown in the histograms below, the bulk of p-values are still quite small.

-
ggplot(res_dif2 %>% 
+
ggplot(res_dif2 %>%
          group_by(model, lead, time_value) %>%
-         summarize(p = binom.test(x = sum(dif > 0, na.rm = TRUE), 
+         summarize(p = binom.test(x = sum(dif > 0, na.rm = TRUE),
                                   n = n(), alt = "greater")$p.val) %>%
          ungroup(), aes(p)) +
-  geom_histogram(aes(color = model, fill = model), alpha = 0.4) + 
+  geom_histogram(aes(color = model, fill = model), alpha = 0.4) +
   scale_color_manual(values = ggplot_colors) +
   scale_fill_manual(values = ggplot_colors) +
-  facet_wrap(vars(lead, model)) + 
+  facet_wrap(vars(lead, model)) +
   labs(x = "P-value", y = "Count") +
   theme_bw() + theme(legend.pos = "none")

 

@@ -912,14 +916,14 @@ 
Varying the Number of Days Ahead*

                cols = -c(geo_value, time_value, lead)) %>%
   mutate(model = factor(model, labels = model_names[1:2])) %>%
   group_by(model, lead) %>%
-  summarize(err = median(err)) %>% 
+  summarize(err = median(err)) %>%
   ungroup()
 
-ggplot(err_by_lead, aes(x = lead, y = err)) + 
-  geom_line(aes(color = model)) + 
-  geom_point(aes(color = model)) + 
+ggplot(err_by_lead, aes(x = lead, y = err)) +
+  geom_line(aes(color = model)) +
+  geom_point(aes(color = model)) +
   scale_color_manual(values = c("black", ggplot_colors)) +
-  geom_hline(yintercept = err_by_lead %>% 
+  geom_hline(yintercept = err_by_lead %>%
                filter(lead %in% 7, model == "Cases") %>% pull(err),
              linetype = 2, color = "gray") +
   labs(title = "Forecasting errors by number of days ahead",
diff --git a/content/blog/2020-10-06-survey-wave-4.Rmd b/content/blog/2020-10-06-survey-wave-4.Rmd
index 21b47837a..7df7f446a 100644
--- a/content/blog/2020-10-06-survey-wave-4.Rmd
+++ b/content/blog/2020-10-06-survey-wave-4.Rmd
@@ -1,8 +1,11 @@
 ---
-title: "New and Improved COVID Symptom Survey Tracks Testing and Mask-Wearing"
-author: "Alex Reinhart"
+title: New and Improved COVID Symptom Survey Tracks Testing and Mask-Wearing
+author: Alex Reinhart
 date: 2020-10-12
-tags: ["symptom surveys", "COVIDcast", "R"]
+tags:
+  - symptom surveys
+  - COVIDcast
+  - R
 summary: |
   Beginning on September 8, 2020, we deployed a new version of our symptom survey. 
   Facebook helps us recruit tens of thousands of respondents daily, and the new survey gives us unprecedented insights into the effects of COVID-19 across the United States. 
diff --git a/content/blog/2020-10-06-survey-wave-4.html b/content/blog/2020-10-06-survey-wave-4.html
index 36f5fdeeb..328895267 100644
--- a/content/blog/2020-10-06-survey-wave-4.html
+++ b/content/blog/2020-10-06-survey-wave-4.html
@@ -1,14 +1,17 @@
 ---
-title: "New and Improved COVID Symptom Survey Tracks Testing and Mask-Wearing"
-author: "Alex Reinhart"
+title: New and Improved COVID Symptom Survey Tracks Testing and Mask-Wearing
+author: Alex Reinhart
 date: 2020-10-12
-tags: ["symptom surveys", "COVIDcast", "R"]
+tags:
+  - symptom surveys
+  - COVIDcast
+  - R
 summary: |
   Beginning on September 8, 2020, we deployed a new version of our symptom survey. 
   Facebook helps us recruit tens of thousands of respondents daily, and the new survey gives us unprecedented insights into the effects of COVID-19 across the United States. 
   Today we release new public datasets and share maps revealing access to COVID testing, test results, and public use of masks.
 authors:
-- alex
+  - alex
 heroImage: /blog/images/blog-lg-img_New and Improved COVID.png
 heroImageThumb: /blog/images/blog-thumb-img_New and Improved COVID.png
 acknowledgments: |
diff --git a/content/blog/2020-10-14-dv-signal.Rmd b/content/blog/2020-10-14-dv-signal.Rmd
index 7cca827d4..a874d9371 100644
--- a/content/blog/2020-10-14-dv-signal.Rmd
+++ b/content/blog/2020-10-14-dv-signal.Rmd
@@ -1,8 +1,11 @@
 ---
-title: "A Syndromic COVID-19 Indicator Based on Insurance Claims of Outpatient Visits"
-author: "Aaron Rumack and Roni Rosenfeld"
+title: A Syndromic COVID-19 Indicator Based on Insurance Claims of Outpatient Visits
+author: Aaron Rumack and Roni Rosenfeld
 date: 2020-11-05
-tags: ["medical records", "COVIDcast", "R"]
+tags:
+  - medical records
+  - COVIDcast
+  - R
 authors:
   - aaron
   - roni
diff --git a/content/blog/2020-10-14-dv-signal.html b/content/blog/2020-10-14-dv-signal.html
index c2c6f77c4..dccdf3a56 100644
--- a/content/blog/2020-10-14-dv-signal.html
+++ b/content/blog/2020-10-14-dv-signal.html
@@ -1,19 +1,22 @@
 ---
-title: "A Syndromic COVID-19 Indicator Based on Insurance Claims of Outpatient Visits"
-author: "Aaron Rumack and Roni Rosenfeld"
+title: A Syndromic COVID-19 Indicator Based on Insurance Claims of Outpatient Visits
+author: Aaron Rumack and Roni Rosenfeld
 date: 2020-11-05
-tags: ["medical records", "COVIDcast", "R"]
+tags:
+  - medical records
+  - COVIDcast
+  - R
 authors:
-- aaron
-- roni
+  - aaron
+  - roni
 heroImage: /blog/images/blog-img_A Syndromic COVID-19.png
 heroImageThumb: /blog/images/blog-thumb-img_A Syndromic COVID-19.png
 related:
-- 2020-09-18-google-survey
-- 2020-08-26-fb-survey
+  - 2020-09-18-google-survey
+  - 2020-08-26-fb-survey
 summary: |
   In previous posts, we discussed our massive ongoing symptom surveys that have reached over 12 million people in the U.S. since April 2020, in partnership with Facebook and Google. Another one of our major data initiatives is based on partnerships with healthcare systems, granting us access to various aggregate statistics from hospital records and insurance claims covering 10-15% of the United States population. From these data, we can extract informative indicators that can be early indicators of COVID activity. This post focuses on one indicator in particular, based on outpatient visits, and demonstrates both the challenges and promises associated with medical records data.
-acknowledgements: | 
+acknowledgements: |
   Maria Jahja contributed immensely to every stage of this project, from determining which ICD codes to use to the final implementation of the indicator. 
   Aaron Rumack devised the weekday adjustment and analyzed the performance of the DV indicator. 
   Roni Rosenfeld worked closely with our health systems partners to get access to the data and provided domain knowledge to ensure that the data was useful. Both Roni and Ryan Tibshirani provided helpful suggestions and insights towards the methodology and analysis.

From 5a4dcdd495cdf7ec9a59e03d34410b73e1e2ad09 Mon Sep 17 00:00:00 2001
From: Samuel Gratzl 
Date: Tue, 24 Nov 2020 10:51:32 -0500
Subject: [PATCH 06/12] build: add vscode settings

---
 .gitignore              | 1 -
 .vscode/extensions.json | 3 +++
 .vscode/settings.json   | 7 +++++++
 3 files changed, 10 insertions(+), 1 deletion(-)
 create mode 100644 .vscode/extensions.json
 create mode 100644 .vscode/settings.json

diff --git a/.gitignore b/.gitignore
index b35019050..c9bd7f824 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,6 @@
 /public
 /resources/_gen
 *.exe
-/.vscode
 /blogdown
 /.Rhistory
 *_cache
diff --git a/.vscode/extensions.json b/.vscode/extensions.json
new file mode 100644
index 000000000..c83e26348
--- /dev/null
+++ b/.vscode/extensions.json
@@ -0,0 +1,3 @@
+{
+  "recommendations": ["esbenp.prettier-vscode"]
+}
diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 000000000..d23056767
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,7 @@
+{
+  "editor.formatOnSave": true,
+  "editor.trimAutoWhitespace": false,
+  "editor.formatOnPaste": true,
+  "editor.formatOnSaveMode": "modifications",
+  "editor.defaultFormatter": "esbenp.prettier-vscode"
+}

From 5ca23201582cd10897df57f77532b88524ada3b8 Mon Sep 17 00:00:00 2001
From: Samuel Gratzl 
Date: Tue, 24 Nov 2020 11:05:05 -0500
Subject: [PATCH 07/12] build: ignore whole directory

---
 .prettierignore | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.prettierignore b/.prettierignore
index 2752b96de..b04bad623 100644
--- a/.prettierignore
+++ b/.prettierignore
@@ -14,12 +14,14 @@
 /LICENSE
 /package-lock.json
 /content/blog/**/*.html
-/static/blog/**/*
+/static/blog
+/static/images
 /public
 /Dockerfile
 /resources
 /static/covidcast
 /static/rmarkdown-libs
+/assets
 *.woff
 *.ttf
 *.woff2

From 7b80ea6f688c48e2767f25180821e7bb37850c9a Mon Sep 17 00:00:00 2001
From: Samuel Gratzl 
Date: Tue, 24 Nov 2020 11:14:20 -0500
Subject: [PATCH 08/12] build: ignore full static dir

---
 .prettierignore | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.prettierignore b/.prettierignore
index b04bad623..3e1c6e1a5 100644
--- a/.prettierignore
+++ b/.prettierignore
@@ -14,13 +14,10 @@
 /LICENSE
 /package-lock.json
 /content/blog/**/*.html
-/static/blog
-/static/images
+/static
 /public
 /Dockerfile
 /resources
-/static/covidcast
-/static/rmarkdown-libs
 /assets
 *.woff
 *.ttf

From 2a52704dc13b0f71dd0dd1e749581d26bad9b7df Mon Sep 17 00:00:00 2001
From: Samuel Gratzl 
Date: Tue, 24 Nov 2020 11:16:25 -0500
Subject: [PATCH 09/12] docs: add prettier info to README

---
 README.md | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 545db57ef..71a21e7bd 100644
--- a/README.md
+++ b/README.md
@@ -14,7 +14,8 @@ This site is based on [Hugo](https://gohugo.io).
 #### Commands
 
 1. run `npm start` to create a development Hugo server
-1. run `npm build` to build a minified build in `/public`
+1. run `npm run format` to run prettier and format files
+1. run `npm run build` to build a minified build in `/public`
 
 ### Blog Editor
 
@@ -38,9 +39,11 @@ In order to convert the Rmd files to HTML files for Hugo you also need to:
    - `local=TRUE` similar to `-D` to process draft files
    - `run_hugo=FALSE` to manually run hugo
    - `build_rmd=TRUE` force a (re)build of the Rmd pages
+1. Alternatively, run `npm run build:blog`
 1. Run Hugo server as usual
 
-blogdown also has an integrated server `blogdown::serve_site()` which will render RMarkdown files on the fly and does a similar thing as `hugo server -D`
+blogdown also has an integrated server `blogdown::serve_site()` which will render RMarkdown files on the fly and does a similar thing as `hugo server -D`. 
+A shortcut is available through `npm run start:blog`.
 
 #### Adding a new blog post
 

From 97a22ce2deb762d600e05321dd3fd504e3bf858a Mon Sep 17 00:00:00 2001
From: Samuel Gratzl 
Date: Tue, 24 Nov 2020 11:34:06 -0500
Subject: [PATCH 10/12] build: try to ignore static directory

---
 README.md    | 2 +-
 package.json | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 71a21e7bd..b6a7d4f56 100644
--- a/README.md
+++ b/README.md
@@ -42,7 +42,7 @@ In order to convert the Rmd files to HTML files for Hugo you also need to:
 1. Alternatively, run `npm run build:blog`
 1. Run Hugo server as usual
 
-blogdown also has an integrated server `blogdown::serve_site()` which will render RMarkdown files on the fly and does a similar thing as `hugo server -D`. 
+blogdown also has an integrated server `blogdown::serve_site()` which will render RMarkdown files on the fly and does a similar thing as `hugo server -D`.
 A shortcut is available through `npm run start:blog`.
 
 #### Adding a new blog post
diff --git a/package.json b/package.json
index 17bf70f2a..f1672bbb9 100644
--- a/package.json
+++ b/package.json
@@ -33,7 +33,7 @@
     "build": "hugo --gc --minify",
     "start": "hugo server -D",
     "start:blog": "Rscript -e \"blogdown::serve_site()\"",
-    "format": "prettier **/* --write",
-    "lint": "prettier **/* --check"
+    "format": "prettier *.* \"(.vscode|content|data|themes)/**\" --write",
+    "lint": "prettier *.* \"(.vscode|content|data|themes)/**\" --check"
   }
 }

From 1a27ac285ea361757c0d8c1fe33acd11fc2c55f8 Mon Sep 17 00:00:00 2001
From: Samuel Gratzl 
Date: Mon, 30 Nov 2020 08:51:51 -0500
Subject: [PATCH 11/12] refactor: npm run format

---
 content/about/_index.md                       |   6 +-
 content/about/publications/index.md           |  96 +++++++-------
 content/covidcast/methodology.md              |   2 +-
 data/supporter.yaml                           |   1 -
 themes/delphi/assets/css/layout/_content.scss |   7 +-
 .../assets/css/layout/_header_footer.scss     |   8 +-
 themes/delphi/assets/css/pages/_about.scss    |  59 +++++----
 themes/delphi/assets/css/pages/_blog.scss     |  78 +++++------
 themes/delphi/assets/css/pages/_landing.scss  |  53 ++++----
 themes/delphi/assets/css/pages/_team.scss     |  20 +--
 themes/delphi/assets/js/blog/imageModal.js    |  44 ++++---
 themes/delphi/assets/js/blog/index.js         |  12 +-
 themes/delphi/assets/js/blog/toc.js           |  62 ++++-----
 themes/delphi/layouts/_default/about.html     |  17 +--
 themes/delphi/layouts/_default/baseof.html    |  17 ++-
 themes/delphi/layouts/_default/single.html    |   2 +-
 themes/delphi/layouts/_default/taxonomy.html  |  44 +++----
 themes/delphi/layouts/_default/team.html      |   9 +-
 themes/delphi/layouts/_default/terms.html     |  14 +-
 themes/delphi/layouts/blog/list.html          |  52 ++++----
 themes/delphi/layouts/blog/single.html        |  60 +++++----
 .../delphi/layouts/covidcast_app/baseof.html  |  15 +--
 themes/delphi/layouts/landing.html            | 121 ++++++++++--------
 .../layouts/partials/about/collaborators.html |  37 +++---
 .../delphi/layouts/partials/arrow-link.html   |   6 +-
 themes/delphi/layouts/partials/blog/card.html |  31 +++--
 .../delphi/layouts/partials/blog/license.html |  10 +-
 themes/delphi/layouts/partials/footer.html    |   2 +-
 .../layouts/partials/footer/desktop.html      |  66 +++++-----
 .../layouts/partials/landing/latest-card.html |  20 +--
 .../layouts/partials/landing/latest-news.html |   6 +-
 .../layouts/partials/menu/breadcrumb.html     |  21 +--
 themes/delphi/layouts/partials/nav.html       |  10 +-
 themes/delphi/layouts/partials/scripts.html   |   6 +-
 themes/delphi/layouts/shortcodes/apiref.html  |   2 +-
 .../layouts/shortcodes/research-papers.html   |  30 ++---
 36 files changed, 537 insertions(+), 509 deletions(-)

diff --git a/content/about/_index.md b/content/about/_index.md
index 2d7101d9e..92df55651 100644
--- a/content/about/_index.md
+++ b/content/about/_index.md
@@ -23,10 +23,10 @@ Public health authorities (federal, state, local), fellow researchers (working o
 
 - Since March 2020, we've created and maintained the [nation's largest public repository of diverse, geographically-detailed, real-time indicators of COVID-19 activity]({{< relref "covidcast" >}}) in the U.S. Our indicators cover every rung of the [severity pyramid](https://docs.google.com/presentation/d/1jvIycxDRMEIozKIowv2UyvSqZyF5y6jR8EAXUEK22D4/edit?usp=sharing), and they're freely available through a [public API]({{< apiref "api/covidcast.html" >}}).
 
-- Several of the underlying data sources (on which these indicators are built) would not exist or be publicly available without our efforts. This includes: 
+- Several of the underlying data sources (on which these indicators are built) would not exist or be publicly available without our efforts. This includes:
 
-    * A massive [national daily survey]({{< relref "surveys">}}) we're running in [partnership with Facebook](https://covid-survey.dataforgood.fb.com/survey_and_map_data.html). This has reached over 12 million Americans since April, providing real-time insights into, e.g., [self-reported symptoms]({{< relref "2020-08-26-fb-survey#whats-in-the-survey" >}}), [mask wearing]({{< relref "2020-10-06-survey-wave-4.html#mask-wearing" >}}), [testing]({{< relref "2020-10-06-survey-wave-4#testing" >}}), and contacts, all broken down by various demographics.
+  - A massive [national daily survey]({{< relref "surveys">}}) we're running in [partnership with Facebook](https://covid-survey.dataforgood.fb.com/survey_and_map_data.html). This has reached over 12 million Americans since April, providing real-time insights into, e.g., [self-reported symptoms]({{< relref "2020-08-26-fb-survey#whats-in-the-survey" >}}), [mask wearing]({{< relref "2020-10-06-survey-wave-4.html#mask-wearing" >}}), [testing]({{< relref "2020-10-06-survey-wave-4#testing" >}}), and contacts, all broken down by various demographics.
 
-    * An enormous database of de-identified medical insurance claims, covering more than half the US population, made possible through health system partners including Change Healthcare. We use this to produce a new [syndromic COVID-19 indicator based on doctor visits]({{< relref "2020-10-14-dv-signal" >}}), and other indicators based on hospitalizations and ICU admissions.
+  - An enormous database of de-identified medical insurance claims, covering more than half the US population, made possible through health system partners including Change Healthcare. We use this to produce a new [syndromic COVID-19 indicator based on doctor visits]({{< relref "2020-10-14-dv-signal" >}}), and other indicators based on hospitalizations and ICU admissions.
 
 - Since April 2020, we've been supporting and advising the CDC in their community-driven COVID-19 forecasting effort, which includes [creating and evaluating an ensemble forecast]({{< relref "2020-09-21-forecast-demo" >}}) from the 70+ forecasts under submission that serves as the basis for the [CDC's official COVID-19 forecast communications](https://www.cdc.gov/coronavirus/2019-ncov/covid-data/forecasting-us.html). We also contribute our own short-term forecasts of COVID-19 cases and deaths, which can be found in the [COVID-19 Forecast Hub](https://covid19forecasthub.org).
diff --git a/content/about/publications/index.md b/content/about/publications/index.md
index 1c4fdcac4..2ba78888b 100644
--- a/content/about/publications/index.md
+++ b/content/about/publications/index.md
@@ -1,54 +1,54 @@
 ---
 title: Research and White Papers
 papers:
-- title: "Pancasting: forecasting epidemics from provisional data"
-  image: pancasting.png
-  authors: Brooks
-  link: https://delphi.cmu.edu/~lcbrooks/brooks2020pancasting.pdf
-  journal: PhD thesis
-  year: 2020
-- title: "Kalman filter, sensor fusion, and constrained regression: equivalences and insights"
-  image: kalman_filter.png
-  authors: Jahja, Farrow, Rosenfeld, Tibshirani
-  link: https://papers.nips.cc/paper/9475-kalman-filter-sensor-fusion-and-constrained-regression-equivalences-and-insights
-  journal: Neural Information Processing Systems
-  year: 2019
-- title: "Nonmechanistic forecasts of seasonal influenza with iterative one-week-ahead distributions"
-  image: nonmechanistic_forecasts.png
-  authors: Brooks, Farrow, Hyun, Tibshirani, Rosenfeld
-  link: https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1006134
-  journal: PLOS Computational Biology
-  year: 2018
-- title: "A human judgment approach to epidemiological forecasting"
-  image: human.png
-  authors: Farrow, Brooks, Hyun, Tibshirani, Burke, Rosenfeld
-  link: https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1005248
-  journal: PLOS Computational Biology
-  year: 2017
-- title: "Modeling the past, present, and future of influenza"
-  image: modeling.png
-  authors: Farrow
-  link: https://delphi.cmu.edu/~dfarrow/thesis.pdf
-  journal: PhD thesis
-  year: 2016
-- title: "Flexible modeling of epidemics with an empirical Bayes framework"
-  image: flexible_modeling.png
-  authors: Brooks, Farrow, Hyun, Tibshirani, Rosenfeld
-  link: https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1004382
-  journal: PLOS Computational Biology
-  year: 2015
-- title: "Predicting the predictable"
-  image: predicting.png
-  authors: Rosenfeld
-  link: https://delphi.cmu.edu/files/PredictingThePredictable_13-04-03.pdf
-  journal: presentation
-  year: 2013
-- title: "A proposal for standardized evaluation of epidemiological models"
-  image: evaluation.png
-  authors: Rosenfeld, Grefenstette, Burke
-  link: http://www.cs.cmu.edu/~roni/standardized-evaluation-of-epi-models-rev-09nov2012.pdf
-  journal: White paper
-  year: 2012
+  - title: "Pancasting: forecasting epidemics from provisional data"
+    image: pancasting.png
+    authors: Brooks
+    link: https://delphi.cmu.edu/~lcbrooks/brooks2020pancasting.pdf
+    journal: PhD thesis
+    year: 2020
+  - title: "Kalman filter, sensor fusion, and constrained regression: equivalences and insights"
+    image: kalman_filter.png
+    authors: Jahja, Farrow, Rosenfeld, Tibshirani
+    link: https://papers.nips.cc/paper/9475-kalman-filter-sensor-fusion-and-constrained-regression-equivalences-and-insights
+    journal: Neural Information Processing Systems
+    year: 2019
+  - title: "Nonmechanistic forecasts of seasonal influenza with iterative one-week-ahead distributions"
+    image: nonmechanistic_forecasts.png
+    authors: Brooks, Farrow, Hyun, Tibshirani, Rosenfeld
+    link: https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1006134
+    journal: PLOS Computational Biology
+    year: 2018
+  - title: "A human judgment approach to epidemiological forecasting"
+    image: human.png
+    authors: Farrow, Brooks, Hyun, Tibshirani, Burke, Rosenfeld
+    link: https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1005248
+    journal: PLOS Computational Biology
+    year: 2017
+  - title: "Modeling the past, present, and future of influenza"
+    image: modeling.png
+    authors: Farrow
+    link: https://delphi.cmu.edu/~dfarrow/thesis.pdf
+    journal: PhD thesis
+    year: 2016
+  - title: "Flexible modeling of epidemics with an empirical Bayes framework"
+    image: flexible_modeling.png
+    authors: Brooks, Farrow, Hyun, Tibshirani, Rosenfeld
+    link: https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1004382
+    journal: PLOS Computational Biology
+    year: 2015
+  - title: "Predicting the predictable"
+    image: predicting.png
+    authors: Rosenfeld
+    link: https://delphi.cmu.edu/files/PredictingThePredictable_13-04-03.pdf
+    journal: presentation
+    year: 2013
+  - title: "A proposal for standardized evaluation of epidemiological models"
+    image: evaluation.png
+    authors: Rosenfeld, Grefenstette, Burke
+    link: http://www.cs.cmu.edu/~roni/standardized-evaluation-of-epi-models-rev-09nov2012.pdf
+    journal: White paper
+    year: 2012
 ---
 
 {{}}
diff --git a/content/covidcast/methodology.md b/content/covidcast/methodology.md
index f36cb7fd1..85d9138df 100644
--- a/content/covidcast/methodology.md
+++ b/content/covidcast/methodology.md
@@ -34,4 +34,4 @@ Generally, we do not report estimates at locations with insufficient data (or in
 
 ### Intensity Heat map
 
-The “Intensity” view presents a heat map of these estimates.  For each indicator, we use a fixed range of values, from a “low” value to a “high” value, and assign a color to each value in between, as shown to the left of the map.  These “low” and “high” values are different for each indicator, but for a given indicator, they are constant across time and geographic hierarchy, meaning that the heat maps are comparable across days.  At the county level, the “rest of state” estimates are plotted in semi-transparent colors, to make the individual counties where estimates are made more easily visually distinguishable.
+The “Intensity” view presents a heat map of these estimates. For each indicator, we use a fixed range of values, from a “low” value to a “high” value, and assign a color to each value in between, as shown to the left of the map. These “low” and “high” values are different for each indicator, but for a given indicator, they are constant across time and geographic hierarchy, meaning that the heat maps are comparable across days. At the county level, the “rest of state” estimates are plotted in semi-transparent colors, to make the individual counties where estimates are made more easily visually distinguishable.
diff --git a/data/supporter.yaml b/data/supporter.yaml
index 00d5a57bf..4a57dc485 100644
--- a/data/supporter.yaml
+++ b/data/supporter.yaml
@@ -14,4 +14,3 @@
   group: collaborator
 - name: Google.org
   group: sponsor
-
diff --git a/themes/delphi/assets/css/layout/_content.scss b/themes/delphi/assets/css/layout/_content.scss
index 3c860bb56..0c197380b 100644
--- a/themes/delphi/assets/css/layout/_content.scss
+++ b/themes/delphi/assets/css/layout/_content.scss
@@ -1,14 +1,13 @@
 // Site-wide
 .uk-container {
-    margin-top: 20px;
-    margin-bottom: 20px;
+  margin-top: 20px;
+  margin-bottom: 20px;
 }
 
-
 .uk-text-bold-600 {
   font-weight: 600;
 }
 
 .uk-container [aria-hidden="true"] {
-    display: none;
+  display: none;
 }
diff --git a/themes/delphi/assets/css/layout/_header_footer.scss b/themes/delphi/assets/css/layout/_header_footer.scss
index aadfee1fb..fdb558f50 100644
--- a/themes/delphi/assets/css/layout/_header_footer.scss
+++ b/themes/delphi/assets/css/layout/_header_footer.scss
@@ -17,13 +17,13 @@
   border-bottom: 2px solid #f03f3f !important;
 }
 .uk-navbar-container {
-    box-shadow: 0 3px 5px -1px rgba(0,0,0,.15);
-    display: block;
+  box-shadow: 0 3px 5px -1px rgba(0, 0, 0, 0.15);
+  display: block;
 }
 
 .uk-navbar-left {
-    margin-top: 40px;
-    margin-bottom: 0;
+  margin-top: 40px;
+  margin-bottom: 0;
 }
 
 .uk-navbar-nav > li {
diff --git a/themes/delphi/assets/css/pages/_about.scss b/themes/delphi/assets/css/pages/_about.scss
index 441a11cbd..44371bbc6 100644
--- a/themes/delphi/assets/css/pages/_about.scss
+++ b/themes/delphi/assets/css/pages/_about.scss
@@ -1,47 +1,46 @@
 .about-mission {
-    background-image: url("../images/red-bubble-map_mission-pg-short.jpg");
-    background-size: auto;
-    height: 200px;
+  background-image: url("../images/red-bubble-map_mission-pg-short.jpg");
+  background-size: auto;
+  height: 200px;
 }
 
 .mission-text {
-    font-weight: 600;
-    font-size: 24px;
-    line-height: 36px;
+  font-weight: 600;
+  font-size: 24px;
+  line-height: 36px;
 }
 
-@media screen and (max-width: $breakpoint-medium) { 
-    .mission-text {
-        font-size: 18px;
-        line-height: 28px;
-    }
-    .about-mission {
-        height: 50px;
-    }
-    .about-description {
-        margin-top: 0px;
-    }
+@media screen and (max-width: $breakpoint-medium) {
+  .mission-text {
+    font-size: 18px;
+    line-height: 28px;
+  }
+  .about-mission {
+    height: 50px;
+  }
+  .about-description {
+    margin-top: 0px;
+  }
 }
 @media screen and (min-width: $breakpoint-medium) {
-    .about-description {
-        background: white;
-        border: 1px solid #D3D4D8;
-        border-radius: 5px;
-        margin-top: -6rem;
-    }
+  .about-description {
+    background: white;
+    border: 1px solid #d3d4d8;
+    border-radius: 5px;
+    margin-top: -6rem;
+  }
 }
 
 .about-description {
-    padding: 2em;
-    margin-bottom: 3rem;
+  padding: 2em;
+  margin-bottom: 3rem;
 
-    h3 {
-        letter-spacing: 3px;
-        text-transform: uppercase;
-    }
+  h3 {
+    letter-spacing: 3px;
+    text-transform: uppercase;
+  }
 }
 
-
 .about-collaborators {
   background: #fafafc;
   padding: 0;
diff --git a/themes/delphi/assets/css/pages/_blog.scss b/themes/delphi/assets/css/pages/_blog.scss
index 0ff3ec54e..a31f0f6ec 100644
--- a/themes/delphi/assets/css/pages/_blog.scss
+++ b/themes/delphi/assets/css/pages/_blog.scss
@@ -61,69 +61,69 @@ $blog-list-date-width: 5em;
 }
 
 .blog-blog {
-    // center align auto generated images
-    p > img:first-of-type {
-        display: block;
-        margin: 0 auto;
-    }
+  // center align auto generated images
+  p > img:first-of-type {
+    display: block;
+    margin: 0 auto;
+  }
 }
 
 .blog-image-wrapper {
-    position: relative;
+  position: relative;
 }
 
 .blog-image-button {
-    position: absolute;
-    right: 0;
-    top: -14px;
-    width: 28px;
-    height: 28px;
-    cursor: pointer;
+  position: absolute;
+  right: 0;
+  top: -14px;
+  width: 28px;
+  height: 28px;
+  cursor: pointer;
 }
 
 .blog-image-modal {
-    height: 100%;
-    background: rgba(0,0,0,0.75);
+  height: 100%;
+  background: rgba(0, 0, 0, 0.75);
 
-    > img {
-        width: 100%;
-        height: 100%;
-    }
+  > img {
+    width: 100%;
+    height: 100%;
+  }
 }
 
 #TOC {
-    // hide original toc
-    display: none;
+  // hide original toc
+  display: none;
 }
 
 .blog-toc-container {
-    margin-top: 2em;
+  margin-top: 2em;
 
-    h5 {
-        letter-spacing: 3px;
-        text-transform: uppercase;
-    }
+  h5 {
+    letter-spacing: 3px;
+    text-transform: uppercase;
+  }
 }
 
 .blog-toc {
-    background: #F5F5F5;
-    border-radius: 5px;
-    padding: 24px;
+  background: #f5f5f5;
+  border-radius: 5px;
+  padding: 24px;
 
-    li {
-        list-style: decimal;
-    }
+  li {
+    list-style: decimal;
+  }
 
-    .uk-nav-default > li.uk-active > a {
-        color: unset;
-        font-weight: 600;
-    }
+  .uk-nav-default > li.uk-active > a {
+    color: unset;
+    font-weight: 600;
+  }
 }
 
 .blog-license {
-    margin-top: 2em;
+  margin-top: 2em;
 
-    .inline-svg-icon {
-        padding: 0;
-    }
+  .inline-svg-icon {
+    padding: 0;
+  }
 }
diff --git a/themes/delphi/assets/css/pages/_landing.scss b/themes/delphi/assets/css/pages/_landing.scss
index 9b843a171..29cb986ab 100644
--- a/themes/delphi/assets/css/pages/_landing.scss
+++ b/themes/delphi/assets/css/pages/_landing.scss
@@ -60,33 +60,33 @@ $carousel-height: 500px;
   background-color: #eee;
 }
 .landing-entry {
+  line-height: 48px;
+  .inline-svg-icon {
+    font-size: x-large;
+  }
+  h2 {
+    font-weight: 600;
+    font-size: 14px;
+    letter-spacing: 3px;
+    text-transform: uppercase;
+  }
+  p {
+    font-weight: 500;
+    font-size: 28px;
     line-height: 48px;
-    .inline-svg-icon{
-        font-size:x-large;
-    }
-    h2 {
-        font-weight: 600;
-        font-size: 14px;
-        letter-spacing: 3px;
-        text-transform: uppercase;
-    }
-    p {
-        font-weight: 500;
-        font-size: 28px;
-        line-height: 48px;
-        color: rgb(35, 39, 53);
-    }
+    color: rgb(35, 39, 53);
+  }
 
-    @media screen and (max-width: $breakpoint-small) { 
-        p {
-         font-size: 18px;
-         line-height: 32px;
-        }
-       }
-    a {
-        color: $global-color;
-        font-weight: bold;
+  @media screen and (max-width: $breakpoint-small) {
+    p {
+      font-size: 18px;
+      line-height: 32px;
     }
+  }
+  a {
+    color: $global-color;
+    font-weight: bold;
+  }
 }
 
 .entry-image {
@@ -97,7 +97,6 @@ $carousel-height: 500px;
   }
 }
 
-
 .latest-news {
-    padding: 0 50px;
-}
\ No newline at end of file
+  padding: 0 50px;
+}
diff --git a/themes/delphi/assets/css/pages/_team.scss b/themes/delphi/assets/css/pages/_team.scss
index 4006e090b..6da2f87ee 100644
--- a/themes/delphi/assets/css/pages/_team.scss
+++ b/themes/delphi/assets/css/pages/_team.scss
@@ -7,15 +7,15 @@
 }
 
 .team-header {
-    line-height: 64px;
-    font-size: 40px;
-    padding-top: .25em;
-    padding-bottom: 1em;
+  line-height: 64px;
+  font-size: 40px;
+  padding-top: 0.25em;
+  padding-bottom: 1em;
 }
 
-@media screen and (max-width: $breakpoint-small) { 
-    .team-header {
-        font-size: 18px;
-        line-height: 24px;
-    }
-   }
\ No newline at end of file
+@media screen and (max-width: $breakpoint-small) {
+  .team-header {
+    font-size: 18px;
+    line-height: 24px;
+  }
+}
diff --git a/themes/delphi/assets/js/blog/imageModal.js b/themes/delphi/assets/js/blog/imageModal.js
index fe6d0a769..90630e732 100644
--- a/themes/delphi/assets/js/blog/imageModal.js
+++ b/themes/delphi/assets/js/blog/imageModal.js
@@ -1,33 +1,37 @@
 function showModal(img) {
-    // based on UIkit.modal.dialog but with full size
+  // based on UIkit.modal.dialog but with full size
 
-    const dialog = UIkit.modal(`

+  const dialog = UIkit.modal(`

                 

                     
                     ${img.outerHTML}
                 

             
`);
-    dialog.show();
-    UIkit.util.on(dialog.$el, 'hidden', () => {
-        return Promise.resolve().then(() => dialog.$destroy(true));
-    }, { self: true });
+  dialog.show();
+  UIkit.util.on(
+    dialog.$el,
+    "hidden",
+    () => {
+      return Promise.resolve().then(() => dialog.$destroy(true));
+    },
+    { self: true }
+  );
 }
 
 export default function imageModal() {
-    const refButton = document.createElement('button');
-    refButton.classList.add('uk-icon-button', 'uk-button-default', 'blog-image-button');
-    refButton.title = 'Show image in fullscreen';
-    refButton.innerHTML = ``
+  const refButton = document.createElement("button");
+  refButton.classList.add("uk-icon-button", "uk-button-default", "blog-image-button");
+  refButton.title = "Show image in fullscreen";
+  refButton.innerHTML = ``;
 
-    Array.from(document.querySelectorAll('.blog-blog p > img')).forEach((elem) => {
-        const button = refButton.cloneNode(true);
-        elem.insertAdjacentElement('beforebegin', button);
-        elem.parentElement.classList.add('blog-image-wrapper');
-        button.addEventListener('click', (e) => {
-            e.preventDefault();
-            e.target.blur();
-            showModal(elem);
-        })
+  Array.from(document.querySelectorAll(".blog-blog p > img")).forEach((elem) => {
+    const button = refButton.cloneNode(true);
+    elem.insertAdjacentElement("beforebegin", button);
+    elem.parentElement.classList.add("blog-image-wrapper");
+    button.addEventListener("click", (e) => {
+      e.preventDefault();
+      e.target.blur();
+      showModal(elem);
     });
+  });
 }
-
diff --git a/themes/delphi/assets/js/blog/index.js b/themes/delphi/assets/js/blog/index.js
index 8315fd6e8..bb4c3931c 100644
--- a/themes/delphi/assets/js/blog/index.js
+++ b/themes/delphi/assets/js/blog/index.js
@@ -1,11 +1,11 @@
-import hljs from 'highlight.js';
-import renderMathInElement from 'katex/dist/contrib/auto-render.mjs';
-import { initializeCodeFolding } from './codeFolding';
-import imageModal from './imageModal';
-import initializeTableOfContent from './toc';
+import hljs from "highlight.js";
+import renderMathInElement from "katex/dist/contrib/auto-render.mjs";
+import { initializeCodeFolding } from "./codeFolding";
+import imageModal from "./imageModal";
+import initializeTableOfContent from "./toc";
 
 hljs.initHighlightingOnLoad();
 initializeCodeFolding();
-Array.from(document.querySelectorAll('.math')).forEach((elem) => renderMathInElement(elem));
+Array.from(document.querySelectorAll(".math")).forEach((elem) => renderMathInElement(elem));
 imageModal();
 initializeTableOfContent();
diff --git a/themes/delphi/assets/js/blog/toc.js b/themes/delphi/assets/js/blog/toc.js
index 5abf879e6..716c6e4de 100644
--- a/themes/delphi/assets/js/blog/toc.js
+++ b/themes/delphi/assets/js/blog/toc.js
@@ -1,37 +1,37 @@
 export default function initializeTableOfContent() {
-    const toc = document.getElementById('TOC');
-    if (!toc) {
-        return;
-    }
-    toc.remove(); // remove from old place
+  const toc = document.getElementById("TOC");
+  if (!toc) {
+    return;
+  }
+  toc.remove(); // remove from old place
 
-    const container = document.querySelector('.blog-toc-container');
-    if (!container) {
-        return;
-    }
+  const container = document.querySelector(".blog-toc-container");
+  if (!container) {
+    return;
+  }
 
-    const source = toc.querySelector('ul');
-    const target = container.querySelector('ol');
+  const source = toc.querySelector("ul");
+  const target = container.querySelector("ol");
 
-    const convert = (target, li) => {
-        target.appendChild(li);
-        const sub = li.querySelector('ul');
-        if (!sub) {
-            // done
-            return;
-        }
-        li.classList.add('uk-parent');
-        sub.remove();
-        const subTarget = document.createElement('ol');
-        subTarget.classList.add('uk-nav-sub');
-        li.appendChild(subTarget);
-        Array.from(sub.children).forEach((subLi) => convert(subTarget, subLi));
-    };
+  const convert = (target, li) => {
+    target.appendChild(li);
+    const sub = li.querySelector("ul");
+    if (!sub) {
+      // done
+      return;
+    }
+    li.classList.add("uk-parent");
+    sub.remove();
+    const subTarget = document.createElement("ol");
+    subTarget.classList.add("uk-nav-sub");
+    li.appendChild(subTarget);
+    Array.from(sub.children).forEach((subLi) => convert(subTarget, subLi));
+  };
 
-    Array.from(source.children).forEach((li) => {
-        convert(target, li);
-    });
+  Array.from(source.children).forEach((li) => {
+    convert(target, li);
+  });
 
-    // done make visible
-    container.classList.remove('uk-hidden');
-}
\ No newline at end of file
+  // done make visible
+  container.classList.remove("uk-hidden");
+}
diff --git a/themes/delphi/layouts/_default/about.html b/themes/delphi/layouts/_default/about.html
index e45dcf8eb..a828d9841 100644
--- a/themes/delphi/layouts/_default/about.html
+++ b/themes/delphi/layouts/_default/about.html
@@ -1,14 +1,11 @@
 {{ define "main" }}
-
-

-

-

+  

+  

     

-        
Delphi Research Group

-        
{{.Site.Params.mission}}

-        {{ .Content }}
+      
Delphi Research Group

+      
{{ .Site.Params.mission }}

+      {{ .Content }}
     

-

-{{partial "about/collaborators.html" .}}
-
+  

+  {{ partial "about/collaborators.html" . }}
 {{ end }}
diff --git a/themes/delphi/layouts/_default/baseof.html b/themes/delphi/layouts/_default/baseof.html
index 7610bdc74..a674e218c 100644
--- a/themes/delphi/layouts/_default/baseof.html
+++ b/themes/delphi/layouts/_default/baseof.html
@@ -1,13 +1,12 @@
 
 
 
-    {{ partial "head.html" . }}
-    
-        {{ partial "nav.html" . }}
-        {{ partial "menu/breadcrumb.html" . }}
-        {{ block "main" . }}
-        {{ end }}
-        {{ partial "footer.html" . }}
-        {{ partial "scripts.html" . }}
-    
+  {{ partial "head.html" . }}
+  
+    {{ partial "nav.html" . }}
+    {{ partial "menu/breadcrumb.html" . }}
+    {{ block "main" . }} {{ end }}
+    {{ partial "footer.html" . }}
+    {{ partial "scripts.html" . }}
+  
 
diff --git a/themes/delphi/layouts/_default/single.html b/themes/delphi/layouts/_default/single.html
index 2acf41d28..20605458d 100644
--- a/themes/delphi/layouts/_default/single.html
+++ b/themes/delphi/layouts/_default/single.html
@@ -1,5 +1,5 @@
 {{ define "main" }}
-

+  

     
{{ .Title }}

     {{ .Content }}
   

diff --git a/themes/delphi/layouts/_default/taxonomy.html b/themes/delphi/layouts/_default/taxonomy.html
index 777565be3..9fbf039c1 100644
--- a/themes/delphi/layouts/_default/taxonomy.html
+++ b/themes/delphi/layouts/_default/taxonomy.html
@@ -1,25 +1,25 @@
 {{ define "main" }}
-

-
-  
Tagged "{{ .Data.Term }}"

-  {{ range .Pages }}
-    
-    

-            

-                
-                    {{ .Title }}
-                
-            

-            

-                {{ .Summary }}
-                {{ if (and (.Site.Params.showReadMore) (.Truncated)) }}
-                
Read more...

-                {{ end }}
-            

-            

-            
-            

-        

+  

+    
+    
Tagged "{{ .Data.Term }}"

+    {{ range .Pages }}
+      
+      

+        

+          
+            {{ .Title }}
+          
+        

+        

+          {{ .Summary }}
+          {{ if (and (.Site.Params.showReadMore) (.Truncated)) }}
+            
Read more...

+          {{ end }}
+        

+        

+          
+        

+      

     {{ end }}
-

+  

 {{ end }}
diff --git a/themes/delphi/layouts/_default/team.html b/themes/delphi/layouts/_default/team.html
index d96610704..589afab80 100644
--- a/themes/delphi/layouts/_default/team.html
+++ b/themes/delphi/layouts/_default/team.html
@@ -1,6 +1,9 @@
 {{ define "main" }}
-

-
Thank you to our {{ len .Params.team }} members around the world, all the students, faculty, staff, and volunteers who have contributed to the COVIDcast project.

+  

+    

+      Thank you to our {{ len .Params.team }} members around the world, all the students, faculty, staff, and volunteers
+      who have contributed to the COVIDcast project.
+    

 
     Thank you for your contributions
         {{ .Params.others }}
       

     

-

+  

 {{ end }}
diff --git a/themes/delphi/layouts/_default/terms.html b/themes/delphi/layouts/_default/terms.html
index cbb175c20..e65b67bdd 100644
--- a/themes/delphi/layouts/_default/terms.html
+++ b/themes/delphi/layouts/_default/terms.html
@@ -1,11 +1,11 @@
 {{ define "main" }}
-
-

+  
+  

     
Tags

     
-

-{{end}}
+  

+{{ end }}
diff --git a/themes/delphi/layouts/blog/list.html b/themes/delphi/layouts/blog/list.html
index 06433dccf..d76ea371d 100644
--- a/themes/delphi/layouts/blog/list.html
+++ b/themes/delphi/layouts/blog/list.html
@@ -1,35 +1,39 @@
 {{ define "main" }}
-

-  
{{ .Title }}

-  
{{ .Description }} 

+  

+    
{{ .Title }}

+    
{{ .Description }}

 
-  

-    {{ range .Paginator.Pages }}
-    

-      
{{ .Date.Format "Jan _2" }}

-      {{ .Title}} - Hero Image
-      

-          

-              
{{.Title}}

+    

+      {{ range .Paginator.Pages }}
+        

+          
{{ .Date.Format "Jan _2" }}

+          {{ .Title }} - Hero Image
+          

+            

+              
{{ .Title }}

               

                 {{ .Summary | safeHTML | truncate 150 }}
               

               

-              {{- range .Params.tags -}}
-                {{ partial "blog/tag.html" . }}
-              {{- end -}}
+                {{- range .Params.tags -}}
+                  {{ partial "blog/tag.html" . }}
+                {{- end -}}
               

-              

-                By {{ .Params.author }}
-              

-          

-          

-              {{partial "arrow-link.html" (dict "alt" "Read More" "link" .RelPermalink)}}
+              
By {{ .Params.author }}

+            

+            

+              {{ partial "arrow-link.html" (dict "alt" "Read More" "link" .RelPermalink) }}
+            

           

-      

+        

+      {{ end }}
     

-    {{end}}
+    {{ template "_internal/pagination.html" . }}
   

-  {{ template "_internal/pagination.html" . }}
-

 {{ end }}
diff --git a/themes/delphi/layouts/blog/single.html b/themes/delphi/layouts/blog/single.html
index 2048150d9..255eb8242 100644
--- a/themes/delphi/layouts/blog/single.html
+++ b/themes/delphi/layouts/blog/single.html
@@ -1,14 +1,14 @@
 {{ define "main" }}
-

-
{{ .Title }}

-{{ partial "blog/tags.html" .}}
-{{if isset .Params "heroimage"}}
-

-    {{ .Title}} - Hero Image
-

-{{end}}
-

-    

+  

+    
{{ .Title }}

+    {{ partial "blog/tags.html" . }}
+    {{ if isset .Params "heroimage" }}
+      

+        {{ .Title }} - Hero Image
+      

+    {{ end }}
+    

+      

         

           

             
@@ -26,14 +26,14 @@ 
{{ .Title }}

           

         

         

-            

-                
Outline

-                
    
-            
    
+          
    
+            
    Outline
    
+            
      
+          
      
         
      
-        
      
-    
      
-    
      
+        
      
+      
      
+      
      
         {{ .Content }}
         {{ if isset .Params "acknowledgements" }}
           
      
@@ -54,19 +54,23 @@ 
      Outline
      
         {{ template "_internal/disqus.html" . }}
       
      
     
      
-
      
-
      
-{{ range .Params.authors}}
-
      
-    {{ range first 1 (where $.Site.Data.authors "key" "eq" .)}}
-    {{ if isset . "link"}}{{.name}}{{ else }}{{.name}}{{ end}} {{.description}}
+    
      
+    {{ range .Params.authors }}
+      
      
+        {{ range first 1 (where $.Site.Data.authors "key" "eq" .) }}
+          {{ if isset . "link" }}
+            {{ .name }}{{ else }}{{ .name }}{{ end }}
+
+          {{ .description }}
+        {{ end }}
+        
      
+        {{ partial "blog/latestblogs.html" . }}
+      
      
     {{ end }}
+    {{ partial "blog/license.html" . }}
     
      
     {{ partial "blog/latestblogs.html" . }}
   
      
 {{ end }}
-{{ partial "blog/license.html" . }}
-
      
-{{ partial "blog/latestblogs.html" . }}
-
      
-{{end}}
diff --git a/themes/delphi/layouts/covidcast_app/baseof.html b/themes/delphi/layouts/covidcast_app/baseof.html
index 885e48aad..3264e53ab 100644
--- a/themes/delphi/layouts/covidcast_app/baseof.html
+++ b/themes/delphi/layouts/covidcast_app/baseof.html
@@ -1,12 +1,11 @@
 
 
 
-    {{ partial "head.html" . }}
-    
-        {{ partial "nav.html" . }}
-        {{ partial "menu/breadcrumb.html" . }}
-        {{ block "main" . }}
-        {{ end }}
-        {{ partial "scripts.html" . }}
-    
+  {{ partial "head.html" . }}
+  
+    {{ partial "nav.html" . }}
+    {{ partial "menu/breadcrumb.html" . }}
+    {{ block "main" . }} {{ end }}
+    {{ partial "scripts.html" . }}
+  
 
diff --git a/themes/delphi/layouts/landing.html b/themes/delphi/layouts/landing.html
index 007bf5bbf..34fd58072 100644
--- a/themes/delphi/layouts/landing.html
+++ b/themes/delphi/layouts/landing.html
@@ -1,65 +1,78 @@
 {{ define "main" }}
-
      
+  
      
     
     
      
-        
      
-            
      
-                
          
-                    Our Mission
-                
      
-                
      
-                    {{.Site.Params.mission}}
-                
      
-            
      
+      
      
+        
      
+          
      Our Mission
      
+          
      
+            {{ .Site.Params.mission }}
+          
      
         
      
+      
      
     
      
 
     
      
-        
      
-            
      
-                
      
-                    
-                
      
-                
      Our Team
      
-                
      Meet the Delphi team
      
-                {{partial "arrow-link.html" (dict "link" (relref . "team") "alt" "View all")}}
-            
      
-            
      
-                
      
-                    
-                
      
-                
      Our API
      
-                
      Access our data and tools
      
-                {{partial "arrow-link.html" (dict "link" .Site.Params.apiUrl "alt" "Learn more")}}
-            
      
+      
      
+        
      
+          
      
+            
+          
      
+          
      Our Team
      
+          
      Meet the Delphi team
      
+          {{ partial "arrow-link.html" (dict "link" (relref . "team") "alt" "View all") }}
+        
      
+        
      
+          
      
+            
+          
      
+          
      Our API
      
+          
      Access our data and tools
      
+          {{ partial "arrow-link.html" (dict "link" .Site.Params.apiUrl "alt" "Learn more") }}
         
      
+      
      
     
      
-    
-    {{partial "landing/latest-news.html" .}}
-
      
+
+    {{ partial "landing/latest-news.html" . }}
+  
      
 {{ end }}
diff --git a/themes/delphi/layouts/partials/about/collaborators.html b/themes/delphi/layouts/partials/about/collaborators.html
index 4fe1b791a..1428997b2 100644
--- a/themes/delphi/layouts/partials/about/collaborators.html
+++ b/themes/delphi/layouts/partials/about/collaborators.html
@@ -1,24 +1,23 @@
 
      
-    
      
-        
      Collaborators
      
-        
      We're grateful for financial and other support from our collaborators and supporters:
      
-        
      
-            
        
-                {{range .Site.Data.supporter}}
-                    
      • 
-                        {{if eq .group "sponsor"}}With support from {{end}}
+  
        
+    
        Collaborators
        
+    
        We're grateful for financial and other support from our collaborators and supporters:
        
+    
        
+      
          
+        {{ range .Site.Data.supporter }}
+          
        • 
+            {{ if eq .group "sponsor" }}With support from {{ end }}
 
-                        {{.name}}
-                    
          • 
-                {{end}}
-            
        
-        
        
-    
        
-    
        With grant support from
        
-    
        
-      {{ range where .Site.Data.supporter "group" "grant" }}
-        {{ partial "about/collaborator-img.html" . }}
-      {{ end }}
+            {{ .name }}
+          
        • 
+        {{ end }}
+      
      
     
      
   
      
+  
      With grant support from
      
+  
      
+    {{ range where .Site.Data.supporter "group" "grant" }}
+      {{ partial "about/collaborator-img.html" . }}
+    {{ end }}
+  
      
 
      
diff --git a/themes/delphi/layouts/partials/arrow-link.html b/themes/delphi/layouts/partials/arrow-link.html
index 9f5c74f18..774d0cee8 100644
--- a/themes/delphi/layouts/partials/arrow-link.html
+++ b/themes/delphi/layouts/partials/arrow-link.html
@@ -1,4 +1,4 @@
-
-    {{partial "font-awesome.html" "solid/arrow-right"}}
-    {{.alt}}
+
+  {{ partial "font-awesome.html" "solid/arrow-right" }}
+  {{ .alt }}
 
diff --git a/themes/delphi/layouts/partials/blog/card.html b/themes/delphi/layouts/partials/blog/card.html
index 005a75c33..9e650dceb 100644
--- a/themes/delphi/layouts/partials/blog/card.html
+++ b/themes/delphi/layouts/partials/blog/card.html
@@ -1,15 +1,20 @@
 
      
-    
      
-        {{ .Title}} - Hero Image
-    
      
-    
      
-        {{ partial "blog/tags.html" . }}
-        
      {{ .Title}}
      
-    
      
-    
      
-        
-        
      
-            By {{ .Params.author }}
-        
      
-    
      
+  
      
+    {{ .Title }} - Hero Image
+  
      
+  
      
+    {{ partial "blog/tags.html" . }}
+    
      {{ .Title }}
      
+  
      
+  
      
+    
+    
      By {{ .Params.author }}
      
+  
      
 
      
diff --git a/themes/delphi/layouts/partials/blog/license.html b/themes/delphi/layouts/partials/blog/license.html
index f22a2bea3..c94c31847 100644
--- a/themes/delphi/layouts/partials/blog/license.html
+++ b/themes/delphi/layouts/partials/blog/license.html
@@ -1,7 +1,7 @@
 
      
-    © {{ now.Format "2006" }} Delphi group authors. 
-    Text and figures released under 
-        CC BY 4.0 {{ partial "font-awesome.html" "brands/creative-commons"}} {{ partial "font-awesome.html" "brands/creative-commons-by"}}
-    ;
-    code under the MIT license.
+  © {{ now.Format "2006" }} Delphi group authors. Text and figures released under
+  
+    CC BY 4.0 {{ partial "font-awesome.html" "brands/creative-commons" }}
+    {{ partial "font-awesome.html" "brands/creative-commons-by" }} ; code under the MIT license.
 
      
diff --git a/themes/delphi/layouts/partials/footer.html b/themes/delphi/layouts/partials/footer.html
index 9d6ed281d..f0912295e 100644
--- a/themes/delphi/layouts/partials/footer.html
+++ b/themes/delphi/layouts/partials/footer.html
@@ -1,3 +1,3 @@
 
      
-	{{ partial "footer/desktop.html" . }}
+  {{ partial "footer/desktop.html" . }}
 
      
diff --git a/themes/delphi/layouts/partials/footer/desktop.html b/themes/delphi/layouts/partials/footer/desktop.html
index 78b6d73c2..d9ff59d37 100644
--- a/themes/delphi/layouts/partials/footer/desktop.html
+++ b/themes/delphi/layouts/partials/footer/desktop.html
@@ -1,34 +1,36 @@
 
      
-    
      
-        {{partial "delphi-text-logo.html" . }}
-    
      
-    
      
-        
      COVIDcast
      
-        
-    
      
-    
      
-        
      Resources
      
-        
-    
      
-    
      
-        
      About
      
-        
-    
      
-    
      
-        
      Contact
      
-        
-    
      
+  
      
+    {{ partial "delphi-text-logo.html" . }}
+  
      
+  
      
+    
      COVIDcast
      
+    
+  
      
+  
      
+    
      Resources
      
+    
+  
      
+  
      
+    
      About
      
+    
+  
      
+  
      
+    
      Contact
      
+    
+  
      
 
      
diff --git a/themes/delphi/layouts/partials/landing/latest-card.html b/themes/delphi/layouts/partials/landing/latest-card.html
index af3e5e49b..d730da0bb 100644
--- a/themes/delphi/layouts/partials/landing/latest-card.html
+++ b/themes/delphi/layouts/partials/landing/latest-card.html
@@ -1,12 +1,12 @@
 
      
-    
      
-        {{
-    
      
-    
      
-        
      {{ .source }}
      
-        
      {{ .title }}
      
-    
      
-    
      
-        Learn more {{partial "font-awesome.html" "solid/long-arrow-alt-right"}}
-    
      
+  
      
+    {{ .title }}
+  
      
+  
      
+    
      {{ .source }}
      
+    
      {{ .title }}
      
+  
      
+  
      
+    Learn more {{ partial "font-awesome.html" "solid/long-arrow-alt-right" }}
+  
      
 
      
diff --git a/themes/delphi/layouts/partials/landing/latest-news.html b/themes/delphi/layouts/partials/landing/latest-news.html
index c097f3038..46026c558 100644
--- a/themes/delphi/layouts/partials/landing/latest-news.html
+++ b/themes/delphi/layouts/partials/landing/latest-news.html
@@ -1,7 +1,7 @@
 
      
-    
      Latest News
      
-    {{- $currentPage := . -}}
-    {{ define "partials/latest-blog" }}
+  
      Latest News
      
+  {{- $currentPage := . -}}
+  {{ define "partials/latest-blog" }}
     {{ return (dict "source" "blog" "image" .Params.heroImageThumb "title" .Title "link" .RelPermalink "date" .PublishDate ) }}
   {{ end }}
   {{ $items := apply (.Site.GetPage "/blog").Pages "partial" "latest-blog" "." }}
diff --git a/themes/delphi/layouts/partials/menu/breadcrumb.html b/themes/delphi/layouts/partials/menu/breadcrumb.html
index 5e3c30ed0..6ff9a0f64 100644
--- a/themes/delphi/layouts/partials/menu/breadcrumb.html
+++ b/themes/delphi/layouts/partials/menu/breadcrumb.html
@@ -1,10 +1,15 @@
 {{ if .Parent }}
-{{ if (not .Parent.IsHome) }}
-
      
-
-
-
      
+  {{ if (not .Parent.IsHome) }}
+    
+    
      
+      
+    
      
+  {{ end }}
 {{ end }}
diff --git a/themes/delphi/layouts/partials/nav.html b/themes/delphi/layouts/partials/nav.html
index 9d0ef516a..349565cc9 100644
--- a/themes/delphi/layouts/partials/nav.html
+++ b/themes/delphi/layouts/partials/nav.html
@@ -1,11 +1,11 @@
 {{- $currentPage := . -}}