clean up zip archive and include readme

Recidiviz · Feb 26, 2021 · aa1eaf4 · aa1eaf4
1 parent 0392f1f
commit aa1eaf4
Show file tree

Hide file tree

Showing 6 changed files with 155 additions and 21 deletions.
diff --git a/spotlight-client/package.json b/spotlight-client/package.json
@@ -80,6 +80,7 @@
     "react-spring": "^8.0.27",
     "react-stickyfill": "^0.2.5",
     "semiotic": "^1.20.6",
+    "string-strip-html": "^8.2.2",
     "styled-components": "^5.2.1",
     "styled-reset": "^4.3.3",
     "topojson": "^3.0.2",

diff --git a/spotlight-client/src/contentModels/Metric.test.ts b/spotlight-client/src/contentModels/Metric.test.ts
@@ -23,6 +23,7 @@ import fetchMock from "jest-fetch-mock";
 import JsZip from "jszip";
 import { when } from "mobx";
 import { fromPromise } from "mobx-utils";
+import { stripHtml } from "string-strip-html";
 import retrieveContent from "../contentApi/retrieveContent";
 import { MetricTypeId, MetricTypeIdList } from "../contentApi/types";
 import { reactImmediately } from "../testUtils";
@@ -201,8 +202,7 @@ describe("data download", () => {
         // see SupervisionSuccessRateMetric tests for coverage
         !["ProbationSuccessHistorical", "ParoleSuccessHistorical"].includes(id)
     )
-  )("for metric %s", async (metricId) => {
-    expect.hasAssertions();
+  )("for metric %s", async (metricId, done) => {
     const metric = getTestMetric(metricId);
     metric.populateAllRecords();
 
@@ -211,22 +211,29 @@ describe("data download", () => {
     expect(downloadjsMock).toHaveBeenCalled();
 
     const [content, filename] = downloadjsMock.mock.calls[0];
-    expect(filename).toBe(`${testTenantId} ${metricId} data`);
+
+    const zipPrefix = `${testTenantId} ${metricId} data`;
+    expect(filename).toBe(`${zipPrefix}.zip`);
 
     // if we read the data in the zip file we should be able to reverse it
     // into something resembling metric.allRecords; we will spot-check
     // the downloaded file by verifying that the first record matches the source
     const zip = await JsZip.loadAsync(content);
-    const csvContents = await zip.file("data.csv")?.async("string");
+    const readmeContents = await zip
+      .file(`${zipPrefix}/README.txt`)
+      ?.async("string");
+    const csvContents = await zip
+      .file(`${zipPrefix}/data.csv`)
+      ?.async("string");
 
-    if (csvContents) {
-      const recordsFromCsv = csvParse(csvContents);
+    reactImmediately(() => {
+      if (csvContents && readmeContents) {
+        const recordsFromCsv = csvParse(csvContents);
 
-      // rather than try to re-typecast the CSV data,
-      // we'll cast the real record value to strings for comparison
-      const expectedRecord: Record<string, string> = {};
+        // rather than try to re-typecast the CSV data,
+        // we'll cast the real record value to strings for comparison
+        const expectedRecord: Record<string, string> = {};
 
-      reactImmediately(() => {
         // in practice, recordsUnfiltered should not be undefined once we've gotten this far
         Object.entries((metric.recordsUnfiltered || [])[0]).forEach(
           ([key, value]) => {
@@ -240,9 +247,15 @@ describe("data download", () => {
             expectedRecord[key] = valueAsString;
           }
         );
-      });
 
-      expect(recordsFromCsv[0]).toEqual(expectedRecord);
-    }
+        expect(recordsFromCsv[0]).toEqual(expectedRecord);
+
+        // the file in the archive is plain text but methodology can contain HTML tags
+        expect(readmeContents).toBe(stripHtml(metric.methodology).result);
+
+        // @ts-expect-error typedefs for `test.each` are wrong, `done` will be a function
+        done();
+      }
+    });
   });
 });
diff --git a/spotlight-client/src/contentModels/Metric.ts b/spotlight-client/src/contentModels/Metric.ts
@@ -26,6 +26,7 @@ import {
   runInAction,
   when,
 } from "mobx";
+import { stripHtml } from "string-strip-html";
 import { ERROR_MESSAGES } from "../constants";
 import { LocalityLabels, MetricTypeId, TenantId } from "../contentApi/types";
 import { DemographicView } from "../demographics";
@@ -227,14 +228,16 @@ export default abstract class Metric<RecordFormat extends MetricRecord> {
         () => this.allRecords !== undefined,
         () => {
           const zip = new JsZip();
+          const zipName = `${this.tenantId} ${this.id} data`;
           // this assertion is safe because we are waiting for it in the reaction above
           const data = csvFormat(this.allRecords as RecordFormat[]);
-          zip.file("data.csv", data);
+          zip.file(`${zipName}/data.csv`, data);
+          zip.file(`${zipName}/README.txt`, stripHtml(this.methodology).result);
 
           zip
             .generateAsync({ type: "blob" })
             .then((content) => {
-              downloadjs(content, `${this.tenantId} ${this.id} data`);
+              downloadjs(content, `${zipName}.zip`);
               resolve();
             })
             .catch((error) => {

diff --git a/spotlight-client/src/contentModels/SupervisionSuccessRateMetric.test.ts b/spotlight-client/src/contentModels/SupervisionSuccessRateMetric.test.ts
@@ -20,6 +20,7 @@ import downloadjs from "downloadjs";
 import JsZip from "jszip";
 import { advanceTo, clear } from "jest-date-mock";
 import { runInAction, when } from "mobx";
+import { stripHtml } from "string-strip-html";
 import { DemographicView } from "../demographics";
 import {
   fetchMetrics,
@@ -276,7 +277,8 @@ test("data download", async (done) => {
   expect(downloadjsMock).toHaveBeenCalled();
 
   const [content, filename] = downloadjsMock.mock.calls[0];
-  expect(filename).toBe(`${testTenantId} ${testMetricId} data`);
+  const prefix = `${testTenantId} ${testMetricId} data`;
+  expect(filename).toBe(`${prefix}.zip`);
 
   // if we read the data in the zip file we should be able to reverse it
   // into something resembling metric.allRecords; we will spot-check
@@ -285,9 +287,12 @@ test("data download", async (done) => {
   reactImmediately(async () => {
     // expecting two files and their respective data sources
     const expectedFiles = [
-      { name: "historical data.csv", source: metric.getOrFetchCohortRecords() },
       {
-        name: "demographic aggregate data.csv",
+        name: `${prefix}/historical data.csv`,
+        source: metric.getOrFetchCohortRecords(),
+      },
+      {
+        name: `${prefix}/demographic aggregate data.csv`,
         source: metric.getOrFetchDemographicRecords(),
       },
     ];
@@ -314,6 +319,12 @@ test("data download", async (done) => {
       })
     );
 
+    const readmeContents = await zip
+      .file(`${prefix}/README.txt`)
+      ?.async("string");
+    // the file in the archive is plain text but methodology can contain HTML tags
+    expect(readmeContents).toBe(stripHtml(metric.methodology).result);
+
     done();
   });
 });
diff --git a/spotlight-client/src/contentModels/SupervisionSuccessRateMetric.ts b/spotlight-client/src/contentModels/SupervisionSuccessRateMetric.ts
@@ -20,6 +20,7 @@ import { csvFormat } from "d3-dsv";
 import downloadjs from "downloadjs";
 import JsZip from "jszip";
 import { computed, makeObservable, observable, runInAction, when } from "mobx";
+import { stripHtml } from "string-strip-html";
 import {
   DemographicView,
   getDemographicCategories,
@@ -282,6 +283,7 @@ export default class SupervisionSuccessRateMetric extends Metric<
           this.allDemographicRecords !== undefined,
         () => {
           const zip = new JsZip();
+          const zipName = `${this.tenantId} ${this.id} data`;
           // these assertions are safe because we are waiting for them in the reaction above
           const cohortData = csvFormat(
             this.allCohortRecords as SupervisionSuccessRateMonthlyRecord[]
@@ -291,13 +293,14 @@ export default class SupervisionSuccessRateMetric extends Metric<
               .allDemographicRecords as SupervisionSuccessRateDemographicsRecord[]
           );
           zip
-            .file("historical data.csv", cohortData)
-            .file("demographic aggregate data.csv", demographicData);
+            .file(`${zipName}/historical data.csv`, cohortData)
+            .file(`${zipName}/demographic aggregate data.csv`, demographicData)
+            .file(`${zipName}/README.txt`, stripHtml(this.methodology).result);
 
           zip
             .generateAsync({ type: "blob" })
             .then((content) => {
-              downloadjs(content, `${this.tenantId} ${this.id} data`);
+              downloadjs(content, `${zipName}.zip`);
               resolve();
             })
             .catch((error) => {

diff --git a/yarn.lock b/yarn.lock
@@ -1138,6 +1138,13 @@
   dependencies:
     regenerator-runtime "^0.13.4"
 
+"@babel/runtime@^7.12.13":
+  version "7.13.7"
+  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.13.7.tgz#d494e39d198ee9ca04f4dcb76d25d9d7a1dc961a"
+  integrity sha512-h+ilqoX998mRVM5FtB5ijRuHUDVt5l3yfoOi2uh18Z/O3hvyaHQ39NpxVkCIG5yFs+mLq/ewFp8Bss6zmWv6ZA==
+  dependencies:
+    regenerator-runtime "^0.13.4"
+
 "@babel/template@^7.10.4", "@babel/template@^7.4.0", "@babel/template@^7.8.6":
   version "7.10.4"
   resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.10.4.tgz#3251996c4200ebc71d1a8fc405fba940f36ba278"
@@ -7326,6 +7333,11 @@ html-entities@^1.3.1:
   resolved "https://registry.yarnpkg.com/html-entities/-/html-entities-1.3.1.tgz#fb9a1a4b5b14c5daba82d3e34c6ae4fe701a0e44"
   integrity sha512-rhE/4Z3hIhzHAUKbW8jVcCyuT5oJCXXqhN/6mXXVCpzTmvJnoH2HL/bt3EZ6p55jbFJBeAe1ZNpL5BugLujxNA==
 
+html-entities@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/html-entities/-/html-entities-2.1.0.tgz#f5de1f8d5e1f16859a74aa73a90f0db502ca723a"
+  integrity sha512-u+OHVGMH5P1HlaTFp3M4HolRnWepgx5rAnYBo+7/TrBZahuJjgQ4TMv2GjQ4IouGDzkgXYeOI/NQuF95VOUOsQ==
+
 html-escaper@^2.0.0:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-2.0.2.tgz#dfd60027da36a36dfcbe236262c00a5822681453"
@@ -9138,6 +9150,11 @@ lodash._reinterpolate@^3.0.0:
   resolved "https://registry.yarnpkg.com/lodash._reinterpolate/-/lodash._reinterpolate-3.0.0.tgz#0ccf2d89166af03b3663c796538b75ac6e114d9d"
   integrity sha1-DM8tiRZq8Ds2Y8eWU4t1rG4RTZ0=
 
+lodash.clonedeep@^4.5.0:
+  version "4.5.0"
+  resolved "https://registry.yarnpkg.com/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz#e23f3f9c4f8fbdde872529c1071857a086e5ccef"
+  integrity sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=
+
 lodash.get@^4.4.2:
   version "4.4.2"
   resolved "https://registry.yarnpkg.com/lodash.get/-/lodash.get-4.4.2.tgz#2d177f652fa31e939b4438d5341499dfa3825e99"
@@ -9148,6 +9165,11 @@ lodash.isempty@^4.4.0:
   resolved "https://registry.yarnpkg.com/lodash.isempty/-/lodash.isempty-4.4.0.tgz#6f86cbedd8be4ec987be9aaf33c9684db1b31e7e"
   integrity sha1-b4bL7di+TsmHvpqvM8loTbGzHn4=
 
+lodash.isplainobject@^4.0.6:
+  version "4.0.6"
+  resolved "https://registry.yarnpkg.com/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz#7c526a52d89b45c45cc690b88163be0497f550cb"
+  integrity sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=
+
 lodash.memoize@^4.1.2:
   version "4.1.2"
   resolved "https://registry.yarnpkg.com/lodash.memoize/-/lodash.memoize-4.1.2.tgz#bcc6c49a42a2840ed997f323eada5ecd182e0bfe"
@@ -9173,11 +9195,21 @@ lodash.templatesettings@^4.0.0:
   dependencies:
     lodash._reinterpolate "^3.0.0"
 
+lodash.trim@^4.5.1:
+  version "4.5.1"
+  resolved "https://registry.yarnpkg.com/lodash.trim/-/lodash.trim-4.5.1.tgz#36425e7ee90be4aa5e27bcebb85b7d11ea47aa57"
+  integrity sha1-NkJefukL5KpeJ7zruFt9EepHqlc=
+
 lodash.uniq@^4.5.0:
   version "4.5.0"
   resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773"
   integrity sha1-0CJTc662Uq3BvILklFM5qEJ1R3M=
 
+lodash.without@^4.4.0:
+  version "4.4.0"
+  resolved "https://registry.yarnpkg.com/lodash.without/-/lodash.without-4.4.0.tgz#3cd4574a00b67bae373a94b748772640507b7aac"
+  integrity sha1-PNRXSgC2e643OpS3SHcmQFB7eqw=
+
 lodash.xor@^4.5.0:
   version "4.5.0"
   resolved "https://registry.yarnpkg.com/lodash.xor/-/lodash.xor-4.5.0.tgz#4d48ed7e98095b0632582ba714d3ff8ae8fb1db6"
@@ -11560,6 +11592,40 @@ range-parser@^1.2.1, range-parser@~1.2.1:
   resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031"
   integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==
 
+ranges-apply@^5.0.5:
+  version "5.0.5"
+  resolved "https://registry.yarnpkg.com/ranges-apply/-/ranges-apply-5.0.5.tgz#06123c70a0c951f0f3e23a3881f53df476cc1ba8"
+  integrity sha512-apSrBvW3FKq0FUksuZumhR5wtLx8K4BqXELzhBVqvmz3dfiBoAsZI57UPhqi0fgBeM2WlY61hMcqgzEPzC75Xg==
+  dependencies:
+    "@babel/runtime" "^7.12.13"
+    ranges-merge "^7.0.5"
+
+ranges-merge@^7.0.5:
+  version "7.0.5"
+  resolved "https://registry.yarnpkg.com/ranges-merge/-/ranges-merge-7.0.5.tgz#b79888129ebee0c8a7d4b2db0a7573b4a6fe599d"
+  integrity sha512-GtI4GsBx8PHCjq5+2DpfgmKW3zvGgBnwvmL8Pcs8q9udYXU/0O5Fi1mUo7UbHr62KCLX0Aw360cBbbKAYuQI3Q==
+  dependencies:
+    "@babel/runtime" "^7.12.13"
+    ranges-push "^5.0.5"
+    ranges-sort "^4.0.5"
+
+ranges-push@^5.0.5:
+  version "5.0.5"
+  resolved "https://registry.yarnpkg.com/ranges-push/-/ranges-push-5.0.5.tgz#29887ed3ebbe7cd3c132e17f3171ccba485081b0"
+  integrity sha512-h46K33Lc+d6kY4GoOLTam1RO53DKJQR/iAAmy3zkTc8RhXBc2uiTmdJzp07nnhq8oX2roPQphsS4kFjM7i+hpQ==
+  dependencies:
+    "@babel/runtime" "^7.12.13"
+    ranges-merge "^7.0.5"
+    string-collapse-leading-whitespace "^5.0.5"
+    string-trim-spaces-only "^3.0.5"
+
+ranges-sort@^4.0.5:
+  version "4.0.5"
+  resolved "https://registry.yarnpkg.com/ranges-sort/-/ranges-sort-4.0.5.tgz#d75199043abae7c6a8f0b4d9fcbe09239b485f2c"
+  integrity sha512-j90F9gbtbdqEUKbQs6u3JZ+BS86RhWIfSvrbikhnXyA0uefq4yBSz/wpNt8Vqk0Tn8XBgnuKdPtu2r4sKjHfDw==
+  dependencies:
+    "@babel/runtime" "^7.12.13"
+
 raw-body@2.4.0:
   version "2.4.0"
   resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.4.0.tgz#a1ce6fb9c9bc356ca52e89256ab59059e13d0332"
@@ -13076,6 +13142,22 @@ string-argv@0.3.1:
   resolved "https://registry.yarnpkg.com/string-argv/-/string-argv-0.3.1.tgz#95e2fbec0427ae19184935f816d74aaa4c5c19da"
   integrity sha512-a1uQGz7IyVy9YwhqjZIZu1c8JO8dNIe20xBmSS6qu9kv++k3JGzCVmprbNN5Kn+BgzD5E7YYwg1CcjuJMRNsvg==
 
+string-collapse-leading-whitespace@^5.0.5:
+  version "5.0.5"
+  resolved "https://registry.yarnpkg.com/string-collapse-leading-whitespace/-/string-collapse-leading-whitespace-5.0.5.tgz#6f4bbbad58b4a40360504a1e37e429a1a5476e48"
+  integrity sha512-cQFJ49yNihmsCTrPh2MpVLqbW4dBiCfZN/WZlfhaFv6bFSOxaD4/GZKZTseM1cqt1xPuemlUjfstC+J+FkCloA==
+  dependencies:
+    "@babel/runtime" "^7.12.13"
+
+string-left-right@^4.0.5:
+  version "4.0.5"
+  resolved "https://registry.yarnpkg.com/string-left-right/-/string-left-right-4.0.5.tgz#101f3ff33bbd975dcad06ff4f288c22ccd3f07de"
+  integrity sha512-FrkHePqfIuCGBa7p2bALCn/sJxp+9ipGMrEC0G/UroL9HZ0+xnWZMIWZGuFX3nacNOTXQJ+oLkUqmrDTSNvTBQ==
+  dependencies:
+    "@babel/runtime" "^7.12.13"
+    lodash.clonedeep "^4.5.0"
+    lodash.isplainobject "^4.0.6"
+
 string-length@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/string-length/-/string-length-2.0.0.tgz#d40dbb686a3ace960c1cffca562bf2c45f8363ed"
@@ -13092,6 +13174,27 @@ string-length@^3.1.0:
     astral-regex "^1.0.0"
     strip-ansi "^5.2.0"
 
+string-strip-html@^8.2.2:
+  version "8.2.2"
+  resolved "https://registry.yarnpkg.com/string-strip-html/-/string-strip-html-8.2.2.tgz#c2e9947f3f8b0ffeb3ed012fd3be2f4d3b4a0bad"
+  integrity sha512-m5f8LWynfo7R6aNrhkInnAmJTmHaNTsaLqsxW3tKf0L16cMdGs0r8we5JGRiyGxzRTw7ZmRK/oXDA4lPayRW+A==
+  dependencies:
+    "@babel/runtime" "^7.12.13"
+    html-entities "^2.1.0"
+    lodash.isplainobject "^4.0.6"
+    lodash.trim "^4.5.1"
+    lodash.without "^4.4.0"
+    ranges-apply "^5.0.5"
+    ranges-push "^5.0.5"
+    string-left-right "^4.0.5"
+
+string-trim-spaces-only@^3.0.5:
+  version "3.0.5"
+  resolved "https://registry.yarnpkg.com/string-trim-spaces-only/-/string-trim-spaces-only-3.0.5.tgz#27d314975f9bb709cca288c794ee33dfd4829d09"
+  integrity sha512-Kb7dHu6eiPq5mh6lzVsCEOFFGbd0Qo4tI6pJQxvWnl0rbwKCRxfT3i+q4Q6XXaqHAJ7kmBBkSaW15MgzH8fMxg==
+  dependencies:
+    "@babel/runtime" "^7.12.13"
+
 string-width@^3.0.0, string-width@^3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-3.1.0.tgz#22767be21b62af1081574306f69ac51b62203961"