Save full lhr report in Firebase storage instead of Firestore

lighthouse-data.mjs

@@ -15,13 +15,15 @@
  */
 
 import fs from 'fs';
-import fetch from 'node-fetch';
 import Firestore from '@google-cloud/firestore';
+import gcs from '@google-cloud/storage';
+const CloudStorage = gcs.Storage;
 import ReportGenerator from 'lighthouse/lighthouse-core/report/report-generator.js';
 import Memcache from './memcache.mjs';
 import LighthouseAPI from './lighthouse-api.mjs';
 
 const SERVICE_ACCOUNT_FILE = './serviceAccount.json';
+const STORAGE_BUCKET = 'webdotdevsite.appspot.com';
 const serviceAccountJSON = JSON.parse(fs.readFileSync(SERVICE_ACCOUNT_FILE));
 
 const MAX_REPORTS = 10;
@@ -53,6 +55,41 @@ function median(numbers) {
   return median;
 }
 
+/**
+ * Uploads the LH report to Firebase cloud storage.
+ * @param {!Object} lhr Full lhr object
+ * @param {string} name Report name.
+ * @return {!Promise<undefined>}
+ * @export
+ */
+async function uploadReport(lhr, name) {
+  const bucket = storage.bucket(STORAGE_BUCKET);
+  const filename = `lhrs/${name}.json`;
+  return await bucket.file(filename).save(JSON.stringify(lhr), {
+    gzip: true,
+    resumable: false,
+  });
+}
+
+/**
+ * Downloads the full LH report from Firebase cloud storage.
+ * @param {string} url Target url for the report.
+ * @return {?Promise<!Object>} Resolves with LHR json.
+ * @export
+ */
+export async function getFullReport(url) {
+  const bucket = storage.bucket(STORAGE_BUCKET);
+  const filename = `lhrs/${slugify(url)}.json`;
+  const file = bucket.file(filename);
+  const fileExists = (await file.exists())[0];
+  if (fileExists) {
+    const data = await file.download();
+    const lhr = JSON.parse(data);
+    return lhr;
+  }
+  return null;
+}
+
 /**
  * Saves Lighthouse report to Firestore.
  * @param {string} url URL to save run under.
@@ -76,7 +113,6 @@ export async function saveReport(url, json, replace) {
 
   const today = new Date();
   const data = {
-    lhr,
     lhrSlim,
     auditedOn: today,
   };
@@ -96,14 +132,11 @@ export async function saveReport(url, json, replace) {
     await memcache.delete('getAllSavedUrls');
   }
 
+  await uploadReport(lhr, slugify(url));
   if (replace && lastDoc) {
     await lastDoc.ref.update(data); // Replace last entry with updated vals.
   } else {
     await collectionRef.add(data); // Add new report.
-    if (lastDoc) {
-      // Delete the full LH report from the last entry to save space over time.
-      await lastDoc.ref.update({lhr: Firestore.FieldValue.delete()});
-    }
   }
 
   await updateLastViewed(url);
@@ -318,6 +351,9 @@ export async function getReports(url,
     return r;
   });
 
+  // Attach full lighthouse report to last entry.
+  runs[runs.length - 1].lhr = await getFullReport(url);
+
   if (useCache) {
     await memcache.set(cacheKey, runs);
   }
@@ -400,3 +436,8 @@ const db = new Firestore({
 });
 
 const memcache = new Memcache();
+
+const storage = new CloudStorage({
+  projectId: serviceAccountJSON.project_id,
+  keyFilename: SERVICE_ACCOUNT_FILE,
+});

package.json

@@ -27,6 +27,7 @@
   },
   "dependencies": {
     "@google-cloud/firestore": "^0.18.0",
+    "@google-cloud/storage": "^2.1.0",
     "@google-cloud/tasks": "^0.2.2",
     "@webcomponents/webcomponentsjs": "^2.1.3",
     "body-parser": "^1.18.3",

server.mjs

@@ -147,22 +147,19 @@ app.get('/lh/urls', async (req, resp) => {
 app.use('/lh/html', requireUrlQueryParam);
 app.get('/lh/html', async (req, resp, next) => {
   const url = req.query.url;
-  const latestRun = (await lighthouse.getReports(url, {
-    useCache: false,
-    maxResults: 1,
-  }))[0];
 
-  if (!latestRun) {
+  const lhr = await lighthouse.getFullReport(url);
+  if (!lhr) {
     return resp.status(404).json({errors: `No results found for "${url}".`});
   }
 
   if ('download' in req.query) {
-    const filename = `${fileNamer.getFilenamePrefix(latestRun.lhr)}.html`;
+    const filename = `${fileNamer.getFilenamePrefix(lhr)}.html`;
     resp.set('Content-Disposition', `attachment; filename=${filename}`);
   }
 
   // Send down LHR html report as response.
-  resp.status(200).send(lighthouse.generateReport(latestRun.lhr, 'html'));
+  resp.status(200).send(lighthouse.generateReport(lhr, 'html'));
 });
 
 app.use('/lh/reports', requireUrlQueryParam);