Merge a0d5a42 into 8938c8b

Geovation · Jun 11, 2021 · 41c6bd3 · 41c6bd3
2 parents 8938c8b + a0d5a42
commit 41c6bd3
Show file tree

Hide file tree

Showing 7 changed files with 124 additions and 112 deletions.
diff --git a/.gitignore b/.gitignore
@@ -71,4 +71,3 @@ scripts/tmp.jpg
 functions/config.json
 public/config.json
 public/config.js
-tmp.jpg
diff --git a/firestore.indexes.json b/firestore.indexes.json
@@ -15,6 +15,14 @@
         { "fieldPath": "moderated", "mode": "ASCENDING" },
         { "fieldPath": "updated", "mode": "DESCENDING" }
       ]
+    },
+    {
+      "collectionGroup": "photos",
+      "queryScope": "COLLECTION",
+      "fields": [
+        { "fieldPath": "published", "mode": "ASCENDING" },
+        { "fieldPath": "updated", "mode": "DESCENDING" }
+      ]
     }
   ]
 }
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -46,7 +46,7 @@
     "node-sass": "^6.0.0",
     "precise-commits": "latest",
     "prettier": "^2.1.2",
-    "random-location": "latest",
+    "random-location": "^1.1.3",
     "react-scripts": "^4.0.3",
     "rimraf": "latest",
     "web-vitals": "^2.0.1",
@@ -67,6 +67,7 @@
     "md5": "^2.3.0",
     "moment": "^2.29.1",
     "prop-types": "^15.7.2",
+    "queue-promise": "^2.2.1",
     "react": "^17.0.2",
     "react-dom": "^17.0.2",
     "react-firebaseui": "latest",

diff --git a/scripts/populateDemoData.js b/scripts/populateDemoData.js
@@ -13,6 +13,11 @@ const storageBucket = require("../public/config.json").FIREBASE.config.storageBu
 const path = require('path');
 const _ = require('lodash');
 const turf = require('@turf/turf');
+const Queue = require('queue-promise');
+
+const fs = require('fs');
+const os = require('os');
+const fsPromises = fs.promises;
 
 const LOCATIONS = {
   "Global": {},
@@ -78,17 +83,16 @@ async function addMetaDataSync(id, locationName) {
     number: Math.round(Math.random()*10)
   };
 
-  console.log(`Adding ${id} with data:`, data);
-  return await db.collection("photos").doc(id).set(data);
+  const rtn = await db.collection("photos").doc(id).set(data);
+  console.log(`Added ${id} with data:`, data);
+  return rtn;
 }
 
-async function addPhotoSync(id) {
-  console.log(`Uploading ${id}`);
-
+async function addPhotoSync(id, filePath) {
   // upload it as "original"
-  return await bucket.upload("tmp.jpg", {
-    destination: `photos/${id}/original.jpg`,
-  });
+  const rtn = await bucket.upload(filePath, { destination: `photos/${id}/original.jpg`, });
+  console.log(`Uploaded ${filePath}`);
+  return rtn;
 }
 
 async function run(num, storage, location) {
@@ -109,15 +113,35 @@ async function run(num, storage, location) {
   const maxWidth = image.getWidth();
   const digits = 12;
 
+  const queue = new Queue({
+    concurrent: 10,
+    interval: 10
+  });
+
+  queue.on("end", () => console.log("The end"));
+  queue.on("resolve", data => console.log(`Resolved ${data}`));
+  queue.on("reject", error => console.error(error));
+  const tempDir = os.tmpdir();
+
   for (let i = 0; i < num; i++) {
-    console.log(`processing photo ${i + 1}/${num}`);
+    console.log(`Enqueuing photo ${i + 1}/${num}`);
 
     const id = `test_${(Math.floor(Math.random() * 10 ** digits) + "").padStart(
       digits,
       "0"
     )}`;
+    const tmpFilePath = path.join(tempDir, `${id}.jpg`);
 
-    // watermark it with id
+    queue.enqueue(() => uploadPhoto(image, tmpFilePath, id, location, i, num, fontWhite, fontBlack, maxWidth, maxHeight)); 
+  }
+  // await Promise.all(queue);
+  while (queue.shouldRun) {
+    const data = await queue.dequeue();
+  }
+}
+
+async function uploadPhoto(image, tmpFilePath, id, location, i, num, fontWhite, fontBlack, maxWidth, maxHeight) {
+        // watermark it with id
     const text = {
       text: id,
       alignmentX: Jimp.HORIZONTAL_ALIGN_CENTER,
@@ -126,13 +150,13 @@ async function run(num, storage, location) {
     const newImage = image.clone();
     newImage.print(fontWhite, 5, 0, text, maxWidth, maxHeight);
     newImage.print(fontBlack, 0, 0, text, maxWidth, maxHeight);
-    await newImage.writeAsync("tmp.jpg");
-
+
+    await newImage.writeAsync(tmpFilePath);
     await addMetaDataSync(id, location);
-    await addPhotoSync(id);
-    // await Promise.all([addPhotoSync(id), addMetaDataSync(id, location)]);
-  }
-}
+    await addPhotoSync(id, tmpFilePath);
+    await fsPromises.unlink(tmpFilePath);
+    console.log(`Completed photo ${i + 1}/${num}`);
+};
 
 run(argv.number, argv.storage, argv.location)
   .then(() => {

diff --git a/src/App.js b/src/App.js
@@ -181,14 +181,13 @@ class App extends Component {
     let { photoId, mapLocation } = this.extractPathnameParams();
     this.setState({ photoId, mapLocation });
 
-    this.unregisterAuthObserver = authFirebase.onAuthStateChanged((user) => {
-      // will do this after the user has been loaded. It should speed up the users login.
-      // not sure if we need this if.
-      if (!this.initDone) {
-        this.initDone = true;
-        this.someInits(photoId, user);
-      }
+    // TODO: test it. Does it slow down starting up ?
+    if (!this.initDone) {
+      this.initDone = true;
+      this.someInits(photoId);
+    }
 
+    this.unregisterAuthObserver = authFirebase.onAuthStateChanged((user) => {
       // lets start fresh if the user logged out
       if (this.state.user && !user) {
         gtagEvent("Signed out", "User");
@@ -225,12 +224,19 @@ class App extends Component {
         }
 
         geojson.features = _.map(this.featuresDict, (f) => f);
-
         // save only if different
         if (!_.isEqual(this.state.geojson, geojson)) {
-          this.setState({ geojson });
           // after the first time, wait for a bit before updating.
           localforage.setItem("cachedGeoJson", geojson);
+
+          const stats = this.props.config.getStats(geojson, this.state.dbStats);
+          this.setState({ geojson, stats });
+
+          this.featuresDict = geojson.features.reduce((acc, feature) => {
+            acc[feature.properties.id] = feature;
+            return acc;
+          }, {});
+
         }
 
         console.debug("Geo Json updated");
@@ -258,103 +264,77 @@ class App extends Component {
     this.delayedSaveGeojson();
   };
 
-  someInits(photoId, user) {
+  async someInits(photoId) {
     this.unregisterConnectionObserver = dbFirebase.onConnectionStateChanged(
       (online) => {
         this.setState({ online });
       }
     );
 
+    // when photoId is defined (when acceding the app with photoid query string), need to get the photo info.
     this.fetchPhotoIfUndefined(photoId).then(async () => {
       // If the selectedFeature is not null, it means that we were able to retrieve a photo from the URL and so we landed
       // into the photoId.
       this.setState({ photoAccessedByUrl: !!this.state.selectedFeature });
 
-      dbFirebase.fetchStats().then((dbStats) => {
-        console.log(dbStats);
-        this.setState({
-          usersLeaderboard: dbStats.users,
-          dbStats,
-          stats: this.props.config.getStats(
-            this.state.geojson,
-            this.state.dbStats
-          ),
-        });
-
-        return dbStats;
-      });
-
       gtagPageView(this.props.location.pathname);
     });
 
-    // use the locals one if we have them: faster boot.
-    localforage
-      .getItem("cachedGeoJson")
-      .then((geojson) => {
-        if (geojson) {
-          this.geojson = geojson;
-          const stats = this.props.config.getStats(geojson, this.state.dbStats);
-          this.setState({ geojson, stats });
-          this.featuresDict = geojson.features.reduce((acc, feature) => {
-            acc[feature.properties.id] = feature;
-            return acc;
-          }, {});
+    // Get the photos from the cache first.
+    const geojson = await localforage.getItem("cachedGeoJson");
+
+    if (geojson) {
+      this.setState({ geojson });
+      this.featuresDict = geojson.features.reduce((acc, feature) => {
+        acc[feature.properties.id] = feature;
+        return acc;
+      }, {});
+    } else {
+      await this.fetchPhotos();
+    }
+
+    this.registerPublishedPhotosRT();     
+  }
 
-          // listen for changes since the last photo in the cache.
-          const latestPhoto = _.maxBy(geojson.features, (photo) => {
-            return photo.properties.updated;
-          });
+  async registerPublishedPhotosRT() {
+    if (this.unregisterPublishedPhotosRT) {
+      await this.unregisterPublishedPhotosRT();
+    }
 
-          const lastUpdated = _.get(latestPhoto, "properties.updated");
-          this.unregisterPublishedPhotosRT = dbFirebase.publishedPhotosRT(
-            this.addFeature,
-            this.modifyFeature,
-            this.removeFeature,
-            (error) => {
-              console.log(error);
-              alert(error);
-              window.location.reload();
-            },
-            lastUpdated
-          );
-        } else {
-          // ??? TODO ????: fetch at least 100 photos, just to show something in the map.... otherwise it will show "LOADING PHOTOS"
-          // ...
-          this.fetchPhotos();
-        }
-      })
-      .catch(console.error);
+    // The following line should speedup things. It reads all the photos until before trigger the RT listener
+    await this.fetchPhotos(false, this.calculateLastUpdate());
+
+    this.unregisterPublishedPhotosRT = dbFirebase.publishedPhotosRT(
+      this.addFeature,
+      this.modifyFeature,
+      this.removeFeature,
+      (error) => {
+        console.log(error);
+        alert(error);
+        window.location.reload();
+      },
+      this.calculateLastUpdate()
+    );
   }
 
-  fetchPhotos(fromAPI = true) {
-    dbFirebase
-      .fetchPhotos(fromAPI)
-      .then(async (photos) => {
-        let lastUpdated = new Date(null);
+  calculateLastUpdate() {
+    let lastUpdated = new Date(null);
+    if (this.state.geojson) {
+      const latestPhoto = _.maxBy(this.state.geojson.features, (photo) => {
+        return photo.properties.updated;
+      });
+      lastUpdated = _.get(latestPhoto, "properties.updated");
+    }
+    return lastUpdated;
+  }
+
+  async fetchPhotos(fromAPI = true, lastUpdate = new Date(null)) {
+    return dbFirebase
+      .fetchPhotos(fromAPI, lastUpdate)
+      .then((photos) => {
         _.forEach(photos, (photo) => {
           this.addFeature(photo);
-          if (photo.updated > lastUpdated) {
-            lastUpdated = photo.updated;
-          }
         });
-        this.delayedSaveGeojson();
-        // at this point we retrieve ALL the photos and we have the date of the last photo comming from the cache.
-        // So we listen for changes since then
-
-        if (this.unregisterPublishedPhotosRT) {
-          await this.unregisterPublishedPhotosRT();
-        }
-        this.unregisterPublishedPhotosRT = dbFirebase.publishedPhotosRT(
-          this.addFeature,
-          this.modifyFeature,
-          this.removeFeature,
-          (error) => {
-            console.log(error);
-            alert(error);
-            window.location.reload();
-          },
-          lastUpdated
-        );
       })
       .catch(console.error);
   }
@@ -662,6 +642,8 @@ class App extends Component {
 
     // it will open the "loading photos" message
     this.setState({ geojson: null });
+
+    // fetch all the photos from firestore instead than from the CDN
     this.fetchPhotos(false);
   };
 

diff --git a/src/features/firebase/dbFirebase.js b/src/features/firebase/dbFirebase.js
@@ -96,18 +96,12 @@ const configObserver = (onNext, onError) => {
     }, onError);
 };
 
-async function fetchStats() {
-  return fetch(appConfig.FIREBASE.apiURL + "/stats", {
-    mode: "cors",
-  }).then((response) => response.json());
-}
-
 /**
  * Open reload all the photos using the REST API. In this way it will laverage CDN caching saving firestore quota.
  *
  * @param {*} fromAPI if true it will get it from the API which is very usefull for caching.
  */
-async function fetchPhotos(fromAPI = true) {
+async function fetchPhotos(fromAPI = true, sinceDate = new Date(null)) {
   let photos = {};
   if (fromAPI) {
     const photosResponse = await axios.get(
@@ -119,9 +113,9 @@ async function fetchPhotos(fromAPI = true) {
     const querySnapshot = await firestore
       .collection("photos")
       .where("published", "==", true)
+      .where("updated", ">=", sinceDate)
       .get();
     querySnapshot.forEach((doc) => {
-      // doc.data() is never undefined for query doc snapshots
       photos[doc.id] = convertFirebaseTimestampFieldsIntoDate(doc.data());
     });
   }
@@ -404,7 +398,6 @@ function buildStorageUrl(path) {
 const rtn = {
   onConnectionStateChanged,
   publishedPhotosRT,
-  fetchStats,
   fetchFeedbacks,
   fetchPhotos,
   getUser,