Skip to content
Permalink
Browse files

Rewrite backend to pull index from Google Cloud Storage + Deploy to F…

…irebase (#6)

* initial WIP geocoding index

* progress logging

* set content type

* Fetch data from GCS

* Sort list by distance from user

* add client side search

* make sorting consistent

* add firebase

* don't load firebase

* remove firebase cache
  • Loading branch information...
hcarnes committed May 6, 2019
1 parent b934791 commit 40ea0759ae863bd5b6b020c28b671b92275cdcbf
Showing with 998 additions and 15 deletions.
  1. +5 −0 .babelrc
  2. +1 −0 .firebaserc
  3. +1 −0 .gitignore
  4. +8 −0 cors-policy.json
  5. +16 −0 firebase.json
  6. +11 −2 package.json
  7. +14 −0 public/index.html
  8. +57 −0 scripts/build-geocode-index.js
  9. +19 −5 src/models/Establishment.js
  10. +866 −8 yarn.lock
@@ -0,0 +1,5 @@
{
"presets": [
"@babel/preset-env"
]
}
@@ -0,0 +1 @@
{}
@@ -21,3 +21,4 @@
npm-debug.log*
yarn-debug.log*
yarn-error.log*
.firebase
@@ -0,0 +1,8 @@
[
{
"origin": ["*"],
"responseHeader": ["Content-Type", "Content-Encoding", "Cache-Control"],
"method": ["GET"],
"maxAgeSeconds": 3600
}
]
@@ -0,0 +1,16 @@
{
"hosting": {
"public": "build",
"ignore": [
"firebase.json",
"**/.*",
"**/node_modules/**"
],
"rewrites": [
{
"source": "**",
"destination": "/index.html"
}
]
}
}
@@ -3,10 +3,12 @@
"version": "0.1.0",
"private": true,
"dependencies": {
"@google-cloud/storage": "^2.4.3",
"axios": "^0.18.0",
"grommet": "^2.3.1",
"grommet-icons": "^4.1.0",
"grommet-styles": "^0.2.0",
"haversine-distance": "^1.1.4",
"polished": "^2.3.3",
"react": "^16.8.2",
"react-dom": "^16.8.2",
@@ -19,7 +21,9 @@
"start": "react-scripts start",
"build": "react-scripts build",
"test": "react-scripts test",
"eject": "react-scripts eject"
"eject": "react-scripts eject",
"build-geocode-index": "babel-node scripts/build-geocode-index.js",
"deploy": "react-scripts build && firebase deploy"
},
"eslintConfig": {
"extends": "react-app"
@@ -29,5 +33,10 @@
"not dead",
"not ie <= 11",
"not op_mini all"
]
],
"devDependencies": {
"@babel/core": "^7.4.0",
"@babel/node": "^7.2.2",
"@babel/preset-env": "^7.4.2"
}
}
@@ -23,6 +23,20 @@
work correctly both with client-side routing and a non-root public URL.
Learn how to configure a non-root public URL by running `npm run build`.
-->

<!-- <script src="https://www.gstatic.com/firebasejs/5.9.2/firebase.js"></script>
<script>
// Initialize Firebase
var config = {
apiKey: "AIzaSyBxd6Af7G-VnHMsD3FVb09sPu1yM7Vt5mk",
authDomain: "filth-finder-9ad86.firebaseapp.com",
databaseURL: "https://filth-finder-9ad86.firebaseio.com",
projectId: "filth-finder",
storageBucket: "filth-finder.appspot.com",
messagingSenderId: "261250091371"
};
firebase.initializeApp(config);
</script> -->
<title>Filth Finder</title>
</head>
<body>
@@ -0,0 +1,57 @@
import axios from "axios"
import {Storage} from '@google-cloud/storage'

async function updateGeocodedEstablishmentsIndex() {
const establishmentListing = await axios.get("https://data.cityofnewyork.us/resource/9w7m-hzhe.json?$group=camis,boro,building,street,zipcode,dba&$select=camis,boro,building,street,zipcode,dba&$limit=50000")

const startTime = Date.now();
const geocodedEstablishments = []
let numberGeocoded = 0

// Use for of so we don't try to send 50k requests at once
for (const establishment of establishmentListing.data) {
try {
const location = await axios.get(`https://api.cityofnewyork.us/geoclient/v1/address.json?houseNumber=${establishment.building}&street=${establishment.street}&borough=${establishment.boro}&app_id=dc074185&app_key=5538af0b4a556296c7b536df853f8697`)
const geocodedEstablishment = {
camis: establishment.camis,
dba: establishment.dba,
latitude: location.data.address.latitude,
longitude: location.data.address.longitude
}
geocodedEstablishments.push(geocodedEstablishment)
numberGeocoded += 1
if (numberGeocoded % 10 == 0) {
const elapsedTime = (Date.now()-startTime)/1000
const rate = (numberGeocoded/elapsedTime).toFixed(2)
console.log("Geocoded %d establishments in %d seconds (Rate: %d/s)", numberGeocoded, elapsedTime, rate)
}
} catch(err) {
console.log(`encountered error: ${err} when geocoding %j`, establishment)
}
}

storeToGCS(geocodedEstablishments)
}

function storeToGCS(indexArray) {
const storage = new Storage({
projectId: 'filth-finder',
});

const bucketName = 'filth-finder';

const indexFile = storage.bucket(bucketName).file("index.json")
const indexWriteStream = indexFile.createWriteStream({
gzip: true,
metadata: {
cacheControl: 'public, max-age=86400',
contentType: 'application/json'
},
})

indexWriteStream.on("finish", () => console.log("Stored %d establishments to the index", indexArray.length))
indexWriteStream.write(JSON.stringify(indexArray))
indexWriteStream.end()
}

updateGeocodedEstablishmentsIndex()
@@ -1,4 +1,5 @@
import axios from "axios";
import haversine from "haversine-distance";

const fetchDetails = async camis => {
const response = await axios.get(
@@ -8,12 +9,25 @@ const fetchDetails = async camis => {
return response.data;
};
class Establishment {
static async near(lng, lat, search = "") {
const establishments = await axios.get(`${process.env.REACT_APP_BACKEND_HOST}/near_me`, {
params: { lat, lng, search }
});
static async near(lng, lat, search = null) {
const establishments = await axios.get(`https://storage.googleapis.com/filth-finder/index.json`);

return establishments.data;
const establishmentsWithDistance = establishments.data.flatMap(e => {
const distance = haversine({lat: e.latitude, lng: e.longitude}, {lat, lng})
if (isNaN(distance)) {
return []
} else {
return {...e, distance}
}
})

const sortedEstablishments = establishmentsWithDistance.sort((a, b) => a.distance - b.distance);

if (search) {
return sortedEstablishments.filter((e) => e.dba && e.dba.toLowerCase().includes(search.toLowerCase())).slice(0, 20)
} else {
return sortedEstablishments.slice(0, 20)
}
}

static async detail(camis) {

0 comments on commit 40ea075

Please sign in to comment.
You can’t perform that action at this time.