Various fixes (#111)

* Fix slack message request id

* Add tokens to requests

* Upload pano with token

* Update los table name

* Refactor

* Reduce cluster precision

* Refactor

* Add los db script

* Update readme

* Add building_data

* Update acuity webhook handler

.gitignore

@@ -1,6 +1,6 @@
 node_modules/
 functions/
-data/
+building_data/
 .env
 .DS_Store
-.netlify
+.netlify

README.md

@@ -68,27 +68,23 @@ Currently, we use node numbers to represent join requests, members, and nodes. T
 
 A physical location.
 
-- id
-- address
-- lat
-- lng
-- alt
-- bin (optional) // NYC Building ID Number
-- notes (optional)
+| id  | address | lat | lng | alt | bin | notes |
+| --- | ------- | --- | --- | --- | --- | ----- |
 
 ### Member
 
 A person in the mesh community. For example, a node-owner, donor or installer.
 
-- id
-- name
-- email
-- phone
+| id  | name | email | phone |
+| --- | ---- | ----- | ----- |
 
 ### Node
 
 A specific location on the network. Typically one per building.
 
+| id  | lat | lng | alt | status | name | location |
+| --- | --- | --- | --- | ------ | ---- | -------- |
+
 - id
 - lat
 - lng
@@ -335,39 +331,52 @@ ORDER BY building_height DESC;
 
 ### DB Setup
 
-Install dependencies:
+Install lxml:
 
-- Python
-- Postgres
-- PostGIS
-
-Create a table in the db:
-
-```sql
-CREATE TABLE ny(gid SERIAL PRIMARY KEY, bldg_id varchar(255), bldg_bin varchar(255), geom GEOMETRY('MULTIPOLYGONZ', 2263))
+```python
+pip3 install lxml
 ```
 
-Download the [building data](https://www1.nyc.gov/site/doitt/initiatives/3d-building.page):
+Set up the db:
 
 ```bash
-curl -o data.zip http://maps.nyc.gov/download/3dmodel/DA_WISE_GML.zip
-unzip data.zip -d data
-rm data.zip
+node scripts/reset-los-db.js
 ```
 
-Insert the data:
+Download the [building data](https://www1.nyc.gov/site/doitt/initiatives/3d-building.page):
 
 ```bash
-# 12 and 13 are Manhattan
-python2 ./scripts/gml_to_pgsql.py ./data/DA_WISE_GMLs/DA12_3D_Buildings_Merged.gml ny | psql db
-python2 ./scripts/gml_to_pgsql.py ./data/DA_WISE_GMLs/DA13_3D_Buildings_Merged.gml ny | psql db
+curl -o building_data.zip http://maps.nyc.gov/download/3dmodel/DA_WISE_GML.zip
+unzip building_data.zip -d building_data
+rm building_data.zip
 ```
 
-Create indices:
+Insert the data
 
-```sql
-CREATE INDEX geom_index ON ny USING GIST (geom);
-CREATE INDEX bin_index ON ny (bldg_bin);
+```bash
+{
+	python3 ./scripts/gml_to_pgsql.py ./building_data/DA_WISE_GMLs/DA1_3D_Buildings_Merged.gml buildings
+	python3 ./scripts/gml_to_pgsql.py ./building_data/DA_WISE_GMLs/DA2_3D_Buildings_Merged.gml buildings
+	python3 ./scripts/gml_to_pgsql.py ./building_data/DA_WISE_GMLs/DA3_3D_Buildings_Merged.gml buildings
+	python3 ./scripts/gml_to_pgsql.py ./building_data/DA_WISE_GMLs/DA4_3D_Buildings_Merged.gml buildings
+	python3 ./scripts/gml_to_pgsql.py ./building_data/DA_WISE_GMLs/DA5_3D_Buildings_Merged.gml buildings
+	python3 ./scripts/gml_to_pgsql.py ./building_data/DA_WISE_GMLs/DA6_3D_Buildings_Merged.gml buildings
+	python3 ./scripts/gml_to_pgsql.py ./building_data/DA_WISE_GMLs/DA7_3D_Buildings_Merged.gml buildings
+	python3 ./scripts/gml_to_pgsql.py ./building_data/DA_WISE_GMLs/DA8_3D_Buildings_Merged.gml buildings
+	python3 ./scripts/gml_to_pgsql.py ./building_data/DA_WISE_GMLs/DA9_3D_Buildings_Merged.gml buildings
+	python3 ./scripts/gml_to_pgsql.py ./building_data/DA_WISE_GMLs/DA10_3D_Buildings_Merged.gml buildings
+	python3 ./scripts/gml_to_pgsql.py ./building_data/DA_WISE_GMLs/DA11_3D_Buildings_Merged.gml buildings
+	python3 ./scripts/gml_to_pgsql.py ./building_data/DA_WISE_GMLs/DA12_3D_Buildings_Merged.gml buildings
+	python3 ./scripts/gml_to_pgsql.py ./building_data/DA_WISE_GMLs/DA13_3D_Buildings_Merged.gml buildings
+	python3 ./scripts/gml_to_pgsql.py ./building_data/DA_WISE_GMLs/DA14_3D_Buildings_Merged.gml buildings
+	python3 ./scripts/gml_to_pgsql.py ./building_data/DA_WISE_GMLs/DA15_3D_Buildings_Merged.gml buildings
+	python3 ./scripts/gml_to_pgsql.py ./building_data/DA_WISE_GMLs/DA16_3D_Buildings_Merged.gml buildings
+	python3 ./scripts/gml_to_pgsql.py ./building_data/DA_WISE_GMLs/DA17_3D_Buildings_Merged.gml buildings
+	python3 ./scripts/gml_to_pgsql.py ./building_data/DA_WISE_GMLs/DA18_3D_Buildings_Merged.gml buildings
+	python3 ./scripts/gml_to_pgsql.py ./building_data/DA_WISE_GMLs/DA19_3D_Buildings_Merged.gml buildings
+	python3 ./scripts/gml_to_pgsql.py ./building_data/DA_WISE_GMLs/DA20_3D_Buildings_Merged.gml buildings
+	python3 ./scripts/gml_to_pgsql.py ./building_data/DA_WISE_GMLs/DA21_3D_Buildings_Merged.gml buildings
+} | psql $LOS_DATABASE_URL
 ```
 
 Now we are ready to make queries!

migrations/1613282969996_request-tokens.js

@@ -0,0 +1,14 @@
+/* eslint-disable camelcase */
+
+exports.shorthands = undefined;
+
+exports.up = (pgm) => {
+  pgm.createTable("request_tokens", {
+    token: { type: "varchar(255)", notNull: true },
+    request_id: { type: "integer", references: "requests(id)", notNull: true },
+  });
+};
+
+exports.down = (pgm) => {
+  pgm.dropTable("request_tokens");
+};

scripts/db.js

@@ -9,15 +9,15 @@ async function createPool(connectionString) {
     connectionString,
     ssl: sslOptions(params.hostname),
   });
-}
 
-// See src/db/index.js
-function sslOptions(host) {
-  if (host === "localhost" || host === "127.0.0.1") return false;
-  return {
-    rejectUnauthorized: false,
-    mode: "require",
-  };
+  // See src/db/index.js
+  function sslOptions(host) {
+    if (host === "localhost" || host === "127.0.0.1") return false;
+    return {
+      rejectUnauthorized: false,
+      mode: "require",
+    };
+  }
 }
 
 async function performQuery(text, values) {

scripts/import-spreadsheet.js

@@ -624,7 +624,7 @@ GROUP BY buildings.id`,
         building = matchingAddress[0];
       }
     } else {
-      [building] = buildings;
+      building = buildings[0];
     }
 
     if (!building) {
@@ -719,7 +719,7 @@ function getClusteredNodes(nodes) {
   return Object.values(clusterMap);
 
   function geoKey(node) {
-    const precision = 4;
+    const precision = 5;
     const [lng, lat] = node.coordinates;
     return `${parseFloat(lat).toFixed(precision)}-${parseFloat(lng).toFixed(
       precision

scripts/reset-los-db.js

@@ -0,0 +1,48 @@
+require("dotenv").config();
+const { Pool } = require("pg");
+const url = require("url");
+
+createTables().then(() => process.exit(0));
+
+async function createTables() {
+  const pool = createPool(process.env.LOS_DATABASE_URL);
+  await performQuery(pool, "CREATE EXTENSION IF NOT EXISTS postgis");
+  await performQuery(pool, "CREATE EXTENSION IF NOT EXISTS postgis_sfcgal");
+  await performQuery(pool, "DROP TABLE IF EXISTS buildings");
+  await performQuery(
+    pool,
+    "CREATE TABLE IF NOT EXISTS buildings(gid SERIAL PRIMARY KEY, bldg_id varchar(255), bldg_bin varchar(255), geom GEOMETRY('MULTIPOLYGONZ', 2263))"
+  );
+  await performQuery(
+    pool,
+    "CREATE INDEX IF NOT EXISTS geom_index ON buildings USING GIST (geom)"
+  );
+  await performQuery(
+    pool,
+    "CREATE INDEX IF NOT EXISTS bin_index ON buildings (bldg_bin)"
+  );
+}
+
+async function performQuery(pool, text, values) {
+  const client = await pool.connect();
+  const result = await client.query(text, values);
+  client.release();
+  return result.rows;
+}
+
+function createPool(connectionString) {
+  const params = url.parse(connectionString);
+  return new Pool({
+    connectionString,
+    ssl: sslOptions(params.hostname),
+  });
+
+  // See src/db/index.js
+  function sslOptions(host) {
+    if (host === "localhost" || host === "127.0.0.1") return false;
+    return {
+      rejectUnauthorized: false,
+      mode: "require",
+    };
+  }
+}

src/db/los.js

@@ -92,7 +92,7 @@ export async function getLos(bin) {
 
 async function getBuildingMidpoint(bin) {
   const text =
-    "SELECT ST_AsText(ST_Centroid((SELECT geom FROM ny WHERE bldg_bin = $1)))";
+    "SELECT ST_AsText(ST_Centroid((SELECT geom FROM buildings WHERE bldg_bin = $1)))";
   const values = [bin];
   const res = await performLosQuery(text, values);
   if (!res.length) throw new Error("Not found");
@@ -105,7 +105,8 @@ async function getBuildingMidpoint(bin) {
 
 export async function getBuildingHeight(bin) {
   try {
-    const text = "SELECT ST_ZMax((SELECT geom FROM ny WHERE bldg_bin = $1))";
+    const text =
+      "SELECT ST_ZMax((SELECT geom FROM buildings WHERE bldg_bin = $1))";
     const values = [bin];
     const res = await performLosQuery(text, values);
     if (!res.length) throw new Error("Not found");
@@ -136,15 +137,15 @@ FROM (
   SELECT
     *
   FROM
-    ny
+    buildings
   WHERE
     bldg_bin = ANY ($1)) AS hubs
 WHERE
   ST_DWithin (ST_Centroid(geom), (
       SELECT
         ST_Centroid(geom)
       FROM
-        ny
+        buildings
       WHERE
         bldg_bin = $2), $3)`,
     [nodeBins, bin, range]

src/db/panos.js

@@ -12,6 +12,8 @@ const s3 = new AWS.S3({
 export async function getUploadURL(name, type) {
   if (!name || !type) throw new Error("Bad params");
 
+  // const timestamp =
+
   // TODO: Validate content type?
 
   const url = await s3.getSignedUrl("putObject", {
@@ -24,20 +26,23 @@ export async function getUploadURL(name, type) {
   return url;
 }
 
-const insertPanoQuery =
-  "INSERT INTO panoramas (url, date, request_id) VALUES ($1, $2, $3) RETURNING *";
+export async function createPano({ url, request_id }, slackClient) {
+  if (!url) throw new Error("Bad params");
+
+  const [
+    pano,
+  ] = await performQuery(
+    "INSERT INTO panoramas (url, date, request_id) VALUES ($1, $2, $3) RETURNING *",
+    [url, new Date(), request_id]
+  );
 
-// TODO: Restrict this somehow. Maybe require a secret?
-export async function savePano(requestId, panoURL, slackClient) {
-  if (!requestId || !panoURL) throw new Error("Bad params");
-  const values = [panoURL, new Date(), requestId];
-  const [pano] = await performQuery(insertPanoQuery, values);
   try {
-    const request = await getRequest(requestId);
+    const request = await getRequest(request_id);
     await panoMessage(slackClient, pano, request);
   } catch (error) {
     console.log("Failed to send pano slack message!");
     console.log(error.message);
   }
+
   return pano;
 }

src/db/requests.js

@@ -1,3 +1,4 @@
+import crypto from "crypto";
 import fetch from "node-fetch";
 import { getLos, getBuildingHeightMeters } from "./los";
 import { requestMessage } from "../slack";
@@ -28,6 +29,29 @@ GROUP BY
   return request;
 }
 
+// Get request without api key using a secret token
+export async function getRequestFromToken(token) {
+  if (!token) throw new Error("Bad params");
+  const [request] = await performQuery(
+    `SELECT
+  requests.*,
+  to_json(buildings) AS building,
+  COALESCE(json_agg(DISTINCT panoramas) FILTER (WHERE panoramas IS NOT NULL), '[]') AS panoramas
+FROM
+  requests
+  JOIN buildings ON requests.building_id = buildings.id
+  LEFT JOIN panoramas ON requests.id = panoramas.request_id
+WHERE
+  requests.id = $1
+GROUP BY
+  requests.id,
+  buildings.id`,
+    [id]
+  );
+  if (!request) throw new Error("Not found");
+  return request;
+}
+
 export async function getRequests() {
   return performQuery(`SELECT
   requests.*,
@@ -129,6 +153,14 @@ export async function createRequest(request, slackClient) {
     [now, apartment, roof_access || roofAccess, member.id, building.id]
   );
 
+  // Create token
+  const buffer = await crypto.randomBytes(8);
+  const token = buffer.toString("hex");
+  await performQuery(
+    "INSERT INTO request_tokens (token, request_id) VALUES ($1, $2) RETURNING *",
+    [token, dbRequest.id]
+  );
+
   // Get los
   let visibleNodes = [];
   try {
@@ -161,7 +193,10 @@ export async function createRequest(request, slackClient) {
 
   dbRequest = await getRequest(dbRequest.id);
 
-  return dbRequest;
+  return {
+    ...dbRequest,
+    token,
+  };
 }
 
 async function sendSlackMessage({