Refactor DB Schema (#10)

* Working admin, broken views

* reset schema to accomodate max flexibility using json field type

* Refactor db schema to accomodate max flexibility using json fields

* create views from user-submitted data from which to create UI elements in front-end

* Auto-generate ERD diagram

* reinstate theme table (instead of view)

* update views

* annotate tables

* remove optional nature of study slug

* convert metrics id into compound id

* convert metrics_metadata id into composite id and add map_display index

* remove unnecessary index

* add missing index to geometries table

* continue removing unnecessary index specifications

* Add step to lint codebase for unstaged ERDs

* Update ERD

* Upload failed prisma builds

* Use slugs rather than IDs, refactor

* Clarify terminology in readme

* Refine models

* Add prototyping data

* Continue example queries

* More examples

* Add functions and seed

Still just for dev, need to be moved to migration and seed.ts

* Rm schema

* Rm shadowDatabaseUrl

As per prisma/prisma#19234 (comment)

* Create migration for updated models

* Create migration for functions

* Working start to seed

* Undo admin logic

* Revert "Working admin, broken views"

This reverts commit 332a54a.

* Cleanup

* Cascade deletes, delete study before ingestion

* Squash migrations

* Update log

* Stage progress

* Refine

* Update to store src_field in aggregations

* Add pre-aggregation to ingestion

* Add units & description to aggregations

* Change term

* Rename field

* Continue buildout

* Working ingestion

* Add ingestion data

* Disable ERD generation

* Update model to fit new DB schema

* Update spatial queries

* Fix build

* Fix attribute name

* Tighten typings to avoid build issues

* Reference origin for tile host

* Delete prisma/views/public/vw_map_fields.sql

* Delete seed.sql

* Cleanup ERD generation

* Add migration to relate themes with scenarios

* Support theme_scenario through table

* Fixup app data model

* Add ERD dependency

* Re-activate scenario control

* Update ERD

* Slim down data

* Simplify

* Update README.md

* Disable buggy ERD generation

* Cleanup

* Avoid creating theme_scenario for baseline scenario

* Avoid global window error

* Fix geo data

* Apply suggestions from code review

Co-authored-by: Tammo Feldmann <31222040+Tammo-Feldmann@users.noreply.github.com>

* Cleanup SQL

* Delete .github/workflows/lint.yaml

---------

Co-authored-by: Emma Paz <emma@developmentseed.org>
Co-authored-by: Tammo Feldmann <31222040+Tammo-Feldmann@users.noreply.github.com>

.gitignore

@@ -36,5 +36,4 @@ yarn-error.log*
 .turbo
 
 # typescript
-*.tsbuildinfo
-data
+*.tsbuildinfo

README.md

@@ -6,6 +6,32 @@ Based off of the [Vercel Postgres + Prisma Next.js Starter](https://vercel.com/t
 - [Prisma](https://www.prisma.io/) - database modeling/ORM
 - [Tailwind](https://tailwindcss.com/) - CSS framework
 
+## Data Submission
+
+Data is modeled with the following concepts:
+
+- Study
+- Theme: Belongs to study, represents a field of research for a given study.
+- Scenario: A way that would modify the outcome of a study.
+
+To provide data, two files must be provided:
+
+1. A XLSX spreadsheet file for a study, named `{study_identifier}.xlsx`
+1. A GeoJSON file containing geometries associated with the study, named `{study_identifier}.geojson`
+
+### Study Spreadsheet
+
+The study spreadsheet must contain the following worksheets:
+
+- `study`: metadata about the study. Column names are case insensitive and asterisks are removed prior to ingestion.
+- `metrics`: raw data. the first column is expected to be the value used for matching geometries
+- `metrics_metadata`
+
+### Study Geometries
+
+- `FeatureCollection` of `Polygon` or `MultiPolygon` values.
+- Each `Feature` must contain a unique `id` property of either a string or integer.
+
 ## Development
 
 ### Install
@@ -35,6 +61,12 @@ Seeding the database is the conventional entry to loading data into our applicat
 pnpm prisma db seed
 ```
 
+### Data Model
+
+![terminology](./.docs/terminology.png)
+
+![entity relationsip diagram](./prisma/ERD.svg)
+
 #### Buildings
 
 To turn our Shapefile of buildings (`shapefile.shp`) in EPSG:32729 into a GeoJSON in EPSG:4326:

app/api/search/[study_slug]/[coordinates]/route.ts

@@ -0,0 +1,29 @@
+import prisma from "@/lib/prisma"
+import { Prisma } from "@prisma/client"
+
+export async function GET(req, { params }) {
+  const { coordinates, study_slug } = params
+  const lineStringCoords = Prisma.raw(decodeURI(coordinates))
+
+  const buildings = await prisma.$queryRaw`
+    SELECT 
+      key
+    FROM 
+      geometries 
+    WHERE
+      ST_Intersects(
+        ST_Transform(geom, 3857),
+        ST_Transform(
+          ST_Polygon(
+            'LINESTRING(${lineStringCoords})'::geometry, 
+            4326
+          ), 
+          3857
+        )
+      )
+      AND
+      study_slug = ${study_slug}
+  `
+
+  return Response.json({ buildings })
+}

app/api/tiles/[z]/[x]/[y]/route.ts → app/api/tiles/[study_slug]/[z]/[x]/[y]/route.ts

@@ -10,7 +10,7 @@ export async function GET(req: NextRequest, { params }: { params: Params }) {
   } catch (e) {
     return new Response((e as Error).message, { status: 400 })
   }
-  const sql = tile.asSql("buildings", "geom", ["properties"])
+  const sql = tile.asSql("geometries", "geom")
   const [{ st_asmvt }] = await prisma.$queryRaw<{ st_asmvt: Buffer }[]>(sql)
 
   return new Response(
@@ -28,12 +28,18 @@ export async function GET(req: NextRequest, { params }: { params: Params }) {
 }
 
 class Tile {
+  public study_slug: string
   // https://github.com/pramsey/minimal-mvt/blob/8b736e342ada89c5c2c9b1c77bfcbcfde7aa8d82/minimal-mvt.py#L50-L150
   public zoom: number
   public x: number
   public y: number
 
-  constructor({ z, x, y }: Params) {
+  constructor({ z, x, y, study_slug }: Params) {
+    if (!study_slug) {
+      throw new Error("study_slug is required")
+    }
+    this.study_slug = study_slug
+
     if ([z, x, y].map(val => parseInt(val)).some(isNaN)) {
       throw new Error("Coordinates must be numbers")
     }
@@ -76,14 +82,23 @@ class Tile {
     const DENSIFY_FACTOR = 4
     const segSize = (env.xmax - env.xmin) / DENSIFY_FACTOR
     return Prisma.sql`
-      ST_Segmentize(ST_MakeEnvelope(${env.xmin}, ${env.ymin}, ${env.xmax}, ${env.ymax}, 3857), ${segSize})
+      ST_Segmentize(
+        ST_MakeEnvelope(
+          ${env.xmin},
+          ${env.ymin},
+          ${env.xmax},
+          ${env.ymax},
+          3857
+        ),
+        ${segSize}
+      )
     `
   }
 
   public asSql(
     table: string,
     geomColumn: string,
-    attrColumns: string[],
+    attrColumns: string[] = [],
     srid: number = 4326
   ): Sql {
     const envSql = this.asEnvelopeSql()
@@ -93,19 +108,34 @@ class Tile {
       Object.entries({ table, geomColumn }).map(([k, v]) => [k, Prisma.raw(v)])
     )
     return Prisma.sql`
-      WITH 
-      bounds AS (
-          SELECT ${envSql} AS geom, 
-                ${envSql}::box2d AS b2d
+      WITH bounds AS (
+        SELECT
+          ${envSql} AS geom,
+          ${envSql}::box2d AS b2d
       ),
       mvtgeom AS (
-          SELECT ST_AsMVTGeom(ST_Transform(t.${rawVals.geomColumn}, 3857), bounds.b2d) AS geom,
-          name,
+        SELECT
+          ST_AsMVTGeom(
+            ST_Transform(t.${rawVals.geomColumn}, 3857),
+            bounds.b2d
+          ) AS geom,
+          key,
           43 AS height
-          FROM ${rawVals.table} t, bounds
-          WHERE ST_Intersects(t.${rawVals.geomColumn}, ST_Transform(bounds.geom, ${srid}::integer))
-      ) 
-      SELECT ST_AsMVT(mvtgeom.*) FROM mvtgeom
+        FROM
+          ${rawVals.table} t,
+          bounds
+        WHERE
+          ST_Intersects(
+            t.${rawVals.geomColumn},
+            ST_Transform(bounds.geom, ${srid}::integer)
+          )
+          AND
+          study_slug = ${this.study_slug}
+      )
+      SELECT
+        ST_AsMVT(mvtgeom.*)
+      FROM
+        mvtgeom
     `
   }
 }
@@ -114,6 +144,7 @@ interface Params {
   x: string
   y: string
   z: string
+  study_slug: string
 }
 
 interface Envelope {

app/explore/[slug]/page.tsx

@@ -1,9 +1,10 @@
 import { notFound } from "next/navigation"
 import { useStore } from "@/app/lib/store"
-import { getStudies, getStudy } from "@/app/lib/data"
+import { getStudy } from "@/app/lib/data"
 import Explore from "@/components/explore"
 import StoreInitialize from "@/components/store-initialize"
 import { Header } from "@/components/header"
+import { scenario } from "@prisma/client"
 
 export default async function ExplorePage({
   params,
@@ -20,8 +21,9 @@ export default async function ExplorePage({
       acc[theme.slug] = {
         ...theme,
         selectedScenario: null,
-        scenarios: theme?.scenarios.reduce((acc, scenario) => {
-          acc[scenario.slug] = scenario
+        scenarios: theme?.scenarios.reduce((acc, themeScenario) => {
+          if (!themeScenario.scenario_slug) return acc
+          acc[themeScenario.scenario_slug] = themeScenario.scenario
           return acc
         }, {}),
       }
@@ -30,12 +32,14 @@ export default async function ExplorePage({
     selectedTheme: {
       ...study.themes[0],
       selectedScenario: { slug: "", name: "", description: "" },
-      scenarios: study.themes[0]?.scenarios,
+      scenarios: study.themes[0]?.scenarios.map(
+        themeScenario => themeScenario.scenario as scenario
+      ),
     },
     selectedThemeId: study.themes[0]?.slug,
     totalSelectedFeatures: 0,
     isDrawing: false,
-    aoi: { feature: null, bbox: null },
+    aoi: { feature: undefined, bbox: [] },
   }
   const stateObject = {
     selectedStudy,

app/lib/data.ts

@@ -1,10 +1,16 @@
 import { cache } from "react"
 import prisma from "@/lib/prisma"
+import { study } from "@prisma/client"
+
+export const getStudies: () => Promise<study[]> = cache(prisma.study.findMany)
 
-export const getStudies = cache(prisma.study.findMany)
 export const getStudy = cache((slug: string) =>
   prisma.study.findUnique({
     where: { slug },
-    include: { themes: { include: { scenarios: true } } },
+    include: {
+      themes: {
+        include: { scenarios: { include: { scenario: true } } },
+      },
+    },
   })
 )

app/lib/store.ts

@@ -7,7 +7,7 @@ interface InitialState {
   setTotalSelectedFeatures: (total: number) => void
   setIsDrawing: (isDrawing: boolean) => void
   setSelectedTheme: (theme: Studies.Theme) => void
-  setSelectedScenario: (scenarioId: Studies.Scenario) => void
+  setSelectedScenario: (scenarioId: Studies.Scenario | null) => void
   // setSelectedStudy: (study: Studies.Study, themes: Studies.Theme[]) => void
 }
 
@@ -28,7 +28,7 @@ export const useStore = create<InitialState>((set, get) => ({
     selectedTheme: {
       name: "",
       slug: "",
-      selectedScenario: { slug: "", name: "", description: "" },
+      selectedScenario: null,
       scenarios: [],
     },
   },
@@ -64,7 +64,7 @@ export const useStore = create<InitialState>((set, get) => ({
           ...state.selectedStudy.themes,
           [state.selectedStudy.selectedTheme.slug]: {
             ...state.selectedStudy.selectedTheme,
-            selectedScenario: scenario,
+            selectedScenario: scenario ? scenario : null,
           },
         },
       },

components/card-grid.tsx

@@ -3,16 +3,17 @@ import { getStudies } from "@/app/lib/data"
 
 export default async function CardGrid() {
   const studies = await getStudies()
+
   return (
     <div className="container pr-4 md:pr-0 md:mx-auto py-8 md:max-w-max">
       <div className="grid grid-cols-1 sm:grid-cols-2 md:grid-cols-3 lg:grid-cols-4 gap-6">
-        {studies?.map(study => (
+        {studies.map(study => (
           <Card
             key={study.slug}
             description={study.description}
             title={study.name}
             href={`explore/${study.slug}`}
-            src={study.imageSrc}
+            imgSrc={study.image_src || undefined}
           />
         ))}
       </div>

components/card.tsx

@@ -2,27 +2,29 @@ import Image from "next/image"
 import Link from "next/link"
 
 interface Props {
-  src: string
+  imgSrc?: string
   title: string
   description: string
   href: string
 }
-const Card: React.FC<Props> = ({ src, title, description, href }) => {
+const Card: React.FC<Props> = ({ imgSrc, title, description, href }) => {
   return (
     <Link
       href={href}
       className="md:max-w-sm rounded shadow-lg p-4 block transition duration-200 ease-in-out hover:shadow-xl border border-transparent hover:border-blue-300 bg-white"
     >
-      <div className="relative h-[150px]">
-        <Image
-          className="top-0 left-0 bottom-0 w-full h-full object-cover rounded-t"
-          src={src}
-          fill={true}
-          alt="Card Image"
-          priority={true}
-          sizes="(max-width: 768px) 100vw, (max-width: 1200px) 50vw, 33vw"
-        />
-      </div>
+      {imgSrc && (
+        <div className="relative h-[150px]">
+          <Image
+            className="top-0 left-0 bottom-0 w-full h-full object-cover rounded-t"
+            src={imgSrc}
+            fill={true}
+            alt="Card Image"
+            priority={true}
+            sizes="(max-width: 768px) 100vw, (max-width: 1200px) 50vw, 33vw"
+          />
+        </div>
+      )}
       <div className="py-4">
         <div className="font-bold text-l mb-2">{title}</div>
         <p className="text-gray-700 text-base">{description}</p>

components/explore.tsx

@@ -29,7 +29,7 @@ const Explore: React.FC<Props> = ({ params, metaData }) => {
     <>
       <SidePane
         {...{
-          src: metaData.imageSrc,
+          imgSrc: metaData.imageSrc,
 
           studyId: params.slug,
         }}
@@ -41,13 +41,16 @@ const Explore: React.FC<Props> = ({ params, metaData }) => {
             zoom: mapZoom,
             center: mapCenter,
             layerType,
+            studySlug: params.slug,
           }}
         >
           <Source
             id="building-footprints"
             promoteId={"name"}
             type="vector"
-            tiles={[`${globalVariables.basePath}/api/tiles/{z}/{x}/{y}`]}
+            tiles={[
+              `${global.window?.location.origin}/api/tiles/${params.slug}/{z}/{x}/{y}`,
+            ]}
             minzoom={6}
             maxzoom={14}
           >

components/map/draw-bbox-control.tsx

@@ -92,8 +92,8 @@ function DrawBboxControl({
   }, [map, handleDraw, handleSelection, handleUpdate])
 
   const resetMapFeatures = useCallback(() => {
-    drawControlRef.current.deleteAll()
-    drawControlRef.current.changeMode("draw_polygon")
+    drawControlRef.current?.deleteAll()
+    drawControlRef.current?.changeMode("draw_polygon")
     map.getCanvas().style.cursor = "crosshair"
   }, [map])