Move from raw SQL access to GraphQL / REST api for now (#40)

* [GraphiQL] Stub in new UI (left old one for now)

* [SQL] Stub in "contact us" page for now

* [SQL] Updated button icon to be email

* [GraphiQL] Update styling & themes

* [GraphiQL] Remove title area

* [APIs] Add back "Query API"

* [Integrations] Add settings button

* [Integrations] Plaid settings page

* [Integrations] Disable plaid settings for now

* [Nav] Ensure nested routes highlight side nav

* localStorage.DISABLE_ZOD_FORM_FOR_NOW = true to bypass the zod form

* Update view definition to work better with graphql

* Hook up graphql editor with the right headers

* Experimental rest API explorer via stoplight elements

* Moving to the new api-access page

* Persist active tab into the query param

* Ensure default ledger no longer depends on createDatabaseUser

* Renaming old and new api access pages

* Add realtime api also

* Remove redundant prefix

* [API] Tweaks

* [Rest] Comment out import since it pollutes global CSS

Even without the page loaded into another page

* [API] Comment old page for now

* [API] Remove Rest explorer for now

* [Export] Remove & style sandbox labels

* [SQL] Remove old access page for now

* [Plaid] Disable preconnect input for now

* [Connections] Tweak UI to be easier to read

* [API] Show API keys & routes in first tab

* [API] Move REST api docs to their own page

* [API docs] Punch out to view REST api docs full page

* Implement proxy for graphql and rest requests against supabase

* [REST] Make docs use dark mode

* [REST] Set info section to override Postgres API name

* Handle apiKey as well during request proxy

* Fix joinPath and add more tests

* Use personal access token in the api explorer

* Remove token param when proxying

* Hooking up the api access page with the correct token

* Do not pass the header onwards

* Create personal access token if needed

* Removing sql endpoint and no more database end user also

* Add migration to remove existing database users

* Fix migration on vanilla postgres

* Declare auth_user as record to make vanilla migration pass

* Actually fix migration test

* Hide the migrations table from api

* Backing up a couple functions we no longer use

* Fix ci again, we should use the supabase postgres image

---------

Co-authored-by: Rob Phillips <rob@robphillips.me>

.github/workflows/validate-workflow.yml

@@ -21,7 +21,7 @@ jobs:
           # optional (defaults to `5432`)
           POSTGRES_PORT: 5432
           # optional (defaults to `postgres`)
-          POSTGRES_USER: test
+          POSTGRES_USER: postgres
         ports:
           # maps tcp port 5432 on service container to the host
           - 5432:5432
@@ -76,7 +76,9 @@ jobs:
         run: POSTGRES_OR_WEBHOOK_URL=noop NEXT_PUBLIC_SUPABASE_URL=noop NEXT_PUBLIC_SUPABASE_ANON_KEY=noop node --loader tsx ./bin/venice.ts health
 
       - name: Run migration check
-        run: POSTGRES_OR_WEBHOOK_URL=postgres://test:test@localhost:5432/test pnpm migration up
+        run: POSTGRES_OR_WEBHOOK_URL=postgres://postgres:test@localhost:5432/test pnpm migration up
+        # To test this with a locally install postgres, run
+        # psql postgres -c 'drop database if exists test;' && psql postgres -c 'create database test;' && POSTGRES_OR_WEBHOOK_URL=postgres://localhost:5432/test pnpm migration up
 
       - name: Run lint
         run: pnpm run lint

apps/app-config/backendConfig.ts

@@ -15,7 +15,7 @@ import {joinPath, R, Rx, zParser} from '@usevenice/util'
 import {veniceCommonConfig} from './commonConfig'
 import type {PROVIDERS} from './env'
 import {parseIntConfigsFromRawEnv, zAllEnv} from './env'
-import {getServerUrl} from './server-url'
+import {getServerUrl} from './constants'
 
 export {Papa} from '@usevenice/integration-import'
 export {makePostgresClient} from '@usevenice/integration-postgres'

apps/app-config/commonConfig.ts

@@ -3,7 +3,7 @@ import {VeniceProvider} from '@usevenice/engine-frontend'
 import {joinPath, zParser} from '@usevenice/util'
 
 import {PROVIDERS, zCommonEnv} from './env'
-import {getServerUrl} from './server-url'
+import {getServerUrl} from './constants'
 
 export {Papa} from '@usevenice/integration-import'
 

apps/app-config/server-url.ts → apps/app-config/constants.ts

@@ -15,3 +15,11 @@ export function getServerUrl(req: GetServerSidePropsContext['req'] | null) {
     }`
   )
 }
+
+export const graphqlEndpoint = new URL('/api/graphql', getServerUrl(null))
+
+export const restEndpoint = new URL('/api/rest', getServerUrl(null))
+
+export const xPatHeaderKey = 'x-token'
+export const xPatUrlParamKey = 'token'
+export const xPatUserMetadataKey = 'apiKey'

apps/cli/admin-queries/2023-02-27_1710 json and plv8 test.sql

@@ -0,0 +1,32 @@
+CREATE OR REPLACE FUNCTION public.xjsonb_object_keys(obj jsonb)
+ RETURNS text[]
+ LANGUAGE plv8
+ IMMUTABLE STRICT
+AS $function$
+    return Object.keys(obj);
+$function$
+
+CREATE OR REPLACE FUNCTION public.jsonb_object_keys_to_text_array(_js jsonb)
+ RETURNS text[]
+ LANGUAGE sql
+ IMMUTABLE PARALLEL SAFE STRICT
+AS $function$SELECT ARRAY(SELECT jsonb_object_keys(_js))$function$
+
+CREATE OR REPLACE FUNCTION public.plv8_test(keys text[], vals text[])
+ RETURNS json
+ LANGUAGE plv8
+ IMMUTABLE STRICT
+AS $function$
+    var o = {};
+    for(var i=0; i<keys.length; i++){
+        o[keys[i]] = vals[i];
+    }
+    return o;
+$function$
+
+
+CREATE OR REPLACE FUNCTION public.jsonb_array_to_text_array(_js jsonb)
+ RETURNS text[]
+ LANGUAGE sql
+ IMMUTABLE PARALLEL SAFE STRICT
+AS $function$SELECT ARRAY(SELECT jsonb_array_elements_text(_js))$function$

apps/cli/admin-queries/createAndDropUser.sql

@@ -0,0 +1,15 @@
+-- Create user
+CREATE USER "usr_deca735c-a043-4394-b12d-33e4b3717a83" PASSWORD "key_test";
+GRANT "usr_deca735c-a043-4394-b12d-33e4b3717a83" TO "postgres";
+GRANT USAGE ON SCHEMA public TO "usr_deca735c-a043-4394-b12d-33e4b3717a83";
+GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO "usr_deca735c-a043-4394-b12d-33e4b3717a83";
+REVOKE ALL PRIVILEGES ON public.migrations FROM "usr_deca735c-a043-4394-b12d-33e4b3717a83";
+REVOKE ALL PRIVILEGES ON public.integration FROM "usr_deca735c-a043-4394-b12d-33e4b3717a83";
+UPDATE auth.users SET raw_user_meta_data = raw_user_meta_data || 'key_test' WHERE id = 'deca735c-a043-4394-b12d-33e4b3717a83';
+
+-- Drop user
+REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM "usr_deca735c-a043-4394-b12d-33e4b3717a83";
+REVOKE USAGE ON SCHEMA public FROM "usr_deca735c-a043-4394-b12d-33e4b3717a83";
+DROP USER "usr_deca735c-a043-4394-b12d-33e4b3717a83";
+UPDATE auth.users SET raw_user_meta_data = raw_user_meta_data - 'apiKey' WHERE id = ${userId};
+

apps/cli/admin-queries/drop graphile worker and pg_cron.sql

@@ -0,0 +1,2 @@
+drop schema graphile_worker CASCADE;
+

apps/cli/admin-queries/lockdown public role.sql

@@ -0,0 +1,29 @@
+
+REVOKE ALL ON pg_catalog.pg_user FROM public;
+-- This will prevent user from being able to query for the existance of other users.
+-- TODO: What other permissions does the `public` role have that they shouldn't have in this context?
+
+
+-- TODO: We ran this query on dev postgres accidentally, how do we revert it back?
+
+
+SELECT grantee, privilege_type
+FROM information_schema.role_table_grants
+WHERE table_name='pg_user';
+
+SELECT
+	*
+FROM
+	information_schema.table_privileges
+WHERE
+	grantee = 'PUBLIC' and "table_name" = 'pg_user';
+
+select * from pg_shadow;
+
+grant select on pg_catalog.pg_user to public;
+
+
+SELECT *
+FROM pg_default_acl;
+
+select * from pg_authid;

apps/web/components/PageLayout/AuthLayout/NavLink.tsx

@@ -14,7 +14,8 @@ interface NavLinkProps {
 
 export function NavLink({name, href, external, icon: Icon}: NavLinkProps) {
   const router = useRouter()
-  const current = router.pathname === href
+  const current =
+    router.pathname === href || router.pathname.startsWith(href + '/')
   const Link = external ? ExternalLink : NextLink
   return (
     <div key={name} className="relative px-4">

apps/web/components/PageLayout/AuthLayout/Sidebar.tsx

@@ -1,8 +1,7 @@
 import {
   ArrowLeftRightIcon,
-  DatabaseIcon,
+  CodeIcon,
   DocsIcon,
-  ExportIcon,
   IntegrationsIcon,
   ProfileIcon,
   SupportIcon,
@@ -17,11 +16,10 @@ const mainNavigation = [
     href: '/connections',
     icon: ArrowLeftRightIcon,
   },
-  {name: 'Export Data', href: '/export', icon: ExportIcon},
   {
     name: 'API Access',
     href: '/api-access',
-    icon: DatabaseIcon,
+    icon: CodeIcon,
   },
   {
     name: 'Integrations',

apps/web/components/ResourceCard.tsx

@@ -16,7 +16,7 @@ export function ResourceCard(props: ResourceCardProps) {
     <Card bgColor={bgColor}>
       <div
         className={clsx(
-          'relative min-h-[6.5rem] rounded-lg',
+          'relative min-h-[7.5rem] rounded-lg',
           // allow children to fill the height or align vertically center
           'grid',
         )}>

apps/web/components/api-access/GraphQLExplorer.tsx

@@ -0,0 +1,44 @@
+import {createGraphiQLFetcher} from '@graphiql/toolkit'
+import {graphqlEndpoint, xPatHeaderKey} from '@usevenice/app-config/constants'
+import {useConstant} from '@usevenice/ui'
+import {GraphiQL} from 'graphiql'
+import 'graphiql/graphiql.css'
+import React from 'react'
+
+export function GraphQLExplorer({pat}: {pat: string}) {
+  const fetcher = useConstant(() =>
+    createGraphiQLFetcher({url: graphqlEndpoint.pathname}),
+  )
+  const headersString = React.useMemo(
+    () => JSON.stringify({[xPatHeaderKey]: pat}, null, 4),
+    [pat],
+  )
+
+  return (
+    <div className="grow">
+      <GraphiQL
+        fetcher={fetcher}
+        defaultHeaders={headersString}
+        defaultQuery={`{
+  transactionCollection {
+    edges {
+      node{
+        id
+        description
+        amountUnit
+        amountQuantity
+        account {
+          id
+          name
+        }
+      }
+    }
+  }
+}`}>
+        <GraphiQL.Logo>
+          <div />
+        </GraphiQL.Logo>
+      </GraphiQL>
+    </div>
+  )
+}