ENG-270, ENG-328 Creation of the supabase embedding schema, integration with turbo and CI/CD

.github/workflows/database-deploy.yaml

@@ -0,0 +1,23 @@
+name: Supabase deploy Function
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    env:
+      SUPABASE_ACCESS_TOKEN: ${{ secrets.SUPABASE_ACCESS_TOKEN }}
+      SUPABASE_PROJECT_ID: ${{ secrets.SUPABASE_PROJECT_ID_PROD }}
+      SUPABASE_DB_PASSWORD: ${{ secrets.SUPABASE_DB_PASSWORD_PROD }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+      - run: npm ci
+      - uses: supabase/setup-cli@v1
+        with:
+          version: latest
+      - run: npx turbo deploy -F @repo/database

packages/database/.sqruff

@@ -0,0 +1,8 @@
+[sqruff]
+dialect = postgres
+exclude_rules = CP05,LT05
+
+[sqruff:indentation]
+indent_unit = space
+tab_space_size = 4
+indented_joins = True

packages/database/README.md

@@ -0,0 +1,25 @@
+This contains the database schema for vector embeddings and concepts.
+All CLI commands below should be run in this directory (`packages/database`.)
+
+1. Setup
+   1. Install [Docker](https://www.docker.com)
+   2. Install the [supabase CLI](https://supabase.com/docs/guides/local-development). (There is a brew version)
+   3. `supabase login` with your (account-specific) supabase access token. (TODO: Create a group access token.)
+   4. `supabase link`. It will ask you for a project name, use `discourse-graphs`. (Production for now.) It will also ask you for the database password (See 1password.)
+   5. Install [sqruff](https://github.com/quarylabs/sqruff)
+2. Usage:
+   1. Use `turbo dev`, (alias for `supabase start`) before you use your local database. URLs will be given for your local supabase database, api endpoint, etc.
+   2. You may need to `supabase db pull` if changes are deployed while you work.
+   3. End you work session with `supabase end` to free docker resources.
+3. Development: We follow the supabase [Declarative Database Schema](https://supabase.com/docs/guides/local-development/declarative-database-schemas) process.
+   1. Assuming you're working on a feature branch.
+   2. Make changes to the schema, by editing files in `packages/database/supabase/schemas`
+   3. If you created a new schema file, make sure to add it to `[db.migrations] schema_paths` in `packages/database/supabase/config.toml`. Schema files are applied in that order, you may need to be strategic in placing your file.
+   4. `turbo build`, which will do the following:
+      1. Check your logic with `sqruff lint supabase/schemas`, and eventually `sqruff fix supabase/schemas`
+      2. Regenerate the types file with `supabase gen types typescript --local > types.gen.ts`
+      3. See if there would be a migration to apply with `supabase db diff`
+   5. If applying the new schema fails, repeat step 4
+   6. If you are satisfied with the migration, create a migration file with `npm run dbdiff:save some_meaningful_migration_name`
+      1. If all goes well, there should be a new file named `supbase/migration/2..._some_meaningful_migration_name.sql` which you should `git add`.
+   10. You can start using your changes again `turbo dev`

packages/database/example.md

@@ -0,0 +1,76 @@
+# example...
+
+Content:
+
+* (nt1pgid) discourse-graphs/nodes/Claim
+* (nt2pgid) discourse-graphs/nodes/Hypothesis
+* (dgpgid) roam/js/discourse-graph
+  * (et1bkid) Opposes
+      * (et1r1bkid) source
+      * (et1r2bkid) destination
+    * (anyid1) If
+      * (et1sr1bkid) Page
+      * (et1sr2bkid) Block
+      * (et1sr3bkid) ParentPage
+      * (et1sr4bkid) PBlock
+      * (et1sr5bkid) SPage
+      * (et1sr6bkid) SBlock
+* (hyp1pgid) [HYP] Some hypothesis
+* (clm1pgid) [CLM] Some claim
+* (somepgid) Some page
+  * (hyp1refbkid) a block referring to [[HYP] Some hypothesis]
+    * (opp1bkid) OpposedBy
+      * (clm1refbkid) a block referring to [[CLM] Some Claim]
+
+Documents:
+
+| id | source_local_id |
+|----|-----------------|
+| 1  | nt1pgid         |
+| 2  | nt2pgid         |
+| 3  | dgpgid         |
+| 22 | hyp1pgid        |
+| 23 | clm1pgid        |
+| 4  | somepgid        |
+
+Content:
+
+| id | source_local_id | page_id | scale    | represents_id | text                                         |
+|----|-------------|-------------|----------|---------------|----------------------------------------------|
+| 5  | nt1pgid         | 1       | document | 16            | discourse-graphs/nodes/Claim                 |
+| 6  | nt2pgid         | 2       | document | 17            | discourse-graphs/nodes/Hypothesis            |
+| 7  | et1bkid         | 3       | document | 18            | discourse-graphs/edges/OpposedBy             |
+| 8  | somepgid        | 4       | document |               | Some page                                    |
+| 24 | hyp1pgid        | 22      | document | 20            | [HYP] Some hypothesis                        |
+| 25 | clm1pgid        | 23      | document | 19            | [HYP] Some claim                             |
+| 9  | hyp1refbkid     | 4       | block    |               | a block referring to [[HYP] Some hypothesis] |
+| 10 | opp1bkid        | 4       | block    | 21            | OpposedBy                                    |
+| 11 | clm1refbkid     | 4       | block    |               | a block referring to [[CLM] Some claim]      |
+| 13 | et1r1bkid       | 3       | block    |               | source                                       |
+| 14 | et1r2bkid       | 3       | block    |               | destination                                  |
+
+Concept:
+
+| id | is_schema | arity | schema | name                  | content   |
+|----|-----------|-------|--------|-----------------------|-----------|
+| 16 | true      | 0     |        | Claim                 | {}        |
+| 17 | true      | 0     |        | Hypothesis            | {}        |
+| 18 | true      | 2     |        | Opposed-by            | { "roles": ["source", "destination"], "representation": ["source", "sourceref", "destination", "destinationref", "predicate"] } |
+| 19 | false     | 0     | 16     | [CLM] Some claim      | {}        |
+| 20 | false     | 0     | 17     | [HYP] Some hypothesis | {}        |
+| 21 | false     | 2     | 18     | OpposedBy             | { "concepts": {"source": 19, "destination": 20}, "occurences": [{"sourceref": 11, "destinationref": 9, "source": 25, "destination": 24, "predicate": 10 }] } |
+
+Note: Open question whether the occurence structure matters, and whether it should be materialized in another table.
+(I would tend to say yes to both.)
+
+ContentLink
+
+| source | destination |
+|--------|-------------|
+| 9      | 24          |
+| 11     | 25          |
+
+Note: I would probably create a sub-Content for the link text and use this as source.
+OR use a char_start, char_end.
+
+Missing: Ontology

packages/database/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "@repo/database",
+  "version": "0.0.0",
+  "private": true,
+  "license": "Apache-2.0",
+  "type": "module",
+  "exports": {
+    "./types.gen.ts": "./types.gen.ts"
+  },
+  "scripts": {
+    "init": "supabase login",
+    "dev": "supabase start",
+    "stop": "supabase stop",
+    "build": "npm run lint && npm run gentypes:local && cp ./types.gen.ts ../../apps/website/app/utils/supabase && npm run dbdiff",
+    "lint": "tsx scripts/lint.ts",
+    "lint:fix": "tsx scripts/lint.ts -f",
+    "gentypes:local": "supabase start && supabase gen types typescript --local --schema public > types.gen.ts",
+    "gentypes:production": "supabase start && supabase gen types typescript --project-id \"$SUPABASE_PROJECT_ID\" --schema public > types.gen.ts",
+    "dbdiff": "supabase stop && supabase db diff",
+    "dbdiff:save": "supabase stop && supabase db diff -f",
+    "deploy": "tsx scripts/deploy.ts",
+    "deploy:functions": "tsx scripts/lint.ts -f"
+  },
+  "devDependencies": {
+    "supabase": "^2.22.12",
+    "tsx": "^4.19.2"
+  },
+  "dependencies": {}
+}

packages/database/schema.puml

@@ -0,0 +1,110 @@
+@startuml
+skinparam nodesep 10
+hide circle
+hide empty members
+class "SpaceAccess" [[{An access control entry for a space}]] {
+    {field} editor : boolean  
+}
+class "Account" [[{A user account on a platform}]] {
+    {field} id : integer  
+    {field} write_permission : boolean  
+    {field} active : boolean  
+}
+class "Space" [[{A space on a platform representing a community engaged in a conversation}]] {
+    {field} id : integer  
+    {field} url : string  
+    {field} name : string  
+}
+"SpaceAccess" --> "1" "Account" : "account"
+"SpaceAccess" --> "0..1" "Space" : "space"
+class "Platform" [[{A data platform where discourse happens}]] {
+    {field} id : integer  
+    {field} name : string  
+    {field} url : string  
+}
+class "Content" [[{A unit of content}]] {
+    {field} id : integer  
+    {field} source_local_id : string  
+    {field} created : datetime  
+    {field} text : string  
+    {field} metadata : JSON  
+    {field} scale : Scale  
+    {field} last_modified : datetime  
+}
+class "Document" [[{None}]] {
+    {field} id : integer  
+    {field} source_local_id : string  
+    {field} url : string  
+    {field} created : datetime  
+    {field} metadata : JSON  
+    {field} last_modified : datetime  
+    {field} contents : blob  
+}
+class "Concept" [[{An abstract concept, claim or relation}]] {
+    {field} id : integer  
+    {field} epistemic_status : EpistemicStatus  
+    {field} name : string  
+    {field} description : string  
+    {field} created : datetime  
+    {field} last_modified : datetime  
+    {field} arity : integer  
+    {field} content : JSON  
+    {field} is_schema : boolean  
+}
+"Space" --> "1" "Platform" : "platform"
+"Content" --> "0..1" "Space" : "space"
+"Document" --> "0..1" "Space" : "space"
+"Concept" --> "0..1" "Space" : "space"
+"Account" --> "1" "Platform" : "platform"
+abstract "Agent" [[{An agent that acts in the system}]] {
+    {field} id : integer  
+    {field} type : EntityType  
+}
+"Document" --> "0..*" "Agent" : "contributors"
+"Document" --> "1" "Agent" : "author"
+"Content" --> "1" "Document" : "document"
+class "ContentEmbedding" [[{None}]] {
+    {field} model : EmbeddingName  
+    {field} vector : vector  
+    {field} obsolete : boolean  
+}
+"ContentEmbedding" --> "1" "Content" : "target"
+"Content" --> "0..1" "Content" : "part_of"
+"Content" --> "0..*" "Agent" : "contributors"
+"Content" --> "1" "Agent" : "creator"
+"Content" --> "1" "Agent" : "author"
+"Concept" --> "0..1" "Content" : "represented_by"
+class "ConceptSchema" [[{None}]] {
+    {field} id(i) : integer  
+    {field} epistemic_status(i) : EpistemicStatus  
+    {field} name(i) : string  
+    {field} description(i) : string  
+    {field} created(i) : datetime  
+    {field} last_modified(i) : datetime  
+    {field} arity(i) : integer  
+    {field} content(i) : JSON  
+    {field} is_schema(i) : boolean  
+}
+"Concept" --> "1" "ConceptSchema" : "schema"
+"Concept" --> "0..*" "Agent" : "contributors"
+"Concept" --> "1" "Agent" : "author"
+"Concept" ^-- "ConceptSchema"
+class "Person" [[{A person using the system}]] {
+    {field} name : string  
+    {field} orcid : string  
+    {field} email : string  
+    {field} id(i) : integer  
+    {field} type(i) : EntityType  
+}
+class "AutomatedAgent" [[{An automated agent}]] {
+    {field} metadata : JSON  
+    {field} name : string  
+    {field} deterministic : boolean  
+    {field} version : string  
+    {field} id(i) : integer  
+    {field} type(i) : EntityType  
+}
+"Account" --> "1" "Agent" : "person"
+"Agent" ^-- "Person"
+"Agent" ^-- "AutomatedAgent"
+@enduml