feat: add support for sqlite

stackedq · stackedq · commit be167324d929 · 2023-03-01T13:41:08.000+03:30
diff --git a/.env b/.env
@@ -1,7 +1,19 @@
 # Default settings, override in .env.local file that is auto-generated on install
+
+
+# Database
+### sqlite (default)
+DATABASE_TYPE=sqlite3
+# sqlite file path, absolute or relative path
+DATABASE_URL=./mydb.sqlite
+### postgres
+
 # NOTE: The default database is provided to get you up and running but is not secure!
+# DATABASE_TYPE=pg
 # DATABASE_URL=postgres://postgres:ZTE*meq1mzh3abn!cmk@db.xbolptvmhqplniydcjcz.supabase.co:5432/postgres
-DATABASE_URL=postgres://postgres:P8Pwj96n38ilb5a2@db.ageqtrdagwgucljhwtab.supabase.co:5432/postgres
+# DATABASE_URL=postgres://postgres:P8Pwj96n38ilb5a2@db.ageqtrdagwgucljhwtab.supabase.co:5432/postgres
+
+# Plugins
 CLIENT_PLUGINS=discord,rest,loop,finetune,ethereum
 SERVER_PLUGINS=discord,rest,loop,finetune,ethereum
 
diff --git a/.gitignore b/.gitignore
@@ -46,4 +46,6 @@ dev.db
 package-lock.json
 yarn.lock
 
-./files/
+./files/
+
+*.sqlite
diff --git a/README.md b/README.md
@@ -49,9 +49,17 @@ There are a few things to keep in mind while this project is actively being work
 
 ## Database
 
+### Local Sqlite (default)
+
+Simply add a relative or absolute path to your sqlite file in the `.env`, run `npm run migrate` and you're all set.
+
+### Supabase public database 
+
 _Please be aware Magick is under heavy development and changes can cause your DB to be wiped. Back up your spells via export regularly._
 
-The default setup of Magick is connected to a test database on Supabase. It will get you going, but it will likely be wiped regularly and is not a good storage for your spells.
+Magick can connect to a test database on Supabase. It will get you going, but it will likely be wiped regularly and is not a good storage for your spells.
+
+just enable the pg database in the `.env` file.
 
 ### Deploy your own database
 
diff --git a/apps/server/knexfile.ts b/apps/server/knexfile.ts
@@ -1,6 +1,20 @@
-const config = {
-    client: 'pg',
-    connection: process.env.DATABASE_URL
+const dbType = process.env.DATABASE_TYPE
+const dbURL = process.env.DATABASE_URL
+const configs = {
+  pg: {
+    client: dbType,
+    connection: dbURL
+  },
+  sqlite3: {
+    client: dbType, 
+    connection: {
+      // handling both absolute and relative paths, 
+      // for relative path resolve back to the main directory,
+      // where .env file exists
+      filename: dbURL.startsWith('/') ? dbURL : `../../${dbURL}`
+    },
+    useNullAsDefault: true
+  }
 }
 
-module.exports = config
+module.exports = configs[dbType]
diff --git a/package.json b/package.json
@@ -139,6 +139,7 @@
     "socket.io": "^4.5.4",
     "socket.io-client": "^4.5.4",
     "solc": "^0.8.18",
+    "sqlite3": "^5.1.4",
     "stream-browserify": "^3.0.0",
     "styled-components": "^5.3.6",
     "swagger-ui-dist": "^4.15.5",
diff --git a/packages/server-core/src/app.ts b/packages/server-core/src/app.ts
@@ -5,7 +5,7 @@ import socketio from '@feathersjs/socketio'
 
 import type { Application } from './declarations'
 import { logError } from './hooks'
-import { postgresql } from './postgresql'
+import { dbClient } from './dbClient'
 import { services } from './services'
 import channels from './channels'
 // import swagger from 'feathers-swagger'
@@ -68,7 +68,7 @@ app.configure(
     handleSockets(app)
   )
 )
-app.configure(postgresql)
+app.configure(dbClient)
 app.configure(services)
 app.configure(channels)
 
diff --git a/packages/server-core/src/dbClient.ts b/packages/server-core/src/dbClient.ts
@@ -0,0 +1,73 @@
+// For more information about this file see https://dove.feathersjs.com/guides/cli/databases.html
+import knex from 'knex'
+import type { Knex } from 'knex'
+import type { Application, SupportedDbs } from './declarations'
+
+declare module './declarations' {
+  interface Configuration {
+    dbClient: Knex
+  }
+
+  type SupportedDbs = 'pg' | 'sqlite3'
+}
+const getDatabaseConfig = () => {
+  const dbType = process.env.DATABASE_TYPE || ''
+  const dbURL = process.env.DATABASE_URL
+
+  if(!dbURL) throw new Error("Missing DATABASE_URL in your .env file.")
+
+  // postgres config 
+  if (dbType === 'pg') 
+    return {
+      client: dbType,
+      connection: dbURL
+    }
+  // sqlite config 
+  if (dbType === 'sqlite3')
+    return {
+      client: dbType, 
+      connection: {
+        filename:  dbURL
+      },
+      // sqlite does not support inserting default values
+      useNullAsDefault: true,
+      pool: {
+        // afterCreate: function (conn, done) {
+        //   // in this example we use pg driver's connection API
+        //   conn.query('SET timezone="UTC";', function (err) {
+        //     if (err) {
+        //       // first query failed, 
+        //       // return error and don't try to make next query
+        //       done(err, conn);
+        //     } else {
+        //       // do the second query...
+        //       conn.query(
+        //         'SELECT set_limit(0.01);', 
+        //         function (err) {
+        //           // if err is not falsy, 
+        //           //  connection is discarded from pool
+        //           // if connection aquire was triggered by a 
+        //           // query the error is passed to query promise
+        //           done(err, conn);
+        //         });
+        //     }
+        //   });
+        // }
+      }
+    }
+
+  throw new Error("Unsupported database type, use `pg` or `sqlite3`")
+}
+
+export const dbClient = (app: Application) => {
+  const config = getDatabaseConfig()
+  const db = knex(config)
+  app.set('dbClient', db)
+}
+
+const dbSupportJson: Record<SupportedDbs, boolean> = {
+  'pg': true,
+  'sqlite3': false
+}
+
+export const doesDbSupportJson = () :boolean => dbSupportJson[process.env.DATABASE_TYPE]
diff --git a/packages/server-core/src/postgresql.ts b/packages/server-core/src/postgresql.ts
diff --git a/packages/server-core/src/services/agents/agents.class.ts b/packages/server-core/src/services/agents/agents.class.ts
@@ -19,7 +19,7 @@ export class AgentService<ServiceParams extends Params = AgentParams> extends Kn
 export const getOptions = (app: Application): KnexAdapterOptions => {
   return {
     paginate: app.get('paginate'),
-    Model: app.get('postgresqlClient'),
+    Model: app.get('dbClient'),
     name: 'agents'
   }
 }
diff --git a/packages/server-core/src/services/agents/agents.schema.ts b/packages/server-core/src/services/agents/agents.schema.ts
@@ -67,3 +67,4 @@ export const agentQuerySchema = Type.Intersect(
 export type AgentQuery = Static<typeof agentQuerySchema>
 export const agentQueryValidator = getValidator(agentQuerySchema, queryValidator)
 export const agentQueryResolver = resolve<AgentQuery, HookContext>({})
+export const agentJsonFields = ['spells', 'data']
diff --git a/packages/server-core/src/services/agents/agents.ts b/packages/server-core/src/services/agents/agents.ts
@@ -10,10 +10,12 @@ import {
   agentDataResolver,
   agentPatchResolver,
   agentQueryResolver,
+  agentJsonFields
 } from './agents.schema'
 
 import type { Application } from '../../declarations'
 import { AgentService, getOptions } from './agents.class'
+import { handleJSONFieldsUpdate, jsonResolver } from '../utils'
 
 export * from './agents.class'
 export * from './agents.schema'
@@ -33,7 +35,8 @@ export const agent = (app: Application) => {
       all: [
         schemaHooks.resolveExternal(agentExternalResolver),
         schemaHooks.resolveResult(agentResolver),
-      ],
+        schemaHooks.resolveResult(jsonResolver(agentJsonFields)),
+      ]
     },
     before: {
       all: [
@@ -42,15 +45,14 @@ export const agent = (app: Application) => {
       ],
       find: [],
       get: [],
-      create: [
-        schemaHooks.validateData(agentDataValidator),
-        schemaHooks.resolveData(agentDataResolver),
-      ],
+      create: [schemaHooks.validateData(agentDataValidator), schemaHooks.resolveData(agentDataResolver)],
       patch: [
         schemaHooks.validateData(agentPatchValidator),
         schemaHooks.resolveData(agentPatchResolver),
+        handleJSONFieldsUpdate(agentJsonFields)
       ],
-      remove: [],
+      update: [handleJSONFieldsUpdate(agentJsonFields)],
+      remove: []
     },
     after: {
       all: [],
diff --git a/packages/server-core/src/services/events/events.class.ts b/packages/server-core/src/services/events/events.class.ts
@@ -20,7 +20,7 @@ export class EventService<ServiceParams extends Params = EventParams> extends Kn
 export const getOptions = (app: Application): KnexAdapterOptions => {
   return {
     paginate: app.get('paginate'),
-    Model: app.get('postgresqlClient'),
+    Model: app.get('dbClient'),
     name: 'events'
   }
 }
diff --git a/packages/server-core/src/services/events/events.ts b/packages/server-core/src/services/events/events.ts
@@ -2,20 +2,15 @@
 //@ts-nocheck
 // For more information about this file see https://dove.feathersjs.com/guides/cli/service.html
 import { authenticate } from '@feathersjs/authentication'
-import {
-  DATABASE_URL
-} from '@magickml/engine'
 import pgvector from 'pgvector/pg'
-import postgres from 'postgres'
+import type { Knex } from 'knex'
 import { hooks as schemaHooks } from '@feathersjs/schema' 
-const sql = postgres(DATABASE_URL)
-async function getUsersOver(embedding) {
-  let users
+async function getUsersOver(db: Knex, embedding) {
+  var users
   try {
-    users = await sql`
-    select * from events order by embedding <-> ${embedding} limit 1;`
-  } catch (error){
-    console.error(error)
+    users = await db.raw(`select * from events order by embedding <-> ${embedding} limit 1;`)
+  } catch (e){
+    console.log(e)
   }
   return users
 }
@@ -67,8 +62,8 @@ export const event = (app: Application) => {
             const ary_buf = new ArrayBuffer( blob.length );
             const dv = new DataView( ary_buf );
             for( let i=0; i < blob.length; i++ ) dv.setUint8( i, blob.charCodeAt(i) );
-            const f32_ary = new Float32Array( ary_buf );
-            const temp = await getUsersOver("["+f32_ary+"]")
+            let f32_ary = new Float32Array( ary_buf );
+            let temp = await getUsersOver(app.get('dbClient'), "["+f32_ary+"]")
             return {
               "result" : temp
             }
diff --git a/packages/server-core/src/services/request/request.class.ts b/packages/server-core/src/services/request/request.class.ts
@@ -19,7 +19,7 @@ export class RequestService<ServiceParams extends Params = RequestParams> extend
 export const getOptions = (app: Application): KnexAdapterOptions => {
   return {
     paginate: app.get('paginate'),
-    Model: app.get('postgresqlClient'),
+    Model: app.get('dbClient'),
     name: 'request'
   }
 }
diff --git a/packages/server-core/src/services/spell-runner/spell-runner.class.ts b/packages/server-core/src/services/spell-runner/spell-runner.class.ts
@@ -131,7 +131,7 @@ export class SpellRunnerService<
 export const getOptions = (app: Application): KnexAdapterOptions => {
   return {
     paginate: app.get('paginate'),
-    Model: app.get('postgresqlClient'),
-    name: 'spell-runner',
+    Model: app.get('dbClient'),
+    name: 'spell-runner'
   }
 }
diff --git a/packages/server-core/src/services/spells/spells.class.ts b/packages/server-core/src/services/spells/spells.class.ts
@@ -52,7 +52,7 @@ export class SpellService<ServiceParams extends Params = SpellParams> extends Kn
 export const getOptions = (app: Application): KnexAdapterOptions => {
   return {
     paginate: app.get('paginate'),
-    Model: app.get('postgresqlClient'),
+    Model: app.get('dbClient'),
     name: 'spells'
   }
 }
diff --git a/packages/server-core/src/services/spells/spells.schema.ts b/packages/server-core/src/services/spells/spells.schema.ts
@@ -39,15 +39,19 @@ export const spellDataSchema = Type.Pick(spellSchema, [
 })
 export type SpellData = Static<typeof spellDataSchema>
 export const spellDataValidator = getDataValidator(spellDataSchema, dataValidator)
-export const spellDataResolver = resolve<Spell, HookContext>({})
+export const spellDataResolver = resolve<Spell, HookContext>({
+  created_at: async () => Date.now().toString()
+})
 
 // Schema for updating existing entries
 export const spellPatchSchema = Type.Partial(spellDataSchema, {
   $id: 'SpellPatch'
 })
 export type SpellPatch = Static<typeof spellPatchSchema>
 export const spellPatchValidator = getDataValidator(spellPatchSchema, dataValidator)
-export const spellPatchResolver = resolve<Spell, HookContext>({})
+export const spellPatchResolver = resolve<Spell, HookContext>({
+  updated_at: async () => Date.now().toString(),
+})
 
 // Schema for allowed query properties
 export const spellQueryProperties = Type.Pick(spellSchema, [
@@ -74,3 +78,4 @@ export const spellQuerySchema = Type.Intersect(
 export type SpellQuery = Static<typeof spellQuerySchema>
 export const spellQueryValidator = getValidator(spellQuerySchema, queryValidator)
 export const spellQueryResolver = resolve<SpellQuery, HookContext>({})
+export const spellJsonFields = ['graph']
diff --git a/packages/server-core/src/services/spells/spells.ts b/packages/server-core/src/services/spells/spells.ts
diff --git a/packages/server-core/src/services/utils.ts b/packages/server-core/src/services/utils.ts

Original file line number	Diff line number	Diff line change
`@@ -19,7 +19,7 @@ export class AgentService<ServiceParams extends Params = AgentParams> extends Kn`
`19`	`19`	`export const getOptions = (app: Application): KnexAdapterOptions => {`
`20`	`20`	`return {`
`21`	`21`	`paginate: app.get('paginate'),`
`22`		`- Model: app.get('postgresqlClient'),`
	`22`	`+ Model: app.get('dbClient'),`
`23`	`23`	`name: 'agents'`
`24`	`24`	`}`
`25`	`25`	`}`