feat(drizzle): abstract shared sql code to new package (#7320)

- Abstract shared sql code to a new drizzle package
- Adds sqlite package, not ready to publish until drizzle patches some
issues
- Add `transactionOptions` to allow customizing or disabling db
transactions
- Adds "experimental" label to the `schemaName` property until drizzle
patches an issue

Author: DanRibbens

.gitignore

@@ -22,6 +22,10 @@ meta_shared.json
 # Ignore test directory media folder/files
 /media
 test/media
+*payloadtests.db
+*payloadtests.db-journal
+*payloadtests.db-shm
+*payloadtests.db-wal
 /versions
 
 # Created by https://www.toptal.com/developers/gitignore/api/node,macos,windows,webstorm,sublimetext,visualstudiocode

.idea/payload.iml

@@ -33,18 +33,18 @@ export default buildConfig({
 
 ## Options
 
-| Option                | Description                                                                                                                                                                                           |
-|-----------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `pool` \*             | [Pool connection options](https://orm.drizzle.team/docs/quick-postgresql/node-postgres) that will be passed to Drizzle and `node-postgres`.                                                           |
-| `push`                | Disable Drizzle's [`db push`](https://orm.drizzle.team/kit-docs/overview#prototyping-with-db-push) in development mode. By default, `push` is enabled for development mode only.                      |
-| `migrationDir`        | Customize the directory that migrations are stored.                                                                                                                                                   |
-| `logger`              | The instance of the logger to be passed to drizzle. By default Payload's will be used.                                                                                                                |
-| `schemaName`   | A string for the postgres schema to use, defaults to 'public'.                                                                                                                   |
-| `localesSuffix`       | A string appended to the end of table names for storing localized fields. Default is '_locales'.                                                                                                      |
-| `relationshipsSuffix` | A string appended to the end of table names for storing relationships. Default is '_rels'.                                                                                                            |
-| `versionsSuffix`      | A string appended to the end of table names for storing versions. Defaults to '_v'.                                                                                                                   |
-
-
+| Option                      | Description                                                                                                                                                                      |
+|-----------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `pool` \*                   | [Pool connection options](https://orm.drizzle.team/docs/quick-postgresql/node-postgres) that will be passed to Drizzle and `node-postgres`.                                      |
+| `push`                      | Disable Drizzle's [`db push`](https://orm.drizzle.team/kit-docs/overview#prototyping-with-db-push) in development mode. By default, `push` is enabled for development mode only. |
+| `migrationDir`              | Customize the directory that migrations are stored.                                                                                                                              |
+| `logger`                    | The instance of the logger to be passed to drizzle. By default Payload's will be used.                                                                                           |
+| `schemaName` (experimental) | A string for the postgres schema to use, defaults to 'public'.                                                                                                                   |
+| `idType`                    | A string of 'serial', or 'uuid' that is used for the data type given to id columns.                                                                                              |
+| `transactionOptions`        | A PgTransactionConfig object for transactions, or set to `false` to disable using transactions. [More details](https://orm.drizzle.team/docs/transactions)                       |
+| `localesSuffix`             | A string appended to the end of table names for storing localized fields. Default is '_locales'.                                                                                 |
+| `relationshipsSuffix`       | A string appended to the end of table names for storing relationships. Default is '_rels'.                                                                                       |
+| `versionsSuffix`            | A string appended to the end of table names for storing versions. Defaults to '_v'.                                                                                              |
 
 ## Access to Drizzle
 

docs/database/postgres.mdx

@@ -0,0 +1,81 @@
+---
+title: SQLite
+label: SQLite
+order: 60
+desc: Payload supports SQLite through an officially supported Drizzle Database Adapter.
+keywords: SQLite, documentation, typescript, Content Management System, cms, headless, javascript, node, react, nextjs
+---
+
+To use Payload with SQLite, install the package `@payloadcms/db-sqlite`. It leverages Drizzle ORM and `libSQL` to interact with a SQLite database that you provide.
+
+It automatically manages changes to your database for you in development mode, and exposes a full suite of migration controls for you to leverage in order to keep other database environments in sync with your schema. DDL transformations are automatically generated.
+
+To configure Payload to use SQLite, pass the `sqliteAdapter` to your Payload Config as follows:
+
+```ts
+import { sqliteAdapter } from '@payloadcms/db-sqlite'
+
+export default buildConfig({
+  // Your config goes here
+  collections: [
+    // Collections go here
+  ],
+  // Configure the SQLite adapter here
+  db: sqliteAdapter({
+    // SQLite-specific arguments go here.
+    // `client.url` is required.
+    client: {
+      url: process.env.DATABASE_URL,
+      authToken: process.env.DATABASE_AUTH_TOKEN,
+    }
+  }),
+})
+```
+
+## Options
+
+| Option                | Description                                                                                                                                                                      |
+|-----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `client` \*           | [Client connection options](https://orm.drizzle.team/docs/get-started-sqlite#turso) that will be passed to `createClient` from `@libsql/client`.                                 |
+| `push`                | Disable Drizzle's [`db push`](https://orm.drizzle.team/kit-docs/overview#prototyping-with-db-push) in development mode. By default, `push` is enabled for development mode only. |
+| `migrationDir`        | Customize the directory that migrations are stored.                                                                                                                              |
+| `logger`              | The instance of the logger to be passed to drizzle. By default Payload's will be used.                                                                                           |
+| `transactionOptions`  | A SQLiteTransactionConfig object for transactions, or set to `false` to disable using transactions. [More details](https://orm.drizzle.team/docs/transactions)                   |
+| `localesSuffix`       | A string appended to the end of table names for storing localized fields. Default is '_locales'.                                                                                 |
+| `relationshipsSuffix` | A string appended to the end of table names for storing relationships. Default is '_rels'.                                                                                       |
+| `versionsSuffix`      | A string appended to the end of table names for storing versions. Defaults to '_v'.                                                                                              |
+
+
+
+## Access to Drizzle
+
+After Payload is initialized, this adapter will expose the full power of Drizzle to you for use if you need it.
+
+You can access Drizzle as follows:
+
+```text
+payload.db.drizzle
+```
+
+## Tables and relations
+
+In addition to exposing Drizzle directly, all of the tables and Drizzle relations are exposed for you via the `payload.db` property as well.
+
+- Tables - `payload.db.tables`
+- Relations - `payload.db.relations`
+
+## Prototyping in development mode
+
+Drizzle exposes two ways to work locally in development mode.
+
+The first is [`db push`](https://orm.drizzle.team/kit-docs/overview#prototyping-with-db-push), which automatically pushes changes you make to your Payload Config (and therefore, Drizzle schema) to your database so you don't have to manually migrate every time you change your Payload Config. This only works in development mode, and should not be mixed with manually running [`migrate`](/docs/database/migrations) commands.
+
+You will be warned if any changes that you make will entail data loss while in development mode. Push is enabled by default, but you can opt out if you'd like.
+
+Alternatively, you can disable `push` and rely solely on migrations to keep your local database in sync with your Payload Config.
+
+## Migration workflows
+
+In SQLite, migrations are a fundamental aspect of working with Payload and you should become familiar with how they work.
+
+For more information about migrations, [click here](/docs/beta/database/migrations#when-to-run-migrations).

docs/database/sqlite.mdx

@@ -68,3 +68,7 @@ The following functions can be used for managing transactions:
 `payload.db.beginTransaction` - Starts a new session and returns a transaction ID for use in other Payload Local API calls.
 `payload.db.commitTransaction` - Takes the identifier for the transaction, finalizes any changes.
 `payload.db.rollbackTransaction` - Takes the identifier for the transaction, discards any changes.
+
+## Disabling Transactions
+
+If you wish to disable transactions entirely, you can do so by passing `false` as the `transactionOptions` in your database adapter configuration. All the official Payload database adapters support this option.

docs/database/transactions.mdx

@@ -15,6 +15,8 @@
     "build:create-payload-app": "turbo build --filter create-payload-app",
     "build:db-mongodb": "turbo build --filter db-mongodb",
     "build:db-postgres": "turbo build --filter db-postgres",
+    "build:db-sqlite": "turbo build --filter db-sqlite",
+    "build:drizzle": "turbo build --filter drizzle",
     "build:email-nodemailer": "turbo build --filter email-nodemailer",
     "build:email-resend": "turbo build --filter email-resend",
     "build:eslint-config": "turbo build --filter eslint-config",
@@ -93,6 +95,7 @@
   },
   "devDependencies": {
     "@jest/globals": "29.7.0",
+    "@libsql/client": "0.6.2",
     "@next/bundle-analyzer": "15.0.0-canary.53",
     "@payloadcms/eslint-config": "workspace:*",
     "@payloadcms/eslint-plugin": "workspace:*",
@@ -116,7 +119,6 @@
     "create-payload-app": "workspace:*",
     "cross-env": "7.0.3",
     "dotenv": "16.4.5",
-    "drizzle-kit": "0.20.14-1f2c838",
     "drizzle-orm": "0.29.4",
     "escape-html": "^1.0.3",
     "execa": "5.1.1",

package.json

@@ -5,7 +5,7 @@ import { fileURLToPath } from 'url'
 const filename = fileURLToPath(import.meta.url)
 const dirname = path.dirname(filename)
 import { commonjs } from '@hyrious/esbuild-plugin-commonjs'
-
+throw new Error('asfdadsf')
 async function build() {
   const resultServer = await esbuild.build({
     entryPoints: ['src/index.ts'],
@@ -18,9 +18,9 @@ async function build() {
       '*.scss',
       '*.css',
       'drizzle-kit',
-      'libsql',
       'pg',
       '@payloadcms/translations',
+      '@payloadcms/drizzle',
       'payload',
       'payload/*',
     ],

packages/db-postgres/bundle.js

@@ -40,11 +40,12 @@
     "build:swc": "swc ./src -d ./dist --config-file .swcrc --strip-leading-paths",
     "build:types": "tsc --emitDeclarationOnly --outDir dist",
     "clean": "rimraf {dist,*.tsbuildinfo}",
+    "prepack": "pnpm clean && pnpm turbo build",
     "prepublishOnly": "pnpm clean && pnpm turbo build",
     "renamePredefinedMigrations": "tsx ./scripts/renamePredefinedMigrations.ts"
   },
   "dependencies": {
-    "@libsql/client": "^0.5.2",
+    "@payloadcms/drizzle": "workspace:*",
     "console-table-printer": "2.11.2",
     "drizzle-kit": "0.20.14-1f2c838",
     "drizzle-orm": "0.29.4",

packages/db-postgres/package.json

@@ -1,13 +1,12 @@
+import type { DrizzleAdapter } from '@payloadcms/drizzle/types'
 import type { Connect, Payload } from 'payload'
 
-import { sql } from 'drizzle-orm'
+import { pushDevSchema } from '@payloadcms/drizzle'
 import { drizzle } from 'drizzle-orm/node-postgres'
 import pg from 'pg'
 
 import type { PostgresAdapter } from './types.js'
 
-import { pushDevSchema } from './utilities/pushDevSchema.js'
-
 const connectWithReconnect = async function ({
   adapter,
   payload,
@@ -71,12 +70,7 @@ export const connect: Connect = async function connect(
     if (!hotReload) {
       if (process.env.PAYLOAD_DROP_DATABASE === 'true') {
         this.payload.logger.info(`---- DROPPING TABLES SCHEMA(${this.schemaName || 'public'}) ----`)
-        await this.drizzle.execute(
-          sql.raw(`
-          drop schema if exists ${this.schemaName || 'public'} cascade;
-          create schema ${this.schemaName || 'public'};
-        `),
-        )
+        await this.dropDatabase({ adapter: this })
         this.payload.logger.info('---- DROPPED TABLES ----')
       }
     }
@@ -92,7 +86,7 @@ export const connect: Connect = async function connect(
     process.env.PAYLOAD_MIGRATING !== 'true' &&
     this.push !== false
   ) {
-    await pushDevSchema(this)
+    await pushDevSchema(this as unknown as DrizzleAdapter)
   }
 
   if (typeof this.resolveInitializing === 'function') this.resolveInitializing()