feat(drizzle): support half-precision, binary, and sparse vectors column types (#12491)

Adds support for `halfvec` and `sparsevec` and `bit` (binary vector)
column types. This is required for supporting indexing of embeddings >
2000 dimensions on postgres using the pg-vector extension.

Author: AlessioGr

.github/workflows/main.yml

@@ -187,7 +187,8 @@ jobs:
 
     services:
       postgres:
-        image: ${{ (startsWith(matrix.database, 'postgres') ) && 'postgis/postgis:16-3.4' || '' }}
+        # Custom postgres 17 docker image that supports both pg-vector and postgis: https://github.com/payloadcms/postgis-vector
+        image: ${{ (startsWith(matrix.database, 'postgres') ) && 'ghcr.io/payloadcms/postgis-vector:latest' || '' }}
         env:
           # must specify password for PG Docker container image, see: https://registry.hub.docker.com/_/postgres?tab=description&page=1&name=10
           POSTGRES_USER: ${{ env.POSTGRES_USER }}

package.json

@@ -151,8 +151,8 @@
     "create-payload-app": "workspace:*",
     "cross-env": "7.0.3",
     "dotenv": "16.4.7",
-    "drizzle-kit": "0.31.0",
-    "drizzle-orm": "0.43.1",
+    "drizzle-kit": "0.31.4",
+    "drizzle-orm": "0.44.2",
     "escape-html": "^1.0.3",
     "execa": "5.1.1",
     "form-data": "3.0.1",
@@ -166,7 +166,7 @@
     "next": "15.3.2",
     "open": "^10.1.0",
     "p-limit": "^5.0.0",
-    "pg": "8.11.3",
+    "pg": "8.16.3",
     "playwright": "1.50.0",
     "playwright-core": "1.50.0",
     "prettier": "3.5.3",

packages/db-postgres/package.json

@@ -78,9 +78,9 @@
     "@payloadcms/drizzle": "workspace:*",
     "@types/pg": "8.10.2",
     "console-table-printer": "2.12.1",
-    "drizzle-kit": "0.31.1",
+    "drizzle-kit": "0.31.4",
     "drizzle-orm": "0.44.2",
-    "pg": "8.11.3",
+    "pg": "8.16.3",
     "prompts": "2.4.2",
     "to-snake-case": "1.0.0",
     "uuid": "10.0.0"

packages/db-sqlite/package.json

@@ -76,7 +76,7 @@
     "@libsql/client": "0.14.0",
     "@payloadcms/drizzle": "workspace:*",
     "console-table-printer": "2.12.1",
-    "drizzle-kit": "0.31.1",
+    "drizzle-kit": "0.31.4",
     "drizzle-orm": "0.44.2",
     "prompts": "2.4.2",
     "to-snake-case": "1.0.0",

packages/db-vercel-postgres/package.json

@@ -78,9 +78,9 @@
     "@payloadcms/drizzle": "workspace:*",
     "@vercel/postgres": "^0.9.0",
     "console-table-printer": "2.12.1",
-    "drizzle-kit": "0.31.1",
+    "drizzle-kit": "0.31.4",
     "drizzle-orm": "0.44.2",
-    "pg": "8.11.3",
+    "pg": "8.16.3",
     "prompts": "2.4.2",
     "to-snake-case": "1.0.0",
     "uuid": "10.0.0"

packages/drizzle/src/postgres/columnToCodeConverter.ts

@@ -24,20 +24,26 @@ export const columnToCodeConverter: ColumnToCodeConverter = ({
 
   const columnBuilderArgsArray: string[] = []
 
-  if (column.type === 'timestamp') {
-    columnBuilderArgsArray.push(`mode: '${column.mode}'`)
-    if (column.withTimezone) {
-      columnBuilderArgsArray.push('withTimezone: true')
+  switch (column.type) {
+    case 'bit':
+    case 'halfvec':
+    case 'sparsevec':
+    case 'vector': {
+      if (column.dimensions) {
+        columnBuilderArgsArray.push(`dimensions: ${column.dimensions}`)
+      }
+      break
     }
-
-    if (typeof column.precision === 'number') {
-      columnBuilderArgsArray.push(`precision: ${column.precision}`)
-    }
-  }
-
-  if (column.type === 'vector') {
-    if (column.dimensions) {
-      columnBuilderArgsArray.push(`dimensions: ${column.dimensions}`)
+    case 'timestamp': {
+      columnBuilderArgsArray.push(`mode: '${column.mode}'`)
+      if (column.withTimezone) {
+        columnBuilderArgsArray.push('withTimezone: true')
+      }
+
+      if (typeof column.precision === 'number') {
+        columnBuilderArgsArray.push(`precision: ${column.precision}`)
+      }
+      break
     }
   }
 

packages/drizzle/src/postgres/schema/buildDrizzleTable.ts

@@ -1,13 +1,16 @@
 import type { ForeignKeyBuilder, IndexBuilder } from 'drizzle-orm/pg-core'
 
 import {
+  bit,
   boolean,
   foreignKey,
+  halfvec,
   index,
   integer,
   jsonb,
   numeric,
   serial,
+  sparsevec,
   text,
   timestamp,
   uniqueIndex,
@@ -44,6 +47,14 @@ export const buildDrizzleTable = ({
 
   for (const [key, column] of Object.entries(rawTable.columns)) {
     switch (column.type) {
+      case 'bit': {
+        const builder = bit(column.name, { dimensions: column.dimensions })
+
+        columns[key] = builder
+
+        break
+      }
+
       case 'enum':
         if ('locale' in column) {
           columns[key] = adapter.enums.enum__locales(column.name)
@@ -56,6 +67,21 @@ export const buildDrizzleTable = ({
         }
         break
 
+      case 'halfvec': {
+        const builder = halfvec(column.name, { dimensions: column.dimensions })
+
+        columns[key] = builder
+        break
+      }
+
+      case 'sparsevec': {
+        const builder = sparsevec(column.name, { dimensions: column.dimensions })
+
+        columns[key] = builder
+
+        break
+      }
+
       case 'timestamp': {
         let builder = timestamp(column.name, {
           mode: column.mode,

packages/drizzle/src/types.ts

@@ -281,12 +281,30 @@ export type VectorRawColumn = {
   type: 'vector'
 } & BaseRawColumn
 
+export type HalfVecRawColumn = {
+  dimensions?: number
+  type: 'halfvec'
+} & BaseRawColumn
+
+export type SparseVecRawColumn = {
+  dimensions?: number
+  type: 'sparsevec'
+} & BaseRawColumn
+
+export type BinaryVecRawColumn = {
+  dimensions?: number
+  type: 'bit'
+} & BaseRawColumn
+
 export type RawColumn =
   | ({
       type: 'boolean' | 'geometry' | 'jsonb' | 'numeric' | 'serial' | 'text' | 'varchar'
     } & BaseRawColumn)
+  | BinaryVecRawColumn
   | EnumRawColumn
+  | HalfVecRawColumn
   | IntegerRawColumn
+  | SparseVecRawColumn
   | TimestampRawColumn
   | UUIDRawColumn
   | VectorRawColumn