From 1f602e0239b0edefcf31714532cf2284020d0dcb Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 24 Sep 2024 15:37:21 +0200 Subject: [PATCH 01/20] connectionId --- apps/browser-proxy/prisma/schema.prisma | 14 + apps/browser-proxy/src/index.ts | 341 ++++++++++-------- apps/postgres-new/components/app-provider.tsx | 21 +- package.json | 7 +- 4 files changed, 238 insertions(+), 145 deletions(-) create mode 100644 apps/browser-proxy/prisma/schema.prisma diff --git a/apps/browser-proxy/prisma/schema.prisma b/apps/browser-proxy/prisma/schema.prisma new file mode 100644 index 00000000..ee282c75 --- /dev/null +++ b/apps/browser-proxy/prisma/schema.prisma @@ -0,0 +1,14 @@ +// This is your Prisma schema file, +// learn more about it in the docs: https://pris.ly/d/prisma-schema + +// Looking for ways to speed up your queries, or scale easily with your serverless or edge functions? +// Try Prisma Accelerate: https://pris.ly/cli/accelerate-init + +generator client { + provider = "prisma-client-js" +} + +datasource db { + provider = "postgresql" + url = env("DATABASE_URL") +} diff --git a/apps/browser-proxy/src/index.ts b/apps/browser-proxy/src/index.ts index f365f31e..fdd1b48b 100644 --- a/apps/browser-proxy/src/index.ts +++ b/apps/browser-proxy/src/index.ts @@ -17,169 +17,228 @@ import { } from './telemetry.ts' const debug = makeDebug('browser-proxy') +try { + type DatabaseId = string + type ConnectionId = string + const tcpConnections = new Map() + const tcpConnectionsByDatabaseId = new Map() + const websocketConnections = new Map() + + const httpsServer = https.createServer({ + SNICallback: (servername, callback) => { + debug('SNICallback', servername) + if (isValidServername(servername)) { + debug('SNICallback', 'valid') + callback(null) + } else { + debug('SNICallback', 'invalid') + callback(new Error('invalid SNI')) + } + }, + }) + await setSecureContext(httpsServer) + // reset the secure context every week to pick up any new TLS certificates + setInterval(() => setSecureContext(httpsServer), 1000 * 60 * 60 * 24 * 7) -const tcpConnections = new Map() -const websocketConnections = new Map() - -const httpsServer = https.createServer({ - SNICallback: (servername, callback) => { - debug('SNICallback', servername) - if (isValidServername(servername)) { - debug('SNICallback', 'valid') - callback(null) - } else { - debug('SNICallback', 'invalid') - callback(new Error('invalid SNI')) - } - }, -}) -await setSecureContext(httpsServer) -// reset the secure context every week to pick up any new TLS certificates -setInterval(() => setSecureContext(httpsServer), 1000 * 60 * 60 * 24 * 7) - -const websocketServer = new WebSocketServer({ - server: httpsServer, -}) + const websocketServer = new WebSocketServer({ + server: httpsServer, + }) -websocketServer.on('error', (error) => { - debug('websocket server error', error) -}) + websocketServer.on('error', (error) => { + debug('websocket server error', error) + }) -websocketServer.on('connection', (socket, request) => { - debug('websocket connection') + websocketServer.on('connection', (socket, request) => { + debug('websocket connection') - const host = request.headers.host + const host = request.headers.host - if (!host) { - debug('No host header present') - socket.close() - return - } + if (!host) { + debug('No host header present') + socket.close() + return + } - const databaseId = extractDatabaseId(host) + const databaseId = extractDatabaseId(host) - if (websocketConnections.has(databaseId)) { - socket.send('sorry, too many clients already') - socket.close() - return - } - - websocketConnections.set(databaseId, socket) + if (websocketConnections.has(databaseId)) { + socket.send('sorry, too many clients already') + socket.close() + return + } - logEvent(new DatabaseShared({ databaseId })) + websocketConnections.set(databaseId, socket) - socket.on('message', (data: Buffer) => { - debug('websocket message', data.toString('hex')) - const tcpConnection = tcpConnections.get(databaseId) - tcpConnection?.streamWriter?.write(data) - }) + logEvent(new DatabaseShared({ databaseId })) - socket.on('close', () => { - websocketConnections.delete(databaseId) - logEvent(new DatabaseUnshared({ databaseId })) - }) -}) - -// we need to use proxywrap to make our tcp server to enable the PROXY protocol support -const net = ( - process.env.PROXIED ? (await import('findhit-proxywrap')).default.proxy(nodeNet) : nodeNet -) as typeof nodeNet - -const tcpServer = net.createServer() - -tcpServer.on('connection', async (socket) => { - let databaseId: string | undefined - - const connection = await fromNodeSocket(socket, { - tls: getTls, - onTlsUpgrade(state) { - if (!state.tlsInfo?.serverName || !isValidServername(state.tlsInfo.serverName)) { - throw BackendError.create({ - code: '08006', - message: 'invalid SNI', - severity: 'FATAL', - }) + socket.on('message', (data: Buffer) => { + if (data.length === 0) { + return } - const _databaseId = extractDatabaseId(state.tlsInfo.serverName!) - - if (!websocketConnections.has(_databaseId!)) { - throw BackendError.create({ - code: 'XX000', - message: 'the browser is not sharing the database', - severity: 'FATAL', - }) + const connectionId = data.slice(0, 8) + const message = data.slice(8) + const tcpConnection = tcpConnections.get(Buffer.from(connectionId).toString('hex')) + if (tcpConnection) { + debug('websocket message', message.toString('hex')) + tcpConnection.streamWriter?.write(message) } + }) - if (tcpConnections.has(_databaseId)) { - throw BackendError.create({ - code: '53300', - message: 'sorry, too many clients already', - severity: 'FATAL', - }) - } + socket.on('close', () => { + websocketConnections.delete(databaseId) + logEvent(new DatabaseUnshared({ databaseId })) + }) + }) - // only set the databaseId after we've verified the connection - databaseId = _databaseId - tcpConnections.set(databaseId!, connection) - logEvent(new UserConnected({ databaseId })) - }, - serverVersion() { - return '16.3' - }, - onAuthenticated() { - const websocket = websocketConnections.get(databaseId!) - - if (!websocket) { - throw BackendError.create({ - code: 'XX000', - message: 'the browser is not sharing the database', - severity: 'FATAL', + // we need to use proxywrap to make our tcp server to enable the PROXY protocol support + const net = ( + process.env.PROXIED ? (await import('findhit-proxywrap')).default.proxy(nodeNet) : nodeNet + ) as typeof nodeNet + + const tcpServer = net.createServer() + + tcpServer.on('connection', async (socket) => { + let databaseId: string | undefined + let connectionId: string | undefined + + const connection = await fromNodeSocket(socket, { + tls: getTls, + onTlsUpgrade(state) { + if (!state.tlsInfo?.serverName || !isValidServername(state.tlsInfo.serverName)) { + throw BackendError.create({ + code: '08006', + message: 'invalid SNI', + severity: 'FATAL', + }) + } + + const _databaseId = extractDatabaseId(state.tlsInfo.serverName!) + + if (!websocketConnections.has(_databaseId!)) { + throw BackendError.create({ + code: 'XX000', + message: 'the browser is not sharing the database', + severity: 'FATAL', + }) + } + + const tcpConnectionCount = tcpConnectionsByDatabaseId.get(_databaseId) ?? 0 + + if (tcpConnectionCount === 1) { + throw BackendError.create({ + code: '53300', + message: 'sorry, too many clients already', + severity: 'FATAL', + }) + } + + tcpConnectionsByDatabaseId.set(_databaseId, 1) + + // only set the databaseId after we've verified the connection + databaseId = _databaseId + }, + serverVersion() { + return '16.3' + }, + onAuthenticated() { + const websocket = websocketConnections.get(databaseId!) + + if (!websocket) { + throw BackendError.create({ + code: 'XX000', + message: 'the browser is not sharing the database', + severity: 'FATAL', + }) + } + + const _connectionId = new Uint8Array(8) + crypto.getRandomValues(_connectionId) + + connectionId = Buffer.from(_connectionId).toString('hex') + tcpConnections.set(connectionId, connection) + + logEvent(new UserConnected({ databaseId: databaseId! })) + + const clientIpMessage = createStartupMessage('postgres', 'postgres', { + client_ip: extractIP(socket.remoteAddress!), }) + websocket.send(wrapMessage(_connectionId, clientIpMessage)) + }, + onMessage(message, state) { + if (message.length === 0) { + return + } + + if (!state.isAuthenticated) { + return + } + + const websocket = websocketConnections.get(databaseId!) + + if (!websocket) { + throw BackendError.create({ + code: 'XX000', + message: 'the browser is not sharing the database', + severity: 'FATAL', + }) + } + + debug('tcp message', { message }) + // wrap the message with the connection id + websocket.send(wrapMessage(hexToUint8Array(connectionId!), message)) + + // return an empty buffer to indicate that the message has been handled + return new Uint8Array() + }, + }) + + socket.on('close', () => { + if (databaseId) { + tcpConnections.delete(connectionId!) + tcpConnectionsByDatabaseId.delete(databaseId) + logEvent(new UserDisconnected({ databaseId })) + const websocket = websocketConnections.get(databaseId) + websocket?.send( + wrapMessage( + hexToUint8Array(connectionId!), + createStartupMessage('postgres', 'postgres', { client_ip: '' }) + ) + ) } + }) + }) - const clientIpMessage = createStartupMessage('postgres', 'postgres', { - client_ip: extractIP(socket.remoteAddress!), - }) - websocket.send(clientIpMessage) - }, - onMessage(message, state) { - if (!state.isAuthenticated) { - return - } + httpsServer.listen(443, () => { + console.log('websocket server listening on port 443') + }) - const websocket = websocketConnections.get(databaseId!) + tcpServer.listen(5432, () => { + console.log('tcp server listening on port 5432') + }) - if (!websocket) { - throw BackendError.create({ - code: 'XX000', - message: 'the browser is not sharing the database', - severity: 'FATAL', - }) - } + function wrapMessage(connectionId: Uint8Array, message: ArrayBuffer | Uint8Array): Uint8Array { + // Convert message to Uint8Array if it's an ArrayBuffer + const messageArray = message instanceof ArrayBuffer ? new Uint8Array(message) : message - debug('tcp message', { message }) - websocket.send(message) + // Create a new Uint8Array to hold the connectionId and the message + const wrappedMessage = new Uint8Array(connectionId.length + messageArray.length) - // return an empty buffer to indicate that the message has been handled - return new Uint8Array() - }, - }) + // Copy the connectionId and the message into the new Uint8Array + wrappedMessage.set(connectionId, 0) + wrappedMessage.set(messageArray, connectionId.length) - socket.on('close', () => { - if (databaseId) { - tcpConnections.delete(databaseId) - logEvent(new UserDisconnected({ databaseId })) - const websocket = websocketConnections.get(databaseId) - websocket?.send(createStartupMessage('postgres', 'postgres', { client_ip: '' })) - } - }) -}) + return wrappedMessage + } -httpsServer.listen(443, () => { - console.log('websocket server listening on port 443') -}) + function uint8ArrayToHex(array: Uint8Array): string { + return Buffer.from(array).toString('hex') + } -tcpServer.listen(5432, () => { - console.log('tcp server listening on port 5432') -}) + function hexToUint8Array(hex: string): Uint8Array { + const buffer = Buffer.from(hex, 'hex') + return new Uint8Array(buffer) + } +} catch (error) { + console.error(error) +} diff --git a/apps/postgres-new/components/app-provider.tsx b/apps/postgres-new/components/app-provider.tsx index 5ffed968..fcee4bc0 100644 --- a/apps/postgres-new/components/app-provider.tsx +++ b/apps/postgres-new/components/app-provider.tsx @@ -134,10 +134,22 @@ export default function AppProvider({ children }: AppProps) { const mutex = new Mutex() let db: PGliteInterface + let connectionId: Uint8Array | undefined ws.onmessage = (event) => { mutex.runExclusive(async () => { - const message = new Uint8Array(await event.data) + const data = new Uint8Array(await event.data) + + const _connectionId = data.slice(0, 8) + if (!connectionId) { + connectionId = _connectionId + } + if (Array.from(connectionId).join('') !== Array.from(_connectionId).join('')) { + console.log('connectionId mismatch', connectionId, _connectionId) + return + } + + const message = data.slice(8) if (isStartupMessage(message)) { const parameters = parseStartupMessage(message) @@ -145,6 +157,7 @@ export default function AppProvider({ children }: AppProps) { // client disconnected if (parameters.client_ip === '') { setConnectedClientIp(null) + connectionId = undefined await dbManager.closeDbInstance(databaseId) } else { db = await dbManager.getDbInstance(databaseId) @@ -156,7 +169,11 @@ export default function AppProvider({ children }: AppProps) { const response = await db.execProtocolRaw(message) - ws.send(response) + const wrappedResponse = new Uint8Array(connectionId.length + response.length) + wrappedResponse.set(connectionId, 0) + wrappedResponse.set(response, connectionId.length) + + ws.send(wrappedResponse) }) } ws.onclose = (event) => { diff --git a/package.json b/package.json index 04bf2e12..a1bfd026 100644 --- a/package.json +++ b/package.json @@ -3,8 +3,11 @@ "scripts": { "dev": "npm run dev --workspace postgres-new" }, - "workspaces": ["apps/*"], + "workspaces": [ + "apps/*" + ], "devDependencies": { "supabase": "^1.191.3" - } + }, + "packageManager": "yarn@1.22.22+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e" } From 08a8a438087817a6829fb1aafd821621b641b71c Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 24 Sep 2024 16:15:03 +0200 Subject: [PATCH 02/20] prevent server crash --- apps/browser-proxy/src/index.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/apps/browser-proxy/src/index.ts b/apps/browser-proxy/src/index.ts index fdd1b48b..4a6dd2b8 100644 --- a/apps/browser-proxy/src/index.ts +++ b/apps/browser-proxy/src/index.ts @@ -16,6 +16,14 @@ import { UserDisconnected, } from './telemetry.ts' +process.on('unhandledRejection', (reason, promise) => { + console.error({ location: 'unhandledRejection', reason, promise }) +}) + +process.on('uncaughtException', (error) => { + console.error({ location: 'uncaughtException', error }) +}) + const debug = makeDebug('browser-proxy') try { type DatabaseId = string From ea23c717c19590d9f6bd81c7972fa2cba1ea4620 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 24 Sep 2024 16:38:13 +0200 Subject: [PATCH 03/20] fix --- apps/browser-proxy/src/index.ts | 383 ++++++++++++++++---------------- 1 file changed, 189 insertions(+), 194 deletions(-) diff --git a/apps/browser-proxy/src/index.ts b/apps/browser-proxy/src/index.ts index 4a6dd2b8..0674cd66 100644 --- a/apps/browser-proxy/src/index.ts +++ b/apps/browser-proxy/src/index.ts @@ -25,228 +25,223 @@ process.on('uncaughtException', (error) => { }) const debug = makeDebug('browser-proxy') -try { - type DatabaseId = string - type ConnectionId = string - const tcpConnections = new Map() - const tcpConnectionsByDatabaseId = new Map() - const websocketConnections = new Map() - - const httpsServer = https.createServer({ - SNICallback: (servername, callback) => { - debug('SNICallback', servername) - if (isValidServername(servername)) { - debug('SNICallback', 'valid') - callback(null) - } else { - debug('SNICallback', 'invalid') - callback(new Error('invalid SNI')) - } - }, - }) - await setSecureContext(httpsServer) - // reset the secure context every week to pick up any new TLS certificates - setInterval(() => setSecureContext(httpsServer), 1000 * 60 * 60 * 24 * 7) - const websocketServer = new WebSocketServer({ - server: httpsServer, - }) +type DatabaseId = string +type ConnectionId = string +const tcpConnections = new Map() +const tcpConnectionsByDatabaseId = new Map() +const websocketConnections = new Map() + +const httpsServer = https.createServer({ + SNICallback: (servername, callback) => { + debug('SNICallback', servername) + if (isValidServername(servername)) { + debug('SNICallback', 'valid') + callback(null) + } else { + debug('SNICallback', 'invalid') + callback(new Error('invalid SNI')) + } + }, +}) +await setSecureContext(httpsServer) +// reset the secure context every week to pick up any new TLS certificates +setInterval(() => setSecureContext(httpsServer), 1000 * 60 * 60 * 24 * 7) - websocketServer.on('error', (error) => { - debug('websocket server error', error) - }) +const websocketServer = new WebSocketServer({ + server: httpsServer, +}) - websocketServer.on('connection', (socket, request) => { - debug('websocket connection') +websocketServer.on('error', (error) => { + debug('websocket server error', error) +}) - const host = request.headers.host +websocketServer.on('connection', (socket, request) => { + debug('websocket connection') - if (!host) { - debug('No host header present') - socket.close() - return - } + const host = request.headers.host + + if (!host) { + debug('No host header present') + socket.close() + return + } + + const databaseId = extractDatabaseId(host) + + if (websocketConnections.has(databaseId)) { + socket.send('sorry, too many clients already') + socket.close() + return + } - const databaseId = extractDatabaseId(host) + websocketConnections.set(databaseId, socket) - if (websocketConnections.has(databaseId)) { - socket.send('sorry, too many clients already') - socket.close() + logEvent(new DatabaseShared({ databaseId })) + + socket.on('message', (data: Buffer) => { + if (data.length === 0) { return } - websocketConnections.set(databaseId, socket) + const connectionId = data.slice(0, 8) + const message = data.slice(8) + const tcpConnection = tcpConnections.get(Buffer.from(connectionId).toString('hex')) + if (tcpConnection) { + debug('websocket message', message.toString('hex')) + tcpConnection.streamWriter?.write(message) + } + }) + + socket.on('close', () => { + websocketConnections.delete(databaseId) + logEvent(new DatabaseUnshared({ databaseId })) + }) +}) + +// we need to use proxywrap to make our tcp server to enable the PROXY protocol support +const net = ( + process.env.PROXIED ? (await import('findhit-proxywrap')).default.proxy(nodeNet) : nodeNet +) as typeof nodeNet + +const tcpServer = net.createServer() + +tcpServer.on('connection', async (socket) => { + let databaseId: string | undefined + let connectionId: string | undefined + + debug('new tcp connection') + + const connection = await fromNodeSocket(socket, { + tls: getTls, + onTlsUpgrade(state) { + if (!state.tlsInfo?.serverName || !isValidServername(state.tlsInfo.serverName)) { + throw BackendError.create({ + code: '08006', + message: 'invalid SNI', + severity: 'FATAL', + }) + } + + const _databaseId = extractDatabaseId(state.tlsInfo.serverName!) + + if (!websocketConnections.has(_databaseId!)) { + throw BackendError.create({ + code: 'XX000', + message: 'the browser is not sharing the database', + severity: 'FATAL', + }) + } + + const tcpConnectionCount = tcpConnectionsByDatabaseId.get(_databaseId) ?? 0 + + if (tcpConnectionCount === 1) { + throw BackendError.create({ + code: '53300', + message: 'sorry, too many clients already', + severity: 'FATAL', + }) + } + + tcpConnectionsByDatabaseId.set(_databaseId, 1) + + // only set the databaseId after we've verified the connection + databaseId = _databaseId + }, + serverVersion() { + return '16.3' + }, + onAuthenticated() { + const websocket = websocketConnections.get(databaseId!) + + if (!websocket) { + throw BackendError.create({ + code: 'XX000', + message: 'the browser is not sharing the database', + severity: 'FATAL', + }) + } - logEvent(new DatabaseShared({ databaseId })) + const _connectionId = new Uint8Array(8) + crypto.getRandomValues(_connectionId) - socket.on('message', (data: Buffer) => { - if (data.length === 0) { + connectionId = Buffer.from(_connectionId).toString('hex') + tcpConnections.set(connectionId, connection) + + logEvent(new UserConnected({ databaseId: databaseId! })) + + const clientIpMessage = createStartupMessage('postgres', 'postgres', { + client_ip: extractIP(socket.remoteAddress!), + }) + websocket.send(wrapMessage(_connectionId, clientIpMessage)) + }, + onMessage(message, state) { + if (message.length === 0) { return } - const connectionId = data.slice(0, 8) - const message = data.slice(8) - const tcpConnection = tcpConnections.get(Buffer.from(connectionId).toString('hex')) - if (tcpConnection) { - debug('websocket message', message.toString('hex')) - tcpConnection.streamWriter?.write(message) + if (!state.isAuthenticated) { + return } - }) - socket.on('close', () => { - websocketConnections.delete(databaseId) - logEvent(new DatabaseUnshared({ databaseId })) - }) - }) + const websocket = websocketConnections.get(databaseId!) - // we need to use proxywrap to make our tcp server to enable the PROXY protocol support - const net = ( - process.env.PROXIED ? (await import('findhit-proxywrap')).default.proxy(nodeNet) : nodeNet - ) as typeof nodeNet - - const tcpServer = net.createServer() - - tcpServer.on('connection', async (socket) => { - let databaseId: string | undefined - let connectionId: string | undefined - - const connection = await fromNodeSocket(socket, { - tls: getTls, - onTlsUpgrade(state) { - if (!state.tlsInfo?.serverName || !isValidServername(state.tlsInfo.serverName)) { - throw BackendError.create({ - code: '08006', - message: 'invalid SNI', - severity: 'FATAL', - }) - } - - const _databaseId = extractDatabaseId(state.tlsInfo.serverName!) - - if (!websocketConnections.has(_databaseId!)) { - throw BackendError.create({ - code: 'XX000', - message: 'the browser is not sharing the database', - severity: 'FATAL', - }) - } - - const tcpConnectionCount = tcpConnectionsByDatabaseId.get(_databaseId) ?? 0 - - if (tcpConnectionCount === 1) { - throw BackendError.create({ - code: '53300', - message: 'sorry, too many clients already', - severity: 'FATAL', - }) - } - - tcpConnectionsByDatabaseId.set(_databaseId, 1) - - // only set the databaseId after we've verified the connection - databaseId = _databaseId - }, - serverVersion() { - return '16.3' - }, - onAuthenticated() { - const websocket = websocketConnections.get(databaseId!) - - if (!websocket) { - throw BackendError.create({ - code: 'XX000', - message: 'the browser is not sharing the database', - severity: 'FATAL', - }) - } - - const _connectionId = new Uint8Array(8) - crypto.getRandomValues(_connectionId) - - connectionId = Buffer.from(_connectionId).toString('hex') - tcpConnections.set(connectionId, connection) - - logEvent(new UserConnected({ databaseId: databaseId! })) - - const clientIpMessage = createStartupMessage('postgres', 'postgres', { - client_ip: extractIP(socket.remoteAddress!), + if (!websocket) { + throw BackendError.create({ + code: 'XX000', + message: 'the browser is not sharing the database', + severity: 'FATAL', }) - websocket.send(wrapMessage(_connectionId, clientIpMessage)) - }, - onMessage(message, state) { - if (message.length === 0) { - return - } - - if (!state.isAuthenticated) { - return - } - - const websocket = websocketConnections.get(databaseId!) - - if (!websocket) { - throw BackendError.create({ - code: 'XX000', - message: 'the browser is not sharing the database', - severity: 'FATAL', - }) - } - - debug('tcp message', { message }) - // wrap the message with the connection id - websocket.send(wrapMessage(hexToUint8Array(connectionId!), message)) - - // return an empty buffer to indicate that the message has been handled - return new Uint8Array() - }, - }) - - socket.on('close', () => { - if (databaseId) { - tcpConnections.delete(connectionId!) - tcpConnectionsByDatabaseId.delete(databaseId) - logEvent(new UserDisconnected({ databaseId })) - const websocket = websocketConnections.get(databaseId) - websocket?.send( - wrapMessage( - hexToUint8Array(connectionId!), - createStartupMessage('postgres', 'postgres', { client_ip: '' }) - ) - ) } - }) - }) - httpsServer.listen(443, () => { - console.log('websocket server listening on port 443') + debug('tcp message', { message }) + // wrap the message with the connection id + websocket.send(wrapMessage(hexToUint8Array(connectionId!), message)) + + // return an empty buffer to indicate that the message has been handled + return new Uint8Array() + }, }) - tcpServer.listen(5432, () => { - console.log('tcp server listening on port 5432') + socket.on('close', () => { + if (databaseId) { + tcpConnections.delete(connectionId!) + tcpConnectionsByDatabaseId.delete(databaseId) + logEvent(new UserDisconnected({ databaseId })) + const websocket = websocketConnections.get(databaseId) + websocket?.send( + wrapMessage( + hexToUint8Array(connectionId!), + createStartupMessage('postgres', 'postgres', { client_ip: '' }) + ) + ) + } }) +}) - function wrapMessage(connectionId: Uint8Array, message: ArrayBuffer | Uint8Array): Uint8Array { - // Convert message to Uint8Array if it's an ArrayBuffer - const messageArray = message instanceof ArrayBuffer ? new Uint8Array(message) : message +httpsServer.listen(443, () => { + console.log('websocket server listening on port 443') +}) - // Create a new Uint8Array to hold the connectionId and the message - const wrappedMessage = new Uint8Array(connectionId.length + messageArray.length) +tcpServer.listen(5432, () => { + console.log('tcp server listening on port 5432') +}) - // Copy the connectionId and the message into the new Uint8Array - wrappedMessage.set(connectionId, 0) - wrappedMessage.set(messageArray, connectionId.length) +function wrapMessage(connectionId: Uint8Array, message: ArrayBuffer | Uint8Array): Uint8Array { + // Convert message to Uint8Array if it's an ArrayBuffer + const messageArray = message instanceof ArrayBuffer ? new Uint8Array(message) : message - return wrappedMessage - } + // Create a new Uint8Array to hold the connectionId and the message + const wrappedMessage = new Uint8Array(connectionId.length + messageArray.length) - function uint8ArrayToHex(array: Uint8Array): string { - return Buffer.from(array).toString('hex') - } + // Copy the connectionId and the message into the new Uint8Array + wrappedMessage.set(connectionId, 0) + wrappedMessage.set(messageArray, connectionId.length) - function hexToUint8Array(hex: string): Uint8Array { - const buffer = Buffer.from(hex, 'hex') - return new Uint8Array(buffer) - } -} catch (error) { - console.error(error) + return wrappedMessage +} + +function hexToUint8Array(hex: string): Uint8Array { + const buffer = Buffer.from(hex, 'hex') + return new Uint8Array(buffer) } From ba37a9b69b1742ffe2d8b908ba130eeae90a7469 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 24 Sep 2024 16:49:43 +0200 Subject: [PATCH 04/20] prisma --- .../prisma/migrations/0_init/migration.sql | 76 +++++++++++++++++++ apps/browser-proxy/prisma/schema.prisma | 57 ++++++++++++-- 2 files changed, 127 insertions(+), 6 deletions(-) create mode 100644 apps/browser-proxy/prisma/migrations/0_init/migration.sql diff --git a/apps/browser-proxy/prisma/migrations/0_init/migration.sql b/apps/browser-proxy/prisma/migrations/0_init/migration.sql new file mode 100644 index 00000000..f15a1511 --- /dev/null +++ b/apps/browser-proxy/prisma/migrations/0_init/migration.sql @@ -0,0 +1,76 @@ +-- CreateTable +CREATE TABLE "comments" ( + "id" BIGSERIAL NOT NULL, + "post_id" BIGINT, + "user_id" BIGINT, + "content" TEXT NOT NULL, + "created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "comments_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "friendships" ( + "id" BIGSERIAL NOT NULL, + "user_id" BIGINT, + "friend_id" BIGINT, + "created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "friendships_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "likes" ( + "id" BIGSERIAL NOT NULL, + "post_id" BIGINT, + "user_id" BIGINT, + "created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "likes_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "posts" ( + "id" BIGSERIAL NOT NULL, + "user_id" BIGINT, + "content" TEXT NOT NULL, + "created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "posts_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "users" ( + "id" BIGSERIAL NOT NULL, + "name" TEXT NOT NULL, + "email" TEXT NOT NULL, + "password" TEXT NOT NULL, + "created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "users_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "users_email_key" ON "users"("email"); + +-- AddForeignKey +ALTER TABLE "comments" ADD CONSTRAINT "comments_post_id_fkey" FOREIGN KEY ("post_id") REFERENCES "posts"("id") ON DELETE NO ACTION ON UPDATE NO ACTION; + +-- AddForeignKey +ALTER TABLE "comments" ADD CONSTRAINT "comments_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE NO ACTION ON UPDATE NO ACTION; + +-- AddForeignKey +ALTER TABLE "friendships" ADD CONSTRAINT "friendships_friend_id_fkey" FOREIGN KEY ("friend_id") REFERENCES "users"("id") ON DELETE NO ACTION ON UPDATE NO ACTION; + +-- AddForeignKey +ALTER TABLE "friendships" ADD CONSTRAINT "friendships_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE NO ACTION ON UPDATE NO ACTION; + +-- AddForeignKey +ALTER TABLE "likes" ADD CONSTRAINT "likes_post_id_fkey" FOREIGN KEY ("post_id") REFERENCES "posts"("id") ON DELETE NO ACTION ON UPDATE NO ACTION; + +-- AddForeignKey +ALTER TABLE "likes" ADD CONSTRAINT "likes_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE NO ACTION ON UPDATE NO ACTION; + +-- AddForeignKey +ALTER TABLE "posts" ADD CONSTRAINT "posts_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE NO ACTION ON UPDATE NO ACTION; + diff --git a/apps/browser-proxy/prisma/schema.prisma b/apps/browser-proxy/prisma/schema.prisma index ee282c75..0677fbd8 100644 --- a/apps/browser-proxy/prisma/schema.prisma +++ b/apps/browser-proxy/prisma/schema.prisma @@ -1,9 +1,3 @@ -// This is your Prisma schema file, -// learn more about it in the docs: https://pris.ly/d/prisma-schema - -// Looking for ways to speed up your queries, or scale easily with your serverless or edge functions? -// Try Prisma Accelerate: https://pris.ly/cli/accelerate-init - generator client { provider = "prisma-client-js" } @@ -12,3 +6,54 @@ datasource db { provider = "postgresql" url = env("DATABASE_URL") } + +model comments { + id BigInt @id @default(autoincrement()) + post_id BigInt? + user_id BigInt? + content String + created_at DateTime? @default(now()) @db.Timestamptz(6) + posts posts? @relation(fields: [post_id], references: [id], onDelete: NoAction, onUpdate: NoAction) + users users? @relation(fields: [user_id], references: [id], onDelete: NoAction, onUpdate: NoAction) +} + +model friendships { + id BigInt @id @default(autoincrement()) + user_id BigInt? + friend_id BigInt? + created_at DateTime? @default(now()) @db.Timestamptz(6) + users_friendships_friend_idTousers users? @relation("friendships_friend_idTousers", fields: [friend_id], references: [id], onDelete: NoAction, onUpdate: NoAction) + users_friendships_user_idTousers users? @relation("friendships_user_idTousers", fields: [user_id], references: [id], onDelete: NoAction, onUpdate: NoAction) +} + +model likes { + id BigInt @id @default(autoincrement()) + post_id BigInt? + user_id BigInt? + created_at DateTime? @default(now()) @db.Timestamptz(6) + posts posts? @relation(fields: [post_id], references: [id], onDelete: NoAction, onUpdate: NoAction) + users users? @relation(fields: [user_id], references: [id], onDelete: NoAction, onUpdate: NoAction) +} + +model posts { + id BigInt @id @default(autoincrement()) + user_id BigInt? + content String + created_at DateTime? @default(now()) @db.Timestamptz(6) + comments comments[] + likes likes[] + users users? @relation(fields: [user_id], references: [id], onDelete: NoAction, onUpdate: NoAction) +} + +model users { + id BigInt @id @default(autoincrement()) + username String + email String @unique + password String + created_at DateTime? @default(now()) @db.Timestamptz(6) + comments comments[] + friendships_friendships_friend_idTousers friendships[] @relation("friendships_friend_idTousers") + friendships_friendships_user_idTousers friendships[] @relation("friendships_user_idTousers") + likes likes[] + posts posts[] +} From 1dfb0ed60c3fe4f22c782ab4f87c5ed32d3c90f9 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 24 Sep 2024 16:51:12 +0200 Subject: [PATCH 05/20] remove prisma --- .../prisma/migrations/0_init/migration.sql | 76 ------------------- apps/browser-proxy/prisma/schema.prisma | 59 -------------- 2 files changed, 135 deletions(-) delete mode 100644 apps/browser-proxy/prisma/migrations/0_init/migration.sql delete mode 100644 apps/browser-proxy/prisma/schema.prisma diff --git a/apps/browser-proxy/prisma/migrations/0_init/migration.sql b/apps/browser-proxy/prisma/migrations/0_init/migration.sql deleted file mode 100644 index f15a1511..00000000 --- a/apps/browser-proxy/prisma/migrations/0_init/migration.sql +++ /dev/null @@ -1,76 +0,0 @@ --- CreateTable -CREATE TABLE "comments" ( - "id" BIGSERIAL NOT NULL, - "post_id" BIGINT, - "user_id" BIGINT, - "content" TEXT NOT NULL, - "created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP, - - CONSTRAINT "comments_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "friendships" ( - "id" BIGSERIAL NOT NULL, - "user_id" BIGINT, - "friend_id" BIGINT, - "created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP, - - CONSTRAINT "friendships_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "likes" ( - "id" BIGSERIAL NOT NULL, - "post_id" BIGINT, - "user_id" BIGINT, - "created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP, - - CONSTRAINT "likes_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "posts" ( - "id" BIGSERIAL NOT NULL, - "user_id" BIGINT, - "content" TEXT NOT NULL, - "created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP, - - CONSTRAINT "posts_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "users" ( - "id" BIGSERIAL NOT NULL, - "name" TEXT NOT NULL, - "email" TEXT NOT NULL, - "password" TEXT NOT NULL, - "created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP, - - CONSTRAINT "users_pkey" PRIMARY KEY ("id") -); - --- CreateIndex -CREATE UNIQUE INDEX "users_email_key" ON "users"("email"); - --- AddForeignKey -ALTER TABLE "comments" ADD CONSTRAINT "comments_post_id_fkey" FOREIGN KEY ("post_id") REFERENCES "posts"("id") ON DELETE NO ACTION ON UPDATE NO ACTION; - --- AddForeignKey -ALTER TABLE "comments" ADD CONSTRAINT "comments_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE NO ACTION ON UPDATE NO ACTION; - --- AddForeignKey -ALTER TABLE "friendships" ADD CONSTRAINT "friendships_friend_id_fkey" FOREIGN KEY ("friend_id") REFERENCES "users"("id") ON DELETE NO ACTION ON UPDATE NO ACTION; - --- AddForeignKey -ALTER TABLE "friendships" ADD CONSTRAINT "friendships_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE NO ACTION ON UPDATE NO ACTION; - --- AddForeignKey -ALTER TABLE "likes" ADD CONSTRAINT "likes_post_id_fkey" FOREIGN KEY ("post_id") REFERENCES "posts"("id") ON DELETE NO ACTION ON UPDATE NO ACTION; - --- AddForeignKey -ALTER TABLE "likes" ADD CONSTRAINT "likes_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE NO ACTION ON UPDATE NO ACTION; - --- AddForeignKey -ALTER TABLE "posts" ADD CONSTRAINT "posts_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE NO ACTION ON UPDATE NO ACTION; - diff --git a/apps/browser-proxy/prisma/schema.prisma b/apps/browser-proxy/prisma/schema.prisma deleted file mode 100644 index 0677fbd8..00000000 --- a/apps/browser-proxy/prisma/schema.prisma +++ /dev/null @@ -1,59 +0,0 @@ -generator client { - provider = "prisma-client-js" -} - -datasource db { - provider = "postgresql" - url = env("DATABASE_URL") -} - -model comments { - id BigInt @id @default(autoincrement()) - post_id BigInt? - user_id BigInt? - content String - created_at DateTime? @default(now()) @db.Timestamptz(6) - posts posts? @relation(fields: [post_id], references: [id], onDelete: NoAction, onUpdate: NoAction) - users users? @relation(fields: [user_id], references: [id], onDelete: NoAction, onUpdate: NoAction) -} - -model friendships { - id BigInt @id @default(autoincrement()) - user_id BigInt? - friend_id BigInt? - created_at DateTime? @default(now()) @db.Timestamptz(6) - users_friendships_friend_idTousers users? @relation("friendships_friend_idTousers", fields: [friend_id], references: [id], onDelete: NoAction, onUpdate: NoAction) - users_friendships_user_idTousers users? @relation("friendships_user_idTousers", fields: [user_id], references: [id], onDelete: NoAction, onUpdate: NoAction) -} - -model likes { - id BigInt @id @default(autoincrement()) - post_id BigInt? - user_id BigInt? - created_at DateTime? @default(now()) @db.Timestamptz(6) - posts posts? @relation(fields: [post_id], references: [id], onDelete: NoAction, onUpdate: NoAction) - users users? @relation(fields: [user_id], references: [id], onDelete: NoAction, onUpdate: NoAction) -} - -model posts { - id BigInt @id @default(autoincrement()) - user_id BigInt? - content String - created_at DateTime? @default(now()) @db.Timestamptz(6) - comments comments[] - likes likes[] - users users? @relation(fields: [user_id], references: [id], onDelete: NoAction, onUpdate: NoAction) -} - -model users { - id BigInt @id @default(autoincrement()) - username String - email String @unique - password String - created_at DateTime? @default(now()) @db.Timestamptz(6) - comments comments[] - friendships_friendships_friend_idTousers friendships[] @relation("friendships_friend_idTousers") - friendships_friendships_user_idTousers friendships[] @relation("friendships_user_idTousers") - likes likes[] - posts posts[] -} From b438b865d767aa72e21c027918367e40c1d56d3d Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 24 Sep 2024 17:26:15 +0200 Subject: [PATCH 06/20] use a Set --- apps/browser-proxy/src/create-message.ts | 9 ++++----- apps/browser-proxy/src/index.ts | 21 ++++++++------------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/apps/browser-proxy/src/create-message.ts b/apps/browser-proxy/src/create-message.ts index bc3365d0..fa8fbf76 100644 --- a/apps/browser-proxy/src/create-message.ts +++ b/apps/browser-proxy/src/create-message.ts @@ -2,7 +2,7 @@ export function createStartupMessage( user: string, database: string, additionalParams: Record = {} -): ArrayBuffer { +): Uint8Array { const encoder = new TextEncoder() // Protocol version number (3.0) @@ -22,9 +22,8 @@ export function createStartupMessage( } messageLength += 1 // Null terminator - const message = new ArrayBuffer(4 + messageLength) - const view = new DataView(message) - const uint8Array = new Uint8Array(message) + const uint8Array = new Uint8Array(4 + messageLength) + const view = new DataView(uint8Array.buffer) let offset = 0 view.setInt32(offset, messageLength + 4, false) // Total message length (including itself) @@ -44,5 +43,5 @@ export function createStartupMessage( uint8Array.set([0], offset) // Final null terminator - return message + return uint8Array } diff --git a/apps/browser-proxy/src/index.ts b/apps/browser-proxy/src/index.ts index 0674cd66..92792529 100644 --- a/apps/browser-proxy/src/index.ts +++ b/apps/browser-proxy/src/index.ts @@ -29,7 +29,7 @@ const debug = makeDebug('browser-proxy') type DatabaseId = string type ConnectionId = string const tcpConnections = new Map() -const tcpConnectionsByDatabaseId = new Map() +const tcpConnectionsByDatabaseId = new Set() const websocketConnections = new Map() const httpsServer = https.createServer({ @@ -84,8 +84,8 @@ websocketServer.on('connection', (socket, request) => { return } - const connectionId = data.slice(0, 8) - const message = data.slice(8) + const connectionId = data.subarray(0, 8) + const message = data.subarray(8) const tcpConnection = tcpConnections.get(Buffer.from(connectionId).toString('hex')) if (tcpConnection) { debug('websocket message', message.toString('hex')) @@ -133,9 +133,7 @@ tcpServer.on('connection', async (socket) => { }) } - const tcpConnectionCount = tcpConnectionsByDatabaseId.get(_databaseId) ?? 0 - - if (tcpConnectionCount === 1) { + if (tcpConnectionsByDatabaseId.has(_databaseId)) { throw BackendError.create({ code: '53300', message: 'sorry, too many clients already', @@ -143,7 +141,7 @@ tcpServer.on('connection', async (socket) => { }) } - tcpConnectionsByDatabaseId.set(_databaseId, 1) + tcpConnectionsByDatabaseId.add(_databaseId) // only set the databaseId after we've verified the connection databaseId = _databaseId @@ -227,16 +225,13 @@ tcpServer.listen(5432, () => { console.log('tcp server listening on port 5432') }) -function wrapMessage(connectionId: Uint8Array, message: ArrayBuffer | Uint8Array): Uint8Array { - // Convert message to Uint8Array if it's an ArrayBuffer - const messageArray = message instanceof ArrayBuffer ? new Uint8Array(message) : message - +function wrapMessage(connectionId: Uint8Array, message: Uint8Array): Uint8Array { // Create a new Uint8Array to hold the connectionId and the message - const wrappedMessage = new Uint8Array(connectionId.length + messageArray.length) + const wrappedMessage = new Uint8Array(connectionId.length + message.length) // Copy the connectionId and the message into the new Uint8Array wrappedMessage.set(connectionId, 0) - wrappedMessage.set(messageArray, connectionId.length) + wrappedMessage.set(message, connectionId.length) return wrappedMessage } From 36f6616e075c80901788eecddb16fa6c11b75734 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 25 Sep 2024 09:22:47 +0200 Subject: [PATCH 07/20] auth gated live sharing --- apps/browser-proxy/.env.example | 4 ++- apps/browser-proxy/package.json | 1 + apps/browser-proxy/src/index.ts | 36 ++++++++++++++++--- apps/browser-proxy/src/telemetry.ts | 8 ++--- apps/postgres-new/components/app-provider.tsx | 12 ++++++- apps/postgres-new/components/sidebar.tsx | 4 +-- package-lock.json | 35 ++++++++++-------- 7 files changed, 72 insertions(+), 28 deletions(-) diff --git a/apps/browser-proxy/.env.example b/apps/browser-proxy/.env.example index 498d756b..c93b222e 100644 --- a/apps/browser-proxy/.env.example +++ b/apps/browser-proxy/.env.example @@ -6,4 +6,6 @@ AWS_REGION=us-east-1 LOGFLARE_SOURCE_URL="" # enable PROXY protocol support #PROXIED=true -WILDCARD_DOMAIN=browser.staging.db.build \ No newline at end of file +SUPABASE_URL="" +SUPABASE_ANON_KEY="" +WILDCARD_DOMAIN=browser.staging.db.build diff --git a/apps/browser-proxy/package.json b/apps/browser-proxy/package.json index d94243da..dcdbcd05 100644 --- a/apps/browser-proxy/package.json +++ b/apps/browser-proxy/package.json @@ -8,6 +8,7 @@ }, "dependencies": { "@aws-sdk/client-s3": "^3.645.0", + "@supabase/supabase-js": "^2.45.4", "debug": "^4.3.7", "expiry-map": "^2.0.0", "findhit-proxywrap": "^0.3.13", diff --git a/apps/browser-proxy/src/index.ts b/apps/browser-proxy/src/index.ts index 92792529..c289c347 100644 --- a/apps/browser-proxy/src/index.ts +++ b/apps/browser-proxy/src/index.ts @@ -4,6 +4,7 @@ import { BackendError, PostgresConnection } from 'pg-gateway' import { fromNodeSocket } from 'pg-gateway/node' import { WebSocketServer, type WebSocket } from 'ws' import makeDebug from 'debug' +import { createClient } from '@supabase/supabase-js' import { extractDatabaseId, isValidServername } from './servername.ts' import { getTls, setSecureContext } from './tls.ts' import { createStartupMessage } from './create-message.ts' @@ -16,6 +17,14 @@ import { UserDisconnected, } from './telemetry.ts' +const supabase = createClient(process.env.SUPABASE_URL!, process.env.SUPABASE_ANON_KEY!, { + auth: { + autoRefreshToken: false, + persistSession: false, + detectSessionInUrl: false, + }, +}) + process.on('unhandledRejection', (reason, promise) => { console.error({ location: 'unhandledRejection', reason, promise }) }) @@ -56,7 +65,7 @@ websocketServer.on('error', (error) => { debug('websocket server error', error) }) -websocketServer.on('connection', (socket, request) => { +websocketServer.on('connection', async (socket, request) => { debug('websocket connection') const host = request.headers.host @@ -67,6 +76,23 @@ websocketServer.on('connection', (socket, request) => { return } + // authenticate the user + const url = new URL(request.url!, `https://${host}`) + const token = url.searchParams.get('token') + if (!token) { + debug('No token present in URL query parameters') + socket.close() + return + } + const { data, error } = await supabase.auth.getUser(token) + if (error) { + debug('Error authenticating user', error) + socket.close() + return + } + + const { user } = data + const databaseId = extractDatabaseId(host) if (websocketConnections.has(databaseId)) { @@ -77,7 +103,7 @@ websocketServer.on('connection', (socket, request) => { websocketConnections.set(databaseId, socket) - logEvent(new DatabaseShared({ databaseId })) + logEvent(new DatabaseShared({ databaseId, userId: user.id })) socket.on('message', (data: Buffer) => { if (data.length === 0) { @@ -95,7 +121,7 @@ websocketServer.on('connection', (socket, request) => { socket.on('close', () => { websocketConnections.delete(databaseId) - logEvent(new DatabaseUnshared({ databaseId })) + logEvent(new DatabaseUnshared({ databaseId, userId: user.id })) }) }) @@ -166,7 +192,7 @@ tcpServer.on('connection', async (socket) => { connectionId = Buffer.from(_connectionId).toString('hex') tcpConnections.set(connectionId, connection) - logEvent(new UserConnected({ databaseId: databaseId! })) + logEvent(new UserConnected({ databaseId: databaseId!, connectionId })) const clientIpMessage = createStartupMessage('postgres', 'postgres', { client_ip: extractIP(socket.remoteAddress!), @@ -205,7 +231,7 @@ tcpServer.on('connection', async (socket) => { if (databaseId) { tcpConnections.delete(connectionId!) tcpConnectionsByDatabaseId.delete(databaseId) - logEvent(new UserDisconnected({ databaseId })) + logEvent(new UserDisconnected({ databaseId, connectionId: connectionId! })) const websocket = websocketConnections.get(databaseId) websocket?.send( wrapMessage( diff --git a/apps/browser-proxy/src/telemetry.ts b/apps/browser-proxy/src/telemetry.ts index 0f18d1e0..2bbf22fe 100644 --- a/apps/browser-proxy/src/telemetry.ts +++ b/apps/browser-proxy/src/telemetry.ts @@ -8,25 +8,25 @@ class BaseEvent { } export class DatabaseShared extends BaseEvent { - constructor(metadata: { databaseId: string }) { + constructor(metadata: { databaseId: string; userId: string }) { super('database-shared', metadata) } } export class DatabaseUnshared extends BaseEvent { - constructor(metadata: { databaseId: string }) { + constructor(metadata: { databaseId: string; userId: string }) { super('database-unshared', metadata) } } export class UserConnected extends BaseEvent { - constructor(metadata: { databaseId: string }) { + constructor(metadata: { databaseId: string; connectionId: string }) { super('user-connected', metadata) } } export class UserDisconnected extends BaseEvent { - constructor(metadata: { databaseId: string }) { + constructor(metadata: { databaseId: string; connectionId: string }) { super('user-disconnected', metadata) } } diff --git a/apps/postgres-new/components/app-provider.tsx b/apps/postgres-new/components/app-provider.tsx index fcee4bc0..7dfb5463 100644 --- a/apps/postgres-new/components/app-provider.tsx +++ b/apps/postgres-new/components/app-provider.tsx @@ -124,7 +124,17 @@ export default function AppProvider({ children }: AppProps) { const databaseHostname = `${databaseId}.${process.env.NEXT_PUBLIC_BROWSER_PROXY_DOMAIN}` - const ws = new WebSocket(`wss://${databaseHostname}`) + const { + data: { session }, + } = await supabase.auth.getSession() + + if (!session) { + throw new Error('You must be signed in to live share') + } + + const ws = new WebSocket( + `wss://${databaseHostname}?token=${encodeURIComponent(session.access_token)}` + ) ws.binaryType = 'arraybuffer' diff --git a/apps/postgres-new/components/sidebar.tsx b/apps/postgres-new/components/sidebar.tsx index c5381071..f519eaf4 100644 --- a/apps/postgres-new/components/sidebar.tsx +++ b/apps/postgres-new/components/sidebar.tsx @@ -523,7 +523,7 @@ type ConnectMenuItemProps = { } function LiveShareMenuItem(props: ConnectMenuItemProps) { - const { liveShare } = useApp() + const { liveShare, user } = useApp() // Only show the connect menu item on fully loaded dashboard if (!props.isActive) { @@ -533,7 +533,7 @@ function LiveShareMenuItem(props: ConnectMenuItemProps) { if (!liveShare.isLiveSharing) { return ( { e.preventDefault() diff --git a/package-lock.json b/package-lock.json index 5b98f519..941f7baf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "name": "@database.build/browser-proxy", "dependencies": { "@aws-sdk/client-s3": "^3.645.0", + "@supabase/supabase-js": "^2.45.4", "debug": "^4.3.7", "expiry-map": "^2.0.0", "findhit-proxywrap": "^0.3.13", @@ -4095,9 +4096,10 @@ } }, "node_modules/@supabase/auth-js": { - "version": "2.64.4", - "resolved": "https://registry.npmjs.org/@supabase/auth-js/-/auth-js-2.64.4.tgz", - "integrity": "sha512-9ITagy4WP4FLl+mke1rchapOH0RQpf++DI+WSG2sO1OFOZ0rW3cwAM0nCrMOxu+Zw4vJ4zObc08uvQrXx590Tg==", + "version": "2.65.0", + "resolved": "https://registry.npmjs.org/@supabase/auth-js/-/auth-js-2.65.0.tgz", + "integrity": "sha512-+wboHfZufAE2Y612OsKeVP4rVOeGZzzMLD/Ac3HrTQkkY4qXNjI6Af9gtmxwccE5nFvTiF114FEbIQ1hRq5uUw==", + "license": "MIT", "dependencies": { "@supabase/node-fetch": "^2.6.14" } @@ -4150,9 +4152,10 @@ } }, "node_modules/@supabase/postgrest-js": { - "version": "1.15.8", - "resolved": "https://registry.npmjs.org/@supabase/postgrest-js/-/postgrest-js-1.15.8.tgz", - "integrity": "sha512-YunjXpoQjQ0a0/7vGAvGZA2dlMABXFdVI/8TuVKtlePxyT71sl6ERl6ay1fmIeZcqxiuFQuZw/LXUuStUG9bbg==", + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/@supabase/postgrest-js/-/postgrest-js-1.16.1.tgz", + "integrity": "sha512-EOSEZFm5pPuCPGCmLF1VOCS78DfkSz600PBuvBND/IZmMciJ1pmsS3ss6TkB6UkuvTybYiBh7gKOYyxoEO3USA==", + "license": "MIT", "dependencies": { "@supabase/node-fetch": "^2.6.14" } @@ -4183,24 +4186,26 @@ } }, "node_modules/@supabase/storage-js": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/@supabase/storage-js/-/storage-js-2.6.0.tgz", - "integrity": "sha512-REAxr7myf+3utMkI2oOmZ6sdplMZZ71/2NEIEMBZHL9Fkmm3/JnaOZVSRqvG4LStYj2v5WhCruCzuMn6oD/Drw==", + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/@supabase/storage-js/-/storage-js-2.7.0.tgz", + "integrity": "sha512-iZenEdO6Mx9iTR6T7wC7sk6KKsoDPLq8rdu5VRy7+JiT1i8fnqfcOr6mfF2Eaqky9VQzhP8zZKQYjzozB65Rig==", + "license": "MIT", "dependencies": { "@supabase/node-fetch": "^2.6.14" } }, "node_modules/@supabase/supabase-js": { - "version": "2.45.0", - "resolved": "https://registry.npmjs.org/@supabase/supabase-js/-/supabase-js-2.45.0.tgz", - "integrity": "sha512-j66Mfs8RhzCQCKxKogAFQYH9oNhRmgIdKk6pexguI2Oc7hi+nL9UNJug5aL1tKnBdaBM3h65riPLQSdL6sWa3Q==", + "version": "2.45.4", + "resolved": "https://registry.npmjs.org/@supabase/supabase-js/-/supabase-js-2.45.4.tgz", + "integrity": "sha512-E5p8/zOLaQ3a462MZnmnz03CrduA5ySH9hZyL03Y+QZLIOO4/Gs8Rdy4ZCKDHsN7x0xdanVEWWFN3pJFQr9/hg==", + "license": "MIT", "dependencies": { - "@supabase/auth-js": "2.64.4", + "@supabase/auth-js": "2.65.0", "@supabase/functions-js": "2.4.1", "@supabase/node-fetch": "2.6.15", - "@supabase/postgrest-js": "1.15.8", + "@supabase/postgrest-js": "1.16.1", "@supabase/realtime-js": "2.10.2", - "@supabase/storage-js": "2.6.0" + "@supabase/storage-js": "2.7.0" } }, "node_modules/@swc/counter": { From 415a47367879a5b1f0ca829180fd86a5edb03647 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 25 Sep 2024 12:14:29 +0200 Subject: [PATCH 08/20] refactor --- apps/browser-proxy/package.json | 1 + apps/browser-proxy/src/connection-manager.ts | 53 ++++ apps/browser-proxy/src/create-message.ts | 8 + apps/browser-proxy/src/debug.ts | 3 + apps/browser-proxy/src/index.ts | 254 +----------------- apps/browser-proxy/src/protocol.ts | 23 ++ apps/browser-proxy/src/tcp-server.ts | 114 ++++++++ apps/browser-proxy/src/websocket-server.ts | 100 +++++++ apps/postgres-new/components/app-provider.tsx | 39 +-- apps/postgres-new/lib/pg-wire-util.ts | 21 ++ apps/postgres-new/lib/websocket-protocol.ts | 21 ++ package-lock.json | 19 ++ 12 files changed, 376 insertions(+), 280 deletions(-) create mode 100644 apps/browser-proxy/src/connection-manager.ts create mode 100644 apps/browser-proxy/src/debug.ts create mode 100644 apps/browser-proxy/src/protocol.ts create mode 100644 apps/browser-proxy/src/tcp-server.ts create mode 100644 apps/browser-proxy/src/websocket-server.ts create mode 100644 apps/postgres-new/lib/websocket-protocol.ts diff --git a/apps/browser-proxy/package.json b/apps/browser-proxy/package.json index dcdbcd05..608cccfe 100644 --- a/apps/browser-proxy/package.json +++ b/apps/browser-proxy/package.json @@ -12,6 +12,7 @@ "debug": "^4.3.7", "expiry-map": "^2.0.0", "findhit-proxywrap": "^0.3.13", + "nanoid": "^5.0.7", "p-memoize": "^7.1.1", "pg-gateway": "^0.3.0-beta.3", "ws": "^8.18.0" diff --git a/apps/browser-proxy/src/connection-manager.ts b/apps/browser-proxy/src/connection-manager.ts new file mode 100644 index 00000000..5f93f22c --- /dev/null +++ b/apps/browser-proxy/src/connection-manager.ts @@ -0,0 +1,53 @@ +import type { PostgresConnection } from 'pg-gateway' +import type { WebSocket } from 'ws' + +type DatabaseId = string +type ConnectionId = string + +class ConnectionManager { + private socketsByDatabase: Map = new Map() + private sockets: Map = new Map() + private websockets: Map = new Map() + + constructor() {} + + public hasSocketForDatabase(databaseId: DatabaseId) { + return this.socketsByDatabase.has(databaseId) + } + + public getSocket(connectionId: ConnectionId) { + return this.sockets.get(connectionId) + } + + public setSocket(databaseId: DatabaseId, connectionId: ConnectionId, socket: PostgresConnection) { + this.sockets.set(connectionId, socket) + this.socketsByDatabase.set(databaseId, connectionId) + } + + public deleteSocketForDatabase(databaseId: DatabaseId) { + const connectionId = this.socketsByDatabase.get(databaseId) + this.socketsByDatabase.delete(databaseId) + if (connectionId) { + this.sockets.delete(connectionId) + } + } + + public hasWebsocket(databaseId: DatabaseId) { + return this.websockets.has(databaseId) + } + + public getWebsocket(databaseId: DatabaseId) { + return this.websockets.get(databaseId) + } + + public setWebsocket(databaseId: DatabaseId, websocket: WebSocket) { + this.websockets.set(databaseId, websocket) + } + + public deleteWebsocket(databaseId: DatabaseId) { + this.websockets.delete(databaseId) + this.deleteSocketForDatabase(databaseId) + } +} + +export const connectionManager = new ConnectionManager() diff --git a/apps/browser-proxy/src/create-message.ts b/apps/browser-proxy/src/create-message.ts index fa8fbf76..c98acbee 100644 --- a/apps/browser-proxy/src/create-message.ts +++ b/apps/browser-proxy/src/create-message.ts @@ -45,3 +45,11 @@ export function createStartupMessage( return uint8Array } + +export function createTerminateMessage(): Uint8Array { + const uint8Array = new Uint8Array(5) + const view = new DataView(uint8Array.buffer) + view.setUint8(0, 'X'.charCodeAt(0)) + view.setUint32(1, 4, false) + return uint8Array +} diff --git a/apps/browser-proxy/src/debug.ts b/apps/browser-proxy/src/debug.ts new file mode 100644 index 00000000..a7a71621 --- /dev/null +++ b/apps/browser-proxy/src/debug.ts @@ -0,0 +1,3 @@ +import makeDebug from 'debug' + +export const debug = makeDebug('browser-proxy') diff --git a/apps/browser-proxy/src/index.ts b/apps/browser-proxy/src/index.ts index c289c347..b9a78294 100644 --- a/apps/browser-proxy/src/index.ts +++ b/apps/browser-proxy/src/index.ts @@ -1,29 +1,5 @@ -import * as nodeNet from 'node:net' -import * as https from 'node:https' -import { BackendError, PostgresConnection } from 'pg-gateway' -import { fromNodeSocket } from 'pg-gateway/node' -import { WebSocketServer, type WebSocket } from 'ws' -import makeDebug from 'debug' -import { createClient } from '@supabase/supabase-js' -import { extractDatabaseId, isValidServername } from './servername.ts' -import { getTls, setSecureContext } from './tls.ts' -import { createStartupMessage } from './create-message.ts' -import { extractIP } from './extract-ip.ts' -import { - DatabaseShared, - DatabaseUnshared, - logEvent, - UserConnected, - UserDisconnected, -} from './telemetry.ts' - -const supabase = createClient(process.env.SUPABASE_URL!, process.env.SUPABASE_ANON_KEY!, { - auth: { - autoRefreshToken: false, - persistSession: false, - detectSessionInUrl: false, - }, -}) +import { httpsServer } from './websocket-server.ts' +import { tcpServer } from './tcp-server.ts' process.on('unhandledRejection', (reason, promise) => { console.error({ location: 'unhandledRejection', reason, promise }) @@ -33,216 +9,6 @@ process.on('uncaughtException', (error) => { console.error({ location: 'uncaughtException', error }) }) -const debug = makeDebug('browser-proxy') - -type DatabaseId = string -type ConnectionId = string -const tcpConnections = new Map() -const tcpConnectionsByDatabaseId = new Set() -const websocketConnections = new Map() - -const httpsServer = https.createServer({ - SNICallback: (servername, callback) => { - debug('SNICallback', servername) - if (isValidServername(servername)) { - debug('SNICallback', 'valid') - callback(null) - } else { - debug('SNICallback', 'invalid') - callback(new Error('invalid SNI')) - } - }, -}) -await setSecureContext(httpsServer) -// reset the secure context every week to pick up any new TLS certificates -setInterval(() => setSecureContext(httpsServer), 1000 * 60 * 60 * 24 * 7) - -const websocketServer = new WebSocketServer({ - server: httpsServer, -}) - -websocketServer.on('error', (error) => { - debug('websocket server error', error) -}) - -websocketServer.on('connection', async (socket, request) => { - debug('websocket connection') - - const host = request.headers.host - - if (!host) { - debug('No host header present') - socket.close() - return - } - - // authenticate the user - const url = new URL(request.url!, `https://${host}`) - const token = url.searchParams.get('token') - if (!token) { - debug('No token present in URL query parameters') - socket.close() - return - } - const { data, error } = await supabase.auth.getUser(token) - if (error) { - debug('Error authenticating user', error) - socket.close() - return - } - - const { user } = data - - const databaseId = extractDatabaseId(host) - - if (websocketConnections.has(databaseId)) { - socket.send('sorry, too many clients already') - socket.close() - return - } - - websocketConnections.set(databaseId, socket) - - logEvent(new DatabaseShared({ databaseId, userId: user.id })) - - socket.on('message', (data: Buffer) => { - if (data.length === 0) { - return - } - - const connectionId = data.subarray(0, 8) - const message = data.subarray(8) - const tcpConnection = tcpConnections.get(Buffer.from(connectionId).toString('hex')) - if (tcpConnection) { - debug('websocket message', message.toString('hex')) - tcpConnection.streamWriter?.write(message) - } - }) - - socket.on('close', () => { - websocketConnections.delete(databaseId) - logEvent(new DatabaseUnshared({ databaseId, userId: user.id })) - }) -}) - -// we need to use proxywrap to make our tcp server to enable the PROXY protocol support -const net = ( - process.env.PROXIED ? (await import('findhit-proxywrap')).default.proxy(nodeNet) : nodeNet -) as typeof nodeNet - -const tcpServer = net.createServer() - -tcpServer.on('connection', async (socket) => { - let databaseId: string | undefined - let connectionId: string | undefined - - debug('new tcp connection') - - const connection = await fromNodeSocket(socket, { - tls: getTls, - onTlsUpgrade(state) { - if (!state.tlsInfo?.serverName || !isValidServername(state.tlsInfo.serverName)) { - throw BackendError.create({ - code: '08006', - message: 'invalid SNI', - severity: 'FATAL', - }) - } - - const _databaseId = extractDatabaseId(state.tlsInfo.serverName!) - - if (!websocketConnections.has(_databaseId!)) { - throw BackendError.create({ - code: 'XX000', - message: 'the browser is not sharing the database', - severity: 'FATAL', - }) - } - - if (tcpConnectionsByDatabaseId.has(_databaseId)) { - throw BackendError.create({ - code: '53300', - message: 'sorry, too many clients already', - severity: 'FATAL', - }) - } - - tcpConnectionsByDatabaseId.add(_databaseId) - - // only set the databaseId after we've verified the connection - databaseId = _databaseId - }, - serverVersion() { - return '16.3' - }, - onAuthenticated() { - const websocket = websocketConnections.get(databaseId!) - - if (!websocket) { - throw BackendError.create({ - code: 'XX000', - message: 'the browser is not sharing the database', - severity: 'FATAL', - }) - } - - const _connectionId = new Uint8Array(8) - crypto.getRandomValues(_connectionId) - - connectionId = Buffer.from(_connectionId).toString('hex') - tcpConnections.set(connectionId, connection) - - logEvent(new UserConnected({ databaseId: databaseId!, connectionId })) - - const clientIpMessage = createStartupMessage('postgres', 'postgres', { - client_ip: extractIP(socket.remoteAddress!), - }) - websocket.send(wrapMessage(_connectionId, clientIpMessage)) - }, - onMessage(message, state) { - if (message.length === 0) { - return - } - - if (!state.isAuthenticated) { - return - } - - const websocket = websocketConnections.get(databaseId!) - - if (!websocket) { - throw BackendError.create({ - code: 'XX000', - message: 'the browser is not sharing the database', - severity: 'FATAL', - }) - } - - debug('tcp message', { message }) - // wrap the message with the connection id - websocket.send(wrapMessage(hexToUint8Array(connectionId!), message)) - - // return an empty buffer to indicate that the message has been handled - return new Uint8Array() - }, - }) - - socket.on('close', () => { - if (databaseId) { - tcpConnections.delete(connectionId!) - tcpConnectionsByDatabaseId.delete(databaseId) - logEvent(new UserDisconnected({ databaseId, connectionId: connectionId! })) - const websocket = websocketConnections.get(databaseId) - websocket?.send( - wrapMessage( - hexToUint8Array(connectionId!), - createStartupMessage('postgres', 'postgres', { client_ip: '' }) - ) - ) - } - }) -}) - httpsServer.listen(443, () => { console.log('websocket server listening on port 443') }) @@ -250,19 +16,3 @@ httpsServer.listen(443, () => { tcpServer.listen(5432, () => { console.log('tcp server listening on port 5432') }) - -function wrapMessage(connectionId: Uint8Array, message: Uint8Array): Uint8Array { - // Create a new Uint8Array to hold the connectionId and the message - const wrappedMessage = new Uint8Array(connectionId.length + message.length) - - // Copy the connectionId and the message into the new Uint8Array - wrappedMessage.set(connectionId, 0) - wrappedMessage.set(message, connectionId.length) - - return wrappedMessage -} - -function hexToUint8Array(hex: string): Uint8Array { - const buffer = Buffer.from(hex, 'hex') - return new Uint8Array(buffer) -} diff --git a/apps/browser-proxy/src/protocol.ts b/apps/browser-proxy/src/protocol.ts new file mode 100644 index 00000000..0ff0a71b --- /dev/null +++ b/apps/browser-proxy/src/protocol.ts @@ -0,0 +1,23 @@ +import { customAlphabet } from 'nanoid' + +const nanoid = customAlphabet('0123456789abcdefghijklmnopqrstuvwxyz', 16) + +export function getConnectionId(): string { + return nanoid() +} + +export function parse(data: T) { + const connectionIdBytes = data.subarray(0, 16) + const connectionId = new TextDecoder().decode(connectionIdBytes) + const message = data.subarray(16) + return { connectionId, message } as { connectionId: string; message: T } +} + +export function serialize(connectionId: string, message: Uint8Array) { + const encoder = new TextEncoder() + const connectionIdBytes = encoder.encode(connectionId) + const data = new Uint8Array(connectionIdBytes.length + message.length) + data.set(connectionIdBytes, 0) + data.set(message, connectionIdBytes.length) + return data +} diff --git a/apps/browser-proxy/src/tcp-server.ts b/apps/browser-proxy/src/tcp-server.ts new file mode 100644 index 00000000..873dab3d --- /dev/null +++ b/apps/browser-proxy/src/tcp-server.ts @@ -0,0 +1,114 @@ +import * as nodeNet from 'node:net' +import { BackendError } from 'pg-gateway' +import { fromNodeSocket } from 'pg-gateway/node' +import { extractDatabaseId, isValidServername } from './servername.ts' +import { getTls } from './tls.ts' +import { createStartupMessage, createTerminateMessage } from './create-message.ts' +import { extractIP } from './extract-ip.ts' +import { logEvent, UserConnected, UserDisconnected } from './telemetry.ts' +import { connectionManager } from './connection-manager.ts' +import { debug as mainDebug } from './debug.ts' +import { getConnectionId, serialize } from './protocol.ts' + +const debug = mainDebug.extend('tcp-server') + +// we need to use proxywrap to make our tcp server to enable the PROXY protocol support +const net = ( + process.env.PROXIED ? (await import('findhit-proxywrap')).default.proxy(nodeNet) : nodeNet +) as typeof nodeNet + +export const tcpServer = net.createServer() + +tcpServer.on('connection', async (socket) => { + let connectionState: { + databaseId: string + connectionId: string + } | null = null + + debug('new tcp connection') + + const connection = await fromNodeSocket(socket, { + tls: getTls, + onTlsUpgrade(state) { + if (!state.tlsInfo?.serverName || !isValidServername(state.tlsInfo.serverName)) { + throw BackendError.create({ + code: '08006', + message: 'invalid SNI', + severity: 'FATAL', + }) + } + + const databaseId = extractDatabaseId(state.tlsInfo.serverName!) + + const websocket = connectionManager.getWebsocket(databaseId) + + if (!websocket) { + throw BackendError.create({ + code: 'XX000', + message: 'the browser is not sharing the database', + severity: 'FATAL', + }) + } + + if (connectionManager.hasSocketForDatabase(databaseId)) { + throw BackendError.create({ + code: '53300', + message: 'sorry, too many clients already', + severity: 'FATAL', + }) + } + + const connectionId = getConnectionId() + connectionManager.setSocket(databaseId, connectionId, connection) + + connectionState = { databaseId, connectionId } + + logEvent(new UserConnected({ databaseId, connectionId })) + + const clientIpMessage = createStartupMessage('postgres', 'postgres', { + client_ip: extractIP(socket.remoteAddress!), + }) + websocket.send(serialize(connectionId, clientIpMessage)) + }, + serverVersion() { + return '16.3' + }, + onMessage(message, state) { + if (!state.isAuthenticated) { + return + } + + const websocket = connectionManager.getWebsocket(connectionState!.databaseId) + + if (!websocket) { + throw BackendError.create({ + code: 'XX000', + message: 'the browser is not sharing the database', + severity: 'FATAL', + }) + } + + debug('tcp message', { message }) + websocket.send(serialize(connectionState!.connectionId, message)) + + // return an empty buffer to indicate that the message has been handled + return new Uint8Array() + }, + }) + + socket.on('close', () => { + if (connectionState) { + connectionManager.deleteSocketForDatabase(connectionState.databaseId) + + logEvent( + new UserDisconnected({ + databaseId: connectionState.databaseId, + connectionId: connectionState.connectionId, + }) + ) + + const websocket = connectionManager.getWebsocket(connectionState.databaseId) + websocket?.send(serialize(connectionState.connectionId, createTerminateMessage())) + } + }) +}) diff --git a/apps/browser-proxy/src/websocket-server.ts b/apps/browser-proxy/src/websocket-server.ts new file mode 100644 index 00000000..43ebddb8 --- /dev/null +++ b/apps/browser-proxy/src/websocket-server.ts @@ -0,0 +1,100 @@ +import * as https from 'node:https' +import { WebSocketServer } from 'ws' +import { debug as mainDebug } from './debug.ts' +import { createClient } from '@supabase/supabase-js' +import { extractDatabaseId, isValidServername } from './servername.ts' +import { setSecureContext } from './tls.ts' +import { connectionManager } from './connection-manager.ts' +import { DatabaseShared, DatabaseUnshared, logEvent } from './telemetry.ts' +import { parse } from './protocol.ts' + +const debug = mainDebug.extend('websocket-server') + +const supabase = createClient(process.env.SUPABASE_URL!, process.env.SUPABASE_ANON_KEY!, { + auth: { + autoRefreshToken: false, + persistSession: false, + detectSessionInUrl: false, + }, +}) + +export const httpsServer = https.createServer({ + SNICallback: (servername, callback) => { + debug('SNICallback', servername) + if (isValidServername(servername)) { + debug('SNICallback', 'valid') + callback(null) + } else { + debug('SNICallback', 'invalid') + callback(new Error('invalid SNI')) + } + }, +}) + +await setSecureContext(httpsServer) + +// reset the secure context every week to pick up any new TLS certificates +setInterval(() => setSecureContext(httpsServer), 1000 * 60 * 60 * 24 * 7) + +const websocketServer = new WebSocketServer({ + server: httpsServer, +}) + +websocketServer.on('error', (error) => { + debug('websocket server error', error) +}) + +websocketServer.on('connection', async (websocket, request) => { + debug('websocket connection') + + const host = request.headers.host + + if (!host) { + debug('No host header present') + websocket.close() + return + } + + // authenticate the user + const url = new URL(request.url!, `https://${host}`) + const token = url.searchParams.get('token') + if (!token) { + debug('No token present in URL query parameters') + websocket.close() + return + } + const { data, error } = await supabase.auth.getUser(token) + if (error) { + debug('Error authenticating user', error) + websocket.close() + return + } + + const { user } = data + + const databaseId = extractDatabaseId(host) + + if (connectionManager.hasWebsocket(databaseId)) { + debug('Database already shared') + websocket.close() + return + } + + connectionManager.setWebsocket(databaseId, websocket) + logEvent(new DatabaseShared({ databaseId, userId: user.id })) + + websocket.on('message', (data: Buffer) => { + const { connectionId, message } = parse(data) + const tcpConnection = connectionManager.getSocket(connectionId) + if (tcpConnection) { + debug('websocket message', message.toString('hex')) + tcpConnection.streamWriter?.write(message) + } + }) + + websocket.on('close', () => { + connectionManager.deleteWebsocket(databaseId) + // TODO: have a way of ending a PostgresConnection + logEvent(new DatabaseUnshared({ databaseId, userId: user.id })) + }) +}) diff --git a/apps/postgres-new/components/app-provider.tsx b/apps/postgres-new/components/app-provider.tsx index 7dfb5463..c102be13 100644 --- a/apps/postgres-new/components/app-provider.tsx +++ b/apps/postgres-new/components/app-provider.tsx @@ -19,7 +19,8 @@ import { } from 'react' import { DbManager } from '~/lib/db' import { useAsyncMemo } from '~/lib/hooks' -import { isStartupMessage, parseStartupMessage } from '~/lib/pg-wire-util' +import { isStartupMessage, isTerminateMessage, parseStartupMessage } from '~/lib/pg-wire-util' +import { parse, serialize } from '~/lib/websocket-protocol' import { createClient } from '~/utils/supabase/client' export type AppProps = PropsWithChildren @@ -144,46 +145,28 @@ export default function AppProvider({ children }: AppProps) { const mutex = new Mutex() let db: PGliteInterface - let connectionId: Uint8Array | undefined ws.onmessage = (event) => { mutex.runExclusive(async () => { const data = new Uint8Array(await event.data) - const _connectionId = data.slice(0, 8) - if (!connectionId) { - connectionId = _connectionId - } - if (Array.from(connectionId).join('') !== Array.from(_connectionId).join('')) { - console.log('connectionId mismatch', connectionId, _connectionId) - return - } - - const message = data.slice(8) + const { connectionId, message } = parse(data) if (isStartupMessage(message)) { const parameters = parseStartupMessage(message) if ('client_ip' in parameters) { - // client disconnected - if (parameters.client_ip === '') { - setConnectedClientIp(null) - connectionId = undefined - await dbManager.closeDbInstance(databaseId) - } else { - db = await dbManager.getDbInstance(databaseId) - setConnectedClientIp(parameters.client_ip) - } + db = await dbManager.getDbInstance(databaseId) + setConnectedClientIp(parameters.client_ip) } return + } else if (isTerminateMessage(message)) { + // TODO: normally a `await db.query('discard all')` would be enough here + await dbManager.closeDbInstance(databaseId) + return } const response = await db.execProtocolRaw(message) - - const wrappedResponse = new Uint8Array(connectionId.length + response.length) - wrappedResponse.set(connectionId, 0) - wrappedResponse.set(response, connectionId.length) - - ws.send(wrappedResponse) + ws.send(serialize(connectionId, response)) }) } ws.onclose = (event) => { @@ -196,7 +179,7 @@ export default function AppProvider({ children }: AppProps) { setLiveShareWebsocket(ws) }, - [dbManager, cleanUp] + [cleanUp, supabase.auth] ) const stopLiveShare = useCallback(() => { liveShareWebsocket?.close() diff --git a/apps/postgres-new/lib/pg-wire-util.ts b/apps/postgres-new/lib/pg-wire-util.ts index 77d5d672..158d4c6e 100644 --- a/apps/postgres-new/lib/pg-wire-util.ts +++ b/apps/postgres-new/lib/pg-wire-util.ts @@ -79,3 +79,24 @@ export function parseStartupMessage(message: Uint8Array): { return params } + +export function isTerminateMessage(message: Uint8Array): boolean { + // A valid Terminate message should be exactly 5 bytes long + if (message.length !== 5) { + return false + } + + const view = new DataView(message.buffer, message.byteOffset, message.byteLength) + + if (message[0] !== 'X'.charCodeAt(0)) { + return false + } + + // Check if the length field (next 4 bytes) is equal to 4 + const length = view.getInt32(1, false) + if (length !== 4) { + return false + } + + return true +} diff --git a/apps/postgres-new/lib/websocket-protocol.ts b/apps/postgres-new/lib/websocket-protocol.ts new file mode 100644 index 00000000..0da0ddb6 --- /dev/null +++ b/apps/postgres-new/lib/websocket-protocol.ts @@ -0,0 +1,21 @@ +// Our protocol structure: +// +------------------+-----------------------------+ +// | connectionId | message | +// | (16 bytes) | (variable length) | +// +------------------+-----------------------------+ + +export function parse(data: Uint8Array) { + const connectionIdBytes = data.subarray(0, 16) + const connectionId = new TextDecoder().decode(connectionIdBytes) + const message = data.subarray(16) + return { connectionId, message } +} + +export function serialize(connectionId: string, message: Uint8Array) { + const encoder = new TextEncoder() + const connectionIdBytes = encoder.encode(connectionId) + const data = new Uint8Array(connectionIdBytes.length + message.length) + data.set(connectionIdBytes, 0) + data.set(message, connectionIdBytes.length) + return data +} diff --git a/package-lock.json b/package-lock.json index 941f7baf..29fbbba6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "debug": "^4.3.7", "expiry-map": "^2.0.0", "findhit-proxywrap": "^0.3.13", + "nanoid": "^5.0.7", "p-memoize": "^7.1.1", "pg-gateway": "^0.3.0-beta.3", "ws": "^8.18.0" @@ -41,6 +42,24 @@ "undici-types": "~6.19.2" } }, + "apps/browser-proxy/node_modules/nanoid": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.0.7.tgz", + "integrity": "sha512-oLxFY2gd2IqnjcYyOXD8XGCftpGtZP2AbHbOkthDkvRywH5ayNtPVy9YlOPcHckXzbLTCHpkb7FB+yuxKV13pQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.js" + }, + "engines": { + "node": "^18 || >=20" + } + }, "apps/browser-proxy/node_modules/pg-gateway": { "version": "0.3.0-beta.3", "resolved": "https://registry.npmjs.org/pg-gateway/-/pg-gateway-0.3.0-beta.3.tgz", From 6cd8637358547eaf19ffc1107924be05f76e2282 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 25 Sep 2024 14:29:13 +0200 Subject: [PATCH 09/20] fix logic --- apps/postgres-new/components/app-provider.tsx | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/apps/postgres-new/components/app-provider.tsx b/apps/postgres-new/components/app-provider.tsx index c102be13..068e6756 100644 --- a/apps/postgres-new/components/app-provider.tsx +++ b/apps/postgres-new/components/app-provider.tsx @@ -143,8 +143,9 @@ export default function AppProvider({ children }: AppProps) { setLiveSharedDatabaseId(databaseId) } + const db = await dbManager.getDbInstance(databaseId) const mutex = new Mutex() - let db: PGliteInterface + let activeConnectionId: string | null = null ws.onmessage = (event) => { mutex.runExclusive(async () => { @@ -153,15 +154,27 @@ export default function AppProvider({ children }: AppProps) { const { connectionId, message } = parse(data) if (isStartupMessage(message)) { + activeConnectionId = connectionId const parameters = parseStartupMessage(message) if ('client_ip' in parameters) { - db = await dbManager.getDbInstance(databaseId) setConnectedClientIp(parameters.client_ip) } return - } else if (isTerminateMessage(message)) { - // TODO: normally a `await db.query('discard all')` would be enough here - await dbManager.closeDbInstance(databaseId) + } + + if (isTerminateMessage(message)) { + activeConnectionId = null + setConnectedClientIp(null) + // reset session state + await db.exec('discard all; set search_path to public;') + return + } + + if (activeConnectionId !== connectionId) { + console.error('received message from inactive connection', { + activeConnectionId, + connectionId, + }) return } From 7962452cadcb8684c77488bb6b436d17a01d40bf Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 25 Sep 2024 15:59:14 +0200 Subject: [PATCH 10/20] lazy evaluation messages when debug is off --- apps/browser-proxy/src/debug.ts | 6 ++++-- apps/browser-proxy/src/tcp-server.ts | 2 +- apps/browser-proxy/src/websocket-server.ts | 2 +- apps/postgres-new/components/app-provider.tsx | 4 +++- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/apps/browser-proxy/src/debug.ts b/apps/browser-proxy/src/debug.ts index a7a71621..2a5265b4 100644 --- a/apps/browser-proxy/src/debug.ts +++ b/apps/browser-proxy/src/debug.ts @@ -1,3 +1,5 @@ -import makeDebug from 'debug' +import createDebug from 'debug' -export const debug = makeDebug('browser-proxy') +createDebug.formatters.e = (fn) => fn() + +export const debug = createDebug('browser-proxy') diff --git a/apps/browser-proxy/src/tcp-server.ts b/apps/browser-proxy/src/tcp-server.ts index 873dab3d..3efff1c1 100644 --- a/apps/browser-proxy/src/tcp-server.ts +++ b/apps/browser-proxy/src/tcp-server.ts @@ -88,7 +88,7 @@ tcpServer.on('connection', async (socket) => { }) } - debug('tcp message', { message }) + debug('tcp message: %e', () => Buffer.from(message).toString('hex')) websocket.send(serialize(connectionState!.connectionId, message)) // return an empty buffer to indicate that the message has been handled diff --git a/apps/browser-proxy/src/websocket-server.ts b/apps/browser-proxy/src/websocket-server.ts index 43ebddb8..91eb086a 100644 --- a/apps/browser-proxy/src/websocket-server.ts +++ b/apps/browser-proxy/src/websocket-server.ts @@ -87,7 +87,7 @@ websocketServer.on('connection', async (websocket, request) => { const { connectionId, message } = parse(data) const tcpConnection = connectionManager.getSocket(connectionId) if (tcpConnection) { - debug('websocket message', message.toString('hex')) + debug('websocket message: %e', () => message.toString('hex')) tcpConnection.streamWriter?.write(message) } }) diff --git a/apps/postgres-new/components/app-provider.tsx b/apps/postgres-new/components/app-provider.tsx index 068e6756..a36a6f9d 100644 --- a/apps/postgres-new/components/app-provider.tsx +++ b/apps/postgres-new/components/app-provider.tsx @@ -144,6 +144,7 @@ export default function AppProvider({ children }: AppProps) { } const db = await dbManager.getDbInstance(databaseId) + const mutex = new Mutex() let activeConnectionId: string | null = null @@ -166,7 +167,8 @@ export default function AppProvider({ children }: AppProps) { activeConnectionId = null setConnectedClientIp(null) // reset session state - await db.exec('discard all; set search_path to public;') + await db.query('discard all') + await db.query('set search_path to public') return } From 05efd7019a157e3e0002b0104fe0bbac698b76e8 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 25 Sep 2024 18:48:39 +0200 Subject: [PATCH 11/20] bump pglite --- apps/postgres-new/components/app-provider.tsx | 2 +- apps/postgres-new/package.json | 2 +- package-lock.json | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/postgres-new/components/app-provider.tsx b/apps/postgres-new/components/app-provider.tsx index a36a6f9d..b105746a 100644 --- a/apps/postgres-new/components/app-provider.tsx +++ b/apps/postgres-new/components/app-provider.tsx @@ -144,7 +144,6 @@ export default function AppProvider({ children }: AppProps) { } const db = await dbManager.getDbInstance(databaseId) - const mutex = new Mutex() let activeConnectionId: string | null = null @@ -167,6 +166,7 @@ export default function AppProvider({ children }: AppProps) { activeConnectionId = null setConnectedClientIp(null) // reset session state + await db.query('rollback').catch(() => {}) await db.query('discard all') await db.query('set search_path to public') return diff --git a/apps/postgres-new/package.json b/apps/postgres-new/package.json index fbb20943..ac2b68c0 100644 --- a/apps/postgres-new/package.json +++ b/apps/postgres-new/package.json @@ -12,7 +12,7 @@ "dependencies": { "@ai-sdk/openai": "^0.0.21", "@dagrejs/dagre": "^1.1.2", - "@electric-sql/pglite": "^0.2.7", + "@electric-sql/pglite": "^0.2.8", "@gregnr/postgres-meta": "^0.82.0-dev.2", "@monaco-editor/react": "^4.6.0", "@radix-ui/react-accordion": "^1.2.0", diff --git a/package-lock.json b/package-lock.json index 29fbbba6..cca449e1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -89,7 +89,7 @@ "dependencies": { "@ai-sdk/openai": "^0.0.21", "@dagrejs/dagre": "^1.1.2", - "@electric-sql/pglite": "^0.2.7", + "@electric-sql/pglite": "^0.2.8", "@gregnr/postgres-meta": "^0.82.0-dev.2", "@monaco-editor/react": "^4.6.0", "@radix-ui/react-accordion": "^1.2.0", @@ -165,9 +165,9 @@ } }, "apps/postgres-new/node_modules/@electric-sql/pglite": { - "version": "0.2.7", - "resolved": "https://registry.npmjs.org/@electric-sql/pglite/-/pglite-0.2.7.tgz", - "integrity": "sha512-8Il//XHTAtZ8VeQF+6P1UjsIoaAJyO4LwOMoXhSFaHpmkwKs63cUhHHNzLzUmcZvP/ZTmlT3+xTiWfU/EyoxwQ==", + "version": "0.2.8", + "resolved": "https://registry.npmjs.org/@electric-sql/pglite/-/pglite-0.2.8.tgz", + "integrity": "sha512-0wSmQu22euBRzR5ghqyIHnBH4MfwlkL5WstOrrA3KOsjEWEglvoL/gH92JajEUA6Ufei/+qbkB2hVloC/K/RxQ==", "license": "Apache-2.0" }, "apps/postgres-new/node_modules/nanoid": { From 49da099d2b800807e3d34f143f94cbb65ede979c Mon Sep 17 00:00:00 2001 From: Greg Richardson Date: Wed, 25 Sep 2024 11:03:27 -0600 Subject: [PATCH 12/20] fix: serialized json type in pglite --- apps/postgres-new/lib/db/index.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/apps/postgres-new/lib/db/index.ts b/apps/postgres-new/lib/db/index.ts index 9c55fb84..8609aede 100644 --- a/apps/postgres-new/lib/db/index.ts +++ b/apps/postgres-new/lib/db/index.ts @@ -85,13 +85,7 @@ export class DbManager { if (message.toolInvocations) { await metaDb.query( 'insert into messages (id, database_id, role, content, tool_invocations) values ($1, $2, $3, $4, $5)', - [ - message.id, - databaseId, - message.role, - message.content, - JSON.stringify(message.toolInvocations), - ] + [message.id, databaseId, message.role, message.content, message.toolInvocations] ) } else { await metaDb.query( From 58fe2713fa941b117b9e13190aad7a0990885c69 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 25 Sep 2024 19:12:55 +0200 Subject: [PATCH 13/20] weird --- package.json | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index a1bfd026..04bf2e12 100644 --- a/package.json +++ b/package.json @@ -3,11 +3,8 @@ "scripts": { "dev": "npm run dev --workspace postgres-new" }, - "workspaces": [ - "apps/*" - ], + "workspaces": ["apps/*"], "devDependencies": { "supabase": "^1.191.3" - }, - "packageManager": "yarn@1.22.22+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e" + } } From 770f337c89f45f6c1f48afc680215fdc35e10ac2 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 30 Sep 2024 13:29:39 +0200 Subject: [PATCH 14/20] upgrade pgglite --- apps/postgres-new/package.json | 2 +- package-lock.json | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/postgres-new/package.json b/apps/postgres-new/package.json index ac2b68c0..eb50df7d 100644 --- a/apps/postgres-new/package.json +++ b/apps/postgres-new/package.json @@ -12,7 +12,7 @@ "dependencies": { "@ai-sdk/openai": "^0.0.21", "@dagrejs/dagre": "^1.1.2", - "@electric-sql/pglite": "^0.2.8", + "@electric-sql/pglite": "^0.2.9", "@gregnr/postgres-meta": "^0.82.0-dev.2", "@monaco-editor/react": "^4.6.0", "@radix-ui/react-accordion": "^1.2.0", diff --git a/package-lock.json b/package-lock.json index cca449e1..d1148808 100644 --- a/package-lock.json +++ b/package-lock.json @@ -89,7 +89,7 @@ "dependencies": { "@ai-sdk/openai": "^0.0.21", "@dagrejs/dagre": "^1.1.2", - "@electric-sql/pglite": "^0.2.8", + "@electric-sql/pglite": "^0.2.9", "@gregnr/postgres-meta": "^0.82.0-dev.2", "@monaco-editor/react": "^4.6.0", "@radix-ui/react-accordion": "^1.2.0", @@ -165,9 +165,9 @@ } }, "apps/postgres-new/node_modules/@electric-sql/pglite": { - "version": "0.2.8", - "resolved": "https://registry.npmjs.org/@electric-sql/pglite/-/pglite-0.2.8.tgz", - "integrity": "sha512-0wSmQu22euBRzR5ghqyIHnBH4MfwlkL5WstOrrA3KOsjEWEglvoL/gH92JajEUA6Ufei/+qbkB2hVloC/K/RxQ==", + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/@electric-sql/pglite/-/pglite-0.2.9.tgz", + "integrity": "sha512-KPItmBmPVZJGOv+qkCXmWyIPUPLQIyN+BVtV/zD+083aWSR/2ReaCUN+HJv6Jw4z9zJ00UCPQkeUXvOLuTlumg==", "license": "Apache-2.0" }, "apps/postgres-new/node_modules/nanoid": { From 87a71168bb0d732c9025365979906aba3b055123 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 30 Sep 2024 23:20:04 +0200 Subject: [PATCH 15/20] it works --- .../src/pg-dump-middleware/constants.ts | 1 + .../get-extension-membership-query.ts | 108 ++++++ .../get-extensions-query.ts | 125 +++++++ .../pg-dump-middleware/pg-dump-middleware.ts | 76 ++++ .../src/pg-dump-middleware/utils.ts | 38 ++ apps/browser-proxy/src/tcp-server.ts | 7 + apps/browser-proxy/src/websocket-server.ts | 11 +- apps/postgres-new/package.json | 2 +- package-lock.json | 333 ++++++++++-------- 9 files changed, 555 insertions(+), 146 deletions(-) create mode 100644 apps/browser-proxy/src/pg-dump-middleware/constants.ts create mode 100644 apps/browser-proxy/src/pg-dump-middleware/get-extension-membership-query.ts create mode 100644 apps/browser-proxy/src/pg-dump-middleware/get-extensions-query.ts create mode 100644 apps/browser-proxy/src/pg-dump-middleware/pg-dump-middleware.ts create mode 100644 apps/browser-proxy/src/pg-dump-middleware/utils.ts diff --git a/apps/browser-proxy/src/pg-dump-middleware/constants.ts b/apps/browser-proxy/src/pg-dump-middleware/constants.ts new file mode 100644 index 00000000..b25d721a --- /dev/null +++ b/apps/browser-proxy/src/pg-dump-middleware/constants.ts @@ -0,0 +1 @@ +export const VECTOR_OID = '99999' diff --git a/apps/browser-proxy/src/pg-dump-middleware/get-extension-membership-query.ts b/apps/browser-proxy/src/pg-dump-middleware/get-extension-membership-query.ts new file mode 100644 index 00000000..4fd51c00 --- /dev/null +++ b/apps/browser-proxy/src/pg-dump-middleware/get-extension-membership-query.ts @@ -0,0 +1,108 @@ +import { VECTOR_OID } from './constants.ts' +import { parseDataRowFields, parseRowDescription } from './utils.ts' + +export function isGetExtensionMembershipQuery(message: Uint8Array): boolean { + // Check if it's a SimpleQuery message (starts with 'Q') + if (message[0] !== 0x51) { + // 'Q' in ASCII + return false + } + + const query = + "SELECT classid, objid, refobjid FROM pg_depend WHERE refclassid = 'pg_extension'::regclass AND deptype = 'e' ORDER BY 3" + + // Skip the message type (1 byte) and message length (4 bytes) + const messageString = new TextDecoder().decode(message.slice(5)) + + // Trim any trailing null character + const trimmedMessage = messageString.replace(/\0+$/, '') + + // Check if the message exactly matches the query + return trimmedMessage === query +} + +export function patchGetExtensionMembershipResult(data: Uint8Array, vectorOid: string): Uint8Array { + let offset = 0 + const messages: Uint8Array[] = [] + let isDependencyTable = false + let objidIndex = -1 + let refobjidIndex = -1 + let patchedRowCount = 0 + let totalRowsProcessed = 0 + + const expectedColumns = ['classid', 'objid', 'refobjid'] + + while (offset < data.length) { + const messageType = data[offset] + const messageLength = new DataView(data.buffer, data.byteOffset + offset + 1, 4).getUint32( + 0, + false + ) + const message = data.subarray(offset, offset + messageLength + 1) + + if (messageType === 0x54) { + // RowDescription + const columnNames = parseRowDescription(message) + isDependencyTable = + columnNames.length === 3 && columnNames.every((col) => expectedColumns.includes(col)) + if (isDependencyTable) { + objidIndex = columnNames.indexOf('objid') + refobjidIndex = columnNames.indexOf('refobjid') + } + } else if (messageType === 0x44 && isDependencyTable) { + // DataRow + const fields = parseDataRowFields(message) + totalRowsProcessed++ + + if (fields.length === 3) { + const refobjid = fields[refobjidIndex]!.value + + if (refobjid === vectorOid) { + const patchedMessage = patchDependencyRow(message, refobjidIndex) + messages.push(patchedMessage) + patchedRowCount++ + offset += messageLength + 1 + continue + } + } + } + + messages.push(message) + offset += messageLength + 1 + } + + return new Uint8Array( + messages.reduce((acc, val) => { + const combined = new Uint8Array(acc.length + val.length) + combined.set(acc) + combined.set(val, acc.length) + return combined + }, new Uint8Array()) + ) +} + +function patchDependencyRow(message: Uint8Array, refobjidIndex: number): Uint8Array { + const newArray = new Uint8Array(message) + let offset = 7 // Start after message type (1 byte), message length (4 bytes), and field count (2 bytes) + + // Navigate to the refobjid field + for (let i = 0; i < refobjidIndex; i++) { + const fieldLength = new DataView(newArray.buffer, offset, 4).getInt32(0) + offset += 4 // Skip the length field + if (fieldLength > 0) { + offset += fieldLength // Skip the field value + } + } + + // Now we're at the start of the refobjid field + const refobjidLength = new DataView(newArray.buffer, offset, 4).getInt32(0) + offset += 4 // Move past the length field + + const encoder = new TextEncoder() + + // Write the new OID value + const newRefobjidBytes = encoder.encode(VECTOR_OID.padStart(refobjidLength, '0')) + newArray.set(newRefobjidBytes, offset) + + return newArray +} diff --git a/apps/browser-proxy/src/pg-dump-middleware/get-extensions-query.ts b/apps/browser-proxy/src/pg-dump-middleware/get-extensions-query.ts new file mode 100644 index 00000000..6a479a9c --- /dev/null +++ b/apps/browser-proxy/src/pg-dump-middleware/get-extensions-query.ts @@ -0,0 +1,125 @@ +import { VECTOR_OID } from './constants.ts' +import { parseDataRowFields, parseRowDescription } from './utils.ts' + +export function isGetExtensionsQuery(message: Uint8Array): boolean { + // Check if it's a SimpleQuery message (starts with 'Q') + if (message[0] !== 0x51) { + // 'Q' in ASCII + return false + } + + const query = + 'SELECT x.tableoid, x.oid, x.extname, n.nspname, x.extrelocatable, x.extversion, x.extconfig, x.extcondition FROM pg_extension x JOIN pg_namespace n ON n.oid = x.extnamespace' + + // Skip the message type (1 byte) and message length (4 bytes) + const messageString = new TextDecoder().decode(message.slice(5)) + + // Trim any trailing null character + const trimmedMessage = messageString.replace(/\0+$/, '') + + // Check if the message exactly matches the query + return trimmedMessage === query +} + +export function patchGetExtensionsResult(data: Uint8Array) { + let offset = 0 + const messages: Uint8Array[] = [] + let isVectorExtensionTable = false + let oidColumnIndex = -1 + let extnameColumnIndex = -1 + let vectorOid: string | null = null + + const expectedColumns = [ + 'tableoid', + 'oid', + 'extname', + 'nspname', + 'extrelocatable', + 'extversion', + 'extconfig', + 'extcondition', + ] + + while (offset < data.length) { + const messageType = data[offset] + const messageLength = new DataView(data.buffer, data.byteOffset + offset + 1, 4).getUint32( + 0, + false + ) + + const message = data.subarray(offset, offset + messageLength + 1) + + if (messageType === 0x54) { + // RowDescription + const columnNames = parseRowDescription(message) + + isVectorExtensionTable = + columnNames.length === expectedColumns.length && + columnNames.every((col) => expectedColumns.includes(col)) + + if (isVectorExtensionTable) { + oidColumnIndex = columnNames.indexOf('oid') + extnameColumnIndex = columnNames.indexOf('extname') + } + } else if (messageType === 0x44 && isVectorExtensionTable) { + // DataRow + const fields = parseDataRowFields(message) + if (fields[extnameColumnIndex]?.value === 'vector') { + vectorOid = fields[oidColumnIndex]!.value! + const patchedMessage = patchOidField(message, oidColumnIndex, fields) + messages.push(patchedMessage) + offset += messageLength + 1 + continue + } + } + + messages.push(message) + offset += messageLength + 1 + } + + return { + message: Buffer.concat(messages), + vectorOid, + } +} + +function patchOidField( + message: Uint8Array, + oidIndex: number, + fields: { value: string | null; length: number }[] +): Uint8Array { + const oldOidField = fields[oidIndex]! + const newOid = VECTOR_OID.padStart(oldOidField.length, '0') + + const newArray = new Uint8Array(message) + + let offset = 7 // Start after message type (1 byte), message length (4 bytes), and field count (2 bytes) + + // Navigate to the OID field + for (let i = 0; i < oidIndex; i++) { + const fieldLength = new DataView(newArray.buffer, offset, 4).getInt32(0) + offset += 4 // Skip the length field + if (fieldLength > 0) { + offset += fieldLength // Skip the field value + } + } + + // Now we're at the start of the OID field + const oidLength = new DataView(newArray.buffer, offset, 4).getInt32(0) + offset += 4 // Move past the length field + + // Ensure the new OID fits in the allocated space + if (newOid.length !== oidLength) { + console.warn( + `New OID length (${newOid.length}) doesn't match the original length (${oidLength}). Skipping patch.` + ) + return message + } + + // Write the new OID value + for (let i = 0; i < oidLength; i++) { + newArray[offset + i] = newOid.charCodeAt(i) + } + + return newArray +} diff --git a/apps/browser-proxy/src/pg-dump-middleware/pg-dump-middleware.ts b/apps/browser-proxy/src/pg-dump-middleware/pg-dump-middleware.ts new file mode 100644 index 00000000..6e141841 --- /dev/null +++ b/apps/browser-proxy/src/pg-dump-middleware/pg-dump-middleware.ts @@ -0,0 +1,76 @@ +import ExpiryMap from 'expiry-map' +import type { ClientParameters } from 'pg-gateway' +import { isGetExtensionsQuery, patchGetExtensionsResult } from './get-extensions-query.ts' +import { + isGetExtensionMembershipQuery, + patchGetExtensionMembershipResult, +} from './get-extension-membership-query.ts' + +type ConnectionId = string + +const state = new ExpiryMap(1000 * 60 * 5) + +type State = + | { step: 'wait-for-get-extensions-query' } + | { step: 'get-extensions-query-received' } + | { step: 'wait-for-get-extension-membership-query'; vectorOid: string } + | { step: 'get-extension-membership-query-received'; vectorOid: string } + | { step: 'complete' } + +export function pgDumpMiddleware( + connectionId: string, + origin: 'client' | 'server', + context: { + clientParams?: ClientParameters + }, + message: Uint8Array +) { + if (context.clientParams?.application_name !== 'pg_dump') { + return message + } + + if (!state.has(connectionId)) { + state.set(connectionId, { step: 'wait-for-get-extensions-query' }) + } + + const connectionState = state.get(connectionId)! + + switch (connectionState.step) { + case 'wait-for-get-extensions-query': + // https://github.com/postgres/postgres/blob/a19f83f87966f763991cc76404f8e42a36e7e842/src/bin/pg_dump/pg_dump.c#L5834-L5837 + if (origin === 'client' && isGetExtensionsQuery(message)) { + state.set(connectionId, { step: 'get-extensions-query-received' }) + } + return message + case 'get-extensions-query-received': + if (origin === 'client') { + return message + } + const patched = patchGetExtensionsResult(message) + if (patched.vectorOid) { + state.set(connectionId, { + step: 'wait-for-get-extension-membership-query', + vectorOid: patched.vectorOid, + }) + } + return patched.message + case 'wait-for-get-extension-membership-query': + // https://github.com/postgres/postgres/blob/a19f83f87966f763991cc76404f8e42a36e7e842/src/bin/pg_dump/pg_dump.c#L18173-L18178 + if (origin === 'client' && isGetExtensionMembershipQuery(message)) { + state.set(connectionId, { + step: 'get-extension-membership-query-received', + vectorOid: connectionState.vectorOid, + }) + } + return message + case 'get-extension-membership-query-received': + if (origin === 'client') { + return message + } + const patchedMessage = patchGetExtensionMembershipResult(message, connectionState.vectorOid) + state.set(connectionId, { step: 'complete' }) + return patchedMessage + case 'complete': + return message + } +} diff --git a/apps/browser-proxy/src/pg-dump-middleware/utils.ts b/apps/browser-proxy/src/pg-dump-middleware/utils.ts new file mode 100644 index 00000000..9d089a8a --- /dev/null +++ b/apps/browser-proxy/src/pg-dump-middleware/utils.ts @@ -0,0 +1,38 @@ +export function parseRowDescription(message: Uint8Array): string[] { + const fieldCount = new DataView(message.buffer, message.byteOffset + 5, 2).getUint16(0) + const names: string[] = [] + let offset = 7 + + for (let i = 0; i < fieldCount; i++) { + const nameEnd = message.indexOf(0, offset) + names.push(new TextDecoder().decode(message.subarray(offset, nameEnd))) + offset = nameEnd + 19 // Skip null terminator and 18 bytes of field info + } + + return names +} + +export function parseDataRowFields( + message: Uint8Array +): { value: string | null; length: number }[] { + const fieldCount = new DataView(message.buffer, message.byteOffset + 5, 2).getUint16(0) + const fields: { value: string | null; length: number }[] = [] + let offset = 7 + + for (let i = 0; i < fieldCount; i++) { + const fieldLength = new DataView(message.buffer, message.byteOffset + offset, 4).getInt32(0) + offset += 4 + + if (fieldLength === -1) { + fields.push({ value: null, length: -1 }) + } else { + fields.push({ + value: new TextDecoder().decode(message.subarray(offset, offset + fieldLength)), + length: fieldLength, + }) + offset += fieldLength + } + } + + return fields +} diff --git a/apps/browser-proxy/src/tcp-server.ts b/apps/browser-proxy/src/tcp-server.ts index 3efff1c1..c14f81f5 100644 --- a/apps/browser-proxy/src/tcp-server.ts +++ b/apps/browser-proxy/src/tcp-server.ts @@ -9,6 +9,7 @@ import { logEvent, UserConnected, UserDisconnected } from './telemetry.ts' import { connectionManager } from './connection-manager.ts' import { debug as mainDebug } from './debug.ts' import { getConnectionId, serialize } from './protocol.ts' +import { pgDumpMiddleware } from './pg-dump-middleware/pg-dump-middleware.ts' const debug = mainDebug.extend('tcp-server') @@ -89,6 +90,12 @@ tcpServer.on('connection', async (socket) => { } debug('tcp message: %e', () => Buffer.from(message).toString('hex')) + message = pgDumpMiddleware( + connectionState!.connectionId, + 'client', + connection.state, + Buffer.from(message) + ) websocket.send(serialize(connectionState!.connectionId, message)) // return an empty buffer to indicate that the message has been handled diff --git a/apps/browser-proxy/src/websocket-server.ts b/apps/browser-proxy/src/websocket-server.ts index 91eb086a..0d586062 100644 --- a/apps/browser-proxy/src/websocket-server.ts +++ b/apps/browser-proxy/src/websocket-server.ts @@ -7,6 +7,7 @@ import { setSecureContext } from './tls.ts' import { connectionManager } from './connection-manager.ts' import { DatabaseShared, DatabaseUnshared, logEvent } from './telemetry.ts' import { parse } from './protocol.ts' +import { pgDumpMiddleware } from './pg-dump-middleware/pg-dump-middleware.ts' const debug = mainDebug.extend('websocket-server') @@ -84,10 +85,18 @@ websocketServer.on('connection', async (websocket, request) => { logEvent(new DatabaseShared({ databaseId, userId: user.id })) websocket.on('message', (data: Buffer) => { - const { connectionId, message } = parse(data) + let { connectionId, message } = parse(data) const tcpConnection = connectionManager.getSocket(connectionId) if (tcpConnection) { debug('websocket message: %e', () => message.toString('hex')) + message = Buffer.from( + pgDumpMiddleware( + connectionId, + 'server', + tcpConnection.state, + new Uint8Array(message.buffer, message.byteOffset, message.byteLength) + ) + ) tcpConnection.streamWriter?.write(message) } }) diff --git a/apps/postgres-new/package.json b/apps/postgres-new/package.json index eb50df7d..743d3488 100644 --- a/apps/postgres-new/package.json +++ b/apps/postgres-new/package.json @@ -12,7 +12,7 @@ "dependencies": { "@ai-sdk/openai": "^0.0.21", "@dagrejs/dagre": "^1.1.2", - "@electric-sql/pglite": "^0.2.9", + "@electric-sql/pglite": "0.2.8", "@gregnr/postgres-meta": "^0.82.0-dev.2", "@monaco-editor/react": "^4.6.0", "@radix-ui/react-accordion": "^1.2.0", diff --git a/package-lock.json b/package-lock.json index d1148808..7a40e018 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,5 @@ { "name": "postgres-new", - "version": "0.1.0", "lockfileVersion": 3, "requires": true, "packages": { @@ -89,7 +88,7 @@ "dependencies": { "@ai-sdk/openai": "^0.0.21", "@dagrejs/dagre": "^1.1.2", - "@electric-sql/pglite": "^0.2.9", + "@electric-sql/pglite": "0.2.8", "@gregnr/postgres-meta": "^0.82.0-dev.2", "@monaco-editor/react": "^4.6.0", "@radix-ui/react-accordion": "^1.2.0", @@ -165,9 +164,9 @@ } }, "apps/postgres-new/node_modules/@electric-sql/pglite": { - "version": "0.2.9", - "resolved": "https://registry.npmjs.org/@electric-sql/pglite/-/pglite-0.2.9.tgz", - "integrity": "sha512-KPItmBmPVZJGOv+qkCXmWyIPUPLQIyN+BVtV/zD+083aWSR/2ReaCUN+HJv6Jw4z9zJ00UCPQkeUXvOLuTlumg==", + "version": "0.2.8", + "resolved": "https://registry.npmjs.org/@electric-sql/pglite/-/pglite-0.2.8.tgz", + "integrity": "sha512-0wSmQu22euBRzR5ghqyIHnBH4MfwlkL5WstOrrA3KOsjEWEglvoL/gH92JajEUA6Ufei/+qbkB2hVloC/K/RxQ==", "license": "Apache-2.0" }, "apps/postgres-new/node_modules/nanoid": { @@ -1441,7 +1440,8 @@ "node_modules/@electric-sql/pglite": { "version": "0.2.0-alpha.9", "resolved": "https://registry.npmjs.org/@electric-sql/pglite/-/pglite-0.2.0-alpha.9.tgz", - "integrity": "sha512-euiFGNa2NtwF2DdXCojZXtbBvhkd1ZgG/jfMimAdHp4h2kzz/bqvRYiLoH41zmFCc4XeaQyMEhuVmbdwb67hBA==" + "integrity": "sha512-euiFGNa2NtwF2DdXCojZXtbBvhkd1ZgG/jfMimAdHp4h2kzz/bqvRYiLoH41zmFCc4XeaQyMEhuVmbdwb67hBA==", + "license": "Apache-2.0" }, "node_modules/@emnapi/runtime": { "version": "0.43.1", @@ -1452,371 +1452,411 @@ } }, "node_modules/@esbuild/aix-ppc64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", - "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.23.1.tgz", + "integrity": "sha512-6VhYk1diRqrhBAqpJEdjASR/+WVRtfjpqKuNw11cLiaWpAT/Uu+nokB+UJnevzy/P9C/ty6AOe0dwueMrGh/iQ==", "cpu": [ "ppc64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "aix" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/android-arm": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz", - "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.23.1.tgz", + "integrity": "sha512-uz6/tEy2IFm9RYOyvKl88zdzZfwEfKZmnX9Cj1BHjeSGNuGLuMD1kR8y5bteYmwqKm1tj8m4cb/aKEorr6fHWQ==", "cpu": [ "arm" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "android" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/android-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz", - "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.23.1.tgz", + "integrity": "sha512-xw50ipykXcLstLeWH7WRdQuysJqejuAGPd30vd1i5zSyKK3WE+ijzHmLKxdiCMtH1pHz78rOg0BKSYOSB/2Khw==", "cpu": [ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "android" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/android-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz", - "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.23.1.tgz", + "integrity": "sha512-nlN9B69St9BwUoB+jkyU090bru8L0NA3yFvAd7k8dNsVH8bi9a8cUAUSEcEEgTp2z3dbEDGJGfP6VUnkQnlReg==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "android" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/darwin-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz", - "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.23.1.tgz", + "integrity": "sha512-YsS2e3Wtgnw7Wq53XXBLcV6JhRsEq8hkfg91ESVadIrzr9wO6jJDMZnCQbHm1Guc5t/CdDiFSSfWP58FNuvT3Q==", "cpu": [ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "darwin" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/darwin-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz", - "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.23.1.tgz", + "integrity": "sha512-aClqdgTDVPSEGgoCS8QDG37Gu8yc9lTHNAQlsztQ6ENetKEO//b8y31MMu2ZaPbn4kVsIABzVLXYLhCGekGDqw==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "darwin" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/freebsd-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz", - "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.23.1.tgz", + "integrity": "sha512-h1k6yS8/pN/NHlMl5+v4XPfikhJulk4G+tKGFIOwURBSFzE8bixw1ebjluLOjfwtLqY0kewfjLSrO6tN2MgIhA==", "cpu": [ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "freebsd" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/freebsd-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz", - "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.23.1.tgz", + "integrity": "sha512-lK1eJeyk1ZX8UklqFd/3A60UuZ/6UVfGT2LuGo3Wp4/z7eRTRYY+0xOu2kpClP+vMTi9wKOfXi2vjUpO1Ro76g==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "freebsd" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/linux-arm": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz", - "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.23.1.tgz", + "integrity": "sha512-CXXkzgn+dXAPs3WBwE+Kvnrf4WECwBdfjfeYHpMeVxWE0EceB6vhWGShs6wi0IYEqMSIzdOF1XjQ/Mkm5d7ZdQ==", "cpu": [ "arm" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/linux-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz", - "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.23.1.tgz", + "integrity": "sha512-/93bf2yxencYDnItMYV/v116zff6UyTjo4EtEQjUBeGiVpMmffDNUyD9UN2zV+V3LRV3/on4xdZ26NKzn6754g==", "cpu": [ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/linux-ia32": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz", - "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.23.1.tgz", + "integrity": "sha512-VTN4EuOHwXEkXzX5nTvVY4s7E/Krz7COC8xkftbbKRYAl96vPiUssGkeMELQMOnLOJ8k3BY1+ZY52tttZnHcXQ==", "cpu": [ "ia32" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/linux-loong64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz", - "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.23.1.tgz", + "integrity": "sha512-Vx09LzEoBa5zDnieH8LSMRToj7ir/Jeq0Gu6qJ/1GcBq9GkfoEAoXvLiW1U9J1qE/Y/Oyaq33w5p2ZWrNNHNEw==", "cpu": [ "loong64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/linux-mips64el": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz", - "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.23.1.tgz", + "integrity": "sha512-nrFzzMQ7W4WRLNUOU5dlWAqa6yVeI0P78WKGUo7lg2HShq/yx+UYkeNSE0SSfSure0SqgnsxPvmAUu/vu0E+3Q==", "cpu": [ "mips64el" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/linux-ppc64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz", - "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.23.1.tgz", + "integrity": "sha512-dKN8fgVqd0vUIjxuJI6P/9SSSe/mB9rvA98CSH2sJnlZ/OCZWO1DJvxj8jvKTfYUdGfcq2dDxoKaC6bHuTlgcw==", "cpu": [ "ppc64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/linux-riscv64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz", - "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.23.1.tgz", + "integrity": "sha512-5AV4Pzp80fhHL83JM6LoA6pTQVWgB1HovMBsLQ9OZWLDqVY8MVobBXNSmAJi//Csh6tcY7e7Lny2Hg1tElMjIA==", "cpu": [ "riscv64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/linux-s390x": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz", - "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.23.1.tgz", + "integrity": "sha512-9ygs73tuFCe6f6m/Tb+9LtYxWR4c9yg7zjt2cYkjDbDpV/xVn+68cQxMXCjUpYwEkze2RcU/rMnfIXNRFmSoDw==", "cpu": [ "s390x" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/linux-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", - "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.23.1.tgz", + "integrity": "sha512-EV6+ovTsEXCPAp58g2dD68LxoP/wK5pRvgy0J/HxPGB009omFPv3Yet0HiaqvrIrgPTBuC6wCH1LTOY91EO5hQ==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "linux" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/netbsd-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", - "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.23.1.tgz", + "integrity": "sha512-aevEkCNu7KlPRpYLjwmdcuNz6bDFiE7Z8XC4CPqExjTvrHugh28QzUXVOZtiYghciKUacNktqxdpymplil1beA==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "netbsd" ], "engines": { - "node": ">=12" + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.23.1.tgz", + "integrity": "sha512-3x37szhLexNA4bXhLrCC/LImN/YtWis6WXr1VESlfVtVeoFJBRINPJ3f0a/6LV8zpikqoUg4hyXw0sFBt5Cr+Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" } }, "node_modules/@esbuild/openbsd-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", - "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.23.1.tgz", + "integrity": "sha512-aY2gMmKmPhxfU+0EdnN+XNtGbjfQgwZj43k8G3fyrDM/UdZww6xrWxmDkuz2eCZchqVeABjV5BpildOrUbBTqA==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "openbsd" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/sunos-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", - "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.23.1.tgz", + "integrity": "sha512-RBRT2gqEl0IKQABT4XTj78tpk9v7ehp+mazn2HbUeZl1YMdaGAQqhapjGTCe7uw7y0frDi4gS0uHzhvpFuI1sA==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "sunos" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/win32-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz", - "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.23.1.tgz", + "integrity": "sha512-4O+gPR5rEBe2FpKOVyiJ7wNDPA8nGzDuJ6gN4okSA1gEOYZ67N8JPk58tkWtdtPeLz7lBnY6I5L3jdsr3S+A6A==", "cpu": [ "arm64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "win32" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/win32-ia32": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz", - "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.23.1.tgz", + "integrity": "sha512-BcaL0Vn6QwCwre3Y717nVHZbAa4UBEigzFm6VdsVdT/MbZ38xoj1X9HPkZhbmaBGUD1W8vxAfffbDe8bA6AKnQ==", "cpu": [ "ia32" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "win32" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/win32-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz", - "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.23.1.tgz", + "integrity": "sha512-BHpFFeslkWrXWyUPnbKm+xYYVYruCinGcftSBaa8zoF9hZO4BcSCFUvHVTtzpIY6YzUnYtuEhZ+C9iEXjxnasg==", "cpu": [ "x64" ], "dev": true, + "license": "MIT", "optional": true, "os": [ "win32" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@eslint-community/eslint-utils": { @@ -3401,18 +3441,6 @@ "react-dom": ">=17" } }, - "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.19.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.19.1.tgz", - "integrity": "sha512-XUXeI9eM8rMP8aGvii/aOOiMvTs7xlCosq9xCjcqI9+5hBxtjDpD+7Abm1ZhVIFE1J2h2VIg0t2DX/gjespC2Q==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ] - }, "node_modules/@rushstack/eslint-patch": { "version": "1.10.3", "resolved": "https://registry.npmjs.org/@rushstack/eslint-patch/-/eslint-patch-1.10.3.tgz", @@ -5636,6 +5664,7 @@ "resolved": "https://registry.npmjs.org/bin-links/-/bin-links-4.0.4.tgz", "integrity": "sha512-cMtq4W5ZsEwcutJrVId+a/tjt8GSbS+h0oNkdl6+6rBuEv8Ot33Bevj5KPm40t309zuhVic8NjpuL42QCiJWWA==", "dev": true, + "license": "ISC", "dependencies": { "cmd-shim": "^6.0.0", "npm-normalize-package-bin": "^3.0.0", @@ -6042,6 +6071,7 @@ "resolved": "https://registry.npmjs.org/cmd-shim/-/cmd-shim-6.0.3.tgz", "integrity": "sha512-FMabTRlc5t5zjdenF6mS0MBeFZm0XqHqeOkcskKFb/LYCcRQ5fVgLOHVc4Lq9CqABd9zhjwPjMBCJvMCziSVtA==", "dev": true, + "license": "ISC", "engines": { "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } @@ -6322,6 +6352,7 @@ "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", "dev": true, + "license": "MIT", "engines": { "node": ">= 12" } @@ -6866,41 +6897,43 @@ } }, "node_modules/esbuild": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", - "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.23.1.tgz", + "integrity": "sha512-VVNz/9Sa0bs5SELtn3f7qhJCDPCF5oMEl5cO9/SSinpE9hbPVvxbd572HH5AKiP7WD8INO53GgfDDhRjkylHEg==", "dev": true, "hasInstallScript": true, + "license": "MIT", "bin": { "esbuild": "bin/esbuild" }, "engines": { - "node": ">=12" + "node": ">=18" }, "optionalDependencies": { - "@esbuild/aix-ppc64": "0.21.5", - "@esbuild/android-arm": "0.21.5", - "@esbuild/android-arm64": "0.21.5", - "@esbuild/android-x64": "0.21.5", - "@esbuild/darwin-arm64": "0.21.5", - "@esbuild/darwin-x64": "0.21.5", - "@esbuild/freebsd-arm64": "0.21.5", - "@esbuild/freebsd-x64": "0.21.5", - "@esbuild/linux-arm": "0.21.5", - "@esbuild/linux-arm64": "0.21.5", - "@esbuild/linux-ia32": "0.21.5", - "@esbuild/linux-loong64": "0.21.5", - "@esbuild/linux-mips64el": "0.21.5", - "@esbuild/linux-ppc64": "0.21.5", - "@esbuild/linux-riscv64": "0.21.5", - "@esbuild/linux-s390x": "0.21.5", - "@esbuild/linux-x64": "0.21.5", - "@esbuild/netbsd-x64": "0.21.5", - "@esbuild/openbsd-x64": "0.21.5", - "@esbuild/sunos-x64": "0.21.5", - "@esbuild/win32-arm64": "0.21.5", - "@esbuild/win32-ia32": "0.21.5", - "@esbuild/win32-x64": "0.21.5" + "@esbuild/aix-ppc64": "0.23.1", + "@esbuild/android-arm": "0.23.1", + "@esbuild/android-arm64": "0.23.1", + "@esbuild/android-x64": "0.23.1", + "@esbuild/darwin-arm64": "0.23.1", + "@esbuild/darwin-x64": "0.23.1", + "@esbuild/freebsd-arm64": "0.23.1", + "@esbuild/freebsd-x64": "0.23.1", + "@esbuild/linux-arm": "0.23.1", + "@esbuild/linux-arm64": "0.23.1", + "@esbuild/linux-ia32": "0.23.1", + "@esbuild/linux-loong64": "0.23.1", + "@esbuild/linux-mips64el": "0.23.1", + "@esbuild/linux-ppc64": "0.23.1", + "@esbuild/linux-riscv64": "0.23.1", + "@esbuild/linux-s390x": "0.23.1", + "@esbuild/linux-x64": "0.23.1", + "@esbuild/netbsd-x64": "0.23.1", + "@esbuild/openbsd-arm64": "0.23.1", + "@esbuild/openbsd-x64": "0.23.1", + "@esbuild/sunos-x64": "0.23.1", + "@esbuild/win32-arm64": "0.23.1", + "@esbuild/win32-ia32": "0.23.1", + "@esbuild/win32-x64": "0.23.1" } }, "node_modules/escalade": { @@ -7701,6 +7734,7 @@ "url": "https://paypal.me/jimmywarting" } ], + "license": "MIT", "dependencies": { "node-domexception": "^1.0.0", "web-streams-polyfill": "^3.0.3" @@ -7835,6 +7869,7 @@ "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", "dev": true, + "license": "MIT", "dependencies": { "fetch-blob": "^3.1.2" }, @@ -10977,6 +11012,7 @@ "url": "https://paypal.me/jimmywarting" } ], + "license": "MIT", "engines": { "node": ">=10.5.0" } @@ -11120,6 +11156,7 @@ "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-3.0.1.tgz", "integrity": "sha512-dMxCf+zZ+3zeQZXKxmyuCKlIDPGuv8EF940xbkC4kQVDTtqoh6rJFO+JTKSA6/Rwi0getWmtuy4Itup0AMcaDQ==", "dev": true, + "license": "ISC", "engines": { "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } @@ -11600,6 +11637,7 @@ "version": "0.2.5-alpha.2", "resolved": "https://registry.npmjs.org/pg-gateway/-/pg-gateway-0.2.5-alpha.2.tgz", "integrity": "sha512-boyO9iC6q5O/SvB7+XLJQrj+0tdf9OblwaQbeXzLnXwBbBb37WS6uWY2Zc+KuMxWq4G+QPGc055Unt9cH545Iw==", + "license": "MIT", "dependencies": { "pg-protocol": "^1.6.1" } @@ -12838,6 +12876,7 @@ "resolved": "https://registry.npmjs.org/read-cmd-shim/-/read-cmd-shim-4.0.0.tgz", "integrity": "sha512-yILWifhaSEEytfXI76kB9xEEiG1AiozaCJZ83A87ytjRiN+jVibXjedjCRNjoZviinhG+4UkalO3mWTd8u5O0Q==", "dev": true, + "license": "ISC", "engines": { "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } @@ -14045,9 +14084,9 @@ } }, "node_modules/supabase": { - "version": "1.191.3", - "resolved": "https://registry.npmjs.org/supabase/-/supabase-1.191.3.tgz", - "integrity": "sha512-5tIG7mPc5lZ9QRbkZssyHiOsx42qGFaVqclauXv+1fJAkZnfA28d0pzEDvfs33+w8YTReO5nNaWAgyzkWQQwfA==", + "version": "1.200.3", + "resolved": "https://registry.npmjs.org/supabase/-/supabase-1.200.3.tgz", + "integrity": "sha512-3NdhqBkfPVlm+rAhWQoVcyr54kykuAlHav/GWaAoQEHBDbbYI1lhbDzugk8ryQg92vSLwr3pWz0s4Hjdte8WyQ==", "dev": true, "hasInstallScript": true, "license": "MIT", @@ -14069,6 +14108,7 @@ "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", "dev": true, + "license": "MIT", "dependencies": { "debug": "^4.3.4" }, @@ -14091,6 +14131,7 @@ "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", "dev": true, + "license": "MIT", "dependencies": { "agent-base": "^7.0.2", "debug": "4" @@ -14134,6 +14175,7 @@ "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", "dev": true, + "license": "MIT", "dependencies": { "data-uri-to-buffer": "^4.0.0", "fetch-blob": "^3.1.4", @@ -14563,12 +14605,13 @@ "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==" }, "node_modules/tsx": { - "version": "4.16.2", - "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.16.2.tgz", - "integrity": "sha512-C1uWweJDgdtX2x600HjaFaucXTilT7tgUZHbOE4+ypskZ1OP8CRCSDkCxG6Vya9EwaFIVagWwpaVAn5wzypaqQ==", + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.19.1.tgz", + "integrity": "sha512-0flMz1lh74BR4wOvBjuh9olbnwqCPc35OOlfyzHba0Dc+QNUeWX/Gq2YTbnwcWPO3BMd8fkzRVrHcsR+a7z7rA==", "dev": true, + "license": "MIT", "dependencies": { - "esbuild": "~0.21.5", + "esbuild": "~0.23.0", "get-tsconfig": "^4.7.5" }, "bin": { @@ -15032,6 +15075,7 @@ "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", "dev": true, + "license": "MIT", "engines": { "node": ">= 8" } @@ -15275,6 +15319,7 @@ "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-5.0.1.tgz", "integrity": "sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==", "dev": true, + "license": "ISC", "dependencies": { "imurmurhash": "^0.1.4", "signal-exit": "^4.0.1" From d809d3e20eec61c21a70c1b980ba4b019251bf86 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Mon, 30 Sep 2024 23:25:24 +0200 Subject: [PATCH 16/20] skip if oid is correct --- .../src/pg-dump-middleware/constants.ts | 3 ++- .../get-extension-membership-query.ts | 2 +- .../pg-dump-middleware/get-extensions-query.ts | 2 +- .../src/pg-dump-middleware/pg-dump-middleware.ts | 15 +++++++++++---- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/apps/browser-proxy/src/pg-dump-middleware/constants.ts b/apps/browser-proxy/src/pg-dump-middleware/constants.ts index b25d721a..b3a03caf 100644 --- a/apps/browser-proxy/src/pg-dump-middleware/constants.ts +++ b/apps/browser-proxy/src/pg-dump-middleware/constants.ts @@ -1 +1,2 @@ -export const VECTOR_OID = '99999' +export const VECTOR_OID = 99999 +export const FIRST_NORMAL_OID = 16384 diff --git a/apps/browser-proxy/src/pg-dump-middleware/get-extension-membership-query.ts b/apps/browser-proxy/src/pg-dump-middleware/get-extension-membership-query.ts index 4fd51c00..74fccd13 100644 --- a/apps/browser-proxy/src/pg-dump-middleware/get-extension-membership-query.ts +++ b/apps/browser-proxy/src/pg-dump-middleware/get-extension-membership-query.ts @@ -101,7 +101,7 @@ function patchDependencyRow(message: Uint8Array, refobjidIndex: number): Uint8Ar const encoder = new TextEncoder() // Write the new OID value - const newRefobjidBytes = encoder.encode(VECTOR_OID.padStart(refobjidLength, '0')) + const newRefobjidBytes = encoder.encode(VECTOR_OID.toString().padStart(refobjidLength, '0')) newArray.set(newRefobjidBytes, offset) return newArray diff --git a/apps/browser-proxy/src/pg-dump-middleware/get-extensions-query.ts b/apps/browser-proxy/src/pg-dump-middleware/get-extensions-query.ts index 6a479a9c..89309074 100644 --- a/apps/browser-proxy/src/pg-dump-middleware/get-extensions-query.ts +++ b/apps/browser-proxy/src/pg-dump-middleware/get-extensions-query.ts @@ -89,7 +89,7 @@ function patchOidField( fields: { value: string | null; length: number }[] ): Uint8Array { const oldOidField = fields[oidIndex]! - const newOid = VECTOR_OID.padStart(oldOidField.length, '0') + const newOid = VECTOR_OID.toString().padStart(oldOidField.length, '0') const newArray = new Uint8Array(message) diff --git a/apps/browser-proxy/src/pg-dump-middleware/pg-dump-middleware.ts b/apps/browser-proxy/src/pg-dump-middleware/pg-dump-middleware.ts index 6e141841..95563d8b 100644 --- a/apps/browser-proxy/src/pg-dump-middleware/pg-dump-middleware.ts +++ b/apps/browser-proxy/src/pg-dump-middleware/pg-dump-middleware.ts @@ -5,6 +5,7 @@ import { isGetExtensionMembershipQuery, patchGetExtensionMembershipResult, } from './get-extension-membership-query.ts' +import { FIRST_NORMAL_OID } from './constants.ts' type ConnectionId = string @@ -48,10 +49,16 @@ export function pgDumpMiddleware( } const patched = patchGetExtensionsResult(message) if (patched.vectorOid) { - state.set(connectionId, { - step: 'wait-for-get-extension-membership-query', - vectorOid: patched.vectorOid, - }) + if (parseInt(patched.vectorOid) >= FIRST_NORMAL_OID) { + state.set(connectionId, { + step: 'complete', + }) + } else { + state.set(connectionId, { + step: 'wait-for-get-extension-membership-query', + vectorOid: patched.vectorOid, + }) + } } return patched.message case 'wait-for-get-extension-membership-query': From b90656017ffa4762452a8079c6ddd7698b9b3c8e Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 1 Oct 2024 09:20:37 +0200 Subject: [PATCH 17/20] use last version of PGlite --- apps/postgres-new/package.json | 2 +- package-lock.json | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/postgres-new/package.json b/apps/postgres-new/package.json index 743d3488..eb50df7d 100644 --- a/apps/postgres-new/package.json +++ b/apps/postgres-new/package.json @@ -12,7 +12,7 @@ "dependencies": { "@ai-sdk/openai": "^0.0.21", "@dagrejs/dagre": "^1.1.2", - "@electric-sql/pglite": "0.2.8", + "@electric-sql/pglite": "^0.2.9", "@gregnr/postgres-meta": "^0.82.0-dev.2", "@monaco-editor/react": "^4.6.0", "@radix-ui/react-accordion": "^1.2.0", diff --git a/package-lock.json b/package-lock.json index 7a40e018..ab77caf9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -88,7 +88,7 @@ "dependencies": { "@ai-sdk/openai": "^0.0.21", "@dagrejs/dagre": "^1.1.2", - "@electric-sql/pglite": "0.2.8", + "@electric-sql/pglite": "^0.2.9", "@gregnr/postgres-meta": "^0.82.0-dev.2", "@monaco-editor/react": "^4.6.0", "@radix-ui/react-accordion": "^1.2.0", @@ -164,9 +164,9 @@ } }, "apps/postgres-new/node_modules/@electric-sql/pglite": { - "version": "0.2.8", - "resolved": "https://registry.npmjs.org/@electric-sql/pglite/-/pglite-0.2.8.tgz", - "integrity": "sha512-0wSmQu22euBRzR5ghqyIHnBH4MfwlkL5WstOrrA3KOsjEWEglvoL/gH92JajEUA6Ufei/+qbkB2hVloC/K/RxQ==", + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/@electric-sql/pglite/-/pglite-0.2.9.tgz", + "integrity": "sha512-KPItmBmPVZJGOv+qkCXmWyIPUPLQIyN+BVtV/zD+083aWSR/2ReaCUN+HJv6Jw4z9zJ00UCPQkeUXvOLuTlumg==", "license": "Apache-2.0" }, "apps/postgres-new/node_modules/nanoid": { From 0ad5bea71072e10f782e57aa7159681de9b96ddd Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 1 Oct 2024 09:42:52 +0200 Subject: [PATCH 18/20] refactor middleware --- .../pg-dump-middleware/pg-dump-middleware.ts | 136 ++++++++++-------- apps/browser-proxy/src/tcp-server.ts | 4 +- apps/browser-proxy/src/websocket-server.ts | 3 +- 3 files changed, 82 insertions(+), 61 deletions(-) diff --git a/apps/browser-proxy/src/pg-dump-middleware/pg-dump-middleware.ts b/apps/browser-proxy/src/pg-dump-middleware/pg-dump-middleware.ts index 95563d8b..f7f9794f 100644 --- a/apps/browser-proxy/src/pg-dump-middleware/pg-dump-middleware.ts +++ b/apps/browser-proxy/src/pg-dump-middleware/pg-dump-middleware.ts @@ -1,4 +1,3 @@ -import ExpiryMap from 'expiry-map' import type { ClientParameters } from 'pg-gateway' import { isGetExtensionsQuery, patchGetExtensionsResult } from './get-extensions-query.ts' import { @@ -6,11 +5,10 @@ import { patchGetExtensionMembershipResult, } from './get-extension-membership-query.ts' import { FIRST_NORMAL_OID } from './constants.ts' +import type { Socket } from 'node:net' type ConnectionId = string -const state = new ExpiryMap(1000 * 60 * 5) - type State = | { step: 'wait-for-get-extensions-query' } | { step: 'get-extensions-query-received' } @@ -18,66 +16,90 @@ type State = | { step: 'get-extension-membership-query-received'; vectorOid: string } | { step: 'complete' } -export function pgDumpMiddleware( - connectionId: string, - origin: 'client' | 'server', - context: { - clientParams?: ClientParameters - }, - message: Uint8Array -) { - if (context.clientParams?.application_name !== 'pg_dump') { - return message - } - - if (!state.has(connectionId)) { - state.set(connectionId, { step: 'wait-for-get-extensions-query' }) - } +class PgDumpMiddleware { + private state: Map = new Map() - const connectionState = state.get(connectionId)! + constructor() {} - switch (connectionState.step) { - case 'wait-for-get-extensions-query': - // https://github.com/postgres/postgres/blob/a19f83f87966f763991cc76404f8e42a36e7e842/src/bin/pg_dump/pg_dump.c#L5834-L5837 - if (origin === 'client' && isGetExtensionsQuery(message)) { - state.set(connectionId, { step: 'get-extensions-query-received' }) - } + client( + socket: Socket, + connectionId: string, + context: { + clientParams?: ClientParameters + }, + message: Uint8Array + ) { + if (context.clientParams?.application_name !== 'pg_dump') { return message - case 'get-extensions-query-received': - if (origin === 'client') { - return message - } - const patched = patchGetExtensionsResult(message) - if (patched.vectorOid) { - if (parseInt(patched.vectorOid) >= FIRST_NORMAL_OID) { - state.set(connectionId, { - step: 'complete', - }) - } else { - state.set(connectionId, { - step: 'wait-for-get-extension-membership-query', - vectorOid: patched.vectorOid, + } + + if (!this.state.has(connectionId)) { + this.state.set(connectionId, { step: 'wait-for-get-extensions-query' }) + socket.on('close', () => { + this.state.delete(connectionId) + }) + } + + const connectionState = this.state.get(connectionId)! + + switch (connectionState.step) { + case 'wait-for-get-extensions-query': + // https://github.com/postgres/postgres/blob/a19f83f87966f763991cc76404f8e42a36e7e842/src/bin/pg_dump/pg_dump.c#L5834-L5837 + if (isGetExtensionsQuery(message)) { + this.state.set(connectionId, { step: 'get-extensions-query-received' }) + } + break + case 'wait-for-get-extension-membership-query': + // https://github.com/postgres/postgres/blob/a19f83f87966f763991cc76404f8e42a36e7e842/src/bin/pg_dump/pg_dump.c#L18173-L18178 + if (isGetExtensionMembershipQuery(message)) { + this.state.set(connectionId, { + step: 'get-extension-membership-query-received', + vectorOid: connectionState.vectorOid, }) } - } - return patched.message - case 'wait-for-get-extension-membership-query': - // https://github.com/postgres/postgres/blob/a19f83f87966f763991cc76404f8e42a36e7e842/src/bin/pg_dump/pg_dump.c#L18173-L18178 - if (origin === 'client' && isGetExtensionMembershipQuery(message)) { - state.set(connectionId, { - step: 'get-extension-membership-query-received', - vectorOid: connectionState.vectorOid, - }) - } + break + } + + return message + } + + server( + connectionId: string, + context: { + clientParams?: ClientParameters + }, + message: Uint8Array + ) { + if (context.clientParams?.application_name !== 'pg_dump' || !this.state.has(connectionId)) { return message - case 'get-extension-membership-query-received': - if (origin === 'client') { + } + + const connectionState = this.state.get(connectionId)! + + switch (connectionState.step) { + case 'get-extensions-query-received': + const patched = patchGetExtensionsResult(message) + if (patched.vectorOid) { + if (parseInt(patched.vectorOid) >= FIRST_NORMAL_OID) { + this.state.set(connectionId, { + step: 'complete', + }) + } else { + this.state.set(connectionId, { + step: 'wait-for-get-extension-membership-query', + vectorOid: patched.vectorOid, + }) + } + } + return patched.message + case 'get-extension-membership-query-received': + const patchedMessage = patchGetExtensionMembershipResult(message, connectionState.vectorOid) + this.state.set(connectionId, { step: 'complete' }) + return patchedMessage + default: return message - } - const patchedMessage = patchGetExtensionMembershipResult(message, connectionState.vectorOid) - state.set(connectionId, { step: 'complete' }) - return patchedMessage - case 'complete': - return message + } } } + +export const pgDumpMiddleware = new PgDumpMiddleware() diff --git a/apps/browser-proxy/src/tcp-server.ts b/apps/browser-proxy/src/tcp-server.ts index c14f81f5..b6eda574 100644 --- a/apps/browser-proxy/src/tcp-server.ts +++ b/apps/browser-proxy/src/tcp-server.ts @@ -90,9 +90,9 @@ tcpServer.on('connection', async (socket) => { } debug('tcp message: %e', () => Buffer.from(message).toString('hex')) - message = pgDumpMiddleware( + message = pgDumpMiddleware.client( + socket, connectionState!.connectionId, - 'client', connection.state, Buffer.from(message) ) diff --git a/apps/browser-proxy/src/websocket-server.ts b/apps/browser-proxy/src/websocket-server.ts index 0d586062..baaf6b1c 100644 --- a/apps/browser-proxy/src/websocket-server.ts +++ b/apps/browser-proxy/src/websocket-server.ts @@ -90,9 +90,8 @@ websocketServer.on('connection', async (websocket, request) => { if (tcpConnection) { debug('websocket message: %e', () => message.toString('hex')) message = Buffer.from( - pgDumpMiddleware( + pgDumpMiddleware.server( connectionId, - 'server', tcpConnection.state, new Uint8Array(message.buffer, message.byteOffset, message.byteLength) ) From b30268e5cd3a88091e60a149beb514eba5c8dcdc Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 1 Oct 2024 09:46:55 +0200 Subject: [PATCH 19/20] comment the middleware --- .../src/pg-dump-middleware/pg-dump-middleware.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apps/browser-proxy/src/pg-dump-middleware/pg-dump-middleware.ts b/apps/browser-proxy/src/pg-dump-middleware/pg-dump-middleware.ts index f7f9794f..ee2adcfa 100644 --- a/apps/browser-proxy/src/pg-dump-middleware/pg-dump-middleware.ts +++ b/apps/browser-proxy/src/pg-dump-middleware/pg-dump-middleware.ts @@ -16,6 +16,12 @@ type State = | { step: 'get-extension-membership-query-received'; vectorOid: string } | { step: 'complete' } +/** + * Middleware to patch pg_dump results for PGlite < v0.2.8 + * PGlite < v0.2.8 has a bug in which userland extensions are not dumped because their oid is lower than FIRST_NORMAL_OID + * This middleware patches the results of the get_extensions and get_extension_membership queries to increase the oid of the `vector` extension so it can be dumped + * For more context, see: https://github.com/electric-sql/pglite/issues/352 + */ class PgDumpMiddleware { private state: Map = new Map() From 68d37be1540a050591fa8c09bdcf31358a2aeb7b Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 1 Oct 2024 10:04:32 +0200 Subject: [PATCH 20/20] gracefully exit for --watch --- apps/browser-proxy/src/index.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/apps/browser-proxy/src/index.ts b/apps/browser-proxy/src/index.ts index b9a78294..b25c763b 100644 --- a/apps/browser-proxy/src/index.ts +++ b/apps/browser-proxy/src/index.ts @@ -16,3 +16,22 @@ httpsServer.listen(443, () => { tcpServer.listen(5432, () => { console.log('tcp server listening on port 5432') }) + +const shutdown = async () => { + await Promise.allSettled([ + new Promise((res) => + httpsServer.close(() => { + res() + }) + ), + new Promise((res) => + tcpServer.close(() => { + res() + }) + ), + ]) + process.exit(0) +} + +process.on('SIGTERM', shutdown) +process.on('SIGINT', shutdown)