test(client): Opt Out Of Providers (#13507)

* test/feat: add optIn options

* test: add coverage for all in the _example

* refactor: change to opt out

* test: add more provider support

* refactor: add missingProvider helper

* temp: remove cockroachdb

* Revert "temp: remove cockroachdb"

This reverts commit fc84e59.

* temp: opt out cockroach for metrics

* fix: add cockroach and mongo to metrics

* temp: remove other tests and add cockroach uri

* Revert "temp: remove other tests and add cockroach uri"

This reverts commit 9ab5b86.

* ci/fix: add cockroachdb envs to client tests

.github/workflows/test.yml

@@ -168,6 +168,9 @@ jobs:
           TEST_MSSQL_URI: mssql://SA:Pr1sm4_Pr1sm4@localhost:1433/master
           TEST_MSSQL_JDBC_URI: sqlserver://localhost:1433;database=master;user=SA;password=Pr1sm4_Pr1sm4;trustServerCertificate=true;
           TEST_MONGO_URI: 'mongodb://root:prisma@localhost:27018/tests?authSource=admin'
+          TEST_COCKROACH_URI: 'postgresql://prisma@localhost:26257/tests'
+          TEST_COCKROACH_URI_MIGRATE: 'postgresql://prisma@localhost:26257/tests-migrate'
+          TEST_COCKROACH_SHADOWDB_URI_MIGRATE: 'postgres://prisma:prisma@localhost:26257/tests-migrate-shadowdb'
           NODE_OPTIONS: '--max-old-space-size=8096'
 
       - uses: codecov/codecov-action@v3

packages/client/tests/functional/_example/_matrix.ts

@@ -7,6 +7,21 @@ export default defineMatrix(() => [
       id: 'Int @id @default(autoincrement())',
       providerFeatures: '',
     },
+    {
+      provider: 'postgresql',
+      id: 'Int @id @default(autoincrement())',
+      providerFeatures: '',
+    },
+    {
+      provider: 'mysql',
+      id: 'Int @id @default(autoincrement())',
+      providerFeatures: '',
+    },
+    {
+      provider: 'cockroachdb',
+      id: 'BigInt @id @default(autoincrement())',
+      providerFeatures: '',
+    },
     {
       provider: 'mongodb',
       id: 'String @id @default(auto()) @map("_id") @db.ObjectId',

packages/client/tests/functional/_example/tests.ts

@@ -6,14 +6,15 @@ import testMatrix from './_matrix'
 // @ts-ignore this is just for type checks
 declare let prisma: import('@prisma/client').PrismaClient
 
-testMatrix.setupTestSuite((suiteConfig, suiteMeta) => {
-  // an example of how to query with the preloaded client
-  test('findMany', async () => {
-    await prisma.user.findMany()
-  })
-
-  test('suiteConfig', () => {
-    /* 
+testMatrix.setupTestSuite(
+  (suiteConfig, suiteMeta) => {
+    // an example of how to query with the preloaded client
+    test('findMany', async () => {
+      await prisma.user.findMany()
+    })
+
+    test('suiteConfig', () => {
+      /* 
       {
         provider: 'sqlite',
         id: 'Int @id @default(autoincrement())',
@@ -22,11 +23,11 @@ testMatrix.setupTestSuite((suiteConfig, suiteMeta) => {
       }
     */
 
-    expect(typeof suiteConfig.provider).toEqual('string')
-  })
+      expect(typeof suiteConfig.provider).toEqual('string')
+    })
 
-  test('suiteMeta', () => {
-    /* 
+    test('suiteMeta', () => {
+      /* 
       {
         testPath: './code/prisma/packages/client/tests/functional/_example/tests.ts',
         testDir: './code/prisma/packages/client/tests/functional/_example',
@@ -38,15 +39,24 @@ testMatrix.setupTestSuite((suiteConfig, suiteMeta) => {
       }
     */
 
-    expect(typeof suiteMeta.testPath).toEqual('string')
-    expect(suiteMeta.testFileName).toEqual(path.basename(__filename))
-  })
-
-  test('getTestSuiteSchema', async () => {
-    const schemaString = await getTestSuiteSchema(suiteMeta, suiteConfig)
-
-    expect(schemaString).toContain('generator')
-    expect(schemaString).toContain('datasource')
-    expect(schemaString).toContain('model')
-  })
-})
+      expect(typeof suiteMeta.testPath).toEqual('string')
+      expect(suiteMeta.testFileName).toEqual(path.basename(__filename))
+    })
+
+    test('getTestSuiteSchema', async () => {
+      const schemaString = await getTestSuiteSchema(suiteMeta, suiteConfig)
+
+      expect(schemaString).toContain('generator')
+      expect(schemaString).toContain('datasource')
+      expect(schemaString).toContain('model')
+    })
+  },
+  // Use `optOut` to opt out from testing the default selected providers
+  // otherwise the suite will require all providers to be specified.
+  // {
+  //   optOut: {
+  //     from: ['sqlite', 'mongodb'],
+  //     reason: 'Only testing xyz provider(s) so opting out of sqlite and mongodb',
+  //   },
+  // },
+)

packages/client/tests/functional/_utils/checkMissingProviders.ts

@@ -0,0 +1,39 @@
+import { getTestSuiteMeta, TestSuiteConfig } from './getTestSuiteInfo'
+import { Providers } from './providers'
+import { MatrixOptions } from './types'
+
+export type TestSuiteMeta = ReturnType<typeof getTestSuiteMeta>
+
+/**
+ * Ensure that we are not forgetting to add a provider to the matrix unless we
+ * explicitly opt out during in the {@link setupTestSuiteMatrix} creation.
+ * @param suiteConfigs
+ * @param suiteMeta
+ * @param options
+ */
+export function checkMissingProviders({
+  suiteConfig,
+  suiteMeta,
+  options,
+}: {
+  suiteConfig: TestSuiteConfig[]
+  suiteMeta: TestSuiteMeta
+  options?: MatrixOptions
+}) {
+  const suiteConfigProviders = suiteConfig.map(({ provider }) => provider)
+
+  const missingProviders = Object.values(Providers).reduce((acc, provider) => {
+    if (suiteConfigProviders.includes(provider)) return acc
+    if (options?.optOut?.from.includes(provider)) return acc
+
+    return [...acc, provider]
+  }, [] as Providers[])
+
+  if (missingProviders.length) {
+    throw new Error(
+      `Test: '${suiteMeta.testDirName}' is missing providers '${missingProviders
+        .map((x) => `'${x}'`)
+        .join(', ')}' out out using options.optOut`,
+    )
+  }
+}

packages/client/tests/functional/_utils/defineMatrix.ts

@@ -1,7 +1,8 @@
 import { U } from 'ts-toolbelt'
 
 import { TestSuiteMatrix } from './getTestSuiteInfo'
-import { MatrixOptions, setupTestSuiteMatrix, TestSuiteMeta } from './setupTestSuiteMatrix'
+import { setupTestSuiteMatrix, TestSuiteMeta } from './setupTestSuiteMatrix'
+import { MatrixOptions } from './types'
 
 type MergedMatrixParams<MatrixT extends TestSuiteMatrix> = U.IntersectOf<MatrixT[number][number]>
 

packages/client/tests/functional/_utils/getTestSuiteInfo.ts

@@ -108,11 +108,13 @@ export function getTestSuiteConfigs(suiteMeta: TestSuiteMeta) {
  * @param suiteMeta
  * @returns [test-suite-title: string, test-suite-config: object]
  */
-export function getTestSuiteTable(suiteMeta: TestSuiteMeta) {
-  return map(
-    getTestSuiteConfigs(suiteMeta),
-    (suiteConfig) => [getTestSuiteFullName(suiteMeta, suiteConfig), suiteConfig] as const,
-  )
+export function getTestSuiteTable(
+  suiteMeta: TestSuiteMeta,
+  suiteConfig: {
+    [x: string]: string
+  }[],
+) {
+  return map(suiteConfig, (suiteConfig) => [getTestSuiteFullName(suiteMeta, suiteConfig), suiteConfig] as const)
 }
 
 /**

packages/client/tests/functional/_utils/idForProvider.ts

@@ -14,6 +14,13 @@ export function idForProvider(provider: string, options: Options = { includeDefa
       strs.push('@map("_id") @db.ObjectId')
 
       break
+    case 'cockroachdb':
+      if (options.includeDefault) {
+        strs.push('@default(cuid())')
+      }
+
+      break
+
     default:
       if (options.includeDefault) {
         strs.push('@default(uuid())')

packages/client/tests/functional/_utils/providers.ts

@@ -0,0 +1,7 @@
+export enum Providers {
+  SQLITE = 'sqlite',
+  POSTGRESQL = 'postgresql',
+  MYSQL = 'mysql',
+  MONGODB = 'mongodb',
+  COCKROACHDB = 'cockroachdb',
+}

packages/client/tests/functional/_utils/setupFilesAfterEnv.ts

@@ -3,5 +3,6 @@ process.env.DATABASE_URI_sqlite = 'file:dev.db'
 process.env.DATABASE_URI_mongodb = process.env.TEST_MONGO_URI
 process.env.DATABASE_URI_postgresql = process.env.TEST_POSTGRES_URI
 process.env.DATABASE_URI_mysql = process.env.TEST_MYSQL_URI
+process.env.DATABASE_URI_cockroachdb = process.env.TEST_COCKROACH_URI
 
 export {}

packages/client/tests/functional/_utils/setupTestSuiteMatrix.ts

@@ -1,13 +1,11 @@
-import { getTestSuiteMeta, getTestSuiteTable, TestSuiteConfig } from './getTestSuiteInfo'
+import { checkMissingProviders } from './checkMissingProviders'
+import { getTestSuiteConfigs, getTestSuiteMeta, getTestSuiteTable, TestSuiteConfig } from './getTestSuiteInfo'
 import { setupTestSuiteClient } from './setupTestSuiteClient'
 import { dropTestSuiteDatabase, setupTestSuiteDbURI } from './setupTestSuiteEnv'
+import { MatrixOptions } from './types'
 
 export type TestSuiteMeta = ReturnType<typeof getTestSuiteMeta>
 
-export type MatrixOptions = {
-  skipDb?: boolean
-}
-
 /**
  * How does this work from a high level? What steps?
  * 1. You create a file that uses `setupTestSuiteMatrix`
@@ -45,9 +43,14 @@ function setupTestSuiteMatrix(
 ) {
   const originalEnv = process.env
   const suiteMeta = getTestSuiteMeta()
-  const suiteTable = getTestSuiteTable(suiteMeta)
+  const suiteConfig = getTestSuiteConfigs(suiteMeta)
+  const suiteTable = getTestSuiteTable(suiteMeta, suiteConfig)
   const forceInlineSnapshot = process.argv.includes('-u')
-
+  checkMissingProviders({
+    suiteConfig,
+    suiteMeta,
+    options,
+  })
   ;(forceInlineSnapshot ? [suiteTable[0]] : suiteTable).forEach((suiteEntry) => {
     const [suiteName, suiteConfig] = suiteEntry
 

packages/client/tests/functional/_utils/types.ts

@@ -0,0 +1,9 @@
+import { Providers } from './providers'
+
+export type MatrixOptions = {
+  optOut?: {
+    from: `${Providers}`[]
+    reason: string
+  }
+  skipDb?: boolean
+}

packages/client/tests/functional/decimal-list/_matrix.ts

@@ -5,5 +5,8 @@ export default defineMatrix(() => [
     {
       provider: 'postgresql',
     },
+    {
+      provider: 'cockroachdb',
+    },
   ],
 ])

packages/client/tests/functional/decimal-list/tests.ts

@@ -3,28 +3,39 @@ import { setupTestSuiteMatrix } from '../_utils/setupTestSuiteMatrix'
 // @ts-ignore this is just for type checks
 declare let prisma: import('@prisma/client').PrismaClient
 
-setupTestSuiteMatrix((suiteConfig, suiteMeta) => {
-  test('with decimal instances', async () => {
-    await prisma.user.create({
-      data: {
-        decimals: [12.3, 45.6],
-      },
+setupTestSuiteMatrix(
+  () => {
+    test('with decimal instances', async () => {
+      await prisma.user.create({
+        data: {
+          decimals: [12.3, 45.6],
+        },
+      })
     })
-  })
 
-  test('with numbers', async () => {
-    await prisma.user.create({
-      data: {
-        decimals: [12.3, 45.6],
-      },
+    test('with numbers', async () => {
+      await prisma.user.create({
+        data: {
+          decimals: [12.3, 45.6],
+        },
+      })
     })
-  })
 
-  test('create with strings', async () => {
-    await prisma.user.create({
-      data: {
-        decimals: ['12.3', '45.6'],
-      },
+    test('create with strings', async () => {
+      await prisma.user.create({
+        data: {
+          decimals: ['12.3', '45.6'],
+        },
+      })
     })
-  })
-})
+  },
+  {
+    optOut: {
+      from: ['mongodb', 'mysql', 'sqlite'],
+      reason: `
+        Mongodb connector does not support the Decimal type. 
+        Mysql & Sqlite connectors do not support lists of primitive types.
+      `,
+    },
+  },
+)

packages/client/tests/functional/decimal/_matrix.ts

@@ -11,5 +11,8 @@ export default defineMatrix(() => [
     {
       provider: 'mysql',
     },
+    {
+      provider: 'cockroachdb',
+    },
   ],
 ])