feat: TS support to views fs I/O (#18419)

* feat(internals): add error-safe fp-ts utilities to deal with fs I/O

* feat(internals): handle fs I/O when receiving view definitions

* feat(migrate): plug in views I/O fs handling in "prisma db pull"

* test(migrate): start testing "postgresql views fs I/O"

* test removal of extraneous files

* test deletion with file perms

* add cli snapshot for views fs

* test without views previewFeature

* remove perms test

* test: assert "views" shape when no views are retrieved

* fix: update IntrospectResult type signature

* fix(tests): postgres views on Windows

* chore: udpate comments

* test(migrate): view fs in several paths, both standard and non-standard

* fix: remove path.posix from globby

* test: complete postgres views fs I/O

* fix: windows support in globby

* fix: globby on windows

* fix: globby on windows

* chore: cannot use replaceAll on Node.js 14

* chore: skip win32

* fix(migrate): update snapshots

---------

Co-authored-by: Sophie Atkins <atkins@prisma.io>
Co-authored-by: Jan Piotrowski <piotrowski+github@gmail.com>
Co-authored-by: Alexey Orlenko <alex@aqrln.net>

packages/internals/src/index.ts

@@ -51,6 +51,7 @@ export { drawBox } from './utils/drawBox'
 export { extractPreviewFeatures } from './utils/extractPreviewFeatures'
 export { formatms } from './utils/formatms'
 export { formatTable } from './utils/formatTable'
+export * as fsFunctional from './utils/fs-functional'
 export { getCommandWithExecutor } from './utils/getCommandWithExecutor'
 export type { EnvPaths } from './utils/getEnvPaths'
 export { getEnvPaths } from './utils/getEnvPaths'
@@ -79,6 +80,8 @@ export { createSpinner } from './utils/spinner'
 export type { Position } from './utils/trimBlocksFromSchema'
 export { trimBlocksFromSchema, trimNewLine } from './utils/trimBlocksFromSchema'
 export { tryLoadEnvs } from './utils/tryLoadEnvs'
+export type { IntrospectionViewDefinition } from './views/handleViewsIO'
+export { handleViewsIO } from './views/handleViewsIO'
 export { warnOnce } from './warnOnce'
 export * as wasm from './wasm'
 export type { Platform } from '@prisma/get-platform'

packages/internals/src/utils/fs-functional.ts

@@ -0,0 +1,75 @@
+import { pipe } from 'fp-ts/lib/function'
+import * as T from 'fp-ts/lib/Task'
+import * as TE from 'fp-ts/lib/TaskEither'
+import fs from 'fs'
+import globby from 'globby'
+import path from 'path'
+
+export const createDirIfNotExists = (dir: string) =>
+  pipe(
+    TE.tryCatch(
+      // Note: { recursive: true } prevents EEEXIST error codes when the directory already exists
+      () => fs.promises.mkdir(dir, { recursive: true }),
+      createTaggedSystemError('fs-create-dir', { dir }),
+    ),
+  )
+
+export const writeFile = ({ path, content }: { path: string; content: string }) =>
+  pipe(
+    TE.tryCatch(
+      () => fs.promises.writeFile(path, content, { encoding: 'utf-8' }),
+      createTaggedSystemError('fs-write-file', { path, content }),
+    ),
+  )
+
+/**
+ * Note to future self: in Node.js, `removeDir` and `removeFile` can both be implemented with a single `fs.promises.rm` call.
+ */
+
+export const removeDir = (dir: string) =>
+  pipe(
+    TE.tryCatch(() => fs.promises.rmdir(dir, { recursive: true }), createTaggedSystemError('fs-remove-dir', { dir })),
+  )
+
+export const removeFile = (filePath: string) =>
+  pipe(TE.tryCatch(() => fs.promises.unlink(filePath), createTaggedSystemError('fs-remove-file', { filePath })))
+
+/**
+ * Removes all backslashes from a possibly Windows path string, which is necessary for globby to work on Windows.
+ * Note: we can't use `dir.replaceAll(path.sep, '/')` because `String.prototype.replaceAll` requires at least Node.js 15.
+ */
+export const normalizePossiblyWindowsDir = (dir: string) => {
+  if (process.platform === 'win32') {
+    return dir.replace(/\\/g, '/')
+  }
+
+  return dir
+}
+
+export const getFoldersInDir =
+  (dir: string): T.Task<string[]> =>
+  () => {
+    const normalizedDir = normalizePossiblyWindowsDir(path.join(dir, '**'))
+    return globby(normalizedDir, { onlyFiles: false, onlyDirectories: true })
+  }
+
+export const getFilesInDir =
+  (dir: string): T.Task<string[]> =>
+  () => {
+    const normalizedDir = normalizePossiblyWindowsDir(path.join(dir, '**'))
+    return globby(normalizedDir, { onlyFiles: true, onlyDirectories: false })
+  }
+
+/**
+ * Closure that creates a tagged system error for a given error callback.
+ * @param type the tag type for the error, e.g. 'fs-create-dir'
+ * @param meta any additional metadata about what caused the error
+ */
+function createTaggedSystemError<Tag extends string, Meta extends Record<string, unknown>>(type: Tag, meta: Meta) {
+  return (e: Error | unknown /* untyped error */) =>
+    ({
+      type,
+      error: e as Error & { code: string },
+      meta,
+    } as const)
+}

packages/internals/src/views/handleViewsIO.ts

@@ -0,0 +1,119 @@
+import * as E from 'fp-ts/lib/Either'
+import { pipe } from 'fp-ts/lib/function'
+import * as T from 'fp-ts/lib/Task'
+import * as TE from 'fp-ts/lib/TaskEither'
+import path from 'path'
+import { match } from 'ts-pattern'
+
+import {
+  createDirIfNotExists,
+  getFilesInDir,
+  getFoldersInDir,
+  normalizePossiblyWindowsDir,
+  removeDir,
+  removeFile,
+  writeFile,
+} from '../utils/fs-functional'
+
+export interface IntrospectionViewDefinition {
+  // The database or schema where the view is located
+  schema: string
+
+  // The name of the view
+  name: string
+
+  // The database SQL query that defines the view
+  definition: string
+}
+
+type HandleViewsIOParams = {
+  views: IntrospectionViewDefinition[]
+  schemaPath: string
+}
+
+/**
+ * For any given view definitions, the CLI must either create or update the corresponding view definition files
+ * in the file system, in `${path.dirname(schemaPath)}/views/{viewDbSchema}/{viewName}.sql`.
+ * If some other files or folders exist within the `views` directory, the CLI must remove them.
+ * These files and folders are deleted silently.
+ */
+export async function handleViewsIO({ views, schemaPath }: HandleViewsIOParams): Promise<void> {
+  const prismaDir = path.dirname(normalizePossiblyWindowsDir(schemaPath))
+  const viewsDir = path.posix.join(prismaDir, 'views')
+
+  // collect the newest view definitions
+  const viewEntries = views.map(({ schema, ...rest }) => {
+    const viewDir = path.posix.join(viewsDir, schema)
+    return [viewDir, rest] as const
+  })
+
+  // collect the paths to the view directories (identified by their db schema name) corresponding to the newest view definitions,
+  // which will be created later if they don't exist
+  const viewPathsToWrite: string[] = viewEntries.map(([viewDir, _]) => viewDir)
+
+  // collect the files paths and content for the newest views' SQL definitions, which will be created later if they don't exist
+  const viewsFilesToWrite = viewEntries.map(([viewDir, { name, definition }]) => {
+    const viewFile = path.posix.join(viewDir, `${name}.sql`)
+    return { path: viewFile, content: definition } as const
+  })
+
+  const updateDefinitionsInViewsDirPipeline = pipe(
+    // create the views directory, idempotently
+    createDirIfNotExists(viewsDir),
+
+    // create the view directories, idempotently and concurrently, collapsing the possible errors
+    TE.chainW(() => TE.traverseArray(createDirIfNotExists)(viewPathsToWrite)),
+
+    // write the view definitions in the directories just created, idempotently and concurrently, collapsing the possible errors
+    TE.chainW(() => TE.traverseArray(writeFile)(viewsFilesToWrite)),
+
+    // remove any view directories related to schemas that no longer exist, concurrently, collapsing the possible errors
+    TE.chainW(() =>
+      pipe(
+        getFoldersInDir(viewsDir),
+        T.chain((directoriesInViewsDir) => {
+          const viewDirsToRemove = directoriesInViewsDir.filter((dir) => !viewPathsToWrite.includes(dir))
+          return TE.traverseArray(removeDir)(viewDirsToRemove)
+        }),
+      ),
+    ),
+
+    // remove any other files in the views directory beyond the ones just created, concurrently, collapsing the possible errors
+    TE.chainW(() =>
+      pipe(
+        getFilesInDir(viewsDir),
+        T.chain((filesInViewsDir) => {
+          const viewFilesToKeep = viewsFilesToWrite.map(({ path }) => path)
+          const viewFilesToRemove = filesInViewsDir.filter((file) => !viewFilesToKeep.includes(file))
+          return TE.traverseArray(removeFile)(viewFilesToRemove)
+        }),
+      ),
+    ),
+  )
+
+  // run the fs views pipeline
+  const updateDefinitionsInViewsDirEither = await updateDefinitionsInViewsDirPipeline()
+
+  if (E.isRight(updateDefinitionsInViewsDirEither)) {
+    // success: no error happened while writing and cleaning up the views directory
+    return
+  }
+
+  // failure: check which error to throw
+  const error = match(updateDefinitionsInViewsDirEither.left)
+    .with({ type: 'fs-create-dir' }, (e) => {
+      throw new Error(`Error creating the directory: ${e.meta.dir}.\n${e.error}.`)
+    })
+    .with({ type: 'fs-write-file' }, (e) => {
+      throw new Error(`Error writing the view definition\n${e.meta.content}\nto file ${e.meta.path}.\n${e.error}.`)
+    })
+    .with({ type: 'fs-remove-dir' }, (e) => {
+      throw new Error(`Error removing the directory: ${e.meta.dir}.\n${e.error}.`)
+    })
+    .with({ type: 'fs-remove-file' }, (e) => {
+      throw new Error(`Error removing the file: ${e.meta.filePath}.\n${e.error}.`)
+    })
+    .exhaustive()
+
+  throw error
+}

packages/migrate/src/MigrateEngine.ts

@@ -1,9 +1,17 @@
 import Debug from '@prisma/debug'
 import type { MigrateEngineLogLine } from '@prisma/internals'
-import { BinaryType, ErrorArea, MigrateEngineExitCode, resolveBinary, RustPanic } from '@prisma/internals'
+import {
+  BinaryType,
+  ErrorArea,
+  handleViewsIO,
+  MigrateEngineExitCode,
+  resolveBinary,
+  RustPanic,
+} from '@prisma/internals'
 import chalk from 'chalk'
 import type { ChildProcess } from 'child_process'
 import { spawn } from 'child_process'
+import path from 'path'
 
 import type { EngineArgs, EngineResults, RPCPayload, RpcSuccessResponse } from './types'
 import byline from './utils/byline'
@@ -170,9 +178,16 @@ export class MigrateEngine {
     this.latestSchema = schema
 
     try {
-      const introspectResult = await this.runCommand(
+      const introspectResult: EngineArgs.IntrospectResult = await this.runCommand(
         this.getRPCPayload('introspect', { schema, force, compositeTypeDepth, schemas }),
       )
+      const { views } = introspectResult
+
+      if (views) {
+        const schemaPath = this.schemaPath ?? path.join(process.cwd(), 'prisma')
+        await handleViewsIO({ views, schemaPath })
+      }
+
       return introspectResult
     } finally {
       // stop the engine after either a successful or failed introspection, to emulate how the