[cli-kit] Support parsing multiline environment variables in .env file

.changeset/perfect-beans-repair.md

@@ -0,0 +1,5 @@
+---
+'@shopify/cli-kit': minor
+---
+
+Support parsing multiline environment variables in .env file

packages/cli-kit/package.json

@@ -123,8 +123,8 @@
     "cross-zip": "4.0.0",
     "deepmerge": "4.3.1",
     "del": "6.1.1",
+    "dotenv": "16.4.5",
     "env-paths": "3.0.0",
-    "envfile": "6.18.0",
     "execa": "7.2.0",
     "fast-glob": "3.3.1",
     "figures": "5.0.0",

packages/cli-kit/src/public/node/dot-env.test.ts

@@ -1,4 +1,4 @@
-import {patchEnvFile, readAndParseDotEnv, writeDotEnv} from './dot-env.js'
+import {patchEnvFile, readAndParseDotEnv, writeDotEnv, createDotEnvFileLine} from './dot-env.js'
 import {inTemporaryDirectory, writeFile, readFile} from './fs.js'
 import {joinPath} from './path.js'
 import {describe, expect, test} from 'vitest'
@@ -26,6 +26,22 @@ describe('readAndParseDotEnv', () => {
       expect(got.variables.FOO).toEqual('BAR')
     })
   })
+
+  test('ensures newline characters are parsed from .env file', async () => {
+    await inTemporaryDirectory(async (tmpDir) => {
+      // Given
+      const dotEnvPath = joinPath(tmpDir, '.env')
+
+      await writeFile(dotEnvPath, `FOO="BAR\nBAR\nBAR"`)
+
+      // When
+      const got = await readAndParseDotEnv(dotEnvPath)
+
+      // Then
+      expect(got.path).toEqual(dotEnvPath)
+      expect(got.variables.FOO).toEqual('BAR\nBAR\nBAR')
+    })
+  })
 })
 
 describe('writeDotEnv', () => {
@@ -49,6 +65,27 @@ describe('writeDotEnv', () => {
     })
   })
 
+  test('creates a file with multiline env vars', async () => {
+    await inTemporaryDirectory(async (tmpDir) => {
+      // Given
+      const dotEnvPath = joinPath(tmpDir, '.env')
+
+      // When
+      await writeDotEnv({
+        path: dotEnvPath,
+        variables: {
+          FOO: 'BAR',
+          MULTI: 'LINE\nVARIABLE',
+        },
+      })
+      const got = await readAndParseDotEnv(dotEnvPath)
+
+      // Then
+      expect(got.path).toEqual(dotEnvPath)
+      expect(got.variables.MULTI).toEqual('LINE\nVARIABLE')
+    })
+  })
+
   test('overrides any existing file', async () => {
     await inTemporaryDirectory(async (tmpDir) => {
       // Given
@@ -112,4 +149,110 @@ describe('patchEnvFile', () => {
       expect(patchedContent).toEqual('FOO=BAR\nABC=123\n#Wow!\n\n  DEF  =GHI\r\nWIN=DOWS')
     })
   })
+
+  test('patches an environment file containing newline characters', async () => {
+    await inTemporaryDirectory(async (tmpDir) => {
+      // Given
+      const dotEnvPath = joinPath(tmpDir, '.env')
+      await writeFile(dotEnvPath, 'FOO="BAR\nBAR\nBAR"\nABC   =XYZ\n#Wow!\n\n  DEF  =GHI')
+
+      // When
+      const got = await readAndParseDotEnv(dotEnvPath)
+      expect(got.variables).toEqual({
+        FOO: 'BAR\nBAR\nBAR',
+        ABC: 'XYZ',
+        DEF: 'GHI',
+      })
+
+      // Then
+      const patchedContent = patchEnvFile(await readFile(dotEnvPath), {ABC: '123'})
+      expect(patchedContent).toEqual('FOO="BAR\nBAR\nBAR"\nABC=123\n#Wow!\n\n  DEF  =GHI')
+    })
+  })
+
+  test('patches env var with newline characters', async () => {
+    await inTemporaryDirectory(async (tmpDir) => {
+      // Given
+      const dotEnvPath = joinPath(tmpDir, '.env')
+      await writeFile(dotEnvPath, 'FOO="BAR\nBAR\nBAR"\nABC   =XYZ\n#Wow!\n\n  DEF  =GHI')
+
+      // When
+      const got = await readAndParseDotEnv(dotEnvPath)
+      expect(got.variables).toEqual({
+        FOO: 'BAR\nBAR\nBAR',
+        ABC: 'XYZ',
+        DEF: 'GHI',
+      })
+
+      // Then
+      const patchedContent = patchEnvFile(await readFile(dotEnvPath), {FOO: 'BAZ\nBAZ\nBAZ'})
+      expect(patchedContent).toEqual('FOO="BAZ\nBAZ\nBAZ"\nABC   =XYZ\n#Wow!\n\n  DEF  =GHI')
+    })
+  })
+
+  test('patches an environment file and creates a new env var with newline characters', async () => {
+    await inTemporaryDirectory(async (tmpDir) => {
+      // Given
+      const dotEnvPath = joinPath(tmpDir, '.env')
+      await writeFile(dotEnvPath, 'FOO=BAR\nABC   =XYZ\n#Wow!\n\n  DEF  =GHI')
+
+      // When
+      const got = await readAndParseDotEnv(dotEnvPath)
+      expect(got.variables).toEqual({
+        FOO: 'BAR',
+        ABC: 'XYZ',
+        DEF: 'GHI',
+      })
+
+      // Then
+      const patchedContent = patchEnvFile(await readFile(dotEnvPath), {MULTI: 'LINE\nVARIABLE'})
+      expect(patchedContent).toEqual('FOO=BAR\nABC   =XYZ\n#Wow!\n\n  DEF  =GHI\nMULTI="LINE\nVARIABLE"')
+    })
+  })
+
+  test(`throws error when multiline environment variable isn't closed`, async () => {
+    await inTemporaryDirectory(async (tmpDir) => {
+      // Given
+      const dotEnvPath = joinPath(tmpDir, '.env')
+      await writeFile(dotEnvPath, 'FOO=BAR\nABC   ="XYZ\n#Wow!\n\n  DEF  =GHI')
+
+      // Then
+      await expect(async () => {
+        patchEnvFile(await readFile(dotEnvPath), {MULTI: 'LINE\nVARIABLE'})
+      }).rejects.toThrow(`Multi-line environment variable 'ABC' is not properly enclosed.`)
+    })
+  })
+})
+
+describe('createDotEnvFileLine', () => {
+  test('creates an env var for a .env file', () => {
+    const line = createDotEnvFileLine('FOO', 'BAR')
+
+    expect(line).toEqual('FOO=BAR')
+  })
+
+  test('creates a multiline env var for a .env file', () => {
+    const line = createDotEnvFileLine('FOO', 'BAR\nBAR\nBAR')
+
+    expect(line).toEqual('FOO="BAR\nBAR\nBAR"')
+  })
+
+  test('creates a multiline env var for a .env file with double-quotes', () => {
+    const line = createDotEnvFileLine('FOO', 'BAR\n"BAR"\nBAR')
+
+    expect(line).toEqual(`FOO='BAR\n"BAR"\nBAR'`)
+  })
+
+  test('creates a multiline env var for a .env file with double-quotes and single-quotes', () => {
+    const line = createDotEnvFileLine('FOO', `BAR\n"BAR"\n'BAR'`)
+
+    expect(line).toEqual(`FOO=\`BAR\n"BAR"\n'BAR'\``)
+  })
+
+  test('throws AbortError when trying to create a multiline env var with single-quote, double-quote and tilde', async () => {
+    const value = `\`BAR\`\n"BAR"\n'BAR'`
+    await expect(async () => {
+      createDotEnvFileLine('FOO', value)
+    }).rejects.toThrow(`The environment file patch has an env value that can't be surrounded by quotes: ${value}`)
+  })
 })

packages/cli-kit/src/public/node/dot-env.ts

@@ -1,9 +1,7 @@
 import {AbortError} from './error.js'
 import {fileExists, readFile, writeFile} from './fs.js'
 import {outputDebug, outputContent, outputToken} from '../../public/node/output.js'
-// eslint-disable-next-line @typescript-eslint/ban-ts-comment
-// @ts-ignore
-import {parse, stringify} from 'envfile'
+import {parse} from 'dotenv'
 
 /**
  * This interface represents a .env file.
@@ -41,7 +39,11 @@ export async function readAndParseDotEnv(path: string): Promise<DotEnvFile> {
  * @param file - .env file to be written.
  */
 export async function writeDotEnv(file: DotEnvFile): Promise<void> {
-  await writeFile(file.path, stringify(file.variables))
+  const fileContent = Object.entries(file.variables)
+    .map(([key, value]) => createDotEnvFileLine(key, value))
+    .join('\n')
+
+  await writeFile(file.path, fileContent)
 }
 
 /**
@@ -55,33 +57,90 @@ export function patchEnvFile(
   updatedValues: {[key: string]: string | undefined},
 ): string {
   const outputLines: string[] = []
-  const lines = envFileContent === null ? [] : envFileContent.split('\n')
+  const envFileLines = envFileContent === null ? [] : envFileContent.split('\n')
 
   const alreadyPresentKeys: string[] = []
 
-  const toLine = (key: string, value?: string) => `${key}=${value}`
+  let multilineVariable:
+    | {
+        key: string
+        value: string
+        quote: string
+      }
+    | undefined
+
+  for (const line of envFileLines) {
+    if (multilineVariable) {
+      if (line.endsWith(multilineVariable.quote)) {
+        let lineToWrite = createDotEnvFileLine(
+          multilineVariable.key,
+          multilineVariable.value + line.slice(0, -1),
+          multilineVariable.quote,
+        )
+        const newValue = updatedValues[multilineVariable.key]
+        if (newValue) {
+          alreadyPresentKeys.push(multilineVariable.key)
+          lineToWrite = createDotEnvFileLine(multilineVariable.key, newValue)
+        }
+        outputLines.push(lineToWrite)
+        multilineVariable = undefined
+      } else {
+        multilineVariable.value += `${line}\n`
+      }
+      continue
+    }
 
-  for (const line of lines) {
     const match = line.match(/^([^=:#]+?)[=:](.*)/)
     let lineToWrite = line
 
     if (match) {
       const key = match[1]!.trim()
+      const value = (match[2] || '')!.trim()
+
+      if (/^["'`]/.test(value) && !value.endsWith(value[0]!)) {
+        multilineVariable = {
+          key,
+          value: `${value.slice(1)}\n`,
+          quote: value[0]!,
+        }
+        continue
+      }
+
       const newValue = updatedValues[key]
       if (newValue) {
         alreadyPresentKeys.push(key)
-        lineToWrite = toLine(key, newValue)
+        lineToWrite = createDotEnvFileLine(key, newValue)
       }
     }
 
     outputLines.push(lineToWrite)
   }
 
+  if (multilineVariable) {
+    throw new AbortError(`Multi-line environment variable '${multilineVariable.key}' is not properly enclosed.`)
+  }
+
   for (const [patchKey, updatedValue] of Object.entries(updatedValues)) {
     if (!alreadyPresentKeys.includes(patchKey)) {
-      outputLines.push(toLine(patchKey, updatedValue))
+      outputLines.push(createDotEnvFileLine(patchKey, updatedValue))
     }
   }
 
   return outputLines.join('\n')
 }
+
+export function createDotEnvFileLine(key: string, value?: string, quote?: string): string {
+  if (quote) {
+    return `${key}=${quote}${value}${quote}`
+  }
+  if (value && value.includes('\n')) {
+    const quoteCharacter = ['"', "'", '`'].find((char) => !value.includes(char))
+
+    if (!quoteCharacter) {
+      throw new AbortError(`The environment file patch has an env value that can't be surrounded by quotes: ${value}`)
+    }
+
+    return `${key}=${quoteCharacter}${value}${quoteCharacter}`
+  }
+  return `${key}=${value}`
+}