feat(fetch-engine): better debug/errors

WIP

/integration

Related
#20302
#20193

packages/fetch-engine/src/__tests__/download.test.ts

@@ -642,6 +642,118 @@ It took ${timeInMsToDownloadAllFromCache2}ms to execute download() for all binar
     })
   })
 
+  describe('retries', () => {
+    test('if fetching of checksums fails with a non 200 code it retries it 2 more times', async () => {
+      mockFetch.mockImplementation((url, opts) => {
+        if (String(url).endsWith('.sha256')) {
+          return Promise.resolve({
+            ok: false,
+            status: 500,
+            statusText: 'KO',
+          } as Response)
+        }
+        return actualFetch(url, opts)
+      })
+
+      await expect(
+        download({
+          binaries: {
+            [BinaryType.QueryEngineLibrary]: baseDirChecksum,
+          },
+          binaryTargets: ['rhel-openssl-3.0.x'],
+          version: CURRENT_ENGINES_HASH,
+        }),
+      ).rejects.toThrow(
+        `Failed to fetch sha256 checksum at https://binaries.prisma.sh/all_commits/b182127de581d0bf6ec89152a6e2fb4652be0f0a/rhel-openssl-3.0.x/libquery_engine.so.node.gz.sha256. 500 KO`,
+      )
+
+      // Because we try to fetch 2 different checksum files
+      // And there are 2 retries for the checksums
+      // 2 checksums * 3 attempts = 6
+      expect(mockFetch).toHaveBeenCalledTimes(6)
+    })
+
+    test('if fetching of a binary fails with a non 200 code it retries it 2 more times', async () => {
+      mockFetch.mockImplementation((url, opts) => {
+        if (!String(url).endsWith('.sha256')) {
+          return Promise.resolve({
+            ok: false,
+            status: 500,
+            statusText: 'KO',
+          } as Response)
+        }
+        return actualFetch(url, opts)
+      })
+
+      await expect(
+        download({
+          binaries: {
+            [BinaryType.QueryEngineLibrary]: baseDirChecksum,
+          },
+          binaryTargets: ['rhel-openssl-3.0.x'],
+          version: CURRENT_ENGINES_HASH,
+        }),
+      ).rejects.toThrow(
+        `Failed to fetch the engine file at https://binaries.prisma.sh/all_commits/b182127de581d0bf6ec89152a6e2fb4652be0f0a/rhel-openssl-3.0.x/libquery_engine.so.node.gz. 500 KO`,
+      )
+
+      // Because we try to fetch 2 different checksum files before we even start downloading the binaries
+      // And there are 2 retries for the binary
+      // 2 checksums + (1 engine * 3 attempts) = 5
+      expect(mockFetch).toHaveBeenCalledTimes(5)
+    })
+
+    test('if fetching of checksums fails with a timeout it retries it 2 more times', async () => {
+      mockFetch.mockImplementation((url, opts) => {
+        opts = opts || {}
+        // This makes everything fail with a timeout
+        opts.timeout = 1
+        return actualFetch(url, opts)
+      })
+
+      await expect(
+        download({
+          binaries: {
+            [BinaryType.QueryEngineLibrary]: baseDirChecksum,
+          },
+          binaryTargets: [platform],
+          version: CURRENT_ENGINES_HASH,
+        }),
+      ).rejects.toThrow(`network timeout at:`)
+
+      // Because we try to fetch 2 different checksum files
+      // And there are 2 retries for the checksums
+      // 2 checksums * 3 attempts = 6
+      expect(mockFetch).toHaveBeenCalledTimes(6)
+    })
+
+    test('if fetching of a binary fails with a timeout it retries it 2 more times', async () => {
+      mockFetch.mockImplementation((url, opts) => {
+        opts = opts || {}
+        // We only make binaries fail with a timeout, not checksums
+        if (!String(url).endsWith('.sha256')) {
+          opts.timeout = 1
+        }
+        return actualFetch(url, opts)
+      })
+
+      await expect(
+        download({
+          binaries: {
+            [BinaryType.QueryEngineLibrary]: baseDirChecksum,
+          },
+          binaryTargets: [platform],
+          version: CURRENT_ENGINES_HASH,
+        }),
+      ).rejects.toThrow(`network timeout at:`)
+
+      // Because we try to fetch 2 different checksum files before we even start downloading the binaries
+      // And there are 2 retries for the binary
+      // 2 checksums + (1 engine * 3 attempts) = 5
+      expect(mockFetch).toHaveBeenCalledTimes(5)
+    })
+  })
+
   describe('env.PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING=1', () => {
     beforeAll(() => {
       process.env.PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING = '1'

packages/fetch-engine/src/__tests__/getProxyAgent.test.ts

@@ -0,0 +1,93 @@
+import { jestConsoleContext, jestContext } from '@prisma/get-platform'
+
+import { getProxyAgent } from '../getProxyAgent'
+
+const originalEnv = { ...process.env }
+const ctx = jestContext.new().add(jestConsoleContext()).assemble()
+
+describe('getProxyAgent', () => {
+  beforeEach(() => {
+    process.env = { ...originalEnv }
+  })
+  afterAll(() => {
+    process.env = { ...originalEnv }
+  })
+
+  test('no proxy / env vars are set - HTTP', () => {
+    expect(getProxyAgent('http://example.com')).toBeUndefined()
+    expect(ctx.mocked['console.warn'].mock.calls.join('\n')).toBe('')
+  })
+  test('no proxy / env vars are set - HTTPS', () => {
+    expect(getProxyAgent('https://example.com')).toBeUndefined()
+    expect(ctx.mocked['console.warn'].mock.calls.join('\n')).toBe('')
+  })
+
+  test('should not use a proxy with NO_PROXY & HTTPS_PROXY set - HTTPS', () => {
+    process.env.NO_PROXY = 'example.com'
+    process.env.HTTPS_PROXY = 'proxy.example.com'
+    expect(getProxyAgent('https://example.com')).toBeUndefined()
+    expect(ctx.mocked['console.warn'].mock.calls.join('\n')).toBe('')
+  })
+
+  // Lowercase env vars
+  test('should warn when http_proxy is not a valid URL - HTTP', () => {
+    process.env.http_proxy = 'proxy.example.com'
+    expect(getProxyAgent('http://example.com')).toBeUndefined()
+    expect(ctx.mocked['console.warn'].mock.calls.join('\n')).toMatchInlineSnapshot(`
+      "An error occurred in getProxyAgent(), no proxy agent will be used.,Error: Error while instantiating HttpProxyAgent with URL: "proxy.example.com"
+      TypeError [ERR_INVALID_URL]: Invalid URL
+      Check the following env vars "http_proxy" or "HTTP_PROXY". The value should be a valid URL starting with "http://""
+    `)
+  })
+  test('should warn when https_proxy is not a valid URL - HTTPS', () => {
+    process.env.https_proxy = 'proxy.example.com'
+    expect(getProxyAgent('https://example.com')).toBeUndefined()
+    expect(ctx.mocked['console.warn'].mock.calls.join('\n')).toMatchInlineSnapshot(`
+      "An error occurred in getProxyAgent(), no proxy agent will be used.,Error: Error while instantiating HttpsProxyAgent with URL: "proxy.example.com"
+      TypeError [ERR_INVALID_URL]: Invalid URL
+      Check the following env vars "https_proxy" or "HTTPS_PROXY". The value should be a valid URL starting with "https://""
+    `)
+  })
+  // Uppercase env vars
+  test('should warn when HTTP_PROXY is not a valid URL - HTTP', () => {
+    process.env.HTTP_PROXY = 'proxy.example.com'
+    expect(getProxyAgent('http://example.com')).toBeUndefined()
+    expect(ctx.mocked['console.warn'].mock.calls.join('\n')).toMatchInlineSnapshot(`
+      "An error occurred in getProxyAgent(), no proxy agent will be used.,Error: Error while instantiating HttpProxyAgent with URL: "proxy.example.com"
+      TypeError [ERR_INVALID_URL]: Invalid URL
+      Check the following env vars "http_proxy" or "HTTP_PROXY". The value should be a valid URL starting with "http://""
+    `)
+  })
+  test('should warn when HTTPS_PROXY is not a valid URL - HTTPS', () => {
+    process.env.HTTPS_PROXY = 'proxy.example.com'
+    expect(getProxyAgent('https://example.com')).toBeUndefined()
+    expect(ctx.mocked['console.warn'].mock.calls.join('\n')).toMatchInlineSnapshot(`
+      "An error occurred in getProxyAgent(), no proxy agent will be used.,Error: Error while instantiating HttpsProxyAgent with URL: "proxy.example.com"
+      TypeError [ERR_INVALID_URL]: Invalid URL
+      Check the following env vars "https_proxy" or "HTTPS_PROXY". The value should be a valid URL starting with "https://""
+    `)
+  })
+
+  // Lowercase env vars
+  test('should use a proxy with http_proxy set - HTTP', () => {
+    process.env.http_proxy = 'http://proxy.example.com'
+    expect(getProxyAgent('http://example.com')?.proxy.toString()).toEqual('http://proxy.example.com/')
+    expect(ctx.mocked['console.warn'].mock.calls.join('\n')).toBe('')
+  })
+  test('should use a proxy with https_proxy set - HTTPS', () => {
+    process.env.https_proxy = 'https://proxy.example.com'
+    expect(getProxyAgent('https://example.com')?.proxy.toString()).toEqual('https://proxy.example.com/')
+    expect(ctx.mocked['console.warn'].mock.calls.join('\n')).toBe('')
+  })
+  // Uppercase env vars
+  test('should use a proxy with HTTP_PROX set - HTTP', () => {
+    process.env.HTTP_PROXY = 'http://proxy.example.com'
+    expect(getProxyAgent('http://example.com')?.proxy.toString()).toEqual('http://proxy.example.com/')
+    expect(ctx.mocked['console.warn'].mock.calls.join('\n')).toBe('')
+  })
+  test('should use a proxy with HTTPS_PROXY set - HTTPS', () => {
+    process.env.HTTPS_PROXY = 'https://proxy.example.com'
+    expect(getProxyAgent('https://example.com')?.proxy.toString()).toEqual('https://proxy.example.com/')
+    expect(ctx.mocked['console.warn'].mock.calls.join('\n')).toBe('')
+  })
+})

packages/fetch-engine/src/download.ts

@@ -20,7 +20,7 @@ import { getCacheDir, getDownloadUrl, overwriteFile } from './utils'
 
 const { enginesOverride } = require('../package.json')
 
-const debug = Debug('prisma:download')
+const debug = Debug('prisma:fetch-engine:download')
 const exists = promisify(fs.exists)
 
 const channel = 'master'
@@ -113,6 +113,7 @@ export async function download(options: DownloadOptions): Promise<BinaryPaths> {
   )
 
   if (process.env.BINARY_DOWNLOAD_VERSION) {
+    debug(`process.env.BINARY_DOWNLOAD_VERSION is set to "${process.env.BINARY_DOWNLOAD_VERSION}"`)
     opts.version = process.env.BINARY_DOWNLOAD_VERSION
   }
 
@@ -146,16 +147,26 @@ export async function download(options: DownloadOptions): Promise<BinaryPaths> {
       setProgress = collectiveBar.setProgress
     }
 
-    await Promise.all(
-      binariesToDownload.map((job) =>
-        downloadBinary({
-          ...job,
-          version: opts.version,
-          failSilent: opts.failSilent,
-          progressCb: setProgress ? setProgress(job.targetFilePath) : undefined,
-        }),
-      ),
-    )
+    const promises = binariesToDownload.map((job) => {
+      const downloadUrl = getDownloadUrl({
+        channel: 'all_commits',
+        version: opts.version,
+        platform: job.binaryTarget,
+        binaryName: job.binaryName,
+      })
+
+      debug(`${downloadUrl} will be downloaded to ${job.targetFilePath}`)
+
+      return downloadBinary({
+        ...job,
+        downloadUrl,
+        version: opts.version,
+        failSilent: opts.failSilent,
+        progressCb: setProgress ? setProgress(job.targetFilePath) : undefined,
+      })
+    })
+
+    await Promise.all(promises)
 
     await cleanupPromise // make sure, that cleanup finished
     if (finishBar) {
@@ -376,13 +387,13 @@ async function getCachedBinaryPath({
 
 type DownloadBinaryOptions = BinaryDownloadJob & {
   version: string
+  downloadUrl: string
   progressCb?: (progress: number) => void
   failSilent?: boolean
 }
 
 async function downloadBinary(options: DownloadBinaryOptions): Promise<void> {
-  const { version, progressCb, targetFilePath, binaryTarget, binaryName } = options
-  const downloadUrl = getDownloadUrl('all_commits', version, binaryTarget, binaryName)
+  const { version, progressCb, targetFilePath, downloadUrl } = options
 
   const targetDir = path.dirname(targetFilePath)
 
@@ -397,7 +408,7 @@ async function downloadBinary(options: DownloadBinaryOptions): Promise<void> {
     }
   }
 
-  debug(`Downloading ${downloadUrl} to ${targetFilePath}`)
+  debug(`Downloading ${downloadUrl} to ${targetFilePath} ...`)
 
   if (progressCb) {
     progressCb(0)