add download tests

packages/fetch-engine/package.json

@@ -11,8 +11,10 @@
     "@types/node": "^12.12.25",
     "@types/node-fetch": "^2.5.4",
     "@types/progress": "^2.0.3",
+    "del": "^5.1.0",
     "jest": "^25.1.0",
     "ncc": "^0.3.6",
+    "prettier": "^1.19.1",
     "ts-jest": "^25.2.1",
     "typescript": "^3.7.5"
   },

packages/fetch-engine/src/__tests__/.gitignore

@@ -0,0 +1 @@
+*engine*

packages/fetch-engine/src/__tests__/download.test.ts

@@ -1,9 +1,19 @@
 import fs from 'fs'
 import path from 'path'
-import { download, getBinaryName } from '../download'
+import { download, getBinaryName, checkVersionCommand } from '../download'
 import { getPlatform } from '@prisma/get-platform'
+import { cleanupCache } from '../cleanupCache'
+import del from 'del'
+
+jest.setTimeout(20000)
 
 describe('download', () => {
+  beforeAll(async () => {
+    // completely clean up the cache and keep nothing
+    await cleanupCache(0)
+    await del(__dirname + '/**/*engine*')
+  })
+
   test('basic download', async () => {
     const platform = await getPlatform()
     const targetPath = path.join(__dirname, getBinaryName('query-engine', platform))
@@ -14,12 +24,215 @@ describe('download', () => {
         console.error(e)
       }
     }
+
     await download({
       binaries: {
         'query-engine': __dirname,
       },
+      version: 'a78fee833bcf4e202645e7cc7df5c3839f658e6a',
     })
 
-    expect(fs.existsSync(targetPath))
+    expect(fs.existsSync(targetPath)).toBe(true)
+
+    expect(await checkVersionCommand(targetPath)).toBe(true)
+  })
+
+  test('auto heal corrupt binary', async () => {
+    const platform = await getPlatform()
+    const baseDir = path.join(__dirname, 'corruption')
+    const targetPath = path.join(baseDir, getBinaryName('query-engine', platform))
+    if (fs.existsSync(targetPath)) {
+      try {
+        fs.unlinkSync(targetPath)
+      } catch (e) {
+        console.error(e)
+      }
+    }
+
+    await download({
+      binaries: {
+        'query-engine': baseDir,
+      },
+      version: 'd20d1e6b1525ae45e3cc39784ad16c97d463f61c',
+    })
+
+    fs.writeFileSync(targetPath, 'incorrect-binary')
+
+    // please heal it
+    await download({
+      binaries: {
+        'query-engine': baseDir,
+      },
+      version: 'd20d1e6b1525ae45e3cc39784ad16c97d463f61c',
+    })
+
+    expect(fs.existsSync(targetPath)).toBe(true)
+
+    expect(await checkVersionCommand(targetPath)).toBe(true)
+  })
+
+  test('handle non-existent binary target', async () => {
+    expect(
+      download({
+        binaries: {
+          'query-engine': __dirname,
+        },
+        version: 'd20d1e6b1525ae45e3cc39784ad16c97d463f61c',
+        binaryTargets: ['darwin', 'marvin'] as any,
+      }),
+    ).rejects.toThrowErrorMatchingInlineSnapshot(`"Unknown binaryTargets marvin"`)
+  })
+
+  test('download all binaries & cache them', async () => {
+    const baseDir = path.join(__dirname, 'all')
+    await download({
+      binaries: {
+        'query-engine': baseDir,
+        'introspection-engine': baseDir,
+        'migration-engine': baseDir,
+      },
+      binaryTargets: [
+        'darwin',
+        'debian-openssl-1.0.x',
+        'debian-openssl-1.1.x',
+        'rhel-openssl-1.0.x',
+        'rhel-openssl-1.1.x',
+        'windows',
+      ],
+      version: 'd20d1e6b1525ae45e3cc39784ad16c97d463f61c',
+    })
+    const files = getFiles(baseDir)
+    expect(files).toMatchInlineSnapshot(`
+      Array [
+        Object {
+          "name": ".gitkeep",
+          "size": 0,
+        },
+        Object {
+          "name": "introspection-engine-darwin",
+          "size": 10818024,
+        },
+        Object {
+          "name": "introspection-engine-debian-openssl-1.0.x",
+          "size": 13686432,
+        },
+        Object {
+          "name": "introspection-engine-debian-openssl-1.1.x",
+          "size": 13672616,
+        },
+        Object {
+          "name": "introspection-engine-rhel-openssl-1.0.x",
+          "size": 13727413,
+        },
+        Object {
+          "name": "introspection-engine-rhel-openssl-1.1.x",
+          "size": 13714821,
+        },
+        Object {
+          "name": "introspection-engine-windows.exe",
+          "size": 22851591,
+        },
+        Object {
+          "name": "migration-engine-darwin",
+          "size": 14451544,
+        },
+        Object {
+          "name": "migration-engine-debian-openssl-1.0.x",
+          "size": 17538960,
+        },
+        Object {
+          "name": "migration-engine-debian-openssl-1.1.x",
+          "size": 17529464,
+        },
+        Object {
+          "name": "migration-engine-rhel-openssl-1.0.x",
+          "size": 17592124,
+        },
+        Object {
+          "name": "migration-engine-rhel-openssl-1.1.x",
+          "size": 17588083,
+        },
+        Object {
+          "name": "migration-engine-windows.exe",
+          "size": 27487354,
+        },
+        Object {
+          "name": "query-engine-darwin",
+          "size": 16302864,
+        },
+        Object {
+          "name": "query-engine-debian-openssl-1.0.x",
+          "size": 19595240,
+        },
+        Object {
+          "name": "query-engine-debian-openssl-1.1.x",
+          "size": 19576464,
+        },
+        Object {
+          "name": "query-engine-rhel-openssl-1.0.x",
+          "size": 19635768,
+        },
+        Object {
+          "name": "query-engine-rhel-openssl-1.1.x",
+          "size": 19622446,
+        },
+        Object {
+          "name": "query-engine-windows.exe",
+          "size": 29855371,
+        },
+      ]
+    `)
+    await del(baseDir + '/*engine*')
+    const before = Date.now()
+    await download({
+      binaries: {
+        'query-engine': baseDir,
+        'introspection-engine': baseDir,
+        'migration-engine': baseDir,
+      },
+      binaryTargets: [
+        'darwin',
+        'debian-openssl-1.0.x',
+        'debian-openssl-1.1.x',
+        'rhel-openssl-1.0.x',
+        'rhel-openssl-1.1.x',
+        'windows',
+      ],
+      version: 'd20d1e6b1525ae45e3cc39784ad16c97d463f61c',
+    })
+    const after = Date.now()
+    // cache should take less than 2s
+    // value on Mac: 1440
+    expect(after - before).toBeLessThan(2000)
+    const before2 = Date.now()
+    await download({
+      binaries: {
+        'query-engine': baseDir,
+        'introspection-engine': baseDir,
+        'migration-engine': baseDir,
+      },
+      binaryTargets: [
+        'darwin',
+        'debian-openssl-1.0.x',
+        'debian-openssl-1.1.x',
+        'rhel-openssl-1.0.x',
+        'rhel-openssl-1.1.x',
+        'windows',
+      ],
+      version: 'd20d1e6b1525ae45e3cc39784ad16c97d463f61c',
+    })
+    const after2 = Date.now()
+    // if binaries are already there, it should take less than 100ms to check all of them
+    // value on Mac: 33ms
+    expect(after2 - before2).toBeLessThan(100)
   })
 })
+
+function getFiles(dir: string): Array<{ name: string; size: number }> {
+  const files = fs.readdirSync(dir, 'utf8')
+  return files.map(name => {
+    const size = fs.statSync(path.join(dir, name)).size
+
+    return { name, size }
+  })
+}

packages/fetch-engine/src/cleanupCache.ts

@@ -10,7 +10,7 @@ const del = promisify(rimraf)
 const readdir = promisify(fs.readdir)
 const stat = promisify(fs.stat)
 
-export async function cleanupCache() {
+export async function cleanupCache(n: number = 5) {
   try {
     const rootCacheDir = await getRootCacheDir()
     const channels = ['master', 'alpha']
@@ -30,7 +30,7 @@ export async function cleanupCache() {
         }),
       )
       dirsWithMeta.sort((a, b) => (a.created < b.created ? 1 : -1))
-      const dirsToRemove = dirsWithMeta.slice(5)
+      const dirsToRemove = dirsWithMeta.slice(n)
       await pMap(dirsToRemove, dir => del(dir.dir), { concurrency: 20 })
     }
   } catch (e) {

packages/fetch-engine/src/download.ts

@@ -13,7 +13,7 @@ import pFilter from 'p-filter'
 import { getBar } from './log'
 import plusxSync from './chmod'
 import { copy } from './copy'
-import { getPlatform, Platform } from '@prisma/get-platform'
+import { getPlatform, Platform, platforms } from '@prisma/get-platform'
 import { downloadZip } from './downloadZip'
 import { getCacheDir, getLocalLastModified, getRemoteLastModified, getDownloadUrl } from './util'
 import { cleanupCache } from './cleanupCache'
@@ -71,6 +71,13 @@ export async function download(options: DownloadOptions): Promise<BinaryPaths> {
     return {}
   }
 
+  if (options.binaryTargets && Array.isArray(options.binaryTargets)) {
+    const unknownTargets = options.binaryTargets.filter(t => !platforms.includes(t))
+    if (unknownTargets.length > 0) {
+      throw new Error(`Unknown binaryTargets ${unknownTargets.join(', ')}`)
+    }
+  }
+
   // merge options
   options = {
     binaryTargets: [platform],
@@ -81,7 +88,7 @@ export async function download(options: DownloadOptions): Promise<BinaryPaths> {
 
   const binaryJobs: Array<BinaryDownloadJob> = flatMap(Object.entries(options.binaries), ([binaryName, targetFolder]) =>
     options.binaryTargets.map(binaryTarget => {
-      const fileName = getBinaryName(binaryName, platform)
+      const fileName = getBinaryName(binaryName, binaryTarget)
       return {
         binaryName,
         targetFolder,
@@ -227,15 +234,15 @@ async function binaryNeedsToBeDownloaded(
 
   // 3. If same platform, always check --version
   if (job.binaryTarget === nativePlatform) {
-    const works = await versionCommandWorks(job.targetFilePath)
+    const works = await checkVersionCommand(job.targetFilePath)
     debug({ works })
     return !works
   }
 
   return false
 }
 
-export async function versionCommandWorks(enginePath: string): Promise<boolean> {
+export async function checkVersionCommand(enginePath: string): Promise<boolean> {
   try {
     const result = await execa(enginePath, ['--version'])
 
@@ -332,24 +339,18 @@ async function downloadBinary(options: DownloadBinaryOptions) {
   const downloadUrl = getDownloadUrl(channel, version, binaryTarget, binaryName)
 
   const targetDir = path.dirname(targetFilePath)
-  try {
-    await makeDir(targetDir)
-  } catch (e) {
-    if (failSilent) {
-      return
-    } else {
-      throw e
-    }
-  }
+
   try {
     fs.accessSync(targetDir, fs.constants.W_OK)
+    await makeDir(targetDir)
   } catch (e) {
     if (options.failSilent || e.code !== 'EACCES') {
       return
     } else {
       throw new Error(`Can't write to ${targetDir} please make sure you install "prisma2" with the right permissions.`)
     }
   }
+
   debug(`Downloading ${downloadUrl} to ${targetFilePath}`)
 
   if (progressCb) {

packages/get-platform/src/index.ts

@@ -1,3 +1,4 @@
 export { getPlatform } from './getPlatform'
 export { Platform } from './platforms'
 export { mayBeCompatible } from './platforms'
+export { platforms } from './platforms'

packages/get-platform/src/platforms.ts

@@ -7,6 +7,15 @@ export type Platform =
   | 'rhel-openssl-1.1.x'
   | 'windows'
 
+export const platforms = [
+  'darwin',
+  'debian-openssl-1.0.x',
+  'debian-openssl-1.1.x',
+  'rhel-openssl-1.0.x',
+  'rhel-openssl-1.1.x',
+  'windows',
+]
+
 export function mayBeCompatible(
   platformA: Platform,
   platformB: Platform,