From 551970475faf9c94961655dd154b6732def31eca Mon Sep 17 00:00:00 2001 From: timsuchanek Date: Thu, 20 Feb 2020 11:19:49 +0100 Subject: [PATCH] add download tests --- packages/fetch-engine/package.json | 2 + .../fetch-engine/src/__tests__/.gitignore | 1 + .../fetch-engine/src/__tests__/all/.gitkeep | 0 .../src/__tests__/corruption/.gitkeep | 0 .../src/__tests__/download.test.ts | 217 +++++++++++++++++- packages/fetch-engine/src/cleanupCache.ts | 4 +- packages/fetch-engine/src/download.ts | 27 +-- packages/get-platform/src/index.ts | 1 + packages/get-platform/src/platforms.ts | 9 + 9 files changed, 244 insertions(+), 17 deletions(-) create mode 100644 packages/fetch-engine/src/__tests__/.gitignore create mode 100644 packages/fetch-engine/src/__tests__/all/.gitkeep create mode 100644 packages/fetch-engine/src/__tests__/corruption/.gitkeep diff --git a/packages/fetch-engine/package.json b/packages/fetch-engine/package.json index 67720b1d..d0f461a1 100644 --- a/packages/fetch-engine/package.json +++ b/packages/fetch-engine/package.json @@ -11,8 +11,10 @@ "@types/node": "^12.12.25", "@types/node-fetch": "^2.5.4", "@types/progress": "^2.0.3", + "del": "^5.1.0", "jest": "^25.1.0", "ncc": "^0.3.6", + "prettier": "^1.19.1", "ts-jest": "^25.2.1", "typescript": "^3.7.5" }, diff --git a/packages/fetch-engine/src/__tests__/.gitignore b/packages/fetch-engine/src/__tests__/.gitignore new file mode 100644 index 00000000..83a45e5e --- /dev/null +++ b/packages/fetch-engine/src/__tests__/.gitignore @@ -0,0 +1 @@ +*engine* \ No newline at end of file diff --git a/packages/fetch-engine/src/__tests__/all/.gitkeep b/packages/fetch-engine/src/__tests__/all/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/packages/fetch-engine/src/__tests__/corruption/.gitkeep b/packages/fetch-engine/src/__tests__/corruption/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/packages/fetch-engine/src/__tests__/download.test.ts b/packages/fetch-engine/src/__tests__/download.test.ts index 1f6beb46..cedbf726 100644 --- a/packages/fetch-engine/src/__tests__/download.test.ts +++ b/packages/fetch-engine/src/__tests__/download.test.ts @@ -1,9 +1,19 @@ import fs from 'fs' import path from 'path' -import { download, getBinaryName } from '../download' +import { download, getBinaryName, checkVersionCommand } from '../download' import { getPlatform } from '@prisma/get-platform' +import { cleanupCache } from '../cleanupCache' +import del from 'del' + +jest.setTimeout(20000) describe('download', () => { + beforeAll(async () => { + // completely clean up the cache and keep nothing + await cleanupCache(0) + await del(__dirname + '/**/*engine*') + }) + test('basic download', async () => { const platform = await getPlatform() const targetPath = path.join(__dirname, getBinaryName('query-engine', platform)) @@ -14,12 +24,215 @@ describe('download', () => { console.error(e) } } + await download({ binaries: { 'query-engine': __dirname, }, + version: 'a78fee833bcf4e202645e7cc7df5c3839f658e6a', }) - expect(fs.existsSync(targetPath)) + expect(fs.existsSync(targetPath)).toBe(true) + + expect(await checkVersionCommand(targetPath)).toBe(true) + }) + + test('auto heal corrupt binary', async () => { + const platform = await getPlatform() + const baseDir = path.join(__dirname, 'corruption') + const targetPath = path.join(baseDir, getBinaryName('query-engine', platform)) + if (fs.existsSync(targetPath)) { + try { + fs.unlinkSync(targetPath) + } catch (e) { + console.error(e) + } + } + + await download({ + binaries: { + 'query-engine': baseDir, + }, + version: 'd20d1e6b1525ae45e3cc39784ad16c97d463f61c', + }) + + fs.writeFileSync(targetPath, 'incorrect-binary') + + // please heal it + await download({ + binaries: { + 'query-engine': baseDir, + }, + version: 'd20d1e6b1525ae45e3cc39784ad16c97d463f61c', + }) + + expect(fs.existsSync(targetPath)).toBe(true) + + expect(await checkVersionCommand(targetPath)).toBe(true) + }) + + test('handle non-existent binary target', async () => { + expect( + download({ + binaries: { + 'query-engine': __dirname, + }, + version: 'd20d1e6b1525ae45e3cc39784ad16c97d463f61c', + binaryTargets: ['darwin', 'marvin'] as any, + }), + ).rejects.toThrowErrorMatchingInlineSnapshot(`"Unknown binaryTargets marvin"`) + }) + + test('download all binaries & cache them', async () => { + const baseDir = path.join(__dirname, 'all') + await download({ + binaries: { + 'query-engine': baseDir, + 'introspection-engine': baseDir, + 'migration-engine': baseDir, + }, + binaryTargets: [ + 'darwin', + 'debian-openssl-1.0.x', + 'debian-openssl-1.1.x', + 'rhel-openssl-1.0.x', + 'rhel-openssl-1.1.x', + 'windows', + ], + version: 'd20d1e6b1525ae45e3cc39784ad16c97d463f61c', + }) + const files = getFiles(baseDir) + expect(files).toMatchInlineSnapshot(` + Array [ + Object { + "name": ".gitkeep", + "size": 0, + }, + Object { + "name": "introspection-engine-darwin", + "size": 10818024, + }, + Object { + "name": "introspection-engine-debian-openssl-1.0.x", + "size": 13686432, + }, + Object { + "name": "introspection-engine-debian-openssl-1.1.x", + "size": 13672616, + }, + Object { + "name": "introspection-engine-rhel-openssl-1.0.x", + "size": 13727413, + }, + Object { + "name": "introspection-engine-rhel-openssl-1.1.x", + "size": 13714821, + }, + Object { + "name": "introspection-engine-windows.exe", + "size": 22851591, + }, + Object { + "name": "migration-engine-darwin", + "size": 14451544, + }, + Object { + "name": "migration-engine-debian-openssl-1.0.x", + "size": 17538960, + }, + Object { + "name": "migration-engine-debian-openssl-1.1.x", + "size": 17529464, + }, + Object { + "name": "migration-engine-rhel-openssl-1.0.x", + "size": 17592124, + }, + Object { + "name": "migration-engine-rhel-openssl-1.1.x", + "size": 17588083, + }, + Object { + "name": "migration-engine-windows.exe", + "size": 27487354, + }, + Object { + "name": "query-engine-darwin", + "size": 16302864, + }, + Object { + "name": "query-engine-debian-openssl-1.0.x", + "size": 19595240, + }, + Object { + "name": "query-engine-debian-openssl-1.1.x", + "size": 19576464, + }, + Object { + "name": "query-engine-rhel-openssl-1.0.x", + "size": 19635768, + }, + Object { + "name": "query-engine-rhel-openssl-1.1.x", + "size": 19622446, + }, + Object { + "name": "query-engine-windows.exe", + "size": 29855371, + }, + ] + `) + await del(baseDir + '/*engine*') + const before = Date.now() + await download({ + binaries: { + 'query-engine': baseDir, + 'introspection-engine': baseDir, + 'migration-engine': baseDir, + }, + binaryTargets: [ + 'darwin', + 'debian-openssl-1.0.x', + 'debian-openssl-1.1.x', + 'rhel-openssl-1.0.x', + 'rhel-openssl-1.1.x', + 'windows', + ], + version: 'd20d1e6b1525ae45e3cc39784ad16c97d463f61c', + }) + const after = Date.now() + // cache should take less than 2s + // value on Mac: 1440 + expect(after - before).toBeLessThan(2000) + const before2 = Date.now() + await download({ + binaries: { + 'query-engine': baseDir, + 'introspection-engine': baseDir, + 'migration-engine': baseDir, + }, + binaryTargets: [ + 'darwin', + 'debian-openssl-1.0.x', + 'debian-openssl-1.1.x', + 'rhel-openssl-1.0.x', + 'rhel-openssl-1.1.x', + 'windows', + ], + version: 'd20d1e6b1525ae45e3cc39784ad16c97d463f61c', + }) + const after2 = Date.now() + // if binaries are already there, it should take less than 100ms to check all of them + // value on Mac: 33ms + expect(after2 - before2).toBeLessThan(100) }) }) + +function getFiles(dir: string): Array<{ name: string; size: number }> { + const files = fs.readdirSync(dir, 'utf8') + return files.map(name => { + const size = fs.statSync(path.join(dir, name)).size + + return { name, size } + }) +} diff --git a/packages/fetch-engine/src/cleanupCache.ts b/packages/fetch-engine/src/cleanupCache.ts index 58c66692..c91d7b4f 100644 --- a/packages/fetch-engine/src/cleanupCache.ts +++ b/packages/fetch-engine/src/cleanupCache.ts @@ -10,7 +10,7 @@ const del = promisify(rimraf) const readdir = promisify(fs.readdir) const stat = promisify(fs.stat) -export async function cleanupCache() { +export async function cleanupCache(n: number = 5) { try { const rootCacheDir = await getRootCacheDir() const channels = ['master', 'alpha'] @@ -30,7 +30,7 @@ export async function cleanupCache() { }), ) dirsWithMeta.sort((a, b) => (a.created < b.created ? 1 : -1)) - const dirsToRemove = dirsWithMeta.slice(5) + const dirsToRemove = dirsWithMeta.slice(n) await pMap(dirsToRemove, dir => del(dir.dir), { concurrency: 20 }) } } catch (e) { diff --git a/packages/fetch-engine/src/download.ts b/packages/fetch-engine/src/download.ts index 04b931e3..28ff37a6 100644 --- a/packages/fetch-engine/src/download.ts +++ b/packages/fetch-engine/src/download.ts @@ -13,7 +13,7 @@ import pFilter from 'p-filter' import { getBar } from './log' import plusxSync from './chmod' import { copy } from './copy' -import { getPlatform, Platform } from '@prisma/get-platform' +import { getPlatform, Platform, platforms } from '@prisma/get-platform' import { downloadZip } from './downloadZip' import { getCacheDir, getLocalLastModified, getRemoteLastModified, getDownloadUrl } from './util' import { cleanupCache } from './cleanupCache' @@ -71,6 +71,13 @@ export async function download(options: DownloadOptions): Promise { return {} } + if (options.binaryTargets && Array.isArray(options.binaryTargets)) { + const unknownTargets = options.binaryTargets.filter(t => !platforms.includes(t)) + if (unknownTargets.length > 0) { + throw new Error(`Unknown binaryTargets ${unknownTargets.join(', ')}`) + } + } + // merge options options = { binaryTargets: [platform], @@ -81,7 +88,7 @@ export async function download(options: DownloadOptions): Promise { const binaryJobs: Array = flatMap(Object.entries(options.binaries), ([binaryName, targetFolder]) => options.binaryTargets.map(binaryTarget => { - const fileName = getBinaryName(binaryName, platform) + const fileName = getBinaryName(binaryName, binaryTarget) return { binaryName, targetFolder, @@ -227,7 +234,7 @@ async function binaryNeedsToBeDownloaded( // 3. If same platform, always check --version if (job.binaryTarget === nativePlatform) { - const works = await versionCommandWorks(job.targetFilePath) + const works = await checkVersionCommand(job.targetFilePath) debug({ works }) return !works } @@ -235,7 +242,7 @@ async function binaryNeedsToBeDownloaded( return false } -export async function versionCommandWorks(enginePath: string): Promise { +export async function checkVersionCommand(enginePath: string): Promise { try { const result = await execa(enginePath, ['--version']) @@ -332,17 +339,10 @@ async function downloadBinary(options: DownloadBinaryOptions) { const downloadUrl = getDownloadUrl(channel, version, binaryTarget, binaryName) const targetDir = path.dirname(targetFilePath) - try { - await makeDir(targetDir) - } catch (e) { - if (failSilent) { - return - } else { - throw e - } - } + try { fs.accessSync(targetDir, fs.constants.W_OK) + await makeDir(targetDir) } catch (e) { if (options.failSilent || e.code !== 'EACCES') { return @@ -350,6 +350,7 @@ async function downloadBinary(options: DownloadBinaryOptions) { throw new Error(`Can't write to ${targetDir} please make sure you install "prisma2" with the right permissions.`) } } + debug(`Downloading ${downloadUrl} to ${targetFilePath}`) if (progressCb) { diff --git a/packages/get-platform/src/index.ts b/packages/get-platform/src/index.ts index f204957e..bcbecae3 100644 --- a/packages/get-platform/src/index.ts +++ b/packages/get-platform/src/index.ts @@ -1,3 +1,4 @@ export { getPlatform } from './getPlatform' export { Platform } from './platforms' export { mayBeCompatible } from './platforms' +export { platforms } from './platforms' diff --git a/packages/get-platform/src/platforms.ts b/packages/get-platform/src/platforms.ts index 0a742ce7..beda6a99 100644 --- a/packages/get-platform/src/platforms.ts +++ b/packages/get-platform/src/platforms.ts @@ -7,6 +7,15 @@ export type Platform = | 'rhel-openssl-1.1.x' | 'windows' +export const platforms = [ + 'darwin', + 'debian-openssl-1.0.x', + 'debian-openssl-1.1.x', + 'rhel-openssl-1.0.x', + 'rhel-openssl-1.1.x', + 'windows', +] + export function mayBeCompatible( platformA: Platform, platformB: Platform,