chore(internal): improve ecosystem tests (#761)

.gitignore

@@ -5,3 +5,8 @@ dist
 /deno
 /*.tgz
 .idea/
+tmp
+.pack
+ecosystem-tests/deno/package.json
+ecosystem-tests/*/openai.tgz
+

.prettierignore

@@ -1,5 +1,5 @@
 CHANGELOG.md
-/ecosystem-tests
+/ecosystem-tests/*/**
 /node_modules
 /deno
 

ecosystem-tests/cli.ts

@@ -5,16 +5,19 @@ import assert from 'assert';
 import path from 'path';
 
 const TAR_NAME = 'openai.tgz';
-const PACK_FILE = `.pack/${TAR_NAME}`;
+const PACK_FOLDER = '.pack';
+const PACK_FILE = `${PACK_FOLDER}/${TAR_NAME}`;
 const IS_CI = Boolean(process.env['CI'] && process.env['CI'] !== 'false');
 
 async function defaultNodeRunner() {
   await installPackage();
   await run('npm', ['run', 'tsc']);
-  if (state.live) await run('npm', ['test']);
+  if (state.live) {
+    await run('npm', ['test']);
+  }
 }
 
-const projects = {
+const projectRunners = {
   'node-ts-cjs': defaultNodeRunner,
   'node-ts-cjs-web': defaultNodeRunner,
   'node-ts-cjs-auto': defaultNodeRunner,
@@ -76,30 +79,17 @@ const projects = {
     }
   },
   deno: async () => {
+    // we don't need to explicitly install the package here
+    // because our deno setup relies on `rootDir/deno` to exist
+    // which is an artifact produced from our build process
     await run('deno', ['task', 'install']);
-    await installPackage();
-    const packFile = getPackFile();
-
-    const openaiDir = path.resolve(
-      process.cwd(),
-      'node_modules',
-      '.deno',
-      'openai@3.3.0',
-      'node_modules',
-      'openai',
-    );
-
-    await run('sh', ['-c', 'rm -rf *'], { cwd: openaiDir, stdio: 'inherit' });
-    await run('tar', ['xzf', path.resolve(packFile)], { cwd: openaiDir, stdio: 'inherit' });
-    await run('sh', ['-c', 'mv package/* .'], { cwd: openaiDir, stdio: 'inherit' });
-    await run('sh', ['-c', 'rm -rf package'], { cwd: openaiDir, stdio: 'inherit' });
-
     await run('deno', ['task', 'check']);
+
     if (state.live) await run('deno', ['task', 'test']);
   },
 };
 
-const projectNames = Object.keys(projects) as Array<keyof typeof projects>;
+let projectNames = Object.keys(projectRunners) as Array<keyof typeof projectRunners>;
 const projectNamesSet = new Set(projectNames);
 
 function parseArgs() {
@@ -118,6 +108,11 @@ function parseArgs() {
         type: 'boolean',
         default: false,
       },
+      skip: {
+        type: 'array',
+        default: [],
+        description: 'Skip one or more projects. Separate project names with a space.',
+      },
       skipPack: {
         type: 'boolean',
         default: false,
@@ -156,6 +151,10 @@ function parseArgs() {
         default: false,
         description: 'run all projects in parallel (jobs = # projects)',
       },
+      noCleanup: {
+        type: 'boolean',
+        default: false,
+      },
     })
     .help().argv;
 }
@@ -165,9 +164,32 @@ type Args = Awaited<ReturnType<typeof parseArgs>>;
 let state: Args & { rootDir: string };
 
 async function main() {
+  if (!process.env['OPENAI_API_KEY']) {
+    console.error(`Error: The environment variable OPENAI_API_KEY must be set. Run the command
+  $echo 'OPENAI_API_KEY = "'"\${OPENAI_API_KEY}"'"' >> ecosystem-tests/cloudflare-worker/wrangler.toml`);
+    process.exit(0);
+  }
+
   const args = (await parseArgs()) as Args;
   console.error(`args:`, args);
 
+  // Some projects, e.g. Deno can be slow to run, so offer the option to skip them. Example:
+  //   --skip=deno node-ts-cjs
+  if (args.skip.length > 0) {
+    args.skip.forEach((projectName, idx) => {
+      // Ensure the inputted project name is lower case
+      args.skip[idx] = (projectName + '').toLowerCase();
+    });
+
+    projectNames = projectNames.filter((projectName) => (args.skip as string[]).indexOf(projectName) < 0);
+
+    args.skip.forEach((projectName) => {
+      projectNamesSet.delete(projectName as any);
+    });
+  }
+
+  const tmpFolderPath = path.resolve(process.cwd(), 'tmp');
+
   const rootDir = await packageDir();
   console.error(`rootDir:`, rootDir);
 
@@ -191,8 +213,63 @@ async function main() {
 
   const failed: typeof projectNames = [];
 
+  let cleanupWasRun = false;
+
+  // Cleanup the various artifacts created as part of executing this script
+  async function runCleanup() {
+    if (cleanupWasRun) {
+      return;
+    }
+    cleanupWasRun = true;
+
+    // Restore the original files in the ecosystem-tests folders from before
+    // npm install was run
+    await fileCache.restoreFiles(tmpFolderPath);
+
+    const packFolderPath = path.join(process.cwd(), PACK_FOLDER);
+
+    try {
+      // Clean up the .pack folder if this was the process that created it.
+      await fs.unlink(PACK_FILE);
+      await fs.rmdir(packFolderPath);
+    } catch (err) {
+      console.log('Failed to delete .pack folder', err);
+    }
+
+    for (let i = 0; i < projectNames.length; i++) {
+      const projectName = (projectNames as any)[i] as string;
+
+      await defaultNodeCleanup(projectName).catch((err: any) => {
+        console.error('Error: Cleanup of file artifacts failed for project', projectName, err);
+      });
+    }
+  }
+
+  async function runCleanupAndExit() {
+    await runCleanup();
+
+    process.exit(1);
+  }
+
+  if (!(await fileExists(tmpFolderPath))) {
+    await fs.mkdir(tmpFolderPath);
+  }
+
   let { jobs } = args;
-  if (args.parallel) jobs = projectsToRun.length;
+  if (args.parallel) {
+    jobs = projectsToRun.length;
+  }
+
+  if (!args.noCleanup) {
+    // The cleanup code is only executed from the parent script that runs
+    // multiple projects.
+    process.on('SIGINT', runCleanupAndExit);
+    process.on('SIGTERM', runCleanupAndExit);
+    process.on('exit', runCleanup);
+
+    await fileCache.cacheFiles(tmpFolderPath);
+  }
+
   if (jobs > 1) {
     const queue = [...projectsToRun];
     const runningProjects = new Set();
@@ -225,7 +302,9 @@ async function main() {
       [...Array(jobs).keys()].map(async () => {
         while (queue.length) {
           const project = queue.shift();
-          if (!project) break;
+          if (!project) {
+            break;
+          }
 
           // preserve interleaved ordering of writes to stdout/stderr
           const chunks: { dest: 'stdout' | 'stderr'; data: string | Buffer }[] = [];
@@ -238,6 +317,7 @@ async function main() {
                 __filename,
                 project,
                 '--skip-pack',
+                '--noCleanup',
                 `--retry=${args.retry}`,
                 ...(args.live ? ['--live'] : []),
                 ...(args.verbose ? ['--verbose'] : []),
@@ -248,14 +328,18 @@ async function main() {
             );
             child.stdout?.on('data', (data) => chunks.push({ dest: 'stdout', data }));
             child.stderr?.on('data', (data) => chunks.push({ dest: 'stderr', data }));
+
             await child;
           } catch (error) {
             failed.push(project);
           } finally {
             runningProjects.delete(project);
           }
 
-          if (IS_CI) console.log(`::group::${failed.includes(project) ? '❌' : '✅'} ${project}`);
+          if (IS_CI) {
+            console.log(`::group::${failed.includes(project) ? '❌' : '✅'} ${project}`);
+          }
+
           for (const { data } of chunks) {
             process.stdout.write(data);
           }
@@ -268,7 +352,7 @@ async function main() {
     clearProgress();
   } else {
     for (const project of projectsToRun) {
-      const fn = projects[project];
+      const fn = projectRunners[project];
 
       await withChdir(path.join(rootDir, 'ecosystem-tests', project), async () => {
         console.error('\n');
@@ -294,6 +378,10 @@ async function main() {
     }
   }
 
+  if (!args.noCleanup) {
+    await runCleanup();
+  }
+
   if (failed.length) {
     console.error(`${failed.length} project(s) failed - ${failed.join(', ')}`);
     process.exit(1);
@@ -340,10 +428,15 @@ async function buildPackage() {
     return;
   }
 
-  if (!(await pathExists('.pack'))) {
-    await fs.mkdir('.pack');
+  if (!(await pathExists(PACK_FOLDER))) {
+    await fs.mkdir(PACK_FOLDER);
   }
 
+  // Run our build script to ensure all of our build artifacts are up to date.
+  // This matters the most for deno as it directly relies on build artifacts
+  // instead of the pack file
+  await run('yarn', ['build']);
+
   const proc = await run('npm', ['pack', '--ignore-scripts', '--json'], {
     cwd: path.join(process.cwd(), 'dist'),
     alwaysPipe: true,
@@ -366,6 +459,11 @@ async function installPackage() {
     return;
   }
 
+  try {
+    // Ensure that there is a clean node_modules folder.
+    await run('rm', ['-rf', `./node_modules`]);
+  } catch (err) {}
+
   const packFile = getPackFile();
   await fs.copyFile(packFile, `./${TAR_NAME}`);
   return await run('npm', ['install', '-D', `./${TAR_NAME}`]);
@@ -440,6 +538,80 @@ export const packageDir = async (): Promise<string> => {
   throw new Error('Package directory not found');
 };
 
+// Caches files that are modified by this script, e.g. package.json,
+// so that they can be restored when the script either finishes or is
+// terminated
+const fileCache = (() => {
+  const filesToCache: Array<string> = ['package.json', 'package-lock.json', 'deno.lock', 'bun.lockb'];
+
+  return {
+    // Copy existing files from each ecosystem-tests project folder to the ./tmp folder
+    cacheFiles: async (tmpFolderPath: string) => {
+      for (let i = 0; i < projectNames.length; i++) {
+        const projectName = (projectNames as any)[i] as string;
+        const projectPath = path.resolve(process.cwd(), 'ecosystem-tests', projectName);
+
+        for (let j = 0; j < filesToCache.length; j++) {
+          const fileName = filesToCache[j] || '';
+
+          const filePath = path.resolve(projectPath, fileName);
+          if (await fileExists(filePath)) {
+            const tmpProjectPath = path.resolve(tmpFolderPath, projectName);
+
+            if (!(await fileExists(tmpProjectPath))) {
+              await fs.mkdir(tmpProjectPath);
+            }
+            await fs.copyFile(filePath, path.resolve(tmpProjectPath, fileName));
+          }
+        }
+      }
+    },
+
+    // Restore the original files to each ecosystem-tests project folder from the ./tmp folder
+    restoreFiles: async (tmpFolderPath: string) => {
+      for (let i = 0; i < projectNames.length; i++) {
+        const projectName = (projectNames as any)[i] as string;
+
+        const projectPath = path.resolve(process.cwd(), 'ecosystem-tests', projectName);
+        const tmpProjectPath = path.resolve(tmpFolderPath, projectName);
+
+        for (let j = 0; j < filesToCache.length; j++) {
+          const fileName = filesToCache[j] || '';
+
+          const filePath = path.resolve(tmpProjectPath, fileName);
+          if (await fileExists(filePath)) {
+            await fs.rename(filePath, path.resolve(projectPath, fileName));
+          }
+        }
+        await fs.rmdir(tmpProjectPath);
+      }
+    },
+  };
+})();
+
+async function defaultNodeCleanup(projectName: string) {
+  try {
+    const projectPath = path.resolve(process.cwd(), 'ecosystem-tests', projectName);
+
+    const packFilePath = path.resolve(projectPath, TAR_NAME);
+
+    if (await fileExists(packFilePath)) {
+      await fs.unlink(packFilePath);
+    }
+  } catch (err) {
+    console.error('Cleanup failed for project', projectName, err);
+  }
+}
+
+async function fileExists(filePath: string) {
+  try {
+    await fs.stat(filePath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 main().catch((err) => {
   console.error(err);
   process.exit(1);

ecosystem-tests/deno/deno.jsonc

@@ -3,5 +3,9 @@
     "install": "deno install --node-modules-dir main_test.ts -f",
     "check": "deno lint && deno check main_test.ts",
     "test": "deno test --allow-env --allow-net --allow-read --node-modules-dir"
+  },
+  "imports": {
+    "openai": "../../deno/mod.ts",
+    "openai/": "../../deno/"
   }
 }