Multi-Build/Package VS Code extensions on single computer. (#1473)

* added try catch on newly added code to buildjs that copies node_sqlite3 to a build directory that may not exist * fix typo extensioins -> extensions * added new npm command to build for all platforms * started adding code to download the ripgrep binaries, will have to finish at a later date * - added platforms global variable - added check for --all parameter - moved alpine check to utils - moved all packaging code to package() - added links to esbuild binaries - chaged code in IIFE to either package for all or package for target * moved alpine config to utils * added the code that will download the ripgrep binaries * added npm run script package-all which will run the package-all.js script * added package-all script which will package all major builds * - commented out old downloadEsbuildBinary and rewrote it - finished the downloadRipgrepBinary() and exported it * - added back some of the old code to make this script reusable - commented out the esbuild download and install in favor of downloading the prebuilt binary - download the ripgrep binary - containerized the all of the package code into package() and call it in the IIFE * added a check for the --target flag and append it to the vsce command * removed unused variable
continuedev · Jun 14, 2024 · 5edc0cb · 5edc0cb
1 parent f3d200a
commit 5edc0cb
Show file tree

Hide file tree

Showing 7 changed files with 184 additions and 38 deletions.
diff --git a/binary/build.js b/binary/build.js
@@ -224,12 +224,17 @@ async function installNodeModuleInTempDirAndCopyToCurrent(packageName, toCopy) {
     );
 
     // Copy to build directory for testing
-    const [platform, arch] = target.split("-");
-    if (platform === currentPlatform && arch === currentArch) {
-      fs.copyFileSync(
-        `${targetDir}/node_sqlite3.node`,
-        `build/node_sqlite3.node`,
-      );
+    try {
+      const [platform, arch] = target.split("-");
+      if (platform === currentPlatform && arch === currentArch) {
+        fs.copyFileSync(
+          `${targetDir}/node_sqlite3.node`,
+          `build/node_sqlite3.node`,
+        );
+      } 
+    } catch (error) {
+      console.log("[warn] Could not copy node_sqlite to build");
+      console.log(error)
     }
 
     fs.unlinkSync(`${targetDir}/build.tar.gz`);

diff --git a/extensions/vscode/package.json b/extensions/vscode/package.json
@@ -488,6 +488,7 @@
     "quick-test": "npm run build-test && node ./out/runTestOnVSCodeHost.js",
     "prepackage": "node scripts/prepackage.js",
     "package": "node scripts/package.js",
+    "package-all": "node scripts/package-all.js",
     "package:pre-release": "node scripts/package.js --pre-release",
     "build:rust": "cargo-cp-artifact -ac sync sync.node -- cargo build --manifest-path ../../sync/Cargo.toml --message-format=json-render-diagnostics",
     "build-debug:rust": "npm run build:rust --",

diff --git a/extensions/vscode/scripts/package-all.js b/extensions/vscode/scripts/package-all.js
@@ -0,0 +1,33 @@
+const { execSync } = require("child_process");
+
+const PLATFORMS = [
+  "win32-x64",
+//   "win32-arm64", can't be built due to no sqlite3 binaries
+  "linux-x64",
+  "linux-arm64",
+  "darwin-x64",
+  "darwin-arm64",
+]
+const args = process.argv.slice(2);
+const isPreRelease = args.includes("--pre-release");
+
+(async () => {
+  for (const i in PLATFORMS) {
+    const platform = PLATFORMS[i];
+    const pkgCommand = isPreRelease
+      ? "node scripts/package.js --pre-release --target " + platform // --yarn"
+      : "node scripts/package.js --target " + platform; // --yarn";
+
+    execSync(
+      "node scripts/prepackage-cross-platform.js --target "+ platform,
+      {stdio: 'inherit'}
+
+    );
+    execSync(
+      pkgCommand,
+      {stdio: 'inherit'}
+    );
+  }
+})();
+
+
diff --git a/extensions/vscode/scripts/package.js b/extensions/vscode/scripts/package.js
@@ -2,15 +2,23 @@ const { exec } = require("child_process");
 const fs = require("fs");
 
 const args = process.argv.slice(2);
-const isPreRelease = args.includes("--pre-release");
+let target;
+
+if (args[0] === "--target") {
+  target = args[1];
+}
 
 if (!fs.existsSync("build")) {
   fs.mkdirSync("build");
 }
 
-const command = isPreRelease
-  ? "npx vsce package --out ./build patch --pre-release --no-dependencies" // --yarn"
-  : "npx vsce package --out ./build patch --no-dependencies"; // --yarn";
+const isPreRelease = args.includes("--pre-release");
+
+let command = isPreRelease
+? "npx vsce package --out ./build patch --pre-release --no-dependencies" // --yarn"
+: "npx vsce package --out ./build patch --no-dependencies"; // --yarn";
+
+if (target) command += ` --target ${target}`;
 
 exec(command, (error) => {
   if (error) throw error;

diff --git a/extensions/vscode/scripts/prepackage-cross-platform.js b/extensions/vscode/scripts/prepackage-cross-platform.js
@@ -20,6 +20,7 @@ const {
   copyTreeSitterTagQryFiles,
   copyNodeModules,
   downloadEsbuildBinary,
+  downloadRipgrepBinary,
   copySqliteBinary,
   installNodeModuleInTempDirAndCopyToCurrent,
   downloadSqliteBinary,
@@ -62,6 +63,8 @@ console.log("[info] Using target: ", target);
 
 const exe = os === "win32" ? ".exe" : "";
 
+console.log("[info] Using target: ", target);
+
 function ghAction() {
   return !!process.env.GITHUB_ACTIONS;
 }
@@ -78,7 +81,7 @@ function isWin() {
   return target?.startsWith("win");
 }
 
-(async () => {
+async function package(target, os, arch, exe) {
   console.log("[info] Packaging extension for target ", target);
 
   // Copy config_schema.json to config.json in docs and intellij
@@ -102,7 +105,7 @@ function isWin() {
   await copyOnnxRuntimeFromNodeModules(target);
 
   // *** Install @lancedb binary ***
-  const packageToInstall = {
+  const lancePackageToInstall = {
     "darwin-arm64": "@lancedb/vectordb-darwin-arm64",
     "darwin-x64": "@lancedb/vectordb-darwin-x64",
     "linux-arm64": "@lancedb/vectordb-linux-arm64-gnu",
@@ -111,20 +114,22 @@ function isWin() {
     "win32-arm64": "@lancedb/vectordb-win32-x64-msvc", // they don't have a win32-arm64 build
   }[target];
   await installNodeModuleInTempDirAndCopyToCurrent(
-    packageToInstall,
+    lancePackageToInstall,
     "@lancedb",
   );
-
   // *** esbuild ***
-  await installNodeModuleInTempDirAndCopyToCurrent(
-    "esbuild@0.17.19",
-    "@esbuild",
-  );
+  // await installNodeModuleInTempDirAndCopyToCurrent(
+  //   "esbuild@0.17.19",
+  //   "@esbuild",
+  // );
+  await downloadEsbuildBinary(target);
 
   // *** sqlite ***
   await downloadSqliteBinary(target);
   await copySqliteBinary();
 
+  await downloadRipgrepBinary(target);
+
   // copy node_modules to out/node_modules
   await copyNodeModules();
 
@@ -199,4 +204,8 @@ function isWin() {
     `out/node_modules/esbuild/lib/main.js`,
     `out/node_modules/esbuild/bin/esbuild`,
   ]);
+}
+
+(async () => {
+  await package(target, os, arch, exe);
 })();
diff --git a/extensions/vscode/scripts/utils.js b/extensions/vscode/scripts/utils.js
@@ -193,7 +193,7 @@ async function copyOnnxRuntimeFromNodeModules(target) {
 }
 
 async function copyTreeSitterWasms() {
-  process.chdir(path.join(continueDir, "extensioins", "vscode"));
+  process.chdir(path.join(continueDir, "extensions", "vscode"));
   fs.mkdirSync("out", { recursive: true });
 
   await new Promise((resolve, reject) => {
@@ -271,27 +271,79 @@ async function copyNodeModules() {
   console.log(`[info] Copied ${NODE_MODULES_TO_COPY.join(", ")}`);
 }
 
-async function downloadEsbuildBinary(isGhAction, isArm, target) {
-  process.chdir(path.join(continueDir, "extensions", "vscode"));
+// async function downloadEsbuildBinary(isGhAction, isArm, target) {
+//   process.chdir(path.join(continueDir, "extensions", "vscode"));
+
+//   if (isGhAction && isArm) {
+//     // Download and unzip esbuild
+//     console.log("[info] Downloading pre-built esbuild binary");
+//     rimrafSync("node_modules/@esbuild");
+//     fs.mkdirSync("node_modules/@esbuild", { recursive: true });
+//     execCmdSync(
+//       `curl -o node_modules/@esbuild/esbuild.zip https://continue-server-binaries.s3.us-west-1.amazonaws.com/${target}/esbuild.zip`,
+//     );
+//     execCmdSync(`cd node_modules/@esbuild && unzip esbuild.zip`);
+//     fs.unlinkSync("node_modules/@esbuild/esbuild.zip");
+//   } else {
+//     // Download esbuild from npm in tmp and copy over
+//     console.log("npm installing esbuild binary");
+//     await installNodeModuleInTempDirAndCopyToCurrent(
+//       "esbuild@0.17.19",
+//       "@esbuild",
+//     );
+//   }
+// }
+
+async function downloadEsbuildBinary(target) {
+  console.log("[info] Downloading pre-built esbuild binary");
+  rimrafSync("out/node_modules/@esbuild");
+  fs.mkdirSync(`out/node_modules/@esbuild/${target}/bin`, { recursive: true });
+  fs.mkdirSync(`out/tmp`, { recursive: true });
+  const downloadUrl = {
+    "darwin-arm64":
+      "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.17.19.tgz",
+    "linux-arm64":
+      "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.17.19.tgz",
+    "win32-arm64":
+      "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.17.19.tgz",
+    "linux-x64":
+      "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.17.19.tgz",
+    "darwin-x64":
+      "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.17.19.tgz",
+    "win32-x64":
+      "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.17.19.tgz",
+  }[target];
+  execCmdSync(
+    `curl -L -o out/tmp/esbuild.tgz ${downloadUrl}`,
+  );
+  execCmdSync("cd out/tmp && tar -xvzf esbuild.tgz");
+  // Copy the installed package back to the current directory
+  let tmpPath = "out/tmp/package/bin";
+  let outPath = `out/node_modules/@esbuild/${target}/bin`;
+  if (target.startsWith("win")) {
+    tmpPath = 'out/tmp/package';
+    outPath = `out/node_modules/@esbuild/${target}`;
+  }
 
-  if (isGhAction && isArm) {
-    // Download and unzip esbuild
-    console.log("[info] Downloading pre-built esbuild binary");
-    rimrafSync("node_modules/@esbuild");
-    fs.mkdirSync("node_modules/@esbuild", { recursive: true });
-    execCmdSync(
-      `curl -o node_modules/@esbuild/esbuild.zip https://continue-server-binaries.s3.us-west-1.amazonaws.com/${target}/esbuild.zip`,
-    );
-    execCmdSync(`cd node_modules/@esbuild && unzip esbuild.zip`);
-    fs.unlinkSync("node_modules/@esbuild/esbuild.zip");
-  } else {
-    // Download esbuild from npm in tmp and copy over
-    console.log("npm installing esbuild binary");
-    await installNodeModuleInTempDirAndCopyToCurrent(
-      "esbuild@0.17.19",
-      "@esbuild",
+  await new Promise((resolve, reject) => {
+    ncp(
+      path.join(tmpPath),
+      path.join(outPath),
+      { dereference: true },
+      (error) => {
+        if (error) {
+          console.error(
+            `[error] Error copying esbuild package`,
+            error,
+          );
+          reject(error);
+        } else {
+          resolve();
+        }
+      },
     );
-  }
+  });
+  rimrafSync("out/tmp")
 }
 
 async function downloadSqliteBinary(target) {
@@ -338,6 +390,41 @@ async function copySqliteBinary() {
   });
 }
 
+async function downloadRipgrepBinary(target) {
+  console.log("[info] Downloading pre-built ripgrep binary");
+  rimrafSync("node_modules/@vscode/ripgrep/bin");
+  fs.mkdirSync("node_modules/@vscode/ripgrep/bin", { recursive: true });4
+  const downloadUrl = {
+    "darwin-arm64":
+      "https://github.com/microsoft/ripgrep-prebuilt/releases/download/v13.0.0-10/ripgrep-v13.0.0-10-aarch64-apple-darwin.tar.gz",
+    "linux-arm64":
+      "https://github.com/microsoft/ripgrep-prebuilt/releases/download/v13.0.0-10/ripgrep-v13.0.0-10-aarch64-unknown-linux-gnu.tar.gz",
+    "win32-arm64":
+      "https://github.com/microsoft/ripgrep-prebuilt/releases/download/v13.0.0-10/ripgrep-v13.0.0-10-aarch64-pc-windows-msvc.zip",
+    "linux-x64":
+      "https://github.com/microsoft/ripgrep-prebuilt/releases/download/v13.0.0-10/ripgrep-v13.0.0-10-x86_64-unknown-linux-musl.tar.gz",
+    "darwin-x64":
+      "https://github.com/microsoft/ripgrep-prebuilt/releases/download/v13.0.0-10/ripgrep-v13.0.0-10-x86_64-apple-darwin.tar.gz",
+    "win32-x64":
+      "https://github.com/microsoft/ripgrep-prebuilt/releases/download/v13.0.0-10/ripgrep-v13.0.0-10-x86_64-pc-windows-msvc.zip",
+  }[target];
+
+
+  if(target.startsWith("win")) {
+    execCmdSync(
+      `curl -L -o node_modules/@vscode/ripgrep/bin/build.zip ${downloadUrl}`,
+    );
+    execCmdSync("cd node_modules/@vscode/ripgrep/bin && unzip build.zip");
+    fs.unlinkSync("node_modules/@vscode/ripgrep/bin/build.zip");
+  } else {
+    execCmdSync(
+      `curl -L -o node_modules/@vscode/ripgrep/bin/build.tar.gz ${downloadUrl}`,
+    );
+    execCmdSync("cd node_modules/@vscode/ripgrep/bin && tar -xvzf build.tar.gz");
+    fs.unlinkSync("node_modules/@vscode/ripgrep/bin/build.tar.gz");
+  }
+}
+
 async function installNodeModuleInTempDirAndCopyToCurrent(packageName, toCopy) {
   console.log(`Copying ${packageName} to ${toCopy}`);
   // This is a way to install only one package without npm trying to install all the dependencies
@@ -409,4 +496,5 @@ module.exports = {
   copySqliteBinary,
   installNodeModuleInTempDirAndCopyToCurrent,
   downloadSqliteBinary,
+  downloadRipgrepBinary,
 };
diff --git a/scripts/util/index.js b/scripts/util/index.js
@@ -14,6 +14,7 @@ function execCmdSync(cmd) {
 function autodetectPlatformAndArch() {
   platform = {
     aix: "linux",
+    alpine: "linux",
     darwin: "darwin",
     freebsd: "linux",
     linux: "linux",
@@ -23,6 +24,7 @@ function autodetectPlatformAndArch() {
   }[process.platform];
   arch = {
     arm: "arm64",
+    armhf: "arm64",
     arm64: "arm64",
     ia32: "x64",
     loong64: "arm64",