feat(cubestore): post-install - improvements

rust/js-wrapper/download.ts

@@ -1,6 +1,6 @@
 /* eslint-disable no-restricted-syntax */
 import tar from 'tar';
-import fs, { mkdirSync, WriteStream } from 'fs';
+import fs, { WriteStream } from 'fs';
 import fetch, { Headers, Request, Response } from 'node-fetch';
 import { throttle } from 'throttle-debounce';
 import { internalExceptions } from '@cubejs-backend/shared';
@@ -9,6 +9,7 @@ import cli from 'cli-ux';
 import process from 'process';
 import { Octokit } from '@octokit/core';
 import * as path from 'path';
+import { mkdirpSync } from 'fs-extra';
 
 import { detectLibc } from './utils';
 
@@ -58,7 +59,13 @@ export async function streamWithProgress(
   );
 }
 
-export async function downloadAndExtractFile(url: string, fileName: string, workingDirectory: string) {
+export function getBinaryPath() {
+  const binaryName = process.platform === 'win32' ? 'cubestored.exe' : 'cubestored';
+
+  return path.join(path.resolve(__dirname, '..'), 'downloaded', 'latest', 'bin', binaryName);
+}
+
+export async function downloadAndExtractFile(url: string) {
   const request = new Request(url, {
     headers: new Headers({
       'Content-Type': 'application/octet-stream'
@@ -75,15 +82,16 @@ export async function downloadAndExtractFile(url: string, fileName: string, work
   });
   bar.start(100, 0);
 
+  const cubestorePath = path.dirname(getBinaryPath());
+
   try {
-    mkdirSync(path.join(workingDirectory, 'downloaded'));
-    mkdirSync(path.join(workingDirectory, 'downloaded', 'latest'));
+    mkdirpSync(cubestorePath);
   } catch (e) {
     internalExceptions(e);
   }
 
   const writer = tar.x({
-    cwd: path.join(workingDirectory, 'downloaded', 'latest'),
+    cwd: cubestorePath,
   });
 
   await streamWithProgress(response, <WriteStream>writer, ({ progress, speed, eta }) => {
@@ -129,6 +137,30 @@ async function fetchRelease(version: string) {
   return data;
 }
 
+function parseInfoFromAssetName(assetName: string): { target: string, type: string, format: string } | null {
+  if (assetName.startsWith('cubestored-')) {
+    const fileName = assetName.slice('cubestored-'.length);
+    const targetAndType = fileName.slice(0, fileName.indexOf('.'));
+    const format = fileName.slice(fileName.indexOf('.') + 1);
+
+    if (targetAndType.endsWith('-shared')) {
+      return {
+        target: targetAndType.substr(0, targetAndType.length - '-shared'.length),
+        format,
+        type: 'shared'
+      };
+    }
+
+    return {
+      target: targetAndType,
+      format,
+      type: 'static'
+    };
+  }
+
+  return null;
+}
+
 export async function downloadBinaryFromRelease() {
   // eslint-disable-next-line global-require
   const { version } = require('../package.json');
@@ -141,19 +173,14 @@ export async function downloadBinaryFromRelease() {
       );
     }
 
-    const target = getTarget();
+    const currentTarget = getTarget();
 
     for (const asset of release.assets) {
-      const fileName = asset.name.substr(0, asset.name.length - 7);
-      if (fileName.startsWith('cubestored-')) {
-        const assetTarget = fileName.substr('cubestored-'.length);
-        if (assetTarget === target) {
-          return downloadAndExtractFile(
-            asset.browser_download_url,
-            asset.name,
-            path.resolve(__dirname, '..')
-          );
-        }
+      const assetInfo = parseInfoFromAssetName(asset.name);
+      if (assetInfo && assetInfo.target === currentTarget
+        && assetInfo.type === 'static' && assetInfo.format === 'tar.gz'
+      ) {
+        return downloadAndExtractFile(asset.browser_download_url);
       }
     }
 

rust/js-wrapper/process.ts

@@ -1,11 +1,8 @@
 import * as fs from 'fs';
-import * as path from 'path';
 import { ChildProcess, spawn } from 'child_process';
 import { withTimeout } from '@cubejs-backend/shared';
 
-import { downloadBinaryFromRelease } from './download';
-
-const binaryName = process.platform === 'win32' ? 'cubestored.exe' : 'cubestored';
+import { downloadBinaryFromRelease, getBinaryPath } from './download';
 
 export interface CubeStoreHandlerOptions {
   stdout: (data: Buffer) => void;
@@ -108,12 +105,8 @@ export class CubeStoreHandler {
     protected readonly config: Readonly<CubeStoreHandlerOptions>
   ) {}
 
-  protected getBinaryPath() {
-    return path.join(__dirname, '..', 'downloaded', 'latest', 'bin', binaryName);
-  }
-
   protected async getBinary() {
-    const pathToExecutable = this.getBinaryPath();
+    const pathToExecutable = getBinaryPath();
 
     if (!fs.existsSync(pathToExecutable)) {
       await downloadBinaryFromRelease();
@@ -139,7 +132,7 @@ export class CubeStoreHandler {
       this.config.onRestart(code);
 
       this.cubeStoreStarting = new Promise<ChildProcess>(
-        (resolve, reject) => startProcess(this.getBinaryPath(), {
+        (resolve, reject) => startProcess(getBinaryPath(), {
           ...this.config,
           onExit,
         }).then((cubeStore) => {

rust/package.json

@@ -38,6 +38,7 @@
     "@octokit/core": "^3.2.5",
     "bytes": "^3.1.0",
     "cli-ux": "^5.5.1",
+    "fs-extra": "^9.1.0",
     "node-fetch": "^2.6.1",
     "tar": "^6.1.0",
     "throttle-debounce": "^3.0.1"

rust/yarn.lock

@@ -35,10 +35,10 @@
     eslint-plugin-import "^2.22.1"
     eslint-plugin-node "^9.2.0"
 
-"@cubejs-backend/shared@^0.26.25":
-  version "0.26.25"
-  resolved "https://registry.yarnpkg.com/@cubejs-backend/shared/-/shared-0.26.25.tgz#6c63d882a8bf917836d3fc4e6e5a27d61e52b761"
-  integrity sha512-cJY9QIx/13Oaf6vtg5/U4t5YesjJQzoB5kTUqXgqf4QkLewi36f2qYFM8JUJkL3d71xRfMf15t7wMIUNTLmJHQ==
+"@cubejs-backend/shared@^0.26.35":
+  version "0.26.35"
+  resolved "https://registry.yarnpkg.com/@cubejs-backend/shared/-/shared-0.26.35.tgz#87806a385282f9e4327259924628e92cc1d95d05"
+  integrity sha512-9cKt/Z9b3mUobks/jlWyF/7xCXWOH+XuvufZuCoAE7It0tUUPnayPiZ3NhoozNl6l2oZeaiAJt1+9ddtPgieWQ==
   dependencies:
     env-var "^6.3.0"
     node-fetch "^2.6.1"
@@ -480,6 +480,11 @@ asynckit@^0.4.0:
   resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
   integrity sha1-x57Zf380y48robyXkLzDZkdLS3k=
 
+at-least-node@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/at-least-node/-/at-least-node-1.0.0.tgz#602cd4b46e844ad4effc92a8011a3c46e0238dc2"
+  integrity sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==
+
 balanced-match@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
@@ -1073,6 +1078,16 @@ fs-extra@^8.1:
     jsonfile "^4.0.0"
     universalify "^0.1.0"
 
+fs-extra@^9.1.0:
+  version "9.1.0"
+  resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.1.0.tgz#5954460c764a8da2094ba3554bf839e6b9a7c86d"
+  integrity sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==
+  dependencies:
+    at-least-node "^1.0.0"
+    graceful-fs "^4.2.0"
+    jsonfile "^6.0.1"
+    universalify "^2.0.0"
+
 fs-minipass@^2.0.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/fs-minipass/-/fs-minipass-2.1.0.tgz#7f5036fdbf12c63c169190cbe4199c852271f9fb"
@@ -1363,6 +1378,15 @@ jsonfile@^4.0.0:
   optionalDependencies:
     graceful-fs "^4.1.6"
 
+jsonfile@^6.0.1:
+  version "6.1.0"
+  resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae"
+  integrity sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==
+  dependencies:
+    universalify "^2.0.0"
+  optionalDependencies:
+    graceful-fs "^4.1.6"
+
 levn@^0.4.1:
   version "0.4.1"
   resolved "https://registry.yarnpkg.com/levn/-/levn-0.4.1.tgz#ae4562c007473b932a6200d403268dd2fffc6ade"
@@ -2043,6 +2067,11 @@ universalify@^0.1.0:
   resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66"
   integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==
 
+universalify@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.0.tgz#75a4984efedc4b08975c5aeb73f530d02df25717"
+  integrity sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==
+
 uri-js@^4.2.2:
   version "4.4.1"
   resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.1.tgz#9b1a52595225859e55f669d928f88c6c57f2a77e"