feat(datastore): change cloning to create project

ulixee · Jan 23, 2023 · 6263ecb · 6263ecb
1 parent 92d9a3b
commit 6263ecb
Show file tree

Hide file tree

Showing 11 changed files with 89 additions and 42 deletions.
diff --git a/datastore/client/cli/cloneDatastore.ts b/datastore/client/cli/cloneDatastore.ts
@@ -1,22 +1,23 @@
 // eslint-disable-next-line import/no-extraneous-dependencies
 import * as ts from 'typescript';
-import { writeFileSync } from 'fs';
+import { copyFileSync, existsSync, mkdirSync, writeFileSync } from 'fs';
 import * as Path from 'path';
 import jsonToSchemaCode from '@ulixee/schema/lib/jsonToSchemaCode';
 import { nanoid } from 'nanoid';
 import DatastoreApiClient from '../lib/DatastoreApiClient';
 
 export default async function cloneDatastore(
   url: string,
-  path: string,
-  options: { emitModules?: boolean; embedCredits?: { id: string; secret: string } } = {
-    emitModules: false,
-  },
+  directoryPath?: string,
+  options: { embedCredits?: { id: string; secret: string } } = {},
 ): Promise<void> {
-  const parsedUrl = new URL(url);
-  const datastoreApiClient = new DatastoreApiClient(parsedUrl.host);
-  const meta = await datastoreApiClient.getMeta(parsedUrl.pathname.slice(1), true);
-  path = Path.resolve(path);
+  const { datastoreVersionHash, host } = await DatastoreApiClient.resolveDatastoreDomain(url);
+  if (url.includes('/free-credits')) {
+    const credit = new URL(url).search.split(':');
+    options.embedCredits = { id: credit[0], secret: credit[1] };
+  }
+  const datastoreApiClient = new DatastoreApiClient(host);
+  const meta = await datastoreApiClient.getMeta(datastoreVersionHash, true);
 
   const schemasByName: Record<string, { isTable: boolean; schemaJson: any }> = {};
   const imports = new Set<string>();
@@ -77,6 +78,7 @@ export default async function cloneDatastore(
   import { ${[...schemaImports].join(', ')} } from '@ulixee/schema';
   
   const datastore = new Datastore({
+    name: ${JSON.stringify(meta.name)},
     affiliateId: "aff${nanoid(12)}",
     remoteDatastores: {
       source: "${url}",
@@ -94,8 +96,26 @@ export default async function cloneDatastore(
 
   export default datastore;`;
 
+  let folder = Path.resolve(directoryPath || `./${meta.name ?? 'datastore'}`);
+  if (!directoryPath) {
+    let counter = 1;
+    const baseFolder = folder;
+    while (existsSync(folder)) folder += `${baseFolder}${counter++}`;
+  }
+
+  if (!existsSync(folder)) {
+    mkdirSync(folder, { recursive: true });
+  }
+  if (!existsSync(Path.join(folder, 'package.json'))) {
+    copyFileSync(`${__dirname}/cloned-package.json`, Path.join(folder, 'package.json'));
+  }
+  if (!existsSync(Path.join(folder, 'tsconfig.json'))) {
+    copyFileSync(`${__dirname}/cloned-tsconfig.json`, Path.join(folder, 'tsconfig.json'));
+  }
+
+  const datastoreFilepath = Path.join(folder, `datastore.ts`);
   const sourceFile = ts.createSourceFile(
-    path,
+    datastoreFilepath,
     script,
     ts.ScriptTarget.ES2020,
     false,
@@ -104,20 +124,5 @@ export default async function cloneDatastore(
 
   const printer = ts.createPrinter({ newLine: ts.NewLineKind.LineFeed, noEmitHelpers: true });
   const tsFile = printer.printFile(sourceFile);
-
-  if (path.endsWith('.ts')) {
-    writeFileSync(path, tsFile);
-  } else {
-    const jsFile = ts.transpileModule(tsFile, {
-      compilerOptions: {
-        target: ts.ScriptTarget.ES2022,
-        module: options.emitModules ? ts.ModuleKind.ES2022 : ts.ModuleKind.CommonJS,
-        esModuleInterop: false,
-        noImplicitUseStrict: true,
-        strict: false,
-        lib: ['es2022'],
-      },
-    });
-    writeFileSync(path, jsFile.outputText);
-  }
+  writeFileSync(datastoreFilepath, tsFile);
 }
diff --git a/datastore/client/cli/cloned-package.json b/datastore/client/cli/cloned-package.json
@@ -0,0 +1,21 @@
+{
+  "name": "datastore",
+  "version": "0.0.1",
+  "private": true,
+  "main": "datastore.js",
+  "scripts": {
+    "postinstall": "tsc",
+    "deploy": "@ulixee/datastore deploy ./datastore.ts",
+    "credits:create": "@ulixee/datastore credits create"
+  },
+  "engines": {
+    "node": ">=14.20.0"
+  },
+  "dependencies": {
+    "@ulixee/datastore": "2.0.0-alpha.18"
+  },
+  "devDependencies": {
+    "@ulixee/datastore-packager": "2.0.0-alpha.18",
+    "typescript": "~4.9.4"
+  }
+}
diff --git a/datastore/client/cli/cloned-tsconfig.json b/datastore/client/cli/cloned-tsconfig.json
@@ -0,0 +1,16 @@
+{
+  "compileOnSave": true,
+  "compilerOptions": {
+    "incremental": true,
+    "target": "ES2022",
+    "module": "CommonJS",
+    "lib": ["ES2022"],
+    "sourceMap": true,
+    "esModuleInterop": false,
+    "declaration": true,
+    "skipLibCheck": true,
+    "resolveJsonModule": true
+  },
+  "exclude": ["**/tsconfig*.json", "node_modules", "**/node_modules"],
+  "include": ["./*.ts"]
+}
diff --git a/datastore/client/cli/index.ts b/datastore/client/cli/index.ts
@@ -40,7 +40,7 @@ export default function datastoreCommands(): Command {
     .command('clone')
     .description('Clone and add onto a Datastore.')
     .argument('<url>', 'The url of the Datastore.')
-    .argument('<path>', 'The path to output your cloned Datastore.')
+    .argument('[path]', 'The directory path to output your cloned Datastore.')
     .action(async (url, path) => {
       await cloneDatastore(url, path);
     });

diff --git a/datastore/core/test/Crawler.test.ts b/datastore/core/test/Crawler.test.ts
@@ -24,6 +24,8 @@ beforeAll(async () => {
   miner.router.datastoreConfiguration = { datastoresDir: storageDir };
   await miner.listen();
   client = new DatastoreApiClient(await miner.address);
+  Helpers.needsClosing.push(miner);
+  Helpers.onClose(() => client.disconnect());
 });
 
 beforeEach(() => {
@@ -34,7 +36,7 @@ afterEach(Helpers.afterEach);
 
 afterAll(async () => {
   await Helpers.afterAll();
-  Fs.rmSync(storageDir, { recursive: true });
+  await Fs.promises.rm(storageDir, { recursive: true }).catch(() => null);
 });
 
 test('should be able to run a crawler', async () => {

diff --git a/datastore/core/test/Datastore.clone.test.ts b/datastore/core/test/Datastore.clone.test.ts
@@ -28,7 +28,7 @@ beforeAll(async () => {
   miner = new UlixeeMiner();
   miner.router.datastoreConfiguration = { datastoresDir: storageDir };
   await miner.listen();
-  client = new DatastoreApiClient(await miner.address);
+  client = new DatastoreApiClient(await miner.address, true);
 
   const packager = new DatastorePackager(`${__dirname}/datastores/cloneme.ts`);
   await packager.build();
@@ -44,11 +44,11 @@ afterAll(async () => {
 });
 
 test('should be able to clone a datastore', async () => {
-  const url = `ulx://${await miner.address}/${versionHash}`;
-  await expect(cloneDatastore(url, `${__dirname}/datastores/cloned.ts`)).resolves.toBeUndefined();
+  const url = `ulx://${await miner.address}/datastore/${versionHash}`;
+  await expect(cloneDatastore(url, `${__dirname}/datastores/cloned`)).resolves.toBeUndefined();
 
-  expect(Fs.existsSync(`${__dirname}/datastores/cloned.ts`)).toBeTruthy();
-  const packager = new DatastorePackager(`${__dirname}/datastores/cloned.ts`);
+  expect(Fs.existsSync(`${__dirname}/datastores/cloned/datastore.ts`)).toBeTruthy();
+  const packager = new DatastorePackager(`${__dirname}/datastores/cloned/datastore.ts`);
   await packager.build();
   await client.upload(await packager.dbx.asBuffer());
 

diff --git a/datastore/core/test/DatastorePayments.test.ts b/datastore/core/test/DatastorePayments.test.ts
@@ -285,12 +285,12 @@ test('should be able to embed Credits in a Datastore', async () => {
   });
 
   await cloneDatastore(
-    `ulx://${await miner.address}/${manifest.versionHash}`,
-    `${__dirname}/datastores/clone-output.js`,
+    `ulx://${await miner.address}/datastore/${manifest.versionHash}`,
+    `${__dirname}/datastores/clone-output`,
     { embedCredits: credits },
   );
   await Fs.writeFileSync(
-    `${__dirname}/datastores/clone-output-manifest.json`,
+    `${__dirname}/datastores/clone-output/datastore-manifest.json`,
     JSON.stringify({
       paymentAddress: encodeBuffer(sha3('payme123'), 'ar'),
       functionsByName: {
@@ -303,7 +303,7 @@ test('should be able to embed Credits in a Datastore', async () => {
   );
 
   {
-    const packager2 = new DatastorePackager(`${__dirname}/datastores/clone-output.js`);
+    const packager2 = new DatastorePackager(`${__dirname}/datastores/clone-output/datastore.ts`);
     const dbx2 = await packager2.build();
     const manifest2 = packager2.manifest;
     await client.upload(await dbx2.asBuffer(), { identity: adminIdentity });

diff --git a/datastore/core/test/PassthroughFunctions.test.ts b/datastore/core/test/PassthroughFunctions.test.ts
@@ -94,6 +94,7 @@ beforeAll(async () => {
   miner.router.datastoreConfiguration = { datastoresDir: storageDir };
   await miner.listen();
   client = new DatastoreApiClient(await miner.address);
+  Helpers.onClose(() => client.disconnect());
 
   const packager = new DatastorePackager(`${__dirname}/datastores/remoteFunction.js`);
   await packager.build();

diff --git a/datastore/core/test/datastores/cloneme.ts b/datastore/core/test/datastores/cloneme.ts
@@ -2,6 +2,7 @@ import Datastore, { Function, Table } from '@ulixee/datastore';
 import { boolean, date, object, string } from '@ulixee/schema';
 
 export default new Datastore({
+  name: 'cloneme',
   functions: {
     cloneUpstream: new Function({
       run(ctx) {

diff --git a/datastore/packager/index.ts b/datastore/packager/index.ts
@@ -193,15 +193,15 @@ export default class DatastorePackager {
       );
     }
 
-    const datastoreVersionHash = remoteUrl.pathname.slice(1);
+    const [datastoreVersionHash] = remoteUrl.pathname.match(/dbx1[ac-hj-np-z02-9]{18}/)
     DatastoreManifest.validateVersionHash(datastoreVersionHash);
 
     const remoteMeta = {
       host: remoteUrl.host,
       datastoreVersionHash,
       functionName,
     };
-    const datastoreApiClient = new DatastoreApiClient(remoteUrl.host);
+    const datastoreApiClient = new DatastoreApiClient(remoteUrl.host, this.logToConsole);
     try {
       const upstreamMeta = await datastoreApiClient.getMeta(datastoreVersionHash);
       const remoteFunctionDetails = upstreamMeta.functionsByName[functionName];
@@ -233,15 +233,15 @@ export default class DatastorePackager {
       );
     }
 
-    const datastoreVersionHash = remoteUrl.pathname.slice(1);
+    const [datastoreVersionHash] = remoteUrl.pathname.match(/dbx1[ac-hj-np-z02-9]{18}/)
     DatastoreManifest.validateVersionHash(datastoreVersionHash);
 
     const remoteMeta = {
       host: remoteUrl.host,
       datastoreVersionHash,
       tableName,
     };
-    const datastoreApiClient = new DatastoreApiClient(remoteUrl.host);
+    const datastoreApiClient = new DatastoreApiClient(remoteUrl.host, this.logToConsole);
     try {
       const upstreamMeta = await datastoreApiClient.getMeta(datastoreVersionHash);
       const remoteDetails = upstreamMeta.tablesByName[tableName];

diff --git a/datastore/packager/package.json b/datastore/packager/package.json
@@ -22,6 +22,7 @@
     "ws": "^7.4.6"
   },
   "devDependencies": {
-    "lodash-es": "^4.17.21"
+    "lodash-es": "^4.17.21",
+    "typescript": "^4.7.4"
   }
 }