Skip to content

Commit

Permalink
fix(datastore): handle rolling up momentjs
Browse files Browse the repository at this point in the history 
fix(datastore): fix datastore stack traces
feat(client): datastore typing
  • Loading branch information
@blakebyrnes
blakebyrnes committed Feb 1, 2023
1 parent a44549b commit 01e29eb
Show file tree
Hide file tree
Showing 10 changed files with 173 additions and 29 deletions.
2 changes: 2 additions & 0 deletions datastore/core/index.ts
View file
Expand Up @@ -203,9 +203,11 @@ export default class DatastoreCore {

this.sidechainClientManager = new SidechainClientManager(this.options);
this.isStarted.resolve();

} catch (error) {
this.isStarted.reject(error, true);
}
return this.isStarted;
}

public static async close(): Promise<void> {
Expand Down
37 changes: 34 additions & 3 deletions datastore/core/lib/DatastoreVm.ts
View file
Expand Up @@ -6,6 +6,8 @@ import Runner from '@ulixee/datastore/lib/Runner';
import { isSemverSatisfied } from '@ulixee/commons/lib/VersionUtils';
import TransportBridge from '@ulixee/net/lib/TransportBridge';
import DatastoreApiClient from '@ulixee/datastore/lib/DatastoreApiClient';
import { SourceMapSupport } from '@ulixee/commons/lib/SourceMapSupport';
import * as Path from 'path';
import DatastoreCore from '../index';

const { version } = require('../package.json');
Expand All @@ -32,7 +34,7 @@ export default class DatastoreVm {
);
}

const script = await this.getVMScript(path, manifest);
const script = await this.getVMScript(path);

let datastore = this.getVm().run(script) as Datastore;
if (datastore instanceof Runner) {
Expand Down Expand Up @@ -68,15 +70,17 @@ export default class DatastoreVm {
return this.apiClientCacheByUrl[host];
}

private static getVMScript(path: string, manifest: IDatastoreManifest): Promise<VMScript> {
private static getVMScript(path: string): Promise<VMScript> {
if (this.compiledScriptsByPath.has(path)) {
return this.compiledScriptsByPath.get(path);
}

SourceMapSupport.clearStackPath(`${Path.dirname(Path.dirname(path))}/`);

const script = new Promise<VMScript>(async resolve => {
const file = await Fs.readFile(path, 'utf8');
const vmScript = new VMScript(file, {
filename: manifest.scriptEntrypoint,
filename: path,
}).compile();
resolve(vmScript);
});
Expand All @@ -101,6 +105,33 @@ export default class DatastoreVm {
strict: true,
require: {
external: Array.from(whitelist),
resolve: moduleName => {
let isAllowed = false;
for (const entry of whitelist) {
if (entry.match(moduleName) || entry.startsWith('@ulixee/')) {
isAllowed = true;
break;
}
}
if (!isAllowed) return;
// eslint-disable-next-line import/no-dynamic-require
return require.resolve(moduleName);
},
builtin: [
'buffer',
'console',
'errors',
'events',
'http',
'https',
'http2',
'net',
'path',
'stream',
'url',
'util',
'zlib',
],
},
});
}
Expand Down
3 changes: 2 additions & 1 deletion datastore/core/package.json
View file
Expand Up @@ -28,6 +28,7 @@
"axios": "^0.26.1",
"ts-node": "^10.8.1",
"zod": "^3.17.10",
"hostile": "^1.3.3"
"hostile": "^1.3.3",
"moment": "^2.29.4"
}
}
66 changes: 48 additions & 18 deletions datastore/core/test/DatastoreCore.test.ts
View file
Expand Up @@ -9,14 +9,16 @@ import SidechainClient from '@ulixee/sidechain';
import DatastoreApiClient from '@ulixee/datastore/lib/DatastoreApiClient';
import * as Hostile from 'hostile';
import UlixeeMiner from '@ulixee/miner';
import * as Moment from 'moment';
import DatastoreRegistry from '../lib/DatastoreRegistry';
import DatastoreCore from '../index';

const storageDir = Path.resolve(process.env.ULX_DATA_DIR ?? '.', 'DatastoreCore.test');
const tmpDir = `${storageDir}/tmp`;
let packager: Packager;
let dbx: DbxFile;
let bootupPackager: Packager;
let bootupDbx: DbxFile;
let miner: UlixeeMiner;
let client: DatastoreApiClient;

beforeAll(async () => {
mkdirSync(storageDir, { recursive: true });
Expand All @@ -26,36 +28,38 @@ beforeAll(async () => {
miner = new UlixeeMiner();
miner.router.datastoreConfiguration = { datastoresDir: storageDir };
await miner.listen();
packager = new Packager(require.resolve('./datastores/bootup.ts'));
dbx = await packager.build();
client = new DatastoreApiClient(await miner.address);
bootupPackager = new Packager(require.resolve('./datastores/bootup.ts'));
bootupDbx = await bootupPackager.build();
if (process.env.CI !== 'true') Hostile.set('127.0.0.1', 'bootup-datastore.com');
}, 30e3);

afterAll(async () => {
await miner.close();
await client.disconnect();
if (process.env.CI !== 'true') Hostile.remove('127.0.0.1', 'bootup-datastore.com');
try {
rmSync(storageDir, { recursive: true });
} catch (err) {}
});

test('should install new datastores on startup', async () => {
await Fs.copyFile(dbx.dbxPath, `${storageDir}/bootup.dbx`);
await Fs.copyFile(bootupDbx.dbxPath, `${storageDir}/bootup.dbx`);
await DatastoreCore.installManuallyUploadedDbxFiles();
const registry = new DatastoreRegistry(storageDir, tmpDir);
expect(registry.hasVersionHash(packager.manifest.versionHash)).toBe(true);
expect(registry.hasVersionHash(bootupPackager.manifest.versionHash)).toBe(true);
// @ts-expect-error
const dbxPath = registry.getDbxPath(
packager.manifest.scriptEntrypoint,
packager.manifest.versionHash,
bootupPackager.manifest.scriptEntrypoint,
bootupPackager.manifest.versionHash,
);
await expect(existsAsync(dbxPath)).resolves.toBeTruthy();
}, 45e3);

test('should be able to lookup a datastore domain', async () => {
await expect(
runApi('Datastore.upload', {
compressedDatastore: await Fs.readFile(dbx.dbxPath),
compressedDatastore: await Fs.readFile(bootupDbx.dbxPath),
allowNewLinkedVersionHistory: false,
}),
).resolves.toEqual({ success: true });
Expand All @@ -64,37 +68,37 @@ test('should be able to lookup a datastore domain', async () => {
DatastoreApiClient.resolveDatastoreDomain(`bootup-datastore.com:${await miner.port}`),
).resolves.toEqual({
host: await miner.address,
datastoreVersionHash: packager.manifest.versionHash,
datastoreVersionHash: bootupPackager.manifest.versionHash,
});
}, 45e3);

test('can load a version from disk if not already open', async () => {
await expect(
runApi('Datastore.upload', {
compressedDatastore: await Fs.readFile(dbx.dbxPath),
compressedDatastore: await Fs.readFile(bootupDbx.dbxPath),
allowNewLinkedVersionHistory: false,
}),
).resolves.toEqual({ success: true });

const registry = new DatastoreRegistry(storageDir, tmpDir);
// @ts-ignore
await Fs.rm(registry.getDatastoreWorkingDirectory(packager.manifest.versionHash), {
await Fs.rm(registry.getDatastoreWorkingDirectory(bootupPackager.manifest.versionHash), {
recursive: true,
});

await runApi('Datastore.query', {
sql: 'SELECT * FROM bootup()',
versionHash: packager.manifest.versionHash,
versionHash: bootupPackager.manifest.versionHash,
});

await expect(
runApi('Datastore.query', {
sql: 'SELECT * FROM bootup()',
versionHash: packager.manifest.versionHash,
versionHash: bootupPackager.manifest.versionHash,
}),
).resolves.toMatchObject({
outputs: [{ success: true }],
latestVersionHash: packager.manifest.versionHash,
latestVersionHash: bootupPackager.manifest.versionHash,
});
});

Expand All @@ -106,14 +110,14 @@ test('can get metadata about an uploaded datastore', async () => {
});
await expect(
runApi('Datastore.upload', {
compressedDatastore: await Fs.readFile(dbx.dbxPath),
compressedDatastore: await Fs.readFile(bootupDbx.dbxPath),
allowNewLinkedVersionHistory: false,
}),
).resolves.toEqual({ success: true });
await expect(
runApi('Datastore.meta', { versionHash: packager.manifest.versionHash }),
runApi('Datastore.meta', { versionHash: bootupPackager.manifest.versionHash }),
).resolves.toEqual(<IDatastoreApiTypes['Datastore.meta']['result']>{
latestVersionHash: packager.manifest.versionHash,
latestVersionHash: bootupPackager.manifest.versionHash,
computePricePerQuery: 0,
runnersByName: {
bootup: {
Expand Down Expand Up @@ -150,6 +154,32 @@ test('can get metadata about an uploaded datastore', async () => {
});
});

test('can run a Datastore with momentjs', async () => {
const packager = new Packager(require.resolve('./datastores/moment.ts'));
const dbx = await packager.build();
await dbx.upload(await miner.address);
await expect(
client.stream(packager.manifest.versionHash, 'moment', { date: '2021/02/01' }),
).rejects.toThrow('input did not match its Schema');

await expect(
client.stream(packager.manifest.versionHash, 'moment', { date: '2021-02-01' }),
).resolves.toEqual([{ date: Moment('2021-02-01').toDate() }]);
}, 45e3);

test('can get the stack trace of a compiled datastore', async () => {
const packager = new Packager(require.resolve('./datastores/errorStackDatastore.ts'));
const dbx = await packager.build();
await dbx.upload(await miner.address);
try {
await client.stream(packager.manifest.versionHash, 'errorStack', {});
} catch (error) {
expect(error.stack).toContain(
`at multiply (${packager.manifest.versionHash}/datastore/core/test/datastores/errorStack.ts:15:25)`,
);
}
}, 45e3);

function runApi<API extends keyof IDatastoreApiTypes & string>(
Api: API,
args: IDatastoreApiTypes[API]['args'],
Expand Down
18 changes: 18 additions & 0 deletions datastore/core/test/datastores/errorStack.ts
View file
@@ -0,0 +1,18 @@
import { Runner } from '@ulixee/datastore';
import { InvalidSignatureError } from '@ulixee/crypto/lib/errors';

export default new Runner({
async run(ctx) {
const y = await multiply(2);
ctx.Output.emit({ y });
},
});

const multiply = async (x: number): Promise<number> => {
for (let i = 0; i <= 100; i += 1) {
await new Promise(process.nextTick);
x += i ** 2;
if (i === 99) throw new InvalidSignatureError('ERROR!!!');
}
return x;
};
8 changes: 8 additions & 0 deletions datastore/core/test/datastores/errorStackDatastore.ts
View file
@@ -0,0 +1,8 @@
import Datastore from '@ulixee/datastore';
import errorStack from './errorStack';

export default new Datastore({
runners: {
errorStack,
},
});
21 changes: 21 additions & 0 deletions datastore/core/test/datastores/moment.ts
View file
@@ -0,0 +1,21 @@
import Datastore, { Runner } from '@ulixee/datastore';
import * as moment from 'moment';
import { date, string } from '@ulixee/schema';

export default new Datastore({
runners: {
moment: new Runner({
run(ctx) {
ctx.Output.emit({ date: moment(ctx.input.date).toDate() });
},
schema: {
input: {
date: string({ format: 'date' }),
},
output: {
date: date(),
},
},
}),
},
});
18 changes: 13 additions & 5 deletions datastore/packager/lib/rollupDatastore.ts
View file
@@ -1,6 +1,6 @@
import { rollup } from 'rollup';
import { nodeResolve } from '@rollup/plugin-node-resolve';
import { terser } from 'rollup-plugin-terser';
import replace from '@rollup/plugin-replace';
import sourcemaps from './sourcemaps';

const commonjs = require('@rollup/plugin-commonjs');
Expand All @@ -13,14 +13,21 @@ export default async function rollupDatastore(
const outDir = options.outDir ?? `${__dirname}/../`;
const plugins = [
nodeResolve({
resolveOnly: module => !module.startsWith('@ulixee'),
resolveOnly: module => {
return !module.startsWith('@ulixee');
},
}),
commonjs({ transformMixedEsModules: true, extensions: ['.js', '.ts'] }), // the ".ts" extension is required }),
terser({ compress: false, mangle: false, format: { comments: false }, ecma: 2020 }),
];

if (scriptPath.endsWith('.ts')) {
plugins.unshift(
replace({
preventAssignment: true,
values: {
'import * as moment': 'import moment',
},
}),
typescript({
compilerOptions: {
composite: false,
Expand All @@ -29,7 +36,7 @@ export default async function rollupDatastore(
inlineSources: true,
declaration: false,
checkJs: false,
target: 'es2020'
target: 'es2020',
},
outputToFilesystem: false,
tsconfig: options?.tsconfig,
Expand Down Expand Up @@ -63,7 +70,8 @@ export default async function rollupDatastore(
const { output } = await bundle[outFn]({
file: outFile,
sourcemap: true,
format: 'commonjs',
sourcemapFile: 'datastore.js.map',
format: 'cjs',
generatedCode: 'es2015',
});

Expand Down
1 change: 1 addition & 0 deletions datastore/packager/package.json
View file
Expand Up @@ -6,6 +6,7 @@
"@rollup/plugin-commonjs": "^22.0.0",
"@rollup/plugin-node-resolve": "^13.3.0",
"@rollup/plugin-typescript": "^8.3.2",
"@rollup/plugin-replace": "^5.0.2",
"@ulixee/commons": "2.0.0-alpha.18",
"@ulixee/crypto": "2.0.0-alpha.18",
"@ulixee/datastore": "2.0.0-alpha.18",
Expand Down

0 comments on commit 01e29eb

Please sign in to comment.