Skip to content

Commit

Permalink
fix(datastore): tweak cli to fix end-to-end test
Browse files Browse the repository at this point in the history
  • Loading branch information
blakebyrnes committed Jan 17, 2023
1 parent c8b6c8a commit 4355cd1
Show file tree
Hide file tree
Showing 18 changed files with 215 additions and 207 deletions.
44 changes: 28 additions & 16 deletions datastore/client/cli/creditsCli.ts
Expand Up @@ -40,14 +40,17 @@ export default function creditsCli(): Command {
)
.addOption(identityPrivateKeyPassphraseOption)
.action(async (url, { identityPath, identityPassphrase, amount }) => {
const parsedUrl = new URL(url);
const client = new DatastoreApiClient(parsedUrl.origin);
const client = new DatastoreApiClient(url);
const microgons = ArgonUtils.parseUnits(amount, 'microgons');
const identity = Identity.loadFromFile(identityPath, { keyPassphrase: identityPassphrase });
const result = await client.createCredits(parsedUrl.pathname, microgons, identity);
try {
const result = await client.createCredits(url.split('/').pop().trim(), microgons, identity);

// TODO: output a url to send the user to
console.log(`Credit created!`, { credit: result });
// TODO: output a url to send the user to
console.log(JSON.stringify({ credit: result }));
} finally {
await client.disconnect();
}
});

cli
Expand All @@ -56,15 +59,20 @@ export default function creditsCli(): Command {
.argument('<url>', 'The url of the Credit.')
.argument('<secret>', 'The Credit secret.')
.action(async (url, secret) => {
if (!url.includes('://')) url = `ulx://${url}`;
const client = new DatastoreApiClient(url);
const parsedUrl = new URL(url);
const [datastoreVersion, id] = parsedUrl.pathname.split('/credit/');
const { balance } = await client.getCreditsBalance(datastoreVersion, id);
await CreditsStore.store(parsedUrl.origin, datastoreVersion.replace(/\//g, ''), {
id,
secret,
remainingCredits: balance,
});
try {
const parsedUrl = new URL(url);
const [datastoreVersion, id] = parsedUrl.pathname.slice(1).split('/credits/');
const { balance } = await client.getCreditsBalance(datastoreVersion, id);
await CreditsStore.store(datastoreVersion.replace(/\//g, ''), {
id,
secret,
remainingCredits: balance,
});
} finally {
await client.disconnect();
}
});

cli
Expand All @@ -74,9 +82,13 @@ export default function creditsCli(): Command {
.argument('<id>', 'The Credit id.')
.action(async (url, id) => {
const client = new DatastoreApiClient(url);
const datastoreVersion = url.pathname.split('/').pop();
const { balance, issuedCredits } = await client.getCreditsBalance(datastoreVersion, id);
console.log({ issuedCredits, balance });
try {
const datastoreVersion = url.split('/').pop().trim();
const { balance, issuedCredits } = await client.getCreditsBalance(datastoreVersion, id);
console.log({ issuedCredits, balance });
} finally {
await client.disconnect();
}
});
return cli;
}
Expand Down
12 changes: 5 additions & 7 deletions datastore/client/lib/CreditsStore.ts
Expand Up @@ -8,25 +8,23 @@ export default class CreditsStore {
private static creditsByDatastore: Promise<ICreditsStore>;

public static async store(
minerHost: string,
datastoreVersionHash: string,
credits: { id: string; secret: string; remainingCredits: number },
): Promise<void> {
const allCredits = await this.load();
allCredits[`${minerHost}/${datastoreVersionHash}`] ??= {};
allCredits[`${minerHost}/${datastoreVersionHash}`][credits.id] = credits;
allCredits[datastoreVersionHash] ??= {};
allCredits[datastoreVersionHash][credits.id] = credits;
await this.writeToDisk(allCredits);
}

public static async getPayment(
minerHost: string,
datastoreVersionHash: string,
microgons: number,
): Promise<
(IPayment & { onFinalized(result: { microgons: number; bytes: number }): void }) | undefined
> {
const credits = await this.load();
const datastoreCredits = credits[`${minerHost}/${datastoreVersionHash}`];
const datastoreCredits = credits[datastoreVersionHash];
if (!datastoreCredits) return;

for (const [creditId, credit] of Object.entries(datastoreCredits)) {
Expand Down Expand Up @@ -54,11 +52,11 @@ export default class CreditsStore {

private static async load(): Promise<ICreditsStore> {
this.creditsByDatastore ??= readFileAsJson<ICreditsStore>(this.storePath).catch(() => ({}));
return await this.creditsByDatastore;
return (await this.creditsByDatastore) ?? {};
}

private static writeToDisk(data: any): Promise<void> {
return safeOverwriteFile(this.storePath, data);
return safeOverwriteFile(this.storePath, JSON.stringify(data));
}
}
interface ICreditsStore {
Expand Down
8 changes: 6 additions & 2 deletions datastore/core/endpoints/Datastore.meta.ts
Expand Up @@ -21,7 +21,9 @@ export default new DatastoreApiHandler('Datastore.meta', {
}
if (minimumPrice > 0) {
settlementFeeMicrogons ??= (
await context.sidechainClientManager.defaultClient.getSettings(false, false)
await context.sidechainClientManager.defaultClient
.getSettings(false, false)
.catch(() => ({ settlementFeeMicrogons: 0 }))
).settlementFeeMicrogons;
minimumPrice += settlementFeeMicrogons;
}
Expand Down Expand Up @@ -52,7 +54,9 @@ export default new DatastoreApiHandler('Datastore.meta', {
}
if (pricePerQuery > 0) {
settlementFeeMicrogons ??= (
await context.sidechainClientManager.defaultClient.getSettings(false, false)
await context.sidechainClientManager.defaultClient
.getSettings(false, false)
.catch(() => ({ settlementFeeMicrogons: 0 }))
).settlementFeeMicrogons;
pricePerQuery += settlementFeeMicrogons;
}
Expand Down
12 changes: 4 additions & 8 deletions datastore/core/test/DatastorePayments.test.ts
Expand Up @@ -224,6 +224,7 @@ test('should be able run a Datastore with Credits', async () => {

await expect(client.getCreditsBalance(manifest.versionHash, credits.id)).resolves.toEqual({
balance: 1,
issuedCredits: 1001,
});

await expect(
Expand All @@ -250,13 +251,9 @@ test('should remove an empty Credits from the local cache', async () => {
const manifest = packager.manifest;
await client.upload(await dbx.asBuffer(), { identity: adminIdentity });
const credits = await client.createCredits(manifest.versionHash, 1250, adminIdentity);
await CreditsStore.store(await miner.address, manifest.versionHash, credits);
await expect(
CreditsStore.getPayment(await miner.address, manifest.versionHash, 1250),
).resolves.toBeTruthy();
await expect(
CreditsStore.getPayment(await miner.address, manifest.versionHash, 1),
).resolves.toBeUndefined();
await CreditsStore.store(manifest.versionHash, credits);
await expect(CreditsStore.getPayment(manifest.versionHash, 1250)).resolves.toBeTruthy();
await expect(CreditsStore.getPayment(manifest.versionHash, 1)).resolves.toBeUndefined();
});

test('should be able to embed Credits in a Datastore', async () => {
Expand Down Expand Up @@ -330,7 +327,6 @@ test('should be able to embed Credits in a Datastore', async () => {
expect(DatastoreVm.apiClientCacheByUrl).toEqual({
[`ulx://${await miner.address}`]: expect.any(DatastoreApiClient),
});

});

async function mockSidechainServer(message: ICoreRequestPayload<ISidechainApis, any>) {
Expand Down
2 changes: 1 addition & 1 deletion datastore/docs/advanced/credits.md
Expand Up @@ -8,7 +8,7 @@ Datastores automatically build-in a table (`ulx_credits`) to grant metered acces

When you clone a Datastore, you can embed Credits you've been issued by an upstream Datastore. This enables you to issue Credits to your own users.

To embed credits, you can configure the [remoteDatastoreEmbeddedCredits](./datastore.md#constructor) parameter on your Datastore.
To embed credits, you can configure the [remoteDatastoreEmbeddedCredits](../basics/datastore.md#constructor) parameter on your Datastore.

## Denominations

Expand Down
2 changes: 1 addition & 1 deletion datastore/docs/overview/configuration.md
Expand Up @@ -52,7 +52,7 @@ The server environment a Datastore runs in will alter a few default settings. Cu

Datastore Core installs some administrative features for built-in Datastores. This currently includes things like:

- [Credits](./credits.md): issuing trial credits to consumers of your Datastore(s). An Admin Identity is required to create new Credits.
- [Credits](../advanced/credits.md): issuing trial credits to consumers of your Datastore(s). An Admin Identity is required to create new Credits.
- Access for an Admin to run private javascript functions on [Tables](../basics/table.md), [Functions](../basics/function.md) and [Crawlers](../basics/crawler.md).
- Ability to "upload" packaged Datastores to a live server in `production` [mode](#env).

Expand Down
4 changes: 2 additions & 2 deletions datastore/packager/index.ts
Expand Up @@ -32,8 +32,8 @@ export default class DatastorePackager {
constructor(entrypoint: string, private readonly outDir?: string, private logToConsole = false) {
this.entrypoint = Path.resolve(entrypoint);
this.outDir ??=
UlixeeConfig.load({ entrypoint, workingDirectory: process.cwd() })?.datastoreOutDir ??
Path.dirname(this.entrypoint);
UlixeeConfig.load({ entrypoint: this.entrypoint, workingDirectory: process.cwd() })
?.datastoreOutDir ?? Path.dirname(this.entrypoint);
this.outDir = Path.resolve(this.outDir);
this.filename = Path.basename(this.entrypoint, Path.extname(this.entrypoint));
this.dbx = new DbxFile(this.dbxPath);
Expand Down
38 changes: 38 additions & 0 deletions end-to-end/credits/dataUser.ts
@@ -0,0 +1,38 @@
import CreditsStore from '@ulixee/datastore/lib/CreditsStore';
import DatastoreApiClient from '@ulixee/datastore/lib/DatastoreApiClient';
import { execAndLog } from '../utils';

export default async function main(
datastore: {
credits: { id: string; secret: string; remainingCredits: number };
minerHost: string;
datastoreHash: string;
},
rootDir: string,
): Promise<void> {
const { datastoreHash, credits, minerHost } = datastore;

execAndLog(
`npx @ulixee/datastore credits install ${minerHost}/${datastoreHash}/credits/${credits.id} ${credits.secret}`,
{
cwd: rootDir,
stdio: 'inherit',
},
);

const datastoreClient = new DatastoreApiClient(minerHost);
const pricing = await datastoreClient.getFunctionPricing(datastoreHash, 'default');
const payment = await CreditsStore.getPayment(datastoreHash, pricing.minimumPrice);

const result = await datastoreClient.query(datastoreHash, 'SELECT * FROM default(test => $1)', {
boundValues: [1],
payment,
});

console.log('Result of datastore query is:', result);

execAndLog(`npx @ulixee/datastore credits get ${minerHost}/${datastoreHash} ${credits.id}`, {
cwd: rootDir,
stdio: 'inherit',
});
}
12 changes: 12 additions & 0 deletions end-to-end/credits/datastore/index.ts
@@ -0,0 +1,12 @@
import Datastore, { Function } from '@ulixee/datastore';

export default new Datastore({
functions: {
default: new Function({
pricePerQuery: 50e4, // ~50 cents
run(ctx) {
ctx.Output.emit({ success: true, input: ctx.input });
},
}),
},
});
75 changes: 75 additions & 0 deletions end-to-end/credits/datastoreDev.ts
@@ -0,0 +1,75 @@
import { spawn } from 'child_process';
import * as Path from 'path';
import * as assert from 'assert';
import { execAndLog, getMinerHost } from '../utils';

export default async function main(
needsClosing: (() => Promise<any> | any)[],
rootDir: string,
): Promise<{
credits: { id: string; secret: string; remainingCredits: number };
minerHost: string;
datastoreHash: string;
}> {
// CREATE IDENTITIES
const identityPath = Path.resolve(`${__dirname}/identities/DatastoreDev.json`);
execAndLog(`npx @ulixee/crypto identity --filename="${identityPath}"`, {
stdio: 'inherit',
});

const identityBech32 = execAndLog(
`npx @ulixee/crypto read-identity --filename="${identityPath}"`,
);
assert(identityBech32, 'Must be a valid identity');

// BOOT UP A MINER WITH GIFT CARD RESTRICTIONS
const miner = spawn(`npx @ulixee/miner start`, {
stdio: 'pipe',
cwd: rootDir,
shell: true,
env: {
...process.env,
ULX_SERVER_ADMIN_IDENTITIES: identityBech32,
ULX_IDENTITY_PATH: identityPath,
ULX_DISABLE_CHROMEALIVE: 'true',
},
});
const minerHost = await getMinerHost(miner);
needsClosing.push(() => miner.kill());

// For some reason, nodejs is taking CWD, but then going to closest package.json, so have to prefix with ./credits
const datastoreResult = execAndLog(
`npx @ulixee/datastore deploy ./credits/datastore/index.js -h ${minerHost}`,
{
cwd: __dirname,
env: {
...process.env,
ULX_IDENTITY_PATH: identityPath,
},
},
);

console.log(datastoreResult);
const datastoreMatch = datastoreResult.match(/'dbx1(?:[0-9a-z]+)'/g);
const datastoreHash = datastoreMatch[0].trim().replace(/'/g, '');
console.log('Datastore VersionHash', datastoreHash);

const creditResult = execAndLog(
`npx @ulixee/datastore credits create ${minerHost}/${datastoreHash} -m 500c`,
{
cwd: __dirname,
env: {
...process.env,
ULX_IDENTITY_PATH: identityPath,
},
},
);
const credits = JSON.parse(creditResult);
console.log('Store Credits:', credits);

return {
credits: credits.credit,
datastoreHash,
minerHost,
};
}
28 changes: 28 additions & 0 deletions end-to-end/credits/index.ts
@@ -0,0 +1,28 @@
import '@ulixee/commons/lib/SourceMapSupport';
import * as Path from 'path';
import datastoreDev from './datastoreDev';
import dataUser from './dataUser';

async function main(): Promise<void> {
const needsClosing: (() => Promise<any> | any)[] = [];
let root = __dirname;
while (!root.endsWith('/ulixee')) {
root = Path.dirname(root);
if (root === '/') throw new Error('Root project not found');
}

const buildDir = Path.join(root, 'build');
try {
const result = await datastoreDev(needsClosing, buildDir);
await dataUser(result, buildDir);
console.log('Completed!')
} catch (error) {
console.error(error);
}
for (const closer of needsClosing) {
await closer();
}
process.exit();
}

main().catch(console.error);
39 changes: 0 additions & 39 deletions end-to-end/gift-cards/dataUser.ts

This file was deleted.

0 comments on commit 4355cd1

Please sign in to comment.