feat: added databox docpage

.eslintignore

@@ -5,3 +5,4 @@ hero
 website
 stream
 sql/ast/syntax/compiled
+databox/docpage/src

databox/client/interfaces/IDataboxComponents.ts

@@ -5,10 +5,12 @@ export default interface IDataboxComponents<
   TTable extends Table<any>,
   TFunction extends Function<any>,
 > {
-  authenticateIdentity?(identity: string, nonce: string): Promise<boolean> | boolean;
+  name?: string;
+  description?: string;
   remoteDataboxes?: { [source: string]: string };
   tables?: Record<string, TTable>;
   functions?: Record<string, TFunction>;
   paymentAddress?: string;
   giftCardIssuerIdentity?: string;
+  authenticateIdentity?(identity: string, nonce: string): Promise<boolean> | boolean;
 }

databox/client/interfaces/IFunctionComponents.ts

@@ -5,6 +5,7 @@ export default interface IFunctionComponents<
   IAfterRunContext = IContext,
 > {
   name?: string;
+  description?: string;
   pricePerQuery?: number;
   addOnPricing?: {
     perKb?: number;

databox/client/interfaces/ITableComponents.ts

@@ -1,5 +1,6 @@
 export default interface ITableComponents<TSchema> {
   name?: string;
+  description?: string;
   schema: TSchema;
   seedlings?: any[];
   isPublic?: boolean;

databox/client/lib/Databox.ts

@@ -15,6 +15,14 @@ export default class Databox<
 > {
   #databoxInternal: DataboxInternal<TTable, TFunction, TComponents>;
 
+  public get name(): string | undefined {
+    return this.#databoxInternal.components.name;
+  }
+
+  public get description(): string | undefined {
+    return this.#databoxInternal.components.description;
+  }
+
   public get paymentAddress(): string | undefined {
     return this.#databoxInternal.components.paymentAddress;
   }

databox/client/lib/Function.ts

@@ -59,6 +59,10 @@ export default class Function<
     return this.components.name;
   }
 
+  public get description(): string | undefined {
+    return this.components.description;
+  }
+
   public get databox(): Databox<any, any> {
     return this.databoxInternal.databox;
   }

databox/client/lib/Table.ts

@@ -24,6 +24,10 @@ export default class Table<
     return this.components.name;
   }
 
+  public get description(): string | undefined {
+    return this.components.description;
+  }
+
   public get databox(): Databox<any, any> {
     return this.#databoxInternal.databox;
   }

databox/core/bin/databox-process.ts

@@ -30,6 +30,7 @@ process.on('message', async (message: IMessage) => {
       databox.functions ?? ({} as Record<string, Function>),
     )) {
       functionsByName[name] = {
+        description: func.description,
         corePlugins: func.plugins.corePlugins ?? {},
         schema: func.schema,
         pricePerQuery: func.pricePerQuery,
@@ -42,6 +43,7 @@ process.on('message', async (message: IMessage) => {
     const tablesByName: IFetchMetaResponseData['tablesByName'] = {};
     for (const [name, table] of Object.entries(databox.tables ?? {})) {
       tablesByName[name] = {
+        description: table.description,
         schema: table.schema,
         seedlings: table.seedlings,
       };
@@ -50,6 +52,8 @@ process.on('message', async (message: IMessage) => {
     return sendToParent({
       responseId: message.messageId,
       data: {
+        name: databox.name,
+        description: databox.description,
         coreVersion: databox.coreVersion,
         remoteDataboxes: databox.remoteDataboxes,
         paymentAddress: databox.paymentAddress,

databox/core/index.ts

@@ -1,6 +1,9 @@
 import * as Os from 'os';
 import * as Path from 'path';
 import { promises as Fs } from 'fs';
+import { IncomingMessage, ServerResponse } from 'http';
+import * as Finalhandler from 'finalhandler';
+import * as ServeStatic from 'serve-static';
 import IFunctionPluginCore from '@ulixee/databox/interfaces/IFunctionPluginCore';
 import ITransportToClient from '@ulixee/net/interfaces/ITransportToClient';
 import Logger from '@ulixee/commons/lib/Logger';
@@ -90,6 +93,17 @@ export default class DataboxCore {
     return connection;
   }
 
+  public static async routeHttp(req: IncomingMessage, res: ServerResponse, params: string[]): Promise<void> {
+    const pathParts = params[0].match(/([^/]+)(\/(.+)?)?/);
+    const versionHash = pathParts[1];
+    const reqPath = pathParts[3] ? pathParts[2] : '/index.html';
+    const { registryEntry } = await this.databoxRegistry.loadVersion(versionHash);
+    const docpagePath = registryEntry.path.replace(/databox.js$/, 'docpage');
+    req.url = reqPath;
+    const done = Finalhandler(req, res);
+    ServeStatic(docpagePath)(req, res, done);
+  }
+
   public static registerPlugin(pluginCore: IFunctionPluginCore): void {
     this.pluginCoresByName[pluginCore.name] = pluginCore;
   }

databox/core/interfaces/ILocalDataboxProcess.ts

@@ -25,12 +25,15 @@ export interface IExecResponseData {
 }
 
 export interface IFetchMetaResponseData {
+  name: string;
+  description: string;
   coreVersion: string;
   remoteDataboxes?: Record<string, string>;
   paymentAddress?: string;
   giftCardIssuerIdentity?: string;
   functionsByName: {
     [name: string]: {
+      description?: string;
       corePlugins: { [name: string]: string };
       schema?: Omit<IFunctionSchema, 'input' | 'output'> & {
         input?: Record<string, IAnySchemaJson>;
@@ -41,6 +44,7 @@ export interface IFetchMetaResponseData {
   };
   tablesByName: {
     [name: string]: {
+      description?: string;
       schema: Record<string, IAnySchemaJson>;
       seedlings: Record<string, any>;
     }

databox/core/package.json

@@ -15,14 +15,17 @@
     "@ulixee/sql-ast": "2.0.0-alpha.17",
     "@ulixee/sql-engine": "2.0.0-alpha.17",
     "better-sqlite3": "^7.5.1",
+    "finalhandler": "^1.2.0",
     "nanoid": "^3.1.32",
+    "serve-static": "^1.15.0",
     "tar": "^6.1.11",
     "vm2": "^3.9.10"
   },
   "devDependencies": {
     "@ulixee/databox-packager": "2.0.0-alpha.17",
     "@ulixee/databox-testing": "2.0.0-alpha.17",
     "@ulixee/miner": "2.0.0-alpha.17",
+    "axios": "^0.26.1",
     "ts-node": "^10.8.1",
     "zod": "^3.17.10"
   }

databox/core/test/Docpage.test.ts

@@ -0,0 +1,41 @@
+import * as Fs from 'fs';
+import * as Path from 'path';
+import Axios from 'axios';
+import DataboxPackager from '@ulixee/databox-packager';
+import UlixeeMiner from '@ulixee/miner';
+import DataboxApiClient from '@ulixee/databox/lib/DataboxApiClient';
+import UlixeeHostsConfig from '@ulixee/commons/config/hosts';
+import IDataboxManifest from '@ulixee/specification/types/IDataboxManifest';
+
+const storageDir = Path.resolve(process.env.ULX_DATA_DIR ?? '.', 'Databox.docpage.test');
+
+let dbxFile: Buffer;
+let miner: UlixeeMiner;
+let manifest: IDataboxManifest;
+let client: DataboxApiClient;
+
+beforeAll(async () => {
+  jest.spyOn<any, any>(UlixeeHostsConfig.global, 'save').mockImplementation(() => null);
+  const packager = new DataboxPackager(`${__dirname}/databoxes/docpage.js`);
+  await packager.build();
+  dbxFile = await packager.dbx.asBuffer();
+  manifest = packager.manifest.toJSON();
+  miner = new UlixeeMiner();
+  miner.router.databoxConfiguration = { databoxesDir: storageDir };
+  await miner.listen();
+  client = new DataboxApiClient(await miner.address);
+});
+
+afterAll(async () => {
+  await miner?.close();
+  if (Fs.existsSync(storageDir)) {
+    if (Fs.existsSync(storageDir)) Fs.rmSync(storageDir, { recursive: true });
+  }
+});
+
+test('should be able to load databox documentation', async () => {
+  await client.upload(dbxFile);
+  const address = await miner.address;
+  const res = await Axios.get(`http://${address}/databox/${manifest.versionHash}/`);
+  expect(res.data.includes('Docpage - Ulixee Datastore')).toBe(true);
+});

databox/core/test/databoxes/docpage.ts

@@ -0,0 +1,37 @@
+import Databox, { Table, Function } from '@ulixee/databox';
+import { boolean, string } from '@ulixee/schema';
+
+export default new Databox({
+  functions: {
+    test: new Function({
+      run(ctx) {
+        ctx.output = { 
+          testerEcho: ctx.input.shouldTest,
+          greeting: 'Hello world',
+        };
+      },
+      schema: {
+        input: {
+          shouldTest: boolean(),
+        },
+        output: {
+          testerEcho: boolean(),
+          greeting: string(),
+        },
+      },
+    }),
+  },
+  tables: {
+    testers: new Table({
+      schema: {
+        firstName: string(),
+        lastName: string(),
+        isTester: boolean(),
+      },
+      seedlings: [
+        { firstName: 'Caleb', lastName: 'Clark', isTester: true },
+        { firstName: 'Blake', lastName: 'Byrnes' }
+      ]
+    }),
+  }
+});

databox/docpage/babel.config.js

@@ -0,0 +1,10 @@
+module.exports = {
+  presets: [
+    '@vue/cli-plugin-babel/preset'
+  ],
+  "plugins": [
+    ["prismjs", {
+        "languages": ["javascript", "shell"],
+    }]
+  ]
+}

databox/docpage/index.ts

@@ -0,0 +1,19 @@
+import * as Fs from 'fs/promises';
+import * as Path from 'path';
+import * as FsExtra from 'fs-extra';
+
+const distOutputDir = Path.resolve(__dirname, './dist');
+
+export async function buildDocpage(config, outputDir): Promise<void> {
+  FsExtra.copySync(distOutputDir, outputDir);
+  const jsDir = `${outputDir}/js`;
+  const indexJsName = (await Fs.readdir(jsDir)).find(x => x.match(/^index\.[^.]+\.js$/));
+
+  let indexHtml = await Fs.readFile(`${outputDir}/index.html`, 'utf8');
+  indexHtml = indexHtml.replace('Vue App', `${config.name} - Ulixee Datastore`);
+  await Fs.writeFile(`${outputDir}/index.html`, indexHtml);
+
+  let indexJs = await Fs.readFile(`${jsDir}/${indexJsName}`, 'utf8');
+  indexJs = indexJs.replace('["$DATABOX_CONFIG_DATA"]', JSON.stringify(config));
+  await Fs.writeFile(`${jsDir}/${indexJsName}`, indexJs);
+}

databox/docpage/lib/ReplaceConfigFile.js

@@ -0,0 +1,30 @@
+module.exports = class ReplaceConfigFile {
+  constructor(configFile) {
+    this.configFile = configFile;
+  }
+
+  apply(compiler) {
+    const pluginName = this.constructor.name;
+
+    compiler.hooks.compilation.tap(pluginName, compilation => {
+      const NormalModule = compiler.webpack?.NormalModule;
+      const isNormalModuleAvailable = Boolean(NormalModule);
+      if (isNormalModuleAvailable && Boolean(NormalModule.getCompilationHooks)) {
+        NormalModule.getCompilationHooks(compilation).beforeLoaders.tap(pluginName, this.tapCallback.bind(this));
+      } else {
+        compilation.hooks.normalModuleLoader.tap(pluginName, this.tapCallback.bind(this));
+      }
+    });
+  }
+
+  tapCallback(_, normalModule) {
+    if (!normalModule.resource.endsWith('docpage/src/data.config.json')) return;
+    const loader = require.resolve('./dataLoader.js');
+    normalModule.loaders.push({
+      loader,
+      options: {
+        data: this.configFile,
+      },
+    });
+  }
+}

databox/docpage/lib/dataLoader.js

@@ -0,0 +1,4 @@
+module.exports = function dataLoader(source) {
+  const options = this.getOptions() || {};
+  return JSON.stringify(options.data || {});
+}