feat(databox): stream output records as available

databox/README.md

@@ -30,7 +30,8 @@ script.ts
 const Databox = require('@ulixee/databox-plugins-hero');
 
 new Databox(async databox => {
-  const { input, Output, hero } = databox;
+  const { input, Output, Hero } = databox;
+  const hero = new Hero();
   await hero.goto('https://example.org');
   Output.emit({ text: `I went to example.org. Your input was: ${input.params.name}` });
 });

databox/client/index.mjs

@@ -1,9 +1,31 @@
 import { setupAutorunMjsHack } from './lib/utils/Autorun.mjs';
 import cjsImport from './index.js';
 
-const { Databox, Function, FunctionContext, Schema, FunctionSchema, Observable, ConnectionToDataboxCore, Table } = cjsImport;
+const {
+  Databox,
+  ExtractorFunction,
+  CrawlerFunction,
+  PassthroughFunction,
+  FunctionContext,
+  Schema,
+  FunctionSchema,
+  Observable,
+  ConnectionToDataboxCore,
+  Table,
+} = cjsImport;
 
-export { Databox, Function, FunctionContext, Schema, FunctionSchema, Observable, ConnectionToDataboxCore, Table };
+export {
+  Databox,
+  ExtractorFunction,
+  Crawler,
+  PassthroughFunction,
+  FunctionContext,
+  Schema,
+  FunctionSchema,
+  Observable,
+  ConnectionToDataboxCore,
+  Table,
+};
 
 export default cjsImport.default;
 

databox/client/index.ts

@@ -1,7 +1,6 @@
 import '@ulixee/commons/lib/SourceMapSupport';
 import Databox from './lib/Databox';
 import FunctionContext from './lib/FunctionContext';
-import Function from './lib/Function';
 import { Observable } from './lib/ObjectObserver';
 import { FunctionPluginStatics } from './interfaces/IFunctionPluginStatics';
 import IFunctionContext from './interfaces/IFunctionContext';
@@ -12,6 +11,9 @@ import IFunctionSchema, { FunctionSchema } from './interfaces/IFunctionSchema';
 import Table from './lib/Table';
 import ConnectionToDataboxCore from './connections/ConnectionToDataboxCore';
 import PassthroughFunction from './lib/PassthroughFunction';
+import Function from './lib/Function';
+import Crawler from './lib/Crawler';
+import './lib/utils/Autorun';
 
 export * as Schema from '@ulixee/schema';
 
@@ -22,6 +24,7 @@ export {
   FunctionSchema,
   FunctionContext,
   Function,
+  Crawler,
   ConnectionToDataboxCore,
   PassthroughFunction,
   IFunctionComponents,

databox/client/interfaces/ICrawlerComponents.ts

@@ -0,0 +1,13 @@
+import ICrawlerOutputSchema from './ICrawlerOutputSchema';
+
+export default interface ICrawlerComponents<TSchema, TContext, TDisableCache extends boolean = false> {
+  name?: string;
+  pricePerQuery?: number;
+  addOnPricing?: {
+    perKb?: never;
+  };
+  minimumPrice?: number;
+  schema?: TSchema;
+  disableCache?: TDisableCache;
+  run(context: TContext): Promise<{ toCrawlerOutput(): Promise<ICrawlerOutputSchema> }>;
+}

databox/client/interfaces/ICrawlerOutputSchema.ts

@@ -0,0 +1,15 @@
+import { string } from '@ulixee/schema';
+
+export const CrawlerOutputSchema = {
+  crawler: string({ description: 'The type of crawler output that has been produced.' }),
+  version: string({ description: 'The semantic version of the crawler output.' }),
+  sessionId: string({
+    description: 'A session id providing context for how to look up the assets',
+  }),
+};
+
+export default interface ICrawlerOutputSchema {
+  crawler: string;
+  version: string;
+  sessionId: string;
+}

databox/client/interfaces/IDataboxComponents.ts

@@ -1,16 +1,37 @@
+import Crawler from '../lib/Crawler';
 import Function from '../lib/Function';
 import Table from '../lib/Table';
 
 export default interface IDataboxComponents<
-  TTable extends Table<any>,
-  TFunction extends Function<any>,
+  TTable extends TTables<any>,
+  TFunction extends TFunctions<any>,
+  TCrawler extends TCrawlers<any>,
 > {
   name?: string;
   description?: string;
   remoteDataboxes?: { [source: string]: string };
-  tables?: Record<string, TTable>;
-  functions?: Record<string, TFunction>;
+  tables?: TTable;
+  functions?: TFunction;
+  crawlers?: TCrawler;
   paymentAddress?: string;
   giftCardIssuerIdentity?: string;
   authenticateIdentity?(identity: string, nonce: string): Promise<boolean> | boolean;
 }
+
+export type TFunctions<T = any> = T extends Function
+  ? {
+      [K in keyof T]: T[K];
+    }
+  : never;
+
+export type TTables<T = any> = T extends Table
+  ? {
+      [K in keyof T]: T[K];
+    }
+  : never;
+
+export type TCrawlers<T = any> = T extends Crawler
+  ? {
+      [K in keyof T]: T[K];
+    }
+  : never;

databox/client/interfaces/IDataboxEvents.ts

@@ -0,0 +1,3 @@
+export default interface IDataboxEvents {
+  'FunctionStream.output': { output: any };
+}

databox/client/interfaces/IDataboxMetadata.ts

@@ -0,0 +1,30 @@
+import { IAnySchemaJson } from '@ulixee/schema/interfaces/ISchemaJson';
+import IFunctionComponents from './IFunctionComponents';
+
+export default interface IDataboxMetadata {
+  coreVersion: string;
+  remoteDataboxes?: Record<string, string>;
+  paymentAddress?: string;
+  giftCardIssuerIdentity?: string;
+  functionsByName: {
+    [name: string]: {
+      corePlugins: { [name: string]: string };
+      remoteFunction?: string;
+      remoteSource?: string;
+      remoteDataboxVersionHash?: string;
+    } & Omit<IFunctionComponents<any, any>, 'run'>;
+  };
+  crawlersByName: {
+    [name: string]: {
+      corePlugins: { [name: string]: string };
+      remoteFunction?: string;
+      remoteSource?: string;
+      remoteDataboxVersionHash?: string;
+    } & Omit<IFunctionComponents<any, any>, 'run'>;
+  };
+  tablesByName: {
+    [name: string]: {
+      schema: Record<string, IAnySchemaJson>;
+    };
+  };
+}

databox/client/interfaces/IFunctionComponents.ts

@@ -1,9 +1,4 @@
-export default interface IFunctionComponents<
-  ISchema,
-  IContext,
-  IBeforeRunContext = IContext,
-  IAfterRunContext = IContext,
-> {
+export default interface IFunctionComponents<ISchema, IContext> {
   name?: string;
   description?: string;
   pricePerQuery?: number;
@@ -12,7 +7,5 @@ export default interface IFunctionComponents<
   };
   minimumPrice?: number;
   schema?: ISchema;
-  beforeRun?(context: IBeforeRunContext): void | Promise<void>;
   run(context: IContext): void | Promise<void>;
-  afterRun?(context: IAfterRunContext): void | Promise<void>;
 }

databox/client/interfaces/IFunctionContext.ts

@@ -1,14 +1,11 @@
 import IFunctionSchema, { ExtractSchemaType } from './IFunctionSchema';
-import Databox from '../lib/Databox';
 import { IOutputClass } from '../lib/Output';
+import IDataboxMetadata from './IDataboxMetadata';
 
-export default interface IFunctionContext<
-  ISchema extends IFunctionSchema,
-  TOutput = ExtractSchemaType<ISchema['output']>,
-> {
-  input?: ExtractSchemaType<ISchema['input']>;
-  readonly outputs?: TOutput[];
-  readonly Output?: IOutputClass<TOutput>;
-  schema?: ISchema;
-  databox: Databox<any, any>;
+export default interface IFunctionContext<TSchema extends IFunctionSchema> {
+  input?: ExtractSchemaType<TSchema['input']>;
+  readonly outputs?: ExtractSchemaType<TSchema['output']>[];
+  readonly Output?: IOutputClass<ExtractSchemaType<TSchema['output']>>;
+  schema?: TSchema;
+  databoxMetadata: IDataboxMetadata;
 }

databox/client/interfaces/IFunctionPlugin.ts

@@ -7,34 +7,12 @@ export default interface IFunctionPlugin<
   ISchema extends IFunctionSchema,
   IOptions extends IFunctionExecOptions<ISchema> = IFunctionExecOptions<ISchema>,
   IContext extends IFunctionContext<ISchema> = IFunctionContext<ISchema>,
-  IBeforeContext extends IFunctionContext<ISchema> = IFunctionContext<ISchema>,
-  IAfterContext extends IFunctionContext<ISchema> = IFunctionContext<ISchema>,
 > {
   name: string;
   version: string;
   run(
     functionInternal: FunctionInternal<ISchema, IOptions>,
-    lifecycle: IFunctionLifecycle<ISchema, IContext, IBeforeContext, IAfterContext>,
+    context: IContext,
     next: () => Promise<IFunctionContext<ISchema>['outputs']>,
   ): Promise<void>;
 }
-
-export interface IFunctionLifecycle<
-  ISchema extends IFunctionSchema,
-  IContext extends IFunctionContext<ISchema> = IFunctionContext<ISchema>,
-  IBeforeContext extends IFunctionContext<ISchema> = IFunctionContext<ISchema>,
-  IAfterContext extends IFunctionContext<ISchema> = IFunctionContext<ISchema>,
-> {
-  beforeRun: {
-    context: IBeforeContext;
-    isEnabled: boolean;
-  };
-  run: {
-    context: IContext;
-    isEnabled: boolean;
-  };
-  afterRun: {
-    context: IAfterContext;
-    isEnabled: boolean;
-  };
-}

databox/client/interfaces/IFunctionPluginStatics.ts

@@ -7,24 +7,18 @@ export type IFunctionPluginConstructor<
   ISchema,
   IExtraAddons = object,
   TContextAddons = object,
-  TBeforeContextAddons = object,
-  TAfterContextAddons = object,
   TComponentAddons = object,
 > = {
   new (
     components?: IFunctionComponents<ISchema, IFunctionContext<ISchema> & TComponentAddons>,
   ): IFunctionPlugin<
     ISchema,
     IFunctionExecOptions<ISchema> & IExtraAddons,
-    IFunctionContext<ISchema> & TContextAddons,
-    IFunctionContext<ISchema> & TBeforeContextAddons,
-    IFunctionContext<ISchema> & TAfterContextAddons
+    IFunctionContext<ISchema> & TContextAddons
   >;
-  readonly execArgAddons: IExtraAddons;
-  readonly componentAddons: TContextAddons;
-  readonly runContextAddons?: TComponentAddons;
-  readonly beforeRunContextAddons?: TBeforeContextAddons;
-  readonly afterRunContextAddons?: TAfterContextAddons;
+  readonly execArgAddons?: IExtraAddons;
+  readonly componentAddons?: TContextAddons;
+  readonly contextAddons?: TComponentAddons;
 };
 
 // eslint-disable-next-line @typescript-eslint/explicit-function-return-type,@typescript-eslint/no-unused-vars

databox/client/interfaces/IFunctionSchema.ts

@@ -1,22 +1,29 @@
-import { DateUtilities, ExtractSchemaType, ISchemaAny, ObjectSchema } from '@ulixee/schema';
+import { DateUtilities, ExtractSchemaType, ISchemaAny } from '@ulixee/schema';
 
 export { ExtractSchemaType };
 
-type IOutputType = Record<string, ISchemaAny> | ObjectSchema<any>;
+export type ISchemaRecordType<T> = T extends Record<string, ISchemaAny>
+  ? {
+      [K in keyof T]: T[K];
+    }
+  : never;
 
 export default interface IFunctionSchema<
-  TInput extends Record<string, ISchemaAny> = Record<string, ISchemaAny>,
-  TOutput extends IOutputType = IOutputType,
+  TInput extends ISchemaRecordType<any> = ISchemaRecordType<any>,
+  TOutput extends ISchemaRecordType<any> = ISchemaRecordType<any>,
 > {
   input?: TInput;
   output?: TOutput;
   inputExamples?: IInputSchemaType<this['input']>[];
 }
 
-export const FunctionSchema = <TInput extends Record<string, ISchemaAny>, TOutput extends IOutputType>(
+export const FunctionSchema = <
+  TInput extends ISchemaRecordType<any>,
+  TOutput extends ISchemaRecordType<any>,
+>(
   schema: IFunctionSchema<TInput, TOutput>,
 ): IFunctionSchema<TInput, TOutput> => schema;
 
-type IInputSchemaType<T extends Record<string, ISchemaAny>> = {
+type IInputSchemaType<T extends ISchemaRecordType<any>> = {
   [P in keyof T]?: T[P]['$type'] | DateUtilities;
 };

databox/client/interfaces/ITableComponents.ts

@@ -1,7 +1,7 @@
-export default interface ITableComponents<TSchema> {
+export default interface ITableComponents<TSchema, TSeedlings> {
   name?: string;
   description?: string;
   schema: TSchema;
-  seedlings?: any[];
+  seedlings?: TSeedlings[];
   isPublic?: boolean;
-}
+}

databox/client/interfaces/ITableSchema.ts

@@ -1,9 +1,6 @@
-import {
-  ExtractSchemaType,
-  ISchemaAny,
-} from '@ulixee/schema';
+import { ExtractSchemaType, ISchemaAny } from '@ulixee/schema';
 
 export { ExtractSchemaType };
 
 type ITableSchema = Record<string, ISchemaAny>;
-export default ITableSchema;
+export default ITableSchema;