Add configurable request limit & flag. (#69)

* Add configurable request limit & flag.
DavidSouther · May 1, 2024 · 6ea72b5 · 6ea72b5
1 parent d00314c
commit 6ea72b5
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 3 deletions.
diff --git a/cli/args.js b/cli/args.js
@@ -19,6 +19,7 @@ export function makeArgs(argv = process.argv) {
       "template-view": { type: "string", default: process.env["AILLY_TEMPLATE_VIEW"] },
       prompt: { type: "string", default: process.env["AILLY_PROMPT"], short: "p" },
       system: { type: "string", default: process.env["AILLY_SYSTEM"], short: "s" },
+      "request-limit": { type: "string", default: process.env["AILLY_REQUEST_LIMIT"] },
       temperature: { type: "string", default: "" },
       "update-db": { type: "boolean", default: false },
       "query-db": { type: "string", default: "" },
@@ -65,6 +66,7 @@ export function help() {
 
     --plugin can load a custom RAG plugin. Specify a path to import with "file://./path/to/plugin.mjs". plugin.mjs must export a single default function that meets the PluginBuilder interface in core/src/plugin/index.ts
     --template-view loads a YAML or JSON file to use as a view for the prompt templates. This view will be merged after global, engine, and plugin views but before system and template views.
+    --request-limit will limit the number of requests per call to the provided value. Default value is 5, except for Opus with a default of 1.
 
     --no-overwrite will not run generation on Content with an existing Response.
     --summary will show a pricing expectation before running and prompt for OK.

diff --git a/cli/fs.js b/cli/fs.js
@@ -26,6 +26,7 @@ export async function loadFs(fs, args) {
   const root = resolve(args.values.root ?? '.');
   fs.cd(root);
 
+
   const settings = await ailly.Ailly.makePipelineSettings({
     root,
     out: resolve(args.values.out ?? root),
@@ -37,6 +38,7 @@ export async function loadFs(fs, args) {
     plugin: args.values.plugin,
     templateView: await loadTemplateView(fs, args.values['template-view']),
     overwrite: !args.values["no-overwrite"],
+    requestLimit: args.values['request-limit'] ?? args.values.model?.includes("opus") ? 1 : undefined,
   });
 
   const positionals = args.positionals.slice(2).map(a => resolve(join(root, a)));

diff --git a/core/src/actions/prompt_thread.ts b/core/src/actions/prompt_thread.ts
@@ -1,4 +1,4 @@
-import { PipelineSettings } from "../ailly.js";
+import { DEFAULT_SCHEDULER_LIMIT, PipelineSettings } from "../ailly.js";
 import { View, type Content } from "../content/content.js";
 import type { Engine } from "../engine/index.js";
 import type { Plugin } from "../plugin/index.js";
@@ -22,7 +22,7 @@ export interface PromptThreadSummary extends PromptThreadsSummary {
 
 export async function scheduler<T>(
   taskQueue: Array<() => Promise<T>>,
-  limit: number = 5
+  limit: number = DEFAULT_SCHEDULER_LIMIT
 ): Promise<PromiseSettledResult<T>[]> {
   taskQueue = [...taskQueue].reverse();
   let finished: Array<Promise<T>> = [];
@@ -130,7 +130,8 @@ export class PromptThread {
   private runIsolated(): Promise<PromiseSettledResult<Content>[]> {
     LOGGER.debug(`Running thread for ${this.content.length} isolated prompts`);
     return scheduler(
-      this.content.map((c, i) => () => this.runOne(c, i))
+      this.content.map((c, i) => () => this.runOne(c, i)),
+      this.settings.requestLimit
     ).finally(() => (this.done = true));
   }
 

diff --git a/core/src/ailly.ts b/core/src/ailly.ts
@@ -12,11 +12,14 @@ export type Thread = Content[];
 
 export { GenerateManager } from "./actions/generate_manager.js";
 
+export const DEFAULT_SCHEDULER_LIMIT = 5;
+
 export interface PipelineSettings {
   root: string;
   out: string;
   engine: string;
   model: string;
+  requestLimit: number;
   context: NonNullable<ContentMeta["context"]>;
   plugin: string;
   isolated: boolean;
@@ -30,6 +33,7 @@ export async function makePipelineSettings({
   out = root,
   engine = DEFAULT_ENGINE,
   model,
+  requestLimit = DEFAULT_SCHEDULER_LIMIT,
   context = "conversation",
   plugin = DEFAULT_PLUGIN,
   overwrite = true,
@@ -46,6 +50,7 @@ export async function makePipelineSettings({
   overwrite?: boolean;
   isolated?: boolean;
   combined?: boolean;
+  requestLimit?: number;
   templateView?: View;
 }): Promise<PipelineSettings> {
   model = model ?? (await getEngine(engine)).DEFAULT_MODEL;
@@ -57,6 +62,7 @@ export async function makePipelineSettings({
     out,
     engine,
     model,
+    requestLimit,
     context: context as NonNullable<ContentMeta["context"]>,
     plugin,
     overwrite,