Last minute fixes for demo. (#54)

* Do not load binary files (closes #41) * Edit the full text of the file, not just the prompt. * Finish rename content to conversation for context
DavidSouther · Apr 22, 2024 · e3ec237 · e3ec237
1 parent 8567b35
commit e3ec237
Show file tree

Hide file tree

Showing 16 changed files with 144 additions and 223 deletions.
diff --git a/cli/fs.js b/cli/fs.js
@@ -70,7 +70,7 @@ export async function loadFs(fs, args) {
       content = [];
     }
     const prompt = assertExists(args.values.prompt);
-    const cliContent = makeCLIContent(prompt, settings.context, system, context, root, edit, content, settings.templateView);
+    const cliContent = makeCLIContent(prompt, settings.context, system, context, root, edit, settings.templateView);
     context[cliContent.path] = cliContent;
     content.push(cliContent.path);
   }
@@ -115,20 +115,20 @@ export function makeEdit(lines, content, hasPrompt) {
  * for this Ailly call to the LLM.
  * 
  * @param {string} prompt 
- * @param {'none'|'folder'|'content'} argContext
+ * @param {'none'|'folder'|'conversation'} argContext
  * @param {string} argSystem 
  * @param {Record<string, Content>} context 
  * @param {string} root 
  * @param {Edit|undefined} edit 
- * @param {string[]} content 
  * @param {*} view 
  * @returns Content
  */
-export function makeCLIContent(prompt, argContext, argSystem, context, root, edit, content, view) {
+export function makeCLIContent(prompt, argContext, argSystem, context, root, edit, view) {
+  const inFolder = Object.keys(context).filter(c => dirname(c) == root);
   // When argContext is folder, `folder` is all files in context in root.
-  const folder = argContext == 'folder' ? Object.keys(context).filter(c => dirname(c) == root) : undefined;
-  // When argContext is `content`, `predecessor` is the last item in the root folder.
-  const predecessor = argContext == 'content' ? content.filter(c => dirname(c) == root).at(-1) : undefined;
+  const folder = argContext == 'folder' ? inFolder : undefined;
+  // When argContext is `conversation`, `predecessor` is the last item in the root folder.
+  const predecessor = argContext == 'conversation' ? inFolder.at(-1) : undefined;
   // When argContext is none, system is empty; otherwise, system is argSystem + predecessor's system.
   const system = argContext == "none" ? [] : [{ content: argSystem ?? "", view: {} }, ...((predecessor ? context[predecessor].context.system : undefined) ?? [])];
   const cliContent = {

diff --git a/cli/fs.test.ts b/cli/fs.test.ts
@@ -73,7 +73,6 @@ describe("makeCLIContent", () => {
       state.context,
       state.root,
       edit,
-      content,
       view
     );
 
@@ -97,7 +96,6 @@ describe("makeCLIContent", () => {
     const argContext = "folder";
     const argSystem = "System";
     const edit = undefined;
-    const content = ["/root/a", "/root/b"];
     const view = { prop: "test" };
 
     const cliContent = cli.makeCLIContent(
@@ -107,7 +105,6 @@ describe("makeCLIContent", () => {
       state.context,
       state.root,
       edit,
-      content,
       view
     );
 
@@ -126,12 +123,11 @@ describe("makeCLIContent", () => {
     });
   });
 
-  it("creates a valid Content object with 'content' context", () => {
+  it("creates a valid Content object with 'conversation' context", () => {
     const prompt = "Prompt";
-    const argContext = "content";
+    const argContext = "conversation";
     const argSystem = "System";
     const edit = undefined;
-    const content = ["/root/a", "/root/b"];
     const view = { prop: "test" };
 
     const cliContent = cli.makeCLIContent(
@@ -141,7 +137,6 @@ describe("makeCLIContent", () => {
       state.context,
       state.root,
       edit,
-      content,
       view
     );
 
@@ -160,12 +155,11 @@ describe("makeCLIContent", () => {
     });
   });
 
-  it("handles empty content array with 'content' context", () => {
+  it("handles empty content array with 'conversation' context", () => {
     const prompt = "Prompt";
-    const argContext = "content";
+    const argContext = "conversation";
     const argSystem = "System";
     const edit = undefined;
-    const content = [];
     const view = { prop: "test" };
 
     const cliContent = cli.makeCLIContent(
@@ -175,7 +169,6 @@ describe("makeCLIContent", () => {
       state.context,
       state.root,
       edit,
-      content,
       view
     );
 
@@ -186,7 +179,7 @@ describe("makeCLIContent", () => {
       prompt: "Prompt",
       context: {
         view: { prop: "test" },
-        predecessor: undefined,
+        predecessor: "/root/b",
         system: [{ content: "System", view: {} }],
         folder: undefined,
         edit: undefined,

diff --git a/cli/index.js b/cli/index.js
@@ -86,7 +86,7 @@ async function main() {
  */
 async function doEdit(fs, loaded, edit, prompt, yes) {
   const out = loaded.context[edit.file];
-  const responseLines = out.prompt?.split("\n") ?? [];
+  const responseLines = (out.meta?.text ?? out.prompt)?.split("\n") ?? [];
   const replaceLines = prompt.response?.split("\n") ?? [];
   const editValue = makeEditConfirmMessage(edit, out.name, responseLines, replaceLines);
   console.log(editValue);

diff --git a/core/package.json b/core/package.json
@@ -31,12 +31,14 @@
     "@dqbd/tiktoken": "^1.0.7",
     "gitignore-parser": "^0.0.2",
     "gray-matter": "^4.0.3",
+    "mime-types": "^2.1.35",
     "mustache": "^4.2.0",
     "openai": "^4.11.1",
     "vectra": "^0.6.0",
     "yaml": "^2.4.1"
   },
   "devDependencies": {
+    "@types/mime-types": "^2.1.4",
     "@types/mustache": "^4.2.5",
     "vitest": "^0.34.6"
   }

diff --git a/core/src/content/content.test.ts b/core/src/content/content.test.ts
@@ -44,7 +44,12 @@ test("it loads content", async () => {
         system: [],
         view: {},
       },
-      meta: { combined: false, root: "/", parent: "root" },
+      meta: {
+        combined: false,
+        root: "/",
+        parent: "root",
+        text: "The quick brown",
+      },
     },
     "/20b/40_part.md": {
       name: "40_part.md",
@@ -56,7 +61,12 @@ test("it loads content", async () => {
         system: [],
         view: {},
       },
-      meta: { combined: false, root: "/", parent: "root" },
+      meta: {
+        combined: false,
+        root: "/",
+        parent: "root",
+        text: "fox jumped",
+      },
     },
     "/20b/56_part.md": {
       name: "56_part.md",
@@ -69,7 +79,12 @@ test("it loads content", async () => {
         view: {},
         predecessor: "/20b/40_part.md",
       },
-      meta: { combined: false, root: "/", parent: "root" },
+      meta: {
+        combined: false,
+        root: "/",
+        parent: "root",
+        text: "over the lazy",
+      },
     },
     "/54_a/12_section.md": {
       name: "12_section.md",
@@ -81,7 +96,7 @@ test("it loads content", async () => {
         system: [],
         view: {},
       },
-      meta: { combined: false, root: "/", parent: "root" },
+      meta: { combined: false, root: "/", parent: "root", text: "dog." },
     },
   });
 });
@@ -128,7 +143,13 @@ test("it loads combined prompt and responses", async () => {
         system: [],
         view: {},
       },
-      meta: { isolated: true, combined: true, root: "/", parent: "root" },
+      meta: {
+        isolated: true,
+        combined: true,
+        root: "/",
+        parent: "root",
+        text: "---\nprompt: content\n---\nResponse",
+      },
     },
     "/prompt.md": {
       name: "prompt.md",
@@ -140,7 +161,13 @@ test("it loads combined prompt and responses", async () => {
         system: [],
         view: {},
       },
-      meta: { isolated: true, combined: true, root: "/", parent: "root" },
+      meta: {
+        isolated: true,
+        combined: true,
+        root: "/",
+        parent: "root",
+        text: "---\nprompt: prompt\n---",
+      },
     },
   });
 });
@@ -193,7 +220,13 @@ test("it loads separate prompt and responses", async () => {
         system: [],
         view: {},
       },
-      meta: { isolated: true, combined: false, root: "/", parent: "root" },
+      meta: {
+        isolated: true,
+        combined: false,
+        root: "/",
+        parent: "root",
+        text: "content",
+      },
     },
     "/prompt.md": {
       name: "prompt.md",
@@ -205,7 +238,13 @@ test("it loads separate prompt and responses", async () => {
         system: [],
         view: {},
       },
-      meta: { isolated: true, combined: false, root: "/", parent: "root" },
+      meta: {
+        isolated: true,
+        combined: false,
+        root: "/",
+        parent: "root",
+        text: "prompt",
+      },
     },
   });
 });
@@ -245,6 +284,7 @@ test("it loads separate prompt and responses in different out directors", async
         root: "/root",
         out: "/out",
         parent: "root",
+        text: "content",
       },
     },
     "/root/prompt.md": {
@@ -263,6 +303,7 @@ test("it loads separate prompt and responses in different out directors", async
         root: "/root",
         out: "/out",
         parent: "root",
+        text: "prompt",
       },
     },
   });
@@ -361,18 +402,6 @@ test("it writes deep java prompts and responses", async () => {
   });
 
   expect(content).toEqual({
-    "/root/.gitignore": {
-      name: ".gitignore",
-      path: "/root/.gitignore",
-      outPath: "/out/.gitignore.ailly.md",
-      prompt: "target",
-      response: "",
-      context: {
-        system: [],
-        view: {},
-      },
-      meta: { combined: false, out: "/out", root: "/root", parent: "root" },
-    },
     "/root/src/com/example/Main.java": {
       name: "Main.java",
       path: "/root/src/com/example/Main.java",
@@ -383,11 +412,16 @@ test("it writes deep java prompts and responses", async () => {
         system: [],
         view: {},
       },
-      meta: { combined: false, out: "/out", root: "/root", parent: "root" },
+      meta: {
+        combined: false,
+        out: "/out",
+        root: "/root",
+        parent: "root",
+        text: "class Main {}\n",
+      },
     },
   });
 
-  content["/root/.gitignore"].response = "Response";
   content["/root/src/com/example/Main.java"].response = "Response";
 
   await writeContent(fs, [...Object.values(content)]);
@@ -396,7 +430,6 @@ test("it writes deep java prompts and responses", async () => {
     "/root/.gitignore": "target",
     "/root/src/com/example/Main.java": "class Main {}\n",
     "/root/target/com/example/Main.class": "0xCAFEBABE",
-    "/out/.gitignore.ailly.md": "---\ncombined: false\n---\nResponse",
     "/out/src/com/example/Main.java.ailly.md":
       "---\ncombined: false\n---\nResponse",
   });
@@ -627,6 +660,7 @@ describe("Load aillyrc", () => {
           context: "none",
           parent: "root",
           root: "/root",
+          text: "a",
         },
       },
       "/root/b": {
@@ -643,6 +677,7 @@ describe("Load aillyrc", () => {
           context: "none",
           parent: "root",
           root: "/root",
+          text: "b",
         },
       },
     });
@@ -683,6 +718,7 @@ describe("Load aillyrc", () => {
           context: "folder",
           parent: "root",
           root: "/root",
+          text: "a",
         },
       },
       "/root/b": {
@@ -701,6 +737,7 @@ describe("Load aillyrc", () => {
           context: "folder",
           parent: "root",
           root: "/root",
+          text: "b",
         },
       },
       "/root/c": {
@@ -719,6 +756,7 @@ describe("Load aillyrc", () => {
           context: "folder",
           parent: "root",
           root: "/root",
+          text: "c",
         },
       },
     });

diff --git a/core/src/content/content.ts b/core/src/content/content.ts
@@ -50,6 +50,7 @@ export interface Context {
 export interface ContentMeta {
   root?: string;
   out?: string;
+  text?: string;
   context?: "conversation" | "folder" | "none";
   parent?: "root" | "always" | "never";
   messages?: Message[];
@@ -127,10 +128,12 @@ async function loadFile(
     head.root = head.root ?? cwd;
     const promptPath = join(cwd, file.name);
 
+    let text: string = "";
     let prompt: string = "";
     let data: Record<string, any> = {};
     try {
-      const parsed = matter(await fs.readFile(promptPath).catch((e) => ""));
+      text = await fs.readFile(promptPath).catch((e) => "");
+      const parsed = matter(text);
       prompt = parsed.content;
       data = parsed.data;
     } catch (e) {
@@ -185,6 +188,7 @@ async function loadFile(
       meta: {
         ...head,
         ...data,
+        text,
       },
     };
   } else {