feat: http(s) support for file readers

packages/core/src/readers/CSVReader.ts

@@ -36,15 +36,15 @@ export class PapaCSVReader implements FileReader {
 
   /**
    * Loads data from csv files
-   * @param {string} file - The path to the file to load.
+   * @param {string | URL} filePath - The path to the file to load.
    * @param {GenericFileSystem} [fs=DEFAULT_FS] - The file system to use for reading the file.
    * @returns {Promise<Document[]>}
    */
   async loadData(
-    file: string,
+    filePath: string | URL,
     fs: GenericFileSystem = defaultFS,
   ): Promise<Document[]> {
-    const fileContent = await fs.readFile(file);
+    const fileContent = await fs.readFile(filePath);
     const result = Papa.parse(fileContent, this.papaConfig);
     const textList = result.data.map((row: any) => {
       // Compatible with header row mode

packages/core/src/readers/DocxReader.ts

@@ -7,11 +7,11 @@ import type { FileReader } from "./type.js";
 export class DocxReader implements FileReader {
   /** DocxParser */
   async loadData(
-    file: string,
+    file: string | URL,
     fs: GenericFileSystem = defaultFS,
   ): Promise<Document[]> {
     const dataBuffer = await fs.readRawFile(file);
     const { value } = await mammoth.extractRawText({ buffer: dataBuffer });
-    return [new Document({ text: value, id_: file })];
+    return [new Document({ text: value, id_: `${file}` })];
   }
 }

packages/core/src/readers/HTMLReader.ts

@@ -19,13 +19,13 @@ export class HTMLReader implements FileReader {
    * @returns Promise<Document[]> A Promise object, eventually yielding zero or one Document parsed from the HTML content of the specified file.
    */
   async loadData(
-    file: string,
+    file: string | URL,
     fs: GenericFileSystem = defaultFS,
   ): Promise<Document[]> {
     const dataBuffer = await fs.readFile(file);
     const htmlOptions = this.getOptions();
     const content = await this.parseContent(dataBuffer, htmlOptions);
-    return [new Document({ text: content, id_: file })];
+    return [new Document({ text: content, id_: `${file}` })];
   }
 
   /**

packages/core/src/readers/ImageReader.ts

@@ -16,11 +16,11 @@ export class ImageReader implements FileReader {
    * @returns Promise<Document[]> A Promise object, eventually yielding zero or one ImageDocument of the specified file.
    */
   async loadData(
-    file: string,
+    file: string | URL,
     fs: GenericFileSystem = defaultFS,
   ): Promise<Document[]> {
     const dataBuffer = await fs.readRawFile(file);
     const blob = new Blob([dataBuffer]);
-    return [new ImageDocument({ image: blob, id_: file })];
+    return [new ImageDocument({ image: blob, id_: `${file}` })];
   }
 }

packages/core/src/readers/LlamaParseReader.ts

@@ -34,10 +34,10 @@ export class LlamaParseReader implements FileReader {
   }
 
   async loadData(
-    file: string,
+    file: string | URL,
     fs: GenericFileSystem = defaultFS,
   ): Promise<Document[]> {
-    if (!file.endsWith(".pdf")) {
+    if (file instanceof URL || !file.endsWith(".pdf")) {
       throw new Error("Currently, only PDF files are supported.");
     }
 

packages/core/src/readers/MarkdownReader.ts

@@ -89,7 +89,7 @@ export class MarkdownReader implements FileReader {
   }
 
   async loadData(
-    file: string,
+    file: string | URL,
     fs: GenericFileSystem = defaultFS,
   ): Promise<Document[]> {
     const content = await fs.readFile(file);

packages/core/src/readers/PDFReader.ts

@@ -8,7 +8,7 @@ import type { BaseReader } from "./type.js";
  */
 export class PDFReader implements BaseReader {
   async loadData(
-    file: string,
+    file: string | URL,
     fs: GenericFileSystem = defaultFS,
   ): Promise<Document[]> {
     const content = await fs.readRawFile(file);

packages/core/src/readers/TextFileReader.ts

@@ -9,10 +9,10 @@ import type { BaseReader } from "./type.js";
 
 export class TextFileReader implements BaseReader {
   async loadData(
-    file: string,
+    file: string | URL,
     fs: CompleteFileSystem = defaultFS,
   ): Promise<Document[]> {
     const dataBuffer = await fs.readFile(file);
-    return [new Document({ text: dataBuffer, id_: file })];
+    return [new Document({ text: dataBuffer, id_: `${file}` })];
   }
 }

packages/core/src/readers/type.ts

@@ -12,5 +12,8 @@ export interface BaseReader {
  * A reader takes file paths and imports data into Document objects.
  */
 export interface FileReader extends BaseReader {
-  loadData(filePath: string, fs?: CompleteFileSystem): Promise<Document[]>;
+  loadData(
+    filePath: string | URL,
+    fs?: CompleteFileSystem,
+  ): Promise<Document[]>;
 }

packages/core/tests/readers/csv-reader.test.ts

@@ -0,0 +1,32 @@
+import { PapaCSVReader } from "llamaindex/readers/CSVReader";
+import type { AddressInfo } from "net";
+import { createServer } from "node:http";
+import { expect, test } from "vitest";
+
+const csv = `title,reviewid,creationdate,criticname,originalscore,reviewstate,reviewtext
+Beavers,1145982,2003-05-23,Ivan M. Lincoln,3.5/4,fresh,"Timed to be just long enough for most youngsters' brief attention spans -- and it's packed with plenty of interesting activity, both on land and under the water."
+Blood Mask,1636744,2007-06-02,The Foywonder,1/5,rotten,"It doesn't matter if a movie costs 300 million or only 300 dollars; good is good and bad is bad, and Bloodmask: The Possession of Nicole Lameroux is just plain bad."`;
+
+test("csv reader with http request", async (context) => {
+  const reader = new PapaCSVReader();
+  const server = createServer((req, res) => {
+    expect(req.url).toBe("/test.csv");
+    res.setHeader("Content-Type", "text/csv");
+    res.end(csv);
+  }).listen();
+  context.onTestFinished(() => {
+    server.close();
+  });
+  const port = (server.address() as AddressInfo).port;
+  const url = new URL("http://localhost:" + port + "/test.csv");
+  const documents = await reader.loadData(url);
+  expect(documents.length).toBe(1);
+  expect(documents[0].text).toMatchInlineSnapshot(`
+    "title, reviewid, creationdate, criticname, originalscore, reviewstate, reviewtext
+    Beavers, 1145982, 2003-05-23, Ivan M. Lincoln, 3.5/4, fresh, Timed to be just long enough for most youngsters' brief attention spans -- and it's packed with plenty of interesting activity, both on land and under the water.
+    Blood Mask, 1636744, 2007-06-02, The Foywonder, 1/5, rotten, It doesn't matter if a movie costs 300 million or only 300 dollars; good is good and bad is bad, and Bloodmask: The Possession of Nicole Lameroux is just plain bad."
+  `);
+  expect(documents[0].hash).toMatchInlineSnapshot(
+    `"HlE1SNFxtti8/Rd16ADNKv1uzLd2MIRfHeNCTuAkwDY="`,
+  );
+});

packages/env/src/index.ts

@@ -20,13 +20,25 @@ export function createSHA256(): SHA256 {
 }
 
 export const defaultFS: CompleteFileSystem = {
-  writeFile: function (path: string, content: string) {
+  writeFile: async function (path: string | URL, content: string) {
     return fs.writeFile(path, content, "utf-8");
   },
-  readRawFile(path: string): Promise<Buffer> {
+  readRawFile: async function (path: string | URL): Promise<Buffer> {
+    if (path instanceof URL) {
+      if (path.protocol === "http:" || path.protocol === "https:") {
+        const response = await fetch(path);
+        return Buffer.from(await response.arrayBuffer());
+      }
+    }
     return fs.readFile(path);
   },
-  readFile: function (path: string) {
+  readFile: async function (path: string | URL): Promise<string> {
+    if (path instanceof URL) {
+      if (path.protocol === "http:" || path.protocol === "https:") {
+        const response = await fetch(path);
+        return response.text();
+      }
+    }
     return fs.readFile(path, "utf-8");
   },
   access: fs.access,

packages/env/src/type.ts

@@ -7,35 +7,35 @@ import _ from "lodash";
  * browsers.
  */
 export type GenericFileSystem = {
-  writeFile(path: string, content: string): Promise<void>;
+  writeFile(path: string | URL, content: string): Promise<void>;
   /**
    * Reads a file and returns its content as a raw buffer.
    */
-  readRawFile(path: string): Promise<Buffer>;
+  readRawFile(path: string | URL): Promise<Buffer>;
   /**
    * Reads a file and returns its content as an utf-8 string.
    */
-  readFile(path: string): Promise<string>;
-  access(path: string): Promise<void>;
+  readFile(path: string | URL): Promise<string>;
+  access(path: string | URL): Promise<void>;
   mkdir(
-    path: string,
+    path: string | URL,
     options: {
       recursive: boolean;
     },
   ): Promise<string | undefined>;
-  mkdir(path: string): Promise<void>;
+  mkdir(path: string | URL): Promise<void>;
 };
 export type WalkableFileSystem = {
-  readdir(path: string): Promise<string[]>;
-  stat(path: string): Promise<any>;
+  readdir(path: string | URL): Promise<string[]>;
+  stat(path: string | URL): Promise<any>;
 };
 export type CompleteFileSystem = GenericFileSystem & WalkableFileSystem;
 
 /**
  * A filesystem implementation that stores files in memory.
  */
 export class InMemoryFileSystem implements CompleteFileSystem {
-  private files: Record<string, any> = {};
+  private files: Record<string, string> = {};
 
   async writeFile(
     path: string,
@@ -59,7 +59,10 @@ export class InMemoryFileSystem implements CompleteFileSystem {
   }
 
   async mkdir(path: string): Promise<undefined> {
-    this.files[path] = _.get(this.files, path, null);
+    const content = _.get(this.files, path, null);
+    if (content) {
+      this.files[path] = content;
+    }
   }
 
   async readdir(path: string): Promise<string[]> {