add getExtractionPolicies and update tests

Author: lucasjacks0n

README.md

@@ -1,13 +1,22 @@
 # Indexify TypeScript Client
 
+[![npm version](https://badge.fury.io/js/getindexify.svg)](https://badge.fury.io/js/getindexify)
+
 ## Installation
 
 This is the TypeScript client for interacting with the Indexify service.
 
 To install it, simply run:
 
 ```shell
-npm install indexify
+npm install getindexify
+```
+
+To initialize a client run the following
+```typescript
+import { IndexifyClient } from "getindexify";
+
+const client = await IndexifyClient.createClient();
 ```
 
 ## Usage

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "getindexify",
-  "version": "0.0.9",
+  "version": "0.0.10",
   "description": "This is the TypeScript client for interacting with the Indexify service.",
   "main": "./dist/index.ts",
   "module": "./dist/index.mjs",

package.json

@@ -1,6 +1,4 @@
 import axios, { AxiosInstance, AxiosResponse } from "axios";
-import * as fs from "fs";
-import FormData from "form-data";
 import Extractor from "./extractor";
 import {
   IContent,
@@ -12,7 +10,7 @@ import {
   ITask,
   IAddExtractorPolicyResponse,
   IDocument,
-  ISearchDocument,
+  ISearchIndexResponse,
 } from "./types";
 
 const DEFAULT_SERVICE_URL = "http://localhost:8900"; // Set your default service URL
@@ -126,7 +124,7 @@ class IndexifyClient {
     name: string,
     query: string,
     topK: number
-  ): Promise<ISearchDocument[]> {
+  ): Promise<ISearchIndexResponse[]> {
     const resp = await this.client.post("search", {
       index: name,
       query,
@@ -143,8 +141,12 @@ class IndexifyClient {
       name: extractionPolicy.name,
       input_params: extractionPolicy.input_params,
       filters_eq: extractionPolicy.labels_eq,
-      content_source: extractionPolicy.content_source,
+      content_source: extractionPolicy.content_source ?? "ingestion",
     });
+
+    // update this.extractor_bindings
+    await this.getExtractionPolicies()
+
     return resp.data;
   }
 
@@ -214,14 +216,41 @@ class IndexifyClient {
     return resp.data.tasks;
   }
 
-  async uploadFile(filePath: string): Promise<any> {
-    const formData = new FormData();
-    formData.append("file", fs.createReadStream(filePath));
-    await this.client.post("upload_file", formData, {
-      headers: {
-        ...formData.getHeaders(),
-      },
-    });
+  async uploadFile(fileInput: string | Blob): Promise<any> {
+    function isBlob(input: any): input is Blob {
+      return input instanceof Blob;
+    }
+
+    if (typeof window === "undefined") {
+      // node
+      if (typeof fileInput !== "string") {
+        throw Error("Expected string")
+      }
+      const FormData = require("form-data");
+      const fs = require("fs");
+      const formData = new FormData();
+      formData.append("file", fs.createReadStream(fileInput as string));
+      await this.client.post("upload_file", formData, {
+        headers: {
+          ...formData.getHeaders(),
+        },
+      });
+    } else {
+      // browser
+      if (!isBlob(fileInput)) {
+        throw Error("Expected blob")
+      }
+      const formData = new FormData();
+      formData.append("file", fileInput);
+      await this.client.post("/upload_file", formData);
+    }
+  }
+
+  async getExtractionPolicies(): Promise<IExtractionPolicy[]> {
+    const resp = await this.client.get("")
+    const policies = resp.data.namespace?.extraction_policies ?? []
+    this.extractionPolicies = policies
+    return policies
   }
 }
 

src/client.ts

@@ -1,3 +1,29 @@
 import IndexifyClient from "./client";
+import Extractor from "./extractor";
+import {
+  INamespace,
+  IEmbeddingSchema,
+  IExtractorSchema,
+  IExtractor,
+  IIndex,
+  IContentMetadata,
+  IContent,
+  IExtractionPolicy,
+  ITask,
+  IDocument,
+} from "./types";
 
-export { IndexifyClient };
+export {
+  IndexifyClient,
+  Extractor,
+  INamespace,
+  IEmbeddingSchema,
+  IExtractorSchema,
+  IExtractor,
+  IIndex,
+  IContentMetadata,
+  IContent,
+  IExtractionPolicy,
+  ITask,
+  IDocument,
+};

src/index.ts

@@ -45,8 +45,8 @@ export interface IExtractionPolicy {
   extractor: string;
   name: string;
   labels_eq?: string;
-  input_params: Record<string, string | number>;
-  content_source: string;
+  input_params?: Record<string, string | number>;
+  content_source?: string;
 }
 
 export interface ITask {
@@ -67,7 +67,7 @@ export interface IDocument {
   labels: Record<string, string>;
 }
 
-export interface ISearchDocument {
+export interface ISearchIndexResponse {
   content_id: string;
   text: string;
   confidence_score: number;

src/types.ts

@@ -9,17 +9,13 @@ test("Create Client", async () => {
 });
 
 test("Create Namespace", async () => {
-  const policy: IExtractionPolicy = {
-    extractor: "tensorlake/minilm-l6",
-    name: "testpolicy",
-    content_source: "ingestion",
-    input_params: {},
-  };
-  const client = await IndexifyClient.createNamespace(
-    "testnamespace",
-    [policy],
-    {}
-  );
+  const client = await IndexifyClient.createNamespace("testnamespace", [
+    {
+      extractor: "tensorlake/minilm-l6",
+      name: "testpolicy",
+    },
+  ]);
+
   expect(client.namespace).toBe("testnamespace");
   // test get namespaces
   const namespaces = await IndexifyClient.namespaces();
@@ -75,8 +71,6 @@ test("Search", async () => {
   const policy: IExtractionPolicy = {
     extractor: "tensorlake/minilm-l6",
     name: "minilml6",
-    content_source: "ingestion",
-    input_params: {},
     labels_eq: "source:test",
   };
 
@@ -87,14 +81,14 @@ test("Search", async () => {
   const indexName = resp.index_names[0];
 
   await client.addDocuments([
-    { text: "This is a test1", labels: {source:"test"} },
-    { text: "This is a test2", labels: {source:"test"} },
+    { text: "This is a test1", labels: { source: "test" } },
+    { text: "This is a test2", labels: { source: "test" } },
   ]);
 
   await new Promise((r) => setTimeout(r, 10000));
 
   const searchResult = await client.searchIndex(indexName, "test", 3);
-  expect(searchResult.length).toBe(2)
+  expect(searchResult.length).toBe(2);
 });
 
 test("Upload file", async () => {
@@ -106,26 +100,39 @@ test("Upload file", async () => {
   };
 
   const client = await IndexifyClient.createNamespace("testuploadfile");
-  client.addExtractionPolicy(policy)
-  await client.uploadFile(`${__dirname}/files/test.txt`)
-})
-
+  client.addExtractionPolicy(policy);
+  await client.uploadFile(`${__dirname}/files/test.txt`);
+  console.log("done");
+});
 
 test("Get content", async () => {
   const client = await IndexifyClient.createNamespace("testgetcontent");
   await client.addDocuments([
-    { text: "This is a test1", labels: {source:"test"} },
-    { text: "This is a test2", labels: {source:"test"} },
+    { text: "This is a test1", labels: { source: "test" } },
+    { text: "This is a test2", labels: { source: "test" } },
   ]);
-  
-  let content
 
-  content = await client.getContent("idontexist")
-  expect(content.length).toBe(0)
+  let content;
 
-  content = await client.getContent(undefined,"source:test")
-  expect(content.length).toBe(2)
+  content = await client.getContent("idontexist");
+  expect(content.length).toBe(0);
+
+  content = await client.getContent(undefined, "source:test");
+  expect(content.length).toBe(2);
+
+  content = await client.getContent(undefined, "source:nothing");
+  expect(content.length).toBe(0);
+});
 
-  content = await client.getContent(undefined,"source:nothing")
-  expect(content.length).toBe(0)
-})
+test("Get Extraction Policies", async () => {
+  const client = await IndexifyClient.createNamespace("testgetpolicies");
+
+  await client.addExtractionPolicy({
+    extractor: "tensorlake/minilm-l6",
+    name: "minilml6",
+  });
+  expect(client.extractionPolicies.length).toBe(1);
+
+  const policies = await client.getExtractionPolicies();
+  expect(policies.length).toBe(1);
+});