Rename to NER

src/lib/helpers/http.js

@@ -185,7 +185,7 @@ function skipLoader(config) {
         new RegExp('http(s*)://(.*?)/knowledge/vector/(.*?)/create', 'g'),
         new RegExp('http(s*)://(.*?)/knowledge/document/(.*?)/page', 'g'),
         new RegExp('http(s*)://(.*?)/knowledge/document/(.*?)/upload', 'g'),
-        new RegExp('http(s*)://(.*?)/knowledge/tokenize', 'g'),
+        new RegExp('http(s*)://(.*?)/knowledge/entity/analyze', 'g'),
         new RegExp('http(s*)://(.*?)/users', 'g'),
         new RegExp('http(s*)://(.*?)/instruct/(.*?)', 'g'),
         new RegExp('http(s*)://(.*?)/agent/(.*?)/code-scripts', 'g'),
@@ -229,9 +229,9 @@ function skipLoader(config) {
         new RegExp('http(s*)://(.*?)/llm-provider/(.*?)/models', 'g'),
         new RegExp('http(s*)://(.*?)/knowledge/vector/collections', 'g'),
         new RegExp('http(s*)://(.*?)/knowledge/vector/(.*?)/exist', 'g'),
-        new RegExp('http(s*)://(.*?)/knowledge/document/processors', 'g'),
-        new RegExp('http(s*)://(.*?)/knowledge/tokenizer/providers', 'g'),
-        new RegExp('http(s*)://(.*?)/knowledge/tokenizer/data-providers', 'g'),
+        new RegExp('http(s*)://(.*?)/knowledge/processors', 'g'),
+        new RegExp('http(s*)://(.*?)/knowledge/entity/analyzers', 'g'),
+        new RegExp('http(s*)://(.*?)/knowledge/entity/data-providers', 'g'),
         new RegExp('http(s*)://(.*?)/logger/instruction/log', 'g'),
         new RegExp('http(s*)://(.*?)/logger/instruction/log/keys', 'g'),
         new RegExp('http(s*)://(.*?)/logger/conversation/(.*?)/content-log', 'g'),

src/lib/helpers/types/knowledgeTypes.js

@@ -162,29 +162,29 @@
  */
 
 /**
- * @typedef {Object} TokenizeRequest
+ * @typedef {Object} EntityAnalysisRequest
  * @property {string} text
  * @property {string?} [provider]
- * @property {TokenizeOptions?} [options]
+ * @property {EntityAnalysisOptions?} [options]
  */
 
 /**
- * @typedef {Object} TokenizeOptions
+ * @typedef {Object} EntityAnalysisOptions
  * @property {string[]?} [data_providers]
  * @property {number?} [max_ngram]
  * @property {number?} [cutoff]
  * @property {number?} [top_k]
  */
 
 /**
- * @typedef {Object} TokenizeResponse
- * @property {TokenizeResult[]} [results]
+ * @typedef {Object} EntityAnalysisResponse
+ * @property {EntityAnalysisResult[]} [results]
  * @property {boolean?} [success]
  * @property {string?} [error_message]
  */
 
 /**
- * @typedef {Object} TokenizeResult
+ * @typedef {Object} EntityAnalysisResult
  * @property {string} token
  * @property {string?} [canonical_text]
  * @property {any} data

src/lib/services/api-endpoints.js

@@ -113,11 +113,11 @@ export const endpoints = {
     knowledgeDocumentDeleteUrl: `${host}/knowledge/document/{collection}/delete/{fileId}`,
     knowledgeDocumentDeleteAllUrl: `${host}/knowledge/document/{collection}/delete`,
     knowledgeDocumentPageListUrl: `${host}/knowledge/document/{collection}/page`,
-    knowledgeDocumentProcessorsUrl: `${host}/knowledge/document/processors`,
+    knowledgeProcessorsUrl: `${host}/knowledge/processors`,
 
-    tokenizersUrl: `${host}/knowledge/tokenizer/providers`,
-    tokenizerDataLoadersUrl: `${host}/knowledge/tokenizer/data-providers`,
-    tokenizeUrl: `${host}/knowledge/tokenize`,
+    entityAnalyzersUrl: `${host}/knowledge/entity/analyzers`,
+    entityDataLoadersUrl: `${host}/knowledge/entity/data-providers`,
+    entityAnalyzeUrl: `${host}/knowledge/entity/analyze`,
 
     // chathub 
     chatHubUrl: `${host}/chatHub`,

src/lib/services/knowledge-base-service.js

@@ -196,8 +196,8 @@ export async function getKnowledgeDocumentPageList(collection, request) {
 /**
  * @returns {Promise<string[]>}
  */
-export async function getKnowledgeDocumentProcessors() {
-    const url = endpoints.knowledgeDocumentProcessorsUrl;
+export async function getKnowledgeProcessors() {
+    const url = endpoints.knowledgeProcessorsUrl;
     const response = await axios.get(url);
     return response.data;
 }
@@ -291,27 +291,27 @@ export async function deleteVectorIndexes(collection, options) {
 /**
  * @returns {Promise<string[]>}
  */
-export async function getTokenizers() {
-    const url = endpoints.tokenizersUrl;
+export async function getEntityAnalyzers() {
+    const url = endpoints.entityAnalyzersUrl;
     const response = await axios.get(url);
     return response.data;
 }
 
 /**
  * @returns {Promise<string[]>}
  */
-export async function getTokenizerDataLoaders() {
-    const url = endpoints.tokenizerDataLoadersUrl;
+export async function getEntityDataLoaders() {
+    const url = endpoints.entityDataLoadersUrl;
     const response = await axios.get(url);
     return response.data;
 }
 
 /**
- * @param {import('$knowledgeTypes').TokenizeRequest} request
- * @returns {Promise<import('$knowledgeTypes').TokenizeResponse>}
+ * @param {import('$knowledgeTypes').EntityAnalysisRequest} request
+ * @returns {Promise<import('$knowledgeTypes').EntityAnalysisResponse>}
  */
-export async function tokenize(request) {
-    const url = endpoints.tokenizeUrl;
+export async function analyzeEntity(request) {
+    const url = endpoints.entityAnalyzeUrl;
     const response = await axios.post(url, {
         ...request
     });

src/routes/page/agent/[agentId]/agent-components/agent-instruction.svelte

@@ -87,6 +87,22 @@
         handleAgentChange();
     }
 
+    /** @param {any} e */
+    function onKeyDown(e) {
+        if (e.key === 'Tab') {
+            e.preventDefault();
+
+            if (e.target) {
+                const start = e.target.selectionStart || 0;
+                const end = e.target.selectionEnd || 0;
+                const value = e.target.value || '';
+                e.target.value = value.substring(0, start) + "\t" + value.substring(end);
+                e.target.selectionStart = start + 1;
+                e.target.selectionEnd = start + 1;
+            }
+        }
+    }
+
     function addChannel() {
         inner_instructions = [
             ...inner_instructions,
@@ -193,6 +209,7 @@
                 value={selected_instruction.instruction}
                 rows={20}
                 on:input={(e) => changePrompt(e)}
+                on:keydown={(e) => onKeyDown(e)}
                 placeholder="Enter your instruction"
             />
         </FormGroup>

src/routes/page/agent/[agentId]/agent-components/agent-template.svelte

@@ -80,6 +80,22 @@
         handleAgentChange();
     }
 
+    /** @param {any} e */
+    function onKeyDown(e) {
+        if (e.key === 'Tab') {
+            e.preventDefault();
+
+            if (e.target) {
+                const start = e.target.selectionStart || 0;
+                const end = e.target.selectionEnd || 0;
+                const value = e.target.value || '';
+                e.target.value = value.substring(0, start) + "\t" + value.substring(end);
+                e.target.selectionStart = start + 1;
+                e.target.selectionEnd = start + 1;
+            }
+        }
+    }
+
     function addTemplate() {
         inner_templates = [
             ...inner_templates,
@@ -164,10 +180,11 @@
                 type="textarea"
                 class="form-control"
                 style="scrollbar-width: thin; resize: none;"
+                placeholder="Enter your content"
                 value={selected_template.content}
                 rows={15}
                 on:input={(e) => changePrompt(e)}
-                placeholder="Enter your content"
+                on:keydown={(e) => onKeyDown(e)}
             />
             {/if}
         </FormGroup>

src/routes/page/knowledge-base/dictionary/+page.svelte

@@ -10,9 +10,9 @@
         Table
     } from '@sveltestrap/sveltestrap';
     import {
-		getTokenizers,
-		getTokenizerDataLoaders,
-		tokenize
+		getEntityAnalyzers,
+		getEntityDataLoaders,
+		analyzeEntity
     } from '$lib/services/knowledge-base-service';
 	import Breadcrumb from '$lib/common/Breadcrumb.svelte';
     import HeadTitle from '$lib/common/HeadTitle.svelte';
@@ -40,19 +40,19 @@
 	let elapsedTime = '';
 
 	/** @type {string | null} */
-	let selectedTokenizer = null;
+	let selectedAnalyzer = null;
 
     /** @type {string[]} */
     let selectedDataLoaders = [];
 
-	/** @type {import('$knowledgeTypes').TokenizeResult[]} */
+	/** @type {import('$knowledgeTypes').EntityAnalysisResult[]} */
 	let items = [];
 
 	/** @type {import('$commonTypes').LabelValuePair[]} */
-	let tokenizers = [];
+	let analyzers = [];
 
     /** @type {import('$commonTypes').LabelValuePair[]} */
-    let tokenizerDataLoaders = [];
+    let dataLoaders = [];
 
 	/** @type {number | null | undefined} */
 	let totalDataCount;
@@ -68,7 +68,7 @@
 
 	$: {
 		disableSearchBtn = false;
-		if (!selectedTokenizer || isSearching) {
+		if (!selectedAnalyzer || isSearching) {
 			disableSearchBtn = true;
 		} else if (!text || util.trim(text).length === 0) {
 			disableSearchBtn = true;
@@ -82,8 +82,8 @@
 	function initData() {
 		isLoading = true;
         Promise.all([
-            getTokenizerProviders(),
-            getTokenizerDataLoaderProviders()
+            getAnalyzerProviders(),
+            getDataLoaderProviders()
         ]).finally(() => {
 			isLoading = false;
 		});
@@ -102,7 +102,7 @@
 		elapsedTime = '';
 		const start = new Date();
 
-        getTokenizeResult().finally(() => {
+        getAnalysisResult().finally(() => {
             isSearching = false;
             searchDone = true;
             const gap = new Date().getTime() - start.getTime();
@@ -137,45 +137,45 @@
         selectedDataLoaders = [];
 	}
 
-	function getTokenizerProviders() {
+	function getAnalyzerProviders() {
 		return new Promise((resolve, reject) => {
-			getTokenizers().then(res => {
+			getEntityAnalyzers().then(res => {
 				const retProviders = res?.map(x => ({  label: x, value: x })) || [];
-				tokenizers = [ ...retProviders ];
-				selectedTokenizer = tokenizers[0]?.value;
+				analyzers = [ ...retProviders ];
+				selectedAnalyzer = analyzers[0]?.value;
 				resolve(res);
 			}).catch(err => {
-				tokenizers = [];
-				selectedTokenizer = tokenizers[0]?.value;
+				analyzers = [];
+				selectedAnalyzer = null;
 				reject(err);
 			});
 		});
 	}
 
-    function getTokenizerDataLoaderProviders() {
+    function getDataLoaderProviders() {
 		return new Promise((resolve, reject) => {
-			getTokenizerDataLoaders().then(res => {
+			getEntityDataLoaders().then(res => {
 				const retProviders = res?.map(x => ({  label: x, value: x })) || [];
-				tokenizerDataLoaders = [ ...retProviders ];
+				dataLoaders = [ ...retProviders ];
 				resolve(res);
 			}).catch(err => {
-				tokenizerDataLoaders = [];
+				dataLoaders = [];
 				reject(err);
 			});
 		});
 	}
 
-    function getTokenizeResult() {
+    function getAnalysisResult() {
         return new Promise((resolve, reject) => {
             const request = {
                 text: util.trim(text),
-                provider: selectedTokenizer,
+                provider: selectedAnalyzer,
                 options: {
                     data_providers: selectedDataLoaders?.length > 0 ? selectedDataLoaders : null
                 }
             };
 
-            tokenize(request).then(res => {
+            analyzeEntity(request).then(res => {
                 items = res?.results || [];
                 totalDataCount = items.length;
                 resolve(res);
@@ -188,13 +188,13 @@
     }
 
 	/** @param {any} e */
-	function changeTokenizer(e) {
+	function changeAnalyzer(e) {
 		const selectedValues = e?.detail?.selecteds || [];
-		selectedTokenizer = selectedValues[0]?.value;
+		selectedAnalyzer = selectedValues[0]?.value;
 	}
 
     /** @param {any} e */
-	function changeTokenizerDataLoaders(e) {
+	function changeDataLoaders(e) {
 		const selectedValues = e?.detail?.selecteds || [];
 		// @ts-ignore
 		selectedDataLoaders = selectedValues.map(x => x.value);
@@ -314,27 +314,27 @@
 								<div class="collection-action-container action-container-padding">
 									<div class="collection-dropdown-container">
 										<div class="line-align-center collection-dropdown">
-                                            <div>Tokenizer</div>
+                                            <div>Analyzer</div>
 											<Select
-												tag={'tokenizer-select'}
-												placeholder={'Select Tokenizer'}
+												tag={'entity-analyzer-select'}
+												placeholder={'Select Analyzer'}
 												searchMode
-												selectedValues={selectedTokenizer ? [selectedTokenizer] : []}
-												options={tokenizers}
-												on:select={e => changeTokenizer(e)}
+												selectedValues={selectedAnalyzer ? [selectedAnalyzer] : []}
+												options={analyzers}
+												on:select={e => changeAnalyzer(e)}
 											/>
 										</div>
                                         <div class="line-align-center collection-dropdown">
-                                            <div>Data Loaders</div>
+                                            <div>Data Providers</div>
 											<Select
-												tag={'tokenizer-data-loader-select'}
-												placeholder={'Select Data Loaders'}
+												tag={'entity-data-loader-select'}
+												placeholder={'Select Data Providers'}
 												searchMode
                                                 selectAll
                                                 multiSelect
 												selectedValues={selectedDataLoaders}
-												options={tokenizerDataLoaders}
-												on:select={e => changeTokenizerDataLoaders(e)}
+												options={dataLoaders}
+												on:select={e => changeDataLoaders(e)}
 											/>
 										</div>
 									</div>

src/routes/page/knowledge-base/documents/+page.svelte

@@ -26,7 +26,7 @@
 		createVectorIndexes,
 		deleteVectorIndexes,
 		getVectorCollectionDetails,
-		getKnowledgeDocumentProcessors
+		getKnowledgeProcessors
     } from '$lib/services/knowledge-base-service';
 	import Breadcrumb from '$lib/common/Breadcrumb.svelte';
     import HeadTitle from '$lib/common/HeadTitle.svelte';
@@ -464,7 +464,7 @@
 
 	function getDocProcessors() {
 		return new Promise((resolve, reject) => {
-			getKnowledgeDocumentProcessors().then(res => {
+			getKnowledgeProcessors().then(res => {
 				const retProcessors = res?.map(x => ({  label: x, value: x })) || [];
 				docProcessors = [ ...retProcessors ];
 				resolve(res);