From eb27485913e1ea0b2fbf091bcc49a93c58fc0fde Mon Sep 17 00:00:00 2001 From: Matthew Ratzke Date: Sun, 4 Jun 2023 12:58:05 -0600 Subject: [PATCH 1/4] update github options --- .../nodes/documentloaders/Github/Github.ts | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/packages/components/nodes/documentloaders/Github/Github.ts b/packages/components/nodes/documentloaders/Github/Github.ts index 552790abf6a..dd0eb4e3b77 100644 --- a/packages/components/nodes/documentloaders/Github/Github.ts +++ b/packages/components/nodes/documentloaders/Github/Github.ts @@ -40,6 +40,12 @@ class Github_DocumentLoaders implements INode { placeholder: '', optional: true }, + { + label: 'Recursive', + name: 'recursive', + type: 'boolean', + optional: true + }, { label: 'Text Splitter', name: 'textSplitter', @@ -59,41 +65,37 @@ class Github_DocumentLoaders implements INode { async init(nodeData: INodeData): Promise { const repoLink = nodeData.inputs?.repoLink as string const branch = nodeData.inputs?.branch as string + const recursive = nodeData.inputs?.recursive as boolean const accessToken = nodeData.inputs?.accessToken as string const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const metadata = nodeData.inputs?.metadata const options: GithubRepoLoaderParams = { branch, - recursive: false, - unknown: 'warn' + recursive, + unknown: 'warn', } if (accessToken) options.accessToken = accessToken - const loader = new GithubRepoLoader(repoLink, options) - let docs = [] + console.log('🤖[GithubRepoLoader]: Initializing!'); - if (textSplitter) { - docs = await loader.loadAndSplit(textSplitter) - } else { - docs = await loader.load() - } + const loader = new GithubRepoLoader(repoLink, options) + const docs = textSplitter ? await loader.loadAndSplit(textSplitter) : await loader.load() + console.log('🤖[GithubRepoLoader]: Documents Loaded!'); + if (metadata) { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) - let finaldocs = [] - for (const doc of docs) { - const newdoc = { + return docs.map(doc => { + return { ...doc, metadata: { ...doc.metadata, ...parsedMetadata } } - finaldocs.push(newdoc) - } - return finaldocs + }) } return docs From 3d1b34bf3df14ab1eab8d6aae9b4d6088a55fb39 Mon Sep 17 00:00:00 2001 From: Matthew Ratzke Date: Mon, 5 Jun 2023 12:33:45 -0600 Subject: [PATCH 2/4] added more logging --- packages/components/nodes/documentloaders/Github/Github.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/components/nodes/documentloaders/Github/Github.ts b/packages/components/nodes/documentloaders/Github/Github.ts index dd0eb4e3b77..64c821e872b 100644 --- a/packages/components/nodes/documentloaders/Github/Github.ts +++ b/packages/components/nodes/documentloaders/Github/Github.ts @@ -81,6 +81,7 @@ class Github_DocumentLoaders implements INode { console.log('🤖[GithubRepoLoader]: Initializing!'); const loader = new GithubRepoLoader(repoLink, options) + console.log('🤖[GithubRepoLoader]: Loading documents from ' + repoLink); const docs = textSplitter ? await loader.loadAndSplit(textSplitter) : await loader.load() console.log('🤖[GithubRepoLoader]: Documents Loaded!'); From b7501a9baae6bc84a0efa6b84b79290de1b1e28c Mon Sep 17 00:00:00 2001 From: Matthew Ratzke Date: Sat, 10 Jun 2023 08:59:19 -0600 Subject: [PATCH 3/4] lint fixes --- .../components/nodes/documentloaders/Github/Github.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/components/nodes/documentloaders/Github/Github.ts b/packages/components/nodes/documentloaders/Github/Github.ts index 64c821e872b..209e6b4030e 100644 --- a/packages/components/nodes/documentloaders/Github/Github.ts +++ b/packages/components/nodes/documentloaders/Github/Github.ts @@ -69,26 +69,21 @@ class Github_DocumentLoaders implements INode { const accessToken = nodeData.inputs?.accessToken as string const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const metadata = nodeData.inputs?.metadata - + const options: GithubRepoLoaderParams = { branch, recursive, - unknown: 'warn', + unknown: 'warn' } if (accessToken) options.accessToken = accessToken - console.log('🤖[GithubRepoLoader]: Initializing!'); - const loader = new GithubRepoLoader(repoLink, options) - console.log('🤖[GithubRepoLoader]: Loading documents from ' + repoLink); const docs = textSplitter ? await loader.loadAndSplit(textSplitter) : await loader.load() - console.log('🤖[GithubRepoLoader]: Documents Loaded!'); - if (metadata) { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) - return docs.map(doc => { + return docs.map((doc) => { return { ...doc, metadata: { From 0d41f6a4bfc2688ea62868a1dcb83251398e6548 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Sat, 10 Jun 2023 17:30:28 +0100 Subject: [PATCH 4/4] Update Github.ts lint fix --- packages/components/nodes/documentloaders/Github/Github.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/components/nodes/documentloaders/Github/Github.ts b/packages/components/nodes/documentloaders/Github/Github.ts index 209e6b4030e..bbaad3cb016 100644 --- a/packages/components/nodes/documentloaders/Github/Github.ts +++ b/packages/components/nodes/documentloaders/Github/Github.ts @@ -69,7 +69,7 @@ class Github_DocumentLoaders implements INode { const accessToken = nodeData.inputs?.accessToken as string const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const metadata = nodeData.inputs?.metadata - + const options: GithubRepoLoaderParams = { branch, recursive,