Add GitLoader

docs/modules/indexes/document_loaders/examples/git.ipynb

@@ -0,0 +1,129 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Git\n",
+    "\n",
+    "This notebook shows how to load text files from Git repository."
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "First, let's clone repository to the local disk."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from git import Repo\n",
+    "\n",
+    "repo = Repo.clone_from(\n",
+    "    \"https://github.com/hwchase17/langchain\", to_path=\"example_data/test_repo\"\n",
+    ")\n",
+    "branch = repo.head.reference"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.document_loaders.git import GitLoader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loader = GitLoader(path=\"./example_data/test_repo/\", branch=branch)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = loader.load()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "1060"
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "page_content='.venv\\n.github\\n.git\\n.mypy_cache\\n.pytest_cache\\nDockerfile' metadata={'file_path': '.dockerignore', 'file_name': '.dockerignore', 'file_type': ''}\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(data[0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "ai",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.6"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

langchain/document_loaders/git.py

@@ -0,0 +1,72 @@
+import os
+from typing import List, Optional
+
+from langchain.docstore.document import Document
+from langchain.document_loaders.base import BaseLoader
+
+
+class GitLoader(BaseLoader):
+    """Loads files from a local Git repository into a list of documents.
+    Currently supports only text files.
+
+    Each document represents one file in the repository. The `path` points to
+    the local Git repository, and the `branch` specifies the branch to load
+    files from. By default, it loads from the `main` branch.
+    """
+
+    def __init__(
+        self,
+        path: str,
+        branch: Optional[str] = "main",
+    ):
+        self.path = path
+        self.branch = branch
+
+    def load(self) -> List[Document]:
+        try:
+            from git import Blob, Repo
+        except ImportError as ex:
+            raise ImportError(
+                "Could not import git python package. "
+                "Please install it with `pip install GitPython`."
+            ) from ex
+
+        repo = Repo(self.path)
+        repo.git.checkout(self.branch)
+
+        docs: List[Document] = []
+
+        for item in repo.tree().traverse():
+            if isinstance(item, Blob):
+                file_path = os.path.join(self.path, item.path)
+                rel_file_path = os.path.relpath(file_path, self.path)
+                try:
+                    with open(file_path, "rb") as f:
+                        content = f.read()
+                        file_type = os.path.splitext(item.name)[1]
+
+                        # loads only text files
+                        if self.is_text_content(content):
+                            metadata = {
+                                "file_path": rel_file_path,
+                                "file_name": item.name,
+                                "file_type": file_type,
+                            }
+                            text_content = content.decode("utf-8", errors="ignore")
+                            doc = Document(page_content=text_content, metadata=metadata)
+                        else:
+                            continue
+                        docs.append(doc)
+                except Exception as e:
+                    print(f"Error reading file {file_path}: {e}")
+
+        return docs
+
+    @staticmethod
+    def is_text_content(content: bytes) -> bool:
+        """Determines if the content is text based on the content bytes."""
+        try:
+            content.decode("utf-8")
+            return True
+        except UnicodeDecodeError:
+            return False