feat(web-ui): add private-GitHub ingestion via PAT

src/gitingest/entrypoint.py

@@ -73,6 +73,7 @@ async def ingest_async(
             from_web=False,
             include_patterns=include_patterns,
             ignore_patterns=exclude_patterns,
+            token=token,
         )
 
         if query.url:

src/gitingest/query_parsing.py

@@ -29,6 +29,7 @@ async def parse_query(
     from_web: bool,
     include_patterns: Optional[Union[str, Set[str]]] = None,
     ignore_patterns: Optional[Union[str, Set[str]]] = None,
+    token: Optional[str] = None,
 ) -> IngestionQuery:
     """
     Parse the input source (URL or path) to extract relevant details for the query.
@@ -49,7 +50,10 @@ async def parse_query(
         Patterns to include, by default None. Can be a set of strings or a single string.
     ignore_patterns : Union[str, Set[str]], optional
         Patterns to ignore, by default None. Can be a set of strings or a single string.
-
+    token : str, optional
+        GitHub personal-access token (PAT). Needed when *source* refers to a
+        **private** repository. Can also be set via the ``GITHUB_TOKEN`` env var.
+        Must start with 'github_pat_' or 'gph_' for GitHub repositories.
     Returns
     -------
     IngestionQuery
@@ -59,7 +63,7 @@ async def parse_query(
     # Determine the parsing method based on the source type
     if from_web or urlparse(source).scheme in ("https", "http") or any(h in source for h in KNOWN_GIT_HOSTS):
         # We either have a full URL or a domain-less slug
-        query = await _parse_remote_repo(source)
+        query = await _parse_remote_repo(source, token=token)
     else:
         # Local path scenario
         query = _parse_local_dir_path(source)

src/server/query_processor.py

@@ -1,6 +1,7 @@
 """Process a query by parsing input, cloning a repository, and generating a summary."""
 
 from functools import partial
+from typing import Optional
 
 from fastapi import Request
 from starlette.templating import _TemplateResponse
@@ -19,6 +20,7 @@ async def process_query(
     pattern_type: str = "exclude",
     pattern: str = "",
     is_index: bool = False,
+    token: Optional[str] = None,
 ) -> _TemplateResponse:
     """
     Process a query by parsing input, cloning a repository, and generating a summary.
@@ -40,6 +42,9 @@ async def process_query(
         Pattern to include or exclude in the query, depending on the pattern type.
     is_index : bool
         Flag indicating whether the request is for the index page (default is False).
+    token : str, optional
+        GitHub personal-access token (PAT). Needed when *input_text* refers to a
+        **private** repository.
 
     Returns
     -------
@@ -71,6 +76,7 @@ async def process_query(
         "default_file_size": slider_position,
         "pattern_type": pattern_type,
         "pattern": pattern,
+        "token": token,
     }
 
     try:
@@ -80,12 +86,13 @@ async def process_query(
             from_web=True,
             include_patterns=include_patterns,
             ignore_patterns=exclude_patterns,
+            token=token,
         )
         if not query.url:
             raise ValueError("The 'url' parameter is required.")
 
         clone_config = query.extract_clone_config()
-        await clone_repo(clone_config)
+        await clone_repo(clone_config, token=token)
         summary, tree, content = ingest_query(query)
         with open(f"{clone_config.local_path}.txt", "w", encoding="utf-8") as f:
             f.write(tree + "\n" + content)

src/server/routers/dynamic.py

@@ -50,6 +50,7 @@ async def process_catch_all(
     max_file_size: int = Form(...),
     pattern_type: str = Form(...),
     pattern: str = Form(...),
+    token: str = Form(...),
 ) -> HTMLResponse:
     """
     Process the form submission with user input for query parameters.
@@ -69,18 +70,22 @@ async def process_catch_all(
         The type of pattern used for the query, specified by the user.
     pattern : str
         The pattern string used in the query, specified by the user.
-
+    token : str
+        GitHub personal-access token (PAT). Needed when *input_text* refers to a
+        **private** repository.
     Returns
     -------
     HTMLResponse
         An HTML response generated after processing the form input and query logic,
         which will be rendered and returned to the user.
     """
+    resolved_token = None if token == "" else token
     return await process_query(
         request,
         input_text,
         max_file_size,
         pattern_type,
         pattern,
         is_index=False,
+        token=resolved_token,
     )

src/server/routers/index.py

@@ -47,6 +47,7 @@ async def index_post(
     max_file_size: int = Form(...),
     pattern_type: str = Form(...),
     pattern: str = Form(...),
+    token: str = Form(...),
 ) -> HTMLResponse:
     """
     Process the form submission with user input for query parameters.
@@ -67,18 +68,22 @@ async def index_post(
         The type of pattern used for the query, specified by the user.
     pattern : str
         The pattern string used in the query, specified by the user.
-
+    token : str
+        GitHub personal-access token (PAT). Needed when *input_text* refers to a
+        **private** repository.
     Returns
     -------
     HTMLResponse
         An HTML response containing the results of processing the form input and query logic,
         which will be rendered and returned to the user.
     """
+    resolved_token = None if token == "" else token
     return await process_query(
         request,
         input_text,
         max_file_size,
         pattern_type,
         pattern,
         is_index=True,
+        token=resolved_token,
     )