diff --git a/src/gitingest/entrypoint.py b/src/gitingest/entrypoint.py index 13dc8170..f9e65dde 100644 --- a/src/gitingest/entrypoint.py +++ b/src/gitingest/entrypoint.py @@ -73,6 +73,7 @@ async def ingest_async( from_web=False, include_patterns=include_patterns, ignore_patterns=exclude_patterns, + token=token, ) if query.url: diff --git a/src/gitingest/query_parsing.py b/src/gitingest/query_parsing.py index d391e184..089a6f96 100644 --- a/src/gitingest/query_parsing.py +++ b/src/gitingest/query_parsing.py @@ -29,6 +29,7 @@ async def parse_query( from_web: bool, include_patterns: Optional[Union[str, Set[str]]] = None, ignore_patterns: Optional[Union[str, Set[str]]] = None, + token: Optional[str] = None, ) -> IngestionQuery: """ Parse the input source (URL or path) to extract relevant details for the query. @@ -49,7 +50,10 @@ async def parse_query( Patterns to include, by default None. Can be a set of strings or a single string. ignore_patterns : Union[str, Set[str]], optional Patterns to ignore, by default None. Can be a set of strings or a single string. - + token : str, optional + GitHub personal-access token (PAT). Needed when *source* refers to a + **private** repository. Can also be set via the ``GITHUB_TOKEN`` env var. + Must start with 'github_pat_' or 'gph_' for GitHub repositories. Returns ------- IngestionQuery @@ -59,7 +63,7 @@ async def parse_query( # Determine the parsing method based on the source type if from_web or urlparse(source).scheme in ("https", "http") or any(h in source for h in KNOWN_GIT_HOSTS): # We either have a full URL or a domain-less slug - query = await _parse_remote_repo(source) + query = await _parse_remote_repo(source, token=token) else: # Local path scenario query = _parse_local_dir_path(source) diff --git a/src/server/query_processor.py b/src/server/query_processor.py index 00b1c640..1440a5e5 100644 --- a/src/server/query_processor.py +++ b/src/server/query_processor.py @@ -1,6 +1,7 @@ """Process a query by parsing input, cloning a repository, and generating a summary.""" from functools import partial +from typing import Optional from fastapi import Request from starlette.templating import _TemplateResponse @@ -19,6 +20,7 @@ async def process_query( pattern_type: str = "exclude", pattern: str = "", is_index: bool = False, + token: Optional[str] = None, ) -> _TemplateResponse: """ Process a query by parsing input, cloning a repository, and generating a summary. @@ -40,6 +42,9 @@ async def process_query( Pattern to include or exclude in the query, depending on the pattern type. is_index : bool Flag indicating whether the request is for the index page (default is False). + token : str, optional + GitHub personal-access token (PAT). Needed when *input_text* refers to a + **private** repository. Returns ------- @@ -71,6 +76,7 @@ async def process_query( "default_file_size": slider_position, "pattern_type": pattern_type, "pattern": pattern, + "token": token, } try: @@ -80,12 +86,13 @@ async def process_query( from_web=True, include_patterns=include_patterns, ignore_patterns=exclude_patterns, + token=token, ) if not query.url: raise ValueError("The 'url' parameter is required.") clone_config = query.extract_clone_config() - await clone_repo(clone_config) + await clone_repo(clone_config, token=token) summary, tree, content = ingest_query(query) with open(f"{clone_config.local_path}.txt", "w", encoding="utf-8") as f: f.write(tree + "\n" + content) diff --git a/src/server/routers/dynamic.py b/src/server/routers/dynamic.py index bfa31f68..57a54a56 100644 --- a/src/server/routers/dynamic.py +++ b/src/server/routers/dynamic.py @@ -50,6 +50,7 @@ async def process_catch_all( max_file_size: int = Form(...), pattern_type: str = Form(...), pattern: str = Form(...), + token: str = Form(...), ) -> HTMLResponse: """ Process the form submission with user input for query parameters. @@ -69,13 +70,16 @@ async def process_catch_all( The type of pattern used for the query, specified by the user. pattern : str The pattern string used in the query, specified by the user. - + token : str + GitHub personal-access token (PAT). Needed when *input_text* refers to a + **private** repository. Returns ------- HTMLResponse An HTML response generated after processing the form input and query logic, which will be rendered and returned to the user. """ + resolved_token = None if token == "" else token return await process_query( request, input_text, @@ -83,4 +87,5 @@ async def process_catch_all( pattern_type, pattern, is_index=False, + token=resolved_token, ) diff --git a/src/server/routers/index.py b/src/server/routers/index.py index 01b84730..8c11aaa8 100644 --- a/src/server/routers/index.py +++ b/src/server/routers/index.py @@ -47,6 +47,7 @@ async def index_post( max_file_size: int = Form(...), pattern_type: str = Form(...), pattern: str = Form(...), + token: str = Form(...), ) -> HTMLResponse: """ Process the form submission with user input for query parameters. @@ -67,13 +68,16 @@ async def index_post( The type of pattern used for the query, specified by the user. pattern : str The pattern string used in the query, specified by the user. - + token : str + GitHub personal-access token (PAT). Needed when *input_text* refers to a + **private** repository. Returns ------- HTMLResponse An HTML response containing the results of processing the form input and query logic, which will be rendered and returned to the user. """ + resolved_token = None if token == "" else token return await process_query( request, input_text, @@ -81,4 +85,5 @@ async def index_post( pattern_type, pattern, is_index=True, + token=resolved_token, ) diff --git a/src/server/templates/components/git_form.jinja b/src/server/templates/components/git_form.jinja index 764fff70..b45d0f92 100644 --- a/src/server/templates/components/git_form.jinja +++ b/src/server/templates/components/git_form.jinja @@ -17,88 +17,156 @@ element.classList.toggle('hover:text-gray-500'); }); } + + function toggleAccessSettings() { + const container = document.getElementById('accessSettingsContainer'); + const checkbox = document.getElementById('showAccessSettings'); + const row = document.getElementById('controlsRow'); + const show = checkbox.checked; + container.classList.toggle('hidden', !show); + row.classList.toggle('mb-8', show); + }
-
+ -
-
- -
-
-
- + +
+ +
+
+ +
+ +
+
+ +
+ - -
- -
-
-
-
-
- - - - + +
+ +
+ +
+
+
+
+ +
+ + + + +
+ + +
+
+
+ +
+ + +
+
+ +
+ +
+ + +
+ +
+
+
+
+ +
+
+ +
+ + Get your token + + + + +
-
-
- - -
-
+ + {% if show_examples %} -

Try these example repositories:

diff --git a/tests/test_flow_integration.py b/tests/test_flow_integration.py index da12ca82..c85f63ae 100644 --- a/tests/test_flow_integration.py +++ b/tests/test_flow_integration.py @@ -63,6 +63,7 @@ async def test_remote_repository_analysis(request): "max_file_size": "243", "pattern_type": "exclude", "pattern": "", + "token": "", } response = client.post("/", data=form_data) @@ -79,6 +80,7 @@ async def test_invalid_repository_url(request): "max_file_size": "243", "pattern_type": "exclude", "pattern": "", + "token": "", } response = client.post("/", data=form_data) @@ -95,6 +97,7 @@ async def test_large_repository(request): "max_file_size": "243", "pattern_type": "exclude", "pattern": "", + "token": "", } response = client.post("/", data=form_data) @@ -113,6 +116,7 @@ def make_request(): "max_file_size": "243", "pattern_type": "exclude", "pattern": "", + "token": "", } response = client.post("/", data=form_data) assert response.status_code == 200, f"Request failed: {response.text}" @@ -133,6 +137,7 @@ async def test_large_file_handling(request): "max_file_size": "1", "pattern_type": "exclude", "pattern": "", + "token": "", } response = client.post("/", data=form_data) @@ -149,6 +154,7 @@ async def test_repository_with_patterns(request): "max_file_size": "243", "pattern_type": "include", "pattern": "*.md", + "token": "", } response = client.post("/", data=form_data) diff --git a/tests/test_repository_clone.py b/tests/test_repository_clone.py index b57d737e..787456b1 100644 --- a/tests/test_repository_clone.py +++ b/tests/test_repository_clone.py @@ -55,7 +55,7 @@ async def test_clone_without_commit() -> None: When `clone_repo` is called, Then only the clone_repo operation should be performed (no checkout). """ - query = CloneConfig( + clone_config = CloneConfig( url="https://github.com/user/repo", local_path="/tmp/repo", commit=None, @@ -68,9 +68,9 @@ async def test_clone_without_commit() -> None: mock_process.communicate.return_value = (b"output", b"error") mock_exec.return_value = mock_process - await clone_repo(query) + await clone_repo(clone_config) - mock_check.assert_called_once_with(query.url, token=None) + mock_check.assert_called_once_with(clone_config.url, token=None) assert mock_exec.call_count == 1 # Only clone call @@ -107,10 +107,10 @@ async def test_clone_nonexistent_repository() -> None: ) async def test_check_repo_exists(mock_stdout: bytes, return_code: int, expected: bool) -> None: """ - Test the `_check_repo_exists` function with different Git HTTP responses. + Test the `check_repo_exists` function with different Git HTTP responses. Given various stdout lines and return codes: - When `_check_repo_exists` is called, + When `check_repo_exists` is called, Then it should correctly indicate whether the repository exists. """ url = "https://github.com/user/repo" @@ -296,8 +296,8 @@ async def test_clone_specific_branch(tmp_path): branch_name = "main" local_path = tmp_path / "gitingest" - config = CloneConfig(url=repo_url, local_path=str(local_path), branch=branch_name) - await clone_repo(config) + clone_config = CloneConfig(url=repo_url, local_path=str(local_path), branch=branch_name) + await clone_repo(clone_config) # Assertions assert local_path.exists(), "The repository was not cloned successfully." @@ -348,10 +348,7 @@ async def test_clone_creates_parent_directory(tmp_path: Path) -> None: Then it should create the parent directories before attempting to clone. """ nested_path = tmp_path / "deep" / "nested" / "path" / "repo" - clone_config = CloneConfig( - url="https://github.com/user/repo", - local_path=str(nested_path), - ) + clone_config = CloneConfig(url="https://github.com/user/repo", local_path=str(nested_path)) with patch("gitingest.cloning.check_repo_exists", return_value=True): with patch("gitingest.cloning.run_command", new_callable=AsyncMock) as mock_exec: