diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a393d1d9..9781d97a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,18 +48,18 @@ Thanks for your interest in contributing to Gitingest! 🚀 Gitingest aims to be pytest ``` -8. Run the app locally using Docker to test your changes (optional): +8. Navigate to src folder 1. Build the Docker image ``` bash - docker build -t gitingest . + cd src ``` - 2. Run the Docker container: + 2. Run the local web server: ``` bash - docker run -d --name gitingest -p 8000:8000 gitingest + uvicorn server.main:app ``` 3. Open your browser and navigate to `http://localhost:8000` to see the app running. diff --git a/Dockerfile b/Dockerfile index cb0eab80..63577a98 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,4 +41,4 @@ USER appuser EXPOSE 8000 -CMD ["python", "-m", "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] +CMD ["python", "-m", "uvicorn", "server.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/src/gitingest/cli.py b/src/gitingest/cli.py index 18a0e106..2163b0e1 100644 --- a/src/gitingest/cli.py +++ b/src/gitingest/cli.py @@ -6,7 +6,7 @@ import click -from config import MAX_FILE_SIZE, OUTPUT_FILE_PATH +from gitingest.config import MAX_FILE_SIZE, OUTPUT_FILE_PATH from gitingest.repository_ingest import ingest diff --git a/src/gitingest/config.py b/src/gitingest/config.py new file mode 100644 index 00000000..291942f3 --- /dev/null +++ b/src/gitingest/config.py @@ -0,0 +1,11 @@ +""" Configuration file for the project. """ + +from pathlib import Path + +MAX_FILE_SIZE = 10 * 1024 * 1024 # 10 MB +MAX_DIRECTORY_DEPTH = 20 # Maximum depth of directory traversal +MAX_FILES = 10_000 # Maximum number of files to process +MAX_TOTAL_SIZE_BYTES = 500 * 1024 * 1024 # 500 MB + +OUTPUT_FILE_PATH = "digest.txt" +TMP_BASE_PATH = Path("/tmp/gitingest") diff --git a/src/gitingest/query_ingestion.py b/src/gitingest/query_ingestion.py index a6f94d23..a0bcfdf7 100644 --- a/src/gitingest/query_ingestion.py +++ b/src/gitingest/query_ingestion.py @@ -6,7 +6,7 @@ import tiktoken -from config import MAX_DIRECTORY_DEPTH, MAX_FILES, MAX_TOTAL_SIZE_BYTES +from gitingest.config import MAX_DIRECTORY_DEPTH, MAX_FILES, MAX_TOTAL_SIZE_BYTES from gitingest.exceptions import ( AlreadyVisitedError, InvalidNotebookError, diff --git a/src/gitingest/query_parser.py b/src/gitingest/query_parser.py index 272ae2d6..be8602f2 100644 --- a/src/gitingest/query_parser.py +++ b/src/gitingest/query_parser.py @@ -9,7 +9,7 @@ from pathlib import Path from urllib.parse import unquote, urlparse -from config import MAX_FILE_SIZE, TMP_BASE_PATH +from gitingest.config import MAX_FILE_SIZE, TMP_BASE_PATH from gitingest.exceptions import InvalidPatternError from gitingest.ignore_patterns import DEFAULT_IGNORE_PATTERNS from gitingest.repository_clone import _check_repo_exists, fetch_remote_branch_list @@ -163,7 +163,7 @@ async def _parse_repo_source(source: str) -> ParsedQuery: _id = str(uuid.uuid4()) slug = f"{user_name}-{repo_name}" - local_path = Path(TMP_BASE_PATH) / _id / slug + local_path = TMP_BASE_PATH / _id / slug url = f"https://{host}/{user_name}/{repo_name}" parsed = ParsedQuery( diff --git a/src/gitingest/repository_ingest.py b/src/gitingest/repository_ingest.py index 64b33ebb..f92c1c2d 100644 --- a/src/gitingest/repository_ingest.py +++ b/src/gitingest/repository_ingest.py @@ -4,7 +4,7 @@ import inspect import shutil -from config import TMP_BASE_PATH +from gitingest.config import TMP_BASE_PATH from gitingest.query_ingestion import run_ingest_query from gitingest.query_parser import ParsedQuery, parse_query from gitingest.repository_clone import CloneConfig, clone_repo diff --git a/src/routers/__init__.py b/src/routers/__init__.py deleted file mode 100644 index d8d24093..00000000 --- a/src/routers/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -""" This module contains the routers for the FastAPI application. """ - -from routers.download import router as download -from routers.dynamic import router as dynamic -from routers.index import router as index - -__all__ = ["download", "dynamic", "index"] diff --git a/src/server/__init__.py b/src/server/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/main.py b/src/server/main.py similarity index 94% rename from src/main.py rename to src/server/main.py index 7bfec181..50fabc92 100644 --- a/src/main.py +++ b/src/server/main.py @@ -10,10 +10,9 @@ from slowapi.errors import RateLimitExceeded from starlette.middleware.trustedhost import TrustedHostMiddleware -from config import templates -from routers import download, dynamic, index -from server_utils import limiter -from utils import lifespan, rate_limit_exception_handler +from server.routers import download, dynamic, index +from server.server_config import templates +from server.server_utils import lifespan, limiter, rate_limit_exception_handler # Load environment variables from .env file load_dotenv() diff --git a/src/query_processor.py b/src/server/query_processor.py similarity index 98% rename from src/query_processor.py rename to src/server/query_processor.py index af7a9079..69fcfc58 100644 --- a/src/query_processor.py +++ b/src/server/query_processor.py @@ -5,11 +5,11 @@ from fastapi import Request from starlette.templating import _TemplateResponse -from config import EXAMPLE_REPOS, MAX_DISPLAY_SIZE, templates from gitingest.query_ingestion import run_ingest_query from gitingest.query_parser import ParsedQuery, parse_query from gitingest.repository_clone import CloneConfig, clone_repo -from server_utils import Colors, log_slider_to_size +from server.server_config import EXAMPLE_REPOS, MAX_DISPLAY_SIZE, templates +from server.server_utils import Colors, log_slider_to_size async def process_query( diff --git a/src/server/routers/__init__.py b/src/server/routers/__init__.py new file mode 100644 index 00000000..ae6666b1 --- /dev/null +++ b/src/server/routers/__init__.py @@ -0,0 +1,7 @@ +""" This module contains the routers for the FastAPI application. """ + +from server.routers.download import router as download +from server.routers.dynamic import router as dynamic +from server.routers.index import router as index + +__all__ = ["download", "dynamic", "index"] diff --git a/src/routers/download.py b/src/server/routers/download.py similarity index 97% rename from src/routers/download.py rename to src/server/routers/download.py index b4da647c..b868444d 100644 --- a/src/routers/download.py +++ b/src/server/routers/download.py @@ -3,7 +3,7 @@ from fastapi import APIRouter, HTTPException from fastapi.responses import Response -from config import TMP_BASE_PATH +from gitingest.config import TMP_BASE_PATH router = APIRouter() diff --git a/src/routers/dynamic.py b/src/server/routers/dynamic.py similarity index 95% rename from src/routers/dynamic.py rename to src/server/routers/dynamic.py index d711655a..74febf8d 100644 --- a/src/routers/dynamic.py +++ b/src/server/routers/dynamic.py @@ -3,9 +3,9 @@ from fastapi import APIRouter, Form, Request from fastapi.responses import HTMLResponse -from config import templates -from query_processor import process_query -from server_utils import limiter +from server.query_processor import process_query +from server.server_config import templates +from server.server_utils import limiter router = APIRouter() diff --git a/src/routers/index.py b/src/server/routers/index.py similarity index 94% rename from src/routers/index.py rename to src/server/routers/index.py index ff226130..5b08a244 100644 --- a/src/routers/index.py +++ b/src/server/routers/index.py @@ -3,9 +3,9 @@ from fastapi import APIRouter, Form, Request from fastapi.responses import HTMLResponse -from config import EXAMPLE_REPOS, templates -from query_processor import process_query -from server_utils import limiter +from server.query_processor import process_query +from server.server_config import EXAMPLE_REPOS, templates +from server.server_utils import limiter router = APIRouter() diff --git a/src/config.py b/src/server/server_config.py similarity index 56% rename from src/config.py rename to src/server/server_config.py index 5be858cd..978a9ba3 100644 --- a/src/config.py +++ b/src/server/server_config.py @@ -1,19 +1,10 @@ -""" Configuration file for the project. """ - -from pathlib import Path +""" Configuration for the server. """ from fastapi.templating import Jinja2Templates -MAX_FILE_SIZE = 10 * 1024 * 1024 # 10 MB -MAX_DIRECTORY_DEPTH = 20 # Maximum depth of directory traversal -MAX_FILES = 10_000 # Maximum number of files to process -MAX_TOTAL_SIZE_BYTES = 500 * 1024 * 1024 # 500 MB - MAX_DISPLAY_SIZE: int = 300_000 DELETE_REPO_AFTER: int = 60 * 60 # In seconds -OUTPUT_FILE_PATH = "digest.txt" -TMP_BASE_PATH = Path("/tmp/gitingest") EXAMPLE_REPOS: list[dict[str, str]] = [ {"name": "Gitingest", "url": "https://github.com/cyclotruc/gitingest"}, @@ -23,4 +14,4 @@ {"name": "ApiAnalytics", "url": "https://github.com/tom-draper/api-analytics"}, ] -templates = Jinja2Templates(directory="templates") +templates = Jinja2Templates(directory="server/templates") diff --git a/src/utils.py b/src/server/server_utils.py similarity index 71% rename from src/utils.py rename to src/server/server_utils.py index 7c968dc0..a316346e 100644 --- a/src/utils.py +++ b/src/server/server_utils.py @@ -1,6 +1,7 @@ -""" Utility functions for the FastAPI server. """ +""" Utility functions for the server. """ import asyncio +import math import shutil import time from contextlib import asynccontextmanager @@ -8,10 +9,15 @@ from fastapi import FastAPI, Request from fastapi.responses import Response -from slowapi import _rate_limit_exceeded_handler +from slowapi import Limiter, _rate_limit_exceeded_handler from slowapi.errors import RateLimitExceeded +from slowapi.util import get_remote_address -from config import DELETE_REPO_AFTER, TMP_BASE_PATH +from gitingest.config import TMP_BASE_PATH +from server.server_config import DELETE_REPO_AFTER + +# Initialize a rate limiter +limiter = Limiter(key_func=get_remote_address) async def rate_limit_exception_handler(request: Request, exc: Exception) -> Response: @@ -136,3 +142,53 @@ async def _process_folder(folder: Path) -> None: shutil.rmtree(folder) except Exception as e: print(f"Error deleting {folder}: {e}") + + +def log_slider_to_size(position: int) -> int: + """ + Convert a slider position to a file size in bytes using a logarithmic scale. + + Parameters + ---------- + position : int + Slider position ranging from 0 to 500. + + Returns + ------- + int + File size in bytes corresponding to the slider position. + """ + maxp = 500 + minv = math.log(1) + maxv = math.log(102_400) + return round(math.exp(minv + (maxv - minv) * pow(position / maxp, 1.5))) * 1024 + + +## Color printing utility +class Colors: + """ANSI color codes""" + + BLACK = "\033[0;30m" + RED = "\033[0;31m" + GREEN = "\033[0;32m" + BROWN = "\033[0;33m" + BLUE = "\033[0;34m" + PURPLE = "\033[0;35m" + CYAN = "\033[0;36m" + LIGHT_GRAY = "\033[0;37m" + DARK_GRAY = "\033[1;30m" + LIGHT_RED = "\033[1;31m" + LIGHT_GREEN = "\033[1;32m" + YELLOW = "\033[1;33m" + LIGHT_BLUE = "\033[1;34m" + LIGHT_PURPLE = "\033[1;35m" + LIGHT_CYAN = "\033[1;36m" + WHITE = "\033[1;37m" + BOLD = "\033[1m" + FAINT = "\033[2m" + ITALIC = "\033[3m" + UNDERLINE = "\033[4m" + BLINK = "\033[5m" + NEGATIVE = "\033[7m" + CROSSED = "\033[9m" + END = "\033[0m" diff --git a/src/templates/api.jinja b/src/server/templates/api.jinja similarity index 100% rename from src/templates/api.jinja rename to src/server/templates/api.jinja diff --git a/src/templates/base.jinja b/src/server/templates/base.jinja similarity index 100% rename from src/templates/base.jinja rename to src/server/templates/base.jinja diff --git a/src/templates/components/footer.jinja b/src/server/templates/components/footer.jinja similarity index 100% rename from src/templates/components/footer.jinja rename to src/server/templates/components/footer.jinja diff --git a/src/templates/components/git_form.jinja b/src/server/templates/components/git_form.jinja similarity index 100% rename from src/templates/components/git_form.jinja rename to src/server/templates/components/git_form.jinja diff --git a/src/templates/components/navbar.jinja b/src/server/templates/components/navbar.jinja similarity index 100% rename from src/templates/components/navbar.jinja rename to src/server/templates/components/navbar.jinja diff --git a/src/templates/components/result.jinja b/src/server/templates/components/result.jinja similarity index 100% rename from src/templates/components/result.jinja rename to src/server/templates/components/result.jinja diff --git a/src/templates/git.jinja b/src/server/templates/git.jinja similarity index 100% rename from src/templates/git.jinja rename to src/server/templates/git.jinja diff --git a/src/templates/index.jinja b/src/server/templates/index.jinja similarity index 100% rename from src/templates/index.jinja rename to src/server/templates/index.jinja diff --git a/src/server_utils.py b/src/server_utils.py deleted file mode 100644 index 432bbb2a..00000000 --- a/src/server_utils.py +++ /dev/null @@ -1,59 +0,0 @@ -""" Utility functions for the server. """ - -import math - -from slowapi import Limiter -from slowapi.util import get_remote_address - -# Initialize a rate limiter -limiter = Limiter(key_func=get_remote_address) - - -def log_slider_to_size(position: int) -> int: - """ - Convert a slider position to a file size in bytes using a logarithmic scale. - - Parameters - ---------- - position : int - Slider position ranging from 0 to 500. - - Returns - ------- - int - File size in bytes corresponding to the slider position. - """ - maxp = 500 - minv = math.log(1) - maxv = math.log(102_400) - return round(math.exp(minv + (maxv - minv) * pow(position / maxp, 1.5))) * 1024 - - -## Color printing utility -class Colors: - """ANSI color codes""" - - BLACK = "\033[0;30m" - RED = "\033[0;31m" - GREEN = "\033[0;32m" - BROWN = "\033[0;33m" - BLUE = "\033[0;34m" - PURPLE = "\033[0;35m" - CYAN = "\033[0;36m" - LIGHT_GRAY = "\033[0;37m" - DARK_GRAY = "\033[1;30m" - LIGHT_RED = "\033[1;31m" - LIGHT_GREEN = "\033[1;32m" - YELLOW = "\033[1;33m" - LIGHT_BLUE = "\033[1;34m" - LIGHT_PURPLE = "\033[1;35m" - LIGHT_CYAN = "\033[1;36m" - WHITE = "\033[1;37m" - BOLD = "\033[1m" - FAINT = "\033[2m" - ITALIC = "\033[3m" - UNDERLINE = "\033[4m" - BLINK = "\033[5m" - NEGATIVE = "\033[7m" - CROSSED = "\033[9m" - END = "\033[0m" diff --git a/tests/test_cli.py b/tests/test_cli.py index 86bd2271..0b652390 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,8 +4,8 @@ from click.testing import CliRunner -from config import MAX_FILE_SIZE, OUTPUT_FILE_PATH from gitingest.cli import main +from gitingest.config import MAX_FILE_SIZE, OUTPUT_FILE_PATH def test_cli_with_default_options():