From 52e0ecb2e6f8195a7fb193d0c7fe97b9a6b83d67 Mon Sep 17 00:00:00 2001 From: Wei Zang Date: Mon, 27 Apr 2026 13:12:08 +0100 Subject: [PATCH 1/2] Add GitHub sync and empty endpoints Introduce endpoints to sync live GitHub data into Postgres and to truncate GitHub tables. Adds app/api/github/sql/sync.py (pulls user, repos, gists, projects and starred resources via the GitHub API with pagination, upserts into github_* tables using direct DB connection, uses httpx and requires GITHUB_TOKEN) and app/api/github/sql/empty_tables.py (POST /api/github/empty truncates GitHub-related tables). Registers both routers in app/api/github/__init__.py and adds GITHUB_TOKEN to .env.sample. Both endpoints require the internal API key dependency and include basic error handling/transaction management. --- .env.sample | 1 + app/__init__.py | 2 +- app/api/github/__init__.py | 4 + app/api/github/sql/empty_tables.py | 39 +++++ app/api/github/sql/sync.py | 251 +++++++++++++++++++++++++++++ 5 files changed, 296 insertions(+), 1 deletion(-) create mode 100644 app/api/github/sql/empty_tables.py create mode 100644 app/api/github/sql/sync.py diff --git a/.env.sample b/.env.sample index b596a82..6b8292c 100644 --- a/.env.sample +++ b/.env.sample @@ -1,4 +1,5 @@ PYTHON_KEY= +GITHUB_TOKEN= GEMINI_API_KEY= RESEND_API_KEY= BASE_URL= diff --git a/app/__init__.py b/app/__init__.py index 9e5f6b2..08e1d23 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,4 +1,4 @@ """Python° - FastAPI, Postgres, tsvector""" # Current Version -__version__ = "3.0.3" +__version__ = "3.0.4" diff --git a/app/api/github/__init__.py b/app/api/github/__init__.py index ccd498c..fd52c5f 100644 --- a/app/api/github/__init__.py +++ b/app/api/github/__init__.py @@ -3,7 +3,11 @@ from fastapi import APIRouter from .github import router as _github_router from .sql.create_tables import router as _create_tables_router +from .sql.sync import router as _sync_router +from .sql.empty_tables import router as _empty_tables_router github_router = APIRouter() github_router.include_router(_github_router) github_router.include_router(_create_tables_router) +github_router.include_router(_sync_router) +github_router.include_router(_empty_tables_router) diff --git a/app/api/github/sql/empty_tables.py b/app/api/github/sql/empty_tables.py new file mode 100644 index 0000000..4515f16 --- /dev/null +++ b/app/api/github/sql/empty_tables.py @@ -0,0 +1,39 @@ +"""Empty all GitHub tables.""" + +from fastapi import APIRouter, HTTPException, Depends +from app.utils.make_meta import make_meta +from app.utils.db import get_db_connection_direct +from app.utils.api_key_auth import get_api_key + +router = APIRouter() + +_TABLES = [ + "github_accounts", + "github_repos", + "github_gists", + "github_projects", + "github_resources", +] + + +@router.post("/api/github/empty") +def empty_github_tables(api_key: str = Depends(get_api_key)) -> dict: + """POST /api/github/empty: Truncate all GitHub tables.""" + conn = None + cur = None + try: + conn = get_db_connection_direct() + cur = conn.cursor() + for table in _TABLES: + cur.execute(f"TRUNCATE TABLE {table} RESTART IDENTITY CASCADE;") + conn.commit() + return {"meta": make_meta("success", "All GitHub tables emptied"), "data": {"tables": _TABLES}} + except Exception as e: + if conn is not None: + conn.rollback() + raise HTTPException(status_code=500, detail=str(e)) + finally: + if cur is not None: + cur.close() + if conn is not None: + conn.close() diff --git a/app/api/github/sql/sync.py b/app/api/github/sql/sync.py new file mode 100644 index 0000000..8097bf1 --- /dev/null +++ b/app/api/github/sql/sync.py @@ -0,0 +1,251 @@ +"""Sync live data from the GitHub API into local Postgres tables.""" + +import os +import json +from fastapi import APIRouter, HTTPException, Depends +from app.utils.make_meta import make_meta +from app.utils.db import get_db_connection_direct +from app.utils.api_key_auth import get_api_key +import httpx + +router = APIRouter() + +_GH_BASE = "https://api.github.com" + + +def _gh_headers() -> dict: + token = os.getenv("GITHUB_TOKEN") + if not token: + raise HTTPException(status_code=500, detail="GITHUB_TOKEN env variable not set") + return { + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + } + + +def _get_all_pages(client: httpx.Client, url: str) -> list: + """Follow GitHub pagination and return all items.""" + results = [] + while url: + resp = client.get(url) + resp.raise_for_status() + results.extend(resp.json()) + url = resp.links.get("next", {}).get("url") or "" + return results + + +@router.post("/api/github/sync") +def sync_github(api_key: str = Depends(get_api_key)) -> dict: + """POST /api/github/sync: Pull live data from GitHub API and upsert into Postgres.""" + headers = _gh_headers() + counts = {} + + with httpx.Client(headers=headers, timeout=30) as client: + # --- Account / user profile --- + user_resp = client.get(f"{_GH_BASE}/user") + user_resp.raise_for_status() + user = user_resp.json() + login = user["login"] + + conn = get_db_connection_direct() + cur = conn.cursor() + + try: + cur.execute( + """ + INSERT INTO github_accounts + (github_user_id, login, name, email, company, blog, location, bio, + avatar_url, html_url, payload, last_synced_at) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s, NOW()) + ON CONFLICT (github_user_id) DO UPDATE SET + login = EXCLUDED.login, + name = EXCLUDED.name, + email = EXCLUDED.email, + company = EXCLUDED.company, + blog = EXCLUDED.blog, + location = EXCLUDED.location, + bio = EXCLUDED.bio, + avatar_url = EXCLUDED.avatar_url, + html_url = EXCLUDED.html_url, + payload = EXCLUDED.payload, + last_synced_at = NOW(), + updated_at = NOW(); + """, + ( + user["id"], login, user.get("name"), user.get("email"), + user.get("company"), user.get("blog"), user.get("location"), + user.get("bio"), user.get("avatar_url"), user.get("html_url"), + json.dumps({ + "public_repos": user.get("public_repos"), + "followers": user.get("followers"), + "following": user.get("following"), + "created_at": user.get("created_at"), + }), + ), + ) + counts["accounts"] = 1 + + # --- Repos --- + repos = _get_all_pages(client, f"{_GH_BASE}/user/repos?per_page=100&type=all") + for r in repos: + cur.execute( + """ + INSERT INTO github_repos + (github_repo_id, account_login, name, full_name, private, fork, + archived, disabled, default_branch, language, stargazers_count, + watchers_count, forks_count, open_issues_count, size_kb, + pushed_at, created_at_github, updated_at_github, + html_url, api_url, payload, synced_at) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s, NOW()) + ON CONFLICT (github_repo_id) DO UPDATE SET + name = EXCLUDED.name, + full_name = EXCLUDED.full_name, + private = EXCLUDED.private, + fork = EXCLUDED.fork, + archived = EXCLUDED.archived, + disabled = EXCLUDED.disabled, + default_branch = EXCLUDED.default_branch, + language = EXCLUDED.language, + stargazers_count = EXCLUDED.stargazers_count, + watchers_count = EXCLUDED.watchers_count, + forks_count = EXCLUDED.forks_count, + open_issues_count = EXCLUDED.open_issues_count, + size_kb = EXCLUDED.size_kb, + pushed_at = EXCLUDED.pushed_at, + updated_at_github = EXCLUDED.updated_at_github, + html_url = EXCLUDED.html_url, + api_url = EXCLUDED.api_url, + payload = EXCLUDED.payload, + synced_at = NOW(), + updated_at = NOW(); + """, + ( + r["id"], login, r["name"], r["full_name"], + r.get("private", False), r.get("fork", False), + r.get("archived", False), r.get("disabled", False), + r.get("default_branch"), r.get("language"), + r.get("stargazers_count", 0), r.get("watchers_count", 0), + r.get("forks_count", 0), r.get("open_issues_count", 0), + r.get("size", 0), r.get("pushed_at"), + r.get("created_at"), r.get("updated_at"), + r.get("html_url"), r.get("url"), + json.dumps({ + "topics": r.get("topics", []), + "license": r.get("license"), + "visibility": r.get("visibility"), + "description": r.get("description"), + }), + ), + ) + counts["repos"] = len(repos) + + # --- Gists --- + gists = _get_all_pages(client, f"{_GH_BASE}/gists?per_page=100") + for g in gists: + cur.execute( + """ + INSERT INTO github_gists + (gist_id, owner_login, description, public, files_count, + comments_count, html_url, api_url, + created_at_github, updated_at_github, payload, synced_at) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s, NOW()) + ON CONFLICT (gist_id) DO UPDATE SET + description = EXCLUDED.description, + public = EXCLUDED.public, + files_count = EXCLUDED.files_count, + comments_count = EXCLUDED.comments_count, + updated_at_github = EXCLUDED.updated_at_github, + payload = EXCLUDED.payload, + synced_at = NOW(), + updated_at = NOW(); + """, + ( + g["id"], login, g.get("description"), g.get("public"), + len(g.get("files", {})), g.get("comments", 0), + g.get("html_url"), g.get("url"), + g.get("created_at"), g.get("updated_at"), + json.dumps({ + "files": list(g.get("files", {}).keys()), + "forks_count": g.get("forks", 0), + }), + ), + ) + counts["gists"] = len(gists) + + # --- Projects (classic REST v3) --- + projects_resp = client.get( + f"{_GH_BASE}/users/{login}/projects?per_page=100", + headers={**headers, "Accept": "application/vnd.github.inertia-preview+json"}, + ) + projects = projects_resp.json() if projects_resp.status_code == 200 and isinstance(projects_resp.json(), list) else [] + for p in projects: + cur.execute( + """ + INSERT INTO github_projects + (github_project_id, owner_login, owner_type, name, body, + state, number, html_url, api_url, + created_at_github, updated_at_github, payload, synced_at) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s, NOW()) + ON CONFLICT (github_project_id) DO UPDATE SET + name = EXCLUDED.name, + body = EXCLUDED.body, + state = EXCLUDED.state, + updated_at_github = EXCLUDED.updated_at_github, + payload = EXCLUDED.payload, + synced_at = NOW(), + updated_at = NOW(); + """, + ( + p["id"], login, "user", p.get("name"), p.get("body"), + p.get("state"), p.get("number"), + p.get("html_url"), p.get("url"), + p.get("created_at"), p.get("updated_at"), + json.dumps({"columns_url": p.get("columns_url")}), + ), + ) + counts["projects"] = len(projects) + + # --- Starred repos → github_resources --- + starred = _get_all_pages(client, f"{_GH_BASE}/user/starred?per_page=100") + for s in starred: + cur.execute( + """ + INSERT INTO github_resources + (account_login, resource_type, resource_id, resource_name, + resource_url, is_private, state, + created_at_github, updated_at_github, payload, synced_at) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s, NOW()) + ON CONFLICT (resource_type, resource_id) DO UPDATE SET + resource_name = EXCLUDED.resource_name, + resource_url = EXCLUDED.resource_url, + payload = EXCLUDED.payload, + synced_at = NOW(), + updated_at = NOW(); + """, + ( + login, "starred_repo", str(s["id"]), s["full_name"], + s.get("html_url"), s.get("private", False), "active", + s.get("created_at"), s.get("updated_at"), + json.dumps({ + "language": s.get("language"), + "stargazers_count": s.get("stargazers_count"), + "description": s.get("description"), + }), + ), + ) + counts["starred"] = len(starred) + + conn.commit() + + except Exception as e: + conn.rollback() + raise HTTPException(status_code=500, detail=str(e)) + finally: + cur.close() + conn.close() + + return { + "meta": make_meta("success", f"GitHub sync complete for @{login}"), + "data": {"login": login, "synced": counts}, + } From 0152dae069ab6fbda88e836de14c973c59034db7 Mon Sep 17 00:00:00 2001 From: Wei Zang Date: Mon, 27 Apr 2026 19:35:45 +0100 Subject: [PATCH 2/2] Add delete_id endpoint and top_record metadata Introduce a new DELETE /prompt/delete_id endpoint (app/api/prompt/delete_id.py) to remove prompt records by id and return the deleted record. Wire the new router into the prompt package and global routes. Enhance the prompt metadata endpoint to include the latest/top record (id, prompt, completion, time as ISO, model). Add a "title" field to make_meta output. Update tests (tests/test_prompt.py) to cover the top_record behavior and the new delete endpoint, and add necessary datetime import for testing. --- app/api/prompt/__init__.py | 1 + app/api/prompt/delete_id.py | 56 ++++++++++++++++++++++++++++ app/api/prompt/prompt.py | 16 ++++++++ app/api/routes.py | 2 + app/utils/make_meta.py | 1 + tests/test_prompt.py | 73 +++++++++++++++++++++++++++++++++++++ 6 files changed, 149 insertions(+) create mode 100644 app/api/prompt/delete_id.py diff --git a/app/api/prompt/__init__.py b/app/api/prompt/__init__.py index de2e21d..6fc2fba 100644 --- a/app/api/prompt/__init__.py +++ b/app/api/prompt/__init__.py @@ -2,3 +2,4 @@ from .prompt import router as prompt_router from .empty import router as empty_router +from .delete_id import router as delete_id_router diff --git a/app/api/prompt/delete_id.py b/app/api/prompt/delete_id.py new file mode 100644 index 0000000..cc6d91a --- /dev/null +++ b/app/api/prompt/delete_id.py @@ -0,0 +1,56 @@ +from fastapi import APIRouter, Depends + +from app.utils.api_key_auth import get_api_key +from app.utils.db import get_db_connection_direct +from app.utils.make_meta import make_meta + +router = APIRouter() + + +@router.delete("/prompt/delete_id") +def delete_prompt_by_id(id: int, api_key: str = Depends(get_api_key)) -> dict: + """DELETE /prompt/delete_id: delete a prompt record by id.""" + conn = None + cur = None + try: + conn = get_db_connection_direct() + cur = conn.cursor() + cur.execute( + """ + SELECT id, prompt + FROM prompt + WHERE id = %s + LIMIT 1; + """, + (id,), + ) + row = cur.fetchone() + if not row: + return { + "meta": make_meta("error", f"id {id} not found"), + "data": {}, + } + + cur.execute("DELETE FROM prompt WHERE id = %s RETURNING id;", (id,)) + deleted = cur.fetchone() + conn.commit() + if not deleted: + return { + "meta": make_meta("error", f"Failed to delete prompt record for id {id}."), + "data": {}, + } + + return { + "meta": make_meta("success", f"Deleted prompt record for id {id}."), + "data": {"id": row[0], "prompt": row[1]}, + } + except Exception as e: + return { + "meta": make_meta("error", f"Failed to delete prompt record: {str(e)}"), + "data": {}, + } + finally: + if cur: + cur.close() + if conn: + conn.close() \ No newline at end of file diff --git a/app/api/prompt/prompt.py b/app/api/prompt/prompt.py index 023f03e..019809d 100644 --- a/app/api/prompt/prompt.py +++ b/app/api/prompt/prompt.py @@ -26,6 +26,15 @@ def get_prompt_table_metadata(api_key: str = Depends(get_api_key)) -> dict: """ ) columns = [row[0] for row in cur.fetchall()] + cur.execute( + """ + SELECT id, prompt, completion, time, model + FROM prompt + ORDER BY id DESC + LIMIT 1; + """ + ) + top_row = cur.fetchone() cur.close() conn.close() meta = make_meta("success", "Prompt table metadata") @@ -34,6 +43,13 @@ def get_prompt_table_metadata(api_key: str = Depends(get_api_key)) -> dict: "data": { "record_count": record_count, "columns": columns, + "top_record": { + "id": top_row[0], + "prompt": top_row[1], + "completion": top_row[2], + "time": top_row[3].isoformat() if top_row and top_row[3] else None, + "model": top_row[4], + } if top_row else None, }, } except Exception as e: diff --git a/app/api/routes.py b/app/api/routes.py index b85cdf8..8af471d 100644 --- a/app/api/routes.py +++ b/app/api/routes.py @@ -10,6 +10,7 @@ from app.utils.notify.resend import router as resend_router from app.api.prompt.prompt import router as prompt_router from app.api.prompt.empty import router as prompts_empty_router +from app.api.prompt.delete_id import router as prompt_delete_id_router from app.api.prospects.prospects import router as prospects_router from app.api.orders.orders import router as orders_router from app.api.queue import router as queue_router @@ -20,6 +21,7 @@ router.include_router(health_router) router.include_router(prompt_router) router.include_router(prompts_empty_router) +router.include_router(prompt_delete_id_router) router.include_router(prospects_router) router.include_router(orders_router) router.include_router(queue_router) diff --git a/app/utils/make_meta.py b/app/utils/make_meta.py index 3107d87..9147b29 100644 --- a/app/utils/make_meta.py +++ b/app/utils/make_meta.py @@ -10,6 +10,7 @@ def make_meta(severity: str, message: str) -> dict: "version": __version__, "time": epoch, "severity": severity, + "title": message, "message": message, "base": base_url, } diff --git a/tests/test_prompt.py b/tests/test_prompt.py index 0129520..4849131 100644 --- a/tests/test_prompt.py +++ b/tests/test_prompt.py @@ -1,4 +1,5 @@ import json +from datetime import datetime, timezone from fastapi.testclient import TestClient from app.main import app @@ -60,6 +61,35 @@ def __init__(self, api_key): self.models = _FakeModels() +def test_prompt_get_includes_top_record(monkeypatch): + conn = _MockConnection( + fetchone_values=[ + (5,), + (501, "Latest prompt", "Latest completion", datetime(2026, 4, 27, 12, 0, tzinfo=timezone.utc), "models/gemini-1.5-pro"), + ] + ) + conn.cursor_obj._fetchall_values = [[("id",), ("prompt",), ("completion",), ("time",), ("model",)]] + + from app.api.prompt import prompt as prompt_module + + monkeypatch.setattr(prompt_module, "get_db_connection_direct", lambda: conn) + + response = client.get("/prompt") + assert response.status_code == 200 + + payload = response.json() + assert payload["meta"]["severity"] == "success" + assert payload["data"]["record_count"] == 5 + assert payload["data"]["columns"] == ["id", "prompt", "completion", "time", "model"] + assert payload["data"]["top_record"] == { + "id": 501, + "prompt": "Latest prompt", + "completion": "Latest completion", + "time": "2026-04-27T12:00:00+00:00", + "model": "models/gemini-1.5-pro", + } + + def test_prompt_post_returns_cached_response(monkeypatch): # Keep auth open for tests and avoid needing a real API key. monkeypatch.setenv("GEMINI_API_KEY", "test-key") @@ -133,3 +163,46 @@ def test_prompt_post_missing_prompt_returns_400(): response = client.post("/prompt", json={}) assert response.status_code == 400 assert response.json()["detail"] == "Missing 'prompt' in request body." + + +def test_prompt_delete_id_deletes_matching_record(monkeypatch): + conn = _MockConnection( + fetchone_values=[ + (303, "Ada Lovelace"), + (303,), + ] + ) + + from app.api.prompt import delete_id as delete_module + + monkeypatch.setattr(delete_module, "get_db_connection_direct", lambda: conn) + + response = client.delete("/prompt/delete_id?id=303") + assert response.status_code == 200 + + payload = response.json() + assert payload["meta"]["severity"] == "success" + assert payload["data"]["id"] == 303 + assert payload["data"]["prompt"] == "Ada Lovelace" + assert conn.committed is True + + +def test_prompt_delete_id_returns_error_when_missing(monkeypatch): + conn = _MockConnection(fetchone_values=[None]) + + from app.api.prompt import delete_id as delete_module + + monkeypatch.setattr(delete_module, "get_db_connection_direct", lambda: conn) + + response = client.delete("/prompt/delete_id?id=999") + assert response.status_code == 200 + + payload = response.json() + assert payload["meta"]["severity"] == "error" + assert payload["data"] == {} + assert conn.committed is False + + +def test_prompt_delete_id_requires_query_string_id(): + response = client.delete("/prompt/delete_id") + assert response.status_code == 422