Add YouTube API integration and routes

.env.sample

@@ -10,4 +10,6 @@ DB_USER=
 DB_PASSWORD=
 FLICKR_USER=@N00
 FLICKR_KEY=
-FLICKR_SECRET=
+FLICKR_SECRET=
+YOUTUBE_API_KEY=
+YOUTUBE_CHANNEL_ID=

app/api/flickr/sql/sync.py

@@ -27,7 +27,7 @@ def sync_flickr(api_key: str = Depends(get_api_key)) -> dict:
         "user_id": flickr_user,
         "format": "json",
         "nojsoncallback": 1,
-        "per_page": 10
+        "per_page": 100
     }
     try:
         resp = requests.get(url, params=params)

app/api/routes.py

@@ -17,6 +17,7 @@
 
 from app.api.github import github_router
 from app.api.flickr import flickr_router
+from app.api.youtube import youtube_router
 
 router.include_router(root_router)
 router.include_router(resend_router)
@@ -28,4 +29,5 @@
 router.include_router(orders_router)
 router.include_router(queue_router)
 router.include_router(github_router)
-router.include_router(flickr_router)
+router.include_router(flickr_router)
+router.include_router(youtube_router)

app/api/youtube/README.md

@@ -0,0 +1,26 @@
+# YouTube API Integration
+
+This module provides API routes for accessing YouTube channel data, similar to the GitHub and Flickr integrations. It expects the following environment variable in your `.env` file:
+
+- YOUTUBE_API_KEY
+
+## Endpoints
+
+- **GET /youtube**: Returns counts and recent records from all YouTube tables.
+
+## Proposed Table Design
+
+1. youtube_channels
+   - One row per YouTube channel.
+   - Stores channel identity fields and full raw payload.
+2. youtube_videos
+   - One row per video.
+   - Stores video metadata plus raw JSON payload.
+3. youtube_playlists
+   - One row per playlist.
+   - Stores playlist metadata plus raw JSON payload.
+4. youtube_resources
+   - Generic catch-all for any future YouTube resource type.
+   - Supports additional YouTube objects through jsonb payload storage.
+
+This structure mirrors the GitHub and Flickr integrations for consistency and flexibility.

app/api/youtube/__init__.py

@@ -0,0 +1,14 @@
+"""YouTube Routes"""
+
+from fastapi import APIRouter
+
+from .youtube import router as _youtube_router
+from .sql.create_tables import router as _create_tables_router
+from .sql.empty_tables import router as _empty_tables_router
+from .sql.sync import router as _sync_router
+
+youtube_router = APIRouter()
+youtube_router.include_router(_youtube_router)
+youtube_router.include_router(_create_tables_router)
+youtube_router.include_router(_empty_tables_router)
+youtube_router.include_router(_sync_router)

app/api/youtube/sql/create_tables.py

@@ -0,0 +1,60 @@
+from fastapi import APIRouter, Depends
+from app.utils.db import get_db_connection_direct
+from app.utils.make_meta import make_meta
+from app.utils.api_key_auth import get_api_key
+
+router = APIRouter()
+
+@router.post("/youtube/createtable")
+def create_youtube_tables(api_key: str = Depends(get_api_key)) -> dict:
+    """POST /youtube/createtable: Drop and create YouTube tables in Postgres."""
+    sql_statements = [
+        'DROP TABLE IF EXISTS youtube_resources;',
+        'DROP TABLE IF EXISTS youtube_playlists;',
+        'DROP TABLE IF EXISTS youtube_videos;',
+        'DROP TABLE IF EXISTS youtube_channels;',
+        '''CREATE TABLE IF NOT EXISTS youtube_channels (
+            id SERIAL PRIMARY KEY,
+            youtube_id TEXT UNIQUE,
+            title TEXT,
+            payload JSONB,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        );''',
+        '''CREATE TABLE IF NOT EXISTS youtube_videos (
+            id SERIAL PRIMARY KEY,
+            youtube_id TEXT UNIQUE,
+            title TEXT,
+            payload JSONB,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        );''',
+        '''CREATE TABLE IF NOT EXISTS youtube_playlists (
+            id SERIAL PRIMARY KEY,
+            youtube_id TEXT,
+            title TEXT,
+            payload JSONB,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        );''',
+        '''CREATE TABLE IF NOT EXISTS youtube_resources (
+            id SERIAL PRIMARY KEY,
+            resource_type TEXT,
+            youtube_id TEXT,
+            payload JSONB,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        );'''
+    ]
+    conn = None
+    cur = None
+    try:
+        conn = get_db_connection_direct()
+        cur = conn.cursor()
+        for stmt in sql_statements:
+            cur.execute(stmt)
+        conn.commit()
+        return {"meta": make_meta("success", "YouTube tables created"), "data": {}}
+    except Exception as e:
+        return {"meta": make_meta("error", f"DB error: {str(e)}"), "data": {}}
+    finally:
+        if cur is not None:
+            cur.close()
+        if conn is not None:
+            conn.close()

app/api/youtube/sql/empty_tables.py

@@ -0,0 +1,32 @@
+from fastapi import APIRouter, Depends
+from app.utils.db import get_db_connection_direct
+from app.utils.make_meta import make_meta
+from app.utils.api_key_auth import get_api_key
+
+router = APIRouter()
+
+@router.post("/youtube/emptytables")
+def empty_youtube_tables(api_key: str = Depends(get_api_key)) -> dict:
+    """POST /youtube/emptytables: Delete all rows from all YouTube tables."""
+    tables = [
+        "youtube_channels",
+        "youtube_videos",
+        "youtube_playlists",
+        "youtube_resources"
+    ]
+    conn = None
+    cur = None
+    try:
+        conn = get_db_connection_direct()
+        cur = conn.cursor()
+        for table in tables:
+            cur.execute(f"DELETE FROM {table};")
+        conn.commit()
+        return {"meta": make_meta("success", "YouTube tables emptied"), "data": {}}
+    except Exception as e:
+        return {"meta": make_meta("error", f"DB error: {str(e)}"), "data": {}}
+    finally:
+        if cur is not None:
+            cur.close()
+        if conn is not None:
+            conn.close()

app/api/youtube/sql/sync.py

@@ -0,0 +1,83 @@
+from fastapi import APIRouter, Depends
+from app.utils.db import get_db_connection_direct
+from app.utils.make_meta import make_meta
+from app.utils.api_key_auth import get_api_key
+import os
+import requests
+import json
+from dotenv import load_dotenv
+
+router = APIRouter()
+
+@router.post("/youtube/sync")
+def sync_youtube(api_key: str = Depends(get_api_key)) -> dict:
+    """POST /youtube/sync: Fetches data from YouTube Data API and stores in DB."""
+    load_dotenv()
+    youtube_key = os.getenv("YOUTUBE_API_KEY")
+    if not youtube_key:
+        return {"meta": make_meta("error", "Missing YouTube API key"), "data": {}}
+
+    # Example: Fetch channel info and latest videos for a given channel ID
+    channel_id = os.getenv("YOUTUBE_CHANNEL_ID")  # Optionally add this to .env
+    if not channel_id:
+        return {"meta": make_meta("error", "Missing YOUTUBE_CHANNEL_ID in .env"), "data": {}}
+
+    try:
+        # Fetch channel details
+        channel_url = "https://www.googleapis.com/youtube/v3/channels"
+        channel_params = {
+            "part": "snippet,contentDetails,statistics",
+            "id": channel_id,
+            "key": youtube_key
+        }
+        channel_resp = requests.get(channel_url, params=channel_params)
+        channel_resp.raise_for_status()
+        channel_data = channel_resp.json()
+        channel_items = channel_data.get("items", [])
+        if channel_items:
+            channel = channel_items[0]
+        else:
+            return {"meta": make_meta("error", "Channel not found"), "data": {}}
+
+        # Insert channel info
+        conn = get_db_connection_direct()
+        cur = conn.cursor()
+        cur.execute(
+            """
+            INSERT INTO youtube_channels (youtube_id, title, payload)
+            VALUES (%s, %s, %s)
+            ON CONFLICT (youtube_id) DO NOTHING;
+            """,
+            (channel.get("id"), channel["snippet"].get("title"), json.dumps(channel))
+        )
+
+        # Fetch latest videos from uploads playlist
+        uploads_playlist_id = channel["contentDetails"]["relatedPlaylists"]["uploads"]
+        playlist_url = "https://www.googleapis.com/youtube/v3/playlistItems"
+        playlist_params = {
+            "part": "snippet,contentDetails",
+            "playlistId": uploads_playlist_id,
+            "maxResults": 100,
+            "key": youtube_key
+        }
+        playlist_resp = requests.get(playlist_url, params=playlist_params)
+        playlist_resp.raise_for_status()
+        playlist_data = playlist_resp.json()
+        videos = playlist_data.get("items", [])
+        for video in videos:
+            video_id = video["contentDetails"]["videoId"]
+            title = video["snippet"].get("title")
+            cur.execute(
+                """
+                INSERT INTO youtube_videos (youtube_id, title, payload)
+                VALUES (%s, %s, %s)
+                ON CONFLICT (youtube_id) DO NOTHING;
+                """,
+                (video_id, title, json.dumps(video))
+            )
+        conn.commit()
+        cur.close()
+        conn.close()
+        return {"meta": make_meta("success", f"Synced {len(videos)} videos from YouTube"), "data": {"count": len(videos)}}
+    except Exception as e:
+        return {"meta": make_meta("error", f"Sync error: {str(e)}"), "data": {}}

app/api/youtube/youtube.py

@@ -0,0 +1,44 @@
+from fastapi import APIRouter, Depends
+from app.utils.make_meta import make_meta
+from app.utils.db import get_db_connection_direct
+from app.utils.api_key_auth import get_api_key
+
+router = APIRouter()
+
+_TABLES = [
+    "youtube_channels",
+    "youtube_videos",
+    "youtube_playlists",
+    "youtube_resources",
+]
+
+def _fetch_table(cur, table: str) -> dict:
+    cur.execute(f"SELECT COUNT(*) FROM {table};")
+    row = cur.fetchone()
+    count = row[0] if row and row[0] is not None else 0
+    cur.execute(f"SELECT * FROM {table} ORDER BY id DESC LIMIT 100;")
+    if cur.description:
+        columns = [desc[0] for desc in cur.description]
+        rows = [dict(zip(columns, r)) for r in cur.fetchall()]
+    else:
+        rows = []
+    return {"count": count, "rows": rows}
+
+
+@router.get("/youtube")
+def get_youtube(api_key: str = Depends(get_api_key)) -> dict:
+    """GET /youtube: Return counts and records from all YouTube tables."""
+    conn = None
+    cur = None
+    try:
+        conn = get_db_connection_direct()
+        cur = conn.cursor()
+        data = {table: _fetch_table(cur, table) for table in _TABLES}
+        return {"meta": make_meta("success", "YouTube data"), "data": data}
+    except Exception as e:
+        return {"meta": make_meta("error", f"DB error: {str(e)}"), "data": {}}
+    finally:
+        if cur is not None:
+            cur.close()
+        if conn is not None:
+            conn.close()