diff --git a/.env.sample b/.env.sample index 9eb3bc3..c533559 100644 --- a/.env.sample +++ b/.env.sample @@ -10,4 +10,6 @@ DB_USER= DB_PASSWORD= FLICKR_USER=@N00 FLICKR_KEY= -FLICKR_SECRET= \ No newline at end of file +FLICKR_SECRET= +YOUTUBE_API_KEY= +YOUTUBE_CHANNEL_ID= \ No newline at end of file diff --git a/app/api/flickr/sql/sync.py b/app/api/flickr/sql/sync.py index 597fb1c..8fda587 100644 --- a/app/api/flickr/sql/sync.py +++ b/app/api/flickr/sql/sync.py @@ -27,7 +27,7 @@ def sync_flickr(api_key: str = Depends(get_api_key)) -> dict: "user_id": flickr_user, "format": "json", "nojsoncallback": 1, - "per_page": 10 + "per_page": 100 } try: resp = requests.get(url, params=params) diff --git a/app/api/routes.py b/app/api/routes.py index 857a3b5..8b6a5fc 100644 --- a/app/api/routes.py +++ b/app/api/routes.py @@ -17,6 +17,7 @@ from app.api.github import github_router from app.api.flickr import flickr_router +from app.api.youtube import youtube_router router.include_router(root_router) router.include_router(resend_router) @@ -28,4 +29,5 @@ router.include_router(orders_router) router.include_router(queue_router) router.include_router(github_router) -router.include_router(flickr_router) \ No newline at end of file +router.include_router(flickr_router) +router.include_router(youtube_router) \ No newline at end of file diff --git a/app/api/youtube/README.md b/app/api/youtube/README.md new file mode 100644 index 0000000..be23a06 --- /dev/null +++ b/app/api/youtube/README.md @@ -0,0 +1,26 @@ +# YouTube API Integration + +This module provides API routes for accessing YouTube channel data, similar to the GitHub and Flickr integrations. It expects the following environment variable in your `.env` file: + +- YOUTUBE_API_KEY + +## Endpoints + +- **GET /youtube**: Returns counts and recent records from all YouTube tables. + +## Proposed Table Design + +1. youtube_channels + - One row per YouTube channel. + - Stores channel identity fields and full raw payload. +2. youtube_videos + - One row per video. + - Stores video metadata plus raw JSON payload. +3. youtube_playlists + - One row per playlist. + - Stores playlist metadata plus raw JSON payload. +4. youtube_resources + - Generic catch-all for any future YouTube resource type. + - Supports additional YouTube objects through jsonb payload storage. + +This structure mirrors the GitHub and Flickr integrations for consistency and flexibility. diff --git a/app/api/youtube/__init__.py b/app/api/youtube/__init__.py new file mode 100644 index 0000000..ec3855a --- /dev/null +++ b/app/api/youtube/__init__.py @@ -0,0 +1,14 @@ +"""YouTube Routes""" + +from fastapi import APIRouter + +from .youtube import router as _youtube_router +from .sql.create_tables import router as _create_tables_router +from .sql.empty_tables import router as _empty_tables_router +from .sql.sync import router as _sync_router + +youtube_router = APIRouter() +youtube_router.include_router(_youtube_router) +youtube_router.include_router(_create_tables_router) +youtube_router.include_router(_empty_tables_router) +youtube_router.include_router(_sync_router) diff --git a/app/api/youtube/sql/create_tables.py b/app/api/youtube/sql/create_tables.py new file mode 100644 index 0000000..73d403b --- /dev/null +++ b/app/api/youtube/sql/create_tables.py @@ -0,0 +1,60 @@ +from fastapi import APIRouter, Depends +from app.utils.db import get_db_connection_direct +from app.utils.make_meta import make_meta +from app.utils.api_key_auth import get_api_key + +router = APIRouter() + +@router.post("/youtube/createtable") +def create_youtube_tables(api_key: str = Depends(get_api_key)) -> dict: + """POST /youtube/createtable: Drop and create YouTube tables in Postgres.""" + sql_statements = [ + 'DROP TABLE IF EXISTS youtube_resources;', + 'DROP TABLE IF EXISTS youtube_playlists;', + 'DROP TABLE IF EXISTS youtube_videos;', + 'DROP TABLE IF EXISTS youtube_channels;', + '''CREATE TABLE IF NOT EXISTS youtube_channels ( + id SERIAL PRIMARY KEY, + youtube_id TEXT UNIQUE, + title TEXT, + payload JSONB, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + );''', + '''CREATE TABLE IF NOT EXISTS youtube_videos ( + id SERIAL PRIMARY KEY, + youtube_id TEXT UNIQUE, + title TEXT, + payload JSONB, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + );''', + '''CREATE TABLE IF NOT EXISTS youtube_playlists ( + id SERIAL PRIMARY KEY, + youtube_id TEXT, + title TEXT, + payload JSONB, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + );''', + '''CREATE TABLE IF NOT EXISTS youtube_resources ( + id SERIAL PRIMARY KEY, + resource_type TEXT, + youtube_id TEXT, + payload JSONB, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + );''' + ] + conn = None + cur = None + try: + conn = get_db_connection_direct() + cur = conn.cursor() + for stmt in sql_statements: + cur.execute(stmt) + conn.commit() + return {"meta": make_meta("success", "YouTube tables created"), "data": {}} + except Exception as e: + return {"meta": make_meta("error", f"DB error: {str(e)}"), "data": {}} + finally: + if cur is not None: + cur.close() + if conn is not None: + conn.close() diff --git a/app/api/youtube/sql/empty_tables.py b/app/api/youtube/sql/empty_tables.py new file mode 100644 index 0000000..ff0f9a5 --- /dev/null +++ b/app/api/youtube/sql/empty_tables.py @@ -0,0 +1,32 @@ +from fastapi import APIRouter, Depends +from app.utils.db import get_db_connection_direct +from app.utils.make_meta import make_meta +from app.utils.api_key_auth import get_api_key + +router = APIRouter() + +@router.post("/youtube/emptytables") +def empty_youtube_tables(api_key: str = Depends(get_api_key)) -> dict: + """POST /youtube/emptytables: Delete all rows from all YouTube tables.""" + tables = [ + "youtube_channels", + "youtube_videos", + "youtube_playlists", + "youtube_resources" + ] + conn = None + cur = None + try: + conn = get_db_connection_direct() + cur = conn.cursor() + for table in tables: + cur.execute(f"DELETE FROM {table};") + conn.commit() + return {"meta": make_meta("success", "YouTube tables emptied"), "data": {}} + except Exception as e: + return {"meta": make_meta("error", f"DB error: {str(e)}"), "data": {}} + finally: + if cur is not None: + cur.close() + if conn is not None: + conn.close() diff --git a/app/api/youtube/sql/sync.py b/app/api/youtube/sql/sync.py new file mode 100644 index 0000000..d724ec8 --- /dev/null +++ b/app/api/youtube/sql/sync.py @@ -0,0 +1,83 @@ +from fastapi import APIRouter, Depends +from app.utils.db import get_db_connection_direct +from app.utils.make_meta import make_meta +from app.utils.api_key_auth import get_api_key +import os +import requests +import json +from dotenv import load_dotenv + +router = APIRouter() + +@router.post("/youtube/sync") +def sync_youtube(api_key: str = Depends(get_api_key)) -> dict: + """POST /youtube/sync: Fetches data from YouTube Data API and stores in DB.""" + load_dotenv() + youtube_key = os.getenv("YOUTUBE_API_KEY") + if not youtube_key: + return {"meta": make_meta("error", "Missing YouTube API key"), "data": {}} + + # Example: Fetch channel info and latest videos for a given channel ID + channel_id = os.getenv("YOUTUBE_CHANNEL_ID") # Optionally add this to .env + if not channel_id: + return {"meta": make_meta("error", "Missing YOUTUBE_CHANNEL_ID in .env"), "data": {}} + + try: + # Fetch channel details + channel_url = "https://www.googleapis.com/youtube/v3/channels" + channel_params = { + "part": "snippet,contentDetails,statistics", + "id": channel_id, + "key": youtube_key + } + channel_resp = requests.get(channel_url, params=channel_params) + channel_resp.raise_for_status() + channel_data = channel_resp.json() + channel_items = channel_data.get("items", []) + if channel_items: + channel = channel_items[0] + else: + return {"meta": make_meta("error", "Channel not found"), "data": {}} + + # Insert channel info + conn = get_db_connection_direct() + cur = conn.cursor() + cur.execute( + """ + INSERT INTO youtube_channels (youtube_id, title, payload) + VALUES (%s, %s, %s) + ON CONFLICT (youtube_id) DO NOTHING; + """, + (channel.get("id"), channel["snippet"].get("title"), json.dumps(channel)) + ) + + # Fetch latest videos from uploads playlist + uploads_playlist_id = channel["contentDetails"]["relatedPlaylists"]["uploads"] + playlist_url = "https://www.googleapis.com/youtube/v3/playlistItems" + playlist_params = { + "part": "snippet,contentDetails", + "playlistId": uploads_playlist_id, + "maxResults": 100, + "key": youtube_key + } + playlist_resp = requests.get(playlist_url, params=playlist_params) + playlist_resp.raise_for_status() + playlist_data = playlist_resp.json() + videos = playlist_data.get("items", []) + for video in videos: + video_id = video["contentDetails"]["videoId"] + title = video["snippet"].get("title") + cur.execute( + """ + INSERT INTO youtube_videos (youtube_id, title, payload) + VALUES (%s, %s, %s) + ON CONFLICT (youtube_id) DO NOTHING; + """, + (video_id, title, json.dumps(video)) + ) + conn.commit() + cur.close() + conn.close() + return {"meta": make_meta("success", f"Synced {len(videos)} videos from YouTube"), "data": {"count": len(videos)}} + except Exception as e: + return {"meta": make_meta("error", f"Sync error: {str(e)}"), "data": {}} diff --git a/app/api/youtube/youtube.py b/app/api/youtube/youtube.py new file mode 100644 index 0000000..02713c2 --- /dev/null +++ b/app/api/youtube/youtube.py @@ -0,0 +1,44 @@ +from fastapi import APIRouter, Depends +from app.utils.make_meta import make_meta +from app.utils.db import get_db_connection_direct +from app.utils.api_key_auth import get_api_key + +router = APIRouter() + +_TABLES = [ + "youtube_channels", + "youtube_videos", + "youtube_playlists", + "youtube_resources", +] + +def _fetch_table(cur, table: str) -> dict: + cur.execute(f"SELECT COUNT(*) FROM {table};") + row = cur.fetchone() + count = row[0] if row and row[0] is not None else 0 + cur.execute(f"SELECT * FROM {table} ORDER BY id DESC LIMIT 100;") + if cur.description: + columns = [desc[0] for desc in cur.description] + rows = [dict(zip(columns, r)) for r in cur.fetchall()] + else: + rows = [] + return {"count": count, "rows": rows} + + +@router.get("/youtube") +def get_youtube(api_key: str = Depends(get_api_key)) -> dict: + """GET /youtube: Return counts and records from all YouTube tables.""" + conn = None + cur = None + try: + conn = get_db_connection_direct() + cur = conn.cursor() + data = {table: _fetch_table(cur, table) for table in _TABLES} + return {"meta": make_meta("success", "YouTube data"), "data": data} + except Exception as e: + return {"meta": make_meta("error", f"DB error: {str(e)}"), "data": {}} + finally: + if cur is not None: + cur.close() + if conn is not None: + conn.close()