fix_links

next/src/components/console/SourceLink.tsx

@@ -1,15 +1,34 @@
 import type { SyntheticEvent } from "react";
 import FadeIn from "../motions/FadeIn";
-import extractMetadata from "../../utils/extractMetadata";
 import { useQuery } from "@tanstack/react-query";
+import axios from "axios";
+import { env } from "../../env/client.mjs";
+import { z } from "zod";
 
 interface LinkInfo {
   link: string;
   index: number;
 }
 
+const MetaDataSchema = z.object({
+  title: z.string().nullish(),
+  favicon: z.string().nullish(),
+  hostname: z.string().nullish(),
+});
+
 const SourceLink = ({ link, index }: LinkInfo) => {
-  const linkMeta = useQuery(["linkMeta", link], () => extractMetadata(link));
+  const linkMeta = useQuery(["linkMeta", link], async () =>
+    MetaDataSchema.parse(
+      (
+        await axios.get(env.NEXT_PUBLIC_BACKEND_URL + "/api/metadata", {
+          params: {
+            url: link,
+          },
+        })
+      ).data
+    )
+  );
+
   const addImageFallback = (event: SyntheticEvent<HTMLImageElement, Event>) => {
     event.currentTarget.src = "/errorFavicon.ico";
   };

next/src/pages/api/metadata.ts

@@ -0,0 +1,13 @@
+import type { NextFetchEvent, NextRequest } from "next/server";
+import { NextResponse } from "next/server";
+
+import extractMetadata from "../../utils/extractMetadata";
+
+export const config = {
+  runtime: "edge",
+};
+
+export default function handler(request: NextRequest, context: NextFetchEvent) {
+  const { searchParams } = new URL(request.url);
+  return NextResponse.json(extractMetadata(searchParams.get("url") || ""));
+}

platform/reworkd_platform/web/api/metadata.py

@@ -0,0 +1,60 @@
+from typing import Optional
+from urllib.parse import urlparse, urljoin
+
+from bs4 import BeautifulSoup
+from fastapi import APIRouter
+from httpx import AsyncClient, RequestError, HTTPStatusError
+from pydantic import BaseModel, Field
+
+router = APIRouter()
+
+
+class Metadata(BaseModel):
+    title: Optional[str] = Field(default=None, description="Title of the page")
+    hostname: Optional[str] = Field(default=None, description="Hostname of the page")
+    favicon: Optional[str] = Field(default=None, description="Favicon of the page")
+
+
+@router.get(
+    "",
+)
+async def extract_metadata(url: str) -> Metadata:
+    try:
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36"
+        }
+
+        async with AsyncClient() as client:
+            res = await client.get(url, headers=headers)
+
+        res.raise_for_status()
+
+        soup = BeautifulSoup(res.text, "html.parser")
+        parsed_url = urlparse(url)
+
+        metadata = Metadata(
+            hostname=parsed_url.hostname,
+            title=soup.title.string.strip() if soup.title else None,
+        )
+
+        favicon = None
+        for link in soup.find_all("link", rel=lambda x: x in ["icon", "shortcut icon"]):
+            favicon = link.get("href")
+            if not favicon.startswith("http"):
+                favicon = urljoin(url, favicon)
+            break
+
+        metadata.favicon = (
+            favicon
+            if favicon
+            else f"{parsed_url.scheme}://{parsed_url.hostname}/favicon.ico"
+        )
+
+        return metadata
+
+    except (RequestError, HTTPStatusError):
+        parsed_url = urlparse(url)
+        return Metadata(
+            hostname=parsed_url.hostname,
+            favicon=f"{parsed_url.scheme}://{parsed_url.hostname}/favicon.ico",
+        )

platform/reworkd_platform/web/api/router.py

@@ -1,9 +1,10 @@
 from fastapi.routing import APIRouter
 
-from reworkd_platform.web.api import agent, auth, models, monitoring
+from reworkd_platform.web.api import agent, auth, models, monitoring, metadata
 
 api_router = APIRouter()
 api_router.include_router(monitoring.router, prefix="/monitoring", tags=["monitoring"])
 api_router.include_router(agent.router, prefix="/agent", tags=["agent"])
 api_router.include_router(models.router, prefix="/models", tags=["models"])
 api_router.include_router(auth.router, prefix="/auth", tags=["auth"])
+api_router.include_router(metadata.router, prefix="/metadata", tags=["metadata"])