feat: 🎉 support YouTube video

.example.env

@@ -1,5 +1,6 @@
 OPENAI_API_KEY=sk-xxx
 BILIBILI_SESSION_TOKEN=
+SAVESUBS_X_AUTH_TOKEN=
 UPSTASH_REDIS_REST_URL=
 UPSTASH_REDIS_REST_TOKEN=
 UPSTASH_RATE_REDIS_REST_URL=

components/Header.tsx

@@ -74,9 +74,9 @@ export default function Header({
                   d="M0 0L3 3L0 6"
                   fill="none"
                   stroke="currentColor"
-                  stroke-width="1.5"
-                  stroke-linecap="round"
-                  stroke-linejoin="round"
+                  strokeWidth="1.5"
+                  strokeLinecap="round"
+                  strokeLinejoin="round"
                 ></path>
               </svg>
             </a>

hooks/useSummarize.ts

@@ -1,6 +1,6 @@
 import { useState } from "react";
 import { useToast } from "~/hooks/use-toast";
-import { UserConfig } from "~/lib/types";
+import { UserConfig, VideoConfig, VideoService } from "~/lib/types";
 import { RATE_LIMIT_COUNT } from "~/utils/constants";
 
 export function useSummarize() {
@@ -13,7 +13,7 @@ export function useSummarize() {
   };
 
   const summarize = async (
-    bvId: string,
+    { videoId, service }: VideoConfig,
     { userKey, shouldShowTimestamp }: UserConfig
   ) => {
     setSummary("");
@@ -27,7 +27,7 @@ export function useSummarize() {
           "Content-Type": "application/json",
         },
         body: JSON.stringify({
-          bvId,
+          videoConfig: { videoId, service },
           userConfig: { userKey, shouldShowTimestamp },
         }),
       });

lib/bilibili.ts

@@ -1,8 +1,14 @@
-import { sample } from "../utils/fp";
+import { find } from "lodash";
+import { VideoService } from "~/lib/types";
+import {
+  fetchYoutubeSubtitle,
+  SUBTITLE_DOWNLOADER_URL,
+} from "~/lib/youtube/fetchYoutubeSubtitle";
+import { sample } from "~/utils/fp";
 
-const run = async (bvId: string) => {
+const fetchBilibiliSubtitles = async (bvId: string) => {
   const requestUrl = `https://api.bilibili.com/x/web-interface/view?bvid=${bvId}`;
-  console.log(`fetch`, requestUrl);
+  console.log(`fetch`, requestUrl, process.env.BILIBILI_SESSION_TOKEN);
   const sessdata = sample(process.env.BILIBILI_SESSION_TOKEN?.split(","));
   const headers = {
     Accept: "application/json",
@@ -23,8 +29,33 @@ const run = async (bvId: string) => {
   return json.data;
 };
 
-export async function fetchSubtitle(bvId: string) {
-  // const res = await pRetry(async () => await run(bvId), {
+export async function fetchSubtitle(
+  videoId: string,
+  service?: VideoService,
+  shouldShowTimestamp?: boolean
+) {
+  if (service === VideoService.Youtube) {
+    const { title, subtitleList } = await fetchYoutubeSubtitle(videoId);
+    if (subtitleList?.length <= 0) {
+      return { title, subtitleList: null };
+    }
+    const betterSubtitle =
+      find(subtitleList, { quality: "English" }) ||
+      find(subtitleList, { quality: "English (auto" }) ||
+      find(subtitleList, { quality: "zh-CN" }) ||
+      subtitleList[0];
+    const format = shouldShowTimestamp ? "" : `?ext=txt`; // ?ext=json
+    const subtitleUrl = `${SUBTITLE_DOWNLOADER_URL}${betterSubtitle.url}${format}`;
+    const response = await fetch(subtitleUrl);
+    const subtitles = await response.text();
+    const transcripts = subtitles
+      .split("\r\n\r\n")
+      ?.map((text: string, index: number) => ({ text, index }));
+    console.log("========subtitleUrl response========", title, transcripts);
+    return { title, subtitles: transcripts };
+  }
+
+  // const res = await pRetry(async () => await fetchBilibiliSubtitles(videoId), {
   //   onFailedAttempt: (error) => {
   //     console.log(
   //       `Attempt ${error.attemptNumber} failed. There are ${error.retriesLeft} retries left.`
@@ -33,7 +64,7 @@ export async function fetchSubtitle(bvId: string) {
   //   retries: 2,
   // });
   // @ts-ignore
-  const res = await run(bvId);
+  const res = await fetchBilibiliSubtitles(videoId);
   const title = res?.title;
   const subtitleList = res?.subtitle?.list;
   if (!subtitleList || subtitleList?.length < 1) {
@@ -48,5 +79,21 @@ export async function fetchSubtitle(bvId: string) {
 
   const subtitleResponse = await fetch(subtitleUrl);
   const subtitles = await subtitleResponse.json();
-  return { title, subtitles };
+  /*{
+      "from": 16.669,
+      "to": 18.619,
+      "sid": 8,
+      "location": 2,
+      "content": "让ppt变得更加精彩",
+      "music": 0.0
+    },*/
+  const transcripts = subtitles?.body.map(
+    (item: { from: number; content: string }, index: number) => {
+      return {
+        text: `${item.from}: ${item.content}`,
+        index,
+      };
+    }
+  );
+  return { title, subtitles: transcripts };
 }

lib/openai/prompt.ts

@@ -1,11 +1,29 @@
-
-export function getSummaryPrompt(title: string, transcript: any, shouldShowTimestamp?: boolean) {
-  console.log('========shouldShowTimestamp========', shouldShowTimestamp);
-  const betterPrompt = `我希望你是一名专业的视频内容编辑，帮我总结视频的内容精华。请你将视频字幕文本进行总结，然后以无序列表的方式返回，不要超过5条。记得不要重复句子，确保所有的句子都足够精简，清晰完整，祝你好运！`
-  const promptWithTimestamp = `我希望你是一名专业的视频内容编辑，帮我总结视频的内容精华。请先用一句简短的话总结视频梗概。然后再请你将视频字幕文本进行总结，在每句话的最前面加上时间戳（类似 10:24），每句话开头只需要一个开始时间。请你以无序列表的方式返回，请注意不要超过5条哦，确保所有的句子都足够精简，清晰完整，祝你好运！`;
+interface PromptConfig {
+  language?: string
+  shouldShowTimestamp?: boolean
+}
+const PROMPT_LANGUAGE_MAP = {
+  'English': "UK English",
+  "中文": "Simplified Chinese",
+  "繁體中文": "Traditional Chinese",
+  "日本語": "Japanese",
+  "Italiano": "Italian",
+  "Deutsch": "German",
+  "Español": "Spanish",
+  "Français": "French",
+  "Nederlands": "Dutch",
+  "한국어": "Korean",
+  "ភាសាខ្មែរ":"Khmer",
+  "हिंदी" : "Hindi"
+}
+export function getSummaryPrompt(title: string, transcript: any, promptConfig: PromptConfig) {
+  console.log('========promptConfig========', promptConfig);
+  const { language = '中文', shouldShowTimestamp } = promptConfig
+  const betterPrompt = `我希望你是一名专业的视频内容编辑，帮我用${language}总结视频的内容精华。请你将视频字幕文本进行总结，然后以无序列表的方式返回，不要超过5条。记得不要重复句子，确保所有的句子都足够精简，清晰完整，祝你好运！`
+  const promptWithTimestamp = `我希望你是一名专业的视频内容编辑，帮我用${language}总结视频的内容精华。请先用一句简短的话总结视频梗概。然后再请你将视频字幕文本进行总结，在每句话的最前面加上时间戳（类似 10:24），每句话开头只需要一个开始时间。请你以无序列表的方式返回，请注意不要超过5条哦，确保所有的句子都足够精简，清晰完整，祝你好运！`;
 
   return `标题: "${title
-    .replace(/\n+/g, " ")
+    ?.replace(/\n+/g, " ")
     .trim()}"\n视频字幕: "${truncateTranscript(transcript)
     .replace(/\n+/g, " ")
     .trim()}"\n${(shouldShowTimestamp ? promptWithTimestamp : betterPrompt)}`;

lib/openai/selectApiKeyAndActivatedLicenseKey.ts

@@ -4,7 +4,7 @@ import { sample } from "~/utils/fp";
 
 export async function selectApiKeyAndActivatedLicenseKey(
   apiKey?: string,
-  bvId?: string
+  videoId?: string
 ) {
   if (apiKey) {
     if (checkOpenaiApiKeys(apiKey)) {
@@ -13,7 +13,7 @@ export async function selectApiKeyAndActivatedLicenseKey(
     }
 
     // user is using validated licenseKey
-    const activated = await activateLicenseKey(apiKey, bvId);
+    const activated = await activateLicenseKey(apiKey, videoId);
     if (!activated) {
       throw new Error("licenseKey is not validated!");
     }

lib/types.ts

@@ -1,8 +1,23 @@
 export type SummarizeParams = {
   bvId: string;
+  videoConfig: VideoConfig;
   userConfig: UserConfig;
 };
 export type UserConfig = {
   userKey?: string;
   shouldShowTimestamp?: boolean;
 };
+export type VideoConfig = {
+  videoId: string;
+  service?: VideoService;
+};
+
+export enum VideoService {
+  Bilibili = "bilibili",
+  Youtube = "youtube",
+  // todo: integrate with whisper API
+  Podcast = "podcast",
+  Meeting = "meeting",
+  LocalVideo = "local-video",
+  LocalAudio = "local-audio",
+}

lib/youtube/fetchYoutubeSubtitle.ts

@@ -0,0 +1,27 @@
+export const SUBTITLE_DOWNLOADER_URL = "https://savesubs.com";
+export async function fetchYoutubeSubtitle(videoId: string) {
+  const response = await fetch(SUBTITLE_DOWNLOADER_URL + "/action/extract", {
+    method: "POST",
+    body: JSON.stringify({
+      data: { url: `https://www.youtube.com/watch?v=${videoId}` },
+    }),
+    headers: {
+      "Content-Type": "text/plain",
+      "User-Agent":
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36",
+      "X-Auth-Token": `${process.env.SAVESUBS_X_AUTH_TOKEN}` || "",
+      "X-Requested-Domain": "savesubs.com",
+      "X-Requested-With": "xmlhttprequest",
+    },
+  });
+  const { response: json = {} } = await response.json();
+  // console.log("========json========", json);
+  /*
+  * "title": "Microsoft vs Google: AI War Explained | tech",
+    "duration": "13 minutes and 15 seconds",
+    "duration_raw": "795",
+    "uploader": "Joma Tech / 2023-02-20",
+    "thumbnail": "//i.ytimg.com/vi/BdHaeczStRA/mqdefault.jpg",
+  * */
+  return { title: json.title, subtitleList: json.formats };
+}

lib/youtube/fetchYoutubeSubtitleOfficially.ts

@@ -0,0 +1,46 @@
+import { find } from "lodash";
+// import { VideoService } from "~/lib/types";
+
+type CaptionSnippet = {
+  videoId: string;
+  language: string;
+  trackKind: "asr" | "standard" | "forced";
+};
+type YouTubeCaptionType = {
+  id: string;
+  snippet: CaptionSnippet;
+};
+
+export async function fetchYoutubeSubtitleOfficially(videoId: string) {
+  // https://github.com/adamrichardson14/youtubestatistics/blob/completed/pages/videos.jsx#L37
+  // https://developers.google.com/youtube/v3/docs/captions/list?hl=zh-cn0拍；、*IK《0拍；、3edc
+  /*
+  * ASR – A caption track generated using automatic speech recognition.
+  forced – A caption track that plays when no other track is selected in the player. For example, a video that shows aliens speaking in an alien language might have a forced caption track to only show subtitles for the alien language.
+  standard – A regular caption track. This is the default value.
+  * */
+  const subtitleListUrl = `https://www.googleapis.com/youtube/v3/captions?part=snippet&videoId=${videoId}`;
+  const response = await fetch(
+    `${subtitleListUrl}&key=${process.env.YOUTUBE_DATA_API_TOKEN}`
+  );
+  const json = await response.json();
+  const subtitles = json.items;
+  console.log("========response========", subtitles);
+  if (subtitles?.length > 0) {
+    /*
+     *   trackKind: 'standard', language: 'en',
+     *   trackKind: 'asr', language: 'en',
+     * */
+    const betterSubtitle =
+      find(subtitles, { trackKind: "standard" }) ||
+      find(subtitles, { language: "zh-CN" }) ||
+      find(subtitles, { language: "en" }) ||
+      subtitles[0];
+    const subtitleUrl = `https://www.googleapis.com/youtube/v3/captions/${betterSubtitle.id}`;
+    const response = await fetch(
+      `${subtitleUrl}&key=${process.env.YOUTUBE_DATA_API_TOKEN}`
+    );
+    // throw oauth2 error
+    console.log("========subtitleUrl response========", await response.json());
+  }
+}

middleware.ts

@@ -11,8 +11,9 @@ import { isDev } from "./utils/env";
 const redis = Redis.fromEnv();
 
 export async function middleware(req: NextRequest, context: NextFetchEvent) {
-  const { userConfig = {}, bvId } = (await req.json()) as SummarizeParams;
-  const { userKey } = userConfig;
+  const { userConfig, videoConfig } = (await req.json()) as SummarizeParams;
+  const { userKey } = userConfig || {};
+  const { videoId: bvId } = videoConfig || {};
 
   function redirectAuth() {
     // return NextResponse.redirect(new URL("/shop", req.url));

package.json

@@ -34,6 +34,7 @@
     "flowbite-react": "^0.4.1",
     "focus-trap-react": "^10.1.0",
     "framer-motion": "^9.0.1",
+    "get-video-id": "^3.6.5",
     "lemonsqueezy.ts": "^0.1.6",
     "lodash": "^4.17.21",
     "lucide-react": "^0.122.0",