feat: add local subtitle support; set plyr as default local media player

- local subtitle support srt and vvt

close #7, #25

package.json

@@ -39,7 +39,7 @@
     "release-it": "^14.5.1",
     "rollup": "^2.32.1",
     "rollup-plugin-import-css": "^2.0.1",
-    "subtitle": "^4.1.0",
+    "srt-webvtt": "^1.0.1",
     "tslib": "^2.0.3",
     "typescript": "^4.0.3"
   },

src/main.css

@@ -8,6 +8,21 @@ iframe.external-video {
   min-width: inherit;
 }
 
+.visuallyhidden:not(:focus):not(:active) {
+  position: absolute;
+
+  width: 1px;
+  height: 1px;
+  margin: -1px;
+  border: 0;
+  padding: 0;
+
+  white-space: nowrap;
+
+  clip-path: inset(100%);
+  overflow: hidden;
+}
+
 div.thumbnail {
   background-repeat: no-repeat;
   background-size: cover;

src/main.ts

@@ -11,7 +11,7 @@ import "./main.css";
 export default class MediaExtended extends Plugin {
   settings: MxSettings = DEFAULT_SETTINGS;
 
-  processInternalEmbeds = processInternalEmbeds;
+  processInternalEmbeds = processInternalEmbeds.bind(this);
   processInternalLinks = processInternalLinks.bind(this);
   processExternalEmbeds = processExternalEmbeds.bind(this);
 

src/modules/handlers.ts

@@ -1,7 +1,9 @@
 import MediaExtended from "main";
 import { FileView, MarkdownPostProcessorContext } from "obsidian";
-import { filterDuplicates, mutationParam } from "./misc";
+import Plyr from "plyr";
+import { Await, filterDuplicates, mutationParam } from "./misc";
 import { getSetupTool } from "./playerSetup";
+import { getSubtitleTracks, SubtitleResource } from "./subtitle";
 import { getPlayer } from "./videohost/getPlayer";
 
 type mediaType = "audio" | "video";
@@ -150,7 +152,11 @@ export function handleLink(
 /**
  * Update media embeds to respond to temporal fragments
  */
-export function handleMedia(span: HTMLSpanElement) {
+export async function handleMedia(
+  span: HTMLSpanElement,
+  plugin: MediaExtended,
+  ctx: MarkdownPostProcessorContext,
+) {
   const srcLinktext = span.getAttr("src");
   if (srcLinktext === null) {
     console.error(span);
@@ -159,32 +165,75 @@ export function handleMedia(span: HTMLSpanElement) {
 
   const { setPlayerTF } = getSetupTool(srcLinktext);
 
-  // skip if timeSpan is missing or invalid
-  if (!setPlayerTF) return;
+  if (!(span.firstElementChild instanceof HTMLMediaElement)) {
+    console.error("first element not player: %o", span.firstElementChild);
+    return;
+  }
+
+  const isWebm = /\.webm$|\.webm#/.test(span.getAttr("src") ?? "");
+
+  function setupPlayer(
+    mediaEl: HTMLMediaElement,
+    info: Await<ReturnType<typeof getSubtitleTracks>>,
+    isWebm = false,
+  ): HTMLDivElement {
+    if (!mediaEl.parentElement) throw new Error("no parentElement");
+    const container = createDiv({ cls: "local-media" });
+    mediaEl.parentElement.appendChild(container);
+
+    let target: HTMLMediaElement;
+    if (!isWebm) target = mediaEl;
+    else {
+      target = mediaEl.cloneNode(true) as typeof mediaEl;
+      mediaEl.addClass("visuallyhidden");
+    }
+    container.appendChild(target);
+    ctx.addChild(new SubtitleResource(container, info?.objUrls ?? []));
+
+    if (info) info.tracks.forEach((t) => target.appendChild(t));
+    const player = new Plyr(target);
+    if (setPlayerTF) setPlayerTF(player);
+    return container;
+  }
+
+  const videoFile = plugin.app.metadataCache.getFirstLinkpathDest(
+    span.getAttr("src") as string,
+    ctx.sourcePath,
+  );
+  const tracks = await getSubtitleTracks(videoFile, plugin);
+
+  const srcMediaEl = span.firstElementChild;
+  /**
+   * div.local-media warped srcMediaEl by default
+   * div.local-media containing cloned srcMediaEl when file is webm
+   */
+  const newMediaContainer = setupPlayer(srcMediaEl, tracks, isWebm);
 
   const webmEmbed: mutationParam = {
-    option: {
-      childList: true,
-    },
-    callback: (list, obs) =>
-      filterDuplicates(list).forEach((m) =>
+    callback: (list, obs) => {
+      list.forEach((m) => {
+        if (m.addedNodes.length)
+          newMediaContainer.parentElement?.removeChild(newMediaContainer);
         m.addedNodes.forEach((node) => {
           if (node instanceof HTMLMediaElement) {
-            setPlayerTF(node);
+            setupPlayer(node, tracks);
             obs.disconnect();
+            return;
           }
-        }),
-      ),
+        });
+      });
+    },
+    option: {
+      childList: true,
+    },
   };
 
-  if (!(span.firstElementChild instanceof HTMLMediaElement)) {
-    console.error("first element not player: %o", span.firstElementChild);
-    return;
-  }
-
-  setPlayerTF(span.firstElementChild);
-  if (span.getAttr("src")?.match(/\.webm$|\.webm#/)) {
+  if (isWebm) {
     const webmObs = new MutationObserver(webmEmbed.callback);
     webmObs.observe(span, webmEmbed.option);
+    setTimeout(() => {
+      if (srcMediaEl.parentElement)
+        srcMediaEl.parentElement.removeChild(srcMediaEl);
+    }, 800);
   }
 }

src/modules/misc.ts

@@ -9,3 +9,9 @@ export function filterDuplicates(list: MutationRecord[]): MutationRecord[] {
     .reverse()
     .filter((item, index) => targets.indexOf(item.target) == index);
 }
+
+export type Await<T> = T extends {
+  then(onfulfilled?: (value: infer U) => unknown): unknown;
+}
+  ? U
+  : T;

src/modules/subtitle.ts

@@ -1,28 +1,31 @@
 import MediaExtended from "main";
-import { TAbstractFile, TFile } from "obsidian";
+import { MarkdownRenderChild, TFile } from "obsidian";
 import iso from "iso-639-1";
-import { Track } from "plyr";
-import { parseSync, stringifySync } from "subtitle";
-import { join } from "node:path";
+import SrtCvt from "srt-webvtt";
 
-export function getSubtitles(video: TFile): TFile[] | null {
+function getSubtitles(video: TFile): TFile[] | null {
   const { basename: videoName, parent: folder } = video;
 
   // for video file "hello.mp4"
-  // vaild subtitle: "./hello.en.srt"
-  const subtitles = folder.children.filter((file) => {
+  // vaild subtitle:
+  // - "./hello.en.srt", "./hello.zh.srt" (muiltple files)
+  // - "./hello.srt" (single file)
+  let subtitles = folder.children.filter((file) => {
     // filter file only (exclude folder)
     if (!(file instanceof TFile)) return false;
     const isSubtitle = file.extension === "srt" || file.extension === "vtt";
     const isSameFile = file.basename.startsWith(videoName);
-    if (isSubtitle && isSameFile) {
+    return isSubtitle && isSameFile;
+  }) as TFile[];
+  if (subtitles.length > 1) {
+    subtitles = subtitles.filter((file) => {
       const languageSuffix = file.basename.slice(videoName.length);
       return (
         languageSuffix.startsWith(".") &&
         iso.validate(languageSuffix.substring(1))
       );
-    } else return false;
-  }) as TFile[];
+    });
+  }
 
   if (subtitles.length === 0) return null;
   else {
@@ -41,63 +44,88 @@ export function getSubtitles(video: TFile): TFile[] | null {
   }
 }
 
-export async function toVtt(
-  srtFile: TFile,
-  plugin: MediaExtended,
-): Promise<TFile> {
-  const srt = await plugin.app.vault.read(srtFile);
-  const vtt = stringifySync(parseSync(srt), { format: "WebVTT" });
-  return plugin.app.vault.create(
-    join(srtFile.parent.path, srtFile.basename + ".vtt"),
-    vtt,
-  );
+async function getSrtUrl(file: TFile, plugin: MediaExtended): Promise<string> {
+  const srt = await plugin.app.vault.read(file);
+  return new SrtCvt(new Blob([srt])).getURL();
+}
+
+async function getVttURL(file: TFile, plugin: MediaExtended) {
+  const blob = new Blob([await plugin.app.vault.read(file)], {
+    type: "text/vtt",
+  });
+  return URL.createObjectURL(blob);
 }
 
 /**
  *
  * @param video
- * @returns empty array if no subtitle exists
+ * @returns empty [objectUrl[], trackEl[]] if no subtitle exists
  */
 export async function getSubtitleTracks(
   video: TFile,
   plugin: MediaExtended,
-): Promise<Track[]> {
+): Promise<{ objUrls: string[]; tracks: HTMLTrackElement[] } | null> {
   const subFiles = getSubtitles(video);
-  if (!subFiles) {
-    console.log("no subtitle found");
-    return [];
-  }
+  if (!subFiles) return null;
+
+  console.log(
+    "found subtitle(s): %o",
+    subFiles.map((v) => v.name),
+  );
 
-  // convert all srt to vtt
-  for (let i = subFiles.length - 1; i >= 0; i--) {
-    const sub = subFiles[i];
-    if (sub.extension !== "srt") continue;
-    try {
-      subFiles[i] = await toVtt(sub, plugin);
-    } catch (error) {
-      console.error(error, sub);
-      subFiles.splice(i, 1);
-    }
-  }
   if (subFiles.length === 0) {
     console.error("no vtt subtitle availble");
-    return [];
+    return null;
   }
 
-  return subFiles.map((file) => {
+  const url: Map<TFile, string> = new Map();
+  for (const file of subFiles) {
+    if (file.extension === "srt") {
+      url.set(file, await getSrtUrl(file, plugin));
+    } else url.set(file, await getVttURL(file, plugin));
+  }
+
+  const tracks = subFiles.map((file, i, array) => {
     const languageCode = file.basename.split(".").pop();
-    if (!languageCode || !iso.validate(languageCode))
-      throw new Error(
-        "languageCode unable to parse, problem with getSubtitles()? ",
-      );
+    let label: string | null = null;
+    let srclang: string | null = null;
+    if (array.length > 1) {
+      if (!languageCode || !iso.validate(languageCode))
+        throw new Error(
+          "languageCode unable to parse, problem with getSubtitles()? ",
+        );
+      label = iso.getNativeName(languageCode);
+      srclang = languageCode;
+    }
 
-    return {
+    const attr = {
       kind: "captions",
-      label: iso.getNativeName(languageCode),
-      srclang: languageCode,
-      src: plugin.app.vault.getResourcePath(file),
-      default:
-        navigator.language.substring(0, 2) === languageCode ? true : undefined,
+      label,
+      srclang,
+      src: url.get(file) as string,
     };
+    return createEl("track", { attr }, (el) => {
+      if (navigator.language.substring(0, 2) === languageCode) {
+        el.setAttr("default", "");
+      }
+    });
   });
+
+  return { objUrls: [...url.values()], tracks };
+}
+
+export class SubtitleResource extends MarkdownRenderChild {
+  objectUrls: string[];
+
+  constructor(containerEl: HTMLDivElement, objectUrls: string[]) {
+    super();
+    this.containerEl = containerEl;
+    this.objectUrls = objectUrls;
+  }
+
+  unload() {
+    for (const url of this.objectUrls) {
+      URL.revokeObjectURL(url);
+    }
+  }
 }

src/processor.ts

@@ -33,15 +33,19 @@ export function processInternalLinks(
 }
 
 /** Process internal media embeds with hash */
-export function processInternalEmbeds(el: HTMLElement) {
+export function processInternalEmbeds(
+  this: MediaExtended,
+  el: HTMLElement,
+  ctx: MarkdownPostProcessorContext,
+) {
   let allEmbeds;
   if ((allEmbeds = el.querySelectorAll("span.internal-embed"))) {
     const internalEmbed: mutationParam = {
       callback: (list, obs) => {
         for (const mutation of filterDuplicates(list)) {
           const span = mutation.target as HTMLSpanElement;
           if (span.hasClass("is-loaded") && !span.hasClass("mod-empty")) {
-            if (span.hasClass("media-embed")) handleMedia(span);
+            if (span.hasClass("media-embed")) handleMedia(span, this, ctx);
             obs.disconnect();
           }
         }