First attempt at not failing archives on common links

hifiwi-fi · Apr 8, 2023 · ef48cb9 · ef48cb9
1 parent f63675a
commit ef48cb9
Show file tree

Hide file tree

Showing 9 changed files with 44 additions and 20 deletions.
diff --git a/plugins/extract-archive.js b/plugins/extract-archive.js
@@ -21,19 +21,37 @@ export default fp(async function (fastify, opts) {
         return cachedRBArchive
       }
 
-      const html = initialHTML ?? await fastify.fetchHTML({ url })
-
-      const { document } = (new JSDOM(html, { url })).window
-      const reader = new Readability(document)
-      const article = reader.parse()
-
-      const dpWindow = new JSDOM('').window
-      const DOMPurify = createDOMPurify(dpWindow)
-      article.content = DOMPurify.sanitize(article.content)
-
-      fastify.siteMetaCache.set(cacheKey, article)
-
-      return article
+      const parsedURL = new URL(url)
+
+      if (parsedURL.hostname === 'youtube.com' || parsedURL.hostname.endsWith('.youtube.com')) {
+        const metadata = await fastify.getYTDLPMetadata({ url, medium: 'video' })
+
+        return {
+          title: metadata.title,
+          textContent: metadata.description,
+          length: metadata.description.length,
+          byline: metadata.channel,
+          dir: 'ltr',
+          siteName: metadata?.uploader_url?.replace(/^https?:\/\//, '') || metadata?.channel_url?.replace(/^https?:\/\//, '') || 'youtube.com',
+          lang: metadata.language
+        }
+      } else {
+        const html = initialHTML ?? await fastify.fetchHTML({ url })
+        const { document } = (new JSDOM(html, { url })).window
+        const reader = new Readability(document)
+        const article = reader.parse()
+        if (article) {
+          const dpWindow = new JSDOM('').window
+          const DOMPurify = createDOMPurify(dpWindow)
+          article.content = DOMPurify.sanitize(article.content)
+
+          fastify.siteMetaCache.set(cacheKey, article)
+
+          return article
+        } else {
+          throw new Error(`Extracting readability archive returned null for ${url}`)
+        }
+      }
     } finally {
       endTimer()
     }

diff --git a/routes/api/archives/_archive_id/get-archive.js b/routes/api/archives/_archive_id/get-archive.js
@@ -45,8 +45,6 @@ export async function getArchive (fastify, opts) {
         fullArchives: true
       })
 
-      console.log({ archiveQuery })
-
       const results = await fastify.pg.query(archiveQuery)
       const archive = results.rows[0]
 

diff --git a/routes/api/archives/archive-props.js b/routes/api/archives/archive-props.js
@@ -3,6 +3,7 @@ export const commonArchiveProps = {
   title: { type: 'string' },
   site_name: { type: 'string' },
   html_content: { type: 'string' },
+  text_content: { type: 'string' },
   length: { type: 'integer' },
   excerpt: { type: 'string' },
   byline: { type: 'string' },

diff --git a/routes/api/archives/archive-query-get.js b/routes/api/archives/archive-query-get.js
@@ -19,6 +19,7 @@ export function getArchivesQuery ({
       coalesce (ar.title, bm.title) as display_title,
       ar.site_name,
       ${fullArchives ? SQL`ar.html_content,` : SQL``}
+      ${fullArchives ? SQL`ar.text_content,` : SQL``}
       ar.length,
       ar.excerpt,
       ar.byline,

diff --git a/web/components/archive/archive-view.css b/web/components/archive/archive-view.css
@@ -32,6 +32,11 @@
 }
 
 
+.bc-archive-text-content,
 .bc-archive-html-content {
   margin-top: 1.2em;
 }
+
+.bc-archive-text-content {
+  white-space: pre-line;
+}
diff --git a/web/components/archive/archive-view.js b/web/components/archive/archive-view.js
@@ -49,7 +49,11 @@ export const archiveView = Component(({
             ${html([ar?.html_content])}
           </div>
         `
-        : null
+        : ar?.text_content // Watch your whitepsace here
+          ? html`
+            <div class="bc-archive-text-content">${ar?.text_content}</div>
+          `
+          : null
       }
     </div>
   `

diff --git a/web/email_confirm/client.js b/web/email_confirm/client.js
@@ -42,7 +42,6 @@ export const page = Component(() => {
 
         if (response.ok && response.status === 202) {
           const body = await response.json()
-          console.log({ body })
           setConfirmed(true)
         } else {
           throw new Error(`${response.status} ${response.statusText} ${await response.text()}`)

diff --git a/web/password_reset/client.js b/web/password_reset/client.js
@@ -33,7 +33,6 @@ export const page = Component(() => {
 
       if (response.ok && response.status === 202) {
         const body = await response.json()
-        console.log({ body })
         setReset(true)
       } else {
         throw new Error(`${response.status} ${response.statusText} ${await response.text()}`)

diff --git a/web/password_reset/confirm/client.js b/web/password_reset/confirm/client.js
@@ -43,8 +43,7 @@ export const page = Component(() => {
       })
 
       if (response.ok && response.status === 202) {
-        const body = await response.json()
-        console.log({ body })
+        await response.json()
         setReset(true)
       } else {
         throw new Error(`${response.status} ${response.statusText} ${await response.text()}`)