Refactor yt-dlp libs

- Remove lib folder
- Refactor some tests
- General cleanup

.dependency-cruiser.json

@@ -5,7 +5,7 @@
       "comment": "Don't allow dependencies from outside the test folder to test",
       "severity": "error",
       "from": {
-        "pathNot": "^(test|spec)"
+        "pathNot": "^(test|spec)|(\\.test\\.js$)"
       },
       "to": {
         "path": "^(test|spec)"

.gitignore

@@ -4,6 +4,6 @@ sandbox.js
 package-lock.json
 coverage
 public
-yt-dlp
+./yt-dlp
 .env
 dependency-graph.html

app.js

@@ -11,11 +11,14 @@ export default async function App (fastify, opts) {
 
   // Do not touch the following lines
 
+  const testPattern = /.*(test|spec).js/
   // This loads all plugins defined in plugins
   // those should be support plugins that are reused
   // through your application
   fastify.register(AutoLoad, {
     dir: join(__dirname, 'plugins'),
+    dirNameRoutePrefix: false,
+    ignorePattern: testPattern,
     options: Object.assign({}, opts)
   })
 
@@ -27,6 +30,7 @@ export default async function App (fastify, opts) {
     autoHooks: true,
     cascadeHooks: true,
     overwriteHooks: true,
+    ignorePattern: testPattern,
     options: Object.assign({}, opts)
   })
 

package.json

@@ -106,7 +106,8 @@
     "version:git": "git add CHANGELOG.md",
     "watch": "npm run clean && run-p watch:*",
     "watch:server": "fastify start -w --ignore-watch='node_modules .git web' -l info -P -p 3000 --options --address 0.0.0.0 app.js",
-    "watch:siteup": "npm run build:siteup -- --watch"
+    "watch:siteup": "npm run build:siteup -- --watch",
+    "print-routes": "fastify print-routes app.js"
   },
   "standard": {
     "ignore": []

plugins/cache.js

@@ -1,5 +1,6 @@
 import fp from 'fastify-plugin'
 import abstractCacheRedis from 'abstract-cache-redis'
+import LRU from 'lru-cache'
 
 /**
  * This plugins adds fastify/fastify-caching
@@ -12,6 +13,25 @@ export default fp(async function (fastify, opts) {
   fastify.register(import('@fastify/caching'), {
     client
   })
+
+  // For caching file URLs
+  const memURLCache = new LRU({
+    max: 10000,
+    ttl: 1000 * 60 * 5, // 20 mins,
+    updateAgeOnGet: false,
+    ttlAutopurge: true
+  })
+  fastify.decorate('memURLCache', memURLCache)
+
+  // For caching url metadata objects
+  // TODO: use this
+  const memMetaCache = new LRU({
+    max: 200,
+    ttl: 1000 * 60 * 5, // 20 mins,
+    updateAgeOnGet: false,
+    ttlAutopurge: true
+  })
+  fastify.decorate('memMetaCache', memMetaCache)
 }, {
   name: 'cache',
   dependencies: ['redis']

plugins/flags/backend-flags.js

@@ -1,7 +1,7 @@
 export const defaultBackendFlags = {
-  foo: {
+  placeholder_url: {
     type: 'string',
-    default: 'bar',
-    description: 'An example backend flag that isn\'t sent to clients'
+    default: 'https://www.youtube.com/watch?v=dQw4w9WgXcQ', // rick roll
+    description: 'The default file to inject into empty feeds'
   }
 }

test/routes/health.test.js → plugins/health.test.js

@@ -1,5 +1,5 @@
 import { test } from 'tap'
-import { build } from '../helper.js'
+import { build } from '../test/helper.js'
 
 test('healthcheck baseline test', async (t) => {
   const app = await build(t)

plugins/prom.js

@@ -94,10 +94,15 @@ export default fp(async function (fastify, opts) {
     }
   }
 
+  if (fastify.config.METRICS) {
+    fastify.addHook('onReady', async () => {
+      await start()
+    })
+  }
+
   fastify.addHook('onClose', async (instance) => {
     await promServer.close()
   })
-  if (fastify.config.METRICS) await start()
 },
 {
   name: 'prom',

plugins/queue.js

@@ -54,6 +54,11 @@ export default fp(async function (fastify, opts) {
   })
 
   fastify.decorate('pqueue', queue)
+
+  fastify.addHook('onClose', async (instance) => {
+    // Wait for the queue to be empty before shutting down
+    await queue.onEmpty()
+  })
 },
 {
   name: 'pqueue',

test/plugins/support.test.js → plugins/support.test.js

@@ -1,6 +1,6 @@
 import { test } from 'tap'
 import Fastify from 'fastify'
-import Support from '../../plugins/support.js'
+import Support from './support.js'
 
 test('support works standalone', async (t) => {
   const fastify = Fastify()

plugins/yt-dlp/index.js

@@ -0,0 +1,76 @@
+import fp from 'fastify-plugin'
+import { request as undiciRequest } from 'undici'
+
+/**
+ * This plugin adds yt-dlp fetching helpeers
+ */
+export default fp(async function (fastify, opts) {
+  fastify.decorate('getYTDLPMetadata', async function getYTDLPMetadata ({
+    url,
+    medium
+  }) {
+    const endTimer = fastify.metrics.ytdlpSeconds.startTimer()
+    try {
+      const formatOpts = getFormatArg(medium)
+      const requestURL = new URL(fastify.config.YT_DLP_API_URL)
+      requestURL.searchParams.set('url', url)
+      requestURL.searchParams.set('format', formatOpts)
+      requestURL.pathname = 'info'
+
+      const response = await undiciRequest(requestURL, {
+        headers: {
+          Accept: 'application/json',
+          Authorization: 'Basic ' + btoa(requestURL.username + ':' + requestURL.password)
+        }
+      })
+
+      if (response.statusCode !== 200) {
+        const text = await response.body.text()
+        throw new Error('yt-dlp error: ' + text)
+      }
+
+      const metadata = await response.body.json()
+
+      return metadata
+    } finally {
+      endTimer()
+    }
+  })
+}, {
+  name: 'yt-dlp',
+  dependencies: ['env', 'prom']
+})
+
+const videoFormat = 'best[ext=mp4]/best[ext=mov]/mp4/mov'
+const audioFormat = 'ba[ext=m4a]/ba[ext=mp4]/ba[ext=mp3]/mp3/m4a'
+
+export function getFormatArg (medium) {
+  if (!['video', 'audio'].includes(medium)) throw new Error('format must be video or audio')
+
+  const formatOpts = medium === 'video'
+    ? [videoFormat, audioFormat].join('/')
+    : medium === 'audio'
+      ? [audioFormat, videoFormat].join('/')
+      : null
+
+  if (!formatOpts) throw new Error('No format options generated. Please report this bug')
+
+  return formatOpts
+}
+
+export function getFileKey ({
+  userId,
+  episodeId,
+  sourceUrl,
+  type,
+  medium
+}) {
+  return [
+    'file',
+    userId,
+    episodeId,
+    sourceUrl,
+    type,
+    medium
+  ].join(':')
+}