fix(tktrex): downloader metadata format, trimmed verbosity, execution duplication

Author: vecna

platforms/tktrex/backend/package.json

@@ -11,7 +11,7 @@
     "tto:start": "DEBUG=\"*,-express:*,-body-parser:*,-send\" ts-node bin/observatory",
     "watch": "key=fuffa DEBUG=\"*,-express:*,-body-parser:*,-send\" ts-node-dev -r tsconfig-paths/register --respawn --transpile-only bin/server",
     "start": "DEBUG=\"*,-body-parser:*,-express:*,-lib:cache,-send\" ts-node bin/server",
-    "parserv": "DEBUG=\"*\" ts-node bin/parser",
+    "parserv": "DEBUG=\"*,-@trex:htmls:debug\" ts-node bin/parser",
     "parserv:watch": "DEBUG=\"*\" ts-node-dev -r tsconfig-paths/register bin/parser"
   },
   "author": "https://github.com/tracking-exposed/tktrex/graphs/contributors",

platforms/tktrex/backend/parsers/description.js

@@ -13,14 +13,14 @@ function videoDescriptionGuess(envelop) {
   });
   let retval = null;
   if (uno) {
-    debug("first condition happened in this 'video'");
+    debug('format kind (1): %s', uno.textContent);
     retval = { description: uno.textContent };
   } else if (due) {
-    debug("second condition happened in this 'video'");
+    debug('format kind (2): %s', due.textContent);
     retval = { description: due.textContent };
   } else if (tre && treTopSize.length) {
     debug(
-      "third condition happened in this 'video', picking the first of %j",
+      'format kind (3), picking the first of %j',
       _.compact(
         _.map(treTopSize, function (o) {
           return o.getAttribute('alt');
@@ -29,7 +29,7 @@ function videoDescriptionGuess(envelop) {
     );
     retval = { description: treTopSize[0].getAttribute('alt') };
   } else {
-    debug("Failure condition in this 'video'!");
+    debug('All the extraction approaches failed');
     return null;
   }
   return retval;

platforms/tktrex/backend/parsers/downloader.ts

@@ -1,11 +1,12 @@
 import { ParserFn } from '@shared/providers/parser.provider';
+import _ from 'lodash';
 import D from 'debug';
 import { HTMLSource } from '../lib/parser';
 import { getUUID, download } from './shared';
 import fs from 'fs';
 import path from 'path';
 
-const debug = D('parser:native');
+const debug = D('parser:downloader');
 
 // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
 async function processLink({ link, linkType }: { link: any; linkType: any }) {
@@ -75,12 +76,21 @@ const downloader: ParserFn<HTMLSource, any> = async (envelop, findings) => {
   const retval = [];
   for (const img of imageNodes) {
     const url = img.getAttribute('src');
+    const alt = img.getAttribute('alt');
+
     const info = await processLink({ link: url, linkType: 'thumbnail' });
+    info.url = url;
+    if (alt?.length) info.alt = alt;
+
     retval.push(info);
   }
 
-  debug('processing native video entry %O %O', envelop, findings);
-  return retval;
+  debug(
+    'reported as downloaded %d links (%O)',
+    retval.length,
+    _.countBy(retval, 'reason')
+  );
+  return { downloader: retval };
 };
 
 /*

platforms/tktrex/backend/parsers/native.ts

@@ -1,9 +1,7 @@
 import { ParserFn } from '@shared/providers/parser.provider';
 import D from 'debug';
-import _ from 'lodash';
 import { HTMLSource } from '../lib/parser';
 import parseAuthor from './author';
-import parseDescription from './description';
 import parseHashtags from './hashtags';
 import parseMusic from './music';
 import parseMetrics from './numbers';
@@ -14,23 +12,21 @@ const parseNativeVideo: ParserFn<HTMLSource, any> = async (
   envelop,
   findings
 ) => {
-  debug(
-    'processing native video entry %O %O',
-    _.omit(envelop, ['html.html', 'jsdom']),
-    findings
-  );
-
   if (envelop.html.type !== 'native') {
     debug('entry is not "native" (%s)', envelop.html.type);
     return null;
   }
+  debug('processing native video entry: %s', envelop.html.href);
 
   const music = await parseMusic(envelop, findings);
   const author = await parseAuthor(envelop, findings);
-  const description = await parseDescription(envelop, findings);
   const metrics = await parseMetrics(envelop, findings);
   const hashtags = await parseHashtags(envelop, findings);
 
+  // description is already available at this point!
+  // const description = await parseDescription(envelop, findings);
+  const description = findings.description;
+
   return {
     nature: { type: 'native' },
     type: 'native',