🐛 Use different UA when scraping Twitter URL and add title fallback

WebCrateApp · Aug 24, 2021 · 7b929cc · 7b929cc
1 parent 3f0cf12
commit 7b929cc
Show file tree

Hide file tree

Showing 4 changed files with 15 additions and 12 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -43,7 +43,7 @@
     "dotenv": "^10.0.0",
     "ejs": "^3.1.6",
     "express": "^4.17.1",
-    "metadata-scraper": "^0.1.1",
+    "metadata-scraper": "^0.2.36",
     "nanoid": "^3.1.23",
     "puppeteer-core": "^10.2.0",
     "running-at": "^0.3.21",

diff --git a/server/models/link.ts b/server/models/link.ts
@@ -51,12 +51,10 @@ export class Link {
 			url: url,
 			crate: crate || 'null',
 			meta: {
-				...(meta && {
-					title: meta.title,
-					description: meta.description,
-					image: meta.image ? makeAbsoluteUrl(url, meta.image) : undefined,
-					icon: meta.icon ? makeAbsoluteUrl(url, meta.icon) : undefined
-				})
+				title: meta?.title || new URL(url).hostname,
+				description: meta?.description,
+				image: meta?.image ? makeAbsoluteUrl(url, meta.image) : undefined,
+				icon: meta?.icon ? makeAbsoluteUrl(url, meta.icon) : undefined
 			},
 			redirect: {
 				enabled: false

diff --git a/server/router/api/link.ts b/server/router/api/link.ts
@@ -25,7 +25,12 @@ router.post('/', async (req: express.Request, res: express.Response, next: expre
 
 		let meta: MetaData | undefined
 		try {
-			meta = await getMetaData(parsedUrl)
+			// Workaround to Twitter to include the right meta tags
+			const useGoogleUa = parsedUrl.includes('twitter.com')
+
+			meta = await getMetaData(parsedUrl, {
+				...(useGoogleUa && { ua: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' })
+			})
 		} catch (err) {
 			log.debug(err)
 		}