From e348f917188ad072e21a6b4ae6bd1d1c617cc08c Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Sun, 19 May 2024 20:09:18 +0200 Subject: [PATCH] refactor: rename twitter into x --- packages/metascraper-twitter/README.md | 22 - packages/metascraper-twitter/src/index.js | 65 --- .../test/fixtures/profile-video.html | 248 -------- .../test/fixtures/profile.html | 125 ---- .../test/fixtures/tweet-gif.html | 549 ------------------ .../test/fixtures/tweet-image.html | 540 ----------------- .../test/fixtures/tweet.html | 117 ---- .../test/snapshots/index.js.md | 92 --- .../test/snapshots/index.js.snap | Bin 1347 -> 0 bytes .../CHANGELOG.md | 60 +- packages/metascraper-x/README.md | 46 ++ .../package.json | 11 +- .../src/index.d.ts | 0 packages/metascraper-x/src/index.js | 73 +++ .../test/fixtures/profile-video.html | 90 +++ .../metascraper-x/test/fixtures/profile.html | 98 ++++ .../test/fixtures/tweet-gif.html | 84 +++ .../test/fixtures/tweet-image.html | 84 +++ .../metascraper-x/test/fixtures/tweet.html | 26 + .../test/index.js | 36 +- .../metascraper-x/test/snapshots/index.js.md | 101 ++++ .../test/snapshots/index.js.snap | Bin 0 -> 1587 bytes .../test/test.js | 4 +- 23 files changed, 657 insertions(+), 1814 deletions(-) delete mode 100644 packages/metascraper-twitter/README.md delete mode 100644 packages/metascraper-twitter/src/index.js delete mode 100644 packages/metascraper-twitter/test/fixtures/profile-video.html delete mode 100644 packages/metascraper-twitter/test/fixtures/profile.html delete mode 100644 packages/metascraper-twitter/test/fixtures/tweet-gif.html delete mode 100644 packages/metascraper-twitter/test/fixtures/tweet-image.html delete mode 100644 packages/metascraper-twitter/test/fixtures/tweet.html delete mode 100644 packages/metascraper-twitter/test/snapshots/index.js.md delete mode 100644 packages/metascraper-twitter/test/snapshots/index.js.snap rename packages/{metascraper-twitter => metascraper-x}/CHANGELOG.md (71%) create mode 100644 packages/metascraper-x/README.md rename packages/{metascraper-twitter => metascraper-x}/package.json (78%) rename packages/{metascraper-twitter => metascraper-x}/src/index.d.ts (100%) create mode 100644 packages/metascraper-x/src/index.js create mode 100644 packages/metascraper-x/test/fixtures/profile-video.html create mode 100644 packages/metascraper-x/test/fixtures/profile.html create mode 100644 packages/metascraper-x/test/fixtures/tweet-gif.html create mode 100644 packages/metascraper-x/test/fixtures/tweet-image.html create mode 100644 packages/metascraper-x/test/fixtures/tweet.html rename packages/{metascraper-twitter => metascraper-x}/test/index.js (65%) create mode 100644 packages/metascraper-x/test/snapshots/index.js.md create mode 100644 packages/metascraper-x/test/snapshots/index.js.snap rename packages/{metascraper-twitter => metascraper-x}/test/test.js (72%) diff --git a/packages/metascraper-twitter/README.md b/packages/metascraper-twitter/README.md deleted file mode 100644 index 48c241dca..000000000 --- a/packages/metascraper-twitter/README.md +++ /dev/null @@ -1,22 +0,0 @@ -
-
- metascraper -
-
-

metascraper-twitter: Metascraper integration with Twitter.

-

See our website for more information.

-
-
- -## Install - -```bash -$ npm install metascraper-twitter --save -``` - -## License - -**metascraper-twitter** © [Microlink](https://microlink.io), released under the [MIT](https://github.com/microlinkhq/metascraper/blob/master/LICENSE.md) License.
-Authored and maintained by [Microlink](https://microlink.io) with help from [contributors](https://github.com/microlinkhq/metascraper/contributors). - -> [microlink.io](https://microlink.io) · GitHub [microlinkhq](https://github.com/microlinkhq) · Twitter [@microlinkhq](https://twitter.com/microlinkhq) diff --git a/packages/metascraper-twitter/src/index.js b/packages/metascraper-twitter/src/index.js deleted file mode 100644 index eb7754b09..000000000 --- a/packages/metascraper-twitter/src/index.js +++ /dev/null @@ -1,65 +0,0 @@ -'use strict' - -const { - $jsonld, - author, - date, - image, - memoizeOne, - parseUrl, - title, - toRule, - video -} = require('@metascraper/helpers') - -const toAuthor = toRule(author) -const toDate = toRule(date) -const toImage = toRule(image) -const toVideo = toRule(video) -const toTitle = toRule(title) - -const test = memoizeOne(url => parseUrl(url).domainWithoutSuffix === 'twitter') - -const REGEX_IMG_MODIFIERS = /_(?:bigger|mini|normal|x96)\./ -const ORIGINAL_IMG_SIZE = '_400x400' - -const isTweet = url => url.includes('/status/') - -const avatarUrl = str => - str?.replace(REGEX_IMG_MODIFIERS, `${ORIGINAL_IMG_SIZE}.`) - -module.exports = () => { - const rules = { - author: [ - toAuthor($jsonld('author.givenName')), - toAuthor($ => { - const author = $('meta[property="og:title"]').attr('content') - return author?.includes(' on X') ? author.split(' on X')[0] : author - }) - ], - title: [toTitle(($, url) => `@${url.split('/')[3]} on X`)], - date: [ - toDate(($, url) => { - const id = url.replace('https://twitter.com', '') - return $(`a[href="${id}"] time`).attr('datetime') - }) - ], - image: [ - toImage( - ($, url) => - isTweet(url) && $('meta[property="og:image"]').attr('content') - ), - toImage(($, url) => isTweet(url) && $('video').attr('poster')), - toImage($ => avatarUrl($jsonld('author.image.contentUrl')($))), - toImage($ => avatarUrl($('article img[src]').attr('src'))) - ], - video: [toVideo(($, url) => isTweet(url) && $('video').attr('src'))], - publisher: () => 'X' - } - - rules.test = ({ url }) => test(url) - - return rules -} - -module.exports.test = test diff --git a/packages/metascraper-twitter/test/fixtures/profile-video.html b/packages/metascraper-twitter/test/fixtures/profile-video.html deleted file mode 100644 index bd68cdde4..000000000 --- a/packages/metascraper-twitter/test/fixtures/profile-video.html +++ /dev/null @@ -1,248 +0,0 @@ - - - - - - - - -Brad, what are you gonna do? (@k4rliky) / Twitter - -
Log in
Sign up
Did someone say … cookies?
Twitter and its partners use cookies to provide you with a better, safer and faster service and to support our business. Some cookies are necessary to use our services, improve our services, and make sure they work properly. Show more about your choices.
Accept all cookies
Refuse non-essential cookies

Brad, what are you gonna do?

95.6K Tweets
Opens profile photo
Follow
Click to Follow k4rliky
Brad, what are you gonna do?
@k4rliky
Research Scientist interested in Quantum Computing, Complexity Science, Reverse Engineering, Programming. - -
@ladyCircus
and me. karliky.dev
Science & Technology***On Map Dungeon***karliky.comJoined May 2010
1,371 Following
1,221 Followers

Brad, what are you gonna do?’s Tweets

Brad, what are you gonna do?
@k4rliky
Taleb talebeando en pleno 2023
Quote Tweet
Nassim Nicholas Taleb
@nntaleb
If you don't get why, between 2019 and 2022, I turned down exactly 10 requests to be on his podcast, this will provide a succinct explanation. twitter.com/lexfridman/sta…
Show this thread
343
diff --git a/packages/metascraper-twitter/test/fixtures/profile.html b/packages/metascraper-twitter/test/fixtures/profile.html deleted file mode 100644 index f10fa081a..000000000 --- a/packages/metascraper-twitter/test/fixtures/profile.html +++ /dev/null @@ -1,125 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - diff --git a/packages/metascraper-twitter/test/fixtures/tweet-gif.html b/packages/metascraper-twitter/test/fixtures/tweet-gif.html deleted file mode 100644 index 421975a15..000000000 --- a/packages/metascraper-twitter/test/fixtures/tweet-gif.html +++ /dev/null @@ -1,549 +0,0 @@ - - - -#!/kiko/beats on X: "Experimenting with Clearbit API + Apple TV 3D Parallax https://t.co/Qsm163k4mJ https://t.co/5bcuqoEyAa" / X
Don’t miss what’s happening
People on X are the first to know.
Log in
Sign up
Did someone say … cookies?
X and its partners use cookies to provide you with a better, safer and faster service and to support our business. Some cookies are necessary to use our services, improve our services, and make sure they work properly. Show more about your choices.
Accept all cookies
Refuse non-essential cookies

Post

Conversation

New to X?

Sign up now to get your own personalized timeline!
Sign up with Apple
Create account
By signing up, you agree to the Terms of Service and Privacy Policy, including Cookie Use.
Trends are unavailable.
\ No newline at end of file diff --git a/packages/metascraper-twitter/test/fixtures/tweet-image.html b/packages/metascraper-twitter/test/fixtures/tweet-image.html deleted file mode 100644 index f6ebfcbd8..000000000 --- a/packages/metascraper-twitter/test/fixtures/tweet-image.html +++ /dev/null @@ -1,540 +0,0 @@ - - - -I don’t know, but I’ll find out. - Marty Sklar on X: "Lo mejor de @codemotion_es #codemotionMadrid es estar con la gente que quieres 😍@ladyCircus https://t.co/NQCYS8Yjt1" / X
Don’t miss what’s happening
People on X are the first to know.
Log in
Sign up
Did someone say … cookies?
X and its partners use cookies to provide you with a better, safer and faster service and to support our business. Some cookies are necessary to use our services, improve our services, and make sure they work properly. Show more about your choices.
Accept all cookies
Refuse non-essential cookies

Post

Conversation

New to X?

Sign up now to get your own personalized timeline!
Sign up with Apple
Create account
By signing up, you agree to the Terms of Service and Privacy Policy, including Cookie Use.
Trends are unavailable.
\ No newline at end of file diff --git a/packages/metascraper-twitter/test/fixtures/tweet.html b/packages/metascraper-twitter/test/fixtures/tweet.html deleted file mode 100644 index 71a2d85c1..000000000 --- a/packages/metascraper-twitter/test/fixtures/tweet.html +++ /dev/null @@ -1,117 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - diff --git a/packages/metascraper-twitter/test/snapshots/index.js.md b/packages/metascraper-twitter/test/snapshots/index.js.md deleted file mode 100644 index 89c7d02e6..000000000 --- a/packages/metascraper-twitter/test/snapshots/index.js.md +++ /dev/null @@ -1,92 +0,0 @@ -# Snapshot report for `test/index.js` - -The actual snapshot is saved in `index.js.snap`. - -Generated by [AVA](https://avajs.dev). - -## from a Twitter profile - -> Snapshot 1 - - { - author: '#!/kiko/beats (Kikobeats)', - date: null, - description: `engineering ▲ @vercel; founder of␊ - https://t.co/4PQvCsVNsA␊ - https://t.co/fpiHwbEPBv␊ - https://t.co/IG8Qq0IDKi␊ - https://t.co/gblDRx1P9D␊ - https://t.co/SmoZi3hAhb␊ - https://t.co/Y0Uk1XU3Eu␊ - https://t.co/PAq3eTEhmI`, - image: null, - lang: 'en', - publisher: 'X', - title: '@Kikobeats on X', - url: 'https://twitter.com/Kikobeats', - video: null, - } - -## from a Twitter profile with tweets with video - -> Snapshot 1 - - { - author: 'Brad, what are you gonna do?', - date: '2010-05-23T16:41:35.000Z', - description: 'Research Scientist interested in Quantum Computing, Complexity Science, Reverse Engineering, Programming. @ladyCircus and me. https://t.co/M5v7b5owoD', - image: 'https://pbs.twimg.com/profile_images/1603675274348040192/y9P6VlyX_400x400.jpg', - lang: 'en', - publisher: 'X', - title: '@k4rliky on X', - url: 'https://twitter.com/k4rliky', - video: null, - } - -## from a tweet - -> Snapshot 1 - - { - author: 'Donald J. Trump (realDonaldTrump)', - date: null, - description: '“Schiff blasted for not focusing on California homeless.” @foxandfriends His District is in terrible shape. He is a corrupt pol who only dreams of the Impeachment Hoax. In my opinion he is mentally deranged!', - image: 'https://pbs.twimg.com/profile_images/874276197357596672/kUuht00m_200x200.jpg', - lang: 'en', - publisher: 'X', - title: '@realDonaldTrump on X', - url: 'https://twitter.com/realDonaldTrump/status/1222907250383245320', - video: null, - } - -## from a tweet with a gif - -> Snapshot 1 - - { - author: '#!/kiko/beats', - date: '2017-06-28T19:01:34.000Z', - description: null, - image: 'https://pbs.twimg.com/tweet_video_thumb/DDbh3WCXYAAZfz9.jpg', - lang: 'en', - publisher: 'X', - title: '@Kikobeats on X', - url: 'https://twitter.com/Kikobeats/status/880139124791029763', - video: 'https://video.twimg.com/tweet_video/DDbh3WCXYAAZfz9.mp4', - } - -## from a tweet with an image - -> Snapshot 1 - - { - author: 'I don’t know, but I’ll find out. - Marty Sklar', - date: '2017-11-25T18:04:12.000Z', - description: null, - image: 'https://pbs.twimg.com/profile_images/1734680668213231616/_iul1bFE_400x400.jpg', - lang: 'en', - publisher: 'X', - title: '@k4rliky on X', - url: 'https://twitter.com/k4rliky/status/934482867480121345', - video: null, - } diff --git a/packages/metascraper-twitter/test/snapshots/index.js.snap b/packages/metascraper-twitter/test/snapshots/index.js.snap deleted file mode 100644 index 2b1e51b2d5efd0440a008c97dadda0517a84b4c8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1347 zcmV-J1-$w}RzV{FGinNJS*}><|A+B}f{FCX}{m zla{m`qOo_@JN53&c6K(7IiPT$ClD8Ys1gVj389`Uf%p%&aRVeIPAE67NZfh=J8`I8 z=LZT@_~JL;**EsS`F!8ZZND0-S|fS)3vPr7`HHVqgbWv(!WgbG*2)(l7l~wX5-5|r zbKCQnVOKNFA$SDZA99EsIS&Af0Pr3Fd<6iX5P)w8zz+oA4+8L92H?#Mz;SYLlpMTJ zz0PZfkgf38v133+fE)&L6poY3V`K=(@F+RRQWGfM9X|Q|iCjZ8RIbXIN$})Z(NLZ6 z#lE})9y7c%7#iuDdeq}dO^etFB{#Lq_rPUcNY1s8bzJ}WYn)u+y2is(=&MwET%+=j z1jfY4L@sBtHI;McFRn}{SI#F>y}BQZ*=BWS{`5+(K6j>c@dll%oE5!#y&6_7ty=Tt zO0T{UscXUsrh;m(ewAKsSWB0knY34*pSt1j#hDwDi}5wZF-qT=q>L zj?-!=l7Me*09oo^y#q~mbaZlSM^H)Ze|RX>VK;tu>)&sHmaVBMx0T)hD=Wg|YV#d7 zL0ugmOe!{o3D7h-H_1laNge?3ok0M9007?sz}p1iHv(`d1MpG?;7SJI{e1)c%xTTM zF>D6RAk!RMD#f~zlA)(w*_PqqQSyXMEjmu~PC6((G3!pUbTrPuaA_eXGvC!Fb8 zfD1Ljr4fliAr03&G2BBT@nXuPNh6$AQJfkf>tmfT-^|?u^5G`&9k>SoeQyxZ9|FMLc2_05tcI+^tFJUlWzbi(G-)?%C2^XC!dF@fhJlKB z$de?yzV;bT`f8O)&)0%WFTq)nU_~TGi<&`^w9z+QYf%k3CIO3i7H4^Tf?-W*oyG=Z z6=E|`sASkePjeO}sC+a5$GIrxtQJIE8k|*ZHH&i+qZX=INTDPKou%5-ENmxp&19W> zCw3(N^*xbaD!O*DV3mtbzL+l;3Pn5DxSR%t(rDSHbk%++;6Faux7)iJ;N?LE=vyO~ z7-rHWXW6!0rbRnXoswg_dB>)E@JRE1k8JdRhIQc&u*;c2c3A^}Zvo(E0H6fm0s+tj z;Pbz}e<%C3%Sg`!X19i49H)hGyR>MPC#W^yxO-~&57?>~uTz8Z>Xd7_Q_p1+q)|0j zsZ;~!jp?PUQ&ZRcx5^KJrhjmIDDHTB=n_;aQOhY?wp%P))Gikb&h`M^zi3AtT$~5W z62)%!Lh{rA^dxqwKY+feLG*nI0KWjhJpiZ^fSUy14gvUy0KBm8W^K=*r{wzDErSiI znqyc^4bH8v-3mkWh4fIRCX3^Ej_LNP-w2uB9Z|~~xATiuX@a^Fmi=!ay2rD>=(vRv zEfh+&<=Bo@unM_lk%m_F)tUd}*`MFx?C%0wc3ih)mkLF +
+ metascraper +
+
+

metascraper-x: Metascraper integration for x.com.

+

See our website for more information.

+
+ + +## Install + +```bash +$ npm install metascraper-x --save +``` + +## API + +### metascraper-x([options]) + +#### options + +##### gotOpts + +Type: `object` + +Any option provided here will passed to [got#options](https://github.com/sindresorhus/got#options). + +##### resolveUrls + +Type: `boolean` + +Set to `true` if you want to resolve `t.co` URLs into the final URL. + +##### resolveUrl + +Type: `function` + +A decorator to be called after `t.co` is resolved. It doesn't do anything by default. + +## License + +**metascraper-x** © [Microlink](https://microlink.io), released under the [MIT](https://github.com/microlinkhq/metascraper/blob/master/LICENSE.md) License.
+Authored and maintained by [Microlink](https://microlink.io) with help from [contributors](https://github.com/microlinkhq/metascraper/contributors). + +> [microlink.io](https://microlink.io) · GitHub [microlinkhq](https://github.com/microlinkhq) · Twitter [@microlinkhq](https://twitter.com/microlinkhq) diff --git a/packages/metascraper-twitter/package.json b/packages/metascraper-x/package.json similarity index 78% rename from packages/metascraper-twitter/package.json rename to packages/metascraper-x/package.json index 4df939f55..676818371 100644 --- a/packages/metascraper-twitter/package.json +++ b/packages/metascraper-x/package.json @@ -1,7 +1,7 @@ { - "name": "metascraper-twitter", - "description": "Metascraper integration with Twitter", - "homepage": "https://github.com/microlinkhq/metascraper/packages/metascraper-twitter", + "name": "metascraper-x", + "description": "Metascraper integration for x.com", + "homepage": "https://github.com/microlinkhq/metascraper/packages/metascraper-x", "version": "5.45.6", "types": "src/index.d.ts", "main": "src/index.js", @@ -11,7 +11,7 @@ "url": "https://microlink.io" }, "repository": { - "directory": "packages/metascraper-twitter", + "directory": "packages/metascraper-x", "type": "git", "url": "git+https://github.com/microlinkhq/metascraper.git" }, @@ -24,7 +24,8 @@ "spotify" ], "dependencies": { - "@metascraper/helpers": "^5.45.6" + "@metascraper/helpers": "^5.45.6", + "get-urls": "~10.0.1" }, "devDependencies": { "ava": "5" diff --git a/packages/metascraper-twitter/src/index.d.ts b/packages/metascraper-x/src/index.d.ts similarity index 100% rename from packages/metascraper-twitter/src/index.d.ts rename to packages/metascraper-x/src/index.d.ts diff --git a/packages/metascraper-x/src/index.js b/packages/metascraper-x/src/index.js new file mode 100644 index 000000000..4a54e2734 --- /dev/null +++ b/packages/metascraper-x/src/index.js @@ -0,0 +1,73 @@ +'use strict' + +const reachableUrl = require('reachable-url') +const getUrls = require('get-urls') + +const { + author, + image, + memoizeOne, + parseUrl, + title, + toRule, + description, + url +} = require('@metascraper/helpers') + +const toAuthor = toRule(author) +const toImage = toRule(image) +const toTitle = toRule(title) +const toDescription = toRule(description) +const toUrl = toRule(url) + +const test = memoizeOne(url => ['twitter', 'x'].includes(parseUrl(url).domainWithoutSuffix)) + +module.exports = ({ gotOpts, resolveUrls = false, resolveUrl = url => url } = {}) => { + const rules = { + author: [ + toAuthor($ => { + const author = $('meta[property="og:title"]').attr('content') + return author?.includes(' on X') ? author.split(' on X')[0] : author + }) + ], + title: [toTitle(($, url) => `@${url.split('/')[3]} on X`)], + url: [toUrl($ => $('link[rel="canonical"]').attr('href').replace('twitter.com', 'x.com'))], + description: [ + toDescription(async $ => { + let description = $('meta[property="og:description"]').attr('content') + if (!resolveUrls) return description + + const urls = Array.from(getUrls(description)) + + const resolvedUrls = await Promise.all( + urls.map(async url => { + const response = await reachableUrl(url, gotOpts) + if (reachableUrl.isReachable(response)) return resolveUrl(response.url) + return url + }) + ) + + for (const [index, url] of resolvedUrls.entries()) { + const original = urls[index] + description = description.replace(original, url) + } + + return description + }) + ], + image: [ + toImage(($) => { + let imageUrl = $('meta[property="og:image"]').attr('content') + if (imageUrl.endsWith('_200x200.jpg')) imageUrl = imageUrl.replace('_200x200.jpg', '_400x400.jpg') + return imageUrl + }), + ], + publisher: () => 'X' + } + + rules.test = ({ url }) => test(url) + + return rules +} + +module.exports.test = test diff --git a/packages/metascraper-x/test/fixtures/profile-video.html b/packages/metascraper-x/test/fixtures/profile-video.html new file mode 100644 index 000000000..2ca6c58b5 --- /dev/null +++ b/packages/metascraper-x/test/fixtures/profile-video.html @@ -0,0 +1,90 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + javilop + + + + +
+ + \ No newline at end of file diff --git a/packages/metascraper-x/test/fixtures/profile.html b/packages/metascraper-x/test/fixtures/profile.html new file mode 100644 index 000000000..db5533419 --- /dev/null +++ b/packages/metascraper-x/test/fixtures/profile.html @@ -0,0 +1,98 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Kikobeats + + + + +
+ + \ No newline at end of file diff --git a/packages/metascraper-x/test/fixtures/tweet-gif.html b/packages/metascraper-x/test/fixtures/tweet-gif.html new file mode 100644 index 000000000..e993e8b87 --- /dev/null +++ b/packages/metascraper-x/test/fixtures/tweet-gif.html @@ -0,0 +1,84 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 880139124791029763 + + + + +
+ + \ No newline at end of file diff --git a/packages/metascraper-x/test/fixtures/tweet-image.html b/packages/metascraper-x/test/fixtures/tweet-image.html new file mode 100644 index 000000000..04c2e5ced --- /dev/null +++ b/packages/metascraper-x/test/fixtures/tweet-image.html @@ -0,0 +1,84 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 934106870834454529 + + + + +
+ + \ No newline at end of file diff --git a/packages/metascraper-x/test/fixtures/tweet.html b/packages/metascraper-x/test/fixtures/tweet.html new file mode 100644 index 000000000..5d24429fd --- /dev/null +++ b/packages/metascraper-x/test/fixtures/tweet.html @@ -0,0 +1,26 @@ +1222907250383245320
\ No newline at end of file diff --git a/packages/metascraper-twitter/test/index.js b/packages/metascraper-x/test/index.js similarity index 65% rename from packages/metascraper-twitter/test/index.js rename to packages/metascraper-x/test/index.js index 89236d3b3..2c4d31a3b 100644 --- a/packages/metascraper-twitter/test/index.js +++ b/packages/metascraper-x/test/index.js @@ -4,13 +4,15 @@ const { readFile } = require('fs/promises') const { resolve } = require('path') const test = require('ava') -const metascraperTwitter = require('metascraper-twitter') +const metascraperX = require('metascraper-x') const createMetascraper = (...args) => require('metascraper')([ - metascraperTwitter(...args), + metascraperX(...args), require('metascraper-author')(), require('metascraper-date')(), + require('metascraper-image')(), + require('metascraper-video')(), require('metascraper-description')(), require('metascraper-lang')(), require('metascraper-publisher')(), @@ -18,52 +20,50 @@ const createMetascraper = (...args) => require('metascraper-url')() ]) -test('from a Twitter profile', async t => { - const url = 'https://twitter.com/Kikobeats' +test('from a X profile', async t => { + const url = 'https://x.com/Kikobeats' const html = await readFile(resolve(__dirname, 'fixtures/profile.html')) - const metascraper = createMetascraper() const metadata = await metascraper({ url, html }) + t.snapshot(metadata) +}) +test('from a X profile resolving URLs', async t => { + const url = 'https://x.com/Kikobeats' + const html = await readFile(resolve(__dirname, 'fixtures/profile.html')) + const metascraper = createMetascraper({ resolveUrls: true }) + const metadata = await metascraper({ url, html }) t.snapshot(metadata) }) -test('from a Twitter profile with tweets with video', async t => { - const url = 'https://twitter.com/k4rliky' +test('from a X profile with tweets with video', async t => { + const url = 'https://x.com/javilop' const html = await readFile(resolve(__dirname, 'fixtures/profile-video.html')) - const metascraper = createMetascraper() const metadata = await metascraper({ url, html }) - t.snapshot(metadata) }) test('from a tweet', async t => { - const url = 'https://twitter.com/realDonaldTrump/status/1222907250383245320' + const url = 'https://x.com/realDonaldTrump/status/1222907250383245320' const html = await readFile(resolve(__dirname, 'fixtures/tweet.html')) - const metascraper = createMetascraper() const metadata = await metascraper({ url, html }) - t.snapshot(metadata) }) test('from a tweet with a gif', async t => { - const url = 'https://twitter.com/Kikobeats/status/880139124791029763' + const url = 'https://x.com/Kikobeats/status/880139124791029763' const html = await readFile(resolve(__dirname, 'fixtures/tweet-gif.html')) - const metascraper = createMetascraper() const metadata = await metascraper({ url, html }) - t.snapshot(metadata) }) test('from a tweet with an image', async t => { - const url = 'https://twitter.com/k4rliky/status/934482867480121345' + const url = 'https://x.com/UaSmart/status/934106870834454529' const html = await readFile(resolve(__dirname, 'fixtures/tweet-image.html')) - const metascraper = createMetascraper() const metadata = await metascraper({ url, html }) - t.snapshot(metadata) }) diff --git a/packages/metascraper-x/test/snapshots/index.js.md b/packages/metascraper-x/test/snapshots/index.js.md new file mode 100644 index 000000000..1facfd433 --- /dev/null +++ b/packages/metascraper-x/test/snapshots/index.js.md @@ -0,0 +1,101 @@ +# Snapshot report for `test/index.js` + +The actual snapshot is saved in `index.js.snap`. + +Generated by [AVA](https://avajs.dev). + +## from a X profile + +> Snapshot 1 + + { + author: '#!/kiko/beats (Kikobeats)', + date: '2024-05-20T09:10:09.000Z', + description: 'engineering ▲ @vercel; founder of https://t.co/4PQvCsVNsA https://t.co/fpiHwbEPBv https://t.co/IG8Qq0IDKi https://t.co/gblDRx1P9D https://t.co/SmoZi3hAhb https://t.co/Y0Uk1XU3Eu https://t.co/PAq3eTEhmI', + image: 'https://pbs.twimg.com/profile_images/1717583638991138816/4HvMeeps_400x400.jpg', + lang: 'en', + publisher: 'X', + title: '@Kikobeats on X', + url: 'https://x.com/Kikobeats', + video: null, + } + +## from a X profile resolving URLs + +> Snapshot 1 + + { + author: '#!/kiko/beats (Kikobeats)', + date: '2024-05-20T09:10:09.000Z', + description: 'engineering ▲ @vercel; founder of https://microlink.io/ https://teslahunt.io/?country=Euro https://unavatar.io/ https://keyvhq.js.org/ https://osom.js.org/ https://browserless.js.org/ https://metascraper.js.org/', + image: 'https://pbs.twimg.com/profile_images/1717583638991138816/4HvMeeps_400x400.jpg', + lang: 'en', + publisher: 'X', + title: '@Kikobeats on X', + url: 'https://x.com/Kikobeats', + video: null, + } + +## from a X profile with tweets with video + +> Snapshot 1 + + { + author: 'Javi López ⛩️ (javilop)', + date: '2024-05-20T09:35:21.000Z', + description: 'Comparto tutoriales, herramientas y noticias de IA. Fundador @Magnific_AI 🔥 Guía IAs: https://t.co/JApwm5Tmfo 🗞️ Newsletter: https://t.co/tMELO1P8Wk', + image: 'https://pbs.twimg.com/profile_images/1581679886267301888/BHGZpOc6_400x400.jpg', + lang: 'en', + publisher: 'X', + title: '@javilop on X', + url: 'https://x.com/javilop/', + video: null, + } + +## from a tweet + +> Snapshot 1 + + { + author: 'Donald J. Trump (realDonaldTrump)', + date: '2024-05-20T09:39:36.000Z', + description: '“Schiff blasted for not focusing on California homeless.” @foxandfriends His District is in terrible shape. He is a corrupt pol who only dreams of the Impeachment Hoax. In my opinion he is mentally deranged!', + image: 'https://pbs.twimg.com/profile_images/874276197357596672/kUuht00m_400x400.jpg', + lang: 'en', + publisher: 'X', + title: '@realDonaldTrump on X', + url: 'https://x.com/realDonaldTrump/status/1222907250383245320', + video: null, + } + +## from a tweet with a gif + +> Snapshot 1 + + { + author: '#!/kiko/beats (Kikobeats)', + date: '2024-05-20T09:40:45.000Z', + description: 'Experimenting with Clearbit API + Apple TV 3D Parallax https://t.co/Qsm163k4mJ', + image: 'https://pbs.twimg.com/tweet_video_thumb/DDbh3WCXYAAZfz9.jpg:large', + lang: 'en', + publisher: 'X', + title: '@Kikobeats on X', + url: 'https://x.com/Kikobeats/status/880139124791029763', + video: null, + } + +## from a tweet with an image + +> Snapshot 1 + + { + author: 'SmartUA (UaSmart)', + date: '2024-05-20T09:48:26.000Z', + description: 'Y terminamos el dia con Cultura de empresa con @patoroco, @flopezluis, Katia, Angélica en @codemotion_es #codemotion2017', + image: 'https://pbs.twimg.com/media/DPadOKpXcAIL-NW.jpg:large', + lang: 'en', + publisher: 'X', + title: '@UaSmart on X', + url: 'https://x.com/UaSmart/status/934106870834454529', + video: null, + } diff --git a/packages/metascraper-x/test/snapshots/index.js.snap b/packages/metascraper-x/test/snapshots/index.js.snap new file mode 100644 index 0000000000000000000000000000000000000000..cb877a440258b00eb90ecc9fded1a9f5ee4f6940 GIT binary patch literal 1587 zcmV-32F&?ERzV$<0&m01;A*0 zLQQXWARmhe00000000B+m(On-#}&ult{Nw4VMIpS^ald;839@cvB~|xA4>*B$%v{& zS(fEkvSlA440mU_Bks) z8olL^6zCsNEJeyC#mW*~7%5PmX1+6T$=Ub$zIl8z?8L(FsaM{jn(3IPp%e)c+{9Fh zkj0cKPEr*Kt$y%k5D-nSn}3^wM7ho;LwrG68;M z0=#1aJZYRbWt_Ow*`~fWjQ!#K{5%*Z!FU{uC*VoL{IW3z#v`YU6C~4-kmKpspE+IZ zv7RV)DA5YfTwpyh96mc0_l1CHI-Y#Zv7FL5t9;I})~#CIw(3@`U|H6Ush}?fRQZyn znh8GA%nyO`F5{F+#=Cg;=Wk=9Po+=e=P?u+52!>DVx)Dd>cyfi_@Y=^yWC$;S65YY zQV&zMH0Ug@z0jZ3+vmN@H?4N-0-Mykow)Vlj=ffEP3l(?af7*0GwMw0*R747y}98o zW|R6_^QKGJ7o(&-b>2rEbvj?Wlqtu%uh?MN%B)e`~l`7452BEqiXo^=dWS zc0JFo6iZ9}is#5@lTH>UVvSaV>Yh zvzSS-za`_OPc)GeaXq@*k8T#WRYAz^Jy|G`91uHF3>1|yRqDW6LN!soBq^2Swg3C$ znEv?o0e=jbj!+LM)oO1%44dJT-_7#Ls{rsG0K8%V{KEj4Hv!zy4f2x*_Q})Bq|b2W z&L2~H3-A8&_NRY(70+ywK8r;JDzC$WNKzuTK%HqJ8HuSnixHKQB#d%R z6z(Dyn)!?<3@Emn1^iyd0}=>{jfKaI)fX36F4=3|wcdXN`LbtMsx{B6IF+hv*`DVWUsyVS zBfaET9wPE58>3X-5BR4hfgeR3F#inR;}Pydz7H}zH$RK%*8t#yy#5;i&zS(rCO~2W z{Kf?M=L0i+x+OS?16(fPy3CRk&qzw*J#i=;3+!6mtsE8XkKpdhzq;Z_EDW&|6QyZ@ zp^!Ps7z#gA`PCpeE|8doLUKkh5(ypF;=3=uj*U?45FUh*Q64B=dxg zG1*0tGR_3YX!xjnnZ$W8m4tU`aQaX?tQ^S>UbW;@D|W5wmaFAjrBZc@y^So=mX$n2 zH+-cr#liht;Z3??D!QmN(U~gRj^osPX75@c*k$32YhnAa z_Q^E>eroLfx%k)sFie2e&qw=Ivg)Ps5v^maF7BjMvK+tMB*Pc75K|&MOk;Dcjn82- zO=F7dSJ7?Z8j&Q9$l4gt;(2A&$Dc|W;>;7&9GaE;Hm*|+W>gq0QkfJc*+DI4-Dw3D+!T$qlsrW$Z&WpFul5S)S7PWUeAke!Z=BU zLKa~Z)?a9U{8~3BO lI7#g&YRqiSE!kGZt6H90DwRuRr*;^#{{qVk={Bel005Vj4#WTe literal 0 HcmV?d00001 diff --git a/packages/metascraper-twitter/test/test.js b/packages/metascraper-x/test/test.js similarity index 72% rename from packages/metascraper-twitter/test/test.js rename to packages/metascraper-x/test/test.js index 4291d4fab..69274cb13 100644 --- a/packages/metascraper-twitter/test/test.js +++ b/packages/metascraper-x/test/test.js @@ -5,9 +5,7 @@ const test = require('ava') const { test: validator } = require('..') test('true', t => { - t.true( - validator('https://twitter.com/realDonaldTrump/status/1222907250383245320') - ) + t.true(validator('https://x.com/realDonaldTrump/status/1222907250383245320')) }) test('false', t => {