Merge df4cd6d into 00a6316

packages/metascraper-helpers/index.js

@@ -233,14 +233,12 @@ const description = (value, opts) =>
 
 const getDescription = (
   str,
-  { truncateLength = TRUNCATE_MAX_LENGTH, ...opts } = {}
+  { truncateLength = TRUNCATE_MAX_LENGTH, ellipsis = '…', ...opts } = {}
 ) => {
-  let truncatedStr = str.slice(0, truncateLength)
-  if (truncatedStr.length < str.length) {
-    truncatedStr = truncatedStr.trim() + '…'
-  }
-  const description = removeLocation(truncatedStr)
-  return titleize(description, opts)
+  let truncated = str.slice(0, truncateLength)
+  if (truncated.length < str.length) truncated = truncated.trim() + ellipsis
+  const description = removeLocation(truncated)
+  return titleize(description, opts).replace(/\s?\.\.\.?$/, ellipsis)
 }
 
 const publisher = value =>

packages/metascraper-helpers/test/index.js

@@ -219,13 +219,32 @@ test('.isImageExtension', t => {
   t.true(isImageExtension('demo.jpeg'))
   t.true(isImageExtension('demo.gif'))
 })
+
 test('.description', t => {
   t.is(
     description(
       'Let me tell you the story of two investors, neither of whom knew each other, but whose paths crossed in an interesting way. Grace Groner was orphaned at age 12. She never married. She never had kids. She never drove a car. She lived most of her life alone in a one-bedroom house and worked her whole career as a secretary. She was, by all accounts, a lovely lady. But she lived a humble and quiet life. That made the $7 million she left to charity after her death in 2010 at age 100 all the more confusing. People who knew her asked: Where did Grace get all that money? But there was no secret. There was no inheritance. Grace took humble savings from a meager salary and enjoyed eighty years of hands-off compounding in the stock market. That was it. Weeks after Grace died, an unrelated investing story hit the news. Richard Fuscone, former vice chairman of Merrill Lynch’s Latin America division, declared personal bankruptcy, fighting off foreclosure on two homes, one of which was nearly 20,000 square feet and had a $66,000 a month mortgage. Fuscone was the opposite of Grace Groner; educated at Harvard and University of Chicago, he became so successful in the investment industry that he retired in his 40s to “pursue personal and charitable interests.” But heavy borrowing and illiquid investments did him in. The same year Grace Goner left a veritable fortune to charity, Richard stood before a bankruptcy judge and declared: “I have been devastated by the financial crisis … The only source of liquidity is whatever my wife is able to sell in terms of personal furnishings.” The purpose of these stories is not to say you should be like Grace and avoid being like Richard. It’s to point out that there is no other field where these stories are even possible. In what other field does someone with no education, no relevant experience, no resources, and no connections vastly outperform someone with the best education, the most relevant experiences, the best resources and the best connections? There will never be a story of a Grace Groner performing heart surgery better than a Harvard-trained cardiologist. Or building a faster chip than Apple’s engineers. Unthinkable. But these stories happen in investing. That’s because investing is not the study of finance. It’s the study of how people behave with money. And behavior is hard to teach, even to really smart people. You can’t sum up behavior with formulas to memorize or spreadsheet models to follow. Behavior is inborn, varies by person, is hard to measure, changes over time, and people are prone to deny its existence, especially when describing themselves. Grace and Richard show that managing money isn’t necessarily about what you know; it’s how you behave. But that’s not how finance is typically taught or discussed. The finance industry talks too much about what to do, and not enough about what happens in your head when you try to do it. This report describes 20 flaws, biases, and causes of bad behavior I’ve seen pop up often when people deal with money. 1. Earned success and deserved failure fallacy: A tendency to underestimate the role of luck and risk, and a failure to recognize that luck and risk are different sides of the same coin. I like to ask people, “What do you want to know about investing that we can’t know?” It’s not a practical question. So few people ask it. But it forces anyone you ask to think about what they intuitively think is true but don’t spend much time trying to answer because it’s futile. Years ago I asked economist Robert Shiller the question. He answered, “The exact role of luck in successful outcomes.” I love that, because no one thinks luck doesn’t play a role in financial success. But since it’s hard to quantify luck, and rude to suggest people’s success is owed to luck, the default stance is often to implicitly ignore luck as a factor. If I say, “There are a billion investors in the world. By sheer chance, would you expect 100 of them to become billionaires predominately off luck?” You would reply, “Of course.” But then if I ask you to name those investors – to their face – you will back down. That’s the problem. The same goes for failure. Did failed businesses not try hard enough? Were bad investments not thought through well enough?'
     ),
     'Let me tell you the story of two investors, neither of whom knew each other, but whose paths crossed in an interesting way. Grace Groner was orphaned at age 12. She never married. She never had kids. She never drove a car. She lived most of her life alone in a one-bedroom house and worked her whole…'
   )
+  t.is(
+    description(
+      'He talks about the ocean and his favorite animal the cowrie...'
+    ),
+    'He talks about the ocean and his favorite animal the cowrie…'
+  )
+  t.is(
+    description(
+      'He talks about the ocean and his favorite animal the cowrie..'
+    ),
+    'He talks about the ocean and his favorite animal the cowrie…'
+  )
+  t.is(
+    description(
+      'He talks about the ocean and his favorite animal the cowrie ...'
+    ),
+    'He talks about the ocean and his favorite animal the cowrie…'
+  )
 })
 
 test('.$jsonld', t => {

packages/metascraper-youtube/index.js

@@ -1,17 +1,10 @@
 'use strict'
 
+const { $filter, author, toRule, memoizeOne } = require('@metascraper/helpers')
 const reachableUrl = require('reachable-url')
 const getVideoId = require('get-video-id')
 const pLocate = require('p-locate')
 
-const {
-  $filter,
-  author,
-  description,
-  toRule,
-  memoizeOne
-} = require('@metascraper/helpers')
-
 const THUMBAILS_RESOLUTIONS = [
   'maxresdefault.jpg',
   'sddefault.jpg',
@@ -32,20 +25,16 @@ const getThumbnailUrl = (id, gotOpts) => {
 
 const toAuthor = toRule(author)
 
-const toDescription = toRule(description)
-
 const getVideoInfo = memoizeOne(getVideoId)
 
 const test = memoizeOne(url => getVideoInfo(url).service === 'youtube')
 
 module.exports = ({ gotOpts } = {}) => {
   const rules = {
     author: [
-      toAuthor($ => $filter($, $('#owner-name'))),
-      toAuthor($ => $filter($, $('#channel-title'))),
-      toAuthor($ => $filter($, $('[class*="user-info" i]')))
+      toAuthor($ => $filter($, $('[class*="user-info" i]'))),
+      toAuthor($ => $('[itemprop="author"] [itemprop="name"]').attr('content'))
     ],
-    description: [toDescription($ => $filter($, $('#description')))],
     publisher: () => 'YouTube',
     image: [
       ({ url }) => {

packages/metascraper-youtube/test/index.js

@@ -42,3 +42,12 @@ test('youtube channel', async t => {
   t.is(typeof date, 'string')
   t.snapshot(metadata)
 })
+
+test('youtube list', async t => {
+  const html = await readFile(resolve(__dirname, 'fixtures/youtube-list.html'))
+  const url =
+    'https://www.youtube.com/watch?v=EAZvxukW8kY&list=PLDB4BCE9817AE7B43&index=11'
+
+  const metadata = await metascraper({ html, url })
+  t.snapshot(metadata)
+})

packages/metascraper-youtube/test/snapshots/index.js.md

@@ -42,7 +42,7 @@ Generated by [AVA](https://avajs.dev).
     {
       author: 'Andrew JRT',
       date: '2017-07-07T00:00:00.000Z',
-      description: '👍 Subscribe - http://bit.do/AndrewJRT 🐦 Twitter - https://twitter.com/andrew_jrt 🎮 Twitch - https://twitch.tv/andrewjrt ❤ Patreon - https://www.patreon.com/a...',
+      description: '👍 Subscribe - http://bit.do/AndrewJRT 🐦 Twitter - https://twitter.com/andrew_jrt 🎮 Twitch - https://twitch.tv/andrewjrt ❤ Patreon - https://www.patreon.com/a…',
       image: 'https://img.youtube.com/vi/GDRd-BFTYIg/maxresdefault.jpg',
       lang: 'en',
       logo: 'https://www.youtube.com/yts/img/favicon_144-vflWmzoXw.png',
@@ -57,12 +57,27 @@ Generated by [AVA](https://avajs.dev).
 
     {
       author: 'ONE Media',
-      description: `GRINGO Official Trailer (2018) Charlize Theron, Amanda Seyfried Action Movie HD␊
-      © 2017 - Amazon Comedy, Kids, Family and Animated Film, Blockbuster, Action Cinema, Blockbuster, Scifi Movie or Fantasy film, Drama... We keep you in the know! Subscribe now to catch the best movie trailers 2017 and the…`,
+      description: 'Thor 3, Star Wars, Valerian, Transformers 5, Baywatch, Despicable Me 3, Fast and Furious 8, Wonder Woman, Justice League... ALL The Must See Movie Trailer, F…',
       image: 'https://yt3.ggpht.com/-q3yclmvjVzk/AAAAAAAAAAI/AAAAAAAAAAA/jVaGJKJjUps/s200-c-k-no-mo-rj-c0xffffff/photo.jpg',
       lang: null,
       logo: 'https://www.youtube.com/yts/img/favicon_144-vfliLAfaB.png',
       publisher: 'YouTube',
       title: 'ONE Media - YouTube',
       url: 'https://www.youtube.com/channel/UCzcRQ3vRNr6fJ1A9rqFn7QA',
     }
+
+## youtube list
+
+> Snapshot 1
+
+    {
+      author: 'USAGov',
+      date: '2023-01-30T21:21:15.000Z',
+      description: 'Meet Chris Meyer, a marine biologist and scientist from the National Museum of Natural History. He talks about the ocean and his favorite animal the cowrie…',
+      image: 'https://img.youtube.com/vi/EAZvxukW8kY/sddefault.jpg',
+      lang: 'en',
+      logo: 'https://www.youtube.com/s/desktop/d8e1215c/img/favicon_144x144.png',
+      publisher: 'YouTube',
+      title: 'Marine Biologist - Career Spotlight',
+      url: 'https://www.youtube.com/watch?v=EAZvxukW8kY',
+    }

packages/metascraper/test/integration/github/snapshots/index.js.md

@@ -11,7 +11,7 @@ Generated by [AVA](https://avajs.dev).
     {
       audio: null,
       author: 'microlinkhq',
-      description: 'Microlink Query Language. The official HTTP client to interact with Microlink API for Node.js, browsers & Deno. - microlinkhq/mql: Microlink Query Language. The official HTTP client to interact...',
+      description: 'Microlink Query Language. The official HTTP client to interact with Microlink API for Node.js, browsers & Deno. - microlinkhq/mql: Microlink Query Language. The official HTTP client to interact…',
       image: 'https://opengraph.githubassets.com/0d2af1011fc53210b9778b88a31fa2f993cc70fad0b68c8dfa4553cfc124f7dc/microlinkhq/mql',
       lang: 'en',
       logo: 'https://github.com/fluidicon.png',

packages/metascraper/test/integration/googleblog/snapshots/index.js.md

@@ -12,7 +12,7 @@ Generated by [AVA](https://avajs.dev).
       audio: null,
       author: 'Google',
       date: '2016-09-29T12:00:00.000Z',
-      description: 'Posted by Brian Stevens, Vice President, Google Cloud As we officially move into the Google Cloud era, Google Cloud Platform (GCP) conti...',
+      description: 'Posted by Brian Stevens, Vice President, Google Cloud As we officially move into the Google Cloud era, Google Cloud Platform (GCP) conti…',
       image: 'https://3.bp.blogspot.com/-m90zG1Qb7vc/Vel5wAn_isI/AAAAAAAARGE/iSOuuYWUXUA/s1600-r/CloudPlatform_128px_Retina.png',
       lang: 'en',
       logo: 'https://cloudplatform.googleblog.com/favicon.ico',

packages/metascraper/test/integration/indiehackers/snapshots/index.js.md

@@ -12,7 +12,7 @@ Generated by [AVA](https://avajs.dev).
       audio: 'https://chrt.fm/track/EB7BD2/media.transistor.fm/70c487ed/f3249e4a.mp3?src=player',
       author: '—Dawson Whitfield of Logojoy ($70K/mo)',
       date: '2017-10-11T12:00:00.000Z',
-      description: 'Why is it so difficult to recognize product-market fit before you find it? And how do you find it, anyway? Learn about the struggles the team behind Seg...',
+      description: 'Why is it so difficult to recognize product-market fit before you find it? And how do you find it, anyway? Learn about the struggles the team behind Seg…',
       image: 'https://storage.googleapis.com/indie-hackers.appspot.com/podcast-shareables/032-peter-and-calvin-of-segment.png',
       lang: 'en',
       logo: 'https://www.indiehackers.com/images/favicons/favicon--192x192.png',

packages/metascraper/test/integration/lean-data/snapshots/index.js.md

@@ -12,7 +12,7 @@ Generated by [AVA](https://avajs.dev).
       audio: null,
       author: 'Mark Emmons',
       date: '2016-02-23T13:37:22.000Z',
-      description: 'Here’s the thing about pendulums. They swing back and forth. It’s just what they do. Grandfather clocks. Amusement park rides. Economic trends. And throughout most of 2015, Silicon Valley observers were engaged in a spirited debate about whether or not the booming tech economy had peaked and if...',
+      description: 'Here’s the thing about pendulums. They swing back and forth. It’s just what they do. Grandfather clocks. Amusement park rides. Economic trends. And throughout most of 2015, Silicon Valley observers were engaged in a spirited debate about whether or not the booming tech economy had peaked and if…',
       image: 'http://www.leandatainc.com/wp-content/uploads/2016/02/Trees_blown.jpg',
       lang: 'en',
       logo: 'http://www.leandatainc.com/wp-content/uploads/2015/11/favicon.ico',

packages/metascraper/test/integration/mac-rumors/snapshots/index.js.md

@@ -12,7 +12,7 @@ Generated by [AVA](https://avajs.dev).
       audio: null,
       author: 'Juli Clover',
       date: '2016-05-24T16:54:45.000Z',
-      description: 'Apple is actively developing a product that would compete with the Amazon Echo and Google Home, reports The Information. Citing a source with direct...',
+      description: 'Apple is actively developing a product that would compete with the Amazon Echo and Google Home, reports The Information. Citing a source with direct…',
       image: 'https://images.macrumors.com/t/iW3I8r5mPGwRKw5nviIrKwFh6WU=/1600x/http://images.macrumors.com/article-new/2016/05/amazonecho2-250x362.jpg',
       lang: 'en',
       logo: 'https://images.macrumors.com/images-new/favicon-32x32.png',

packages/metascraper/test/integration/pr-newswire/snapshots/index.js.md

@@ -12,7 +12,7 @@ Generated by [AVA](https://avajs.dev).
       audio: null,
       author: 'HackerRank',
       date: '2016-04-25T15:39:00.000Z',
-      description: 'PALO ALTO, Calif., April 25, 2016 /PRNewswire/ -- HackerRank, a platform that ranks engineers based on their...',
+      description: 'PALO ALTO, Calif., April 25, 2016 /PRNewswire/ -- HackerRank, a platform that ranks engineers based on their…',
       image: 'http://content.prnewswire.com/images/prn_facebook_sharing_logo.jpg',
       lang: 'en',
       logo: 'http://content.prnewswire.com/designimages/favicon_PRN.ico',

packages/metascraper/test/integration/techcrunch/snapshots/index.js.md

@@ -12,7 +12,7 @@ Generated by [AVA](https://avajs.dev).
       audio: null,
       author: 'Frederic Lardinois',
       date: '2016-01-12T17:00:44.000Z',
-      description: 'Recruiting software engineers is a massive headache for both startups and established companies. For a while now, HackerRank has tried to make both applying..',
+      description: 'Recruiting software engineers is a massive headache for both startups and established companies. For a while now, HackerRank has tried to make both applying…',
       image: 'https://tctechcrunch2011.files.wordpress.com/2015/08/10-interviewed.png?w=720&h=388&crop=1',
       lang: 'en',
       logo: 'https://s0.wp.com/wp-content/themes/vip/techcrunch-2013/assets/images/homescreen_TCIcon_ipad_2x.png',

packages/metascraper/test/integration/the-verge/snapshots/index.js.md

@@ -12,7 +12,7 @@ Generated by [AVA](https://avajs.dev).
       audio: null,
       author: 'Nick Statt',
       date: '2016-05-24T20:49:03.000Z',
-      description: 'Apple is working on its own competitor to the Amazon Echo to give consumers a Siri-powered speaker for the home, according to a report from The Information today. Not only that, but Apple is also...',
+      description: 'Apple is working on its own competitor to the Amazon Echo to give consumers a Siri-powered speaker for the home, according to a report from The Information today. Not only that, but Apple is also…',
       image: 'https://cdn2.vox-cdn.com/thumbor/PXH0xicJLovP7mMyslRK15uJ8MM=/0x78:1500x922/1600x900/cdn0.vox-cdn.com/uploads/chorus_image/image/49685249/siri_stock.0.0.jpg',
       lang: 'en',
       logo: 'https://cdn0.vox-cdn.com/images/verge/2.0/verge-icon-196x196.v503bbf1.png',