Make domains configurable for hc score dsa1-score

[shawnmjones]

Scoring mementos by Padia's work depends upon a list of domains for the categories of news, image sharing sites, video sharing sites, blog sites, and social media sites. These lists will likely change over time and should be configurable by the end user.

The scoring is handled on lines 431 - 451 of hypercane/score/dsa1_ranking. We will have to make it generic.

hypercane/hypercane/score/dsa1_ranking.py

Lines 431 to 451 in 2e071d7

	if domain in blog_sources:
	return 0.4
	elif domain in wikipedia_imagesharing_sources:
	return 0.6
	elif domain.upper() in pew_news_sources:
	return 0.7
	elif domain in w3newspapers_sources:
	return 0.7
	elif 'news' in domain:
	return 0.7
	elif domain in wikipedia_video_sources:
	return 0.7
	elif domain in adobe_socialmedia_sources:
	return 0.5

The lists themselves are on lines 15 - 386 of that same file.

hypercane/hypercane/score/dsa1_ranking.py

Lines 15 to 386 in 2e071d7

	blog_sources = [
	'blogger.com',
	'blogspot.com',
	'wordpress.com',
	'typepad.com'
	]
	# image sharing websites as per https://en.wikipedia.org/wiki/List_of_image-sharing_websites
	wikipedia_imagesharing_sources = [
	'500px.com',
	'album2.com',
	'bilddagboken.se',
	'myphotodiary.com',
	'kuvapaivakirja.fi',
	'bildedagboka.no',
	'billeddagbog.dk',
	'deviantart.com',
	'dronestagr.am',
	'flickr.com',
	'fotki.com',
	'fotolog.com',
	'fotolog.net',
	'geograph.org.uk',
	'photos.google.com',
	'instagram.com',
	'imgur.com',
	'ipernity.com',
	'jalbum.net',
	'photobucket.com',
	'pinterest.com',
	'pixabay.com',
	'securetribeapp.com',
	'shutterflyinc.com',
	'smugmug.com',
	'snapfish.com',
	'unsplash.com'
	]
	# video domains as per https://en.wikipedia.org/wiki/List_of_video_hosting_services#Specifically_dedicated_video_hosting_websites
	wikipedia_video_sources = [
	'acfun.cn',
	'afreecatv.com',
	'aparat.com',
	'bigo.tv',
	'bilibili.com',
	'bitchute.com',
	'dailymotion.com',
	'godtube.com',
	'iqiyi.com',
	'liveleak.com',
	'metacafe.com',
	'mixer.com',
	'nicovideo.jp',
	'periscope.tv',
	'rutube.ru',
	'schooltube.com',
	'smashcast.tv',
	'trilulilu.ro',
	'tudou.com',
	'tune.pk',
	'twitch.tv',
	'vbox7.com',
	'veoh.com',
	'vimeo.com',
	'youku.com',
	'younow.com',
	'youtube.com'
	]
	# social media domains as per https://helpx.adobe.com/analytics/kb/list-social-networks.html
	adobe_socialmedia_sources = [
	'12seconds.tv',
	'4travel.jp',
	'advogato.org',
	'ameba.jp',
	'anobii.com',
	'answers.yahoo.com',
	'asmallworld.net',
	'avforums.com',
	'backtype.com',
	'badoo.com',
	'bebo.com',
	'bigadda.com',
	'bigtent.com',
	'biip.no',
	'blackplanet.com',
	'blog.seesaa.jp',
	'blogspot.com',
	'blogster.com',
	'blomotion.jp',
	'bolt.com',
	'brightkite.com',
	'buzznet.com',
	'cafemom.com',
	'care2.com',
	'classmates.com',
	'cloob.com',
	'collegeblender.com',
	'cyworld.co.kr',
	'cyworld.com.cn',
	'dailymotion.com',
	'delicious.com',
	'deviantart.com',
	'digg.com',
	'diigo.com',
	'disqus.com',
	'draugiem.lv',
	'facebook.com',
	'faceparty.com',
	'fc2.com',
	'flickr.com',
	'flixster.com',
	'fotolog.com',
	'foursquare.com',
	'friendfeed.com',
	'friendsreunited.co.uk',
	'friendsreunited.com',
	'friendster.com',
	'fubar.com',
	'gaiaonline.com',
	'geni.com',
	'goodreads.com',
	'grono.net',
	'habbo.com',
	'hatena.ne.jp',
	'hi5.com',
	'hotnews.infoseek.co.jp',
	'hyves.nl',
	'ibibo.com',
	'identi.ca',
	'imeem.com',
	'instagram.com',
	'intensedebate.com',
	'irc-galleria.net',
	'iwiw.hu',
	'jaiku.com',
	'jp.myspace.com',
	'kaixin001.com',
	'kaixin002.com',
	'kakaku.com',
	'kanshin.com',
	'kozocom.com',
	'last.fm',
	'linkedin.com',
	'livejournal.com',
	'lnkd.in',
	'matome.naver.jp',
	'me2day.net',
	'meetup.com',
	'mister-wong.com',
	'mixi.jp',
	'mixx.com',
	'mouthshut.com',
	'mp.weixin.qq.com',
	'multiply.com',
	'mumsnet.com',
	'myheritage.com',
	'mylife.com',
	'myspace.com',
	'myyearbook.com',
	'nasza-klasa.pl',
	'netlog.com',
	'nettby.no',
	'netvibes.com',
	'nextdoor.com',
	'nicovideo.jp',
	'ning.com',
	'odnoklassniki.ru',
	'ok.ru',
	'orkut.com',
	'pakila.jp',
	'photobucket.com',
	'pinterest.at',
	'pinterest.be',
	'pinterest.ca',
	'pinterest.ch',
	'pinterest.cl',
	'pinterest.co',
	'pinterest.co.kr',
	'pinterest.co.uk',
	'pinterest.com',
	'pinterest.de',
	'pinterest.dk',
	'pinterest.es',
	'pinterest.fr',
	'pinterest.hu',
	'pinterest.ie',
	'pinterest.in',
	'pinterest.jp',
	'pinterest.nz',
	'pinterest.ph',
	'pinterest.pt',
	'pinterest.se',
	'plaxo.com',
	'plurk.com',
	'plus.google.com',
	'plus.url.google.com',
	'po.st',
	'reddit.com',
	'renren.com',
	'skyrock.com',
	'slideshare.net',
	'smcb.jp',
	'smugmug.com',
	'sonico.com',
	'studivz.net',
	'stumbleupon.com',
	't.163.com',
	't.co',
	't.hexun.com',
	't.ifeng.com',
	't.people.com.cn',
	't.qq.com',
	't.sina.com.cn',
	't.sohu.com',
	'tabelog.com',
	'tagged.com',
	'taringa.net',
	'thefancy.com',
	'toutiao.com',
	'tripit.com',
	'trombi.com',
	'trytrend.jp',
	'tuenti.com',
	'tumblr.com',
	'twine.com',
	'twitter.com',
	'uhuru.jp',
	'viadeo.com',
	'vimeo.com',
	'vk.com',
	'wayn.com',
	'weibo.com',
	'weourfamily.com',
	'wer-kennt-wen.de',
	'wordpress.com',
	'xanga.com',
	'xing.com',
	'yammer.com',
	'yaplog.jp',
	'yelp.co.uk',
	'yelp.com',
	'youku.com',
	'youtube.com',
	'yozm.daum.net',
	'yuku.com',
	'zhihu.com',
	'zooomr.com'
	]
	# news domains from https://pewresearch-org-preprod.go-vip.co/journalism/2019/07/23/state-of-the-news-media-methodology/#digital-native-news-outlet-audit
	pew_news_sources = [
	'12UP.COM',
	'247SPORTS.COM',
	'90MIN.COM',
	'APLUS.COM',
	'BGR.COM',
	'BLEACHERREPORT.COM',
	'BREITBART.COM',
	'BUSINESSINSIDER.COM',
	'BUSTLE.COM',
	'BUZZFEED.COM',
	'BUZZFEEDNEWS.COM',
	'CHEATSHEET.COM',
	'CINEMABLEND.COM',
	'CNET.COM',
	'COMICBOOK.COM',
	'DAILYDOT.COM',
	'DEADSPIN.COM',
	'DIGITALTRENDS.COM',
	'EATER.COM',
	'ELITEDAILY.COM',
	'ENGADGET.COM',
	'FIVETHIRTYEIGHT.COM',
	'GAMESPOT.COM',
	'GIZMODO.COM',
	'HELLOGIGGLES.COM',
	'HOLLYWOODLIFE.COM',
	'HUFFINGTONPOST.COM',
	'IBTIMES.COM',
	'IFLSCIENCE.COM',
	'IGN.COM',
	'IJR.COM',
	'IJREVIEW.COM',
	'INVESTOPEDIA.COM',
	'JEZEBEL.COM',
	'MARKETWATCH.COM',
	'MASHABLE.COM',
	'MAXPREPS.COM',
	'MIC.COM',
	'OPPOSINGVIEWS.COM',
	'POLITICO.COM',
	'POLYGON.COM',
	'QZ.COM',
	'RARE.US',
	'RAWSTORY.COM',
	'REFINERY29.COM',
	'SALON.COM',
	'SBNATION.COM',
	'SLATE.COM',
	'TECHRADAR.COM',
	'THEBLAZE.COM',
	'THEDAILYBEAST.COM',
	'THEROOT.COM',
	'THEVERGE.COM',
	'THISISINSIDER.COM',
	'THRILLIST.COM',
	'TMZ.COM',
	'TOPIX.COM',
	'TOPIX.NET',
	'UPROXX.COM',
	'UPWORTHY.COM',
	'VOX.COM'
	]
	# sources from https://www.w3newspapers.com/newssites/
	w3newspapers_sources = [
	'aljazeera.com',
	'nytimes.com',
	'wsj.com',
	'huffpost.com',
	'washingtonpost.com',
	'latimes.com',
	'reuters.com',
	'abcnews.go.com',
	'usatoday.com',
	'bloomberg.com',
	'nbcnews.com',
	'dailymail.co.uk',
	'theguardian.com',
	'thesun.co.uk',
	'mirror.co.uk',
	'telegraph.co.uk',
	'bbc.com',
	'thestar.com',
	'theglobeandmail.com',
	'news.com.au',
	'forbes.com',
	'cnbc.com',
	'chinadaily.com.cn',
	'chron.com',
	'nypost.com',
	'usnews.com',
	'dw.com',
	'indiatimes.com',
	'thehindu.com',
	'indianexpress.com',
	'hindustantimes.com',
	'cbsnews.com',
	'time.com',
	'sfgate.com',
	'thehill.com',
	'thedailybeast.com',
	'newsweek.com',
	'theatlantic.com',
	'nzherald.co.nz',
	'herald.co.zw',
	'vanguardngr.com',
	'dailysun.co.za',
	'thejakartapost.com',
	'thestar.com.my',
	'straitstimes.com',
	'bangkokpost.com',
	'japantimes.co.jp',
	'thedailystar.net',
	'dawn.com',
	'alarabiya.net',
	'hollywoodreporter.com',
	'scmp.com',
	'aljazeera.com',
	'voanews.com'
	]

[shawnmjones]

This is a duplicate of #10. Closing.