Refactor reddit interface to use PRAW 4+

Add Amazon and Anime Network link support
Add lots of type annotations to critical and confusing sections (database, services)
Begin rewrite on show finding

.gitignore

@@ -1,5 +1,5 @@
 # Custom
-config.ini
+config_debug.ini
 *.sqlite
 *.sqlite-journal
 *.sqlite-backup

config-example.ini → config.ini

@@ -28,6 +28,14 @@ filter = 2
 new_show_types = tv
 record_scores = true
 
+[options.discovery]
+# Service used 
+primary_source = mal
+# Services for additional data/links
+secondary_sources = anidb
+# Stream services
+stream_sources = crunchyroll
+
 [post]
 title = [Spoilers] {show_name} - Episode {episode} discussion
 title_postfix_final = - FINAL

requirements.txt

@@ -1,7 +1,6 @@
-praw<4
-praw-script-oauth>=0.1.1
-beautifulsoup4==4.4.1
+praw>=4.1.0
+beautifulsoup4==4.5.1
 feedparser==5.2.1
-requests==2.9.1
+requests==2.12.4
 Unidecode==0.4.19
-PyYAML==3.11
+PyYAML==3.12

src/config.py

@@ -25,19 +25,26 @@ def __init__(self):
 		self.new_show_types = list()
 		self.record_scores = False
 
+		self.discovery_primary_source = None
+		self.discovery_secondary_sources = list()
+		self.discovery_stream_sources = list()
+
 		self.post_title = None
 		self.post_title_postfix_final = None
 		self.post_body = None
 		self.post_formats = dict()
 
 def from_file(file_path):
-	config = Config()
+	if file_path.find(".") < 0:
+		file_path += ".ini"
 
 	parsed = WhitespaceFriendlyConfigParser()
 	success = parsed.read(file_path)
 	if len(success) == 0:
 		print("Failed to load config file")
-		return config
+		return None
+
+	config = Config()
 
 	if "data" in parsed:
 		sec = parsed["data"]
@@ -60,9 +67,15 @@ def from_file(file_path):
 		sec = parsed["options"]
 		config.debug = sec.getboolean("debug", False)
 		from data.models import str_to_showtype
-		config.new_show_types.extend(map(lambda s: str_to_showtype(s.strip()), sec.get("new_show_types", "").split(",")))
+		config.new_show_types.extend(map(lambda s: str_to_showtype(s.strip()), sec.get("new_show_types", "").split(" ")))
 		config.record_scores = sec.getboolean("record_scores", False)
 
+	if "options.discovery" in parsed:
+		sec = parsed["options.discovery"]
+		config.discovery_primary_source = sec.get("primary_source", None)
+		config.discovery_secondary_sources = sec.get("secondary_sources", "").split(" ")
+		config.discovery_stream_sources = sec.get("stream_sources", "").split(" ")
+
 	if "post" in parsed:
 		sec = parsed["post"]
 		config.post_title = sec.get("title", None)

src/data/database.py

@@ -2,12 +2,16 @@
 import sqlite3, re
 from functools import wraps, lru_cache
 from unidecode import unidecode
-from typing import Set
+from typing import Set, List, Optional
 
 from .models import Show, ShowType, Stream, Service, LinkSite, Link, Episode, EpisodeScore, UnprocessedStream, UnprocessedShow
 
 def living_in(the_database):
-	# wow wow
+	"""
+	wow wow
+	:param the_database: 
+	:return: 
+	"""
 	try:
 		db = sqlite3.connect(the_database)
 		db.execute("PRAGMA foreign_keys=ON")
@@ -163,7 +167,7 @@ def register_link_sites(self, sites):
 
 	@db_error_default(None)
 	@lru_cache(10)
-	def get_service(self, id=None, key=None) -> Service:
+	def get_service(self, id=None, key=None) -> Optional[Service]:
 		if id is not None:
 			self.q.execute("SELECT id, key, name, enabled, use_in_post FROM Services WHERE id = ?", (id,))
 		elif key is not None:
@@ -175,7 +179,7 @@ def get_service(self, id=None, key=None) -> Service:
 		return Service(*service)
 
 	@db_error_default(list())
-	def get_services(self, enabled=True, disabled=False) -> [Service]:
+	def get_services(self, enabled=True, disabled=False) -> List(Service):
 		services = list()
 		if enabled:
 			self.q.execute("SELECT id, key, name, enabled, use_in_post FROM Services WHERE enabled = 1")
@@ -188,7 +192,7 @@ def get_services(self, enabled=True, disabled=False) -> [Service]:
 		return services
 
 	@db_error_default(None)
-	def get_stream(self, id=None, service_tuple=None) -> Stream:
+	def get_stream(self, id=None, service_tuple=None) -> Optional[Stream]:
 		if id is not None:
 			debug("Getting stream for id {}".format(id))
 
@@ -215,7 +219,7 @@ def get_stream(self, id=None, service_tuple=None) -> Stream:
 			return None
 
 	@db_error_default(list())
-	def get_streams(self, service=None, show=None, active=True, unmatched=False, missing_name=False) -> [Stream]:
+	def get_streams(self, service=None, show=None, active=True, unmatched=False, missing_name=False) -> List(Stream):
 		# Not the best combination of options, but it's only the usage needed
 		if service is not None:
 			debug("Getting all streams for service {}".format(service.key))
@@ -279,7 +283,7 @@ def update_stream(self, stream: Stream, show=None, active=None, name=None, show_
 	# Links
 
 	@db_error_default(None)
-	def get_link_site(self, id=None, key=None) -> LinkSite:
+	def get_link_site(self, id:str=None, key:str=None) -> Optional[LinkSite]:
 		if id is not None:
 			self.q.execute("SELECT id, key, name, enabled FROM LinkSites WHERE id = ?", (id,))
 		elif key is not None:
@@ -293,7 +297,7 @@ def get_link_site(self, id=None, key=None) -> LinkSite:
 		return LinkSite(*site)
 
 	@db_error_default(list())
-	def get_link_sites(self, enabled=True, disabled=False) -> [LinkSite]:
+	def get_link_sites(self, enabled=True, disabled=False) -> List(LinkSite):
 		sites = list()
 		if enabled:
 			self.q.execute("SELECT id, key, name, enabled FROM LinkSites WHERE enabled = 1")
@@ -306,7 +310,7 @@ def get_link_sites(self, enabled=True, disabled=False) -> [LinkSite]:
 		return sites
 
 	@db_error_default(list())
-	def get_links(self, show=None) -> [Link]:
+	def get_links(self, show:Show=None) -> List(Link):
 		if show is not None:
 			debug("Getting all links for show {}".format(show.id))
 
@@ -320,7 +324,7 @@ def get_links(self, show=None) -> [Link]:
 			return list()
 
 	@db_error_default(None)
-	def get_link(self, show, link_site) -> Link:
+	def get_link(self, show: Show, link_site: LinkSite) -> Optional[Link]:
 		debug("Getting link for show {} and site {}".format(show.id, link_site.key))
 
 		self.q.execute("SELECT site, show, site_key FROM Links WHERE show = ? AND site = ?", (show.id, link_site.id))
@@ -373,7 +377,7 @@ def get_shows(self, missing_length=False, missing_stream=False, enabled=True, de
 		return shows
 
 	@db_error_default(None)
-	def get_show(self, id=None, stream=None) -> Show:
+	def get_show(self, id=None, stream=None) -> Optional[Show]:
 		#debug("Getting show from database")
 
 		# Get show ID
@@ -409,7 +413,7 @@ def add_show(self, raw_show: UnprocessedShow, commit=True) -> int:
 		return show_id
 
 	@db_error_default(None)
-	def update_show(self, show_id, raw_show, commit=True):
+	def update_show(self, show_id: str, raw_show: UnprocessedShow, commit=True):
 		debug("Updating show: {}".format(raw_show))
 
 		#name = raw_show.name
@@ -458,7 +462,7 @@ def stream_has_episode(self, stream: Stream, episode_num) -> bool:
 		return num_found > 0
 
 	@db_error_default(None)
-	def get_latest_episode(self, show: Show) -> Episode:
+	def get_latest_episode(self, show: Show) -> Optional[Episode]:
 		self.q.execute("SELECT episode, post_url FROM Episodes WHERE show = ? ORDER BY episode DESC LIMIT 1", (show.id,))
 		data = self.q.fetchone()
 		if data is not None:
@@ -472,7 +476,7 @@ def add_episode(self, show_id, episode_num, post_url):
 		self.commit()
 
 	@db_error_default(list())
-	def get_episodes(self, show, ensure_sorted=True) -> [Episode]:
+	def get_episodes(self, show, ensure_sorted=True) -> List[Episode]:
 		episodes = list()
 		self.q.execute("SELECT episode, post_url FROM Episodes WHERE show = ?", (show.id,))
 		for data in self.q.fetchall():
@@ -485,24 +489,24 @@ def get_episodes(self, show, ensure_sorted=True) -> [Episode]:
 	# Scores
 
 	@db_error_default(list())
-	def get_show_scores(self, show: Show) -> [EpisodeScore]:
+	def get_show_scores(self, show: Show) -> List[EpisodeScore]:
 		self.q.execute("SELECT episode, site, score FROM Scores WHERE show=?", (show.id,))
 		return [EpisodeScore(show.id, *s) for s in self.q.fetchall()]
 
 	@db_error_default(list())
-	def get_episode_scores(self, show: Show, episode: Episode) -> [EpisodeScore]:
+	def get_episode_scores(self, show: Show, episode: Episode) -> List[EpisodeScore]:
 		self.q.execute("SELECT site, score FROM Scores WHERE show=? AND episode=?", (show.id, episode.number))
 		return [EpisodeScore(show.id, episode.number, *s) for s in self.q.fetchall()]
 
 	@db_error_default(None)
-	def get_episode_score_avg(self, show: Show, episode: Episode) -> EpisodeScore:
+	def get_episode_score_avg(self, show: Show, episode: Episode) -> Optional[EpisodeScore]:
 		debug("Calculating avg score for {} ({})".format(show.name, show.id))
 		self.q.execute("SELECT score FROM Scores WHERE show=? AND episode=?", (show.id, episode.number))
 		scores = [s[0] for s in self.q.fetchall()]
 		if len(scores) > 0:
 			score = sum(scores)/len(scores)
 			debug("  Score: {} (from {} scores)".format(score, len(scores)))
-			return score
+			return EpisodeScore(show.id, episode.number, None, score)
 		return None
 
 	@db_error
@@ -538,7 +542,7 @@ def to_show_type(db_val: str) -> ShowType:
 			return st
 	return ShowType.UNKNOWN
 
-def from_show_type(st: ShowType) -> str:
+def from_show_type(st: ShowType) -> Optional[str]:
 	if st is None:
 		return None
 	return st.value

src/holo.py

@@ -7,7 +7,7 @@
 # Metadata
 name = "Holo"
 description = "episode discussion bot"
-version = "0.1.3"
+version = "0.1.4"
 
 # Ensure proper files can be access if running with cron
 import os
@@ -86,6 +86,9 @@ def main(config, args, extra_args):
 	import config as config_loader
 	config_file = os.environ["HOLO_CONFIG"] if "HOLO_CONFIG" in os.environ else args.config_file[0]
 	c = config_loader.from_file(config_file)
+	if c is None:
+		print("Cannot start without a valid configuration file")
+		sys.exit(2)
 
 	# Override config with args
 	c.debug |= args.debug

src/module_find_episodes.py

@@ -191,7 +191,7 @@ def _gen_text_discussions(db, formats, show, stream):
 		for episode in episodes:
 			episode = stream.to_display_episode(episode)
 			score = db.get_episode_score_avg(show, episode)
-			table.append(safe_format(formats["discussion"], episode_num=episode.number, episode_link=episode.link, episode_score=score if score else ""))
+			table.append(safe_format(formats["discussion"], episode_num=episode.number, episode_link=episode.link, episode_score=score.score if score else ""))
 		return "\n".join(table)
 	else:
 		return formats["discussion_none"]

src/module_find_shows.py

@@ -25,41 +25,46 @@ def main(config, db, output_yaml, output_file=None, **kwargs):
 
 def create_season_config(config, db, output_file):
 	info("Checking for new shows")
-	shows = []
+	shows = _get_primary_source_shows(config)
 
+	debug("Outputting new shows")
+	with open(output_file, "w", encoding="utf-8") as f:
+		yaml.dump_all(shows, f, explicit_start=True, default_flow_style=False)
+
+def _get_primary_source_shows(config):
+	debug("Retrieving primary show list")
 	link_handlers = services.get_link_handlers()
 	service_handlers = services.get_service_handlers()
 
-	for site in db.get_link_sites():
-		if site.key not in link_handlers:
-			warning("Link site handler for {} not installed".format(site.key))
+	site_key = config.discovery_primary_source
+	if site_key not in link_handlers:
+		warning("Primary source site handler for {} not installed".format(site_key))
+		return
+
+	site_handler = link_handlers.get(site_key)
+	shows = []
+	for raw_show in site_handler.get_seasonal_shows(useragent=config.useragent):
+		if raw_show.show_type is not ShowType.UNKNOWN and raw_show.show_type not in config.new_show_types:
+			debug("  Show isn't an allowed type ({})".format(raw_show.show_type))
+			debug("    name={}".format(raw_show.name))
 			continue
 
-		site_handler = link_handlers.get(site.key)
-		for raw_show in site_handler.get_seasonal_shows(useragent=config.useragent):
-			if raw_show.show_type is not ShowType.UNKNOWN and raw_show.show_type not in config.new_show_types:
-				debug("  Show isn't an allowed type ({})".format(raw_show.show_type))
-				debug("    name={}".format(raw_show.name))
-				continue
-			#if raw_show.name in shows:
-			#	debug("  Show already seen")
-			#	debug("    name={}".format(raw_show.name))
-			#	continue
-
-			debug("New show: {}".format(raw_show.name))
-
-			d = OrderedDict([
-				("title", raw_show.name),
-				("type", raw_show.show_type.name.lower()),
-				("has_source", raw_show.has_source),
-				("info", OrderedDict([(i, "") for i in sorted(link_handlers.keys())])),
-				("streams", OrderedDict([(s, "") for s in sorted(service_handlers.keys()) if not service_handlers[s].is_generic and s in ["crunchyroll", "funimation"]]))
-			])
-			shows.append(d)
+		debug("New show: {}".format(raw_show.name))
+
+		d = OrderedDict([
+			("title", raw_show.name),
+			("type", raw_show.show_type.name.lower()),
+			("has_source", raw_show.has_source),
+			("info", OrderedDict([(i, "") for i in sorted(link_handlers.keys()) if i in config.discovery_secondary_sources])),
+			("streams", OrderedDict([(s, "") for s in sorted(service_handlers.keys()) if not service_handlers[s].is_generic and s in config.discovery_stream_sources]))
+		])
+		shows.append(d)
 
-	debug("Outputting new shows")
-	with open(output_file, "w", encoding="utf-8") as f:
-		yaml.dump_all(shows, f, explicit_start=True, default_flow_style=False)
+	return shows
+
+#############
+# OLD STUFF #
+#############
 
 def check_new_shows(config, db, update_db=True):
 	info("Checking for new shows")

src/module_update_shows.py

@@ -34,7 +34,7 @@ def _check_show_lengths(config, db, update_db=True):
 				continue
 
 			# Validate length
-			new_length = handler.get_episode_count(show, link, useragent=config.useragent)
+			new_length = handler.get_episode_count(link, useragent=config.useragent)
 			if new_length is not None:
 				debug("    Lists length: {}".format(new_length))
 				if length is not None and new_length != length: