Skip to content

Commit

Permalink
Reworked AniDB balance functionality, Fixed search slowdown, Fixed no…
Browse files Browse the repository at this point in the history 
… data bug, Added AniDB bad episode title ignore

The previously introduced AniDB balance code was being two aggressive
when left on by default resulting in performance loss, its now only used
if a needed.
  • Loading branch information
@Dingmatt
Dingmatt committed Apr 13, 2017
1 parent f4bba1c commit 1ebc71a
Show file tree
Hide file tree
Showing 5 changed files with 79 additions and 38 deletions.
5 changes: 3 additions & 2 deletions Plug-ins/Amsa_Test.bundle/Contents/Code/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,11 +90,12 @@ def scoreTitle(anime=anime, maxi=maxi, anidb=anidb, perfectScore=perfectScore):
elif anime.Score >= 90:
elite.append(isValid)
if isValid:
if anime.Score == 100: perfectScore.append(True)
if anime.Score == 100: perfectScore.append(anime.Id)
Log.Debug("Init - Search() - find - id: '%s-%s', title: '%s', score: '%s'" % ("anidb", anime.Id, anime.Title, anime.Score))
results.Append(MetadataSearchResult(id="%s-%s" % ("anidb", anime.Id), name="%s [%s-%s]" % (anime.Title, "anidb", anime.Id), year=startdate, lang=Locale.Language.English, score=anime.Score))

if len(perfectScore) > 1:

if len(list(set(perfectScore))) > 1:
for result in results:
if result.score == 100:
show = anidb.AniDB(result.id.split("-")[1])
Expand Down
37 changes: 21 additions & 16 deletions Plug-ins/Amsa_Test.bundle/Contents/Code/common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,8 @@ def MapSeries(mappingData):
status = ""
tvdbParse = "S%sE%s" % (str(ScudLee.DefaultTvdbSeason).zfill(2), str(i + ScudLee.EpisodeOffset).zfill(2))
SubElement(seriesMap, "Episode", anidb="%s" % (anidb.ParseNoFromSeason(1, i)), tvdb=tvdbParse, status=status)

Log("Common - MapSeries() - Season - AniDB: '%s', TvDB: '%s'" % (anidb.ParseNoFromSeason(1, i), tvdbParse))

for i in range(1, AniDB.SpecialCount+1):
status = "scudlee missing"
tvdbParse = "S00E00"
Expand All @@ -202,7 +203,8 @@ def MapSeries(mappingData):
status = ""
tvdbParse = "S%sE%s" % ("00", str(i + ScudLee.EpisodeOffset).zfill(2))
SubElement(seriesMap, "Episode", anidb="%s" % (anidb.ParseNoFromType(2, i)), tvdb=tvdbParse, status=status)

Log("Common - MapSeries() - Season - AniDB: '%s', TvDB: '%s'" % (anidb.ParseNoFromType(2, i), tvdbParse))

for opening in AniDB.OpList:
if not seriesMap.xpath("""./Episode[@anidb="%s"]""" % (opening)):
SubElement(seriesMap, "Episode", anidb="%s" % (opening), tvdb="%s" % (anidb.ParseLocalNoFromType(3, openingEpsNo, "op")), status="")
Expand All @@ -220,7 +222,8 @@ def MapSeries(mappingData):
SubElement(seriesMap, "Episode", anidb="%s" % (anidb.ParseNoFromSeason(1, episode.Absolute_Index - ScudLee.EpisodeOffset))
, tvdb="S%sE%s" % (episode.Season, episode.Number)
, status="")

Log("Common - MapSeries() - Season - AniDB: '%s', TvDB: 'S%sE%s'" % (anidb.ParseNoFromSeason(1, episode.Absolute_Index - ScudLee.EpisodeOffset), episode.Season, episode.Number))

seriesMap[:] = sorted(seriesMap, key=lambda x: (0 if re.sub('[^A-Z]','', x.get("anidb")) else 1, int(re.sub('[^0-9]','', x.get("anidb")))))

unmappedlist = sorted(mapping.xpath("""./Series/Episode[@tvdb="S00E00" and @missing!=""]"""), key=lambda x: x.getparent().get("anidbid"))
Expand Down Expand Up @@ -309,7 +312,7 @@ def Provider_Par():
def Provider_Task(root=root, providers=providers, provider=provider):
#Log("Provider: %s" %(providers))
data = None
for map in sorted(root.xpath("""./Season/Episode/Mapped/%s""" % (providers[0].lower())), key=lambda x: int(x.get("series") if x.get("series") else 0)):
for map in sorted(root.xpath("""./Season/Episode/Mapped/%s""" % (providers[0].lower())), key=lambda x: int(x.get("series") if x.get("series") and x.get("series") != "None" else 0)):
if map.get("series"):
for existing in map.getparent().getparent().xpath(""".//%s""" % (providers[0])):
if not existing.getparent().tag == "Mapped":
Expand Down Expand Up @@ -353,12 +356,13 @@ def Provider_Task(root=root, providers=providers, provider=provider):
def SearchMap(root, media, anidbId, tvdbId):
logging.Log_Milestone("SearchMap")
streamTag = []
seasonMap = root.xpath("""./Season[@AnidbId="%s" and @num!="0"]""" % (anidbId))[0]
seasonNo = 0
seasonMap = root.xpath("""./Season[@AnidbId="%s" and @num!="0"]""" % (anidbId))
if not seasonMap:
seasonMap = root.xpath("""./Season[@TvdbId="%s" and @num!="0"]""" % (tvdbId))[0]
seasonNo = int(seasonMap.get("num"))
if not seasonNo:
seasonNo = 0
seasonMap = root.xpath("""./Season[@TvdbId="%s" and @num!="0"]""" % (tvdbId))
if seasonMap:
seasonMap = seasonMap [0]
seasonNo = int(seasonMap.get("num"))

season = str(int(map.getparent().get('num')) - seasonNo + 1)

Expand Down Expand Up @@ -390,12 +394,13 @@ def SearchMap(root, media, anidbId, tvdbId):
def MapMedia(root, metadata, anidbId, tvdbId):
logging.Log_Milestone("MapMedia")
streamTag = []
seasonMap = root.xpath("""./Season[@AnidbId="%s" and @num!="0"]""" % (anidbId))[0]
seasonNo = 0
seasonMap = root.xpath("""./Season[@AnidbId="%s" and @num!="0"]""" % (anidbId))
if not seasonMap:
seasonMap = root.xpath("""./Season[@TvdbId="%s" and @num!="0"]""" % (tvdbId))[0]
seasonNo = int(seasonMap.get("num"))
if not seasonNo:
seasonNo = 0
seasonMap = root.xpath("""./Season[@TvdbId="%s" and @num!="0"]""" % (tvdbId))
if seasonMap:
seasonMap = seasonMap [0]
seasonNo = int(seasonMap.get("num"))

#Log("Populate Season")
logging.Log_Milestone("MapMedia_Season")
Expand All @@ -420,7 +425,7 @@ def MapMedia(root, metadata, anidbId, tvdbId):

@parallelize
def Episode_Par():
for map in sorted(root.xpath("""./Season[@num>=%s]/Episode""" % (seasonNo)), key=lambda x: x.getparent().get("num"), reverse=False) :
for map in sorted(root.xpath("""./Season[@num>=%s or @num=0]/Episode""" % (seasonNo)), key=lambda x: x.getparent().get("num"), reverse=False) :

@task
def Episode_Task(map=map, metadata=metadata, anidbId=anidbId, tvdbId=tvdbId, seasonNo=seasonNo):
Expand All @@ -430,7 +435,7 @@ def Episode_Task(map=map, metadata=metadata, anidbId=anidbId, tvdbId=tvdbId, sea
#Log("Episode: '%s', '%s', %s, %s" % (season, episode, anidbId, map.getparent().get('AnidbId')))

logging.Log_Milestone("MapMedia_Episode_S" + season + "E" + episode)
metadata.seasons[season].episodes[episode].title = functions.PopulateMetadata(map.xpath("""./Title/*[node()]"""), str, constants.EPISODE_TITLE_PRIORITY)
metadata.seasons[season].episodes[episode].title = functions.PopulateMetadata(map.xpath("""./Title/*[node()]"""), str, constants.EPISODE_TITLE_PRIORITY, None, "EpisodeTitle")
metadata.seasons[season].episodes[episode].summary = functions.PopulateMetadata(map.xpath("""./Summary/*[node()]"""), str, constants.EPISODE_SUMMARY_PRIORITY)
metadata.seasons[season].episodes[episode].originally_available_at = functions.PopulateMetadata(map.xpath("""./Originally_Available_At/*[node()]"""), datetime.date, constants.EPISODE_ORIGINALLYAVAILABLEAT_PRIORITY)
metadata.seasons[season].episodes[episode].rating = functions.PopulateMetadata(map.xpath("""./Rating/*[node()]"""), float, constants.EPISODE_RATING_PRIORITY)
Expand Down
1 change: 1 addition & 0 deletions Plug-ins/Amsa_Test.bundle/Contents/Code/constants.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@
SERIES_TYPE_PRIORITY = ["main", "official", "syn", "synonym", "short"]
ANIDB_THROTTLE_THRESHOLD = 100
ANIDB_ANTIBAN_WAIT = 2
ANIDB_BADTITLES = ["^TV Special$", "^Part . of .$", "^Episode .$"]
#-------------------ANIDB------------------#

#-------------------TVDB-------------------#
Expand Down
69 changes: 50 additions & 19 deletions Plug-ins/Amsa_Test.bundle/Contents/Code/functions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

def XMLFromURL (url, filename="", directory="", cache=constants.DefaultCache, timeout=constants.DefaultTimeout):
Log.Debug("Functions - XMLFromURL() - url: '%s', filename: '%s'" % (url, filename))
global AniDB_WaitUntil, req_proxy, AniDB_RequestCount
global AniDB_WaitUntil, req_proxy #, AniDB_RequestCount
#try:
# netLock.acquire()
result = LoadFile(filename, directory, cache)
Expand All @@ -30,28 +30,53 @@ def XMLFromURL (url, filename="", directory="", cache=constants.DefaultCache, ti
runOnce = True
while AniDB_WaitUntil > dt.now():
if runOnce:
Log("Functions - XMLFromURL() - AniDB AntiBan Delay: %s" % (AniDB_RequestCount))
AniDB_RequestCount += 1
Log("Functions - XMLFromURL() - AniDB AntiBan Delay: %s" % (AniDB_WaitUntil))#% (AniDB_RequestCount))
#AniDB_RequestCount += 1
runOnce = False
sleep(0.1)
AniDB_WaitUntil = dt.now() + timedelta(seconds=constants.ANIDB_ANTIBAN_WAIT)
if AniDB_RequestCount >= constants.ANIDB_THROTTLE_THRESHOLD:
AniDB_RequestCount = 0
req_proxy = RequestProxy(sustain=True)
# if AniDB_RequestCount >= constants.ANIDB_THROTTLE_THRESHOLD:
# req_proxy.randomize_proxy()
# AniDB_RequestCount = 0
# result = None
# attempts = 0
# while result is None:
# if len(req_proxy.get_proxy_list()) == 0: RequestProxy(sustain=True)
# current_proxy = req_proxy.current_proxy_ip()
# request = req_proxy.generate_proxied_request(url, params={}, req_timeout=2)
# if request is not None and request.status_code == 200 and len(request.text) > 1024:
# result = request.text
# #else:
# #sleep(0.5)
# #Log("Test: %s" % (proxy == req_proxy.current_proxy))
# if current_proxy == req_proxy.current_proxy_ip():
# req_proxy.randomize_proxy()
# AniDB_RequestCount = 0
# attempts = 0
# Log("Attempt 1: %s, %s" % (attempts, len(req_proxy.get_proxy_list())))
# attempts +=1
# else:
result = str(HTTP.Request(url, headers={"Accept-Encoding":"gzip, deflate", "content-type":"charset=utf8"}, cacheTime=cache, timeout=timeout))
if str(result).startswith("<error>"):
# req_proxy.randomize_proxy()
# AniDB_RequestCount = 0
result = None
attempts = 0
while result is None:
request = req_proxy.generate_proxied_request(url, params={}, req_timeout=2)
if request is not None and request.status_code == 200:
result = request.text
else:
result = str(HTTP.Request(url, headers={"Accept-Encoding":"gzip, deflate", "content-type":"charset=utf8"}, cacheTime=cache, timeout=timeout))
if str(result).startswith("<error>"):
result = None
while result is None:
req_proxy = RequestProxy(sustain=True)
if len(req_proxy.get_proxy_list()) == 0: req_proxy = RequestProxy(sustain=True)
current_proxy = req_proxy.current_proxy_ip()
request = req_proxy.generate_proxied_request(url, params={}, req_timeout=2)
if request is not None and request.status_code == 200:
if request is not None and request.status_code == 200 and len(request.text) > 1024:
result = request.text
#else:
#sleep(0.5)
#Log("Test: %s" % (proxy == req_proxy.current_proxy))
if current_proxy == req_proxy.current_proxy_ip():
req_proxy.randomize_proxy()
# AniDB_RequestCount = 0
attempts = 0
Log("Attempt 2: %s, %s" % (attempts, len(req_proxy.get_proxy_list())))
attempts +=1

Log("Functions - XMLFromURL() - AniDB AntiBan Proxy")
logging.Log_AniDB(url)
Expand Down Expand Up @@ -220,7 +245,7 @@ def CleanTitle(title, filter = False):
def GetElementText(el, xp, default=None):
return el.xpath(xp)[0].text if el is not None and el.xpath(xp) and el.xpath(xp)[0].text else ("" if default == None else default)

def GetByPriority(metaList, priorityList, metaType):
def GetByPriority(metaList, priorityList, metaType, secondType=None):
try:
if metaType is list:
return ast.literal_eval(sorted(filter(lambda i: i.text != None and i.text != "None", metaList), key=lambda x: priorityList.index(x.tag.lower()), reverse=False)[0].text)
Expand All @@ -238,13 +263,19 @@ def GetByPriority(metaList, priorityList, metaType):

return dataList
else:
return sorted(filter(lambda i: i.text != None and i.text != "None", metaList), key=lambda x: priorityList.index(x.tag.lower()), reverse=False)[0].text
result = sorted(filter(lambda i: i.text != None and i.text != "None", metaList), key=lambda x: priorityList.index(x.tag.lower()), reverse=False)[0].text
if secondType == "EpisodeTitle":
for pattern in constants.ANIDB_BADTITLES:
if re.search(r"%s" % (pattern), result, re.IGNORECASE) and priorityList.count > 1:
result = sorted(filter(lambda i: i.text != None and i.text != "None", metaList), key=lambda x: priorityList.index(x.tag.lower()), reverse=False)[1].text
return result

except:
return ""

def PopulateMetadata(map, metaType, priorityList, metaList=None, secondType=None):
if map:
data = GetByPriority(map, priorityList, metaType)
data = GetByPriority(map, priorityList, metaType, secondType)
if data:
#Log("Data: %s, %s" % (data, metaList))
if metaType is datetime.date:
Expand Down
5 changes: 4 additions & 1 deletion Plug-ins/Amsa_Test.bundle/Contents/Libraries/Shared/http/requests/proxy/requestProxy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,10 @@ def set_logger_level(self, level):

def get_proxy_list(self):
return self.proxy_list


def current_proxy_ip(self):
return str(self.current_proxy)

def generate_random_request_headers(self):
headers = {
"Connection": "close", # another way to cover tracks
Expand Down

0 comments on commit 1ebc71a

Please sign in to comment.