Permalink
Browse files

fix(worker) adjust zombie threshold according to timeout

  • Loading branch information...
fffonion committed Mar 19, 2018
1 parent eea87ab commit 597f020a3efe0d9231866c413bded750cc9bdee6
Showing with 11 additions and 8 deletions.
  1. +3 −3 xeHentai/core.py
  2. +1 −1 xeHentai/i18n/en_us.py
  3. +1 −1 xeHentai/i18n/zh_hans.py
  4. +1 −1 xeHentai/i18n/zh_hant.py
  5. +5 −2 xeHentai/worker.py
View
@@ -259,8 +259,8 @@ def _do_task(self, task_guid):
tid = 'scan-%d' % (i + 1)
_ = self._get_httpworker(tid, task.page_q,
filters.flt_imgurl_wrapper(task.config['download_ori'] and self.has_login),
lambda x, tid = tid: (task.img_q.put(x[0]),
task.set_reload_url(x[0], x[1], x[2]),
lambda x, tid = tid: (task.set_reload_url(x[0], x[1], x[2]),
task.img_q.put(x[0]),
mon.vote(tid, 0)),
lambda x, tid = tid: (mon.vote(tid, x[0])),
mon.wrk_keepalive,
@@ -282,7 +282,7 @@ def _do_task(self, task_guid):
_ = self._get_httpworker(tid, task.img_q,
filters.download_file_wrapper(task.config['dir']),
lambda x, tid = tid: (task.save_file(x[1], x[2], x[0]) and \
(self.logger.debug(i18n.XEH_FILE_DOWNLOADED % (task.get_fname(x[1]))),
(self.logger.debug(i18n.XEH_FILE_DOWNLOADED.format(tid, *task.get_fname(x[1]))),
mon.vote(tid, 0))),
lambda x, tid = tid: (
task.page_q.put(task.get_reload_url(x[1])),# if x[0] != ERR_QUOTA_EXCEEDED else None,
View
@@ -109,7 +109,7 @@
XEH_CLEANUP = "cleaning up..."
XEH_CRITICAL_ERROR = "xeHentai throws critical error:\n%s"
XEH_DOWNLOAD_ORI_NEED_LOGIN = "haven't login, so I won't download original images"
XEH_FILE_DOWNLOADED = "file downloaded #%d %s"
XEH_FILE_DOWNLOADED = "file downloaded by thread-{} #{} {}"
XEH_RENAME_HAS_ERRORS = "some files are not renamed:\n%s"
XEH_DOWNLOAD_HAS_ERROR = "thread-%s retry because of error: %s"
View
@@ -108,7 +108,7 @@
XEH_CLEANUP = "擦干净..."
XEH_CRITICAL_ERROR = "xeHentai 抽风啦:\n%s"
XEH_DOWNLOAD_ORI_NEED_LOGIN = "下载原图需要登录"
XEH_FILE_DOWNLOADED = "图片已下载 #%d %s"
XEH_FILE_DOWNLOADED = "绅士-{} 已下载图片 #{} {}"
XEH_RENAME_HAS_ERRORS = "部分图片重命名失败:\n%s"
XEH_DOWNLOAD_HAS_ERROR = "绅士-%s 下载图片时出错: %s, 将在稍后重试"
View
@@ -108,7 +108,7 @@
XEH_CLEANUP = "擦乾淨..."
XEH_CRITICAL_ERROR = "xeHentai 抽風啦:\n%s"
XEH_DOWNLOAD_ORI_NEED_LOGIN = "下載原圖需要登錄"
XEH_FILE_DOWNLOADED = "圖片已下載 #%d %s"
XEH_FILE_DOWNLOADED = "紳士-{} 已下載圖片 #{} {}"
XEH_RENAME_HAS_ERRORS = "部分圖片重命名失敗:\n%s"
XEH_DOWNLOAD_HAS_ERROR = "紳士-%s 下載圖片時出錯: %s, 將在稍後重試"
View
@@ -171,6 +171,8 @@ def __init__(self, tname, task_queue, flt, suc, fail, headers={}, proxy=None, pr
self.f_suc = suc
self.f_fail = fail
self.stream_mode = stream_mode
# if we don't checkin in this zombie_threshold time, monitor will regard us as zombie
self.zombie_threshold = timeout * (retry + 1)
self.run_once = False
def _finish_queue(self, *args):
@@ -334,8 +336,9 @@ def run(self):
intv += 1
self._check_vote()
for k in list(self.thread_last_seen.keys()):
if time.time() - self.thread_last_seen[k] > 30:
if k in self.thread_ref and self.thread_ref[k].is_alive():
_zombie_threshold = self.thread_ref[k].zombie_threshold if k in self.thread_ref else 30
if time.time() - self.thread_last_seen[k] > _zombie_threshold:
if k not in self.thread_ref and self.thread_ref[k].is_alive():
self.logger.warning(i18n.THREAD_MAY_BECOME_ZOMBIE % k)
self.thread_zombie.add(k)
else:

0 comments on commit 597f020

Please sign in to comment.