Permalink
Browse files

Merge branch 'dev'

  • Loading branch information...
fffonion committed Nov 10, 2017
2 parents f0c39b5 + 0527195 commit be6b56e814c5d072ad51883686ff0a9803655367
View
@@ -5,6 +5,7 @@
- 增加`download_range`选项,选择下载范围
- 增加`timeout`选项,设置下载图片的超时
- 增加`proxy_image_only`选项,设置仅使用代理下载图片
- 命令行增加`--force`选项,设置忽略配额继续下载; 配置增加`ignored_errors`选项,设置忽略的错误码
- 检查下载的图片是否完整
- 识别`socks5h`代理字符串
View
@@ -47,6 +47,7 @@ xeH
- **scan_thread_cnt** 扫描线程数,默认为1
- **download_thread_cnt** 下载线程数,默认为5
- **download_timeout** 设置下载图片的超时,默认为10
- **ignored_errors** 设置忽略的错误码, 默认为空, 错误码可以从`const.py`中获得
- **log_path** 日志路径,默认为`eh.log`
- **log_verbose** 日志等级,可选1-3,值越大输出越详细,默认为2
- **save_tasks** 是否保存任务到`h.json`,默认为否
@@ -94,7 +95,8 @@ xeH
设置下载的图片范围, 格式为 开始位置-结束位置, 或者单张图片的位置, 使用逗号来分隔多个范围, 例如
5-10,15,20-25, 默认为下载所有
-t N, --thread N 下载线程数 (默认: 5)
--timeout N 设置下载图片的超时 (默认: 10s)
--timeout N 设置下载图片的超时 (默认: 10秒)
-f, --force 忽略配额判断,继续下载 (默认: False)
-l /path/to/eh.log, --logpath /path/to/eh.log
保存日志的路径 (默认: eh.log)
-v, --verbose 设置日志装逼等级 (默认: 2)
View
@@ -106,6 +106,21 @@ def _parse_range(s):
rg.append((start, end))
return sorted(rg)
class _AddToListAction(argparse.Action):
''' This action add a value 'add_value' to the list 'dest' '''
def __init__(self, option_strings, dest, add_value=None, current=None, nargs=None, **kwargs):
super(_AddToListAction, self).__init__(option_strings, dest, default=None, nargs=0, **kwargs)
self.add_value = add_value
# to use in formatting output
self.current = current
def __call__(self, parser, namespace, values, option_string=None):
if getattr(namespace, self.dest, None) is None:
setattr(namespace, self.dest, [])
items = list(getattr(namespace, self.dest))
items.append(self.add_value)
setattr(namespace, self.dest, items)
def parse_opt():
_def = {k:v for k,v in default_config.__dict__.items() if not k.startswith("_")}
_def.update({k:v for k,v in config.__dict__.items() if not k.startswith("_")})
@@ -158,6 +173,11 @@ def parse_opt():
help = i18n.XEH_OPT_t)
parser.add_argument('--timeout', type = int, metavar = "N", default = _def['download_timeout'],
dest = 'download_timeout', help = i18n.XEH_OPT_timeout)
parser.add_argument('-f', '--force', action = _AddToListAction,
current = ERR_QUOTA_EXCEEDED in _def['ignored_errors'],
add_value = ERR_QUOTA_EXCEEDED, dest='ignored_errors',
help = i18n.XEH_OPT_f)
parser.add_argument('-l', '--logpath', metavar = '/path/to/eh.log',
default = os.path.abspath(_def['log_path']), help = i18n.XEH_OPT_l)
View
@@ -46,6 +46,12 @@
# set image download timeout
download_timeout = 10
# ignore these error codes, continue download
# to use predefined error codes, use:
# import const as __c
# ignored_errors = [__c.ERR_QUOTA_EXCEEDED]
ignored_errors = []
# define log path
log_path = "eh.log"
# set log level
View
@@ -44,6 +44,7 @@ def __init__(self):
self._all_tasks = {} # for saving states
self._all_threads = [[] for i in range(20)]
self.cfg = {k:v for k,v in default_config.__dict__.items() if not k.startswith("_")}
# note that ignored_errors are overwritten using val from custom config
self.cfg.update({k:v for k,v in config.__dict__.items() if not k.startswith("_")})
self.proxy = None
self.cookies = {"nw": "1"}
@@ -58,7 +59,10 @@ def __init__(self):
self.rpc = None
def update_config(self, cfg_dict):
self.cfg.update({k:v for k, v in cfg_dict.items() if k in cfg_dict})
self.cfg.update({k:v for k, v in cfg_dict.items() if k in cfg_dict and k not in ('ignored_errors',)})
# merge ignored errors list
if 'ignored_errors' in cfg_dict and cfg_dict['ignored_errors']:
self.cfg['ignored_errors'] = list(set(self.cfg['ignored_errors'] + cfg_dict['ignored_errors']))
self.logger.set_level(logger.Logger.WARNING - self.cfg['log_verbose'])
self.logger.verbose("cfg %s" % self.cfg)
if cfg_dict['proxy']:
@@ -97,7 +101,7 @@ def add_task(self, url, cfg_dict = {}):
url = url.strip()
cfg = {k:v for k, v in self.cfg.items() if k in (
"dir", "download_ori", "download_thread_cnt", "scan_thread_cnt",
"proxy", "proxy_image", "proxy_image_only",
"proxy", "proxy_image", "proxy_image_only", "ignored_errors",
"rename_ori", "make_archive", "jpn_title", "download_range", "download_timeout")}
cfg.update(cfg_dict)
if cfg['download_ori'] and not self.has_login:
@@ -179,7 +183,7 @@ def _do_task(self, task_guid):
if task.state >= TASK_STATE_SCAN_IMG and not monitor_started:
self.logger.verbose("state %d >= %d, bring up montior" % (task.state, TASK_STATE_SCAN_IMG))
# bring up the monitor here, ahead of workers
mon = Monitor(req, self.proxy, self.logger, task)
mon = Monitor(req, self.proxy, self.logger, task, ignored_errors=task.config['ignored_errors'])
_ = ['down-%d' % (i + 1) for i in range(task.config['download_thread_cnt'])]
# if we jumpstart from a saved session to DOQNLOAD
# there will be no scan_thread
View
@@ -112,10 +112,10 @@ def _(r, suc, fail):
elif r.status_code == 403:
fail((ERR_KEY_EXPIRED, r._real_url))
elif r.status_code == 509 or len(r.content) in [925, 28658, 144, 210, 1009] or '509.gif' in r.url:
fail((ERR_QUOTA_EXCEEDED, None))
fail((ERR_QUOTA_EXCEEDED, r._real_url))
# will not call the decorated filter
elif len(r.content) < 200 and re.findall("exceeded your image viewing limits", r.text):
fail((ERR_QUOTA_EXCEEDED, None))
fail((ERR_QUOTA_EXCEEDED, r._real_url))
# will not call the decorated filter
else:
func(r, suc, fail)
@@ -25,7 +25,8 @@ def c(cls, code):
(cls.ERR_NOMSG % code) or \
lng_fallback.err_msg[code] ) or \
lng.err_msg[code]
return _ if PY3K else _.decode('utf-8')
return _ if PY3K else (
_ if isinstance(_, unicode) else _.decode('utf-8')) # cls.ERR_NOMSG % code is unicode
def __getattr__(cls, idx):
_ = not hasattr(lng, idx) and \
View
@@ -13,6 +13,7 @@
ERR_CONNECTION_ERROR: "a connection problem occurs",
ERR_IP_BANNED: "IP has been banned, retry in %s",
ERR_IMAGE_BROKEN: "downloaded image is broken",
ERR_QUOTA_EXCEEDED: "quota exceeded",
ERR_TASK_NOT_FOUND: "no such task guid",
ERR_TASK_LEVEL_UNDEF: "task filter level unknown",
ERR_DELETE_RUNNING_TASK: "can't delete a running task",
@@ -62,6 +63,7 @@
XEH_OPT_download_range = "specify ranges of images to be downloaded, in format start-end, or single index, " \
"use comma to concat multiple ranges, e.g.: 5-10,15,20-25, default to download all images"
XEH_OPT_timeout = "set image download timeout (current: %(default)ss)"
XEH_OPT_f = "download regardless of quota exceeded warning (current: %(current)s)"
XEH_OPT_h = "show this help message and exit"
XEH_OPT_version = "show program's version number and exit"
XEH_OPT_IGNORING_I = "ignoring -i option in daemon mode"
View
@@ -13,6 +13,7 @@
ERR_CONNECTION_ERROR: "连接有问题?",
ERR_IP_BANNED: "IP被ban了, 恢复时间: %s",
ERR_IMAGE_BROKEN: "下载的图片有猫饼",
ERR_QUOTA_EXCEEDED: "配额超限",
ERR_TASK_NOT_FOUND: "没有该GUID对应的任务",
ERR_TASK_LEVEL_UNDEF: "任务过滤等级不存在",
ERR_DELETE_RUNNING_TASK: "无法删除运行中的任务",
@@ -59,7 +60,8 @@
XEH_OPT_j = "使用日语标题, 如果关闭则使用英文或罗马字标题 (当前: %(default)s)"
XEH_OPT_download_range = "设置下载的图片范围, 格式为 开始位置-结束位置, 或者单张图片的位置, " \
"使用逗号来分隔多个范围, 例如 5-10,15,20-25, 默认为下载所有"
XEH_OPT_timeout = "设置下载图片的超时 (当前: %(default)ss)"
XEH_OPT_timeout = "设置下载图片的超时 (当前: %(default)s秒)"
XEH_OPT_f = "忽略配额判断,继续下载 (当前: %(current)s)"
XEH_OPT_h = "显示本帮助信息"
XEH_OPT_version = "显示版本信息"
XEH_OPT_IGNORING_I = "后台模式已忽略 -i 参数"
View
@@ -13,6 +13,7 @@
ERR_CONNECTION_ERROR: "連接有問題?",
ERR_IP_BANNED: "IP被ban了, 恢復時間: %s",
ERR_IMAGE_BROKEN: "下載的圖片有貓餅",
ERR_QUOTA_EXCEEDED: "配額超限",
ERR_TASK_NOT_FOUND: "沒有該GUID對應的任務",
ERR_TASK_LEVEL_UNDEF: "任務過濾等級不存在",
ERR_DELETE_RUNNING_TASK: "無法刪除運行中的任務",
@@ -59,7 +60,8 @@
XEH_OPT_j = "使用日語標題, 如果關閉則使用英文或羅馬字標題 (當前: %(default)s)"
XEH_OPT_download_range = "設置下載的圖片範圍, 格式為 開始位置-結束位置, 或者單張圖片的位置, " \
"使用逗號來分隔多個範圍, 例如 5-10,15,20-25, 默認為下載所有"
XEH_OPT_timeout = "設置下載圖片的超時 (當前: %(default)ss)"
XEH_OPT_timeout = "設置下載圖片的超時 (當前: %(default)s秒)"
XEH_OPT_f = "忽略配額判斷,繼續下載 (當前: %(current)s)"
XEH_OPT_h = "顯示本幫助信息"
XEH_OPT_version = "顯示版本信息"
XEH_OPT_IGNORING_I = "後台模式已忽略 -i 參數"
View
@@ -154,11 +154,13 @@ def run(self):
class Monitor(Thread):
def __init__(self, req, proxy, logger, task, exit_check = None):
def __init__(self, req, proxy, logger, task, exit_check=None, ignored_errors=[]):
Thread.__init__(self, name = "monitor%s" % task.guid)
Thread.setDaemon(self, True)
# the count of votes per error code
self.vote_result = {}
self.vote_cleared = set()
# the error code to be ignored
self.vote_cleared = set().union(ignored_errors)
self.thread_last_seen = {}
self.dctlock = RLock()
self.votelock = RLock()
@@ -187,7 +189,7 @@ def set_vote_ns(self, tnames):
def vote(self, tname, code):
# thread_id, result_code
self.votelock.acquire()
if code != 0:
if code != ERR_NO_ERROR:
self.logger.verbose("t-%s vote:%s" % (tname, code))
if code not in self.vote_result:
self.vote_result[code] = 1
@@ -241,6 +243,7 @@ def _check_vote(self):
self.task.meta['has_ori'] = True
self.vote_cleared.add(ERR_IMAGE_RESAMPLED)
elif ERR_QUOTA_EXCEEDED in self.vote_result and \
ERR_QUOTA_EXCEEDED not in self.vote_cleared and \
self.vote_result[ERR_QUOTA_EXCEEDED] >= len(self.thread_last_seen):
self.logger.error(i18n.TASK_STOP_QUOTA_EXCEEDED % self.task.guid)
self.task.state = TASK_STATE_FAILED

0 comments on commit be6b56e

Please sign in to comment.