Skip to content
Permalink
Browse files

Merge branch 'dev'

  • Loading branch information
fffonion committed Dec 3, 2019
2 parents beb71fb + dc4cf86 commit c87edd892880d573cc266427431653f43bde20b8
Showing with 106 additions and 27 deletions.
  1. +41 −11 .travis.yml
  2. +4 −4 xeHentai/const.py
  3. +14 −6 xeHentai/core.py
  4. +29 −2 xeHentai/filters.py
  5. +1 −1 xeHentai/rpc.py
  6. +11 −3 xeHentai/task.py
  7. +6 −0 xeHentai/util/logger.py
@@ -1,27 +1,57 @@
os:
- linux

language: python
sudo: required
dist: xenial
python:
- 2.7
- 3.4
- 3.5
- 3.6
- 3.7

matrix:
include:
- python: 3.3
dist: trusty
- os: linux
language: python
python: "2.7"
- os: linux
language: python
python: "3.5"
- os: linux
language: python
python: "3.6"
- os: linux
language: python
python: "3.7"
# https://chocolatey.org/packages/python2
# - os: windows
# language: sh
# python: "2.7"
# before_install:
# - choco install python2
# - export PATH="/c/Python26:/c/Python27/Scripts:$PATH"
# - wget https://bootstrap.pypa.io/get-pip.py
# - python ./get-pip.py
# https://chocolatey.org/packages/python/3.7.4
- os: windows
language: sh
python: "3.5"
before_install:
- choco install python --version 3.5.4
- export PATH="/c/Python35:/c/Python35/Scripts:$PATH"
- os: windows
language: sh
python: "3.6"
before_install:
- choco install python --version 3.6.8
- export PATH="/c/Python36:/c/Python36/Scripts:$PATH"
- os: windows
language: sh
python: "3.7"
before_install:
- choco install python --version 3.7.4
- export PATH="/c/Python37:/c/Python37/Scripts:$PATH"

install:
- python setup.py install

script:
- xeH --help
- xeH $TEST_URL_E --dir test1
# nested env currently have no effect on windows
- LANG=zh_CN.utf-8 LC_ALL=zh_CN.utf-8 xeH $TEST_URL_E --dir test1
- LANG=zh_TW.utf-8 LC_ALL=zh_TW.utf-8 xeH $TEST_URL_E --dir test1
- xeH $TEST_URL_E --dir test1 --archive true
@@ -9,14 +9,14 @@
import sys
import locale

PY3K = sys.version_info[0] == 3
PY3K = sys.version_info.major == 3
IRONPYTHON = sys.platform == 'cli'
EXEBUNDLE = getattr(sys, 'frozen', False)
LOCALE = locale.getdefaultlocale()[0]
CODEPAGE = locale.getdefaultlocale()[1] or 'ascii'
ANDROID = 'ANDROID_ARGUMENT' in os.environ

__version__ = 2.020
__version__ = 2.021
DEVELOPMENT = False

SCRIPT_NAME = "xeHentai"
@@ -48,11 +48,11 @@

RESTR_SITE = "https*://(?:[g\.]*e\-|ex)hentai\.org"

FALLBACK_CF_IP = ("104.24.255.11", "104.24.254.11")
FALLBACK_CF_IP = ("104.20.26.25", "104.20.27.25")
FALLBACK_IP_MAP = {
'e-hentai.org': FALLBACK_CF_IP,
'forums.e-hentai.org': ("94.100.18.243", ),
'exhentai.org': ("217.23.13.91","217.23.13.45","109.236.84.136","109.236.92.143","109.236.84.145","109.236.92.166")
'exhentai.org': ("178.175.129.254", "178.175.128.252", "178.175.132.20", "178.175.129.252", "178.175.128.254", "178.175.132.22")
}

DEFAULT_MAX_REDIRECTS = 30
@@ -238,15 +238,23 @@ def _do_task(self, task_guid):
task.scan_downloaded()
if task.state == TASK_STATE_FINISHED:
continue
for x in range(0,
int(math.ceil(1.0 * task.meta['total'] / int(task.meta['thumbnail_cnt'])))):
if not task.meta['use_multipage_viewer']:
for x in range(0,
int(math.ceil(1.0 * task.meta['total'] / int(task.meta['thumbnail_cnt'])))):
r = req.request("GET",
"%s/?p=%d" % (task.url, x),
filters.flt_pageurl,
lambda x: task.queue_wrapper(task.page_q.put, url = x),
lambda x: task.set_fail(x))
if task.failcode:
break
elif task.meta['finished'] < task.meta['total']:
# use multipage viewer
r = req.request("GET",
"%s/?p=%d" % (task.url, x),
filters.flt_pageurl,
task.mpv_url(),
filters.flt_pageurl_mpv,
lambda x: task.queue_wrapper(task.page_q.put, url = x),
lambda x: task.set_fail(x))
if task.failcode:
break
elif task.state == TASK_STATE_SCAN_IMG:
# print here so that see it after we can join former threads
self.logger.info(i18n.TASK_TITLE % (
@@ -5,6 +5,7 @@

import os
import re
import json
from . import util
from .const import *

@@ -42,7 +43,6 @@ def flt_metadata(r, suc, fail):
fail(ERR_IP_BANNED)
return re.findall("The ban expires in (.+)", r.text)[0]
meta = {}
# print(r.text)
# sample_hash = re.findall('<a href="%s/./([a-f0-9]{10})/\d+\-\d+"><img' % RESTR_SITE, r.text)
# meta['sample_hash'] = sample_hash
# meta['resampled'] = {}
@@ -58,6 +58,15 @@ def flt_metadata(r, suc, fail):
_ = re.findall("Showing (\d+) \- (\d+) of ([\d,]+) images", r.text)[0]
meta['thumbnail_cnt'] = int(_[1]) - int(_[0]) + 1

# check multi page viewer status in order to call proper flt_pageurl
mpv_urls = re.findall(
'<a href="(%s/mpv/(\d+)/[a-f0-9]{10})/#page\d+"><img alt="\d+" title="Page' % RESTR_SITE,
r.text)
if mpv_urls:
meta['use_multipage_viewer'] = True
else:
meta['use_multipage_viewer'] = False

suc(meta)
# _ = re.findall(
# '%s/[^/]+/(\d+)/[^/]+/\?p=\d*" onclick="return false"(.*?)</a>' % RESTR_SITE,
@@ -105,13 +114,31 @@ def flt_pageurl(r, suc, fail):
for p in picpage:
suc(p)

def flt_pageurl_mpv(r, suc, fail):
# input mpv gallaery response
# add per image urls if suc; finish task if fail
# construct single page gallery view
imagelist = re.findall("imagelist\s=\s([^;]+)", r.text)
if not imagelist:
fail(ERR_NO_PAGEURL_FOUND)
try:
imagelist = json.loads(imagelist[0])
except:
fail(ERR_NO_PAGEURL_FOUND)
baseurl = re.findall("https?://[^/]+", r._real_url)[0]
gid, _ = RE_INDEX.findall(r._real_url)[0]
for i in range(len(imagelist)):
suc("%s/s/%s/%s-%d" % (baseurl, imagelist[i]['k'], gid, i+1))


def flt_quota_check(func):
def _(r, suc, fail):
if r.status_code == 600:# tcp layer error
fail((ERR_CONNECTION_ERROR, r._real_url))
elif r.status_code == 403:
fail((ERR_KEY_EXPIRED, r._real_url))
elif r.status_code == 509 or r.content_length in [925, 28658, 144, 210, 1009] or '509.gif' in r.url:
elif r.status_code == 509 or r.content_length in [925, 28658, 144, 210, 1009] or \
'509.gif' in r.url or '509.gif' in r._real_url:
fail((ERR_QUOTA_EXCEEDED, r._real_url))
# will not call the decorated filter
elif r.content_length < 200 and \
@@ -71,7 +71,7 @@ def gen_thumbnail(fh, args):
# returns a new file handler if resized
# and a boolean indicates there'e error
try:
from PIL import Image
import PIL.Image as Image
except:
return fh, True
if 'w' not in args and 'h' not in args:
@@ -15,6 +15,7 @@
from . import util
from .const import *
from .const import __version__
from .util.logger import safestr
if PY3K:
from queue import Queue, Empty
else:
@@ -81,6 +82,13 @@ def migrate_exhentai(self):
self.failcode = 0
return True

def mpv_url(self):
return re.sub(
"/./%s/%s" % (self.gid, self.sethash),
"/mpv/%s/%s" % (self.gid, self.sethash),
self.url
)

def update_meta(self, meta):
self.meta.update(meta)
if self.config['jpn_title'] and self.meta['gjname']:
@@ -235,7 +243,7 @@ def save_file(self, imgurl, redirect_url, binary_iter):
self._f_lock.acquire()
try:
try:
os.rename(fn_tmp, fn)
shutil.move(fn_tmp, fn)
except WindowsError as ex:
# file is used by another process
# do a copy and delete, WindowsError[32]
@@ -323,8 +331,8 @@ def rename_fname(self):
os.rename(fname_to, os.path.join(tmppath, os.path.split(fname_to)[1]))
break
if _ :# if ...(1) exists, use ...(2)
print(_base)
_base = re.sub("\((\d+)\)$", lambda x:"(%d)" % (int(x.group(1)) + 1), _base)
print(safestr(_base))
_base = re.sub("\((\d+)\)$", _base, lambda x:"(%d)" % (int(x.group(1)) + 1))
else:
_base = "%s(1)" % _base
fname_to = "".join((_base, _ext))
@@ -18,10 +18,16 @@ def utcoffset(self, dt):
def dst(self, dt):
return datetime.timedelta(0)


def safestr(s):
if (PY3K and isinstance(s, bytes)) or (not PY3K and not isinstance(s, unicode)):
s = s.decode("utf-8")
if PY3K:
# python<=3.5 hack
if sys.version_info.minor <= 5:
return s \
.encode(locale.getdefaultlocale()[1] or 'utf-8', 'replace') \
.decode(locale.getdefaultlocale()[1] or 'utf-8', 'replace')
return s
return s.encode(locale.getdefaultlocale()[1] or 'utf-8', 'replace')
#return _.decode('utf-8') if PY3K else _

0 comments on commit c87edd8

Please sign in to comment.
You can’t perform that action at this time.