Permalink
Browse files

Revert "V0.30 dev"

  • Loading branch information...
aploium committed Oct 8, 2016
1 parent f378016 commit 965b6e0ab6323960d0ce589e5b3273686ead2044
@@ -16,4 +16,3 @@ exclude_paths:
- "external_pkgs/"
- "more_configs/"
- "zmirror/external_pkgs/"
- "zmirror/tests/"
@@ -4,7 +4,6 @@ omit =
tests/*
more_configs/*
zmirror/external_pkgs/*
zmirror/tests/*
test.py
wsgi.py
config.py
@@ -329,7 +329,7 @@ config_zjumail.py
custom_func_.py
custom_func_t.py
img/
/zmirror/tests/sample/
/tests/sample/
/config.py._unittest_raw
/custom_func.py._unittest_raw
/error_dump/
@@ -1,5 +1,5 @@
# zmirror
[![zmirror version](https://img.shields.io/badge/version-0.30.0-blue.svg)](https://github.com/aploium/zmirror)
[![zmirror version](https://img.shields.io/badge/version-0.29.4-blue.svg)](https://github.com/aploium/zmirror)
[![zmirror Build Status](https://travis-ci.org/aploium/zmirror.svg?branch=master)](https://travis-ci.org/aploium/zmirror)
[![zmirror unittest coverage](https://codecov.io/gh/aploium/zmirror/branch/master/graph/badge.svg)](https://codecov.io/gh/aploium/zmirror)
[![zmirror Dependency Status](https://www.versioneye.com/user/projects/57addd5358ae9200345e108c/badge.svg?style=flat-square)](https://www.versioneye.com/user/projects/57addd5358ae9200345e108c)
@@ -12,7 +12,6 @@ coverage:
range: "0...100"
ignore:
- zmirror/external_pkgs/.*
- zmirror/tests/.*
- external_pkgs/.*
- more_configs/.*
- tests/.*
@@ -146,7 +146,7 @@ def demo__custom_identity_verify(identity_dict):
"""
import hashlib
import requests
from zmirror import cfg
import config

if 'cc98_username' not in identity_dict or 'cc98_password' not in identity_dict:
return False
@@ -155,8 +155,8 @@ def demo__custom_identity_verify(identity_dict):
pass_md5 = hashlib.md5()
pass_md5.update(identity_dict['cc98_password'].encode())
pass_md5 = pass_md5.hexdigest()
if cfg.is_use_proxy:
proxy = cfg.requests_proxies
if config.is_use_proxy:
proxy = config.requests_proxies
else:
proxy = None
r = requests.post('http://www.cc98.org/sign.asp', data={
@@ -175,14 +175,14 @@ def demo__custom_identity_verify(identity_dict):

# Demo for Twitter
def demo__handle_expand_url(mobj):
import config
from zmirror.zmirror import add_ssrf_allowed_domain, get_group
from zmirror import cfg

domain = get_group('domain', mobj)
if not domain:
return mobj.group()
add_ssrf_allowed_domain(domain)
if 'https' in get_group('scheme', mobj) or cfg.force_https_domains == 'ALL':
if 'https' in get_group('scheme', mobj) or config.force_https_domains == 'ALL':
scheme_prefix = 'https-'
else:
scheme_prefix = ''
@@ -7,8 +7,8 @@
"""
import re
from zmirror.zmirror import add_ssrf_allowed_domain, get_group, \
encode_mirror_url, decode_mirror_url
from zmirror import cfg
force_https_domains, my_host_scheme, my_host_name, encode_mirror_url, \
decode_mirror_url

regex_twitter_data_expanded = re.compile(
r'''data-expanded-url\s*=\s*'''
@@ -21,7 +21,7 @@ def handle_expand_url(mobj):
if not domain:
return mobj.group()
add_ssrf_allowed_domain(domain)
if 'https' in get_group('scheme', mobj) or cfg.force_https_domains == 'ALL':
if 'https' in get_group('scheme', mobj) or force_https_domains == 'ALL':
scheme_prefix = 'https-'
else:
scheme_prefix = ''
@@ -34,7 +34,7 @@ def custom_response_text_rewriter(raw_text, content_mime, remote_url):
regex_twitter_data_expanded.sub(handle_expand_url, raw_text)

# For twitter t.co redirect
raw_text = raw_text.replace('https://t.co/', cfg.my_host_scheme + cfg.my_host_name + '/extdomains/https-t.co/')
raw_text = raw_text.replace('https://t.co/', my_host_scheme + my_host_name + '/extdomains/https-t.co/')

# For twitter video
if decode_mirror_url()["domain"] == 'video.twimg.com':
@@ -10,7 +10,6 @@
import traceback
from urllib.parse import urlsplit
from zmirror.zmirror import *
from zmirror import cfg

# 如果你想视频服务器和网页服务器分开, 通过多个视频服务器来进行负载均衡,
# 请设置在主网页服务器上设置 is_master = True ,在视频服务器上部署后设置 is_master = False
@@ -21,7 +20,7 @@
is_master = False

# 注: 如果使用了非标准端口, 请在下面所有需要填写域名的地方后面加上端口号, 如 'videocdn1.mycdn.com:20822'
videocdn_this_site_name = cfg.my_host_name
videocdn_this_site_name = my_host_name
# 如果你想视频服务器和网页服务器分开, 请注释掉上面一行, 去掉下面一行的注释, 并把域名改成这一台视频服务器的域名
# videocdn_this_site_name = 'videocdn1.mycdn.com' # 使用标准端口
# videocdn_this_site_name = 'videocdn1.mycdn.com:20822' # 使用非标准端口
@@ -40,7 +39,7 @@

# get videocdn domain's root domain
try:
if videocdn_this_site_name != cfg.my_host_name:
if videocdn_this_site_name != my_host_name:
temp0 = videocdn_domain_list[0]
else:
temp0 = videocdn_this_site_name
@@ -65,30 +64,30 @@
REGEX_OF_URL = r'(https?|ftp):\/\/[^\s/$.?#].[^\s]*'

regex_youtube_video_videoplayback_resolve = re.compile(
('https(%|%25)3A(%|%25)2F(%|%25)2F' if cfg.my_host_scheme == 'http://' else '') +
('https(%|%25)3A(%|%25)2F(%|%25)2F' if my_host_scheme == 'http://' else '') +
r'''(?P<prefix>r\d+---sn-[a-z0-9]{8})\.googlevideo\.com(?P<percent>%|%25)2Fvideoplayback(%|%25)3F''',
flags=re.IGNORECASE)

regex_youtube_video_url_resolve = re.compile(
(r'https:(?P<escape_slash>\\?)/\\?/' if cfg.my_host_scheme == 'http://' else '') +
(r'https:(?P<escape_slash>\\?)/\\?/' if my_host_scheme == 'http://' else '') +
r'''(?P<prefix>r\d+---sn-[a-z0-9]{8})\.googlevideo\.com''')

regex_youtube_video_c_videoplayback_resolve = re.compile(
('https://' if cfg.my_host_scheme == 'http://' else '') +
('https://' if my_host_scheme == 'http://' else '') +
r'''(?P<prefix>r\d+---sn-[a-z0-9]{8})\.c\.youtube\.com/videoplayback\?''',
flags=re.IGNORECASE)


def custom_response_text_rewriter(raw_text, content_mime, remote_url):
# if 'html' in content_mime or 'x-www-form-urlencoded' in content_mime:
raw_text = regex_youtube_video_videoplayback_resolve.sub(
('http\g<percent>3A\g<percent>2F\g<percent>2F' if cfg.my_host_scheme == 'http://' else '') +
('http\g<percent>3A\g<percent>2F\g<percent>2F' if my_host_scheme == 'http://' else '') +
video_cdn_domain + '\g<percent>2Fvideoplayback\g<percent>3Fewmytbserver\g<percent>3D\g<prefix>\g<percent>26', raw_text)
raw_text = regex_youtube_video_url_resolve.sub(
('http:\g<escape_slash>/\g<escape_slash>/' if cfg.my_host_scheme == 'http://' else '') + video_cdn_domain, raw_text)
('http:\g<escape_slash>/\g<escape_slash>/' if my_host_scheme == 'http://' else '') + video_cdn_domain, raw_text)

raw_text = regex_youtube_video_c_videoplayback_resolve.sub(
('http://' if cfg.my_host_scheme == 'http://' else '') +
('http://' if my_host_scheme == 'http://' else '') +
video_cdn_domain + '/videoplayback?ewmytbserver=\g<prefix>&', raw_text)

if 'javascript' in content_mime:
@@ -97,7 +96,7 @@ def custom_response_text_rewriter(raw_text, content_mime, remote_url):
+ videocdn_video_root_domain.replace('.', r'\\.')
+ '$')

_buff = re.escape(videocdn_video_root_domain) + '|' + re.escape(cfg.my_host_name_root)
_buff = re.escape(videocdn_video_root_domain) + '|' + re.escape(my_host_name_root)
raw_text = raw_text.replace(r'-nocookie)?\.com\/|(m\.)?[a-z0-9\-]',
r'-nocookie)?\.com\/|' + _buff + r'|(m\.)?[a-z0-9\-]') # xp

@@ -1,9 +1,8 @@
# coding=utf-8
import os
import unittest
from zmirror.tests import *

os.environ['ZMIRROR_UNITTEST'] = "True"
from tests import *

if __name__ == '__main__':
os.environ['ZMIRROR_UNITTEST'] = "True"
unittest.main()
File renamed without changes.
@@ -56,18 +56,6 @@ def tearDownClass(cls):
def reload_zmirror(self, configs_dict=None):
self.del_temp_var()

for m in [x for x in sys.modules if x.startswith("zmirror")]:
# print("ref_counter {} {}".format(m, sys.getrefcount(sys.modules[m])))
del sys.modules[m]

# importlib.reload(importlib.import_module("config_default"))
# importlib.reload(importlib.import_module("config"))
# importlib.reload(importlib.import_module("zmirror.cfg"))
# importlib.reload(importlib.import_module("zmirror.utils_complex"))
# importlib.reload(importlib.import_module("zmirror.cache_system"))
# importlib.reload(importlib.import_module("zmirror.zmirror"))
# importlib.reload(importlib.import_module("zmirror"))
# importlib.reload(importlib.import_module("zmirror.cfg"))
import config
importlib.reload(config)

@@ -80,10 +68,6 @@ def reload_zmirror(self, configs_dict=None):
for config_name, config_value in configs_dict.items():
setattr(config, config_name, config_value)

import zmirror.cfg as cfg

# print(self.C.target_domain, cfg.target_domain, cfg.allowed_domains_set)

import zmirror.cache_system as cache_system
import zmirror.zmirror as zmirror
importlib.reload(cache_system)
@@ -53,7 +53,7 @@ def test_img_cdn(self):
) # type: Response

self.assertEqual( # 此时应出现重定向
self.zmirror.cfg.cdn_redirect_code_if_cannot_hard_rewrite
self.zmirror.cdn_redirect_code_if_cannot_hard_rewrite
, self.rv2.status_code, msg=self.dump()
)
self.assertEqual(
@@ -82,7 +82,7 @@ def test_img_cdn(self):
) # type: Response

self.assertEqual( # 此时应出现重定向
self.zmirror.cfg.cdn_redirect_code_if_cannot_hard_rewrite
self.zmirror.cdn_redirect_code_if_cannot_hard_rewrite
, self.rv2.status_code, msg=self.dump())
self.assertEqual(
# 查询参数会被编码进url
@@ -119,7 +119,7 @@ def test_long_query_str_encode_compress(self):
) # type: Response

self.assertEqual( # 此时应出现重定向
self.zmirror.cfg.cdn_redirect_code_if_cannot_hard_rewrite
self.zmirror.cdn_redirect_code_if_cannot_hard_rewrite
, self.rv2.status_code, msg=self.dump())
self.assertIn(
"/image/jpeg_{salt}z_.".format(salt=self.zmirror.cdn_url_query_encode_salt),
@@ -130,7 +130,7 @@ def test_long_query_str_encode_compress(self):
self.rv2.location,
environ_base=env(),
headers=headers(
user_agent=DEFAULT_USER_AGENT + " " + self.zmirror.cfg.spider_ua_white_list[0]
user_agent=DEFAULT_USER_AGENT + " " + self.zmirror.spider_ua_white_list[0]
)
) # type: Response
self.assertEqual("image/jpeg", self.rv3.content_type, msg=self.dump())
@@ -159,7 +159,7 @@ def test_cdn_excluded_ua(self):
) # type: Response

self.assertEqual( # 此时应出现重定向
self.zmirror.cfg.cdn_redirect_code_if_cannot_hard_rewrite
self.zmirror.cdn_redirect_code_if_cannot_hard_rewrite
, self.rv2.status_code, msg=self.dump())
self.assertEqual(
# 查询参数会被编码进url
@@ -174,7 +174,7 @@ def test_cdn_excluded_ua(self):
self.url("/image/jpeg?love=luciaz"),
environ_base=env(),
headers=headers(
user_agent=DEFAULT_USER_AGENT + " " + self.zmirror.cfg.spider_ua_white_list[0]
user_agent=DEFAULT_USER_AGENT + " " + self.zmirror.spider_ua_white_list[0]
)
) # type: Response

@@ -189,7 +189,7 @@ def test_cdn_excluded_ua(self):
),
environ_base=env(),
headers=headers(
user_agent=DEFAULT_USER_AGENT + " " + self.zmirror.cfg.spider_ua_white_list[0]
user_agent=DEFAULT_USER_AGENT + " " + self.zmirror.spider_ua_white_list[0]
)
) # type: Response

@@ -43,8 +43,8 @@ def test_keep_alive(self):
) # type: Response
time_non_alive = self.zmirror.parse.remote_response.elapsed

max_fail_count = 5
for _ in range(15):
max_fail_count = 3
for _ in range(10):
self.rv2 = self.client.head(
self.url("/"),
environ_base=env(),
File renamed without changes.
@@ -171,12 +171,12 @@ def test__is_target_domain_use_https(self):

def test__add_ssrf_allowed_domain(self):
self.zmirror.add_ssrf_allowed_domain("www.example.com")
self.assertIn("www.example.com", self.zmirror.cfg.allowed_domains_set)
self.assertIn("www.example.com", self.zmirror.allowed_domains_set)

def test__check_global_ua_pass(self):
self.assertFalse(self.zmirror.check_global_ua_pass(None))
self.assertTrue(self.zmirror.check_global_ua_pass(
self.zmirror.cfg.global_ua_white_name)
self.zmirror.global_ua_white_name)
)

def test__is_content_type_using_cdn(self):
@@ -235,27 +235,22 @@ def test__embed_real_url_to_embedded_url(self):
)

def test__encoding_detect(self):
self.assertEqual(
"UTF-8",
self.zmirror.encoding_detect("测试中文".encode(encoding="utf-8"))
)

self.zmirror.cfg.force_decode_remote_using_encode = "utf-8"
self.zmirror.force_decode_remote_using_encode = "utf-8"
self.assertEqual(
"utf-8",
self.zmirror.encoding_detect("测试中文".encode(encoding="gbk"))
)

self.zmirror.cfg.force_decode_remote_using_encode = None
self.zmirror.cfg.possible_charsets = ["gbk", "utf-8"]
self.zmirror.force_decode_remote_using_encode = None
self.zmirror.possible_charsets = ["gbk", "utf-8"]
self.assertEqual(
"utf-8",
self.zmirror.encoding_detect("测试中文".encode(encoding="utf-8"))
)
self.assertEqual(
"gbk",
self.zmirror.encoding_detect("测试中文".encode(encoding="gbk"))
)

self.zmirror.possible_charsets = None
self.zmirror.cchardet_available = False
self.assertIsNone(self.zmirror.encoding_detect("测试中文".encode(encoding="utf-8")))

def test__get_group(self):
import re
File renamed without changes.
Oops, something went wrong.

0 comments on commit 965b6e0

Please sign in to comment.