diff --git a/README.md b/README.md index fddf47d..fa46899 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # google-tr-free -Google translate for free -- local cache plus throttling. Let's hope it lasts. +Google translate for free -- local cache and throttling (averag 1.5 calls/s, first 1000 calls exempted).. Let's hope it lasts. ### Installation diff --git a/google_tr/__init__.py b/google_tr/__init__.py index e0b82b5..c5a7e01 100644 --- a/google_tr/__init__.py +++ b/google_tr/__init__.py @@ -1,5 +1,8 @@ from .google_tr import google_tr -__version__ = '0.0.1' -__date__ = '2019.7.18' +# __version__ = '0.0.2' +# __date__ = '2019.7.23' + +__version__ = '0.0.4' +__date__ = '2019.7.28' VERSION = __version__ \ No newline at end of file diff --git a/google_tr/__pycache__/__init__.cpython-36.pyc b/google_tr/__pycache__/__init__.cpython-36.pyc index e1adc05..2db9d63 100644 Binary files a/google_tr/__pycache__/__init__.cpython-36.pyc and b/google_tr/__pycache__/__init__.cpython-36.pyc differ diff --git a/google_tr/__pycache__/google_tr.cpython-36.pyc b/google_tr/__pycache__/google_tr.cpython-36.pyc index 47f7713..479f391 100644 Binary files a/google_tr/__pycache__/google_tr.cpython-36.pyc and b/google_tr/__pycache__/google_tr.cpython-36.pyc differ diff --git a/google_tr/google_tr.py b/google_tr/google_tr.py index 9cb6bf4..94d775a 100644 --- a/google_tr/google_tr.py +++ b/google_tr/google_tr.py @@ -4,14 +4,22 @@ ''' # pylint: disable=line-too-long, broad-except +import logging from pathlib import Path +from time import sleep +from random import random -import requests import js2py import requests_cache -CACHE_NAME = (Path().home() / Path(__file__).stem).as_posix() +LOGGER = logging.getLogger(__name__) +LOGGER.addHandler(logging.NullHandler()) + +HOME_FOLDER = Path.home() +__FILE__ = globals().get('__file__') or 'test' +CACHE_NAME = (Path(HOME_FOLDER) / (Path(__FILE__)).stem).as_posix() +EXPIRE_AFTER = 3600 requests_cache.configure(cache_name=CACHE_NAME, expire_after=36000) # 10 hrs @@ -60,8 +68,54 @@ HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0'} -SESS = requests.Session() -SESS.get(URL) + +def make_throttle_hook(timeout=0.67, exempt=1000): + """ + Returns a response hook function which sleeps for `timeout` seconds if + response is not cached + + the first exempt calls exempted from throttling + """ + + try: + timeout = float(timeout) + except Exception as _: + timeout = .67 + + try: + exempt = int(exempt) + except Exception as _: + exempt = 100 + + def hook(response, *args, **kwargs): # pylint: disable=unused-argument + if not getattr(response, 'from_cache', False): + timeout_ = timeout + random() - 0.5 + timeout_ = max(0, timeout_) + + try: + hook.flag + except AttributeError: + hook.flag = -1 + finally: + hook.flag += 1 + quo, _ = divmod(hook.flag, exempt) + # quo is 0 only for the first exempt calls + + LOGGER.debug('avg delay: %s, sleeping %s s, flag: %s', timeout, timeout_, bool(quo)) + + # will not sleep (timeout_ * bool(quo)=0) for the first exempt calls + sleep(timeout_ * bool(quo)) + + return response + return hook + +SESS = requests_cache.CachedSession( + cache_name=CACHE_NAME, + expire_after=EXPIRE_AFTER, + allowable_methods=('GET', 'POST'), +) + +SESS.hooks = {'response': make_throttle_hook()} def google_tr(content, from_lang='auto', to_lang='zh-CN', cache=True): diff --git a/setup.py b/setup.py index c089dee..b0d45dc 100644 --- a/setup.py +++ b/setup.py @@ -9,10 +9,10 @@ name = """google-tr-free""" # description = ' '.join(name.split('-')) -description = name.replace('-tr-', 'translate for') +description = name.replace('-tr-', ' translate for ') dir_name, = find_packages() -version, = re.findall(r"__version__\W*=\W*'([^']+)'", open(Path(__file__).parent / f'{dir_name}/__init__.py').read()) +version, = re.findall(r"\n__version__\W*=\W*'([^']+)'", open(Path(__file__).parent / f'{dir_name}/__init__.py').read()) README_rst = f'{Path(__file__).parent}/README.md' long_description = open(README_rst, encoding='utf-8').read() if Path(README_rst).exists() else '' @@ -23,7 +23,7 @@ version=version, description=description, long_description=long_description, - long_description_content_type='text/markdown', + long_description_content_type='text/markdown', keywords=['machine translation', 'free', 'scraping', ], author="mikeee", url=f'http://github.com/ffreemt/{name}',