minor update

README.md

@@ -1,6 +1,6 @@
 # google-tr-free
 
-Google translate for free -- local cache plus throttling. Let's hope it lasts.
+Google translate for free -- local cache and throttling (averag 1.5 calls/s, first 1000 calls exempted).. Let's hope it lasts.
 
 ### Installation
 

google_tr/__init__.py

@@ -1,5 +1,8 @@
 from .google_tr import google_tr
 
-__version__ = '0.0.1'
-__date__ = '2019.7.18'
+# __version__ = '0.0.2'
+# __date__ = '2019.7.23'
+
+__version__ = '0.0.4'
+__date__ = '2019.7.28'
 VERSION = __version__

google_tr/google_tr.py

@@ -4,14 +4,22 @@
 '''
 # pylint: disable=line-too-long, broad-except
 
+import logging
 from pathlib import Path
+from time import sleep
+from random import random
 
-import requests
 import js2py
 
 import requests_cache
 
-CACHE_NAME = (Path().home() / Path(__file__).stem).as_posix()
+LOGGER = logging.getLogger(__name__)
+LOGGER.addHandler(logging.NullHandler())
+
+HOME_FOLDER = Path.home()
+__FILE__ = globals().get('__file__') or 'test'
+CACHE_NAME = (Path(HOME_FOLDER) / (Path(__FILE__)).stem).as_posix()
+EXPIRE_AFTER = 3600
 
 requests_cache.configure(cache_name=CACHE_NAME, expire_after=36000)  # 10 hrs
 
@@ -60,8 +68,54 @@
 
 HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0'}
 
-SESS = requests.Session()
-SESS.get(URL)
+
+def make_throttle_hook(timeout=0.67, exempt=1000):
+    """
+    Returns a response hook function which sleeps for `timeout` seconds if
+    response is not cached
+
+    the first exempt calls exempted from throttling
+    """
+
+    try:
+        timeout = float(timeout)
+    except Exception as _:
+        timeout = .67
+
+    try:
+        exempt = int(exempt)
+    except Exception as _:
+        exempt = 100
+
+    def hook(response, *args, **kwargs):  # pylint: disable=unused-argument
+        if not getattr(response, 'from_cache', False):
+            timeout_ = timeout + random() - 0.5
+            timeout_ = max(0, timeout_)
+
+            try:
+                hook.flag
+            except AttributeError:
+                hook.flag = -1
+            finally:
+                hook.flag += 1
+                quo, _ = divmod(hook.flag, exempt)
+            # quo is 0 only for the first exempt calls
+
+            LOGGER.debug('avg delay: %s, sleeping %s s, flag: %s', timeout, timeout_, bool(quo))
+
+            # will not sleep (timeout_ * bool(quo)=0) for the first exempt calls
+            sleep(timeout_ * bool(quo))
+
+        return response
+    return hook
+
+SESS = requests_cache.CachedSession(
+    cache_name=CACHE_NAME,
+    expire_after=EXPIRE_AFTER,
+    allowable_methods=('GET', 'POST'),
+)
+
+SESS.hooks = {'response': make_throttle_hook()}
 
 
 def google_tr(content, from_lang='auto', to_lang='zh-CN', cache=True):

setup.py

@@ -9,10 +9,10 @@
 
 name = """google-tr-free"""
 # description = ' '.join(name.split('-'))
-description = name.replace('-tr-', 'translate for')
+description = name.replace('-tr-', ' translate for ')
 dir_name, = find_packages()
 
-version, = re.findall(r"__version__\W*=\W*'([^']+)'", open(Path(__file__).parent / f'{dir_name}/__init__.py').read())
+version, = re.findall(r"\n__version__\W*=\W*'([^']+)'", open(Path(__file__).parent / f'{dir_name}/__init__.py').read())
 
 README_rst = f'{Path(__file__).parent}/README.md'
 long_description = open(README_rst, encoding='utf-8').read() if Path(README_rst).exists() else ''
@@ -23,7 +23,7 @@
     version=version,
     description=description,
     long_description=long_description,
-    long_description_content_type='text/markdown'，
+    long_description_content_type='text/markdown',
     keywords=['machine translation', 'free', 'scraping', ],
     author="mikeee",
     url=f'http://github.com/ffreemt/{name}',