API keys support, bug fixes, improvements

.gitignore

@@ -129,3 +129,5 @@ dmypy.json
 .pyre/
 installed_models/
 
+# Misc
+api_keys.db

README.md

@@ -112,22 +112,49 @@ docker-compose up -d --build
 | --frontend-language-source | Set frontend default language - source | `en`          |
 | --frontend-language-target | Set frontend default language - target | `es`          |
 | --frontend-timeout | Set frontend translation timeout | `500`         |
+| --offline | Run user-interface entirely offline (don't use internet CDNs) | `false` |
+| --api-keys | Enable API keys database for per-user rate limits lookup | `Don't use API keys` |
 
+## Manage API Keys
+
+LibreTranslate supports per-user limit quotas, e.g. you can issue API keys to users so that they can enjoy higher requests limits per minute (if you also set `--req-limit`). By default all users are rate-limited based on `--req-limit`, but passing an optional `api_key` parameter to the REST endpoints allows a user to enjoy higher request limits.
+
+To use API keys simply start LibreTranslate with the `--api-keys` option.
+
+### Add New Keys
+
+To issue a new API key with 120 requests per minute limits:
+
+```bash
+ltmanage keys add 120
+```
+
+### Remove Keys
+
+```bash
+ltmanage keys remove <api-key>
+```
+
+### View Keys
+
+```bash
+ltmanage keys
+```
 
 ## Roadmap
 
 Help us by opening a pull request!
 
 - [x] A docker image (thanks [@vemonet](https://github.com/vemonet) !)
 - [x] Auto-detect input language (thanks [@vemonet](https://github.com/vemonet) !)
-- [ ] User authentication / tokens
+- [X] User authentication / tokens
 - [ ] Language bindings for every computer language
 
 ## FAQ
 
 ### Can I use your API server at libretranslate.com for my application in production?
 
-The API on libretranslate.com should be used for testing, personal or infrequent use. If you're going to run an application in production, please [get in touch](https://uav4geo.com/contact) to discuss options.
+The API on libretranslate.com should be used for testing, personal or infrequent use. If you're going to run an application in production, please [get in touch](https://uav4geo.com/contact) to get an API key or discuss other options.
 
 ## Credits
 

app/__init__.py

@@ -1 +1,2 @@
 from .main import main
+from .manage import manage

app/api_keys.py

@@ -0,0 +1,54 @@
+import sqlite3
+import uuid
+from expiringdict import ExpiringDict
+
+DEFAULT_DB_PATH = "api_keys.db"
+
+class Database:
+    def __init__(self, db_path = DEFAULT_DB_PATH, max_cache_len=1000, max_cache_age=30):
+        self.db_path = db_path
+        self.cache = ExpiringDict(max_len=max_cache_len, max_age_seconds=max_cache_age)
+
+        # Make sure to do data synchronization on writes!
+        self.c = sqlite3.connect(db_path, check_same_thread=False)
+        self.c.execute('''CREATE TABLE IF NOT EXISTS api_keys (
+            "api_key"	TEXT NOT NULL,
+            "req_limit"	INTEGER NOT NULL,
+            PRIMARY KEY("api_key")
+        );''')
+
+    def lookup(self, api_key):
+        req_limit = self.cache.get(api_key)
+        if req_limit is None:
+            # DB Lookup
+            stmt = self.c.execute('SELECT req_limit FROM api_keys WHERE api_key = ?', (api_key, ))
+            row = stmt.fetchone()
+            if row is not None:
+                self.cache[api_key] = row[0]
+                req_limit = row[0]
+            else:
+                self.cache[api_key] = False
+                req_limit = False
+
+        if isinstance(req_limit, bool):
+            req_limit = None
+
+        return req_limit
+
+    def add(self, req_limit, api_key = "auto"):
+        if api_key == "auto":
+            api_key = str(uuid.uuid4())
+
+        self.remove(api_key)
+        self.c.execute("INSERT INTO api_keys (api_key, req_limit) VALUES (?, ?)", (api_key, req_limit))
+        self.c.commit()
+        return (api_key, req_limit)
+
+    def remove(self, api_key):
+        self.c.execute('DELETE FROM api_keys WHERE api_key = ?', (api_key, ))
+        self.c.commit()
+        return api_key
+
+    def all(self):
+        row = self.c.execute("SELECT api_key, req_limit FROM api_keys")
+        return row.fetchall()

app/app.py

@@ -1,11 +1,16 @@
+import os
 from flask import Flask, render_template, jsonify, request, abort, send_from_directory
 from flask_swagger import swagger
 from flask_swagger_ui import get_swaggerui_blueprint
 from langdetect import detect_langs
 from langdetect import DetectorFactory
 from pkg_resources import resource_filename
+from .api_keys import Database
+
 DetectorFactory.seed = 0 # deterministic
 
+api_keys_db = None
+
 def get_remote_address():
     if request.headers.getlist("X-Forwarded-For"):
         ip = request.headers.getlist("X-Forwarded-For")[0]
@@ -14,8 +19,32 @@ def get_remote_address():
 
     return ip
 
-def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=False, frontend_language_source="en", frontend_language_target="en", frontend_timeout=500, offline=False):
-    if not offline:
+def get_routes_limits(default_req_limit, api_keys_db):
+    if default_req_limit == -1:
+        # TODO: better way?
+        default_req_limit = 9999999999999
+
+    def limits():
+        req_limit = default_req_limit
+
+        if api_keys_db:
+            if request.is_json:
+                json = request.get_json()
+                api_key = json.get('api_key')
+            else:
+                api_key = request.values.get("api_key")
+
+            if api_key:
+                db_req_limit = api_keys_db.lookup(api_key)
+                if db_req_limit is not None:
+                    req_limit = db_req_limit
+
+        return "%s per minute" % req_limit
+
+    return [limits]
+
+def create_app(args):
+    if not args.offline:
         from app.init import boot
         boot()
 
@@ -27,32 +56,32 @@ def create_app(char_limit=-1, req_limit=-1, batch_limit=-1, ga_id=None, debug=Fa
     for l in languages:
         language_map[l.code] = l.name
 
-    if debug:
+    if args.debug:
         app.config['TEMPLATES_AUTO_RELOAD'] = True
 
     # Map userdefined frontend languages to argos language object.
-    if frontend_language_source == "auto":
+    if args.frontend_language_source == "auto":
         frontend_argos_language_source = type('obj', (object,), {
             'code': 'auto',
             'name': 'Auto Detect'
         })
     else:
-        frontend_argos_language_source = next(iter([l for l in languages if l.code == frontend_language_source]), None)
+        frontend_argos_language_source = next(iter([l for l in languages if l.code == args.frontend_language_source]), None)
 
-    frontend_argos_language_target = next(iter([l for l in languages if l.code == frontend_language_target]), None)
+    frontend_argos_language_target = next(iter([l for l in languages if l.code == args.frontend_language_target]), None)
 
     # Raise AttributeError to prevent app startup if user input is not valid.
     if frontend_argos_language_source is None:
-        raise AttributeError(f"{frontend_language_source} as frontend source language is not supported.")
+        raise AttributeError(f"{args.frontend_language_source} as frontend source language is not supported.")
     if frontend_argos_language_target is None:
-        raise AttributeError(f"{frontend_language_target} as frontend target language is not supported.")
+        raise AttributeError(f"{args.frontend_language_target} as frontend target language is not supported.")
 
-    if req_limit > 0:
+    if args.req_limit > 0 or args.api_keys:
         from flask_limiter import Limiter
         limiter = Limiter(
             app,
             key_func=get_remote_address,
-            default_limits=["%s per minute" % req_limit]
+            default_limits=get_routes_limits(args.req_limit, Database() if args.api_keys else None)
         )
 
     @app.errorhandler(400)
@@ -68,10 +97,12 @@ def slow_down_error(e):
         return jsonify({"error": "Slowdown: " + str(e.description)}), 429
 
     @app.route("/")
+    @limiter.exempt
     def index():
-        return render_template('index.html', gaId=ga_id, frontendTimeout=frontend_timeout, offline=offline)
+        return render_template('index.html', gaId=args.ga_id, frontendTimeout=args.frontend_timeout, offline=args.offline, api_keys=args.api_keys, web_version=os.environ.get('LT_WEB') is not None)
 
     @app.route("/languages", methods=['GET', 'POST'])
+    @limiter.exempt
     def langs():
         """
         Retrieve list of supported languages
@@ -149,6 +180,13 @@ def translate():
               example: es
             required: true
             description: Target language code
+          - in: formData
+            name: api_key
+            schema:
+              type: string
+              example: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+            required: false
+            description: API key
         responses:
           200:
             description: Translated text
@@ -209,27 +247,27 @@ def translate():
 
         batch = isinstance(q, list)
 
-        if batch and batch_limit != -1:
+        if batch and args.batch_limit != -1:
           batch_size = len(q)
-          if batch_limit < batch_size:
-            abort(400, description="Invalid request: Request (%d) exceeds text limit (%d)" % (batch_size, batch_limit))
+          if args.batch_limit < batch_size:
+            abort(400, description="Invalid request: Request (%d) exceeds text limit (%d)" % (batch_size, args.batch_limit))
 
-        if char_limit != -1:
+        if args.char_limit != -1:
             if batch:
               chars = sum([len(text) for text in q])
             else:
               chars = len(q)
 
-            if char_limit < chars:
-              abort(400, description="Invalid request: Request (%d) exceeds character limit (%d)" % (chars, char_limit))
+            if args.char_limit < chars:
+              abort(400, description="Invalid request: Request (%d) exceeds character limit (%d)" % (chars, args.char_limit))
 
         if source_lang == 'auto':
             candidate_langs = list(filter(lambda l: l.lang in language_map, detect_langs(q)))
 
             if len(candidate_langs) > 0:
                 candidate_langs.sort(key=lambda l: l.prob, reverse=True)
 
-                if debug:
+                if args.debug:
                     print(candidate_langs)
 
                 source_lang = next(iter([l.code for l in languages if l.code == candidate_langs[0].lang]), None)
@@ -238,7 +276,7 @@ def translate():
             else:
                 source_lang = 'en'
 
-            if debug:
+            if args.debug:
                 print("Auto detected: %s" % source_lang)
 
         src_lang = next(iter([l for l in languages if l.code == source_lang]), None)
@@ -274,6 +312,13 @@ def detect():
               example: Hello world!
             required: true
             description: Text to detect
+          - in: formData
+            name: api_key
+            schema:
+              type: string
+              example: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+            required: false
+            description: API key
         responses:
           200:
             description: Detections
@@ -340,6 +385,7 @@ def detect():
 
 
     @app.route("/frontend/settings")
+    @limiter.exempt
     def frontend_settings():
         """
         Retrieve frontend specific settings
@@ -381,18 +427,19 @@ def frontend_settings():
                           type: string
                           description: Human-readable language name (in English)
         """
-        return jsonify({'charLimit': char_limit,
-                        'frontendTimeout': frontend_timeout,
+        return jsonify({'charLimit': args.char_limit,
+                        'frontendTimeout': args.frontend_timeout,
                         'language': {
                             'source': {'code': frontend_argos_language_source.code, 'name': frontend_argos_language_source.name},
                             'target': {'code': frontend_argos_language_target.code, 'name': frontend_argos_language_target.name}}
                        })
 
     swag = swagger(app)
-    swag['info']['version'] = "1.0"
+    swag['info']['version'] = "1.2"
     swag['info']['title'] = "LibreTranslate"
 
     @app.route("/spec")
+    @limiter.exempt
     def spec():
         return jsonify(swag)
 

app/main.py

@@ -10,7 +10,7 @@ def main():
     parser.add_argument('--char-limit', default=-1, type=int, metavar="<number of characters>",
                         help='Set character limit (%(default)s)')
     parser.add_argument('--req-limit', default=-1, type=int, metavar="<number>",
-                        help='Set maximum number of requests per minute per client (%(default)s)')
+                        help='Set the default maximum number of requests per minute per client (%(default)s)')
     parser.add_argument('--batch-limit', default=-1, type=int, metavar="<number of texts>",
                         help='Set maximum number of texts to translate in a batch request (%(default)s)')
     parser.add_argument('--ga-id', type=str, default=None, metavar="<GA ID>",
@@ -27,18 +27,13 @@ def main():
                         help='Set frontend translation timeout (%(default)s)')
     parser.add_argument('--offline', default=False, action="store_true",
                         help="Use offline")
+    parser.add_argument('--api-keys', default=False, action="store_true",
+                        help="Enable API keys database for per-user rate limits lookup")
+
 
     args = parser.parse_args()
+    app = create_app(args)
 
-    app = create_app(char_limit=args.char_limit,
-                     req_limit=args.req_limit,
-                     batch_limit=args.batch_limit,
-                     ga_id=args.ga_id,
-                     debug=args.debug,
-                     frontend_language_source=args.frontend_language_source,
-                     frontend_language_target=args.frontend_language_target,
-                     frontend_timeout=args.frontend_timeout,
-                     offline=args.offline)
     if args.debug:
         app.run(host=args.host, port=args.port)
     else: