Merge pull request #212 from JWCook/limiter-param

Update all API request functions to optionally take a Limiter object and dry_run option

HISTORY.md

@@ -1,6 +1,10 @@
 # History
 
-## 0.14.1 (2021-07-TBD)
+## 0.15.0 (2021-TBD)
+* Allow all API request functions to accept a `limiter` argument to override rate-limiting settings
+* Allow all API request functions to accept a `dry_run` argument to dry-run an individual request
+
+## 0.14.1 (2021-07-21)
 * Added new function for **Posts** endpoint: `get_posts()`
 * Fix broken `response_format` parameter in `v0.get_observations()`
 

docs/conf.py

@@ -33,7 +33,7 @@
 # Add project path so we can import our package
 sys.path.insert(0, '..')
 from pyinaturalist import __version__
-from pyinaturalist.constants import DOCS_DIR, PROJECT_DIR, EXAMPLES_DIR, SAMPLE_DATA_DIR
+from pyinaturalist.constants import DOCS_DIR, EXAMPLES_DIR, PROJECT_DIR, SAMPLE_DATA_DIR
 from pyinaturalist.docs.model_docs import document_models
 
 # Relevant doc directories used in extension settings

docs/user_guide.md

@@ -333,46 +333,101 @@ macOS and Linux systems. See this guide for setup info:
 Credentials storage with keyring + KeePassXC
 ```
 
+## Rate Limiting
+By default, pyinaturalist applies rate limiting to all requests so they stay within the rates specified by
+iNaturalist's [API Recommended Practices](https://www.inaturalist.org/pages/api+recommended+practices).
+If you want to customize these rate limits, all API request functions take an optional `limiter` argument,
+which takes a [`pyrate_limiter.Limiter`](https://github.com/vutran1710/PyrateLimiter) object.
+
+For example, to increase the rate to 75 requests per minute:
+```python
+>>> from pyinaturalist import get_taxa
+>>> from pyrate_limiter import Duration, Limiter, RequestRate
+>>>
+>>> limiter = Limiter(RequestRate(75, Duration.MINUTE))
+>>> get_taxa(q='warbler', locale=1, limiter=limiter)
+```
+
 ## Dry-run mode
 While developing and testing, it can be useful to temporarily mock out HTTP requests, especially
 requests that add, modify, or delete real data. Pyinaturalist has some settings to make this easier.
 
-### Dry-run all requests
-To enable dry-run mode, set the `DRY_RUN_ENABLED` variable. When set, requests will not be sent
-but will be logged instead:
-
+### Dry-run individual requests
+All API request functions take an optional `dry_run` argument. When set to `True`, requests will not
+be sent but will be logged instead:
 ```python
 >>> import logging
->>> import pyinaturalist
->>>
->>> # Enable at least INFO-level logging
+>>> from pyinaturalist import get_taxa
+>>> # Enable at least INFO-level logging to see the request info
 >>> logging.basicConfig(level='INFO')
 >>>
->>> pyinaturalist.DRY_RUN_ENABLED = True
->>> get_taxa(q='warbler', locale=1)
-{'results': \[\], 'total_results': 0}
+>>> get_taxa(q='warbler', locale=1, dry_run=True)
+{'results': [], 'total_results': 0}
 INFO:pyinaturalist.api_requests:Request: GET, https://api.inaturalist.org/v1/taxa,
     params={'q': 'warbler', 'locale': 1},
     headers={'Accept': 'application/json', 'User-Agent': 'Pyinaturalist/0.9.1'}
 ```
 
-You can also set this as an environment variable (case-insensitive):
+### Dry-run all requests
+To enable dry-run mode for all requests, set the `DRY_RUN_ENABLED` environment variable:
 
+:::{tab} Python
+```python
+>>> import os
+>>> os.environ['DRY_RUN_ENABLED'] = 'true'
+```
+:::
+:::{tab} Unix (MacOS / Linux)
 ```bash
-$ export DRY_RUN_ENABLED=true
-$ python my_script.py
+export DRY_RUN_ENABLED=true
+```
+:::
+:::{tab} Windows CMD
+```bat
+set DRY_RUN_ENABLED="true"
+```
+:::
+:::{tab} PowerShell
+```powershell
+$Env:DRY_RUN_ENABLED="true"
+```
+:::
+
+You can also set this as a global variable:
+
+:::{warning}
+This usage is deprecated and will be removed in a future release.
+```python
+>>> import pyinaturalist
+>>> pyinaturalist.DRY_RUN_ENABLED = True
 ```
+:::
 
 ### Dry-run only write requests
 If you would like to send real `GET` requests but mock out any requests that modify data
-(`POST`, `PUT`, `DELETE`, etc.), you can use the `DRY_RUN_WRITE_ONLY` variable
-instead:
+(`POST`, `PUT`, and `DELETE`), you can use the `DRY_RUN_WRITE_ONLY` variable instead:
+
+:::{tab} Python
 ```python
->>> pyinaturalist.DRY_RUN_WRITE_ONLY = True
->>> # Also works as an environment variable
 >>> import os
->>> os.environ\["DRY_RUN_WRITE_ONLY"\] = 'True'
+>>> os.environ['DRY_RUN_WRITE_ONLY'] = 'true'
+```
+:::
+:::{tab} Unix (MacOS / Linux)
+```bash
+export DRY_RUN_WRITE_ONLY=true
+```
+:::
+:::{tab} Windows CMD
+```bat
+set DRY_RUN_WRITE_ONLY="true"
+```
+:::
+:::{tab} PowerShell
+```powershell
+$Env:DRY_RUN_WRITE_ONLY="true"
 ```
+:::
 
 ## User Agent
 While not mandatory, it's good practice to include a [user-agent](https://en.wikipedia.org/wiki/User_agent) in

pyinaturalist/__init__.py

@@ -1,6 +1,6 @@
 # flake8: noqa: F401, F403
-__version__ = '0.14.1'
-DEFAULT_USER_AGENT = f'pyinaturalist/{__version__}'
+__version__ = '0.15.0'
+DEFAULT_USER_AGENT: str = f'pyinaturalist/{__version__}'
 user_agent = DEFAULT_USER_AGENT
 
 # Ignore ImportErrors if this is imported outside a virtualenv

pyinaturalist/api_requests.py

@@ -1,11 +1,13 @@
 """Some common functions for HTTP requests used by all API modules"""
+# TODO: Default retry and backoff settings
 import threading
 from contextlib import contextmanager
 from logging import getLogger
 from os import getenv
 from typing import Dict
 from unittest.mock import Mock
 
+from pyrate_limiter import Duration, Limiter, RequestRate
 from requests import Response, Session
 
 import pyinaturalist
@@ -21,6 +23,13 @@
 from pyinaturalist.docs import copy_signature
 from pyinaturalist.request_params import prepare_request
 
+# Default rate-limiting settings
+RATE_LIMITER = Limiter(
+    RequestRate(REQUESTS_PER_SECOND, Duration.SECOND),
+    RequestRate(REQUESTS_PER_MINUTE, Duration.MINUTE),
+    RequestRate(REQUESTS_PER_DAY, Duration.DAY),
+)
+
 # Mock response content to return in dry-run mode
 MOCK_RESPONSE = Mock(spec=Response)
 MOCK_RESPONSE.json.return_value = {'results': [], 'total_results': 0, 'access_token': ''}
@@ -33,29 +42,32 @@ def request(
     method: str,
     url: str,
     access_token: str = None,
-    user_agent: str = None,
-    ids: MultiInt = None,
+    dry_run: bool = False,
     headers: Dict = None,
+    ids: MultiInt = None,
     json: Dict = None,
+    limiter: Limiter = None,
     session: Session = None,
     raise_for_status: bool = True,
     timeout: float = 5,
+    user_agent: str = None,
     **params: RequestParams,
 ) -> Response:
-    """Wrapper around :py:func:`requests.request` that supports dry-run mode and rate-limiting,
-    and adds appropriate headers.
+    """Wrapper around :py:func:`requests.request` with additional options specific to iNat API requests
 
     Args:
         method: HTTP method
         url: Request URL
         access_token: access_token: the access token, as returned by :func:`get_access_token()`
-        user_agent: A custom user-agent string to provide to the iNaturalist API
+        dry_run: Just log the request instead of sending a real request
         ids: One or more integer IDs used as REST resource(s) to request
         headers: Request headers
         json: JSON request body
-        session: Existing Session object to use instead of creating a new one
+        limiter: Custom rate limits to apply to this request
+        session: An existing Session object to use instead of creating a new one
         timeout: Time (in seconds) to wait for a response from the server; if exceeded, a
             :py:exc:`requests.exceptions.Timeout` will be raised.
+        user_agent: A custom user-agent string to provide to the iNaturalist API
         params: All other keyword arguments are interpreted as request parameters
 
     Returns:
@@ -70,10 +82,11 @@ def request(
         headers,
         json,
     )
+    limiter = limiter or RATE_LIMITER
     session = session or get_session()
 
     # Run either real request or mock request depending on settings
-    if is_dry_run_enabled(method):
+    if dry_run or is_dry_run_enabled(method):
         logger.debug('Dry-run mode enabled; mocking request')
         log_request(method, url, params=params, headers=headers)
         return MOCK_RESPONSE
@@ -89,25 +102,25 @@ def request(
 
 @copy_signature(request, exclude='method')
 def delete(url: str, **kwargs) -> Response:
-    """Wrapper around :py:func:`requests.delete` that supports dry-run mode and rate-limiting"""
+    """Wrapper around :py:func:`requests.delete` with additional options specific to iNat API requests"""
     return request('DELETE', url, **kwargs)
 
 
 @copy_signature(request, exclude='method')
 def get(url: str, **kwargs) -> Response:
-    """Wrapper around :py:func:`requests.get` that supports dry-run mode and rate-limiting"""
+    """Wrapper around :py:func:`requests.get` with additional options specific to iNat API requests"""
     return request('GET', url, **kwargs)
 
 
 @copy_signature(request, exclude='method')
 def post(url: str, **kwargs) -> Response:
-    """Wrapper around :py:func:`requests.post` that supports dry-run mode and rate-limiting"""
+    """Wrapper around :py:func:`requests.post` with additional options specific to iNat API requests"""
     return request('POST', url, **kwargs)
 
 
 @copy_signature(request, exclude='method')
 def put(url: str, **kwargs) -> Response:
-    """Wrapper around :py:func:`requests.put` that supports dry-run mode and rate-limiting"""
+    """Wrapper around :py:func:`requests.put` with additional options specific to iNat API requests"""
     return request('PUT', url, **kwargs)
 
 
@@ -124,21 +137,6 @@ def ratelimit(bucket=pyinaturalist.user_agent):
         yield
 
 
-def get_limiter():
-    """Get a rate limiter object, if pyrate-limiter is installed"""
-    try:
-        from pyrate_limiter import Duration, Limiter, RequestRate
-
-        requst_rates = [
-            RequestRate(REQUESTS_PER_SECOND, Duration.SECOND),
-            RequestRate(REQUESTS_PER_MINUTE, Duration.MINUTE),
-            RequestRate(REQUESTS_PER_DAY, Duration.DAY),
-        ]
-        return Limiter(*requst_rates)
-    except ImportError:
-        return None
-
-
 def get_session() -> Session:
     """Get a Session object that will be reused across requests to take advantage of connection
     pooling. This is especially relevant for large paginated requests. If used in a multi-threaded
@@ -173,6 +171,3 @@ def log_request(*args, **kwargs):
     """Log all relevant information about an HTTP request"""
     kwargs_strs = [f'{k}={v}' for k, v in kwargs.items()]
     logger.info('Request: {}'.format(', '.join(list(args) + kwargs_strs)))
-
-
-RATE_LIMITER = get_limiter()

pyinaturalist/docs/forge_utils.py

@@ -7,14 +7,15 @@
 from logging import getLogger
 from typing import Callable, List, get_type_hints
 
+from pyrate_limiter import Limiter
 from requests import Session
 
 from pyinaturalist.constants import TemplateFunction
 
 logger = getLogger(__name__)
 
 
-def copy_doc_signature(template_functions: List[TemplateFunction]) -> Callable:
+def copy_doc_signature(*template_functions: TemplateFunction, add_common_args: bool = True) -> Callable:
     """Document a function with docstrings, function signatures, and type annotations from
     one or more template functions.
 
@@ -50,8 +51,10 @@ def copy_doc_signature(template_functions: List[TemplateFunction]) -> Callable:
     Args:
         template_functions: Template functions containing docstrings and params to apply to the
             wrapped function
+        add_common_args: Add additional keyword arguments common to most functions
     """
-    template_functions += [_user_agent, _session]
+    if add_common_args:
+        template_functions = list(template_functions) + [_dry_run, _limiter, _session, _user_agent]
 
     def wrapper(func):
         # Modify annotations and docstring
@@ -159,13 +162,25 @@ def _get_combined_revision(target_function: Callable, template_functions: List[T
 
 
 # Param templates that are added to every function signature by default
-def _user_agent(user_agent: str = None):
+def _dry_run(dry_run: bool = False):
     """
-    user_agent: A custom user-agent string to provide to the iNaturalist API
+    dry_run: Just log the request instead of sending a real request
+    """
+
+
+def _limiter(limiter: Limiter = None):
+    """
+    limiter: Custom rate limits to apply to this request
     """
 
 
 def _session(session: Session = None):
     """
-    session: Allows managing your own `Session object <https://docs.python-requests.org/en/latest/user/advanced/>`_
+    session: An existing `Session object <https://docs.python-requests.org/en/latest/user/advanced/>`_  to use instead of creating a new one
+    """
+
+
+def _user_agent(user_agent: str = None):
+    """
+    user_agent: A custom user-agent string to provide to the iNaturalist API
     """

pyinaturalist/v0/observation_fields.py

@@ -8,7 +8,7 @@
 from pyinaturalist.pagination import add_paginate_all
 
 
-@document_request_params([docs._search_query, docs._pagination])
+@document_request_params(docs._search_query, docs._pagination)
 @add_paginate_all(method='page')
 def get_observation_fields(**params) -> JsonResponse:
     """Search observation fields. Observation fields are basically typed data fields that