Merge pull request #552 from pyinat/ratelimit

Allow setting lockfile path used for multiprocess rate limiting

.pre-commit-config.yaml

@@ -8,13 +8,13 @@ repos:
       - id: mixed-line-ending
       - id: trailing-whitespace
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.14
+    rev: v0.3.5
     hooks:
       - id: ruff
         args: [ --fix ]
       - id: ruff-format
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.8.0
+    rev: v1.9.0
     hooks:
       - id: mypy
         additional_dependencies: [attrs, types-python-dateutil, types-requests, types-ujson]

HISTORY.md

@@ -5,7 +5,8 @@
 * Fix `KeyError` when using `create_observation()` in dry-run mode
 * Increase default request timeout from 10 to 20 seconds
 * Add `validate_token()` function to manually check if an access token is valid
-* Support rate limits less than one request per second (e.g.: `ClientSession(per_second=0.5)`)
+* Support rate limits less than one request per second (example: `ClientSession(per_second=0.5)`)
+* Allow setting lockfile path used for multiprocess rate limiting (example: `ClientSession(lock_path='/tmp/pyinat.lock')`)
 
 ## 0.19.0 (2023-12-12)
 

docs/conf.py

@@ -14,6 +14,7 @@
 * Some CSS and JS adds collapsible drop-down container
 * On Readthedocs, CSS and JS is automatically added for a version dropdown
 """
+
 # ruff: noqa: E402
 import sys
 from importlib.metadata import version as pkg_version

docs/user_guide/advanced.md

@@ -94,11 +94,10 @@ Float values also work, for example to slow it down to less than 1 request per s
 The default rate-limiting backend is thread-safe, and persistent across application restarts. If
 you have a larger application running from multiple processes, you will need an additional locking
 mechanism to make sure these processes don't conflict with each other. This is available with
-{py:class}`pyrate_limter.FileLockSQLiteBucket`, which can be passed as the session's `bucket_class`:
+{py:class}`.FileLockSQLiteBucket`, which can be passed as the session's `bucket_class`:
 
 ```python
->>> from pyinaturalist import ClientSession
->>> from pyrate_limter import FileLockSQLiteBucket
+>>> from pyinaturalist import ClientSession, FileLockSQLiteBucket
 >>> session = ClientSession(bucket_class=FileLockSQLiteBucket)
 ```
 

noxfile.py

@@ -3,6 +3,7 @@
 * 'lint' command: tools and environments are managed by pre-commit
 * All other commands: the current environment will be used instead of creating new ones
 """
+
 from os.path import join
 from shutil import rmtree
 

pyinaturalist/__init__.py

@@ -9,7 +9,7 @@
 from pyinaturalist.models import *
 from pyinaturalist.paginator import Paginator, IDPaginator, WrapperPaginator
 from pyinaturalist.request_params import get_interval_ranges
-from pyinaturalist.session import ClientSession, clear_cache
+from pyinaturalist.session import ClientSession, FileLockSQLiteBucket, clear_cache
 from pyinaturalist.v0 import *
 from pyinaturalist.v1 import *
 

pyinaturalist/constants.py

@@ -1,10 +1,12 @@
+# ruff: noqa: F401
 from datetime import date, datetime, timedelta
 from os.path import abspath, dirname, join
 from pathlib import Path
 from typing import IO, TYPE_CHECKING, Any, BinaryIO, Dict, Iterable, List, Optional, Tuple, Union
 
 from dateutil.relativedelta import relativedelta
 from platformdirs import user_data_dir
+from pyrate_limiter.sqlite_bucket import LOCK_PATH as DEFAULT_LOCK_PATH
 
 # iNaturalist URLs
 API_V0 = 'https://www.inaturalist.org'

pyinaturalist/controllers/__init__.py

@@ -1,6 +1,7 @@
 """Controller classes for :py:class:`.iNatClient`. These contain all the request functions used by
 the client, grouped by resource type.
 """
+
 # ruff: noqa: F401
 # isort: skip_file
 from pyinaturalist.controllers.base_controller import BaseController

pyinaturalist/converters.py

@@ -1,4 +1,5 @@
 """Type conversion utilities used for both requests and responses"""
+
 import re
 from datetime import date, datetime
 from io import BytesIO

pyinaturalist/docs/__init__.py

@@ -4,6 +4,7 @@
 * Dynamic docs: Function signatures + docstrings based on API request params
 * Static docs: Sphinx documentation on readthedocs.io
 """
+
 from pyinaturalist.docs.docstrings import ApiDocstring, copy_annotations, copy_docstrings
 from pyinaturalist.docs.emoji import EMOJI
 from pyinaturalist.docs.signatures import (

pyinaturalist/docs/docstrings.py

@@ -1,4 +1,5 @@
 """Utilities for copying and modifying docstrings with type annotations"""
+
 import re
 from inspect import cleandoc
 from typing import Callable, Dict, Iterable, List, Optional, get_type_hints

pyinaturalist/docs/emoji.py

@@ -1,4 +1,5 @@
 """Extended list of emoji to represent taxa"""
+
 EMOJI = {
     # Birds
     3: '🐦',

pyinaturalist/docs/model_docs.py

@@ -1,6 +1,7 @@
 """Utilities for generating documentation for model classes. This outputs CSV files that are then
 rendered in the docs as tables.
 """
+
 import csv
 from inspect import getmembers, isclass
 from os import makedirs

pyinaturalist/docs/signatures.py

@@ -1,4 +1,5 @@
 """Utilities for modifying function signatures using ``python-forge``"""
+
 # ruff: noqa: E501
 from functools import partial
 from inspect import Parameter, ismethod, signature

pyinaturalist/docs/templates.py

@@ -10,6 +10,7 @@
 Note: Since the templates are applied dynamically at import time, this adds a tiny amount of overhead
 (about 20 milliseconds as of v0.14) to the import time of the library.
 """
+
 # ruff: noqa: E501
 from typing import Any, Dict, List, Optional, Union
 

pyinaturalist/formatters.py

@@ -7,6 +7,7 @@
 * A list of response objects
 * A single response object
 """
+
 import json
 from datetime import date, datetime, timedelta
 from logging import basicConfig, getLogger

pyinaturalist/models/__init__.py

@@ -1,6 +1,7 @@
 """Data models that represent iNaturalist API response objects.
 See :ref:`data-models` section for usage details.
 """
+
 # ruff: noqa: F401, E402
 # isort: skip_file
 from datetime import datetime, timezone

pyinaturalist/models/base.py

@@ -1,4 +1,5 @@
 """Base class and utilities for data models"""
+
 from collections import UserList
 from copy import deepcopy
 from datetime import datetime

pyinaturalist/models/checklist.py

@@ -1,4 +1,5 @@
 """Models for taxon checklists (aka "original life lists")"""
+
 from typing import List, Optional
 
 from pyinaturalist.constants import ESTABLISTMENT_MEANS, DateTime, TableRow

pyinaturalist/models/conservation_status.py

@@ -1,4 +1,5 @@
 """Models for additional taxon conservation statuses"""
+
 import re
 from logging import getLogger
 from typing import List, Optional, Union

pyinaturalist/node_api.py

@@ -1,4 +1,5 @@
 """Placeholder module for backwards-compatibility"""
+
 # ruff: noqa: F401, F403, F405
 from typing import List
 from warnings import warn

pyinaturalist/paginator.py

@@ -1,4 +1,5 @@
 """Classes to handle pagination of API requests"""
+
 from asyncio import AbstractEventLoop, get_running_loop
 from collections import deque
 from concurrent.futures import ThreadPoolExecutor

pyinaturalist/request_params.py

@@ -5,6 +5,7 @@
 Also see :py:mod:`pyinaturalist.converters` for type conversion utilities not specific to request
 params.
 """
+
 import re
 from datetime import date, datetime, timedelta
 from inspect import signature

pyinaturalist/rest_api.py

@@ -1,4 +1,5 @@
 """Placeholder module for backwards-compatibility"""
+
 # ruff: noqa: F401, F403, F405
 from warnings import warn
 

pyinaturalist/session.py

@@ -1,4 +1,5 @@
 """Session class and related functions for preparing and sending API requests"""
+
 import json
 import threading
 from collections import defaultdict
@@ -7,6 +8,7 @@
 from os import getenv
 from typing import Dict, Optional, Type
 
+import pyrate_limiter
 from requests import PreparedRequest, Request, Response, Session
 from requests.adapters import HTTPAdapter
 from requests.utils import default_user_agent
@@ -32,6 +34,7 @@
     CACHE_EXPIRATION,
     CACHE_FILE,
     CONNECT_TIMEOUT,
+    DEFAULT_LOCK_PATH,
     MAX_DELAY,
     RATELIMIT_FILE,
     REQUEST_BURST_RATE,
@@ -121,6 +124,13 @@ def __init__(
             url_patterns.update(urls_expire_after)
         self.timeout = timeout
 
+        # Extra args to pass to rate limiter backend
+        bucket_kwargs = kwargs.pop('bucket_kwargs', {})
+        if ratelimit_path := kwargs.pop('ratelimit_path', None):
+            bucket_kwargs['path'] = ratelimit_path
+        if lock_path := kwargs.pop('lock_path', None):
+            bucket_kwargs['lock_path'] = lock_path
+
         super().__init__(  # type: ignore  # false positive
             # Cache settings
             cache_name=cache_file,
@@ -132,7 +142,7 @@ def __init__(
             stale_if_error=True,
             # Rate limit settings
             bucket_class=bucket_class,
-            bucket_kwargs={'path': RATELIMIT_FILE},
+            bucket_kwargs=bucket_kwargs,
             per_second=per_second,
             per_minute=per_minute,
             per_day=per_day,
@@ -359,6 +369,23 @@ def _validate_json(
             return response
 
 
+class FileLockSQLiteBucket(pyrate_limiter.FileLockSQLiteBucket):
+    """Bucket backed by a SQLite database and file lock. Suitable for usage from multiple processes
+    with no shared state. Requires installing [py-filelock](https://py-filelock.readthedocs.io).
+
+    The file lock is reentrant and shared across buckets, allowing a process to access multiple
+    buckets at once.
+
+    This modified class allows setting a custom lock path.
+    """
+
+    def __init__(self, lock_path: str = DEFAULT_LOCK_PATH, **kwargs):
+        from filelock import FileLock
+
+        super().__init__(**kwargs)
+        self._lock = FileLock(lock_path)
+
+
 def delete(url: str, session: Optional[ClientSession] = None, **kwargs) -> Response:
     """Wrapper around :py:func:`requests.delete` with additional options specific to iNat API requests"""
     session = session or get_local_session()

pyinaturalist/v1/__init__.py

@@ -4,6 +4,7 @@
 
 Most recent API version tested: 1.3.0
 """
+
 # ruff: noqa: F401, F403
 from pyinaturalist.v1.controlled_terms import get_controlled_terms, get_controlled_terms_for_taxon
 from pyinaturalist.v1.identifications import get_identifications, get_identifications_by_id

pyinaturalist/v2/__init__.py

@@ -2,5 +2,6 @@
 Functions to access the iNaturalist API v2
 See: http://api.inaturalist.org/v2/docs/
 """
+
 # ruff: noqa: F401, F403
 from pyinaturalist.v2.observations import get_observations

pyproject.toml

@@ -144,13 +144,15 @@ fix = true
 unsafe-fixes = true
 line-length = 100
 output-format = 'grouped'
-select = ['B', 'C4', 'C90', 'E', 'F']
 target-version = 'py37'
 exclude = ['examples/', 'test/sample_data/']
 
 [tool.ruff.format]
 quote-style = 'single'
 
+[tool.ruff.lint]
+select = ['B', 'C4', 'C90', 'E', 'F']
+
 [tool.ruff.lint.isort]
 known-first-party = ['test']
 

scripts/map_fips_codes.py

@@ -2,6 +2,7 @@
 """A script for mapping US county FIPS codes to iNaturalist place IDs.
 This is a bit complicated, since we only have a text search endpoint to work with for iNat places.
 """
+
 import json
 import logging
 import re

scripts/observation_crud_test.py

@@ -14,6 +14,7 @@
 python scripts/obs_crud_test.py
 ```
 """
+
 from datetime import datetime
 from logging import getLogger
 from os.path import join

scripts/parse_openapi_spec.py

@@ -4,6 +4,7 @@
 Extra dependencies:
     ``pip install prance[osv] rich``
 """
+
 import json
 from os.path import isfile, join
 from typing import Dict, List, Tuple
@@ -30,9 +31,9 @@ def download_spec(force=False):
     spec = requests.get(SPEC_URL).json()
 
     spec['parameters']['ids_without_taxon_id']['name'] = 'ids_without_taxon_id'
-    spec['parameters']['ids_without_observation_taxon_id'][
-        'name'
-    ] = 'ids_without_observation_taxon_id'
+    spec['parameters']['ids_without_observation_taxon_id']['name'] = (
+        'ids_without_observation_taxon_id'
+    )
     spec['parameters']['projects_order_by']['default'] = 'created'
 
     with open(SPEC_FILE, 'w') as f:

scripts/unique_response_keys.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # ruff: noqa: E402, F401, F402, F403, F405
 """A script used to determine unique response keys for each response type"""
+
 import sys
 from itertools import chain
 from pprint import pprint

test/conftest.py

@@ -2,6 +2,7 @@
 Shared unit test-related utilities.
 Pytest will also automatically pick up any fixtures defined here.
 """
+
 import json
 import os
 import re

test/sample_data.py

@@ -1,4 +1,5 @@
-'Sample responses representing different variations on all supported resource types'
+"Sample responses representing different variations on all supported resource types"
+
 # ruff: noqa: F401, F403
 import json
 from glob import glob

test/test_compat.py

@@ -1,7 +1,6 @@
 """Tests for backwards-compatible (but deprecated) imports"""
 # ruff: noqa: F401
 
-
 from test.conftest import ignore_deprecation
 
 

test/test_models.py

@@ -9,6 +9,7 @@
 * Any additional properties or aliases on the model
 * Formatting in the model's __str__ method
 """
+
 from copy import deepcopy
 from datetime import date, datetime