change setuptools build to flit

require python 3.7
support python 3.12
add more linters
improve typing
passthrough kwargs to lmdb.open

.github/workflows/pythonpackage.yml

@@ -8,8 +8,8 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: 3.8
     - name: Install dependencies
@@ -28,12 +28,12 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-20.04, macos-latest, windows-latest]
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
     runs-on: ${{ matrix.os }}
 
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
@@ -50,16 +50,16 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: 3.8
     - name: Build dists
       run: |
         python -m pip install -U pip wheel
         python setup.py sdist bdist_wheel
     - name: Publish a Python distribution to PyPI
-      uses: pypa/gh-action-pypi-publish@master
+      uses: pypa/gh-action-pypi-publish@release/v1
       with:
         user: __token__
         password: ${{ secrets.pypi_password }}

.gitignore

@@ -1,6 +1,5 @@
 *.bak
 
-# python
 __pycache__/
 *.py[cod]
 *.egg-info

.pre-commit-config.yaml

@@ -1,45 +1,69 @@
 repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
-    hooks:
-      - id: check-json
-      - id: check-toml
-      - id: check-yaml
-      - id: check-case-conflict
-      - id: check-added-large-files
-      - id: debug-statements
-      - id: mixed-line-ending
-        args: ["--fix=no"]
-      - id: requirements-txt-fixer
-      - id: trailing-whitespace
-        args: ["--markdown-linebreak-ext=md"]
-      - id: end-of-file-fixer
-  - repo: https://github.com/asottile/pyupgrade
-    rev: 'v3.3.1'
-    hooks:
-      - id: pyupgrade
-        args: ["--py36-plus"]
-  - repo: https://github.com/psf/black
-    rev: '23.1.0'
-    hooks:
-      - id: black
-  - repo: https://github.com/PyCQA/isort
-    rev: '5.12.0'
-    hooks:
-      - id: isort
-  - repo: https://github.com/PyCQA/bandit
-    rev: '1.7.4'
-    hooks:
-      - id: bandit
-        args: ['--ini', '.bandit']
-  - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
-    hooks:
-      - id: flake8
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.0.0
-    hooks:
-      - id: mypy
-        args: ["--ignore-missing-imports", "--install-types", "--non-interactive"]
-        additional_dependencies:
-          - lmdb==1.3.0
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.6.0
+  hooks:
+  - id: check-added-large-files
+  - id: check-case-conflict
+  - id: check-json
+  - id: check-merge-conflict
+  - id: check-symlinks
+  - id: check-toml
+  - id: check-yaml
+  - id: debug-statements
+  - id: detect-private-key
+  - id: end-of-file-fixer
+  - id: mixed-line-ending
+    args: [--fix=no]
+  - id: requirements-txt-fixer
+  - id: trailing-whitespace
+    args: [--markdown-linebreak-ext=md]
+- repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
+  rev: v2.13.0
+  hooks:
+  - id: pretty-format-yaml
+    args: [--autofix]
+- repo: https://github.com/tox-dev/pyproject-fmt
+  rev: 1.8.0
+  hooks:
+  - id: pyproject-fmt
+- repo: https://github.com/asottile/pyupgrade
+  rev: v3.15.2
+  hooks:
+  - id: pyupgrade
+    args: [--py37-plus]
+- repo: https://github.com/psf/black-pre-commit-mirror
+  rev: 24.4.2
+  hooks:
+  - id: black
+- repo: https://github.com/PyCQA/isort
+  rev: 5.13.2
+  hooks:
+  - id: isort
+- repo: https://github.com/PyCQA/bandit
+  rev: 1.7.8
+  hooks:
+  - id: bandit
+    args: [-c, pyproject.toml]
+    additional_dependencies: ['.[toml]']
+- repo: https://github.com/pycqa/flake8
+  rev: 7.0.0
+  hooks:
+  - id: flake8
+    additional_dependencies:
+    - flake8-annotations
+    - flake8-bugbear
+    - flake8-eradicate
+    - flake8-mutable
+    - flake8-simplify
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v1.10.0
+  hooks:
+  - id: mypy
+    args: [--ignore-missing-imports, --install-types, --non-interactive]
+    additional_dependencies:
+    - lmdb==1.4.1
+    - typing-extensions>=4.0.0
+- repo: https://github.com/Yelp/detect-secrets
+  rev: v1.4.0
+  hooks:
+  - id: detect-secrets

benchmark.py

@@ -9,7 +9,7 @@
 from collections import defaultdict
 from contextlib import closing, suppress
 from random import randrange
-from typing import Any, Callable, DefaultDict, Dict, Iterable, List, Sequence, TextIO
+from typing import Any, Callable, ContextManager, DefaultDict, Dict, Iterable, List, Sequence, TextIO
 
 import pysos
 import rocksdict
@@ -44,25 +44,30 @@ def __init__(self, db_tpl, db_type):
         self.combined = -1
 
     @abstractmethod
-    def open(self) -> None:
-        # Open the database
+    def open(self) -> ContextManager:
+        """Open the database"""
+
         pass
 
-    def commit(self) -> None:
-        # Commit the changes, if it is not done automatically
+    def commit(self) -> None:  # noqa: B027
+        """Commit the changes, if it is not done automatically"""
+
         pass
 
     def purge(self) -> None:
-        # Remove the database file(s)
+        """Remove the database file(s)"""
+
         with suppress(FileNotFoundError):
             os.unlink(self.path)
 
     def encode(self, value: Any) -> Any:
-        # Convert Python objects to database-capable ones
+        """Convert Python objects to database-capable ones"""
+
         return value
 
     def decode(self, value: Any) -> Any:
-        # Convert database values to Python objects
+        """Convert database values to Python objects"""
+
         return value
 
     def measure_writes(self, N: int) -> None:
@@ -363,7 +368,7 @@ def bench(base: str, nums: Iterable[int]) -> ResultsDict:
 
 
 def write_markdown_table(stream: TextIO, results: ResultsDict, method: str):
-    for k, v in results.items():
+    for v in results.values():
         headers = list(v.keys())
         break
 
@@ -411,7 +416,8 @@ def merge_results(results: Sequence[ResultsDict], func: Callable = min) -> Resul
 if __name__ == "__main__":
     from argparse import ArgumentParser
 
-    from genutility.iter import progress
+    from genutility.rich import Progress
+    from rich.progress import Progress as RichProgress
 
     parser = ArgumentParser()
     parser.add_argument("--outpath", default="bench-dbs", help="Directory to store temporary benchmarking databases")
@@ -430,9 +436,10 @@ def merge_results(results: Sequence[ResultsDict], func: Callable = min) -> Resul
 
     results: List[ResultsDict] = []
 
-    for _ in progress(range(args.bestof)):
-        print("")
-        results.append(bench(args.outpath, args.sizes))
+    with RichProgress() as progress:
+        p = Progress(progress)
+        for _ in p.track(range(args.bestof)):
+            results.append(bench(args.outpath, args.sizes))
 
     if args.bestof == 1:
         best_results = results[0]

lmdbm/__init__.py

@@ -1,5 +1,7 @@
+"""Python DBM style wrapper around LMDB (Lightning Memory-Mapped Database)"""
+
 from .lmdbm import Lmdb, LmdbGzip, error, open
 
-__version__ = "0.0.4"
+__version__ = "0.0.5"
 
 __all__ = ["Lmdb", "LmdbGzip", "error", "open", "__version__"]

lmdbm/lmdbm.py

@@ -1,5 +1,3 @@
-from __future__ import generator_stop
-
 import logging
 from collections.abc import Mapping, MutableMapping
 from gzip import compress, decompress
@@ -8,6 +6,7 @@
 from typing import Any, Generic, Iterator, List, Optional, Tuple, TypeVar, Union
 
 import lmdb
+from typing_extensions import Self
 
 T = TypeVar("T")
 KT = TypeVar("KT")
@@ -28,7 +27,7 @@ class MissingOk:
     def __init__(self, ok: bool) -> None:
         self.ok = ok
 
-    def __enter__(self):
+    def __enter__(self) -> Self:
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
@@ -56,7 +55,7 @@ def __init__(self, env: lmdb.Environment, autogrow: bool) -> None:
 
     @classmethod
     def open(
-        cls, file: str, flag: str = "r", mode: int = 0o755, map_size: int = 2**20, autogrow: bool = True
+        cls, file: str, flag: str = "r", mode: int = 0o755, map_size: int = 2**20, autogrow: bool = True, **kwargs
     ) -> "Lmdb":
         """
         Opens the database `file`.
@@ -65,17 +64,18 @@ def open(
         `map_size`: Initial database size. Defaults to 2**20 (1MB).
         `autogrow`: Automatically grow the database size when `map_size` is exceeded.
                 WARNING: Set this to `False` for multi-process write access.
+        `**kwargs`: All other keyword arguments are passed through to `lmdb.open`.
         """
 
         if flag == "r":  # Open existing database for reading only (default)
-            env = lmdb.open(file, map_size=map_size, max_dbs=1, readonly=True, create=False, mode=mode)
+            env = lmdb.open(file, map_size=map_size, max_dbs=1, readonly=True, create=False, mode=mode, **kwargs)
         elif flag == "w":  # Open existing database for reading and writing
-            env = lmdb.open(file, map_size=map_size, max_dbs=1, readonly=False, create=False, mode=mode)
+            env = lmdb.open(file, map_size=map_size, max_dbs=1, readonly=False, create=False, mode=mode, **kwargs)
         elif flag == "c":  # Open database for reading and writing, creating it if it doesn't exist
-            env = lmdb.open(file, map_size=map_size, max_dbs=1, readonly=False, create=True, mode=mode)
+            env = lmdb.open(file, map_size=map_size, max_dbs=1, readonly=False, create=True, mode=mode, **kwargs)
         elif flag == "n":  # Always create a new, empty database, open for reading and writing
             remove_lmdbm(file)
-            env = lmdb.open(file, map_size=map_size, max_dbs=1, readonly=False, create=True, mode=mode)
+            env = lmdb.open(file, map_size=map_size, max_dbs=1, readonly=False, create=True, mode=mode, **kwargs)
         else:
             raise ValueError("Invalid flag")
 
@@ -121,7 +121,7 @@ def __getitem__(self, key: KT) -> VT:
     def __setitem__(self, key: KT, value: VT) -> None:
         k = self._pre_key(key)
         v = self._pre_value(value)
-        for i in range(12):
+        for _i in range(12):
             try:
                 with self.env.begin(write=True) as txn:
                     txn.put(k, v)
@@ -187,7 +187,7 @@ def update(self, __other: Any = (), **kwds: VT) -> None:  # python3.8 only: upda
         pairs_other: Optional[List[Tuple[bytes, bytes]]] = None
         pairs_kwds: Optional[List[Tuple[bytes, bytes]]] = None
 
-        for i in range(12):
+        for _i in range(12):
             try:
                 with self.env.begin(write=True) as txn:
                     with txn.cursor() as curs:
@@ -228,7 +228,7 @@ def sync(self) -> None:
     def close(self) -> None:
         self.env.close()
 
-    def __enter__(self):
+    def __enter__(self) -> Self:
         return self
 
     def __exit__(self, *args):
@@ -248,5 +248,5 @@ def _post_value(self, value: bytes) -> VT:
         return decompress(value)
 
 
-def open(file, flag="r", mode=0o755):
-    return Lmdb.open(file, flag, mode)
+def open(file, flag="r", mode=0o755, **kwargs):
+    return Lmdb.open(file, flag, mode, **kwargs)