Feature/weak refs collection

AGENTS.md

@@ -0,0 +1,41 @@
+# dbzero
+
+dbzero is a state management system for persisting process state without a database. It lets Python processes keep their in-memory state durable across restarts — no separate DB server, schemas, or ORM. The core is C++ with Python bindings. See https://docs.dbzero.io for user-facing documentation.
+
+## Development workflow
+
+### TDD is required
+
+When implementing new features, follow test-driven development:
+
+1. Write a failing test first (Python tests in `python_tests/`, C++ tests under `tests/` / `subprojects/`).
+2. Implement the minimum code to make the test pass.
+3. Refactor while keeping tests green.
+
+All tests must pass before a change is considered complete.
+
+### Building
+
+- Debug build: `./docker/dbzero-build.sh`
+- Release build: `./docker/dbzero-build.sh -r`
+
+### Running tests
+
+- Python tests: `./scripts/run_tests.sh`
+- If any C++ source under the native/core part of the project was modified, also run the C++ test suite (do not rely on the Python tests alone to cover native changes).
+
+Never mark a task done while tests are failing.
+
+## Implementation notes
+
+### MorphingBIndex: address and type can change on mutation
+
+A `MorphingBIndex` does not behave like a typical container. On mutation (`insert`, `erase`) it may morph into a different internal storage variant (itty / array_2..4 / vector / bindex), and the morph can change both its **address** and its **type**.
+
+Consequences for any code that mutates a `MorphingBIndex`:
+
+- Any externally stored `{address, type}` pair referring to the bindex is potentially invalidated after every `insert` or `erase` call. Lookups through a stale pair read pre-mutation storage and return wrong data.
+- A live handle to the bindex remains valid across the mutation and reflects the new storage; prefer re-reading `bindex.getAddress()` / `bindex.getIndexType()` from the handle over trusting any previously captured copy.
+- Destructive shortcuts (destroying and rebuilding the whole bindex, or erasing it entirely from its parent) avoid the issue since no stale reference remains.
+
+When adding a new mutating path that operates on a `MorphingBIndex`, treat re-syncing any externally held `{address, type}` as mandatory, not an optimization. Collection-specific handling (where these pairs live, which paths must re-sync) is documented at the top of the relevant `.cpp` files.

dbzero/dbzero/dbzero.py

@@ -10,7 +10,7 @@ def load_dynamic(name, path):
 
 def __bootstrap__():
     global __bootstrap__, __loader__, __file__
-    paths = [os.path.join(os.path.split(__file__)[0]), "/src/dev/build/release", "/usr/local/lib/python3/dist-packages/dbzero/"]
+    paths = [os.path.join(os.path.split(__file__)[0]), "/src/dev/build/", "/usr/local/lib/python3/dist-packages/dbzero/"]
     __file__ = None
     for path in paths:
         if os.path.isdir(path):

dbzero/setup.py

@@ -10,7 +10,7 @@
 
 setup(
     name='dbzero',
-    version='0.1.12',
+    version='0.2.1',
     description='DBZero community edition',
     packages=['dbzero'],
     python_requires='>=3.9',

docker/Dockerfile-claude

@@ -0,0 +1,88 @@
+FROM python:3.12-bullseye
+
+ARG NODE_MAJOR=20
+ARG CLAUDE_CODE_VERSION=latest
+ARG DEV_USER=claude
+ARG DEV_UID=1000
+ARG DEV_GID=1000
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV HOME=/home/${DEV_USER}
+ENV CLAUDE_CONFIG_DIR=/home/${DEV_USER}/.claude
+ENV PATH=/home/${DEV_USER}/.local/bin:${PATH}
+
+# Install development dependencies, Node.js 20, and Claude Code.
+RUN apt-get update && apt-get install -y \
+    ca-certificates \
+    cmake \
+    curl \
+    gdb \
+    gettext-base \
+    git \
+    gnupg \
+    jq \
+    less \
+    meson \
+    ninja-build \
+    psmisc \
+    python3-dbg \
+    python3-pip \
+    python3-venv \
+    ripgrep \
+    rsync \
+    screen \
+    unzip \
+    valgrind \
+    && mkdir -p /etc/apt/keyrings \
+    && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
+        | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_MAJOR}.x nodistro main" \
+        > /etc/apt/sources.list.d/nodesource.list \
+    && apt-get update && apt-get install -y nodejs \
+    && npm install -g "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}" \
+    && node "$(npm root -g)/@anthropic-ai/claude-code/install.cjs" \
+    && npm cache clean --force \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN groupadd --gid ${DEV_GID} ${DEV_USER} \
+    && useradd --uid ${DEV_UID} --gid ${DEV_GID} --create-home --shell /bin/bash ${DEV_USER}
+
+# Install Python build tooling and project requirements.
+RUN python3 -m pip install --no-cache-dir --upgrade pip setuptools wheel build
+
+COPY requirements.txt /usr/src/dbzero/
+RUN python3 -m pip install --no-cache-dir --upgrade -r /usr/src/dbzero/requirements.txt
+
+# Seed Claude Code defaults and make them available in interactive shells.
+COPY docker/claude-settings.json /opt/claude/settings.json
+COPY docker/claude-shell-init.sh /usr/local/bin/claude-shell-init
+COPY docker/dbzero-build.sh /usr/local/bin/dbzero-build
+COPY docker/dbzero-build-package.sh /usr/local/bin/dbzero-build-package
+RUN chmod +x /usr/local/bin/claude-shell-init \
+    && chmod +x /usr/local/bin/dbzero-build /usr/local/bin/dbzero-build-package \
+    && mkdir -p ${CLAUDE_CONFIG_DIR} /etc/profile.d \
+    && chmod 700 ${CLAUDE_CONFIG_DIR} \
+    && cp /opt/claude/settings.json ${CLAUDE_CONFIG_DIR}/settings.json \
+    && chmod 600 ${CLAUDE_CONFIG_DIR}/settings.json \
+    && chown -R ${DEV_UID}:${DEV_GID} ${HOME} /opt/claude \
+    && printf '%s\n' \
+        'export CLAUDE_CONFIG_DIR="${CLAUDE_CONFIG_DIR:-$HOME/.claude}"' \
+        > /etc/profile.d/claude-code.sh \
+    && printf '%s\n' \
+        'export CLAUDE_CONFIG_DIR="${CLAUDE_CONFIG_DIR:-$HOME/.claude}"' \
+        >> ${HOME}/.bashrc \
+    && claude --version
+
+RUN ulimit -c unlimited
+
+COPY --chown=${DEV_UID}:${DEV_GID} . /usr/src/dbzero
+WORKDIR /usr/src/dbzero
+
+RUN python3 scripts/generate_meson.py ./src/dbzero/ core
+RUN python3 scripts/generate_meson_tests.py tests/
+
+USER ${DEV_USER}
+
+ENTRYPOINT ["/usr/local/bin/claude-shell-init"]
+CMD ["/bin/bash"]

docker/claude-settings.json

@@ -0,0 +1,8 @@
+{
+  "$schema": "https://json.schemastore.org/claude-code-settings.json",
+  "autoUpdatesChannel": "stable",
+  "defaultShell": "bash",
+  "env": {
+    "DISABLE_AUTOUPDATER": "1"
+  }
+}

docker/claude-shell-init.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+CLAUDE_CONFIG_DIR="${CLAUDE_CONFIG_DIR:-$HOME/.claude}"
+export CLAUDE_CONFIG_DIR
+
+mkdir -p "$CLAUDE_CONFIG_DIR"
+chmod 700 "$CLAUDE_CONFIG_DIR"
+
+if [[ ! -f "$CLAUDE_CONFIG_DIR/settings.json" && -f /opt/claude/settings.json ]]; then
+    cp /opt/claude/settings.json "$CLAUDE_CONFIG_DIR/settings.json"
+    chmod 600 "$CLAUDE_CONFIG_DIR/settings.json"
+fi
+
+if [[ $# -eq 0 ]]; then
+    exec /bin/bash
+fi
+
+exec "$@"

docker/dbzero-build-package.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+show_help() {
+    echo "Build dbzero python package in the ./build directory and optionally install it"
+    echo "Use: dbzero-build-package [options]"
+    echo " -h, --help   Shows this help screen."
+    echo " --install    Install the package locally"
+    exit 0
+}
+
+if [[ ! -f setup.py || ! -d dbzero ]]; then
+    echo "Run dbzero-build-package from the repository root." >&2
+    exit 1
+fi
+
+install_package="false"
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        -h|--help) show_help ;;
+        --install) install_package="true" ; shift ;;
+        --) shift ; break ;;
+        *) break ;;
+    esac
+done
+
+pip_install_args=()
+if python3 -m pip install --help 2>/dev/null | grep -q -- "--break-system-packages"; then
+    pip_install_args+=(--break-system-packages)
+fi
+if [[ "$(id -u)" -ne 0 && -z "${VIRTUAL_ENV:-}" ]]; then
+    pip_install_args+=(--user)
+fi
+
+rm -rf ./.build
+mkdir -p ./.build/dbzero
+cp ./dbzero/* ./.build/dbzero
+cp setup.py ./.build/setup.py
+cp LICENSE ./.build/LICENSE
+cp README.md ./.build/README.md
+
+cd ./.build
+python3 setup.py sdist
+
+if [[ "$install_package" == "true" ]]; then
+    python3 -m pip install "${pip_install_args[@]}" "$(ls ./dist/*.tar.gz | sort | tail -n 1)"
+fi
+
+cd ..
+rm -rf ./.build

docker/dbzero-build.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+show_help() {
+    echo "Builds dbzero for the Claude development container"
+    echo "Use: dbzero-build [options]"
+    echo " -h, --help                   Shows this help screen."
+    echo " -j, --jobs                   Threads number. Max by default."
+    echo " -r, --release                Compile as release. Note: debug build is by default."
+    echo " -s, --sanitize               Compile with sanitizers."
+    echo " -p, --prefix                 Install build under the specified prefix."
+    echo " -t, --tests                  Build tests."
+    echo " -e, --disable_debug_exceptions      Disable debug exceptions."
+    echo ""
+    exit 0
+}
+
+if [[ ! -f scripts/generate_meson.py || ! -d dbzero ]]; then
+    echo "Run dbzero-build from the repository root." >&2
+    exit 1
+fi
+
+export CPLUS_INCLUDE_PATH="${CPLUS_INCLUDE_PATH:-}:/usr/include/python3.9/"
+
+cores=$(grep -c ^processor /proc/cpuinfo)
+build_type="debug"
+sanitizer="false"
+enable_debug_exceptions="true"
+build_tests="false"
+
+default_prefix="/usr/local"
+if [[ "$(id -u)" -ne 0 ]]; then
+    default_prefix="$HOME/.local"
+fi
+install_prefix="${DBZERO_INSTALL_PREFIX:-$default_prefix}"
+
+temp=$(getopt -o hj:rtsep: --long help,jobs:,release,tests,sanitize,disable_debug_exceptions,prefix: -n 'dbzero-build' -- "$@")
+if [[ $? -ne 0 ]]; then
+    exit 1
+fi
+eval set -- "$temp"
+
+while true; do
+    case "$1" in
+        -h|--help) show_help ;;
+        -s|--sanitize) sanitizer="true" ; shift ;;
+        -r|--release) build_type="release" ; shift ;;
+        -t|--tests) build_tests="true" ; shift ;;
+        -e|--disable_debug_exceptions) enable_debug_exceptions="false" ; shift ;;
+        -p|--prefix) install_prefix="$2" ; shift 2 ;;
+        -j|--jobs) cores="$2" ; shift 2 ;;
+        --) shift ; break ;;
+        *) echo "Argument parsing error: $1" >&2 ; exit 1 ;;
+    esac
+done
+
+if [[ "$cores" -lt 1 ]]; then
+    echo "Argument parsing error: Wrong jobs number: $cores" >&2
+    exit 1
+fi
+
+python3 scripts/generate_meson.py ./src/dbzero/ core
+python3 scripts/generate_meson_tests.py tests/
+python3 scripts/generate_meson_dbzero.py dbzero/
+
+mkdir -p build
+
+build_dir="build/debug"
+if [[ "$build_type" == "release" ]]; then
+    build_dir="build/release"
+fi
+
+options=(
+    "-Denable_debug_exceptions=$enable_debug_exceptions"
+    "-Denable_sanitizers=$sanitizer"
+    "-Dbuild_tests=$build_tests"
+)
+
+if [[ -f "$build_dir/meson-private/coredata.dat" ]]; then
+    meson setup --reconfigure --prefix="$install_prefix" --buildtype="$build_type" "${options[@]}" "$build_dir"
+else
+    meson setup --prefix="$install_prefix" --buildtype="$build_type" "${options[@]}" "$build_dir"
+fi
+
+ninja -C "$build_dir" -j "$cores"
+meson install -C "$build_dir"
+
+cd dbzero
+envsubst < dbzero/dbzero.template > dbzero/dbzero.py
+dbzero-build-package --install
+echo "$build_type"

pyproject.toml

@@ -4,7 +4,7 @@ requires = ['meson-python']
 
 [project]
 name = 'dbzero'
-version = '0.1.12'
+version = '0.2.1'
 description = 'A state management system for Python 3.x that unifies your applications business logic, data persistence, and caching into a single, efficient layer.'
 readme = 'README.md'
 requires-python = '>=3.9'

python_tests/test_auto_weak_proxy.py

@@ -53,7 +53,6 @@ def test_auto_wrap_expires_when_source_deleted(db0_fixture):
         _ = obj_2.value.value
 
 
-
 def test_auto_wrap_on_cross_prefix_assignment(db0_fixture):
     """Cross-prefix assignment is silently wrapped as weak_proxy by default."""
     px_1 = db0.get_current_prefix().name

python_tests/test_set.py

@@ -658,3 +658,36 @@ def test_db0_set_iterator_type_valid(db0_fixture):
     s = db0.set()
     it = iter(s)
     assert type(type(it)) is type
+
+
+def test_db0_set_remove_with_hash_collision(db0_fixture):
+    # The TUPLE hash in PyHash.cpp is xor-of-element-hashes, so any
+    # two permutations of the same elements collide. That puts them
+    # in the same m_index bucket, which is the only path that exercises
+    # Set::remove's bindex.erase(*it) branch (size > 1).
+    #
+    # If that branch fails to re-sync m_index after the underlying
+    # bindex morphs to a smaller container, the bucket's stale
+    # {address, type} will keep reporting pre-erase data on later
+    # lookups — wrong len(), wrong `in`, stale iteration.
+    a = (1, 2)
+    b = (2, 1)
+
+    s = db0.set()
+    s.add(a)
+    s.add(b)
+    assert len(s) == 2
+    assert a in s
+    assert b in s
+
+    s.remove(a)
+    assert a not in s
+    assert b in s
+    assert len(s) == 1
+    # Iteration should see only b.
+    assert list(s) == [b]
+
+    # Removing the last colliding element must fully clean the bucket.
+    s.remove(b)
+    assert b not in s
+    assert len(s) == 0

python_tests/test_weak_refs.py

@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 # Copyright (c) 2025 DBZero Software sp. z o.o.
 
+import random
+
 import pytest
 import dbzero as db0
 from .memo_test_types import MemoTestPxClass
@@ -178,7 +180,7 @@ def test_long_weak_ref_inside_set(db0_fixture):
     obj_1 = MemoTestPxClass(123, prefix=px_1)
     set_1 = db0.set([db0.weak_proxy(obj_1)])
     for obj in set_1:
-        assert obj == obj_1    
+        assert obj == obj_1
 
 
 def test_long_weak_ref_inside_dict(db0_fixture):