refactor: HAMT to use async and other performance optimizations

.github/workflows/pages-main.yaml

@@ -1,35 +1,49 @@
 name: Deploy static site generated by pdoc to GitHub Pages
+
 on:
   push:
     branches: ["main"]
+
 permissions:
   contents: read
   pages: write
   id-token: write
+
 jobs:
   build:
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
+
     permissions:
       contents: read
       pages: write
       id-token: write
+
     runs-on: ubuntu-latest
+
     steps:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+
       - name: Setup Pages
         uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5
+
       - name: Install uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v6
+        with:
+          version: "latest"
+
       - name: Create project environment
         run: uv sync
+
       - name: Build with pdoc
         run: uv run pdoc py_hamt -o ./_site
+
       - name: Upload artifact
         # Automatically uploads an artifact from the './_site' directory by default
         uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3
+
       - name: Deploy to GitHub Pages
         id: deployment
         uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4

.github/workflows/run-checks.yaml

@@ -1,108 +1,54 @@
 name: Run checks
 run-name: Triggered on push from ${{ github.actor }} to branch/tag ${{ github.ref_name }}
 on: push
-# Should be the same as py-hamt/run-checks.sh
 
 jobs:
-  run_checks:
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: pre-commit/action@v3.0.1
+  test:
     name: Create project environment, run all checks
+    needs:
+      - validate
     runs-on: ubuntu-latest
-
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: ["3.12"]
     steps:
-      - name: Checkout repo
+      - name: Check out repository
         uses: actions/checkout@v4
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v6
+        with:
+          version: "latest"
 
       - name: Create project environment
         run: uv sync
 
       - name: Install IPFS
-        uses: ibnesayeed/setup-ipfs@master
+        uses: oduwsdl/setup-ipfs@e92fedca9f61ab9184cb74940254859f4d7af4d9 # v0.6.3
         with:
-          ipfs_version: "0.34.1"
+          ipfs_version: "0.35.0"
           run_daemon: true
-        id: ipfs_setup
-
-      - name: Install and configure Nginx
-        run: |
-          # Install Nginx
-          sudo apt-get update
-          sudo apt-get install -y nginx
-
-          # Create Nginx config for reverse proxy with auth
-          cat <<EOF | sudo tee /etc/nginx/sites-available/ipfs
-          server {
-              listen 5002;
-              server_name localhost;
-
-              # Default deny unless authenticated
-              set \$auth_valid 0;
-
-              location /api/v0/ {
-                  # Enforce X-API-Key for API key auth
-                  if (\$http_x_api_key = "test") {
-                      set \$auth_valid 1;
-                  }
-
-                  # Check Bearer token
-                  if (\$http_authorization = "Bearer test") {
-                      set \$auth_valid 1;
-                  }
-
-                  # Check Basic Auth (test:test = dGVzdDp0ZXN0)
-                  if (\$http_authorization = "Basic dGVzdDp0ZXN0") {
-                      set \$auth_valid 1;
-                  }
-
-                  # Deny if no valid auth method
-                  if (\$auth_valid = 0) {
-                      return 401 "Unauthorized: Invalid or missing authentication";
-                  }
-
-                  # Proxy to IPFS RPC API
-                  proxy_pass http://127.0.0.1:5001;
-                  proxy_set_header Host \$host;
-                  proxy_set_header X-Real-IP \$remote_addr;
-                  proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
-                  proxy_set_header X-Forwarded-Proto \$scheme;
-              }
-          }
-          EOF
-
-          # Enable the site and remove default
-          sudo ln -s /etc/nginx/sites-available/ipfs /etc/nginx/sites-enabled/
-          sudo rm -f /etc/nginx/sites-enabled/default
-
-          # Test Nginx config
-          sudo nginx -t
-
-      - name: Start Nginx and restart IPFS daemon
-        run: |
-          # Start Nginx
-          sudo systemctl start nginx
-
-          # Restart IPFS daemon to ensure it’s running
-          ipfs shutdown
-          ipfs daemon &
-
-          # Wait for IPFS and Nginx to be ready
-          sleep 5
 
       - name: Run pytest with coverage
-        run: uv run pytest --cov=py_hamt tests/ --cov-report=xml
+        run: uv run pytest --ipfs --cov=py_hamt tests/ --cov-report=xml
 
       - name: Upload coverage reports to Codecov
-        uses: codecov/codecov-action@v5
+        uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
       - name: Check coverage
         run: uv run coverage report --fail-under=100 --show-missing
-
-      - name: Check linting with ruff
-        run: uv run ruff check
-
-      - name: Check formatting with ruff
-        run: uv run ruff format --check

.gitignore

@@ -1,3 +1,4 @@
+.DS_Store
 pyrightconfig.json
 
 *.prof

.pre-commit-config.yaml

@@ -0,0 +1,32 @@
+repos:
+  - repo: https://github.com/compilerla/conventional-pre-commit
+    rev: v4.0.0
+    hooks:
+      - id: conventional-pre-commit
+        stages: [commit-msg]
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: check-ast
+      - id: check-case-conflict
+      - id: check-merge-conflict
+      - id: check-toml
+      - id: debug-statements
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+      - id: trailing-whitespace
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: v0.11.11
+    hooks:
+      - id: ruff-check
+      - id: ruff-format
+  - repo: https://github.com/ariebovenberg/slotscheck
+    rev: v0.17.1
+    hooks:
+      - id: slotscheck
+        name: slotscheck
+        entry: bash -c 'env PYTHONPATH=src slotscheck'
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: "v1.15.0"
+    hooks:
+      - id: mypy

AGENTS.md

@@ -0,0 +1,127 @@
+# Project Agents.md Guide for py-hamt
+
+This AGENTS.md file provides comprehensive guidance for OpenAI Codex and other AI agents working with this codebase.
+
+## Project Description
+
+This library is a python implementation of a HAMT, inspired by [rvagg's IAMap project written in JavaScript](https://github.com/rvagg/iamap).
+
+py-hamt provides efficient storage and retrieval of large sets of key-value mappings in a content-addressed storage system. The main target is IPFS, and the data model used is IPLD.
+
+dClimate primarily created this for storing large [zarrs](https://zarr.dev/) on IPFS. To see this in action, see our [data ETLs](https://github.com/dClimate/etl-scripts).
+
+## Project Architecture
+
+A user instantiates a ContentAddressedStore such as KuboCAS which is then used to instantiate a HAMT. This HAMT can then be passed to a ZarrHAMTStore to read/write zarr files. An IPFS daemon running locally can be used or a remote one can be accessed via gateway(reads) & rpc endpoints(writes) on their IPFS defaults. For tests, a local docker container is spun up if docker is available.
+
+## Project Structure for OpenAI Codex Navigation
+
+- `/py_hamt`: Source code of the py-hamt library
+    - `encryption_hamt_store.py` - Example using total encryption
+    - `hamt.py`- where the HAMT data structure is constructed and accesed
+    - `store.py`- Where various stores live that writes the data. Primarily used KuboCAS
+    - `zarr_hamt_store.py` - ZarrHAMTStore class used to directly write or read zarrs leveraging the HAMT data structure onto a Store usually a Content Addressed store like KuboCAS located in store.py.
+- `/tests`: Test files that should be maintained and extended where possible.
+
+## Coding Conventions
+
+### General Conventions for AGENTS.md Implementation
+
+- Use the latest version of Python (3.12) for all new code generated by OpenAI Codex
+- The AI Agent should follow the existing code style in each file
+- Agents.md requires meaningful variable and function names
+- The AI Agent should add comments for complex logic as guided by Agents.md
+
+### Python Guidelines
+- Use functional easy to understand functions wherever possible.
+- All functions need to be fully typed with mypy.
+- All functions require tests to be written and coverage must be full.
+- Example usage can be found in the `/tests` folder
+
+### Development Setup
+
+`py-hamt` uses uv for package management. If uv is not already present which can be checked with `uv` then it can be installed via
+
+`curl -LsSf https://astral.sh/uv/install.sh | sh`
+
+otherwise if uv is already present you can run
+
+`uv sync` to install all dependencies
+
+and then
+
+`source .venv/bin/activate` to activate the venv created by uv.
+
+Lastly while still setting up you can run
+
+```
+pre-commit install
+```
+
+to ensure pre-commit installed.
+
+Docker is also used for integration tests to test ipfs. Try to have docker installed to be able to run all tests with` pytest --ipfs`
+
+## Testing Requirements for the Agent
+
+All tests should be created within the `/tests` directory.
+
+The agent should run tests with the following commands:
+
+```bash
+# Run all tests
+pytest --ipfs
+
+# Run specific test file
+pytest test tests/<insert_file_here>
+
+# Run tests with coverage
+pytest --ipfs --cov
+
+# If IPFS is not present use
+pytest --cov
+```
+
+Tests should be kept as closed to 100% as possible.
+
+## Pull Request Guidelines for the Agent
+
+When the Agent helps create a PR, please ensure it:
+
+1. Includes a clear description of the changes as guided by AGENTS.md
+2. References any related issues that the Agent is addressing
+3. Ensures all tests pass for code generated by the Agent
+4. Keeps PRs focused on a single concern as specified in AGENTS.md
+
+## Programmatic Checks for the Agent
+
+Before submitting changes generated by the Agent, run:
+
+```bash
+bash run-checks.sh
+```
+
+**Note:** This assumes ipfs is running as `run-checks.sh` runs with the ``--ipfs` flag. If kubo ipfs is not running in a local daemon or docker was unable to instantiate simply run
+
+```
+pytest --cov
+```
+
+and
+
+```
+uv run pre-commit run --all-files --show-diff-on-failure
+```
+
+instead.
+
+
+If there is an error with formatting for ruff and it can be autofixed you can normally run
+
+```
+uv run ruff check --fix
+```
+
+to fix it.
+
+All checks must pass before the Agent generated code can be merged. AGENTS.md helps ensure the Agent follows these requirements.