diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..2cc9f6d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +# Ignore junk for Docker builds (helps with the duplicated layout issue) +engram-parser/ +target/ +**/.git +**/*.rs.bk diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 0000000..daa2a2d --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,17 @@ +# Minimal Codecov config for Rust (per https://about.codecov.io/language/rust/ and #11 AC) +# We will use codecov https://about.codecov.io/language/rust/ +codecov: + require_ci_to_pass: true + +coverage: + precision: 2 + round: down + range: "70...100" + + status: + project: true + patch: true + changes: false + +# CI generates lcov format (cargo llvm-cov --lcov); no gcov parser needed. +comment: false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e37a9b6..771064b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,12 @@ on: branches: [main] pull_request: branches: [main] + schedule: + - cron: '0 0 * * *' # Run daily at midnight + +concurrency: + group: ci-cpu-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read @@ -13,6 +19,9 @@ jobs: validate: name: Build & Test runs-on: ubuntu-latest + timeout-minutes: 20 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} steps: # actions/checkout@v4.2.2 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 @@ -23,19 +32,14 @@ jobs: - name: Install Rust stable uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 with: - components: clippy, rustfmt + components: clippy, rustfmt, llvm-tools-preview - # actions/cache@v4 - - name: Cache cargo registry - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 + # Swatinem/rust-cache@v2 (adopted from corinth-canal for better caching) + - name: Cache Cargo + target + uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo- + shared-key: "cpu-ci-v1" + cache-on-failure: true - name: Check formatting run: cargo fmt --check @@ -48,3 +52,31 @@ jobs: - name: Test run: cargo test --all-features + + # Verify build/test leaves no unexpected artifacts (clean-tree guard per qodo review) + - name: Check working tree is clean + run: | + if [ -n "$(git status --porcelain)" ]; then + echo "❌ Working tree has unexpected changes after build+test:" + git status --short + exit 1 + fi + echo "✅ Working tree is clean" + + # Coverage using Codecov (per review: "We will use codecov https://about.codecov.io/language/rust/") + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@16b05812d776ae1dfaabc8277e421fb6d2506419 # v2 + with: + tool: cargo-llvm-cov + + - name: Generate coverage report + run: cargo llvm-cov --all-targets --all-features --locked --lcov --output-path lcov.info + + - name: Upload coverage to Codecov + if: ${{ env.CODECOV_TOKEN != '' }} + uses: codecov/codecov-action@04b047e8bb82a0c002c8312c1c880fbc6a999d45 # v5 + with: + token: ${{ env.CODECOV_TOKEN }} + files: lcov.info + fail_ci_if_error: false + verbose: true diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml new file mode 100644 index 0000000..b943ee6 --- /dev/null +++ b/.github/workflows/docker-build.yml @@ -0,0 +1,70 @@ +name: Docker Build + +on: + pull_request: + branches: + - '**' + push: + branches: + - main + +concurrency: + group: docker-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + build: + name: Build Docker Image (CPU-only) + permissions: + contents: read + packages: write + runs-on: ubuntu-latest + + steps: + # actions/checkout@v4.2.2 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 + + - name: Set Docker tags + id: tags + env: + EVENT_NAME: ${{ github.event_name }} + REF: ${{ github.ref }} + PR_NUMBER: ${{ github.event.number }} + COMMIT_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + REPO_RAW: ${{ github.repository }} + run: | + REPO="ghcr.io/$(echo "$REPO_RAW" | tr '[:upper:]' '[:lower:]')" + TAGS="${REPO}:${COMMIT_SHA}" + if [ "$EVENT_NAME" = "push" ] && [ "$REF" = "refs/heads/main" ]; then + TAGS="$TAGS,${REPO}:main" + fi + if [ "$EVENT_NAME" = "pull_request" ]; then + TAGS="$TAGS,${REPO}:pr-${PR_NUMBER}" + fi + echo "tags=$TAGS" >> $GITHUB_OUTPUT + + - name: Login to GitHub Container Registry + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build Docker Image + uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5 + with: + context: . + push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + tags: ${{ steps.tags.outputs.tags }} + # No build-args needed (CPU-only, stable Rust in Dockerfile) + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore index 74b27f1..eeb1374 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ /target /Cargo.lock.bak **/*.rs.bk +.mimocode/ +engram-parser/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ad06eaa --- /dev/null +++ b/Dockerfile @@ -0,0 +1,40 @@ +# syntax=docker/dockerfile:1.4 +# +# Dockerfile for engram-parser (pure-Rust, zero-dep GGUF/MoE parser). +# Single-stage build for CI verification and reproducible builds. +# +# engram-parser is a library crate with no binary target, so there is no +# standalone artifact to deploy to a runtime image. This image is used for: +# - CI build/test verification +# - Reproducible build environment +# - Base image for downstream crates that depend on engram-parser +# +# Usage: +# docker build -t engram-parser . +# docker run --rm engram-parser cargo test --all-features +# +# See .github/workflows/docker-build.yml and issue #9 for CI (GHCR on main). + +ARG RUST_VERSION=1.87 + +FROM rust:${RUST_VERSION}-slim + +RUN useradd -m -u 10001 appuser + +WORKDIR /app + +# Copy manifests and lock file for reproducibility +COPY Cargo.toml Cargo.lock ./ + +# Copy source +COPY . . + +# Build and test the crate (zero external deps, no system packages needed) +RUN cargo build --release --all-features && \ + cargo test --release --all-features + +RUN chown -R appuser:appuser /app + +USER appuser + +CMD ["cargo", "test", "--release", "--all-features"] diff --git a/README.md b/README.md index e0b5d9b..7fb10b9 100644 --- a/README.md +++ b/README.md @@ -92,14 +92,62 @@ is returned as raw `Vec`. `DType`, `extract_expert`, `list_experts`, `MoeExpertWeights`, `RawTensor`, `ParserError`, `Result`. +## Ecosystem / Sibling parsers (LIM-9) + +- **engram-parser** (this crate): canonical zero-dep GGUF v3 deserializer + per-expert MoE raw weight ripper. +- Safetensors extraction (header inspection, deterministic manifest, MoE router/expert candidate discovery via classify + groups + layout families) from `rmems/corinth-canal` (experimental source of inspiration) is tracked as a **separate issue** in this repo: #10 (parallel to the GGUF work in #7). + - Source-side bootstrap/supporting: rmems/corinth-canal#116. + - Coordination for consumers (e.g. future multi-format in cortex): Limen-Neural/cortex-tensor#9. + - The reusable implementation will target a dedicated Limen-Neural crate (per org boundary matrix LIM-9); engram-parser charter remains GGUF-only. +- **Clarification**: one-way extraction/copy of code from inspiration. We are not adding any dependency from corinth-canal. corinth-canal keeps an unmodified reference copy (per its PROMOTION_RULES "frozen" status). See #10, #7, and the plan for full cross-links and "no dep on corinth-canal" language. + +Cross-links and updates performed when #10 was created. + ## Development +This is a pure-Rust, zero-dependency crate. Build, lint, and test commands use `--all-features`. + ```bash +# Format cargo fmt --check + +# Lint (fail on warnings) cargo clippy --all-targets --all-features -- -D warnings + +# Build +cargo build --all-features + +# Test cargo test --all-features + +# Coverage (local; requires cargo-llvm-cov: cargo install cargo-llvm-cov) +cargo llvm-cov --all-targets --all-features --locked --lcov --output-path lcov.info +``` + +## Docker + +```bash +# Build the image locally (includes build + test verification) +docker build -t engram-parser . + +# Run tests in the container +docker run --rm engram-parser + +# Pull from GHCR (published on merges to main) +docker pull ghcr.io/limen-neural/engram-parser:main ``` +## CI + +- GitHub Actions: `.github/workflows/ci.yml` (hardened via #11; uses Codecov per ) +- Azure Pipelines: `azure-pipelines.yml` (tracked in #8 for cross-platform ubuntu/mac/windows) +- Docker: `Dockerfile` + `.github/workflows/docker-build.yml` (tracked in #9 for GHCR reproducible builds; use user's Docker CLI for local verification) +- Other CI/DX issues: #12 (security), #13 (releases on tags w/ sentry option), #14 (MSRV), #15 (Dependabot no auto-merge), #16 (layout clean) + +See the issue bodies for full ACs and corinth-canal inspiration patterns (one-way copy only; no dep on corinth-canal). + +Cross-reference: #11, #8, #9, #7, #5, LIM-9. + ## License Licensed under either of diff --git a/src/moe/extract.rs b/src/moe/extract.rs index 02b0f1c..9e01d05 100644 --- a/src/moe/extract.rs +++ b/src/moe/extract.rs @@ -202,7 +202,12 @@ fn stacked_slice_range( path: layout.path.clone(), reason: format!("stacked stride overflow for tensor '{}'", tensor.name), })?; - let end = start + stride; + let end = start + .checked_add(stride) + .ok_or_else(|| ParserError::InvalidLayout { + path: layout.path.clone(), + reason: format!("stacked end overflow for tensor '{}'", tensor.name), + })?; if end > buffer_len { return Err(ParserError::InvalidLayout { path: layout.path.clone(),