Claude/qwen claude reverse eng v hu hv

crates/bge-m3/src/lib.rs

@@ -5,3 +5,4 @@
 
 pub mod weights;
 pub mod embed;
+pub mod tokenizer;

crates/bge-m3/src/tokenizer.rs

@@ -0,0 +1,94 @@
+//! Simple BPE tokenizer stub for BGE-M3 (XLM-RoBERTa).
+//!
+//! STUB: uses deterministic hashing instead of real SentencePiece.
+//! TODO: load sentencepiece.bpe.model for production accuracy.
+//! The real tokenizer requires loading the 5MB SentencePiece BPE model
+//! and performing proper byte-pair encoding. This stub provides
+//! deterministic token IDs for testing and development.
+
+const CLS_TOKEN: u32 = 0;
+const SEP_TOKEN: u32 = 2;
+#[allow(dead_code)]
+const UNK_TOKEN: u32 = 3;
+
+/// Tokenize text into token IDs.
+/// Adds \[CLS\] at start and \[SEP\] at end.
+///
+/// Deterministic: same input always produces the same token sequence.
+pub fn tokenize(text: &str) -> Vec<u32> {
+    let mut tokens = vec![CLS_TOKEN];
+    for word in text.split(|c: char| {
+        c.is_whitespace() || c == ',' || c == '.' || c == '!' || c == '?' || c == ';' || c == ':'
+    }) {
+        let word = word.trim();
+        if word.is_empty() {
+            continue;
+        }
+        // Deterministic hash to token ID (within vocab range, skip special tokens 0-3)
+        let hash = word
+            .bytes()
+            .fold(5381u64, |h, b| h.wrapping_mul(33).wrapping_add(b as u64));
+        let token_id = (hash % 250000) as u32 + 4;
+        tokens.push(token_id);
+    }
+    tokens.push(SEP_TOKEN);
+    tokens
+}
+
+/// Tokenize and return token count (for usage stats).
+pub fn token_count(text: &str) -> usize {
+    tokenize(text).len()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_cls_sep() {
+        let tokens = tokenize("hello");
+        assert_eq!(tokens[0], CLS_TOKEN);
+        assert_eq!(*tokens.last().unwrap(), SEP_TOKEN);
+    }
+
+    #[test]
+    fn test_deterministic() {
+        let a = tokenize("hello world");
+        let b = tokenize("hello world");
+        assert_eq!(a, b);
+    }
+
+    #[test]
+    fn test_different_texts() {
+        let a = tokenize("hello");
+        let b = tokenize("world");
+        assert_ne!(a, b);
+    }
+
+    #[test]
+    fn test_token_range() {
+        let tokens = tokenize("The quick brown fox jumps over the lazy dog");
+        for &tok in &tokens[1..tokens.len() - 1] {
+            assert!(tok >= 4, "non-special tokens must be >= 4");
+            assert!(tok < 250004, "tokens must be in vocab range");
+        }
+    }
+
+    #[test]
+    fn test_empty() {
+        let tokens = tokenize("");
+        assert_eq!(tokens, vec![CLS_TOKEN, SEP_TOKEN]);
+    }
+
+    #[test]
+    fn test_punctuation_split() {
+        let tokens = tokenize("hello,world");
+        // Should split on comma: CLS + hello + world + SEP
+        assert_eq!(tokens.len(), 4);
+    }
+
+    #[test]
+    fn test_token_count() {
+        assert_eq!(token_count("a b c"), 5); // CLS + a + b + c + SEP
+    }
+}

crates/bgz-tensor/data/manifest.json

@@ -93,6 +93,26 @@
       "total_bytes_bgz7": 37400000,
       "release_tag": "v0.1.0-bgz-data",
       "sha256": {}
+    },
+    "reader-lm-1.5b": {
+      "source": "jinaai/reader-lm-1.5b",
+      "format": "safetensors",
+      "shards": 1,
+      "total_bytes_bgz7": 27262976,
+      "release_tag": "v0.1.0-bgz-data",
+      "sha256": {
+        "reader-lm-1.5b.bgz7": "ec576cdd37c6ee2f1e138970bf75528c4214491c432c5403a0d9f9e3c025a6ba"
+      }
+    },
+    "bge-m3-f16": {
+      "source": "CompendiumLabs/bge-m3-gguf",
+      "format": "gguf",
+      "shards": 1,
+      "total_bytes_bgz7": 7654400,
+      "release_tag": "v0.1.0-bgz-data",
+      "sha256": {
+        "bge-m3-f16.bgz7": "970daa4d248df76f7d28cf830158ea1261cbb8a2066ffb9cff53e84704a6a50b"
+      }
     }
   },
   "savants": {

crates/bgz-tensor/src/hydrate.rs

@@ -115,17 +115,20 @@ fn cmd_download(manifest: &manifest::Manifest, model: &str) {
     let tag = &entry.release_tag;
 
     for shard in 0..entry.shards {
-        let filename = format!("shard-{shard:02}.bgz7");
-        let dest = dir.join(&filename);
+        let local_filename = format!("shard-{shard:02}.bgz7");
+        let dest = dir.join(&local_filename);
 
         if dest.exists() && fs::metadata(&dest).map(|m| m.len() > 0).unwrap_or(false) {
-            println!("  {filename}: already present, skipping");
+            println!("  {local_filename}: already present, skipping");
             continue;
         }
 
-        let asset_name = format!("{model}--{filename}");
+        // Release assets are 1-indexed (shard-01..shard-11),
+        // local storage is 0-indexed (shard-00..shard-10) matching manifest.
+        let release_shard = shard + 1;
+        let asset_name = format!("{model}--shard-{release_shard:02}.bgz7");
         let url = format!("https://github.com/{repo}/releases/download/{tag}/{asset_name}");
-        println!("  Downloading {filename} from release {tag}...");
+        println!("  Downloading {local_filename} (from asset {asset_name})...");
 
         let status = process::Command::new("curl")
             .args(["-fSL", "--retry", "4", "--retry-delay", "2",
@@ -134,8 +137,7 @@ fn cmd_download(manifest: &manifest::Manifest, model: &str) {
             .expect("curl not found");
 
         if !status.success() {
-            eprintln!("  FAILED to download {filename}");
-            // Clean up partial file
+            eprintln!("  FAILED to download {local_filename}");
             let _ = fs::remove_file(&dest);
             process::exit(1);
         }

crates/lance-graph-planner/src/serve.rs

@@ -68,9 +68,20 @@ mod server {
         Json(json!({
             "object": "list",
             "data": [
-                {"id": "qwen35-opus46", "object": "model", "owned_by": "ada", "created": timestamp()},
-                {"id": "qwen35-opus45", "object": "model", "owned_by": "ada", "created": timestamp()},
-                {"id": "qwen35-9b", "object": "model", "owned_by": "ada", "created": timestamp()},
+                {"id": "qwen35-opus46", "object": "model", "owned_by": "ada", "created": timestamp(),
+                 "description": "Qwen3.5-27B + Opus 4.6 reasoning scaffold (174 MB bgz7)"},
+                {"id": "qwen35-opus45", "object": "model", "owned_by": "ada", "created": timestamp(),
+                 "description": "Qwen3.5-27B + Opus 4.5 behavioral traits (174 MB bgz7)"},
+                {"id": "qwen35-9b", "object": "model", "owned_by": "ada", "created": timestamp(),
+                 "description": "Qwen3.5-9B distilled, scale-invariant core (80 MB bgz7)"},
+                {"id": "reader-lm", "object": "model", "owned_by": "ada", "created": timestamp(),
+                 "description": "jinaai/reader-lm-1.5b HTML→Markdown (26 MB bgz7)"},
+                {"id": "bge-m3", "object": "model", "owned_by": "ada", "created": timestamp(),
+                 "description": "BAAI/bge-m3 multilingual embeddings (7.3 MB bgz7)"},
+                {"id": "llama4-scout", "object": "model", "owned_by": "ada", "created": timestamp(),
+                 "description": "Llama-4-Scout-17B MoE (37 MB bgz7)"},
+                {"id": "openchat-3.5", "object": "model", "owned_by": "ada", "created": timestamp(),
+                 "description": "OpenChat 3.5 Mistral-7B (41 MB bgz7)"},
             ]
         }))
     }
@@ -82,6 +93,21 @@ mod server {
         let model = req.get("model").and_then(|v| v.as_str()).unwrap_or("qwen35-opus46");
         let messages = req.get("messages").and_then(|v| v.as_array()).cloned().unwrap_or_default();
 
+        // Validate model name
+        const VALID_MODELS: &[&str] = &[
+            "qwen35-opus46", "qwen35-opus45", "qwen35-9b",
+            "reader-lm", "bge-m3", "llama4-scout", "openchat-3.5",
+        ];
+        if !VALID_MODELS.contains(&model) {
+            return Err((StatusCode::NOT_FOUND, Json(json!({
+                "error": {
+                    "message": format!("Model '{}' not found. Available: {}", model, VALID_MODELS.join(", ")),
+                    "type": "invalid_request_error",
+                    "code": "model_not_found"
+                }
+            }))));
+        }
+
         if messages.is_empty() {
             return Err((StatusCode::BAD_REQUEST, Json(json!({
                 "error": {"message": "messages array is empty", "type": "invalid_request_error"}
@@ -248,6 +274,42 @@ mod server {
             cache.triple.self_model.matrix.gestalt.l1(&cache.triple.user_model.matrix.gestalt));
     }
 
+    async fn embeddings(
+        State(_state): State<AppState>,
+        Json(req): Json<Value>,
+    ) -> Result<Json<Value>, (StatusCode, Json<Value>)> {
+        let model = req.get("model").and_then(|v| v.as_str()).unwrap_or("bge-m3");
+        let input = req.get("input").and_then(|v| v.as_str())
+            .or_else(|| req.get("input").and_then(|v| v.as_array())
+                .and_then(|a| a.first())
+                .and_then(|v| v.as_str()))
+            .unwrap_or("");
+
+        if input.is_empty() {
+            return Err((StatusCode::BAD_REQUEST, Json(json!({
+                "error": {"message": "input is empty", "type": "invalid_request_error"}
+            }))));
+        }
+
+        // Embed as Base17 fingerprint (17 dims, golden-step folding)
+        let fp = message_to_headprint(input);
+        let embedding: Vec<f64> = fp.dims.iter().map(|d| *d as f64 / 10000.0).collect();
+
+        Ok(Json(json!({
+            "object": "list",
+            "data": [{
+                "object": "embedding",
+                "index": 0,
+                "embedding": embedding,
+            }],
+            "model": model,
+            "usage": {
+                "prompt_tokens": input.split_whitespace().count(),
+                "total_tokens": input.split_whitespace().count(),
+            }
+        })))
+    }
+
     pub async fn run(port: u16) {
         let mut cache = AutocompleteCache::new();
 
@@ -267,11 +329,13 @@ mod server {
             .route("/health", get(health))
             .route("/v1/models", get(list_models))
             .route("/v1/chat/completions", post(chat_completions))
+            .route("/v1/embeddings", post(embeddings))
             .with_state(state);
 
         let addr = format!("0.0.0.0:{port}");
         eprintln!("lance-graph-planner serve listening on {addr}");
         eprintln!("  POST /v1/chat/completions  (OpenAI compatible)");
+        eprintln!("  POST /v1/embeddings         (Base17 fingerprints)");
         eprintln!("  GET  /v1/models");
         eprintln!("  GET  /health");
         let listener = tokio::net::TcpListener::bind(&addr).await.unwrap();