diff --git a/README.md b/README.md index d512686..6918e5d 100644 --- a/README.md +++ b/README.md @@ -521,6 +521,7 @@ mini-a "Refactor the parser and keep iterating until validation passes" \ | `wikiuseversion1` | Use S3 signature v1/path-style compatibility for wiki access | `false` | | `wikiignorecertcheck` | Disable TLS certificate checks for wiki S3 access | `false` | | `wikiindexdir` | Override local index/cache root for non-filesystem wiki indexes | - | +| `wikilexical` | SLON/JSON lexical configuration for Lucene (`language` defaults to `english`; explicit `synonyms` and opt-in enhanced features supported) | `{ language: "english" }` | | `wikis3artifactprefix` | Optional S3 prefix containing a published `.mini-a-wiki-lucene/` cache and, for `mcp-wiki`, `.mini-a-wiki-graph/graph.json`; downloaded into `wikiindexdir` on startup | - | | `wikirestrictprofile` | `mcp-wiki-safe` restricted retrieval defaults profile (`tight`, `moderate`, or `relaxed`); `tight` preserves legacy defaults and individual `wikirestrict*` settings override profile values | `tight` | | `wikimetacache` | Enable sharded wiki page metadata cache | `true` | diff --git a/USAGE.md b/USAGE.md index e3ad40c..7393ab7 100644 --- a/USAGE.md +++ b/USAGE.md @@ -884,6 +884,8 @@ The `start()` method accepts various configuration options: - **`wikiuseversion1`** (boolean, default: `false`): Use S3 path-style/signature-v1 compatibility for `s3`/`s3fs`. - **`wikiignorecertcheck`** (boolean, default: `false`): Disable TLS certificate checks for the S3 endpoint. - **`wikiindexdir`** (string, optional): Override the local index/cache directory used for non-filesystem wiki indexes. +- **`wikilexical`** (SLON/JSON, optional): Lucene lexical configuration. It defaults to `{ language: "english" }`; choose another supported Lucene language and add explicit synonym rules (for example `{ language: "portuguese", synonyms: [["carro", "automovel"]] }`). `shingles`, `ngrams`, `queryExpansion`, and `pseudoRelevanceFeedback` are disabled unless explicitly enabled. The matching configuration is recorded in `.mini-a-wiki-lucene/mini-a-lexical.json`; run writable `/wiki reindex` after changing it. Read-only or hydrated legacy indexes continue ordinary Lucene retrieval and log one publisher-reindex warning. +- **`OAF_MINI_A_WIKI_LEXICAL`** (environment variable, optional): Environment equivalent of `wikilexical`; an explicit runtime `wikilexical` value takes precedence. - **`wikis3artifactprefix`** (string, optional): For an `s3` wiki, download a separately published artifact tree into `wikiindexdir` at startup. Place Lucene files below `.mini-a-wiki-lucene/`; `mcp-wiki` can also consume `.mini-a-wiki-graph/graph.json` when `usewikigraph=true`. The MCP servers remain read-only and never publish artifacts back to S3. - **`wikirestrictprofile`** (string, default: `tight`): For `mcp-wiki-safe`, selects restricted retrieval defaults: `tight`, `moderate`, or `relaxed` (case-insensitive). `tight` preserves the previous defaults; individual `wikirestrict*` settings override profile values; hard disclosure ceilings remain enforced and the safe server stays read-only with only `search` and `read`. - **`wikimetacache`** (boolean, default: `true`): Enable the sharded metadata cache used by wiki search/list/read helpers. diff --git a/mcps/mcp-wiki-ops.yaml b/mcps/mcp-wiki-ops.yaml index 427eae0..b16b77c 100644 --- a/mcps/mcp-wiki-ops.yaml +++ b/mcps/mcp-wiki-ops.yaml @@ -84,6 +84,10 @@ help: desc : If true, disable write operations in this MCP server. example : "false" mandatory: false + - name : wikilexical + desc : "SLON/JSON Lucene lexical configuration. Defaults to {language: 'english'}; supports explicit synonym rules." + example : "{ language: 'english', synonyms: [['js', 'javascript']] }" + mandatory: false - name : wikimounts desc : "SLON/JSON array of read-only wiki mounts: [{name, backend, root|bucket|prefix|url|...}]" example : "[{name: 'team', backend: 'fs', root: './team-wiki'}]" @@ -295,6 +299,7 @@ jobs: label : isString.default("wiki") wikiaccess : isString.default("rw") wikiopsreadonly : toBoolean.isBoolean.default(false) + wikilexical : isString.default(__) wikimounts : isString.default(__) exec : | #js loadLib("mini-a-mcp-wiki.js") diff --git a/mcps/mcp-wiki-safe.yaml b/mcps/mcp-wiki-safe.yaml index e7279b8..ab26d6e 100644 --- a/mcps/mcp-wiki-safe.yaml +++ b/mcps/mcp-wiki-safe.yaml @@ -55,6 +55,10 @@ help: desc : Local cache directory used for downloaded S3 Lucene artifacts example : "/var/cache/mini-a/wiki" mandatory: false + - name : wikilexical + desc : "SLON/JSON Lucene lexical configuration. Defaults to {language: 'english'}; supports explicit synonym rules." + example : "{ language: 'english', synonyms: [['js', 'javascript']] }" + mandatory: false - name : toolPrefix desc : Optional prefix prepended to all tool names (e.g. "docs-" -> docs-search, docs-read) example : "wiki-" @@ -213,6 +217,7 @@ jobs: wikiignorecertcheck : toBoolean.isBoolean.default(false) wikis3artifactprefix : isString.default(__) wikiindexdir : isString.default(__) + wikilexical : isString.default(__) usewikigraph : toBoolean.isBoolean.default(false) wikirestrictprofile : isString.default("tight") wikirestrictsearchlimit : toNumber.isNumber.default(__) diff --git a/mcps/mcp-wiki.yaml b/mcps/mcp-wiki.yaml index 3743f38..e274338 100644 --- a/mcps/mcp-wiki.yaml +++ b/mcps/mcp-wiki.yaml @@ -51,6 +51,10 @@ help: desc : Local cache directory used for downloaded S3 search and graph artifacts example : "/var/cache/mini-a/wiki" mandatory: false + - name : wikilexical + desc : "SLON/JSON Lucene lexical configuration. Defaults to {language: 'english'}; supports explicit synonym rules." + example : "{ language: 'english', synonyms: [['js', 'javascript']] }" + mandatory: false - name : toolPrefix desc : Optional prefix prepended to all tool names (e.g. "docs-" -> docs-list, docs-read, ...) example : "wiki-" @@ -318,6 +322,7 @@ jobs: wikiignorecertcheck : toBoolean.isBoolean.default(false) wikis3artifactprefix: isString.default(__) wikiindexdir : isString.default(__) + wikilexical : isString.default(__) usewikigraph : toBoolean.isBoolean.default(false) wikigraphfalkorhost : isString.default(__) wikigraphfalkorport : toNumber.isNumber.default(__) diff --git a/mini-a-con.js b/mini-a-con.js index 09d1b4b..9af586c 100644 --- a/mini-a-con.js +++ b/mini-a-con.js @@ -637,6 +637,7 @@ try { wikiignorecertcheck: { type: "boolean", default: false, description: "Disable TLS certificate checks for the wiki S3 backend." }, wikilintstaleddays: { type: "number", default: 90, description: "Default stale-page threshold in days for wiki lint." }, wikimounts : { type: "string", description: "SLON/JSON array of read-only wiki mounts; fs roots may be directories or local .zip/.okt archives." }, + wikilexical : { type: "string", description: "SLON/JSON Lucene lexical configuration; defaults to {language:'english'}." }, usewikigraph : { type: "boolean", default: false, description: "Enable the wiki knowledge graph for structural and semantic page relationships." }, wikigraphsemantic: { type: "boolean", default: false, description: "Build semantic (embedding-based) edges in addition to structural links when running /graph build." }, wikigraphcommunity: { type: "string", description: "Community detection algorithm for the wiki graph (louvain|leiden)." }, @@ -1417,6 +1418,8 @@ try { var wikiCfg = { access : sessionOptions.wikiaccess, backend: sessionOptions.wikibackend, + indexdir: sessionOptions.wikiindexdir, + wikilexical: sessionOptions.wikilexical, usegraph: toBoolean(sessionOptions.usewikigraph) === true, wikigraphsemantic: toBoolean(sessionOptions.wikigraphsemantic) === true, wikigraphcommunity: sessionOptions.wikigraphcommunity, diff --git a/mini-a-mcp-wiki.js b/mini-a-mcp-wiki.js index 6c280b6..97b47fb 100644 --- a/mini-a-mcp-wiki.js +++ b/mini-a-mcp-wiki.js @@ -381,6 +381,7 @@ function __miniAMcpWikiBuildConfig(args, options) { access : access, backend : backend, indexdir : isString(args.wikiindexdir) && args.wikiindexdir.trim().length > 0 ? args.wikiindexdir.trim() : __, + wikilexical : isDef(args.wikilexical) ? args.wikilexical : __, s3artifactprefix : isString(args.wikis3artifactprefix) && args.wikis3artifactprefix.trim().length > 0 ? args.wikis3artifactprefix.trim() : __, wikimetacache : isDef(args.wikimetacache) ? toBoolean(args.wikimetacache) : true, usegraph : (isDef(args.usewikigraph) ? toBoolean(args.usewikigraph) : false) || isString(wikiGraphFalkorHost), diff --git a/mini-a-web.yaml b/mini-a-web.yaml index 20057be..df4afc6 100644 --- a/mini-a-web.yaml +++ b/mini-a-web.yaml @@ -714,6 +714,10 @@ help: example : "false" mandatory: false options : ["true", "false"] + - name : wikilexical + desc : SLON/JSON Lucene lexical configuration (default `{language: 'english'}`) + example : "{ language: 'english', synonyms: [['js', 'javascript']] }" + mandatory: false - name : wikilintstaleddays desc : Default stale-page threshold in days for wiki lint example : "90" @@ -1879,6 +1883,7 @@ jobs: wikiregion : isString().default(__) wikiuseversion1: toBoolean().isBoolean().default(false) wikiignorecertcheck: toBoolean().isBoolean().default(false) + wikilexical : isString().default(__) wikilintstaleddays: toNumber().isNumber().default(__) wikimounts : isString().default(__) promptprofile : isString().default(__) diff --git a/mini-a-wiki.js b/mini-a-wiki.js index 3eb7b5f..402f302 100644 --- a/mini-a-wiki.js +++ b/mini-a-wiki.js @@ -5,6 +5,55 @@ // ── Template version & helpers ──────────────────────────────────────────────── var __MINI_A_WIKI_AGENTS_VERSION = 4 +var __MINI_A_WIKI_LEXICAL_SCHEMA_VERSION = 1 +var __MINI_A_WIKI_LEXICAL_LANGUAGES = [ + "arabic", "armenian", "basque", "bengali", "brazilian", "bulgarian", "catalan", "chinese", "cjk", "czech", "danish", "dutch", "english", "estonian", "finnish", "french", "galician", "german", "greek", "hindi", "hungarian", "indonesian", "irish", "italian", "latvian", "lithuanian", "norwegian", "persian", "polish", "portuguese", "romanian", "russian", "sorani", "spanish", "swedish", "tamil", "telugu", "thai", "turkish" +] + +// Keep the on-disk index contract deliberately small and model-free. More +// expensive lexical features remain explicit opt-ins in wikilexical. +var __miniAWikiLexicalConfig = function(raw) { + var value = raw + if (isUnDef(value) || value === null || (isString(value) && value.trim().length === 0)) value = { language: "english" } + if (isString(value)) { + try { value = af.fromJSSLON(value) } catch(e) { throw new Error("Invalid wikilexical configuration: expected SLON/JSON object: " + __miniAErrMsg(e)) } + } + if (!isMap(value)) throw new Error("Invalid wikilexical configuration: expected a SLON/JSON object") + var allowed = { language: true, synonyms: true, shingles: true, ngrams: true, queryexpansion: true, pseudorelevancefeedback: true } + Object.keys(value).forEach(function(key) { + if (allowed[String(key).toLowerCase()] !== true) throw new Error("Invalid wikilexical configuration: unsupported option '" + key + "'") + }) + var language = isString(value.language) ? value.language.toLowerCase().trim() : "english" + if (__MINI_A_WIKI_LEXICAL_LANGUAGES.indexOf(language) < 0) throw new Error("Invalid wikilexical language '" + String(value.language) + "'. Supported Lucene languages include english, french, german, portuguese and spanish.") + var cfg = { language: language, synonyms: [], shingles: true, ngrams: false, queryExpansion: false, pseudoRelevanceFeedback: false } + if (isDef(value.synonyms)) { + if (!isArray(value.synonyms)) throw new Error("Invalid wikilexical synonyms: expected an array of synonym rules") + cfg.synonyms = value.synonyms.map(function(rule, i) { + var terms = isString(rule) ? rule.split(",") : rule + if (!isArray(terms) || terms.length < 2 || !terms.every(function(t) { return isString(t) && t.trim().length > 0 })) throw new Error("Invalid wikilexical synonym rule at index " + i + ": use a comma-separated string or an array with at least two terms") + return terms.map(function(t) { return t.trim().toLowerCase() }) + }) + } + ;["shingles", "ngrams", "queryExpansion", "pseudoRelevanceFeedback"].forEach(function(key) { + var sourceKey = key.toLowerCase() + var provided = isDef(value[key]) ? value[key] : value[sourceKey] + if (isDef(provided) && typeof provided !== "boolean") throw new Error("Invalid wikilexical " + key + ": expected boolean") + if (isDef(provided)) cfg[key] = provided + }) + return cfg +} + +var __miniAWikiLexicalFingerprint = function(cfg) { + var normalized = { + language: cfg.language, + synonyms: cfg.synonyms, + shingles: cfg.shingles === true, + ngrams: cfg.ngrams === true, + queryExpansion: cfg.queryExpansion === true, + pseudoRelevanceFeedback: cfg.pseudoRelevanceFeedback === true + } + return sha1(stringify(normalized, __, "")) +} // v1 stock fingerprint phrase — if AGENTS.md contains this verbatim it was never user-edited var __MINI_A_WIKI_V1_STOCK_PHRASE = "This file defines how agents should read, distil, and contribute knowledge to this wiki." @@ -280,6 +329,7 @@ MiniAWikiManager.prototype._ensureIndexRuntime = function() { this._luceneChannel = isString(this._luceneChannel) ? this._luceneChannel : "" this._luceneFallbackWarned = this._luceneFallbackWarned === true this._luceneNeedsRebuild = this._luceneNeedsRebuild === true + this._lexicalReadOnlyWarned = this._lexicalReadOnlyWarned === true this._stats = isMap(this._stats) ? this._stats : { luceneFullRebuilds: 0, luceneSets: 0, luceneUnsets: 0, metaHits: 0, metaMisses: 0 } } @@ -538,7 +588,7 @@ MiniAWikiManager.prototype._openLucene = function(forceEphemeral) { var chName = this._luceneChName() if (forceEphemeral !== true && this._luceneChannel === chName) return chName try { - $ch(chName).create("searchdb", { path: this._getLuceneIndexPath(), idField: "id", contentField: "content" }) + $ch(chName).create("searchdb", this._luceneOptions()) if (forceEphemeral !== true) this._luceneChannel = chName return chName } catch(e) { @@ -652,7 +702,7 @@ MiniAWikiManager.prototype._mountGraph = function(mount) { } MiniAWikiManager.prototype._rebuildSearchIndex = function(options) { - if (this._access !== 'rw') return + if (this._access !== 'rw') return { ok: false, error: "wiki is read-only" } try { var opts = isObject(options) ? options : {} var self = this @@ -665,8 +715,11 @@ MiniAWikiManager.prototype._rebuildSearchIndex = function(options) { docs.push({ path: pages[i], title: isString(parsed.meta.title) ? parsed.meta.title : pages[i], raw: raw, body: isString(parsed.body) ? parsed.body : "" }) this._metaUpdate(pages[i], raw, parsed) } - this._ensureSearchIndex().rebuild(docs, opts) - } catch(e) { this._logFn('warn', 'Failed to rebuild wiki index: ' + __miniAErrMsg(e)) } + return this._ensureSearchIndex().rebuild(docs, opts) + } catch(e) { + this._logFn('warn', 'Failed to rebuild wiki index: ' + __miniAErrMsg(e)) + return { ok: false, error: __miniAErrMsg(e) } + } } MiniAWikiManager.prototype._getGraphPath = function() { @@ -745,6 +798,73 @@ MiniAWikiManager.prototype._getLuceneIndexPath = function() { return this._ensureIndexRoot() + "/.mini-a-wiki-lucene" } +MiniAWikiManager.prototype._getLexicalManifestPath = function() { + return this._getLuceneIndexPath() + "/mini-a-lexical.json" +} + +MiniAWikiManager.prototype._lexicalManifest = function() { + try { + var path = this._getLexicalManifestPath() + if (!io.fileExists(path)) return __ + var manifest = af.fromJson(io.readFileString(path)) + return isMap(manifest) ? manifest : __ + } catch(e) { return __ } +} + +MiniAWikiManager.prototype._lexicalManifestStatus = function() { + var manifest = this._lexicalManifest() + if (!isMap(manifest)) return { compatible: false, reason: "missing" } + if (manifest.schemaVersion !== __MINI_A_WIKI_LEXICAL_SCHEMA_VERSION) return { compatible: false, reason: "schema" } + if (manifest.fingerprint !== this._lexicalFingerprint) return { compatible: false, reason: "configuration" } + return { compatible: true, manifest: manifest } +} + +MiniAWikiManager.prototype._writeLexicalManifest = function() { + if (this._access !== "rw") return false + var indexPath = this._getLuceneIndexPath() + if (!io.fileExists(indexPath)) io.mkdir(indexPath) + var manifest = { + schemaVersion: __MINI_A_WIKI_LEXICAL_SCHEMA_VERSION, + lexical: this._lexicalConfig, + fingerprint: this._lexicalFingerprint + } + io.writeFileString(this._getLexicalManifestPath(), stringify(manifest, __, "")) + return true +} + +MiniAWikiManager.prototype._hasEnhancedLexicalSupport = function() { + try { + if (isUnDef(ow.ch.__types.searchdb) || !isFunction(ow.ch.__types.searchdb.search)) return false + // searchdb.search existed before enhanced lexical retrieval. Check the + // adapter implementation rather than mistaking that plain API for support. + return String(ow.ch.__types.searchdb.search).toLowerCase().indexOf("lexicalenhanced") >= 0 + } catch(e) { return false } +} + +MiniAWikiManager.prototype._luceneOptions = function() { + return { + path: this._getLuceneIndexPath(), + idField: "id", + contentField: "content", + analyzer: this._lexicalConfig.language, + lexical: this._luceneLexicalOptions() + } +} + +MiniAWikiManager.prototype._luceneLexicalOptions = function() { + return { + language: this._lexicalConfig.language, + synonyms: { + enabled: this._lexicalConfig.synonyms.length > 0, + rules: this._lexicalConfig.synonyms.map(function(rule) { return rule.join(",") }) + }, + shingles: { enabled: this._lexicalConfig.shingles === true }, + characterNGrams: { enabled: this._lexicalConfig.ngrams === true }, + queryExpansion: { enabled: this._lexicalConfig.queryExpansion === true }, + pseudoRelevanceFeedback: { enabled: this._lexicalConfig.pseudoRelevanceFeedback === true } + } +} + MiniAWikiManager.prototype._ensureLucene = function() { if (this._luceneReady === true) return true try { @@ -760,8 +880,8 @@ MiniAWikiManager.prototype._ensureLucene = function() { } MiniAWikiManager.prototype._rebuildLuceneIndex = function(docs, options) { - if (this._access !== "rw") return - if (!this._ensureLucene()) return + if (this._access !== "rw") return { ok: false, error: "wiki is read-only" } + if (!this._ensureLucene()) return { ok: false, error: "Lucene oPack is not available" } try { var opts = isObject(options) ? options : {} this._ensureIndexRuntime() @@ -772,7 +892,7 @@ MiniAWikiManager.prototype._rebuildLuceneIndex = function(docs, options) { try { if (io.fileExists(idxPath)) io.rm(idxPath) } catch(ignoreRm) {} } try { - $ch(chName).create("searchdb", { path: idxPath, idField: "id", contentField: "content" }) + $ch(chName).create("searchdb", this._luceneOptions()) ;(isArray(docs) ? docs : []).forEach(function(d) { $ch(chName).set({ id: d.path }, { content: d.raw, payload: { path: d.path, title: d.title } }) }) @@ -792,8 +912,10 @@ MiniAWikiManager.prototype._rebuildLuceneIndex = function(docs, options) { } this._stats.luceneFullRebuilds++ this._luceneNeedsRebuild = false + return { ok: true } } catch(e) { this._logFn("warn", "Failed to rebuild Lucene index: " + __miniAErrMsg(e)) + return { ok: false, error: __miniAErrMsg(e) } } } @@ -813,17 +935,31 @@ MiniAWikiManager.prototype._makeLuceneSearchIndex = function() { available: function() { return self._ensureLucene() }, exists : function() { return self._luceneIndexExists() }, query : function(q, limit) { - if (self._access !== "rw") return self._luceneQueryReadOnly(q, limit) + var manifestStatus = self._lexicalManifestStatus() + if (self._access !== "rw") { + if (!manifestStatus.compatible) { + if (self._luceneIndexExists() && self._lexicalReadOnlyWarned !== true) { + self._logFn("warn", "Wiki Lucene index uses a legacy or mismatched lexical schema; using ordinary Lucene search. The publisher must run writable reindex and republish the artifacts.") + self._lexicalReadOnlyWarned = true + } + return self._luceneQueryReadOnly(q, limit) + } + // The enhanced searchdb API is channel-backed and opening it takes an + // IndexWriter lock. Keep read-only/hydrated managers mutation-free. + return self._luceneQueryReadOnly(q, limit) + } var chName = self._openLucene(false) if (chName === "__ephemeral__") chName = self._openLucene(true) if (!isString(chName) || chName.length === 0) return [] - var hits = $ch(chName).getAll({ query: q, limit: limit }) + var hits = manifestStatus.compatible && self._hasEnhancedLexicalSupport() + ? ow.ch.__types.searchdb.search(chName, { mode: "lexicalEnhanced", query: q, lexical: self._luceneLexicalOptions(), limit: limit }) + : $ch(chName).getAll({ query: q, limit: limit }) if (chName !== self._luceneChannel) self._closeLucene(chName) return isArray(hits) ? hits : [] }, set : function(path, raw, title) { self._luceneSet(path, raw, title) }, unset : function(path) { self._luceneUnset(path) }, - rebuild : function(docs, opts) { self._rebuildLuceneIndex(docs, opts) }, + rebuild : function(docs, opts) { return self._rebuildLuceneIndex(docs, opts) }, close : function() { self._closeLucene() } } } @@ -900,7 +1036,11 @@ MiniAWikiManager.prototype._removePageIndexes = function(path) { MiniAWikiManager.prototype.reindex = function() { if (this._access !== "rw") return { ok: false, error: "wiki is read-only" } try { - this._rebuildSearchIndex() + if (!this._ensureLucene() || !this._hasEnhancedLexicalSupport()) return { ok: false, error: "Lucene oPack does not support lexicalEnhanced search; upgrade the lucene oPack before publishing an enhanced wiki index." } + var manifestStatus = this._lexicalManifestStatus() + var rebuilt = this._rebuildSearchIndex({ resetLucene: !manifestStatus.compatible }) + if (!isMap(rebuilt) || rebuilt.ok !== true) return isMap(rebuilt) ? rebuilt : { ok: false, error: "failed to rebuild lexical index" } + this._writeLexicalManifest() this._rebuildGraphIndex() return { ok: true } } catch(e) { @@ -1055,6 +1195,12 @@ MiniAWikiManager.prototype._resolveMountPath = function(path) { MiniAWikiManager.prototype.configure = function(config) { var cfg = isMap(config) ? config : {} + // The explicit runtime value wins. The environment form is intentionally + // read here too so direct manager/MCP construction has the same behaviour as + // the Mini-A launcher. + if (isUnDef(cfg.wikilexical) && isString(getEnv("OAF_MINI_A_WIKI_LEXICAL"))) cfg.wikilexical = getEnv("OAF_MINI_A_WIKI_LEXICAL") + this._lexicalConfig = __miniAWikiLexicalConfig(cfg.wikilexical) + this._lexicalFingerprint = __miniAWikiLexicalFingerprint(this._lexicalConfig) var accessRaw = isDef(cfg.access) ? String(cfg.access).toLowerCase().trim() : "ro" var backendRaw = isDef(cfg.backend) ? String(cfg.backend).toLowerCase().trim() : "fs" this._access = accessRaw === "rw" ? "rw" : "ro" @@ -2991,6 +3137,10 @@ MiniAWikiManager.prototype.attach = function(name, config) { this._mounts = this._mounts.filter(function(m) { return m.name !== name }) var cfg = isMap(config) ? config : {} cfg.access = "ro" + // Mounts inherit the caller's lexical contract unless they explicitly select + // another language/rule set. This makes a single wikilexical setting apply + // consistently to federated retrieval. + if (isUnDef(cfg.wikilexical)) cfg.wikilexical = this._lexicalConfig try { var manager = new MiniAWikiManager(cfg, this._logFn) var count = manager._safeListPages("").length diff --git a/mini-a.js b/mini-a.js index 7ae6557..4c8e9cd 100644 --- a/mini-a.js +++ b/mini-a.js @@ -7405,6 +7405,7 @@ MiniA.prototype._initWiki = function(args) { backend: args.wikibackend, usegraph: wikiGraphEnabled, indexdir: args.wikiindexdir, + wikilexical: args.wikilexical, wikimetacache: args.wikimetacache, wikigraphsemantic: toBoolean(args.wikigraphsemantic) === true, wikigraphcommunity: args.wikigraphcommunity, @@ -13589,7 +13590,7 @@ MiniA._KNOWN_ARGUMENT_NAMES = (function() { "lcescalatedefer", "lcbudget", "llmcomplexity", "usewiki", "wikiaccess", "wikibackend", "wikiroot", "wikibucket", "wikiprefix", "wikiurl", "wikiaccesskey", "wikisecret", "wikiregion", "wikiuseversion1", - "wikiignorecertcheck", "wikilintstaleddays", "wikimounts", "usewikigraph", "wikigraphsemantic", "wikigraphcommunity", "wikigraphsearchhints", "wikigraphhintcap", "wikigraphfalkorhost", "wikigraphfalkorport", "wikigraphfalkorgraph", "wikigraphfalkoruser", "wikigraphfalkorpass", "dreammode", "dreamwiki", + "wikiignorecertcheck", "wikilintstaleddays", "wikimounts", "wikilexical", "usewikigraph", "wikigraphsemantic", "wikigraphcommunity", "wikigraphsearchhints", "wikigraphhintcap", "wikigraphfalkorhost", "wikigraphfalkorport", "wikigraphfalkorgraph", "wikigraphfalkoruser", "wikigraphfalkorpass", "dreammode", "dreamwiki", "dreamwikimode", "dreammemorymode", "dreamwikidryrun", "dreamwikiapproval", "dreamwikireorg", "dreamwikiminpages", "dreamwikimaxdepth", "dreamreport" ].forEach(function(name) { @@ -15567,6 +15568,8 @@ MiniA.prototype._startInternal = function(args, sessionStartTime) { args.wikiuseversion1 = _$(toBoolean(args.wikiuseversion1), "args.wikiuseversion1").isBoolean().default(false) args.wikiignorecertcheck = _$(toBoolean(args.wikiignorecertcheck), "args.wikiignorecertcheck").isBoolean().default(false) args.wikiindexdir = _$(args.wikiindexdir, "args.wikiindexdir").isString().default(__) + if (isUnDef(args.wikilexical) && isString(getEnv("OAF_MINI_A_WIKI_LEXICAL"))) args.wikilexical = getEnv("OAF_MINI_A_WIKI_LEXICAL") + if (isDef(args.wikilexical) && !isString(args.wikilexical) && !isMap(args.wikilexical)) throw new Error("args.wikilexical must be a SLON/JSON string or object") args.wikimetacache = _$(toBoolean(args.wikimetacache), "args.wikimetacache").isBoolean().default(true) var _wikiLintStaleDays = isNumber(args.wikilintstaleddays) ? args.wikilintstaleddays : Number(args.wikilintstaleddays) if (isNaN(_wikiLintStaleDays)) _wikiLintStaleDays = __ diff --git a/mini-a.yaml b/mini-a.yaml index 88c59f6..36571c0 100644 --- a/mini-a.yaml +++ b/mini-a.yaml @@ -914,6 +914,10 @@ help: example : "false" mandatory: false options : ["true", "false"] + - name : wikilexical + desc : #SLON/JSON Lucene lexical configuration (default `{language: 'english'}`); supports language, explicit synonyms, shingles, ngrams, queryExpansion and pseudoRelevanceFeedback" + example : "{ language: 'english', synonyms: [['js', 'javascript']] }" + mandatory: false - name : wikilintstaleddays desc : Default stale-days threshold for wiki lint example : "90" @@ -1249,6 +1253,7 @@ jobs: wikiregion : wikiregion wikiuseversion1: wikiuseversion1 wikiignorecertcheck: wikiignorecertcheck + wikilexical : wikilexical wikilintstaleddays: wikilintstaleddays wikimounts : wikimounts mode : mode @@ -1482,6 +1487,7 @@ jobs: wikiregion : isString.default(__) wikiuseversion1: toBoolean.isBoolean.default(false) wikiignorecertcheck: toBoolean.isBoolean.default(false) + wikilexical : isString.default(__) wikilintstaleddays: toNumber.isNumber.default(__) wikimounts : isString.default(__) mode : isString.default(__) diff --git a/tests/wiki.js b/tests/wiki.js index bd7538d..62147cb 100644 --- a/tests/wiki.js +++ b/tests/wiki.js @@ -775,7 +775,7 @@ ow.test.assert(isString(page.meta.updated), true, "updated timestamp set") ow.test.assert(page.body.trim().indexOf("# Round Trip") === 0, true, "body preserved") var reindexResult = wm.reindex() - ow.test.assert(isObject(reindexResult) && reindexResult.ok === true, true, "reindex should succeed in rw mode") + ow.test.assert(isObject(reindexResult) && reindexResult.ok === wm._hasEnhancedLexicalSupport(), true, "reindex should require lexicalEnhanced Lucene support") } finally { cleanupTestDir(dir) } @@ -1076,9 +1076,10 @@ ow.test.assert(denied.windowSeconds, 3600, "restricted budget errors should report the budget window") ow.test.assert(denied.retryAfterSeconds > 0, true, "restricted budget errors should give retry guidance") ow.test.assert(denied.message.indexOf("avoid parallel fallback requests") >= 0, true, "restricted budget errors should discourage repeated fallback requests") - var cfg = __miniAMcpWikiBuildConfig({ usewikigraph: true, wikigraphsearchhints: true, wikis3artifactprefix: "published-cache/" }, { access: "ro" }) + var cfg = __miniAMcpWikiBuildConfig({ usewikigraph: true, wikigraphsearchhints: true, wikis3artifactprefix: "published-cache/", wikilexical: "{ language: 'french', synonyms: [['velo', 'bicyclette']] }" }, { access: "ro" }) ow.test.assert(cfg.wikigraphsearchhints, true, "default configuration must preserve graph search hints") ow.test.assert(cfg.s3artifactprefix, "published-cache/", "MCP configuration should pass the S3 artifact prefix to the wiki manager") + ow.test.assert(cfg.wikilexical.indexOf("french") >= 0, true, "MCP configuration should pass lexical configuration to the wiki manager") } exports.testMcpWikiRestrictedGraphHintsAreOptInOpaqueReferences = function() { @@ -1896,13 +1897,50 @@ return after.filter(function(p) { return before.indexOf(p) < 0 }) } + exports.testLexicalConfigDefaultsAndValidation = function() { + var wm = new MiniAWikiManager({ backend: "fs", root: "." }) + ow.test.assert(wm._lexicalConfig.language, "english", "lexical search should default to English") + ow.test.assert(wm._lexicalConfig.synonyms.length, 0, "default lexical search should not add synonyms") + var portuguese = new MiniAWikiManager({ backend: "fs", root: ".", wikilexical: "{ language: 'portuguese', synonyms: [['carro', 'automovel']] }" }) + ow.test.assert(portuguese._lexicalConfig.language, "portuguese", "configured Lucene language should be preserved") + ow.test.assert(portuguese._lexicalConfig.synonyms[0][1], "automovel", "configured synonym rules should be normalized") + var invalid = false + try { new MiniAWikiManager({ backend: "fs", root: ".", wikilexical: { language: "klingon" } }) } catch(e) { invalid = String(e).indexOf("Invalid wikilexical language") >= 0 } + ow.test.assert(invalid, true, "invalid lexical languages should fail initialization clearly") + } + + exports.testLexicalManifestUpgradeContract = function() { + var dir = createTestDir() + try { + var wm = new MiniAWikiManager({ backend: "fs", root: dir, access: "rw" }) + var resetSeen = __ + wm._hasEnhancedLexicalSupport = function() { return true } + wm._rebuildSearchIndex = function(opts) { resetSeen = opts.resetLucene; return { ok: true } } + wm._rebuildGraphIndex = function() {} + var first = wm.reindex() + ow.test.assert(first.ok, true, "a supported lexical reindex should succeed") + ow.test.assert(resetSeen, true, "a legacy index without manifest should be reset") + var manifest = af.fromJson(io.readFileString(wm._getLexicalManifestPath())) + ow.test.assert(manifest.fingerprint, wm._lexicalFingerprint, "successful reindex should write the lexical manifest") + resetSeen = __ + wm.reindex() + ow.test.assert(resetSeen, false, "matching manifests should retain the normal rebuild path") + var changed = new MiniAWikiManager({ backend: "fs", root: dir, access: "rw", wikilexical: { language: "french" } }) + changed._hasEnhancedLexicalSupport = function() { return true } + changed._rebuildSearchIndex = function(opts) { resetSeen = opts.resetLucene; return { ok: true } } + changed._rebuildGraphIndex = function() {} + ow.test.assert(changed.reindex().ok, true, "changed lexical configuration should reindex") + ow.test.assert(resetSeen, true, "a changed lexical fingerprint should reset the legacy fields") + } finally { cleanupTestDir(dir) } + } + exports.testReadOnlyConsumesExistingLuceneIndex = function() { var dir = createTestDir() try { var rw = new MiniAWikiManager({ backend: "fs", root: dir, access: "rw" }) rw.write("notes/alpha.md", { title: "Alpha", description: "About alpha" }, "# Alpha\n\nThe quick brown zebrafish jumps.") rw.write("notes/beta.md", { title: "Beta", description: "About beta" }, "# Beta\n\nA different marmoset topic.") - ow.test.assert(rw.reindex().ok, true, "rw reindex should succeed") + ow.test.assert(rw.reindex().ok, rw._hasEnhancedLexicalSupport(), "rw reindex should require lexicalEnhanced support") rw.close() var ro = new MiniAWikiManager({ backend: "fs", root: dir, access: "ro" }) diff --git a/tests/wiki.yaml b/tests/wiki.yaml index d430e1d..bbaf523 100644 --- a/tests/wiki.yaml +++ b/tests/wiki.yaml @@ -441,6 +441,14 @@ jobs: to : oJob Test exec: args.func = require("tests/wiki.js").testReadOnlyConsumesExistingLuceneIndex +- name: MiniA Wiki Tests::LexicalConfigDefaultsAndValidation + to : oJob Test + exec: args.func = require("tests/wiki.js").testLexicalConfigDefaultsAndValidation + +- name: MiniA Wiki Tests::LexicalManifestUpgradeContract + to : oJob Test + exec: args.func = require("tests/wiki.js").testLexicalManifestUpgradeContract + - name: MiniA Wiki Tests::ReadOnlySessionWritesNothingWhenIndexExists to : oJob Test exec: args.func = require("tests/wiki.js").testReadOnlySessionWritesNothingWhenIndexExists @@ -587,6 +595,8 @@ todo: - MiniA Wiki Tests::MovePreservesRootedLinks - MiniA Wiki Tests::DriftGuard - MiniA Wiki Tests::ReadOnlyConsumesExistingLuceneIndex +- MiniA Wiki Tests::LexicalConfigDefaultsAndValidation +- MiniA Wiki Tests::LexicalManifestUpgradeContract - MiniA Wiki Tests::ReadOnlySessionWritesNothingWhenIndexExists - MiniA Wiki Tests::ReadOnlySessionWritesNothingWhenNoIndexExists - MiniA Wiki Tests::ReadOnlySearchWorksWhileWriterLockHeld