diff --git a/package.json b/package.json index cf13fb7..1ea31a5 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,8 @@ }, "scripts": { "start": "node bin/moshcode.mjs", - "demo": "node bin/moshcode.mjs run examples/alive.mosh" + "demo": "node bin/moshcode.mjs run examples/alive.mosh", + "test": "node --test" }, "files": ["bin", "src", "examples", "install.sh", "README.md"], "license": "MIT" diff --git a/src/interpreter.mjs b/src/interpreter.mjs index df3b338..dd1c647 100644 --- a/src/interpreter.mjs +++ b/src/interpreter.mjs @@ -15,7 +15,12 @@ export function tokenize(src) { const re = /("(?:[^"\\]|\\.)*")|(\/\/[^\n]*)|(\d+(?:\.\d+)?)|([A-Za-z_]\w*)|([(){};,])|(\s+)/g; const tokens = []; let m; + let lastIndex = 0; while ((m = re.exec(src)) !== null) { + if (m.index !== lastIndex) { + throw new Error(`moshscript: unexpected character near "${src.slice(lastIndex, lastIndex + 12)}"`); + } + lastIndex = re.lastIndex; if (m[2] || m[6]) continue; // comment or whitespace if (m[1]) tokens.push({ t: "str", v: JSON.parse(m[1]) }); else if (m[3]) tokens.push({ t: "num", v: Number(m[3]) }); @@ -23,6 +28,9 @@ export function tokenize(src) { else if (m[5]) tokens.push({ t: "punc", v: m[5] }); else throw new Error(`moshscript: unexpected character near "${src.slice(m.index, m.index + 12)}"`); } + if (lastIndex !== src.length) { + throw new Error(`moshscript: unexpected character near "${src.slice(lastIndex, lastIndex + 12)}"`); + } return tokens; } diff --git a/test/interpreter.test.mjs b/test/interpreter.test.mjs new file mode 100644 index 0000000..399cf04 --- /dev/null +++ b/test/interpreter.test.mjs @@ -0,0 +1,16 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { compile, tokenize } from "../src/interpreter.mjs"; + +test("tokenize rejects unexpected characters between valid tokens", () => { + assert.throws(() => tokenize("say(@);"), /unexpected character/); + assert.throws(() => tokenize("say(\"ok\") @"), /unexpected character/); +}); + +test("compile preserves valid moshscript behavior", () => { + assert.deepEqual(compile("say(\"hi\");").body[0], { + type: "call", + name: "say", + args: ["hi"], + }); +});