Bump forthic-ts to v0.18.0

Handle quadrulple quote case for nested quotes

Author: rjose

forthic-ts/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forthic/interp",
-  "version": "0.17.0",
+  "version": "0.18.0",
   "description": "Forthic interpreter",
   "main": "dist/cjs/index.js",
   "module": "dist/esm/index.js",

forthic-ts/src/forthic/tests/streamingRun.test.ts

@@ -358,10 +358,11 @@ test("Nested string issue", async () => {
   const gen = interp.streamingRun(`"""Reply saying "Thanks`, false);
   await gen.next();
 
-  // Should throw an error because we're done
+  // Should now handle nested quotes correctly (no longer throws error)
   interp = new Interpreter();
   const gen2 = interp.streamingRun(`"""Reply saying "Thanks""""`, true);
-  await expect(gen2.next()).rejects.toThrow(UnterminatedStringError);
+  const result = await gen2.next();
+  expect(result.done).toBe(true);
 });
 
 class SampleModule extends Module {

forthic-ts/src/forthic/tests/tokenizer.test.ts

@@ -235,4 +235,111 @@ test("Unterminated string", () => {
     expect(e).toBeInstanceOf(UnterminatedStringError);
     console.log(e.message);
   }
-})
+})
+
+describe("Triple quote string with nested quotes", () => {
+  const reference_location = new CodeLocation({
+    screen_name: "test",
+    line: 1,
+    column: 1,
+    start_pos: 0,
+  });
+
+  test("Basic case: '''I said 'Hello''''", () => {
+    const input = "'''I said 'Hello''''";
+    const tokenizer = new Tokenizer(input, reference_location);
+    const token = tokenizer.next_token();
+    
+    expect(token.string).toEqual("I said 'Hello'");
+  });
+
+  test("Normal triple quote behavior (no 4+ consecutive quotes)", () => {
+    const input = "'''Hello'''";
+    const tokenizer = new Tokenizer(input, reference_location);
+    const token = tokenizer.next_token();
+    
+    expect(token.string).toEqual("Hello");
+  });
+
+  test("Double quotes with greedy mode", () => {
+    const input = '"""I said "Hello""""';
+    const tokenizer = new Tokenizer(input, reference_location);
+    const token = tokenizer.next_token();
+    
+    expect(token.string).toEqual('I said "Hello"');
+  });
+
+  test("Six consecutive quotes (empty string case)", () => {
+    const input = "''''''";
+    const tokenizer = new Tokenizer(input, reference_location);
+    const token = tokenizer.next_token();
+    
+    expect(token.string).toEqual("");
+  });
+
+  test("Eight consecutive quotes (two quote content)", () => {
+    const input = "''''''''";
+    const tokenizer = new Tokenizer(input, reference_location);
+    const token = tokenizer.next_token();
+    
+    expect(token.string).toEqual("''");
+  });
+
+  test("Multiple nested quotes", () => {
+    const input = `"""He said "I said 'Hello' to you""""`;
+    const tokenizer = new Tokenizer(input, reference_location);
+    const token = tokenizer.next_token();
+    
+    expect(token.string).toEqual(`He said "I said 'Hello' to you"`);
+  });
+
+  test("No greedy mode when triple quote not followed by quote", () => {
+    const input = "'''Hello''' world'''";
+    const tokenizer = new Tokenizer(input, reference_location);
+    const token = tokenizer.next_token();
+    
+    // Should close at first ''' since it's not followed by another quote
+    expect(token.string).toEqual("Hello");
+  });
+
+  test("Content with apostrophes (contractions)", () => {
+    const input = "'''It's a beautiful day, isn't it?''''";
+    const tokenizer = new Tokenizer(input, reference_location);
+    const token = tokenizer.next_token();
+    
+    expect(token.string).toEqual("It's a beautiful day, isn't it?'");
+  });
+
+  test("Mixed quote types don't trigger greedy mode", () => {
+    const input = "'''Hello\"\"\"";
+    const tokenizer = new Tokenizer(input, reference_location);
+    
+    try {
+      tokenizer.next_token();
+    } catch (e) {
+      expect(e).toBeInstanceOf(UnterminatedStringError);
+    }
+  });
+
+  test("Backward compatibility: normal strings unchanged", () => {
+    const inputs = [
+      "'''simple'''",
+      "'''multi\nline\nstring'''",
+      "'''string with \"double quotes\"'''",
+      "'''string with 'single quotes'''''"
+    ];
+    
+    const expected = [
+      "simple",
+      "multi\nline\nstring", 
+      'string with "double quotes"',
+      "string with 'single quotes''"
+    ];
+    
+    inputs.forEach((input, i) => {
+      const tokenizer = new Tokenizer(input, reference_location);
+      const token = tokenizer.next_token();
+      expect(token.string).toEqual(expected[i]);
+    });
+  });
+});

forthic-ts/src/forthic/tokenizer.ts

@@ -364,6 +364,19 @@ export class Tokenizer {
         char === string_delimiter &&
         this.is_triple_quote(this.input_pos, char)
       ) {
+        // Check if this triple quote is followed by at least one more quote (greedy mode trigger)
+        if (
+          this.input_pos + 3 < this.input_string.length &&
+          this.input_string[this.input_pos + 3] === string_delimiter
+        ) {
+          // Greedy mode: include this quote as content and continue looking for the end
+          this.advance_position(1); // Advance by 1 to catch overlapping sequences
+          this.token_string += string_delimiter;
+          this.string_delta.end = this.input_pos;
+          continue;
+        }
+        
+        // Normal behavior: close at first triple quote
         this.advance_position(3);
         const token = new Token(
           TokenType.STRING,
@@ -422,6 +435,7 @@ export class Tokenizer {
     );
   }
 
+
   transition_from_GATHER_WORD(): Token {
     this.note_start_token();
     while (this.input_pos < this.input_string.length) {