From 18cf8513a51ee7253c30e88d938814990c503de1 Mon Sep 17 00:00:00 2001
From: ChristianStadelmann
 <46969952+ChristianStadelmann@users.noreply.github.com>
Date: Tue, 19 Jan 2021 18:09:23 +0100
Subject: [PATCH] Tokenizer: Fix line continuation after punctuation

Prior to this change, any line ending with [punctuation + '...'], for
example `||...`, would cause the tokenizer to fail.
---
 tokenize_code.m | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tokenize_code.m b/tokenize_code.m
index dd5d628..94e5f51 100644
--- a/tokenize_code.m
+++ b/tokenize_code.m
@@ -99,6 +99,12 @@
             % any other operator:
             else
                 symbol = skip(punctuation);
+                % ends with '...'
+                if length(symbol) > 3 && endsWith(symbol, '...')
+                    % unskip '...'
+                    pos = pos - 3;
+                    symbol = symbol(1:end-3);
+                end
                 % one operator:
                 if any(strcmp(symbol, operators))
                     add_token('punctuation', symbol);