Syntax improvements for inline sequences and lists

- Improved syntax for inline blocks and sequences (`|`-operator) - Improved list syntax - `()` the empty list - `(1,)` list with one item (explicit comma require Squashed commit of the following: commit 667b627 Author: Jan Max Meyer <jmm@phorward.de> Date: Thu Jun 16 21:06:40 2022 +0200 Improving inline sequence syntax for lists - `()` becomes the empty list - `(1,)` becomes a list with one item - `(1)` is just value 1 commit f7014ea Author: Jan Max Meyer <jmm@phorward.de> Date: Thu Jun 16 19:44:45 2022 +0200 Improving first draft for inline sequences commit 65163ba Merge: c375fc3 cf68ab9 Author: Jan Max Meyer <jmm@phorward.de> Date: Tue Jun 14 00:54:30 2022 +0200 Merge branch 'main' into syntax/inline-sequences commit c375fc3 Merge: 6e776b3 75ba7fc Author: Jan Max Meyer <jmm@phorward.de> Date: Tue May 31 01:30:21 2022 +0200 Merge branch 'main' into syntax/inline-sequences commit 6e776b3 Author: Jan Max Meyer <jmm@phorward.de> Date: Wed May 25 22:12:10 2022 +0200 Drafting a syntax for inline sequences This draft implemented in tokay.tok replaces Collection by InlineSequence and allows for ('inline' | 'sequences') as an alternative syntax for blocks.
tokay-lang · Jun 16, 2022 · b48b420 · b48b420
1 parent cf68ab9
commit b48b420
Show file tree

Hide file tree

Showing 8 changed files with 89 additions and 39 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,11 @@ Current main branch.
 
 - General
   - Moved `macros` into separate repository [tokay-macros](https://github.com/tokay-lang/tokay-macros)
+- Syntax
+  - Improved syntax for inline blocks and sequences (`|`-operator)
+  - Improved list syntax
+    - `()` the empty list
+    - `(1,)` list with one item (explicit comma required)
 - Compiler
   - Include `prelude.tok` with default parselets
     - `Number` matches either `Float` or `Int`

diff --git a/examples/tokay.tok b/examples/tokay.tok
@@ -156,16 +156,26 @@ Literal : @{
     T_Integer
 }
 
-CollectionItem : @{
+InlineSequenceItem : @{
     T_Alias _ '=>' _ expect Expression  ast("alias")
     Expression '=>' _ expect Expression  ast("alias")
     Expression
 }
 
-Collection : @{
-    '(' _ (T_EOL _)* ')'  ast("value_void", null)
-    '(' _ (T_EOL _)* (Expression (',' _)? (T_EOL _)*)+ ')'  ast("sequence")
-    '(' _ (T_EOL _)* (CollectionItem (',' _)? (T_EOL _)*)+ expect ')'  ast("sequence")
+InlineSequence : @{
+    # Special case: Expression followed by "," is considered as a list with a single item (syntactic sugar)
+    Expression (T_EOL _)* (',' _)? (T_EOL _)* peek ')'  ast("list")
+
+    # A sequence is a list of items optionally separated by ","
+    (InlineSequenceItem (T_EOL _)* (',' _)? (T_EOL _)*)+  ast("sequence")
+
+    # The empty sequences generates an empty list
+    Void  ast("list")
+}
+
+InlineSequences : @{
+    '(' _ (T_EOL _)* InlineSequence ((T_EOL _)* '|' _ (T_EOL _)* InlineSequence)+ (T_EOL _)* expect ')'  ast("block")
+    '(' _ (T_EOL _)* InlineSequence (T_EOL _)* expect ')'
 }
 
 TokenLiteral : @{
@@ -180,7 +190,7 @@ TokenCall : @{
     T_Consumable '(' _ (T_EOL _)* CallParameters? (T_EOL _)* expect ')'  ast("call")
     T_Consumable  ast("call")
     Parselet
-    Collection
+    InlineSequences
     Block
 }
 
@@ -324,15 +334,18 @@ Sequences : @{
     Sequence
 }
 
-SequenceOrExpression : @{
-    Expression peek T_EOL
+SequencesOrExpression : @{
+    Expression peek {
+        T_EOL
+        EOF
+    }
     Sequences
 }
 
 Instruction : @{
     'begin' ___ Sequence expect T_EOL  ast("begin")
     'end' ___ Sequence expect T_EOL  ast("end")
-    T_Identifier _ ':' _ expect SequenceOrExpression expect T_EOL  ast("constant")
+    T_Identifier _ ':' _ expect SequencesOrExpression   ast("constant")
     Sequences
     T_EOL
 }

diff --git a/src/compiler/ast.rs b/src/compiler/ast.rs
@@ -1228,21 +1228,30 @@ fn traverse_node(compiler: &mut Compiler, node: &Dict) -> ImlResult {
         }
 
         // sequence  ------------------------------------------------------
-        "sequence" => {
-            let children = List::from(&node["children"]);
+        "sequence" | "list" => {
+            let children = if let Some(children) = node.get("children") {
+                List::from(children)
+            } else {
+                List::new()
+            };
 
             let mut ops = Vec::new();
 
             for node in children.iter() {
                 ops.extend(traverse_node_or_list(compiler, node).into_ops(compiler, true))
             }
 
-            if ops.len() == 1 {
-                ImlResult::Ops(ops)
-            } else if ops.len() > 0 {
-                ImlResult::Ops(vec![ImlSequence::new(ops)])
+            if emit == "sequence" {
+                if ops.len() == 1 {
+                    ImlResult::Ops(ops)
+                } else if ops.len() > 0 {
+                    ImlResult::Ops(vec![ImlSequence::new(ops)])
+                } else {
+                    ImlResult::Empty
+                }
             } else {
-                ImlResult::Empty
+                ops.push(Op::MakeList(children.len()).into());
+                ImlResult::Ops(ops)
             }
         }
 

diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs
@@ -638,7 +638,7 @@ fn test_error_reporting() {
     );
 
     // Test empty sequence
-    assert_eq!(crate::run("()", ""), Ok(None));
+    assert_eq!(crate::run("()", ""), Ok(Some(crate::value::List::new().into())));
 
     // Tests on filled and empty blocks and empty blocks
     assert_eq!(

diff --git a/src/compiler/parser.rs b/src/compiler/parser.rs
@@ -215,20 +215,30 @@ impl Parser {
             T_Integer
         }),
 
-        // Collections (are at least embedded sequences)
+        // Inline sequences are used to construct lists and dicts as well
 
-        (CollectionItem = {
+        (InlineSequenceItem = {
             [T_Alias, _, "=>", _, (expect Expression), (call ast[(value "alias")])],
             [Expression, "=>", _, (expect Expression), (call ast[(value "alias")])],
             Expression
         }),
 
-        (Collection = {
-            ["(", _, (kle [T_EOL, _]), ")", (call ast[(value "value_void")])],
-            ["(", _, (kle [T_EOL, _]), (expect (pos [Expression, (opt [",", _]), (kle [T_EOL, _])])), ")", // no expect ")" here!
-                (call ast[(value "sequence")])],
-            ["(", _, (kle [T_EOL, _]), (pos [CollectionItem, (opt [",", _]), (kle [T_EOL, _])]), (expect ")"),
-                (call ast[(value "sequence")])]
+        (InlineSequence = {
+            // Special case: Expression followed by "," is considered as a list with a single item (syntactic sugar)
+            [Expression, (kle [T_EOL, _]), ",", _, (kle [T_EOL, _]), (peek ")"), (call ast[(value "list")])],
+            // A sequence is a list of items optionally separated by ","
+            [(pos [InlineSequenceItem, (kle [T_EOL, _]), (opt [",", _]), (kle [T_EOL, _])]), (call ast[(value "sequence")])],
+            // The empty sequences generates an empty list
+            [Void, (call ast[(value "list")])]
+        }),
+
+        (InlineSequences = {
+            // Multiple sequences delimited by "|" are an alternative form of the block syntax
+            ["(", _, (kle [T_EOL, _]), InlineSequence,
+                (pos [(kle [T_EOL, _]), "|", _, (kle [T_EOL, _]), InlineSequence]), (expect ")"),
+                    (call ast[(value "block")])],
+            // In case there's only a single sequence, handle it just as a sequence without a block
+            ["(", _, (kle [T_EOL, _]), InlineSequence, (expect ")")]
         }),
 
         // Tokens
@@ -246,7 +256,7 @@ impl Parser {
                 (call ast[(value "call")])],
             [T_Consumable, (call ast[(value "call")])],
             Parselet,
-            Collection,
+            InlineSequences,
             Block
         }),
 

diff --git a/src/test.rs b/src/test.rs
@@ -417,8 +417,11 @@ fn scoping() {
 // Tests for dicts and lists ----------------------------------------------------------------------
 
 #[test]
-// Test for collection (list, dict) parsing
-fn collections() {
+// Test for parsing inline-sequences, which may result in lists or dicts.
+fn inline_sequences() {
+    // Inline alternation
+    assert_eq!(run("('a' | 'b' | 'c')", "b"), Ok(Some(value!("b"))));
+
     // Lists
     assert_eq!(
         run(

diff --git a/src/value/list.rs b/src/value/list.rs
@@ -31,12 +31,11 @@ impl Object for List {
             ret.push_str(&item.borrow().repr());
         }
 
-        ret.push(')');
-
-        if self.len() <= 1 {
-            ret = format!("list{}", ret);
+        if self.len() == 1 {
+            ret.push(',');
         }
 
+        ret.push(')');
         ret
     }
 
@@ -283,23 +282,23 @@ fn test_list_len() {
 #[test]
 fn test_list_iadd() {
     assert_eq!(
-        crate::run("l = list(1); l += 2; l += (3, 4); l", ""),
+        crate::run("l = (1,); l += 2; l += (3, 4); l", ""),
         Ok(Some(crate::value!([1, 2, 3, 4])))
     )
 }
 
 #[test]
 fn test_list_add() {
     assert_eq!(
-        crate::run("l = list(1); l + (2, 3) l", ""),
+        crate::run("l = (1,); l + (2, 3) l", ""),
         Ok(Some(crate::value!([[1, 2, 3], [1]])))
     )
 }
 
 #[test]
 fn test_list_push() {
     assert_eq!(
-        crate::run("l = list(1); l.push(2); l.push((3, 4)); l", ""),
+        crate::run("l = (1,); l.push(2); l.push((3, 4)); l", ""),
         Ok(Some(crate::value!([1, 2, [3, 4]])))
     );
 
@@ -345,8 +344,8 @@ fn test_list_repr() {
     */
 
     assert_eq!(
-        crate::run("l = list(); l += 1; repr(l)", ""),
-        Ok(Some(crate::value!("list(1)")))
+        crate::run("l = (); l += 1; repr(l)", ""),
+        Ok(Some(crate::value!("(1,)")))
     );
 
     assert_eq!(

diff --git a/src/vm/op.rs b/src/vm/op.rs
@@ -2,7 +2,7 @@ use super::*;
 use crate::error::Error;
 use crate::reader::Offset;
 use crate::value;
-use crate::value::{Dict, Object, Str, Value};
+use crate::value::{Dict, List, Object, Str, Value};
 use std::io;
 use std::io::prelude::*;
 use std::rc::Rc;
@@ -99,7 +99,8 @@ pub enum Op {
     StoreIndexHold,
 
     MakeAlias,       // Make key-value-Capture from last two stack items
-    MakeDict(usize), // Make a Dict from specified amount of key-value-pairs
+    MakeList(usize), // Make a List from specified amount of items on stack
+    MakeDict(usize), // Make a Dict from specified amount of key-value-pairs on the stack
 
     // Operations
     Drop,  // drop TOS
@@ -626,6 +627,16 @@ impl Op {
                     Ok(Accept::Next)
                 }
 
+                Op::MakeList(count) => {
+                    let mut list = List::new();
+
+                    for _ in 0..*count {
+                        list.insert(0, context.pop());
+                    }
+
+                    context.push(RefValue::from(list))
+                }
+
                 Op::MakeDict(count) => {
                     let mut dict = Dict::new();