Feat/everything is an expr

CHANGELOG.md

@@ -15,6 +15,10 @@
 - display a warning to `stderr` when using a deprecated function/value (checks for `@deprecated` inside the attached comment of functions / values)
 
 ### Changed
+- `pop!` can return the removed value
+- `@=` and `@@=` return the inserted value
+- `append!` and `concat!` return the modified list
+- `let`, `mut` and `set` can return the assigned value
 
 ### Removed
 

docs/arkdoc/List.txt

@@ -35,7 +35,7 @@
 
 --#
 * @name append!
-* @brief Add an element to a list, modifying it in place. It doesn't return anything
+* @brief Add an element to a list, modifying it in place. Return the modified list
 * @param lst a list. Must be mutable
 * @param element
 * =begin
@@ -60,7 +60,7 @@
 
 --#
 * @name concat!
-* @brief Concatenate two lists in place, modifying the first one it in place. It doesn't return anything
+* @brief Concatenate two lists in place, modifying the first one it in place. Return the modified list
 * @param lst a list. Must be mutable
 * @param more another list
 * =begin
@@ -87,7 +87,7 @@
 
 --#
 * @name pop!
-* @brief Remove an element from a list in place, given its index. It doesn't return anything
+* @brief Remove an element from a list in place, given its index. Return the removed element
 * @details Supports negative indices, -1 being the end.
 * @param lst a list. Must be mutable
 * @param index number
@@ -157,3 +157,25 @@
 * (print (@@ ["abc" "def" "ghi"] 0 -1))  # c
 * =end
 #--
+
+--#
+* @name @=
+* @brief Set an element in a list, in place
+* @details Return the newly added element
+* @param lst list
+* @param index number (can be negative to start from the end)
+* @param x value
+* =begin
+* (mut lst [1 2 3 4 5])
+* (print (@= lst 0 "x"))  # "x"
+* (print lst)  # ["x" 2 3 4 5]
+* (print (@= lst 1 "y"))  # "y"
+* (print lst)  # ["x" "y" 3 4 5]
+* (print (@= lst 2 "z"))  # "z"
+* (print lst)  # ["x" "y" "z" 4 5]
+* (@= lst -1 "f")
+* (print lst)  # ["x" "y" "z" 4 "f"]
+* (@= lst -2 "g")
+* (print lst)  # ["x" "y" "z" "g" "f"]
+* =end
+#--

docs/arkdoc/String.txt

@@ -77,3 +77,25 @@
 * (print (@ "abc" -2))  # "b"
 * =end
 #--
+
+--#
+* @name @=
+* @brief Set a character in a string, in place
+* @details Return the newly added character
+* @param str string
+* @param index number (can be negative to start from the end)
+* @param char character
+* =begin
+* (mut str "abcde")
+* (print (@= str 0 "x"))  # "x"
+* (print str)  # "xbcde"
+* (print (@= str 1 "y"))  # "y"
+* (print str)  # "xycde"
+* (print (@= str 2 "z"))  # "z"
+* (print str)  # "xyzde"
+* (@= str -1 "f")
+* (print str)  # "xyzdf"
+* (@= str -2 "g")
+* (print str)  # "xyzgf"
+* =end
+#--

include/Ark/Compiler/Lowerer/ASTLowerer.hpp

@@ -257,7 +257,7 @@ namespace Ark::internal
         void compileApplyInstruction(Node& x, Page p, bool is_result_unused);
         void compileIf(Node& x, Page p, bool is_result_unused, bool is_terminal);
         void compileFunction(Node& x, Page p, bool is_result_unused);
-        void compileLetMutSet(Keyword n, Node& x, Page p);
+        void compileLetMutSet(Keyword n, Node& x, Page p, bool is_result_unused);
         void compileWhile(Node& x, Page p);
         void compilePluginImport(const Node& x, Page p);
         void pushFunctionCallArguments(Node& call, Page p, bool is_tail_call);

src/arkreactor/Compiler/BytecodeReader.cpp

@@ -513,6 +513,10 @@ namespace Ark
             { CONCAT, ArgKind::Raw },
             { APPEND_IN_PLACE, ArgKind::Raw },
             { CONCAT_IN_PLACE, ArgKind::Raw },
+            { POP_LIST, ArgKind::Raw },
+            { POP_LIST_IN_PLACE, ArgKind::Raw },
+            { SET_AT_INDEX, ArgKind::Raw },
+            { SET_AT_2_INDEX, ArgKind::Raw },
             { RESET_SCOPE_JUMP, ArgKind::Raw },
             { GET_CURRENT_PAGE_ADDR, ArgKind::Symbol },
             { LOAD_CONST_LOAD_CONST, ArgKind::ConstConst },

src/arkreactor/Compiler/Lowerer/ASTLowerer.cpp

@@ -37,9 +37,11 @@ namespace Ark::internal
         // gather symbols, values, and start to create code segments
         compileExpression(
             ast,
-            /* current_page */ global,
-            /* is_result_unused */ false,
-            /* is_terminal */ false);
+            /* current_page= */ global,
+            // the offset is non-zero when coming from the debugger, and setting is_result_unused to
+            // true will avoid adding a LOAD_FAST/LOAD_FAST_FROM_INDEX after a let/mut/set
+            /* is_result_unused= */ m_start_page_at_offset != 0,
+            /* is_terminal= */ false);
         m_logger.traceEnd();
     }
 
@@ -95,18 +97,24 @@ namespace Ark::internal
     bool ASTLowerer::nodeProducesOutput(const Node& node)
     {
         if (node.nodeType() == NodeType::List && !node.constList().empty() && node.constList()[0].nodeType() == NodeType::Keyword)
-            // a begin node produces a value if the last node in it produces a value
+            // a 'begin' node produces a value if the last node in it produces a value
             return (node.constList()[0].keyword() == Keyword::Begin && node.constList().size() > 1 && nodeProducesOutput(node.constList().back())) ||
                 // a function always produces a value ; even if it ends with a node not producing one, the VM returns nil
                 node.constList()[0].keyword() == Keyword::Fun ||
+                // a let/mut/set pushes the value that was assigned
+                node.constList()[0].keyword() == Keyword::Let ||
+                node.constList()[0].keyword() == Keyword::Mut ||
+                node.constList()[0].keyword() == Keyword::Set ||
                 // a condition produces a value if all its branches produce a value
                 (node.constList()[0].keyword() == Keyword::If &&
                  nodeProducesOutput(node.constList()[2]) &&
                  (node.constList().size() == 3 || nodeProducesOutput(node.constList()[3])));
-        // in place list instruction, as well as breakpoint, do not produce values
+        // breakpoint do not produce values
         if (node.nodeType() == NodeType::List && !node.constList().empty() && node.constList()[0].nodeType() == NodeType::Symbol)
-            return std::ranges::find(Language::UpdateRef, node.constList().front().string()) == Language::UpdateRef.end() &&
-                node.constList().front().string() != "breakpoint";
+        {
+            const std::string& name = node.constList().front().string();
+            return name != "breakpoint";
+        }
         return true;  // any other node, function call, symbol, number...
     }
 
@@ -248,7 +256,7 @@ namespace Ark::internal
                     case Keyword::Let:
                         [[fallthrough]];
                     case Keyword::Mut:
-                        compileLetMutSet(keyword, x, p);
+                        compileLetMutSet(keyword, x, p, is_result_unused);
                         break;
 
                     case Keyword::Fun:
@@ -332,15 +340,15 @@ namespace Ark::internal
     void ASTLowerer::compileListInstruction(Node& x, const Page p, const bool is_result_unused)
     {
         const Node head = x.constList()[0];
-        std::string name = x.constList()[0].string();
-        Instruction inst = getListInstruction(name).value();
+        const std::string& name = head.string();
+        const Instruction inst = getListInstruction(name).value();
 
         // length of at least 1 since we got a symbol name
         const auto argc = x.constList().size() - 1u;
         // error, can not use append/concat/pop (and their in place versions) with a <2 length argument list
-        if (argc < 2 && APPEND <= inst && inst <= POP)
+        if (argc < 2 && APPEND <= inst && inst <= SET_AT_2_INDEX)
             buildAndThrowError(fmt::format("Can not use {} with less than 2 arguments", name), head);
-        if (inst <= POP && std::cmp_greater(argc, MaxValue16Bits))
+        if (std::cmp_greater(argc, MaxValue16Bits))
             buildAndThrowError(fmt::format("Too many arguments ({}), exceeds {}", argc, MaxValue16Bits), x);
         if (argc != 3 && inst == SET_AT_INDEX)
             buildAndThrowError(fmt::format("Expected 3 arguments (list, index, value) for {}, got {}", name, argc), head);
@@ -366,24 +374,42 @@ namespace Ark::internal
                 break;
 
             case APPEND:
+                [[fallthrough]];
             case APPEND_IN_PLACE:
+                [[fallthrough]];
             case CONCAT:
+                [[fallthrough]];
             case CONCAT_IN_PLACE:
                 inst_argc = argc - 1;
                 break;
 
             case POP_LIST:
-            case POP_LIST_IN_PLACE:
                 inst_argc = 0;
                 break;
 
+            case SET_AT_INDEX:
+                [[fallthrough]];
+            case SET_AT_2_INDEX:
+                [[fallthrough]];
+            case POP_LIST_IN_PLACE:
+                inst_argc = is_result_unused ? 0 : 1;
+                break;
+
             default:
                 break;
         }
         page(p).emplace_back(inst, static_cast<uint16_t>(inst_argc));
         page(p).back().setSourceLocation(head.filename(), head.position().start.line);
 
-        if (is_result_unused && name.back() != '!' && inst <= POP_LIST_IN_PLACE)  // in-place functions never push a value
+        if (!is_result_unused && (inst == APPEND_IN_PLACE || inst == CONCAT_IN_PLACE))
+        {
+            // Load the first argument which should be a symbol (or field),
+            // that append!/concat! write to, so that we have its new value available.
+            compileExpression(x.list()[1], p, false, false);
+        }
+
+        // append!, concat!, pop!, @= and @@= can push to the stack, but not using its returned value isn't an error
+        if (is_result_unused && (inst == LIST || inst == APPEND || inst == CONCAT || inst == POP_LIST))
         {
             warning("Ignoring return value of function", x);
             page(p).emplace_back(POP);
@@ -535,7 +561,7 @@ namespace Ark::internal
         }
     }
 
-    void ASTLowerer::compileLetMutSet(const Keyword n, Node& x, const Page p)
+    void ASTLowerer::compileLetMutSet(const Keyword n, Node& x, const Page p, const bool is_result_unused)
     {
         if (const auto sym = x.constList()[1]; sym.nodeType() != NodeType::Symbol)
             buildAndThrowError(fmt::format("Expected a symbol, got a {}", typeToString(sym)), sym);
@@ -557,17 +583,24 @@ namespace Ark::internal
 
         // put value before symbol id
         // starting at index = 2 because x is a (let|mut|set variable ...) node
-        for (std::size_t idx = 2, end = x.constList().size(); idx < end; ++idx)
-            compileExpression(x.list()[idx], p, false, false);
+        compileExpression(x.list()[2], p, false, false);
 
         if (n == Keyword::Let || n == Keyword::Mut)
         {
             page(p).emplace_back(STORE, i);
             m_locals_locator.addLocal(name);
+
+            if (!is_result_unused)
+                page(p).emplace_back(LOAD_FAST_BY_INDEX, 0);
         }
         else
+        {
             page(p).emplace_back(SET_VAL, i);
 
+            if (!is_result_unused)
+                page(p).emplace_back(LOAD_FAST, i);
+        }
+
         if (is_function)
             m_opened_vars.pop();
         page(p).back().setSourceLocation(x.filename(), x.position().start.line);

src/arkreactor/VM/Debugger.cpp

@@ -179,7 +179,7 @@ namespace Ark::internal
         if (limit > 0 && count > 0)
         {
             fmt::println(m_os, "scope size: {}", limit);
-            fmt::println(m_os, "index |  id |    type   | value");
+            fmt::println(m_os, "index |  id |      name      |    type   | value");
             std::size_t i = 0;
 
             do
@@ -192,9 +192,10 @@ namespace Ark::internal
 
                 fmt::println(
                     m_os,
-                    "{:>5} | {:3} | {:>9} | {}",
+                    "{:>5} | {:3} | {:14} | {:>9} | {}",
                     fmt::styled(limit - i - 1, color),
                     fmt::styled(id, color),
+                    fmt::styled(vm.m_state.m_symbols[id], color),
                     fmt::styled(std::to_string(value.valueType()), color),
                     fmt::styled(value.toString(vm, /* show_as_code= */ true), color));
                 ++i;
@@ -257,7 +258,7 @@ namespace Ark::internal
                 m_os,
                 "dbg[{},{}]:{:0>3}{} ",
                 fmt::format("pp:{}", fmt::styled(pp, m_colorize ? fmt::fg(fmt::color::green) : fmt::text_style())),
-                fmt::format("ip:{}", fmt::styled(ip, m_colorize ? fmt::fg(fmt::color::cyan) : fmt::text_style())),
+                fmt::format("ip:{}", fmt::styled(ip / 4, m_colorize ? fmt::fg(fmt::color::cyan) : fmt::text_style())),
                 m_line_count,
                 unfinished_block ? ":" : ">");
 

src/arkreactor/VM/VM.cpp

@@ -920,7 +920,14 @@ namespace Ark
                                     ErrorKind::Index,
                                     fmt::format("pop! index ({}) out of range (list size: {})", idx, list->list().size()));
 
+                            // Save the value we're removing to push it later.
+                            // We need to save the value and push later because we're using a pointer to 'list', and pushing before erasing
+                            // would overwrite values from the stack.
+                            if (arg)
+                                number = list->list()[static_cast<std::size_t>(idx)];
                             list->list().erase(list->list().begin() + idx);
+                            if (arg)
+                                push(number, context);
                         }
                         DISPATCH();
                     }
@@ -954,9 +961,17 @@ namespace Ark
                                     fmt::format("@= index ({}) out of range (indexable size: {})", idx, size));
 
                             if (list->valueType() == ValueType::List)
+                            {
                                 list->list()[static_cast<std::size_t>(idx)] = new_value;
+                                if (arg)
+                                    push(new_value, context);
+                            }
                             else
+                            {
                                 list->stringRef()[static_cast<std::size_t>(idx)] = new_value.string()[0];
+                                if (arg)
+                                    push(Value(new_value.string()[0]), context);
+                            }
                         }
                         DISPATCH();
                     }
@@ -1016,9 +1031,17 @@ namespace Ark
                                     fmt::format("@@= index (x: {}) out of range (inner indexable size: {})", idx_x, size));
 
                             if (is_list)
+                            {
                                 list->list()[static_cast<std::size_t>(idx_y)].list()[static_cast<std::size_t>(idx_x)] = new_value;
+                                if (arg)
+                                    push(new_value, context);
+                            }
                             else
+                            {
                                 list->list()[static_cast<std::size_t>(idx_y)].stringRef()[static_cast<std::size_t>(idx_x)] = new_value.string()[0];
+                                if (arg)
+                                    push(Value(new_value.string()[0]), context);
+                            }
                         }
                         DISPATCH();
                     }

tests/benchmarks/compare.py

@@ -127,24 +127,26 @@ def main(files: List[str]):
             runs_by_name[run.name].append(run)
 
     data = []
-    times = colorize_time_type("real_time", 0) + "\n" + colorize_time_type("cpu_time", 1)
 
-    for (name, runs) in runs_by_name.items():
+    for i, (name, runs) in enumerate(runs_by_name.items()):
         baseline: Run = runs[0]
         baseline_index = headers.index(baseline.benchmark) - 2
         diffs = [r.diff_from_baseline(baseline) for r in runs[1:]]
 
         padding = ["" for _ in range(baseline_index)] if baseline_index > 0 else []
         data.append(
-            [name, times] +
+            [
+                colorize_time_type(name, i % 2),
+                colorize_time_type("real_time", i % 2) + "\n" + colorize_time_type("cpu_time", i % 2)
+            ] +
             padding +
             [
-                colorize_time_type(f"{baseline.real_time:.3f}{baseline.time_unit}", 0) + "\n" +
-                colorize_time_type(f"{baseline.cpu_time:.3f}{baseline.time_unit}", 1)
+                colorize_time_type(f"{baseline.real_time:.3f}{baseline.time_unit}", i % 2) + "\n" +
+                colorize_time_type(f"{baseline.cpu_time:.3f}{baseline.time_unit}", i % 2)
             ] +
             [
-                f"{colorize_time_type(diff.dt_real_time, 0)} ({colorize_diff(diff.dt_rt_percent)}%)\n" +
-                f"{colorize_time_type(diff.dt_cpu_time, 1)} ({colorize_diff(diff.dt_ct_percent)}%)"
+                f"{colorize_time_type(diff.dt_real_time, i % 2)} ({colorize_diff(diff.dt_rt_percent)}%)\n" +
+                f"{colorize_time_type(diff.dt_cpu_time, i % 2)} ({colorize_diff(diff.dt_ct_percent)}%)"
                 for
                 diff in diffs
             ]

tests/benchmarks/results/013-74e64c67.csv

@@ -0,0 +1,11 @@
+name,iterations,real_time,cpu_time,time_unit,bytes_per_second,items_per_second,label,error_occurred,error_message
+"quicksort",7431,0.0933847,0.0930486,ms,,,,,
+"ackermann/iterations:50",50,32.3309,32.1861,ms,,,,,
+"fibonacci/iterations:100",100,3.16125,3.14818,ms,,,,,
+"builtins",1205,0.581715,0.580722,ms,,,,,
+"binary_trees",1,855.337,853.891,ms,,,,,
+"for_sum",8,88.2647,88.0911,ms,,,,,
+"create_closure/iterations:500",500,0.771819,0.770728,ms,,,,,
+"create_list/iterations:500",500,1.7033,1.70064,ms,,,,,
+"create_list_with_ref/iterations:500",500,1.46902,1.46652,ms,,,,,
+"n_queens/iterations:50",50,13.0718,13.0555,ms,,,,,