convert PRIORITY_REDUCE_LISTS into a context flag

ir/trace.c

@@ -1215,7 +1215,8 @@ unsigned int trace_reduce(tcell_t *entry, cell_t **cp) {
       bool has_lists = false;
       FORLIST(a, *p, true) { // ***
         collapse_row(a);
-        response rsp = WITH(x, &CTX(any), x->priority = PRIORITY_MAX,
+        response rsp = WITH(x, &CTX(any),
+                            x->priority = PRIORITY_TOP,
                             reduce(a, x));
         if(rsp != SUCCESS) goto loop_start; // ***
         if(is_value(*a) &&
@@ -1231,7 +1232,10 @@ unsigned int trace_reduce(tcell_t *entry, cell_t **cp) {
       }
       if(has_lists && !entry->entry.parent) {
         LOG("reducing outer lists in %C", *p);
-        if(WITH(x, &CTX(return), x->priority = PRIORITY_REDUCE_LISTS, func_list(p, x)) != SUCCESS) goto loop_start;
+        if(WITH(x, &CTX(return),
+                x->priority = PRIORITY_TOP,
+                x->flags |= CONTEXT_REDUCE_LISTS,
+                func_list(p, x)) != SUCCESS) goto loop_start;
       }
       int x = trace_return(entry, *p);
       tcell_t *r = &entry[x];

list.c

@@ -100,6 +100,7 @@ response func_list(cell_t **cp, context_t *ctx) {
 
     // commit - force everything in this branch ***
     ctx->priority = PRIORITY_TOP;
+    ctx->flags &= ~CONTEXT_REDUCE_LISTS;
     c->priority = 0;
   }
 
@@ -112,6 +113,7 @@ response func_list(cell_t **cp, context_t *ctx) {
     context_t *arg_ctx = row && i == n - 1 ?
       &CTX(list, 0, 0) :
       &CTX(any);
+    if(ctx->t != T_RETURN) arg_ctx->flags &= ~CONTEXT_REDUCE_LISTS;
     if(c->value.ptr[i] && !closure_is_ready(c->value.ptr[i])) continue;
     response r = reduce_ptr(c, i, arg_ctx);
     if(ctx->depth > 0 && r == FAIL) {

primitive/core.c

@@ -145,7 +145,9 @@ OP(seq) {
   cell_t *q = NULL;
   bool q_var = false;
   int pos = c->pos;
-  CHECK(WITH(x, &CTX(any), x->flags |= CONTEXT_SEQ, reduce_arg(c, 1, x))); // don't split arg here?
+  CHECK(WITH(x, &CTX(any),
+             x->flags |= CONTEXT_SEQ,
+             reduce_arg(c, 1, x))); // don't split arg here?
   if(rsp == SUCCESS) {
     q = c->expr.arg[1];
     q_var = is_var(q);
@@ -227,7 +229,9 @@ OP(unless) {
   cell_t **p = &c->expr.arg[0];
   cell_t **q = &c->expr.arg[1];
   while(*q) {
-    response rsp0 = WITH(x, &CTX(any), x->flags ^= CONTEXT_INV, reduce(q, x));
+    response rsp0 = WITH(x, &CTX(any),
+                         x->flags ^= CONTEXT_INV,
+                         reduce(q, x));
     CHECK_IF(rsp0 == RETRY, RETRY);
     if(rsp0 == DELAY) {
       rsp = DELAY;
@@ -409,9 +413,11 @@ response func_compose_ap(cell_t **cp, context_t *ctx, bool row) {
     bs = compose_size_b(arg_in, as, cs);
   }
 
-  CHECK(reduce_arg(c, in, &CTX(list,
-                               out ? arg_in : bs.in, // only account for known inputs when there are outputs
-                               bs.out)));
+  CHECK(WITH(x, &CTX(list,
+                     out ? arg_in : bs.in, // only account for known inputs when there are outputs
+                     bs.out),
+             x->flags &= ~CONTEXT_REDUCE_LISTS,
+             reduce_arg(c, in, x)));
   bs = quote_size(c->expr.arg[in], false);
   as = row ? compose_size_a(arg_in, bs, cs) : (qsize_t) {0, 0};
 

rt.c

@@ -360,7 +360,7 @@ response reduce(cell_t **cp, context_t *ctx) {
     }
 
     if(r == SUCCESS &&
-       ctx->priority == PRIORITY_REDUCE_LISTS &&
+       ctx->flags & CONTEXT_REDUCE_LISTS &&
        is_list(*cp) &&
        closure_is_ready(*leftmost(cp))) {
       r = func_list(cp, ctx);
@@ -387,7 +387,9 @@ response reduce(cell_t **cp, context_t *ctx) {
 
 response simplify_2(cell_t **cp, context_t *ctx) {
   CONTEXT("simplify %C", *cp);
-  return WITH(x, ctx, x->priority = PRIORITY_SIMPLIFY,
+  return WITH(x, ctx,
+              x->flags &= ~CONTEXT_REDUCE_LISTS,
+              x->priority = PRIORITY_SIMPLIFY,
               reduce(cp, x));
 }
 

rt_types.h

@@ -106,8 +106,7 @@ typedef enum __attribute__((packed)) priority {
   PRIORITY_DELAY,
   PRIORITY_EXEC_SELF,
   PRIORITY_UNLESS,
-  PRIORITY_MAX,
-  PRIORITY_REDUCE_LISTS
+  PRIORITY_MAX
 } priority_t;
 #define PRIORITY_TOP (PRIORITY_MAX - 1)
 static_assert(sizeof(priority_t) == 1, "priority_t too big");
@@ -128,10 +127,11 @@ struct context {
   uint8_t flags; // see below [up/down]
 };
 
-#define CONTEXT_RETRY 0x01
-#define CONTEXT_INV   0x02
-#define CONTEXT_SEQ   0x04
-#define CONTEXT_DOWN  (CONTEXT_INV | CONTEXT_SEQ)
+#define CONTEXT_RETRY         0x01
+#define CONTEXT_INV           0x02
+#define CONTEXT_SEQ           0x04
+#define CONTEXT_REDUCE_LISTS  0x08
+#define CONTEXT_DOWN  (CONTEXT_INV | CONTEXT_SEQ | CONTEXT_REDUCE_LISTS)
 
 typedef enum response {
   SUCCESS = 0, // continue reduction

user_func.c

@@ -1010,7 +1010,9 @@ response func_exec_trace(cell_t **cp, context_t *ctx) {
       context_t arg_ctx = t == T_OPAQUE ?
         CTX(opaque, in_opaque[i]) :
         CTX(t, t);
-      CHECK_IF(WITH(x, &arg_ctx, x->priority = PRIORITY_ASSERT - 1,
+      CHECK_IF(WITH(x, &arg_ctx,
+                    x->flags &= ~CONTEXT_REDUCE_LISTS,
+                    x->priority = PRIORITY_ASSERT - 1,
                     reduce(&c->expr.arg[i], x)) == FAIL, FAIL);
     }
 
@@ -1031,7 +1033,9 @@ response func_exec_trace(cell_t **cp, context_t *ctx) {
     if(is_list(*ap) &&
        closure_is_ready(left = *leftmost(ap))) {
       LOG(HACK " forced cells[%C].expr.arg[%d]", c, i);
-      CHECK(WITH(x, &CTX(return), x->priority = PRIORITY_TOP, func_list(ap, x)));
+      CHECK(WITH(x, &CTX(return),
+                 x->priority = PRIORITY_TOP,
+                 func_list(ap, x)));
       CHECK_DELAY();
 
       // ensure quotes are stored first
@@ -1273,6 +1277,7 @@ OP(exec) {
       } else if(FLAG(*entry, entry, RECURSIVE)) {
         CHECK_PRIORITY(PRIORITY_EXEC_SELF);
         if(c->pos && c->pos <= trace_current_entry()->pos) { // lift recursion out of loop
+          assert_error(!(ctx->flags & CONTEXT_REDUCE_LISTS));
           assert_error(FLAG(*entry, entry, ROW)); // TODO
           csize_t in = closure_in(c);
           int n = 0;