Refactor handling of named flattening args.

They now count as part of the named arguments section of a callsite,
and can appear between named args. This will allow us to take order
into account when arguments are flattened in, then overridden with
another value of the same name.

src/core/args.c

@@ -22,7 +22,7 @@ void MVM_args_proc_init(MVMThreadContext *tc, MVMArgProcContext *ctx, MVMCallsit
     /* Stash callsite and argument counts/pointers. */
     ctx->callsite = callsite;
     /* initial counts and values; can be altered by flatteners */
-    init_named_used(tc, ctx, (callsite->arg_count - callsite->num_pos) / 2);
+    init_named_used(tc, ctx, MVM_callsite_num_nameds(tc, callsite));
     ctx->args     = args;
     ctx->num_pos  = callsite->num_pos;
     ctx->arg_count = callsite->arg_count;
@@ -62,12 +62,13 @@ void MVM_args_proc_cleanup(MVMThreadContext *tc, MVMArgProcContext *ctx) {
 MVMCallsite * MVM_args_proc_to_callsite(MVMThreadContext *tc, MVMArgProcContext *ctx) {
     if (ctx->arg_flags) {
         MVMCallsite      *res   = MVM_malloc(sizeof(MVMCallsite));
-        MVMint32          fsize = ctx->num_pos + (ctx->arg_count - ctx->num_pos) / 2;
+        MVMint32          fsize = ctx->flag_count;
         MVMCallsiteEntry *flags = NULL;
         if (fsize) {
             flags = MVM_malloc(fsize);
             memcpy(flags, ctx->arg_flags, fsize);
         }
+        res->flag_count = fsize;
         res->arg_flags = flags;
         res->arg_count = ctx->arg_count;
         res->num_pos   = ctx->num_pos;
@@ -734,7 +735,7 @@ static void flatten_args(MVMThreadContext *tc, MVMArgProcContext *ctx) {
                 }
             }
         }
-        else if (!(arg_info.flags & MVM_CALLSITE_ARG_FLAT_NAMED)) {
+        else {
             if (new_arg_pos == new_args_size) {
                 new_args = MVM_realloc(new_args, (new_args_size *= 2) * sizeof(MVMRegister));
             }
@@ -749,28 +750,40 @@ static void flatten_args(MVMThreadContext *tc, MVMArgProcContext *ctx) {
     new_num_pos = new_arg_pos;
 
     /* then append any nameds from the original */
-    for ( flag_pos = arg_pos; arg_pos < ctx->arg_count; flag_pos++, arg_pos += 2) {
-
-        if (new_arg_pos + 1 >=  new_args_size) {
-            new_args = MVM_realloc(new_args, (new_args_size *= 2) * sizeof(MVMRegister));
-        }
-        if (new_flag_pos == new_arg_flags_size) {
-            new_arg_flags = MVM_realloc(new_arg_flags, (new_arg_flags_size *= 2) * sizeof(MVMCallsiteEntry));
+    flag_pos = arg_pos;
+    while (arg_pos < ctx->arg_count) {
+        if (ctx->callsite->arg_flags[flag_pos] & MVM_CALLSITE_ARG_FLAT_NAMED) {
+            flag_pos++;
+            arg_pos++;
         }
+        else {
+            if (new_arg_pos + 1 >= new_args_size) {
+                new_args = MVM_realloc(new_args, (new_args_size *= 2) * sizeof(MVMRegister));
+            }
+            if (new_flag_pos == new_arg_flags_size) {
+                new_arg_flags = MVM_realloc(new_arg_flags, (new_arg_flags_size *= 2) * sizeof(MVMCallsiteEntry));
+            }
+
+            (new_args + new_arg_pos++)->s = (ctx->args + arg_pos)->s;
+            *(new_args + new_arg_pos++) = *(ctx->args + arg_pos + 1);
+            new_arg_flags[new_flag_pos++] = ctx->callsite->arg_flags[flag_pos];
 
-        (new_args + new_arg_pos++)->s = (ctx->args + arg_pos)->s;
-        *(new_args + new_arg_pos++) = *(ctx->args + arg_pos + 1);
-        new_arg_flags[new_flag_pos++] = ctx->callsite->arg_flags[flag_pos];
+            flag_pos++;
+            arg_pos += 2;
+        }
     }
 
     /* now flatten any flattening hashes */
-    for (arg_pos = 0; arg_pos < ctx->num_pos; arg_pos++) {
+    arg_pos = flag_pos = ctx->num_pos;
+    while (arg_pos < ctx->arg_count) {
+        arg_info.flags = ctx->callsite->arg_flags[flag_pos];
+        if (!(arg_info.flags & MVM_CALLSITE_ARG_FLAT_NAMED)) {
+            arg_pos += 2;
+            flag_pos++;
+            continue;
+        }
 
         arg_info.arg = ctx->args[arg_pos];
-        arg_info.flags = ctx->callsite->arg_flags[arg_pos];
-
-        if (!(arg_info.flags & MVM_CALLSITE_ARG_FLAT_NAMED))
-            continue;
 
         if (arg_info.arg.o && REPR(arg_info.arg.o)->ID == MVM_REPR_ID_MVMHash) {
             MVMHashBody *body = &((MVMHash *)arg_info.arg.o)->body;
@@ -794,13 +807,17 @@ static void flatten_args(MVMThreadContext *tc, MVMArgProcContext *ctx) {
         else if (arg_info.arg.o) {
             MVM_exception_throw_adhoc(tc, "flattening of other hash reprs NYI.");
         }
+
+        arg_pos++;
+        flag_pos++;
     }
 
     init_named_used(tc, ctx, (new_arg_pos - new_num_pos) / 2);
     ctx->args = new_args;
     ctx->arg_count = new_arg_pos;
     ctx->num_pos = new_num_pos;
     ctx->arg_flags = new_arg_flags;
+    ctx->flag_count = new_flag_pos;
 }
 
 /* Does the common setup work when we jump the interpreter into a chosen

src/core/args.h

@@ -3,7 +3,8 @@ struct MVMArgProcContext {
     /* The callsite we're processing. */
     MVMCallsite *callsite;
 
-    /* The set of flags. */
+    /* The set of flags (only set if we flattened, otherwise we use the ones
+     * from callsite). */
     MVMCallsiteEntry *arg_flags;
 
     /* The arguments. */
@@ -21,6 +22,9 @@ struct MVMArgProcContext {
 
     /* Number of positionals. */
     MVMuint16 num_pos;
+
+    /* The number of arg flags; only valid if arg_flags isn't NULL. */
+    MVMuint16 flag_count;
 };
 
 /* Expected return type flags. */

src/core/bytecode.c

@@ -752,7 +752,7 @@ MVMuint8 MVM_bytecode_find_static_lexical_scref(MVMThreadContext *tc, MVMCompUni
 static MVMCallsite ** deserialize_callsites(MVMThreadContext *tc, MVMCompUnit *cu, ReaderState *rs) {
     MVMCallsite **callsites;
     MVMuint8     *pos;
-    MVMuint32     i, j, elems, positionals, nameds;
+    MVMuint32     i, j, elems;
     MVMCompUnitBody *cu_body = &cu->body;
 
     /* Allocate space for callsites. */
@@ -764,8 +764,9 @@ static MVMCallsite ** deserialize_callsites(MVMThreadContext *tc, MVMCompUnit *c
     pos = rs->callsite_seg;
     for (i = 0; i < rs->expected_callsites; i++) {
         MVMuint8 has_flattening = 0;
-        positionals = 0;
-        nameds = 0;
+        MVMuint32 positionals = 0;
+        MVMuint32 nameds_slots = 0;
+        MVMuint32 nameds_non_flattening = 0;
 
         /* Ensure we can read at least an element count. */
         ensure_can_read(tc, cu, rs, pos, 2);
@@ -774,6 +775,7 @@ static MVMCallsite ** deserialize_callsites(MVMThreadContext *tc, MVMCompUnit *c
 
         /* Allocate space for the callsite. */
         callsites[i] = MVM_malloc(sizeof(MVMCallsite));
+        callsites[i]->flag_count = elems;
         if (elems)
             callsites[i]->arg_flags = MVM_malloc(elems);
 
@@ -786,37 +788,44 @@ static MVMCallsite ** deserialize_callsites(MVMThreadContext *tc, MVMCompUnit *c
         /* Add alignment. */
         pos += elems % 2;
 
-        /* Count positional arguments. */
-        /* Validate that all positionals come before all nameds. */
+        /* Count positional arguments, and validate that all positionals come
+         * before all nameds (flattening named counts as named). */
         for (j = 0; j < elems; j++) {
-            if (callsites[i]->arg_flags[j] & (MVM_CALLSITE_ARG_FLAT | MVM_CALLSITE_ARG_FLAT_NAMED)) {
-                if (!(callsites[i]->arg_flags[j] & MVM_CALLSITE_ARG_OBJ)) {
-                    MVM_exception_throw_adhoc(tc, "Flattened args must be objects");
-                }
-                if (nameds) {
-                    MVM_exception_throw_adhoc(tc, "All positional args must appear first");
-                }
+            if (callsites[i]->arg_flags[j] & MVM_CALLSITE_ARG_FLAT) {
+                if (!(callsites[i]->arg_flags[j] & MVM_CALLSITE_ARG_OBJ))
+                    MVM_exception_throw_adhoc(tc, "Flattened positional args must be objects");
+                if (nameds_slots)
+                    MVM_exception_throw_adhoc(tc, "Flattened positional args must appear before named args");
                 has_flattening = 1;
                 positionals++;
             }
+            else if (callsites[i]->arg_flags[j] & MVM_CALLSITE_ARG_FLAT_NAMED) {
+                if (!(callsites[i]->arg_flags[j] & MVM_CALLSITE_ARG_OBJ))
+                    MVM_exception_throw_adhoc(tc, "Flattened named args must be objects");
+                has_flattening = 1;
+                nameds_slots++;
+            }
             else if (callsites[i]->arg_flags[j] & MVM_CALLSITE_ARG_NAMED) {
-                nameds += 2;
+                nameds_slots += 2;
+                nameds_non_flattening++;
             }
-            else if (nameds) { /* positional appearing after a named one */
-                MVM_exception_throw_adhoc(tc, "All positional args must appear first");
+            else if (nameds_slots) {
+                MVM_exception_throw_adhoc(tc, "All positional args must appear before named args");
+            }
+            else {
+                positionals++;
             }
-            else positionals++;
         }
         callsites[i]->num_pos        = positionals;
-        callsites[i]->arg_count      = positionals + nameds;
+        callsites[i]->arg_count      = positionals + nameds_slots;
         callsites[i]->has_flattening = has_flattening;
         callsites[i]->is_interned    = 0;
         callsites[i]->with_invocant  = NULL;
 
-        if (rs->version >= 3 && nameds) {
-            ensure_can_read(tc, cu, rs, pos, (nameds / 2) * 4);
-            callsites[i]->arg_names = MVM_malloc((nameds / 2) * sizeof(MVMString));
-            for (j = 0; j < nameds / 2; j++) {
+        if (rs->version >= 3 && nameds_non_flattening) {
+            ensure_can_read(tc, cu, rs, pos, nameds_non_flattening * 4);
+            callsites[i]->arg_names = MVM_malloc(nameds_non_flattening * sizeof(MVMString));
+            for (j = 0; j < nameds_non_flattening; j++) {
                 callsites[i]->arg_names[j] = get_heap_string(tc, cu, rs, pos, 0);
                 pos += 4;
             }
@@ -826,8 +835,8 @@ static MVMCallsite ** deserialize_callsites(MVMThreadContext *tc, MVMCompUnit *c
 
         /* Track maximum callsite size we've seen. (Used for now, though
          * in the end we probably should calculate it by frame.) */
-        if (positionals + nameds > cu_body->max_callsite_size)
-            cu_body->max_callsite_size = positionals + nameds;
+        if (callsites[i]->arg_count > cu_body->max_callsite_size)
+            cu_body->max_callsite_size = callsites[i]->arg_count;
 
         /* Try to intern the callsite (that is, see if it matches one the
          * VM already knows about). If it does, it will free the memory

src/core/callsite.c

@@ -15,40 +15,40 @@ static MVMint32 callsites_equal(MVMThreadContext *tc, MVMCallsite *cs1, MVMCalls
     return 1;
 }
 
-static MVMCallsite   null_args_callsite = { NULL, 0, 0, 0, 0, 0, 0 };
+static MVMCallsite   null_args_callsite = { NULL, 0, 0, 0, 0, 0, 0, 0 };
 
 static MVMCallsiteEntry obj_arg_flags[] = { MVM_CALLSITE_ARG_OBJ };
-static MVMCallsite     inv_arg_callsite = { obj_arg_flags, 1, 1, 0, 0, 0, 0 };
+static MVMCallsite     inv_arg_callsite = { obj_arg_flags, 1, 1, 1, 0, 0, 0, 0 };
 
 static MVMCallsiteEntry two_obj_arg_flags[] = { MVM_CALLSITE_ARG_OBJ,
                                                 MVM_CALLSITE_ARG_OBJ };
-static MVMCallsite    two_args_callsite = { two_obj_arg_flags, 2, 2, 0, 0, 0 };
+static MVMCallsite    two_args_callsite = { two_obj_arg_flags, 2, 2, 2, 0, 0, 0 };
 
 static MVMCallsiteEntry mnfe_flags[] = { MVM_CALLSITE_ARG_OBJ,
                                          MVM_CALLSITE_ARG_STR };
-static MVMCallsite     methnotfound_callsite = { mnfe_flags, 2, 2, 0 };
+static MVMCallsite     methnotfound_callsite = { mnfe_flags, 2, 2, 2, 0 };
 
 static MVMCallsiteEntry fm_flags[] = { MVM_CALLSITE_ARG_OBJ,
                                        MVM_CALLSITE_ARG_OBJ,
                                        MVM_CALLSITE_ARG_STR };
-static MVMCallsite     findmeth_callsite = { fm_flags, 3, 3, 0 };
+static MVMCallsite     findmeth_callsite = { fm_flags, 3, 3, 3, 0 };
 
 static MVMCallsiteEntry tc_flags[] = { MVM_CALLSITE_ARG_OBJ,
                                        MVM_CALLSITE_ARG_OBJ,
                                        MVM_CALLSITE_ARG_OBJ };
-static MVMCallsite     typecheck_callsite = { tc_flags, 3, 3, 0 };
+static MVMCallsite     typecheck_callsite = { tc_flags, 3, 3, 3, 0 };
 
 static MVMCallsiteEntry obj_int_flags[] = { MVM_CALLSITE_ARG_OBJ,
                                             MVM_CALLSITE_ARG_INT };
-static MVMCallsite    obj_int_callsite = { obj_int_flags, 2, 2, 0, 0, 0 };
+static MVMCallsite    obj_int_callsite = { obj_int_flags, 2, 2, 2, 0, 0, 0 };
 
 static MVMCallsiteEntry obj_num_flags[] = { MVM_CALLSITE_ARG_OBJ,
                                             MVM_CALLSITE_ARG_NUM };
-static MVMCallsite    obj_num_callsite = { obj_num_flags, 2, 2, 0, 0, 0 };
+static MVMCallsite    obj_num_callsite = { obj_num_flags, 2, 2, 2, 0, 0, 0 };
 
 static MVMCallsiteEntry obj_str_flags[] = { MVM_CALLSITE_ARG_OBJ,
                                             MVM_CALLSITE_ARG_STR };
-static MVMCallsite    obj_str_callsite = { obj_str_flags, 2, 2, 0, 0, 0 };
+static MVMCallsite    obj_str_callsite = { obj_str_flags, 2, 2, 2, 0, 0, 0 };
 
 MVM_PUBLIC MVMCallsite *MVM_callsite_get_common(MVMThreadContext *tc, MVMCommonCallsiteID id) {
     switch (id) {
@@ -98,22 +98,22 @@ void MVM_callsite_initialize_common(MVMThreadContext *tc) {
 MVM_PUBLIC void MVM_callsite_try_intern(MVMThreadContext *tc, MVMCallsite **cs_ptr) {
     MVMCallsiteInterns *interns    = tc->instance->callsite_interns;
     MVMCallsite        *cs         = *cs_ptr;
-    MVMint32            num_nameds = (cs->arg_count - cs->num_pos) / 2;
-    MVMint32            num_flags  = cs->num_pos + num_nameds;
+    MVMint32            num_flags  = cs->flag_count;
+    MVMint32            num_nameds = MVM_callsite_num_nameds(tc, cs);
     MVMint32 i, found;
 
     /* Can't intern anything with flattening. */
     if (cs->has_flattening)
         return;
 
-    /* Can intern things with nameds, provided we know the names. */
-    if (num_nameds > 0 && !cs->arg_names)
-        return;
-
     /* Also can't intern past the max arity. */
     if (num_flags >= MVM_INTERN_ARITY_LIMIT)
         return;
 
+    /* Can intern things with nameds, provided we know the names. */
+    if (num_nameds > 0 && !cs->arg_names)
+        return;
+
     /* Obtain mutex protecting interns store. */
     uv_mutex_lock(&tc->instance->mutex_callsite_interns);
 

src/core/callsite.h

@@ -69,10 +69,14 @@ struct MVMCallsite {
     /* The set of flags. */
     MVMCallsiteEntry *arg_flags;
 
+    /* The number of arg flags. */
+    MVMuint16 flag_count;
+
     /* The total argument count (including 2 for each named arg). */
     MVMuint16 arg_count;
 
-    /* Number of positionals. */
+    /* Number of positionals, including flattening positionals but
+     * excluding named positionals. */
     MVMuint16 num_pos;
 
     /* Whether it has any flattening args. */
@@ -86,7 +90,8 @@ struct MVMCallsite {
     MVMCallsite *with_invocant;
 
     /* Names of named arguments, in the order that they are passed (and thus
-     * matching the flags). */
+     * matching the flags). Note that named flattening args do not have an
+     * entry here, even though they come in the nameds section. */
     MVMString **arg_names;
 };
 
@@ -114,3 +119,15 @@ MVM_PUBLIC MVMCallsite *MVM_callsite_get_common(MVMThreadContext *tc, MVMCommonC
 
 /* Callsite interning function. */
 MVM_PUBLIC void MVM_callsite_try_intern(MVMThreadContext *tc, MVMCallsite **cs);
+
+/* Count the number of nameds (excluding flattening). */
+MVM_STATIC_INLINE MVMuint16 MVM_callsite_num_nameds(MVMThreadContext *tc, MVMCallsite *cs) {
+    MVMuint16 i = cs->num_pos;
+    MVMuint16 nameds = 0;
+    while (i < cs->flag_count) {
+        if (!(cs->arg_flags[i] & MVM_CALLSITE_ARG_FLAT_NAMED))
+            nameds++;
+        i++;
+    }
+    return nameds;
+}

src/core/frame.c

@@ -1488,8 +1488,9 @@ static MVMObject * find_invokee_internal(MVMThreadContext *tc, MVMObject *code,
             }
             else {
                 MVMCallsite *new   = MVM_malloc(sizeof(MVMCallsite));
-                MVMint32     fsize = orig->num_pos + (orig->arg_count - orig->num_pos) / 2;
-                new->arg_flags     = MVM_malloc((fsize + 1) * sizeof(MVMCallsiteEntry));
+                MVMint32     fsize = orig->flag_count;
+                new->flag_count    = fsize + 1;
+                new->arg_flags     = MVM_malloc(new->flag_count * sizeof(MVMCallsiteEntry));
                 new->arg_flags[0]  = MVM_CALLSITE_ARG_OBJ;
                 memcpy(new->arg_flags + 1, orig->arg_flags, fsize);
                 new->arg_count      = orig->arg_count + 1;

src/core/nativecall_dyncall.c

@@ -133,6 +133,7 @@ static void * unmarshal_callback(MVMThreadContext *tc, MVMObject *callback, MVMO
 
         /* We'll also build up a MoarVM callsite as we go. */
         cs                 = MVM_malloc(sizeof(MVMCallsite));
+        cs->flag_count     = num_info - 1;
         cs->arg_flags      = MVM_malloc(num_info * sizeof(MVMCallsiteEntry));
         cs->arg_count      = num_info - 1;
         cs->num_pos        = num_info - 1;

src/core/nativecall_libffi.c

@@ -116,6 +116,7 @@ static void * unmarshal_callback(MVMThreadContext *tc, MVMObject *callback, MVMO
 
         /* We'll also build up a MoarVM callsite as we go. */
         cs                 = MVM_malloc(sizeof(MVMCallsite));
+        cs->flag_count     = num_info - 1;
         cs->arg_flags      = MVM_malloc(num_info * sizeof(MVMCallsiteEntry));
         cs->arg_count      = num_info - 1;
         cs->num_pos        = num_info - 1;