Add a primop ‘concatLists’

This can serve as a generic efficient list builder.  For instance, the
function ‘catAttrs’ in Nixpkgs can be rewritten from

  attr: l: fold (s: l: if hasAttr attr s then [(getAttr attr s)] ++ l else l) [] l

to

  attr: l: builtins.concatLists (map (s: if hasAttr attr s then [(getAttr attr s)] else []) l)

Statistics before:

  time elapsed: 1.08683
  size of a value: 24
  environments allocated: 1384376 (35809568 bytes)
  list elements: 6946783 (55574264 bytes)
  list concatenations: 37434
  values allocated: 1760440 (42250560 bytes)
  attribute sets allocated: 392040
  right-biased unions: 186334
  values copied in right-biased unions: 591137
  symbols in symbol table: 18273
  number of thunks: 1297673
  number of thunks avoided: 1380759
  number of attr lookups: 430802
  number of primop calls: 628912
  number of function calls: 1333544

Statistics after (including new catAttrs):

  time elapsed: 0.959854
  size of a value: 24
  environments allocated: 1010198 (26829296 bytes)
  list elements: 1984878 (15879024 bytes)
  list concatenations: 30488
  values allocated: 1589760 (38154240 bytes)
  attribute sets allocated: 392040
  right-biased unions: 186334
  values copied in right-biased unions: 591137
  symbols in symbol table: 18274
  number of thunks: 1040925
  number of thunks avoided: 1038428
  number of attr lookups: 438419
  number of primop calls: 474844
  number of function calls: 959366

src/libexpr/eval.cc

@@ -912,16 +912,29 @@ void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v)
 
 void ExprOpConcatLists::eval(EvalState & state, Env & env, Value & v)
 {
-    state.nrListConcats++;
     Value v1; e1->eval(state, env, v1);
-    state.forceList(v1);
     Value v2; e2->eval(state, env, v2);
-    state.forceList(v2);
-    state.mkList(v, v1.list.length + v2.list.length);
-    for (unsigned int n = 0; n < v1.list.length; ++n)
-        v.list.elems[n] = v1.list.elems[n];
-    for (unsigned int n = 0; n < v2.list.length; ++n)
-        v.list.elems[n + v1.list.length] = v2.list.elems[n];
+    Value * lists[2] = { &v1, &v2 };
+    state.concatLists(v, 2, lists);
+}
+
+
+void EvalState::concatLists(Value & v, unsigned int nrLists, Value * * lists)
+{
+    nrListConcats++;
+
+    unsigned int len = 0;
+    for (unsigned int n = 0; n < nrLists; ++n) {
+        forceList(*lists[n]);
+        len += lists[n]->list.length;
+    }
+
+    mkList(v, len);
+    for (unsigned int n = 0, pos = 0; n < nrLists; ++n) {
+        unsigned int l = lists[n]->list.length;
+        memcpy(v.list.elems + pos, lists[n]->list.elems, l * sizeof(Value *));
+        pos += l;
+    }
 }
 
 

src/libexpr/eval.hh

@@ -232,6 +232,8 @@ public:
     void mkAttrs(Value & v, unsigned int expected);
     void mkThunk_(Value & v, Expr * expr);
 
+    void concatLists(Value & v, unsigned int nrLists, Value * * lists);
+
     /* Print statistics. */
     void printStats();
 

src/libexpr/primops.cc

@@ -931,6 +931,7 @@ static void prim_filter(EvalState & state, Value * * args, Value & v)
 }
 
 
+/* Return true if a list contains a given element. */
 static void prim_elem(EvalState & state, Value * * args, Value & v)
 {
     bool res = false;
@@ -944,6 +945,14 @@ static void prim_elem(EvalState & state, Value * * args, Value & v)
 }
 
 
+/* Concatenate a list of lists. */
+static void prim_concatLists(EvalState & state, Value * * args, Value & v)
+{
+    state.forceList(*args[0]);
+    state.concatLists(v, args[0]->list.length, args[0]->list.elems);
+}
+
+
 /* Return the length of a list.  This is an O(1) time operation. */
 static void prim_length(EvalState & state, Value * * args, Value & v)
 {
@@ -1160,6 +1169,7 @@ void EvalState::createBaseEnv()
     addPrimOp("map", 2, prim_map);
     addPrimOp("__filter", 2, prim_filter);
     addPrimOp("__elem", 2, prim_elem);
+    addPrimOp("__concatLists", 1, prim_concatLists);
     addPrimOp("__length", 1, prim_length);
 
     // Integer arithmetic