Merge pull request #53 from modusintegration/master

Syntactical Ordering of Object Keys

sjsonnet/src-js/sjsonnet/SjsonnetMain.scala

@@ -12,7 +12,8 @@ object SjsonnetMain {
                 extVars: js.Any,
                 tlaVars: js.Any,
                 wd0: String,
-                importer: js.Function2[String, String, js.Array[String]]): js.Any = {
+                importer: js.Function2[String, String, js.Array[String]],
+                preserveOrder: Boolean = false): js.Any = {
     val interp = new Interpreter(
       mutable.Map.empty,
       ujson.WebJson.transform(extVars, ujson.Value).obj.toMap,
@@ -23,7 +24,8 @@ object SjsonnetMain {
           case null => None
           case arr => Some((JsVirtualPath(arr(0)), arr(1)))
         }
-      }
+      },
+      preserveOrder
     )
     interp.interpret0(text, JsVirtualPath("(memory)"), ujson.WebJson.Builder) match{
       case Left(msg) => throw new js.JavaScriptException(msg)

sjsonnet/src-jvm/sjsonnet/Cli.scala

@@ -20,7 +20,8 @@ object Cli{
                     expectString: Boolean = false,
                     varBinding: Map[String, ujson.Value] = Map(),
                     tlaBinding: Map[String, ujson.Value] = Map(),
-                    indent: Int = 3)
+                    indent: Int = 3,
+                    preserveOrder: Boolean = false)
 
 
   def genericSignature(wd: os.Path) = Seq(
@@ -127,7 +128,13 @@ object Cli{
         case Array(x, v) =>
           c.copy(tlaBinding = c.tlaBinding ++ Seq(x -> ujson.read(os.read(os.Path(v, wd)))))
       }
-    )
+    ),
+    Arg[Config, Unit](
+      "preserve-order", Some('p'),
+      "Preserves order of keys in the resulting JSON",
+      (c, v) => c.copy(preserveOrder = true)
+    ),
+
   )
   def showArg(arg: Arg[_, _]) =
     "  " + arg.shortName.fold("")("-" + _ + ", ") + "--" + arg.name

sjsonnet/src-jvm/sjsonnet/SjsonnetMain.scala

@@ -101,7 +101,8 @@ object SjsonnetMain {
       importer = resolveImport(
         config.jpaths.map(os.Path(_, wd)).map(OsPath(_)),
         allowedInputs
-      )
+      ),
+      config.preserveOrder
     )
 
     def writeFile(f: os.RelPath, contents: String): Either[String, Unit] = {

sjsonnet/src/sjsonnet/Evaluator.scala

@@ -19,7 +19,8 @@ import scala.collection.mutable
 class Evaluator(parseCache: collection.mutable.Map[String, fastparse.Parsed[(Expr, Map[String, Int])]],
                 val extVars: Map[String, ujson.Value],
                 val wd: Path,
-                importer: (Path, String) => Option[(Path, String)]) extends EvalScope{
+                importer: (Path, String) => Option[(Path, String)],
+                override val preserveOrder: Boolean = false) extends EvalScope{
   implicit def evalScope: EvalScope = this
 
   val loadedFileContents = mutable.Map.empty[Path, String]
@@ -435,7 +436,7 @@ class Evaluator(parseCache: collection.mutable.Map[String, fastparse.Parsed[(Exp
       ).toArray
 
       lazy val newSelf: Val.Obj = {
-        val builder = Map.newBuilder[String, Val.Obj.Member]
+        val builder = mutable.LinkedHashMap.newBuilder[String, Val.Obj.Member]
         value.foreach {
           case Member.Field(offset, fieldName, plus, None, sep, rhs) =>
             visitFieldName(fieldName, offset).map(_ -> Val.Obj.Member(plus, sep, (self: Val.Obj, sup: Option[Val.Obj], _, _) => {
@@ -461,7 +462,7 @@ class Evaluator(parseCache: collection.mutable.Map[String, fastparse.Parsed[(Exp
       )
 
       lazy val newSelf: Val.Obj = {
-        val builder = Map.newBuilder[String, Val.Obj.Member]
+        val builder = mutable.LinkedHashMap.newBuilder[String, Val.Obj.Member]
         for(s <- visitComp(first :: rest.toList, Seq(compScope))){
           lazy val newScope: ValScope = s.extend(
             newBindings,

sjsonnet/src/sjsonnet/Interpreter.scala

@@ -13,13 +13,15 @@ class Interpreter(parseCache: collection.mutable.Map[String, fastparse.Parsed[(E
                   extVars: Map[String, ujson.Value],
                   tlaVars: Map[String, ujson.Value],
                   wd: Path,
-                  importer: (Path, String) => Option[(Path, String)]) {
+                  importer: (Path, String) => Option[(Path, String)],
+                  preserveOrder: Boolean = false) {
 
   val evaluator = new Evaluator(
     parseCache,
     extVars,
     wd,
-    importer
+    importer,
+    preserveOrder
   )
 
   def interpret(txt: String, path: Path): Either[String, ujson.Value] = {

sjsonnet/src/sjsonnet/Materializer.scala

@@ -4,6 +4,8 @@ import sjsonnet.Expr.{FieldName, Member, ObjBody}
 import sjsonnet.Expr.Member.Visibility
 import upickle.core.Visitor
 
+import scala.collection.mutable
+
 /**
   * Serializes the given [[Val]] out to the given [[upickle.core.Visitor]],
   * which can transform it into [[ujson.Value]]s or directly serialize it
@@ -36,12 +38,13 @@ object Materializer {
       case obj: Val.Obj =>
         obj.triggerAllAsserts(obj)
 
-        val keys = obj.getVisibleKeys().toArray.sortBy(_._1)
+        val keysUnsorted = obj.getVisibleKeys().toArray
+        val keys = if (!evaluator.preserveOrder) keysUnsorted.sortBy(_._1) else keysUnsorted
         val objVisitor = visitor.visitObject(keys.length , -1)
 
         for(t <- keys) {
           val (k, hidden) = t
-          if (!hidden){
+          if (!hidden){ 
             objVisitor.visitKeyValue(objVisitor.visitKey(-1).visitString(k, -1))
             objVisitor.visitValue(
               apply0(
@@ -73,7 +76,7 @@ object Materializer {
     case ujson.Str(s) => Val.Str(s)
     case ujson.Arr(xs) => Val.Arr(xs.map(x => Val.Lazy(reverse(x))).toArray[Val.Lazy])
     case ujson.Obj(xs) =>
-      val builder = Map.newBuilder[String, Val.Obj.Member]
+      val builder = mutable.LinkedHashMap.newBuilder[String, Val.Obj.Member]
       for(x <- xs){
         val v = Val.Obj.Member(false, Visibility.Normal,
           (_: Val.Obj, _: Option[Val.Obj], _, _) => reverse(x._2)

sjsonnet/src/sjsonnet/Std.scala

@@ -8,7 +8,7 @@ import java.util.zip.GZIPOutputStream
 import sjsonnet.Expr.Member.Visibility
 import sjsonnet.Expr.{BinaryOp, False, Params}
 
-import scala.collection.mutable.ArrayBuffer
+import scala.collection.mutable
 import scala.collection.compat._
 import sjsonnet.Std.builtinWithDefaults
 import ujson.Value
@@ -53,22 +53,26 @@ object Std {
       v1.getVisibleKeys().get(v2).isDefined
     },
     builtin("objectFields", "o"){ (ev, fs, v1: Val.Obj) =>
-      Val.Arr(
-        v1.getVisibleKeys()
-          .collect{case (k, false) => k}
-          .toSeq
-          .sorted
-          .map(k => Val.Lazy(Val.Str(k)))
-      )
+      val keys = v1.getVisibleKeys()
+        .collect{case (k, false) => k}
+        .toSeq
+      val maybeSorted = if(ev.preserveOrder) {
+        keys
+      } else {
+        keys.sorted
+      }
+      Val.Arr(maybeSorted.map(k => Val.Lazy(Val.Str(k))))
     },
     builtin("objectFieldsAll", "o"){ (ev, fs, v1: Val.Obj) =>
-      Val.Arr(
-        v1.getVisibleKeys()
-          .collect{case (k, _) => k}
-          .toSeq
-          .sorted
-          .map(k => Val.Lazy(Val.Str(k)))
-      )
+      val keys = v1.getVisibleKeys()
+        .collect{case (k, _) => k}
+        .toSeq
+      val maybeSorted = if(ev.preserveOrder) {
+        keys
+      } else {
+        keys.sorted
+      }
+      Val.Arr(maybeSorted.map(k => Val.Lazy(Val.Str(k))))
     },
     builtin("type", "x"){ (ev, fs, v1: Val) =>
       v1 match{
@@ -250,14 +254,15 @@ object Std {
     builtin("mapWithKey", "func", "obj"){ (ev, fs, func: Applyer, obj: Val.Obj) =>
       val allKeys = obj.getVisibleKeys()
       new Val.Obj(
+        mutable.LinkedHashMap() ++
         allKeys.map{ k =>
           k._1 -> (Val.Obj.Member(false, Visibility.Normal, (self: Val.Obj, sup: Option[Val.Obj], _, _) =>
             func.apply(
               Val.Lazy(Val.Str(k._1)),
               Val.Lazy(obj.value(k._1, -1)(fs,ev))
             )
           ))
-        }.toMap,
+        },
         _ => (),
         None
       )
@@ -289,7 +294,7 @@ object Std {
     builtin("findSubstr", "pat", "str") { (ev, fs, pat: String, str: String) =>
       if (pat.length == 0) Val.Arr(Seq())
       else {
-        val indices = ArrayBuffer[Int]()
+        val indices = mutable.ArrayBuffer[Int]()
         var matchIndex = str.indexOf(pat)
         while (0 <= matchIndex && matchIndex < str.length) {
           indices.append(matchIndex)
@@ -561,14 +566,14 @@ object Std {
         } else {
           val keyFFunc = keyF.asInstanceOf[Val.Func]
           val keyFApplyer = Applyer(keyFFunc, ev, null)
-          val appliedX = keyFApplyer.apply(v)
+          val appliedX = Materializer(keyFApplyer.apply(v))(ev)
 
           if (b.exists(value => {
             val appliedValue = keyFApplyer.apply(value)
-            appliedValue == appliedX
+            Materializer(appliedValue)(ev) == appliedX
           }) && !out.exists(value => {
             val mValue = keyFApplyer.apply(value)
-            mValue == appliedX
+            Materializer(mValue)(ev) == appliedX
           })) {
             out.append(v)
           }
@@ -608,14 +613,14 @@ object Std {
         } else {
           val keyFFunc = keyF.asInstanceOf[Val.Func]
           val keyFApplyer = Applyer(keyFFunc, ev, null)
-          val appliedX = keyFApplyer.apply(v)
+          val appliedX = Materializer(keyFApplyer.apply(v))(ev)
 
           if (!b.exists(value => {
             val appliedValue = keyFApplyer.apply(value)
-            appliedValue == appliedX
+            Materializer(appliedValue)(ev) == appliedX
           }) && !out.exists(value => {
             val mValue = keyFApplyer.apply(value)
-            mValue == appliedX
+            Materializer(mValue)(ev) == appliedX
           })) {
             out.append(v)
           }
@@ -639,7 +644,7 @@ object Std {
         val appliedX = keyFApplyer.apply(x)
         arr.exists(value => {
           val appliedValue = keyFApplyer.apply(value)
-          appliedValue == appliedX
+          Materializer(appliedValue)(ev) == Materializer(appliedX)(ev)
         })
       }
     },
@@ -675,10 +680,10 @@ object Std {
             val transformedValue: Seq[Val.Lazy] = values.map(v => Val.Lazy(recursiveTransform(v))).toSeq
             Val.Arr(transformedValue)
           case ujson.Obj(valueMap) =>
-            val transformedValue = valueMap
+            val transformedValue = mutable.LinkedHashMap() ++ valueMap
               .mapValues { v =>
                 Val.Obj.Member(false, Expr.Member.Visibility.Normal, (_, _, _, _) => recursiveTransform(v))
-              }.toMap
+              }
             new Val.Obj(transformedValue , (x: Val.Obj) => (), None)
         }
       }
@@ -702,7 +707,7 @@ object Std {
             v = rec(o.value(k, -1)(fs, ev))
             if filter(v)
           }yield (k, Val.Obj.Member(false, Visibility.Normal, (_, _, _, _) => v))
-          new Val.Obj(bindings.toMap, _ => (), None)
+          new Val.Obj(mutable.LinkedHashMap() ++ bindings, _ => (), None)
         case a: Val.Arr =>
           Val.Arr(a.value.map(x => rec(x.force)).filter(filter).map(Val.Lazy(_)))
         case _ => x
@@ -737,6 +742,7 @@ object Std {
     )
   )
   val Std = new Val.Obj(
+    mutable.LinkedHashMap() ++
     functions
       .map{
         case (k, v) =>
@@ -748,8 +754,7 @@ object Std {
               (self: Val.Obj, sup: Option[Val.Obj], _, _) => v
             )
           )
-      }
-      .toMap ++ Seq(
+      } ++ Seq(
       (
         "thisFile",
         Val.Obj.Member(
@@ -891,19 +896,19 @@ object Std {
             if (keyF == Val.False) {
               throw new Error.Delegate("Unable to sort array of objects without key function")
             } else {
+              val objs = vs.map(_.force.cast[Val.Obj])
+
               val keyFFunc = keyF.asInstanceOf[Val.Func]
               val keyFApplyer = Applyer(keyFFunc, ev, null)
-              vs.map(_.force.cast[Val.Obj]).sortWith((o1, o2) => {
-                val o1Key = keyFApplyer.apply(Val.Lazy(o1))
-                val o2Key = keyFApplyer.apply(Val.Lazy(o2))
-                val o1KeyExpr = Materializer.toExpr(Materializer.apply(o1Key)(ev))
-                val o2KeyExpr = Materializer.toExpr(Materializer.apply(o2Key)(ev))
-
-                val comparisonExpr = Expr.BinaryOp(0, o1KeyExpr, BinaryOp.`<`, o2KeyExpr)
-                val exprResult = ev.visitExpr(comparisonExpr)(scope(0), new FileScope(null, Map.empty))
-                val res = Materializer.apply(exprResult)(ev).asInstanceOf[ujson.Bool]
-                res.value
-              }).map(Val.Lazy(_))
+              val keys = objs.map((v) => keyFApplyer(Val.Lazy(v)))
+
+              if (keys.forall(_.isInstanceOf[Val.Str])){
+                objs.sortBy((v) => keyFApplyer(Val.Lazy(v)).cast[Val.Str].value).map(Val.Lazy(_))
+              } else if (keys.forall(_.isInstanceOf[Val.Num])) {
+                objs.sortBy((v) => keyFApplyer(Val.Lazy(v)).cast[Val.Num].value).map(Val.Lazy(_))
+              } else {
+                throw new Error.Delegate("Cannot sort with key values that are " + keys(0).prettyName + "s")
+              }
             }
           }else {
             ???

sjsonnet/src/sjsonnet/Val.scala

@@ -73,7 +73,7 @@ object Val{
 
 
   }
-  final class Obj(value0: Map[String, Obj.Member],
+  final class Obj(value0: mutable.Map[String, Obj.Member],
                   triggerAsserts: Val.Obj => Unit,
                   `super`: Option[Obj]) extends Val{
 
@@ -102,7 +102,7 @@ object Val{
     }
 
     def getVisibleKeys() = {
-      val mapping = collection.mutable.Map.empty[String, Boolean]
+      val mapping = mutable.LinkedHashMap.empty[String, Boolean]
       foreachVisibleKey{ (k, sep) =>
         (mapping.get(k), sep) match{
           case (None, Visibility.Hidden) => mapping(k) = true
@@ -314,6 +314,8 @@ trait EvalScope extends EvalErrorScope{
   def materialize(v: Val): ujson.Value
 
   val emptyMaterializeFileScope = new FileScope(wd / "(materialize)", Map())
+
+  val preserveOrder: Boolean = false
 }
 object ValScope{
   def empty(size: Int) = new ValScope(None, None, None, new Array(size))