Std lib improvements: add std.slice and std.manifestJsonMinified, fix handling of numbers in manifestXmlJsonml, handling of code in extCode

bench/src/main/scala/sjsonnet/MainBenchmark.scala

@@ -21,7 +21,7 @@ object MainBenchmark {
     val path = OsPath(os.Path(file, wd))
     val parseCache = new DefaultParseCache
     val interp = new Interpreter(
-      Map.empty[String, ujson.Value],
+      Map.empty[String, String],
       Map.empty[String, ujson.Value],
       OsPath(wd),
       importer = SjsonnetMain.resolveImport(config.jpaths.map(os.Path(_, wd)).map(OsPath(_)), None),

bench/src/main/scala/sjsonnet/MaterializerBenchmark.scala

@@ -28,7 +28,7 @@ class MaterializerBenchmark {
     val path = os.Path(file, wd)
     var currentPos: Position = null
     this.interp = new Interpreter(
-      Map.empty[String, ujson.Value],
+      Map.empty[String, String],
       Map.empty[String, ujson.Value],
       OsPath(wd),
       importer = SjsonnetMain.resolveImport(config.jpaths.map(os.Path(_, wd)).map(OsPath(_)), None),

bench/src/main/scala/sjsonnet/ProfilingEvaluator.scala

@@ -6,7 +6,7 @@ import scala.collection.mutable
 import scala.jdk.CollectionConverters._
 
 class ProfilingEvaluator(resolver: CachedResolver,
-                         extVars: Map[String, ujson.Value],
+                         extVars: String => Option[Expr],
                          wd: Path,
                          settings: Settings,
                          warn: Error => Unit)

bench/src/main/scala/sjsonnet/RunProfiler.scala

@@ -10,13 +10,13 @@ object RunProfiler extends App {
   val path = OsPath(os.Path(file, wd))
   val parseCache = new DefaultParseCache
   val interp = new Interpreter(
-    Map.empty[String, ujson.Value],
+    Map.empty[String, String],
     Map.empty[String, ujson.Value],
     OsPath(wd),
     importer = SjsonnetMain.resolveImport(config.jpaths.map(os.Path(_, wd)).map(OsPath(_)), None),
     parseCache = parseCache
   ) {
-    override def createEvaluator(resolver: CachedResolver, extVars: Map[String, ujson.Value], wd: Path,
+    override def createEvaluator(resolver: CachedResolver, extVars: String => Option[Expr], wd: Path,
                                  settings: Settings, warn: Error => Unit): Evaluator =
       new ProfilingEvaluator(resolver, extVars, wd, settings, warn)
   }

sjsonnet/src-js/sjsonnet/SjsonnetMain.scala

@@ -15,7 +15,7 @@ object SjsonnetMain {
                 importLoader: js.Function1[String, String],
                 preserveOrder: Boolean = false): js.Any = {
     val interp = new Interpreter(
-      ujson.WebJson.transform(extVars, ujson.Value).obj.toMap,
+      ujson.WebJson.transform(extVars, ujson.Value).obj.toMap.map{case (k, ujson.Str(v)) => (k, v)},
       ujson.WebJson.transform(tlaVars, ujson.Value).obj.toMap,
       JsVirtualPath(wd0),
       new Importer {

sjsonnet/src-jvm-native/sjsonnet/SjsonnetMain.scala

@@ -141,22 +141,22 @@ object SjsonnetMain {
                      importer: Option[(Path, String) => Option[os.Path]] = None,
                      warnLogger: String => Unit = null): Either[String, String] = {
     val path = os.Path(file, wd)
-    var varBinding = Map.empty[String, ujson.Value]
+    var varBinding = Map.empty[String, String]
     config.extStr.map(_.split("=", 2)).foreach{
-      case Array(x) => varBinding = varBinding ++ Seq(x -> ujson.Str(System.getenv(x)))
-      case Array(x, v) => varBinding = varBinding ++ Seq(x -> ujson.Str(v))
+      case Array(x) => varBinding = varBinding ++ Seq(x -> ujson.write(System.getenv(x)))
+      case Array(x, v) => varBinding = varBinding ++ Seq(x -> ujson.write(v))
     }
     config.extStrFile.map(_.split("=", 2)).foreach {
       case Array(x, v) =>
-        varBinding = varBinding ++ Seq(x -> ujson.Str(os.read(os.Path(v, wd))))
+        varBinding = varBinding ++ Seq(x -> ujson.write(os.read(os.Path(v, wd))))
     }
     config.extCode.map(_.split("=", 2)).foreach {
-      case Array(x) => varBinding = varBinding ++ Seq(x -> ujson.read(System.getenv(x)))
-      case Array(x, v) => varBinding = varBinding ++ Seq(x -> ujson.read(v))
+      case Array(x) => varBinding = varBinding ++ Seq(x -> System.getenv(x))
+      case Array(x, v) => varBinding = varBinding ++ Seq(x -> v)
     }
     config.extCodeFile.map(_.split("=", 2)).foreach {
       case Array(x, v) =>
-        varBinding = varBinding ++ Seq(x -> ujson.read(os.read(os.Path(v, wd))))
+        varBinding = varBinding ++ Seq(x -> os.read(os.Path(v, wd)))
     }
 
     var tlaBinding = Map.empty[String, ujson.Value]

sjsonnet/src/sjsonnet/Error.scala

@@ -96,7 +96,7 @@ object StaticError {
 }
 
 trait EvalErrorScope {
-  def extVars: Map[String, ujson.Value]
+  def extVars: String => Option[Expr]
   def importer: CachedImporter
   def wd: Path
 

sjsonnet/src/sjsonnet/Evaluator.scala

@@ -16,7 +16,7 @@ import scala.collection.mutable
   * `parseCache`.
   */
 class Evaluator(resolver: CachedResolver,
-                val extVars: Map[String, ujson.Value],
+                val extVars: String => Option[Expr],
                 val wd: Path,
                 val settings: Settings,
                 warnLogger: Error => Unit = null) extends EvalScope {
@@ -251,15 +251,28 @@ class Evaluator(resolver: CachedResolver,
   private def visitSlice(e: Slice)(implicit scope: ValScope): Val = {
     visitExpr(e.value) match {
       case a: Val.Arr =>
-        a.slice(e.start.fold(0)(visitExpr(_).cast[Val.Num].value.toInt),
-          e.end.fold(a.length)(visitExpr(_).cast[Val.Num].value.toInt),
-          e.stride.fold(1)(visitExpr(_).cast[Val.Num].value.toInt))
+        new Val.Arr(
+          e.pos,
+          Util.sliceArr(
+            a.asLazyArray,
+            e.start.fold(0)(visitExpr(_).cast[Val.Num].value.toInt),
+            e.end.fold(a.length)(visitExpr(_).cast[Val.Num].value.toInt),
+            e.stride.fold(1)(visitExpr(_).cast[Val.Num].value.toInt)
+          )
+        )
+
+
       case Val.Str(_, s) =>
-        val range =
-          e.start.fold(0)(visitExpr(_).cast[Val.Num].value.toInt) until
-            e.end.fold(s.length)(visitExpr(_).cast[Val.Num].value.toInt) by
+        Val.Str(
+          e.pos,
+          Util.sliceStr(
+            s,
+            e.start.fold(0)(visitExpr(_).cast[Val.Num].value.toInt),
+            e.end.fold(s.length)(visitExpr(_).cast[Val.Num].value.toInt),
             e.stride.fold(1)(visitExpr(_).cast[Val.Num].value.toInt)
-        Val.Str(e.pos, range.dropWhile(_ < 0).takeWhile(_ < s.length).map(s).mkString)
+          )
+        )
+
       case x => Error.fail("Can only slice array or string, not " + x.prettyName, e.pos)
     }
   }

sjsonnet/src/sjsonnet/Interpreter.scala

@@ -11,7 +11,7 @@ import scala.util.control.NonFatal
   * Wraps all the machinery of evaluating Jsonnet source code, from parsing to
   * evaluation to materialization, into a convenient wrapper class.
   */
-class Interpreter(extVars: Map[String, ujson.Value],
+class Interpreter(extVars: Map[String, String],
                   tlaVars: Map[String, ujson.Value],
                   wd: Path,
                   importer: Importer,
@@ -29,11 +29,22 @@ class Interpreter(extVars: Map[String, ujson.Value],
 
   private def warn(e: Error): Unit = warnLogger("[warning] " + formatError(e))
 
-  def createEvaluator(resolver: CachedResolver, extVars: Map[String, ujson.Value], wd: Path,
+  def createEvaluator(resolver: CachedResolver, extVars: String => Option[Expr], wd: Path,
                       settings: Settings, warn: Error => Unit): Evaluator =
     new Evaluator(resolver, extVars, wd, settings, warn)
 
-  val evaluator: Evaluator = createEvaluator(resolver, extVars, wd, settings, warn)
+  lazy val evaluator: Evaluator = createEvaluator(
+    resolver,
+    // parse extVars lazily, because they can refer to each other and be recursive
+    extVars
+      .mapValues{ v => resolver.parse(wd / s"<ext-var $v>", v)(evaluator).fold(throw _, _._1) }
+      .lift,
+    wd,
+    settings,
+    warn
+  )
+
+  evaluator // force the lazy val
 
   def formatError(e: Error): String = {
     val s = new StringWriter()

sjsonnet/src/sjsonnet/Std.scala

@@ -515,10 +515,7 @@ class Std {
   private object ExtVar extends Val.Builtin1("x") {
     def evalRhs(_x: Val, ev: EvalScope, pos: Position): Val = {
       val Val.Str(_, x) = _x
-      Materializer.reverse(
-        pos,
-        ev.extVars.getOrElse(x, Error.fail("Unknown extVar: " + x))
-      )
+      ev.visitExpr(ev.extVars(x).getOrElse(Error.fail("Unknown extVar: " + x)))(ValScope.empty)
     }
     override def staticSafe = false
   }
@@ -569,6 +566,11 @@ class Std {
       Val.Str(pos, Materializer.apply0(v, new MaterializeJsonRenderer())(ev).toString)
   }
 
+  private object ManifestJsonMinified extends Val.Builtin1("v") {
+    def evalRhs(v: Val, ev: EvalScope, pos: Position): Val =
+      Val.Str(pos, Materializer.apply0(v, new MaterializeJsonRenderer(indent = -1))(ev).toString)
+  }
+
   private object ManifestJsonEx extends Val.Builtin2("value", "indent") {
     def evalRhs(v: Val, i: Val, ev: EvalScope, pos: Position): Val =
       Val.Str(pos, Materializer
@@ -742,6 +744,14 @@ class Std {
     builtin("clamp", "x", "minVal", "maxVal"){ (pos, ev, x: Double, minVal: Double, maxVal: Double) =>
       math.max(minVal, math.min(x, maxVal))
     },
+    builtin("slice", "indexable", "index", "end", "step"){ (pos, ev, indexable: Val, index: Int, end: Int, step: Int) =>
+      val res = indexable match {
+        case Val.Str(pos0, s) => Val.Str(pos, Util.sliceStr(s, index, end, step))
+        case arr: Val.Arr => new Val.Arr(pos, Util.sliceArr(arr.asLazyArray, index, end, step))
+        case _ => Error.fail("std.slice first argument must be indexable")
+      }
+      res: Val
+    },
 
     builtin("makeArray", "sz", "func"){ (pos, ev, sz: Int, func: Val.Func) =>
       new Val.Arr(
@@ -950,6 +960,7 @@ class Std {
       Materializer.apply0(v, new PythonRenderer())(ev).toString
     },
     "manifestJson" -> ManifestJson,
+    "manifestJsonMinified" -> ManifestJsonMinified,
     "manifestJsonEx" -> ManifestJsonEx,
     builtinWithDefaults("manifestYamlDoc",
                         "v" -> null,
@@ -1000,6 +1011,12 @@ class Std {
             tag(t)(
               attrs.value.map {
                 case (k, ujson.Str(v)) => attr(k) := v
+
+                // use ujson.write to make sure output number format is same as
+                // google/jsonnet, e.g. whole numbers are printed without the
+                // decimal point and trailing zero
+                case (k, ujson.Num(v)) => attr(k) := ujson.write(v)
+
                 case (k, v) => Error.fail("Cannot call manifestXmlJsonml on " + v.getClass)
               }.toSeq,
               children.map(rec)
@@ -1221,6 +1238,21 @@ class Std {
     })
   }
 
+  def builtin[R: ReadWriter, T1: ReadWriter, T2: ReadWriter, T3: ReadWriter, T4: ReadWriter]
+             (name: String, p1: String, p2: String, p3: String, p4: String)
+             (eval: (Position, EvalScope, T1, T2, T3, T4) => R): (String, Val.Func) = {
+    (name, new Val.Builtin4(p1, p2, p3, p4) {
+      def evalRhs(arg1: Val, arg2: Val, arg3: Val, arg4: Val, ev: EvalScope, outerPos: Position): Val = {
+        //println("--- calling builtin: "+name)
+        val v1: T1 = implicitly[ReadWriter[T1]].apply(arg1)
+        val v2: T2 = implicitly[ReadWriter[T2]].apply(arg2)
+        val v3: T3 = implicitly[ReadWriter[T3]].apply(arg3)
+        val v4: T4 = implicitly[ReadWriter[T4]].apply(arg4)
+        implicitly[ReadWriter[R]].write(outerPos, eval(outerPos, ev, v1, v2, v3, v4))
+      }
+    })
+  }
+
   /**
     * Helper function that can define a built-in function with default parameters
     *

sjsonnet/src/sjsonnet/Util.scala

@@ -16,4 +16,21 @@ object Util{
     val col = index - lineStarts(line)
     s"${line+1}:${col+1}"
   }
+
+  def sliceArr[T: scala.reflect.ClassTag](arr: Array[T], start: Int, end: Int, step: Int): Array[T] = {
+    step match{
+      case 1 => arr.slice(start, end)
+      case _ =>
+        val range = start until end by step
+        range.dropWhile(_ < 0).takeWhile(_ < arr.length).map(arr).toArray
+    }
+  }
+  def sliceStr(s: String, start: Int, end: Int, step: Int): String = {
+    step match{
+      case 1 => s.slice(start, end)
+      case _ =>
+        val range = start until end by step
+        new String(range.dropWhile(_ < 0).takeWhile(_ < s.length).map(s).toArray)
+    }
+  }
 }

sjsonnet/src/sjsonnet/Val.scala

@@ -102,11 +102,6 @@ object Val{
     def concat(newPos: Position, rhs: Arr): Arr =
       new Arr(newPos, value ++ rhs.value)
 
-    def slice(start: Int, end: Int, stride: Int): Arr = {
-      val range = start until end by stride
-      new Arr(pos, range.dropWhile(_ < 0).takeWhile(_ < value.length).map(value).toArray)
-    }
-
     def iterator: Iterator[Val] = value.iterator.map(_.force)
     def foreach[U](f: Val => U) = {
       var i = 0
@@ -500,6 +495,18 @@ object Val{
         evalRhs(argVals(0).force, argVals(1).force, argVals(2).force, ev, outerPos)
       else super.apply(argVals, namedNames, outerPos)
   }
+
+  abstract class Builtin4(pn1: String, pn2: String, pn3: String, pn4: String, defs: Array[Expr] = null) extends Builtin(Array(pn1, pn2, pn3, pn4), defs) {
+    final def evalRhs(args: Array[Val], ev: EvalScope, pos: Position): Val =
+      evalRhs(args(0), args(1), args(2), args(3), ev, pos)
+
+    def evalRhs(arg1: Val, arg2: Val, arg3: Val, arg4: Val, ev: EvalScope, pos: Position): Val
+
+    override def apply(argVals: Array[_ <: Lazy], namedNames: Array[String], outerPos: Position)(implicit ev: EvalScope): Val =
+      if(namedNames == null && argVals.length == 4)
+        evalRhs(argVals(0).force, argVals(1).force, argVals(2).force, argVals(3).force, ev, outerPos)
+      else super.apply(argVals, namedNames, outerPos)
+  }
 }
 
 /**

sjsonnet/test/src-jvm-native/sjsonnet/FileTests.scala

@@ -6,7 +6,7 @@ object FileTests extends TestSuite{
   val testSuiteRoot = os.pwd / "sjsonnet" / "test" / "resources" / "test_suite"
   def eval(p: os.Path) = {
     val interp = new Interpreter(
-      Map("var1" -> "test", "var2" -> ujson.Obj("x" -> 1, "y" -> 2)),
+      Map("var1" -> "\"test\"", "var2" -> """local f(a, b) = {[a]: b, "y": 2}; f("x", 1)"""),
       Map("var1" -> "test", "var2" -> ujson.Obj("x" -> 1, "y" -> 2)),
       OsPath(testSuiteRoot),
       importer = sjsonnet.SjsonnetMain.resolveImport(Array(OsPath(testSuiteRoot))),

sjsonnet/test/src/sjsonnet/FormatTests.scala

@@ -10,7 +10,7 @@ object FormatTests extends TestSuite{
     val json = ujson.read(jsonStr)
     val formatted = Format.format(fmt, Materializer.reverse(null, json), dummyPos)(
       new EvalScope{
-        def extVars: Map[String, Value] = Map()
+        def extVars = _ => None
         def wd: Path = DummyPath()
         def visitExpr(expr: Expr)(implicit scope: ValScope): Val = ???
         def materialize(v: Val): Value = ???

sjsonnet/test/src/sjsonnet/Std0150FunctionsTests.scala

@@ -41,5 +41,67 @@ object Std0150FunctionsTests extends TestSuite {
       eval("std.join(' ', ['', 'foo'])") ==> ujson.Str(" foo")
       eval("std.join(' ', [null, 'foo'])") ==> ujson.Str("foo")
     }
+
+    test("slice"){
+      eval("std.slice([1, 2, 3, 4, 5, 6], 0, 4, 1)") ==> ujson.read("[ 1, 2, 3, 4 ]")
+      eval("std.slice([1, 2, 3, 4, 5, 6], 1, 6, 2)") ==> ujson.read("[ 2, 4, 6 ]")
+      eval("""std.slice("jsonnet", 0, 4, 1)""") ==> ujson.Str("json")
+    }
+
+    test("manifestJsonMinified"){
+      eval("""std.manifestJsonMinified( { x: [1, 2, 3, true, false, null, "string\nstring"], y: { a: 1, b: 2, c: [1, 2] }, })""") ==>
+        ujson.Str("{\"x\":[1,2,3,true,false,null,\"string\\nstring\"],\"y\":{\"a\":1,\"b\":2,\"c\":[1,2]}}")
+    }
+
+    test("manifestXmlJsonml"){
+      eval(
+        """std.manifestXmlJsonml([
+          |  'svg', { height: 100, width: 100 },
+          |  [
+          |    'circle', {
+          |      cx: 50, cy: 50, r: 40,
+          |       stroke: 'black', 'stroke-width': 3,
+          |       fill: 'red',
+          |    }
+          |  ],
+          |])
+          |""".stripMargin
+      ) ==>
+        ujson.Str("""<svg height="100" width="100"><circle cx="50" cy="50" fill="red" r="40" stroke="black" stroke-width="3"></circle></svg>""".stripMargin)
+    }
+
+    test("extVars"){
+      val interpreter = new Interpreter(
+        Map(
+          "num" -> "1",
+          "str" -> "\"hello\"",
+          "bool" -> "true",
+          "jsonArrNums" -> """[1, 2, 3]""",
+          "jsonObjBools" -> """{"hello": false}""",
+          "code" -> """local f(a, b) = {[a]: b, "y": 2}; f("x", 1)""",
+          "std" -> """std.length("hello")""",
+          "stdExtVar" -> """std.extVar("std") + 10""",
+          "stdExtVarRecursive" -> """std.extVar("stdExtVar") + 100""",
+        ),
+        Map(),
+        DummyPath(),
+        Importer.empty,
+        parseCache = new DefaultParseCache,
+      )
+
+      def check(s: String, expected: ujson.Value) =
+        interpreter.interpret(s, DummyPath("(memory)")) ==> Right(expected)
+
+      check("""std.extVar("num")""", 1)
+      check("""std.extVar("str")""", "hello")
+      check("""std.extVar("bool")""", ujson.True)
+      check("""std.extVar("jsonArrNums")""", ujson.Arr(1, 2, 3))
+      check("""std.extVar("jsonObjBools")""", ujson.Obj("hello" -> false))
+      check("""std.extVar("code")""", ujson.Obj("x" -> 1, "y" -> 2))
+      check("""std.extVar("std")""", 5)
+      check("""std.extVar("stdExtVar")""", 15)
+      check("""std.extVar("stdExtVarRecursive")""", 115)
+    }
+
   }
 }