optimize the graph shape to not create sub-graphs when possible

lang/funcs/core/iter/map_func.go

@@ -794,7 +794,7 @@ func (obj *MapFunc) replaceSubGraph(subgraphInput interfaces.Func) error {
 		)
 		obj.init.Txn.AddVertex(inputElemFunc)
 
-		outputElemFunc, err := obj.lastFuncValue.Call(obj.init.Txn, []interfaces.Func{inputElemFunc})
+		outputElemFunc, err := structs.CallFuncValue(obj.lastFuncValue, obj.init.Txn, []interfaces.Func{inputElemFunc})
 		if err != nil {
 			return errwrap.Wrapf(err, "could not call obj.lastFuncValue.Call()")
 		}

lang/funcs/facts/facts.go

@@ -86,3 +86,16 @@ type Fact interface {
 	Init(*Init) error
 	Stream(context.Context) error
 }
+
+// CallableFunc is a function that can be called statically if we want to do it
+// speculatively or from a resource.
+type CallableFact interface {
+	Fact // implement everything in Fact but add the additional requirements
+
+	// Call this function with the input args and return the value if it is
+	// possible to do so at this time. To transform from the single value,
+	// graph representation of the callable values into a linear, standard
+	// args list for use here, you can use the StructToCallableArgs
+	// function.
+	Call() (types.Value, error)
+}

lang/funcs/facts/func.go

@@ -79,3 +79,15 @@ func (obj *FactFunc) Init(init *interfaces.Init) error {
 func (obj *FactFunc) Stream(ctx context.Context) error {
 	return obj.Fact.Stream(ctx)
 }
+
+// Call means this function implements the CallableFunc interface and can be
+// called statically if we want to do it speculatively or from a resource.
+func (obj *FactFunc) Call() (types.Value, error) {
+
+	callableFact, ok := obj.Fact.(CallableFact)
+	if !ok {
+		return nil, fmt.Errorf("fact is not a CallableFact")
+	}
+
+	return callableFact.Call()
+}

lang/funcs/simple/simple.go

@@ -153,15 +153,18 @@ func (obj *WrappedFunc) Stream(ctx context.Context) error {
 				obj.last = input // store for next
 			}
 
-			values := []types.Value{}
+			// Use the existing implementation instead of this one
+			// which can't handle this case at the moment.
+			//args, err := interfaces.StructToCallableArgs(input)
+			args := []types.Value{}
 			for _, name := range obj.Fn.Type().Ord {
 				x := input.Struct()[name]
-				values = append(values, x)
+				args = append(args, x)
 			}
 
-			result, err := obj.Fn.Call(values) // (Value, error)
+			result, err := obj.Call(args)
 			if err != nil {
-				return errwrap.Wrapf(err, "simple function errored")
+				return err
 			}
 
 			// TODO: do we want obj.result to be a pointer instead?
@@ -184,3 +187,13 @@ func (obj *WrappedFunc) Stream(ctx context.Context) error {
 		}
 	}
 }
+
+// Call means this function implements the CallableFunc interface and can be
+// called statically if we want to do it speculatively or from a resource.
+func (obj *WrappedFunc) Call(args []types.Value) (types.Value, error) {
+	result, err := obj.Fn.Call(args) // (Value, error)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "simple function errored")
+	}
+	return result, err
+}

lang/funcs/simplepoly/simplepoly.go

@@ -581,24 +581,18 @@ func (obj *WrappedFunc) Stream(ctx context.Context) error {
 				obj.last = input // store for next
 			}
 
-			values := []types.Value{}
+			// Use the existing implementation instead of this one
+			// which can't handle this case at the moment.
+			//args, err := interfaces.StructToCallableArgs(input)
+			args := []types.Value{}
 			for _, name := range obj.fn.Type().Ord {
 				x := input.Struct()[name]
-				values = append(values, x)
+				args = append(args, x)
 			}
 
-			if obj.init.Debug {
-				obj.init.Logf("Calling function with: %+v", values)
-			}
-			result, err := obj.fn.Call(values) // (Value, error)
+			result, err := obj.Call(args)
 			if err != nil {
-				if obj.init.Debug {
-					obj.init.Logf("Function returned error: %+v", err)
-				}
-				return errwrap.Wrapf(err, "simple poly function errored")
-			}
-			if obj.init.Debug {
-				obj.init.Logf("Function returned with: %+v", result)
+				return err
 			}
 
 			// TODO: do we want obj.result to be a pointer instead?
@@ -621,3 +615,13 @@ func (obj *WrappedFunc) Stream(ctx context.Context) error {
 		}
 	}
 }
+
+// Call means this function implements the CallableFunc interface and can be
+// called statically if we want to do it speculatively or from a resource.
+func (obj *WrappedFunc) Call(args []types.Value) (types.Value, error) {
+	result, err := obj.fn.Call(args) // (Value, error)
+	if err != nil {
+		return nil, errwrap.Wrapf(err, "simple poly function errored")
+	}
+	return result, err
+}

lang/funcs/structs/call.go

@@ -211,7 +211,7 @@ func (obj *CallFunc) replaceSubGraph(newFuncValue *full.FuncValue) error {
 	// methods called on it. Nothing else. It will _not_ call Commit or
 	// Reverse. It adds to the graph, and our Commit and Reverse operations
 	// are the ones that actually make the change.
-	outputFunc, err := newFuncValue.Call(obj.init.Txn, obj.ArgVertices)
+	outputFunc, err := CallFuncValue(newFuncValue, obj.init.Txn, obj.ArgVertices)
 	if err != nil {
 		return errwrap.Wrapf(err, "could not call newFuncValue.Call()")
 	}

lang/funcs/structs/const.go

@@ -90,3 +90,19 @@ func (obj *ConstFunc) Stream(ctx context.Context) error {
 	close(obj.init.Output) // signal that we're done sending
 	return nil
 }
+
+// Call means this function implements the CallableFunc interface and can be
+// called statically if we want to do it speculatively or from a resource.
+func (obj *ConstFunc) Call(args []types.Value) (types.Value, error) {
+	if obj.Info() == nil {
+		return nil, fmt.Errorf("info is empty")
+	}
+	if obj.Info().Sig == nil {
+		return nil, fmt.Errorf("sig is empty")
+	}
+	if i, j := len(args), len(obj.Info().Sig.Ord); i != j {
+		return nil, fmt.Errorf("arg length doesn't match, got %d, exp: %d", i, j)
+	}
+
+	return obj.Value, nil
+}

lang/funcs/structs/if.go

@@ -103,12 +103,13 @@ func (obj *IfFunc) Stream(ctx context.Context) error {
 			}
 			obj.last = input // store for next
 
-			var result types.Value
-
-			if input.Struct()["c"].Bool() {
-				result = input.Struct()["a"] // true branch
-			} else {
-				result = input.Struct()["b"] // false branch
+			args, err := interfaces.StructToCallableArgs(input) // []types.Value, error
+			if err != nil {
+				return err
+			}
+			result, err := obj.Call(args)
+			if err != nil {
+				return err
 			}
 
 			// skip sending an update...
@@ -129,3 +130,22 @@ func (obj *IfFunc) Stream(ctx context.Context) error {
 		}
 	}
 }
+
+// Call means this function implements the CallableFunc interface and can be
+// called statically if we want to do it speculatively or from a resource.
+func (obj *IfFunc) Call(args []types.Value) (types.Value, error) {
+	if obj.Info() == nil {
+		return nil, fmt.Errorf("info is empty")
+	}
+	if obj.Info().Sig == nil {
+		return nil, fmt.Errorf("sig is empty")
+	}
+	if i, j := len(args), len(obj.Info().Sig.Ord); i != j {
+		return nil, fmt.Errorf("arg length doesn't match, got %d, exp: %d", i, j)
+	}
+
+	if args[0].Bool() { // condition
+		return args[1], nil // true branch
+	}
+	return args[2], nil // false branch
+}

lang/funcs/structs/util.go

@@ -18,6 +18,8 @@
 package structs
 
 import (
+	"fmt"
+
 	"github.com/purpleidea/mgmt/lang/funcs/simple"
 	"github.com/purpleidea/mgmt/lang/interfaces"
 	"github.com/purpleidea/mgmt/lang/types"
@@ -51,18 +53,9 @@ func SimpleFnToDirectFunc(name string, fv *types.FuncValue) interfaces.Func {
 // *full.FuncValue.
 func SimpleFnToFuncValue(name string, fv *types.FuncValue) *full.FuncValue {
 	return &full.FuncValue{
-		V: func(txn interfaces.Txn, args []interfaces.Func) (interfaces.Func, error) {
-			wrappedFunc := SimpleFnToDirectFunc(name, fv)
-			txn.AddVertex(wrappedFunc)
-			for i, arg := range args {
-				argName := fv.T.Ord[i]
-				txn.AddEdge(arg, wrappedFunc, &interfaces.FuncEdge{
-					Args: []string{argName},
-				})
-			}
-			return wrappedFunc, nil
-		},
-		T: fv.T,
+		Name:     &name,
+		Timeless: fv,
+		T:        fv.T,
 	}
 }
 
@@ -72,3 +65,50 @@ func SimpleFnToFuncValue(name string, fv *types.FuncValue) *full.FuncValue {
 func SimpleFnToConstFunc(name string, fv *types.FuncValue) interfaces.Func {
 	return FuncValueToConstFunc(SimpleFnToFuncValue(name, fv))
 }
+
+// FuncToFullFuncValue creates a *full.FuncValue which adds the given
+// interfaces.Func to the graph. Note that this means the *full.FuncValue
+// can only be called once.
+func FuncToFullFuncValue(name string, valueTransformingFunc interfaces.Func, typ *types.Type) *full.FuncValue {
+	return &full.FuncValue{
+		Name: &name,
+		Timeful: func(txn interfaces.Txn, args []interfaces.Func) (interfaces.Func, error) {
+			for i, arg := range args {
+				argName := typ.Ord[i]
+				txn.AddEdge(arg, valueTransformingFunc, &interfaces.FuncEdge{
+					Args: []string{argName},
+				})
+			}
+			return valueTransformingFunc, nil
+		},
+		T: typ,
+	}
+}
+
+// Call calls the function with the provided txn and args.
+func CallFuncValue(obj *full.FuncValue, txn interfaces.Txn, args []interfaces.Func) (interfaces.Func, error) {
+	if obj.Timeful != nil {
+		return obj.Timeful(txn, args)
+	}
+
+	wrappedFunc := SimpleFnToDirectFunc(*obj.Name, obj.Timeless)
+	txn.AddVertex(wrappedFunc)
+	for i, arg := range args {
+		argName := obj.T.Ord[i]
+		txn.AddEdge(arg, wrappedFunc, &interfaces.FuncEdge{
+			Args: []string{argName},
+		})
+	}
+	return wrappedFunc, nil
+}
+
+// Speculatively call the function with the provided arguments.
+// Only makes sense if the function is timeless (produces a single Value, not a
+// stream of values).
+func CallTimelessFuncValue(obj *full.FuncValue, args []types.Value) (types.Value, error) {
+	if obj.Timeless != nil {
+		return obj.Timeless.V(args)
+	}
+
+	return nil, fmt.Errorf("cannot call CallIfTimeless on a Timeful function")
+}

lang/interfaces/ast.go

@@ -109,6 +109,10 @@ type Expr interface {
 	// SetScope sets the scope here and propagates it downwards.
 	SetScope(*Scope, map[string]Expr) error
 
+	// Ensure that only the specified ExprParams are free in this expression.
+	// XXX: JAMES: REPLACE WITH WITH A CALL TO APPLY() AFTER EVERYTHING WORKS.
+	CheckParamScope(map[Expr]struct{}) error
+
 	// SetType sets the type definitively, and errors if it is incompatible.
 	SetType(*types.Type) error
 

lang/interfaces/func.go

@@ -190,6 +190,19 @@ type OldPolyFunc interface {
 	Build(*types.Type) (*types.Type, error)
 }
 
+// CallableFunc is a function that can be called statically if we want to do it
+// speculatively or from a resource.
+type CallableFunc interface {
+	Func // implement everything in Func but add the additional requirements
+
+	// Call this function with the input args and return the value if it is
+	// possible to do so at this time. To transform from the single value,
+	// graph representation of the callable values into a linear, standard
+	// args list for use here, you can use the StructToCallableArgs
+	// function.
+	Call(args []types.Value) (types.Value, error)
+}
+
 // NamedArgsFunc is a function that uses non-standard function arg names. If you
 // don't implement this, then the argnames (if specified) must correspond to the
 // a, b, c...z, aa, ab...az, ba...bz, and so on sequence.
@@ -344,3 +357,33 @@ type Txn interface {
 	// committed.
 	Graph() *pgraph.Graph
 }
+
+// StructToCallableArgs transforms the single value, graph representation of the
+// callable values into a linear, standard args list.
+func StructToCallableArgs(st types.Value) ([]types.Value, error) {
+	if st == nil {
+		return nil, fmt.Errorf("empty struct")
+	}
+	typ := st.Type()
+	if typ == nil {
+		return nil, fmt.Errorf("empty type")
+	}
+	if kind := typ.Kind; kind != types.KindStruct {
+		return nil, fmt.Errorf("incorrect kind, got: %s", kind)
+	}
+	structValues := st.Struct() // map[string]types.Value
+	if structValues == nil {
+		return nil, fmt.Errorf("empty values")
+	}
+
+	args := []types.Value{}
+	for i, x := range typ.Ord { // in the correct order
+		v, exists := structValues[x]
+		if !exists {
+			return nil, fmt.Errorf("invalid input value at %d", i)
+		}
+
+		args = append(args, v)
+	}
+	return args, nil
+}

lang/interfaces/structs.go

@@ -78,6 +78,8 @@ func (obj *ExprAny) Ordering(produces map[string]Node) (*pgraph.Graph, map[Node]
 // does not need to know about the parent scope.
 func (obj *ExprAny) SetScope(*Scope, map[string]Expr) error { return nil }
 
+func (obj *ExprAny) CheckParamScope(freeVars map[Expr]struct{}) error { return nil }
+
 // SetType is used to set the type of this expression once it is known. This
 // usually happens during type unification, but it can also happen during
 // parsing if a type is specified explicitly. Since types are static and don't

lang/interpret_test/TestAstFunc1/optimized-higher-order-function.txtar

@@ -0,0 +1,7 @@
+func apply($f, $x) {
+  $f($x)
+}
+$add1 = fn($x) {
+  $x + 1
+}
+$z = apply($add, 1)