feat(config): support for-loops for lists

You can now map through a list of values by using the special
`$forEach/$return` object.

You specify an object with two keys, `$forEach: <some list>` and
`$return: <any value>`. You can also optionally add a
`$filter: <expression>` key, which if evaluates to `false`
for a particular value, it will be omitted.

See the added docs for more details and examples.

core/src/config/common.ts

@@ -16,6 +16,9 @@ import { DEFAULT_API_VERSION } from "../constants"
 
 export const objectSpreadKey = "$merge"
 export const arrayConcatKey = "$concat"
+export const arrayForEachKey = "$forEach"
+export const arrayForEachReturnKey = "$return"
+export const arrayForEachFilterKey = "$filter"
 
 const ajv = new Ajv({ allErrors: true, useDefaults: true })
 
@@ -315,9 +318,14 @@ joi = joi.extend({
   //   return { value }
   // },
   args(schema: any, keys: any) {
-    // Always allow the $merge key, which we resolve and collapse in resolveTemplateStrings()
+    // Always allow the special $merge, $forEach etc. keys, which we resolve and collapse in resolveTemplateStrings()
+    // Note: we allow both the expected schema and strings, since they may be templates resolving to the expected type.
     return schema.keys({
       [objectSpreadKey]: joi.alternatives(joi.object(), joi.string()),
+      [arrayConcatKey]: joi.alternatives(joi.array(), joi.string()),
+      [arrayForEachKey]: joi.alternatives(joi.array(), joi.string()),
+      [arrayForEachFilterKey]: joi.any(),
+      [arrayForEachReturnKey]: joi.any(),
       ...(keys || {}),
     })
   },

core/src/config/template-contexts/base.ts

@@ -53,7 +53,7 @@ export function schema(joiSchema: Joi.Schema) {
 // Note: we're using classes here to be able to use decorators to describe each context node and key
 export abstract class ConfigContext {
   private readonly _rootContext: ConfigContext
-  private readonly _resolvedValues: { [path: string]: string }
+  private readonly _resolvedValues: { [path: string]: any }
 
   // This is used for special-casing e.g. runtime.* resolution
   protected _alwaysAllowPartial: boolean

core/src/template-string/template-string.ts

@@ -13,11 +13,21 @@ import {
   ScanContext,
   ContextResolveOutput,
   ContextKeySegment,
+  GenericContext,
 } from "../config/template-contexts/base"
-import { difference, uniq, isPlainObject, isNumber } from "lodash"
-import { Primitive, StringMap, isPrimitive, objectSpreadKey, arrayConcatKey } from "../config/common"
+import { difference, uniq, isPlainObject, isNumber, cloneDeep } from "lodash"
+import {
+  Primitive,
+  StringMap,
+  isPrimitive,
+  objectSpreadKey,
+  arrayConcatKey,
+  arrayForEachKey,
+  arrayForEachReturnKey,
+  arrayForEachFilterKey,
+} from "../config/common"
 import { profile } from "../util/profiling"
-import { dedent, deline, truncate } from "../util/string"
+import { dedent, deline, naturalList, truncate } from "../util/string"
 import { ObjectWithName } from "../util/util"
 import { LogEntry } from "../logger/log-entry"
 import { ModuleConfigContext } from "../config/template-contexts/module"
@@ -206,9 +216,23 @@ export const resolveTemplateStrings = profile(function $resolveTemplateStrings<T
     const output: unknown[] = []
 
     for (const v of value) {
-      if (isPlainObject(v) && v.hasOwnProperty(arrayConcatKey)) {
+      if (isPlainObject(v) && v[arrayConcatKey] !== undefined) {
+        if (Object.keys(v).length > 1) {
+          const extraKeys = naturalList(
+            Object.keys(v)
+              .filter((k) => k !== arrayConcatKey)
+              .map((k) => JSON.stringify(k))
+          )
+          throw new ConfigurationError(
+            `A list item with a ${arrayConcatKey} key cannot have any other keys (found ${extraKeys})`,
+            {
+              value: v,
+            }
+          )
+        }
+
         // Handle array concatenation via $concat
-        const resolved = resolveTemplateStrings(v.$concat, context, opts)
+        const resolved = resolveTemplateStrings(v[arrayConcatKey], context, opts)
 
         if (Array.isArray(resolved)) {
           output.push(...resolved)
@@ -230,37 +254,124 @@ export const resolveTemplateStrings = profile(function $resolveTemplateStrings<T
 
     return <T>(<unknown>output)
   } else if (isPlainObject(value)) {
-    // Resolve $merge keys, depth-first, leaves-first
-    let output = {}
-
-    for (const [k, v] of Object.entries(value)) {
-      const resolved = resolveTemplateStrings(v, context, opts)
-
-      if (k === objectSpreadKey) {
-        if (isPlainObject(resolved)) {
-          output = { ...output, ...resolved }
-        } else if (opts.allowPartial) {
-          output[k] = resolved
+    if (value[arrayForEachKey] !== undefined) {
+      // Handle $forEach loop
+      return handleForEachObject(value, context, opts)
+    } else {
+      // Resolve $merge keys, depth-first, leaves-first
+      let output = {}
+
+      for (const [k, v] of Object.entries(value)) {
+        const resolved = resolveTemplateStrings(v, context, opts)
+
+        if (k === objectSpreadKey) {
+          if (isPlainObject(resolved)) {
+            output = { ...output, ...resolved }
+          } else if (opts.allowPartial) {
+            output[k] = resolved
+          } else {
+            throw new ConfigurationError(
+              `Value of ${objectSpreadKey} key must be (or resolve to) a mapping object (got ${typeof resolved})`,
+              {
+                value,
+                resolved,
+              }
+            )
+          }
         } else {
-          throw new ConfigurationError(
-            `Value of ${objectSpreadKey} key must be (or resolve to) a mapping object (got ${typeof resolved})`,
-            {
-              value,
-              resolved,
-            }
-          )
+          output[k] = resolved
         }
-      } else {
-        output[k] = resolved
       }
-    }
 
-    return <T>output
+      return <T>output
+    }
   } else {
     return <T>value
   }
 })
 
+const expectedKeys = [arrayForEachKey, arrayForEachReturnKey, arrayForEachFilterKey]
+
+function handleForEachObject(value: any, context: ConfigContext, opts: ContextResolveOpts) {
+  // Validate input object
+  if (value[arrayForEachReturnKey] === undefined) {
+    throw new ConfigurationError(`Missing ${arrayForEachReturnKey} field next to ${arrayForEachKey} field.`, {
+      value,
+    })
+  }
+
+  const unexpectedKeys = Object.keys(value).filter((k) => !expectedKeys.includes(k))
+
+  if (unexpectedKeys.length > 0) {
+    const extraKeys = naturalList(unexpectedKeys.map((k) => JSON.stringify(k)))
+
+    throw new ConfigurationError(`Found one or more unexpected keys on $forEach object: ${extraKeys}`, {
+      value,
+      expectedKeys,
+      unexpectedKeys,
+    })
+  }
+
+  // Try resolving the value of the $forEach key
+  let resolvedInput = resolveTemplateStrings(value[arrayForEachKey], context, opts)
+  const isObject = isPlainObject(resolvedInput)
+
+  if (!Array.isArray(resolvedInput) && !isObject) {
+    if (opts.allowPartial) {
+      return value
+    } else {
+      throw new ConfigurationError(
+        `Value of ${arrayForEachKey} key must be (or resolve to) an array or mapping object (got ${typeof resolvedInput})`,
+        {
+          value,
+          resolved: resolvedInput,
+        }
+      )
+    }
+  }
+
+  const filterExpression = value[arrayForEachFilterKey]
+
+  // TODO: maybe there's a more efficient way to do the cloning/extending?
+  const loopContext = cloneDeep(context)
+
+  const output: unknown[] = []
+
+  for (const i of Object.keys(resolvedInput)) {
+    const itemValue = resolvedInput[i]
+
+    loopContext["item"] = new GenericContext({ key: i, value: itemValue })
+
+    // Have to override the cache in the parent context here
+    // TODO: make this a little less hacky :P
+    delete loopContext["_resolvedValues"]["item.key"]
+    delete loopContext["_resolvedValues"]["item.value"]
+
+    // Check $filter clause output, if applicable
+    if (filterExpression !== undefined) {
+      const filterResult = resolveTemplateStrings(value[arrayForEachFilterKey], loopContext, opts)
+
+      if (filterResult === false) {
+        continue
+      } else if (filterResult !== true) {
+        throw new ConfigurationError(
+          `${arrayForEachFilterKey} clause in ${arrayForEachKey} loop must resolve to a boolean value (got ${typeof resolvedInput})`,
+          {
+            itemValue,
+            filterExpression,
+            filterResult,
+          }
+        )
+      }
+    }
+
+    output.push(resolveTemplateStrings(value[arrayForEachReturnKey], loopContext, opts))
+  }
+
+  // Need to resolve once more to handle e.g. $concat expressions
+  return resolveTemplateStrings(output, context, opts)
+}
+
 /**
  * Scans for all template strings in the given object and lists the referenced keys.
  */