[#1034] made TM placeholders resolvement required, failing with a 400…

… bad request

* reworked where placeholders are loaded from - from attributes/model-placeholders for Things and from properties/model-placeholders for Features
* use the Ditto config only as fallback when not finding the placeholder in the Thing/Feature
* support all possible Json types

Signed-off-by: Thomas Jaeckle <thomas.jaeckle@bosch.io>

documentation/src/main/resources/pages/ditto/basic-wot-integration.md

@@ -250,30 +250,34 @@ curl -u ditto:ditto 'http://localhost:8080/api/2/things/io.eclipserpojects.ditto
 #### Resolving Thing Model placeholders
 
 WoT Thing Models may contain [placeholders](https://www.w3.org/TR/wot-thing-description11/#thing-model-td-placeholder)
-which must be resolved during generation of the TD from a TM.  
-Those placeholders can be configured in Ditto via the 
-[things.conf](https://github.com/eclipse/ditto/blob/master/things/service/src/main/resources/things.conf) configuration
-file of the [things](architecture-services-things.html) service at path 
-`ditto.things.wot.to-thing-description.placeholders`.  
-This map may either contain static values, or also contain dynamic values which are then resolved by the Thing's or Feature's 
-JSON payload:
-
-```
-TITLE = "json:/attributes/title" # dynamically resolve variables via thing content
-```
-
-This way, the Thing Model must not contain Ditto specific paths, but only define placeholders like `{{ TITLE }}` and via
-Ditto configuration, such placeholders may dynamically be resolved by e.g. a Thing attribute value.
-
-{% include warning.html content="Please be aware that such dynamically resolved placeholders from the Thing are not 
+which **must** be resolved during generation of the TD from a TM.  
+
+In order to resolve TM placeholders, Ditto applies the following strategy:
+* when generating a TD for a Thing, it looks in the Things' attribute `"model-placeholders"` (being a JsonObject) in
+   order to lookup placeholders
+* when generating a TD for a Feature, it looks in the Feature's property `"model-placeholders"` (being a JsonObject) in
+   order to lookup placeholders
+* when a placeholder was not found in the `"model-placeholders"` of the Thing/Feature, a fallback to the Ditto 
+  configuration is done:
+    * placeholder fallbacks can be configured in Ditto via the 
+      [things.conf](https://github.com/eclipse/ditto/blob/master/things/service/src/main/resources/things.conf) 
+      configuration file of the [things](architecture-services-things.html) service at path 
+      `ditto.things.wot.to-thing-description.placeholders`.<br/>  
+      This map may contain static values, but use and Json type as value (e.g. also a JsonObject), e.g.:
+      ```
+      FOO = "bar"
+      TM_REQUIRED = [
+        "#/properties/status",
+        "#/actions/toggle"
+      ]
+      ```
+
+{% include warning.html content="Please be aware that placeholders put into the `"model-placeholders"` attribute/property 
+    of a Thing/Feature may be used in TM placeholders and therefore are not 
     protected by any authorization check based on the Thing's [Policy](basic-policy.html) as TDs are available for all
     authenticated users which know the Thing ID."
 %}
 
-{% include note.html content="Currently, Ditto only supports string placeholders, the datatype is not converted to e.g. 
-    an integer or a boolean after resolving placeholders."
-%}
-
 ### Thing skeleton generation upon Thing creation
 
 Prerequisites to use the skeleton generation during Thing creation:

things/service/src/main/resources/things.conf

@@ -114,7 +114,10 @@ ditto {
         placeholders {
           # add arbitrary placeholders to be resolved, e.g.:
           # FOO = "bar"
-          # TITLE = "json:/attributes/title" # dynamically resolve variables via thing content
+          # TM_REQUIRED = [
+          #  "#/properties/status",
+          #  "#/actions/toggle"
+          # ]
         }
 
         add-created = true

wot/integration/src/main/java/org/eclipse/ditto/wot/integration/config/DefaultToThingDescriptionConfig.java

@@ -21,7 +21,9 @@
 import org.eclipse.ditto.internal.utils.config.ConfigWithFallback;
 import org.eclipse.ditto.internal.utils.config.ScopedConfig;
 import org.eclipse.ditto.json.JsonFactory;
+import org.eclipse.ditto.json.JsonField;
 import org.eclipse.ditto.json.JsonObject;
+import org.eclipse.ditto.json.JsonValue;
 
 import com.typesafe.config.Config;
 import com.typesafe.config.ConfigRenderOptions;
@@ -36,7 +38,7 @@ final class DefaultToThingDescriptionConfig implements ToThingDescriptionConfig
 
     private final String basePrefix;
     private final JsonObject jsonTemplate;
-    private final Map<String, String> placeholders;
+    private final Map<String, JsonValue> placeholders;
     private final boolean addCreated;
     private final boolean addModified;
 
@@ -49,7 +51,7 @@ private DefaultToThingDescriptionConfig(final ScopedConfig scopedConfig) {
                 scopedConfig.getValue(ConfigValue.PLACEHOLDERS.getConfigPath()).render(ConfigRenderOptions.concise())
         ).asObject()
                 .stream()
-                .collect(Collectors.toMap(f -> f.getKey().toString(), f -> f.getValue().formatAsString()));
+                .collect(Collectors.toMap(f -> f.getKey().toString(), JsonField::getValue));
         addCreated = scopedConfig.getBoolean(ConfigValue.ADD_CREATED.getConfigPath());
         addModified = scopedConfig.getBoolean(ConfigValue.ADD_MODIFIED.getConfigPath());
     }
@@ -77,7 +79,7 @@ public JsonObject getJsonTemplate() {
     }
 
     @Override
-    public Map<String, String> getPlaceholders() {
+    public Map<String, JsonValue> getPlaceholders() {
         return placeholders;
     }
 

wot/integration/src/main/java/org/eclipse/ditto/wot/integration/config/ToThingDescriptionConfig.java

@@ -18,6 +18,7 @@
 
 import org.eclipse.ditto.internal.utils.config.KnownConfigValue;
 import org.eclipse.ditto.json.JsonObject;
+import org.eclipse.ditto.json.JsonValue;
 
 /**
  * Provides configuration settings for WoT (Web of Things) integration regarding the Thing Description transformation
@@ -49,7 +50,7 @@ public interface ToThingDescriptionConfig {
      *
      * @return the map containing placeholder constants to use in order to resolve WoT TM placeholders.
      */
-    Map<String, String> getPlaceholders();
+    Map<String, JsonValue> getPlaceholders();
 
     /**
      * Returns whether to add {@code "created"} to each TD reflecting the created timestamp of the Thing.

wot/integration/src/main/java/org/eclipse/ditto/wot/integration/generator/DefaultWotThingDescriptionGenerator.java

@@ -95,6 +95,7 @@
 import org.eclipse.ditto.wot.model.UriVariables;
 import org.eclipse.ditto.wot.model.Version;
 import org.eclipse.ditto.wot.model.WotThingModelInvalidException;
+import org.eclipse.ditto.wot.model.WotThingModelPlaceholderUnresolvedException;
 
 import akka.actor.ActorSystem;
 
@@ -177,7 +178,8 @@ public ThingDescription generateThingDescription(final ThingId thingId,
                 Map.of(SCHEMA_DITTO_ERROR, buildDittoErrorSchema())
         ));
 
-        final ThingDescription thingDescription = resolvePlaceholders(tdBuilder.build(), placeholderLookupObject);
+        final ThingDescription thingDescription = resolvePlaceholders(tdBuilder.build(), placeholderLookupObject,
+                dittoHeaders);
         LOGGER.withCorrelationId(dittoHeaders)
                 .info("Created ThingDescription for thingId <{}> and featureId <{}>: <{}>", thingId, featureId,
                         thingDescription);
@@ -804,66 +806,73 @@ private EventFormElement buildEventFormElement(final SingleEventFormElementOp op
     }
 
     private ThingDescription resolvePlaceholders(final ThingDescription tdWithPotentialPlaceholders,
-            @Nullable final JsonObject o) {
-        return ThingDescription.fromJson(resolvePlaceholders(tdWithPotentialPlaceholders.toJson(), o));
+            @Nullable final JsonObject modelPlaceholders, final DittoHeaders dittoHeaders) {
+        return ThingDescription.fromJson(
+                resolvePlaceholders(tdWithPotentialPlaceholders.toJson(), modelPlaceholders, dittoHeaders)
+        );
     }
 
-    private JsonObject resolvePlaceholders(final JsonObject jsonObject, @Nullable final JsonObject o) {
+    private JsonObject resolvePlaceholders(final JsonObject jsonObject, @Nullable final JsonObject modelPlaceholders,
+            final DittoHeaders dittoHeaders) {
         return jsonObject.stream()
                 .map(field -> {
                     final JsonKey key = field.getKey();
                     final JsonValue value = field.getValue();
                     if (value.isString()) {
-                        return JsonField.newInstance(key, JsonValue.of(resolvePlaceholder(value.asString(), o)));
+                        return JsonField.newInstance(key, resolvePlaceholder(value.asString(), modelPlaceholders)
+                                .orElseThrow(() -> WotThingModelPlaceholderUnresolvedException
+                                        .newBuilder(value.asString())
+                                        .dittoHeaders(dittoHeaders)
+                                        .build()));
                     } else if (value.isObject()) {
-                        return JsonField.newInstance(key, resolvePlaceholders(value.asObject(), o)); // recurse!
+                        return JsonField.newInstance(key,
+                                resolvePlaceholders(value.asObject(), modelPlaceholders, dittoHeaders) // recurse!
+                        );
                     } else if (value.isArray()) {
-                        return JsonField.newInstance(key, resolvePlaceholders(value.asArray(), o));
+                        return JsonField.newInstance(key,
+                                resolvePlaceholders(value.asArray(), modelPlaceholders, dittoHeaders)
+                        );
                     } else {
                         return field;
                     }
                 })
                 .collect(JsonCollectors.fieldsToObject());
     }
 
-    private JsonArray resolvePlaceholders(final JsonArray jsonArray, @Nullable final JsonObject o) {
+    private JsonArray resolvePlaceholders(final JsonArray jsonArray, @Nullable final JsonObject modelPlaceholders,
+            final DittoHeaders dittoHeaders) {
         return jsonArray.stream()
                 .map(arrValue -> {
                     if (arrValue.isString()) {
-                        return JsonValue.of(resolvePlaceholder(arrValue.asString(), o));
+                        return resolvePlaceholder(arrValue.asString(), modelPlaceholders)
+                                .orElseThrow(() -> WotThingModelPlaceholderUnresolvedException
+                                        .newBuilder(arrValue.asString())
+                                        .dittoHeaders(dittoHeaders)
+                                        .build());
                     } else if (arrValue.isObject()) {
-                        return resolvePlaceholders(arrValue.asObject(), o);
+                        return resolvePlaceholders(arrValue.asObject(), modelPlaceholders, dittoHeaders);
                     } else if (arrValue.isArray()) {
-                        return resolvePlaceholders(arrValue.asArray(), o); // recurse!
+                        return resolvePlaceholders(arrValue.asArray(), modelPlaceholders, dittoHeaders); // recurse!
                     } else {
                         return arrValue;
                     }
                 })
                 .collect(JsonCollectors.valuesToArray());
     }
 
-    private String resolvePlaceholder(final String value, @Nullable final JsonObject o) {
+    private Optional<JsonValue> resolvePlaceholder(final String value, @Nullable final JsonObject modelPlaceholders) {
         final Matcher matcher = TM_PLACEHOLDER_PATTERN.matcher(value);
         if (matcher.matches()) {
             final String placeholderToResolve = matcher.group(TM_PLACEHOLDER_PL_GROUP).trim();
-            final String staticallyResolvedPlaceholder = toThingDescriptionConfig.getPlaceholders()
-                    .getOrDefault(placeholderToResolve, placeholderToResolve);
-
-            if (null != o && staticallyResolvedPlaceholder.startsWith("json:/")) {
-                return resolvePlaceholderInThingPayload(o, staticallyResolvedPlaceholder);
+            if (null != modelPlaceholders) {
+                return modelPlaceholders.getValue(placeholderToResolve)
+                        .or(() -> Optional.ofNullable(toThingDescriptionConfig.getPlaceholders().get(placeholderToResolve)));
             } else {
-                return "{{" + staticallyResolvedPlaceholder + "}}";
+                return Optional.ofNullable(toThingDescriptionConfig.getPlaceholders().get(placeholderToResolve));
             }
         } else {
-            return value;
+            return Optional.of(JsonValue.of(value));
         }
     }
 
-    private String resolvePlaceholderInThingPayload(final JsonObject jsonObject, final String placeholderToResolve) {
-        final JsonPointer pointerToResolve = JsonPointer.of(placeholderToResolve.substring("json:".length()));
-        return jsonObject.getValue(pointerToResolve)
-                .map(JsonValue::formatAsString)
-                .orElse("{{" + placeholderToResolve + "}}");
-    }
-
 }

wot/integration/src/main/java/org/eclipse/ditto/wot/integration/provider/DefaultWotThingDescriptionProvider.java

@@ -27,6 +27,7 @@
 import org.eclipse.ditto.internal.utils.akka.logging.DittoLoggerFactory;
 import org.eclipse.ditto.internal.utils.akka.logging.ThreadSafeDittoLogger;
 import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig;
+import org.eclipse.ditto.json.JsonValue;
 import org.eclipse.ditto.things.model.DefinitionIdentifier;
 import org.eclipse.ditto.things.model.Feature;
 import org.eclipse.ditto.things.model.FeatureDefinition;
@@ -54,6 +55,8 @@ final class DefaultWotThingDescriptionProvider implements WotThingDescriptionPro
     private static final ThreadSafeDittoLogger LOGGER =
             DittoLoggerFactory.getThreadSafeLogger(DefaultWotThingDescriptionProvider.class);
 
+    public static final String MODEL_PLACEHOLDERS_KEY = "model-placeholders";
+
     private final WotConfig wotConfig;
     private final WotThingModelFetcher thingModelFetcher;
     private final WotThingDescriptionGenerator thingDescriptionGenerator;
@@ -194,7 +197,12 @@ private ThingDescription getWotThingDescriptionForThing(final ThingDefinition de
                         .thenApply(thingModel -> thingDescriptionGenerator
                                 .generateThingDescription(thingId,
                                         thing,
-                                        Optional.ofNullable(thing).map(Thing::toJson).orElse(null),
+                                        Optional.ofNullable(thing)
+                                                .flatMap(Thing::getAttributes)
+                                                .flatMap(a -> a.getValue(MODEL_PLACEHOLDERS_KEY))
+                                                .filter(JsonValue::isObject)
+                                                .map(JsonValue::asObject)
+                                                .orElse(null),
                                         null,
                                         thingModel,
                                         url,
@@ -235,7 +243,11 @@ private ThingDescription getWotThingDescriptionForFeature(final ThingId thingId,
                         .thenApply(thingModel -> thingDescriptionGenerator
                                 .generateThingDescription(thingId,
                                         thing,
-                                        feature.toJson(),
+                                        feature.getProperties()
+                                                .flatMap(p -> p.getValue(MODEL_PLACEHOLDERS_KEY))
+                                                .filter(JsonValue::isObject)
+                                                .map(JsonValue::asObject)
+                                                .orElse(null),
                                         feature.getId(),
                                         thingModel,
                                         url,

wot/model/src/main/java/org/eclipse/ditto/wot/model/WotThingModelInvalidException.java

@@ -31,6 +31,7 @@
 
 /**
  * Thrown if a downloaded WoT (Web of Things) ThingModel was invalid.
+ *
  * @since 2.4.0
  */
 @Immutable

wot/model/src/main/java/org/eclipse/ditto/wot/model/WotThingModelNotAccessibleException.java

@@ -32,6 +32,7 @@
 /**
  * Thrown if a WoT (Web of Things) ThingModel could not be accessed (e.g. because the URL is not accessible or public
  * access to it is restricted).
+ *
  * @since 2.4.0
  */
 @Immutable
@@ -44,7 +45,7 @@ public final class WotThingModelNotAccessibleException extends DittoRuntimeExcep
     public static final String ERROR_CODE = ERROR_CODE_PREFIX + "tm.notfound";
 
     private static final String MESSAGE_TEMPLATE =
-            "The ThingModel at URI ''{0}'' could not be accessed.";
+            "The WoT ThingModel at URI ''{0}'' could not be accessed.";
 
     private static final String DEFAULT_DESCRIPTION =
             "Please ensure that the linked ThingModel is publicly available in order to be downloaded.";