Skip to content

Commit

Permalink
enable loading additional extra JavaScript libraries which can be use…
Browse files Browse the repository at this point in the history 
…d in Ditto's JS based payload mapping

* Rhino's "CommonJS" loading feature is used, enabling use of `require("module")`
* connectivity service can optionally be configured to load additional JS modules from a file path
* via Docker container volume mount, additional libraries can be provided that way
* no packaging of the JS modules in Ditto is required
* added unit test using protobufjs via "require"

Signed-off-by: Thomas Jaeckle <thomas.jaeckle@bosch.io>
  • Loading branch information
@thjaeckle
thjaeckle committed Oct 17, 2021
1 parent 632bcca commit 96cfbe4
Show file tree
Hide file tree
Showing 8 changed files with 517 additions and 19 deletions.
40 changes: 40 additions & 0 deletions connectivity/service/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,18 @@ jmh-generator-annprocess). jmh-generator-annprocess overwrites the whole META-IN
<artifactId>akka-stream-kafka-testkit_${scala.version}</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.webjars.npm</groupId>
<artifactId>github-com-protobufjs-protobuf-js</artifactId>
<version>6.11.1</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.webjars.npm</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>

<build>
Expand Down Expand Up @@ -324,6 +336,34 @@ jmh-generator-annprocess). jmh-generator-annprocess overwrites the whole META-IN
</compilerArgs>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>unpack-webjars</id>
<phase>generate-test-resources</phase>
<goals>
<goal>unpack</goal>
</goals>
<configuration>
<artifactItems>
<artifactItem>
<groupId>org.webjars.npm</groupId>
<artifactId>github-com-protobufjs-protobuf-js</artifactId>
<version>6.11.1</version>
<type>jar</type>
<includes>META-INF/resources/webjars/github-com-protobufjs-protobuf-js/6.11.1/protobuf.min.js</includes>
<fileMappers>
<org.codehaus.plexus.components.io.filemappers.FlattenFileMapper/>
</fileMappers>
</artifactItem>
</artifactItems>
<outputDirectory>${project.build.testOutputDirectory}/unpacked-test-webjars</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
Expand Down
23 changes: 21 additions & 2 deletions ...ava/org/eclipse/ditto/connectivity/service/config/javascript/DefaultJavaScriptConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,12 @@
*/
package org.eclipse.ditto.connectivity.service.config.javascript;

import java.nio.file.Path;
import java.time.Duration;
import java.util.Objects;
import java.util.Optional;

import javax.annotation.Nullable;
import javax.annotation.concurrent.Immutable;

import org.eclipse.ditto.internal.utils.config.ConfigWithFallback;
Expand All @@ -34,6 +37,7 @@ public final class DefaultJavaScriptConfig implements JavaScriptConfig {
private final Duration maxScriptExecutionTime;
private final int maxScriptStackDepth;
private final boolean allowUnsafeStandardObjects;
@Nullable private final Path commonJsModulesPath;

private DefaultJavaScriptConfig(final ScopedConfig config) {
maxScriptSizeBytes = config.getPositiveIntOrThrow(JavaScriptConfigValue.MAX_SCRIPT_SIZE_BYTES);
Expand All @@ -42,6 +46,13 @@ private DefaultJavaScriptConfig(final ScopedConfig config) {
maxScriptStackDepth = config.getPositiveIntOrThrow(JavaScriptConfigValue.MAX_SCRIPT_STACK_DEPTH);
allowUnsafeStandardObjects = config.getBoolean(JavaScriptConfigValue.ALLOW_UNSAFE_STANDARD_OBJECTS
.getConfigPath());
final String commonJsModulesPathString = config.getString(
JavaScriptConfigValue.COMMON_JS_MODULE_PATH.getConfigPath());
if (commonJsModulesPathString.isEmpty()) {
commonJsModulesPath = null;
} else {
commonJsModulesPath = Path.of(commonJsModulesPathString);
}
}

/**
Expand Down Expand Up @@ -76,6 +87,11 @@ public boolean isAllowUnsafeStandardObjects() {
return allowUnsafeStandardObjects;
}

@Override
public Optional<Path> getCommonJsModulesPath() {
return Optional.ofNullable(commonJsModulesPath);
}

@Override
public boolean equals(final Object o) {
if (this == o) {
Expand All @@ -88,12 +104,14 @@ public boolean equals(final Object o) {
return maxScriptSizeBytes == that.maxScriptSizeBytes &&
maxScriptStackDepth == that.maxScriptStackDepth &&
allowUnsafeStandardObjects == that.allowUnsafeStandardObjects &&
Objects.equals(maxScriptExecutionTime, that.maxScriptExecutionTime);
Objects.equals(maxScriptExecutionTime, that.maxScriptExecutionTime) &&
Objects.equals(commonJsModulesPath, that.commonJsModulesPath);
}

@Override
public int hashCode() {
return Objects.hash(maxScriptSizeBytes, maxScriptExecutionTime, maxScriptStackDepth, allowUnsafeStandardObjects);
return Objects.hash(maxScriptSizeBytes, maxScriptExecutionTime, maxScriptStackDepth, allowUnsafeStandardObjects,
commonJsModulesPath);
}

@Override
Expand All @@ -103,6 +121,7 @@ public String toString() {
", maxScriptExecutionTime=" + maxScriptExecutionTime +
", maxScriptStackDepth=" + maxScriptStackDepth +
", allowUnsafeStandardObjects=" + allowUnsafeStandardObjects +
", commonJsModulesPath=" + commonJsModulesPath +
"]";
}

Expand Down
22 changes: 19 additions & 3 deletions .../main/java/org/eclipse/ditto/connectivity/service/config/javascript/JavaScriptConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@
*/
package org.eclipse.ditto.connectivity.service.config.javascript;

import java.nio.file.Path;
import java.time.Duration;
import java.util.Optional;

import javax.annotation.concurrent.Immutable;

Expand Down Expand Up @@ -56,6 +58,14 @@ public interface JavaScriptConfig {
*/
boolean isAllowUnsafeStandardObjects();

/**
* Returns an optional file Path where to load additional Javascript libraries via {@code require()} from, utilizing
* the Rhino engine feature for CommonJS
*
* @return the optional path to resolve JS modules via commonJS from.
*/
Optional<Path> getCommonJsModulesPath();

/**
* An enumeration of the known config path expressions and their associated default values for
* {@code JavaScriptConfig}.
Expand All @@ -75,17 +85,23 @@ enum JavaScriptConfigValue implements KnownConfigValue {
/**
* The maximum call stack depth in the mapping script.
*/
MAX_SCRIPT_STACK_DEPTH("maxScriptStackDepth", 10),
MAX_SCRIPT_STACK_DEPTH("maxScriptStackDepth", 25),

/**
* Whether to allow using 'print', 'exit', 'quit' in JavaScript executions, only intended for debugging purposes.
*/
ALLOW_UNSAFE_STANDARD_OBJECTS("allowUnsafeStandardObjects", false);
ALLOW_UNSAFE_STANDARD_OBJECTS("allowUnsafeStandardObjects", false),

/**
* The filesystem path where to load CommonJS modules from, by default empty indicating to not load any CommonJS
* modules.
*/
COMMON_JS_MODULE_PATH("commonJsModulePath", "");

private final String path;
private final Object defaultValue;

private JavaScriptConfigValue(final String thePath, final Object theDefaultValue) {
JavaScriptConfigValue(final String thePath, final Object theDefaultValue) {
path = thePath;
defaultValue = theDefaultValue;
}
Expand Down
27 changes: 25 additions & 2 deletions ...g/eclipse/ditto/connectivity/service/mapping/javascript/JavaScriptMessageMapperRhino.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,10 @@
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
Expand All @@ -33,6 +37,9 @@
import org.mozilla.javascript.ContextFactory;
import org.mozilla.javascript.RhinoException;
import org.mozilla.javascript.Scriptable;
import org.mozilla.javascript.commonjs.module.RequireBuilder;
import org.mozilla.javascript.commonjs.module.provider.SoftCachingModuleScriptProvider;
import org.mozilla.javascript.commonjs.module.provider.UrlModuleSourceProvider;

/**
* This mapper executes its mapping methods on the <b>current thread</b>. The caller should be aware of that.
Expand Down Expand Up @@ -99,7 +106,7 @@ public void doConfigure(final MappingConfig mappingConfig, final MessageMapperCo
} else {
scope = cx.initSafeStandardObjects(); // that one disables "print, exit, quit", etc.
}
initLibraries(cx, scope);
initLibraries(cx, scope, javaScriptConfig.getCommonJsModulesPath().orElse(null));
return scope;
});
} catch (final RhinoException e) {
Expand All @@ -124,7 +131,7 @@ public List<ExternalMessage> map(final Adaptable adaptable) {
return outgoingMapping.apply(adaptable);
}

private void initLibraries(final Context cx, final Scriptable scope) {
private void initLibraries(final Context cx, final Scriptable scope, @Nullable final Path commonJsModulePath) {
if (getConfiguration().map(JavaScriptMessageMapperConfiguration::isLoadLongJS).orElse(false)) {
loadJavascriptLibrary(cx, scope, new InputStreamReader(getClass().getResourceAsStream(WEBJARS_LONG)),
WEBJARS_LONG);
Expand All @@ -134,6 +141,22 @@ private void initLibraries(final Context cx, final Scriptable scope) {
WEBJARS_BYTEBUFFER);
}

final List<URI> paths = new ArrayList<>();
try {
paths.add(getClass().getResource(WEBJARS_LONG).toURI());
paths.add(getClass().getResource(WEBJARS_BYTEBUFFER).toURI());
} catch (URISyntaxException e) {
throw new IllegalStateException("Could not webjars", e);
}
if (null != commonJsModulePath) {
paths.add(commonJsModulePath.toUri());
}
new RequireBuilder().setModuleScriptProvider(
new SoftCachingModuleScriptProvider(new UrlModuleSourceProvider(paths, null)))
.setSandboxed(true)
.createRequire(cx, scope)
.install(scope);

loadJavascriptLibrary(cx, scope, new InputStreamReader(getClass().getResourceAsStream(DITTO_SCOPE_SCRIPT)),
DITTO_SCOPE_SCRIPT);
loadJavascriptLibrary(cx, scope, new InputStreamReader(getClass().getResourceAsStream(INCOMING_SCRIPT)),
Expand Down
2 changes: 2 additions & 0 deletions connectivity/service/src/main/resources/connectivity.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -512,6 +512,8 @@ ditto {
maxScriptStackDepth = 25
# Whether to allow using 'print', 'exit', 'quit' in JavaScript executions, only intended for debugging purposes
allowUnsafeStandardObjects = false
# The filesystem path where to load CommonJS modules from, by default empty indicating to not load any CommonJS modules
commonJsModulePath = ""
}

mapper-limits {
Expand Down
3 changes: 2 additions & 1 deletion ...rg/eclipse/ditto/connectivity/service/mapping/javascript/DefaultJavaScriptConfigTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import static org.mutabilitydetector.unittesting.MutabilityAssert.assertInstancesOf;
import static org.mutabilitydetector.unittesting.MutabilityMatchers.areImmutable;

import java.nio.file.Path;
import java.time.Duration;

import org.assertj.core.api.JUnitSoftAssertions;
Expand Down Expand Up @@ -49,7 +50,7 @@ public static void initTestFixture() {
public void assertImmutability() {
assertInstancesOf(DefaultJavaScriptConfig.class,
areImmutable(),
provided(JavaScriptConfig.class).isAlsoImmutable());
provided(JavaScriptConfig.class, Path.class).isAlsoImmutable());
}

@Test
Expand Down

0 comments on commit 96cfbe4

Please sign in to comment.