Implementing issue #2167 : Create a new REST API

.dockerignore

@@ -8,4 +8,5 @@
 !/assemblies/web/target
 !/assemblies/plugins/dist/target
 !/docker
+!/rest
 !google-key-apache-hop-it.json

assemblies/plugins/dist/pom.xml

@@ -2493,6 +2493,19 @@
       </exclusions>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.hop</groupId>
+      <artifactId>hop-assemblies-plugins-transforms-validator</artifactId>
+      <version>${project.version}</version>
+      <type>zip</type>
+      <exclusions>
+        <exclusion>
+          <groupId>*</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
     <dependency>
       <groupId>org.apache.hop</groupId>
       <artifactId>hop-assemblies-plugins-transforms-valuemapper</artifactId>

assemblies/plugins/transforms/pom.xml

@@ -163,6 +163,7 @@
     <module>uniquerows</module>
     <module>uniquerowsbyhashset</module>
     <module>update</module>
+    <module>validator</module>
     <module>valuemapper</module>
     <module>verticabulkloader</module>
     <module>webserviceavailable</module>

assemblies/plugins/transforms/validator/pom.xml

@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~       http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.apache.hop</groupId>
+        <artifactId>hop-assemblies-plugins-transforms</artifactId>
+        <version>2.4.0-SNAPSHOT</version>
+    </parent>
+
+
+    <artifactId>hop-assemblies-plugins-transforms-validator</artifactId>
+    <version>2.4.0-SNAPSHOT</version>
+    <packaging>pom</packaging>
+
+    <name>Hop Assemblies Plugins Transforms Data Validator</name>
+    <description />
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.hop</groupId>
+            <artifactId>hop-transform-validator</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+    </dependencies>
+</project>

assemblies/plugins/transforms/validator/src/assembly/assembly.xml

@@ -0,0 +1,50 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~       http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
+    <id>hop-assemblies-plugins-transforms-validator</id>
+    <formats>
+        <format>zip</format>
+    </formats>
+    <baseDirectory>transforms/validator</baseDirectory>
+    <files>
+        <file>
+            <source>${project.basedir}/src/main/resources/version.xml</source>
+            <outputDirectory>.</outputDirectory>
+            <filtered>true</filtered>
+        </file>
+    </files>
+    <fileSets>
+        <fileSet>
+            <outputDirectory>lib</outputDirectory>
+            <excludes>
+                <exclude>**/*</exclude>
+            </excludes>
+        </fileSet>
+    </fileSets>
+    <dependencySets>
+        <dependencySet>
+            <useProjectArtifact>false</useProjectArtifact>
+            <includes>
+                <include>org.apache.hop:hop-transform-validator:jar</include>
+            </includes>
+        </dependencySet>
+    </dependencySets>
+</assembly>

assemblies/plugins/transforms/validator/src/main/resources/version.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~       http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+
+<version>${project.version}</version>

core/src/main/java/org/apache/hop/core/config/IRestServicesProvider.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hop.core.config;
+
+import java.util.Properties;
+import org.apache.hop.metadata.api.IHasHopMetadataProvider;
+
+/** This signals that the implementing class provides a Properties object.
+ * It also needs to be able to handle metadata providers so that plugins can add/remove providers. */
+public interface IRestServicesProvider {
+  Properties getProperties();
+
+  IHasHopMetadataProvider getHasHopMetadataProvider();
+}

core/src/main/java/org/apache/hop/core/extension/HopExtensionPoint.java

@@ -147,6 +147,8 @@ public enum HopExtensionPoint {
 
   HopImportStart("Executed at the start of the 'hop-import' command line tool"),
   HopImportEnd("Executed at the end of the 'hop-import' command line tool"),
+
+  HopRestServiceStart("Called during Hop REST services startup"),
   ;
 
   public String id;

core/src/main/java/org/apache/hop/metadata/serializer/json/JsonMetadataSerializer.java

@@ -22,6 +22,13 @@
 import com.google.gson.GsonBuilder;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonParser;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.vfs2.FileObject;
 import org.apache.commons.vfs2.FileSystemException;
@@ -34,15 +41,9 @@
 import org.apache.hop.metadata.api.IHopMetadataSerializer;
 import org.json.simple.JSONObject;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-/** @param <T> */
+/**
+ * @param <T>
+ */
 public class JsonMetadataSerializer<T extends IHopMetadata> implements IHopMetadataSerializer<T> {
 
   protected IHopMetadataProvider metadataProvider;
@@ -104,17 +105,21 @@ public T load(String name) throws HopException {
       try {
         fileInputStream = HopVfs.getInputStream(filename);
         JsonFactory jsonFactory = new JsonFactory();
-        com.fasterxml.jackson.core.JsonParser jsonParser =
-            jsonFactory.createParser(fileInputStream);
+        try (com.fasterxml.jackson.core.JsonParser jsonParser =
+            jsonFactory.createParser(fileInputStream)) {
 
-        jsonParser.nextToken(); // skip {
+          // skip opening '{'
+          jsonParser.nextToken();
 
-        T t = parser.loadJsonObject(managedClass, jsonParser);
-        inheritVariables(t);
-        t.setMetadataProviderName(metadataProvider.getDescription());
-        return t;
+          T t = parser.loadJsonObject(managedClass, jsonParser);
+          inheritVariables(t);
+          t.setMetadataProviderName(metadataProvider.getDescription());
+          return t;
+        }
       } finally {
-        fileInputStream.close();
+        if (fileInputStream != null) {
+          fileInputStream.close();
+        }
       }
     } catch (Exception e) {
       throw new HopException(
@@ -147,8 +152,7 @@ public void save(T t) throws HopException {
       try (OutputStream outputStream = HopVfs.getOutputStream(filename, false)) {
         String jsonString = jObject.toJSONString();
         Gson gson = (new GsonBuilder()).setPrettyPrinting().create();
-        JsonParser jp = new JsonParser();
-        JsonElement je = jp.parse(jsonString);
+        JsonElement je = JsonParser.parseString(jsonString);
 
         String formattedJson = gson.toJson(je);
         outputStream.write(formattedJson.getBytes(StandardCharsets.UTF_8));
@@ -244,22 +248,30 @@ public IHopMetadataProvider getMetadataProvider() {
     return metadataProvider;
   }
 
-  /** @param metadataProvider The metadataProvider to set */
+  /**
+   * @param metadataProvider The metadataProvider to set
+   */
   public void setMetadataProvider(IHopMetadataProvider metadataProvider) {
     this.metadataProvider = metadataProvider;
   }
 
-  /** @param baseFolder The baseFolder to set */
+  /**
+   * @param baseFolder The baseFolder to set
+   */
   public void setBaseFolder(String baseFolder) {
     this.baseFolder = baseFolder;
   }
 
-  /** @param managedClass The managedClass to set */
+  /**
+   * @param managedClass The managedClass to set
+   */
   public void setManagedClass(Class<T> managedClass) {
     this.managedClass = managedClass;
   }
 
-  /** @param description The description to set */
+  /**
+   * @param description The description to set
+   */
   public void setDescription(String description) {
     this.description = description;
   }

docker/Dockerfile.rest

@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+FROM tomcat:10-jdk11-openjdk
+LABEL maintainer="Apache Hop Team"
+ENV HOP_CONFIG_FOLDER=""
+ENV HOP_AES_ENCODER_KEY=""
+ENV HOP_AUDIT_FOLDER="${CATALINA_HOME}/webapps/ROOT/audit"
+ENV HOP_CONFIG_FOLDER="${CATALINA_HOME}/webapps/ROOT/config"
+# specify the hop log level
+ENV HOP_LOG_LEVEL="Basic"
+# any JRE settings you want to pass on
+# The “-XX:+AggressiveHeap” tells the container to use all memory assigned to the container. 
+# this removed the need to calculate the necessary heap Xmx
+ENV HOP_OPTIONS="-Xmx4g"
+ENV HOP_PASSWORD_ENCODER_PLUGIN="Hop"
+ENV HOP_PLUGIN_BASE_FOLDERS="plugins"
+# path to jdbc drivers
+ENV HOP_SHARED_JDBC_FOLDER=""
+ENV HOP_REST_CONFIG_FOLDER="/config"
+
+# Set TOMCAT start variables
+ENV CATALINA_OPTS='${HOP_OPTIONS} \
+  -DHOP_AES_ENCODER_KEY="${HOP_AES_ENCODER_KEY}" \
+  -DHOP_AUDIT_FOLDER="${HOP_AUDIT_FOLDER}" \
+  -DHOP_CONFIG_FOLDER="${HOP_CONFIG_FOLDER}" \
+  -DHOP_LOG_LEVEL="${HOP_LOG_LEVEL}" \
+  -DHOP_PASSWORD_ENCODER_PLUGIN="${HOP_PASSWORD_ENCODER_PLUGIN}" \
+  -DHOP_PLUGIN_BASE_FOLDERS="${HOP_PLUGIN_BASE_FOLDERS}" \
+  -DHOP_REST_CONFIG_FOLDER="${HOP_REST_CONFIG_FOLDER}" \
+  -DHOP_SHARED_JDBC_FOLDER="${HOP_SHARED_JDBC_FOLDER}"\'
+
+# Cleanup and create folder
+#
+RUN    rm -rf webapps/*
+
+# Copy resources
+#
+COPY ./assemblies/plugins/dist/target/plugins "${CATALINA_HOME}"/plugins
+COPY ./rest/target/hop-rest*.war "${CATALINA_HOME}"/webapps/hop.war
+
+# Copy the run script
+#
+COPY ./docker/resources/run-rest.sh /tmp/
+
+RUN mkdir -p "$CATALINA_HOME"/lib/swt/linux/x86_64
+
+CMD ["/bin/bash", "/tmp/run-rest.sh"]