Implementing issue apache#2167 : Create a new REST API (location serv…

…ice)
mattcasters · Mar 7, 2023 · d3e0bd4 · d3e0bd4
1 parent 6ba39e8
commit d3e0bd4
Show file tree

Hide file tree

Showing 8 changed files with 412 additions and 47 deletions.
diff --git a/core/src/main/java/org/apache/hop/metadata/serializer/json/JsonMetadataSerializer.java b/core/src/main/java/org/apache/hop/metadata/serializer/json/JsonMetadataSerializer.java
@@ -22,6 +22,13 @@
 import com.google.gson.GsonBuilder;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonParser;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.vfs2.FileObject;
 import org.apache.commons.vfs2.FileSystemException;
@@ -34,15 +41,9 @@
 import org.apache.hop.metadata.api.IHopMetadataSerializer;
 import org.json.simple.JSONObject;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-/** @param <T> */
+/**
+ * @param <T>
+ */
 public class JsonMetadataSerializer<T extends IHopMetadata> implements IHopMetadataSerializer<T> {
 
   protected IHopMetadataProvider metadataProvider;
@@ -104,17 +105,21 @@ public T load(String name) throws HopException {
       try {
         fileInputStream = HopVfs.getInputStream(filename);
         JsonFactory jsonFactory = new JsonFactory();
-        com.fasterxml.jackson.core.JsonParser jsonParser =
-            jsonFactory.createParser(fileInputStream);
+        try (com.fasterxml.jackson.core.JsonParser jsonParser =
+            jsonFactory.createParser(fileInputStream)) {
 
-        jsonParser.nextToken(); // skip {
+          // skip opening '{'
+          jsonParser.nextToken();
 
-        T t = parser.loadJsonObject(managedClass, jsonParser);
-        inheritVariables(t);
-        t.setMetadataProviderName(metadataProvider.getDescription());
-        return t;
+          T t = parser.loadJsonObject(managedClass, jsonParser);
+          inheritVariables(t);
+          t.setMetadataProviderName(metadataProvider.getDescription());
+          return t;
+        }
       } finally {
-        fileInputStream.close();
+        if (fileInputStream != null) {
+          fileInputStream.close();
+        }
       }
     } catch (Exception e) {
       throw new HopException(
@@ -147,8 +152,7 @@ public void save(T t) throws HopException {
       try (OutputStream outputStream = HopVfs.getOutputStream(filename, false)) {
         String jsonString = jObject.toJSONString();
         Gson gson = (new GsonBuilder()).setPrettyPrinting().create();
-        JsonParser jp = new JsonParser();
-        JsonElement je = jp.parse(jsonString);
+        JsonElement je = JsonParser.parseString(jsonString);
 
         String formattedJson = gson.toJson(je);
         outputStream.write(formattedJson.getBytes(StandardCharsets.UTF_8));
@@ -244,22 +248,30 @@ public IHopMetadataProvider getMetadataProvider() {
     return metadataProvider;
   }
 
-  /** @param metadataProvider The metadataProvider to set */
+  /**
+   * @param metadataProvider The metadataProvider to set
+   */
   public void setMetadataProvider(IHopMetadataProvider metadataProvider) {
     this.metadataProvider = metadataProvider;
   }
 
-  /** @param baseFolder The baseFolder to set */
+  /**
+   * @param baseFolder The baseFolder to set
+   */
   public void setBaseFolder(String baseFolder) {
     this.baseFolder = baseFolder;
   }
 
-  /** @param managedClass The managedClass to set */
+  /**
+   * @param managedClass The managedClass to set
+   */
   public void setManagedClass(Class<T> managedClass) {
     this.managedClass = managedClass;
   }
 
-  /** @param description The description to set */
+  /**
+   * @param description The description to set
+   */
   public void setDescription(String description) {
     this.description = description;
   }

diff --git a/docs/hop-user-manual/modules/ROOT/pages/hop-rest/index.adoc b/docs/hop-user-manual/modules/ROOT/pages/hop-rest/index.adoc
@@ -184,9 +184,9 @@ This call retrieves all the information about all the pipeline engine plugins.
 
 |===
 
-== Service services
+== Execution services
 
-The 'Service' services are deployed under sub-path `services/`.
+The 'Execution' services are deployed under sub-path `execution/`.
 
 === Execute a pipeline synchronously
 
@@ -196,13 +196,13 @@ The 'Service' services are deployed under sub-path `services/`.
 |`POST`
 
 |Path
-|`services/sync`
+|`execution/sync`
 
 |Produces
 |`application/json`
 
 |Example call
-|`curl -X POST  http://localhost:8080/hop/api/v1/service/sync/  -H 'Content-Type: application/json'  -d '{ "service" : "test", "runConfig" : "local", "variables" : { "VAR1" : "value1", "VAR2" : "value2" }, "bodyContent" : "This is body content" }'`
+|`curl -X POST  http://localhost:8080/hop/api/v1/execution/sync/  -H 'Content-Type: application/json'  -d '{ "service" : "test", "runConfig" : "local", "variables" : { "VAR1" : "value1", "VAR2" : "value2" }, "bodyContent" : "This is body content" }'`
 `
 |Example output
 |The executing pipeline can produce output like this:
@@ -221,8 +221,102 @@ The 'Service' services are deployed under sub-path `services/`.
 
 The body to post can contain the following options (see also: the example above)
 
-* `service`: the name of the Web Service metadata element to use
+* `execute`: the name of the Web Service metadata element to use
 * `runConfig`: the name of the pipeline run configuration to use
 * `variables`: a map with variables (or parameters) with their names and values
 * `bodyContent`: this will be set as a variable using the body content variable option in the Web Service metadata.
 
+
+
+== Execution Information Location services
+
+The execution information location services are deployed under sub-path `location/`.
+
+=== Get execution IDs
+
+[cols="1,2"]
+|===
+|Type
+|`POST`
+
+|Path
+|`location/executions/{locationName}`
+
+|Arguments
+|`locationName` : the name of the execution information location to query.
+
+|Consumes
+|`application/json`
+
+|Produces
+|`application/json`
+
+|Example call
+|`curl -X GET  http://localhost:8080/hop/api/v1/location/executions/local/  -H 'Content-Type: application/json'  -d '{ "includeChildren" : "true", "limit" : 100 }'`
+`
+|Example output
+|The list of execution IDs looks like this:
+
+`["af84cbc2-0166-4dea-956f-72b73cf66d0d", "bf84cbc2-0166-4dea-956f-72b73cf66d0e", "cf84cbc2-0166-4dea-956f-72b73cf66d0f"]`
+
+|===
+
+The body to post can contain the following options (see also: the example above)
+
+* `includeChildren`: Set this to true if you want to see child executions of workflows and pipelines
+* `limit`: the maximum number of IDs to retrieve or a value <=0 to get all IDs
+
+=== Get execution
+
+[cols="1,2"]
+|===
+|Type
+|`GET`
+
+|Path
+|`location/executions/{locationName}/{executionId}`
+
+|Arguments
+|`locationName`: the name of the execution information location to query. `executionId`: the ID of the execution to retrieve.
+
+|Produces
+|`application/json`
+
+|Example call
+|`curl -X GET  http://localhost:8080/hop/api/v1/location/executions/local/df84cbc2-0166-4dea-956f-72b73cf66d0d/`
+`
+|Example output
+|The execution details look like this:
+
+`{"name":"pipeline-name","filename":"/path/filename.hpl","id":"df84cbc2-0166-4dea-956f-72b73cf66d0d","parentId":null,"executionType":"Pipeline", ... }`
+
+|===
+
+=== Get execution state
+
+[cols="1,2"]
+|===
+|Type
+|`GET`
+
+|Path
+|`location/state/{locationName}/{executionId}`
+
+|Arguments
+|`locationName`: the name of the execution information location to query. `executionId`: the ID of the execution state to retrieve.
+
+|Produces
+|`application/json`
+
+|Example call
+|`curl -X GET  http://localhost:8080/hop/api/v1/location/state/local/df84cbc2-0166-4dea-956f-72b73cf66d0d/`
+`
+|Example output
+|The execution state looks like this:
+
+`{"executionType":"Pipeline","parentId":null,"id":"df84cbc2-0166-4dea-956f-72b73cf66d0d","name":"test-service","copyNr":null,"loggingText":"logging-text","lastLogLineNr":14,"metrics":[{"componentName":"id","componentCopy":"0","metrics":{"Read":5,"Buffers Output":0,"Errors":0,"Input":0,"Written":5,"Updated":0,"Output":0,"Rejected":0,"Buffers Input":0}},{"componentName":"uuid","componentCopy":"0","metrics":{"Read":5,"Buffers Output":0,"Errors":0,"Input":0,"Written":5,"Updated":0,"Output":0,"Rejected":0,"Buffers Input":0}},{"componentName":"Enhanced JSON Output","componentCopy":"0","metrics":{"Read":5,"Buffers Output":0,"Errors":0,"Input":0,"Written":1,"Updated":0,"Output":1,"Rejected":0,"Buffers Input":0}},{"componentName":"OUTPUT","componentCopy":"0","metrics":{"Read":1,"Buffers Output":0,"Errors":0,"Input":0,"Written":1,"Updated":0,"Output":0,"Rejected":0,"Buffers Input":0}},{"componentName":"5 rows","componentCopy":"0","metrics":{"Read":0,"Buffers Output":0,"Errors":0,"Input":0,"Written":5,"Updated":0,"Output":0,"Rejected":0,"Buffers Input":0}},{"componentName":"Get variables","componentCopy":"0","metrics":{"Read":5,"Buffers Output":0,"Errors":0,"Input":0,"Written":5,"Updated":0,"Output":0,"Rejected":0,"Buffers Input":0}}],"statusDescription":"Finished","updateTime":1678191016156,"childIds":["46690405-4b06-4353-973a-06aff689afe0","ec974060-e228-4438-8408-b049803eb316","f7344a76-06b0-4499-95a1-1b30bd987561","32837620-df15-4602-b32c-69111689767f","db2e3feb-3ab1-4491-a4dc-7036cc0c3a3f","4653fb73-ac94-4ac3-adef-a14e6129aa14"],"details":{},"failed":false,"containerId":"c7c90ed5-6684-408a-be11-d1fd1c65164a"}`
+
+|===
+
+
+
diff --git a/docs/hop-user-manual/modules/ROOT/pages/hop-server/web-service.adoc b/docs/hop-user-manual/modules/ROOT/pages/hop-server/web-service.adoc
@@ -106,7 +106,7 @@ http://<hop-server-url>/hop/webService
 |===
 |Parameter |Description
 
-|`service`
+|`execute`
 |The name of the service.
 This corresponds to the Web Service metadata object.
 

diff --git a/rest/pom.xml b/rest/pom.xml
@@ -97,10 +97,6 @@
                     <groupId>org.glassfish.jersey.connectors</groupId>
                     <artifactId>*</artifactId>
                 </exclusion>
-                <exclusion>
-                    <groupId>org.glassfish.metro</groupId>
-                    <artifactId>*</artifactId>
-                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
@@ -116,10 +112,6 @@
                     <groupId>org.glassfish.jersey.connectors</groupId>
                     <artifactId>*</artifactId>
                 </exclusion>
-                <exclusion>
-                    <groupId>org.glassfish.metro</groupId>
-                    <artifactId>*</artifactId>
-                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
@@ -135,10 +127,6 @@
                     <groupId>org.glassfish.jersey.connectors</groupId>
                     <artifactId>*</artifactId>
                 </exclusion>
-                <exclusion>
-                    <groupId>org.glassfish.metro</groupId>
-                    <artifactId>*</artifactId>
-                </exclusion>
             </exclusions>
         </dependency>
         <dependency>

diff --git a/...rest/v1/resources/WebServiceResource.java → .../rest/v1/resources/ExecutionResource.java b/...rest/v1/resources/WebServiceResource.java → .../rest/v1/resources/ExecutionResource.java
@@ -43,16 +43,16 @@
 import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
 import org.apache.hop.pipeline.transform.RowAdapter;
 import org.apache.hop.rest.Hop;
-import org.apache.hop.rest.v1.resources.service.SyncRequest;
+import org.apache.hop.rest.v1.resources.execute.SyncRequest;
 import org.apache.hop.www.service.WebService;
 
 /** The Synchronous and Asynchronous web services to execute a pipeline or a workflow. */
-@Path("/service")
-public class WebServiceResource extends BaseResource {
+@Path("/execute")
+public class ExecutionResource extends BaseResource {
   private final Hop hop = Hop.getInstance();
 
   /**
-   * Run a pipeline by referencing the Web Service metadata object name.
+   * Execute a pipeline synchronously, by referencing the Web Service metadata object name.
    *
    * @param request The JSON containing the request details.
    * @return The output of the web service.
@@ -61,7 +61,7 @@ public class WebServiceResource extends BaseResource {
   @Path("/sync")
   @Consumes(MediaType.APPLICATION_JSON)
   @Produces(MediaType.APPLICATION_JSON)
-  public Response listPlugins(SyncRequest request) {
+  public Response executeSynchronously(SyncRequest request) {
     try {
       if (StringUtils.isEmpty(request.getService())) {
         throw new HopException(