diff --git a/.dockerignore b/.dockerignore index 91fbeb0f80f..d39a869dcf5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,4 +8,5 @@ !/assemblies/web/target !/assemblies/plugins/dist/target !/docker +!/rest !google-key-apache-hop-it.json \ No newline at end of file diff --git a/core/src/main/java/org/apache/hop/core/config/IRestServicesProvider.java b/core/src/main/java/org/apache/hop/core/config/IRestServicesProvider.java new file mode 100644 index 00000000000..91fa85deb43 --- /dev/null +++ b/core/src/main/java/org/apache/hop/core/config/IRestServicesProvider.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hop.core.config; + +import java.util.Properties; +import org.apache.hop.metadata.api.IHasHopMetadataProvider; + +/** This signals that the implementing class provides a Properties object. + * It also needs to be able to handle metadata providers so that plugins can add/remove providers. */ +public interface IRestServicesProvider { + Properties getProperties(); + + IHasHopMetadataProvider getHasHopMetadataProvider(); +} diff --git a/core/src/main/java/org/apache/hop/core/extension/HopExtensionPoint.java b/core/src/main/java/org/apache/hop/core/extension/HopExtensionPoint.java index ecfc83a60d9..bbc5b7b7193 100644 --- a/core/src/main/java/org/apache/hop/core/extension/HopExtensionPoint.java +++ b/core/src/main/java/org/apache/hop/core/extension/HopExtensionPoint.java @@ -147,6 +147,8 @@ public enum HopExtensionPoint { HopImportStart("Executed at the start of the 'hop-import' command line tool"), HopImportEnd("Executed at the end of the 'hop-import' command line tool"), + + HopRestServiceStart("Called during Hop REST services startup"), ; public String id; diff --git a/docker/Dockerfile.rest b/docker/Dockerfile.rest new file mode 100644 index 00000000000..2442dc3d1d5 --- /dev/null +++ b/docker/Dockerfile.rest @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +FROM tomcat:10-jdk11-openjdk +LABEL maintainer="Apache Hop Team" +ENV HOP_CONFIG_FOLDER="" +ENV HOP_AES_ENCODER_KEY="" +ENV HOP_AUDIT_FOLDER="${CATALINA_HOME}/webapps/ROOT/audit" +ENV HOP_CONFIG_FOLDER="${CATALINA_HOME}/webapps/ROOT/config" +# specify the hop log level +ENV HOP_LOG_LEVEL="Basic" +# any JRE settings you want to pass on +# The “-XX:+AggressiveHeap” tells the container to use all memory assigned to the container. +# this removed the need to calculate the necessary heap Xmx +ENV HOP_OPTIONS="-Xmx4g" +ENV HOP_PASSWORD_ENCODER_PLUGIN="Hop" +ENV HOP_PLUGIN_BASE_FOLDERS="plugins" +# path to jdbc drivers +ENV HOP_SHARED_JDBC_FOLDER="" +ENV HOP_REST_CONFIG_FOLDER="/config" + +# Set TOMCAT start variables +ENV CATALINA_OPTS='${HOP_OPTIONS} \ + -DHOP_AES_ENCODER_KEY="${HOP_AES_ENCODER_KEY}" \ + -DHOP_AUDIT_FOLDER="${HOP_AUDIT_FOLDER}" \ + -DHOP_CONFIG_FOLDER="${HOP_CONFIG_FOLDER}" \ + -DHOP_LOG_LEVEL="${HOP_LOG_LEVEL}" \ + -DHOP_PASSWORD_ENCODER_PLUGIN="${HOP_PASSWORD_ENCODER_PLUGIN}" \ + -DHOP_PLUGIN_BASE_FOLDERS="${HOP_PLUGIN_BASE_FOLDERS}" \ + -DHOP_REST_CONFIG_FOLDER="${HOP_REST_CONFIG_FOLDER}" \ + -DHOP_SHARED_JDBC_FOLDER="${HOP_SHARED_JDBC_FOLDER}"\' + +# Cleanup and create folder +# +RUN rm -rf webapps/* + +# Copy resources +# +COPY ./assemblies/plugins/dist/target/plugins "${CATALINA_HOME}"/plugins +COPY ./rest/target/hop-rest*.war "${CATALINA_HOME}"/webapps/hop.war + +# Copy the run script +# +COPY ./docker/resources/run-rest.sh /tmp/ + +RUN mkdir -p "$CATALINA_HOME"/lib/swt/linux/x86_64 + +CMD ["/bin/bash", "/tmp/run-rest.sh"] diff --git a/docker/create_hop_rest_container.sh b/docker/create_hop_rest_container.sh new file mode 100755 index 00000000000..604f8e6c46d --- /dev/null +++ b/docker/create_hop_rest_container.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# set working dir to current location +# +cd "${0%/*}" + +REST_TARGET=../rest/target/webapp + +# unzip files for docker image +# +unzip -qu ../rest/target/hop-rest*.war -d ${REST_TARGET} +unzip -qu ../assemblies/plugins/dist/target/hop-assemblies-*.zip -d ../assemblies/plugins/dist/target/ + +# Copy recent changes in libraries. +# +echo "Copying Hop jar files from the target folders." +cp ../core/target/hop-core-*SNAPSHOT.jar ${REST_TARGET}/WEB-INF/lib/ +cp ../engine/target/hop-engine-*SNAPSHOT.jar ${REST_TARGET}/WEB-INF/lib +cp ../ui/target/hop-ui-*SNAPSHOT.jar ${REST_TARGET}/WEB-INF/lib/ +cp ../rap/target/hop-*SNAPSHOT.jar ${REST_TARGET}/WEB-INF/lib/ + +# Copy recent changes to a few plugins. +# +cp ../plugins/engines/beam/target/hop-plugins*.jar ../assemblies/plugins/dist/target/plugins/engines/beam/ +cp ../plugins/misc/projects/target/hop-plugins*.jar ../assemblies/plugins/dist/target/plugins/misc/projects/ + +# Build the docker image. +# +docker build ../ -f Dockerfile.rest -t hop-rest # --progress=plain --no-cache + +# Cleanup +# +# rm -rf ${REST_TARGET}/ +# rm -rf ../assemblies/plugins/dist/target/plugins diff --git a/docker/integration-tests/integration-tests-doris.yaml b/docker/integration-tests/integration-tests-doris.yaml deleted file mode 100644 index 763ddd83410..00000000000 --- a/docker/integration-tests/integration-tests-doris.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -version: '2.4' -services: - integration_test_kafka: - extends: - file: integration-tests-base.yaml - service: integration_test - links: - - doris-fe - - doris-be - - docker-fe: - image: "apache/doris:1.2.1-fe-x86_64" - container_name: "doris-fe" - hostname: "fe" - environment: - - FE_SERVERS=fe1:local_host:9010 - - FE_ID=1 - ports: - - 8030:8030 - - 9030:9030 - volumes: - - /data/fe/doris-meta:/opt/apache-doris/fe/doris-meta - - /data/fe/conf:/opt/apache-doris/fe/conf - network_mode: host - - docker-be: - image: "apache/doris:1.2.1-be-x86_64" - container_name: "doris-be" - hostname: "be" - depends_on: - - docker-fe - environment: - - FE_SERVERS=fe1:local_host:9010 - - BE_ADDR=local_host:9050 - ports: - - 8040:8040 - volumes: - - /data/be/storage:/opt/apache-doris/be/storage - - /data/be/conf:/opt/apache-doris/be/conf - - /data/be/script:/docker-entrypoint-initdb.d - network_mode: host diff --git a/docker/resources/run-rest.sh b/docker/resources/run-rest.sh new file mode 100755 index 00000000000..514d47d9656 --- /dev/null +++ b/docker/resources/run-rest.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +log() { + echo `date '+%Y/%m/%d %H:%M:%S'`" - ${1}" +} + +# +# Stopping a running hop web container with 'docker stop' is obviously possible. +# Doing it with CTRL-C is just more convenient. +# So we'll start the catalina.sh script in the background and wait until +# we trap SIGINT or SIGTERM. At that point we'll simply stop Tomcat. +# +catalina.sh run & +pid="$!" +log "Running Apache Tomcat / Hop REST with PID ${pid}" +trap "log 'Stopping Tomcat'; catalina.sh stop" SIGINT SIGTERM + +while kill -0 $pid > /dev/null 2>&1; do + wait +done diff --git a/docs/hop-user-manual/modules/ROOT/pages/hop-rest/index.adoc b/docs/hop-user-manual/modules/ROOT/pages/hop-rest/index.adoc new file mode 100644 index 00000000000..8c24b5f9236 --- /dev/null +++ b/docs/hop-user-manual/modules/ROOT/pages/hop-rest/index.adoc @@ -0,0 +1,228 @@ +//// +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +//// +[[HopRest]] +:imagesdir: ../assets/images +:description: hop-rest is a web application (and docker container) which is capable of working with the Hop ecosystem. + +:toc: + += Hop REST Web Application + +== Description + +The Hop REST web application is deployed as a WAR file `hop-rest-.war`. You can put this archive in the `webapps` folder of your favorite server (e.g. Apache Tomcat). We suggest that you rename it to `hop.war` or that you unzip it in a `hop` folder. + +== The base URL + +When deployed under a `webapps/hop` folder you will get the services, detailed below, under the base URL of `/hop/api/v1/`. + +== Metadata services + +The metadata services are deployed under sub-path `metadata/`. + +=== List the metadata type keys + +[cols="1,2"] +|=== +|Type +|`GET` + +|Path +|`metadata/types` + +|Produces +|`application/json` + +|Example call +|`curl -X GET http://localhost:8080/hop/api/v1/metadata/types` + +|Example output +| +`["dataset","workflow-log","async-web-service","execution-info-location","web-service","cassandra-connection","workflow-run-configuration","unit-test","rdbms","mongodb-connection","file-definition","splunk","pipeline-run-configuration","pipeline-log","server","partition","execution-data-profile","neo4j-connection","pipeline-probe","neo4j-graph-model"]` + +|=== + +=== List the metadata elements for a key + +[cols="1,2"] +|=== +|Type +|`GET` + +|Path +|`metadata/list/{key}` + +|Arguments +|`key` : the key of the metadata type + +|Produces +|`application/json` + +|Example call +|`curl -X GET http://localhost:8080/hop/api/v1/metadata/list/pipeline-run-configuration/` + +|Example output +| +`["local"]` + +|=== + +=== Get a metadata element + +[cols="1,2"] +|=== +|Type +|`GET` + +|Path +|`metadata/{key}/{name}` + +|Arguments +|`key` : the key of the metadata type, `{name}` : the name of the metadata element to get. + +|Produces +|`application/json` + +|Example call +|`curl -X GET http://localhost:8080/hop/api/v1/metadata/pipeline-run-configuration/local/` + +|Example output +| +`{"engineRunConfiguration":{"Local":{"feedback_size":"50000","sample_size":"1000","sample_type_in_gui":"None","wait_time":"2","rowset_size":"10000","safe_mode":false,"show_feedback":false,"topo_sort":false,"gather_metrics":false,"transactional":false}},"defaultSelection":false,"configurationVariables":[],"name":"local","description":"Runs your pipelines locally with the standard local Hop pipeline engine","dataProfile":"first-last","executionInfoLocationName":"local"}` + +|=== + +=== Save a metadata element + +[cols="1,2"] +|=== +|Type +|`POST` + +|Path +|`metadata/{key}` + +|Arguments +|`key` : the key of the metadata type to save + +|Consumes +|`application/json` + +|Produces +|`application/json` + +|Example call +|`curl -X POST http://localhost:8080/hop/api/v1/metadata/pipeline-run-configuration/ -d '{"engineRunConfiguration":{"Local":{"feedback_size":"50000","sample_size":"1000","sample_type_in_gui":"None","wait_time":"2","rowset_size":"10000","safe_mode":false,"show_feedback":false,"topo_sort":false,"gather_metrics":false,"transactional":false}},"defaultSelection":false,"configurationVariables":[],"name":"local","description":"Runs your pipelines locally with the standard local Hop pipeline engine","dataProfile":"first-last","executionInfoLocationName":"local"}'` + +|=== + + +== Plugins services + +The plugins services are deployed under sub-path `plugins/`. + +=== List all the plugin type classes + +[cols="1,2"] +|=== +|Type +|`GET` + +|Path +|`plugins/types` + +|Produces +|`application/json` + +|Example call +|`curl -X GET http://localhost:8080/hop/api/v1/plugins/types` + +|Example output +| +`["org.apache.hop.core.compress.CompressionPluginType","org.apache.hop.core.row.value.ValueMetaPluginType","org.apache.hop.core.database.DatabasePluginType","org.apache.hop.core.plugins.TransformPluginType","org.apache.hop.core.auth.AuthenticationConsumerPluginType","org.apache.hop.imp.ImportPluginType","org.apache.hop.core.encryption.TwoWayPasswordEncoderPluginType","org.apache.hop.core.auth.AuthenticationProviderPluginType","org.apache.hop.core.plugins.PartitionerPluginType","org.apache.hop.core.logging.LoggingPluginType","org.apache.hop.execution.plugin.ExecutionInfoLocationPluginType","org.apache.hop.core.config.plugin.ConfigPluginType","org.apache.hop.core.plugins.HopServerPluginType","org.apache.hop.core.plugins.ActionPluginType","org.apache.hop.core.extension.ExtensionPointPluginType","org.apache.hop.pipeline.engine.PipelineEnginePluginType","org.apache.hop.pipeline.transform.RowDistributionPluginType","org.apache.hop.workflow.engine.WorkflowEnginePluginType","org.apache.hop.core.vfs.plugin.VfsPluginType","org.apache.hop.execution.sampler.ExecutionDataSamplerPluginType","org.apache.hop.metadata.plugin.MetadataPluginType"]` + +|=== + +=== List all the plugins for a given type class + +[cols="1,2"] +|=== +|Type +|`GET` + +|Path +|`metadata/list/{typeClassName}/` + +|Arguments +|`key` : the class name of the plugin type + +|Produces +|`application/json` + +|Example call +|`curl -X GET http://localhost:8080/hop/api/v1/plugins/list/org.apache.hop.pipeline.engine.PipelineEnginePluginType` + +This call retrieves all the information about all the pipeline engine plugins. + +|Example output +| +`[{"category":null,"name":"Beam DataFlow pipeline engine","description":"This allows you to run your pipeline on Google Cloud Platform DataFlow, provided by the Apache Beam community","ids":["BeamDataFlowPipelineEngine"],"pluginType":"org.apache.hop.pipeline.engine.PipelineEnginePluginType","imageFile":null,"separateClassLoaderNeeded":false,"classLoaderGroup":null,"nativePlugin":false, ... }` + +|=== + +== Service services + +The 'Service' services are deployed under sub-path `services/`. + +=== Execute a pipeline synchronously + +[cols="1,2"] +|=== +|Type +|`POST` + +|Path +|`services/sync` + +|Produces +|`application/json` + +|Example call +|`curl -X POST http://localhost:8080/hop/api/v1/service/sync/ -H 'Content-Type: application/json' -d '{ "service" : "test", "runConfig" : "local", "variables" : { "VAR1" : "value1", "VAR2" : "value2" }, "bodyContent" : "This is body content" }'` +` +|Example output +|The executing pipeline can produce output like this: + +`{ +"rows" : [ { +"id" : 1, +"uuid" : "93b534f0-31ef-4992-8012-ffbfd3d7d489", +"var1" : "value1", +"var2" : "value2", +"bodyContent" : "This is body content" +}, ... ] +}` + +|=== + +The body to post can contain the following options (see also: the example above) + +* `service`: the name of the Web Service metadata element to use +* `runConfig`: the name of the pipeline run configuration to use +* `variables`: a map with variables (or parameters) with their names and values +* `bodyContent`: this will be set as a variable using the body content variable option in the Web Service metadata. + diff --git a/integration-tests/hop_server/project-config.json b/integration-tests/hop_server/project-config.json index df9d54f0243..01ddf10c949 100644 --- a/integration-tests/hop_server/project-config.json +++ b/integration-tests/hop_server/project-config.json @@ -1,16 +1,14 @@ { - "metadataBaseFolder": "${PROJECT_HOME}/metadata", - "unitTestsBasePath": "${PROJECT_HOME}", - "dataSetsCsvFolder": "${PROJECT_HOME}/datasets", - "enforcingExecutionInHome": true, - "parentProjectName": "base-project", - "config": { - "variables": [ - { - "name": "HOP_LICENSE_HEADER_FILE", - "value": "${PROJECT_HOME}/../asf-header.txt", - "description": "This will automatically serialize the ASF license header into pipelines and workflows in the integration test projects" - } - ] + "metadataBaseFolder" : "${PROJECT_HOME}/metadata", + "unitTestsBasePath" : "${PROJECT_HOME}", + "dataSetsCsvFolder" : "${PROJECT_HOME}/datasets", + "enforcingExecutionInHome" : true, + "parentProjectName" : "default", + "config" : { + "variables" : [ { + "name" : "HOP_LICENSE_HEADER_FILE", + "value" : "${PROJECT_HOME}/../asf-header.txt", + "description" : "This will automatically serialize the ASF license header into pipelines and workflows in the integration test projects" + } ] } } \ No newline at end of file diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/config/ProjectsOptionPlugin.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/config/ProjectsOptionPlugin.java index fcd608b8e77..ec7b09c7232 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/config/ProjectsOptionPlugin.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/config/ProjectsOptionPlugin.java @@ -17,6 +17,8 @@ package org.apache.hop.projects.config; +import java.util.ArrayList; +import java.util.List; import org.apache.commons.lang.StringUtils; import org.apache.hop.core.config.plugin.IConfigOptions; import org.apache.hop.core.exception.HopException; @@ -29,9 +31,6 @@ import org.apache.hop.projects.util.ProjectsUtil; import picocli.CommandLine; -import java.util.ArrayList; -import java.util.List; - public class ProjectsOptionPlugin implements IConfigOptions { @CommandLine.Option( @@ -44,24 +43,30 @@ public class ProjectsOptionPlugin implements IConfigOptions { description = "The name of the project to use") private String projectOption = null; - protected ProjectsConfig config; - protected ProjectConfig projectConfig; - protected List configurationFiles; - protected Project project; protected String projectName; protected String environmentName; - protected LifecycleEnvironment environment; @Override public boolean handleOption( ILogChannel log, IHasHopMetadataProvider hasHopMetadataProvider, IVariables variables) throws HopException { - config = ProjectsConfigSingleton.getConfig(); - projectConfig = null; - configurationFiles = new ArrayList<>(); projectName = projectOption; environmentName = environmentOption; + return configure(log, variables, hasHopMetadataProvider, projectName, environmentName); + } + + public static final boolean configure( + ILogChannel log, + IVariables variables, + IHasHopMetadataProvider hasHopMetadataProvider, + String projectName, + String environmentName) + throws HopException { + ProjectsConfig config = ProjectsConfigSingleton.getConfig(); + ProjectConfig projectConfig; + List configurationFiles = new ArrayList<>(); + LifecycleEnvironment environment; // You can specify the project using -p (project) or -e (lifecycle environment) // The main difference is that the environment provides extra configuration files to consider. @@ -104,7 +109,7 @@ public boolean handleOption( if (StringUtils.isEmpty(projectName)) { throw new HopException( - "Lifecycle environment '" + environmentOption + "' is not referencing a project."); + "Lifecycle environment '" + environmentName + "' is not referencing a project."); } projectConfig = config.findProjectConfig(projectName); if (projectConfig == null) { @@ -118,7 +123,7 @@ public boolean handleOption( log.logBasic( "Referencing environment '" - + environmentOption + + environmentName + "' for project " + projectName + "' in " @@ -155,7 +160,6 @@ public boolean handleOption( hasHopMetadataProvider); return true; - } catch (Exception e) { throw new HopException("Error enabling project '" + projectName + "'", e); } diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/config/ProjectsSearchOptionPlugin.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/config/ProjectsSearchOptionPlugin.java index 6763f374c75..44c99689211 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/config/ProjectsSearchOptionPlugin.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/config/ProjectsSearchOptionPlugin.java @@ -24,6 +24,7 @@ import org.apache.hop.core.logging.ILogChannel; import org.apache.hop.core.variables.IVariables; import org.apache.hop.metadata.api.IHasHopMetadataProvider; +import org.apache.hop.projects.project.ProjectConfig; import org.apache.hop.projects.search.ProjectsSearchablesLocation; import org.apache.hop.search.HopSearch; import org.apache.hop.ui.core.gui.HopNamespace; @@ -47,6 +48,10 @@ public boolean handleOption( if (StringUtils.isNotEmpty(projectName)) { log.logBasic("Searching in project : " + projectName); + // Re-load the project config + // + ProjectConfig projectConfig = + ProjectsConfigSingleton.getConfig().findProjectConfig(projectName); ProjectsSearchablesLocation projectsSearchablesLocation = new ProjectsSearchablesLocation(projectConfig); ((HopSearch) hasHopMetadataProvider) diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/HopRestServicesStartupProjectsConfig.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/HopRestServicesStartupProjectsConfig.java new file mode 100644 index 00000000000..e6cda116c47 --- /dev/null +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/HopRestServicesStartupProjectsConfig.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.projects.xp; + +import java.util.Properties; +import org.apache.commons.lang.StringUtils; +import org.apache.hop.core.config.IRestServicesProvider; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.extension.ExtensionPoint; +import org.apache.hop.core.extension.IExtensionPoint; +import org.apache.hop.core.logging.ILogChannel; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.metadata.api.IHasHopMetadataProvider; +import org.apache.hop.projects.config.ProjectsConfig; +import org.apache.hop.projects.config.ProjectsConfigSingleton; +import org.apache.hop.projects.config.ProjectsOptionPlugin; + +@ExtensionPoint( + id = "HopRestServicesStartupProjectsConfig", + extensionPointId = "HopRestServiceStart", + description = "Configures a selected environment or project by name") +public class HopRestServicesStartupProjectsConfig + implements IExtensionPoint { + + public static final String OPTION_ENVIRONMENT_NAME = "environmentName"; + public static final String OPTION_PROJECT_NAME = "projectName"; + + @Override + public void callExtensionPoint( + ILogChannel log, IVariables variables, IRestServicesProvider provider) throws HopException { + + Properties properties = provider.getProperties(); + IHasHopMetadataProvider hasHopMetadataProvider = provider.getHasHopMetadataProvider(); + ProjectsConfig config = ProjectsConfigSingleton.getConfig(); + + String environmentName = (String) properties.get(OPTION_ENVIRONMENT_NAME); + if (StringUtils.isEmpty(environmentName)) { + environmentName = config.getDefaultEnvironment(); + } + String projectName = (String) properties.get(OPTION_PROJECT_NAME); + if (StringUtils.isEmpty(projectName)) { + projectName = config.getDefaultProject(); + } + + try { + // Configure the project and environment if needed + if (StringUtils.isNotEmpty(projectName) || StringUtils.isNotEmpty(environmentName)) { + if (ProjectsOptionPlugin.configure( + log, variables, hasHopMetadataProvider, projectName, environmentName)) { + log.logBasic("Configured project or environment for the Hop REST services"); + } + } + } catch (Exception e) { + throw new HopException( + "Error configuring project or environment for the Hop REST services", e); + } + } +} diff --git a/pom.xml b/pom.xml index 736a57f3827..77297c9cdda 100644 --- a/pom.xml +++ b/pom.xml @@ -143,6 +143,7 @@ 4.4.15 4.0.4 2.14.2 + 3.1.0 2.0.1 3.0.5 4.0.1 @@ -867,6 +868,7 @@ ui rap rcp + rest plugins diff --git a/rest/pom.xml b/rest/pom.xml new file mode 100644 index 00000000000..ebb7e318f61 --- /dev/null +++ b/rest/pom.xml @@ -0,0 +1,150 @@ + + + + 4.0.0 + + org.apache.hop + hop-rest + war + 2.4.0-SNAPSHOT + + Hop REST + + + org.apache.hop + hop + 2.4.0-SNAPSHOT + + + + 3.1.0 + + + + + + org.apache.maven.plugins + maven-war-plugin + 3.3.1 + + false + + + + + + + + jakarta.ws.rs + jakarta.ws.rs-api + ${jakarta.version} + + + + org.glassfish.jersey.containers + jersey-container-servlet + ${jakarta.version} + + + org.glassfish.jersey.inject + jersey-hk2 + ${jakarta.version} + + + org.glassfish.jersey.media + jersey-media-json-jackson + ${jakarta.version} + + + org.glassfish.jersey + jersey-bom + 3.1.0 + pom + + + com.fasterxml.jackson.module + jackson-module-jaxb-annotations + ${jackson.version} + + + + org.apache.hop + hop-core + ${project.version} + + + org.glassfish.jersey.core + * + + + org.glassfish.jersey.connectors + * + + + org.glassfish.metro + * + + + + + org.apache.hop + hop-engine + ${project.version} + + + org.glassfish.jersey.core + * + + + org.glassfish.jersey.connectors + * + + + org.glassfish.metro + * + + + + + org.apache.hop + hop-engine-beam + ${project.version} + + + org.glassfish.jersey.core + * + + + org.glassfish.jersey.connectors + * + + + org.glassfish.metro + * + + + + + org.apache.hop + hop-ui + ${project.version} + + + diff --git a/rest/src/main/java/org/apache/hop/rest/Hop.java b/rest/src/main/java/org/apache/hop/rest/Hop.java new file mode 100644 index 00000000000..23b9a163a16 --- /dev/null +++ b/rest/src/main/java/org/apache/hop/rest/Hop.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hop.rest; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Properties; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.hop.core.Const; +import org.apache.hop.core.HopEnvironment; +import org.apache.hop.core.config.IRestServicesProvider; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.extension.ExtensionPointHandler; +import org.apache.hop.core.extension.HopExtensionPoint; +import org.apache.hop.core.logging.HopLogStore; +import org.apache.hop.core.logging.LogChannel; +import org.apache.hop.core.logging.LogLevel; +import org.apache.hop.core.logging.LoggingObject; +import org.apache.hop.core.metadata.SerializableMetadataProvider; +import org.apache.hop.core.plugins.JarCache; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.variables.Variables; +import org.apache.hop.core.vfs.HopVfs; +import org.apache.hop.metadata.api.IHasHopMetadataProvider; +import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider; +import org.apache.hop.metadata.util.HopMetadataUtil; + +/** Singleton class serving as the basis for the Web Application */ +public class Hop implements IHasHopMetadataProvider, IRestServicesProvider { + public static final String OPTION_LOG_LEVEL = "logLevel"; + public static final String OPTION_METADATA_EXPORT_FILE = "metadataExportFile"; + + private static Hop instance; + + private final Properties properties; + private final LoggingObject loggingObject; + private final IVariables variables; + private final LogChannel log; + + private MultiMetadataProvider metadataProvider; + + /** + * Gets instance + * + * @return value of instance + */ + public static Hop getInstance() { + if (instance == null) { + instance = new Hop(); + } + return instance; + } + + private Hop() { + try { + // Initialize Hop + // + System.err.println("=====> STARTING <======"); + HopEnvironment.init(); + + loggingObject = new LoggingObject("Hop REST API v1"); + log = new LogChannel("Hop REST"); + + // Load configuration details. We need to know where the metadata is, and so on. + // + properties = new Properties(); + String propertyPath; + try { + if (StringUtils.isEmpty(System.getProperty("HOP_REST_CONFIG_FOLDER"))) { + propertyPath = "/config"; + } else { + propertyPath = System.getProperty("HOP_REST_CONFIG_FOLDER"); + } + String configFileName = propertyPath + "/hop-rest.properties"; + log.logBasic("The configuration filename we're looking for is: " + configFileName); + File configFile = new File(configFileName); + if (configFile.exists()) { + try (InputStream inputStream = new FileInputStream(configFile)) { + properties.load(inputStream); + log.logBasic("Configuration file " + configFileName+" was successfully loaded."); + } + } else { + try (InputStream inputStream = getClass().getResourceAsStream("/hop-rest.properties")) { + properties.load(inputStream); + } + } + } catch (IOException e) { + throw new HopException("Error initializing Lean Rest: ", e); + } + + // Set a logging level for the application + // + log.setLogLevel( + LogLevel.getLogLevelForCode( + Const.NVL(properties.getProperty(OPTION_LOG_LEVEL), "BASIC"))); + log.logBasic("Starting the Apache Hop REST services application."); + + variables = new Variables(); + + // Initialize the logging backend + // + HopLogStore.init(); + + // Clear the jar file cache so that we don't waste memory... + // + JarCache.getInstance().clear(); + + // Set up the metadata to use + // + metadataProvider = HopMetadataUtil.getStandardHopMetadataProvider(variables); + + // Allow plugins to modify the elements loaded so far, before a pipeline or workflow is even + // loaded + // + ExtensionPointHandler.callExtensionPoint( + log, variables, HopExtensionPoint.HopRestServiceStart.id, this); + + // Optionally we can configure metadata to come from a JSON export file. + // + String metadataExportFile = properties.getProperty(OPTION_METADATA_EXPORT_FILE); + if (StringUtils.isNotEmpty(metadataExportFile)) { + // Load the JSON from the specified file: + // + try (InputStream inputStream = HopVfs.getInputStream(metadataExportFile)) { + String json = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + SerializableMetadataProvider exportedProvider = new SerializableMetadataProvider(json); + metadataProvider.getProviders().add(exportedProvider); + + log.logBasic( + "Added metadata provider from export file: " + metadataProvider.getDescription()); + } + } + + } catch (Exception e) { + throw new RuntimeException("Error initializing the Apache Hop REST services", e); + } + } + + /** + * Sets instance + * + * @param instance value of instance + */ + public static void setInstance(Hop instance) { + Hop.instance = instance; + } + + /** + * Gets properties + * + * @return value of properties + */ + public Properties getProperties() { + return properties; + } + + /** + * Gets loggingObject + * + * @return value of loggingObject + */ + public LoggingObject getLoggingObject() { + return loggingObject; + } + + /** + * Gets variables + * + * @return value of variables + */ + public IVariables getVariables() { + return variables; + } + + /** + * Gets log + * + * @return value of log + */ + public LogChannel getLog() { + return log; + } + + /** + * Gets metadataProvider + * + * @return value of metadataProvider + */ + @Override + public MultiMetadataProvider getMetadataProvider() { + return metadataProvider; + } + + /** + * Sets metadataProvider + * + * @param metadataProvider value of metadataProvider + */ + @Override + public void setMetadataProvider(MultiMetadataProvider metadataProvider) { + this.metadataProvider = metadataProvider; + } + + @Override + public IHasHopMetadataProvider getHasHopMetadataProvider() { + return this; + } +} diff --git a/rest/src/main/java/org/apache/hop/rest/v1/HopApplication.java b/rest/src/main/java/org/apache/hop/rest/v1/HopApplication.java new file mode 100644 index 00000000000..9d0d04fc9f7 --- /dev/null +++ b/rest/src/main/java/org/apache/hop/rest/v1/HopApplication.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hop.rest.v1; + +import jakarta.ws.rs.ApplicationPath; +import jakarta.ws.rs.core.Application; +import org.apache.hop.rest.Hop; + +@ApplicationPath("/api/v1/") +public class HopApplication extends Application { + public HopApplication() { + Hop.getInstance(); + } +} diff --git a/rest/src/main/java/org/apache/hop/rest/v1/resources/BaseResource.java b/rest/src/main/java/org/apache/hop/rest/v1/resources/BaseResource.java new file mode 100644 index 00000000000..dab9773c356 --- /dev/null +++ b/rest/src/main/java/org/apache/hop/rest/v1/resources/BaseResource.java @@ -0,0 +1,37 @@ +package org.apache.hop.rest.v1.resources; + +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import org.apache.hop.core.Const; +import org.apache.hop.rest.Hop; + +public abstract class BaseResource { + protected final Hop hop = Hop.getInstance(); + + protected Response getServerError(String errorMessage) { + return getServerError(errorMessage, null, true); + } + + protected Response getServerError(String errorMessage, boolean logOnServer) { + return getServerError(errorMessage, null, logOnServer); + } + + protected Response getServerError(String errorMessage, Exception e) { + return getServerError(errorMessage, e, true); + } + + protected Response getServerError(String errorMessage, Exception e, boolean logOnServer) { + if (logOnServer) { + if (e != null) { + hop.getLog().logError(errorMessage, e); + } else { + hop.getLog().logError(errorMessage); + } + } + return Response.serverError() + .status(Response.Status.INTERNAL_SERVER_ERROR) + .entity(errorMessage + (e == null ? "" : ("\n" + Const.getSimpleStackTrace(e)))) + .type(MediaType.TEXT_PLAIN) + .build(); + } +} diff --git a/rest/src/main/java/org/apache/hop/rest/v1/resources/MetadataResource.java b/rest/src/main/java/org/apache/hop/rest/v1/resources/MetadataResource.java new file mode 100644 index 00000000000..2dc08ff169d --- /dev/null +++ b/rest/src/main/java/org/apache/hop/rest/v1/resources/MetadataResource.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hop.rest.v1.resources; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import java.util.ArrayList; +import java.util.List; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.metadata.api.HopMetadata; +import org.apache.hop.metadata.api.IHopMetadata; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.metadata.api.IHopMetadataSerializer; +import org.apache.hop.metadata.serializer.json.JsonMetadataParser; +import org.json.simple.JSONObject; + +@Path("/metadata") +public class MetadataResource extends BaseResource { + + /** + * List all the type keys + * + * @return A list with all the type keys in the metadata + */ + @GET + @Path("/types") + @Produces(MediaType.APPLICATION_JSON) + public Response getTypes() { + List types = new ArrayList<>(); + IHopMetadataProvider provider = hop.getMetadataProvider(); + List> metadataClasses = provider.getMetadataClasses(); + for (Class metadataClass : metadataClasses) { + HopMetadata metadata = metadataClass.getAnnotation(HopMetadata.class); + types.add(metadata.key()); + } + return Response.ok(types).build(); + } + + /** + * List all the element names for a given type + * + * @param key the metadata key to use + * @return A list with all the metadata element names + * @throws HopException + */ + @GET + @Path("/list/{key}/") + @Produces(MediaType.APPLICATION_JSON) + public Response listNames(@PathParam("key") String key) throws HopException { + IHopMetadataProvider provider = hop.getMetadataProvider(); + Class metadataClass = provider.getMetadataClassForKey(key); + IHopMetadataSerializer serializer = provider.getSerializer(metadataClass); + return Response.ok(serializer.listObjectNames()).build(); + } + + /** + * Get a metadata element with a given type and name + * + * @param key The key of the metadata type + * @param name The name to look up + * @return The metadata element + * @throws HopException + */ + @GET + @Path("/{key}/{name}") + @Produces(MediaType.APPLICATION_JSON) + public Response getElement(@PathParam("key") String key, @PathParam("name") String name) + throws HopException { + IHopMetadataProvider provider = hop.getMetadataProvider(); + Class metadataClass = provider.getMetadataClassForKey(key); + IHopMetadataSerializer serializer = provider.getSerializer(metadataClass); + IHopMetadata metadata = serializer.load(name); + // We want to serialize this exactly like we do on the filesystem to avoid confusion. + // + JsonMetadataParser parser = new JsonMetadataParser<>(metadataClass, provider); + JSONObject jsonObject = parser.getJsonObject(metadata); + + return Response.ok().entity(jsonObject.toJSONString()).build(); + } + + /** + * Save a metadata element + * + * @param key + * @param metadataJson + * @return + * @throws HopException + */ + @POST + @Path("/{key}/") + @Produces(MediaType.APPLICATION_JSON) + public Response saveElement(@PathParam("key") String key, String metadataJson) + throws HopException { + try { + IHopMetadataProvider provider = hop.getMetadataProvider(); + Class metadataClass = provider.getMetadataClassForKey(key); + IHopMetadataSerializer serializer = provider.getSerializer(metadataClass); + JsonMetadataParser parser = new JsonMetadataParser<>(metadataClass, provider); + JsonFactory jsonFactory = new JsonFactory(); + JsonParser jsonParser = jsonFactory.createParser(metadataJson); + IHopMetadata metadata = parser.loadJsonObject(metadataClass, jsonParser); + serializer.save(metadata); + return Response.ok().entity(metadata.getName()).build(); + } catch (Exception e) { + return getServerError("Error saving element of type " + key, e, true); + } + } +} diff --git a/rest/src/main/java/org/apache/hop/rest/v1/resources/PluginsResource.java b/rest/src/main/java/org/apache/hop/rest/v1/resources/PluginsResource.java new file mode 100644 index 00000000000..d2163523908 --- /dev/null +++ b/rest/src/main/java/org/apache/hop/rest/v1/resources/PluginsResource.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hop.rest.v1.resources; + +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import java.util.List; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.plugins.IPlugin; +import org.apache.hop.core.plugins.IPluginType; +import org.apache.hop.core.plugins.PluginRegistry; +import org.apache.hop.rest.Hop; + +@Path("/plugins") +public class PluginsResource extends BaseResource { + private final Hop hop = Hop.getInstance(); + + /** + * List all the plugin type classes + * + * @return A list with all the plugin types from the plugin registry + */ + @GET + @Path("/types") + @Produces(MediaType.APPLICATION_JSON) + public Response getTypes() { + PluginRegistry registry = PluginRegistry.getInstance(); + List> types = registry.getPluginTypes(); + return Response.ok(types).build(); + } + + /** + * List all the plugins for a given type class + * + * @param typeClassName the metadata key to use + * @return A list with all the available plugins + * @throws HopException + */ + @GET + @Path("/list/{typeClassName}/") + @Produces(MediaType.APPLICATION_JSON) + public Response listPlugins(@PathParam("typeClassName") String typeClassName) { + try { + PluginRegistry registry = PluginRegistry.getInstance(); + Class> typeClass = (Class>) Class.forName(typeClassName); + List plugins = registry.getPlugins(typeClass); + return Response.ok(plugins).build(); + } catch (Exception e) { + String errorMessage = + "Unexpected error retrieving the list of plugins for plugin type class " + typeClassName; + return getServerError(errorMessage, e); + } + } +} diff --git a/rest/src/main/java/org/apache/hop/rest/v1/resources/WebServiceResource.java b/rest/src/main/java/org/apache/hop/rest/v1/resources/WebServiceResource.java new file mode 100644 index 00000000000..e71338b695f --- /dev/null +++ b/rest/src/main/java/org/apache/hop/rest/v1/resources/WebServiceResource.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hop.rest.v1.resources; + +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import java.util.UUID; +import org.apache.commons.lang.StringUtils; +import org.apache.hop.core.Const; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.exception.HopTransformException; +import org.apache.hop.core.exception.HopValueException; +import org.apache.hop.core.logging.LoggingObjectType; +import org.apache.hop.core.logging.SimpleLoggingObject; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.metadata.api.IHopMetadataSerializer; +import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.engine.IEngineComponent; +import org.apache.hop.pipeline.engine.IPipelineEngine; +import org.apache.hop.pipeline.engine.PipelineEngineFactory; +import org.apache.hop.pipeline.engines.local.LocalPipelineEngine; +import org.apache.hop.pipeline.transform.RowAdapter; +import org.apache.hop.rest.Hop; +import org.apache.hop.rest.v1.resources.service.SyncRequest; +import org.apache.hop.www.service.WebService; + +/** The Synchronous and Asynchronous web services to execute a pipeline or a workflow. */ +@Path("/service") +public class WebServiceResource extends BaseResource { + private final Hop hop = Hop.getInstance(); + + /** + * Run a pipeline by referencing the Web Service metadata object name. + * + * @param request The JSON containing the request details. + * @return The output of the web service. + */ + @POST + @Path("/sync") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + public Response listPlugins(SyncRequest request) { + try { + if (StringUtils.isEmpty(request.getService())) { + throw new HopException( + "Please specify the name of the service in the JSON you post: service"); + } + + MultiMetadataProvider metadataProvider = hop.getMetadataProvider(); + IHopMetadataSerializer serializer = + metadataProvider.getSerializer(WebService.class); + WebService service = serializer.load(request.getService()); + if (service == null) { + throw new HopException("Unable to find service with name '" + request.getService() + "'"); + } + + if (!service.isEnabled()) { + throw new HopException("Web service '" + service.getName() + "' is disabled."); + } + + IVariables variables = hop.getVariables(); + + String runConfigurationName = request.getRunConfig(); + if (StringUtils.isEmpty(runConfigurationName)) { + runConfigurationName = variables.resolve(service.getRunConfigurationName()); + } + + if (StringUtils.isEmpty(runConfigurationName)) { + throw new HopException( + "Please specify the name of the run configuration to use, " + + "either in the request (runConfig) or in the Web Service metadata element '" + + service.getName() + + "'"); + } + + String filename = variables.resolve(service.getFilename()); + String transformName = variables.resolve(service.getTransformName()); + String fieldName = variables.resolve(service.getFieldName()); + String contentType = variables.resolve(service.getContentType()); + String bodyContentVariable = variables.resolve(service.getBodyContentVariable()); + + String bodyContent = request.getBodyContent(); + + if (StringUtils.isEmpty(contentType)) { + contentType = "text/plain"; + } + + // Load and execute the pipeline. + // + String serverObjectId = UUID.randomUUID().toString(); + SimpleLoggingObject servletLoggingObject = + new SimpleLoggingObject("/service/sync/", LoggingObjectType.HOP_SERVER, null); + servletLoggingObject.setContainerObjectId(serverObjectId); + + // Load and start the pipeline + // Output the data to the response output stream... + // + PipelineMeta pipelineMeta = new PipelineMeta(filename, metadataProvider, variables); + IPipelineEngine pipeline; + if (StringUtils.isEmpty(runConfigurationName)) { + pipeline = new LocalPipelineEngine(pipelineMeta, variables, servletLoggingObject); + } else { + pipeline = + PipelineEngineFactory.createPipelineEngine( + variables, runConfigurationName, metadataProvider, pipelineMeta); + } + pipeline.setContainerId(serverObjectId); + + if (StringUtils.isNotEmpty(bodyContentVariable)) { + pipeline.setVariable(bodyContentVariable, Const.NVL(bodyContent, "")); + } + + // Set all the other parameters as variables/parameters... + // + String[] pipelineParameters = pipelineMeta.listParameters(); + pipeline.copyParametersFromDefinitions(pipelineMeta); + for (String requestParameter : request.getVariables().keySet()) { + if ("service".equals(requestParameter)) { + continue; + } + String requestParameterValue = request.getVariables().get(requestParameter); + if (Const.indexOfString(requestParameter, pipelineParameters) < 0) { + pipeline.setVariable(requestParameter, Const.NVL(requestParameterValue, "")); + } else { + pipeline.setParameterValue(requestParameter, Const.NVL(requestParameterValue, "")); + } + } + pipeline.activateParameters(pipeline); + + // Allocate the threads... + pipeline.prepareExecution(); + + // Add the row listener to the transform/field... + // + final StringBuilder output = new StringBuilder(); + IEngineComponent component = pipeline.findComponent(transformName, 0); + component.addRowListener( + new RowAdapter() { + @Override + public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) + throws HopTransformException { + try { + output.append(rowMeta.getString(row, fieldName, "")); + } catch (HopValueException e) { + throw new HopTransformException( + "Error getting output field '" + + fieldName + + " from row: " + + rowMeta.toStringMeta(), + e); + } + } + }); + + pipeline.startThreads(); + pipeline.waitUntilFinished(); + + // For now just give back the request as JSON + // + return Response.ok(output.toString()).type(contentType).encoding(Const.XML_ENCODING).build(); + } catch (Exception e) { + String errorMessage = + "Unexpected error executing synchronous web service (pipeline) with name " + + request.getService(); + return getServerError(errorMessage, e); + } + } +} diff --git a/rest/src/main/java/org/apache/hop/rest/v1/resources/service/SyncRequest.java b/rest/src/main/java/org/apache/hop/rest/v1/resources/service/SyncRequest.java new file mode 100644 index 00000000000..e4aec357f0e --- /dev/null +++ b/rest/src/main/java/org/apache/hop/rest/v1/resources/service/SyncRequest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hop.rest.v1.resources.service; + +import java.util.HashMap; +import java.util.Map; + +public class SyncRequest { + private String service; + private String runConfig; + private Map variables; + private String bodyContent; + + public SyncRequest() { + variables = new HashMap<>(); + } + + /** + * Gets service + * + * @return value of service + */ + public String getService() { + return service; + } + + /** + * Sets service + * + * @param service value of service + */ + public void setService(String service) { + this.service = service; + } + + /** + * Gets runConfig + * + * @return value of runConfig + */ + public String getRunConfig() { + return runConfig; + } + + /** + * Sets runConfig + * + * @param runConfig value of runConfig + */ + public void setRunConfig(String runConfig) { + this.runConfig = runConfig; + } + + /** + * Gets variables + * + * @return value of variables + */ + public Map getVariables() { + return variables; + } + + /** + * Sets variables + * + * @param variables value of variables + */ + public void setVariables(Map variables) { + this.variables = variables; + } + + /** + * Gets bodyContent + * + * @return value of bodyContent + */ + public String getBodyContent() { + return bodyContent; + } + + /** + * Sets bodyContent + * + * @param bodyContent value of bodyContent + */ + public void setBodyContent(String bodyContent) { + this.bodyContent = bodyContent; + } +} diff --git a/rest/src/main/resources/hop-rest.properties b/rest/src/main/resources/hop-rest.properties new file mode 100644 index 00000000000..ee497e2826a --- /dev/null +++ b/rest/src/main/resources/hop-rest.properties @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +# The logging level: Nothing, Error, Minimal, Basic, Detailed, Debug, Rowlevel +# +logLevel=Detailed + +# The name of the metadata export file to use as metadata for the services +# +metadataExportFile= + +# The name of the project to use +# +projectName= + +# The name of the environment to enable +# +environmentName=hop2 local + diff --git a/rest/src/test/resources/hop-rest.properties b/rest/src/test/resources/hop-rest.properties new file mode 100644 index 00000000000..ee497e2826a --- /dev/null +++ b/rest/src/test/resources/hop-rest.properties @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +# The logging level: Nothing, Error, Minimal, Basic, Detailed, Debug, Rowlevel +# +logLevel=Detailed + +# The name of the metadata export file to use as metadata for the services +# +metadataExportFile= + +# The name of the project to use +# +projectName= + +# The name of the environment to enable +# +environmentName=hop2 local +