diff --git a/assemblies/plugins/pom.xml b/assemblies/plugins/pom.xml index c53aac32534..ee02975c130 100644 --- a/assemblies/plugins/pom.xml +++ b/assemblies/plugins/pom.xml @@ -1291,6 +1291,12 @@ ${project.version} zip + + org.apache.hop + hop-transform-odata + ${project.version} + zip + org.apache.hop hop-transform-pgp diff --git a/docker/integration-tests/integration-tests-odata.yaml b/docker/integration-tests/integration-tests-odata.yaml new file mode 100644 index 00000000000..0a61c9227e1 --- /dev/null +++ b/docker/integration-tests/integration-tests-odata.yaml @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +services: + integration_test_database: + extends: + file: integration-tests-base.yaml + service: integration_test + depends_on: + odata_mock: + condition: service_started + links: + - odata_mock + + odata_mock: + image: mockserver/mockserver:5.15.0 + hostname: odata_mock + ports: + - "1080" + environment: + - MOCKSERVER_INITIALIZATION_JSON_PATH=/config/initializerJson.json + - MOCKSERVER_LOG_LEVEL=WARN + volumes: + - ../../integration-tests/odata/import/initializerJson.json:/config/initializerJson.json + - ../../integration-tests/odata/import/metadata.xml:/config/metadata.xml diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/transforms/odata-input-connection-tab.png b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/odata-input-connection-tab.png new file mode 100644 index 00000000000..53f7bb8191d Binary files /dev/null and b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/odata-input-connection-tab.png differ diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/transforms/odata-input-fields-tab.png b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/odata-input-fields-tab.png new file mode 100644 index 00000000000..d54f81ac527 Binary files /dev/null and b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/odata-input-fields-tab.png differ diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/transforms/odata-input-query-tab.png b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/odata-input-query-tab.png new file mode 100644 index 00000000000..edaf6fa5507 Binary files /dev/null and b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/odata-input-query-tab.png differ diff --git a/docs/hop-user-manual/modules/ROOT/nav.adoc b/docs/hop-user-manual/modules/ROOT/nav.adoc index e8d04fc0585..972aeac08de 100644 --- a/docs/hop-user-manual/modules/ROOT/nav.adoc +++ b/docs/hop-user-manual/modules/ROOT/nav.adoc @@ -209,6 +209,7 @@ under the License. *** xref:pipeline/transforms/neo4j-split-graph.adoc[Neo4j Split Graph] *** xref:pipeline/transforms/nullif.adoc[Null If] *** xref:pipeline/transforms/numberrange.adoc[Number range] +*** xref:pipeline/transforms/odata-input.adoc[OData Input] *** xref:pipeline/transforms/orabulkloader.adoc[Oracle Bulk Loader] *** xref:pipeline/transforms/parquet-file-input.adoc[Parquet File Input] *** xref:pipeline/transforms/parquet-file-output.adoc[Parquet File Output] diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms.adoc index 4933b31be74..40e598f0b1f 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms.adoc @@ -166,6 +166,7 @@ The pages nested under this topic contain information on how to use the transfor * xref:pipeline/transforms/neo4j-split-graph.adoc[Neo4j Split Graph] * xref:pipeline/transforms/nullif.adoc[Null If] * xref:pipeline/transforms/numberrange.adoc[Number range] +* xref:pipeline/transforms/odata-input.adoc[OData Input] * xref:pipeline/transforms/orabulkloader.adoc[Oracle Bulk Loader] * xref:pipeline/transforms/parquet-file-input.adoc[Parquet File Input] * xref:pipeline/transforms/parquet-file-output.adoc[Parquet File Output] diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/odata-input.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/odata-input.adoc new file mode 100644 index 00000000000..55783c0c7e6 --- /dev/null +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/odata-input.adoc @@ -0,0 +1,104 @@ +//// +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +//// +:documentationPath: /pipeline/transforms/ +:language: en_US +:description: The OData Input transform retrieves data from OData V2 and V4 service endpoints. + += image:transforms/icons/odata.svg[OData Input transform Icon, role="image-doc-icon"] OData Input + +[%noheader,cols="3a,1a", role="table-no-borders" ] +|=== +| +== Description + +The OData Input transform enables you to query OData (Open Data Protocol) services. OData is an OASIS standard that defines a set of best practices for building and consuming RESTful APIs. For more information, visit the official link:https://www.odata.org/[OData Documentation]. + +This transform performs GET requests against OData Entity Sets, maps JSON properties to Apache Hop rows, and automatically handles relative pagination using standard `@odata.nextLink` (OData V4) and `__next` (OData V2) metadata attributes. + +| +== Supported Engines +[%noheader,cols="2,1a",frame=none, role="table-supported-engines"] +!=== +!Hop Engine! image:check_mark.svg[Supported, 24] +!Spark! image:question_mark.svg[Maybe Supported, 24] +!Flink! image:question_mark.svg[Maybe Supported, 24] +!Dataflow! image:question_mark.svg[Maybe Supported, 24] +!=== +|=== + +== Connection Options + +The Connection tab allows you to configure the service endpoint and optional credentials. + +image::transforms/odata-input-connection-tab.png[OData Input Connection Tab, align="center"] + +[options="header"] +|=== +|Option|Description +|Transform name|Name of this transform as it appears in the pipeline workspace. +|OData Service Root URL|The base URL of the OData service (e.g., `https://services.odata.org/V4/TripPinServiceRW/`). +|Authentication Type|The authentication type to use: +* `No Authentication`: Anonymous connection. +* `Basic Authentication`: Use standard Username/Password credentials. +* `Bearer Token`: Use a Bearer token in the `Authorization` header. +|Username (Basic Auth)|The username for Basic authentication. +|Password (Basic Auth)|The password for Basic authentication. +|Token (Bearer / OAuth2)|The bearer token value. +|OData Entity Set|The OData entity set to query (e.g., `People` or `Airports`). +|Get Entity Sets|Retrieves and displays a drop-down list of available entity sets from the service endpoint. +|=== + +== Query Options + +The OData Query tab allows you to configure filtering, ordering, and field selection properties. + +image::transforms/odata-input-query-tab.png[OData Input Query Tab, align="center"] + +[options="header"] +|=== +|Option|Description +|$select|A comma-separated list of properties to select (e.g., `UserName,FirstName,LastName`). +|$filter|An OData expression to filter matching entities (e.g., `Gender eq 'Male'`). +|$orderby|Specifies the sorting order of the results (e.g., `LastName asc, FirstName desc`). +|$top|Limits the number of entities returned on each page request (e.g., `50`). +|$skip|Specifies the number of entities to skip from the beginning of the list. +|=== + +== Fields Mapping + +The Fields tab defines how properties in the returned OData JSON response map to Apache Hop row fields. + +image::transforms/odata-input-fields-tab.png[OData Input Fields Tab, align="center"] + +[options="header"] +|=== +|Option|Description +|Name in Hop|The name of the field as it will be produced in the Apache Hop pipeline. +|OData Path|The JSON property path of the field within the entity JSON object (e.g., `UserName`, `AddressInfo/0/City/Name` for nested structures). +|Type|The Hop data type (String, Integer, Number, Date, Boolean, BigNumber). +|Format|Optional parsing format for date or numeric fields. +|Get Fields|Connects to the `$metadata` endpoint of the OData service and automatically populates the fields list based on the selected Entity Set's schema properties. +|=== + +== Example + +To query a public OData service: +1. Set the **Service Root URL** to `https://services.odata.org/V4/TripPinServiceRW/`. +2. Set the **Entity Set** to `People`. +3. Go to the **Fields** tab and click **Get Fields** to pull all metadata fields (`UserName`, `FirstName`, `LastName`, etc.). +4. (Optional) In the **OData Query** tab, add a `$filter` such as `FirstName eq 'John'` or a `$select` of `UserName,Emails`. +5. Run the pipeline to stream entities into downstream transforms. diff --git a/integration-tests/odata/0001-odata-query.hpl b/integration-tests/odata/0001-odata-query.hpl new file mode 100644 index 00000000000..85c89395a63 --- /dev/null +++ b/integration-tests/odata/0001-odata-query.hpl @@ -0,0 +1,163 @@ + + + + + 0001-odata-query + Y + + + + Normal + + + + + + OData Input + count + Y + + + count + check count + Y + + + check count + Abort + Y + + + + OData Input + ODataInput + + Y + + 1 + + none + + + http://odata_mock:1080/odata + Customers + NONE + + + Id + Id + Integer + + + Name + Name + String + + + Balance + Balance + Number + + + + 100 + 100 + + + + count + MemoryGroupBy + + Y + + 1 + + none + + + + + count + COUNT_ANY + + + N + + + + 250 + 100 + + + + check count + FilterRows + + Y + + 1 + + none + + + + + + + <> + count + N + - + + N + -1 + ####0;-####0 + constant + 0 + 3 + Integer + + + + + 400 + 100 + + + + Abort + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + Incorrect number of OData records received + 0 + + 550 + 100 + + + diff --git a/integration-tests/odata/dev-env-config.json b/integration-tests/odata/dev-env-config.json new file mode 100644 index 00000000000..41e7cbaee08 --- /dev/null +++ b/integration-tests/odata/dev-env-config.json @@ -0,0 +1,9 @@ +{ + "variables" : [ + { + "name" : "HOSTNAME", + "value" : "odata_mock", + "description" : "hostname of odata_mock container" + } + ] +} diff --git a/integration-tests/odata/hop-config.json b/integration-tests/odata/hop-config.json new file mode 100644 index 00000000000..4ae5d9cd607 --- /dev/null +++ b/integration-tests/odata/hop-config.json @@ -0,0 +1,34 @@ +{ + "variables": [], + "LocaleDefault": "en_US", + "guiProperties": { + "DarkMode": "Y" + }, + "projectsConfig": { + "enabled": true, + "projectMandatory": true, + "environmentMandatory": false, + "defaultProject": "odata", + "defaultEnvironment": null, + "standardParentProject": "odata", + "standardProjectsFolder": null, + "projectConfigurations": [ + { + "projectName": "odata", + "projectHome": "${HOP_CONFIG_FOLDER}", + "configFilename": "project-config.json" + } + ], + "lifecycleEnvironments": [ + { + "name": "dev", + "purpose": "Testing", + "projectName": "odata", + "configurationFiles": [ + "${PROJECT_HOME}/dev-env-config.json" + ] + } + ], + "projectLifecycles": [] + } +} diff --git a/integration-tests/odata/import/initializerJson.json b/integration-tests/odata/import/initializerJson.json new file mode 100644 index 00000000000..ae6d4893c34 --- /dev/null +++ b/integration-tests/odata/import/initializerJson.json @@ -0,0 +1,58 @@ +[ + { + "httpRequest": { + "method": "GET", + "path": "/odata" + }, + "httpResponse": { + "statusCode": 200, + "headers": { + "Content-Type": ["application/json; charset=utf-8"] + }, + "body": "{\"value\": [{\"name\": \"Customers\", \"kind\": \"EntitySet\", \"url\": \"Customers\"}]}" + } + }, + { + "httpRequest": { + "method": "GET", + "path": "/odata/$metadata" + }, + "httpResponse": { + "statusCode": 200, + "headers": { + "Content-Type": ["application/xml; charset=utf-8"] + }, + "body": "" + } + }, + { + "httpRequest": { + "method": "GET", + "path": "/odata/Customers" + }, + "httpResponse": { + "statusCode": 200, + "headers": { + "Content-Type": ["application/json; charset=utf-8"] + }, + "body": "{\"value\": [{\"Id\": 1, \"Name\": \"Alice\", \"Balance\": 100.50}, {\"Id\": 2, \"Name\": \"Bob\", \"Balance\": 250.00}], \"@odata.nextLink\": \"http://odata_mock:1080/odata/Customers?%24skip=2\"}" + } + }, + { + "priority": 10, + "httpRequest": { + "method": "GET", + "path": "/odata/Customers", + "queryStringParameters": { + "$skip": ["2"] + } + }, + "httpResponse": { + "statusCode": 200, + "headers": { + "Content-Type": ["application/json; charset=utf-8"] + }, + "body": "{\"value\": [{\"Id\": 3, \"Name\": \"Charlie\", \"Balance\": 0.00}]}" + } + } +] diff --git a/integration-tests/odata/import/metadata.xml b/integration-tests/odata/import/metadata.xml new file mode 100644 index 00000000000..5582adc51d3 --- /dev/null +++ b/integration-tests/odata/import/metadata.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + + + diff --git a/integration-tests/odata/main-0001-odata-query.hwf b/integration-tests/odata/main-0001-odata-query.hwf new file mode 100644 index 00000000000..8e4034574b8 --- /dev/null +++ b/integration-tests/odata/main-0001-odata-query.hwf @@ -0,0 +1,106 @@ + + + + main-0001-odata-query + Y + + + + 0 + + + + + Start + + SPECIAL + + N + 0 + 0 + 60 + 12 + 0 + 1 + 1 + N + 50 + 50 + + + + 0001-odata-query.hpl + + PIPELINE + + ${PROJECT_HOME}/0001-odata-query.hpl + N + N + N + N + N + + + N + N + Basic + N + Y + N + N + local + + Y + + N + 200 + 50 + + + + Abort workflow + + ABORT + + N + 200 + 180 + + + + + + Start + 0001-odata-query.hpl + Y + Y + Y + + + 0001-odata-query.hpl + Abort workflow + Y + N + N + + + + + + diff --git a/integration-tests/odata/metadata/pipeline-run-configuration/local.json b/integration-tests/odata/metadata/pipeline-run-configuration/local.json new file mode 100644 index 00000000000..5d38d84e258 --- /dev/null +++ b/integration-tests/odata/metadata/pipeline-run-configuration/local.json @@ -0,0 +1,18 @@ +{ + "engineRunConfiguration": { + "Local": { + "feedback_size": "50000", + "sample_size": "100", + "sample_type_in_gui": "Last", + "rowset_size": "10000", + "safe_mode": false, + "show_feedback": false, + "topo_sort": false, + "gather_metrics": false, + "transactional": false + } + }, + "configurationVariables": [], + "name": "local", + "description": "Runs your pipelines locally with the standard local Hop pipeline engine" +} diff --git a/integration-tests/odata/metadata/workflow-run-configuration/local.json b/integration-tests/odata/metadata/workflow-run-configuration/local.json new file mode 100644 index 00000000000..eec94f94e89 --- /dev/null +++ b/integration-tests/odata/metadata/workflow-run-configuration/local.json @@ -0,0 +1,10 @@ +{ + "engineRunConfiguration": { + "Local": { + "safe_mode": false, + "transactional": false + } + }, + "name": "local", + "description": "Runs your workflows locally with the standard local Hop workflow engine" +} diff --git a/integration-tests/odata/project-config.json b/integration-tests/odata/project-config.json new file mode 100644 index 00000000000..793afe289e1 --- /dev/null +++ b/integration-tests/odata/project-config.json @@ -0,0 +1,13 @@ +{ + "metadataBaseFolder" : "${PROJECT_HOME}/metadata", + "unitTestsBasePath" : "${PROJECT_HOME}", + "dataSetsCsvFolder" : "${PROJECT_HOME}/datasets", + "enforcingExecutionInHome" : true, + "config" : { + "variables" : [ { + "name" : "HOP_LICENSE_HEADER_FILE", + "value" : "${PROJECT_HOME}/../asf-header.txt", + "description" : "This will automatically serialize the ASF license header into pipelines and workflows in the integration test projects" + } ] + } +} diff --git a/plugins/transforms/odata/pom.xml b/plugins/transforms/odata/pom.xml new file mode 100644 index 00000000000..7eb4841124e --- /dev/null +++ b/plugins/transforms/odata/pom.xml @@ -0,0 +1,32 @@ + + + + 4.0.0 + + + org.apache.hop + hop-plugins-transforms + 2.19.0-SNAPSHOT + + + hop-transform-odata + jar + Hop Plugins Transforms OData + + diff --git a/plugins/transforms/odata/src/assembly/assembly.xml b/plugins/transforms/odata/src/assembly/assembly.xml new file mode 100644 index 00000000000..5d6b0b0869b --- /dev/null +++ b/plugins/transforms/odata/src/assembly/assembly.xml @@ -0,0 +1,44 @@ + + + + + hop-transform-odata + + zip + + . + + + ${project.basedir}/src/main/resources/version.xml + ${hop.plugin.libdir} + true + + + ${project.basedir}/src/main/resources/dependencies.xml + ${hop.plugin.libdir} + true + + + + + ${maven.multiModuleProjectDirectory}/assemblies/shared/hop-plugin-libs.xml + + diff --git a/plugins/transforms/odata/src/main/java/org/apache/hop/pipeline/transforms/odata/ODataAuthType.java b/plugins/transforms/odata/src/main/java/org/apache/hop/pipeline/transforms/odata/ODataAuthType.java new file mode 100644 index 00000000000..fc22c605082 --- /dev/null +++ b/plugins/transforms/odata/src/main/java/org/apache/hop/pipeline/transforms/odata/ODataAuthType.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.odata; + +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.IEnumHasCodeAndDescription; + +public enum ODataAuthType implements IEnumHasCodeAndDescription { + NONE("NONE", "ODataAuthType.None.Description"), + BASIC("BASIC", "ODataAuthType.Basic.Description"), + BEARER("BEARER", "ODataAuthType.Bearer.Description"); + + private static final Class PKG = ODataAuthType.class; + + private final String code; + private final String descriptionKey; + + ODataAuthType(String code, String descriptionKey) { + this.code = code; + this.descriptionKey = descriptionKey; + } + + @Override + public String getCode() { + return code; + } + + @Override + public String getDescription() { + return BaseMessages.getString(PKG, descriptionKey); + } + + public static String[] getDescriptions() { + return IEnumHasCodeAndDescription.getDescriptions(ODataAuthType.class); + } + + public static ODataAuthType lookupDescription(String description) { + return IEnumHasCodeAndDescription.lookupDescription( + ODataAuthType.class, description, ODataAuthType.NONE); + } + + public static ODataAuthType lookupCode(String code) { + for (ODataAuthType val : values()) { + if (val.getCode().equalsIgnoreCase(code)) { + return val; + } + } + return NONE; + } +} diff --git a/plugins/transforms/odata/src/main/java/org/apache/hop/pipeline/transforms/odata/ODataField.java b/plugins/transforms/odata/src/main/java/org/apache/hop/pipeline/transforms/odata/ODataField.java new file mode 100644 index 00000000000..241e9adc59f --- /dev/null +++ b/plugins/transforms/odata/src/main/java/org/apache/hop/pipeline/transforms/odata/ODataField.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.odata; + +import java.util.Objects; +import lombok.Getter; +import lombok.Setter; +import org.apache.hop.core.exception.HopPluginException; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.value.ValueMetaBase; +import org.apache.hop.core.row.value.ValueMetaFactory; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.metadata.api.HopMetadataProperty; + +@Getter +@Setter +public class ODataField { + @HopMetadataProperty(key = "name") + private String name; + + @HopMetadataProperty(key = "path") + private String path; + + @HopMetadataProperty(key = "type", intCodeConverter = ValueMetaBase.ValueTypeCodeConverter.class) + private int type; + + @HopMetadataProperty(key = "format") + private String format; + + public ODataField() { + this.name = ""; + this.path = ""; + this.type = IValueMeta.TYPE_STRING; + this.format = ""; + } + + public ODataField(String name, String path, int type, String format) { + this.name = name; + this.path = path; + this.type = type; + this.format = format; + } + + public ODataField(ODataField other) { + this.name = other.name; + this.path = other.path; + this.type = other.type; + this.format = other.format; + } + + @Override + public ODataField clone() { + return new ODataField(this); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof ODataField other)) { + return false; + } + return Objects.equals(name, other.name) + && Objects.equals(path, other.path) + && type == other.type + && Objects.equals(format, other.format); + } + + @Override + public int hashCode() { + return Objects.hash(name, path, type, format); + } + + public IValueMeta toValueMeta(String originTransformName, IVariables variables) + throws HopPluginException { + int hopType = getType(); + if (hopType == IValueMeta.TYPE_NONE) { + hopType = IValueMeta.TYPE_STRING; + } + IValueMeta v = + ValueMetaFactory.createValueMeta( + variables != null ? variables.resolve(getName()) : getName(), hopType); + v.setOrigin(originTransformName); + v.setConversionMask(getFormat()); + return v; + } +} diff --git a/plugins/transforms/odata/src/main/java/org/apache/hop/pipeline/transforms/odata/ODataInput.java b/plugins/transforms/odata/src/main/java/org/apache/hop/pipeline/transforms/odata/ODataInput.java new file mode 100644 index 00000000000..261b45c0242 --- /dev/null +++ b/plugins/transforms/odata/src/main/java/org/apache/hop/pipeline/transforms/odata/ODataInput.java @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.odata; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.net.URI; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; +import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.RowDataUtil; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.util.HttpClientManager; +import org.apache.hop.core.util.Utils; +import org.apache.hop.pipeline.Pipeline; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransform; +import org.apache.hop.pipeline.transform.TransformMeta; + +public class ODataInput extends BaseTransform { + private static final Class PKG = ODataInputMeta.class; + + public ODataInput( + TransformMeta transformMeta, + ODataInputMeta meta, + ODataInputData data, + int copyNr, + PipelineMeta pipelineMeta, + Pipeline pipeline) { + super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline); + } + + @Override + public boolean init() { + if (super.init()) { + try { + HttpClientManager.HttpClientBuilderFacade builder = + HttpClientManager.getInstance().createBuilder(); + if ("BASIC".equalsIgnoreCase(meta.getAuthType()) && !Utils.isEmpty(meta.getUsername())) { + builder.setCredentials(resolve(meta.getUsername()), resolve(meta.getPassword())); + } + data.httpClient = builder.build(); + + String serviceUrl = resolve(meta.getUrl()); + String entitySet = resolve(meta.getEntitySet()); + if (Utils.isEmpty(serviceUrl)) { + logError("Service URL is empty."); + return false; + } + if (Utils.isEmpty(entitySet)) { + logError("Entity Set is empty."); + return false; + } + + String rootUrl = serviceUrl; + if (!rootUrl.endsWith("/")) { + rootUrl += "/"; + } + String requestUrl = rootUrl + entitySet; + + List queryParams = new ArrayList<>(); + if (!Utils.isEmpty(meta.getQuerySelect())) { + queryParams.add( + "$select=" + + URLEncoder.encode( + resolve(meta.getQuerySelect()), StandardCharsets.UTF_8.name())); + } + if (!Utils.isEmpty(meta.getQueryFilter())) { + queryParams.add( + "$filter=" + + URLEncoder.encode( + resolve(meta.getQueryFilter()), StandardCharsets.UTF_8.name())); + } + if (!Utils.isEmpty(meta.getQueryOrder())) { + queryParams.add( + "$orderby=" + + URLEncoder.encode( + resolve(meta.getQueryOrder()), StandardCharsets.UTF_8.name())); + } + if (!Utils.isEmpty(meta.getQueryTop())) { + queryParams.add( + "$top=" + + URLEncoder.encode(resolve(meta.getQueryTop()), StandardCharsets.UTF_8.name())); + } + if (!Utils.isEmpty(meta.getQuerySkip())) { + queryParams.add( + "$skip=" + + URLEncoder.encode(resolve(meta.getQuerySkip()), StandardCharsets.UTF_8.name())); + } + + if (!queryParams.isEmpty()) { + requestUrl += "?" + String.join("&", queryParams); + } + data.nextPageUrl = requestUrl; + data.isFinishedReading = false; + + return true; + } catch (Exception e) { + logError("Error initializing OData HTTP Client", e); + return false; + } + } + return false; + } + + @Override + public boolean processRow() throws HopException { + if (first) { + first = false; + data.outputRowMeta = new RowMeta(); + meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); + } + + if (data.recordIndex < data.recordBuffer.size()) { + Object[] row = data.recordBuffer.get(data.recordIndex++); + putRow(data.outputRowMeta, row); + return true; + } + + if (data.isFinishedReading && data.recordIndex >= data.recordBuffer.size()) { + setOutputDone(); + return false; + } + + // Fetch next page of data + fetchNextPage(); + + if (data.recordBuffer.isEmpty()) { + setOutputDone(); + return false; + } + + Object[] row = data.recordBuffer.get(data.recordIndex++); + putRow(data.outputRowMeta, row); + return true; + } + + private void fetchNextPage() throws HopException { + if (data.nextPageUrl == null || data.nextPageUrl.isEmpty()) { + data.isFinishedReading = true; + return; + } + + try { + HttpGet get = new HttpGet(data.nextPageUrl); + get.setHeader("Accept", "application/json"); + if ("BEARER".equalsIgnoreCase(meta.getAuthType()) && !Utils.isEmpty(meta.getToken())) { + get.setHeader("Authorization", "Bearer " + resolve(meta.getToken())); + } + + try (CloseableHttpResponse response = data.httpClient.execute(get)) { + int statusCode = response.getCode(); + if (statusCode != 200) { + throw new HopException("OData service returned non-200 status code: " + statusCode); + } + + HttpEntity entity = response.getEntity(); + String body = entity != null ? EntityUtils.toString(entity, StandardCharsets.UTF_8) : ""; + + ObjectMapper mapper = new ObjectMapper(); + JsonNode rootNode = mapper.readTree(body); + + JsonNode recordsNode = null; + String nextLink = null; + + if (rootNode.has("value")) { + // OData V4 + recordsNode = rootNode.get("value"); + if (rootNode.has("@odata.nextLink")) { + nextLink = rootNode.get("@odata.nextLink").asText(); + } + } else if (rootNode.has("d")) { + // OData V2 + JsonNode dNode = rootNode.get("d"); + if (dNode.has("results")) { + recordsNode = dNode.get("results"); + } else if (dNode.isArray()) { + recordsNode = dNode; + } else { + // Single object wrapper + recordsNode = mapper.createArrayNode(); + ((com.fasterxml.jackson.databind.node.ArrayNode) recordsNode).add(dNode); + } + + if (dNode.has("__next")) { + nextLink = dNode.get("__next").asText(); + } + } else { + // General json backup + if (rootNode.isArray()) { + recordsNode = rootNode; + } else { + recordsNode = mapper.createArrayNode(); + ((com.fasterxml.jackson.databind.node.ArrayNode) recordsNode).add(rootNode); + } + } + + data.recordBuffer.clear(); + data.recordIndex = 0; + + if (recordsNode != null && recordsNode.isArray()) { + for (JsonNode record : recordsNode) { + Object[] row = RowDataUtil.allocateRowData(data.outputRowMeta.size()); + for (int i = 0; i < meta.getFields().size(); i++) { + ODataField field = meta.getFields().get(i); + String path = resolve(field.getPath()); + JsonNode valueNode = getValueByPath(record, path); + IValueMeta valueMeta = data.outputRowMeta.getValueMeta(i); + + Object val = null; + if (valueNode != null && !valueNode.isNull()) { + if (valueMeta.isBoolean()) { + val = valueNode.asBoolean(); + } else if (valueMeta.isInteger()) { + val = valueNode.asLong(); + } else if (valueMeta.isNumber()) { + val = valueNode.asDouble(); + } else if (valueMeta.isDate()) { + String text = valueNode.asText(); + try { + val = valueMeta.convertDataFromString(text, null, null, null, 0); + } catch (Exception ex) { + logError("Failed to parse date: " + text + ", field: " + field.getName(), ex); + } + } else { + val = valueNode.asText(); + } + } + row[i] = val; + } + data.recordBuffer.add(row); + } + } + + if (nextLink != null && !nextLink.isEmpty()) { + // Resolve nextLink relative to base if needed + if (!nextLink.startsWith("http://") && !nextLink.startsWith("https://")) { + URI uri = URI.create(data.nextPageUrl); + String portStr = uri.getPort() != -1 ? ":" + uri.getPort() : ""; + String base = uri.getScheme() + "://" + uri.getHost() + portStr; + if (nextLink.startsWith("/")) { + nextLink = base + nextLink; + } else { + String path = uri.getPath(); + int idx = path.lastIndexOf("/"); + if (idx != -1) { + nextLink = base + path.substring(0, idx + 1) + nextLink; + } else { + nextLink = base + "/" + nextLink; + } + } + } + data.nextPageUrl = nextLink; + } else { + data.nextPageUrl = null; + data.isFinishedReading = true; + } + } + } catch (Exception e) { + throw new HopException("Error requesting OData data page", e); + } + } + + private JsonNode getValueByPath(JsonNode root, String path) { + if (root == null || path == null || path.isEmpty()) { + return root; + } + String[] parts = path.split("/"); + JsonNode current = root; + for (String part : parts) { + if (current == null) { + return null; + } + current = current.get(part); + } + return current; + } + + @Override + public void dispose() { + if (data.httpClient != null) { + try { + data.httpClient.close(); + } catch (Exception e) { + logError("Error closing OData HTTP Client", e); + } + } + super.dispose(); + } +} diff --git a/plugins/transforms/odata/src/main/java/org/apache/hop/pipeline/transforms/odata/ODataInputData.java b/plugins/transforms/odata/src/main/java/org/apache/hop/pipeline/transforms/odata/ODataInputData.java new file mode 100644 index 00000000000..566527bda9f --- /dev/null +++ b/plugins/transforms/odata/src/main/java/org/apache/hop/pipeline/transforms/odata/ODataInputData.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.odata; + +import java.util.ArrayList; +import java.util.List; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.pipeline.transform.BaseTransformData; +import org.apache.hop.pipeline.transform.ITransformData; + +public class ODataInputData extends BaseTransformData implements ITransformData { + public IRowMeta outputRowMeta; + public CloseableHttpClient httpClient; + public String nextPageUrl; + public List recordBuffer; + public int recordIndex; + public boolean isFinishedReading; + + public ODataInputData() { + super(); + recordBuffer = new ArrayList<>(); + recordIndex = 0; + isFinishedReading = false; + } +} diff --git a/plugins/transforms/odata/src/main/java/org/apache/hop/pipeline/transforms/odata/ODataInputDialog.java b/plugins/transforms/odata/src/main/java/org/apache/hop/pipeline/transforms/odata/ODataInputDialog.java new file mode 100644 index 00000000000..dcf3acadf03 --- /dev/null +++ b/plugins/transforms/odata/src/main/java/org/apache/hop/pipeline/transforms/odata/ODataInputDialog.java @@ -0,0 +1,730 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.odata; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.apache.hop.core.Const; +import org.apache.hop.core.Props; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.row.value.ValueMetaFactory; +import org.apache.hop.core.util.HttpClientManager; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.ui.core.PropsUi; +import org.apache.hop.ui.core.dialog.ErrorDialog; +import org.apache.hop.ui.core.widget.ColumnInfo; +import org.apache.hop.ui.core.widget.ComboVar; +import org.apache.hop.ui.core.widget.PasswordTextVar; +import org.apache.hop.ui.core.widget.TableView; +import org.apache.hop.ui.core.widget.TextVar; +import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; +import org.eclipse.swt.SWT; +import org.eclipse.swt.custom.CTabFolder; +import org.eclipse.swt.custom.CTabItem; +import org.eclipse.swt.events.ModifyListener; +import org.eclipse.swt.graphics.Point; +import org.eclipse.swt.layout.FormAttachment; +import org.eclipse.swt.layout.FormData; +import org.eclipse.swt.layout.FormLayout; +import org.eclipse.swt.widgets.Button; +import org.eclipse.swt.widgets.Composite; +import org.eclipse.swt.widgets.Label; +import org.eclipse.swt.widgets.Shell; +import org.eclipse.swt.widgets.TableItem; +import org.eclipse.swt.widgets.Text; + +public class ODataInputDialog extends BaseTransformDialog { + private static final Class PKG = ODataInputMeta.class; + + private final ODataInputMeta input; + + private TextVar wUrl; + private ComboVar wEntitySet; + private ComboVar wAuthType; + private TextVar wUsername; + private PasswordTextVar wPassword; + private PasswordTextVar wToken; + + private TextVar wQuerySelect; + private TextVar wQueryFilter; + private TextVar wQueryOrder; + private TextVar wQueryTop; + private TextVar wQuerySkip; + + private TableView wFields; + + public ODataInputDialog( + Shell parent, IVariables variables, ODataInputMeta transformMeta, PipelineMeta pipelineMeta) { + super(parent, variables, transformMeta, pipelineMeta); + input = transformMeta; + } + + @Override + public String open() { + Shell parent = getParent(); + shell = new Shell(parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MAX | SWT.MIN); + PropsUi.setLook(shell); + setShellImage(shell, input); + + ModifyListener lsMod = e -> input.setChanged(); + changed = input.hasChanged(); + + FormLayout formLayout = new FormLayout(); + formLayout.marginWidth = PropsUi.getFormMargin(); + formLayout.marginHeight = PropsUi.getFormMargin(); + + shell.setLayout(formLayout); + shell.setText(BaseMessages.getString(PKG, "ODataInput.Name")); + + middle = props.getMiddlePct(); + margin = PropsUi.getFormMargin(); + + // TransformName line + wlTransformName = new Label(shell, SWT.RIGHT); + wlTransformName.setText(BaseMessages.getString(PKG, "ODataInputDialog.TransformName.Label")); + PropsUi.setLook(wlTransformName); + fdlTransformName = new FormData(); + fdlTransformName.left = new FormAttachment(0, 0); + fdlTransformName.right = new FormAttachment(middle, -margin); + fdlTransformName.top = new FormAttachment(0, margin); + wlTransformName.setLayoutData(fdlTransformName); + wTransformName = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wTransformName.setText(transformName); + PropsUi.setLook(wTransformName); + wTransformName.addModifyListener(lsMod); + fdTransformName = new FormData(); + fdTransformName.left = new FormAttachment(middle, 0); + fdTransformName.top = new FormAttachment(0, margin); + fdTransformName.right = new FormAttachment(100, 0); + wTransformName.setLayoutData(fdTransformName); + + // Button bar at the bottom + wOk = new Button(shell, SWT.PUSH); + wOk.setText(BaseMessages.getString(PKG, "ODataInputDialog.Button.OK")); + wOk.addListener(SWT.Selection, e -> ok()); + wCancel = new Button(shell, SWT.PUSH); + wCancel.setText(BaseMessages.getString(PKG, "ODataInputDialog.Button.Cancel")); + wCancel.addListener(SWT.Selection, e -> cancel()); + setButtonPositions(new Button[] {wOk, wCancel}, margin, null); + + // Tab Folder + CTabFolder wTabFolder = new CTabFolder(shell, SWT.BORDER); + PropsUi.setLook(wTabFolder, Props.WIDGET_STYLE_TAB); + + // //////////////////////// + // START OF CONNECTION TAB + // //////////////////////// + CTabItem wGeneralTab = new CTabItem(wTabFolder, SWT.NONE); + wGeneralTab.setText(BaseMessages.getString(PKG, "ODataInputDialog.GeneralTab.Title")); + Composite wGeneralComp = new Composite(wTabFolder, SWT.NONE); + PropsUi.setLook(wGeneralComp); + FormLayout generalLayout = new FormLayout(); + generalLayout.marginWidth = margin; + generalLayout.marginHeight = margin; + wGeneralComp.setLayout(generalLayout); + + // URL line + Label wlUrl = new Label(wGeneralComp, SWT.RIGHT); + wlUrl.setText(BaseMessages.getString(PKG, "ODataInputDialog.ServiceUrl.Label")); + PropsUi.setLook(wlUrl); + FormData fdlUrl = new FormData(); + fdlUrl.left = new FormAttachment(0, 0); + fdlUrl.right = new FormAttachment(middle, -margin); + fdlUrl.top = new FormAttachment(0, margin); + wlUrl.setLayoutData(fdlUrl); + wUrl = new TextVar(variables, wGeneralComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wUrl); + wUrl.addModifyListener(lsMod); + FormData fdUrl = new FormData(); + fdUrl.left = new FormAttachment(middle, 0); + fdUrl.right = new FormAttachment(100, 0); + fdUrl.top = new FormAttachment(0, margin); + wUrl.setLayoutData(fdUrl); + + // AuthType line + Label wlAuthType = new Label(wGeneralComp, SWT.RIGHT); + wlAuthType.setText(BaseMessages.getString(PKG, "ODataInputDialog.AuthType.Label")); + PropsUi.setLook(wlAuthType); + FormData fdlAuthType = new FormData(); + fdlAuthType.left = new FormAttachment(0, 0); + fdlAuthType.right = new FormAttachment(middle, -margin); + fdlAuthType.top = new FormAttachment(wUrl, margin); + wlAuthType.setLayoutData(fdlAuthType); + wAuthType = new ComboVar(variables, wGeneralComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wAuthType.setItems(ODataAuthType.getDescriptions()); + PropsUi.setLook(wAuthType); + wAuthType.addModifyListener(lsMod); + FormData fdAuthType = new FormData(); + fdAuthType.left = new FormAttachment(middle, 0); + fdAuthType.right = new FormAttachment(100, 0); + fdAuthType.top = new FormAttachment(wUrl, margin); + wAuthType.setLayoutData(fdAuthType); + + // Username line + Label wlUsername = new Label(wGeneralComp, SWT.RIGHT); + wlUsername.setText(BaseMessages.getString(PKG, "ODataInputDialog.Username.Label")); + PropsUi.setLook(wlUsername); + FormData fdlUsername = new FormData(); + fdlUsername.left = new FormAttachment(0, 0); + fdlUsername.right = new FormAttachment(middle, -margin); + fdlUsername.top = new FormAttachment(wAuthType, margin); + wlUsername.setLayoutData(fdlUsername); + wUsername = new TextVar(variables, wGeneralComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wUsername); + wUsername.addModifyListener(lsMod); + FormData fdUsername = new FormData(); + fdUsername.left = new FormAttachment(middle, 0); + fdUsername.right = new FormAttachment(100, 0); + fdUsername.top = new FormAttachment(wAuthType, margin); + wUsername.setLayoutData(fdUsername); + + // Password line + Label wlPassword = new Label(wGeneralComp, SWT.RIGHT); + wlPassword.setText(BaseMessages.getString(PKG, "ODataInputDialog.Password.Label")); + PropsUi.setLook(wlPassword); + FormData fdlPassword = new FormData(); + fdlPassword.left = new FormAttachment(0, 0); + fdlPassword.right = new FormAttachment(middle, -margin); + fdlPassword.top = new FormAttachment(wUsername, margin); + wlPassword.setLayoutData(fdlPassword); + wPassword = new PasswordTextVar(variables, wGeneralComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wPassword); + wPassword.addModifyListener(lsMod); + FormData fdPassword = new FormData(); + fdPassword.left = new FormAttachment(middle, 0); + fdPassword.right = new FormAttachment(100, 0); + fdPassword.top = new FormAttachment(wUsername, margin); + wPassword.setLayoutData(fdPassword); + + // Token line + Label wlToken = new Label(wGeneralComp, SWT.RIGHT); + wlToken.setText(BaseMessages.getString(PKG, "ODataInputDialog.Token.Label")); + PropsUi.setLook(wlToken); + FormData fdlToken = new FormData(); + fdlToken.left = new FormAttachment(0, 0); + fdlToken.right = new FormAttachment(middle, -margin); + fdlToken.top = new FormAttachment(wPassword, margin); + wlToken.setLayoutData(fdlToken); + wToken = new PasswordTextVar(variables, wGeneralComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wToken); + wToken.addModifyListener(lsMod); + FormData fdToken = new FormData(); + fdToken.left = new FormAttachment(middle, 0); + fdToken.right = new FormAttachment(100, 0); + fdToken.top = new FormAttachment(wPassword, margin); + wToken.setLayoutData(fdToken); + + // EntitySet line + Label wlEntitySet = new Label(wGeneralComp, SWT.RIGHT); + wlEntitySet.setText(BaseMessages.getString(PKG, "ODataInputDialog.EntitySet.Label")); + PropsUi.setLook(wlEntitySet); + FormData fdlEntitySet = new FormData(); + fdlEntitySet.left = new FormAttachment(0, 0); + fdlEntitySet.right = new FormAttachment(middle, -margin); + fdlEntitySet.top = new FormAttachment(wToken, margin); + wlEntitySet.setLayoutData(fdlEntitySet); + + Button wGetEntitySets = new Button(wGeneralComp, SWT.PUSH); + wGetEntitySets.setText(BaseMessages.getString(PKG, "ODataInputDialog.Button.GetEntitySets")); + wGetEntitySets.addListener(SWT.Selection, e -> getEntitySets()); + FormData fdGetEntitySets = new FormData(); + fdGetEntitySets.right = new FormAttachment(100, 0); + fdGetEntitySets.top = new FormAttachment(wToken, margin); + wGetEntitySets.setLayoutData(fdGetEntitySets); + + wEntitySet = new ComboVar(variables, wGeneralComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wEntitySet); + wEntitySet.addModifyListener(lsMod); + FormData fdEntitySet = new FormData(); + fdEntitySet.left = new FormAttachment(middle, 0); + fdEntitySet.right = new FormAttachment(wGetEntitySets, -margin); + fdEntitySet.top = new FormAttachment(wToken, margin); + wEntitySet.setLayoutData(fdEntitySet); + + FormData fdGeneralComp = new FormData(); + fdGeneralComp.left = new FormAttachment(0, 0); + fdGeneralComp.right = new FormAttachment(100, 0); + fdGeneralComp.top = new FormAttachment(0, 0); + fdGeneralComp.bottom = new FormAttachment(100, 0); + wGeneralComp.setLayoutData(fdGeneralComp); + wGeneralComp.layout(); + wGeneralTab.setControl(wGeneralComp); + + // //////////////////////// + // START OF QUERY TAB + // //////////////////////// + CTabItem wQueryTab = new CTabItem(wTabFolder, SWT.NONE); + wQueryTab.setText(BaseMessages.getString(PKG, "ODataInputDialog.QueryTab.Title")); + Composite wQueryComp = new Composite(wTabFolder, SWT.NONE); + PropsUi.setLook(wQueryComp); + FormLayout queryLayout = new FormLayout(); + queryLayout.marginWidth = margin; + queryLayout.marginHeight = margin; + wQueryComp.setLayout(queryLayout); + + // Select line + Label wlQuerySelect = new Label(wQueryComp, SWT.RIGHT); + wlQuerySelect.setText(BaseMessages.getString(PKG, "ODataInputDialog.QuerySelect.Label")); + PropsUi.setLook(wlQuerySelect); + FormData fdlQuerySelect = new FormData(); + fdlQuerySelect.left = new FormAttachment(0, 0); + fdlQuerySelect.right = new FormAttachment(middle, -margin); + fdlQuerySelect.top = new FormAttachment(0, margin); + wlQuerySelect.setLayoutData(fdlQuerySelect); + wQuerySelect = new TextVar(variables, wQueryComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wQuerySelect); + wQuerySelect.addModifyListener(lsMod); + FormData fdQuerySelect = new FormData(); + fdQuerySelect.left = new FormAttachment(middle, 0); + fdQuerySelect.right = new FormAttachment(100, 0); + fdQuerySelect.top = new FormAttachment(0, margin); + wQuerySelect.setLayoutData(fdQuerySelect); + + // Filter line + Label wlQueryFilter = new Label(wQueryComp, SWT.RIGHT); + wlQueryFilter.setText(BaseMessages.getString(PKG, "ODataInputDialog.QueryFilter.Label")); + PropsUi.setLook(wlQueryFilter); + FormData fdlQueryFilter = new FormData(); + fdlQueryFilter.left = new FormAttachment(0, 0); + fdlQueryFilter.right = new FormAttachment(middle, -margin); + fdlQueryFilter.top = new FormAttachment(wQuerySelect, margin); + wlQueryFilter.setLayoutData(fdlQueryFilter); + wQueryFilter = new TextVar(variables, wQueryComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wQueryFilter); + wQueryFilter.addModifyListener(lsMod); + FormData fdQueryFilter = new FormData(); + fdQueryFilter.left = new FormAttachment(middle, 0); + fdQueryFilter.right = new FormAttachment(100, 0); + fdQueryFilter.top = new FormAttachment(wQuerySelect, margin); + wQueryFilter.setLayoutData(fdQueryFilter); + + // OrderBy line + Label wlQueryOrder = new Label(wQueryComp, SWT.RIGHT); + wlQueryOrder.setText(BaseMessages.getString(PKG, "ODataInputDialog.QueryOrder.Label")); + PropsUi.setLook(wlQueryOrder); + FormData fdlQueryOrder = new FormData(); + fdlQueryOrder.left = new FormAttachment(0, 0); + fdlQueryOrder.right = new FormAttachment(middle, -margin); + fdlQueryOrder.top = new FormAttachment(wQueryFilter, margin); + wlQueryOrder.setLayoutData(fdlQueryOrder); + wQueryOrder = new TextVar(variables, wQueryComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wQueryOrder); + wQueryOrder.addModifyListener(lsMod); + FormData fdQueryOrder = new FormData(); + fdQueryOrder.left = new FormAttachment(middle, 0); + fdQueryOrder.right = new FormAttachment(100, 0); + fdQueryOrder.top = new FormAttachment(wQueryFilter, margin); + wQueryOrder.setLayoutData(fdQueryOrder); + + // Top line + Label wlQueryTop = new Label(wQueryComp, SWT.RIGHT); + wlQueryTop.setText(BaseMessages.getString(PKG, "ODataInputDialog.QueryTop.Label")); + PropsUi.setLook(wlQueryTop); + FormData fdlQueryTop = new FormData(); + fdlQueryTop.left = new FormAttachment(0, 0); + fdlQueryTop.right = new FormAttachment(middle, -margin); + fdlQueryTop.top = new FormAttachment(wQueryOrder, margin); + wlQueryTop.setLayoutData(fdlQueryTop); + wQueryTop = new TextVar(variables, wQueryComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wQueryTop); + wQueryTop.addModifyListener(lsMod); + FormData fdQueryTop = new FormData(); + fdQueryTop.left = new FormAttachment(middle, 0); + fdQueryTop.right = new FormAttachment(100, 0); + fdQueryTop.top = new FormAttachment(wQueryOrder, margin); + wQueryTop.setLayoutData(fdQueryTop); + + // Skip line + Label wlQuerySkip = new Label(wQueryComp, SWT.RIGHT); + wlQuerySkip.setText(BaseMessages.getString(PKG, "ODataInputDialog.QuerySkip.Label")); + PropsUi.setLook(wlQuerySkip); + FormData fdlQuerySkip = new FormData(); + fdlQuerySkip.left = new FormAttachment(0, 0); + fdlQuerySkip.right = new FormAttachment(middle, -margin); + fdlQuerySkip.top = new FormAttachment(wQueryTop, margin); + wlQuerySkip.setLayoutData(fdlQuerySkip); + wQuerySkip = new TextVar(variables, wQueryComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wQuerySkip); + wQuerySkip.addModifyListener(lsMod); + FormData fdQuerySkip = new FormData(); + fdQuerySkip.left = new FormAttachment(middle, 0); + fdQuerySkip.right = new FormAttachment(100, 0); + fdQuerySkip.top = new FormAttachment(wQueryTop, margin); + wQuerySkip.setLayoutData(fdQuerySkip); + + FormData fdQueryComp = new FormData(); + fdQueryComp.left = new FormAttachment(0, 0); + fdQueryComp.right = new FormAttachment(100, 0); + fdQueryComp.top = new FormAttachment(0, 0); + fdQueryComp.bottom = new FormAttachment(100, 0); + wQueryComp.setLayoutData(fdQueryComp); + wQueryComp.layout(); + wQueryTab.setControl(wQueryComp); + + // //////////////////////// + // START OF FIELDS TAB + // //////////////////////// + CTabItem wFieldsTab = new CTabItem(wTabFolder, SWT.NONE); + wFieldsTab.setText(BaseMessages.getString(PKG, "ODataInputDialog.FieldsTab.Title")); + Composite wFieldsComp = new Composite(wTabFolder, SWT.NONE); + PropsUi.setLook(wFieldsComp); + FormLayout fieldsLayout = new FormLayout(); + fieldsLayout.marginWidth = margin; + fieldsLayout.marginHeight = margin; + wFieldsComp.setLayout(fieldsLayout); + + Button wGetFields = new Button(wFieldsComp, SWT.PUSH); + wGetFields.setText(BaseMessages.getString(PKG, "ODataInputDialog.Button.GetFields")); + wGetFields.addListener(SWT.Selection, e -> getFields()); + FormData fdGetFields = new FormData(); + fdGetFields.right = new FormAttachment(100, 0); + fdGetFields.top = new FormAttachment(0, margin); + wGetFields.setLayoutData(fdGetFields); + + ColumnInfo[] colinf = + new ColumnInfo[] { + new ColumnInfo( + BaseMessages.getString(PKG, "ODataInputDialog.Fields.Column.HopName"), + ColumnInfo.COLUMN_TYPE_TEXT, + false), + new ColumnInfo( + BaseMessages.getString(PKG, "ODataInputDialog.Fields.Column.ODataPath"), + ColumnInfo.COLUMN_TYPE_TEXT, + false), + new ColumnInfo( + BaseMessages.getString(PKG, "ODataInputDialog.Fields.Column.HopType"), + ColumnInfo.COLUMN_TYPE_CCOMBO, + ValueMetaFactory.getValueMetaNames()), + new ColumnInfo( + BaseMessages.getString(PKG, "ODataInputDialog.Fields.Column.Format"), + ColumnInfo.COLUMN_TYPE_FORMAT, + 3) + }; + + wFields = + new TableView( + variables, + wFieldsComp, + SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI, + colinf, + 1, + lsMod, + props); + FormData fdFields = new FormData(); + fdFields.left = new FormAttachment(0, 0); + fdFields.right = new FormAttachment(100, 0); + fdFields.top = new FormAttachment(wGetFields, margin); + fdFields.bottom = new FormAttachment(100, 0); + wFields.setLayoutData(fdFields); + + FormData fdFieldsComp = new FormData(); + fdFieldsComp.left = new FormAttachment(0, 0); + fdFieldsComp.right = new FormAttachment(100, 0); + fdFieldsComp.top = new FormAttachment(0, 0); + fdFieldsComp.bottom = new FormAttachment(100, 0); + wFieldsComp.setLayoutData(fdFieldsComp); + wFieldsComp.layout(); + wFieldsTab.setControl(wFieldsComp); + + // Layout Tab Folder + FormData fdTabFolder = new FormData(); + fdTabFolder.left = new FormAttachment(0, 0); + fdTabFolder.right = new FormAttachment(100, 0); + fdTabFolder.top = new FormAttachment(wTransformName, margin); + fdTabFolder.bottom = new FormAttachment(wOk, -margin); + wTabFolder.setLayoutData(fdTabFolder); + + wTabFolder.setSelection(0); + + // Populate data + getData(); + input.setChanged(changed); + + shell.open(); + Point size = shell.computeSize(SWT.DEFAULT, SWT.DEFAULT); + shell.setSize(Math.max(600, size.x), Math.max(500, size.y)); + while (!shell.isDisposed()) { + if (!shell.getDisplay().readAndDispatch()) { + shell.getDisplay().sleep(); + } + } + return transformName; + } + + private void getData() { + wUrl.setText(Const.NVL(input.getUrl(), "")); + wEntitySet.setText(Const.NVL(input.getEntitySet(), "")); + wAuthType.setText(ODataAuthType.lookupCode(input.getAuthType()).getDescription()); + wUsername.setText(Const.NVL(input.getUsername(), "")); + wPassword.setText(Const.NVL(input.getPassword(), "")); + wToken.setText(Const.NVL(input.getToken(), "")); + + wQuerySelect.setText(Const.NVL(input.getQuerySelect(), "")); + wQueryFilter.setText(Const.NVL(input.getQueryFilter(), "")); + wQueryOrder.setText(Const.NVL(input.getQueryOrder(), "")); + wQueryTop.setText(Const.NVL(input.getQueryTop(), "")); + wQuerySkip.setText(Const.NVL(input.getQuerySkip(), "")); + + for (int i = 0; i < input.getFields().size(); i++) { + ODataField field = input.getFields().get(i); + TableItem item = new TableItem(wFields.table, SWT.NONE); + item.setText(1, Const.NVL(field.getName(), "")); + item.setText(2, Const.NVL(field.getPath(), "")); + item.setText(3, Const.NVL(ValueMetaFactory.getValueMetaName(field.getType()), "")); + item.setText(4, Const.NVL(field.getFormat(), "")); + } + wFields.removeEmptyRows(); + wFields.setRowNums(); + wFields.optWidth(true); + } + + private void cancel() { + transformName = null; + input.setChanged(changed); + dispose(); + } + + private void ok() { + if (Utils.isEmpty(wTransformName.getText())) { + return; + } + transformName = wTransformName.getText(); + + input.setUrl(wUrl.getText()); + input.setEntitySet(wEntitySet.getText()); + input.setAuthType(ODataAuthType.lookupDescription(wAuthType.getText()).getCode()); + input.setUsername(wUsername.getText()); + input.setPassword(wPassword.getText()); + input.setToken(wToken.getText()); + + input.setQuerySelect(wQuerySelect.getText()); + input.setQueryFilter(wQueryFilter.getText()); + input.setQueryOrder(wQueryOrder.getText()); + input.setQueryTop(wQueryTop.getText()); + input.setQuerySkip(wQuerySkip.getText()); + + input.getFields().clear(); + int nrFields = wFields.nrNonEmpty(); + for (int i = 0; i < nrFields; i++) { + TableItem item = wFields.getNonEmpty(i); + String name = item.getText(1); + String path = item.getText(2); + int type = ValueMetaFactory.getIdForValueMeta(item.getText(3)); + String format = item.getText(4); + input.getFields().add(new ODataField(name, path, type, format)); + } + + input.setChanged(); + dispose(); + } + + private void getEntitySets() { + String urlStr = wUrl.getText(); + if (Utils.isEmpty(urlStr)) { + return; + } + urlStr = variables.resolve(urlStr); + try { + HttpClientManager.HttpClientBuilderFacade builder = + HttpClientManager.getInstance().createBuilder(); + ODataAuthType authType = ODataAuthType.lookupDescription(wAuthType.getText()); + if (ODataAuthType.BASIC == authType && !Utils.isEmpty(wUsername.getText())) { + builder.setCredentials( + variables.resolve(wUsername.getText()), variables.resolve(wPassword.getText())); + } + try (org.apache.hc.client5.http.impl.classic.CloseableHttpClient client = builder.build()) { + HttpGet get = new HttpGet(urlStr); + get.setHeader("Accept", "application/json"); + if (ODataAuthType.BEARER == authType && !Utils.isEmpty(wToken.getText())) { + get.setHeader("Authorization", "Bearer " + variables.resolve(wToken.getText())); + } + try (CloseableHttpResponse response = client.execute(get)) { + int code = response.getCode(); + if (code != 200) { + throw new HopException("HTTP " + code); + } + String body = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); + ObjectMapper mapper = new ObjectMapper(); + JsonNode root = mapper.readTree(body); + List list = new ArrayList<>(); + if (root.has("value")) { + JsonNode val = root.get("value"); + if (val.isArray()) { + for (JsonNode it : val) { + if (it.has("name")) { + list.add(it.get("name").asText()); + } + } + } + } else if (root.has("d")) { + JsonNode d = root.get("d"); + if (d.has("EntitySets") && d.get("EntitySets").isArray()) { + for (JsonNode it : d.get("EntitySets")) { + list.add(it.asText()); + } + } else if (d.isArray()) { + for (JsonNode it : d) { + if (it.has("name")) { + list.add(it.get("name").asText()); + } + } + } else { + d.fieldNames() + .forEachRemaining( + name -> { + if (!name.startsWith("__")) { + list.add(name); + } + }); + } + } + if (!list.isEmpty()) { + wEntitySet.setItems(list.toArray(new String[0])); + } + } + } + } catch (Exception e) { + new ErrorDialog(shell, "Error", "Error fetching entity sets", e); + } + } + + private void getFields() { + String urlStr = wUrl.getText(); + String entitySetName = wEntitySet.getText(); + if (Utils.isEmpty(urlStr) || Utils.isEmpty(entitySetName)) { + return; + } + urlStr = variables.resolve(urlStr); + entitySetName = variables.resolve(entitySetName); + if (!urlStr.endsWith("/")) { + urlStr += "/"; + } + String metadataUrl = urlStr + "$metadata"; + try { + HttpClientManager.HttpClientBuilderFacade builder = + HttpClientManager.getInstance().createBuilder(); + ODataAuthType authType = ODataAuthType.lookupDescription(wAuthType.getText()); + if (ODataAuthType.BASIC == authType && !Utils.isEmpty(wUsername.getText())) { + builder.setCredentials( + variables.resolve(wUsername.getText()), variables.resolve(wPassword.getText())); + } + try (org.apache.hc.client5.http.impl.classic.CloseableHttpClient client = builder.build()) { + HttpGet get = new HttpGet(metadataUrl); + if (ODataAuthType.BEARER == authType && !Utils.isEmpty(wToken.getText())) { + get.setHeader("Authorization", "Bearer " + variables.resolve(wToken.getText())); + } + try (CloseableHttpResponse response = client.execute(get)) { + int code = response.getCode(); + if (code != 200) { + throw new HopException("HTTP " + code); + } + String body = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); + + javax.xml.parsers.DocumentBuilderFactory factory = + javax.xml.parsers.DocumentBuilderFactory.newInstance(); + factory.setNamespaceAware(true); + javax.xml.parsers.DocumentBuilder db = factory.newDocumentBuilder(); + org.w3c.dom.Document doc = + db.parse(new java.io.ByteArrayInputStream(body.getBytes(StandardCharsets.UTF_8))); + + org.w3c.dom.NodeList entitySets = doc.getElementsByTagNameNS("*", "EntitySet"); + String entityType = null; + for (int i = 0; i < entitySets.getLength(); i++) { + org.w3c.dom.Element set = (org.w3c.dom.Element) entitySets.item(i); + if (entitySetName.equalsIgnoreCase(set.getAttribute("Name"))) { + entityType = set.getAttribute("EntityType"); + break; + } + } + + if (entityType == null) { + throw new HopException("Entity Set not found in metadata: " + entitySetName); + } + + String entityTypeName = entityType; + int lastDot = entityType.lastIndexOf("."); + if (lastDot != -1) { + entityTypeName = entityType.substring(lastDot + 1); + } + + org.w3c.dom.NodeList entityTypes = doc.getElementsByTagNameNS("*", "EntityType"); + org.w3c.dom.Element targetEntityType = null; + for (int i = 0; i < entityTypes.getLength(); i++) { + org.w3c.dom.Element type = (org.w3c.dom.Element) entityTypes.item(i); + if (entityTypeName.equalsIgnoreCase(type.getAttribute("Name"))) { + targetEntityType = type; + break; + } + } + + if (targetEntityType == null) { + throw new HopException( + "EntityType definition not found in metadata: " + entityTypeName); + } + + org.w3c.dom.NodeList properties = + targetEntityType.getElementsByTagNameNS("*", "Property"); + wFields.table.removeAll(); + for (int i = 0; i < properties.getLength(); i++) { + org.w3c.dom.Element prop = (org.w3c.dom.Element) properties.item(i); + String name = prop.getAttribute("Name"); + String type = prop.getAttribute("Type"); + + String hopType = "String"; + if (type != null) { + if (type.contains("Int16") + || type.contains("Int32") + || type.contains("Int64") + || type.contains("Byte")) { + hopType = "Integer"; + } else if (type.contains("Decimal") + || type.contains("Double") + || type.contains("Single")) { + hopType = "Number"; + } else if (type.contains("Boolean")) { + hopType = "Boolean"; + } else if (type.contains("DateTime") || type.contains("Date")) { + hopType = "Date"; + } + } + + TableItem item = new TableItem(wFields.table, SWT.NONE); + item.setText(1, name); + item.setText(2, name); + item.setText(3, hopType); + item.setText(4, ""); + } + wFields.removeEmptyRows(); + wFields.setRowNums(); + wFields.optWidth(true); + } + } + } catch (Exception e) { + new ErrorDialog(shell, "Error", "Error fetching fields metadata", e); + } + } +} diff --git a/plugins/transforms/odata/src/main/java/org/apache/hop/pipeline/transforms/odata/ODataInputMeta.java b/plugins/transforms/odata/src/main/java/org/apache/hop/pipeline/transforms/odata/ODataInputMeta.java new file mode 100644 index 00000000000..14d5a185e5f --- /dev/null +++ b/plugins/transforms/odata/src/main/java/org/apache/hop/pipeline/transforms/odata/ODataInputMeta.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.odata; + +import java.util.ArrayList; +import java.util.List; +import lombok.Getter; +import lombok.Setter; +import org.apache.hop.core.CheckResult; +import org.apache.hop.core.ICheckResult; +import org.apache.hop.core.annotations.Transform; +import org.apache.hop.core.exception.HopTransformException; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.metadata.api.HopMetadataProperty; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransformMeta; +import org.apache.hop.pipeline.transform.TransformMeta; + +@Getter +@Setter +@Transform( + id = "ODataInput", + image = "odata.svg", + name = "i18n::ODataInput.Name", + description = "i18n::ODataInput.Description", + categoryDescription = "i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Input", + keywords = "i18n::ODataInputMeta.keyword", + documentationUrl = "/pipeline/transforms/odata-input.html") +public class ODataInputMeta extends BaseTransformMeta { + private static final Class PKG = ODataInputMeta.class; + + @HopMetadataProperty(key = "url") + private String url; + + @HopMetadataProperty(key = "entity_set") + private String entitySet; + + @HopMetadataProperty(key = "auth_type") + private String authType; + + @HopMetadataProperty(key = "username") + private String username; + + @HopMetadataProperty(key = "password", password = true) + private String password; + + @HopMetadataProperty(key = "token", password = true) + private String token; + + @HopMetadataProperty(key = "query_select") + private String querySelect; + + @HopMetadataProperty(key = "query_filter") + private String queryFilter; + + @HopMetadataProperty(key = "query_order") + private String queryOrder; + + @HopMetadataProperty(key = "query_top") + private String queryTop; + + @HopMetadataProperty(key = "query_skip") + private String querySkip; + + @HopMetadataProperty(key = "field", groupKey = "fields") + private List fields; + + public ODataInputMeta() { + super(); + fields = new ArrayList<>(); + authType = "NONE"; + } + + public ODataInputMeta(ODataInputMeta other) { + this(); + this.url = other.url; + this.entitySet = other.entitySet; + this.authType = other.authType; + this.username = other.username; + this.password = other.password; + this.token = other.token; + this.querySelect = other.querySelect; + this.queryFilter = other.queryFilter; + this.queryOrder = other.queryOrder; + this.queryTop = other.queryTop; + this.querySkip = other.querySkip; + if (other.fields != null) { + for (ODataField f : other.fields) { + this.fields.add(new ODataField(f)); + } + } + } + + @Override + public void setDefault() { + this.url = ""; + this.entitySet = ""; + this.authType = "NONE"; + this.username = ""; + this.password = ""; + this.token = ""; + this.querySelect = ""; + this.queryFilter = ""; + this.queryOrder = ""; + this.queryTop = ""; + this.querySkip = ""; + this.fields = new ArrayList<>(); + } + + @Override + public void getFields( + IRowMeta rowMeta, + String name, + IRowMeta[] info, + TransformMeta nextTransform, + IVariables variables, + IHopMetadataProvider metadataProvider) + throws HopTransformException { + for (ODataField field : fields) { + try { + rowMeta.addValueMeta(field.toValueMeta(name, variables)); + } catch (Exception e) { + throw new HopTransformException( + "Error generating value meta for field: " + field.getName(), e); + } + } + } + + @Override + public void check( + List remarks, + PipelineMeta pipelineMeta, + TransformMeta transformMeta, + IRowMeta prev, + String[] input, + String[] output, + IRowMeta info, + IVariables variables, + IHopMetadataProvider metadataProvider) { + if (url == null || url.trim().isEmpty()) { + remarks.add( + new CheckResult( + ICheckResult.TYPE_RESULT_ERROR, "Service URL is missing.", transformMeta)); + } else { + remarks.add( + new CheckResult(ICheckResult.TYPE_RESULT_OK, "Service URL is specified.", transformMeta)); + } + + if (entitySet == null || entitySet.trim().isEmpty()) { + remarks.add( + new CheckResult(ICheckResult.TYPE_RESULT_ERROR, "Entity Set is missing.", transformMeta)); + } else { + remarks.add( + new CheckResult(ICheckResult.TYPE_RESULT_OK, "Entity Set is specified.", transformMeta)); + } + } +} diff --git a/plugins/transforms/odata/src/main/resources/dependencies.xml b/plugins/transforms/odata/src/main/resources/dependencies.xml new file mode 100644 index 00000000000..4e93773f89e --- /dev/null +++ b/plugins/transforms/odata/src/main/resources/dependencies.xml @@ -0,0 +1,21 @@ + + + + + diff --git a/plugins/transforms/odata/src/main/resources/odata.svg b/plugins/transforms/odata/src/main/resources/odata.svg new file mode 100644 index 00000000000..cde9c9b82b7 --- /dev/null +++ b/plugins/transforms/odata/src/main/resources/odata.svg @@ -0,0 +1,40 @@ + + + + + + + + OD + + diff --git a/plugins/transforms/odata/src/main/resources/org/apache/hop/pipeline/transforms/odata/messages/messages_en_US.properties b/plugins/transforms/odata/src/main/resources/org/apache/hop/pipeline/transforms/odata/messages/messages_en_US.properties new file mode 100644 index 00000000000..854ab755090 --- /dev/null +++ b/plugins/transforms/odata/src/main/resources/org/apache/hop/pipeline/transforms/odata/messages/messages_en_US.properties @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ODataInput.Name=OData Input +ODataInput.Description=Retrieve data from an OData service (V2 and V4). +ODataInputMeta.keyword=odata,input,rest,service,sap,microsoft + +ODataInputDialog.TransformName.Label=Transform name +ODataInputDialog.GeneralTab.Title=Connection +ODataInputDialog.QueryTab.Title=OData Query +ODataInputDialog.FieldsTab.Title=Fields + +ODataInputDialog.ServiceUrl.Label=OData Service Root URL +ODataInputDialog.EntitySet.Label=OData Entity Set +ODataInputDialog.AuthType.Label=Authentication Type +ODataInputDialog.Username.Label=Username (Basic Auth) +ODataInputDialog.Password.Label=Password (Basic Auth) +ODataInputDialog.Token.Label=Token (Bearer / OAuth2) + +ODataInputDialog.QuerySelect.Label=$select (comma separated fields) +ODataInputDialog.QueryFilter.Label=$filter condition +ODataInputDialog.QueryOrder.Label=$orderby field desc/asc +ODataInputDialog.QueryTop.Label=$top count +ODataInputDialog.QuerySkip.Label=$skip count + +ODataInputDialog.Fields.Column.HopName=Name in Hop +ODataInputDialog.Fields.Column.ODataPath=OData Path +ODataInputDialog.Fields.Column.HopType=Type +ODataInputDialog.Fields.Column.Format=Format + +ODataInputDialog.Button.GetEntitySets=Get Entity Sets +ODataInputDialog.Button.GetFields=Get Fields +ODataInputDialog.Button.OK=&OK +ODataInputDialog.Button.Cancel=&Cancel + +ODataAuthType.None.Description=No Authentication +ODataAuthType.Basic.Description=Basic Authentication (Username / Password) +ODataAuthType.Bearer.Description=Bearer Token (Authorization Header) diff --git a/plugins/transforms/odata/src/main/resources/version.xml b/plugins/transforms/odata/src/main/resources/version.xml new file mode 100644 index 00000000000..36ab20e22eb --- /dev/null +++ b/plugins/transforms/odata/src/main/resources/version.xml @@ -0,0 +1,20 @@ + + + +${project.version} diff --git a/plugins/transforms/pom.xml b/plugins/transforms/pom.xml index a481cedd4a4..ab621dd3f26 100644 --- a/plugins/transforms/pom.xml +++ b/plugins/transforms/pom.xml @@ -112,6 +112,7 @@ normaliser nullif numberrange + odata pgp pipelineexecutor processfiles