Skip to content

issue #7308 : OData Input #7362

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hansva merged 1 commit into from
Jun 29, 2026
+2,229 −0
Merged

issue #7308 : OData Input #7362

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions assemblies/plugins/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1291,6 +1291,12 @@
<version>${project.version}</version>
<type>zip</type>
</dependency>
<dependency>
<groupId>org.apache.hop</groupId>
<artifactId>hop-transform-odata</artifactId>
<version>${project.version}</version>
<type>zip</type>
</dependency>
<dependency>
<groupId>org.apache.hop</groupId>
<artifactId>hop-transform-pgp</artifactId>
Expand Down
39 changes: 39 additions & 0 deletions docker/integration-tests/integration-tests-odata.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

services:
integration_test_database:
extends:
file: integration-tests-base.yaml
service: integration_test
depends_on:
odata_mock:
condition: service_started
links:
- odata_mock

odata_mock:
image: mockserver/mockserver:5.15.0
hostname: odata_mock
ports:
- "1080"
environment:
- MOCKSERVER_INITIALIZATION_JSON_PATH=/config/initializerJson.json
- MOCKSERVER_LOG_LEVEL=WARN
volumes:
- ../../integration-tests/odata/import/initializerJson.json:/config/initializerJson.json
- ../../integration-tests/odata/import/metadata.xml:/config/metadata.xml
Binary file added ...ser-manual/modules/ROOT/assets/images/transforms/odata-input-connection-tab.png
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added ...op-user-manual/modules/ROOT/assets/images/transforms/odata-input-fields-tab.png
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added ...hop-user-manual/modules/ROOT/assets/images/transforms/odata-input-query-tab.png
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
1 change: 1 addition & 0 deletions docs/hop-user-manual/modules/ROOT/nav.adoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,7 @@ under the License.
*** xref:pipeline/transforms/neo4j-split-graph.adoc[Neo4j Split Graph]
*** xref:pipeline/transforms/nullif.adoc[Null If]
*** xref:pipeline/transforms/numberrange.adoc[Number range]
*** xref:pipeline/transforms/odata-input.adoc[OData Input]
*** xref:pipeline/transforms/orabulkloader.adoc[Oracle Bulk Loader]
*** xref:pipeline/transforms/parquet-file-input.adoc[Parquet File Input]
*** xref:pipeline/transforms/parquet-file-output.adoc[Parquet File Output]
Expand Down
1 change: 1 addition & 0 deletions docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms.adoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,7 @@ The pages nested under this topic contain information on how to use the transfor
* xref:pipeline/transforms/neo4j-split-graph.adoc[Neo4j Split Graph]
* xref:pipeline/transforms/nullif.adoc[Null If]
* xref:pipeline/transforms/numberrange.adoc[Number range]
* xref:pipeline/transforms/odata-input.adoc[OData Input]
* xref:pipeline/transforms/orabulkloader.adoc[Oracle Bulk Loader]
* xref:pipeline/transforms/parquet-file-input.adoc[Parquet File Input]
* xref:pipeline/transforms/parquet-file-output.adoc[Parquet File Output]
Expand Down
104 changes: 104 additions & 0 deletions docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/odata-input.adoc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
////
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
////
:documentationPath: /pipeline/transforms/
:language: en_US
:description: The OData Input transform retrieves data from OData V2 and V4 service endpoints.

= image:transforms/icons/odata.svg[OData Input transform Icon, role="image-doc-icon"] OData Input

[%noheader,cols="3a,1a", role="table-no-borders" ]
|===
|
== Description

The OData Input transform enables you to query OData (Open Data Protocol) services. OData is an OASIS standard that defines a set of best practices for building and consuming RESTful APIs. For more information, visit the official link:https://www.odata.org/[OData Documentation].

This transform performs GET requests against OData Entity Sets, maps JSON properties to Apache Hop rows, and automatically handles relative pagination using standard `@odata.nextLink` (OData V4) and `__next` (OData V2) metadata attributes.

|
== Supported Engines
[%noheader,cols="2,1a",frame=none, role="table-supported-engines"]
!===
!Hop Engine! image:check_mark.svg[Supported, 24]
!Spark! image:question_mark.svg[Maybe Supported, 24]
!Flink! image:question_mark.svg[Maybe Supported, 24]
!Dataflow! image:question_mark.svg[Maybe Supported, 24]
!===
|===

== Connection Options

The Connection tab allows you to configure the service endpoint and optional credentials.

image::transforms/odata-input-connection-tab.png[OData Input Connection Tab, align="center"]

[options="header"]
|===
|Option|Description
|Transform name|Name of this transform as it appears in the pipeline workspace.
|OData Service Root URL|The base URL of the OData service (e.g., `https://services.odata.org/V4/TripPinServiceRW/`).
|Authentication Type|The authentication type to use:
* `No Authentication`: Anonymous connection.
* `Basic Authentication`: Use standard Username/Password credentials.
* `Bearer Token`: Use a Bearer token in the `Authorization` header.
|Username (Basic Auth)|The username for Basic authentication.
|Password (Basic Auth)|The password for Basic authentication.
|Token (Bearer / OAuth2)|The bearer token value.
|OData Entity Set|The OData entity set to query (e.g., `People` or `Airports`).
|Get Entity Sets|Retrieves and displays a drop-down list of available entity sets from the service endpoint.
|===

== Query Options

The OData Query tab allows you to configure filtering, ordering, and field selection properties.

image::transforms/odata-input-query-tab.png[OData Input Query Tab, align="center"]

[options="header"]
|===
|Option|Description
|$select|A comma-separated list of properties to select (e.g., `UserName,FirstName,LastName`).
|$filter|An OData expression to filter matching entities (e.g., `Gender eq 'Male'`).
|$orderby|Specifies the sorting order of the results (e.g., `LastName asc, FirstName desc`).
|$top|Limits the number of entities returned on each page request (e.g., `50`).
|$skip|Specifies the number of entities to skip from the beginning of the list.
|===

== Fields Mapping

The Fields tab defines how properties in the returned OData JSON response map to Apache Hop row fields.

image::transforms/odata-input-fields-tab.png[OData Input Fields Tab, align="center"]

[options="header"]
|===
|Option|Description
|Name in Hop|The name of the field as it will be produced in the Apache Hop pipeline.
|OData Path|The JSON property path of the field within the entity JSON object (e.g., `UserName`, `AddressInfo/0/City/Name` for nested structures).
|Type|The Hop data type (String, Integer, Number, Date, Boolean, BigNumber).
|Format|Optional parsing format for date or numeric fields.
|Get Fields|Connects to the `$metadata` endpoint of the OData service and automatically populates the fields list based on the selected Entity Set's schema properties.
|===

== Example

To query a public OData service:
1. Set the **Service Root URL** to `https://services.odata.org/V4/TripPinServiceRW/`.
2. Set the **Entity Set** to `People`.
3. Go to the **Fields** tab and click **Get Fields** to pull all metadata fields (`UserName`, `FirstName`, `LastName`, etc.).
4. (Optional) In the **OData Query** tab, add a `$filter` such as `FirstName eq 'John'` or a `$select` of `UserName,Emails`.
5. Run the pipeline to stream entities into downstream transforms.
163 changes: 163 additions & 0 deletions integration-tests/odata/0001-odata-query.hpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
~
-->
<pipeline>
<info>
<name>0001-odata-query</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
<parameters>
</parameters>
</info>
<order>
<hop>
<from>OData Input</from>
<to>count</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>count</from>
<to>check count</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>check count</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>OData Input</name>
<type>ODataInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<url>http://odata_mock:1080/odata</url>
<entity_set>Customers</entity_set>
<auth_type>NONE</auth_type>
<fields>
<field>
<name>Id</name>
<path>Id</path>
<type>Integer</type>
</field>
<field>
<name>Name</name>
<path>Name</path>
<type>String</type>
</field>
<field>
<name>Balance</name>
<path>Balance</path>
<type>Number</type>
</field>
</fields>
<GUI>
<xloc>100</xloc>
<yloc>100</yloc>
</GUI>
</transform>
<transform>
<name>count</name>
<type>MemoryGroupBy</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<fields>
<field>
<aggregate>count</aggregate>
<type>COUNT_ANY</type>
</field>
</fields>
<give_back_row>N</give_back_row>
<group>
</group>
<GUI>
<xloc>250</xloc>
<yloc>100</yloc>
</GUI>
</transform>
<transform>
<name>check count</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>&lt;&gt;</function>
<leftvalue>count</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<mask>####0;-####0</mask>
<name>constant</name>
<precision>0</precision>
<text>3</text>
<type>Integer</type>
</value>
</condition>
</compare>
<GUI>
<xloc>400</xloc>
<yloc>100</yloc>
</GUI>
</transform>
<transform>
<name>Abort</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT_WITH_ERROR</abort_option>
<always_log_rows>Y</always_log_rows>
<message>Incorrect number of OData records received</message>
<row_threshold>0</row_threshold>
<GUI>
<xloc>550</xloc>
<yloc>100</yloc>
</GUI>
</transform>
</pipeline>
9 changes: 9 additions & 0 deletions integration-tests/odata/dev-env-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"variables" : [
{
"name" : "HOSTNAME",
"value" : "odata_mock",
"description" : "hostname of odata_mock container"
}
]
}
34 changes: 34 additions & 0 deletions integration-tests/odata/hop-config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
{
"variables": [],
"LocaleDefault": "en_US",
"guiProperties": {
"DarkMode": "Y"
},
"projectsConfig": {
"enabled": true,
"projectMandatory": true,
"environmentMandatory": false,
"defaultProject": "odata",
"defaultEnvironment": null,
"standardParentProject": "odata",
"standardProjectsFolder": null,
"projectConfigurations": [
{
"projectName": "odata",
"projectHome": "${HOP_CONFIG_FOLDER}",
"configFilename": "project-config.json"
}
],
"lifecycleEnvironments": [
{
"name": "dev",
"purpose": "Testing",
"projectName": "odata",
"configurationFiles": [
"${PROJECT_HOME}/dev-env-config.json"
]
}
],
"projectLifecycles": []
}
}
Loading
Loading