Skip to content

Improving Data Validation Performance, fixes #7589 #7812

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hansva merged 1 commit into from
Aug 6, 2026
Merged

Improving Data Validation Performance, fixes #7589 #7812

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/validator.adoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,12 @@ Once you select a validation on the left hand side of the dialog you can specify
|Output one row, concatenate errors with a separator
|If this is enabled error codes and descriptions are concatenated with the specified separator string.

|Do not log failed validation data
|When enabled, the failed field and row values are left out of the validation error messages and shown as `?` instead. The error messages themselves are still logged and error rows are still sent to the error handling hop.

|Do not log a line for every rejected row
|When enabled, the transform no longer writes an error line to the log for every row it rejects. Error rows are still sent to the error handling hop. On streams where a large share of the rows is rejected, writing that log line costs considerably more than the validation itself, so enabling this option can speed up such pipelines by an order of magnitude.

|Validation description
|Optionally you can describe what you're testing for here.

Expand Down
380 changes: 380 additions & 0 deletions integration-tests/transforms/0067-data-validator-source-multicopy.hpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,380 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--

Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

-->
<pipeline>
<info>
<name>0067-data-validator-source-multicopy</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
<parameters>
</parameters>
<capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
<created_user>-</created_user>
<created_date>2023/03/08 15:11:16.589</created_date>
<modified_user>-</modified_user>
<modified_date>2023/03/08 15:11:16.589</modified_date>
</info>
<notepads>
</notepads>
<order>
<hop>
<from>Validate data</from>
<to>sort errors</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Validate data</from>
<to>sort output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sort output</from>
<to>Output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sort errors</from>
<to>Errors</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sample data</from>
<to>Validate data</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>allowed values</from>
<to>Validate data</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Errors</name>
<type>Dummy</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<attributes/>
<GUI>
<xloc>496</xloc>
<yloc>176</yloc>
</GUI>
</transform>
<transform>
<name>Output</name>
<type>Dummy</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<attributes/>
<GUI>
<xloc>496</xloc>
<yloc>80</yloc>
</GUI>
</transform>
<transform>
<name>Validate data</name>
<type>Validator</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>4</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<concat_errors>N</concat_errors>
<validate_all>Y</validate_all>
<validator_field>
<allowed_value>
</allowed_value>
<data_type>String</data_type>
<data_type_verified>N</data_type_verified>
<error_code>STR1_DICTIONARY</error_code>
<error_description>Field string1 is not matching with lookup values.</error_description>
<is_sourcing_values>Y</is_sourcing_values>
<max_length/>
<min_length/>
<name>string1</name>
<null_allowed>Y</null_allowed>
<only_null_allowed>N</only_null_allowed>
<only_numeric_allowed>N</only_numeric_allowed>
<sourcing_field>allowed_value</sourcing_field>
<sourcing_transform>allowed values</sourcing_transform>
<validation_name>string1: verify against allowed values</validation_name>
</validator_field>
<validator_field>
<allowed_value>
</allowed_value>
<data_type>String</data_type>
<data_type_verified>N</data_type_verified>
<error_code>STR2_DICTIONARY</error_code>
<error_description>Field string2 is not matching with lookup values.</error_description>
<is_sourcing_values>Y</is_sourcing_values>
<max_length/>
<min_length/>
<name>string2</name>
<null_allowed>Y</null_allowed>
<only_null_allowed>N</only_null_allowed>
<only_numeric_allowed>N</only_numeric_allowed>
<sourcing_field>allowed_value</sourcing_field>
<sourcing_transform>allowed values</sourcing_transform>
<validation_name>string2: verify against allowed values</validation_name>
</validator_field>
<attributes/>
<GUI>
<xloc>304</xloc>
<yloc>80</yloc>
</GUI>
</transform>
<transform>
<name>allowed values</name>
<type>DataGrid</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<data>
<line>
<item>A</item>
</line>
<line>
<item>B</item>
</line>
<line>
<item>C</item>
</line>
</data>
<fields>
<field>
<length>-1</length>
<precision>-1</precision>
<set_empty_string>N</set_empty_string>
<name>allowed_value</name>
<type>String</type>
</field>
</fields>
<attributes/>
<GUI>
<xloc>128</xloc>
<yloc>208</yloc>
</GUI>
</transform>
<transform>
<name>sample data</name>
<type>DataGrid</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<data>
<line>
<item>A</item>
<item>B</item>
</line>
<line>
<item>B</item>
<item>X</item>
</line>
<line>
<item>C</item>
<item>A</item>
</line>
<line>
<item>X</item>
<item>C</item>
</line>
<line>
<item>B</item>
<item>Z</item>
</line>
<line>
<item>A</item>
<item>A</item>
</line>
<line>
<item>Z</item>
<item>B</item>
</line>
<line>
<item>A</item>
<item>A</item>
</line>
<line>
<item>B</item>
<item>C</item>
</line>
<line>
<item>C</item>
<item>Y</item>
</line>
</data>
<fields>
<field>
<length>-1</length>
<precision>-1</precision>
<set_empty_string>N</set_empty_string>
<name>string1</name>
<type>String</type>
</field>
<field>
<length>-1</length>
<precision>-1</precision>
<set_empty_string>N</set_empty_string>
<name>string2</name>
<type>String</type>
</field>
</fields>
<attributes/>
<GUI>
<xloc>128</xloc>
<yloc>80</yloc>
</GUI>
</transform>
<transform>
<name>sort output</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>N</unique_rows>
<fields>
<field>
<name>string1</name>
<ascending>Y</ascending>
<case_sensitive>Y</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
<field>
<name>string2</name>
<ascending>Y</ascending>
<case_sensitive>Y</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>400</xloc>
<yloc>80</yloc>
</GUI>
</transform>
<transform>
<name>sort errors</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>N</unique_rows>
<fields>
<field>
<name>string1</name>
<ascending>Y</ascending>
<case_sensitive>Y</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
<field>
<name>string2</name>
<ascending>Y</ascending>
<case_sensitive>Y</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
<field>
<name>errorCodes</name>
<ascending>Y</ascending>
<case_sensitive>Y</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>400</xloc>
<yloc>176</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Validate data</source_transform>
<target_transform>sort errors</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename>errorCount</nr_valuename>
<descriptions_valuename>errorDescriptions</descriptions_valuename>
<fields_valuename>errorFields</fields_valuename>
<codes_valuename>errorCodes</codes_valuename>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
</transform_error_handling>
<attributes/>
</pipeline>
Loading