diff --git a/.mvn/jvm.config b/.mvn/jvm.config index 19d754323c1..f061e563ec8 100644 --- a/.mvn/jvm.config +++ b/.mvn/jvm.config @@ -16,4 +16,6 @@ --add-opens java.base/sun.security.action=ALL-UNNAMED --add-opens java.base/sun.util.calendar=ALL-UNNAMED --add-opens java.security.jgss/sun.security.krb5=ALL-UNNAMED ---add-exports java.base/sun.nio.ch=ALL-UNNAMED \ No newline at end of file +--add-exports java.base/sun.nio.ch=ALL-UNNAMED +-Dorg.slf4j.simpleLogger.log.org.jacoco.maven.AgentMojo=warn +-Dorg.slf4j.simpleLogger.log.org.jacoco.maven.AgentITMojo=warn \ No newline at end of file diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/postgresbulkloader.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/postgresbulkloader.adoc index 31b61757323..d6eb17e4496 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/postgresbulkloader.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/postgresbulkloader.adoc @@ -27,7 +27,7 @@ under the License. The PostgreSQL Bulk Loader transform streams data from Hop to Postgresql using https://www.postgresql.org/docs/current/sql-copy.html["COPY DATA FROM STDIN"^] into the database. -TIP: replace boolean fields in your pipeline stream by string fields with "Y" or "N" values to avoid errors. +TIP: boolean stream fields are serialized as `t` or `f`, which PostgreSQL `COPY` accepts for boolean columns. | == Supported Engines diff --git a/integration-tests/database/0039-postgresql-bulkloader-boolean.hpl b/integration-tests/database/0039-postgresql-bulkloader-boolean.hpl new file mode 100644 index 00000000000..fb84d5b79dd --- /dev/null +++ b/integration-tests/database/0039-postgresql-bulkloader-boolean.hpl @@ -0,0 +1,133 @@ + + + + + 0039-postgresql-bulkloader-boolean + Y + + + + Normal + + + N + 1000 + 100 + - + 2026/04/13 12:00:00.000 + - + 2026/04/13 12:00:00.000 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + + test data + PostgreSQL Bulk Loader + Y + + + + PostgreSQL Bulk Loader + PGBulkLoader + + Y + + 1 + + none + + + unit-test-db + ; + " + TRUNCATE + + + id + id + + + + active + is_active + + public + N + pg_bulk_bool_test
+ + + 464 + 128 + + + + test data + DataGrid + + Y + + 1 + + none + + + + + N + -1 + id + -1 + String + + + N + -1 + active + -1 + Boolean + + + + + row1 + true + + + row2 + false + + + row3 + true + + + + + 144 + 128 + + + + + + diff --git a/integration-tests/database/datasets/golden-table-compare-general.csv b/integration-tests/database/datasets/golden-table-compare-general.csv index eb4a7b37894..366dee37766 100644 --- a/integration-tests/database/datasets/golden-table-compare-general.csv +++ b/integration-tests/database/datasets/golden-table-compare-general.csv @@ -1,2 +1,2 @@ -ref_schema,ref_table,cmp_schema,cmp_table,id_fields,excl_fields,key_description,ref_value,compare_value,nrErrors,nrRecordsReferenceTable,nrRecordsCompareTable,nrErrorsLeftJoin,nrErrorsInnerJoin,nrErrorsRightJoin -public,reference_table,,compare_table,id,,,,,3,3,3,0,1,2 +ref_schema,ref_table,cmp_schema,cmp_table,id_fields,excl_fields,key_description,ref_value,compare_value,nrErrors,nrRecordsReferenceTable,nrRecordsCompareTable,nrErrorsLeftJoin,nrErrorsInnerJoin,nrErrorsRightJoin +public,reference_table,,compare_table,id,,,,,3,3,3,1,1,1 diff --git a/integration-tests/database/main-0039-postgresql-bulkloader-boolean.hwf b/integration-tests/database/main-0039-postgresql-bulkloader-boolean.hwf new file mode 100644 index 00000000000..aa1f54e7c86 --- /dev/null +++ b/integration-tests/database/main-0039-postgresql-bulkloader-boolean.hwf @@ -0,0 +1,133 @@ + + + + main-0039-postgresql-bulkloader-boolean + Y + + + + - + 2026/04/13 12:00:00.000 + - + 2026/04/13 12:00:00.000 + + + + + Start + + SPECIAL + + 1 + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 80 + 80 + + + + SQL + + SQL + + + F + T + ${PROJECT_HOME}/scripts/script_postgresql_bulkloader_boolean.sql + F + unit-test-db + N + 288 + 80 + + + + 0039-postgresql-bulkloader-boolean.hpl + + PIPELINE + + ${PROJECT_HOME}/0039-postgresql-bulkloader-boolean.hpl + N + N + N + N + N + + + N + N + Basic + N + Y + N + local + + Y + + N + 496 + 80 + + + + Abort workflow + + ABORT + + N + N + 752 + 80 + + + + + + SQL + 0039-postgresql-bulkloader-boolean.hpl + Y + Y + N + + + Start + SQL + Y + Y + Y + + + 0039-postgresql-bulkloader-boolean.hpl + Abort workflow + Y + N + N + + + + + + diff --git a/integration-tests/database/scripts/script_postgresql_bulkloader_boolean.sql b/integration-tests/database/scripts/script_postgresql_bulkloader_boolean.sql new file mode 100644 index 00000000000..b3e132f3aad --- /dev/null +++ b/integration-tests/database/scripts/script_postgresql_bulkloader_boolean.sql @@ -0,0 +1,24 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file to +you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +DROP TABLE IF EXISTS public.pg_bulk_bool_test; + +CREATE TABLE public.pg_bulk_bool_test +( + id varchar NULL, + is_active boolean NULL +); diff --git a/plugins/transforms/pgbulkloader/src/main/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoader.java b/plugins/transforms/pgbulkloader/src/main/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoader.java index 8aaa6405048..cc8ec34f086 100644 --- a/plugins/transforms/pgbulkloader/src/main/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoader.java +++ b/plugins/transforms/pgbulkloader/src/main/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoader.java @@ -37,6 +37,7 @@ import org.apache.hop.core.database.Database; import org.apache.hop.core.database.DatabaseMeta; import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.exception.HopValueException; import org.apache.hop.core.logging.ILoggingObject; import org.apache.hop.core.row.IRowMeta; import org.apache.hop.core.row.IValueMeta; @@ -267,6 +268,20 @@ public boolean processRow() throws HopException { } } + /** + * Encodes a boolean for PostgreSQL COPY text format. {@code t}/{@code f} are accepted; literals + * like {@code 1.0} from a numeric conversion are not. + */ + @VisibleForTesting + static byte[] booleanFieldBytesForPgCopyText( + IValueMeta valueMeta, Object valueData, Charset charset) throws HopValueException { + Boolean bool = valueMeta.getBoolean(valueData); + if (bool == null) { + return null; + } + return (bool ? "t" : "f").getBytes(charset); + } + private void writeRowToPostgres(IRowMeta rowMeta, Object[] r) throws HopException { try { @@ -392,11 +407,10 @@ private void writeRowToPostgres(IRowMeta rowMeta, Object[] r) throws HopExceptio } break; case IValueMeta.TYPE_BOOLEAN: - if (valueMeta.isStorageBinaryString()) { - pgCopyOut.write((byte[]) valueData); - } else { - pgCopyOut.write( - Double.toString(valueMeta.getNumber(valueData)).getBytes(clientEncoding)); + byte[] boolBytes = + booleanFieldBytesForPgCopyText(valueMeta, valueData, clientEncoding); + if (boolBytes != null) { + pgCopyOut.write(boolBytes); } break; case IValueMeta.TYPE_NUMBER: diff --git a/plugins/transforms/pgbulkloader/src/test/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderTest.java b/plugins/transforms/pgbulkloader/src/test/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderTest.java index b3bd38e4be2..7e386f292ae 100644 --- a/plugins/transforms/pgbulkloader/src/test/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderTest.java +++ b/plugins/transforms/pgbulkloader/src/test/java/org/apache/hop/pipeline/transforms/pgbulkloader/PGBulkLoaderTest.java @@ -17,9 +17,11 @@ package org.apache.hop.pipeline.transforms.pgbulkloader; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; @@ -30,6 +32,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import org.apache.hop.core.HopClientEnvironment; import org.apache.hop.core.database.Database; @@ -37,9 +40,11 @@ import org.apache.hop.core.database.DatabaseMetaPlugin; import org.apache.hop.core.database.DatabasePluginType; import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.exception.HopValueException; import org.apache.hop.core.exception.HopXmlException; import org.apache.hop.core.logging.ILoggingObject; import org.apache.hop.core.plugins.PluginRegistry; +import org.apache.hop.core.row.value.ValueMetaBoolean; import org.apache.hop.databases.postgresql.PostgreSqlDatabaseMeta; import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension; import org.apache.hop.pipeline.transforms.mock.TransformMockHelper; @@ -101,6 +106,18 @@ void tearDown() throws Exception { transformMockHelper.cleanUp(); } + @Test + void booleanFieldBytesForPgCopyText_usesPostgresAcceptableLiterals() throws HopValueException { + ValueMetaBoolean meta = new ValueMetaBoolean("active"); + assertArrayEquals( + "t".getBytes(StandardCharsets.UTF_8), + PGBulkLoader.booleanFieldBytesForPgCopyText(meta, Boolean.TRUE, StandardCharsets.UTF_8)); + assertArrayEquals( + "f".getBytes(StandardCharsets.UTF_8), + PGBulkLoader.booleanFieldBytesForPgCopyText(meta, Boolean.FALSE, StandardCharsets.UTF_8)); + assertNull(PGBulkLoader.booleanFieldBytesForPgCopyText(meta, null, StandardCharsets.UTF_8)); + } + @Test void testCreateCommandLine() throws Exception { PGBulkLoaderMeta meta = mock(PGBulkLoaderMeta.class);