From 862292f7ccba3448831a17b363fe88ab5872aca0 Mon Sep 17 00:00:00 2001 From: Kathik Narayanan Date: Sun, 20 Nov 2016 20:18:45 -0500 Subject: [PATCH] NiFI-3604 onvert to Avro fails when a MySQL query returns two columns with the same name even with different aliases --- .../processors/standard/util/JdbcCommon.java | 7 ++++- .../processors/standard/TestExecuteSQL.java | 29 +++++++++++++++++++ .../standard/util/TestJdbcCommon.java | 2 +- 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JdbcCommon.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JdbcCommon.java index bb058d4e5800..22301b021ec4 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JdbcCommon.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JdbcCommon.java @@ -260,7 +260,12 @@ public static Schema createSchema(final ResultSet rs, String recordName, boolean * Some missing Avro types - Decimal, Date types. May need some additional work. */ for (int i = 1; i <= nrOfColumns; i++) { - String nameOrLabel = StringUtils.isNotEmpty(meta.getColumnName(i)) ? meta.getColumnName(i) : meta.getColumnLabel(i); + /** + * as per jdbc 4 specs, getColumnLabel will have the alias for the column, if not it will have the column name. + * so it may be a better option to check for columnlabel first and if in case it is null is someimplementation, + * check for alias. Postgres is the one that has the null column names for calculated fields. + */ + String nameOrLabel = StringUtils.isNotEmpty(meta.getColumnLabel(i)) ? meta.getColumnLabel(i) :meta.getColumnName(i); String columnName = convertNames ? normalizeNameForAvro(nameOrLabel) : nameOrLabel; switch (meta.getColumnType(i)) { case CHAR: diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestExecuteSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestExecuteSQL.java index ebca87e09d4f..5659e4a1b1b6 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestExecuteSQL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestExecuteSQL.java @@ -171,6 +171,35 @@ public void testWithNullIntColumn() throws SQLException { runner.getFlowFilesForRelationship(ExecuteSQL.REL_SUCCESS).get(0).assertAttributeEquals(ExecuteSQL.RESULT_ROW_COUNT, "2"); } + @Test + public void testWithduplicateColumns() throws SQLException { + // remove previous test database, if any + final File dbLocation = new File(DB_LOCATION); + dbLocation.delete(); + + // load test data to database + final Connection con = ((DBCPService) runner.getControllerService("dbcp")).getConnection(); + Statement stmt = con.createStatement(); + + try { + stmt.execute("drop table host1"); + stmt.execute("drop table host2"); + } catch (final SQLException sqle) { + } + + stmt.execute("create table host1 (id integer not null, host varchar(45))"); + stmt.execute("create table host2 (id integer not null, host varchar(45))"); + stmt.execute("insert into host1 values(1,'host1')"); + stmt.execute("insert into host2 values(1,'host2')"); + stmt.execute("select a.host as hostA,b.host as hostB from host1 a join host2 b on b.id=a.id"); + runner.setIncomingConnection(false); + runner.setProperty(ExecuteSQL.SQL_SELECT_QUERY, "select a.host as hostA,b.host as hostB from host1 a join host2 b on b.id=a.id"); + runner.run(); + + runner.assertAllFlowFilesTransferred(ExecuteSQL.REL_SUCCESS, 1); + runner.getFlowFilesForRelationship(ExecuteSQL.REL_SUCCESS).get(0).assertAttributeEquals(ExecuteSQL.RESULT_ROW_COUNT, "1"); + } + @Test public void testWithSqlException() throws SQLException { // remove previous test database, if any diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestJdbcCommon.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestJdbcCommon.java index e05844f452ea..18621d641726 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestJdbcCommon.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestJdbcCommon.java @@ -169,7 +169,7 @@ public void testCreateSchemaOnlyColumnLabel() throws ClassNotFoundException, SQL assertNotNull(schema); assertNotNull(schema.getField("ID")); - assertNotNull(schema.getField("VCHARC")); + assertNotNull(schema.getField("NOT_VCHARC")); // records name, should be result set first column table name assertEquals("TEST", schema.getName());