From 5f4fc1df685b8aa4b7bbc8ba09e09814cb91368b Mon Sep 17 00:00:00 2001 From: Peter Wicks Date: Mon, 22 Aug 2016 11:45:20 -0600 Subject: [PATCH 01/26] NIFI-2603 --- .../nifi-web-ui/src/main/webapp/css/flow-status.css | 12 ++++++++++++ .../nifi-web-ui/src/main/webapp/css/graph.css | 10 +++++++++- .../src/main/webapp/js/nf/canvas/nf-port.js | 10 ++++++++-- .../src/main/webapp/js/nf/canvas/nf-processor.js | 10 ++++++++-- 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/flow-status.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/flow-status.css index c13344db4af3..e67e666683c5 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/flow-status.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/flow-status.css @@ -38,6 +38,18 @@ color: #728E9B; /*base-color*/ } +#flow-status .fa.fa-play { + color: #5cb85c; +} + +#flow-status .fa.fa-stop { + color: #d9534f; +} + +#flow-status .fa.fa-warning { + color: #f0ad4e; +} + #flow-status .icon span { font-size: 15px; font-weight: 500; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/graph.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/graph.css index 641b02d9871e..617b7d6bd87a 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/graph.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/graph.css @@ -114,7 +114,15 @@ rect.bulletin-background { } text.process-group-invalid.has-validation-errors { - fill: #ba554a; + fill: #f0ad4e; +} + +text.process-group-stopped.process-group-contents-icon { + fill: #d9534f; +} + +text.process-group-running.process-group-contents-icon{ + fill: #5cb85c; } text.active-thread-count-icon { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-port.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-port.js index 75aed00dffc8..31b87f190c85 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-port.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-port.js @@ -330,9 +330,15 @@ nf.Port = (function () { .attr({ 'fill': function (d) { var fill = '#728e9b'; - if (d.status.aggregateSnapshot.runStatus === 'Invalid') { - fill = '#ba554a'; + + if (d.status.aggregateSnapshot.runStatus === 'Invalid') { + fill = '#f0ad4e'; + } else if (d.status.aggregateSnapshot.runStatus === 'Running') { + fill = '#5cb85c'; + } else if (d.status.aggregateSnapshot.runStatus === 'Stopped') { + fill = '#d9534f'; } + return fill; }, 'font-family': function (d) { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-processor.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-processor.js index 536f222949b5..29ade59e6243 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-processor.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-processor.js @@ -581,9 +581,15 @@ nf.Processor = (function () { .attr({ 'fill': function (d) { var fill = '#728e9b'; - if (d.status.aggregateSnapshot.runStatus === 'Invalid') { - fill = '#ba554a'; + + if (d.status.aggregateSnapshot.runStatus === 'Invalid') { + fill = '#f0ad4e'; + } else if (d.status.aggregateSnapshot.runStatus === 'Running') { + fill = '#5cb85c'; + } else if (d.status.aggregateSnapshot.runStatus === 'Stopped') { + fill = '#d9534f'; } + return fill; }, 'font-family': function (d) { From ce027bef7029e47f75febecc28dbb0188decea55 Mon Sep 17 00:00:00 2001 From: Paul Gibeault Date: Mon, 22 Aug 2016 17:19:29 -0600 Subject: [PATCH 02/26] Initial Check-in 3 Unit tests still failing --- .../processors/standard/ConvertJSONToSQL.java | 148 ++++++++- .../standard/TestConvertJSONToSQL.java | 287 ++++++++++++++++++ 2 files changed, 430 insertions(+), 5 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java index 194aa1953a45..b8be13eddd70 100755 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java @@ -68,8 +68,8 @@ @SupportsBatching @SeeAlso(PutSQL.class) @InputRequirement(Requirement.INPUT_REQUIRED) -@Tags({"json", "sql", "database", "rdbms", "insert", "update", "relational", "flat"}) -@CapabilityDescription("Converts a JSON-formatted FlowFile into an UPDATE or INSERT SQL statement. The incoming FlowFile is expected to be " +@Tags({"json", "sql", "database", "rdbms", "insert", "update", "merge", "relational", "flat"}) +@CapabilityDescription("Converts a JSON-formatted FlowFile into an UPDATE, INSERT or MERGE SQL statement. The incoming FlowFile is expected to be " + "\"flat\" JSON message, meaning that it consists of a single JSON element and each field maps to a simple type. If a field maps to " + "a JSON object, that JSON object will be interpreted as Text. If the input is an array of JSON elements, each element in the array is " + "output as a separate FlowFile to the 'sql' relationship. Upon successful conversion, the original FlowFile is routed to the 'original' " @@ -96,6 +96,7 @@ public class ConvertJSONToSQL extends AbstractProcessor { private static final String UPDATE_TYPE = "UPDATE"; private static final String INSERT_TYPE = "INSERT"; + private static final String MERGE_TYPE = "MERGE"; static final AllowableValue IGNORE_UNMATCHED_FIELD = new AllowableValue("Ignore Unmatched Fields", "Ignore Unmatched Fields", "Any field in the JSON document that cannot be mapped to a column in the database is ignored"); @@ -122,7 +123,7 @@ public class ConvertJSONToSQL extends AbstractProcessor { .name("Statement Type") .description("Specifies the type of SQL Statement to generate") .required(true) - .allowableValues(UPDATE_TYPE, INSERT_TYPE) + .allowableValues(UPDATE_TYPE, INSERT_TYPE, MERGE_TYPE) .build(); static final PropertyDescriptor TABLE_NAME = new PropertyDescriptor.Builder() .name("Table Name") @@ -167,7 +168,7 @@ public class ConvertJSONToSQL extends AbstractProcessor { static final PropertyDescriptor UPDATE_KEY = new PropertyDescriptor.Builder() .name("Update Keys") .description("A comma-separated list of column names that uniquely identifies a row in the database for UPDATE statements. " - + "If the Statement Type is UPDATE and this property is not set, the table's Primary Keys are used. " + + "If the Statement Type is UPDATE or MERGE and this property is not set, the table's Primary Keys are used. " + "In this case, if no Primary Key exists, the conversion to SQL will fail if Unmatched Column Behaviour is set to FAIL. " + "This property is ignored if the Statement Type is INSERT") .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) @@ -248,7 +249,7 @@ public void onTrigger(final ProcessContext context, final ProcessSession session final String schemaName = context.getProperty(SCHEMA_NAME).evaluateAttributeExpressions(flowFile).getValue(); final String tableName = context.getProperty(TABLE_NAME).evaluateAttributeExpressions(flowFile).getValue(); final SchemaKey schemaKey = new SchemaKey(catalog, tableName); - final boolean includePrimaryKeys = UPDATE_TYPE.equals(statementType) && updateKeys == null; + final boolean includePrimaryKeys = (UPDATE_TYPE.equals(statementType) || MERGE_TYPE.equals(statementType)) && updateKeys == null; // Is the unmatched column behaviour fail or warning? final boolean failUnmappedColumns = FAIL_UNMATCHED_COLUMN.getValue().equalsIgnoreCase(context.getProperty(UNMATCHED_COLUMN_BEHAVIOR).getValue()); @@ -331,6 +332,8 @@ public void process(final InputStream in) throws IOException { if (INSERT_TYPE.equals(statementType)) { sql = generateInsert(jsonNode, attributes, fqTableName, schema, translateFieldNames, ignoreUnmappedFields, failUnmappedColumns, warningUnmappedColumns); + } else if (MERGE_TYPE.equals(statementType)) { + sql = generateMerge(jsonNode, attributes, fqTableName, updateKeys, schema, translateFieldNames, ignoreUnmappedFields, failUnmappedColumns, warningUnmappedColumns); } else { sql = generateUpdate(jsonNode, attributes, fqTableName, updateKeys, schema, translateFieldNames, ignoreUnmappedFields, failUnmappedColumns, warningUnmappedColumns); } @@ -578,6 +581,141 @@ private String generateUpdate(final JsonNode rootNode, final Map return sqlBuilder.toString(); } + private String generateMerge(final JsonNode rootNode, final Map attributes, final String tableName, final String updateKeys, + final TableSchema schema, final boolean translateFieldNames, final boolean ignoreUnmappedFields, final boolean failUnmappedColumns, + final boolean warningUnmappedColumns) { + + /////////////////////////////////////////// + // This is the SQL Format we are building + /////////////////////////////////////////// + // MERGE target_t + // USING ( + // VALUES () + // ) AS source_t () + // ON + // WHEN MATCHED THEN + // UPDATE SET + // WHEN NOT MATCHED THEN + // INSERT () + // VALUES () + // ; --A MERGE statement must be terminated by a semi-colon (;). + // + // We first need to collect: ValueList, ColumnList, MatchClause, SetList and NamedValueList + final String source_table = "source_t"; + final String target_table = "target_t"; + + final Set updateKeyNames; + if (updateKeys == null) { + updateKeyNames = schema.getPrimaryKeyColumnNames(); + } else { + updateKeyNames = new HashSet<>(); + for (final String updateKey : updateKeys.split(",")) { + updateKeyNames.add(updateKey.trim()); + } + } + + if (updateKeyNames.isEmpty()) { + throw new ProcessException("Table '" + tableName + "' does not have a Primary Key and no Update Keys were specified"); + } + + // Create a Set of all normalized Update Key names, and ensure that there is a field in the JSON + // for each of the Update Key fields. + final Set normalizedFieldNames = getNormalizedColumnNames(rootNode, translateFieldNames); + final Set normalizedUpdateNames = new HashSet<>(); + for (final String uk : updateKeyNames) { + final String normalizedUK = normalizeColumnName(uk, translateFieldNames); + normalizedUpdateNames.add(normalizedUK); + + if (!normalizedFieldNames.contains(normalizedUK)) { + String missingColMessage = "JSON does not have a value for the " + (updateKeys == null ? "Primary" : "Update") + "Key column '" + uk + "'"; + if (failUnmappedColumns) { + getLogger().error(missingColMessage); + throw new ProcessException(missingColMessage); + } else if (warningUnmappedColumns) { + getLogger().warn(missingColMessage); + } + } + } + + // iterate over all of the elements in the JSON, building the SQL statement by adding the column names, as well as + // adding the column value to a "sql.args.N.value" attribute and the type of a "sql.args.N.type" attribute add the + // columns that we are inserting into + final StringBuilder valueListBuilder = new StringBuilder(); + final StringBuilder namedValueListBuilder = new StringBuilder(); + final StringBuilder columnListBuilder = new StringBuilder(); + final StringBuilder setListBuilder = new StringBuilder(); + final StringBuilder matchClauseBuilder = new StringBuilder(); + int fieldCount = 0; + + Iterator fieldNames = rootNode.getFieldNames(); + while (fieldNames.hasNext()) { + final String fieldName = fieldNames.next(); + + final String normalizedColName = normalizeColumnName(fieldName, translateFieldNames); + final ColumnDescription desc = schema.getColumns().get(normalizedColName); + if (desc == null) { + if (!ignoreUnmappedFields) { + throw new ProcessException("Cannot map JSON field '" + fieldName + "' to any column in the database"); + } else { + continue; + } + } + + // Build Named Value List, Column List, and Value list + valueListBuilder.append("?,"); + columnListBuilder.append(normalizedColName).append(","); + namedValueListBuilder.append(String.format("%s.%s,",source_table,normalizedColName)); + + // Check if this column is an Update Key. If so, + // Use it to build the Match Clause + if (normalizedUpdateNames.contains(normalizedColName)) { + matchClauseBuilder.append(String.format("%s.%s = %s.%s,", target_table,normalizedColName,source_table,normalizedColName)); + } + else + { + setListBuilder.append(String.format("%s = %s.%s,",normalizedColName, source_table,normalizedColName)); + } + + // Create parameter attributes + fieldCount++; + final int sqlType = desc.getDataType(); + attributes.put("sql.args." + fieldCount + ".type", String.valueOf(sqlType)); + + final Integer colSize = desc.getColumnSize(); + + final JsonNode fieldNode = rootNode.get(fieldName); + if (!fieldNode.isNull()) { + String fieldValue = rootNode.get(fieldName).asText(); + if (colSize != null && fieldValue.length() > colSize) { + fieldValue = fieldValue.substring(0, colSize); + } + attributes.put("sql.args." + fieldCount + ".value", fieldValue); + } + } + + // Trim trailing , characters + final String valueList = valueListBuilder.toString().substring(0, valueListBuilder.length() -1); + final String columnList = columnListBuilder.toString().substring(0, columnListBuilder.length() -1); + final String matchClause = matchClauseBuilder.toString().substring(0, matchClauseBuilder.length() -1); + final String setList = setListBuilder.toString().substring(0, setListBuilder.length() -1); + final String namedValueList = namedValueListBuilder.toString().substring(0, namedValueListBuilder.length() -1); + + // Build the SQL statement from the pieces we gathered + return String.format("MERGE %s target_t " + + "USING VALUES (%s) " + + "AS source_t (%s) " + + "ON %s " + + "WHEN MATCHED THEN " + + "UPDATE SET %s " + + "WHEN NOT MATCHED THEN " + + "INSERT (%s) " + + "VALUES (%s) " + + ";", // --MERGE requires a trailing semi-colon + tableName, valueList, columnList, matchClause, + setList, columnList, namedValueList); + } + + private static String normalizeColumnName(final String colName, final boolean translateColumnNames) { return translateColumnNames ? colName.toUpperCase().replace("_", "") : colName; } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java index 60d993ef0538..8b6a010aded4 100755 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java @@ -596,6 +596,293 @@ public void testUpdateWithMissingColumnIgnore() throws InitializationException, } // End testUpdateWithMissingColumnIgnore() + @Test + public void testMergeWithNullValue() throws InitializationException, ProcessException, SQLException, IOException { + final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); + final File tempDir = folder.getRoot(); + final File dbDir = new File(tempDir, "db"); + final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate(createPersons); + } + } + + runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); + runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); + runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "MERGE"); + runner.enqueue(Paths.get("src/test/resources/TestConvertJSONToSQL/person-with-null-code.json")); + runner.run(); + + runner.assertTransferCount(ConvertJSONToSQL.REL_ORIGINAL, 1); + runner.assertTransferCount(ConvertJSONToSQL.REL_SQL, 1); + final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL).get(0); + out.assertAttributeEquals("sql.args.1.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.1.value", "1"); + out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.VARCHAR)); + out.assertAttributeEquals("sql.args.2.value", "Mark"); + out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeNotExists("sql.args.3.value"); + + out.assertContentEquals("UPDATE PERSONS SET NAME = ?, CODE = ? WHERE ID = ?"); + } + + @Test + public void testMergeBasedOnPrimaryKey() throws InitializationException, ProcessException, SQLException, IOException { + final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); + final File tempDir = folder.getRoot(); + final File dbDir = new File(tempDir, "db"); + final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate(createPersons); + } + } + + runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); + runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); + runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "MERGE"); + runner.enqueue(Paths.get("src/test/resources/TestConvertJSONToSQL/person-1.json")); + runner.run(); + + runner.assertTransferCount(ConvertJSONToSQL.REL_ORIGINAL, 1); + runner.assertTransferCount(ConvertJSONToSQL.REL_SQL, 1); + final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL).get(0); + + out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.ID = source_t.ID WHEN MATCHED THEN UPDATE SET NAME = source_t.NAME,CODE = source_t.CODE WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + out.assertAttributeEquals("sql.args.1.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.1.value", "1"); + out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.VARCHAR)); + out.assertAttributeEquals("sql.args.2.value", "Mark"); + out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.3.value", "48"); + } + + @Test + public void testMergeBasedOnUpdateKey() throws InitializationException, ProcessException, SQLException, IOException { + final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); + final File tempDir = folder.getRoot(); + final File dbDir = new File(tempDir, "db"); + final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate(createPersons); + } + } + + runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); + runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); + runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "MERGE"); + runner.setProperty(ConvertJSONToSQL.UPDATE_KEY, "code"); + runner.enqueue(Paths.get("src/test/resources/TestConvertJSONToSQL/person-1.json")); + runner.run(); + + runner.assertTransferCount(ConvertJSONToSQL.REL_ORIGINAL, 1); + runner.assertTransferCount(ConvertJSONToSQL.REL_SQL, 1); + final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL).get(0); + out.assertAttributeEquals("sql.args.1.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.1.value", "48"); + out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.2.value", "1"); + out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.VARCHAR)); + out.assertAttributeEquals("sql.args.3.value", "Mark"); + + out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.ID = source_t.ID WHEN MATCHED THEN UPDATE SET NAME = source_t.NAME,CODE = source_t.CODE WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + } + + @Test + public void testMergeBasedOnCompoundUpdateKey() throws InitializationException, ProcessException, SQLException, IOException { + final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); + final File tempDir = folder.getRoot(); + final File dbDir = new File(tempDir, "db"); + final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate(createPersons); + } + } + + runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); + runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); + runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "MERGE"); + runner.setProperty(ConvertJSONToSQL.UPDATE_KEY, "name, code"); + runner.enqueue(Paths.get("src/test/resources/TestConvertJSONToSQL/person-1.json")); + runner.run(); + + runner.assertTransferCount(ConvertJSONToSQL.REL_ORIGINAL, 1); + runner.assertTransferCount(ConvertJSONToSQL.REL_SQL, 1); + final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL).get(0); + out.assertAttributeEquals("sql.args.1.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.1.value", "1"); + out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.2.value", "48"); + out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.VARCHAR)); + out.assertAttributeEquals("sql.args.3.value", "Mark"); + + out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.ID = source_t.ID WHEN MATCHED THEN UPDATE SET NAME = source_t.NAME,CODE = source_t.CODE WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + } + + @Test + public void testMergeWithMissingFieldBasedOnCompoundUpdateKey() throws InitializationException, ProcessException, SQLException, IOException { + final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); + final File tempDir = folder.getRoot(); + final File dbDir = new File(tempDir, "db"); + final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate(createPersons); + } + } + + runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); + runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); + runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "MERGE"); + runner.setProperty(ConvertJSONToSQL.UPDATE_KEY, "name, code"); + runner.enqueue(Paths.get("src/test/resources/TestConvertJSONToSQL/person-without-code.json")); + runner.run(); + + runner.assertAllFlowFilesTransferred(ConvertJSONToSQL.REL_FAILURE, 1); + } + + @Test + public void testMergeWithMalformedJson() throws InitializationException, ProcessException, SQLException, IOException { + final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); + final File tempDir = folder.getRoot(); + final File dbDir = new File(tempDir, "db"); + final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate(createPersons); + } + } + + runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); + runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); + runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "MERGE"); + runner.setProperty(ConvertJSONToSQL.UPDATE_KEY, "name, code"); + runner.enqueue(Paths.get("src/test/resources/TestConvertJSONToSQL/malformed-person-extra-comma.json")); + runner.run(); + + runner.assertAllFlowFilesTransferred(ConvertJSONToSQL.REL_FAILURE, 1); + } + @Test + public void testMergeWithMissingColumnFail() throws InitializationException, ProcessException, SQLException, IOException { + final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); + final File tempDir = folder.getRoot(); + final File dbDir = new File(tempDir, "db"); + final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate(createPersons); + } + } + + runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); + runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); + runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "MERGE"); + runner.setProperty(ConvertJSONToSQL.UPDATE_KEY, "name, code, extra"); + runner.setProperty(ConvertJSONToSQL.UNMATCHED_COLUMN_BEHAVIOR, ConvertJSONToSQL.FAIL_UNMATCHED_COLUMN); + runner.enqueue(Paths.get("src/test/resources/TestConvertJSONToSQL/person-1.json")); + runner.run(); + + runner.assertAllFlowFilesTransferred(ConvertJSONToSQL.REL_FAILURE, 1); + } // End testMergeWithMissingColumnFail() + + @Test + public void testMergeWithMissingColumnWarning() throws InitializationException, ProcessException, SQLException, IOException { + final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); + final File tempDir = folder.getRoot(); + final File dbDir = new File(tempDir, "db"); + final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate(createPersons); + } + } + + runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); + runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); + runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "MERGE"); + runner.setProperty(ConvertJSONToSQL.UPDATE_KEY, "name, code, extra"); + runner.setProperty(ConvertJSONToSQL.UNMATCHED_COLUMN_BEHAVIOR, ConvertJSONToSQL.WARNING_UNMATCHED_COLUMN); + runner.enqueue(Paths.get("src/test/resources/TestConvertJSONToSQL/person-1.json")); + runner.run(); + + runner.assertTransferCount(ConvertJSONToSQL.REL_ORIGINAL, 1); + runner.assertTransferCount(ConvertJSONToSQL.REL_SQL, 1); + final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL).get(0); + out.assertAttributeEquals("sql.args.1.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.1.value", "48"); + out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.2.value", "1"); + out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.VARCHAR)); + out.assertAttributeEquals("sql.args.3.value", "Mark"); + + out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.ID = source_t.ID WHEN MATCHED THEN UPDATE SET NAME = source_t.NAME,CODE = source_t.CODE WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + + } // End testMergeWithMissingColumnWarning() + + @Test + public void testMergeWithMissingColumnIgnore() throws InitializationException, ProcessException, SQLException, IOException { + final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); + final File tempDir = folder.getRoot(); + final File dbDir = new File(tempDir, "db"); + final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate(createPersons); + } + } + + runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); + runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); + runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "UPDATE"); + runner.setProperty(ConvertJSONToSQL.UPDATE_KEY, "name, code, extra"); + runner.setProperty(ConvertJSONToSQL.UNMATCHED_COLUMN_BEHAVIOR, "Ignore Unmatched Columns"); + runner.enqueue(Paths.get("src/test/resources/TestConvertJSONToSQL/person-1.json")); + runner.run(); + + runner.assertTransferCount(ConvertJSONToSQL.REL_ORIGINAL, 1); + runner.assertTransferCount(ConvertJSONToSQL.REL_SQL, 1); + final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL).get(0); + out.assertAttributeEquals("sql.args.1.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.1.value", "1"); + out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.VARCHAR)); + out.assertAttributeEquals("sql.args.2.value", "Mark"); + out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.3.value", "48"); + + out.assertContentEquals("UPDATE PERSONS SET ID = ? WHERE NAME = ? AND CODE = ?"); + + } // End testMergeWithMissingColumnIgnore() + /** * Simple implementation only for testing purposes */ From 0632774e403194a29ca7a49d7c63bc8b5d9873d5 Mon Sep 17 00:00:00 2001 From: Peter Wicks Date: Tue, 23 Aug 2016 11:47:46 -0600 Subject: [PATCH 03/26] Gui Icons --- .../nifi-web-ui/src/main/webapp/css/common-ui.css | 8 ++++++++ .../nifi-web-ui/src/main/webapp/css/navigation.css | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css index 9eac22615e6b..522f3fd87bf0 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css @@ -520,6 +520,14 @@ div.context-menu-item.hover { left: 2px; } +div.context-menu-item.hover > div.context-menu-item-img.fa.fa-play { + color:#5cb85c; +} + +div.context-menu-item.hover > div.context-menu-item-img.fa.fa-stop { + color:#5cb85c; +} + .context-menu-item-img.icon { position: relative; top: 1px; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/navigation.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/navigation.css index 2439d229e1c5..6ad2a9babd79 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/navigation.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/navigation.css @@ -213,6 +213,14 @@ div.action-button { text-transform:uppercase; } +#operate-start button:hover { + color:#5cb85c; +} + +#operate-stop button:hover { + color:#5cb85c; +} + div.graph-control div.icon-disabled { color: #ddd; } From e89fed9292d88cab4d0cabe4de1153fba27aecf3 Mon Sep 17 00:00:00 2001 From: Paul Gibeault Date: Wed, 24 Aug 2016 09:04:06 -0600 Subject: [PATCH 04/26] All 10 Unit Tests pass now --- .../standard/TestConvertJSONToSQL.java | 71 ++++++++++++++----- 1 file changed, 53 insertions(+), 18 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java index 8b6a010aded4..93b5e1ad44a3 100755 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java @@ -627,7 +627,7 @@ public void testMergeWithNullValue() throws InitializationException, ProcessExce out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.INTEGER)); out.assertAttributeNotExists("sql.args.3.value"); - out.assertContentEquals("UPDATE PERSONS SET NAME = ?, CODE = ? WHERE ID = ?"); + out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.ID = source_t.ID WHEN MATCHED THEN UPDATE SET NAME = source_t.NAME,CODE = source_t.CODE WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); } @Test @@ -690,13 +690,13 @@ public void testMergeBasedOnUpdateKey() throws InitializationException, ProcessE runner.assertTransferCount(ConvertJSONToSQL.REL_SQL, 1); final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL).get(0); out.assertAttributeEquals("sql.args.1.type", String.valueOf(java.sql.Types.INTEGER)); - out.assertAttributeEquals("sql.args.1.value", "48"); - out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.INTEGER)); - out.assertAttributeEquals("sql.args.2.value", "1"); - out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.VARCHAR)); - out.assertAttributeEquals("sql.args.3.value", "Mark"); + out.assertAttributeEquals("sql.args.1.value", "1"); + out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.VARCHAR)); + out.assertAttributeEquals("sql.args.2.value", "Mark"); + out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.3.value", "48"); - out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.ID = source_t.ID WHEN MATCHED THEN UPDATE SET NAME = source_t.NAME,CODE = source_t.CODE WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.CODE = source_t.CODE WHEN MATCHED THEN UPDATE SET ID = source_t.ID,NAME = source_t.NAME WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); } @Test @@ -726,12 +726,12 @@ public void testMergeBasedOnCompoundUpdateKey() throws InitializationException, final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL).get(0); out.assertAttributeEquals("sql.args.1.type", String.valueOf(java.sql.Types.INTEGER)); out.assertAttributeEquals("sql.args.1.value", "1"); - out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.INTEGER)); - out.assertAttributeEquals("sql.args.2.value", "48"); - out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.VARCHAR)); - out.assertAttributeEquals("sql.args.3.value", "Mark"); + out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.VARCHAR)); + out.assertAttributeEquals("sql.args.2.value", "Mark"); + out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.3.value", "48"); - out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.ID = source_t.ID WHEN MATCHED THEN UPDATE SET NAME = source_t.NAME,CODE = source_t.CODE WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.NAME = source_t.NAME,target_t.CODE = source_t.CODE WHEN MATCHED THEN UPDATE SET ID = source_t.ID WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); } @Test @@ -836,13 +836,13 @@ public void testMergeWithMissingColumnWarning() throws InitializationException, runner.assertTransferCount(ConvertJSONToSQL.REL_SQL, 1); final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL).get(0); out.assertAttributeEquals("sql.args.1.type", String.valueOf(java.sql.Types.INTEGER)); - out.assertAttributeEquals("sql.args.1.value", "48"); - out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.INTEGER)); - out.assertAttributeEquals("sql.args.2.value", "1"); - out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.VARCHAR)); - out.assertAttributeEquals("sql.args.3.value", "Mark"); + out.assertAttributeEquals("sql.args.1.value", "1"); + out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.VARCHAR)); + out.assertAttributeEquals("sql.args.2.value", "Mark"); + out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.3.value", "48"); - out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.ID = source_t.ID WHEN MATCHED THEN UPDATE SET NAME = source_t.NAME,CODE = source_t.CODE WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.NAME = source_t.NAME,target_t.CODE = source_t.CODE WHEN MATCHED THEN UPDATE SET ID = source_t.ID WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); } // End testMergeWithMissingColumnWarning() @@ -883,6 +883,41 @@ public void testMergeWithMissingColumnIgnore() throws InitializationException, P } // End testMergeWithMissingColumnIgnore() + @Test + public void testMultipleMerges() throws InitializationException, ProcessException, SQLException, IOException { + final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); + final File tempDir = folder.getRoot(); + final File dbDir = new File(tempDir, "db"); + final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate(createPersons); + } + } + + runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); + runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); + runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "MERGE"); + runner.enqueue(Paths.get("src/test/resources/TestConvertJSONToSQL/persons.json")); + runner.run(); + + runner.assertTransferCount(ConvertJSONToSQL.REL_ORIGINAL, 1); + runner.assertTransferCount(ConvertJSONToSQL.REL_SQL, 5); + final List mffs = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL); + for (final MockFlowFile mff : mffs) { + mff.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.ID = source_t.ID WHEN MATCHED THEN UPDATE SET NAME = source_t.NAME,CODE = source_t.CODE WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + + for (int i=1; i <= 3; i++) { + mff.assertAttributeExists("sql.args." + i + ".type"); + mff.assertAttributeExists("sql.args." + i + ".value"); + } + } + } + + /** * Simple implementation only for testing purposes */ From dbde098af7fa8f76a7d35dcb22aa028f497ff7be Mon Sep 17 00:00:00 2001 From: Peter Wicks Date: Wed, 24 Aug 2016 10:01:37 -0600 Subject: [PATCH 05/26] Gui Icons - Fixing Orange Color --- .../src/main/webapp/css/common-ui.css | 2 +- .../nifi-web-ui/src/main/webapp/css/graph.css | 12 ++++---- .../nifi-web-ui/src/main/webapp/css/main.css | 8 +++--- .../src/main/webapp/css/navigation.css | 2 +- .../webapp/css/processor-configuration.css | 2 +- .../src/main/webapp/css/provenance.css | 4 +-- .../src/main/webapp/css/queue-listing.css | 2 +- .../remote-process-group-configuration.css | 2 +- .../slickgrid/css/slick-default-theme.css | 2 +- .../nf-ng-canvas-flow-status-controller.js | 6 ++-- .../src/main/webapp/js/nf/canvas/nf-canvas.js | 2 +- .../src/main/webapp/js/nf/nf-ng-app-config.js | 28 +++++++++---------- .../js/nf/provenance/nf-provenance-lineage.js | 2 +- .../src/main/webapp/css/main.css | 2 +- 14 files changed, 38 insertions(+), 38 deletions(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css index 522f3fd87bf0..7ab85d850697 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css @@ -525,7 +525,7 @@ div.context-menu-item.hover > div.context-menu-item-img.fa.fa-play { } div.context-menu-item.hover > div.context-menu-item-img.fa.fa-stop { - color:#5cb85c; + color:#d9534f; } .context-menu-item-img.icon { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/graph.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/graph.css index 617b7d6bd87a..9bbf05f79845 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/graph.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/graph.css @@ -73,7 +73,7 @@ g.component rect.border { g.component rect.border.unauthorized { stroke-width: 1.5; - stroke: #ba554a; + stroke: #f0ad4e; stroke-dasharray: 3,3; } @@ -110,7 +110,7 @@ text.bulletin-icon { } rect.bulletin-background { - fill: #ba554a; + fill: #f0ad4e; } text.process-group-invalid.has-validation-errors { @@ -201,7 +201,7 @@ g.connection rect.body.unauthorized { g.connection rect.border.unauthorized { stroke-width: 1.5; - stroke: #ba554a; + stroke: #f0ad4e; stroke-dasharray: 3,3; } @@ -234,7 +234,7 @@ g.connection path.connection-path { } g.connection path.connection-path.unauthorized { - stroke: #ba554a; + stroke: #f0ad4e; stroke-dasharray: 3,3; } @@ -245,7 +245,7 @@ text.connection-from-run-status, text.connection-to-run-status, text.expiration- } text.connection-from-run-status.is-missing-port, text.connection-to-run-status.is-missing-port { - fill: #ba554a; + fill: #f0ad4e; } /* grouped connection */ @@ -392,7 +392,7 @@ text.remote-process-group-transmission-status { } text.remote-process-group-transmission-status.has-authorization-errors { - fill: #ba554a; + fill: #f0ad4e; } text.remote-process-group-transmission-secure { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/main.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/main.css index b78a14df79f2..9ef82beabb46 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/main.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/main.css @@ -104,13 +104,13 @@ div.running { div.has-errors, div.invalid { float: left; - color: #ba554a !important; + color: #f0ad4e !important; } div.has-errors:before, div.invalid:before { font-family: FontAwesome; content: "\f071"; - color: #ba554a; + color: #f0ad4e; } div.transmitting { @@ -128,7 +128,7 @@ div.valid { } div.has-bulletins { - color: #ba554a !important; + color: #f0ad4e !important; } /* @@ -233,7 +233,7 @@ span.details-title { font-weight: bold; font-family: Roboto; font-size: 13px; - color: #ba554a; + color: #f0ad4e; } #upload-template-container button { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/navigation.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/navigation.css index 6ad2a9babd79..e986a9bea8df 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/navigation.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/navigation.css @@ -218,7 +218,7 @@ div.action-button { } #operate-stop button:hover { - color:#5cb85c; + color:#d9534f; } div.graph-control div.icon-disabled { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/processor-configuration.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/processor-configuration.css index e03a6982618f..b8b6bcffb54d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/processor-configuration.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/processor-configuration.css @@ -75,7 +75,7 @@ div.processor-configuration-warning-icon:before { font-family: FontAwesome; content: "\f071"; font-size: 16px; - color: #ba554a; + color: #f0ad4e; } #auto-terminate-relationship-names { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/provenance.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/provenance.css index f65011d51f03..8e39997d0196 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/provenance.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/provenance.css @@ -463,11 +463,11 @@ div.lineage-collapse-children:before { } path.link.selected { - stroke: #ba554a; + stroke: #f0ad4e; } g.event circle.selected { - fill: #ba554a; + fill: #f0ad4e; } text.event-type { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/queue-listing.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/queue-listing.css index 92d9997a594c..0c45c42d45cc 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/queue-listing.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/queue-listing.css @@ -75,7 +75,7 @@ #queue-listing-message { position: absolute; top: 36px; - color: #ba554a; + color: #f0ad4e; font-family: Roboto; font-size: 13px; font-weight: 500; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/remote-process-group-configuration.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/remote-process-group-configuration.css index 847bdb760598..44d24a3337fe 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/remote-process-group-configuration.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/remote-process-group-configuration.css @@ -100,7 +100,7 @@ div.remote-port-removed:before { font-family: FontAwesome; content: "\f071"; font-size: 16px; - color: #ba554a; + color: #f0ad4e; } div.remote-port-edit-container { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/slickgrid/css/slick-default-theme.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/slickgrid/css/slick-default-theme.css index 780d0d30caa8..1b8503d4afb0 100755 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/slickgrid/css/slick-default-theme.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/slickgrid/css/slick-default-theme.css @@ -77,7 +77,7 @@ classes should alter those! } .slick-cell.invalid { - border-color: #ba554a; + border-color: #f0ad4e; -moz-animation-duration: 0.2s; -webkit-animation-duration: 0.2s; -moz-animation-name: slickgrid-invalid-hilite; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/controllers/nf-ng-canvas-flow-status-controller.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/controllers/nf-ng-canvas-flow-status-controller.js index 9e340270a2b4..5b8b4fe4c805 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/controllers/nf-ng-canvas-flow-status-controller.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/controllers/nf-ng-canvas-flow-status-controller.js @@ -360,14 +360,14 @@ nf.ng.Canvas.FlowStatusCtrl = function (serviceProvider, $sanitize) { var connectedNodes = clusterSummary.connectedNodes.split(' / '); if (connectedNodes.length === 2 && connectedNodes[0] !== connectedNodes[1]) { this.clusterConnectionWarning = true; - color = '#BA554A'; + color = '#f0ad4e'; } } this.connectedNodesCount = nf.Common.isDefinedAndNotNull(clusterSummary.connectedNodes) ? $sanitize(clusterSummary.connectedNodes) : '-'; } else { this.connectedNodesCount = 'Disconnected'; - color = '#BA554A'; + color = '#f0ad4e'; } // update the color @@ -382,7 +382,7 @@ nf.ng.Canvas.FlowStatusCtrl = function (serviceProvider, $sanitize) { update: function (status) { var controllerInvalidCountColor = (nf.Common.isDefinedAndNotNull(status.invalidCount) && (status.invalidCount > 0)) ? - '#BA554A' : '#728E9B'; + '#f0ad4e' : '#728E9B'; $('#controller-invalid-count').parent().css('color', controllerInvalidCountColor); // update the report values diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas.js index 790d10d91d5b..617da92b594c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas.js @@ -213,7 +213,7 @@ nf.Canvas = (function () { if (d === 'ghost') { return '#aaaaaa'; } else if (d === 'unauthorized') { - return '#ba554a'; + return '#f0ad4e'; } else { return '#000000'; } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-ng-app-config.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-ng-app-config.js index 54dccde026e4..a0bcfd8e5d2e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-ng-app-config.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-ng-app-config.js @@ -61,20 +61,20 @@ nf.ng.AppConfig = function ($mdThemingProvider, $compileProvider) { 'contrastLightColors': undefined }); $mdThemingProvider.definePalette('warnPalette', { - '50': 'BA554A', - '100': 'BA554A', - '200': 'BA554A', - '300': 'BA554A', - '400': 'BA554A', - '500': 'BA554A', /* warn-color */ - '600': 'BA554A', - '700': 'BA554A', - '800': 'BA554A', - '900': 'BA554A', - 'A100': 'BA554A', - 'A200': 'BA554A', - 'A400': 'BA554A', - 'A700': 'BA554A', + '50': 'f0ad4e', + '100': 'f0ad4e', + '200': 'f0ad4e', + '300': 'f0ad4e', + '400': 'f0ad4e', + '500': 'f0ad4e', /* warn-color */ + '600': 'f0ad4e', + '700': 'f0ad4e', + '800': 'f0ad4e', + '900': 'f0ad4e', + 'A100': 'f0ad4e', + 'A200': 'f0ad4e', + 'A400': 'f0ad4e', + 'A700': 'f0ad4e', 'contrastDefaultColor': 'light', 'contrastDarkColors': ['A100'], 'contrastLightColors': undefined diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-lineage.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-lineage.js index 6b038c7342d8..f687efeb6210 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-lineage.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-lineage.js @@ -586,7 +586,7 @@ nf.ng.ProvenanceLineage = function () { 'orient': 'auto', 'fill': function (d) { if (d.indexOf('SELECTED') >= 0) { - return '#ba554a'; + return '#f0ad4e'; } else { return '#000000'; } diff --git a/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-ui/src/main/webapp/css/main.css b/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-ui/src/main/webapp/css/main.css index c56eafa09cac..02d13d1f7e42 100644 --- a/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-ui/src/main/webapp/css/main.css +++ b/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-ui/src/main/webapp/css/main.css @@ -247,7 +247,7 @@ div.large-label-container { } #message { - color: #ba554a; + color: #f0ad4e; margin-left: 20px; float: right; margin-right: 20px; From 87191e214869f4d7e86d6de9a0ab9af896df7954 Mon Sep 17 00:00:00 2001 From: Paul Gibeault Date: Wed, 24 Aug 2016 11:11:23 -0600 Subject: [PATCH 06/26] Corrected MatchClause of Merge statement Included line-returns in MERGE SQL for readability --- .../processors/standard/ConvertJSONToSQL.java | 22 +++++++++---------- .../standard/TestConvertJSONToSQL.java | 12 +++++----- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java index b8be13eddd70..89842d4329df 100755 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java @@ -669,7 +669,7 @@ private String generateMerge(final JsonNode rootNode, final Map // Check if this column is an Update Key. If so, // Use it to build the Match Clause if (normalizedUpdateNames.contains(normalizedColName)) { - matchClauseBuilder.append(String.format("%s.%s = %s.%s,", target_table,normalizedColName,source_table,normalizedColName)); + matchClauseBuilder.append(String.format("%s.%s = %s.%s and ", target_table,normalizedColName,source_table,normalizedColName)); } else { @@ -696,20 +696,20 @@ private String generateMerge(final JsonNode rootNode, final Map // Trim trailing , characters final String valueList = valueListBuilder.toString().substring(0, valueListBuilder.length() -1); final String columnList = columnListBuilder.toString().substring(0, columnListBuilder.length() -1); - final String matchClause = matchClauseBuilder.toString().substring(0, matchClauseBuilder.length() -1); + final String matchClause = matchClauseBuilder.toString().substring(0, matchClauseBuilder.length() -5); final String setList = setListBuilder.toString().substring(0, setListBuilder.length() -1); final String namedValueList = namedValueListBuilder.toString().substring(0, namedValueListBuilder.length() -1); // Build the SQL statement from the pieces we gathered - return String.format("MERGE %s target_t " + - "USING VALUES (%s) " + - "AS source_t (%s) " + - "ON %s " + - "WHEN MATCHED THEN " + - "UPDATE SET %s " + - "WHEN NOT MATCHED THEN " + - "INSERT (%s) " + - "VALUES (%s) " + + return String.format("MERGE %s target_t \n" + + "USING VALUES (%s) \n" + + "AS source_t (%s) \n" + + "ON %s \n" + + "WHEN MATCHED THEN \n" + + "UPDATE SET %s \n" + + "WHEN NOT MATCHED THEN \n" + + "INSERT (%s) \n" + + "VALUES (%s) \n" + ";", // --MERGE requires a trailing semi-colon tableName, valueList, columnList, matchClause, setList, columnList, namedValueList); diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java index 93b5e1ad44a3..97a4c4d0c266 100755 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java @@ -627,7 +627,7 @@ public void testMergeWithNullValue() throws InitializationException, ProcessExce out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.INTEGER)); out.assertAttributeNotExists("sql.args.3.value"); - out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.ID = source_t.ID WHEN MATCHED THEN UPDATE SET NAME = source_t.NAME,CODE = source_t.CODE WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + out.assertContentEquals("MERGE PERSONS target_t \nUSING VALUES (?,?,?) \nAS source_t (ID,NAME,CODE) \nON target_t.ID = source_t.ID \nWHEN MATCHED THEN \nUPDATE SET NAME = source_t.NAME,CODE = source_t.CODE \nWHEN NOT MATCHED THEN \nINSERT (ID,NAME,CODE) \nVALUES (source_t.ID,source_t.NAME,source_t.CODE) \n;"); } @Test @@ -655,7 +655,7 @@ public void testMergeBasedOnPrimaryKey() throws InitializationException, Process runner.assertTransferCount(ConvertJSONToSQL.REL_SQL, 1); final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL).get(0); - out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.ID = source_t.ID WHEN MATCHED THEN UPDATE SET NAME = source_t.NAME,CODE = source_t.CODE WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + out.assertContentEquals("MERGE PERSONS target_t \nUSING VALUES (?,?,?) \nAS source_t (ID,NAME,CODE) \nON target_t.ID = source_t.ID \nWHEN MATCHED THEN \nUPDATE SET NAME = source_t.NAME,CODE = source_t.CODE \nWHEN NOT MATCHED THEN \nINSERT (ID,NAME,CODE) \nVALUES (source_t.ID,source_t.NAME,source_t.CODE) \n;"); out.assertAttributeEquals("sql.args.1.type", String.valueOf(java.sql.Types.INTEGER)); out.assertAttributeEquals("sql.args.1.value", "1"); out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.VARCHAR)); @@ -696,7 +696,7 @@ public void testMergeBasedOnUpdateKey() throws InitializationException, ProcessE out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.INTEGER)); out.assertAttributeEquals("sql.args.3.value", "48"); - out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.CODE = source_t.CODE WHEN MATCHED THEN UPDATE SET ID = source_t.ID,NAME = source_t.NAME WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + out.assertContentEquals("MERGE PERSONS target_t \nUSING VALUES (?,?,?) \nAS source_t (ID,NAME,CODE) \nON target_t.CODE = source_t.CODE \nWHEN MATCHED THEN \nUPDATE SET ID = source_t.ID,NAME = source_t.NAME \nWHEN NOT MATCHED THEN \nINSERT (ID,NAME,CODE) \nVALUES (source_t.ID,source_t.NAME,source_t.CODE) \n;"); } @Test @@ -731,7 +731,7 @@ public void testMergeBasedOnCompoundUpdateKey() throws InitializationException, out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.INTEGER)); out.assertAttributeEquals("sql.args.3.value", "48"); - out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.NAME = source_t.NAME,target_t.CODE = source_t.CODE WHEN MATCHED THEN UPDATE SET ID = source_t.ID WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + out.assertContentEquals("MERGE PERSONS target_t \nUSING VALUES (?,?,?) \nAS source_t (ID,NAME,CODE) \nON target_t.NAME = source_t.NAME and target_t.CODE = source_t.CODE \nWHEN MATCHED THEN \nUPDATE SET ID = source_t.ID \nWHEN NOT MATCHED THEN \nINSERT (ID,NAME,CODE) \nVALUES (source_t.ID,source_t.NAME,source_t.CODE) \n;"); } @Test @@ -842,7 +842,7 @@ public void testMergeWithMissingColumnWarning() throws InitializationException, out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.INTEGER)); out.assertAttributeEquals("sql.args.3.value", "48"); - out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.NAME = source_t.NAME,target_t.CODE = source_t.CODE WHEN MATCHED THEN UPDATE SET ID = source_t.ID WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + out.assertContentEquals("MERGE PERSONS target_t \nUSING VALUES (?,?,?) \nAS source_t (ID,NAME,CODE) \nON target_t.NAME = source_t.NAME and target_t.CODE = source_t.CODE \nWHEN MATCHED THEN \nUPDATE SET ID = source_t.ID \nWHEN NOT MATCHED THEN \nINSERT (ID,NAME,CODE) \nVALUES (source_t.ID,source_t.NAME,source_t.CODE) \n;"); } // End testMergeWithMissingColumnWarning() @@ -908,7 +908,7 @@ public void testMultipleMerges() throws InitializationException, ProcessExceptio runner.assertTransferCount(ConvertJSONToSQL.REL_SQL, 5); final List mffs = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL); for (final MockFlowFile mff : mffs) { - mff.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.ID = source_t.ID WHEN MATCHED THEN UPDATE SET NAME = source_t.NAME,CODE = source_t.CODE WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + mff.assertContentEquals("MERGE PERSONS target_t \nUSING VALUES (?,?,?) \nAS source_t (ID,NAME,CODE) \nON target_t.ID = source_t.ID \nWHEN MATCHED THEN \nUPDATE SET NAME = source_t.NAME,CODE = source_t.CODE \nWHEN NOT MATCHED THEN \nINSERT (ID,NAME,CODE) \nVALUES (source_t.ID,source_t.NAME,source_t.CODE) \n;"); for (int i=1; i <= 3; i++) { mff.assertAttributeExists("sql.args." + i + ".type"); From 0ce0ae4855f061fb078fed409e0266ee0a3f1762 Mon Sep 17 00:00:00 2001 From: Peter Wicks Date: Wed, 24 Aug 2016 15:45:54 -0600 Subject: [PATCH 07/26] Gui Icons - Fixing misc. colors --- .../nifi-web-ui/src/main/webapp/css/common-ui.css | 12 ++++++++++++ .../main/webapp/js/nf/summary/nf-summary-table.js | 6 +++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css index 7ab85d850697..facb3a5e61b8 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css @@ -524,10 +524,22 @@ div.context-menu-item.hover > div.context-menu-item-img.fa.fa-play { color:#5cb85c; } +.running { + color:#5cb85c; +} + div.context-menu-item.hover > div.context-menu-item-img.fa.fa-stop { color:#d9534f; } +.stopped { + color:#d9534f; +} + +.invalid { + color:#f0ad4e; +} + .context-menu-item-img.icon { position: relative; top: 1px; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary-table.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary-table.js index a43390ac4ec2..29c23949119e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary-table.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary-table.js @@ -266,10 +266,10 @@ nf.SummaryTable = (function () { var classes = nf.Common.escapeHtml(value.toLowerCase()); switch(nf.Common.escapeHtml(value.toLowerCase())) { case 'running': - classes += ' fa fa-play'; + classes += ' fa fa-play running'; break; case 'stopped': - classes += ' fa fa-stop'; + classes += ' fa fa-stop stopped'; break; case 'enabled': classes += ' fa fa-flash'; @@ -277,7 +277,7 @@ nf.SummaryTable = (function () { case 'disabled': classes += ' icon icon-enable-false'; case 'invalid': - classes += ' fa fa-warning'; + classes += ' fa fa-warning invalid'; break; default: classes += ''; From d03d1c5aa1d63cbb7b5f9f1baab20ba4db6a5bbd Mon Sep 17 00:00:00 2001 From: Peter Wicks Date: Wed, 24 Aug 2016 17:38:10 -0600 Subject: [PATCH 08/26] Gui Icons - Summary page + spinner --- .../nifi-web-ui/src/main/webapp/css/flow-status.css | 4 ++-- .../nifi-web-ui/src/main/webapp/js/nf/nf-ng-app-config.js | 2 +- .../src/main/webapp/js/nf/summary/nf-summary-table.js | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/flow-status.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/flow-status.css index e67e666683c5..d91ab01a4d91 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/flow-status.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/flow-status.css @@ -106,7 +106,7 @@ } #bulletin-button.has-bulletins { - background-color: #ba554a; /*warm-color*/ + background-color: #f0ad4e; /*warm-color*/ } #bulletin-button i.fa { @@ -124,7 +124,7 @@ } #connected-nodes-count.connection-warning { - color: #BA554A; + color: #f0ad4e; } /* search field */ diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-ng-app-config.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-ng-app-config.js index a0bcfd8e5d2e..effc909b59ab 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-ng-app-config.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-ng-app-config.js @@ -66,7 +66,7 @@ nf.ng.AppConfig = function ($mdThemingProvider, $compileProvider) { '200': 'f0ad4e', '300': 'f0ad4e', '400': 'f0ad4e', - '500': 'f0ad4e', /* warn-color */ + '500': '2B5C76', /* warn-color */ '600': 'f0ad4e', '700': 'f0ad4e', '800': 'f0ad4e', diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary-table.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary-table.js index 29c23949119e..218ecf6695d0 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary-table.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary-table.js @@ -266,10 +266,10 @@ nf.SummaryTable = (function () { var classes = nf.Common.escapeHtml(value.toLowerCase()); switch(nf.Common.escapeHtml(value.toLowerCase())) { case 'running': - classes += ' fa fa-play running'; + classes = ' fa fa-play running'; break; case 'stopped': - classes += ' fa fa-stop stopped'; + classes = ' fa fa-stop stopped'; break; case 'enabled': classes += ' fa fa-flash'; @@ -277,7 +277,7 @@ nf.SummaryTable = (function () { case 'disabled': classes += ' icon icon-enable-false'; case 'invalid': - classes += ' fa fa-warning invalid'; + classes = ' fa fa-warning invalid'; break; default: classes += ''; From 0aa91fd80e82241adfa658b6c2329ec6705260fb Mon Sep 17 00:00:00 2001 From: Peter Wicks Date: Wed, 24 Aug 2016 19:15:02 -0600 Subject: [PATCH 09/26] Gui Icons - Summary page again --- .../nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css index facb3a5e61b8..65fce6b6ec8f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css @@ -533,11 +533,11 @@ div.context-menu-item.hover > div.context-menu-item-img.fa.fa-stop { } .stopped { - color:#d9534f; + color:#d9534f !important; } .invalid { - color:#f0ad4e; + color:#f0ad4e !important; } .context-menu-item-img.icon { From 36614be5a47f448dee50d6ba27c279b6db562064 Mon Sep 17 00:00:00 2001 From: Paul Gibeault Date: Thu, 25 Aug 2016 11:10:50 -0600 Subject: [PATCH 10/26] Added guards and error handling to catch common Flow File errors --- .../processors/standard/ConvertJSONToSQL.java | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java index 89842d4329df..606a6c026348 100755 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java @@ -589,13 +589,12 @@ private String generateMerge(final JsonNode rootNode, final Map // This is the SQL Format we are building /////////////////////////////////////////// // MERGE target_t - // USING ( - // VALUES () - // ) AS source_t () + // USING VALUES () + // AS source_t () // ON // WHEN MATCHED THEN // UPDATE SET - // WHEN NOT MATCHED THEN + // WHEN NOT MATCHED THEN // INSERT () // VALUES () // ; --A MERGE statement must be terminated by a semi-colon (;). @@ -693,12 +692,18 @@ private String generateMerge(final JsonNode rootNode, final Map } } - // Trim trailing , characters - final String valueList = valueListBuilder.toString().substring(0, valueListBuilder.length() -1); - final String columnList = columnListBuilder.toString().substring(0, columnListBuilder.length() -1); - final String matchClause = matchClauseBuilder.toString().substring(0, matchClauseBuilder.length() -5); - final String setList = setListBuilder.toString().substring(0, setListBuilder.length() -1); - final String namedValueList = namedValueListBuilder.toString().substring(0, namedValueListBuilder.length() -1); + // Trim trailing delimiters + final String valueList = trimTrailingDelimiter(valueListBuilder.toString(), ","); + final String columnList = trimTrailingDelimiter(columnListBuilder.toString(), ","); + final String matchClause = trimTrailingDelimiter(matchClauseBuilder.toString(), " and "); + final String setList = trimTrailingDelimiter(setListBuilder.toString(), ","); + final String namedValueList = trimTrailingDelimiter(namedValueListBuilder.toString(),","); + + // We need all of these lists to be non-empty to proceed + if (valueList.isEmpty() || columnList.isEmpty() || matchClause.isEmpty() || setList.isEmpty() || namedValueList.isEmpty()) + { + throw new ProcessException("Unable to generate MERGE statement. There were no columns in the target table that matched fields in the JSON message."); + } // Build the SQL statement from the pieces we gathered return String.format("MERGE %s target_t \n" + @@ -715,6 +720,17 @@ private String generateMerge(final JsonNode rootNode, final Map setList, columnList, namedValueList); } + // Helper utility to trim trailing delimiter if present + private static String trimTrailingDelimiter(final String value, final String delimiter) { + int delimiterIndex = value.length() - delimiter.length(); + if (value.isEmpty() || + value.length() < delimiter.length() || + ! delimiter.equals(value.substring(delimiterIndex))) { + return value; + } else { + return value.substring(0,delimiterIndex); + } + } private static String normalizeColumnName(final String colName, final boolean translateColumnNames) { return translateColumnNames ? colName.toUpperCase().replace("_", "") : colName; From 57f9635ce75af5e55f45d02f184bc2485a710621 Mon Sep 17 00:00:00 2001 From: Paul Gibeault Date: Thu, 25 Aug 2016 16:06:38 -0600 Subject: [PATCH 11/26] Updated to follow NiFi code style guidelines --- .../nifi/processors/standard/ConvertJSONToSQL.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java index 606a6c026348..bc7868759b73 100755 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java @@ -669,9 +669,7 @@ private String generateMerge(final JsonNode rootNode, final Map // Use it to build the Match Clause if (normalizedUpdateNames.contains(normalizedColName)) { matchClauseBuilder.append(String.format("%s.%s = %s.%s and ", target_table,normalizedColName,source_table,normalizedColName)); - } - else - { + } else { setListBuilder.append(String.format("%s = %s.%s,",normalizedColName, source_table,normalizedColName)); } @@ -700,8 +698,7 @@ private String generateMerge(final JsonNode rootNode, final Map final String namedValueList = trimTrailingDelimiter(namedValueListBuilder.toString(),","); // We need all of these lists to be non-empty to proceed - if (valueList.isEmpty() || columnList.isEmpty() || matchClause.isEmpty() || setList.isEmpty() || namedValueList.isEmpty()) - { + if (valueList.isEmpty() || columnList.isEmpty() || matchClause.isEmpty() || setList.isEmpty() || namedValueList.isEmpty()) { throw new ProcessException("Unable to generate MERGE statement. There were no columns in the target table that matched fields in the JSON message."); } @@ -723,9 +720,9 @@ private String generateMerge(final JsonNode rootNode, final Map // Helper utility to trim trailing delimiter if present private static String trimTrailingDelimiter(final String value, final String delimiter) { int delimiterIndex = value.length() - delimiter.length(); - if (value.isEmpty() || - value.length() < delimiter.length() || - ! delimiter.equals(value.substring(delimiterIndex))) { + if (value.isEmpty() + || value.length() < delimiter.length() + || ! delimiter.equals(value.substring(delimiterIndex))) { return value; } else { return value.substring(0,delimiterIndex); From 5a37e1cf7854edf23fef66a6b7be5e82c9e9f174 Mon Sep 17 00:00:00 2001 From: Paul Gibeault Date: Tue, 27 Sep 2016 12:39:27 -0600 Subject: [PATCH 12/26] Implemented changes to address Jira NIFI-2829 Now Date and Time fields can be provided in String or EPOCH (Long) formats Implemented Unit Tests to validate new functionality --- .../nifi/processors/standard/PutSQL.java | 40 ++++++++- .../nifi/processors/standard/TestPutSQL.java | 85 +++++++++++++++++++ 2 files changed, 121 insertions(+), 4 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSQL.java index 8e87c567276b..e013661595da 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSQL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSQL.java @@ -783,18 +783,50 @@ private void setParameter(final PreparedStatement stmt, final String attrName, f stmt.setBigDecimal(parameterIndex, new BigDecimal(parameterValue)); break; case Types.DATE: - stmt.setDate(parameterIndex, new Date(Long.parseLong(parameterValue))); + long lDate; + + if(LONG_PATTERN.matcher(parameterValue).matches()){ + lDate = Long.parseLong(parameterValue); + }else { + String dateFormatString = "yyyy-MM-dd"; + if (!valueFormat.isEmpty()) { + dateFormatString = valueFormat; + } + SimpleDateFormat dateFormat = new SimpleDateFormat(dateFormatString); + java.util.Date parsedDate = dateFormat.parse(parameterValue); + lDate = parsedDate.getTime(); + } + + stmt.setDate(parameterIndex, new Date(lDate)); break; case Types.TIME: - stmt.setTime(parameterIndex, new Time(Long.parseLong(parameterValue))); + long lTime; + + if(LONG_PATTERN.matcher(parameterValue).matches()){ + lTime = Long.parseLong(parameterValue); + }else { + String timeFormatString = "HH:mm:ss.SSS"; + if (!valueFormat.isEmpty()) { + timeFormatString = valueFormat; + } + SimpleDateFormat dateFormat = new SimpleDateFormat(timeFormatString); + java.util.Date parsedDate = dateFormat.parse(parameterValue); + lTime = parsedDate.getTime(); + } + + stmt.setTime(parameterIndex, new Time(lTime)); break; case Types.TIMESTAMP: - long lTimestamp=0L; + long lTimestamp; if(LONG_PATTERN.matcher(parameterValue).matches()){ lTimestamp = Long.parseLong(parameterValue); }else { - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); + String dateTimeFormatString = "yyyy-MM-dd HH:mm:ss.SSS"; + if (!valueFormat.isEmpty()) { + dateTimeFormatString = valueFormat; + } + SimpleDateFormat dateFormat = new SimpleDateFormat(dateTimeFormatString); java.util.Date parsedDate = dateFormat.parse(parameterValue); lTimestamp = parsedDate.getTime(); } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutSQL.java index 321bac783fbc..bde3a2fb92fa 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutSQL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutSQL.java @@ -309,6 +309,91 @@ public void testUsingTimestampValuesEpochAndString() throws InitializationExcept } } + @Test + public void testUsingTimeValuesEpochAndString() throws InitializationException, ProcessException, SQLException, IOException, ParseException { + final TestRunner runner = TestRunners.newTestRunner(PutSQL.class); + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate("CREATE TABLE TIMETESTS (id integer primary key, ts1 time, ts2 time)"); + } + } + + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + runner.setProperty(PutSQL.CONNECTION_POOL, "dbcp"); + + final String arg2TS = "00:01:01"; + final String art3TS = "12:02:02"; + final String timeFormatString = "HH:mm:ss"; + SimpleDateFormat dateFormat = new SimpleDateFormat(timeFormatString); + java.util.Date parsedDate = dateFormat.parse(arg2TS); + + final Map attributes = new HashMap<>(); + attributes.put("sql.args.1.type", String.valueOf(Types.TIME)); + attributes.put("sql.args.1.value", Long.toString(parsedDate.getTime())); + attributes.put("sql.args.1.format", timeFormatString); + attributes.put("sql.args.2.type", String.valueOf(Types.TIME)); + attributes.put("sql.args.2.value", art3TS); + attributes.put("sql.args.2.format", timeFormatString); + + runner.enqueue("INSERT INTO TIMETESTS (ID, ts1, ts2) VALUES (1, ?, ?)".getBytes(), attributes); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutSQL.REL_SUCCESS, 1); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + final ResultSet rs = stmt.executeQuery("SELECT * FROM TIMETESTS"); + assertTrue(rs.next()); + assertEquals(1, rs.getInt(1)); + assertEquals(arg2TS, rs.getString(2)); + assertEquals(art3TS, rs.getString(3)); + assertFalse(rs.next()); + } + } + } + + @Test + public void testUsingDateValuesEpochAndString() throws InitializationException, ProcessException, SQLException, IOException, ParseException { + final TestRunner runner = TestRunners.newTestRunner(PutSQL.class); + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate("CREATE TABLE DATETESTS (id integer primary key, ts1 date, ts2 date)"); + } + } + + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + runner.setProperty(PutSQL.CONNECTION_POOL, "dbcp"); + + final String arg2TS = "2001-01-01"; + final String art3TS = "2002-02-02"; + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + java.util.Date parsedDate = dateFormat.parse(arg2TS); + + final Map attributes = new HashMap<>(); + attributes.put("sql.args.1.type", String.valueOf(Types.DATE)); + attributes.put("sql.args.1.value", Long.toString(parsedDate.getTime())); + attributes.put("sql.args.2.type", String.valueOf(Types.DATE)); + attributes.put("sql.args.2.value", art3TS); + + runner.enqueue("INSERT INTO DATETESTS (ID, ts1, ts2) VALUES (1, ?, ?)".getBytes(), attributes); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutSQL.REL_SUCCESS, 1); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + final ResultSet rs = stmt.executeQuery("SELECT * FROM DATETESTS"); + assertTrue(rs.next()); + assertEquals(1, rs.getInt(1)); + assertEquals(arg2TS, rs.getString(2)); + assertEquals(art3TS, rs.getString(3)); + assertFalse(rs.next()); + } + } + } + @Test public void testBinaryColumnTypes() throws InitializationException, ProcessException, SQLException, IOException, ParseException { final TestRunner runner = TestRunners.newTestRunner(PutSQL.class); From 5e2f0c0315bc775dbf0dedc018cb665102a690ed Mon Sep 17 00:00:00 2001 From: Paul Gibeault Date: Tue, 27 Sep 2016 12:39:27 -0600 Subject: [PATCH 13/26] Implemented changes to address Jira NIFI-2829 Now Date and Time fields can be provided in String or EPOCH (Long) formats Implemented Unit Tests to validate new functionality --- .../nifi/processors/standard/PutSQL.java | 40 ++++++++- .../nifi/processors/standard/TestPutSQL.java | 85 +++++++++++++++++++ 2 files changed, 121 insertions(+), 4 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSQL.java index 8e87c567276b..e013661595da 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSQL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSQL.java @@ -783,18 +783,50 @@ private void setParameter(final PreparedStatement stmt, final String attrName, f stmt.setBigDecimal(parameterIndex, new BigDecimal(parameterValue)); break; case Types.DATE: - stmt.setDate(parameterIndex, new Date(Long.parseLong(parameterValue))); + long lDate; + + if(LONG_PATTERN.matcher(parameterValue).matches()){ + lDate = Long.parseLong(parameterValue); + }else { + String dateFormatString = "yyyy-MM-dd"; + if (!valueFormat.isEmpty()) { + dateFormatString = valueFormat; + } + SimpleDateFormat dateFormat = new SimpleDateFormat(dateFormatString); + java.util.Date parsedDate = dateFormat.parse(parameterValue); + lDate = parsedDate.getTime(); + } + + stmt.setDate(parameterIndex, new Date(lDate)); break; case Types.TIME: - stmt.setTime(parameterIndex, new Time(Long.parseLong(parameterValue))); + long lTime; + + if(LONG_PATTERN.matcher(parameterValue).matches()){ + lTime = Long.parseLong(parameterValue); + }else { + String timeFormatString = "HH:mm:ss.SSS"; + if (!valueFormat.isEmpty()) { + timeFormatString = valueFormat; + } + SimpleDateFormat dateFormat = new SimpleDateFormat(timeFormatString); + java.util.Date parsedDate = dateFormat.parse(parameterValue); + lTime = parsedDate.getTime(); + } + + stmt.setTime(parameterIndex, new Time(lTime)); break; case Types.TIMESTAMP: - long lTimestamp=0L; + long lTimestamp; if(LONG_PATTERN.matcher(parameterValue).matches()){ lTimestamp = Long.parseLong(parameterValue); }else { - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); + String dateTimeFormatString = "yyyy-MM-dd HH:mm:ss.SSS"; + if (!valueFormat.isEmpty()) { + dateTimeFormatString = valueFormat; + } + SimpleDateFormat dateFormat = new SimpleDateFormat(dateTimeFormatString); java.util.Date parsedDate = dateFormat.parse(parameterValue); lTimestamp = parsedDate.getTime(); } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutSQL.java index 321bac783fbc..bde3a2fb92fa 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutSQL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutSQL.java @@ -309,6 +309,91 @@ public void testUsingTimestampValuesEpochAndString() throws InitializationExcept } } + @Test + public void testUsingTimeValuesEpochAndString() throws InitializationException, ProcessException, SQLException, IOException, ParseException { + final TestRunner runner = TestRunners.newTestRunner(PutSQL.class); + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate("CREATE TABLE TIMETESTS (id integer primary key, ts1 time, ts2 time)"); + } + } + + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + runner.setProperty(PutSQL.CONNECTION_POOL, "dbcp"); + + final String arg2TS = "00:01:01"; + final String art3TS = "12:02:02"; + final String timeFormatString = "HH:mm:ss"; + SimpleDateFormat dateFormat = new SimpleDateFormat(timeFormatString); + java.util.Date parsedDate = dateFormat.parse(arg2TS); + + final Map attributes = new HashMap<>(); + attributes.put("sql.args.1.type", String.valueOf(Types.TIME)); + attributes.put("sql.args.1.value", Long.toString(parsedDate.getTime())); + attributes.put("sql.args.1.format", timeFormatString); + attributes.put("sql.args.2.type", String.valueOf(Types.TIME)); + attributes.put("sql.args.2.value", art3TS); + attributes.put("sql.args.2.format", timeFormatString); + + runner.enqueue("INSERT INTO TIMETESTS (ID, ts1, ts2) VALUES (1, ?, ?)".getBytes(), attributes); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutSQL.REL_SUCCESS, 1); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + final ResultSet rs = stmt.executeQuery("SELECT * FROM TIMETESTS"); + assertTrue(rs.next()); + assertEquals(1, rs.getInt(1)); + assertEquals(arg2TS, rs.getString(2)); + assertEquals(art3TS, rs.getString(3)); + assertFalse(rs.next()); + } + } + } + + @Test + public void testUsingDateValuesEpochAndString() throws InitializationException, ProcessException, SQLException, IOException, ParseException { + final TestRunner runner = TestRunners.newTestRunner(PutSQL.class); + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate("CREATE TABLE DATETESTS (id integer primary key, ts1 date, ts2 date)"); + } + } + + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + runner.setProperty(PutSQL.CONNECTION_POOL, "dbcp"); + + final String arg2TS = "2001-01-01"; + final String art3TS = "2002-02-02"; + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + java.util.Date parsedDate = dateFormat.parse(arg2TS); + + final Map attributes = new HashMap<>(); + attributes.put("sql.args.1.type", String.valueOf(Types.DATE)); + attributes.put("sql.args.1.value", Long.toString(parsedDate.getTime())); + attributes.put("sql.args.2.type", String.valueOf(Types.DATE)); + attributes.put("sql.args.2.value", art3TS); + + runner.enqueue("INSERT INTO DATETESTS (ID, ts1, ts2) VALUES (1, ?, ?)".getBytes(), attributes); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutSQL.REL_SUCCESS, 1); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + final ResultSet rs = stmt.executeQuery("SELECT * FROM DATETESTS"); + assertTrue(rs.next()); + assertEquals(1, rs.getInt(1)); + assertEquals(arg2TS, rs.getString(2)); + assertEquals(art3TS, rs.getString(3)); + assertFalse(rs.next()); + } + } + } + @Test public void testBinaryColumnTypes() throws InitializationException, ProcessException, SQLException, IOException, ParseException { final TestRunner runner = TestRunners.newTestRunner(PutSQL.class); From d6be36fabdff3d863cf6bd037518eb7cf216916c Mon Sep 17 00:00:00 2001 From: Peter Wicks Date: Mon, 22 Aug 2016 11:45:20 -0600 Subject: [PATCH 14/26] NIFI-2603 --- .../nifi-web-ui/src/main/webapp/css/flow-status.css | 12 ++++++++++++ .../nifi-web-ui/src/main/webapp/css/graph.css | 10 +++++++++- .../src/main/webapp/js/nf/canvas/nf-port.js | 10 ++++++++-- .../src/main/webapp/js/nf/canvas/nf-processor.js | 10 ++++++++-- 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/flow-status.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/flow-status.css index c13344db4af3..e67e666683c5 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/flow-status.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/flow-status.css @@ -38,6 +38,18 @@ color: #728E9B; /*base-color*/ } +#flow-status .fa.fa-play { + color: #5cb85c; +} + +#flow-status .fa.fa-stop { + color: #d9534f; +} + +#flow-status .fa.fa-warning { + color: #f0ad4e; +} + #flow-status .icon span { font-size: 15px; font-weight: 500; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/graph.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/graph.css index 641b02d9871e..617b7d6bd87a 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/graph.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/graph.css @@ -114,7 +114,15 @@ rect.bulletin-background { } text.process-group-invalid.has-validation-errors { - fill: #ba554a; + fill: #f0ad4e; +} + +text.process-group-stopped.process-group-contents-icon { + fill: #d9534f; +} + +text.process-group-running.process-group-contents-icon{ + fill: #5cb85c; } text.active-thread-count-icon { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-port.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-port.js index 5b58d443ae78..bd9064039207 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-port.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-port.js @@ -337,9 +337,15 @@ nf.Port = (function () { .attr({ 'fill': function (d) { var fill = '#728e9b'; - if (d.status.aggregateSnapshot.runStatus === 'Invalid') { - fill = '#ba554a'; + + if (d.status.aggregateSnapshot.runStatus === 'Invalid') { + fill = '#f0ad4e'; + } else if (d.status.aggregateSnapshot.runStatus === 'Running') { + fill = '#5cb85c'; + } else if (d.status.aggregateSnapshot.runStatus === 'Stopped') { + fill = '#d9534f'; } + return fill; }, 'font-family': function (d) { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-processor.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-processor.js index 183d1b5d85ff..1679b9b19097 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-processor.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-processor.js @@ -588,9 +588,15 @@ nf.Processor = (function () { .attr({ 'fill': function (d) { var fill = '#728e9b'; - if (d.status.aggregateSnapshot.runStatus === 'Invalid') { - fill = '#ba554a'; + + if (d.status.aggregateSnapshot.runStatus === 'Invalid') { + fill = '#f0ad4e'; + } else if (d.status.aggregateSnapshot.runStatus === 'Running') { + fill = '#5cb85c'; + } else if (d.status.aggregateSnapshot.runStatus === 'Stopped') { + fill = '#d9534f'; } + return fill; }, 'font-family': function (d) { From 6d19dba085e7baee5c70ada8e594dcb837214424 Mon Sep 17 00:00:00 2001 From: Paul Gibeault Date: Mon, 22 Aug 2016 17:19:29 -0600 Subject: [PATCH 15/26] Initial Check-in 3 Unit tests still failing --- .../processors/standard/ConvertJSONToSQL.java | 148 ++++++++- .../standard/TestConvertJSONToSQL.java | 287 ++++++++++++++++++ 2 files changed, 430 insertions(+), 5 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java index 48da57f1436a..6e3b79d62c44 100755 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java @@ -68,8 +68,8 @@ @SupportsBatching @SeeAlso(PutSQL.class) @InputRequirement(Requirement.INPUT_REQUIRED) -@Tags({"json", "sql", "database", "rdbms", "insert", "update", "relational", "flat"}) -@CapabilityDescription("Converts a JSON-formatted FlowFile into an UPDATE or INSERT SQL statement. The incoming FlowFile is expected to be " +@Tags({"json", "sql", "database", "rdbms", "insert", "update", "merge", "relational", "flat"}) +@CapabilityDescription("Converts a JSON-formatted FlowFile into an UPDATE, INSERT or MERGE SQL statement. The incoming FlowFile is expected to be " + "\"flat\" JSON message, meaning that it consists of a single JSON element and each field maps to a simple type. If a field maps to " + "a JSON object, that JSON object will be interpreted as Text. If the input is an array of JSON elements, each element in the array is " + "output as a separate FlowFile to the 'sql' relationship. Upon successful conversion, the original FlowFile is routed to the 'original' " @@ -96,6 +96,7 @@ public class ConvertJSONToSQL extends AbstractProcessor { private static final String UPDATE_TYPE = "UPDATE"; private static final String INSERT_TYPE = "INSERT"; + private static final String MERGE_TYPE = "MERGE"; static final AllowableValue IGNORE_UNMATCHED_FIELD = new AllowableValue("Ignore Unmatched Fields", "Ignore Unmatched Fields", "Any field in the JSON document that cannot be mapped to a column in the database is ignored"); @@ -122,7 +123,7 @@ public class ConvertJSONToSQL extends AbstractProcessor { .name("Statement Type") .description("Specifies the type of SQL Statement to generate") .required(true) - .allowableValues(UPDATE_TYPE, INSERT_TYPE) + .allowableValues(UPDATE_TYPE, INSERT_TYPE, MERGE_TYPE) .build(); static final PropertyDescriptor TABLE_NAME = new PropertyDescriptor.Builder() .name("Table Name") @@ -167,7 +168,7 @@ public class ConvertJSONToSQL extends AbstractProcessor { static final PropertyDescriptor UPDATE_KEY = new PropertyDescriptor.Builder() .name("Update Keys") .description("A comma-separated list of column names that uniquely identifies a row in the database for UPDATE statements. " - + "If the Statement Type is UPDATE and this property is not set, the table's Primary Keys are used. " + + "If the Statement Type is UPDATE or MERGE and this property is not set, the table's Primary Keys are used. " + "In this case, if no Primary Key exists, the conversion to SQL will fail if Unmatched Column Behaviour is set to FAIL. " + "This property is ignored if the Statement Type is INSERT") .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) @@ -248,7 +249,7 @@ public void onTrigger(final ProcessContext context, final ProcessSession session final String schemaName = context.getProperty(SCHEMA_NAME).evaluateAttributeExpressions(flowFile).getValue(); final String tableName = context.getProperty(TABLE_NAME).evaluateAttributeExpressions(flowFile).getValue(); final SchemaKey schemaKey = new SchemaKey(catalog, tableName); - final boolean includePrimaryKeys = UPDATE_TYPE.equals(statementType) && updateKeys == null; + final boolean includePrimaryKeys = (UPDATE_TYPE.equals(statementType) || MERGE_TYPE.equals(statementType)) && updateKeys == null; // Is the unmatched column behaviour fail or warning? final boolean failUnmappedColumns = FAIL_UNMATCHED_COLUMN.getValue().equalsIgnoreCase(context.getProperty(UNMATCHED_COLUMN_BEHAVIOR).getValue()); @@ -331,6 +332,8 @@ public void process(final InputStream in) throws IOException { if (INSERT_TYPE.equals(statementType)) { sql = generateInsert(jsonNode, attributes, fqTableName, schema, translateFieldNames, ignoreUnmappedFields, failUnmappedColumns, warningUnmappedColumns); + } else if (MERGE_TYPE.equals(statementType)) { + sql = generateMerge(jsonNode, attributes, fqTableName, updateKeys, schema, translateFieldNames, ignoreUnmappedFields, failUnmappedColumns, warningUnmappedColumns); } else { sql = generateUpdate(jsonNode, attributes, fqTableName, updateKeys, schema, translateFieldNames, ignoreUnmappedFields, failUnmappedColumns, warningUnmappedColumns); } @@ -578,6 +581,141 @@ private String generateUpdate(final JsonNode rootNode, final Map return sqlBuilder.toString(); } + private String generateMerge(final JsonNode rootNode, final Map attributes, final String tableName, final String updateKeys, + final TableSchema schema, final boolean translateFieldNames, final boolean ignoreUnmappedFields, final boolean failUnmappedColumns, + final boolean warningUnmappedColumns) { + + /////////////////////////////////////////// + // This is the SQL Format we are building + /////////////////////////////////////////// + // MERGE target_t + // USING ( + // VALUES () + // ) AS source_t () + // ON + // WHEN MATCHED THEN + // UPDATE SET + // WHEN NOT MATCHED THEN + // INSERT () + // VALUES () + // ; --A MERGE statement must be terminated by a semi-colon (;). + // + // We first need to collect: ValueList, ColumnList, MatchClause, SetList and NamedValueList + final String source_table = "source_t"; + final String target_table = "target_t"; + + final Set updateKeyNames; + if (updateKeys == null) { + updateKeyNames = schema.getPrimaryKeyColumnNames(); + } else { + updateKeyNames = new HashSet<>(); + for (final String updateKey : updateKeys.split(",")) { + updateKeyNames.add(updateKey.trim()); + } + } + + if (updateKeyNames.isEmpty()) { + throw new ProcessException("Table '" + tableName + "' does not have a Primary Key and no Update Keys were specified"); + } + + // Create a Set of all normalized Update Key names, and ensure that there is a field in the JSON + // for each of the Update Key fields. + final Set normalizedFieldNames = getNormalizedColumnNames(rootNode, translateFieldNames); + final Set normalizedUpdateNames = new HashSet<>(); + for (final String uk : updateKeyNames) { + final String normalizedUK = normalizeColumnName(uk, translateFieldNames); + normalizedUpdateNames.add(normalizedUK); + + if (!normalizedFieldNames.contains(normalizedUK)) { + String missingColMessage = "JSON does not have a value for the " + (updateKeys == null ? "Primary" : "Update") + "Key column '" + uk + "'"; + if (failUnmappedColumns) { + getLogger().error(missingColMessage); + throw new ProcessException(missingColMessage); + } else if (warningUnmappedColumns) { + getLogger().warn(missingColMessage); + } + } + } + + // iterate over all of the elements in the JSON, building the SQL statement by adding the column names, as well as + // adding the column value to a "sql.args.N.value" attribute and the type of a "sql.args.N.type" attribute add the + // columns that we are inserting into + final StringBuilder valueListBuilder = new StringBuilder(); + final StringBuilder namedValueListBuilder = new StringBuilder(); + final StringBuilder columnListBuilder = new StringBuilder(); + final StringBuilder setListBuilder = new StringBuilder(); + final StringBuilder matchClauseBuilder = new StringBuilder(); + int fieldCount = 0; + + Iterator fieldNames = rootNode.getFieldNames(); + while (fieldNames.hasNext()) { + final String fieldName = fieldNames.next(); + + final String normalizedColName = normalizeColumnName(fieldName, translateFieldNames); + final ColumnDescription desc = schema.getColumns().get(normalizedColName); + if (desc == null) { + if (!ignoreUnmappedFields) { + throw new ProcessException("Cannot map JSON field '" + fieldName + "' to any column in the database"); + } else { + continue; + } + } + + // Build Named Value List, Column List, and Value list + valueListBuilder.append("?,"); + columnListBuilder.append(normalizedColName).append(","); + namedValueListBuilder.append(String.format("%s.%s,",source_table,normalizedColName)); + + // Check if this column is an Update Key. If so, + // Use it to build the Match Clause + if (normalizedUpdateNames.contains(normalizedColName)) { + matchClauseBuilder.append(String.format("%s.%s = %s.%s,", target_table,normalizedColName,source_table,normalizedColName)); + } + else + { + setListBuilder.append(String.format("%s = %s.%s,",normalizedColName, source_table,normalizedColName)); + } + + // Create parameter attributes + fieldCount++; + final int sqlType = desc.getDataType(); + attributes.put("sql.args." + fieldCount + ".type", String.valueOf(sqlType)); + + final Integer colSize = desc.getColumnSize(); + + final JsonNode fieldNode = rootNode.get(fieldName); + if (!fieldNode.isNull()) { + String fieldValue = rootNode.get(fieldName).asText(); + if (colSize != null && fieldValue.length() > colSize) { + fieldValue = fieldValue.substring(0, colSize); + } + attributes.put("sql.args." + fieldCount + ".value", fieldValue); + } + } + + // Trim trailing , characters + final String valueList = valueListBuilder.toString().substring(0, valueListBuilder.length() -1); + final String columnList = columnListBuilder.toString().substring(0, columnListBuilder.length() -1); + final String matchClause = matchClauseBuilder.toString().substring(0, matchClauseBuilder.length() -1); + final String setList = setListBuilder.toString().substring(0, setListBuilder.length() -1); + final String namedValueList = namedValueListBuilder.toString().substring(0, namedValueListBuilder.length() -1); + + // Build the SQL statement from the pieces we gathered + return String.format("MERGE %s target_t " + + "USING VALUES (%s) " + + "AS source_t (%s) " + + "ON %s " + + "WHEN MATCHED THEN " + + "UPDATE SET %s " + + "WHEN NOT MATCHED THEN " + + "INSERT (%s) " + + "VALUES (%s) " + + ";", // --MERGE requires a trailing semi-colon + tableName, valueList, columnList, matchClause, + setList, columnList, namedValueList); + } + + private static String normalizeColumnName(final String colName, final boolean translateColumnNames) { return translateColumnNames ? colName.toUpperCase().replace("_", "") : colName; } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java index 60d993ef0538..8b6a010aded4 100755 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java @@ -596,6 +596,293 @@ public void testUpdateWithMissingColumnIgnore() throws InitializationException, } // End testUpdateWithMissingColumnIgnore() + @Test + public void testMergeWithNullValue() throws InitializationException, ProcessException, SQLException, IOException { + final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); + final File tempDir = folder.getRoot(); + final File dbDir = new File(tempDir, "db"); + final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate(createPersons); + } + } + + runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); + runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); + runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "MERGE"); + runner.enqueue(Paths.get("src/test/resources/TestConvertJSONToSQL/person-with-null-code.json")); + runner.run(); + + runner.assertTransferCount(ConvertJSONToSQL.REL_ORIGINAL, 1); + runner.assertTransferCount(ConvertJSONToSQL.REL_SQL, 1); + final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL).get(0); + out.assertAttributeEquals("sql.args.1.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.1.value", "1"); + out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.VARCHAR)); + out.assertAttributeEquals("sql.args.2.value", "Mark"); + out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeNotExists("sql.args.3.value"); + + out.assertContentEquals("UPDATE PERSONS SET NAME = ?, CODE = ? WHERE ID = ?"); + } + + @Test + public void testMergeBasedOnPrimaryKey() throws InitializationException, ProcessException, SQLException, IOException { + final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); + final File tempDir = folder.getRoot(); + final File dbDir = new File(tempDir, "db"); + final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate(createPersons); + } + } + + runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); + runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); + runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "MERGE"); + runner.enqueue(Paths.get("src/test/resources/TestConvertJSONToSQL/person-1.json")); + runner.run(); + + runner.assertTransferCount(ConvertJSONToSQL.REL_ORIGINAL, 1); + runner.assertTransferCount(ConvertJSONToSQL.REL_SQL, 1); + final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL).get(0); + + out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.ID = source_t.ID WHEN MATCHED THEN UPDATE SET NAME = source_t.NAME,CODE = source_t.CODE WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + out.assertAttributeEquals("sql.args.1.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.1.value", "1"); + out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.VARCHAR)); + out.assertAttributeEquals("sql.args.2.value", "Mark"); + out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.3.value", "48"); + } + + @Test + public void testMergeBasedOnUpdateKey() throws InitializationException, ProcessException, SQLException, IOException { + final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); + final File tempDir = folder.getRoot(); + final File dbDir = new File(tempDir, "db"); + final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate(createPersons); + } + } + + runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); + runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); + runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "MERGE"); + runner.setProperty(ConvertJSONToSQL.UPDATE_KEY, "code"); + runner.enqueue(Paths.get("src/test/resources/TestConvertJSONToSQL/person-1.json")); + runner.run(); + + runner.assertTransferCount(ConvertJSONToSQL.REL_ORIGINAL, 1); + runner.assertTransferCount(ConvertJSONToSQL.REL_SQL, 1); + final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL).get(0); + out.assertAttributeEquals("sql.args.1.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.1.value", "48"); + out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.2.value", "1"); + out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.VARCHAR)); + out.assertAttributeEquals("sql.args.3.value", "Mark"); + + out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.ID = source_t.ID WHEN MATCHED THEN UPDATE SET NAME = source_t.NAME,CODE = source_t.CODE WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + } + + @Test + public void testMergeBasedOnCompoundUpdateKey() throws InitializationException, ProcessException, SQLException, IOException { + final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); + final File tempDir = folder.getRoot(); + final File dbDir = new File(tempDir, "db"); + final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate(createPersons); + } + } + + runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); + runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); + runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "MERGE"); + runner.setProperty(ConvertJSONToSQL.UPDATE_KEY, "name, code"); + runner.enqueue(Paths.get("src/test/resources/TestConvertJSONToSQL/person-1.json")); + runner.run(); + + runner.assertTransferCount(ConvertJSONToSQL.REL_ORIGINAL, 1); + runner.assertTransferCount(ConvertJSONToSQL.REL_SQL, 1); + final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL).get(0); + out.assertAttributeEquals("sql.args.1.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.1.value", "1"); + out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.2.value", "48"); + out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.VARCHAR)); + out.assertAttributeEquals("sql.args.3.value", "Mark"); + + out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.ID = source_t.ID WHEN MATCHED THEN UPDATE SET NAME = source_t.NAME,CODE = source_t.CODE WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + } + + @Test + public void testMergeWithMissingFieldBasedOnCompoundUpdateKey() throws InitializationException, ProcessException, SQLException, IOException { + final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); + final File tempDir = folder.getRoot(); + final File dbDir = new File(tempDir, "db"); + final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate(createPersons); + } + } + + runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); + runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); + runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "MERGE"); + runner.setProperty(ConvertJSONToSQL.UPDATE_KEY, "name, code"); + runner.enqueue(Paths.get("src/test/resources/TestConvertJSONToSQL/person-without-code.json")); + runner.run(); + + runner.assertAllFlowFilesTransferred(ConvertJSONToSQL.REL_FAILURE, 1); + } + + @Test + public void testMergeWithMalformedJson() throws InitializationException, ProcessException, SQLException, IOException { + final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); + final File tempDir = folder.getRoot(); + final File dbDir = new File(tempDir, "db"); + final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate(createPersons); + } + } + + runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); + runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); + runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "MERGE"); + runner.setProperty(ConvertJSONToSQL.UPDATE_KEY, "name, code"); + runner.enqueue(Paths.get("src/test/resources/TestConvertJSONToSQL/malformed-person-extra-comma.json")); + runner.run(); + + runner.assertAllFlowFilesTransferred(ConvertJSONToSQL.REL_FAILURE, 1); + } + @Test + public void testMergeWithMissingColumnFail() throws InitializationException, ProcessException, SQLException, IOException { + final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); + final File tempDir = folder.getRoot(); + final File dbDir = new File(tempDir, "db"); + final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate(createPersons); + } + } + + runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); + runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); + runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "MERGE"); + runner.setProperty(ConvertJSONToSQL.UPDATE_KEY, "name, code, extra"); + runner.setProperty(ConvertJSONToSQL.UNMATCHED_COLUMN_BEHAVIOR, ConvertJSONToSQL.FAIL_UNMATCHED_COLUMN); + runner.enqueue(Paths.get("src/test/resources/TestConvertJSONToSQL/person-1.json")); + runner.run(); + + runner.assertAllFlowFilesTransferred(ConvertJSONToSQL.REL_FAILURE, 1); + } // End testMergeWithMissingColumnFail() + + @Test + public void testMergeWithMissingColumnWarning() throws InitializationException, ProcessException, SQLException, IOException { + final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); + final File tempDir = folder.getRoot(); + final File dbDir = new File(tempDir, "db"); + final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate(createPersons); + } + } + + runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); + runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); + runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "MERGE"); + runner.setProperty(ConvertJSONToSQL.UPDATE_KEY, "name, code, extra"); + runner.setProperty(ConvertJSONToSQL.UNMATCHED_COLUMN_BEHAVIOR, ConvertJSONToSQL.WARNING_UNMATCHED_COLUMN); + runner.enqueue(Paths.get("src/test/resources/TestConvertJSONToSQL/person-1.json")); + runner.run(); + + runner.assertTransferCount(ConvertJSONToSQL.REL_ORIGINAL, 1); + runner.assertTransferCount(ConvertJSONToSQL.REL_SQL, 1); + final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL).get(0); + out.assertAttributeEquals("sql.args.1.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.1.value", "48"); + out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.2.value", "1"); + out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.VARCHAR)); + out.assertAttributeEquals("sql.args.3.value", "Mark"); + + out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.ID = source_t.ID WHEN MATCHED THEN UPDATE SET NAME = source_t.NAME,CODE = source_t.CODE WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + + } // End testMergeWithMissingColumnWarning() + + @Test + public void testMergeWithMissingColumnIgnore() throws InitializationException, ProcessException, SQLException, IOException { + final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); + final File tempDir = folder.getRoot(); + final File dbDir = new File(tempDir, "db"); + final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate(createPersons); + } + } + + runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); + runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); + runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "UPDATE"); + runner.setProperty(ConvertJSONToSQL.UPDATE_KEY, "name, code, extra"); + runner.setProperty(ConvertJSONToSQL.UNMATCHED_COLUMN_BEHAVIOR, "Ignore Unmatched Columns"); + runner.enqueue(Paths.get("src/test/resources/TestConvertJSONToSQL/person-1.json")); + runner.run(); + + runner.assertTransferCount(ConvertJSONToSQL.REL_ORIGINAL, 1); + runner.assertTransferCount(ConvertJSONToSQL.REL_SQL, 1); + final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL).get(0); + out.assertAttributeEquals("sql.args.1.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.1.value", "1"); + out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.VARCHAR)); + out.assertAttributeEquals("sql.args.2.value", "Mark"); + out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.3.value", "48"); + + out.assertContentEquals("UPDATE PERSONS SET ID = ? WHERE NAME = ? AND CODE = ?"); + + } // End testMergeWithMissingColumnIgnore() + /** * Simple implementation only for testing purposes */ From 3aab286409c1eb871376d998119145fbad85d3f6 Mon Sep 17 00:00:00 2001 From: Peter Wicks Date: Tue, 23 Aug 2016 11:47:46 -0600 Subject: [PATCH 16/26] Gui Icons --- .../nifi-web-ui/src/main/webapp/css/common-ui.css | 8 ++++++++ .../nifi-web-ui/src/main/webapp/css/navigation.css | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css index cf6a7a6558cf..c57a14979955 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css @@ -523,6 +523,14 @@ div.context-menu-item.hover { left: 2px; } +div.context-menu-item.hover > div.context-menu-item-img.fa.fa-play { + color:#5cb85c; +} + +div.context-menu-item.hover > div.context-menu-item-img.fa.fa-stop { + color:#5cb85c; +} + .context-menu-item-img.icon { position: relative; top: 1px; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/navigation.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/navigation.css index 2439d229e1c5..6ad2a9babd79 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/navigation.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/navigation.css @@ -213,6 +213,14 @@ div.action-button { text-transform:uppercase; } +#operate-start button:hover { + color:#5cb85c; +} + +#operate-stop button:hover { + color:#5cb85c; +} + div.graph-control div.icon-disabled { color: #ddd; } From 4e6c3c1469d02b857f2870c35671c65a48490902 Mon Sep 17 00:00:00 2001 From: Paul Gibeault Date: Wed, 24 Aug 2016 09:04:06 -0600 Subject: [PATCH 17/26] All 10 Unit Tests pass now --- .../standard/TestConvertJSONToSQL.java | 71 ++++++++++++++----- 1 file changed, 53 insertions(+), 18 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java index 8b6a010aded4..93b5e1ad44a3 100755 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java @@ -627,7 +627,7 @@ public void testMergeWithNullValue() throws InitializationException, ProcessExce out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.INTEGER)); out.assertAttributeNotExists("sql.args.3.value"); - out.assertContentEquals("UPDATE PERSONS SET NAME = ?, CODE = ? WHERE ID = ?"); + out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.ID = source_t.ID WHEN MATCHED THEN UPDATE SET NAME = source_t.NAME,CODE = source_t.CODE WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); } @Test @@ -690,13 +690,13 @@ public void testMergeBasedOnUpdateKey() throws InitializationException, ProcessE runner.assertTransferCount(ConvertJSONToSQL.REL_SQL, 1); final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL).get(0); out.assertAttributeEquals("sql.args.1.type", String.valueOf(java.sql.Types.INTEGER)); - out.assertAttributeEquals("sql.args.1.value", "48"); - out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.INTEGER)); - out.assertAttributeEquals("sql.args.2.value", "1"); - out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.VARCHAR)); - out.assertAttributeEquals("sql.args.3.value", "Mark"); + out.assertAttributeEquals("sql.args.1.value", "1"); + out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.VARCHAR)); + out.assertAttributeEquals("sql.args.2.value", "Mark"); + out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.3.value", "48"); - out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.ID = source_t.ID WHEN MATCHED THEN UPDATE SET NAME = source_t.NAME,CODE = source_t.CODE WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.CODE = source_t.CODE WHEN MATCHED THEN UPDATE SET ID = source_t.ID,NAME = source_t.NAME WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); } @Test @@ -726,12 +726,12 @@ public void testMergeBasedOnCompoundUpdateKey() throws InitializationException, final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL).get(0); out.assertAttributeEquals("sql.args.1.type", String.valueOf(java.sql.Types.INTEGER)); out.assertAttributeEquals("sql.args.1.value", "1"); - out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.INTEGER)); - out.assertAttributeEquals("sql.args.2.value", "48"); - out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.VARCHAR)); - out.assertAttributeEquals("sql.args.3.value", "Mark"); + out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.VARCHAR)); + out.assertAttributeEquals("sql.args.2.value", "Mark"); + out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.3.value", "48"); - out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.ID = source_t.ID WHEN MATCHED THEN UPDATE SET NAME = source_t.NAME,CODE = source_t.CODE WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.NAME = source_t.NAME,target_t.CODE = source_t.CODE WHEN MATCHED THEN UPDATE SET ID = source_t.ID WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); } @Test @@ -836,13 +836,13 @@ public void testMergeWithMissingColumnWarning() throws InitializationException, runner.assertTransferCount(ConvertJSONToSQL.REL_SQL, 1); final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL).get(0); out.assertAttributeEquals("sql.args.1.type", String.valueOf(java.sql.Types.INTEGER)); - out.assertAttributeEquals("sql.args.1.value", "48"); - out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.INTEGER)); - out.assertAttributeEquals("sql.args.2.value", "1"); - out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.VARCHAR)); - out.assertAttributeEquals("sql.args.3.value", "Mark"); + out.assertAttributeEquals("sql.args.1.value", "1"); + out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.VARCHAR)); + out.assertAttributeEquals("sql.args.2.value", "Mark"); + out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.INTEGER)); + out.assertAttributeEquals("sql.args.3.value", "48"); - out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.ID = source_t.ID WHEN MATCHED THEN UPDATE SET NAME = source_t.NAME,CODE = source_t.CODE WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.NAME = source_t.NAME,target_t.CODE = source_t.CODE WHEN MATCHED THEN UPDATE SET ID = source_t.ID WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); } // End testMergeWithMissingColumnWarning() @@ -883,6 +883,41 @@ public void testMergeWithMissingColumnIgnore() throws InitializationException, P } // End testMergeWithMissingColumnIgnore() + @Test + public void testMultipleMerges() throws InitializationException, ProcessException, SQLException, IOException { + final TestRunner runner = TestRunners.newTestRunner(ConvertJSONToSQL.class); + final File tempDir = folder.getRoot(); + final File dbDir = new File(tempDir, "db"); + final DBCPService service = new MockDBCPService(dbDir.getAbsolutePath()); + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate(createPersons); + } + } + + runner.setProperty(ConvertJSONToSQL.CONNECTION_POOL, "dbcp"); + runner.setProperty(ConvertJSONToSQL.TABLE_NAME, "PERSONS"); + runner.setProperty(ConvertJSONToSQL.STATEMENT_TYPE, "MERGE"); + runner.enqueue(Paths.get("src/test/resources/TestConvertJSONToSQL/persons.json")); + runner.run(); + + runner.assertTransferCount(ConvertJSONToSQL.REL_ORIGINAL, 1); + runner.assertTransferCount(ConvertJSONToSQL.REL_SQL, 5); + final List mffs = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL); + for (final MockFlowFile mff : mffs) { + mff.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.ID = source_t.ID WHEN MATCHED THEN UPDATE SET NAME = source_t.NAME,CODE = source_t.CODE WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + + for (int i=1; i <= 3; i++) { + mff.assertAttributeExists("sql.args." + i + ".type"); + mff.assertAttributeExists("sql.args." + i + ".value"); + } + } + } + + /** * Simple implementation only for testing purposes */ From b0cb964a7e6ca6c6f968882ac277affaf1b724cd Mon Sep 17 00:00:00 2001 From: Peter Wicks Date: Wed, 24 Aug 2016 10:01:37 -0600 Subject: [PATCH 18/26] Gui Icons - Fixing Orange Color --- .../src/main/webapp/css/common-ui.css | 2 +- .../nifi-web-ui/src/main/webapp/css/graph.css | 12 ++++---- .../nifi-web-ui/src/main/webapp/css/main.css | 8 +++--- .../src/main/webapp/css/navigation.css | 2 +- .../webapp/css/processor-configuration.css | 2 +- .../src/main/webapp/css/provenance.css | 4 +-- .../src/main/webapp/css/queue-listing.css | 2 +- .../remote-process-group-configuration.css | 2 +- .../slickgrid/css/slick-default-theme.css | 2 +- .../nf-ng-canvas-flow-status-controller.js | 6 ++-- .../src/main/webapp/js/nf/canvas/nf-canvas.js | 2 +- .../src/main/webapp/js/nf/nf-ng-app-config.js | 28 +++++++++---------- .../js/nf/provenance/nf-provenance-lineage.js | 2 +- .../src/main/webapp/css/main.css | 2 +- 14 files changed, 38 insertions(+), 38 deletions(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css index c57a14979955..a0e9aa114191 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css @@ -528,7 +528,7 @@ div.context-menu-item.hover > div.context-menu-item-img.fa.fa-play { } div.context-menu-item.hover > div.context-menu-item-img.fa.fa-stop { - color:#5cb85c; + color:#d9534f; } .context-menu-item-img.icon { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/graph.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/graph.css index 617b7d6bd87a..9bbf05f79845 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/graph.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/graph.css @@ -73,7 +73,7 @@ g.component rect.border { g.component rect.border.unauthorized { stroke-width: 1.5; - stroke: #ba554a; + stroke: #f0ad4e; stroke-dasharray: 3,3; } @@ -110,7 +110,7 @@ text.bulletin-icon { } rect.bulletin-background { - fill: #ba554a; + fill: #f0ad4e; } text.process-group-invalid.has-validation-errors { @@ -201,7 +201,7 @@ g.connection rect.body.unauthorized { g.connection rect.border.unauthorized { stroke-width: 1.5; - stroke: #ba554a; + stroke: #f0ad4e; stroke-dasharray: 3,3; } @@ -234,7 +234,7 @@ g.connection path.connection-path { } g.connection path.connection-path.unauthorized { - stroke: #ba554a; + stroke: #f0ad4e; stroke-dasharray: 3,3; } @@ -245,7 +245,7 @@ text.connection-from-run-status, text.connection-to-run-status, text.expiration- } text.connection-from-run-status.is-missing-port, text.connection-to-run-status.is-missing-port { - fill: #ba554a; + fill: #f0ad4e; } /* grouped connection */ @@ -392,7 +392,7 @@ text.remote-process-group-transmission-status { } text.remote-process-group-transmission-status.has-authorization-errors { - fill: #ba554a; + fill: #f0ad4e; } text.remote-process-group-transmission-secure { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/main.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/main.css index b78a14df79f2..9ef82beabb46 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/main.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/main.css @@ -104,13 +104,13 @@ div.running { div.has-errors, div.invalid { float: left; - color: #ba554a !important; + color: #f0ad4e !important; } div.has-errors:before, div.invalid:before { font-family: FontAwesome; content: "\f071"; - color: #ba554a; + color: #f0ad4e; } div.transmitting { @@ -128,7 +128,7 @@ div.valid { } div.has-bulletins { - color: #ba554a !important; + color: #f0ad4e !important; } /* @@ -233,7 +233,7 @@ span.details-title { font-weight: bold; font-family: Roboto; font-size: 13px; - color: #ba554a; + color: #f0ad4e; } #upload-template-container button { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/navigation.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/navigation.css index 6ad2a9babd79..e986a9bea8df 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/navigation.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/navigation.css @@ -218,7 +218,7 @@ div.action-button { } #operate-stop button:hover { - color:#5cb85c; + color:#d9534f; } div.graph-control div.icon-disabled { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/processor-configuration.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/processor-configuration.css index e03a6982618f..b8b6bcffb54d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/processor-configuration.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/processor-configuration.css @@ -75,7 +75,7 @@ div.processor-configuration-warning-icon:before { font-family: FontAwesome; content: "\f071"; font-size: 16px; - color: #ba554a; + color: #f0ad4e; } #auto-terminate-relationship-names { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/provenance.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/provenance.css index d787c4b081c3..c8e26ae6f6bd 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/provenance.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/provenance.css @@ -466,11 +466,11 @@ div.lineage-collapse-children:before { } path.link.selected { - stroke: #ba554a; + stroke: #f0ad4e; } g.event circle.selected { - fill: #ba554a; + fill: #f0ad4e; } text.event-type { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/queue-listing.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/queue-listing.css index 92d9997a594c..0c45c42d45cc 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/queue-listing.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/queue-listing.css @@ -75,7 +75,7 @@ #queue-listing-message { position: absolute; top: 36px; - color: #ba554a; + color: #f0ad4e; font-family: Roboto; font-size: 13px; font-weight: 500; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/remote-process-group-configuration.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/remote-process-group-configuration.css index 847bdb760598..44d24a3337fe 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/remote-process-group-configuration.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/remote-process-group-configuration.css @@ -100,7 +100,7 @@ div.remote-port-removed:before { font-family: FontAwesome; content: "\f071"; font-size: 16px; - color: #ba554a; + color: #f0ad4e; } div.remote-port-edit-container { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/slickgrid/css/slick-default-theme.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/slickgrid/css/slick-default-theme.css index 780d0d30caa8..1b8503d4afb0 100755 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/slickgrid/css/slick-default-theme.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/jquery/slickgrid/css/slick-default-theme.css @@ -77,7 +77,7 @@ classes should alter those! } .slick-cell.invalid { - border-color: #ba554a; + border-color: #f0ad4e; -moz-animation-duration: 0.2s; -webkit-animation-duration: 0.2s; -moz-animation-name: slickgrid-invalid-hilite; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/controllers/nf-ng-canvas-flow-status-controller.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/controllers/nf-ng-canvas-flow-status-controller.js index 32bdecadc119..7155e86943cc 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/controllers/nf-ng-canvas-flow-status-controller.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/controllers/nf-ng-canvas-flow-status-controller.js @@ -360,14 +360,14 @@ nf.ng.Canvas.FlowStatusCtrl = function (serviceProvider) { var connectedNodes = clusterSummary.connectedNodes.split(' / '); if (connectedNodes.length === 2 && connectedNodes[0] !== connectedNodes[1]) { this.clusterConnectionWarning = true; - color = '#BA554A'; + color = '#f0ad4e'; } } this.connectedNodesCount = nf.Common.isDefinedAndNotNull(clusterSummary.connectedNodes) ? clusterSummary.connectedNodes : '-'; } else { this.connectedNodesCount = 'Disconnected'; - color = '#BA554A'; + color = '#f0ad4e'; } // update the color @@ -382,7 +382,7 @@ nf.ng.Canvas.FlowStatusCtrl = function (serviceProvider) { update: function (status) { var controllerInvalidCountColor = (nf.Common.isDefinedAndNotNull(status.invalidCount) && (status.invalidCount > 0)) ? - '#BA554A' : '#728E9B'; + '#f0ad4e' : '#728E9B'; $('#controller-invalid-count').parent().css('color', controllerInvalidCountColor); // update the report values diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas.js index 66c1c6f0f121..a402a59f45b9 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas.js @@ -213,7 +213,7 @@ nf.Canvas = (function () { if (d === 'ghost') { return '#aaaaaa'; } else if (d === 'unauthorized') { - return '#ba554a'; + return '#f0ad4e'; } else { return '#000000'; } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-ng-app-config.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-ng-app-config.js index 54dccde026e4..a0bcfd8e5d2e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-ng-app-config.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-ng-app-config.js @@ -61,20 +61,20 @@ nf.ng.AppConfig = function ($mdThemingProvider, $compileProvider) { 'contrastLightColors': undefined }); $mdThemingProvider.definePalette('warnPalette', { - '50': 'BA554A', - '100': 'BA554A', - '200': 'BA554A', - '300': 'BA554A', - '400': 'BA554A', - '500': 'BA554A', /* warn-color */ - '600': 'BA554A', - '700': 'BA554A', - '800': 'BA554A', - '900': 'BA554A', - 'A100': 'BA554A', - 'A200': 'BA554A', - 'A400': 'BA554A', - 'A700': 'BA554A', + '50': 'f0ad4e', + '100': 'f0ad4e', + '200': 'f0ad4e', + '300': 'f0ad4e', + '400': 'f0ad4e', + '500': 'f0ad4e', /* warn-color */ + '600': 'f0ad4e', + '700': 'f0ad4e', + '800': 'f0ad4e', + '900': 'f0ad4e', + 'A100': 'f0ad4e', + 'A200': 'f0ad4e', + 'A400': 'f0ad4e', + 'A700': 'f0ad4e', 'contrastDefaultColor': 'light', 'contrastDarkColors': ['A100'], 'contrastLightColors': undefined diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-lineage.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-lineage.js index 4e15cd21fa25..b6da60bf589a 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-lineage.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-lineage.js @@ -608,7 +608,7 @@ nf.ng.ProvenanceLineage = function () { 'orient': 'auto', 'fill': function (d) { if (d.indexOf('SELECTED') >= 0) { - return '#ba554a'; + return '#f0ad4e'; } else { return '#000000'; } diff --git a/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-ui/src/main/webapp/css/main.css b/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-ui/src/main/webapp/css/main.css index b5d315cb2726..e95ecb061735 100644 --- a/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-ui/src/main/webapp/css/main.css +++ b/nifi-nar-bundles/nifi-update-attribute-bundle/nifi-update-attribute-ui/src/main/webapp/css/main.css @@ -249,7 +249,7 @@ div.large-label-container { } #message { - color: #ba554a; + color: #f0ad4e; margin-left: 20px; margin-right: 20px; line-height: 32px; From d2a7d2311b2ede2c804b5bd8e201c38cd358bda7 Mon Sep 17 00:00:00 2001 From: Paul Gibeault Date: Wed, 24 Aug 2016 11:11:23 -0600 Subject: [PATCH 19/26] Corrected MatchClause of Merge statement Included line-returns in MERGE SQL for readability --- .../processors/standard/ConvertJSONToSQL.java | 22 +++++++++---------- .../standard/TestConvertJSONToSQL.java | 12 +++++----- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java index 6e3b79d62c44..aefeaaf2f360 100755 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java @@ -669,7 +669,7 @@ private String generateMerge(final JsonNode rootNode, final Map // Check if this column is an Update Key. If so, // Use it to build the Match Clause if (normalizedUpdateNames.contains(normalizedColName)) { - matchClauseBuilder.append(String.format("%s.%s = %s.%s,", target_table,normalizedColName,source_table,normalizedColName)); + matchClauseBuilder.append(String.format("%s.%s = %s.%s and ", target_table,normalizedColName,source_table,normalizedColName)); } else { @@ -696,20 +696,20 @@ private String generateMerge(final JsonNode rootNode, final Map // Trim trailing , characters final String valueList = valueListBuilder.toString().substring(0, valueListBuilder.length() -1); final String columnList = columnListBuilder.toString().substring(0, columnListBuilder.length() -1); - final String matchClause = matchClauseBuilder.toString().substring(0, matchClauseBuilder.length() -1); + final String matchClause = matchClauseBuilder.toString().substring(0, matchClauseBuilder.length() -5); final String setList = setListBuilder.toString().substring(0, setListBuilder.length() -1); final String namedValueList = namedValueListBuilder.toString().substring(0, namedValueListBuilder.length() -1); // Build the SQL statement from the pieces we gathered - return String.format("MERGE %s target_t " + - "USING VALUES (%s) " + - "AS source_t (%s) " + - "ON %s " + - "WHEN MATCHED THEN " + - "UPDATE SET %s " + - "WHEN NOT MATCHED THEN " + - "INSERT (%s) " + - "VALUES (%s) " + + return String.format("MERGE %s target_t \n" + + "USING VALUES (%s) \n" + + "AS source_t (%s) \n" + + "ON %s \n" + + "WHEN MATCHED THEN \n" + + "UPDATE SET %s \n" + + "WHEN NOT MATCHED THEN \n" + + "INSERT (%s) \n" + + "VALUES (%s) \n" + ";", // --MERGE requires a trailing semi-colon tableName, valueList, columnList, matchClause, setList, columnList, namedValueList); diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java index 93b5e1ad44a3..97a4c4d0c266 100755 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestConvertJSONToSQL.java @@ -627,7 +627,7 @@ public void testMergeWithNullValue() throws InitializationException, ProcessExce out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.INTEGER)); out.assertAttributeNotExists("sql.args.3.value"); - out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.ID = source_t.ID WHEN MATCHED THEN UPDATE SET NAME = source_t.NAME,CODE = source_t.CODE WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + out.assertContentEquals("MERGE PERSONS target_t \nUSING VALUES (?,?,?) \nAS source_t (ID,NAME,CODE) \nON target_t.ID = source_t.ID \nWHEN MATCHED THEN \nUPDATE SET NAME = source_t.NAME,CODE = source_t.CODE \nWHEN NOT MATCHED THEN \nINSERT (ID,NAME,CODE) \nVALUES (source_t.ID,source_t.NAME,source_t.CODE) \n;"); } @Test @@ -655,7 +655,7 @@ public void testMergeBasedOnPrimaryKey() throws InitializationException, Process runner.assertTransferCount(ConvertJSONToSQL.REL_SQL, 1); final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL).get(0); - out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.ID = source_t.ID WHEN MATCHED THEN UPDATE SET NAME = source_t.NAME,CODE = source_t.CODE WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + out.assertContentEquals("MERGE PERSONS target_t \nUSING VALUES (?,?,?) \nAS source_t (ID,NAME,CODE) \nON target_t.ID = source_t.ID \nWHEN MATCHED THEN \nUPDATE SET NAME = source_t.NAME,CODE = source_t.CODE \nWHEN NOT MATCHED THEN \nINSERT (ID,NAME,CODE) \nVALUES (source_t.ID,source_t.NAME,source_t.CODE) \n;"); out.assertAttributeEquals("sql.args.1.type", String.valueOf(java.sql.Types.INTEGER)); out.assertAttributeEquals("sql.args.1.value", "1"); out.assertAttributeEquals("sql.args.2.type", String.valueOf(java.sql.Types.VARCHAR)); @@ -696,7 +696,7 @@ public void testMergeBasedOnUpdateKey() throws InitializationException, ProcessE out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.INTEGER)); out.assertAttributeEquals("sql.args.3.value", "48"); - out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.CODE = source_t.CODE WHEN MATCHED THEN UPDATE SET ID = source_t.ID,NAME = source_t.NAME WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + out.assertContentEquals("MERGE PERSONS target_t \nUSING VALUES (?,?,?) \nAS source_t (ID,NAME,CODE) \nON target_t.CODE = source_t.CODE \nWHEN MATCHED THEN \nUPDATE SET ID = source_t.ID,NAME = source_t.NAME \nWHEN NOT MATCHED THEN \nINSERT (ID,NAME,CODE) \nVALUES (source_t.ID,source_t.NAME,source_t.CODE) \n;"); } @Test @@ -731,7 +731,7 @@ public void testMergeBasedOnCompoundUpdateKey() throws InitializationException, out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.INTEGER)); out.assertAttributeEquals("sql.args.3.value", "48"); - out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.NAME = source_t.NAME,target_t.CODE = source_t.CODE WHEN MATCHED THEN UPDATE SET ID = source_t.ID WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + out.assertContentEquals("MERGE PERSONS target_t \nUSING VALUES (?,?,?) \nAS source_t (ID,NAME,CODE) \nON target_t.NAME = source_t.NAME and target_t.CODE = source_t.CODE \nWHEN MATCHED THEN \nUPDATE SET ID = source_t.ID \nWHEN NOT MATCHED THEN \nINSERT (ID,NAME,CODE) \nVALUES (source_t.ID,source_t.NAME,source_t.CODE) \n;"); } @Test @@ -842,7 +842,7 @@ public void testMergeWithMissingColumnWarning() throws InitializationException, out.assertAttributeEquals("sql.args.3.type", String.valueOf(java.sql.Types.INTEGER)); out.assertAttributeEquals("sql.args.3.value", "48"); - out.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.NAME = source_t.NAME,target_t.CODE = source_t.CODE WHEN MATCHED THEN UPDATE SET ID = source_t.ID WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + out.assertContentEquals("MERGE PERSONS target_t \nUSING VALUES (?,?,?) \nAS source_t (ID,NAME,CODE) \nON target_t.NAME = source_t.NAME and target_t.CODE = source_t.CODE \nWHEN MATCHED THEN \nUPDATE SET ID = source_t.ID \nWHEN NOT MATCHED THEN \nINSERT (ID,NAME,CODE) \nVALUES (source_t.ID,source_t.NAME,source_t.CODE) \n;"); } // End testMergeWithMissingColumnWarning() @@ -908,7 +908,7 @@ public void testMultipleMerges() throws InitializationException, ProcessExceptio runner.assertTransferCount(ConvertJSONToSQL.REL_SQL, 5); final List mffs = runner.getFlowFilesForRelationship(ConvertJSONToSQL.REL_SQL); for (final MockFlowFile mff : mffs) { - mff.assertContentEquals("MERGE PERSONS target_t USING VALUES (?,?,?) AS source_t (ID,NAME,CODE) ON target_t.ID = source_t.ID WHEN MATCHED THEN UPDATE SET NAME = source_t.NAME,CODE = source_t.CODE WHEN NOT MATCHED THEN INSERT (ID,NAME,CODE) VALUES (source_t.ID,source_t.NAME,source_t.CODE) ;"); + mff.assertContentEquals("MERGE PERSONS target_t \nUSING VALUES (?,?,?) \nAS source_t (ID,NAME,CODE) \nON target_t.ID = source_t.ID \nWHEN MATCHED THEN \nUPDATE SET NAME = source_t.NAME,CODE = source_t.CODE \nWHEN NOT MATCHED THEN \nINSERT (ID,NAME,CODE) \nVALUES (source_t.ID,source_t.NAME,source_t.CODE) \n;"); for (int i=1; i <= 3; i++) { mff.assertAttributeExists("sql.args." + i + ".type"); From d816e581ff4454d29b2e8b0dcebc00a2ed43ebd9 Mon Sep 17 00:00:00 2001 From: Peter Wicks Date: Wed, 24 Aug 2016 15:45:54 -0600 Subject: [PATCH 20/26] Gui Icons - Fixing misc. colors --- .../nifi-web-ui/src/main/webapp/css/common-ui.css | 12 ++++++++++++ .../main/webapp/js/nf/summary/nf-summary-table.js | 6 +++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css index a0e9aa114191..b43309272f21 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css @@ -527,10 +527,22 @@ div.context-menu-item.hover > div.context-menu-item-img.fa.fa-play { color:#5cb85c; } +.running { + color:#5cb85c; +} + div.context-menu-item.hover > div.context-menu-item-img.fa.fa-stop { color:#d9534f; } +.stopped { + color:#d9534f; +} + +.invalid { + color:#f0ad4e; +} + .context-menu-item-img.icon { position: relative; top: 1px; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary-table.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary-table.js index 60fea88fc1c1..ea568694f202 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary-table.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary-table.js @@ -266,10 +266,10 @@ nf.SummaryTable = (function () { var classes = nf.Common.escapeHtml(value.toLowerCase()); switch(nf.Common.escapeHtml(value.toLowerCase())) { case 'running': - classes += ' fa fa-play'; + classes += ' fa fa-play running'; break; case 'stopped': - classes += ' fa fa-stop'; + classes += ' fa fa-stop stopped'; break; case 'enabled': classes += ' fa fa-flash'; @@ -278,7 +278,7 @@ nf.SummaryTable = (function () { classes += ' icon icon-enable-false'; break; case 'invalid': - classes += ' fa fa-warning'; + classes += ' fa fa-warning invalid'; break; default: classes += ''; From 1e711d4b303137b630fdd9d8eb41ca7ae1d54bc7 Mon Sep 17 00:00:00 2001 From: Peter Wicks Date: Wed, 24 Aug 2016 17:38:10 -0600 Subject: [PATCH 21/26] Gui Icons - Summary page + spinner --- .../nifi-web-ui/src/main/webapp/css/flow-status.css | 4 ++-- .../nifi-web-ui/src/main/webapp/js/nf/nf-ng-app-config.js | 2 +- .../src/main/webapp/js/nf/summary/nf-summary-table.js | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/flow-status.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/flow-status.css index e67e666683c5..d91ab01a4d91 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/flow-status.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/flow-status.css @@ -106,7 +106,7 @@ } #bulletin-button.has-bulletins { - background-color: #ba554a; /*warm-color*/ + background-color: #f0ad4e; /*warm-color*/ } #bulletin-button i.fa { @@ -124,7 +124,7 @@ } #connected-nodes-count.connection-warning { - color: #BA554A; + color: #f0ad4e; } /* search field */ diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-ng-app-config.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-ng-app-config.js index a0bcfd8e5d2e..effc909b59ab 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-ng-app-config.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/nf-ng-app-config.js @@ -66,7 +66,7 @@ nf.ng.AppConfig = function ($mdThemingProvider, $compileProvider) { '200': 'f0ad4e', '300': 'f0ad4e', '400': 'f0ad4e', - '500': 'f0ad4e', /* warn-color */ + '500': '2B5C76', /* warn-color */ '600': 'f0ad4e', '700': 'f0ad4e', '800': 'f0ad4e', diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary-table.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary-table.js index ea568694f202..2c788219d089 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary-table.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/summary/nf-summary-table.js @@ -266,10 +266,10 @@ nf.SummaryTable = (function () { var classes = nf.Common.escapeHtml(value.toLowerCase()); switch(nf.Common.escapeHtml(value.toLowerCase())) { case 'running': - classes += ' fa fa-play running'; + classes = ' fa fa-play running'; break; case 'stopped': - classes += ' fa fa-stop stopped'; + classes = ' fa fa-stop stopped'; break; case 'enabled': classes += ' fa fa-flash'; @@ -278,7 +278,7 @@ nf.SummaryTable = (function () { classes += ' icon icon-enable-false'; break; case 'invalid': - classes += ' fa fa-warning invalid'; + classes = ' fa fa-warning invalid'; break; default: classes += ''; From de5ad09296753c37ff2e750f3eea70084415b1be Mon Sep 17 00:00:00 2001 From: Peter Wicks Date: Wed, 24 Aug 2016 19:15:02 -0600 Subject: [PATCH 22/26] Gui Icons - Summary page again --- .../nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css index b43309272f21..d6fc4d346b7d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/common-ui.css @@ -536,11 +536,11 @@ div.context-menu-item.hover > div.context-menu-item-img.fa.fa-stop { } .stopped { - color:#d9534f; + color:#d9534f !important; } .invalid { - color:#f0ad4e; + color:#f0ad4e !important; } .context-menu-item-img.icon { From 215bb035a5ed0a8f2233c69734230a1556549ab6 Mon Sep 17 00:00:00 2001 From: Paul Gibeault Date: Thu, 25 Aug 2016 11:10:50 -0600 Subject: [PATCH 23/26] Added guards and error handling to catch common Flow File errors --- .../processors/standard/ConvertJSONToSQL.java | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java index aefeaaf2f360..276359635cda 100755 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java @@ -589,13 +589,12 @@ private String generateMerge(final JsonNode rootNode, final Map // This is the SQL Format we are building /////////////////////////////////////////// // MERGE target_t - // USING ( - // VALUES () - // ) AS source_t () + // USING VALUES () + // AS source_t () // ON // WHEN MATCHED THEN // UPDATE SET - // WHEN NOT MATCHED THEN + // WHEN NOT MATCHED THEN // INSERT () // VALUES () // ; --A MERGE statement must be terminated by a semi-colon (;). @@ -693,12 +692,18 @@ private String generateMerge(final JsonNode rootNode, final Map } } - // Trim trailing , characters - final String valueList = valueListBuilder.toString().substring(0, valueListBuilder.length() -1); - final String columnList = columnListBuilder.toString().substring(0, columnListBuilder.length() -1); - final String matchClause = matchClauseBuilder.toString().substring(0, matchClauseBuilder.length() -5); - final String setList = setListBuilder.toString().substring(0, setListBuilder.length() -1); - final String namedValueList = namedValueListBuilder.toString().substring(0, namedValueListBuilder.length() -1); + // Trim trailing delimiters + final String valueList = trimTrailingDelimiter(valueListBuilder.toString(), ","); + final String columnList = trimTrailingDelimiter(columnListBuilder.toString(), ","); + final String matchClause = trimTrailingDelimiter(matchClauseBuilder.toString(), " and "); + final String setList = trimTrailingDelimiter(setListBuilder.toString(), ","); + final String namedValueList = trimTrailingDelimiter(namedValueListBuilder.toString(),","); + + // We need all of these lists to be non-empty to proceed + if (valueList.isEmpty() || columnList.isEmpty() || matchClause.isEmpty() || setList.isEmpty() || namedValueList.isEmpty()) + { + throw new ProcessException("Unable to generate MERGE statement. There were no columns in the target table that matched fields in the JSON message."); + } // Build the SQL statement from the pieces we gathered return String.format("MERGE %s target_t \n" + @@ -715,6 +720,17 @@ private String generateMerge(final JsonNode rootNode, final Map setList, columnList, namedValueList); } + // Helper utility to trim trailing delimiter if present + private static String trimTrailingDelimiter(final String value, final String delimiter) { + int delimiterIndex = value.length() - delimiter.length(); + if (value.isEmpty() || + value.length() < delimiter.length() || + ! delimiter.equals(value.substring(delimiterIndex))) { + return value; + } else { + return value.substring(0,delimiterIndex); + } + } private static String normalizeColumnName(final String colName, final boolean translateColumnNames) { return translateColumnNames ? colName.toUpperCase().replace("_", "") : colName; From 43b374b27aab334cc286d0f1a1941dca05326288 Mon Sep 17 00:00:00 2001 From: Paul Gibeault Date: Thu, 25 Aug 2016 16:06:38 -0600 Subject: [PATCH 24/26] Updated to follow NiFi code style guidelines --- .../nifi/processors/standard/ConvertJSONToSQL.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java index 276359635cda..eea91880f612 100755 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ConvertJSONToSQL.java @@ -669,9 +669,7 @@ private String generateMerge(final JsonNode rootNode, final Map // Use it to build the Match Clause if (normalizedUpdateNames.contains(normalizedColName)) { matchClauseBuilder.append(String.format("%s.%s = %s.%s and ", target_table,normalizedColName,source_table,normalizedColName)); - } - else - { + } else { setListBuilder.append(String.format("%s = %s.%s,",normalizedColName, source_table,normalizedColName)); } @@ -700,8 +698,7 @@ private String generateMerge(final JsonNode rootNode, final Map final String namedValueList = trimTrailingDelimiter(namedValueListBuilder.toString(),","); // We need all of these lists to be non-empty to proceed - if (valueList.isEmpty() || columnList.isEmpty() || matchClause.isEmpty() || setList.isEmpty() || namedValueList.isEmpty()) - { + if (valueList.isEmpty() || columnList.isEmpty() || matchClause.isEmpty() || setList.isEmpty() || namedValueList.isEmpty()) { throw new ProcessException("Unable to generate MERGE statement. There were no columns in the target table that matched fields in the JSON message."); } @@ -723,9 +720,9 @@ private String generateMerge(final JsonNode rootNode, final Map // Helper utility to trim trailing delimiter if present private static String trimTrailingDelimiter(final String value, final String delimiter) { int delimiterIndex = value.length() - delimiter.length(); - if (value.isEmpty() || - value.length() < delimiter.length() || - ! delimiter.equals(value.substring(delimiterIndex))) { + if (value.isEmpty() + || value.length() < delimiter.length() + || ! delimiter.equals(value.substring(delimiterIndex))) { return value; } else { return value.substring(0,delimiterIndex); From d2c8255da76156e9869ce7e02fc1aacd30822e1e Mon Sep 17 00:00:00 2001 From: Paul Gibeault Date: Tue, 27 Sep 2016 12:39:27 -0600 Subject: [PATCH 25/26] Implemented changes to address Jira NIFI-2829 Now Date and Time fields can be provided in String or EPOCH (Long) formats Implemented Unit Tests to validate new functionality --- .../nifi/processors/standard/PutSQL.java | 40 ++++++++- .../nifi/processors/standard/TestPutSQL.java | 85 +++++++++++++++++++ 2 files changed, 121 insertions(+), 4 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSQL.java index eb27d407f824..3862b4eeb222 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSQL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSQL.java @@ -783,18 +783,50 @@ private void setParameter(final PreparedStatement stmt, final String attrName, f stmt.setBigDecimal(parameterIndex, new BigDecimal(parameterValue)); break; case Types.DATE: - stmt.setDate(parameterIndex, new Date(Long.parseLong(parameterValue))); + long lDate; + + if(LONG_PATTERN.matcher(parameterValue).matches()){ + lDate = Long.parseLong(parameterValue); + }else { + String dateFormatString = "yyyy-MM-dd"; + if (!valueFormat.isEmpty()) { + dateFormatString = valueFormat; + } + SimpleDateFormat dateFormat = new SimpleDateFormat(dateFormatString); + java.util.Date parsedDate = dateFormat.parse(parameterValue); + lDate = parsedDate.getTime(); + } + + stmt.setDate(parameterIndex, new Date(lDate)); break; case Types.TIME: - stmt.setTime(parameterIndex, new Time(Long.parseLong(parameterValue))); + long lTime; + + if(LONG_PATTERN.matcher(parameterValue).matches()){ + lTime = Long.parseLong(parameterValue); + }else { + String timeFormatString = "HH:mm:ss.SSS"; + if (!valueFormat.isEmpty()) { + timeFormatString = valueFormat; + } + SimpleDateFormat dateFormat = new SimpleDateFormat(timeFormatString); + java.util.Date parsedDate = dateFormat.parse(parameterValue); + lTime = parsedDate.getTime(); + } + + stmt.setTime(parameterIndex, new Time(lTime)); break; case Types.TIMESTAMP: - long lTimestamp=0L; + long lTimestamp; if(LONG_PATTERN.matcher(parameterValue).matches()){ lTimestamp = Long.parseLong(parameterValue); }else { - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); + String dateTimeFormatString = "yyyy-MM-dd HH:mm:ss.SSS"; + if (!valueFormat.isEmpty()) { + dateTimeFormatString = valueFormat; + } + SimpleDateFormat dateFormat = new SimpleDateFormat(dateTimeFormatString); java.util.Date parsedDate = dateFormat.parse(parameterValue); lTimestamp = parsedDate.getTime(); } diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutSQL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutSQL.java index 321bac783fbc..bde3a2fb92fa 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutSQL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPutSQL.java @@ -309,6 +309,91 @@ public void testUsingTimestampValuesEpochAndString() throws InitializationExcept } } + @Test + public void testUsingTimeValuesEpochAndString() throws InitializationException, ProcessException, SQLException, IOException, ParseException { + final TestRunner runner = TestRunners.newTestRunner(PutSQL.class); + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate("CREATE TABLE TIMETESTS (id integer primary key, ts1 time, ts2 time)"); + } + } + + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + runner.setProperty(PutSQL.CONNECTION_POOL, "dbcp"); + + final String arg2TS = "00:01:01"; + final String art3TS = "12:02:02"; + final String timeFormatString = "HH:mm:ss"; + SimpleDateFormat dateFormat = new SimpleDateFormat(timeFormatString); + java.util.Date parsedDate = dateFormat.parse(arg2TS); + + final Map attributes = new HashMap<>(); + attributes.put("sql.args.1.type", String.valueOf(Types.TIME)); + attributes.put("sql.args.1.value", Long.toString(parsedDate.getTime())); + attributes.put("sql.args.1.format", timeFormatString); + attributes.put("sql.args.2.type", String.valueOf(Types.TIME)); + attributes.put("sql.args.2.value", art3TS); + attributes.put("sql.args.2.format", timeFormatString); + + runner.enqueue("INSERT INTO TIMETESTS (ID, ts1, ts2) VALUES (1, ?, ?)".getBytes(), attributes); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutSQL.REL_SUCCESS, 1); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + final ResultSet rs = stmt.executeQuery("SELECT * FROM TIMETESTS"); + assertTrue(rs.next()); + assertEquals(1, rs.getInt(1)); + assertEquals(arg2TS, rs.getString(2)); + assertEquals(art3TS, rs.getString(3)); + assertFalse(rs.next()); + } + } + } + + @Test + public void testUsingDateValuesEpochAndString() throws InitializationException, ProcessException, SQLException, IOException, ParseException { + final TestRunner runner = TestRunners.newTestRunner(PutSQL.class); + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + stmt.executeUpdate("CREATE TABLE DATETESTS (id integer primary key, ts1 date, ts2 date)"); + } + } + + runner.addControllerService("dbcp", service); + runner.enableControllerService(service); + runner.setProperty(PutSQL.CONNECTION_POOL, "dbcp"); + + final String arg2TS = "2001-01-01"; + final String art3TS = "2002-02-02"; + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + java.util.Date parsedDate = dateFormat.parse(arg2TS); + + final Map attributes = new HashMap<>(); + attributes.put("sql.args.1.type", String.valueOf(Types.DATE)); + attributes.put("sql.args.1.value", Long.toString(parsedDate.getTime())); + attributes.put("sql.args.2.type", String.valueOf(Types.DATE)); + attributes.put("sql.args.2.value", art3TS); + + runner.enqueue("INSERT INTO DATETESTS (ID, ts1, ts2) VALUES (1, ?, ?)".getBytes(), attributes); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutSQL.REL_SUCCESS, 1); + + try (final Connection conn = service.getConnection()) { + try (final Statement stmt = conn.createStatement()) { + final ResultSet rs = stmt.executeQuery("SELECT * FROM DATETESTS"); + assertTrue(rs.next()); + assertEquals(1, rs.getInt(1)); + assertEquals(arg2TS, rs.getString(2)); + assertEquals(art3TS, rs.getString(3)); + assertFalse(rs.next()); + } + } + } + @Test public void testBinaryColumnTypes() throws InitializationException, ProcessException, SQLException, IOException, ParseException { final TestRunner runner = TestRunners.newTestRunner(PutSQL.class); From 9569e91adceac17d7c790605f85b0db0679d4508 Mon Sep 17 00:00:00 2001 From: patricker Date: Thu, 20 Oct 2016 09:56:22 -0600 Subject: [PATCH 26/26] NIFI-2918 JDBC getColumnName may return empty string --- .../processors/standard/util/JdbcCommon.java | 3 ++- .../standard/util/TestJdbcCommon.java | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JdbcCommon.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JdbcCommon.java index 0aa4c60e375e..f19f7e0bcd3f 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JdbcCommon.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JdbcCommon.java @@ -258,7 +258,8 @@ public static Schema createSchema(final ResultSet rs, String recordName, boolean * Some missing Avro types - Decimal, Date types. May need some additional work. */ for (int i = 1; i <= nrOfColumns; i++) { - String columnName = convertNames ? normalizeNameForAvro(meta.getColumnName(i)) : meta.getColumnName(i); + String nameOrLabel = StringUtils.isNotEmpty(meta.getColumnName(i)) ? meta.getColumnName(i) : meta.getColumnLabel(i); + String columnName = convertNames ? normalizeNameForAvro(nameOrLabel) : nameOrLabel; switch (meta.getColumnType(i)) { case CHAR: case LONGNVARCHAR: diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestJdbcCommon.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestJdbcCommon.java index dd375aa7f703..1e0fe2fd54bd 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestJdbcCommon.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestJdbcCommon.java @@ -150,6 +150,31 @@ public void testCreateSchemaNoTableName() throws ClassNotFoundException, SQLExce } + @Test + public void testCreateSchemaOnlyColumnLabel() throws ClassNotFoundException, SQLException { + + final ResultSet resultSet = mock(ResultSet.class); + final ResultSetMetaData resultSetMetaData = mock(ResultSetMetaData.class); + when(resultSet.getMetaData()).thenReturn(resultSetMetaData); + when(resultSetMetaData.getColumnCount()).thenReturn(2); + when(resultSetMetaData.getTableName(1)).thenReturn("TEST"); + when(resultSetMetaData.getColumnType(1)).thenReturn(Types.INTEGER); + when(resultSetMetaData.getColumnName(1)).thenReturn(""); + when(resultSetMetaData.getColumnLabel(1)).thenReturn("ID"); + when(resultSetMetaData.getColumnType(2)).thenReturn(Types.VARCHAR); + when(resultSetMetaData.getColumnName(2)).thenReturn("VCHARC"); + when(resultSetMetaData.getColumnLabel(2)).thenReturn("NOT_VCHARC"); + + final Schema schema = JdbcCommon.createSchema(resultSet); + assertNotNull(schema); + + assertNotNull(schema.getField("ID")); + assertNotNull(schema.getField("VCHARC")); + + // records name, should be result set first column table name + assertEquals("TEST", schema.getName()); + } + @Test public void testConvertToBytes() throws ClassNotFoundException, SQLException, IOException { final Statement st = con.createStatement();