From 7e9f08c362d24a2e7a32807d9286e2658fa7858b Mon Sep 17 00:00:00 2001 From: Oleg Smelov Date: Tue, 30 Aug 2022 15:55:31 +0400 Subject: [PATCH] configuration parameter for header publication --- README.md | 5 ++- .../com/exactpro/th2/codec/csv/CsvCodec.java | 32 ++++++--------- .../codec/csv/cfg/CsvCodecConfiguration.java | 14 ++++++- .../exactpro/th2/codec/csv/TestCsvCodec.java | 39 +++++++++++++++++-- 4 files changed, 65 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 2a85fc6..0044210 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ delimiter: ',' encoding: UTF-8 display-name: CodecCsv validate-length: true +publish-header: false ``` **default-header** - the default header for this codec. It will be used if no header found in the received batch. @@ -75,7 +76,9 @@ validate-length: true **display-name** - the name to set in the root event sent to the event store. All errors during decoding will be attached to that root event. The default value for the name is `CodecCsv`. -**validate-length** - check if csv have different count of values against header's count +**validate-length** - check if csv have different count of values against header's count. + +**publish-header** - set to publish decoded header. The default value is `false`. ## Full configuration example diff --git a/src/main/java/com/exactpro/th2/codec/csv/CsvCodec.java b/src/main/java/com/exactpro/th2/codec/csv/CsvCodec.java index c5b699e..140f3d9 100644 --- a/src/main/java/com/exactpro/th2/codec/csv/CsvCodec.java +++ b/src/main/java/com/exactpro/th2/codec/csv/CsvCodec.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2020 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,32 +32,21 @@ import org.slf4j.LoggerFactory; import com.csvreader.CsvReader; -import com.exactpro.th2.codec.CodecException; import com.exactpro.th2.codec.DecodeException; import com.exactpro.th2.codec.api.IPipelineCodec; import com.exactpro.th2.codec.api.IReportingContext; import com.exactpro.th2.codec.csv.cfg.CsvCodecConfiguration; -import com.exactpro.th2.codec.util.MessageUtilKt; -import com.exactpro.th2.common.event.Event; -import com.exactpro.th2.common.event.Event.Status; -import com.exactpro.th2.common.event.EventUtils; import com.exactpro.th2.common.grpc.AnyMessage; -import com.exactpro.th2.common.grpc.EventBatch; -import com.exactpro.th2.common.grpc.EventID; import com.exactpro.th2.common.grpc.Message; import com.exactpro.th2.common.grpc.Message.Builder; import com.exactpro.th2.common.grpc.MessageGroup; -import com.exactpro.th2.common.grpc.MessageGroupBatch; import com.exactpro.th2.common.grpc.MessageID; import com.exactpro.th2.common.grpc.MessageMetadata; import com.exactpro.th2.common.grpc.RawMessage; import com.exactpro.th2.common.grpc.RawMessageMetadata; import com.exactpro.th2.common.message.MessageUtils; -import com.exactpro.th2.common.schema.message.MessageListener; -import com.exactpro.th2.common.schema.message.MessageRouter; import com.exactpro.th2.common.value.ValueUtils; import com.google.protobuf.ByteString; -import com.google.protobuf.TextFormat; public class CsvCodec implements IPipelineCodec { @@ -168,12 +157,16 @@ private void decodeCsvData(Collection errors, MessageGroup.Builder LOGGER.debug("Set header to: " + Arrays.toString(strings)); } header = strings; - AnyMessage.Builder messageBuilder = groupBuilder.addMessagesBuilder(); - Builder headerMsg = Message.newBuilder(); - // Not set message type - setMetadata(originalMetadata, headerMsg, HEADER_MSG_TYPE, currentIndex); - headerMsg.putFields(HEADER_FIELD_NAME, ValueUtils.toValue(strings)); - messageBuilder.setMessage(headerMsg); + + if (configuration.isPublishHeader()) { + AnyMessage.Builder messageBuilder = groupBuilder.addMessagesBuilder(); + Builder headerMsg = Message.newBuilder(); + // Not set message type + setMetadata(originalMetadata, headerMsg, HEADER_MSG_TYPE, currentIndex); + headerMsg.putFields(HEADER_FIELD_NAME, ValueUtils.toValue(strings)); + messageBuilder.setMessage(headerMsg); + } + continue; } @@ -207,7 +200,6 @@ private void decodeCsvData(Collection errors, MessageGroup.Builder } } - messageBuilder.setMessage(builder); } } @@ -279,4 +271,4 @@ private void trimEachElement(String[] elements) { elements[i] = element.trim(); } } -} +} \ No newline at end of file diff --git a/src/main/java/com/exactpro/th2/codec/csv/cfg/CsvCodecConfiguration.java b/src/main/java/com/exactpro/th2/codec/csv/cfg/CsvCodecConfiguration.java index 729223f..f1afa3d 100644 --- a/src/main/java/com/exactpro/th2/codec/csv/cfg/CsvCodecConfiguration.java +++ b/src/main/java/com/exactpro/th2/codec/csv/cfg/CsvCodecConfiguration.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2020 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -44,6 +44,10 @@ public class CsvCodecConfiguration implements IPipelineCodecSettings { @JsonPropertyDescription("Set to validate length of columns or not") private boolean validateLength = true; + @JsonProperty("publish-header") + @JsonPropertyDescription("Set to enable header publication") + private boolean publishHeader = false; + public List getDefaultHeader() { return defaultHeader; } @@ -83,4 +87,12 @@ public boolean getValidateLength() { public void setValidateLength(boolean validateLength) { this.validateLength = validateLength; } + + public boolean isPublishHeader() { + return publishHeader; + } + + public void setPublishHeader(boolean publishHeader) { + this.publishHeader = publishHeader; + } } diff --git a/src/test/java/com/exactpro/th2/codec/csv/TestCsvCodec.java b/src/test/java/com/exactpro/th2/codec/csv/TestCsvCodec.java index b0611be..c474b7c 100644 --- a/src/test/java/com/exactpro/th2/codec/csv/TestCsvCodec.java +++ b/src/test/java/com/exactpro/th2/codec/csv/TestCsvCodec.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2020 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2022 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -53,6 +53,7 @@ class TestPositive { void decodeArrayWithDifferentLength() throws IOException { CsvCodecConfiguration configuration = new CsvCodecConfiguration(); configuration.setValidateLength(false); + configuration.setPublishHeader(true); CsvCodec codec = createCodec(configuration); MessageGroup group = MessageGroup.newBuilder() .addMessages(createCsvMessage("A,B, , ,", "1,2,3,4")) @@ -95,6 +96,7 @@ void decodeArrayWithDifferentLength() throws IOException { void decodeArrayInEnd() throws IOException { CsvCodecConfiguration configuration = new CsvCodecConfiguration(); configuration.setValidateLength(false); + configuration.setPublishHeader(true); CsvCodec codec = createCodec(configuration); MessageGroup group = MessageGroup.newBuilder() .addMessages(createCsvMessage("A,B,C ,", "1,2,3")) @@ -176,7 +178,7 @@ void decodeArrayInMiddle() throws IOException { void decodesDataAndSkipsHeader() { CsvCodec codec = createCodec(); MessageGroup group = MessageGroup.newBuilder() - .addMessages(createCsvMessage("A,B,C", "1,2,3")) + .addMessages(createCsvMessage("A,B,C", "1,2,3")) .build(); MessageGroup value = codec.decode(group); @@ -208,6 +210,34 @@ void decodesDataAndSkipsHeader() { ); } + @Test + void skipsHeaderPublishing() { + final var config = new CsvCodecConfiguration(); + config.setPublishHeader(false); + CsvCodec codec = createCodec(config); + MessageGroup group = MessageGroup.newBuilder() + .addMessages(createCsvMessage("A,B,C", "1,2,3")) + .build(); + + MessageGroup value = codec.decode(group); + assertEquals(1, value.getMessagesCount()); + + Message message = getMessage(value, 0); + assertFieldCount(3, message); + + assertAll( + () -> assertAll("Current message: " + message, + () -> { + assertEquals(1, message.getMetadata().getId().getSubsequenceCount()); + assertEquals(2, message.getMetadata().getId().getSubsequence(0)); + }, + () -> assertEquals("1", getFieldValue(message, "A", () -> "No field A. " + message)), + () -> assertEquals("2", getFieldValue(message, "B", () -> "No field B. " + message)), + () -> assertEquals("3", getFieldValue(message, "C", () -> "No field C. " + message)) + ) + ); + } + @Test void trimsEndOfTheLine() { CsvCodec codec = createCodec(); @@ -311,6 +341,7 @@ void decodesDataWithEscapedCharacters() { void decodesDataCustomDelimiter() { CsvCodecConfiguration configuration = new CsvCodecConfiguration(); configuration.setDelimiter(';'); + configuration.setPublishHeader(true); CsvCodec codec = createCodec(configuration); MessageGroup group = MessageGroup.newBuilder() @@ -420,7 +451,9 @@ void reportsErrorIfDefaultHeaderAndDataHaveDifferentSize() { } private CsvCodec createCodec() { - return createCodec(new CsvCodecConfiguration()); + final var configuration = new CsvCodecConfiguration(); + configuration.setPublishHeader(true); + return createCodec(configuration); } private CsvCodec createCodec(CsvCodecConfiguration configuration) {