Added flatpack component support #1541

docs/modules/ROOT/pages/reference/extensions/flatpack.adoc

@@ -28,3 +28,22 @@ Please refer to the above links for usage and configuration details.
 ----
 
 Check the xref:user-guide/index.adoc[User guide] for more information about writing Camel Quarkus applications.
+
+== Additional Camel Quarkus configuration
+
+Beyond standard usages described above, a trick is needed when using flatpack mappings from classpath resources in native mode. In such a situation, one needs to explicitly embed the resources in the native executable by specifying the `include-patterns` option.
+
+For instance, the route below would load the flatpack mapping from a classpath resource named _mappings/simple.pzmap.xml_:
+[source,java]
+----
+from("direct:start").to("flatpack:delim:mappings/simple.pzmap.xml");
+----
+
+In order to work in native mode the `include-patterns` configuration should be set. For instance, in the `application.properties` file as below :
+[source,properties]
+----
+quarkus.camel.native.resources.include-patterns = mappings/*.pzmap.xml
+----
+
+More information about selecting resources for inclusion in the native executable could be found at xref:user-guide/native-mode.adoc#embedding-resource-in-native-executable[Embedding resource in native executable].
+

extensions/flatpack/deployment/src/main/java/org/apache/camel/quarkus/component/flatpack/deployment/FlatpackProcessor.java

@@ -19,6 +19,8 @@
 import io.quarkus.deployment.annotations.BuildStep;
 import io.quarkus.deployment.builditem.FeatureBuildItem;
 import io.quarkus.deployment.builditem.nativeimage.NativeImageResourceBuildItem;
+import io.quarkus.deployment.builditem.nativeimage.ReflectiveClassBuildItem;
+import org.apache.camel.component.flatpack.FlatpackException;
 
 class FlatpackProcessor {
 
@@ -33,4 +35,9 @@ FeatureBuildItem feature() {
     NativeImageResourceBuildItem registerNativeImageResources() {
         return new NativeImageResourceBuildItem("net/sf/flatpack/xml/flatpack.dtd", "fpconvert.properties");
     }
+
+    @BuildStep
+    ReflectiveClassBuildItem registerReflectiveClasses() {
+        return new ReflectiveClassBuildItem(false, false, FlatpackException.class);
+    }
 }

extensions/flatpack/runtime/src/main/doc/configuration.adoc

@@ -0,0 +1,15 @@
+Beyond standard usages described above, a trick is needed when using flatpack mappings from classpath resources in native mode. In such a situation, one needs to explicitly embed the resources in the native executable by specifying the `include-patterns` option.
+
+For instance, the route below would load the flatpack mapping from a classpath resource named _mappings/simple.pzmap.xml_:
+[source,java]
+----
+from("direct:start").to("flatpack:delim:mappings/simple.pzmap.xml");
+----
+
+In order to work in native mode the `include-patterns` configuration should be set. For instance, in the `application.properties` file as below :
+[source,properties]
+----
+quarkus.camel.native.resources.include-patterns = mappings/*.pzmap.xml
+----
+
+More information about selecting resources for inclusion in the native executable could be found at xref:user-guide/native-mode.adoc#embedding-resource-in-native-executable[Embedding resource in native executable].

integration-tests/flatpack/pom.xml

@@ -39,7 +39,7 @@
         <!-- explicit dependencies of this module in the Maven sense, although they are required by the Quarkus Maven plugin. -->
         <!-- Please update the rule whenever you change the dependencies of this module by running -->
         <!--     mvn process-resources -Pformat    from the root directory -->
-        <mvnd.builder.rule>camel-quarkus-direct-deployment,camel-quarkus-flatpack-deployment,camel-quarkus-support-policy-deployment</mvnd.builder.rule>
+        <mvnd.builder.rule>camel-quarkus-bean-deployment,camel-quarkus-direct-deployment,camel-quarkus-flatpack-deployment,camel-quarkus-support-policy-deployment</mvnd.builder.rule>
     </properties>
 
     <dependencyManagement>
@@ -55,6 +55,10 @@
     </dependencyManagement>
 
     <dependencies>
+        <dependency>
+            <groupId>org.apache.camel.quarkus</groupId>
+            <artifactId>camel-quarkus-bean</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.apache.camel.quarkus</groupId>
             <artifactId>camel-quarkus-direct</artifactId>

integration-tests/flatpack/src/main/java/org/apache/camel/quarkus/component/flatpack/it/FlatPackRoutes.java

@@ -16,26 +16,69 @@
  */
 package org.apache.camel.quarkus.component.flatpack.it;
 
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import io.quarkus.runtime.annotations.RegisterForReflection;
+import org.apache.camel.Exchange;
 import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.component.flatpack.FlatpackException;
 import org.apache.camel.dataformat.flatpack.FlatpackDataFormat;
 
 public class FlatPackRoutes extends RouteBuilder {
 
     @Override
     public void configure() {
+        FlatpackRowStore flatpackRowStore = new FlatpackRowStore();
+        bindToRegistry("flatpackRowStore", flatpackRowStore);
+
+        onException(FlatpackException.class).maximumRedeliveries(1).handled(true).process(e -> {
+            FlatpackException cause = e.getProperty(Exchange.EXCEPTION_CAUGHT, FlatpackException.class);
+            e.getMessage().setBody(cause.getMessage());
+        });
+
         // Testing delimited data format
         FlatpackDataFormat delimitedDataFormat = new FlatpackDataFormat();
-        delimitedDataFormat.setDefinition("mappings/INVENTORY-Delimited.pzmap.xml");
+        delimitedDataFormat.setDefinition("INVENTORY-Delimited.pzmap.xml");
         from("direct:delimited-unmarshal").unmarshal(delimitedDataFormat);
         from("direct:delimited-marshal").marshal(delimitedDataFormat);
 
         // Testing fixed length data format
         FlatpackDataFormat fixedLengthDataFormat = new FlatpackDataFormat();
-        fixedLengthDataFormat.setDefinition("mappings/PEOPLE-FixedLength.pzmap.xml");
+        fixedLengthDataFormat.setDefinition("PEOPLE-FixedLength.pzmap.xml");
         fixedLengthDataFormat.setFixed(true);
         fixedLengthDataFormat.setIgnoreFirstRecord(false);
         from("direct:fixed-length-unmarshal").unmarshal(fixedLengthDataFormat);
         from("direct:fixed-length-marshal").marshal(fixedLengthDataFormat);
+
+        from("direct:delimited").to("flatpack:delim:INVENTORY-Delimited.pzmap.xml").bean(flatpackRowStore, "flush");
+        from("flatpack:delim:INVENTORY-Delimited.pzmap.xml").bean(flatpackRowStore, "store");
+
+        from("direct:fixed").to("flatpack:fixed:PEOPLE-FixedLength.pzmap.xml").bean(flatpackRowStore, "flush");
+        from("flatpack:fixed:PEOPLE-FixedLength.pzmap.xml").bean(flatpackRowStore, "store");
+
+        from("direct:header-and-trailer").to("flatpack:fixed:PEOPLE-HeaderAndTrailer.pzmap.xml").bean(flatpackRowStore,
+                "flush");
+        from("flatpack:fixed:PEOPLE-HeaderAndTrailer.pzmap.xml").bean(flatpackRowStore, "store");
+
+        from("direct:no-descriptor").to("flatpack:foo").bean(flatpackRowStore, "flush");
+        from("flatpack:foo").bean(flatpackRowStore, "store");
+    }
+
+    @RegisterForReflection
+    public static class FlatpackRowStore {
+        List<Map<?, ?>> rows = new LinkedList<>();
+
+        public void store(Map<?, ?> m) {
+            rows.add(m);
+        }
+
+        public List<Map<?, ?>> flush() {
+            List<Map<?, ?>> current = rows;
+            rows = new LinkedList<>();
+            return current;
+        }
     }
 
 }

integration-tests/flatpack/src/main/java/org/apache/camel/quarkus/component/flatpack/it/FlatpackResource.java

@@ -16,6 +16,7 @@
  */
 package org.apache.camel.quarkus.component.flatpack.it;
 
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 
@@ -78,4 +79,49 @@ public String fixedLengthMarshal(List<Map<String, String>> object) {
         return producerTemplate.requestBody("direct:fixed-length-marshal", object, String.class);
     }
 
+    @Path("/delimited")
+    @GET
+    @Consumes(MediaType.TEXT_PLAIN)
+    @Produces(MediaType.APPLICATION_JSON)
+    public LinkedList<?> delimited(String data) {
+        LOG.infof("Invoking delimited with data: %s", data);
+        return producerTemplate.requestBody("direct:delimited", data, LinkedList.class);
+    }
+
+    @Path("/fixed")
+    @GET
+    @Consumes(MediaType.TEXT_PLAIN)
+    @Produces(MediaType.APPLICATION_JSON)
+    public LinkedList<?> fixed(String data) {
+        LOG.infof("Invoking fixed with data: %s", data);
+        return producerTemplate.requestBody("direct:fixed", data, LinkedList.class);
+    }
+
+    @Path("/headerAndTrailer")
+    @GET
+    @Consumes(MediaType.TEXT_PLAIN)
+    @Produces(MediaType.APPLICATION_JSON)
+    public LinkedList<?> headerAndTrailer(String data) {
+        LOG.infof("Invoking headerAndTrailer with data: %s", data);
+        return producerTemplate.requestBody("direct:header-and-trailer", data, LinkedList.class);
+    }
+
+    @Path("/noDescriptor")
+    @GET
+    @Consumes(MediaType.TEXT_PLAIN)
+    @Produces(MediaType.APPLICATION_JSON)
+    public LinkedList<?> noDescriptor(String data) {
+        LOG.infof("Invoking noDescriptor with data: %s", data);
+        return producerTemplate.requestBody("direct:no-descriptor", data, LinkedList.class);
+    }
+
+    @Path("/invalid")
+    @GET
+    @Consumes(MediaType.TEXT_PLAIN)
+    @Produces(MediaType.TEXT_PLAIN)
+    public String invalid(String data) {
+        LOG.infof("Invoking invalid with data: %s", data);
+        return producerTemplate.requestBody("direct:fixed", data, String.class);
+    }
+
 }

integration-tests/flatpack/src/main/resources/PEOPLE-HeaderAndTrailer.pzmap.xml

@@ -0,0 +1,40 @@
+<?xml version='1.0'?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+-->
+<!DOCTYPE PZMAP SYSTEM "flatpack.dtd" >
+
+<PZMAP>
+    <RECORD id="header" startPosition="1" endPosition="3" indicator="HBT">
+        <COLUMN name="INDICATOR" length="3"/>
+        <COLUMN name="DATE" length="8"/>
+    </RECORD>
+
+    <COLUMN name="FIRSTNAME" length="35" />
+    <COLUMN name="LASTNAME" length="35" />
+    <COLUMN name="ADDRESS" length="100" />
+    <COLUMN name="CITY" length="100" />
+    <COLUMN name="STATE" length="2" />
+    <COLUMN name="ZIP" length="5" />
+
+    <RECORD id="trailer" startPosition="1" endPosition="3" indicator="FBT">
+        <COLUMN name="INDICATOR" length="3"/>
+        <COLUMN name="STATUS" length="7"/>
+    </RECORD>
+
+</PZMAP>

integration-tests/flatpack/src/main/resources/application.properties

@@ -14,4 +14,4 @@
 ## See the License for the specific language governing permissions and
 ## limitations under the License.
 ## ---------------------------------------------------------------------------
-quarkus.camel.native.resources.include-patterns = mappings/*.pzmap.xml
+quarkus.camel.native.resources.include-patterns = *.pzmap.xml

integration-tests/flatpack/src/test/java/org/apache/camel/quarkus/component/flatpack/it/FlatpackTest.java

@@ -29,15 +29,18 @@
 import org.junit.jupiter.api.Test;
 
 import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.startsWith;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 
 @QuarkusTest
 class FlatpackTest {
 
     @Test
     public void delimitedUnmarshalShouldSucceed() throws IOException {
-        String data = IOUtils.toString(getClass().getResourceAsStream("/delim/INVENTORY-CommaDelimitedWithQualifier.txt"),
+        String data = IOUtils.toString(getClass().getResourceAsStream("/INVENTORY-CommaDelimitedWithQualifier.txt"),
                 StandardCharsets.UTF_8);
 
         given().body(data).when().get("/flatpack/delimited-unmarshal").then().statusCode(200).body(is("4-SOME VALVE"));
@@ -67,7 +70,7 @@ public void delimitedMarshalShouldSucceed() {
 
     @Test
     public void fixedLengthUnmarshalShouldSucceed() throws IOException {
-        String data = IOUtils.toString(getClass().getResourceAsStream("/fixed/PEOPLE-FixedLength.txt"), StandardCharsets.UTF_8);
+        String data = IOUtils.toString(getClass().getResourceAsStream("/PEOPLE-FixedLength.txt"), StandardCharsets.UTF_8);
         given().body(data).when().get("/flatpack/fixed-length-unmarshal").then().statusCode(200).body(is("4-JOHN"));
     }
 
@@ -87,4 +90,100 @@ public void fixedLengthMarshalShouldSucceed() {
                 .body(startsWith("JOHN                               DOE"));
     }
 
+    @Test
+    @SuppressWarnings("unchecked")
+    public void delimitedShouldSucceed() throws IOException {
+        String data = IOUtils.toString(getClass().getResourceAsStream("/INVENTORY-CommaDelimitedWithQualifier.txt"),
+                StandardCharsets.UTF_8);
+        Map<String, String>[] rows = given().body(data).when().get("/flatpack/delimited").then().statusCode(200).extract()
+                .as(Map[].class);
+        assertNotNull(rows);
+        assertEquals(4, rows.length);
+        assertNotNull(rows[0]);
+        assertEquals("SOME VALVE", rows[0].get("ITEM_DESC"));
+        assertNotNull(rows[1]);
+        assertEquals("AN ENGINE", rows[1].get("ITEM_DESC"));
+        assertNotNull(rows[2]);
+        assertEquals("A BELT", rows[2].get("ITEM_DESC"));
+        assertNotNull(rows[3]);
+        assertEquals("A BOLT", rows[3].get("ITEM_DESC"));
+    }
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void fixedShouldSucceed() throws IOException {
+        String data = IOUtils.toString(getClass().getResourceAsStream("/PEOPLE-FixedLength.txt"), StandardCharsets.UTF_8);
+        Map<String, String>[] rows = given().body(data).when().get("/flatpack/fixed").then().statusCode(200).extract()
+                .as(Map[].class);
+        assertNotNull(rows);
+        assertEquals(4, rows.length);
+        assertNotNull(rows[0]);
+        assertEquals("JOHN", rows[0].get("FIRSTNAME"));
+        assertNotNull(rows[1]);
+        assertEquals("JIMMY", rows[1].get("FIRSTNAME"));
+        assertNotNull(rows[2]);
+        assertEquals("JANE", rows[2].get("FIRSTNAME"));
+        assertNotNull(rows[3]);
+        assertEquals("FRED", rows[3].get("FIRSTNAME"));
+    }
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void fixedHeaderAndTrailerShouldSucceed() throws IOException {
+        String data = IOUtils.toString(getClass().getResourceAsStream("/PEOPLE-HeaderAndTrailer.txt"),
+                StandardCharsets.UTF_8);
+        Map<String, String>[] rows = given().body(data).when().get("/flatpack/headerAndTrailer").then().statusCode(200)
+                .extract().as(Map[].class);
+        assertNotNull(rows);
+        assertEquals(6, rows.length);
+
+        // Assert Flatpack Header
+        assertNotNull(rows[0]);
+        assertEquals("HBT", rows[0].get("INDICATOR"));
+        assertEquals("20080817", rows[0].get("DATE"));
+
+        // Assert Flatpack Main Body
+        assertNotNull(rows[1]);
+        assertEquals("JOHN", rows[1].get("FIRSTNAME"));
+        assertNotNull(rows[2]);
+        assertEquals("JIMMY", rows[2].get("FIRSTNAME"));
+        assertNotNull(rows[3]);
+        assertEquals("JANE", rows[3].get("FIRSTNAME"));
+        assertNotNull(rows[4]);
+        assertEquals("FRED", rows[4].get("FIRSTNAME"));
+
+        // Assert Flatpack Trailer
+        assertNotNull(rows[5]);
+        assertEquals("FBT", rows[5].get("INDICATOR"));
+        assertEquals("SUCCESS", rows[5].get("STATUS"));
+    }
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void noDescriptorShouldSucceed() throws IOException {
+        String data = IOUtils.toString(getClass().getResourceAsStream("/foo.csv"), StandardCharsets.UTF_8);
+        Map<String, String>[] rows = given().body(data).when().get("/flatpack/noDescriptor").then().statusCode(200).extract()
+                .as(Map[].class);
+        assertNotNull(rows);
+        assertEquals(4, rows.length);
+
+        assertNotNull(rows[0]);
+        assertEquals("James", rows[0].get("NAME"));
+        assertNotNull(rows[1]);
+        assertEquals("Claus", rows[1].get("NAME"));
+        assertNotNull(rows[2]);
+        assertEquals("Antoine", rows[2].get("NAME"));
+        assertNotNull(rows[3]);
+        assertEquals("Xavier", rows[3].get("NAME"));
+    }
+
+    @Test
+    public void invalidShouldFail() throws IOException {
+        String data = IOUtils.toString(getClass().getResourceAsStream("/PEOPLE-FixedLength-Invalid.txt"),
+                StandardCharsets.UTF_8);
+        given().body(data).when().get("/flatpack/invalid").then().statusCode(200)
+                .body(containsString("Flatpack has found 4 errors while parsing."))
+                .body(containsString("Line:4 Level:2 Desc:LINE TOO LONG. LINE IS 278 LONG. SHOULD BE 277"));
+    }
+
 }

integration-tests/flatpack/src/test/resources/PEOPLE-FixedLength-Invalid.txt

@@ -0,0 +1,4 @@
+JOHN-LONG                          DOE                                1234 CIRCLE CT                                                                                      ELYRIA                                                                                              OH44035*
+JIMMY-LONG                         SMITH                              180 SOME ST                                                                                         AVON                                                                                                OH44011*
+JANE-LONG                          DOE                                111 MILKY WY                                                                                        AMHERST                                                                                             OH44001*
+FRED-LONG                          FLINTSTONE                         123 ROCKY WY                                                                                        BEDROCK                                                                                             AZ12345*

integration-tests/flatpack/src/test/resources/PEOPLE-HeaderAndTrailer.txt

@@ -0,0 +1,6 @@
+HBT20080817
+JOHN                               DOE                                1234 CIRCLE CT                                                                                      ELYRIA                                                                                              OH44035
+JIMMY                              SMITH                              180 SOME ST                                                                                         AVON                                                                                                OH44011
+JANE                               DOE                                111 MILKY WY                                                                                        AMHERST                                                                                             OH44001
+FRED                               FLINTSTONE                         123 ROCKY WY                                                                                        BEDROCK                                                                                             AZ12345
+FBTSUCCESS