IGNITE-28972 Generation must fail Message to byte[] marshalling

modules/codegen/src/main/java/org/apache/ignite/internal/MessageMarshallerGenerator.java

@@ -1156,18 +1156,82 @@ private enum MarshalledKind {
             return null;
         }
 
+        MarshalledKind res;
+
         if (map)
-            return MarshalledKind.MAP;
+            res = MarshalledKind.MAP;
+        else {
+            TypeMirror wire = requireEnclosed(enclosed, ann.value(), "@Marshalled").asType();
 
-        TypeMirror wire = requireEnclosed(enclosed, ann.value(), "@Marshalled").asType();
+            if (wire.getKind() == TypeKind.ARRAY) {
+                res = ((ArrayType)wire).getComponentType().getKind() == TypeKind.BYTE
+                    ? MarshalledKind.BLOB
+                    : MarshalledKind.ELEMENTS;
+            }
+            else
+                res = MarshalledKind.ELEMENT_BLOBS;
+        }
 
-        if (wire.getKind() == TypeKind.ARRAY) {
-            return ((ArrayType)wire).getComponentType().getKind() == TypeKind.BYTE
-                ? MarshalledKind.BLOB
-                : MarshalledKind.ELEMENTS;
+        /*
+         * Ensures that field annotated with {@link Marshalled} doesn't perform {@code Message} -> {@code byte[]} transformation
+         * which escapes {@link Order} and other rules implemented on top of communication {@code MessageWriter, MessageReader} logic.
+         */
+        if (messageToBytesTransformation(field.asType(), field, ann)) {
+            env.getMessager().printMessage(Diagnostic.Kind.ERROR,
+                "Message must be written by dedicated message serializers. " +
+                "Remove @" + Marshalled.class.getSimpleName() + " annotation and remove companion field " +
+                "and set @" + Order.class.getSimpleName(), field);
         }
 
-        return MarshalledKind.ELEMENT_BLOBS;
+        return res;
+    }
+
+    /**
+     * Recursively checks no {@code Message} -> {@code byte[]} transformation.
+     * @return {@code True} in case error transformation found.
+     */
+    private boolean messageToBytesTransformation(TypeMirror type, VariableElement field, Marshalled ann) {
+        if (assignableFrom(type, msgType))
+            return true;
+
+        if (isCollection(type)) {
+            DeclaredType colType = (DeclaredType)type;
+
+            List<? extends TypeMirror> typeArgs = colType.getTypeArguments();
+
+            if (typeArgs.size() != 1) {
+                env.getMessager().printMessage(Diagnostic.Kind.ERROR, "Raw collection not supported.", field);
+
+                return false;
+            }
+
+            return messageToBytesTransformation(typeArgs.get(0), field, ann);
+        }
+
+        if (type.getKind() == TypeKind.ARRAY)
+            return messageToBytesTransformation(((ArrayType)type).getComponentType(), field, ann);
+
+        if (isMap(type) && !ann.value().isEmpty()) {
+            DeclaredType mapType = (DeclaredType)type;
+
+            List<? extends TypeMirror> typeArgs = mapType.getTypeArguments();
+
+            if (typeArgs.size() != 2) {
+                env.getMessager().printMessage(Diagnostic.Kind.ERROR, "Raw Map not supported.", field);
+
+                return false;
+            }
+
+            TypeMirror keyType = typeArgs.get(0);
+            TypeMirror valType = typeArgs.get(1);
+
+            return assignableFrom(keyType, msgType)
+                || assignableFrom(valType, msgType)
+                || messageToBytesTransformation(keyType, field, ann)
+                || messageToBytesTransformation(valType, field, ann);
+        }
+
+        return false;
     }
 
     /** Returns the enclosed field named {@code name}, or throws if absent. */

modules/core/src/test/java/org/apache/ignite/internal/codegen/MessageProcessorTest.java

@@ -22,6 +22,8 @@
 import java.lang.reflect.Modifier;
 import java.net.URI;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -683,6 +685,53 @@ public void testNonMarshallableSelfMarshallingFailed() {
             "or SelfMarshallingMessage, nor declare @Marshalled fields");
     }
 
+    /** Test that {@code @Marshalled} annotation on {@link Message} field will fail generation. */
+    @Test
+    public void testMarshalledOnMessageFieldFailGeneration() {
+        List<String> cases = Arrays.asList(
+            "IncorrectMarshalledOnMessage.java",
+            "IncorrectMarshalledOnMessageCollection.java",
+            "IncorrectMarshalledOnMessageCollection2.java",
+            "IncorrectMarshalledOnMessageCollection3.java",
+            "IncorrectMarshalledOnMessageMap.java",
+            "IncorrectMarshalledOnMessageMap2.java",
+            "IncorrectMarshalledOnMessageMap3.java",
+            "IncorrectMarshalledOnMessageArray.java",
+            "IncorrectMarshalledOnMessageCollectionArray.java",
+            "IncorrectMarshalledOnMessageSet.java",
+            "IncorrectMarshalledOnMessageList.java",
+            "IncorrectMarshalledOnMessageList2.java"
+        );
+
+        for (String file : cases) {
+            Compilation compilation = compile("TestMessage.java", file);
+
+            assertThat(compilation).failed();
+            assertThat(compilation).hadErrorContaining("Message must be written by dedicated message serializers");
+        }
+    }
+
+    /** Test that {@code @Marshalled} annotation on raw {@link Collection} or {@link Map} fail generation. */
+    @Test
+    public void testRawClassesFailGeneration() {
+        List<String> cases = Arrays.asList(
+            "IncorrectRawListMessage.java",
+            "IncorrectRawCollectionMessage.java"
+        );
+
+        for (String file : cases) {
+            Compilation compilation = compile("TestMessage.java", file);
+
+            assertThat(compilation).failed();
+            assertThat(compilation).hadErrorContaining("Raw collection not supported");
+        }
+
+        Compilation compilation = compile("TestMessage.java", "IncorrectRawMapMessage.java");
+
+        assertThat(compilation).failed();
+        assertThat(compilation).hadErrorContaining("Raw Map not supported");
+    }
+
     /** */
     private Compilation compile(String... srcFiles) {
         return compile(new MessageProcessor(), srcFiles);

modules/core/src/test/resources/codegen/IncorrectMarshalledOnMessage.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import org.apache.ignite.plugin.extensions.communication.Message;
+
+/** */
+public class IncorrectMarshalledOnMessage implements Message {
+    /** */
+    @Marshalled("msgBytes")
+    TestMessage msg;
+
+    /** */
+    @Order(0)
+    byte[] msgBytes;
+}

modules/core/src/test/resources/codegen/IncorrectMarshalledOnMessageArray.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import org.apache.ignite.plugin.extensions.communication.Message;
+
+/** */
+public class IncorrectMarshalledOnMessageArray implements Message {
+    /** */
+    @Marshalled("msgBytes")
+    TestMessage[] msgArr;
+
+    /** */
+    @Order(0)
+    byte[] msgBytes;
+}

modules/core/src/test/resources/codegen/IncorrectMarshalledOnMessageCollection.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.Collection;
+import org.apache.ignite.plugin.extensions.communication.Message;
+
+/** */
+public class IncorrectMarshalledOnMessageCollection implements Message {
+    /** */
+    @Marshalled("msgBytes")
+    Collection<TestMessage> msgColl;
+
+    /** */
+    @Order(0)
+    byte[] msgBytes;
+}

modules/core/src/test/resources/codegen/IncorrectMarshalledOnMessageCollection2.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.Collection;
+import org.apache.ignite.plugin.extensions.communication.Message;
+
+/** */
+public class IncorrectMarshalledOnMessageCollection2 implements Message {
+    /** */
+    @Marshalled("msgBytes")
+    Collection<TestMessage> msgColl;
+
+    /** */
+    @Order(0)
+    Collection<byte[]> msgBytes;
+}

modules/core/src/test/resources/codegen/IncorrectMarshalledOnMessageCollection3.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.Collection;
+import org.apache.ignite.plugin.extensions.communication.Message;
+
+/** */
+public class IncorrectMarshalledOnMessageCollection3 implements Message {
+    /** */
+    @Marshalled("msgBytes")
+    Collection<Collection<TestMessage>> msgColl;
+
+    /** */
+    @Order(0)
+    byte[] msgBytes;
+}

modules/core/src/test/resources/codegen/IncorrectMarshalledOnMessageCollectionArray.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.Collection;
+import org.apache.ignite.plugin.extensions.communication.Message;
+
+/** */
+public class IncorrectMarshalledOnMessageCollectionArray implements Message {
+    /** */
+    @Marshalled("msgBytes")
+    Collection<TestMessage>[] msgArr;
+
+    /** */
+    @Order(0)
+    byte[] msgBytes;
+}

modules/core/src/test/resources/codegen/IncorrectMarshalledOnMessageList.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.List;
+import org.apache.ignite.plugin.extensions.communication.Message;
+
+/** */
+public class IncorrectMarshalledOnMessageList implements Message {
+    /** */
+    @Marshalled("msgBytes")
+    List<TestMessage> msgColl;
+
+    /** */
+    @Order(0)
+    byte[] msgBytes;
+}