[ISSUE #10832] fix(remoting): reject malformed command frames

remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java

@@ -190,13 +190,21 @@ public static RemotingCommand decode(final ByteBuffer byteBuffer) throws Remotin
 
     public static RemotingCommand decode(final ByteBuf byteBuffer) throws RemotingCommandException {
         int length = byteBuffer.readableBytes();
+        if (length < 4) {
+            throw new RemotingCommandException("decode error, frame is too short: " + length);
+        }
         int oriHeaderLen = byteBuffer.readInt();
         int headerLength = getHeaderLength(oriHeaderLen);
         if (headerLength > length - 4) {
             throw new RemotingCommandException("decode error, bad header length: " + headerLength);
         }
 
-        RemotingCommand cmd = headerDecode(byteBuffer, headerLength, getProtocolType(oriHeaderLen));
+        SerializeType protocolType = getProtocolType(oriHeaderLen);
+        if (protocolType == null) {
+            throw new RemotingCommandException("decode error, unknown serialize type: "
+                + ((oriHeaderLen >> 24) & 0xFF));
+        }
+        RemotingCommand cmd = headerDecode(byteBuffer, headerLength, protocolType);
 
         int bodyLength = length - 4 - headerLength;
         byte[] bodyData = null;

remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RemotingCommandTest.java

@@ -30,6 +30,18 @@
 import static org.assertj.core.api.Assertions.assertThat;
 
 public class RemotingCommandTest {
+    @Test
+    public void testDecodeRejectsShortFrame() {
+        Assert.assertThrows(RemotingCommandException.class,
+            () -> RemotingCommand.decode(new byte[] {0, 0, 0}));
+    }
+
+    @Test
+    public void testDecodeRejectsUnknownSerializeType() {
+        Assert.assertThrows(RemotingCommandException.class,
+            () -> RemotingCommand.decode(new byte[] {Byte.MAX_VALUE, 0, 0, 0}));
+    }
+
     @Test
     public void testMarkProtocolType_JSONProtocolType() {
         int source = 261;