Merge branch 'i2p.i2p.2.2.1-xor-messageIDs-as-interface-change' into …

…'master'

Patches all INMP.add() calls to use replay "Contexts" specific to where they were called from.

See merge request i2p-hackers/i2p.i2p!91

history.txt

@@ -1,3 +1,8 @@
+2023-05-29 idk
+ * adds "virtual contexts" to bloom filter, where each entity that
+  passes an i2np message to the bloom filter xor's the messageID with a random, local value.
+  credit Xe Iaso for discovering the issue, obscuratus for the solution
+
 2023-04-12 idk
  * Fix missing Java options in docker/rootfs/startapp.sh
  * Detect when running in Podman instead of regular Docker

router/java/src/net/i2p/data/i2np/I2NPMessage.java

@@ -87,7 +87,8 @@ public interface I2NPMessage {
     /**
      * Replay resistant message ID
      */
-    public long getUniqueId(); 
+    public long getUniqueId(long msgIDBloomXor); 
+    public long getUniqueId();
     public void setUniqueId(long id);
 
     /**

router/java/src/net/i2p/data/i2np/I2NPMessageImpl.java

@@ -180,6 +180,10 @@ public void writeBytes(OutputStream out) {
     /**
      * Replay resistant message Id
      */
+    public synchronized long getUniqueId(long msgIDBloomXor) {
+        return getUniqueId() ^ msgIDBloomXor;
+    }
+
     public synchronized long getUniqueId() {
         // Lazy initialization of value
         if (_uniqueId < 0) {

router/java/src/net/i2p/router/InNetMessagePool.java

@@ -120,9 +120,9 @@ public synchronized HandlerJobBuilder registerHandlerJobBuilder(int i2npMessageT
         return old;
     }
 
-    public int add(I2NPMessage messageBody, RouterIdentity fromRouter, Hash fromRouterHash) {
-        return add(messageBody, fromRouter, fromRouterHash, 0);
-    }
+    //public int add(I2NPMessage messageBody, RouterIdentity fromRouter, Hash fromRouterHash) {
+        //return add(messageBody, fromRouter, fromRouterHash, 0);
+    //}
 
     /**
      * Add a new message to the pool.
@@ -134,6 +134,7 @@ public int add(I2NPMessage messageBody, RouterIdentity fromRouter, Hash fromRout
      * @param messageBody non-null
      * @param fromRouter may be null
      * @param fromRouterHash may be null, calculated from fromRouter if null
+     * @param msgIDBloomXor constant value to XOR with the messageID before passing to the bloom filter.
      *
      * @return -1 for some types of errors but not all; 0 otherwise
      *         (was queue length, long ago)
@@ -150,6 +151,7 @@ public int add(I2NPMessage messageBody,
         if (_log.shouldDebug())
                 _log.debug("Rcvd" 
                           + " ID " + messageBody.getUniqueId()
+                          + " xor-ed ID " + messageBody.getUniqueId(msgIDBloomXor)
                           + " exp. " + new Date(exp)
                           + " type " + messageBody.getClass().getSimpleName());
 
@@ -165,27 +167,23 @@ public int add(I2NPMessage messageBody,
             // just validate the expiration
             invalidReason = _context.messageValidator().validateMessage(exp);
         } else {
-            if (msgIDBloomXor == 0)
-                invalidReason = _context.messageValidator().validateMessage(messageBody.getUniqueId(), exp);
-            else
-                invalidReason = _context.messageValidator().validateMessage(messageBody.getUniqueId()
-                                                                            ^ msgIDBloomXor, exp);
+            invalidReason = _context.messageValidator().validateMessage(messageBody.getUniqueId(msgIDBloomXor), exp);
         }
 
         if (invalidReason != null) {
             int level = Log.WARN;
             //if (messageBody instanceof TunnelCreateMessage)
             //    level = Log.INFO;
             if (_log.shouldLog(level))
-                _log.log(level, "Dropping message [" + messageBody.getUniqueId() 
+                _log.log(level, "Dropping message ID [" + messageBody.getUniqueId() + " xor-ed: " + messageBody.getUniqueId(msgIDBloomXor)
                           + " expiring on " + exp + "]: " + messageBody.getClass().getSimpleName() + ": " + invalidReason 
                           + ": " + messageBody);
             _context.statManager().addRateData("inNetPool.dropped", 1);
             // FIXME not necessarily a duplicate, could be expired too long ago / too far in future
             _context.statManager().addRateData("inNetPool.duplicate", 1);
             if (doHistory) {
                 history.droppedOtherMessage(messageBody, (fromRouter != null ? fromRouter.calculateHash() : fromRouterHash));
-                history.messageProcessingError(messageBody.getUniqueId(), 
+                history.messageProcessingError(messageBody.getUniqueId(msgIDBloomXor), 
                                                                 messageBody.getClass().getSimpleName(), 
                                                                 "Duplicate/expired");
             }
@@ -309,7 +307,7 @@ public int add(I2NPMessage messageBody,
                 } else {
                     if (doHistory) {
                         String mtype = messageBody.getClass().getName();
-                        history.receiveMessage(mtype, messageBody.getUniqueId(), 
+                        history.receiveMessage(mtype, messageBody.getUniqueId(msgIDBloomXor), 
                                                              messageBody.getMessageExpiration(), 
                                                              fromRouterHash, true);	
                     }
@@ -320,7 +318,7 @@ public int add(I2NPMessage messageBody,
 
         if (doHistory) {
             String mtype = messageBody.getClass().getName();
-            history.receiveMessage(mtype, messageBody.getUniqueId(), 
+            history.receiveMessage(mtype, messageBody.getUniqueId(msgIDBloomXor), 
                                                  messageBody.getMessageExpiration(), 
                                                  fromRouterHash, true);	
         }

router/java/src/net/i2p/router/TunnelPoolSettings.java

@@ -86,7 +86,7 @@ public class TunnelPoolSettings {
     private static final int MAX_PRIORITY = 25;
     private static final int EXPLORATORY_PRIORITY = 30;
 
-    private final long _msgIdBloomXor;
+    private final long _msgIDBloomXor;
 
     /**
      *  Exploratory tunnel
@@ -118,7 +118,7 @@ public TunnelPoolSettings(Hash dest, boolean isInbound) {
         _IPRestriction = DEFAULT_IP_RESTRICTION;
         _unknownOptions = new Properties();
         _randomKey = generateRandomKey();
-        _msgIdBloomXor = RandomSource.getInstance().nextLong();
+        _msgIDBloomXor = RandomSource.getInstance().nextLong();
 
         if (_isExploratory && !_isInbound)
             _priority = EXPLORATORY_PRIORITY;
@@ -290,7 +290,7 @@ public void setAliasOf(Hash h) {
      */
     public Properties getUnknownOptions() { return _unknownOptions; }
 
-    public long getMsgIdBloomXor() { return _msgIdBloomXor; }
+    public long getMsgIdBloomXor() { return _msgIDBloomXor; }
 
     /**
      *  Defaults in props are NOT honored.

router/java/src/net/i2p/router/dummy/VMCommSystem.java

@@ -165,7 +165,7 @@ else if (size <= 4096)
                 else
                     ReceiveJob.this.getContext().statManager().addRateData("transport.receiveMessageLarge", 1, 1);
 
-                _ctx.inNetMessagePool().add(msg, null, _from);
+                _ctx.inNetMessagePool().add(msg, null, _from, 0);
             } catch (I2NPMessageException e) {
                 _log.error("Error reading/formatting a VM message? Something is not right...", e);
             }

router/java/src/net/i2p/router/message/GarlicMessageHandler.java

@@ -8,6 +8,8 @@
  *
  */
 
+import java.util.Random;
+
 import net.i2p.data.Hash;
 import net.i2p.data.router.RouterIdentity;
 import net.i2p.data.i2np.GarlicMessage;
@@ -27,13 +29,26 @@
  */
 public class GarlicMessageHandler implements HandlerJobBuilder {
     private final RouterContext _context;
+    private final long _msgIDBloomXorLocal;
+    private final long _msgIDBloomXorRouter;
+    private final long _msgIDBloomXorTunnel;
 
     public GarlicMessageHandler(RouterContext context) {
         _context = context;
+        _msgIDBloomXorLocal = new Random().nextLong();
+        _msgIDBloomXorRouter = new Random().nextLong();
+        _msgIDBloomXorTunnel = new Random().nextLong();
+    }
+
+    public GarlicMessageHandler(RouterContext context, long msgIDBloomXorLocal, long msgIDBloomXorRouter, long msgIDBloomXorTunnel) {
+        _context = context;
+        _msgIDBloomXorLocal = msgIDBloomXorLocal;
+        _msgIDBloomXorRouter = msgIDBloomXorRouter;
+        _msgIDBloomXorTunnel = msgIDBloomXorTunnel;
     }
 
     public Job createJob(I2NPMessage receivedMessage, RouterIdentity from, Hash fromHash) {
-        HandleGarlicMessageJob job = new HandleGarlicMessageJob(_context, (GarlicMessage)receivedMessage, from, fromHash);
+        HandleGarlicMessageJob job = new HandleGarlicMessageJob(_context, (GarlicMessage)receivedMessage, from, fromHash, _msgIDBloomXorLocal, _msgIDBloomXorRouter, _msgIDBloomXorTunnel);
         return job;
     }
 

router/java/src/net/i2p/router/message/HandleGarlicMessageJob.java

@@ -36,6 +36,9 @@
 public class HandleGarlicMessageJob extends JobImpl implements GarlicMessageReceiver.CloveReceiver {
     private final Log _log;
     private final GarlicMessage _message;
+    private final long _msgIDBloomXorLocal;
+    private final long _msgIDBloomXorRouter;
+    private final long _msgIDBloomXorTunnel;
     //private RouterIdentity _from;
     //private Hash _fromHash;
     //private Map _cloves; // map of clove Id --> Expiration of cloves we've already seen
@@ -49,12 +52,15 @@ public class HandleGarlicMessageJob extends JobImpl implements GarlicMessageRece
      *  @param from ignored
      *  @param fromHash ignored
      */
-    public HandleGarlicMessageJob(RouterContext context, GarlicMessage msg, RouterIdentity from, Hash fromHash) {
+    public HandleGarlicMessageJob(RouterContext context, GarlicMessage msg, RouterIdentity from, Hash fromHash, long msgIDBloomXorLocal, long msgIDBloomXorRouter, long msgIDBloomXorTunnel) {
         super(context);
         _log = context.logManager().getLog(HandleGarlicMessageJob.class);
         if (_log.shouldLog(Log.DEBUG))
             _log.debug("Garlic Message not down a tunnel from [" + from + "]");
         _message = msg;
+        _msgIDBloomXorLocal = msgIDBloomXorLocal;
+        _msgIDBloomXorRouter = msgIDBloomXorRouter;
+        _msgIDBloomXorTunnel = msgIDBloomXorTunnel;
         //_from = from;
         //_fromHash = fromHash;
         //_cloves = new HashMap();
@@ -74,7 +80,9 @@ public void handleClove(DeliveryInstructions instructions, I2NPMessage data) {
             case DeliveryInstructions.DELIVERY_MODE_LOCAL:
                 if (_log.shouldLog(Log.DEBUG))
                     _log.debug("local delivery instructions for clove: " + data);
-                getContext().inNetMessagePool().add(data, null, null);
+                // Here we are adding the message to the InNetMessagePool and it is Local. Xor the messageID with
+                // a long unique to the router/session.
+                getContext().inNetMessagePool().add(data, null, null, _msgIDBloomXorLocal);
                 return;
             case DeliveryInstructions.DELIVERY_MODE_DESTINATION:
                 // i2pd bug with DLM to ratchet router
@@ -86,14 +94,18 @@ public void handleClove(DeliveryInstructions instructions, I2NPMessage data) {
                 if (getContext().routerHash().equals(instructions.getRouter())) {
                     if (_log.shouldLog(Log.DEBUG))
                         _log.debug("router delivery instructions targetting us");
-                    getContext().inNetMessagePool().add(data, null, null);
+                    // Here we are adding the message to the InNetMessagePool and it is for us. Xor the messageID with
+                    // a long unique to the router/session.
+                    getContext().inNetMessagePool().add(data, null, null, _msgIDBloomXorRouter);
                 } else {
                     if (_log.shouldLog(Log.DEBUG))
                         _log.debug("router delivery instructions targetting " 
                                    + instructions.getRouter().toBase64().substring(0,4) + " for " + data);
+                    // we don't need to use the msgIDBloomXorRouter here because we have already handled the case
+                    // where the message will be added to the InNetMessagePool(see SendMessageDirectJob 159-179)
                     SendMessageDirectJob j = new SendMessageDirectJob(getContext(), data, 
                                                                       instructions.getRouter(), 
-                                                                      10*1000, ROUTER_PRIORITY);
+                                                                      10*1000, ROUTER_PRIORITY, _msgIDBloomXorRouter);
                     // run it inline (adds to the outNetPool if it has the router info, otherwise queue a lookup)
                     j.runJob(); 
                     //getContext().jobQueue().addJob(j);
@@ -107,9 +119,10 @@ public void handleClove(DeliveryInstructions instructions, I2NPMessage data) {
                 if (_log.shouldLog(Log.DEBUG))
                     _log.debug("tunnel delivery instructions targetting " 
                                + instructions.getRouter().toBase64().substring(0,4) + " for " + data);
+                // Here we do Xor the messageID in case it is added to the InNetMessagePool(see SendMessageDirectJob 159-179)
                 SendMessageDirectJob job = new SendMessageDirectJob(getContext(), gw, 
                                                                     instructions.getRouter(), 
-                                                                    10*1000, TUNNEL_PRIORITY);
+                                                                    10*1000, TUNNEL_PRIORITY, _msgIDBloomXorTunnel);
                 // run it inline (adds to the outNetPool if it has the router info, otherwise queue a lookup)
                 job.runJob(); 
                 // getContext().jobQueue().addJob(job);

router/java/src/net/i2p/router/message/SendMessageDirectJob.java

@@ -42,12 +42,21 @@ public class SendMessageDirectJob extends JobImpl {
     private boolean _alreadySearched;
     private boolean _sent;
     private long _searchOn;
+    private final long _msgIDBloomXor;
 
     /**
      * @param toPeer may be ourselves
      */
     public SendMessageDirectJob(RouterContext ctx, I2NPMessage message, Hash toPeer, int timeoutMs, int priority) {
-        this(ctx, message, toPeer, null, null, null, null, timeoutMs, priority);
+        this(ctx, message, toPeer, null, null, null, null, timeoutMs, priority, 0);
+    }
+
+    /**
+     * @param toPeer may be ourselves
+     * @param msgIDBloomXor value to xor the messageID with before passing to the InNetMessagePool, may be 0
+     */
+    public SendMessageDirectJob(RouterContext ctx, I2NPMessage message, Hash toPeer, int timeoutMs, int priority, long msgIDBloomXor) {
+        this(ctx, message, toPeer, null, null, null, null, timeoutMs, priority, msgIDBloomXor);
     }
 
     /**
@@ -58,7 +67,19 @@ public SendMessageDirectJob(RouterContext ctx, I2NPMessage message, Hash toPeer,
      */
     public SendMessageDirectJob(RouterContext ctx, I2NPMessage message, Hash toPeer, ReplyJob onSuccess,
                                 Job onFail, MessageSelector selector, int timeoutMs, int priority) {
-        this(ctx, message, toPeer, null, onSuccess, onFail, selector, timeoutMs, priority);
+        this(ctx, message, toPeer, null, onSuccess, onFail, selector, timeoutMs, priority, 0);
+    }
+
+    /**
+     * @param toPeer may be ourselves
+     * @param onSuccess may be null
+     * @param onFail may be null
+     * @param selector be null
+     * @param msgIDBloomXor value to xor the messageID with before passing to the InNetMessagePool, may be 0
+     */
+    public SendMessageDirectJob(RouterContext ctx, I2NPMessage message, Hash toPeer, ReplyJob onSuccess,
+                                Job onFail, MessageSelector selector, int timeoutMs, int priority, long msgIDBloomXor) {
+        this(ctx, message, toPeer, null, onSuccess, onFail, selector, timeoutMs, priority, msgIDBloomXor);
     }
 
     /**
@@ -67,11 +88,13 @@ public SendMessageDirectJob(RouterContext ctx, I2NPMessage message, Hash toPeer,
      * @param onSuccess may be null
      * @param onFail may be null
      * @param selector be null
+     * @param msgIDBloomXor value to xor the messageID with before passing to the InNetMessagePool, may be 0
      */
     public SendMessageDirectJob(RouterContext ctx, I2NPMessage message, Hash toPeer, Job onSend, ReplyJob onSuccess,
-                                Job onFail, MessageSelector selector, int timeoutMs, int priority) {
+                                Job onFail, MessageSelector selector, int timeoutMs, int priority, long msgIDBloomXor) {
         super(ctx);
         _log = getContext().logManager().getLog(SendMessageDirectJob.class);
+        _msgIDBloomXor = msgIDBloomXor;
         _message = message;
         _targetHash = toPeer;
         if (timeoutMs < 10*1000) {
@@ -159,7 +182,7 @@ private void send() {
             if (_onSend != null)
                 getContext().jobQueue().addJob(_onSend);
 
-            getContext().inNetMessagePool().add(_message, _router.getIdentity(), null);
+            getContext().inNetMessagePool().add(_message, _router.getIdentity(), null, _msgIDBloomXor);
 
             if (_log.shouldLog(Log.DEBUG))
                 _log.debug("Adding " + _message.getClass().getName() 

router/java/src/net/i2p/router/networkdb/HandleDatabaseLookupMessageJob.java

@@ -45,6 +45,7 @@ public class HandleDatabaseLookupMessageJob extends JobImpl {
     private final DatabaseLookupMessage _message;
     private boolean _replyKeyConsumed;
     private final Hash _us;
+    private final long _msgIDBloomXor;
 
     private final static int MAX_ROUTERS_RETURNED = 3;
     private final static int CLOSENESS_THRESHOLD = 8; // FNDF.MAX_TO_FLOOD + 1
@@ -57,11 +58,12 @@ public class HandleDatabaseLookupMessageJob extends JobImpl {
      */
     public final static long EXPIRE_DELAY = 60*60*1000;
 
-    public HandleDatabaseLookupMessageJob(RouterContext ctx, DatabaseLookupMessage receivedMessage, RouterIdentity from, Hash fromHash) {
+    public HandleDatabaseLookupMessageJob(RouterContext ctx, DatabaseLookupMessage receivedMessage, RouterIdentity from, Hash fromHash, long msgIDBloomXor) {
         super(ctx);
         _log = ctx.logManager().getLog(HandleDatabaseLookupMessageJob.class);
         _message = receivedMessage;
         _us = ctx.routerHash();
+        _msgIDBloomXor = msgIDBloomXor;
     }
 
     protected boolean answerAllQueries() { return false; }
@@ -295,7 +297,7 @@ protected void sendMessage(I2NPMessage message, Hash toPeer, TunnelId replyTunne
         } else {
             if (_log.shouldLog(Log.DEBUG))
                 _log.debug("Sending reply directly to " + toPeer);
-            Job send = new SendMessageDirectJob(getContext(), message, toPeer, REPLY_TIMEOUT, MESSAGE_PRIORITY);
+            Job send = new SendMessageDirectJob(getContext(), message, toPeer, REPLY_TIMEOUT, MESSAGE_PRIORITY, _msgIDBloomXor);
             send.runJob();
             //getContext().netDb().lookupRouterInfo(toPeer, send, null, REPLY_TIMEOUT);
         }
@@ -338,7 +340,7 @@ private void sendThroughTunnel(I2NPMessage message, Hash toPeer, TunnelId replyT
             m.setMessage(message);
             m.setMessageExpiration(message.getMessageExpiration());
             m.setTunnelId(replyTunnel);
-            SendMessageDirectJob j = new SendMessageDirectJob(getContext(), m, toPeer, 10*1000, MESSAGE_PRIORITY);
+            SendMessageDirectJob j = new SendMessageDirectJob(getContext(), m, toPeer, 10*1000, MESSAGE_PRIORITY, _msgIDBloomXor);
             j.runJob();
             //getContext().jobQueue().addJob(j);
         }