IPFilterRules: add anylocal and multicast keywords, fail on unparseable rules

core/src/main/java/org/apache/stormcrawler/protocol/IPFilterRules.java

@@ -49,10 +49,18 @@
  *       {@link InetAddress#isLoopbackAddress()} is true
  *   <li><code>sitelocal</code> applies to all IP addresses for which {@link
  *       InetAddress#isSiteLocalAddress()} is true
+ *   <li><code>linklocal</code> applies to all IP addresses for which {@link
+ *       InetAddress#isLinkLocalAddress()} is true
+ *   <li><code>anylocal</code> applies to all IP addresses for which {@link
+ *       InetAddress#isAnyLocalAddress()} is true
+ *   <li><code>multicast</code> applies to all IP addresses for which {@link
+ *       InetAddress#isMulticastAddress()} is true
  * </ul>
  *
  * <p>Multiple IP ranges can be given either as a comma-separated string, e.g. <code>
- * loopback,sitelocal,fd00::/8</code>, or as a list in the configuration.
+ * loopback,sitelocal,fd00::/8</code>, or as a list in the configuration. A value which is neither a
+ * keyword, a CIDR block nor a single IP address is a configuration error and throws an {@link
+ * IllegalArgumentException}.
  */
 public class IPFilterRules {
 
@@ -119,15 +127,24 @@ private static List<Predicate<InetAddress>> parseIPRules(
                     case "linklocal":
                         rules.add(InetAddress::isLinkLocalAddress);
                         break;
+                    case "anylocal":
+                        rules.add(InetAddress::isAnyLocalAddress);
+                        break;
+                    case "multicast":
+                        rules.add(InetAddress::isMulticastAddress);
+                        break;
                     default:
                         try {
                             CIDR cidr = new CIDR(ipRule);
                             rules.add(cidr::contains);
                         } catch (IllegalArgumentException e) {
-                            LOG.error(
-                                    "Failed to parse {} as CIDR, ignoring it while configuring IP rules ({})",
-                                    ipRule,
-                                    ipRuleProperty);
+                            throw new IllegalArgumentException(
+                                    "Failed to parse "
+                                            + ipRule
+                                            + " as CIDR while configuring IP rules ("
+                                            + ipRuleProperty
+                                            + ")",
+                                    e);
                         }
                 }
             }

core/src/main/resources/crawler-default.yaml

@@ -159,10 +159,14 @@ config:
   #   - "localhost" / "loopback" (matches InetAddress.isLoopbackAddress())
   #   - "sitelocal" (matches InetAddress.isSiteLocalAddress())
   #   - "linklocal" (matches InetAddress.isLinkLocalAddress())
+  #   - "anylocal" (matches InetAddress.isAnyLocalAddress())
+  #   - "multicast" (matches InetAddress.isMulticastAddress())
+  # A value which is neither of these is a configuration error and stops the
+  # topology from starting.
   # Only addresses matching an include rule are fetched (empty means all are
   # allowed), addresses matching an exclude rule are always blocked.
   # http.filter.ipaddress.include:
-  # http.filter.ipaddress.exclude: "localhost,sitelocal,linklocal"
+  # http.filter.ipaddress.exclude: "localhost,sitelocal,linklocal,100.64.0.0/10,fd00::/8"
 
   # Allow all if robots.txt cannot be parsed due to code 403 (Forbidden):
   http.robots.403.allow: true

core/src/test/java/org/apache/stormcrawler/protocol/IPFilterRulesTest.java

@@ -25,6 +25,7 @@
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
 class IPFilterRulesTest {
@@ -104,9 +105,32 @@ void excludeTakesPrecedenceOverInclude() {
     }
 
     @Test
-    void invalidRuleIsIgnored() {
-        IPFilterRules r = rules(null, "not-an-ip,localhost");
-        assertFalse(r.accept(ip("127.0.0.1")));
+    void invalidRuleFailsConfiguration() {
+        Assertions.assertThrows(
+                IllegalArgumentException.class, () -> rules(null, "not-an-ip,localhost"));
+    }
+
+    @Test
+    void documentedExampleBlocksTheNonRoutableRanges() {
+        // the example shipped in crawler-default.yaml
+        IPFilterRules r = rules(null, "localhost,sitelocal,linklocal,100.64.0.0/10,fd00::/8");
+        assertFalse(r.accept(ip("127.0.0.1")), "loopback");
+        assertFalse(r.accept(ip("10.0.0.1")), "RFC1918");
+        assertFalse(r.accept(ip("192.168.1.10")), "RFC1918");
+        assertFalse(r.accept(ip("172.16.0.1")), "RFC1918");
+        assertFalse(r.accept(ip("169.254.169.254")), "IPv4 link-local");
+        assertFalse(r.accept(ip("100.64.0.1")), "CGNAT");
+        assertFalse(r.accept(ip("fd00::1")), "IPv6 unique local");
+        assertFalse(r.accept(ip("fe80::1")), "IPv6 link-local");
+        assertTrue(r.accept(ip("8.8.8.8")), "public IPv4");
+    }
+
+    @Test
+    void anylocalAndMulticastKeywords() {
+        IPFilterRules r = rules(null, "anylocal,multicast");
+        assertFalse(r.accept(ip("0.0.0.0")), "wildcard");
+        assertFalse(r.accept(ip("::")), "IPv6 wildcard");
+        assertFalse(r.accept(ip("224.0.0.1")), "multicast");
         assertTrue(r.accept(ip("8.8.8.8")));
     }