From 373d4b7809eb96767eb5f03866884a07f7ad0d1a Mon Sep 17 00:00:00 2001 From: abhinav-phi Date: Sun, 6 Sep 2026 03:51:35 +0530 Subject: [PATCH] IPFilterRules: add anylocal and multicast keywords, fail on unparseable rules (#2086) A rule that is neither a keyword, a CIDR block nor a single address was logged and dropped, silently widening what the crawler will connect to. Configuration now fails with an IllegalArgumentException instead, so a typo in an exclude list stops the topology rather than removing a rule the operator believed was in force. Adds the anylocal and multicast keywords, documents linklocal (already present but missing from the javadoc), and updates the example in crawler-default.yaml to also cover CGNAT and IPv6 unique-local space. --- .../stormcrawler/protocol/IPFilterRules.java | 27 +++++++++++++---- core/src/main/resources/crawler-default.yaml | 6 +++- .../protocol/IPFilterRulesTest.java | 30 +++++++++++++++++-- 3 files changed, 54 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/IPFilterRules.java b/core/src/main/java/org/apache/stormcrawler/protocol/IPFilterRules.java index 2fbb5aecd..5b35ea2c6 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/IPFilterRules.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/IPFilterRules.java @@ -49,10 +49,18 @@ * {@link InetAddress#isLoopbackAddress()} is true *
  • sitelocal applies to all IP addresses for which {@link * InetAddress#isSiteLocalAddress()} is true + *
  • linklocal applies to all IP addresses for which {@link + * InetAddress#isLinkLocalAddress()} is true + *
  • anylocal applies to all IP addresses for which {@link + * InetAddress#isAnyLocalAddress()} is true + *
  • multicast applies to all IP addresses for which {@link + * InetAddress#isMulticastAddress()} is true * * *

    Multiple IP ranges can be given either as a comma-separated string, e.g. - * loopback,sitelocal,fd00::/8, or as a list in the configuration. + * loopback,sitelocal,fd00::/8, or as a list in the configuration. A value which is neither a + * keyword, a CIDR block nor a single IP address is a configuration error and throws an {@link + * IllegalArgumentException}. */ public class IPFilterRules { @@ -119,15 +127,24 @@ private static List> parseIPRules( case "linklocal": rules.add(InetAddress::isLinkLocalAddress); break; + case "anylocal": + rules.add(InetAddress::isAnyLocalAddress); + break; + case "multicast": + rules.add(InetAddress::isMulticastAddress); + break; default: try { CIDR cidr = new CIDR(ipRule); rules.add(cidr::contains); } catch (IllegalArgumentException e) { - LOG.error( - "Failed to parse {} as CIDR, ignoring it while configuring IP rules ({})", - ipRule, - ipRuleProperty); + throw new IllegalArgumentException( + "Failed to parse " + + ipRule + + " as CIDR while configuring IP rules (" + + ipRuleProperty + + ")", + e); } } } diff --git a/core/src/main/resources/crawler-default.yaml b/core/src/main/resources/crawler-default.yaml index 6945b2c4c..1beb14e12 100644 --- a/core/src/main/resources/crawler-default.yaml +++ b/core/src/main/resources/crawler-default.yaml @@ -159,10 +159,14 @@ config: # - "localhost" / "loopback" (matches InetAddress.isLoopbackAddress()) # - "sitelocal" (matches InetAddress.isSiteLocalAddress()) # - "linklocal" (matches InetAddress.isLinkLocalAddress()) + # - "anylocal" (matches InetAddress.isAnyLocalAddress()) + # - "multicast" (matches InetAddress.isMulticastAddress()) + # A value which is neither of these is a configuration error and stops the + # topology from starting. # Only addresses matching an include rule are fetched (empty means all are # allowed), addresses matching an exclude rule are always blocked. # http.filter.ipaddress.include: - # http.filter.ipaddress.exclude: "localhost,sitelocal,linklocal" + # http.filter.ipaddress.exclude: "localhost,sitelocal,linklocal,100.64.0.0/10,fd00::/8" # Allow all if robots.txt cannot be parsed due to code 403 (Forbidden): http.robots.403.allow: true diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/IPFilterRulesTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/IPFilterRulesTest.java index fa42d190f..9b996721e 100644 --- a/core/src/test/java/org/apache/stormcrawler/protocol/IPFilterRulesTest.java +++ b/core/src/test/java/org/apache/stormcrawler/protocol/IPFilterRulesTest.java @@ -25,6 +25,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; class IPFilterRulesTest { @@ -104,9 +105,32 @@ void excludeTakesPrecedenceOverInclude() { } @Test - void invalidRuleIsIgnored() { - IPFilterRules r = rules(null, "not-an-ip,localhost"); - assertFalse(r.accept(ip("127.0.0.1"))); + void invalidRuleFailsConfiguration() { + Assertions.assertThrows( + IllegalArgumentException.class, () -> rules(null, "not-an-ip,localhost")); + } + + @Test + void documentedExampleBlocksTheNonRoutableRanges() { + // the example shipped in crawler-default.yaml + IPFilterRules r = rules(null, "localhost,sitelocal,linklocal,100.64.0.0/10,fd00::/8"); + assertFalse(r.accept(ip("127.0.0.1")), "loopback"); + assertFalse(r.accept(ip("10.0.0.1")), "RFC1918"); + assertFalse(r.accept(ip("192.168.1.10")), "RFC1918"); + assertFalse(r.accept(ip("172.16.0.1")), "RFC1918"); + assertFalse(r.accept(ip("169.254.169.254")), "IPv4 link-local"); + assertFalse(r.accept(ip("100.64.0.1")), "CGNAT"); + assertFalse(r.accept(ip("fd00::1")), "IPv6 unique local"); + assertFalse(r.accept(ip("fe80::1")), "IPv6 link-local"); + assertTrue(r.accept(ip("8.8.8.8")), "public IPv4"); + } + + @Test + void anylocalAndMulticastKeywords() { + IPFilterRules r = rules(null, "anylocal,multicast"); + assertFalse(r.accept(ip("0.0.0.0")), "wildcard"); + assertFalse(r.accept(ip("::")), "IPv6 wildcard"); + assertFalse(r.accept(ip("224.0.0.1")), "multicast"); assertTrue(r.accept(ip("8.8.8.8"))); }