Issue 5: Fix bug where some IPv6 addresses blew up domain loading.

MarginaliaSearch · Apr 15, 2023 · d42ab19 · d42ab19
1 parent 2ab26f3
commit d42ab19
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 3 deletions.
diff --git a/code/common/db/src/main/resources/sql/current/00-base.sql b/code/common/db/src/main/resources/sql/current/00-base.sql
@@ -10,7 +10,7 @@ CREATE TABLE IF NOT EXISTS EC_DOMAIN (
 
     RANK DOUBLE,
     DOMAIN_ALIAS INTEGER,
-    IP VARCHAR(32),
+    IP VARCHAR(48),
 
     INDEX_DATE TIMESTAMP DEFAULT NOW(),
     DISCOVER_DATE TIMESTAMP DEFAULT NOW(),

diff --git a/code/common/db/src/main/resources/sql/migrations/01-domain.sql b/code/common/db/src/main/resources/sql/migrations/01-domain.sql
@@ -0,0 +1 @@
+ALTER TABLE EC_DOMAIN MODIFY COLUMN IP VARCHAR(48);
diff --git a/...es/loading-process/src/main/java/nu/marginalia/loading/loader/SqlLoadProcessedDomain.java b/...es/loading-process/src/main/java/nu/marginalia/loading/loader/SqlLoadProcessedDomain.java
@@ -5,6 +5,7 @@
 import nu.marginalia.model.crawl.DomainIndexingState;
 import nu.marginalia.converting.instruction.instructions.DomainLink;
 import nu.marginalia.model.EdgeDomain;
+import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -29,7 +30,7 @@ CREATE PROCEDURE INITIALIZE_DOMAIN (
                             IN ST ENUM('ACTIVE', 'EXHAUSTED', 'SPECIAL', 'SOCIAL_MEDIA', 'BLOCKED', 'REDIR', 'ERROR', 'UNKNOWN'),
                             IN IDX INT,
                             IN DID INT,
-                            IN IP VARCHAR(32))
+                            IN IP VARCHAR(48))
                         BEGIN
                             UPDATE EC_DOMAIN SET INDEX_DATE=NOW(), STATE=ST, DOMAIN_ALIAS=NULL, INDEXED=GREATEST(INDEXED,IDX), IP=IP WHERE ID=DID;
                             DELETE FROM EC_DOMAIN_LINK WHERE SOURCE_DOMAIN_ID=DID;
@@ -53,7 +54,7 @@ public void load(LoaderData data, EdgeDomain domain, DomainIndexingState state,
             initCall.setString(1, state.name());
             initCall.setInt(2, 1 + data.sizeHint / 100);
             initCall.setInt(3, data.getDomainId(domain));
-            initCall.setString(4, ip);
+            initCall.setString(4, StringUtils.truncate(ip, 48));
             int rc = initCall.executeUpdate();
             conn.commit();
             if (rc < 1) {

diff --git a/...cesses/loading-process/src/test/java/nu/marginalia/loader/SqlLoadProcessedDomainTest.java b/...cesses/loading-process/src/test/java/nu/marginalia/loader/SqlLoadProcessedDomainTest.java
@@ -15,6 +15,9 @@
 import org.testcontainers.junit.jupiter.Container;
 import org.testcontainers.junit.jupiter.Testcontainers;
 
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
 @Tag("slow")
 @Testcontainers
 class SqlLoadProcessedDomainTest {
@@ -50,6 +53,16 @@ public void loadProcessedDomain() {
         var loader = new SqlLoadProcessedDomain(dataSource, new SqlLoadDomains(dataSource));
         loader.load(loaderData, new EdgeDomain("www.marginalia.nu"), DomainIndexingState.BLOCKED, "127.0.0.1");
     }
+
+    @Test
+    public void loadProcessedDomaiWithExtremelyLongIP() {
+        var loader = new SqlLoadProcessedDomain(dataSource, new SqlLoadDomains(dataSource));
+
+        String ip = Stream.generate(() -> "127.").limit(1024).collect(Collectors.joining());
+
+        loader.load(loaderData, new EdgeDomain("www.marginalia.nu"), DomainIndexingState.BLOCKED, ip);
+    }
+
     @Test
     public void loadDomainAlias() {
         var loader = new SqlLoadProcessedDomain(dataSource, new SqlLoadDomains(dataSource));