diff --git a/code/common/db/src/main/resources/sql/current/00-base.sql b/code/common/db/src/main/resources/sql/current/00-base.sql index 935ae991b..cec6e928b 100644 --- a/code/common/db/src/main/resources/sql/current/00-base.sql +++ b/code/common/db/src/main/resources/sql/current/00-base.sql @@ -10,7 +10,7 @@ CREATE TABLE IF NOT EXISTS EC_DOMAIN ( RANK DOUBLE, DOMAIN_ALIAS INTEGER, - IP VARCHAR(32), + IP VARCHAR(48), INDEX_DATE TIMESTAMP DEFAULT NOW(), DISCOVER_DATE TIMESTAMP DEFAULT NOW(), diff --git a/code/common/db/src/main/resources/sql/migrations/01-domain.sql b/code/common/db/src/main/resources/sql/migrations/01-domain.sql new file mode 100644 index 000000000..0402fecb2 --- /dev/null +++ b/code/common/db/src/main/resources/sql/migrations/01-domain.sql @@ -0,0 +1 @@ +ALTER TABLE EC_DOMAIN MODIFY COLUMN IP VARCHAR(48); \ No newline at end of file diff --git a/code/processes/loading-process/src/main/java/nu/marginalia/loading/loader/SqlLoadProcessedDomain.java b/code/processes/loading-process/src/main/java/nu/marginalia/loading/loader/SqlLoadProcessedDomain.java index 4c8be7081..1e1998c7a 100644 --- a/code/processes/loading-process/src/main/java/nu/marginalia/loading/loader/SqlLoadProcessedDomain.java +++ b/code/processes/loading-process/src/main/java/nu/marginalia/loading/loader/SqlLoadProcessedDomain.java @@ -5,6 +5,7 @@ import nu.marginalia.model.crawl.DomainIndexingState; import nu.marginalia.converting.instruction.instructions.DomainLink; import nu.marginalia.model.EdgeDomain; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,7 +30,7 @@ CREATE PROCEDURE INITIALIZE_DOMAIN ( IN ST ENUM('ACTIVE', 'EXHAUSTED', 'SPECIAL', 'SOCIAL_MEDIA', 'BLOCKED', 'REDIR', 'ERROR', 'UNKNOWN'), IN IDX INT, IN DID INT, - IN IP VARCHAR(32)) + IN IP VARCHAR(48)) BEGIN UPDATE EC_DOMAIN SET INDEX_DATE=NOW(), STATE=ST, DOMAIN_ALIAS=NULL, INDEXED=GREATEST(INDEXED,IDX), IP=IP WHERE ID=DID; DELETE FROM EC_DOMAIN_LINK WHERE SOURCE_DOMAIN_ID=DID; @@ -53,7 +54,7 @@ public void load(LoaderData data, EdgeDomain domain, DomainIndexingState state, initCall.setString(1, state.name()); initCall.setInt(2, 1 + data.sizeHint / 100); initCall.setInt(3, data.getDomainId(domain)); - initCall.setString(4, ip); + initCall.setString(4, StringUtils.truncate(ip, 48)); int rc = initCall.executeUpdate(); conn.commit(); if (rc < 1) { diff --git a/code/processes/loading-process/src/test/java/nu/marginalia/loader/SqlLoadProcessedDomainTest.java b/code/processes/loading-process/src/test/java/nu/marginalia/loader/SqlLoadProcessedDomainTest.java index 2b28f6a29..df61cf506 100644 --- a/code/processes/loading-process/src/test/java/nu/marginalia/loader/SqlLoadProcessedDomainTest.java +++ b/code/processes/loading-process/src/test/java/nu/marginalia/loader/SqlLoadProcessedDomainTest.java @@ -15,6 +15,9 @@ import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; +import java.util.stream.Collectors; +import java.util.stream.Stream; + @Tag("slow") @Testcontainers class SqlLoadProcessedDomainTest { @@ -50,6 +53,16 @@ public void loadProcessedDomain() { var loader = new SqlLoadProcessedDomain(dataSource, new SqlLoadDomains(dataSource)); loader.load(loaderData, new EdgeDomain("www.marginalia.nu"), DomainIndexingState.BLOCKED, "127.0.0.1"); } + + @Test + public void loadProcessedDomaiWithExtremelyLongIP() { + var loader = new SqlLoadProcessedDomain(dataSource, new SqlLoadDomains(dataSource)); + + String ip = Stream.generate(() -> "127.").limit(1024).collect(Collectors.joining()); + + loader.load(loaderData, new EdgeDomain("www.marginalia.nu"), DomainIndexingState.BLOCKED, ip); + } + @Test public void loadDomainAlias() { var loader = new SqlLoadProcessedDomain(dataSource, new SqlLoadDomains(dataSource));