use urlcanon instead of crawler4j because we only used it for url can…

…onization
Athou · Apr 29, 2023 · 0a99dac · 0a99dac
1 parent 00f6c04
commit 0a99dac
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 11 deletions.
diff --git a/commafeed-server/pom.xml b/commafeed-server/pom.xml
@@ -426,15 +426,9 @@
 			<version>0.9.30</version>
 		</dependency>
 		<dependency>
-			<groupId>edu.uci.ics</groupId>
-			<artifactId>crawler4j</artifactId>
-			<version>3.5</version>
-			<exclusions>
-				<exclusion>
-					<groupId>log4j</groupId>
-					<artifactId>log4j</artifactId>
-				</exclusion>
-			</exclusions>
+			<groupId>org.netpreserve</groupId>
+			<artifactId>urlcanon</artifactId>
+			<version>0.4.0</version>
 		</dependency>
 		<dependency>
 			<groupId>com.google.gwt</groupId>

diff --git a/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUtils.java b/commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUtils.java
@@ -28,6 +28,8 @@
 import org.jsoup.safety.Cleaner;
 import org.jsoup.safety.Safelist;
 import org.jsoup.select.Elements;
+import org.netpreserve.urlcanon.Canonicalizer;
+import org.netpreserve.urlcanon.ParsedUrl;
 import org.w3c.css.sac.InputSource;
 import org.w3c.dom.css.CSSStyleDeclaration;
 
@@ -41,7 +43,6 @@
 import com.ibm.icu.text.CharsetMatch;
 import com.steadystate.css.parser.CSSOMParser;
 
-import edu.uci.ics.crawler4j.url.URLCanonicalizer;
 import lombok.extern.slf4j.Slf4j;
 
 /**
@@ -179,7 +180,10 @@ public static String normalizeURL(String url) {
 		if (url == null) {
 			return null;
 		}
-		String normalized = URLCanonicalizer.getCanonicalURL(url);
+
+		ParsedUrl parsedUrl = ParsedUrl.parseUrl(url);
+		Canonicalizer.AGGRESSIVE.canonicalize(parsedUrl);
+		String normalized = parsedUrl.toString();
 		if (normalized == null) {
 			normalized = url;
 		}