new URI instead of new URL.

RipMeApp · Mar 20, 2024 · 91c0652 · 91c0652
1 parent c2d1472
commit 91c0652
Show file tree

Hide file tree

Showing 13 changed files with 62 additions and 70 deletions.
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
@@ -72,7 +72,7 @@ public static List<ChanSite> getChansFromConfig(String rawChanString) {
         );
 
     private ChanSite chanSite;
-    private Boolean generalChanSite = true;
+    private boolean generalChanSite = true;
 
     public ChanRipper(URL url) throws IOException {
         super(url);

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java
@@ -94,13 +94,6 @@ public String getGID(URL url) throws MalformedURLException {
                         + " Got: " + url);
     }
 
-    /**
-     * Attempts to get page, checks for IP ban, waits.
-     *
-     * @param url
-     * @return Page document
-     * @throws IOException If page loading errors, or if retries are exhausted
-     */
     private Document getPageWithRetries(URL url) throws IOException {
         Document doc;
         int retries = 3;
@@ -251,16 +244,16 @@ private void fetchImage() {
                         savePath += String.format("%03d_", index);
                     }
                     savePath += m.group(1);
-                    addURLToDownload(new URL(imgsrc), Paths.get(savePath));
+                    addURLToDownload(new URI(imgsrc).toURL(), Paths.get(savePath));
                 } else {
                     // Provide prefix and let the AbstractRipper "guess" the filename
                     String prefix = "";
                     if (Utils.getConfigBoolean("download.save_order", true)) {
                         prefix = String.format("%03d_", index);
                     }
-                    addURLToDownload(new URL(imgsrc), prefix);
+                    addURLToDownload(new URI(imgsrc).toURL(), prefix);
                 }
-            } catch (IOException e) {
+            } catch (IOException | URISyntaxException e) {
                 LOGGER.error("[!] Exception while loading/parsing " + this.url, e);
             }
         }

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java
@@ -1,9 +1,7 @@
 package com.rarchives.ripme.ripper.rippers;
 
 import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.MalformedURLException;
-import java.net.URL;
+import java.net.*;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
@@ -164,8 +162,8 @@ private String getUserID(String username) throws IOException {
     }
 
     @Override
-    public JSONObject getFirstPage() throws IOException {
-        URL apiURL = new URL(baseURL + "&consumer_key=" + CONSUMER_KEY);
+    public JSONObject getFirstPage() throws IOException, URISyntaxException {
+        URL apiURL = new URI(baseURL + "&consumer_key=" + CONSUMER_KEY).toURL();
         LOGGER.debug("apiURL: " + apiURL);
         JSONObject json = Http.url(apiURL).getJSON();
 
@@ -232,7 +230,7 @@ else if (baseURL.contains("/blogs?")) {
     }
 
     @Override
-    public JSONObject getNextPage(JSONObject json) throws IOException {
+    public JSONObject getNextPage(JSONObject json) throws IOException, URISyntaxException {
         if (isThisATest()) {
             return null;
         }
@@ -249,9 +247,9 @@ public JSONObject getNextPage(JSONObject json) throws IOException {
 
         sleep(500);
         ++page;
-        URL apiURL = new URL(baseURL
+        URL apiURL = new URI(baseURL
                              + "&page=" + page
-                             + "&consumer_key=" + CONSUMER_KEY);
+                             + "&consumer_key=" + CONSUMER_KEY).toURL();
         return Http.url(apiURL).getJSON();
     }
 
@@ -296,28 +294,23 @@ public List<String> getURLsFromJSON(JSONObject json) {
                     }
                 }
             }
-            if (imageURL == null) {
-                LOGGER.error("Failed to find image for photo " + photo.toString());
-            }
-            else {
-                imageURLs.add(imageURL);
-                if (isThisATest()) {
-                    break;
-                }
+            imageURLs.add(imageURL);
+            if (isThisATest()) {
+                break;
             }
         }
         return imageURLs;
     }
 
     private boolean urlExists(String url) {
         try {
-            HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
+            HttpURLConnection connection = (HttpURLConnection) new URI(url).toURL().openConnection();
             connection.setRequestMethod("HEAD");
             if (connection.getResponseCode() != 200) {
                 throw new IOException("Couldn't find full-size image at " + url);
             }
             return true;
-        } catch (IOException e) {
+        } catch (IOException | URISyntaxException e) {
             return false;
         }
     }

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuskatorRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuskatorRipper.java
@@ -2,6 +2,8 @@
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.List;
@@ -40,15 +42,15 @@ public String getDomain() {
     }
 
     @Override
-    public URL sanitizeURL(URL url) throws MalformedURLException {
+    public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException {
         String u = url.toExternalForm();
         if (u.contains("/thumbs/")) {
             u = u.replace("/thumbs/", "/full/");
         }
         if (u.contains("/expanded/")) {
             u = u.replaceAll("/expanded/", "/full/");
         }
-        return new URL(u);
+        return new URI(u).toURL();
     }
 
     @Override

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java
@@ -2,6 +2,8 @@
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Base64;
@@ -80,16 +82,15 @@ protected List<String> getURLsFromJSON(JSONObject json) throws JSONException {
     }
 
     @Override
-    protected JSONObject getFirstPage() throws IOException {
+    protected JSONObject getFirstPage() throws IOException, URISyntaxException {
         String jsonEncodedString = getJsonEncodedStringFromPage();
         String jsonDecodedString = decodeJsonString(jsonEncodedString);
         return new JSONObject(jsonDecodedString);
     }
 
-    public String getJsonEncodedStringFromPage() throws MalformedURLException, IOException
-    {
+    public String getJsonEncodedStringFromPage() throws MalformedURLException, IOException, URISyntaxException {
         // Image data only appears on the /read/ page and not on the /view/ one.
-        URL readUrl = new URL(String.format("http://hentainexus.com/read/%s",getGID(url)));
+        URL readUrl = new URI(String.format("http://hentainexus.com/read/%s",getGID(url))).toURL();
         Document document = Http.url(readUrl).response().parse();
 
         for (Element scripts : document.getElementsByTag("script")) {
@@ -143,7 +144,7 @@ The following code is a Java adaptation of the initRender() JavaScript function
         }
 
         magicByte = (byte) (magicByte & 0x7);
-        ArrayList<Integer> newArray = new ArrayList();
+        ArrayList<Integer> newArray = new ArrayList<>();
 
         for (int i = 0x0; i < 0x100; i++) {
             newArray.add(i);

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java
@@ -2,6 +2,7 @@
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
@@ -14,7 +15,6 @@
 
 import com.rarchives.ripme.ripper.AbstractHTMLRipper;
 import com.rarchives.ripme.utils.Http;
-import org.jsoup.nodes.Element;
 
 public class HitomiRipper extends AbstractHTMLRipper {
 
@@ -47,9 +47,9 @@ public String getGID(URL url) throws MalformedURLException {
     }
 
     @Override
-    public Document getFirstPage() throws IOException {
+    public Document getFirstPage() throws IOException, URISyntaxException {
         // if we go to /GALLERYID.js we get a nice json array of all images in the gallery
-        return Http.url(new URL(url.toExternalForm().replaceAll("hitomi", "ltn.hitomi").replaceAll(".html", ".js"))).ignoreContentType().get();
+        return Http.url(new URI(url.toExternalForm().replaceAll("hitomi", "ltn.hitomi").replaceAll(".html", ".js")).toURL()).ignoreContentType().get();
     }
 
 

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NsfwXxxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NsfwXxxRipper.java
@@ -8,6 +8,8 @@
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.List;
@@ -34,7 +36,7 @@ public String getHost() {
 
 
     @Override
-    public URL sanitizeURL(URL url) throws MalformedURLException {
+    public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException {
         String u = url.toExternalForm();
         // https://nsfw.xxx/user/kelly-kat/foo -> https://nsfw.xxx/user/kelly-kat
         // https://nsfw.xxx/user/kelly-kat -> https://nsfw.xxx/user/kelly-kat
@@ -44,15 +46,15 @@ public URL sanitizeURL(URL url) throws MalformedURLException {
             throw new MalformedURLException("Invalid URL: " + url);
         }
 
-        return new URL(u);
+        return new URI(u).toURL();
     }
 
     String getUser() throws MalformedURLException {
         return getGID(url);
     }
 
-    URL getPage(int page) throws MalformedURLException {
-        return new URL("https://nsfw.xxx/slide-page/" + page + "?nsfw%5B%5D=0&types%5B%5D=image&types%5B%5D=video&types%5B%5D=gallery&slider=1&jsload=1&user=" + getUser());
+    URL getPage(int page) throws MalformedURLException, URISyntaxException {
+        return new URI("https://nsfw.xxx/slide-page/" + page + "?nsfw%5B%5D=0&types%5B%5D=image&types%5B%5D=video&types%5B%5D=gallery&slider=1&jsload=1&user=" + getUser()).toURL();
     }
 
 
@@ -71,18 +73,18 @@ public String getGID(URL url) throws MalformedURLException {
     int currentPage = 1;
 
     @Override
-    protected JSONObject getFirstPage() throws IOException {
+    protected JSONObject getFirstPage() throws IOException, URISyntaxException {
         return Http.url(getPage(1)).getJSON();
     }
 
     List<String> descriptions = new ArrayList<>();
 
     @Override
-    protected JSONObject getNextPage(JSONObject doc) throws IOException {
+    protected JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxException {
         currentPage++;
         JSONObject nextPage = Http.url(getPage(doc.getInt("page") + 1)).getJSON();
         JSONArray items = nextPage.getJSONArray("items");
-        if (items.length() == 0) {
+        if (items.isEmpty()) {
             throw new IOException("No more pages");
         }
         return nextPage;
@@ -120,7 +122,7 @@ protected List<String> getURLsFromJSON(JSONObject json) {
 
                     return new ApiEntry(srcUrl, o.getString("author"), o.getString("title"));
                 })
-                .collect(Collectors.toList());
+                .toList();
 
         data.forEach(e -> descriptions.add(e.title));
         return data.stream().map(e -> e.srcUrl).collect(Collectors.toList());

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java
@@ -2,6 +2,7 @@
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
@@ -44,13 +45,13 @@ public String getDomain() {
     }
 
     @Override
-    public URL sanitizeURL(URL url) throws MalformedURLException {
+    public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException {
         if (isVideoUrl(url)) {
             return url;
         }
         String URLToReturn = url.toExternalForm();
         URLToReturn = URLToReturn.replaceAll("https?://\\w?\\w?\\.?xhamster([^<]*)\\.", "https://m.xhamster$1.");
-        URL san_url = new URL(URLToReturn);
+        URL san_url = new URI(URLToReturn).toURL();
         LOGGER.info("sanitized URL is " + san_url.toExternalForm());
         return san_url;
     }
@@ -168,10 +169,10 @@ public List<String> getURLsFromPage(Document doc) {
                         // This works around some redirect fuckery xhamster likes to do where visiting m.xhamster.com sends to
                         // the page chamster.com but displays the mobile site from m.xhamster.com
                         pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster([^<]*)\\.", "://m.xhamster$1.");
-                        String image = Http.url(new URL(pageWithImageUrl)).get().select("a > img#photoCurr").attr("src");
+                        String image = Http.url(new URI(pageWithImageUrl).toURL()).get().select("a > img#photoCurr").attr("src");
                         result.add(image);
                         downloadFile(image);
-                    } catch (IOException e) {
+                    } catch (IOException | URISyntaxException e) {
                         LOGGER.error("Was unable to load page " + pageWithImageUrl);
               }
               if (isStopped() || isThisATest()) {

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java
@@ -2,6 +2,8 @@
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -56,7 +58,7 @@ public String getGID(URL url) throws MalformedURLException {
     }
 
     @Override
-    public void rip() throws IOException {
+    public void rip() throws IOException, URISyntaxException {
         String vidUrl = "";
         LOGGER.info("    Retrieving " + this.url.toExternalForm());
         Document doc = Http.url(this.url).get();
@@ -146,7 +148,7 @@ public void rip() throws IOException {
         if (vidUrl.equals("")) {
             throw new IOException("Unable to find encrypted video URL at " + this.url);
         }
-        addURLToDownload(new URL(vidUrl), HOST + "_" + bestQuality + "p_" + getGID(this.url));
+        addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + bestQuality + "p_" + getGID(this.url));
 
         waitForThreads();
     }

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/StickyXXXRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/StickyXXXRipper.java
@@ -2,6 +2,8 @@
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -32,11 +34,6 @@ public boolean canRip(URL url) {
         return m.matches();
     }
 
-    @Override
-    public URL sanitizeURL(URL url) throws MalformedURLException {
-        return url;
-    }
-
     @Override
     public String getGID(URL url) throws MalformedURLException {
         Pattern p = Pattern.compile("^https?://.*stickyxxx\\.com(/)(.*)/$");
@@ -52,15 +49,15 @@ public String getGID(URL url) throws MalformedURLException {
     }
 
     @Override
-    public void rip() throws IOException {
+    public void rip() throws IOException, URISyntaxException {
         LOGGER.info("Retrieving " + this.url);
         Document doc = Http.url(url).get();
         Elements videos = doc.select(".wp-video > video > source");
         if (videos.isEmpty()) {
             throw new IOException("Could not find Embed code at " + url);
         }
         String vidUrl = videos.attr("src");
-        addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url));
+        addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url));
         waitForThreads();
     }
 }