diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml new file mode 100644 index 000000000..38ad74fa8 --- /dev/null +++ b/.github/workflows/gradle.yml @@ -0,0 +1,44 @@ +name: CI + release + +on: [push, pull_request] + +jobs: + build: + + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest] + java: [1.8] + include: # test newest java on one os only, upload from ubuntu java8 + - os: ubuntu-latest + java: 1.15 + - os: ubuntu-latest + upload: true + + steps: + - uses: actions/checkout@v1 + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + - name: Cache Gradle packages + # speed up the build by caching dependencies, downloaded versions + uses: actions/cache@v2 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + - name: Build with Gradle + run: ./gradlew build + - name: upload jar as asset + if: matrix.upload + uses: actions/upload-artifact@v2 + with: + name: zipped-ripme-jar + path: build/libs/*.jar + +# vim:set ts=2 sw=2 et: diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index bb44b0c84..038c890f6 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -1,6 +1,6 @@ name: Java CI -on: [push, pull_request] +on: workflow_dispatch jobs: build: @@ -9,7 +9,12 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macOS-latest] - java: [1.8, 1.14] + java: [1.8] + include: # test newest java on one os only, upload from ubuntu java8 + - os: ubuntu-latest + java: 1.15 + - os: ubuntu-latest + upload: true steps: - uses: actions/checkout@v1 @@ -18,4 +23,12 @@ jobs: with: java-version: ${{ matrix.java }} - name: Build with Maven - run: mvn package --file pom.xml + run: mvn -B package assembly:single --file pom.xml + - name: upload jar as asset + if: matrix.upload + uses: actions/upload-artifact@v2 + with: + name: zipped-ripme-jar + path: target/*dependencies.jar + +# vim:set ts=2 sw=2 et: diff --git a/.gitignore b/.gitignore index e7813bc7f..fb5ed2101 100644 --- a/.gitignore +++ b/.gitignore @@ -80,6 +80,12 @@ buildNumber.properties # Avoid ignoring Maven wrapper jar file (.jar files are usually ignored) !/.mvn/wrapper/maven-wrapper.jar +### gradle ### +/.gradle +/build +# Avoid ignoring gradle wrapper jar file (.jar files are usually ignored) +!/gradle/wrapper/gradle-wrapper.jar + ### Windows ### # Windows thumbnail cache files Thumbs.db diff --git a/README.md b/README.md index 1b2525c53..be7739873 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ For information about running the `.jar` file, see [the How To Run wiki](https:/ On macOS, there is a [cask](https://github.com/Homebrew/homebrew-cask/blob/master/Casks/ripme.rb). ``` -brew cask install ripme && xattr -d com.apple.quarantine /Applications/ripme.jar +brew install --cask ripme && xattr -d com.apple.quarantine /Applications/ripme.jar ``` ## Changelog @@ -77,14 +77,21 @@ If you're a developer, you can add your own Ripper by following the wiki guide: # Compiling & Building -The project uses [Maven](http://maven.apache.org/). -To build the .jar file using Maven, navigate to the root project directory and run: +The project uses [Gradle](https://gradle.org) or [Maven](http://maven.apache.org/). +Therefor both commands are given. To build the .jar file, navigate to the root +project directory and run: ```bash mvn clean compile assembly:single +mvn -B package assembly:single -Dmaven.test.skip=true +``` +```bash +./gradlew clean build +./gradlew clean build -x test --warning-mode all ``` -This will include all dependencies in the JAR. +This will include all dependencies in the JAR. One can skip executing the tests +as well. # Running Tests @@ -98,6 +105,12 @@ mvn test -DexcludedGroups= -Dgroups=flaky,slow mvn test '-Dgroups=!slow' ``` +```bash +./gradlew test +./gradlew test -DexcludeTags= -DincludeTags=flaky,slow +./gradlew test '-DincludeTags=!slow' +``` + Please note that some tests may fail as sites change and our rippers become out of date. Start by building and testing a released version of RipMe and then ensure that any changes you make do not cause more tests to break. diff --git a/build.gradle.kts b/build.gradle.kts new file mode 100644 index 000000000..4acf0119e --- /dev/null +++ b/build.gradle.kts @@ -0,0 +1,100 @@ +plugins { + id("fr.brouillard.oss.gradle.jgitver") version "0.9.1" + id("jacoco") + id("java") + id("maven-publish") +} + +repositories { + mavenLocal() + mavenCentral() +} + +dependencies { + implementation("org.java-websocket:Java-WebSocket:1.5.1") + implementation("org.jsoup:jsoup:1.8.1") + implementation("org.json:json:20190722") + implementation("commons-configuration:commons-configuration:1.7") + implementation("log4j:log4j:1.2.17") + implementation("commons-cli:commons-cli:1.2") + implementation("commons-io:commons-io:1.3.2") + implementation("org.apache.httpcomponents:httpclient:4.3.6") + implementation("org.apache.httpcomponents:httpmime:4.3.3") + implementation("org.graalvm.js:js:20.1.0") + testImplementation(enforcedPlatform("org.junit:junit-bom:5.6.2")) + testImplementation("org.junit.jupiter:junit-jupiter") + testImplementation("junit:junit:4.13") +} + +group = "com.rarchives.ripme" +version = "1.7.94" +description = "ripme" + +jgitver { + gitCommitIDLength = 8 + nonQualifierBranches = "main,master" + useGitCommitID = true +} + +java { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 +} + +tasks.withType { + manifest { + attributes["Main-Class"] = "com.rarchives.ripme.App" + } + + // To add all of the dependencies otherwise a "NoClassDefFoundError" error + from(sourceSets.main.get().output) + + dependsOn(configurations.runtimeClasspath) + from({ + configurations.runtimeClasspath.get().filter { it.name.endsWith("jar") }.map { zipTree(it) } + }) +} + +publishing { + publications { + create("maven") { + from(components["java"]) + } + } +} + +tasks.withType { + options.encoding = "UTF-8" +} + +tasks.test { + useJUnitPlatform { + // gradle-6.5.1 not yet allows passing this as parameter, so exclude it + excludeTags("flaky","slow") + includeEngines("junit-jupiter") + includeEngines("junit-vintage") + } + finalizedBy(tasks.jacocoTestReport) // report is always generated after tests run +} + +tasks.register("testAll") { + useJUnitPlatform { + includeTags("flaky","slow") + } +} + +// make all archive tasks in the build reproducible +tasks.withType().configureEach { + isPreserveFileTimestamps = false + isReproducibleFileOrder = true +} + +tasks.jacocoTestReport { + dependsOn(tasks.test) // tests are required to run before generating the report + reports { + xml.isEnabled = false + csv.isEnabled = false + html.destination = file("${buildDir}/jacocoHtml") + } +} + diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..62d4c0535 Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..442d9132e --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.3-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100755 index 000000000..fbd7c5158 --- /dev/null +++ b/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 000000000..a9f778a7a --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,104 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/java b/java new file mode 100644 index 000000000..e69de29bb diff --git a/pom.xml b/pom.xml index fb1bb42e5..ccfa46a94 100644 --- a/pom.xml +++ b/pom.xml @@ -83,6 +83,11 @@ httpmime 4.3.3 + + org.java-websocket + Java-WebSocket + 1.5.1 + @@ -132,7 +137,7 @@ org.jacoco jacoco-maven-plugin - 0.8.5 + 0.8.6 prepare-agent diff --git a/settings.gradle.kts b/settings.gradle.kts new file mode 100644 index 000000000..25d894516 --- /dev/null +++ b/settings.gradle.kts @@ -0,0 +1 @@ +rootProject.name = "ripme" diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index ba1104eb1..c05fe0f97 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -282,7 +282,14 @@ public void run() { logger.debug("IOException", e); logger.error("[!] " + Utils.getLocalizedString("exception.while.downloading.file") + ": " + url + " - " + e.getMessage()); - } finally { + } catch (NullPointerException npe){ + + logger.error("[!] " + Utils.getLocalizedString("failed.to.download") + " for URL " + url); + observer.downloadErrored(url, + Utils.getLocalizedString("failed.to.download") + " " + url.toExternalForm()); + return; + + }finally { // Close any open streams try { if (bis != null) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/CyberdropRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/CyberdropRipper.java new file mode 100644 index 000000000..f288592a1 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/CyberdropRipper.java @@ -0,0 +1,60 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.*; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +public class CyberdropRipper extends AbstractHTMLRipper { + + public CyberdropRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "cyberdrop"; + } + + @Override + protected Document getFirstPage() throws IOException { + return Http.url(url).get(); + } + + @Override + public String getDomain() { + return "cyberdrop.me"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^https?://cyberdrop\\.me/a/([a-zA-Z0-9]+).*?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected cyberdrop.me URL format: " + + "https://cyberdrop.me/a/xxxxxxxx - got " + url + "instead"); + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } + + @Override + protected List getURLsFromPage(Document page) { + ArrayList urls = new ArrayList<>(); + for (Element element: page.getElementsByClass("image")) { + urls.add(element.attr("href")); + } + return urls; + } +} \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java index 9b586b9af..b44d34d47 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java @@ -4,10 +4,12 @@ import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.rarchives.ripme.utils.Utils; import org.jsoup.Connection.Response; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -23,7 +25,7 @@ public class EromeRipper extends AbstractHTMLRipper { boolean rippingProfile; - + private HashMap cookies = new HashMap<>(); public EromeRipper (URL url) throws IOException { super(url); @@ -31,12 +33,12 @@ public EromeRipper (URL url) throws IOException { @Override public String getDomain() { - return "erome.com"; + return "erome.com"; } @Override public String getHost() { - return "erome"; + return "erome"; } @Override @@ -67,19 +69,19 @@ public List getAlbumsToQueue(Document doc) { @Override public String getAlbumTitle(URL url) throws MalformedURLException { - try { - // Attempt to use album title as GID - Element titleElement = getFirstPage().select("meta[property=og:title]").first(); - String title = titleElement.attr("content"); - title = title.substring(title.lastIndexOf('/') + 1); - return getHost() + "_" + getGID(url) + "_" + title.trim(); - } catch (IOException e) { - // Fall back to default album naming convention - LOGGER.info("Unable to find title at " + url); - } catch (NullPointerException e) { - return getHost() + "_" + getGID(url); - } - return super.getAlbumTitle(url); + try { + // Attempt to use album title as GID + Element titleElement = getFirstPage().select("meta[property=og:title]").first(); + String title = titleElement.attr("content"); + title = title.substring(title.lastIndexOf('/') + 1); + return getHost() + "_" + getGID(url) + "_" + title.trim(); + } catch (IOException e) { + // Fall back to default album naming convention + LOGGER.info("Unable to find title at " + url); + } catch (NullPointerException e) { + return getHost() + "_" + getGID(url); + } + return super.getAlbumTitle(url); } @Override @@ -96,9 +98,11 @@ public List getURLsFromPage(Document doc) { @Override public Document getFirstPage() throws IOException { + this.setAuthCookie(); Response resp = Http.url(this.url) - .ignoreContentType() - .response(); + .cookies(cookies) + .ignoreContentType() + .response(); return resp.parse(); } @@ -124,18 +128,18 @@ public String getGID(URL url) throws MalformedURLException { private List getMediaFromPage(Document doc) { List results = new ArrayList<>(); for (Element el : doc.select("img.img-front")) { - if (el.hasAttr("src")) { - if (el.attr("src").startsWith("https:")) { - results.add(el.attr("src")); - } else { - results.add("https:" + el.attr("src")); - } - } else if (el.hasAttr("data-src")) { - //to add images that are not loaded( as all images are lasyloaded as we scroll). - results.add(el.attr("data-src")); - } - - } + if (el.hasAttr("src")) { + if (el.attr("src").startsWith("https:")) { + results.add(el.attr("src")); + } else { + results.add("https:" + el.attr("src")); + } + } else if (el.hasAttr("data-src")) { + //to add images that are not loaded( as all images are lasyloaded as we scroll). + results.add(el.attr("data-src")); + } + + } for (Element el : doc.select("source[label=HD]")) { if (el.attr("src").startsWith("https:")) { results.add(el.attr("src")); @@ -152,7 +156,22 @@ private List getMediaFromPage(Document doc) { results.add("https:" + el.attr("src")); } } + + if (results.size() == 0) { + if (cookies.isEmpty()) { + LOGGER.warn("You might try setting erome.laravel_session manually " + + "if you think this page definitely contains media."); + } + } + return results; } + private void setAuthCookie() { + String sessionId = Utils.getConfigString("erome.laravel_session", null); + if (sessionId != null) { + cookies.put("laravel_session", sessionId); + } + } + } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java index 37b2d5ae1..c542c6dcf 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java @@ -57,7 +57,7 @@ public URL sanitizeURL(URL url) throws MalformedURLException { } public boolean isProfile() { - Pattern p = Pattern.compile("^https?://[wm.]*gfycat\\.com/@([a-zA-Z0-9]+).*$"); + Pattern p = Pattern.compile("^https?://[wm.]*gfycat\\.com/@([a-zA-Z0-9\\.\\-\\_]+).*$"); Matcher m = p.matcher(url.toExternalForm()); return m.matches(); } @@ -79,11 +79,11 @@ public void downloadURL(URL url, int index) { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://(thumbs\\.|[wm\\.]*)gfycat\\.com/@?([a-zA-Z0-9]+).*$"); + Pattern p = Pattern.compile("^https?://(?:thumbs\\.|[wm\\.]*)gfycat\\.com/@?([a-zA-Z0-9\\.\\-\\_]+).*$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) - return m.group(2); + return m.group(1); throw new MalformedURLException( "Expected gfycat.com format: " diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java index 56ce0d2fa..ca709418a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaiNexusRipper.java @@ -4,27 +4,22 @@ import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; +import java.util.Base64; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.rarchives.ripme.utils.Http; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import com.rarchives.ripme.ripper.AbstractJSONRipper; +import org.jsoup.nodes.DataNode; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; -import com.rarchives.ripme.utils.Http; -import com.rarchives.ripme.utils.Utils; - -public class HentaiNexusRipper extends AbstractHTMLRipper { - - private Document firstPage; - private DownloadThreadPool hentainexusThreadPool = new DownloadThreadPool("hentainexus"); - @Override - public DownloadThreadPool getThreadPool() { - return hentainexusThreadPool; - } +public class HentaiNexusRipper extends AbstractJSONRipper { public HentaiNexusRipper(URL url) throws IOException { super(url); @@ -34,7 +29,6 @@ public HentaiNexusRipper(URL url) throws IOException { public String getHost() { return "hentainexus"; } - @Override public String getDomain() { return "hentainexus.com"; @@ -42,88 +36,148 @@ public String getDomain() { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("https?://hentainexus\\.com/view/([a-zA-Z0-9_\\-%]*)/?$"); + /* + Valid URLs are /view/id, /read/id and those 2 with #pagenumber + https://hentainexus.com/view/9202 + https://hentainexus.com/read/9202 + https://hentainexus.com/view/9202#001 + https://hentainexus.com/read/9202#001 + */ + + Pattern p = Pattern.compile("^https?://hentainexus\\.com/(?:view|read)/([0-9]+)(?:\\#[0-9]+)*$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); } throw new MalformedURLException("Expected hentainexus.com URL format: " + - "hentainexus.com/view/NUMBER - got " + url + " instead"); + "hentainexus.com/view/id OR hentainexus.com/read/id - got " + url + "instead"); } @Override - public Document getFirstPage() throws IOException { - // "url" is an instance field of the superclass - if (firstPage == null) { - firstPage = Http.url(url).get(); - } - return firstPage; + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); } - @Override - public List getURLsFromPage(Document doc) { - List imageURLs = new ArrayList<>(); - Elements thumbs = doc.select("div.is-multiline > div.column > a"); - for (Element el : thumbs) { - imageURLs.add("https://" + getDomain() + el.attr("href")); - } - return imageURLs; - } @Override - public String getAlbumTitle(URL url) throws MalformedURLException { - try { - Document gallery = Http.url(url).get(); - return getHost() + "_" + gallery.select("h1.title").text(); - } catch (IOException e) { - LOGGER.info("Falling back"); + protected List getURLsFromJSON(JSONObject json) throws JSONException { + + List urlList = new ArrayList<>(); + + JSONArray imagesList = json.getJSONArray("f"); + String host = json.getString("b"); + String folder = json.getString("r"); + String id = json.getString("i"); + + for (Object singleImage : imagesList) { + String hashTMP = ((JSONObject) singleImage).getString("h"); + String fileNameTMP = ((JSONObject) singleImage).getString("p"); + String imageUrlTMP = String.format("%s%s%s/%s/%s",host,folder,hashTMP,id,fileNameTMP); + urlList.add(imageUrlTMP); } - return super.getAlbumTitle(url); + return urlList; } @Override - public void downloadURL(URL url, int index) { - HentaiNexusImageThread t = new HentaiNexusImageThread(url, index); - hentainexusThreadPool.addThread(t); + protected JSONObject getFirstPage() throws IOException { + String jsonEncodedString = getJsonEncodedStringFromPage(); + String jsonDecodedString = decodeJsonString(jsonEncodedString); + return new JSONObject(jsonDecodedString); } - /** - * Helper class to find and download images found on "image" pages - */ - private class HentaiNexusImageThread extends Thread { - private URL url; - private int index; - - HentaiNexusImageThread(URL url, int index) { - super(); - this.url = url; - this.index = index; + public String getJsonEncodedStringFromPage() throws MalformedURLException, IOException + { + // Image data only appears on the /read/ page and not on the /view/ one. + URL readUrl = new URL(String.format("http://hentainexus.com/read/%s",getGID(url))); + Document document = Http.url(readUrl).response().parse(); + + for (Element scripts : document.getElementsByTag("script")) { + for (DataNode dataNode : scripts.dataNodes()) { + if (dataNode.getWholeData().contains("initReader")) { + // Extract JSON encoded string from the JavaScript initReader() call. + String data = dataNode.getWholeData().trim().replaceAll("\\r|\\n|\\t",""); + + Pattern p = Pattern.compile(".*?initReader\\(\"(.*?)\",.*?\\).*?"); + Matcher m = p.matcher(data); + if (m.matches()) { + return m.group(1); + } + } + } } + return ""; + } - @Override - public void run() { - fetchImage(); + public String decodeJsonString(String jsonEncodedString) + { + /* + The initReader() JavaScript function accepts 2 parameters: a weird string and the window title (we can ignore this). + The weird string is a JSON string with some bytes shifted and swapped around and then encoded in base64. + The following code is a Java adaptation of the initRender() JavaScript function after manual deobfuscation. + */ + + byte[] jsonBytes = Base64.getDecoder().decode(jsonEncodedString); + + ArrayList unknownArray = new ArrayList(); + ArrayList indexesToUse = new ArrayList<>(); + + for (int i = 0x2; unknownArray.size() < 0x10; ++i) { + if (!indexesToUse.contains(i)) { + unknownArray.add(i); + for (int j = i << 0x1; j <= 0x100; j += i) { + if (!indexesToUse.contains(j)) { + indexesToUse.add(j); + } + } + } } - private void fetchImage() { - try { - Document doc = Http.url(url).retries(3).get(); - Elements images = doc.select("figure.image > img"); - if (images.isEmpty()) { - LOGGER.warn("Image not found at " + this.url); - return; - } - Element image = images.first(); - String imgsrc = image.attr("src"); - String prefix = ""; - if (Utils.getConfigBoolean("download.save_order", true)) { - prefix = String.format("%03d_", index); - } - addURLToDownload(new URL(imgsrc), prefix); - } catch (IOException e) { - LOGGER.error("[!] Exception while loading/parsing " + this.url, e); + byte magicByte = 0x0; + for (int i = 0x0; i < 0x40; i++) { + magicByte = (byte) (signedToUnsigned(magicByte) ^ signedToUnsigned(jsonBytes[i])); + for (int j = 0x0; j < 0x8; j++) { + long unsignedMagicByteTMP = signedToUnsigned(magicByte); + magicByte = (byte) ((unsignedMagicByteTMP & 0x1) == 1 ? unsignedMagicByteTMP >>> 0x1 ^ 0xc : unsignedMagicByteTMP >>> 0x1); } } + + magicByte = (byte) (magicByte & 0x7); + ArrayList newArray = new ArrayList(); + + for (int i = 0x0; i < 0x100; i++) { + newArray.add(i); + } + + int newIndex = 0, backup = 0; + for (int i = 0x0; i < 0x100; i++) { + newIndex = (newIndex + newArray.get(i) + (int) signedToUnsigned(jsonBytes[i % 0x40])) % 0x100; + backup = newArray.get(i); + newArray.set(i, newArray.get(newIndex)); + newArray.set(newIndex, backup); + } + + int magicByteTranslated = (int) unknownArray.get(magicByte); + int index1 = 0x0, index2 = 0x0, index3 = 0x0, swap1 = 0x0, xorNumber = 0x0; + String decodedJsonString = ""; + + for (int i = 0x0; i + 0x40 < jsonBytes.length; i++) { + index1 = (index1 + magicByteTranslated) % 0x100; + index2 = (index3 + newArray.get((index2 + newArray.get(index1)) % 0x100)) % 0x100; + index3 = (index3 + index1 + newArray.get(index1)) % 0x100; + swap1 = newArray.get(index1); + newArray.set(index1, newArray.get(index2)); + newArray.set(index2,swap1); + xorNumber = newArray.get((index2 + newArray.get((index1 + newArray.get((xorNumber + index3) % 0x100)) % 0x100)) % 0x100); + decodedJsonString += Character.toString((char) signedToUnsigned((jsonBytes[i + 0x40] ^ xorNumber))); + } + + return decodedJsonString; } -} + + + private static long signedToUnsigned(int signed) { + return (byte) signed & 0xFF; + } + +} \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java new file mode 100644 index 000000000..7e0c1c46e --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java @@ -0,0 +1,293 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.net.*; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.java_websocket.client.WebSocketClient; + +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URLEncodedUtils; +import org.java_websocket.handshake.ServerHandshake; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import com.rarchives.ripme.ripper.AbstractJSONRipper; + +public class ScrolllerRipper extends AbstractJSONRipper { + + public ScrolllerRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return "scrolller"; + } + @Override + public String getDomain() { + return "scrolller.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + // Typical URL is: https://scrolller.com/r/subreddit + // Parameters like "filter" and "sort" can be passed (ex: https://scrolller.com/r/subreddit?filter=xxx&sort=yyyy) + Pattern p = Pattern.compile("^https?://scrolller\\.com/r/([a-zA-Z0-9]+).*?$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + throw new MalformedURLException("Expected scrolller.com URL format: " + + "scrolller.com/r/subreddit OR scroller.com/r/subreddit?filter= - got " + url + "instead"); + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } + + + private JSONObject prepareQuery(String iterator, String gid, String sortByString) throws IOException, URISyntaxException { + + String QUERY_NOSORT = "query SubredditQuery( $url: String! $filter: SubredditPostFilter $iterator: String ) { getSubreddit(url: $url) { children( limit: 50 iterator: $iterator filter: $filter ) { iterator items { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } } } }"; + String QUERY_SORT = "subscription SubredditSubscription( $url: String! $sortBy: SubredditSortBy $timespan: SubredditTimespan $iterator: String $limit: Int $filter: SubredditPostFilter ) { fetchSubreddit( url: $url sortBy: $sortBy timespan: $timespan iterator: $iterator limit: $limit filter: $filter ) { __typename ... on Subreddit { __typename url title secondaryTitle description createdAt isNsfw subscribers isComplete itemCount videoCount pictureCount albumCount isFollowing } ... on SubredditPost { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } ... on Iterator { iterator } ... on Error { message } } }"; + + String filterString = convertFilterString(getParameter(this.url,"filter")); + + JSONObject variablesObject = new JSONObject().put("url", String.format("/r/%s", gid)).put("sortBy", sortByString.toUpperCase()); + JSONObject finalQueryObject = new JSONObject().put("variables", variablesObject).put("query", sortByString.equals("") ? QUERY_NOSORT : QUERY_SORT); + + if (iterator != null) { + // Iterator is not present on the first page + variablesObject.put("iterator", iterator); + } + if (!filterString.equals("NOFILTER")) { + variablesObject.put("filter", filterString); + } + + return sortByString.equals("") ? getPosts(finalQueryObject) : getPostsSorted(finalQueryObject); + + } + + + public String convertFilterString(String filterParameter) { + // Converts the ?filter= parameter of the URL to one that can be used in the GraphQL query + // I could basically remove the last "s" and call toUpperCase instead of this switch statement but this looks easier to read. + switch (filterParameter.toLowerCase()) { + case "pictures": + return "PICTURE"; + case "videos": + return "VIDEO"; + case "albums": + return "ALBUM"; + case "": + return "NOFILTER"; + default: + LOGGER.error(String.format("Invalid filter %s using no filter",filterParameter)); + return ""; + } + } + + public String getParameter(URL url, String parameter) throws MalformedURLException { + // Gets passed parameters from the URL + String toReplace = String.format("https://scrolller.com/r/%s?",getGID(url)); + List args= URLEncodedUtils.parse(url.toExternalForm(), Charset.defaultCharset()); + for (NameValuePair arg:args) { + // First parameter contains part of the url so we have to remove it + // Ex: for the url https://scrolller.com/r/CatsStandingUp?filter=xxxx&sort=yyyy + // 1) arg.getName() => https://scrolller.com/r/CatsStandingUp?filter + // 2) arg.getName() => sort + + if (arg.getName().replace(toReplace,"").toLowerCase().equals((parameter))) { + return arg.getValue(); + } + } + return ""; + } + + private JSONObject getPosts(JSONObject data) { + // The actual GraphQL query call + + try { + String url = "https://api.scrolller.com/api/v2/graphql"; + + URL obj = new URL(url); + HttpURLConnection conn = (HttpURLConnection) obj.openConnection(); + conn.setReadTimeout(5000); + conn.addRequestProperty("Accept-Language", "en-US,en;q=0.8"); + conn.addRequestProperty("User-Agent", "Mozilla"); + conn.addRequestProperty("Referer", "scrolller.com"); + + conn.setDoOutput(true); + + OutputStreamWriter w = new OutputStreamWriter(conn.getOutputStream(), "UTF-8"); + + w.write(data.toString()); + w.close(); + + BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream())); + String inputLine; + StringBuffer jsonString = new StringBuffer(); + + while ((inputLine = in.readLine()) != null) { + jsonString.append(inputLine); + } + + in.close(); + conn.disconnect(); + + return new JSONObject(jsonString.toString()); + + } catch (Exception e) { + e.printStackTrace(); + } + + return new JSONObject("{}"); + } + + private JSONObject getPostsSorted(JSONObject data) throws MalformedURLException { + + // The actual GraphQL query call (if sort parameter is present) + try { + + ArrayList postsJsonStrings = new ArrayList<>(); + + WebSocketClient wsc = new WebSocketClient(new URI("wss://api.scrolller.com/api/v2/graphql")) { + @Override + public void onOpen(ServerHandshake serverHandshake) { + // As soon as the WebSocket connects send our query + this.send(data.toString()); + } + + @Override + public void onMessage(String s) { + postsJsonStrings.add(s); + if (new JSONObject(s).getJSONObject("data").getJSONObject("fetchSubreddit").has("iterator")) { + this.close(); + } + } + + @Override + public void onClose(int i, String s, boolean b) { + } + + @Override + public void onError(Exception e) { + LOGGER.error(String.format("WebSocket error, server reported %s", e.getMessage())); + } + }; + wsc.connect(); + + while (!wsc.isClosed()) { + // Posts list is not over until the connection closes. + } + + JSONObject finalObject = new JSONObject(); + JSONArray posts = new JSONArray(); + + // Iterator is the last object in the post list, let's duplicate it in his own object for clarity. + finalObject.put("iterator", new JSONObject(postsJsonStrings.get(postsJsonStrings.size()-1))); + + for (String postString : postsJsonStrings) { + posts.put(new JSONObject(postString)); + } + finalObject.put("posts", posts); + + if (finalObject.getJSONArray("posts").length() == 1 && !finalObject.getJSONArray("posts").getJSONObject(0).getJSONObject("data").getJSONObject("fetchSubreddit").has("mediaSources")) { + // Only iterator, no posts. + return null; + } + + return finalObject; + + + } catch (URISyntaxException ue) { + // Nothing to catch, it's an hardcoded URI. + } + + return null; + } + + + @Override + protected List getURLsFromJSON(JSONObject json) throws JSONException { + + boolean sortRequested = json.has("posts"); + + int bestArea = 0; + String bestUrl = ""; + List list = new ArrayList<>(); + + JSONArray itemsList = sortRequested ? json.getJSONArray("posts") : json.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").getJSONArray("items"); + + for (Object item : itemsList) { + + if (sortRequested && !((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").has("mediaSources")) { + continue; + } + + JSONArray sourcesTMP = sortRequested ? ((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").getJSONArray("mediaSources") : ((JSONObject) item).getJSONArray("mediaSources"); + for (Object sourceTMP : sourcesTMP) + { + int widthTMP = ((JSONObject) sourceTMP).getInt("width"); + int heightTMP = ((JSONObject) sourceTMP).getInt("height"); + int areaTMP = widthTMP * heightTMP; + + if (areaTMP > bestArea) { + bestArea = widthTMP; + bestUrl = ((JSONObject) sourceTMP).getString("url"); + } + } + list.add(bestUrl); + bestUrl = ""; + bestArea = 0; + } + + return list; + } + + @Override + protected JSONObject getFirstPage() throws IOException { + try { + return prepareQuery(null, this.getGID(url), getParameter(url,"sort")); + } catch (URISyntaxException e) { + LOGGER.error(String.format("Error obtaining first page: %s", e.getMessage())); + return null; + } + } + + @Override + public JSONObject getNextPage(JSONObject source) throws IOException { + // Every call the the API contains an "iterator" string that we need to pass to the API to get the next page + // Checking if iterator is null is not working for some reason, hence why the weird "iterator.toString().equals("null")" + + Object iterator = null; + if (source.has("iterator")) { + // Sort requested, custom JSON. + iterator = source.getJSONObject("iterator").getJSONObject("data").getJSONObject("fetchSubreddit").get("iterator"); + } else { + iterator = source.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").get("iterator"); + } + + if (!iterator.toString().equals("null")) { + // Need to change page. + try { + return prepareQuery(iterator.toString(), this.getGID(url), getParameter(url,"sort")); + } catch (URISyntaxException e) { + LOGGER.error(String.format("Error changing page: %s", e.getMessage())); + return null; + } + } else { + return null; + } + } +} \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SoundgasmRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SoundgasmRipper.java new file mode 100644 index 000000000..65a1f1deb --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SoundgasmRipper.java @@ -0,0 +1,68 @@ +package com.rarchives.ripme.ripper.rippers; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class SoundgasmRipper extends AbstractHTMLRipper { + + private static final String HOST = "soundgasm.net"; + + public SoundgasmRipper(URL url) throws IOException { + super(new URL(url.toExternalForm())); + } + + @Override + protected String getDomain() { return "soundgasm.net"; } + + @Override + public String getHost() { return "soundgasm"; } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^/u/([a-zA-Z0-9_-]+)/([a-zA-Z0-9_-]+).*$"); + Matcher m = p.matcher(url.getFile()); + if (m.find()) { + return m.group(m.groupCount()); + } + throw new MalformedURLException( + "Expected soundgasm.net format: " + + "soundgasm.net/u/username/id or " + + " Got: " + url); + } + + @Override + public Document getFirstPage() throws IOException { + return Http.url(url).get(); + } + + @Override + public List getURLsFromPage(Document page) { + List res = new ArrayList<>(); + + Elements script = page.select("script"); + Pattern p = Pattern.compile("m4a\\:\\s\"(https?:.*)\\\""); + + for (Element e: script) { + Matcher m = p.matcher(e.data()); + if (m.find()) { res.add(m.group(1)); } + } + return res; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } + +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index 7ae570f37..10ff7d8ef 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -48,8 +48,7 @@ public URL sanitizeURL(URL url) throws MalformedURLException { return url; } String URLToReturn = url.toExternalForm(); - URLToReturn = URLToReturn.replaceAll("https?://\\w?\\w?\\.?xhamster\\.", "https://m.xhamster."); - URLToReturn = URLToReturn.replaceAll("https?://xhamster2\\.", "https://m.xhamster2."); + URLToReturn = URLToReturn.replaceAll("https?://\\w?\\w?\\.?xhamster([^<]*)\\.", "https://m.xhamster$1."); URL san_url = new URL(URLToReturn); LOGGER.info("sanitized URL is " + san_url.toExternalForm()); return san_url; @@ -57,23 +56,23 @@ public URL sanitizeURL(URL url) throws MalformedURLException { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://[\\w\\w.]*xhamster2?\\.com/photos/gallery/.*?(\\d+)$"); + Pattern p = Pattern.compile("^https?://([\\w\\w]*\\.)?xhamster([^<]*)\\.(com|one|desi)/photos/gallery/.*?(\\d+)$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { - return m.group(1); + return m.group(4); } - p = Pattern.compile("^https?://[\\w\\w.]*xhamster2?\\.com/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); + p = Pattern.compile("^https?://[\\w\\w.]*xhamster([^<]*)\\.(com|one|desi)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); m = p.matcher(url.toExternalForm()); if (m.matches()) { return "user_" + m.group(1); } - p = Pattern.compile("^https?://.*xhamster2?\\.com/(movies|videos)/(.*)$"); + p = Pattern.compile("^https?://.*xhamster([^<]*)\\.(com|one|desi)/(movies|videos)/(.*$)"); m = p.matcher(url.toExternalForm()); if (m.matches()) { - return m.group(2); + return m.group(4); } - throw new MalformedURLException( + throw new MalformedURLException( "Expected xhamster.com gallery formats: " + "xhamster.com/photos/gallery/xxxxx-#####" + " Got: " + url); @@ -97,7 +96,7 @@ public boolean hasQueueSupport() { @Override public boolean pageContainsAlbums(URL url) { - Pattern p = Pattern.compile("^https?://[\\w\\w.]*xhamster\\.com/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); + Pattern p = Pattern.compile("^https?://[\\w\\w.]*xhamster([^<]*)\\.(com|one|desi)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); Matcher m = p.matcher(url.toExternalForm()); LOGGER.info("Checking if page has albums"); LOGGER.info(m.matches()); @@ -113,17 +112,17 @@ public Document getFirstPage() throws IOException { @Override public boolean canRip(URL url) { - Pattern p = Pattern.compile("^https?://([\\w\\w]*\\.)?xhamster2?\\.(com|one|desi)/photos/gallery/.*?(\\d+)$"); + Pattern p = Pattern.compile("^https?://([\\w\\w]*\\.)?xhamster([^<]*)\\.(com|one|desi)/photos/gallery/.*?(\\d+)$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return true; } - p = Pattern.compile("^https?://[\\w\\w.]*xhamster2?\\.(com|one|desi)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); + p = Pattern.compile("^https?://[\\w\\w.]*xhamster([^<]*)\\.(com|one|desi)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); m = p.matcher(url.toExternalForm()); if (m.matches()) { return true; } - p = Pattern.compile("^https?://.*xhamster2?\\.(com|one|desi)/(movies|videos)/.*$"); + p = Pattern.compile("^https?://.*xhamster([^<]*)\\.(com|one|desi)/(movies|videos)/(.*$)"); m = p.matcher(url.toExternalForm()); if (m.matches()) { return true; @@ -132,18 +131,17 @@ public boolean canRip(URL url) { } private boolean isVideoUrl(URL url) { - Pattern p = Pattern.compile("^https?://.*xhamster2?\\.(com|one|desi)/(movies|videos)/.*$"); + Pattern p = Pattern.compile("^https?://.*xhamster([^<]*)\\.(com|one|desi)/(movies|videos)/(.*$)"); Matcher m = p.matcher(url.toExternalForm()); return m.matches(); } @Override public Document getNextPage(Document doc) throws IOException { - if (doc.select("a[data-page=next]").first() != null) { - String nextPageUrl = doc.select("a[data-page=next]").first().attr("href"); + if (doc.select("a.prev-next-list-link").first() != null) { + String nextPageUrl = doc.select("a.prev-next-list-link").first().attr("href"); if (nextPageUrl.startsWith("http")) { - nextPageUrl = nextPageUrl.replaceAll("https?://\\w?\\w?\\.?xhamster\\.", "https://m.xhamster."); - nextPageUrl = nextPageUrl.replaceAll("https?://xhamster2\\.", "https://m.xhamster2."); + nextPageUrl = nextPageUrl.replaceAll("https?://\\w?\\w?\\.?xhamster([^<]*)\\.", "https://m.xhamster$1."); return Http.url(nextPageUrl).get(); } } @@ -156,25 +154,43 @@ public List getURLsFromPage(Document doc) { LOGGER.debug("Checking for urls"); List result = new ArrayList<>(); if (!isVideoUrl(url)) { - for (Element page : doc.select("div.picture_view > div.pictures_block > div.items > div.item-container > a.item")) { - // Make sure we don't waste time running the loop if the ripper has been stopped - if (isStopped()) { - break; - } - String pageWithImageUrl = page.attr("href"); - try { - // This works around some redirect fuckery xhamster likes to do where visiting m.xhamster.com sends to - // the page chamster.com but displays the mobile site from m.xhamster.com - pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster\\.", "://m.xhamster."); - pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster2\\.", "://m.xhamster."); - String image = Http.url(new URL(pageWithImageUrl)).get().select("a > img#photoCurr").attr("src"); - downloadFile(image); - } catch (IOException e) { - LOGGER.error("Was unable to load page " + pageWithImageUrl); - } - } + if (!doc.select("div.picture_view > div.pictures_block > div.items > div.item-container > a.item").isEmpty()) { + // Old HTML structure is still present at some places + for (Element page : doc.select("div.picture_view > div.pictures_block > div.items > div.item-container > a.item")) { + // Make sure we don't waste time running the loop if the ripper has been stopped + if (isStopped()) { + break; + } + String pageWithImageUrl = page.attr("href"); + try { + // This works around some redirect fuckery xhamster likes to do where visiting m.xhamster.com sends to + // the page chamster.com but displays the mobile site from m.xhamster.com + pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster([^<]*)\\.", "://m.xhamster$1."); + String image = Http.url(new URL(pageWithImageUrl)).get().select("a > img#photoCurr").attr("src"); + result.add(image); + downloadFile(image); + } catch (IOException e) { + LOGGER.error("Was unable to load page " + pageWithImageUrl); + } + } + } else { + // New HTML structure + for (Element page : doc.select("div#photo-slider > div#photo_slider > a")) { + // Make sure we don't waste time running the loop if the ripper has been stopped + if (isStopped()) { + break; + } + String image = page.attr("href"); + // This works around some redirect fuckery xhamster likes to do where visiting m.xhamster.com sends to + // the page chamster.com but displays the mobile site from m.xhamster.com + image = image.replaceAll("://xhamster([^<]*)\\.", "://m.xhamster$1."); + result.add(image); + downloadFile(image); + } + } } else { String imgUrl = doc.select("div.player-container > a").attr("href"); + result.add(imgUrl); downloadFile(imgUrl); } return result; @@ -193,7 +209,7 @@ private void downloadFile(String url) { LOGGER.error("The url \"" + url + "\" is malformed"); } } - + @Override public String getAlbumTitle(URL url) throws MalformedURLException { try { diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index 3fcb71c2e..503a81653 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -14,6 +14,7 @@ import com.rarchives.ripme.ripper.rippers.RedgifsRipper; import com.rarchives.ripme.ripper.rippers.VidbleRipper; import com.rarchives.ripme.ripper.rippers.GfycatRipper; +import com.rarchives.ripme.ripper.rippers.SoundgasmRipper; import org.apache.commons.lang.math.NumberUtils; import org.apache.log4j.Logger; import org.jsoup.Jsoup; @@ -127,6 +128,20 @@ else if (url.toExternalForm().contains("erome.com")) { } return result; } + else if (url.toExternalForm().contains("soundgasm.net")) { + try { + logger.info("Getting soundgasm page " + url); + SoundgasmRipper r = new SoundgasmRipper(url); + Document tempDoc = r.getFirstPage(); + for (String u : r.getURLsFromPage(tempDoc)) { + result.add(new URL(u)); + } + } catch (IOException e) { + // Do nothing + logger.warn("Exception while retrieving soundgasm page:", e); + } + return result; + } Pattern p = Pattern.compile("https?://i.reddituploads.com/([a-zA-Z0-9]+)\\?.*"); Matcher m = p.matcher(url.toExternalForm()); diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index a009c7a1e..7f9d99aa7 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -2,7 +2,6 @@ import java.io.File; import java.io.FileNotFoundException; -import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; @@ -22,7 +21,6 @@ import java.util.Collections; import java.util.Enumeration; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; @@ -32,7 +30,6 @@ import java.util.jar.JarFile; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Stream; import javax.sound.sampled.AudioSystem; import javax.sound.sampled.Clip; @@ -359,7 +356,7 @@ public static String stripURLParameter(String url, String parameter) { if (wasFirstParam) { c = "?"; } - url = url.substring(0, paramIndex) + c + url.substring(nextParam + 1, url.length()); + url = url.substring(0, paramIndex) + c + url.substring(nextParam + 1); } else { url = url.substring(0, paramIndex); } @@ -486,8 +483,15 @@ public static String filesystemSanitized(String text) { return text; } + /** + * Removes any potentially unsafe characters from a string and truncates it on a maximum length of 100 characters. + * Characters considered safe are alpha numerical characters as well as minus, dot, comma, underscore and whitespace. + * + * @param text The potentially unsafe text + * @return a filesystem safe string + */ public static String filesystemSafe(String text) { - text = text.replaceAll("[^a-zA-Z0-9.-]", "_").replaceAll("__", "_").replaceAll("_+$", ""); + text = text.replaceAll("[^a-zA-Z0-9-.,_ ]", ""); if (text.length() > 100) { text = text.substring(0, 99); } @@ -510,8 +514,7 @@ public static String getOriginalDirectory(String path) { return path; } - String original = path; // needs to be checked if lowercase exists - String lastPart = original.substring(index + 1).toLowerCase(); // setting lowercase to check if it exists + String lastPart = path.substring(index + 1).toLowerCase(); // setting lowercase to check if it exists // Get a List of all Directories and check its lowercase // if file exists return it @@ -525,7 +528,8 @@ public static String getOriginalDirectory(String path) { } } - return original; + // otherwise return original path + return path; } /** @@ -771,7 +775,7 @@ public static String[] getSupportedLanguages() { Path myPath; if (uri.getScheme().equals("jar")) { - FileSystem fileSystem = FileSystems.newFileSystem(uri, Collections.emptyMap()); + FileSystem fileSystem = FileSystems.newFileSystem(uri, Collections.emptyMap()); myPath = fileSystem.getPath("/"); } else { myPath = Paths.get(uri).getParent(); @@ -813,7 +817,7 @@ public static String getLocalizedString(String key) { * bar */ public static String getByteStatusText(int completionPercentage, int bytesCompleted, int bytesTotal) { - return String.valueOf(completionPercentage) + "% - " + Utils.bytesToHumanReadable(bytesCompleted) + " / " + return completionPercentage + "% - " + Utils.bytesToHumanReadable(bytesCompleted) + " / " + Utils.bytesToHumanReadable(bytesTotal); } @@ -847,7 +851,7 @@ public static boolean fuzzyExists(File folder, String fileName) { for (File file : listOfFiles) { if (file.isFile()) { - String[] filename = file.getName().split("\\.(?=[^\\.]+$)"); // split filename from it's extension + String[] filename = file.getName().split("\\.(?=[^.]+$)"); // split filename from it's extension if (filename[0].equalsIgnoreCase(fileName)) { return true; } @@ -861,15 +865,11 @@ public static String sanitizeSaveAs(String fileNameToSan) { } public static File shortenSaveAsWindows(String ripsDirPath, String fileName) throws FileNotFoundException { - // int ripDirLength = ripsDirPath.length(); - // int maxFileNameLength = 260 - ripDirLength; - // LOGGER.info(maxFileNameLength); LOGGER.error("The filename " + fileName + " is to long to be saved on this file system."); LOGGER.info("Shortening filename"); String fullPath = ripsDirPath + File.separator + fileName; // How long the path without the file name is int pathLength = ripsDirPath.length(); - int fileNameLength = fileName.length(); if (pathLength == 260) { // We've reached the max length, there's nothing more we can do throw new FileNotFoundException("File path is too long for this OS"); @@ -879,7 +879,6 @@ public static File shortenSaveAsWindows(String ripsDirPath, String fileName) thr // file extension String fileExt = saveAsSplit[saveAsSplit.length - 1]; // The max limit for paths on Windows is 260 chars - LOGGER.info(fullPath.substring(0, 259 - pathLength - fileExt.length() + 1) + "." + fileExt); fullPath = fullPath.substring(0, 259 - pathLength - fileExt.length() + 1) + "." + fileExt; LOGGER.info(fullPath); LOGGER.info(fullPath.length()); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java new file mode 100644 index 000000000..847f2abf6 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CyberdropRipperTest.java @@ -0,0 +1,53 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import com.rarchives.ripme.ripper.rippers.CyberdropRipper; +import com.rarchives.ripme.utils.Http; +import org.jsoup.nodes.Document; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class CyberdropRipperTest extends RippersTest { + @Test + public void testScrolllerGID() throws IOException { + Map testURLs = new HashMap<>(); + + testURLs.put(new URL("https://cyberdrop.me/a/n4umdBjw"), "n4umdBjw"); + testURLs.put(new URL("https://cyberdrop.me/a/iLtp4BjW"), "iLtp4BjW"); + for (URL url : testURLs.keySet()) { + CyberdropRipper ripper = new CyberdropRipper(url); + ripper.setup(); + Assertions.assertEquals(testURLs.get(url), ripper.getGID(ripper.getURL())); + deleteDir(ripper.getWorkingDir()); + } + } + + @Test + @Tag("flaky") + public void testCyberdropNumberOfFiles() throws IOException { + List testURLs = new ArrayList(); + + testURLs.add(new URL("https://cyberdrop.me/a/n4umdBjw")); + testURLs.add(new URL("https://cyberdrop.me/a/iLtp4BjW")); + for (URL url : testURLs) { + Assertions.assertTrue(willDownloadAllFiles(url)); + } + } + + public boolean willDownloadAllFiles(URL url) throws IOException { + Document doc = Http.url(url).get(); + long numberOfLinks = doc.getElementsByClass("image").stream().count(); + int numberOfFiles = Integer.parseInt(doc.getElementById("totalFilesAmount").text()); + return numberOfLinks == numberOfFiles; + } + + + +} \ No newline at end of file diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java index 019350ad4..39c146734 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java @@ -44,4 +44,13 @@ public void testGfycatAmp() throws IOException { GfycatRipper ripper = new GfycatRipper(new URL("https://gfycat.com/amp/TemptingExcellentIchthyosaurs")); testRipper(ripper); } + + /** + * Rips a Gfycat profile with special characters in username + * @throws IOException + */ + public void testGfycatSpecialChar() throws IOException { + GfycatRipper ripper = new GfycatRipper(new URL("https://gfycat.com/@rsss.kr")); + testRipper(ripper); + } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java index f448f0def..c6e2d3def 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/Hentai2readRipperTest.java @@ -4,10 +4,12 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.Hentai2readRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class Hentai2readRipperTest extends RippersTest { @Test + @Tag("flaky") public void testHentai2readAlbum() throws IOException { Hentai2readRipper ripper = new Hentai2readRipper(new URL("https://hentai2read.com/sm_school_memorial/1/")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java index cfe540fb1..00340eba7 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/HentainexusRipperTest.java @@ -2,14 +2,45 @@ import java.io.IOException; import java.net.URL; +import java.util.ArrayList; +import java.util.List; import com.rarchives.ripme.ripper.rippers.HentaiNexusRipper; +import org.json.JSONObject; +import org.junit.Assert; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class HentainexusRipperTest extends RippersTest { @Test - public void testHentaiNexusAlbum() throws IOException { - HentaiNexusRipper ripper = new HentaiNexusRipper(new URL("https://hentainexus.com/view/44")); - testRipper(ripper); + @Tag("flaky") + public void testHentaiNexusJson() throws IOException { + List testURLs = new ArrayList<>(); + testURLs.add(new URL("https://hentainexus.com/view/9202")); + testURLs.add(new URL("https://hentainexus.com/read/9202")); + testURLs.add(new URL("https://hentainexus.com/view/9202#001")); + testURLs.add(new URL("https://hentainexus.com/read/9202#001")); + + for (URL url : testURLs) { + + HentaiNexusRipper ripper = new HentaiNexusRipper(url); + + boolean testOK = false; + try { + + String jsonEncodedString = ripper.getJsonEncodedStringFromPage(); + String jsonDecodedString = ripper.decodeJsonString(jsonEncodedString); + JSONObject json = new JSONObject(jsonDecodedString); + // Fail test if JSON empty + testOK = !json.isEmpty(); + + } catch (Exception e) { + // Fail test if JSON invalid, not present or other errors + testOK = false; + } + + Assert.assertEquals(true, testOK); + } + } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java index efe57b96d..5ecfe3f61 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagebamRipperTest.java @@ -4,10 +4,12 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.ImagebamRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class ImagebamRipperTest extends RippersTest { @Test + @Tag("flaky") public void testImagebamRip() throws IOException { ImagebamRipper ripper = new ImagebamRipper(new URL("http://www.imagebam.com/gallery/488cc796sllyf7o5srds8kpaz1t4m78i")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java index 0ed0add25..19061e347 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImagefapRipperTest.java @@ -30,6 +30,7 @@ public void testImagefapAlbums() throws IOException { } } @Test + @Tag("flaky") public void testImagefapGetAlbumTitle() throws IOException { URL url = new URL("https://www.imagefap.com/gallery.php?gid=7789753"); ImagefapRipper ripper = new ImagefapRipper(url); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java index 2739f9da1..97f48a5fd 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/MotherlessRipperTest.java @@ -5,10 +5,12 @@ import com.rarchives.ripme.ripper.rippers.MotherlessRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; public class MotherlessRipperTest extends RippersTest { @Test + @Tag("flaky") public void testMotherlessAlbumRip() throws IOException { MotherlessRipper ripper = new MotherlessRipper(new URL("https://motherless.com/G1168D90")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixDotOneRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixDotOneRipperTest.java index 9d4df122e..1edf7b800 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixDotOneRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PorncomixDotOneRipperTest.java @@ -4,10 +4,12 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.PorncomixDotOneRipper; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class PorncomixDotOneRipperTest extends RippersTest { @Test + @Disabled("website down?") public void testPorncomixAlbum() throws IOException { PorncomixDotOneRipper ripper = new PorncomixDotOneRipper(new URL("https://www.porncomix.one/gallery/blacknwhite-make-america-great-again")); testRipper(ripper); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java index d0e8dd6ae..354b4e62b 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/PornhubRipperTest.java @@ -29,7 +29,7 @@ public void testGetGID() throws IOException { @Test public void testGetNextPage() throws IOException { - String baseURL = "https://www.pornhub.com/album/43902391"; + String baseURL = "https://www.pornhub.com/album/30687901"; PornhubRipper ripper = new PornhubRipper(new URL(baseURL)); Document page = Http.url(baseURL).get(); int numPagesRemaining = 1; diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java index e99df56cf..f4dbe3278 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java @@ -61,6 +61,7 @@ public void testRedditGfyBadURL() throws IOException { } @Test + @Tag("flaky") public void testRedditGallery() throws IOException{ RedditRipper ripper = new RedditRipper( new URL("https://www.reddit.com/gallery/hrrh23")); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java index 9789417df..8b45594d4 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java @@ -46,6 +46,7 @@ public void testRedgifsProfile() throws IOException { * @throws IOException */ @Test + @Disabled("test or ripper broken") public void testRedgifsSearch() throws IOException { RedgifsRipper ripper = new RedgifsRipper(new URL("https://redgifs.com/gifs/browse/little-caprice")); Document doc = ripper.getFirstPage(); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java new file mode 100644 index 000000000..c7bf3d7d8 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ScrolllerRipperTest.java @@ -0,0 +1,53 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import com.rarchives.ripme.ripper.rippers.ScrolllerRipper; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; + +public class ScrolllerRipperTest extends RippersTest { + @Test + public void testScrolllerGID() throws IOException { + Map testURLs = new HashMap<>(); + + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp"), "CatsStandingUp"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=pictures"), "CatsStandingUp"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?sort=top&filter=pictures"), "CatsStandingUp"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=pictures&sort=top"), "CatsStandingUp"); + for (URL url : testURLs.keySet()) { + ScrolllerRipper ripper = new ScrolllerRipper(url); + ripper.setup(); + Assertions.assertEquals(testURLs.get(url), ripper.getGID(ripper.getURL())); + deleteDir(ripper.getWorkingDir()); + } + } + + @Test + public void testScrolllerFilterRegex() throws IOException { + Map testURLs = new HashMap<>(); + + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp"), "NOFILTER"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=pictures"), "PICTURE"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=videos"), "VIDEO"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=albums"), "ALBUM"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?sort=top&filter=pictures"), "PICTURE"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?sort=top&filter=videos"), "VIDEO"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?sort=top&filter=albums"), "ALBUM"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=pictures&sort=top"), "PICTURE"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=videos&sort=top"), "VIDEO"); + testURLs.put(new URL("https://scrolller.com/r/CatsStandingUp?filter=albums&sort=top"), "ALBUM"); + for (URL url : testURLs.keySet()) { + ScrolllerRipper ripper = new ScrolllerRipper(url); + ripper.setup(); + Assertions.assertEquals(testURLs.get(url), ripper.convertFilterString(ripper.getParameter(ripper.getURL(),"filter"))); + deleteDir(ripper.getWorkingDir()); + } + } + + + +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SoundgasmRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SoundgasmRipperTest.java new file mode 100644 index 000000000..753e7b786 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/SoundgasmRipperTest.java @@ -0,0 +1,23 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import com.rarchives.ripme.ripper.rippers.RedditRipper; +import org.junit.Test; +import com.rarchives.ripme.ripper.rippers.SoundgasmRipper; + +import java.io.IOException; +import java.net.URL; + +public class SoundgasmRipperTest extends RippersTest { + + @Test + public void testSoundgasmURLs() throws IOException { + SoundgasmRipper ripper = new SoundgasmRipper(new URL("https://soundgasm.net/u/_Firefly_xoxo/Rambles-with-my-Lovense")); + testRipper(ripper); + } + + @Test + public void testRedditSoundgasmURL() throws IOException { + RedditRipper ripper = new RedditRipper(new URL("https://www.reddit.com/r/gonewildaudio/comments/kn1bvj/f4m_mistress_controlled_my_lovense_while_i_tried/")); + testRipper(ripper); + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java index 6a647286e..e8f217264 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/WordpressComicRipperTest.java @@ -51,6 +51,7 @@ public void test_prismblush() throws IOException { testRipper(ripper); } @Test + @Tag("flaky") public void test_konradokonski_1() throws IOException { WordpressComicRipper ripper = new WordpressComicRipper( new URL("http://www.konradokonski.com/sawdust/comic/get-up/")); @@ -58,6 +59,7 @@ public void test_konradokonski_1() throws IOException { } @Test + @Tag("flaky") public void test_konradokonski_2() throws IOException { WordpressComicRipper ripper = new WordpressComicRipper( new URL("http://www.konradokonski.com/wiory/comic/08182008/")); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java index a5f1beaa7..aaccf47c6 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java @@ -18,6 +18,7 @@ public void testXhamsterAlbum1() throws IOException { testRipper(ripper); } @Test + @Tag("flaky") public void testXhamster2Album() throws IOException { XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster2.com/photos/gallery/sexy-preggo-girls-9026608")); testRipper(ripper); @@ -28,6 +29,7 @@ public void testXhamsterAlbum2() throws IOException { testRipper(ripper); } @Test + @Tag("flaky") public void testXhamsterAlbumOneDomain() throws IOException { XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.one/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664")); testRipper(ripper); @@ -35,7 +37,7 @@ public void testXhamsterAlbumOneDomain() throws IOException { @Test @Tag("flaky") public void testXhamsterAlbumDesiDomain() throws IOException { - XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.desi/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664")); + XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster5.desi/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664")); testRipper(ripper); } @Test @@ -49,9 +51,9 @@ public void testBrazilianXhamster() throws IOException { XhamsterRipper ripper = new XhamsterRipper(new URL("https://pt.xhamster.com/photos/gallery/silvana-7105696")); testRipper(ripper); } - + @Test public void testGetGID() throws IOException { - URL url = new URL("https://xhamster.com/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664"); + URL url = new URL("https://xhamster5.desi/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664"); XhamsterRipper ripper = new XhamsterRipper(url); Assertions.assertEquals("7254664", ripper.getGID(url)); } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YoupornRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YoupornRipperTest.java index c8640cad0..bce22d628 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YoupornRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/YoupornRipperTest.java @@ -1,6 +1,7 @@ package com.rarchives.ripme.tst.ripper.rippers; import com.rarchives.ripme.ripper.rippers.YoupornRipper; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -10,6 +11,7 @@ public class YoupornRipperTest extends RippersTest { @Test + @Tag("flaky") public void testYoupornRipper() throws IOException { List contentURLs = new ArrayList<>(); contentURLs.add(new URL("http://www.youporn.com/watch/7669155/mrs-li-amateur-69-orgasm/?from=categ"));