✨ Add option to clear cookies or browser data

This could help to avoid the bot detection. This is untested.

README.md

@@ -165,6 +165,11 @@ personalDataEmail = impfMePlease@lasthope.de
 # It seems that non windows agents are blocked. You can set it to "default" to use the default one of chrome.
 # Could be something like: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36"
 userAgent = default
+
+# Clears browser cookies to avoid bot detection. Disabled by default.
+clearCookies = false
+# Experimental feature, which tries to clear the browser cache to avoid bot detection. Disabled by default.
+experimentalClearBrowser = false
 ```
 
 ### Setup Slack

src/main/kotlin/de/tfr/impf/ReportJob.kt

@@ -16,7 +16,6 @@ import org.openqa.selenium.TakesScreenshot
 import org.openqa.selenium.WebDriver
 import java.io.File
 import java.lang.System.currentTimeMillis
-import java.util.concurrent.TimeUnit
 
 val log = KotlinLogging.logger("ReportJob")
 
@@ -68,11 +67,14 @@ class ReportJob {
                 log.error(e) { "Failed to check location: $location\n" + e.message }
             }
 
-            if (System.currentTimeMillis() > startTime + (1000 * 60 * 60)) {
-                log.info { "Restarting session" }
+            if (Config.experimentalClearBrowser) {
+                log.debug { "Clearing Browser Cache (Experimental)" }
                 driver.quit()
                 driver = createDriver()
-                startTime = System.currentTimeMillis()
+            }
+            if (Config.clearCookies) {
+                log.debug { "Clearing Browser Cookies" }
+                driver.manage().deleteAllCookies()
             }
 
             Thread.sleep(Config.waitingTime())

src/main/kotlin/de/tfr/impf/config/Config.kt

@@ -126,6 +126,16 @@ object Config : KProperties() {
      */
     val personalDataEmail: String by lazyProperty()
 
+    /**
+     * Clears browser cookies to avoid bot detection. Disabled as default.
+     */
+    val clearCookies = false
+
+    /**
+     * Experimental feature, which tries to clear the browser cache to avoid bot detection. Disabled as default.
+     */
+    val experimentalClearBrowser: Boolean by lazyBoolProperty()
+
     fun isSlackEnabled() = slackEnabled
     val slackBotApiToken: String by lazyProperty()
     private val slackEnabled: Boolean by lazyBoolProperty()

src/main/kotlin/de/tfr/impf/selenium/ClearBrowserData.kt

@@ -0,0 +1,54 @@
+package de.tfr.impf.selenium
+
+import org.openqa.selenium.By
+import org.openqa.selenium.JavascriptExecutor
+import org.openqa.selenium.WebElement
+import org.openqa.selenium.chrome.ChromeDriver
+
+/**
+ * Experimental support for resetting the browser to avoid bot detection.
+ * Disabled by default.
+ * Check out the merge request for discussion and alternatives: https://github.com/TobseF/impf-bot/pull/88
+ */
+fun clearBrowserData(chromeDriver: ChromeDriver) {
+
+    chromeDriver.get("chrome://settings/clearBrowserData")
+
+    val root1 = chromeDriver.findElement(By.cssSelector("settings-ui"))
+    // get 1st shadow root element
+    val shadowRoot1 = expandRootElement(chromeDriver, root1)
+
+    // get 2nd shadow root element
+    val shadowRoot2 = shadowRoot1.findAndExpandShadowElement("settings-main", chromeDriver)
+
+    // get 3rd shadow root element
+    val shadowRoot3 = shadowRoot2.findAndExpandShadowElement("settings-basic-page", chromeDriver)
+
+    // get 4th shadow root element
+    val shadowRoot4 = shadowRoot3.findAndExpandShadowElement("settings-section > settings-privacy-page", chromeDriver)
+
+    // get 5th shadow root element
+    val shadowRoot5 = shadowRoot4.findAndExpandShadowElement("settings-clear-browsing-data-dialog", chromeDriver)
+
+    // get 6th parent
+    val root6 = shadowRoot5.findElement(By.cssSelector("#clearBrowsingDataDialog"))
+
+    // get button (finally!)
+    val clearDataButton = root6.findElement(By.cssSelector("#clearBrowsingDataConfirm"))
+    // end identify clear data button via nested Shadow Dom elements
+
+    clearDataButton.click(); // click that hard to reach button!
+
+    Thread.sleep(5000)
+
+    chromeDriver.manage().deleteAllCookies()
+}
+
+fun WebElement.findAndExpandShadowElement(cssSelector: String, driver: ChromeDriver): WebElement {
+    return expandRootElement(driver, this.findElement(By.cssSelector(cssSelector)))
+}
+
+private fun expandRootElement(driver: ChromeDriver, element: WebElement): WebElement {
+    val executor = driver as JavascriptExecutor
+    return executor.executeScript("return arguments[0].shadowRoot", element) as WebElement
+}

src/main/kotlin/de/tfr/impf/selenium/DriverFactory.kt

@@ -6,9 +6,6 @@ import org.openqa.selenium.WebDriver
 import org.openqa.selenium.chrome.ChromeDriver
 import org.openqa.selenium.chrome.ChromeOptions
 import java.util.concurrent.TimeUnit
-import org.openqa.selenium.By;
-import org.openqa.selenium.JavascriptExecutor;
-import org.openqa.selenium.WebElement
 
 fun createDriver(): ChromeDriver {
     WebDriverManager.chromedriver().setup();
@@ -20,52 +17,17 @@ fun createDriver(): ChromeDriver {
     val chromeDriver = ChromeDriver(chromeOptions)
     chromeDriver.setTimeOut(Config.searchElementTimeout())
 
-    chromeDriver.get("chrome://settings/clearBrowserData")
-
-    val root1 = chromeDriver.findElement(By.cssSelector("settings-ui"))
-    // get 1st shadowroot element
-    val shadowRoot1 = expandRootElement(chromeDriver, root1)
-
-    // get 2nd parent
-    val root2 = shadowRoot1.findElement(By.cssSelector("settings-main"))
-    // get 2nd shadowroot element
-    val shadowRoot2 = expandRootElement(chromeDriver, root2)
-
-    // get 3rd parent
-    val root3 = shadowRoot2.findElement(By.cssSelector("settings-basic-page"))
-    // get 3rd shadowroot element
-    val shadowRoot3 = expandRootElement(chromeDriver, root3)
-
-    // get 4th parent
-    val root4 = shadowRoot3.findElement(By.cssSelector("settings-section > settings-privacy-page"))
-    // get 4th shadowroot element
-    val shadowRoot4 = expandRootElement(chromeDriver, root4)
-
-    // get 5th parent
-    val root5 = shadowRoot4.findElement(By.cssSelector("settings-clear-browsing-data-dialog"))
-    // get 5th shadowroot element
-    val shadowRoot5 = expandRootElement(chromeDriver, root5)
-
-    // get 6th parent
-    val root6 = shadowRoot5.findElement(By.cssSelector("#clearBrowsingDataDialog"))
-
-    // get button (finally!)
-    val clearDataButton = root6.findElement(By.cssSelector("#clearBrowsingDataConfirm"))
-    // end identify clear data button via nested Shadow Dom elements
-
-    clearDataButton.click(); // click that hard to reach button!
-
-    Thread.sleep(5000)
+    if (Config.clearCookies) {
+        chromeDriver.manage().deleteAllCookies()
+    }
 
-    chromeDriver.manage().deleteAllCookies()
+    if (Config.experimentalClearBrowser) {
+        clearBrowserData(chromeDriver)
+    }
 
     return chromeDriver
 }
 
-private fun expandRootElement(driver: ChromeDriver, element: WebElement): WebElement {
-    val executor = driver as JavascriptExecutor
-    return executor.executeScript("return arguments[0].shadowRoot", element) as WebElement
-}
 
 fun WebDriver.setTimeOut(milliseconds: Long) {
     this.manage()?.timeouts()?.implicitlyWait(milliseconds, TimeUnit.MILLISECONDS)

src/main/resources/config.properties

@@ -101,3 +101,8 @@ personalDataEmail = impfMePlease@lasthope.de
 # It seems that non windows agents are blocked. You can set it to "default" to use the default one of chrome.
 # Could be something like: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36"
 userAgent = default
+
+# Clears browser cookies to avoid bot detection. Disabled by default.
+clearCookies = false
+# Experimental feature, which tries to clear the browser cache to avoid bot detection. Disabled by default.
+experimentalClearBrowser = false