Configurable browser steps (enter text, scroll, wait for text, etc)

.gitignore

@@ -8,6 +8,7 @@ __pycache__
 build
 dist
 venv
+test-datastore/*
 test-datastore
 *.egg-info*
 .vscode/settings.json

Dockerfile

@@ -25,7 +25,7 @@ RUN pip install --target=/dependencies -r /requirements.txt
 # Playwright is an alternative to Selenium
 # Excluded this package from requirements.txt to prevent arm/v6 and arm/v7 builds from failing
 # https://github.com/dgtlmoon/changedetection.io/pull/1067 also musl/alpine (not supported)
-RUN pip install --target=/dependencies playwright~=1.26 \
+RUN pip install --target=/dependencies playwright~=1.27.1 \
     || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled."
 
 # Final image stage

MANIFEST.in

@@ -4,7 +4,10 @@ recursive-include changedetectionio/static *
 recursive-include changedetectionio/model *
 recursive-include changedetectionio/tests *
 recursive-include changedetectionio/res *
+prune changedetectionio/static/package-lock.json
+prune changedetectionio/static/styles/node_modules
+prune changedetectionio/static/styles/package-lock.json
 include changedetection.py
 global-exclude *.pyc
 global-exclude node_modules
-global-exclude venv
+global-exclude venv

README.md

@@ -69,6 +69,17 @@ Available when connected to a <a href="https://github.com/dgtlmoon/changedetecti
 
 <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/visualselector-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Self-hosted web page change monitoring context difference " />
 
+### Perform interactive browser steps
+
+Fill in text boxes, click buttons and more, setup your changedetection scenario. 
+
+Using the **Browser Steps** configuration, add basic steps before performing change detection, such as logging into websites, adding a product to a cart, refining searches.
+
+<img src="docs/browsersteps-anim.gif" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference "  title="Website change detection with interactive browser steps, login, cookies etc" />
+
+After **Browser Steps** have been run, then visit the **Visual Selector** tab to refine the content you're interested in.
+Requires Playwright to be enabled.
+
 ## Installation
 
 ### Docker

changedetectionio/__init__.py

@@ -1,18 +1,20 @@
 #!/usr/bin/python3
 
 import datetime
+import flask_login
+import logging
 import os
+import pytz
 import queue
 import threading
 import time
+import timeago
+
 from copy import deepcopy
+from distutils.util import strtobool
+from feedgen.feed import FeedGenerator
 from threading import Event
 
-import flask_login
-import logging
-import pytz
-import timeago
-from feedgen.feed import FeedGenerator
 from flask import (
     Flask,
     abort,
@@ -27,7 +29,6 @@
 )
 from flask_login import login_required
 from flask_restful import abort, Api
-
 from flask_wtf import CSRFProtect
 
 from changedetectionio import html_tools
@@ -44,7 +45,6 @@
 extra_stylesheets = []
 
 update_q = queue.PriorityQueue()
-
 notification_q = queue.Queue()
 
 app = Flask(__name__,
@@ -97,7 +97,7 @@ def _jinja2_filter_datetime(watch_obj, format="%Y-%m-%d %H:%M:%S"):
     # Worker thread tells us which UUID it is currently processing.
     for t in running_update_threads:
         if t.current_uuid == watch_obj['uuid']:
-            return '<span class="loader"></span><span> Checking now</span>'
+            return '<span class="spinner"></span><span> Checking now</span>'
 
     if watch_obj['last_checked'] == 0:
         return 'Not yet'
@@ -525,6 +525,7 @@ def get_current_checksum_include_ignore_text(uuid):
 
     def edit_page(uuid):
         from changedetectionio import forms
+        from changedetectionio.blueprint.browser_steps.browser_steps import browser_step_ui_config
 
         using_default_check_time = True
         # More for testing, possible to return the first/only
@@ -558,6 +559,8 @@ def edit_page(uuid):
                                data=default,
                                )
 
+        # form.browser_steps[0] can be assumed that we 'goto url' first
+
         if datastore.proxy_list is None:
             # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead
             del form.proxy
@@ -650,6 +653,7 @@ def edit_page(uuid):
                     watch.get('fetch_backend', None) is None and system_uses_webdriver) else False
 
             output = render_template("edit.html",
+                                     browser_steps_config=browser_step_ui_config,
                                      current_base_url=datastore.data['settings']['application']['base_url'],
                                      emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
                                      form=form,
@@ -661,7 +665,6 @@ def edit_page(uuid):
                                      settings_application=datastore.data['settings']['application'],
                                      using_global_webdriver_wait=default['webdriver_delay'] is None,
                                      uuid=uuid,
-                                     visualselector_data_is_ready=visualselector_data_is_ready,
                                      visualselector_enabled=visualselector_enabled,
                                      watch=watch
                                      )
@@ -1190,7 +1193,6 @@ def form_watch_checknow():
         else:
             # No tag, no uuid, add everything.
             for watch_uuid, watch in datastore.data['watching'].items():
-
                 if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
                     update_q.put((1, watch_uuid))
                     i += 1
@@ -1308,9 +1310,11 @@ def form_share_put_watch():
         # paste in etc
         return redirect(url_for('index'))
 
+    import changedetectionio.blueprint.browser_steps as browser_steps
+    app.register_blueprint(browser_steps.construct_blueprint(datastore), url_prefix='/browser-steps')
+
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
-
     threading.Thread(target=notification_runner).start()
 
     # Check for new release version, but not when running in test/build or pytest

changedetectionio/blueprint/browser_steps/__init__.py

@@ -0,0 +1,226 @@
+
+# HORRIBLE HACK BUT WORKS :-) PR anyone?
+#
+# Why?
+# `browsersteps_playwright_browser_interface.chromium.connect_over_cdp()` will only run once without async()
+# - this flask app is not async()
+# - browserless has a single timeout/keepalive which applies to the session made at .connect_over_cdp()
+#
+# So it means that we must unfortunately for now just keep a single timer since .connect_over_cdp() was run
+# and know when that reaches timeout/keepalive :( when that time is up, restart the connection and tell the user
+# that their time is up, insert another coin. (reload)
+#
+# Bigger picture
+# - It's horrible that we have this click+wait deal, some nice socket.io solution using something similar
+# to what the browserless debug UI already gives us would be smarter..
+#
+# OR
+# - Some API call that should be hacked into browserless or playwright that we can "/api/bump-keepalive/{session_id}/60"
+# So we can tell it that we need more time (run this on each action)
+#
+# OR
+# - use multiprocessing to bump this over to its own process and add some transport layer (queue/pipes)
+
+
+
+
+from distutils.util import strtobool
+from flask import Blueprint, request, make_response
+from flask_login import login_required
+import os
+import logging
+from changedetectionio.store import ChangeDetectionStore
+
+browsersteps_live_ui_o = {}
+browsersteps_playwright_browser_interface = None
+browsersteps_playwright_browser_interface_start_time = None
+browsersteps_playwright_browser_interface_browser = None
+browsersteps_playwright_browser_interface_end_time = None
+
+
+def cleanup_playwright_session():
+    print("Cleaning up old playwright session because time was up")
+    global browsersteps_playwright_browser_interface
+    global browsersteps_live_ui_o
+    global browsersteps_playwright_browser_interface_browser
+    global browsersteps_playwright_browser_interface
+    global browsersteps_playwright_browser_interface_start_time
+    global browsersteps_playwright_browser_interface_end_time
+
+    import psutil
+
+    current_process = psutil.Process()
+    children = current_process.children(recursive=True)
+    for child in children:
+        print (child)
+        print('Child pid is {}'.format(child.pid))
+
+    # .stop() hangs sometimes if its called when there are no children to process
+    # but how do we know this is our child? dunno
+    if children:
+        browsersteps_playwright_browser_interface.stop()
+
+    browsersteps_live_ui_o = {}
+    browsersteps_playwright_browser_interface = None
+    browsersteps_playwright_browser_interface_start_time = None
+    browsersteps_playwright_browser_interface_browser = None
+    browsersteps_playwright_browser_interface_end_time = None
+    print ("Cleaning up old playwright session because time was up - done")
+
+def construct_blueprint(datastore: ChangeDetectionStore):
+
+    browser_steps_blueprint = Blueprint('browser_steps', __name__, template_folder="templates")
+
+    @login_required
+    @browser_steps_blueprint.route("/browsersteps_update", methods=['GET', 'POST'])
+    def browsersteps_ui_update():
+        import base64
+        import playwright._impl._api_types
+        import time
+
+        from changedetectionio.blueprint.browser_steps import browser_steps
+
+        global browsersteps_live_ui_o, browsersteps_playwright_browser_interface_end_time
+        global browsersteps_playwright_browser_interface_browser
+        global browsersteps_playwright_browser_interface
+        global browsersteps_playwright_browser_interface_start_time
+
+        step_n = None
+        remaining =0
+        uuid = request.args.get('uuid')
+
+        browsersteps_session_id = request.args.get('browsersteps_session_id')
+
+        if not browsersteps_session_id:
+            return make_response('No browsersteps_session_id specified', 500)
+
+        # Because we don't "really" run in a context manager ( we make the playwright interface global/long-living )
+        # We need to manage the shutdown when the time is up
+        if browsersteps_playwright_browser_interface_end_time:
+            remaining = browsersteps_playwright_browser_interface_end_time-time.time()
+            if browsersteps_playwright_browser_interface_end_time and remaining <= 0:
+
+
+                cleanup_playwright_session()
+
+                return make_response('Browser session expired, please reload the Browser Steps interface', 500)
+
+
+        # Actions - step/apply/etc, do the thing and return state
+        if request.method == 'POST':
+            # @todo - should always be an existing session
+            step_operation = request.form.get('operation')
+            step_selector = request.form.get('selector')
+            step_optional_value = request.form.get('optional_value')
+            step_n = int(request.form.get('step_n'))
+            is_last_step = strtobool(request.form.get('is_last_step'))
+
+            if step_operation == 'Goto site':
+                step_operation = 'goto_url'
+                step_optional_value = None
+                step_selector = datastore.data['watching'][uuid].get('url')
+
+            # @todo try.. accept.. nice errors not popups..
+            try:
+
+                this_session = browsersteps_live_ui_o.get(browsersteps_session_id)
+                if not this_session:
+                    print("Browser exited")
+                    return make_response('Browser session ran out of time :( Please reload this page.', 401)
+
+                this_session.call_action(action_name=step_operation,
+                                         selector=step_selector,
+                                         optional_value=step_optional_value)
+            except playwright._impl._api_types.TimeoutError as e:
+                print("Element wasnt found :-(", step_operation)
+                return make_response("Element was not found on page", 401)
+
+            except playwright._impl._api_types.Error as e:
+                # Browser/playwright level error
+                print("Browser error - got playwright._impl._api_types.Error, try reloading the session/browser")
+                print (str(e))
+
+                # Try to find something of value to give back to the user
+                for l in str(e).splitlines():
+                    if 'DOMException' in l:
+                        return make_response(l, 401)
+
+                return make_response('Browser session ran out of time :( Please reload this page.', 401)
+
+            # Get visual selector ready/update its data (also use the current filter info from the page?)
+            # When the last 'apply' button was pressed
+            # @todo this adds overhead because the xpath selection is happening twice
+            u = this_session.page.url
+            if is_last_step and u:
+                (screenshot, xpath_data) = this_session.request_visualselector_data()
+                datastore.save_screenshot(watch_uuid=uuid, screenshot=screenshot)
+                datastore.save_xpath_data(watch_uuid=uuid, data=xpath_data)
+
+        # Setup interface
+        if request.method == 'GET':
+
+            if not browsersteps_playwright_browser_interface:
+                print("Starting connection with playwright")
+                logging.debug("browser_steps.py connecting")
+                from playwright.sync_api import sync_playwright
+
+                browsersteps_playwright_browser_interface = sync_playwright().start()
+
+
+                time.sleep(1)
+                # At 20 minutes, some other variable is closing it
+                # @todo find out what it is and set it
+                seconds_keepalive = int(os.getenv('BROWSERSTEPS_MINUTES_KEEPALIVE', 10)) * 60
+
+                # keep it alive for 10 seconds more than we advertise, sometimes it helps to keep it shutting down cleanly
+                keepalive = "&timeout={}".format(((seconds_keepalive+3) * 1000))
+                try:
+                    browsersteps_playwright_browser_interface_browser = browsersteps_playwright_browser_interface.chromium.connect_over_cdp(
+                        os.getenv('PLAYWRIGHT_DRIVER_URL', '') + keepalive)
+                except Exception as e:
+                    if 'ECONNREFUSED' in str(e):
+                        return make_response('Unable to start the Playwright session properly, is it running?', 401)
+
+                browsersteps_playwright_browser_interface_end_time = time.time() + (seconds_keepalive-3)
+                print("Starting connection with playwright - done")
+
+            if not browsersteps_live_ui_o.get(browsersteps_session_id):
+                # Boot up a new session
+                proxy_id = datastore.get_preferred_proxy_for_watch(uuid=uuid)
+                proxy = None
+                if proxy_id:
+                    proxy_url = datastore.proxy_list.get(proxy_id).get('url')
+                    if proxy_url:
+                        proxy = {'server': proxy_url}
+                        print("Browser Steps: UUID {} Using proxy {}".format(uuid, proxy_url))
+
+                # Begin the new "Playwright Context" that re-uses the playwright interface
+                # Each session is a "Playwright Context" as a list, that uses the playwright interface
+                browsersteps_live_ui_o[browsersteps_session_id] = browser_steps.browsersteps_live_ui(
+                    playwright_browser=browsersteps_playwright_browser_interface_browser,
+                    proxy=proxy)
+                this_session = browsersteps_live_ui_o[browsersteps_session_id]
+
+        if not this_session.page:
+            cleanup_playwright_session()
+            return make_response('Browser session ran out of time :( Please reload this page.', 401)
+
+        try:
+            state = this_session.get_current_state()
+        except playwright._impl._api_types.Error as e:
+            return make_response("Browser session ran out of time :( Please reload this page."+str(e), 401)
+
+        p = {'screenshot': "data:image/png;base64,{}".format(
+            base64.b64encode(state[0]).decode('ascii')),
+            'xpath_data': state[1],
+            'session_age_start': this_session.age_start,
+            'browser_time_remaining': round(remaining)
+        }
+
+
+        # @todo BSON/binary JSON, faster xfer, OR pick it off the disk
+        return p
+
+    return browser_steps_blueprint
+
+