Added cleanup script for hanging Chrome processes and temp files (Sel…

…eniumHQ#2173)

NodeBase/Dockerfile

@@ -133,6 +133,14 @@ RUN chmod -R 775 ${HOME} /tmp/.X11-unix \
   && chgrp -R 0 ${HOME} /tmp/.X11-unix \
   && chmod -R g=u ${HOME} /tmp/.X11-unix
 
+#============================================
+# Shared cleanup script environment variables
+#============================================
+ENV SE_ENABLE_BROWSER_LEFTOVERS_CLEANUP false
+ENV SE_BROWSER_LEFTOVERS_INTERVAL_SECS 3600
+ENV SE_BROWSER_LEFTOVERS_PROCESSES_SECS 1200
+ENV SE_BROWSER_LEFTOVERS_TEMPFILES_DAYS 1
+
 #===================================================
 # Run the following commands as non-privileged user
 #===================================================

NodeChrome/Dockerfile

@@ -53,6 +53,12 @@ RUN if [ ! -z "$CHROME_DRIVER_VERSION" ]; \
   && chmod 755 /opt/selenium/chromedriver-$CHROME_DRIVER_VERSION \
   && ln -fs /opt/selenium/chromedriver-$CHROME_DRIVER_VERSION /usr/bin/chromedriver
 
+#============================================
+# Chrome cleanup script and supervisord file
+#============================================
+COPY chrome-cleanup.sh /opt/bin/chrome-cleanup.sh
+COPY chrome-cleanup.conf /etc/supervisor/conf.d/chrome-cleanup.conf
+
 USER ${SEL_UID}
 
 #============================================

NodeChrome/chrome-cleanup.conf

@@ -0,0 +1,21 @@
+; Documentation of this file format -> http://supervisord.org/configuration.html
+
+; Priority 0 - xvfb & fluxbox, 5 - x11vnc, 10 - noVNC, 15 - selenium-node
+
+[program:browserleftoverscleanup]
+priority=0
+command=bash -c "if [ ${SE_ENABLE_BROWSER_LEFTOVERS_CLEANUP} = "true" ]; then /opt/bin/chrome-cleanup.sh; fi"
+autostart=true
+exitcodes=0
+autorestart=unexpected
+
+;Logs
+redirect_stderr=false
+stdout_logfile=/var/log/supervisor/browser-leftover-cleanup-stdout.log
+stderr_logfile=/var/log/supervisor/browser-leftover-cleanup-stderr.log
+stdout_logfile_maxbytes=50MB
+stderr_logfile_maxbytes=50MB
+stdout_logfile_backups=5
+stderr_logfile_backups=5
+stdout_capture_maxbytes=50MB
+stderr_capture_maxbytes=50MB

NodeChrome/chrome-cleanup.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Return error exit code in case of any failure, so supervisord will restart the script
+set -e
+
+cleanup_stuck_chrome_processes() {
+  echo -n "Killing Chrome processes older than ${SE_BROWSER_LEFTOVERS_PROCESSES_SECS} seconds... "
+  ps -e -o pid,etimes,command | grep -v grep | grep chrome/chrome | awk '{if($2>'${SE_BROWSER_LEFTOVERS_PROCESSES_SECS}') print $0}' | awk '{print $1}' | xargs -r kill -9
+  echo "DONE."
+}
+
+cleanup_tmp_chrome_files() {
+  echo -n "Deleting all Chrome files in /tmp... "
+  find /tmp -name ".com.google.Chrome.*" -type d -mtime +${SE_BROWSER_LEFTOVERS_TEMPFILES_DAYS} -exec rm -rf "{}" +
+  echo "DONE."
+}
+
+echo "Chrome cleanup script init with parameters: SE_BROWSER_LEFTOVERS_TEMPFILES_DAYS=${SE_BROWSER_LEFTOVERS_TEMPFILES_DAYS}, SE_BROWSER_LEFTOVERS_PROCESSES_SECS=${SE_BROWSER_LEFTOVERS_PROCESSES_SECS}, SE_BROWSER_LEFTOVERS_INTERVAL_SECS=${SE_BROWSER_LEFTOVERS_INTERVAL_SECS}."
+
+# Start the main loop
+while :
+do
+  echo "Starting cleanup daemon script."
+
+  # Clean up stuck processes
+  cleanup_stuck_chrome_processes
+
+  # Wait a few seconds for the processes to stop before removing files
+  sleep 5
+
+  # Clean up temporary files
+  cleanup_tmp_chrome_files
+
+  # Go to sleep for 1 hour
+  echo "Cleanup daemon sleeping for ${SE_BROWSER_LEFTOVERS_INTERVAL_SECS} seconds."
+  sleep ${SE_BROWSER_LEFTOVERS_INTERVAL_SECS}
+done

NodeEdge/Dockerfile

@@ -46,6 +46,12 @@ RUN if [ -z "$EDGE_DRIVER_VERSION" ]; \
   && chmod 755 /opt/selenium/msedgedriver-$EDGE_DRIVER_VERSION \
   && ln -fs /opt/selenium/msedgedriver-$EDGE_DRIVER_VERSION /usr/bin/msedgedriver
 
+#============================================
+# Edge cleanup script and supervisord file
+#============================================
+COPY edge-cleanup.sh /opt/bin/edge-cleanup.sh
+COPY edge-cleanup.conf /etc/supervisor/conf.d/edge-cleanup.conf
+
 USER ${SEL_UID}
 
 #============================================

NodeEdge/edge-cleanup.conf

@@ -0,0 +1,21 @@
+; Documentation of this file format -> http://supervisord.org/configuration.html
+
+; Priority 0 - xvfb & fluxbox, 5 - x11vnc, 10 - noVNC, 15 - selenium-node
+
+[program:browserleftoverscleanup]
+priority=0
+command=bash -c "if [ ${SE_ENABLE_BROWSER_LEFTOVERS_CLEANUP} = "true" ]; then /opt/bin/edge-cleanup.sh; fi"
+autostart=true
+exitcodes=0
+autorestart=unexpected
+
+;Logs
+redirect_stderr=false
+stdout_logfile=/var/log/supervisor/browser-leftover-cleanup-stdout.log
+stderr_logfile=/var/log/supervisor/browser-leftover-cleanup-stderr.log
+stdout_logfile_maxbytes=50MB
+stderr_logfile_maxbytes=50MB
+stdout_logfile_backups=5
+stderr_logfile_backups=5
+stdout_capture_maxbytes=50MB
+stderr_capture_maxbytes=50MB

NodeEdge/edge-cleanup.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Return error exit code in case of any failure, so supervisord will restart the script
+set -e
+
+cleanup_stuck_edge_processes() {
+  echo -n "Killing Edge processes older than ${SE_BROWSER_LEFTOVERS_PROCESSES_SECS} seconds... "
+  ps -e -o pid,etimes,command | grep -v grep | grep msedge/msedge | awk '{if($2>'${SE_BROWSER_LEFTOVERS_PROCESSES_SECS}') print $0}' | awk '{print $1}' | xargs -r kill -9
+  echo "DONE."
+}
+
+cleanup_tmp_edge_files() {
+  echo -n "Deleting all Edge files in /tmp... "
+  find /tmp -name ".com.microsoft.Edge.*" -type d -mtime +${SE_BROWSER_LEFTOVERS_TEMPFILES_DAYS} -exec rm -rf "{}" +
+  echo "DONE."
+}
+
+echo "Edge cleanup script init with parameters: SE_BROWSER_LEFTOVERS_TEMPFILES_DAYS=${SE_BROWSER_LEFTOVERS_TEMPFILES_DAYS}, SE_BROWSER_LEFTOVERS_PROCESSES_SECS=${SE_BROWSER_LEFTOVERS_PROCESSES_SECS}, SE_BROWSER_LEFTOVERS_INTERVAL_SECS=${SE_BROWSER_LEFTOVERS_INTERVAL_SECS}."
+
+# Start the main loop
+while :
+do
+  echo "Starting cleanup daemon script."
+
+  # Clean up stuck processes
+  cleanup_stuck_edge_processes
+
+  # Wait a few seconds for the processes to stop before removing files
+  sleep 5
+
+  # Clean up temporary files
+  cleanup_tmp_edge_files
+
+  # Go to sleep for 1 hour
+  echo "Cleanup daemon sleeping for ${SE_BROWSER_LEFTOVERS_INTERVAL_SECS} seconds."
+  sleep ${SE_BROWSER_LEFTOVERS_INTERVAL_SECS}
+done

NodeFirefox/Dockerfile

@@ -36,6 +36,12 @@ RUN GK_VERSION=$(if [ ${GECKODRIVER_VERSION:-latest} = "latest" ]; then echo "0.
   && chmod 755 /opt/geckodriver-$GK_VERSION \
   && ln -fs /opt/geckodriver-$GK_VERSION /usr/bin/geckodriver
 
+#============================================
+# Firefox cleanup script and supervisord file
+#============================================
+COPY firefox-cleanup.sh /opt/bin/firefox-cleanup.sh
+COPY firefox-cleanup.conf /etc/supervisor/conf.d/firefox-cleanup.conf
+
 USER ${SEL_UID}
 
 #============================================

NodeFirefox/firefox-cleanup.conf

@@ -0,0 +1,21 @@
+; Documentation of this file format -> http://supervisord.org/configuration.html
+
+; Priority 0 - xvfb & fluxbox, 5 - x11vnc, 10 - noVNC, 15 - selenium-node
+
+[program:browserleftoverscleanup]
+priority=0
+command=bash -c "if [ ${SE_ENABLE_BROWSER_LEFTOVERS_CLEANUP} = "true" ]; then /opt/bin/firefox-cleanup.sh; fi"
+autostart=true
+exitcodes=0
+autorestart=unexpected
+
+;Logs
+redirect_stderr=false
+stdout_logfile=/var/log/supervisor/browser-leftover-cleanup-stdout.log
+stderr_logfile=/var/log/supervisor/browser-leftover-cleanup-stderr.log
+stdout_logfile_maxbytes=50MB
+stderr_logfile_maxbytes=50MB
+stdout_logfile_backups=5
+stderr_logfile_backups=5
+stdout_capture_maxbytes=50MB
+stderr_capture_maxbytes=50MB

NodeFirefox/firefox-cleanup.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Return error exit code in case of any failure, so supervisord will restart the script
+set -e
+
+cleanup_stuck_firefox_processes() {
+  echo -n "Killing Firefox processes older than ${SE_BROWSER_LEFTOVERS_PROCESSES_SECS} seconds... "
+  ps -e -o pid,etimes,command | grep -v grep | grep firefox-bin | awk '{if($2>'${SE_BROWSER_LEFTOVERS_PROCESSES_SECS}') print $0}' | awk '{print $1}' | xargs -r kill -9
+  echo "DONE."
+}
+
+echo "Firefox cleanup script init with parameters: SE_BROWSER_LEFTOVERS_PROCESSES_SECS=${SE_BROWSER_LEFTOVERS_PROCESSES_SECS}, SE_BROWSER_LEFTOVERS_INTERVAL_SECS=${SE_BROWSER_LEFTOVERS_INTERVAL_SECS}."
+
+# Start the main loop
+while :
+do
+  echo "Starting cleanup daemon script."
+
+  # Clean up stuck processes
+  cleanup_stuck_firefox_processes
+
+  # Go to sleep for 1 hour
+  echo "Cleanup daemon sleeping for ${SE_BROWSER_LEFTOVERS_INTERVAL_SECS} seconds."
+  sleep ${SE_BROWSER_LEFTOVERS_INTERVAL_SECS}
+done

README.md

@@ -944,6 +944,38 @@ $ docker run -e SE_DRAIN_AFTER_SESSION_COUNT=5 --shm-size="2g" selenium/standalo
 
 With the previous command, the Standalone container will shut down after 5 sessions have been executed.
 
+### Automatic browser leftovers cleanup
+
+In long-running containers, it can happen that browsers leave some leftovers. These can be stuck browser processes
+of jobs that have already finished but failed to fully stop the browser, or temporary files written to the `/tmp`
+file system (notably on Chrome-based browsers). To avoid these filling up resources like process IDs and file system
+usage in the container, there is an automatic cleanup script running every hour in the node containers. This will
+clean up old processes and old temporary files. By default, this is disabled. When enabled, this will clean up browsers
+running for longer than 20 minutes, and files older than 1 day. These can be enabled and tweaked with the following
+environment variables:
+
+* `SE_ENABLE_BROWSER_LEFTOVERS_CLEANUP`: default value `false`, set to `true` to enable the cleanup.
+* `SE_BROWSER_LEFTOVERS_INTERVAL_SECS`: default value `3600` (1 hour), cleanup interval in seconds.
+* `SE_BROWSER_LEFTOVERS_PROCESSES_SECS`: default value `1200` (2 hours), browsers running for longer than this time will be killed.
+* `SE_BROWSER_LEFTOVERS_TEMPFILES_DAYS`: default value `1` (1 day), files generated by Chrome-based browsers in `/tmp` will be removed after these number of days (ignored when using Firefox).
+
+If you use Selenium for long-running sessions and expect browsers to be running for longer than 20 minutes, either do
+not set `SE_ENABLE_BROWSER_LEFTOVERS_CLEANUP` to `true` (leave the default value of `false`), or tweak
+`SE_BROWSER_LEFTOVERS_PROCESSES_SECS` to set a value higher than your expected long-running browser processes.
+
+``` bash
+$ docker run -e SE_ENABLE_BROWSER_LEFTOVERS_CLEANUP=true --shm-size="2g" selenium/node-chrome:4.18.1-20240224
+```
+
+With the previous command, the cleanup will be enabled with the default timings.
+
+``` bash
+$ docker run -e SE_ENABLE_BROWSER_LEFTOVERS_CLEANUP=true -e SE_BROWSER_LEFTOVERS_INTERVAL_SECS=7200 -e SE_BROWSER_LEFTOVERS_PROCESSES_SECS=3600 -e SE_BROWSER_LEFTOVERS_TEMPFILES_DAYS=2 --shm-size="2g" selenium/node-chrome:4.18.1-20240224
+```
+
+With the previous command, the cleanup will be enabled, but will run every 2 hours (instead of 1), will kill browsers
+running longer than 1 hour (instead of 20 minutes), and will remove temp files older than 2 days (instead of 1).
+
 ___
 
 ## Building the images