Merge pull request #4 from wowitsjack/patch-1

Create sage_getwebkit.sh

Author: ChijinZ

browser_builders/sage_getwebkit.sh

@@ -0,0 +1,184 @@
+#!/bin/bash
+
+# Check for and install missing dependencies
+install_missing_deps() {
+    local missing_deps=""
+
+    # List of essential commands and their corresponding packages
+    declare -A essential_cmds=( ["dialog"]="dialog" ["sudo"]="sudo" ["git"]="git" ["wget"]="wget" ["cmake"]="cmake" ["ninja-build"]="ninja-build" ["apt-get"]="apt" )
+
+    # Check each command and record missing ones
+    for cmd in "${!essential_cmds[@]}"; do
+        if ! command -v $cmd &> /dev/null; then
+            missing_deps+="${essential_cmds[$cmd]} "
+        fi
+    done
+
+    # Install missing dependencies, if any
+    if [ -n "$missing_deps" ]; then
+        echo "The following dependencies are missing and will be installed: $missing_deps"
+        sudo apt-get update
+        sudo apt-get install -y $missing_deps
+    else
+        echo "All essential dependencies are already installed."
+    fi
+}
+
+# Initial setup
+clear
+echo "Checking and installing missing dependencies..."
+install_missing_deps
+
+# Request sudo access at the beginning
+echo "Requesting administrative access for initial setup..."
+sudo -v
+while true; do sudo -n true; sleep 60; kill -0 "$$" || exit; done 2>/dev/null &
+
+# Function to execute commands within dialog boxes, showing output in a progress box
+execute_command() {
+    local cmd=$1
+    local title=$2
+    dialog --title "$title" --infobox "Preparing execution..." 10 70
+    eval "$cmd" 2>&1 | dialog --title "$title" --progressbox 50 100
+}
+
+# Function to display welcome message using dialog
+welcome_message() {
+    dialog --clear --title "Welcome" --msgbox "Welcome to the Setup Wizard. This will guide you through setting up the environment." 10 50
+}
+
+# Function to get WebKitGTK version from the user
+get_webkitgtk_version() {
+    WEBKITGTK_VERSION=$(dialog --title "WebKitGTK Version" --inputbox "Enter the version of WebKitGTK you want to build:" 8 40 "2.42.5" 2>&1 >/dev/tty)
+}
+
+# Function to show build options menu and capture selections
+show_build_options_menu() {
+    BUILD_OPTIONS=$(dialog --checklist "Choose build options:" 22 76 15 \
+    "ENABLE_BUBBLEWRAP_SANDBOX" "Bubblewrap Sandbox" ON \
+    "ENABLE_DOCUMENTATION" "Documentation" OFF \
+    "ENABLE_DRAG_SUPPORT" "Drag Support" ON \
+    "ENABLE_GAMEPAD" "Gamepad Support" ON \
+    "ENABLE_INTROSPECTION" "Introspection" ON \
+    "ENABLE_JOURNALD_LOG" "Journald Log" ON \
+    "ENABLE_MINIBROWSER" "Mini Browser" ON \
+    "ENABLE_PDFJS" "PDF.js" ON \
+    "ENABLE_QUARTZ_TARGET" "Quartz Target" OFF \
+    "ENABLE_SPELLCHECK" "Spellcheck" ON \
+    "ENABLE_TOUCH_EVENTS" "Touch Events" ON \
+    "ENABLE_VIDEO" "Video" ON \
+    "ENABLE_WAYLAND_TARGET" "Wayland Target" ON \
+    "ENABLE_WEBDRIVER" "WebDriver" ON \
+    "ENABLE_WEB_AUDIO" "Web Audio" ON \
+    "ENABLE_WEB_CRYPTO" "Web Crypto" ON \
+    "ENABLE_X11_TARGET" "X11 Target" ON \
+    "USE_AVIF" "AVIF Images" ON \
+    "USE_GBM" "GBM" ON \
+    "USE_GSTREAMER_TRANSCODER" "GStreamer Transcoder" ON \
+    "USE_GSTREAMER_WEBRTC" "GStreamer WebRTC" OFF \
+    "USE_GTK4" "GTK4" OFF \
+    "USE_JPEGXL" "JPEG XL" ON \
+    "USE_LCMS" "Little CMS" ON \
+    "USE_LIBHYPHEN" "LibHyphen" ON \
+    "USE_LIBSECRET" "Libsecret" ON \
+    "USE_OPENGL_OR_ES" "OpenGL or ES" ON \
+    "USE_OPENJPEG" "OpenJPEG" ON \
+    "USE_SOUP2" "Soup2" OFF \
+    "USE_WOFF2" "WOFF2 Fonts" ON 2>&1 >/dev/tty)
+}
+
+# Install all required packages at once, silently, with no user interaction
+install_packages() {
+    packages="libgcrypt20 libgcrypt20-dev libtasn1-6 libtasn1-6-dev unifdef libwebp-dev libgtk-4-dev libsoup3-dev libsoup3 libsoup-3.0-dev libmanette-0.2-dev libxslt1-dev libsecret-1-dev libdrm-dev libgbm-dev libenchant-2-dev libjxl-dev afl++ libstdc++-11-dev build-essential clang llvm-17 libstdc++-12-dev libhyphen-dev libwoff-dev libavif-dev libsystemd-dev liblcms2-dev libgcc-11-dev libseccomp-dev libgstreamer-plugins-base1.0-dev gstreamer1.0-plugins-base gstreamer1.0-plugins-good libgstreamer1.0-dev gstreamer1.0-libav gstreamer1.0-plugins-bad libgstreamer-plugins-bad1.0-dev gstreamer1.0-plugins-good libgstrtspserver-1.0-dev gperf gettext libxt-dev libopenjp2-7-dev gi-docgen libwebkit2gtk-4.1-dev ninja-build"
+    execute_command "sudo apt-get update && sudo apt-get install -y $packages" "Installing required packages"
+}
+
+# Function to clone, build, and install dependencies
+git_clone_and_setup() {
+    local git_url=$1
+    local folder_name=$2
+    local setup_commands=$3
+    local title=$4
+    execute_command "rm -rf $folder_name && git clone $git_url $folder_name --recursive --shallow-submodules && cd $folder_name && $setup_commands" "$title"
+}
+
+# Welcome the user and get necessary input
+welcome_message
+
+# Function to create a swapfile
+create_swapfile() {
+    # Ask user for the swapfile size and location
+    SWAPFILE_DETAILS=$(dialog --title "Swapfile Configuration" --form "Enter the details for the swapfile:" 15 50 0 \
+    "Size (e.g., 128G):" 1 1 "128G" 1 25 25 0 \
+    "Location (path):" 2 1 "$(pwd)/swapfile" 2 25 100 0 \
+    2>&1 >/dev/tty)
+
+    # Parse input
+    SWAPFILE_SIZE=$(echo "$SWAPFILE_DETAILS" | sed -n 1p)
+    SWAPFILE_LOCATION=$(echo "$SWAPFILE_DETAILS" | sed -n 2p)
+
+    # Remove existing swapfile if it exists
+    [ -f "$SWAPFILE_LOCATION" ] && sudo swapoff "$SWAPFILE_LOCATION" && rm -f "$SWAPFILE_LOCATION"
+
+    # Create new swapfile
+    execute_command "sudo fallocate -l $SWAPFILE_SIZE $SWAPFILE_LOCATION && sudo chmod 600 $SWAPFILE_LOCATION && sudo mkswap $SWAPFILE_LOCATION && sudo swapon $SWAPFILE_LOCATION" "Creating Swapfile"
+    
+    # Confirmation message
+    dialog --title "Swapfile Creation" --msgbox "Swapfile created successfully at $SWAPFILE_LOCATION with a size of $SWAPFILE_SIZE." 10 50
+}
+
+# Create swapfile as part of the setup process
+create_swapfile
+
+get_webkitgtk_version
+show_build_options_menu
+
+# Install essential packages
+install_packages
+
+# Clone and setup libjxl
+git_clone_and_setup "https://github.com/libjxl/libjxl.git" "libjxl" "sudo apt install -y cmake pkg-config libbrotli-dev libgif-dev libjpeg-dev libopenexr-dev libpng-dev libwebp-dev clang && export CC=clang CXX=clang++ && mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF .. && cmake --build . -- -j\$(nproc) && sudo cmake --install ." "Setting up libjxl"
+
+# Clone and setup libbacktrace
+git_clone_and_setup "https://github.com/ianlancetaylor/libbacktrace" "libbacktrace" "./configure && make && sudo make install" "Installing libbacktrace"
+
+# Convert dialog output to CMake and Ninja build options
+CONVERTED_OPTIONS=""
+IFS=' ' read -ra ADDR <<< "$BUILD_OPTIONS"
+for option in "${ADDR[@]}"; do
+    if [[ "$option" == "\"USE_WPE_RENDERER\"" ]]; then
+        CONVERTED_OPTIONS+="-DUSE_WPE_RENDERER=ON "
+    else
+        CONVERTED_OPTIONS+="-D${option//\"/}=ON "
+    fi
+done
+
+# Build and install WebKitGTK using Ninja as per user-selected options, removing any previous extracted folder
+BUILD_COMMAND="rm -rf webkitgtk-${WEBKITGTK_VERSION} && wget https://webkitgtk.org/releases/webkitgtk-${WEBKITGTK_VERSION}.tar.xz && tar xf webkitgtk-${WEBKITGTK_VERSION}.tar.xz && cd webkitgtk-${WEBKITGTK_VERSION} && mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_SKIP_RPATH=ON -DPORT=GTK -DLIB_INSTALL_DIR=/usr $CONVERTED_OPTIONS -Wno-dev -G Ninja .. && ninja && sudo ninja install"
+execute_command "$BUILD_COMMAND" "Building and installing WebKitGTK ${WEBKITGTK_VERSION} with Ninja"
+
+# Function to remove a swapfile
+remove_swapfile() {
+    local location=$(dialog --title "Remove Swapfile" --inputbox "Enter the location of the swap file to remove:" 8 40 "./swapfile" 2>&1 >/dev/tty)
+
+    if [ -f "$location" ]; then
+        # Deactivate the swapfile
+        sudo swapoff "$location"
+
+        # Remove the swapfile
+        rm -f "$location"
+
+        # Display completion message
+        dialog --title "Swapfile Removal" --msgbox "Swapfile removed successfully from $location." 10 50
+    else
+        dialog --title "Swapfile Removal" --msgbox "Swapfile not found at $location." 10 50
+    fi
+}
+
+# Optionally remove the swapfile as part of the cleanup process
+# remove_swapfile
+
+# Final message to indicate completion
+dialog --clear --title "Completion" --msgbox "All steps completed successfully. Your environment is now set up." 10 50
+
+clear

demo.png

@@ -1,19 +1,28 @@
-# SaGe browser fuzzer
+
+# SaGe Browser Fuzzer 🌐💻
 
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.8328742.svg)](https://doi.org/10.5281/zenodo.8328742)
 
-SaGe is a browser fuzzer that can effectively explore browsers' semantics.
+Welcome to SaGe Browser Fuzzer, a cutting-edge tool designed for the intricate exploration of web browser semantics 🚀. 
+
+Developed with a focus on identifying and cataloging internal browser JS vulnerabilities, SaGe offers an automated, comprehensive suite of features for thorough browser testing via PCSG-guided fuzzing.
+
+## System Requirements 📋
+
+- **Operating System**: Linux (Ubuntu 20.04 LTS and 22.04 LTS highly recommended; other distributions may function but are less tested), MacOS (experimental support), Windows (limited testing).
+- **Python**: Version 3.8 or newer.
+- **Selenium**: Install via `pip3 install selenium==3.141.0` and `pip3 install urllib3==1.26.5`. Newer versions may also be compatible.
+- **Xvfb**: Required for headless operation on Linux, install with `apt install xvfb`.
 
-## Requirement
+## Installation 🛠
 
-- OS: Linux (recommended, well tested in Ubuntu 20.04 LTS and 22.04 LTS); MacOS (supposed to work but not well tested); Windows (not well tested)
-- Python 3.8+
-- Selenium (``pip3 install selenium==3.141.0`` and ``pip3 install urllib3==1.26.5``. The versions are well tested, but the latest versions should also work.)
-- Xvfb (apt install xvfb)
+Ensure all dependencies are met. 
 
-## Usage
+Linux users benefit from an automatic dependency check and installation feature upon initiating SaGe Launcher.
 
-Before running the tool, first set some environment variables to enable PCSG-guided fuzzing:
+## Configuration and Usage 🖥
+
+Set environment variables crucial for PCSG-guided fuzzing:
 
 ```shell
 export COLLECT_TREE_INFO=true
@@ -23,64 +32,95 @@ export INVALID_TREE_PATH="$SAGE_PATH/invalid_tree/invalid_tree.pickle"
 export RULE_INFO_PATH="$SAGE_PATH/invalid_tree/global_info.pickle"
 ```
 
-Next, use ```python main.py --help``` to show how it works.
+### Executing SaGe with `sage_launcher.sh`
 
-```
-Usage: python main [-options] -o output_dir
-
-Options:
-  -h, --help            show this help message and exit
-  -f FUZZER, --fuzzer=FUZZER
-                        choose a fuzzer (default: sage)
-  -b BROWSER, --browser=BROWSER
-                        choose a browser (default: webkitgtk)
-  -t TIMEOUT, --timeout=TIMEOUT
-                        timeout of each test (ms) (default: 5000ms)
-  -p PARALLEL, --parallel=PARALLEL
-                        how many instances in parallel (default: 1)
-  -o OUTPUT_DIR, --output_dir=OUTPUT_DIR
-                        where the result should output
-  -e TIME_TO_EXIT, --time_to_exit=TIME_TO_EXIT
-                        time to exit the fuzzing (hour)
-  -x EXECUTION_ITERATION, --execution_iteration=EXECUTION_ITERATION
-                        exit after this iteration
-```
+Utilize the `sage_launcher.sh` script for streamlined execution. 
+
+This script encompasses dependency checks, environmental setup, and execution controls in one command. 
+
+
+### Monitoring live with `sage_watcher.sh`
+
+To monitor fuzzing in real-time, run `./sage_watcher.sh` after starting `sage_launcher.sh` in a second terminal tab/window.
+
+![sage_watcher.sh Demo Image](demo.png)
+
+#### Command-Line Options 🔍
+
+- **Browser Selection**: Choose among `--firefox`, `--webkitgtk`, and `--chromium` for targeted fuzzing.
+- **Fuzzer Choice**: Select a fuzzer through `--fuzzer` with options including `domato`, `minerva`, `freedom`, `sage`, `favocado`.
+- **Clean Start**: Utilize `--kill-old` to terminate existing instances for a fresh testing environment.
+- **Resource Management**: Activate the `--watchdog` for intelligent resource monitoring and management.
+- **Session Timing**: Control the fuzzing duration with `--timerpurge`, specifying a numeric value for the session limit.
+
+### Examples for Each Browser 🌍
 
-For each kind of browser, we need to set environment variables to specify the path of the target browser and the path of the corresponding webdriver. What follows are the commands for fuzzing WebKit, Chrome, and Firefox:
+#### Fuzzing WebKit
 
-### Fuzzing WebKit
+For users interested in WebKit, an example WebKit builder is located in the `browserbuilders` folder. Configure environment variables accordingly:
 
 ```shell
 export WEBKIT_BINARY_PATH="$WEBKIT_PATH/MiniBrowser"
 export WEBKIT_WEBDRIVER_PATH="$WEBKIT_PATH/WebKitWebDriver"
-# max timeout for each input is 10000 ms, 10 instances are created for parallel, target browser is webkitgtk, fuzzing outputs are save in $PWD/output
-python3 main.py -t 10000 -b webkitgtk -p 10 -o $PWD/output
+./sage_launcher.sh --webkitgtk 5
 ```
 
-### Fuzzing Chrome
+#### Fuzzing Chrome
+
+Set up Chrome for fuzzing with the following environment variables and execute the script:
 
 ```shell
 export CHROMIUM_PATH="$C_PATH/chrome"
 export CHROMEDRIVER_PATH="$C_PATH/chromedriver"
-# max timeout for each input is 10000 ms, 10 instances are created for parallel, target browser is chrome, fuzzing outputs are save in $PWD/output
-python3 main.py -t 10000 -b chromium -p 10 -o $PWD/output
+./sage_launcher.sh --chromium 5
 ```
 
-### Fuzzing Firefox
+#### Fuzzing Firefox
+
+Prepare Firefox for fuzzing by setting up its environment variables and start the process:
 
 ```shell
 export FIREFOXDRIVER_PATH="$F_PATH/geckodriver"
 export FIREFOX_PATH="$F_PATH/firefox"
-# max timeout for each input is 10000 ms, 10 instances are created for parallel, target browser is firefox, fuzzing outputs are save in $PWD/output
-python3 main.py -t 10000 -b firefox -p 10 -o $PWD/output
+./sage_launcher.sh --firefox 5
 ```
 
+
+## DEMO: WebKit Builder Setup 🛠️
+
+The included WebKitGTK Builder Setup script automates the setup process for building WebKitGTK for use with SaGe, and other dependencies. 
+
+It simplifies the process with dialog boxes for user input and displays progress in a clean and interactive manner.
+
+
+![WebKitBuilder Demo Image](webkitbuilder.png)
+
+### Features:
+
+- **Interactive Dialogs**: Utilizes dialog boxes for inputs and displays like welcome message, WebKitGTK version input, and build options.
+- **Dynamic Build Options**: Allows selecting various build options through a checklist dialog.
+- **Package Installation**: Installs required packages automatically based on the build environment needs.
+- **Dependency Management**: Clones and sets up necessary dependencies like libjxl and libbacktrace.
+- **Swapfile Management**: Includes functionality to create and remove a swapfile to facilitate builds on systems with limited RAM.
+- **Custom Build Commands**: Converts selected build options into commands for cmake and Ninja, facilitating a tailored build process.
+
+### Usage:
+
+1. **Start the Script**: Run the script with `./browser_builders/sage_getwebkit.sh.sh`.
+2. **Follow Dialog Prompts**: Input the desired WebKitGTK version, select build options, and proceed through the setup as guided by the dialogs.
+3. **Monitor Progress**: The script displays progress in dialog boxes for each step, including package installations and builds.
+4. **Completion**: Upon successful completion, the environment will be set up with the selected configurations.
+
+## Academic Contributions 🎓
+
+Our approach and the detailed workings of SaGe are described in a publication accepted by OOPSLA 2023. For enthusiasts and researchers, a Docker environment is available for replicating our experiments, ensuring a seamless experience in understanding the tool's capabilities.
+
 ### Fuzzing other browsers
 
 This tool can adapt to any other browsers that are based on the three browsers. Generally speaking, almost all browsers are built on the top of the above three browsers. If users want to test a browser other than the three, they need to implement a subclass of ``FuzzedBrowser`` in the ``browser_adapters`` directory, and register itself in the ``get_browser()`` function of ``browser_selenium.py``.
 
 ## Implementation/Configuration Details
-- If you don’t want to use Xvfb, set ``export NO_XVFB=true`` before running ``main.py``.
+- If you don’t want to use Xvfb, set ``export NO_XVFB=true`` before running ``go.sh``.
 - During fuzzing, suppose we set p=2, this means that we create two (almost) separated browser instances, and each of them creates a new tab for handling one fuzzing input. If a browser crashes, the fuzzer will close it and create a new browser instance for testing.
 - Browsers may crash because of long-term running. For stability, the fuzzer will close an instance with 1% probability. Users can use ``CLOSE_BROWSER_PROB`` to change this setting. For example, ``export CLOSE_BROWSER_PROB=0.05`` will set the probability to 5%.
 
@@ -103,6 +143,6 @@ The paper which describes the design detail of this browser fuzzer is accepted b
 
 The artifact of this paper is available at [zenodo](https://doi.org/10.5281/zenodo.8328742), which includes a docker environment for reproducing the experitmental results in the paper. The artifact passed the OOPSLA'23 Artifact Evaluation and earned all badges.
 
-## Acknowledgement
+## Acknowledgements 👏
 
-We build the fuzzer on the top of [Domato](https://github.com/googleprojectzero/domato) for input generation.
+We extend our heartfelt gratitude to the Domato project, and Google for their foundational input generation techniques, which have significantly contributed to enhancing SaGe's fuzzing methodologies.