Build binaries for Linux x64 and Windows x64

.github/workflows/release-docker.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       -
         name: Checkout

.github/workflows/release.yml

@@ -6,26 +6,15 @@ on:
       - 'v*.*.*'
 
 jobs:
-  build:
+  create-release:
     name: Create release
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
         with:
           fetch-depth: 0 # get all commits, branches and tags (required for the changelog)
 
-      - name: Setup Node
-        uses: actions/setup-node@v3
-        with:
-          node-version: '16'
-
-      - name: Build artifacts
-        run: |
-          npm install
-          npm run build
-          npm run package
-
       - name: Build changelog
         id: github_changelog
         run: |
@@ -47,9 +36,60 @@ jobs:
           draft: false
           prerelease: false
 
+  build-linux:
+    name: Build Linux binary
+    needs: create-release
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # get all commits, branches and tags (required for the changelog)
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Build artifacts
+        run: |
+          python -m pip install -r requirements.txt
+          python -m pip install pyinstaller==5.9.0
+          cd src
+          python build_package.py
+
+      - name: Upload release artifacts
+        uses: alexellis/upload-assets@0.4.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT }}
+        with:
+          asset_paths: '["./dist/flaresolverr_*"]'
+
+  build-windows:
+    name: Build Windows binary
+    needs: create-release
+    runs-on: windows-2022
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # get all commits, branches and tags (required for the changelog)
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Build artifacts
+        run: |
+          python -m pip install -r requirements.txt
+          python -m pip install pyinstaller==5.9.0
+          cd src
+          python build_package.py
+
       - name: Upload release artifacts
-        uses: alexellis/upload-assets@0.2.2
+        uses: alexellis/upload-assets@0.4.0
         env:
           GITHUB_TOKEN: ${{ secrets.GH_PAT }}
         with:
-          asset_paths: '["./bin/*.zip"]'
+          asset_paths: '["./dist/flaresolverr_*"]'

.gitignore

@@ -25,6 +25,7 @@ __pycache__/
 build/
 develop-eggs/
 dist/
+dist_chrome/
 downloads/
 eggs/
 .eggs/

README.md

@@ -64,12 +64,13 @@ Remember to restart the Docker daemon and the container after the update.
 
 ### Precompiled binaries
 
-Precompiled binaries are not currently available for v3. Please see https://github.com/FlareSolverr/FlareSolverr/issues/660 for updates,
-or below for instructions of how to build FlareSolverr from source code.
+This is the recommended way for Windows users.
+* Download the [FlareSolverr executable](https://github.com/FlareSolverr/FlareSolverr/releases) from the release's page. It is available for Windows x64 and Linux x64.
+* Execute FlareSolverr binary. In the environment variables section you can find how to change the configuration.
 
 ### From source code
 
-* Install [Python 3.10](https://www.python.org/downloads/).
+* Install [Python 3.11](https://www.python.org/downloads/).
 * Install [Chrome](https://www.google.com/intl/en_us/chrome/) or [Chromium](https://www.chromium.org/getting-involved/download-chromium/) web browser.
 * (Only in Linux / macOS) Install [Xvfb](https://en.wikipedia.org/wiki/Xvfb) package.
 * Clone this repository and open a shell in that path.

requirements.txt

@@ -4,6 +4,9 @@ selenium==4.8.2
 func-timeout==4.3.5
 # required by undetected_chromedriver
 requests==2.28.2
+certifi==2022.12.7
 websockets==10.4
 # only required for linux
 xvfbwrapper==0.2.9
+# only required for windows
+pefile==2023.2.7

src/build_package.py

@@ -0,0 +1,86 @@
+import os
+import platform
+import shutil
+import subprocess
+import sys
+import zipfile
+
+import requests
+
+
+def clean_files():
+    try:
+        shutil.rmtree(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'build'))
+    except Exception:
+        pass
+    try:
+        shutil.rmtree(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist'))
+    except Exception:
+        pass
+    try:
+        shutil.rmtree(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist_chrome'))
+    except Exception:
+        pass
+
+
+def download_chromium():
+    # https://commondatastorage.googleapis.com/chromium-browser-snapshots/index.html?prefix=Linux_x64/
+    revision = "1090006" if os.name == 'nt' else '1090007'
+    arch = 'Win' if os.name == 'nt' else 'Linux_x64'
+    dl_file = 'chrome-win' if os.name == 'nt' else 'chrome-linux'
+    dl_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist_chrome')
+    dl_path_folder = os.path.join(dl_path, dl_file)
+    dl_path_zip = dl_path_folder + '.zip'
+
+    # response = requests.get(
+    #     f'https://commondatastorage.googleapis.com/chromium-browser-snapshots/{arch}/LAST_CHANGE',
+    #     timeout=30)
+    # revision = response.text.strip()
+    print("Downloading revision: " + revision)
+
+    os.mkdir(dl_path)
+    with requests.get(
+            f'https://commondatastorage.googleapis.com/chromium-browser-snapshots/{arch}/{revision}/{dl_file}.zip',
+            stream=True) as r:
+        r.raise_for_status()
+        with open(dl_path_zip, 'wb') as f:
+            for chunk in r.iter_content(chunk_size=8192):
+                f.write(chunk)
+    print("File downloaded: " + dl_path_zip)
+    with zipfile.ZipFile(dl_path_zip, 'r') as zip_ref:
+        zip_ref.extractall(dl_path)
+    os.remove(dl_path_zip)
+    shutil.move(dl_path_folder, os.path.join(dl_path, "chrome"))
+
+
+def run_pyinstaller():
+    sep = ';' if os.name == 'nt' else ':'
+    subprocess.check_call([sys.executable, "-m", "PyInstaller",
+                           "--onefile",
+                           "--add-data", f"package.json{sep}.",
+                           "--add-data", f"{os.path.join('dist_chrome', 'chrome')}{sep}chrome",
+                           os.path.join("src", "flaresolverr.py")],
+                          cwd=os.pardir)
+    exe_folder = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist')
+    exe_name = 'flaresolverr.exe' if os.name == 'nt' else 'flaresolverr'
+    exe_new_name = 'flaresolverr_windows_x64.exe' if os.name == 'nt' else 'flaresolverr_linux_x64'
+    exe_path = os.path.join(exe_folder, exe_name)
+    exe_new_path = os.path.join(exe_folder, exe_new_name)
+    shutil.move(exe_path, exe_new_path)
+    print("Executable path: " + exe_new_path)
+
+
+if __name__ == "__main__":
+    print("Building package...")
+    print("Platform: " + platform.platform())
+
+    print("Cleaning previous build...")
+    clean_files()
+
+    print("Downloading Chromium...")
+    download_chromium()
+
+    print("Building pyinstaller executable... ")
+    run_pyinstaller()
+
+# NOTE: python -m pip install pyinstaller

src/flaresolverr.py

@@ -3,6 +3,7 @@
 import os
 import sys
 
+import certifi
 from bottle import run, response, Bottle, request, ServerAdapter
 
 from bottle_plugins.error_plugin import error_plugin
@@ -60,6 +61,12 @@ def controller_v1():
 
 
 if __name__ == "__main__":
+    # fix ssl certificates for compiled binaries
+    # https://github.com/pyinstaller/pyinstaller/issues/7229
+    # https://stackoverflow.com/questions/55736855/how-to-change-the-cafile-argument-in-the-ssl-module-in-python3
+    os.environ["REQUESTS_CA_BUNDLE"] = certifi.where()
+    os.environ["SSL_CERT_FILE"] = certifi.where()
+
     # validate configuration
     log_level = os.environ.get('LOG_LEVEL', 'info').upper()
     log_html = utils.get_config_log_html()

src/utils.py

@@ -8,6 +8,7 @@
 import undetected_chromedriver as uc
 
 FLARESOLVERR_VERSION = None
+CHROME_EXE_PATH = None
 CHROME_MAJOR_VERSION = None
 USER_AGENT = None
 XVFB_DISPLAY = None
@@ -28,6 +29,8 @@ def get_flaresolverr_version() -> str:
         return FLARESOLVERR_VERSION
 
     package_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'package.json')
+    if not os.path.isfile(package_path):
+        package_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'package.json')
     with open(package_path) as f:
         FLARESOLVERR_VERSION = json.loads(f.read())['version']
         return FLARESOLVERR_VERSION
@@ -72,9 +75,13 @@ def get_webdriver() -> WebDriver:
         if PATCHED_DRIVER_PATH is not None:
             driver_exe_path = PATCHED_DRIVER_PATH
 
+    # detect chrome path
+    browser_executable_path = get_chrome_exe_path()
+
     # downloads and patches the chromedriver
     # if we don't set driver_executable_path it downloads, patches, and deletes the driver each time
-    driver = uc.Chrome(options=options, driver_executable_path=driver_exe_path, version_main=version_main,
+    driver = uc.Chrome(options=options, browser_executable_path=browser_executable_path,
+                       driver_executable_path=driver_exe_path, version_main=version_main,
                        windows_headless=windows_headless)
 
     # save the patched driver to avoid re-downloads
@@ -94,7 +101,22 @@ def get_webdriver() -> WebDriver:
 
 
 def get_chrome_exe_path() -> str:
-    return uc.find_chrome_executable()
+    global CHROME_EXE_PATH
+    if CHROME_EXE_PATH is not None:
+        return CHROME_EXE_PATH
+    # linux pyinstaller bundle
+    chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome")
+    if os.path.exists(chrome_path) and os.access(chrome_path, os.X_OK):
+        CHROME_EXE_PATH = chrome_path
+        return CHROME_EXE_PATH
+    # windows pyinstaller bundle
+    chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome.exe")
+    if os.path.exists(chrome_path) and os.access(chrome_path, os.X_OK):
+        CHROME_EXE_PATH = chrome_path
+        return CHROME_EXE_PATH
+    # system
+    CHROME_EXE_PATH = uc.find_chrome_executable()
+    return CHROME_EXE_PATH
 
 
 def get_chrome_major_version() -> str:
@@ -103,17 +125,17 @@ def get_chrome_major_version() -> str:
         return CHROME_MAJOR_VERSION
 
     if os.name == 'nt':
+        # Example: '104.0.5112.79'
         try:
-            stream = os.popen(
-                'reg query "HKLM\\SOFTWARE\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\Google Chrome"')
-            output = stream.read()
-            # Example: '104.0.5112.79'
-            complete_version = extract_version_registry(output)
+            complete_version = extract_version_nt_executable(get_chrome_exe_path())
         except Exception:
-            # Example: '104.0.5112.79'
-            complete_version = extract_version_folder()
+            try:
+                complete_version = extract_version_nt_registry()
+            except Exception:
+                # Example: '104.0.5112.79'
+                complete_version = extract_version_nt_folder()
     else:
-        chrome_path = uc.find_chrome_executable()
+        chrome_path = get_chrome_exe_path()
         process = os.popen(f'"{chrome_path}" --version')
         # Example 1: 'Chromium 104.0.5112.79 Arch Linux\n'
         # Example 2: 'Google Chrome 104.0.5112.79 Arch Linux\n'
@@ -124,20 +146,29 @@ def get_chrome_major_version() -> str:
     return CHROME_MAJOR_VERSION
 
 
-def extract_version_registry(output) -> str:
-    try:
-        google_version = ''
-        for letter in output[output.rindex('DisplayVersion    REG_SZ') + 24:]:
-            if letter != '\n':
-                google_version += letter
-            else:
-                break
-        return google_version.strip()
-    except TypeError:
-        return ''
-
-
-def extract_version_folder() -> str:
+def extract_version_nt_executable(exe_path: str) -> str:
+    import pefile
+    pe = pefile.PE(exe_path, fast_load=True)
+    pe.parse_data_directories(
+        directories=[pefile.DIRECTORY_ENTRY["IMAGE_DIRECTORY_ENTRY_RESOURCE"]]
+    )
+    return pe.FileInfo[0][0].StringTable[0].entries[b"FileVersion"].decode('utf-8')
+
+
+def extract_version_nt_registry() -> str:
+    stream = os.popen(
+        'reg query "HKLM\\SOFTWARE\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\Google Chrome"')
+    output = stream.read()
+    google_version = ''
+    for letter in output[output.rindex('DisplayVersion    REG_SZ') + 24:]:
+        if letter != '\n':
+            google_version += letter
+        else:
+            break
+    return google_version.strip()
+
+
+def extract_version_nt_folder() -> str:
     # Check if the Chrome folder exists in the x32 or x64 Program Files folders.
     for i in range(2):
         path = 'C:\\Program Files' + (' (x86)' if i else '') + '\\Google\\Chrome\\Application'