diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 49f3f209..27eb0dee 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -14,45 +14,162 @@ permissions: contents: write env: - NSS_VERSION: nss-3.77 + NSS_VERSION: nss-3.92 BORING_SSL_COMMIT: d24a38200fef19150eef00cad35b138936c08767 jobs: - build-and-test: - name: Build curl-impersonate and run the tests - runs-on: ${{ matrix.os }} + build-and-test-linux: + name: (Linux ${{ matrix.arch }}) Build curl-impersonate and run the tests + runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - os: [ubuntu-20.04, macos-12] arch: [x86_64] include: - - os: ubuntu-20.04 - arch: x86_64 + - arch: x86_64 + go_arch: amd64 + docker_arch: linux/amd64 host: x86_64-linux-gnu capture_interface: eth0 - make: make - - os: ubuntu-20.04 - arch: aarch64 + image: quay.io/pypa/manylinux2014_x86_64:latest + - arch: aarch64 + go_arch: arm64 + docker_arch: linux/arm/v8 host: aarch64-linux-gnu capture_interface: eth0 - make: make - - os: ubuntu-20.04 - arch: arm - host: arm-linux-gnueabihf + image: quay.io/pypa/manylinux2014_aarch64:latest + - arch: arm + go_arch: armv6l + docker_arch: linux/arm/v7 + host: arm-linux-gnu capture_interface: eth0 - make: make - - os: macos-12 - arch: x86_64 - host: x86_64-macos - capture_interface: en0 - make: gmake + image: ghcr.io/bjia56/armv7l-wheel-builder:main steps: - - uses: actions/setup-python@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Install dependencies + run: | + python_bin_dir=$(docker run ${{ matrix.image }} python3.10 -c 'import sys; import os; print(os.path.dirname(os.path.realpath(sys.executable)))') + echo "runner_uid=$(id -u)" >> $GITHUB_ENV + echo "runner_gid=$(id -g)" >> $GITHUB_ENV + echo "runner_home=$HOME" >> $GITHUB_ENV + docker build -t curl-impersonate-builder -f - . < + -v ${{ env.runner_home }}:${{ env.runner_home }} + --workdir ${{ github.workspace }} + --user ${{ env.runner_uid }}:${{ env.runner_gid }} + run: | + set -e + mkdir ${{ runner.temp }}/install + ./configure --prefix=${{ runner.temp }}/install \ + --with-ca-path=/etc/ssl/certs \ + --with-ca-bundle=/etc/ssl/certs/ca-certificates.crt \ + --with-libnssckbi=/usr/lib/${{ matrix.host }}/nss + + # Cache the build of BoringSSL, which is the longest part of the build + # We must cache the .zip as well, otherwise the Makefile will + # rebuild BoringSSL. This whole thing is a bit hacky, but necessary to + # reduce the insanely long build times. + - name: Cache BoringSSL source + uses: actions/cache@v3 + with: + path: boringssl.zip + key: ${{ runner.os }}-${{ matrix.arch }}-boring-source-${{ env.BORING_SSL_COMMIT }} + + - name: Cache BoringSSL build + id: cache-boringssl + uses: actions/cache@v3 + with: + path: boringssl/build + key: ${{ runner.os }}-${{ matrix.arch }}-boring-build-${{ env.BORING_SSL_COMMIT }}-${{ hashFiles('chrome/patches/boringssl*.patch') }} - - name: Install Ubuntu dependencies - if: matrix.os == 'ubuntu-20.04' + # Trick the Makefile into skipping the BoringSSL build step + # if it was found in the cache. See Makefile.in + - name: Post BoringSSL cache restore + if: ${{ steps.cache-boringssl.outputs.cache-hit != false }} + run: | + touch boringssl.zip + touch boringssl/.patched + find boringssl/build -type f | xargs touch + + - name: Build the Chrome version of curl-impersonate + uses: addnab/docker-run-action@v3 + with: + image: curl-impersonate-builder + options: > + -v ${{ env.runner_home }}:${{ env.runner_home }} + --workdir ${{ github.workspace }} + --user ${{ env.runner_uid }}:${{ env.runner_gid }} + run: | + set -e + make chrome-build + make chrome-checkbuild + make chrome-install + + # Cache the build of NSS, which is the longest part of the build + # We must cache the .tar.gz as well, otherwise the Makefile will + # rebuild NSS. + - name: Cache NSS source + uses: actions/cache@v3 + with: + path: ${{ env.NSS_VERSION }}.tar.gz + key: ${{ runner.os }}-${{ matrix.arch }}-nss-source-${{ env.NSS_VERSION }} + + - name: Cache NSS build + id: cache-nss + uses: actions/cache@v3 + with: + path: ${{ env.NSS_VERSION }}/dist + key: ${{ runner.os }}-${{ matrix.arch }}-nss-build-${{ env.NSS_VERSION }} + + # Trick the Makefile into skipping the NSS build step + # if it was found in the cache. + - name: Post NSS cache restore + if: ${{ steps.cache-nss.outputs.cache-hit != false }} + run: | + touch ${{ env.NSS_VERSION }}.tar.gz + find ${{ env.NSS_VERSION }}/dist -type f | xargs touch + + - name: Build the Firefox version of curl-impersonate + uses: addnab/docker-run-action@v3 + with: + image: curl-impersonate-builder + options: > + -v ${{ env.runner_home }}:${{ env.runner_home }} + --workdir ${{ github.workspace }} + --user ${{ env.runner_uid }}:${{ env.runner_gid }} + run: | + set -e + make firefox-build + make firefox-checkbuild + make firefox-install + + - name: Install test dependencies + if: matrix.arch == 'x86_64' run: | sudo apt-get update sudo apt-get install build-essential pkg-config cmake ninja-build curl autoconf automake libtool @@ -63,13 +180,122 @@ jobs: # More dependencies for the tests sudo apt-get install tcpdump nghttp2-server libnss3 - - name: Install Ubuntu cross-compile dependencies (${{ matrix.arch }}) - if: matrix.os == 'ubuntu-20.04' && matrix.arch != 'x86_64' + - name: Prepare the tests + if: matrix.arch == 'x86_64' + run: | + # Compile 'minicurl' which is used by the tests + gcc -Wall -Werror -o ${{ runner.temp }}/install/bin/minicurl tests/minicurl.c `curl-config --libs` + + - uses: actions/setup-python@v4 + if: matrix.arch == 'x86_64' + with: + python-version: '3.10' + + - name: Install dependencies for the tests script + if: matrix.arch == 'x86_64' + run: | + pip3 install -r tests/requirements.txt + + # For now we can only run the tests when native + # tests run the curl-impersonate binary locally. + - name: Run the tests + if: matrix.arch == 'x86_64' + run: | + cd tests + # sudo is needed for capturing packets + python_bin=$(which python3) + sudo $python_bin -m pytest . --log-cli-level DEBUG --install-dir ${{ runner.temp }}/install --capture-interface ${{ matrix.capture_interface }} + + # Upload pre-compiled binaries to GitHub releases page. + - name: Create tar release files for libcurl-impersonate + if: startsWith(github.ref, 'refs/tags/') run: | - sudo apt-get install gcc-${{ matrix.host }} g++-${{ matrix.host }} + cd ${{ runner.temp }}/install/lib + tar -c -z -f ${{ runner.temp }}/libcurl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz libcurl-impersonate* + echo "release_file_lib=${{ runner.temp }}/libcurl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz" >> $GITHUB_ENV + + - name: Clean build + if: startsWith(github.ref, 'refs/tags/') + uses: addnab/docker-run-action@v3 + with: + image: curl-impersonate-builder + options: > + -v ${{ env.runner_home }}:${{ env.runner_home }} + --workdir ${{ github.workspace }} + --user ${{ env.runner_uid }}:${{ env.runner_gid }} + run: | + set -e + make chrome-clean + make firefox-clean + rm -Rf ${{ runner.temp }}/install + mkdir ${{ runner.temp }}/install + + # Recompile curl-impersonate statically when doing a release. + - name: Reconfigure statically + if: startsWith(github.ref, 'refs/tags/') && matrix.arch == 'x86_64' + uses: addnab/docker-run-action@v3 + with: + image: curl-impersonate-builder + options: > + -v ${{ env.runner_home }}:${{ env.runner_home }} + --workdir ${{ github.workspace }} + --user ${{ env.runner_uid }}:${{ env.runner_gid }} + run: | + set -e + ./configure --prefix=${{ runner.temp }}/install --enable-static + + - name: Rebuild statically + if: startsWith(github.ref, 'refs/tags/') + uses: addnab/docker-run-action@v3 + with: + image: curl-impersonate-builder + options: > + -v ${{ env.runner_home }}:${{ env.runner_home }} + --workdir ${{ github.workspace }} + --user ${{ env.runner_uid }}:${{ env.runner_gid }} + run: | + set -e + make chrome-build + make chrome-checkbuild + make chrome-install-strip + make firefox-build + make firefox-checkbuild + make firefox-install-strip + + - name: Create tar release files for curl-impersonate + if: startsWith(github.ref, 'refs/tags/') + run: | + cd ${{ runner.temp }}/install/bin + tar -c -z -f ${{ runner.temp }}/curl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz curl-impersonate-ff curl-impersonate-chrome curl_* + echo "release_file_bin=${{ runner.temp }}/curl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz" >> $GITHUB_ENV + + - name: Upload release files + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') + with: + files: | + ${{ env.release_file_lib }} + ${{ env.release_file_bin }} + + build-and-test-macos: + name: (MacOS ${{ matrix.arch }}) Build curl-impersonate and run the tests + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [macos-12] + arch: [x86_64] + include: + - os: macos-12 + arch: x86_64 + host: x86_64-macos + capture_interface: en0 + make: gmake + steps: + - uses: actions/setup-python@v4 + with: + python-version: '3.10' - name: Install macOS dependencies - if: matrix.os == 'macos-12' run: | brew install pkg-config make cmake ninja autoconf automake libtool # Chrome version dependencies @@ -79,9 +305,6 @@ jobs: brew install curl # More dependencies for the tests brew install tcpdump nghttp2 nss - - - name: Install common dependencies - run: | # Firefox version dependencies pip3 install gyp-next @@ -92,7 +315,6 @@ jobs: run: | pip3 install -r tests/requirements.txt - # When cross compiling we need to build zlib first. - name: Build zlib run: | curl -LO https://zlib.net/zlib-1.3.tar.gz @@ -110,20 +332,6 @@ jobs: mkdir ${{ runner.temp }}/install ./configure --prefix=${{ runner.temp }}/install - # When cross compiling a more complicated configuration is needed, since - # curl's configure script can't figure out where some files and libraries - # are located. The locations used here are the ones used by Ubuntu. - - name: Run configure script (cross compiling) - if: matrix.arch != 'x86_64' - run: | - mkdir ${{ runner.temp }}/install - ./configure --prefix=${{ runner.temp }}/install \ - --host=${{ matrix.host }} \ - --with-zlib=${{ runner.temp }}/zlib \ - --with-ca-path=/etc/ssl/certs \ - --with-ca-bundle=/etc/ssl/certs/ca-certificates.crt \ - --with-libnssckbi=/usr/lib/${{ matrix.host }}/nss - # Cache the build of BoringSSL, which is the longest part of the build # We must cache the .zip as well, otherwise the Makefile will # rebuild BoringSSL. This whole thing is a bit hacky, but necessary to diff --git a/.github/workflows/build-win.yaml b/.github/workflows/build-win.yaml index 1c386a5e..e8f644ca 100644 --- a/.github/workflows/build-win.yaml +++ b/.github/workflows/build-win.yaml @@ -75,6 +75,7 @@ jobs: run: ./win/dll2lib.bat ${{ matrix.env }} build\dist\libcurl.dll - name: Build tarball + if: startsWith(github.ref, 'refs/tags/') shell: msys2 {0} run: tar cvzf libcurl-impersonate-${{ github.head_ref || github.ref_name }}.${{ matrix.env }}-win32.tar.gz -C ./build/dist . diff --git a/Makefile.in b/Makefile.in index 3190fabe..95ba5182 100644 --- a/Makefile.in +++ b/Makefile.in @@ -7,6 +7,7 @@ SHELL := bash .DELETE_ON_ERROR: # MAKEFLAGS += --warn-undefined-variables # MAKEFLAGS += --no-builtin-rules +SUBJOBS := 4 BROTLI_VERSION := 1.0.9 # In case this is changed, update build-and-test-make.yml as well @@ -67,7 +68,7 @@ help: ## Show this help message firefox-build: $(CURL_VERSION)/.firefox ## Build the Firefox version of curl-impersonate cd $(CURL_VERSION) # Don't pass this Makefile's MAKEFLAGS - $(MAKE) MAKEFLAGS= + $(MAKE) MAKEFLAGS=-j$(SUBJOBS) .PHONY: firefox-build firefox-checkbuild: ## Run basic checks on the built binary @@ -116,7 +117,7 @@ firefox-clean: ## Clean build artifacts of the Firefox version. Use after re-run chrome-build: $(CURL_VERSION)/.chrome ## Build the Chrome version of curl-impersonate cd $(CURL_VERSION) # Don't pass this Makefile's MAKEFLAGS - $(MAKE) MAKEFLAGS= + $(MAKE) MAKEFLAGS=-j$(SUBJOBS) .PHONY: chrome-build chrome-checkbuild: ## Run basic checks on the built binary @@ -201,7 +202,7 @@ $(brotli_static_libs): brotli-$(BROTLI_VERSION).tar.gz -DCMAKE_SYSTEM_PROCESSOR=$(host_cpu) \ .. - @cmake@ --build . --config Release --target install + @cmake@ --build . --config Release --target install --parallel $(SUBJOBS) $(NSS_VERSION).tar.gz: @@ -213,7 +214,7 @@ $(nss_static_libs): $(NSS_VERSION).tar.gz ifeq ($(host),$(build)) # Native build, use NSS' build script. cd $(NSS_VERSION)/nss - ./build.sh -o --disable-tests --static --python=python3 + ./build.sh -o --disable-tests --static --python=python3 -j $(SUBJOBS) else # We are cross compiling. # Cross compiling NSS is not supported by its build script and is poorly @@ -294,7 +295,7 @@ $(boringssl_static_libs): boringssl.zip boringssl/.patched -DCMAKE_SYSTEM_PROCESSOR=$(host_cpu) \ -GNinja \ .. - @ninja@ + @ninja@ -j$(SUBJOBS) # Fix the directory structure so that curl can compile against it. # See https://everything.curl.dev/source/build/tls/boringssl mkdir -p lib @@ -323,7 +324,7 @@ $(nghttp2_static_libs): $(NGHTTP2_VERSION).tar.bz2 } ./configure $$config_flags - $(MAKE) MAKEFLAGS= + $(MAKE) MAKEFLAGS=-j$(SUBJOBS) $(MAKE) install MAKEFLAGS= $(CURL_VERSION).tar.xz: diff --git a/chrome/patches/boringssl-old-ciphers.patch b/chrome/patches/boringssl-old-ciphers.patch index 2b278299..b3821999 100644 --- a/chrome/patches/boringssl-old-ciphers.patch +++ b/chrome/patches/boringssl-old-ciphers.patch @@ -207,3 +207,34 @@ index 57116cd6c..fa1652832 100644 } // Check for invalid algorithms, and filter out |SSL_SIGN_RSA_PKCS1_MD5_SHA1|. +diff --git a/crypto/fipsmodule/rand/urandom_test.cc b/crypto/fipsmodule/rand/urandom_test.cc +index 08e4183..d486c01 100644 +--- a/crypto/fipsmodule/rand/urandom_test.cc ++++ b/crypto/fipsmodule/rand/urandom_test.cc +@@ -38,10 +38,26 @@ + #include "fork_detect.h" + #include "getrandom_fillin.h" + ++#if !defined(NT_ARM_SYSTEM_CALL) ++// https://elixir.bootlin.com/linux/v6.6.8/source/include/uapi/linux/elf.h#L433 ++#define NT_ARM_SYSTEM_CALL 0x404 /* ARM system call number */ ++#endif ++ + #if !defined(PTRACE_O_EXITKILL) + #define PTRACE_O_EXITKILL (1 << 20) + #endif + ++#if defined(OPENSSL_AARCH64) && defined(__linux__) ++// https://elixir.bootlin.com/glibc/glibc-2.38/source/sysdeps/unix/sysv/linux/aarch64/sys/user.h#L22 ++struct user_regs_struct ++{ ++ unsigned long long regs[31]; ++ unsigned long long sp; ++ unsigned long long pc; ++ unsigned long long pstate; ++}; ++#endif ++ + #if defined(OPENSSL_ANDROID) + static const bool kIsAndroid = true; + #else diff --git a/firefox/patches/curl-impersonate.patch b/firefox/patches/curl-impersonate.patch index fe461274..c0a71d04 100644 --- a/firefox/patches/curl-impersonate.patch +++ b/firefox/patches/curl-impersonate.patch @@ -1452,7 +1452,7 @@ index 2916c3613..f07fa30ac 100644 + esac + + case $host_cpu in -+ arm) ++ arm|armv7l) + addlib="$addlib -larmv8_c_lib -lgcm-aes-arm32-neon_c_lib" + ;; + aarch64) diff --git a/tests/test_impersonate.py b/tests/test_impersonate.py index cb600b84..cb407f20 100644 --- a/tests/test_impersonate.py +++ b/tests/test_impersonate.py @@ -160,7 +160,7 @@ async def nghttpd(): # Otherwise fail. started = await asyncio.wait_for(_wait_nghttpd(proc), timeout=3) if not started: - raise Exception("nghttpd failed to start on time") + raise Exception("nghttpd failed to start") except asyncio.TimeoutError: raise Exception("nghttpd failed to start on time")