Skip to content

Build and test GYB #1537

Build and test GYB

Build and test GYB #1537

Workflow file for this run

name: Build and test GYB
permissions:
id-token: write
attestations: write
contents: read
on:
push:
pull_request:
schedule:
- cron: '37 22 * * *'
defaults:
run:
shell: bash
env:
OPENSSL_CONFIG_OPTS: no-fips --api=3.0.0
OPENSSL_INSTALL_PATH: ${{ github.workspace }}/ssl
OPENSSL_SOURCE_PATH: ${{ github.workspace }}/openssl
PYTHON_INSTALL_PATH: ${{ github.workspace }}/python
PYTHON_SOURCE_PATH: ${{ github.workspace }}/cpython
jobs:
build:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-20.04
jid: 1
goal: build
arch: x86_64
- os: ubuntu-20.04
jid: 2
goal: build
arch: x86_64
staticx: yes
- os: macos-12
jid: 3
goal: build
arch: x86_64
- os: macos-14
jid: 4
goal: build
arch: aarch64
- os: windows-2022
jid: 5
goal: build
arch: Win64
- os: [self-hosted, linux, arm64]
jid: 6
goal: build
arch: aarch64
- os: [self-hosted, linux, arm64]
jid: 7
goal: build
arch: aarch64
staticx: yes
- os: ubuntu-22.04
goal: test
python: "3.9"
jid: 8
- os: ubuntu-22.04
goal: test
python: "3.10"
jid: 9
- os: ubuntu-22.04
goal: test
python: "3.11"
jid: 10
- os: ubuntu-22.04
goal: test
python: "3.12"
jid: 11
- os: ubuntu-22.04
goal: test
python: "3.13.0-beta.1"
jid: 12
steps:
- uses: actions/checkout@master
with:
persist-credentials: false
fetch-depth: 0
- name: Cache the build
if: matrix.goal == 'build'
id: cache-python-ssl
uses: actions/cache@v4
with:
path: |
cache.tar.xz
key: ${{ matrix.jid }}-2024-05-03
- name: Untar Cache archive
if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit == 'true'
working-directory: ${{ github.workspace }}
run: |
tar xvvf cache.tar.xz
- name: Use pre-compiled Python
if: matrix.python != ''
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: Set env variables for test
if: matrix.goal == 'test'
env:
JID: ${{ matrix.jid }}
ACTIONS_CACHE: ${{ steps.cache-python-ssl.outputs.cache-hit }}
ACTIONS_GOAL: ${{ matrix.goal }}
run: |
export PYTHON=$(which python3)
export gyb="${PYTHON} -m gyb"
export gybpath="$(readlink -e .)"
echo -e "PYTHON: ${PYTHON}\nPIP: ${PIP}\ngyb: ${gyb}\ngybpath: ${gybpath}"
echo "PYTHON=${PYTHON}" >> $GITHUB_ENV
echo "gyb=${gyb}" >> $GITHUB_ENV
echo "gybpath=${gybpath}" >> $GITHUB_ENV
echo "JID=${JID}" >> $GITHUB_ENV
echo "ACTIONS_CACHE=${ACTIONS_CACHE}" >> $GITHUB_ENV
echo "ACTIONS_GOAL=${ACTIONS_GOAL}" >> $GITHUB_ENV
- name: Set Env Variables for build
if: matrix.goal == 'build'
env:
arch: ${{ matrix.arch }}
jid: ${{ matrix.jid }}
staticx: ${{ matrix.staticx }}
run: |
echo "We are running on ${RUNNER_OS}"
if [[ "${arch}" == "Win64" ]]; then
PYEXTERNALS_PATH="amd64"
PYBUILDRELEASE_ARCH="x64"
OPENSSL_CONFIG_TARGET="VC-WIN64A"
CHOC_OPS=""
elif [[ "${arch}" == "Win32" ]]; then
PYEXTERNALS_PATH="win32"
PYBUILDRELEASE_ARCH="Win32"
OPENSSL_CONFIG_TARGET="VC-WIN32"
CHOC_OPS="--forcex86"
fi
if [[ "${RUNNER_OS}" == "macOS" ]]; then
MAKE=make
OS=macos
MAKEOPT="-j$(sysctl -n hw.logicalcpu)"
PERL=perl
# We only care about non-deprecated OSes
MACOSX_DEPLOYMENT_TARGET="10.15"
echo "MACOSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET}" >> $GITHUB_ENV
echo "PYTHON=${PYTHON_INSTALL_PATH}/bin/python3" >> $GITHUB_ENV
elif [[ "${RUNNER_OS}" == "Linux" ]]; then
MAKE=make
OS=linux
MAKEOPT="-j$(nproc)"
PERL=perl
echo "PYTHON=${PYTHON_INSTALL_PATH}/bin/python3" >> $GITHUB_ENV
elif [[ "${RUNNER_OS}" == "Windows" ]]; then
MAKE=nmake
OS=windows
MAKEOPT=""
PERL="c:\strawberry\perl\bin\perl.exe"
echo "PYTHON=${PYTHON_SOURCE_PATH}/PCbuild/${PYEXTERNALS_PATH}/python.exe" >> $GITHUB_ENV
fi
echo "We'll run make with: ${MAKEOPT}"
echo "JID=${jid}" >> $GITHUB_ENV
echo "OS=${OS}" >> $GITHUB_ENV
echo "arch=${arch}" >> $GITHUB_ENV
echo "staticx=${staticx}" >> $GITHUB_ENV
echo "MAKE=${MAKE}" >> $GITHUB_ENV
echo "MAKEOPT=${MAKEOPT}" >> $GITHUB_ENV
echo "PERL=${PERL}" >> $GITHUB_ENV
echo "PYEXTERNALS_PATH=${PYEXTERNALS_PATH}" >> $GITHUB_ENV
echo "PYBUILDRELEASE_ARCH=${PYBUILDRELEASE_ARCH}" >> $GITHUB_ENV
echo "OPENSSL_CONFIG_TARGET=${OPENSSL_CONFIG_TARGET}" >> $GITHUB_ENV
echo "CHOC_OPS=${CHOC_OPS}" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=${OPENSSL_INSTALL_PATH}/lib:${PYTHON_INSTALL_PATH}/lib:/usr/local/lib" >> $GITHUB_ENV
- name: Get latest stable OpenSSL source
if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
run: |
git clone https://github.com/openssl/openssl.git
cd "${OPENSSL_SOURCE_PATH}"
export LATEST_STABLE_TAG=$(git tag --list openssl-* | grep -v alpha | grep -v beta | sort -Vr | head -n1)
echo "Checking out version ${LATEST_STABLE_TAG}"
git checkout "${LATEST_STABLE_TAG}"
export COMPILED_OPENSSL_VERSION=${LATEST_STABLE_TAG:8} # Trim the openssl- prefix
echo "COMPILED_OPENSSL_VERSION=${COMPILED_OPENSSL_VERSION}" >> $GITHUB_ENV
- name: Windows Configure VCode
uses: ilammy/msvc-dev-cmd@v1
if: matrix.os == 'windows-2022' && steps.cache-python-ssl.outputs.cache-hit != 'true'
with:
arch: ${{ matrix.arch }}
- name: Windows Choco Packages Install
if: runner.os == 'Windows'
run: |
choco install sqlite $CHOC_OPS
- name: Windows NASM Install
uses: ilammy/setup-nasm@v1
if: matrix.goal == 'build' && runner.os == 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
- name: Config OpenSSL
if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
run: |
cd "${OPENSSL_SOURCE_PATH}"
# --libdir=lib is needed so Python can find OpenSSL libraries
"${PERL}" ./Configure "${OPENSSL_CONFIG_TARGET}" --libdir=lib --prefix="${OPENSSL_INSTALL_PATH}" $OPENSSL_CONFIG_OPTS
- name: Rename GNU link on Windows
if: matrix.goal == 'build' && runner.os == 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
shell: bash
run: mv /usr/bin/link /usr/bin/gnulink
- name: Make OpenSSL
if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
run: |
cd "${OPENSSL_SOURCE_PATH}"
$MAKE "${MAKEOPT}"
- name: Install OpenSSL
if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
run: |
cd "${OPENSSL_SOURCE_PATH}"
# install_sw saves us ages processing man pages :-)
$MAKE install_sw
- name: Run OpenSSL
if: matrix.goal == 'build'
run: |
"${OPENSSL_INSTALL_PATH}/bin/openssl" version
- name: Get latest stable Python source
if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
run: |
git clone https://github.com/python/cpython.git
cd "${PYTHON_SOURCE_PATH}"
export LATEST_STABLE_TAG=$(git tag --list | grep -v a | grep -v rc | grep -v b | sort -Vr | head -n1)
git checkout "${LATEST_STABLE_TAG}"
export COMPILED_PYTHON_VERSION=${LATEST_STABLE_TAG:1} # Trim the "v" prefix
echo "COMPILED_PYTHON_VERSION=${COMPILED_PYTHON_VERSION}" >> $GITHUB_ENV
- name: Mac/Linux Configure Python
if: matrix.goal == 'build' && runner.os != 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
run: |
cd "${PYTHON_SOURCE_PATH}"
./configure --with-openssl="${OPENSSL_INSTALL_PATH}" \
--prefix="${PYTHON_INSTALL_PATH}" \
--enable-shared \
--with-ensurepip=upgrade \
--enable-optimizations \
--with-lto
- name: Windows Get External Python deps
if: matrix.goal == 'build' && runner.os == 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
shell: powershell
run: |
cd "${env:PYTHON_SOURCE_PATH}"
PCBuild\get_externals.bat
- name: Windows overwrite external OpenSSL with local
if: matrix.goal == 'build' && runner.os == 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
shell: powershell
run: |
cd "${env:PYTHON_SOURCE_PATH}"
$env:OPENSSL_EXT_PATH = "$(Get-Item externals\openssl-bin-* | Select -exp FullName)\"
echo "External OpenSSL was downloaded to ${env:OPENSSL_EXT_PATH}"
Remove-Item -recurse -force "${env:OPENSSL_EXT_PATH}*"
# Emulate what this script does:
# https://github.com/python/cpython/blob/main/PCbuild/openssl.vcxproj
$env:OPENSSL_EXT_TARGET_PATH = "${env:OPENSSL_EXT_PATH}${env:PYEXTERNALS_PATH}"
echo "Copying our OpenSSL to ${env:OPENSSL_EXT_TARGET_PATH}"
mkdir "${env:OPENSSL_EXT_TARGET_PATH}\include\openssl\"
Copy-Item -Path "${env:OPENSSL_SOURCE_PATH}\LICENSE.txt" -Destination "${env:OPENSSL_EXT_TARGET_PATH}\LICENSE" -verbose
cp "$env:OPENSSL_INSTALL_PATH\lib\*" "${env:OPENSSL_EXT_TARGET_PATH}"
cp "$env:OPENSSL_INSTALL_PATH\bin\*" "${env:OPENSSL_EXT_TARGET_PATH}"
cp "$env:OPENSSL_INSTALL_PATH\include\openssl\*" "${env:OPENSSL_EXT_TARGET_PATH}\include\openssl\"
cp "$env:OPENSSL_INSTALL_PATH\include\openssl\applink.c" "${env:OPENSSL_EXT_TARGET_PATH}\include\"
- name: Windows Install sphinx-build
if: matrix.goal == 'build' && runner.os == 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
shell: powershell
run: |
pip install --upgrade pip
pip install --upgrade sphinx
sphinx-build --version
- name: Windows Config/Build Python
if: matrix.goal == 'build' && runner.os == 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
shell: powershell
run: |
cd "${env:PYTHON_SOURCE_PATH}"
# We need out custom openssl.props which uses OpenSSL 3 DLL names
Copy-Item -Path "${env:GITHUB_WORKSPACE}\openssl.props" -Destination PCBuild\ -verbose
echo "Building for ${env:PYBUILDRELEASE_ARCH}..."
PCBuild\build.bat -m --pgo -c Release -p "${env:PYBUILDRELEASE_ARCH}"
- name: Mac/Linux Build Python
if: matrix.goal == 'build' && runner.os != 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
run: |
cd "${PYTHON_SOURCE_PATH}"
echo "Running: ${MAKE} ${MAKEOPT}"
$MAKE $MAKEOPT
- name: Mac/Linux Install Python
if: matrix.goal == 'build' && runner.os != 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
run: |
cd "${PYTHON_SOURCE_PATH}"
$MAKE altinstall
$MAKE bininstall
- name: Run Python
run: |
"${PYTHON}" -V
- name: Upgrade pip, wheel, etc
run: |
curl -O https://bootstrap.pypa.io/get-pip.py
"${PYTHON}" get-pip.py
"${PYTHON}" -m pip install --upgrade pip
"${PYTHON}" -m pip install --upgrade wheel
"${PYTHON}" -m pip install --upgrade setuptools
- name: Install pip requirements
run: |
set +e
outdated=$("$PYTHON" -m pip list --outdated | grep -v 'Package' | grep -v '\-\-\-' | cut -d ' ' -f 1)
"$PYTHON" -m pip install --upgrade --force-reinstall "$outdated"
"$PYTHON" -m pip install --upgrade -r requirements.txt
- name: Install PyInstaller
if: matrix.goal == 'build'
run: |
git clone https://github.com/pyinstaller/pyinstaller.git
cd pyinstaller
export latest_release=$(git tag --list | grep -v dev | grep -v rc | sort -Vr | head -n1)
echo $latest_release
git checkout "${latest_release}"
# remove pre-compiled bootloaders so we fail if bootloader compile fails
rm -rf PyInstaller/bootloader/*-*/*
cd bootloader
if [[ "${arch}" == "Win64" ]]; then
export PYINSTALLER_BUILD_ARGS="--target-arch=64bit"
fi
echo "PyInstaller build arguments: ${PYINSTALLER_BUILD_ARGS}"
"${PYTHON}" ./waf all $PYINSTALLER_BUILD_ARGS
cd ..
"${PYTHON}" -m pip install .
"${PYTHON}" -m PyInstaller --version
- name: Build GYB with PyInstaller
if: matrix.goal == 'build'
run: |
"${PYTHON}" -m PyInstaller --clean --distpath=gyb gyb.spec
if [ -x "$(command -v realpath)" ]; then
realpath=realpath
else
brew install coreutils
realpath=grealpath
fi
echo "gybpath=$(${realpath} ./gyb/)" >> $GITHUB_ENV
gyb=$(${realpath} ./gyb/gyb)
if [[ "${RUNNER_OS}" == "Windows" ]]; then
gyb=$(cygpath -w "$gyb")
fi
echo "gyb=${gyb}" >> $GITHUB_ENV
echo -e "GYB: ${gyb}\nGYBPATH: ${gybpath}"
- name: Linux install patchelf/staticx
if: matrix.staticx == 'yes'
run: |
"${PYTHON}" -m pip install --upgrade patchelf-wrapper
"${PYTHON}" -m pip install --upgrade staticx
- name: Linux Make Static GYB
if: matrix.staticx == 'yes'
run: |
case $RUNNER_ARCH in
X64)
ldlib=/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2
;;
ARM64)
ldlib=/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1
;;
esac
echo "ldlib=${ldlib}"
$PYTHON -m staticx -l "${ldlib}" gyb/gyb gyb/gyb-staticx
- name: Attest Binary Provenance
uses: actions/attest-build-provenance@v1
if: matrix.goal == 'build'
with:
subject-path: ${{ env.gyb }}
- name: Basic Tests all jobs
run: |
echo -e "PYTHON: $PYTHON\ngyb: $gyb\ngybpath: $gybpath\n"
touch "${gybpath}/nobrowser.txt"
$gyb --version
export GYBVERSION=$($gyb --short-version )
echo "GYB Version ${GYBVERSION}"
echo "GYBVERSION=${GYBVERSION}" >> $GITHUB_ENV
if [ -d "gyb" ]; then
cp LICENSE gyb/
fi
rm -f "${gybpath}/nobrowser.txt"
rm -f "${gybpath}/lastcheck.txt"
- name: Linux package GYB
if: runner.os == 'Linux' && matrix.goal == 'build'
run: |
if [[ "${staticx}" == "yes" ]]; then
this_glibc_ver="legacy"
else
this_glibc_ver=glibc$(ldd --version | awk '/ldd/{print $NF}')
fi
GYB_ARCHIVE="gyb-${GYBVERSION}-linux-$(arch)-${this_glibc_ver}.tar.xz"
tar cfJ $GYB_ARCHIVE gyb/
- name: MacOS package GYB
if: runner.os == 'macOS' && matrix.goal == 'build'
run: |
MACOSVERSION=$(defaults read loginwindow SystemVersionStampAsString)
GYB_ARCHIVE="gyb-${GYBVERSION}-macos-${arch}.tar.xz"
tar cfJ $GYB_ARCHIVE gyb/
- name: Windows package GYB
if: runner.os == 'Windows' && matrix.goal == 'build'
run: |
cp gyb-setup.bat gyb
GYB_ARCHIVE=gyb-$GYBVERSION-windows-x86_64.zip
/c/Program\ Files/7-Zip/7z.exe a -tzip $GYB_ARCHIVE gyb -xr!.svn
mkdir gyb-64
cp -rf gyb/* gyb-64/;
/c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.14/bin/candle.exe -arch x64 windows-gyb.wxs
/c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.14/bin/light.exe -ext /c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.14/bin/WixUIExtension.dll windows-gyb.wixobj -o gyb-$GYBVERSION-windows-x86_64.msi || true;
rm -f *.wixpdb
- name: Basic Tests build jobs only
if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
run: |
export voutput=$($gyb --version)
export python_line=$(echo -e "${voutput}" | grep "Python ")
export python_arr=($python_line)
export this_python=${python_arr[1]}
if [[ "${this_python}" != "${COMPILED_PYTHON_VERSION}" ]]; then
echo "ERROR: Tried to compile Python ${COMPILED_PYTHON_VERSION} but ended up with ${this_python}"
exit 1
fi
export openssl_line=$(echo -e "${voutput}" | grep "OpenSSL ")
export openssl_arr=($openssl_line)
export this_openssl=${openssl_arr[1]}
if [[ "${this_openssl}" != "${COMPILED_OPENSSL_VERSION}" ]]; then
echo "ERROR: Tried to compile OpenSSL ${COMPILED_OPENSSL_VERSION} but ended up with ${this_openssl}"
exit 1
fi
echo "We successfully compiled Python ${this_python} and OpenSSL ${this_openssl}"
- name: Live API tests push only
if: (github.event_name == 'push' || github.event_name == 'schedule') && github.repository_owner == 'GAM-team'
env:
PASSCODE: ${{ secrets.PASSCODE }}
run: |
export gyb_user="gyb-gha-${JID}@pdl.jaylee.us"
export gyb_group="gyb-travis-group-${JID}@pdl.jaylee.us"
source .github/actions/decrypt.sh .github/actions/creds.tar.gpg creds.tar
$gyb --action check-service-account --email $gyb_user
$gyb --action purge --email $gyb_user
$gyb --action purge-labels --email $gyb_user
$gyb --action restore --local-folder samples/gyb-format --email $gyb_user --cleanup
$gyb --action restore --local-folder samples/gyb-format --email $gyb_user --service-account --noresume
$gyb --action restore-group --local-folder samples/gyb-format --email $gyb_group --use-admin $gyb_user --cleanup --cleanup-from "God of Thunder <thor@avengers.com>"
$gyb --action restore-group --local-folder samples/gyb-format --email $gyb_group --use-admin $gyb_user --service-account --noresume
$gyb --action restore-group --local-folder samples/google-takeout --email $gyb_group --use-admin $gyb_user
$gyb --action restore-group --local-folder samples/vault-export-mbox --email $gyb_group --use-admin $gyb_user --service-account
$gyb --action restore-mbox --local-folder samples/historic-public-mbox --email $gyb_user --cleanup
$gyb --action restore-mbox --local-folder samples/historic-public-mbox --email $gyb_user --service-account --noresume
$gyb --action restore-mbox --local-folder samples/google-takeout --email $gyb_user --cleanup --cleanup-from "God of Thunder <thor@avengers.com>"
$gyb --action restore-mbox --local-folder samples/vault-export-mbox --email $gyb_user
$gyb --action count --email $gyb_user
$gyb --action count --email $gyb_user --service-account
$gyb --action quota --email $gyb_user
$gyb --action quota --email $gyb_user --service-account
$gyb --action estimate --email $gyb_user
$gyb --action estimate --email $gyb_user --service-account
$gyb --action print-labels --email $gyb_user
$gyb --action print-labels --email $gyb_user --service-account
$gyb --action backup --email $gyb_user
$gyb --action backup --email $gyb_user --service-account --local-folder sa-backup
$gyb --action backup-chat --email $gyb_user
- name: Attest Build Archive Provenance
uses: actions/attest-build-provenance@v1
if: (github.event_name == 'push' || github.event_name == 'schedule') && matrix.goal == 'build'
with:
subject-path: |
gyb*.tar.xz
gyb*.zip
gyb*.msi
- name: VirusTotal Scan
uses: crazy-max/ghaction-virustotal@v4
if: (github.event_name == 'push' || github.event_name == 'schedule') && matrix.goal == 'build'
with:
vt_api_key: ${{ secrets.VT_API_KEY }}
files: |
gyb*.tar.xz
gyb*.zip
gyb*.msi
- name: Archive artifacts
uses: actions/upload-artifact@v4
if: (github.event_name == 'push' || github.event_name == 'schedule') && matrix.goal == 'build'
with:
name: gyb-binaries-${{ env.OS }}-${{ env.arch }}-${{ matrix.jid }}
path: |
gyb*.tar.xz
gyb*.zip
gyb*.msi
- name: Tar Cache archive
if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
working-directory: ${{ github.workspace }}
run: |
if [[ "${RUNNER_OS}" == "Windows" ]]; then
tar_folder="cpython/ ssl/"
else
tar_folder="ssl/ python/"
fi
tar cJvvf cache.tar.xz $tar_folder