CI Maintenance

.ci/build-linux.sh

@@ -0,0 +1,65 @@
+#!/bin/sh -ex
+
+# Setup Qt variables
+export QT_BASE_DIR=/opt/qt${QTVERMIN}
+export PATH=$QT_BASE_DIR/bin:$PATH
+export LD_LIBRARY_PATH=$QT_BASE_DIR/lib/x86_64-linux-gnu:$QT_BASE_DIR/lib
+
+cd rpcs3 || exit 1
+
+# Pull all the submodules except llvm, since it is built separately and we just download that build
+# Note: Tried to use git submodule status, but it takes over 20 seconds
+# shellcheck disable=SC2046
+git submodule -q update --init $(awk '/path/ && !/llvm/ { print $3 }' .gitmodules)
+
+# Download pre-compiled llvm libs
+curl -sLO https://github.com/RPCS3/llvm-mirror/releases/download/custom-build/llvmlibs-linux.tar.gz
+mkdir llvmlibs
+tar -xzf ./llvmlibs-linux.tar.gz -C llvmlibs
+
+mkdir build && cd build || exit 1
+
+if [ "$COMPILER" = "gcc" ]; then
+    # These are set in the dockerfile
+    export CC=${GCC_BINARY}
+    export CXX=${GXX_BINARY}
+    export LINKER=gold
+    # We need to set the following variables for LTO to link properly
+    export AR=/usr/bin/gcc-ar-9
+    export RANLIB=/usr/bin/gcc-ranlib-9
+    export CFLAGS="-fuse-linker-plugin"
+else
+    export CC=${CLANG_BINARY}
+    export CXX=${CLANGXX_BINARY}
+    export LINKER=lld
+    export AR=/usr/bin/llvm-ar-$LLVMVER
+    export RANLIB=/usr/bin/llvm-ranlib-$LLVMVER
+fi
+
+export CFLAGS="$CFLAGS -fuse-ld=${LINKER}"
+
+cmake ..                                               \
+    -DCMAKE_INSTALL_PREFIX=/usr                        \
+    -DBUILD_LLVM_SUBMODULE=OFF -DUSE_COTIRE=OFF        \
+    -DLLVM_DIR=llvmlibs/lib/cmake/llvm/                \
+    -DUSE_NATIVE_INSTRUCTIONS=OFF                      \
+    -DCMAKE_C_FLAGS="$CFLAGS"                          \
+    -DCMAKE_CXX_FLAGS="$CFLAGS"                        \
+    -DCMAKE_AR="$AR"                                   \
+    -DCMAKE_RANLIB="$RANLIB"                           \
+    -G Ninja
+
+ninja; build_status=$?;
+
+cd ..
+
+# If it compiled succesfully let's deploy depending on the build pipeline (Travis, Azure Pipelines).
+# Travis only deploys on master, and it publishes to GitHub releases. Azure publishes PRs as artifacts
+# only.
+{   [ "$IS_AZURE" = "true" ] ||
+    { [ "$TRAVIS_BRANCH" = "master" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ]; };
+} && SHOULD_DEPLOY="true" || SHOULD_DEPLOY="false"
+
+if [ "$build_status" -eq 0 ] && [ "$SHOULD_DEPLOY" = "true" ]; then
+    .ci/deploy-linux.sh
+fi

.travis/build-mac.bash → .ci/build-mac.sh

@@ -1,3 +1,5 @@
+#!/bin/sh -ex
+
 export CCACHE_SLOPPINESS=pch_defines,time_macros
 export CMAKE_PREFIX_PATH=/usr/local/opt/qt5/
 export PATH="/usr/local/opt/ccache/libexec:$PATH"
@@ -8,15 +10,18 @@ unzip -: gfx-portability-macos-latest.zip
 curl -sLO https://github.com/KhronosGroup/Vulkan-Headers/archive/sdk-1.1.106.0.zip
 unzip -: sdk-*.zip
 mkdir vulkan-sdk
-ln -s ${PWD}/Vulkan-Headers*/include vulkan-sdk/include
+ln -s "${PWD}"/Vulkan-Headers*/include vulkan-sdk/include
 mkdir vulkan-sdk/lib
 cp target/release/libportability.dylib vulkan-sdk/lib/libVulkan.dylib
 # Let macdeployqt locate and install Vulkan library
-install_name_tool -id ${PWD}/vulkan-sdk/lib/libVulkan.dylib vulkan-sdk/lib/libVulkan.dylib
-export VULKAN_SDK=${PWD}/vulkan-sdk
+install_name_tool -id "${PWD}"/vulkan-sdk/lib/libVulkan.dylib vulkan-sdk/lib/libVulkan.dylib
+export VULKAN_SDK="${PWD}/vulkan-sdk"
 
-git submodule update --quiet --init asmjit 3rdparty/ffmpeg 3rdparty/pugixml 3rdparty/span 3rdparty/libpng 3rdparty/cereal 3rdparty/hidapi 3rdparty/libusb 3rdparty/xxHash 3rdparty/yaml-cpp 3rdparty/FAudio Vulkan/glslang
+# Pull all the submodules except llvm
+# Note: Tried to use git submodule status, but it takes over 20 seconds
+# shellcheck disable=SC2046
+git submodule -q update --init $(awk '/path/ && !/llvm/ { print $3 }' .gitmodules)
 
-mkdir build; cd build
+mkdir build && cd build || exit 1
 cmake .. -DWITH_LLVM=OFF -DUSE_NATIVE_INSTRUCTIONS=OFF -G Ninja
 ninja

.ci/deploy-linux.sh

@@ -0,0 +1,67 @@
+#!/bin/sh -ex
+
+cd build || exit 1
+
+if [ "$DEPLOY_APPIMAGE" = "true" ]; then
+    DESTDIR=appdir ninja install
+    QT_APPIMAGE="linuxdeployqt.AppImage"
+
+    curl -sL "https://github.com/probonopd/linuxdeployqt/releases/download/continuous/linuxdeployqt-continuous-x86_64.AppImage" > "$QT_APPIMAGE"
+    chmod a+x "$QT_APPIMAGE"
+    "./$QT_APPIMAGE" --appimage-extract
+    ./squashfs-root/AppRun ./appdir/usr/share/applications/*.desktop -bundle-non-qt-libs
+    ls ./appdir/usr/lib/
+    rm -r ./appdir/usr/share/doc
+    rm ./appdir/usr/lib/libxcb*
+    cp "$(readlink -f /lib/x86_64-linux-gnu/libnsl.so.1)" ./appdir/usr/lib/libnsl.so.1
+    export PATH=/rpcs3/build/squashfs-root/usr/bin/:${PATH}
+
+    # Embed newer libstdc++ for distros that don't come with it (ubuntu 16.04)
+    mkdir -p appdir/usr/optional/ ; mkdir -p appdir/usr/optional/libstdc++/
+    cp /usr/lib/x86_64-linux-gnu/libstdc++.so.6 ./appdir/usr/optional/libstdc++/
+    rm ./appdir/AppRun
+    curl -sL https://github.com/RPCS3/AppImageKit-checkrt/releases/download/continuous2/AppRun-patched-x86_64 -o ./appdir/AppRun
+    chmod a+x ./appdir/AppRun
+    curl -sL https://github.com/RPCS3/AppImageKit-checkrt/releases/download/continuous2/exec-x86_64.so -o ./appdir/usr/optional/exec.so
+
+    # Compile checker binary for AppImageKit-checkrt
+
+    # This may need updating if you update the compiler or rpcs3 uses newer c++ features
+    # See https://github.com/gcc-mirror/gcc/blob/master/libstdc%2B%2B-v3/config/abi/pre/gnu.ver
+    # for which definitions correlate to which CXXABI version.
+    # Currently we target a minimum of GLIBCXX_3.4.26 and CXXABI_1.3.11
+    printf "#include <bits/stdc++.h>\nint main(){std::make_exception_ptr(0);std::pmr::get_default_resource();}" | $CXX -x c++ -std=c++2a -o ./appdir/usr/optional/checker -
+
+    # Package it up and send it off
+    ./squashfs-root/usr/bin/appimagetool /rpcs3/build/appdir
+    ls
+
+    COMM_TAG="$(grep 'version{.*}' ../rpcs3/rpcs3_version.cpp | awk -F[\{,] '{printf "%d.%d.%d", $2, $3, $4}')"
+    COMM_COUNT="$(git rev-list --count HEAD)"
+    COMM_HASH="$(git rev-parse --short=8 HEAD)"
+    RPCS3_APPIMAGE="rpcs3-v${COMM_TAG}-${COMM_COUNT}-${COMM_HASH}_linux64.AppImage"
+
+    curl -sLO https://github.com/hcorion/uploadtool/raw/master/upload.sh
+    mv ./RPCS3*.AppImage "$RPCS3_APPIMAGE"
+
+    # If we're building using Azure Pipelines, let's copy over the AppImage artifact
+    if [ -n "$BUILD_ARTIFACTSTAGINGDIRECTORY" ]; then
+        cp "$RPCS3_APPIMAGE" ~/artifacts
+    fi
+
+    FILESIZE=$(stat -c %s ./rpcs3*.AppImage)
+    SHA256SUM=$(sha256sum ./rpcs3*.AppImage)
+    if [ -n "$GITHUB_TOKEN" ]; then
+        unset TRAVIS_REPO_SLUG
+        REPO_SLUG=RPCS3/rpcs3-binaries-linux \
+            UPLOADTOOL_BODY="$SHA256SUM;${FILESIZE}B"\
+            RELEASE_NAME=build-${TRAVIS_COMMIT}\
+            RELEASE_TITLE=${COMM_TAG}-${COMM_COUNT}\
+            REPO_COMMIT=d812f1254a1157c80fd402f94446310560f54e5f\
+            bash upload.sh rpcs3*.AppImage
+    fi
+fi
+
+if [ "$DEPLOY_PPA" = "true" ]; then
+    export DEBFULLNAME="RPCS3 Build Bot"
+fi

.travis/export-azure-vars.sh → .ci/export-azure-vars.sh

@@ -10,4 +10,4 @@ while IFS='=' read -r key val; do
     # Skip over lines containing comments.
     [ "${key##\#*}" ] || continue
     echo "##vso[task.setvariable variable=$key]$val"
-done < ".travis/azure-vars.env"
+done < ".ci/azure-vars.env"

.travis/setup-windows.sh → .ci/setup-windows.sh

@@ -46,7 +46,7 @@ rev()
 }
 
 # Usage: download_and_verify url checksum algo file
-# Check to see if its already cached, and the checksum matches. If not, download it.
+# Check to see if a file is already cached, and the checksum matches. If not, download it.
 # Tries up to 3 times
 download_and_verify()
 {
@@ -59,7 +59,7 @@ download_and_verify()
         [ -e "$CACHE_DIR/$fileName" ] || curl -L -o "$CACHE_DIR/$fileName" "$url"
         fileChecksum=$("${algo}sum" "$CACHE_DIR/$fileName" | awk '{ print $1 }')
         [ "$fileChecksum" = "$correctChecksum" ] && return 0
-        rm
+        rm "$CACHE_DIR/$fileName"
     done
 
     return 1;
@@ -111,6 +111,6 @@ fi
 # BUILD is the name of the release artifact
 # AVVER is used for GitHub releases, it is the version number.
 BRANCH="${REPO_NAME}/${REPO_BRANCH}"
-echo "BRANCH=$BRANCH" > .travis/azure-vars.env
-echo "BUILD=$BUILD" >> .travis/azure-vars.env
-echo "AVVER=$AVVER" >> .travis/azure-vars.env
+echo "BRANCH=$BRANCH" > .ci/azure-vars.env
+echo "BUILD=$BUILD" >> .ci/azure-vars.env
+echo "AVVER=$AVVER" >> .ci/azure-vars.env

.travis/travis.env → .ci/travis.env

@@ -3,6 +3,7 @@ TRAVIS_PULL_REQUEST
 TRAVIS_BRANCH
 TRAVIS_COMMIT
 # Variables set by Azure Pipelines
+IS_AZURE
 BUILD_REASON
 BUILD_SOURCEVERSION
 BUILD_ARTIFACTSTAGINGDIRECTORY

.travis.yml

@@ -11,7 +11,7 @@ jobs:
           cache: ccache
           compiler: gcc
           install: "docker pull rpcs3/rpcs3-travis-xenial:1.2"
-          script: 'travis_wait docker run -v $(pwd):/rpcs3 -v "$HOME/.ccache":/root/.ccache --env-file .travis/travis.env rpcs3/rpcs3-travis-xenial:1.2 /bin/bash -ex /rpcs3/.travis/build-linux.bash'
+          script: 'travis_wait docker run -v $(pwd):/rpcs3 -v "$HOME/.ccache":/root/.ccache --env-file .ci/travis.env rpcs3/rpcs3-travis-xenial:1.2 /rpcs3/.ci/build-linux.sh'
         - os: linux
           dist: xenial
           env:
@@ -21,7 +21,7 @@ jobs:
           cache: ccache
           compiler: clang
           install: "docker pull rpcs3/rpcs3-travis-xenial:1.2"
-          script: 'travis_wait docker run -v $(pwd):/rpcs3 -v "$HOME/.ccache":/root/.ccache --env-file .travis/travis.env rpcs3/rpcs3-travis-xenial:1.2 /bin/bash -ex /rpcs3/.travis/build-linux.bash'
+          script: 'travis_wait docker run -v $(pwd):/rpcs3 -v "$HOME/.ccache":/root/.ccache --env-file .ci/travis.env rpcs3/rpcs3-travis-xenial:1.2 /rpcs3/.ci/build-linux.sh'
         # - os: osx
         #   osx_image: xcode11.3
         #   addons:
@@ -31,7 +31,7 @@ jobs:
         #             - glew
         #             - ninja
         #             - qt
-        #   script: "/bin/bash -ex .travis/build-mac.bash"
+        #   script: "/bin/bash -ex .ci/build-mac.sh"
         #   cache: ccache
     allow_failures:
         - os: osx

azure-pipelines.yml

@@ -20,6 +20,7 @@ jobs:
         DEPLOY_APPIMAGE: true
   variables:
     CCACHE_DIR: $(Pipeline.Workspace)/ccache
+    IS_AZURE: true
   pool:
     vmImage: 'ubuntu-latest'
   steps:
@@ -33,14 +34,15 @@ jobs:
       docker pull --quiet rpcs3/rpcs3-travis-xenial:1.2
       docker run                      \
         -v $(pwd):/rpcs3              \
-        --env-file .travis/travis.env \
+        --env-file .ci/travis.env \
         -v $CCACHE_DIR:/root/.ccache  \
         -v $BUILD_ARTIFACTSTAGINGDIRECTORY:/root/artifacts \
         rpcs3/rpcs3-travis-xenial:1.2 \
-        /bin/bash -ex /rpcs3/.travis/build-linux.bash
+        /rpcs3/.ci/build-linux.sh
     displayName: Docker setup and build
 
   - publish: $(Build.ArtifactStagingDirectory)
+    condition: eq(variables['COMPILER'], 'gcc')
 
 - job: Windows_Build
   variables:
@@ -63,10 +65,10 @@ jobs:
         path: $(CACHE_DIR)
       displayName: Cache
 
-    - bash: .travis/setup-windows.sh
+    - bash: .ci/setup-windows.sh
       displayName: Download and unpack dependencies
 
-    - bash: .travis/export-azure-vars.sh
+    - bash: .ci/export-azure-vars.sh
       displayName: Export Variables
 
     - task: VSBuild@1
@@ -77,7 +79,7 @@ jobs:
         configuration: 'Release - LLVM'
       displayName: Compile RPCS3
 
-    - bash: .travis/deploy-windows.sh
+    - bash: .ci/deploy-windows.sh
       displayName: Pack up build artifacts
 
     - publish: $(Build.ArtifactStagingDirectory)