azure-pipelines.yml

# https://aka.ms/yaml

trigger:
  - master

variables:
    regression: 1

jobs:
  - job: build_ubuntu_2004
    displayName: Ubuntu 2004
    pool:
      vmImage: 'Ubuntu-20.04'
    timeoutInMinutes: 120
    strategy:
      matrix:
        avx2:
          SIMD: 'AVX2'
          STATIC: 1
          MPI: 0
          BUILD_TYPE: RelWithDebInfo
        sse41:
          SIMD: 'SSE4_1'
          STATIC: 1
          MPI: 0
          BUILD_TYPE: RelWithDebInfo
        sse2:
          SIMD: 'SSE2'
          STATIC: 1
          MPI: 0
          BUILD_TYPE: RelWithDebInfo
        avx2_mpi:
          SIMD: 'AVX2'
          STATIC: 0
          MPI: 1
          BUILD_TYPE: RelWithDebInfo
        asan:
          SIMD: 'AVX2'
          STATIC: 0
          MPI: 0
          BUILD_TYPE: ASanOpt
    steps:
      - checkout: self
      - script: |
          sudo add-apt-repository ppa:ubuntu-toolchain-r/test
          sudo apt-get update
          sudo apt-get install -y gcc-11 g++-11
          mkdir build-opt && cd build-opt
          CC=gcc-11 CXX=g++-11 cmake -DCMAKE_BUILD_TYPE=Release ..
          make -j $(nproc --all)
        condition: eq(variables['BUILD_TYPE'], 'ASanOpt')
        displayName: Install newer GCC
      - script: |
          sudo apt-get update
          sudo apt-get -y install mpi-default-dev mpi-default-bin
        displayName: Install Dependencies
        condition: eq(variables['MPI'], 1)
      - script: |
          mkdir build && cd build
          if [ "${BUILD_TYPE}" = "ASanOpt" ]; then
              export CC=gcc-11 ; export CXX=g++-11
          fi
          if [ "${STATIC}" -eq "1" ]; then
              cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF \
                -DCMAKE_EXE_LINKER_FLAGS="-static -static-libgcc \
                -static-libstdc++" -DCMAKE_FIND_LIBRARY_SUFFIXES=".a" \
                -DENABLE_WERROR=1 -DHAVE_${SIMD}=1 -DHAVE_MPI=${MPI} ..
          else
              cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
                -DENABLE_WERROR=1 -DHAVE_${SIMD}=1 -DHAVE_MPI=${MPI} ..
          fi

          make -j $(nproc --all)
        displayName: Build Plass
      - script: |
          cd ${BUILD_SOURCESDIRECTORY}
          export TTY=0
          if [ "${BUILD_TYPE}" = "ASanOpt" ]; then
              echo "leak:libgomp1" > ${BUILD_SOURCESDIRECTORY}/ASan.supp
              export ASAN_OPTIONS=suppressions=${BUILD_SOURCESDIRECTORY}/ASan.supp
              ./util/run_regression.sh ./build/src/plass ./build-opt/src/plass REGRESSION
          else
              ./util/run_regression.sh ./build/src/plass ./build/src/plass REGRESSION
          fi
        displayName: Run Regression Suite
        condition: eq(variables['regression'], 1)
      - task: PublishPipelineArtifact@0
        condition: eq(variables['STATIC'], 1)
        inputs:
          targetPath: $(Build.SourcesDirectory)/build/src/plass
          artifactName: plass-linux-$(SIMD)
      - task: PublishPipelineArtifact@0
        condition: eq(variables['STATIC'], 1)
        inputs:
          targetPath: $(Build.SourcesDirectory)/build/src/penguin
          artifactName: penguin-linux-$(SIMD)

  - job: build_ubuntu_cross_2004
    displayName: Ubuntu 2004 Cross-Compile
    pool:
      vmImage: 'Ubuntu-20.04'
    strategy:
      matrix:
        power8:
          SIMD: POWER8
          ARCH: ppc64el
          CPREF: powerpc64le
        power9:
          SIMD: POWER9
          ARCH: ppc64el
          CPREF: powerpc64le
        arm64:
          SIMD: ARM8
          ARCH: arm64
          CPREF: aarch64
        s390x:
          SIMD: S390X
          ARCH: s390x
          CPREF: s390x
    steps:
      - script: |
          sudo dpkg --add-architecture $ARCH
          cat << HEREDOC | sudo tee /etc/apt/sources.list
          deb [arch=amd64,i386] http://archive.ubuntu.com/ubuntu/ focal main universe multiverse
          deb [arch=amd64,i386] http://archive.ubuntu.com/ubuntu/ focal-updates main universe multiverse
          deb [arch=amd64,i386] http://archive.ubuntu.com/ubuntu/ focal-backports main universe multiverse
          deb [arch=amd64,i386] http://security.ubuntu.com/ubuntu/ focal-security main universe multiverse
          deb [arch=$ARCH] http://ports.ubuntu.com/ubuntu-ports/ focal main  universe multiverse
          deb [arch=$ARCH] http://ports.ubuntu.com/ubuntu-ports/ focal-updates main universe multiverse
          deb [arch=$ARCH] http://ports.ubuntu.com/ubuntu-ports/ focal-backports main universe multiverse
          deb [arch=$ARCH] http://ports.ubuntu.com/ubuntu-ports/ focal-security main universe multiverse
          HEREDOC
          sudo apt-get update
          sudo apt-get -y install -o APT::Immediate-Configure=false crossbuild-essential-$ARCH zlib1g-dev:$ARCH libbz2-dev:$ARCH
        displayName: Install Toolchain
      - script: |
          mkdir build && cd build
          CC=${CPREF}-linux-gnu-gcc CXX=${CPREF}-linux-gnu-g++ \
           cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DHAVE_TESTS=1 \
            -DBUILD_SHARED_LIBS=OFF \
            -DCMAKE_EXE_LINKER_FLAGS="-static -static-libgcc \
            -static-libstdc++" -DCMAKE_FIND_LIBRARY_SUFFIXES=".a" \
            -DENABLE_WERROR=1 -DHAVE_${SIMD}=1 ..
          make -j $(nproc --all)
        displayName: Build Plass
      - task: PublishPipelineArtifact@0
        inputs:
          targetPath: $(Build.SourcesDirectory)/build/src/plass
          artifactName: plass-linux-$(SIMD)
      - task: PublishPipelineArtifact@0
        inputs:
          targetPath: $(Build.SourcesDirectory)/build/src/penguin
          artifactName: penguin-linux-$(SIMD)

  - job: build_macos_11
    displayName: macOS 11
    pool:
      vmImage: 'macos-11'
    steps:
      - checkout: self
      - script: |
          cd ${BUILD_SOURCESDIRECTORY}
          ./lib/mmseqs/util/build_osx.sh . build plass
          lipo -create -arch x86_64 "build/build_sse41/src/penguin" -arch x86_64h "build/build_avx2/src/penguin" -arch arm64 "build/build_arm64/src/penguin" -output "build/penguin"
        displayName: Build Plass
      - script: |
          cd ${BUILD_SOURCESDIRECTORY}
          ./util/run_regression.sh ./build/plass ./build/plass REGRESSION
        displayName: Run Regression Suite
        condition: eq(variables['regression'], 1)
      - task: PublishPipelineArtifact@0
        inputs:
          targetPath: $(Build.SourcesDirectory)/build/plass
          artifactName: plass-darwin-universal
      - task: PublishPipelineArtifact@0
        inputs:
          targetPath: $(Build.SourcesDirectory)/build/penguin
          artifactName: penguin-darwin-universal

  - job: upload_artifacts
    displayName: Upload Artifacts
    condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
    pool:
      vmImage: 'ubuntu-latest'
    dependsOn:
      - build_macos_11
      - build_ubuntu_2004
      - build_ubuntu_cross_2004
    steps:
      - script: |
          cd "${BUILD_SOURCESDIRECTORY}"
          mkdir plass
          cp -f README.md LICENCE.md plass
          mkdir plass/bin
      - task: DownloadPipelineArtifact@1
        inputs:
          artifactName: plass-darwin-universal
          targetPath: $(Build.SourcesDirectory)/plass/bin
      - task: DownloadPipelineArtifact@1
        inputs:
          artifactName: penguin-darwin-universal
          targetPath: $(Build.SourcesDirectory)/plass/bin
      - script: |
          chmod +x "${BUILD_SOURCESDIRECTORY}/plass/bin/plass"
          chmod +x "${BUILD_SOURCESDIRECTORY}/plass/bin/penguin"
      - task: ArchiveFiles@2
        inputs:
          rootFolderOrFile: $(Build.SourcesDirectory)/plass
          archiveFile: $(Build.SourcesDirectory)/plass-osx-universal.tar.gz
          includeRootFolder: true
          archiveType: tar
      - task: DownloadPipelineArtifact@1
        inputs:
          artifactName: plass-linux-SSE2
          targetPath: $(Build.SourcesDirectory)/plass/bin
      - task: DownloadPipelineArtifact@1
        inputs:
          artifactName: penguin-linux-SSE2
          targetPath: $(Build.SourcesDirectory)/plass/bin
      - script: |
          chmod +x "${BUILD_SOURCESDIRECTORY}/plass/bin/plass"
          chmod +x "${BUILD_SOURCESDIRECTORY}/plass/bin/penguin"
      - task: ArchiveFiles@2
        inputs:
          rootFolderOrFile: $(Build.SourcesDirectory)/plass
          archiveFile: $(Build.SourcesDirectory)/plass-linux-sse2.tar.gz
          includeRootFolder: true
          archiveType: tar
      - task: DownloadPipelineArtifact@1
        inputs:
          artifactName: plass-linux-SSE4_1
          targetPath: $(Build.SourcesDirectory)/plass/bin
      - task: DownloadPipelineArtifact@1
        inputs:
          artifactName: penguin-linux-SSE4_1
          targetPath: $(Build.SourcesDirectory)/plass/bin
      - script: |
          chmod +x "${BUILD_SOURCESDIRECTORY}/plass/bin/plass"
          chmod +x "${BUILD_SOURCESDIRECTORY}/plass/bin/penguin"
      - task: ArchiveFiles@2
        inputs:
          rootFolderOrFile: $(Build.SourcesDirectory)/plass
          archiveFile: $(Build.SourcesDirectory)/plass-linux-sse41.tar.gz
          includeRootFolder: true
          archiveType: tar
      - task: DownloadPipelineArtifact@1
        inputs:
          artifactName: plass-linux-AVX2
          targetPath: $(Build.SourcesDirectory)/plass/bin
      - task: DownloadPipelineArtifact@1
        inputs:
          artifactName: penguin-linux-AVX2
          targetPath: $(Build.SourcesDirectory)/plass/bin
      - script: |
          chmod +x "${BUILD_SOURCESDIRECTORY}/plass/bin/plass"
          chmod +x "${BUILD_SOURCESDIRECTORY}/plass/bin/penguin"
      - task: ArchiveFiles@2
        inputs:
          rootFolderOrFile: $(Build.SourcesDirectory)/plass
          archiveFile: $(Build.SourcesDirectory)/plass-linux-avx2.tar.gz
          includeRootFolder: true
          archiveType: tar
      - task: DownloadPipelineArtifact@1
        inputs:
          artifactName: plass-linux-ARM8
          targetPath: $(Build.SourcesDirectory)/plass/bin
      - task: DownloadPipelineArtifact@1
        inputs:
          artifactName: penguin-linux-ARM8
          targetPath: $(Build.SourcesDirectory)/plass/bin
      - script: |
          chmod +x "${BUILD_SOURCESDIRECTORY}/plass/bin/plass"
          chmod +x "${BUILD_SOURCESDIRECTORY}/plass/bin/penguin"
      - task: ArchiveFiles@2
        inputs:
          rootFolderOrFile: $(Build.SourcesDirectory)/plass
          archiveFile: $(Build.SourcesDirectory)/plass-linux-arm64.tar.gz
          includeRootFolder: true
          archiveType: tar
      - task: DownloadPipelineArtifact@1
        inputs:
          artifactName: plass-linux-POWER8
          targetPath: $(Build.SourcesDirectory)/plass/bin
      - task: DownloadPipelineArtifact@1
        inputs:
          artifactName: penguin-linux-POWER8
          targetPath: $(Build.SourcesDirectory)/plass/bin
      - script: |
          chmod +x "${BUILD_SOURCESDIRECTORY}/plass/bin/plass"
          chmod +x "${BUILD_SOURCESDIRECTORY}/plass/bin/penguin"
      - task: ArchiveFiles@2
        inputs:
          rootFolderOrFile: $(Build.SourcesDirectory)/plass
          archiveFile: $(Build.SourcesDirectory)/plass-linux-ppc64le-POWER8.tar.gz
          includeRootFolder: true
          archiveType: tar
      - task: DownloadPipelineArtifact@1
        inputs:
          artifactName: plass-linux-POWER9
          targetPath: $(Build.SourcesDirectory)/plass/bin
      - task: DownloadPipelineArtifact@1
        inputs:
          artifactName: penguin-linux-POWER9
          targetPath: $(Build.SourcesDirectory)/plass/bin          
      - script: |
          chmod +x "${BUILD_SOURCESDIRECTORY}/plass/bin/plass"
          chmod +x "${BUILD_SOURCESDIRECTORY}/plass/bin/penguin"
      - task: ArchiveFiles@2
        inputs:
          rootFolderOrFile: $(Build.SourcesDirectory)/plass
          archiveFile: $(Build.SourcesDirectory)/plass-linux-ppc64le-POWER9.tar.gz
          includeRootFolder: true
          archiveType: tar
      - task: DownloadSecureFile@1
        inputs:
          secureFile: azure_upload_plass
        displayName: Get Deployment Key
      - script: |
          mkdir ~/.ssh && mv $DOWNLOADSECUREFILE_SECUREFILEPATH ~/.ssh/id_rsa
          chmod 700 ~/.ssh && chmod 600 ~/.ssh/id_rsa
          ssh-keygen -f ~/.ssh/id_rsa -y > ~/.ssh/id_rsa.pub
          cd "${BUILD_SOURCESDIRECTORY}"
          ssh-keygen -Y sign -f ~/.ssh/id_rsa -n file \
             plass-osx-universal.tar.gz  \
             plass-linux-sse2.tar.gz \
             plass-linux-sse41.tar.gz \
             plass-linux-avx2.tar.gz \
             plass-linux-arm64.tar.gz \
             plass-linux-ppc64le-POWER8.tar.gz \
             plass-linux-ppc64le-POWER9.tar.gz
          curl --retry 5 --retry-all-errors -X POST \
            -F file[]=@plass-osx-universal.tar.gz -F signature[]=@plass-osx-universal.tar.gz.sig \
            -F file[]=@plass-linux-sse2.tar.gz -F signature[]=@plass-linux-sse2.tar.gz.sig \
            -F file[]=@plass-linux-sse41.tar.gz  -F signature[]=@plass-linux-sse41.tar.gz.sig \
            -F file[]=@plass-linux-avx2.tar.gz -F signature[]=@plass-linux-avx2.tar.gz.sig \
            -F file[]=@plass-linux-arm64.tar.gz -F signature[]=@plass-linux-arm64.tar.gz.sig \
            -F file[]=@plass-linux-ppc64le-POWER8.tar.gz -F signature[]=@plass-linux-ppc64le-POWER8.tar.gz.sig \
            -F file[]=@plass-linux-ppc64le-POWER9.tar.gz -F signature[]=@plass-linux-ppc64le-POWER9.tar.gz.sig \
            -F identifier="plass" -F directory="${BUILD_SOURCEVERSION}" \
              https://mmseqs.com/upload