Add: [AzurePipeline] introducing a release pipeline

This release pipeline creates all the official release binaries,
and publishes them as artifacts. Currently it can only produce
nightlies and custom builds; stable/testing release binaries are
untested.

This commit also splits up the pipeline in small bits, to both
improve readability, and to share code with the CI pipeline where
possible.

azure-pipelines/changelog.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+tag=$(git describe --tags 2>/dev/null)
+
+# If we are a tag, show the part of the changelog that matches the tag.
+# In case of a stable, also show all betas and RCs.
+if [ -n "$tag" ]; then
+    grep="."
+    if [ "$(echo $tag | grep '-')" = "" ]; then
+            grep='^[0-9]\.[0-9]\.[0-9][^-]'
+    fi
+    next=$(cat changelog.txt | grep '^[0-9]' | awk 'BEGIN { show="false" } // { if (show=="true") print $0; if ($1=="'$tag'") show="true"} ' | grep "$grep" | head -n1 | sed 's/ .*//')
+    cat changelog.txt | awk 'BEGIN { show="false" } /^[0-9].[0-9].[0-9]/ { if ($1=="'$next'") show="false"; if ($1=="'$tag'") show="true";} // { if (show=="true") print $0 }'
+    exit 0
+fi
+
+# In all other cases, show the git log of the last 7 days
+revdate=$(git log -1 --pretty=format:"%ci")
+last_week=$(date -u -d "$revdate -7days" +"%Y-%m-%d %H:%M")
+git log --after="${last_week}" --pretty=fuller

azure-pipelines/ci.yml

@@ -0,0 +1,64 @@
+trigger:
+- master
+pr:
+- master
+
+jobs:
+- job: windows
+  displayName: 'Windows'
+  pool:
+    vmImage: 'VS2017-Win2016'
+
+  strategy:
+    matrix:
+      Win32:
+        BuildPlatform: 'Win32'
+      Win64:
+        BuildPlatform: 'x64'
+
+  steps:
+  - template: templates/ci-git-rebase.yml
+  - template: templates/windows-dependencies.yml
+  - template: templates/ci-opengfx.yml
+  - template: templates/windows-build.yml
+    parameters:
+      BuildPlatform: $(BuildPlatform)
+  - script: |
+      call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x86
+      cd projects
+      call regression.bat
+    displayName: 'Test'
+
+- job: linux
+  displayName: 'Linux'
+  pool:
+    vmImage: 'ubuntu-16.04'
+
+  strategy:
+    matrix:
+      commit-checker: {}
+      linux-amd64-clang-3.8: {}
+      linux-amd64-gcc-6: {}
+      linux-i386-gcc-6: {}
+
+  steps:
+  - template: templates/ci-git-rebase.yml
+  # The dockers already have the dependencies installed
+  # The dockers already have OpenGFX installed
+  - template: templates/linux-build.yml
+    parameters:
+      Image: compile-farm-ci
+      Tag: $(Agent.JobName)
+
+- job: macos
+  displayName: 'MacOS'
+  pool:
+    vmImage: 'macOS-10.13'
+
+  steps:
+  - template: templates/ci-git-rebase.yml
+  - template: templates/osx-dependencies.yml
+  - template: templates/ci-opengfx.yml
+  - template: templates/osx-build.yml
+  - script: 'make regression'
+    displayName: 'Test'

azure-pipelines/release-stable.yml

@@ -0,0 +1,7 @@
+trigger: none
+pr: none
+
+jobs:
+- template: templates/release.yml
+  parameters:
+    IsStableRelease: true

azure-pipelines/release.yml

@@ -0,0 +1,7 @@
+trigger: none
+pr: none
+
+jobs:
+- template: templates/release.yml
+  parameters:
+    IsStableRelease: false

azure-pipelines/templates/ci-git-rebase.yml

@@ -0,0 +1,9 @@
+steps:
+# Rebase to origin/master for every PR. This means users don't have to
+# rebase every time master changes. As long as the PR applies cleanly, we
+# will validate it.
+- script: |
+    git config user.email 'info@openttd.org'
+    git config user.name 'OpenTTD CI'
+    git rebase origin/master
+  displayName: 'Rebase to master'

azure-pipelines/templates/ci-opengfx.yml

@@ -0,0 +1,8 @@
+steps:
+- bash: |
+    set -ex
+    cd bin/baseset
+    curl -L https://binaries.openttd.org/extra/opengfx/0.5.2/opengfx-0.5.2-all.zip > opengfx-0.5.2-all.zip
+    unzip opengfx-0.5.2-all.zip
+    rm -f opengfx-0.5.2-all.zip
+  displayName: 'Install OpenGFX'

azure-pipelines/templates/linux-build.yml

@@ -0,0 +1,18 @@
+parameters:
+  Image: ''
+  Tag: ''
+  ContainerCommand: ''
+
+steps:
+- task: Docker@1
+  ${{ if eq(parameters.Image, 'compile-farm') }}:
+    displayName: 'Build'
+  ${{ if eq(parameters.Image, 'compile-farm-ci') }}:
+    displayName: 'Build and test'
+  inputs:
+    command: 'Run an image'
+    imageName: openttd/${{ parameters.Image }}:${{ parameters.Tag }}
+    volumes: '$(Build.SourcesDirectory):$(Build.SourcesDirectory)'
+    workingDirectory: '$(Build.SourcesDirectory)'
+    containerCommand: ${{ parameters.ContainerCommand }}
+    runInBackground: false

azure-pipelines/templates/linux-claim-bundles.yml

@@ -0,0 +1,5 @@
+steps:
+# Because we run the compile in a docker (under root), we are not owner
+# of the 'bundles' folder. Fix that by executing a chown on it.
+- bash: sudo chown -R $(id -u):$(id -g) bundles
+  displayName: 'Claim bundles folder back'

azure-pipelines/templates/osx-build.yml

@@ -0,0 +1,5 @@
+steps:
+- script: './configure PKG_CONFIG_PATH=/usr/local/lib/pkgconfig --enable-static'
+  displayName: 'Configure'
+- script: 'make -j2'
+  displayName: 'Build'

azure-pipelines/templates/osx-dependencies.yml

@@ -0,0 +1,11 @@
+steps:
+- script: |
+    set -ex
+    HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config lzo xz libpng
+    # Remove the dynamic libraries of these libraries, to ensure we use
+    # the static versions. That is important, as it is unlikely any
+    # end-user has these brew libraries installed.
+    rm /usr/local/Cellar/lzo/*/lib/*.dylib
+    rm /usr/local/Cellar/xz/*/lib/*.dylib
+    rm /usr/local/Cellar/libpng/*/lib/*.dylib
+  displayName: 'Install dependencies'

azure-pipelines/templates/release-bundles.yml

@@ -0,0 +1,19 @@
+parameters:
+  CalculateChecksums: true
+
+steps:
+- ${{ if eq(parameters.CalculateChecksums, true) }}:
+  - bash: |
+      set -ex
+      cd bundles
+      for i in $(ls); do
+        openssl dgst -r -md5 -hex $i > $i.md5sum
+        openssl dgst -r -sha1 -hex $i > $i.sha1sum
+        openssl dgst -r -sha256 -hex $i > $i.sha256sum
+      done
+    displayName: 'Calculate checksums'
+- task: PublishBuildArtifacts@1
+  displayName: 'Publish bundles'
+  inputs:
+    PathtoPublish: bundles/
+    ArtifactName: bundles

azure-pipelines/templates/release-fetch-source.yml

@@ -0,0 +1,20 @@
+# Fetch the source tarball as prepared by an earlier job. In there is the
+# version predefined. This ensures we are all going to compile the same
+# source with the same version.
+
+steps:
+- checkout: none
+- task: DownloadBuildArtifacts@0
+  displayName: 'Download source'
+  inputs:
+    downloadType: specific
+    itemPattern: 'bundles/openttd-*-source.tar.xz'
+    downloadPath: '$(Build.ArtifactStagingDirectory)'
+- bash: tar --xz -xf ../a/bundles/openttd-*-source.tar.xz --strip-components=1
+  displayName: 'Extracting source'
+- bash: |
+    set -e
+    VERSION=$(cat .version)
+    echo "${VERSION}"
+    echo "##vso[build.updatebuildnumber]${VERSION}"
+  displayName: 'Change BuildNumber to version'

azure-pipelines/templates/release-manifest.yml

@@ -0,0 +1,59 @@
+steps:
+- task: DownloadBuildArtifacts@0
+  displayName: 'Download all bundles'
+  inputs:
+    downloadType: specific
+    itemPattern: 'bundles/*'
+    downloadPath: '$(Build.ArtifactStagingDirectory)'
+- script: |
+    set -ex
+
+    # Find the name based on the version
+    if [ "${ISSTABLERELEASE}" = "true" ]; then
+      isTesting=$(cat .version | grep "RC\|beta" || true)
+      if [ -z "${isTesting}" ]; then
+        NAME="stable"
+      else
+        NAME="testing"
+      fi
+    else
+      NAME=$(cat .version | cut -d- -f2 | cut -d- -f-2)
+    fi
+
+    # Convert the date to a YAML date
+    DATE=$(cat .release_date | tr ' ' T | sed 's/TUTC/:00-00:00/')
+    VERSION=$(cat .version)
+    BASE="openttd-${VERSION}"
+
+    echo "name: ${NAME}" >> manifest.yaml
+    echo "date: ${DATE}" >> manifest.yaml
+    echo "base: ${BASE}-" >> manifest.yaml
+    echo "files:" >> manifest.yaml
+
+    FOLDER="../a/bundles"
+    error=""
+    for i in $(ls ${FOLDER} | grep -v ".txt$\|.md$\|sum$" | sort); do
+      if [ -n "$(echo $i | grep pdb.xz)" ]; then continue; fi
+      if [ -n "$(echo $i | grep dbg.deb)" ]; then continue; fi
+
+      if [ ! -e ${FOLDER}/$i.md5sum ] || [ ! -e ${FOLDER}/$i.sha1sum ] || [ ! -e ${FOLDER}/$i.sha256sum ]; then
+        echo "ERROR: missing checksum file for ${i}" 1>&2
+        error="y"
+        continue
+      fi
+
+      echo "- id: $i" >> manifest.yaml
+      echo "  size: $(stat -c"%s" ${FOLDER}/$i)" >> manifest.yaml
+      echo "  md5sum: $(cat ${FOLDER}/$i.md5sum | cut -d\  -f1)" >> manifest.yaml
+      echo "  sha1sum: $(cat ${FOLDER}/$i.sha1sum | cut -d\  -f1)" >> manifest.yaml
+      echo "  sha256sum: $(cat ${FOLDER}/$i.sha256sum | cut -d\  -f1)" >> manifest.yaml
+    done
+
+    if [ -n "${error}" ]; then
+      echo "ERROR: exiting due to earlier errors" 1>&2
+      exit 1
+    fi
+
+    mkdir -p bundles
+    mv manifest.yaml bundles/
+  displayName: 'Create manifest.yaml'