Add: [AzurePipeline] introducing a release pipeline

Makefile.bundle.in

@@ -54,7 +54,7 @@ ifdef OSXAPP
 	$(Q)mkdir -p "$(BUNDLE_DIR)/$(OSXAPP)/Contents/Resources"
 	$(Q)echo "APPL????" >                                          "$(BUNDLE_DIR)/$(OSXAPP)/Contents/PkgInfo"
 	$(Q)cp    "$(ROOT_DIR)/os/macosx/openttd.icns"                 "$(BUNDLE_DIR)/$(OSXAPP)/Contents/Resources/openttd.icns"
-	$(Q)$(ROOT_DIR)/os/macosx/plistgen.sh                          "$(BUNDLE_DIR)/$(OSXAPP)" "$(REV)"
+	$(Q)$(ROOT_DIR)/os/macosx/plistgen.sh                          "$(BUNDLE_DIR)/$(OSXAPP)" "$(VERSION)"
 	$(Q)cp    "$(ROOT_DIR)/os/macosx/splash.png"                   "$(BASESET_DIR)"
 endif
 ifeq ($(OS),UNIX)
@@ -151,10 +151,10 @@ bundle_lha: bundle
 
 bundle_dmg: bundle
 	@echo '[BUNDLE] Creating $(BUNDLE_NAME).dmg'
-	$(Q)mkdir -p "$(BUNDLES_DIR)/OpenTTD $(REV)"
-	$(Q)cp -R "$(BUNDLE_DIR)/" "$(BUNDLES_DIR)/OpenTTD $(REV)"
-	$(Q)hdiutil create -ov -format UDZO -srcfolder "$(BUNDLES_DIR)/OpenTTD $(REV)" "$(BUNDLES_DIR)/$(BUNDLE_NAME).dmg"
-	$(Q)rm -fr "$(BUNDLES_DIR)/OpenTTD $(REV)"
+	$(Q)mkdir -p "$(BUNDLES_DIR)/OpenTTD $(VERSION)"
+	$(Q)cp -R "$(BUNDLE_DIR)/" "$(BUNDLES_DIR)/OpenTTD $(VERSION)"
+	$(Q)hdiutil create -ov -format UDZO -srcfolder "$(BUNDLES_DIR)/OpenTTD $(VERSION)" "$(BUNDLES_DIR)/$(BUNDLE_NAME).dmg"
+	$(Q)rm -fr "$(BUNDLES_DIR)/OpenTTD $(VERSION)"
 
 bundle_exe: all
 	@echo '[BUNDLE] Creating $(BUNDLE_NAME).exe'

azure-pipelines-ci.yml

@@ -17,37 +17,12 @@ jobs:
         BuildPlatform: 'x64'
 
   steps:
-  # Rebase to origin/master for every PR. This means users don't have to
-  # rebase every time master changes. As long as the PR applies cleanly, we
-  # will validate it.
-  - script: |
-      git config user.email 'info@openttd.org'
-      git config user.name 'OpenTTD CI'
-      git rebase origin/master
-    displayName: 'Rebase to master'
-  - bash: |
-      set -ex
-      curl -L https://github.com/OpenTTD/OpenTTD-CF/releases/download/latest/windows-dependencies.zip > windows-dependencies.zip
-      unzip windows-dependencies.zip
-      rm -f windows-dependencies.zip
-    displayName: 'Download dependencies'
-    workingDirectory: $(Build.ArtifactStagingDirectory)
-  - script: $(Build.ArtifactStagingDirectory)\windows-dependencies\vcpkg.exe integrate install
-    displayName: 'Install dependencies'
-  - bash: |
-      set -ex
-      cd bin/baseset
-      curl -L https://binaries.openttd.org/extra/opengfx/0.5.2/opengfx-0.5.2-all.zip > opengfx-0.5.2-all.zip
-      unzip opengfx-0.5.2-all.zip
-      rm -f opengfx-0.5.2-all.zip
-    displayName: 'Install OpenGFX'
-  - task: VSBuild@1
-    displayName: 'Build'
-    inputs:
-      solution: 'projects/openttd_vs141.sln'
-      platform: $(BuildPlatform)
-      configuration: Release
-      maximumCpuCount: true
+  - template: azure-pipelines/templates/ci-git-rebase.yml
+  - template: azure-pipelines/templates/windows-dependencies.yml
+  - template: azure-pipelines/templates/ci-opengfx.yml
+  - template: azure-pipelines/templates/windows-build.yml
+    parameters:
+      BuildPlatform: $(BuildPlatform)
   - script: |
       call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x86
       cd projects
@@ -67,50 +42,23 @@ jobs:
       linux-i386-gcc-6: {}
 
   steps:
-  # Rebase to origin/master for every PR. This means users don't have to
-  # rebase every time master changes. As long as the PR applies cleanly, we
-  # will validate it.
-  - script: |
-      git config user.email 'info@openttd.org'
-      git config user.name 'OpenTTD CI'
-      git rebase origin/master
-    displayName: 'Rebase to master'
-  - task: Docker@1
-    displayName: 'Build and test'
-    inputs:
-      command: 'Run an image'
-      imageName: openttd/compile-farm-ci:$(Agent.JobName)
-      volumes: '$(Build.SourcesDirectory):$(Build.SourcesDirectory)'
-      workingDirectory: '$(Build.SourcesDirectory)'
-      runInBackground: false
+  - template: azure-pipelines/templates/ci-git-rebase.yml
+  # The dockers already have the dependencies installed
+  # The dockers already have OpenGFX installed
+  - template: azure-pipelines/templates/linux-build.yml
+    parameters:
+      Image: compile-farm-ci
+      Tag: $(Agent.JobName)
 
 - job: macos
   displayName: 'MacOS'
   pool:
     vmImage: 'macOS-10.13'
 
   steps:
-  # Rebase to origin/master for every PR. This means users don't have to
-  # rebase every time master changes. As long as the PR applies cleanly, we
-  # will validate it.
-  - script: |
-      git config user.email 'info@openttd.org'
-      git config user.name 'OpenTTD CI'
-      git rebase origin/master
-    displayName: 'Rebase to master'
-  - script: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config lzo xz libpng
-    displayName: 'Install dependencies'
-  - bash: |
-      set -ex
-      mkdir -p ~/Documents/OpenTTD/baseset
-      cd ~/Documents/OpenTTD/baseset
-      wget https://binaries.openttd.org/extra/opengfx/0.5.2/opengfx-0.5.2-all.zip
-      unzip opengfx-0.5.2-all.zip
-      rm -f opengfx-0.5.2-all.zip
-    displayName: 'Install OpenGFX'
-  - script: './configure PKG_CONFIG_PATH=/usr/local/lib/pkgconfig --enable-static'
-    displayName: 'Configure'
-  - script: 'make -j2'
-    displayName: 'Build'
+  - template: azure-pipelines/templates/ci-git-rebase.yml
+  - template: azure-pipelines/templates/osx-dependencies.yml
+  - template: azure-pipelines/templates/ci-opengfx.yml
+  - template: azure-pipelines/templates/osx-build.yml
   - script: 'make regression'
     displayName: 'Test'

azure-pipelines-release-stable.yml

@@ -0,0 +1,7 @@
+trigger: none
+pr: none
+
+jobs:
+- template: azure-pipelines/templates/release.yml
+  parameters:
+    IsStableRelease: true

azure-pipelines-release.yml

@@ -0,0 +1,7 @@
+trigger: none
+pr: none
+
+jobs:
+- template: azure-pipelines/templates/release.yml
+  parameters:
+    IsStableRelease: false

azure-pipelines/changelog.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+tag=$(git describe --tags 2>/dev/null)
+
+# If we are a tag, show the part of the changelog that matches the tag.
+# In case of a stable, also show all betas and RCs.
+if [ -n "$tag" ]; then
+    grep="."
+    if [ "$(echo $tag | grep '-')" = "" ]; then
+            grep='^[0-9]\.[0-9]\.[0-9][^-]'
+    fi
+    next=$(cat changelog.txt | grep '^[0-9]' | awk 'BEGIN { show="false" } // { if (show=="true") print $0; if ($1=="'$tag'") show="true"} ' | grep "$grep" | head -n1 | sed 's/ .*//')
+    cat changelog.txt | awk 'BEGIN { show="false" } /^[0-9].[0-9].[0-9]/ { if ($1=="'$next'") show="false"; if ($1=="'$tag'") show="true";} // { if (show=="true") print $0 }'
+    exit 0
+fi
+
+# In all other cases, show the git log of the last 7 days
+revdate=$(git log -1 --pretty=format:"%ci")
+last_week=$(date -u -d "$revdate -7days" +"%Y-%m-%d %H:%M")
+git log --after="${last_week}" --pretty=fuller

azure-pipelines/manifest.sh

@@ -0,0 +1,60 @@
+#!/bin/sh
+
+set -ex
+
+if [ -z "$1" ]; then
+    echo "Usage: $0 <folder-with-bundles>"
+    exit 1
+fi
+
+FOLDER=$1
+
+if [ ! -e .version ] || [ ! -e .release_date ]; then
+    echo "This script should be executed in the root of an extracted source tarball"
+    exit 1
+fi
+
+# Find the name based on the version
+if [ "${ISSTABLERELEASE}" = "true" ]; then
+    isTesting=$(cat .version | grep "RC\|beta" || true)
+    if [ -z "${isTesting}" ]; then
+        NAME="stable"
+    else
+        NAME="testing"
+    fi
+else
+    NAME=$(cat .version | cut -d- -f2 | cut -d- -f-2)
+fi
+
+# Convert the date to a YAML date
+DATE=$(cat .release_date | tr ' ' T | sed 's/TUTC/:00-00:00/')
+VERSION=$(cat .version)
+BASE="openttd-${VERSION}"
+
+echo "name: ${NAME}" >> manifest.yaml
+echo "date: ${DATE}" >> manifest.yaml
+echo "base: ${BASE}-" >> manifest.yaml
+echo "files:" >> manifest.yaml
+
+error=""
+for i in $(ls ${FOLDER} | grep -v ".txt$\|.md$\|sum$" | sort); do
+    if [ -n "$(echo $i | grep pdb.xz)" ]; then continue; fi
+    if [ -n "$(echo $i | grep dbg.deb)" ]; then continue; fi
+
+    if [ ! -e ${FOLDER}/$i.md5sum ] || [ ! -e ${FOLDER}/$i.sha1sum ] || [ ! -e ${FOLDER}/$i.sha256sum ]; then
+        echo "ERROR: missing checksum file for ${i}" 1>&2
+        error="y"
+        continue
+    fi
+
+    echo "- id: $i" >> manifest.yaml
+    echo "  size: $(stat -c"%s" ${FOLDER}/$i)" >> manifest.yaml
+    echo "  md5sum: $(cat ${FOLDER}/$i.md5sum | cut -d\  -f1)" >> manifest.yaml
+    echo "  sha1sum: $(cat ${FOLDER}/$i.sha1sum | cut -d\  -f1)" >> manifest.yaml
+    echo "  sha256sum: $(cat ${FOLDER}/$i.sha256sum | cut -d\  -f1)" >> manifest.yaml
+done
+
+if [ -n "${error}" ]; then
+    echo "ERROR: exiting due to earlier errors" 1>&2
+    exit 1
+fi

azure-pipelines/templates/ci-git-rebase.yml

@@ -0,0 +1,9 @@
+steps:
+# Rebase to origin/master for every PR. This means users don't have to
+# rebase every time master changes. As long as the PR applies cleanly, we
+# will validate it.
+- script: |
+    git config user.email 'info@openttd.org'
+    git config user.name 'OpenTTD CI'
+    git rebase origin/master
+  displayName: 'Rebase to master'

azure-pipelines/templates/ci-opengfx.yml

@@ -0,0 +1,8 @@
+steps:
+- bash: |
+    set -ex
+    cd bin/baseset
+    curl -L https://binaries.openttd.org/extra/opengfx/0.5.2/opengfx-0.5.2-all.zip > opengfx-0.5.2-all.zip
+    unzip opengfx-0.5.2-all.zip
+    rm -f opengfx-0.5.2-all.zip
+  displayName: 'Install OpenGFX'

azure-pipelines/templates/linux-build.yml

@@ -0,0 +1,18 @@
+parameters:
+  Image: ''
+  Tag: ''
+  ContainerCommand: ''
+
+steps:
+- task: Docker@1
+  ${{ if eq(parameters.Image, 'compile-farm') }}:
+    displayName: 'Build'
+  ${{ if eq(parameters.Image, 'compile-farm-ci') }}:
+    displayName: 'Build and test'
+  inputs:
+    command: 'Run an image'
+    imageName: openttd/${{ parameters.Image }}:${{ parameters.Tag }}
+    volumes: '$(Build.SourcesDirectory):$(Build.SourcesDirectory)'
+    workingDirectory: '$(Build.SourcesDirectory)'
+    containerCommand: ${{ parameters.ContainerCommand }}
+    runInBackground: false

azure-pipelines/templates/linux-claim-bundles.yml

@@ -0,0 +1,5 @@
+steps:
+# Because we run the compile in a docker (under root), we are not owner
+# of the 'bundles' folder. Fix that by executing a chown on it.
+- bash: sudo chown -R $(id -u):$(id -g) bundles
+  displayName: 'Claim bundles folder back'

azure-pipelines/templates/osx-build.yml

@@ -0,0 +1,5 @@
+steps:
+- script: './configure PKG_CONFIG_PATH=/usr/local/lib/pkgconfig --enable-static'
+  displayName: 'Configure'
+- script: 'make -j2'
+  displayName: 'Build'

azure-pipelines/templates/osx-dependencies.yml

@@ -0,0 +1,11 @@
+steps:
+- script: |
+    set -ex
+    HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config lzo xz libpng
+    # Remove the dynamic libraries of these libraries, to ensure we use
+    # the static versions. That is important, as it is unlikely any
+    # end-user has these brew libraries installed.
+    rm /usr/local/Cellar/lzo/*/lib/*.dylib
+    rm /usr/local/Cellar/xz/*/lib/*.dylib
+    rm /usr/local/Cellar/libpng/*/lib/*.dylib
+  displayName: 'Install dependencies'

azure-pipelines/templates/release-bundles.yml

@@ -0,0 +1,19 @@
+parameters:
+  CalculateChecksums: true
+
+steps:
+- ${{ if eq(parameters.CalculateChecksums, true) }}:
+  - bash: |
+      set -ex
+      cd bundles
+      for i in $(ls); do
+        openssl dgst -r -md5 -hex $i > $i.md5sum
+        openssl dgst -r -sha1 -hex $i > $i.sha1sum
+        openssl dgst -r -sha256 -hex $i > $i.sha256sum
+      done
+    displayName: 'Calculate checksums'
+- task: PublishBuildArtifacts@1
+  displayName: 'Publish bundles'
+  inputs:
+    PathtoPublish: bundles/
+    ArtifactName: bundles

azure-pipelines/templates/release-fetch-source.yml

@@ -0,0 +1,20 @@
+# Fetch the source tarball as prepared by an earlier job. In there is the
+# version predefined. This ensures we are all going to compile the same
+# source with the same version.
+
+steps:
+- checkout: none
+- task: DownloadBuildArtifacts@0
+  displayName: 'Download source'
+  inputs:
+    downloadType: specific
+    itemPattern: 'bundles/openttd-*-source.tar.xz'
+    downloadPath: '$(Build.ArtifactStagingDirectory)'
+- bash: tar --xz -xf ../a/bundles/openttd-*-source.tar.xz --strip-components=1
+  displayName: 'Extracting source'
+- bash: |
+    set -e
+    VERSION=$(cat .version)
+    echo "${VERSION}"
+    echo "##vso[build.updatebuildnumber]${VERSION}"
+  displayName: 'Change BuildNumber to version'

azure-pipelines/templates/release-manifest.yml

@@ -0,0 +1,13 @@
+steps:
+- task: DownloadBuildArtifacts@0
+  displayName: 'Download all bundles'
+  inputs:
+    downloadType: specific
+    itemPattern: 'bundles/*'
+    downloadPath: '$(Build.ArtifactStagingDirectory)'
+- script: |
+    set -ex
+    ./azure-pipelines/manifest.sh ../a/bundles/
+    mkdir -p bundles
+    mv manifest.yaml bundles/
+  displayName: 'Create manifest.yaml'

azure-pipelines/templates/release-prepare-source.yml

@@ -0,0 +1,24 @@
+# Set the revisions, and remove the VCS files.
+# This ensures everything else picks up on the predefined versions, and not
+# that because of some build process the version all of a sudden changes.
+
+steps:
+- script: |
+    set -ex
+    git checkout -B ${BUILD_SOURCEBRANCHNAME}
+    ./findversion.sh > .ottdrev
+    ./azure-pipelines/changelog.sh > .changelog
+    TZ='UTC' date +"%Y-%m-%d %H:%M UTC" > .release_date
+    cat .ottdrev | cut -f 1 -d$'\t' > .version
+    echo "Release Date: $(cat .release_date)"
+    echo "Revision: $(cat .ottdrev)"
+    echo "Version: $(cat .version)"
+  displayName: 'Create version files'
+- script: |
+    set -e
+    VERSION=$(cat .version)
+    echo "${VERSION}"
+    echo "##vso[build.updatebuildnumber]${VERSION}"
+  displayName: 'Change BuildNumber to version'
+- script: find . -iname .hg -or -iname .git -or -iname .svn | xargs rm -rf
+  displayName: 'Remove VCS information'