Add: [AzurePipeline] introducing a release pipeline

This release pipeline creates all the official release binaries,
and publishes them as artifacts. Currently it can only produce
nightlies and custom builds; stable/testing release binaries are
untested.

This commit also splits up the pipeline in small bits, to both
improve readability, and to share code with the CI pipeline where
possible.
This commit is contained in:
Patric Stout 2019-01-05 20:11:29 +01:00 committed by Charles Pigott
parent 52a66e4dd3
commit 750927372f
21 changed files with 466 additions and 71 deletions

View File

@ -17,37 +17,12 @@ jobs:
BuildPlatform: 'x64'
steps:
# Rebase to origin/master for every PR. This means users don't have to
# rebase every time master changes. As long as the PR applies cleanly, we
# will validate it.
- script: |
git config user.email 'info@openttd.org'
git config user.name 'OpenTTD CI'
git rebase origin/master
displayName: 'Rebase to master'
- bash: |
set -ex
curl -L https://github.com/OpenTTD/OpenTTD-CF/releases/download/latest/windows-dependencies.zip > windows-dependencies.zip
unzip windows-dependencies.zip
rm -f windows-dependencies.zip
displayName: 'Download dependencies'
workingDirectory: $(Build.ArtifactStagingDirectory)
- script: $(Build.ArtifactStagingDirectory)\windows-dependencies\vcpkg.exe integrate install
displayName: 'Install dependencies'
- bash: |
set -ex
cd bin/baseset
curl -L https://binaries.openttd.org/extra/opengfx/0.5.2/opengfx-0.5.2-all.zip > opengfx-0.5.2-all.zip
unzip opengfx-0.5.2-all.zip
rm -f opengfx-0.5.2-all.zip
displayName: 'Install OpenGFX'
- task: VSBuild@1
displayName: 'Build'
inputs:
solution: 'projects/openttd_vs141.sln'
platform: $(BuildPlatform)
configuration: Release
maximumCpuCount: true
- template: azure-pipelines/templates/ci-git-rebase.yml
- template: azure-pipelines/templates/windows-dependencies.yml
- template: azure-pipelines/templates/ci-opengfx.yml
- template: azure-pipelines/templates/windows-build.yml
parameters:
BuildPlatform: $(BuildPlatform)
- script: |
call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x86
cd projects
@ -67,22 +42,13 @@ jobs:
linux-i386-gcc-6: {}
steps:
# Rebase to origin/master for every PR. This means users don't have to
# rebase every time master changes. As long as the PR applies cleanly, we
# will validate it.
- script: |
git config user.email 'info@openttd.org'
git config user.name 'OpenTTD CI'
git rebase origin/master
displayName: 'Rebase to master'
- task: Docker@1
displayName: 'Build and test'
inputs:
command: 'Run an image'
imageName: openttd/compile-farm-ci:$(Agent.JobName)
volumes: '$(Build.SourcesDirectory):$(Build.SourcesDirectory)'
workingDirectory: '$(Build.SourcesDirectory)'
runInBackground: false
- template: azure-pipelines/templates/ci-git-rebase.yml
# The dockers already have the dependencies installed
# The dockers already have OpenGFX installed
- template: azure-pipelines/templates/linux-build.yml
parameters:
Image: compile-farm-ci
Tag: $(Agent.JobName)
- job: macos
displayName: 'MacOS'
@ -90,27 +56,9 @@ jobs:
vmImage: 'macOS-10.13'
steps:
# Rebase to origin/master for every PR. This means users don't have to
# rebase every time master changes. As long as the PR applies cleanly, we
# will validate it.
- script: |
git config user.email 'info@openttd.org'
git config user.name 'OpenTTD CI'
git rebase origin/master
displayName: 'Rebase to master'
- script: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config lzo xz libpng
displayName: 'Install dependencies'
- bash: |
set -ex
mkdir -p ~/Documents/OpenTTD/baseset
cd ~/Documents/OpenTTD/baseset
wget https://binaries.openttd.org/extra/opengfx/0.5.2/opengfx-0.5.2-all.zip
unzip opengfx-0.5.2-all.zip
rm -f opengfx-0.5.2-all.zip
displayName: 'Install OpenGFX'
- script: './configure PKG_CONFIG_PATH=/usr/local/lib/pkgconfig --enable-static'
displayName: 'Configure'
- script: 'make -j2'
displayName: 'Build'
- template: azure-pipelines/templates/ci-git-rebase.yml
- template: azure-pipelines/templates/osx-dependencies.yml
- template: azure-pipelines/templates/ci-opengfx.yml
- template: azure-pipelines/templates/osx-build.yml
- script: 'make regression'
displayName: 'Test'

View File

@ -0,0 +1,7 @@
trigger: none
pr: none
jobs:
- template: azure-pipelines/templates/release.yml
parameters:
IsStableRelease: true

View File

@ -0,0 +1,7 @@
trigger: none
pr: none
jobs:
- template: azure-pipelines/templates/release.yml
parameters:
IsStableRelease: false

20
azure-pipelines/changelog.sh Executable file
View File

@ -0,0 +1,20 @@
#!/bin/sh
tag=$(git describe --tags 2>/dev/null)
# If we are a tag, show the part of the changelog that matches the tag.
# In case of a stable, also show all betas and RCs.
if [ -n "$tag" ]; then
grep="."
if [ "$(echo $tag | grep '-')" = "" ]; then
grep='^[0-9]\.[0-9]\.[0-9][^-]'
fi
next=$(cat changelog.txt | grep '^[0-9]' | awk 'BEGIN { show="false" } // { if (show=="true") print $0; if ($1=="'$tag'") show="true"} ' | grep "$grep" | head -n1 | sed 's/ .*//')
cat changelog.txt | awk 'BEGIN { show="false" } /^[0-9].[0-9].[0-9]/ { if ($1=="'$next'") show="false"; if ($1=="'$tag'") show="true";} // { if (show=="true") print $0 }'
exit 0
fi
# In all other cases, show the git log of the last 7 days
revdate=$(git log -1 --pretty=format:"%ci")
last_week=$(date -u -d "$revdate -7days" +"%Y-%m-%d %H:%M")
git log --after="${last_week}" --pretty=fuller

60
azure-pipelines/manifest.sh Executable file
View File

@ -0,0 +1,60 @@
#!/bin/sh
set -ex
if [ -z "$1" ]; then
echo "Usage: $0 <folder-with-bundles>"
exit 1
fi
FOLDER=$1
if [ ! -e .version ] || [ ! -e .release_date ]; then
echo "This script should be executed in the root of an extracted source tarball"
exit 1
fi
# Find the name based on the version
if [ "${ISSTABLERELEASE}" = "true" ]; then
isTesting=$(cat .version | grep "RC\|beta" || true)
if [ -z "${isTesting}" ]; then
NAME="stable"
else
NAME="testing"
fi
else
NAME=$(cat .version | cut -d- -f2 | cut -d- -f-2)
fi
# Convert the date to a YAML date
DATE=$(cat .release_date | tr ' ' T | sed 's/TUTC/:00-00:00/')
VERSION=$(cat .version)
BASE="openttd-${VERSION}"
echo "name: ${NAME}" >> manifest.yaml
echo "date: ${DATE}" >> manifest.yaml
echo "base: ${BASE}-" >> manifest.yaml
echo "files:" >> manifest.yaml
error=""
for i in $(ls ${FOLDER} | grep -v ".txt$\|.md$\|sum$" | sort); do
if [ -n "$(echo $i | grep pdb.xz)" ]; then continue; fi
if [ -n "$(echo $i | grep dbg.deb)" ]; then continue; fi
if [ ! -e ${FOLDER}/$i.md5sum ] || [ ! -e ${FOLDER}/$i.sha1sum ] || [ ! -e ${FOLDER}/$i.sha256sum ]; then
echo "ERROR: missing checksum file for ${i}" 1>&2
error="y"
continue
fi
echo "- id: $i" >> manifest.yaml
echo " size: $(stat -c"%s" ${FOLDER}/$i)" >> manifest.yaml
echo " md5sum: $(cat ${FOLDER}/$i.md5sum | cut -d\ -f1)" >> manifest.yaml
echo " sha1sum: $(cat ${FOLDER}/$i.sha1sum | cut -d\ -f1)" >> manifest.yaml
echo " sha256sum: $(cat ${FOLDER}/$i.sha256sum | cut -d\ -f1)" >> manifest.yaml
done
if [ -n "${error}" ]; then
echo "ERROR: exiting due to earlier errors" 1>&2
exit 1
fi

View File

@ -0,0 +1,9 @@
steps:
# Rebase to origin/master for every PR. This means users don't have to
# rebase every time master changes. As long as the PR applies cleanly, we
# will validate it.
- script: |
git config user.email 'info@openttd.org'
git config user.name 'OpenTTD CI'
git rebase origin/master
displayName: 'Rebase to master'

View File

@ -0,0 +1,8 @@
steps:
- bash: |
set -ex
cd bin/baseset
curl -L https://binaries.openttd.org/extra/opengfx/0.5.2/opengfx-0.5.2-all.zip > opengfx-0.5.2-all.zip
unzip opengfx-0.5.2-all.zip
rm -f opengfx-0.5.2-all.zip
displayName: 'Install OpenGFX'

View File

@ -0,0 +1,18 @@
parameters:
Image: ''
Tag: ''
ContainerCommand: ''
steps:
- task: Docker@1
${{ if eq(parameters.Image, 'compile-farm') }}:
displayName: 'Build'
${{ if eq(parameters.Image, 'compile-farm-ci') }}:
displayName: 'Build and test'
inputs:
command: 'Run an image'
imageName: openttd/${{ parameters.Image }}:${{ parameters.Tag }}
volumes: '$(Build.SourcesDirectory):$(Build.SourcesDirectory)'
workingDirectory: '$(Build.SourcesDirectory)'
containerCommand: ${{ parameters.ContainerCommand }}
runInBackground: false

View File

@ -0,0 +1,5 @@
steps:
# Because we run the compile in a docker (under root), we are not owner
# of the 'bundles' folder. Fix that by executing a chown on it.
- bash: sudo chown -R $(id -u):$(id -g) bundles
displayName: 'Claim bundles folder back'

View File

@ -0,0 +1,5 @@
steps:
- script: './configure PKG_CONFIG_PATH=/usr/local/lib/pkgconfig --enable-static'
displayName: 'Configure'
- script: 'make -j2'
displayName: 'Build'

View File

@ -0,0 +1,11 @@
steps:
- script: |
set -ex
HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config lzo xz libpng
# Remove the dynamic libraries of these libraries, to ensure we use
# the static versions. That is important, as it is unlikely any
# end-user has these brew libraries installed.
rm /usr/local/Cellar/lzo/*/lib/*.dylib
rm /usr/local/Cellar/xz/*/lib/*.dylib
rm /usr/local/Cellar/libpng/*/lib/*.dylib
displayName: 'Install dependencies'

View File

@ -0,0 +1,19 @@
parameters:
CalculateChecksums: true
steps:
- ${{ if eq(parameters.CalculateChecksums, true) }}:
- bash: |
set -ex
cd bundles
for i in $(ls); do
openssl dgst -r -md5 -hex $i > $i.md5sum
openssl dgst -r -sha1 -hex $i > $i.sha1sum
openssl dgst -r -sha256 -hex $i > $i.sha256sum
done
displayName: 'Calculate checksums'
- task: PublishBuildArtifacts@1
displayName: 'Publish bundles'
inputs:
PathtoPublish: bundles/
ArtifactName: bundles

View File

@ -0,0 +1,20 @@
# Fetch the source tarball as prepared by an earlier job. In there is the
# version predefined. This ensures we are all going to compile the same
# source with the same version.
steps:
- checkout: none
- task: DownloadBuildArtifacts@0
displayName: 'Download source'
inputs:
downloadType: specific
itemPattern: 'bundles/openttd-*-source.tar.xz'
downloadPath: '$(Build.ArtifactStagingDirectory)'
- bash: tar --xz -xf ../a/bundles/openttd-*-source.tar.xz --strip-components=1
displayName: 'Extracting source'
- bash: |
set -e
VERSION=$(cat .version)
echo "${VERSION}"
echo "##vso[build.updatebuildnumber]${VERSION}"
displayName: 'Change BuildNumber to version'

View File

@ -0,0 +1,13 @@
steps:
- task: DownloadBuildArtifacts@0
displayName: 'Download all bundles'
inputs:
downloadType: specific
itemPattern: 'bundles/*'
downloadPath: '$(Build.ArtifactStagingDirectory)'
- script: |
set -ex
./azure-pipelines/manifest.sh ../a/bundles/
mkdir -p bundles
mv manifest.yaml bundles/
displayName: 'Create manifest.yaml'

View File

@ -0,0 +1,24 @@
# Set the revisions, and remove the VCS files.
# This ensures everything else picks up on the predefined versions, and not
# that because of some build process the version all of a sudden changes.
steps:
- script: |
set -ex
git checkout -B ${BUILD_SOURCEBRANCHNAME}
./findversion.sh > .ottdrev
./azure-pipelines/changelog.sh > .changelog
TZ='UTC' date +"%Y-%m-%d %H:%M UTC" > .release_date
cat .ottdrev | cut -f 1 -d$'\t' > .version
echo "Release Date: $(cat .release_date)"
echo "Revision: $(cat .ottdrev)"
echo "Version: $(cat .version)"
displayName: 'Create version files'
- script: |
set -e
VERSION=$(cat .version)
echo "${VERSION}"
echo "##vso[build.updatebuildnumber]${VERSION}"
displayName: 'Change BuildNumber to version'
- script: find . -iname .hg -or -iname .git -or -iname .svn | xargs rm -rf
displayName: 'Remove VCS information'

View File

@ -0,0 +1,169 @@
parameters:
# If this is false, not all targets are triggered. For example:
# The NSIS installer for Windows and the creation of debs only work for
# releases. Not for any other type of binary. So they are skilled if this
# is set to false.
IsStableRelease: false
jobs:
- job: source
displayName: 'Source'
pool:
vmImage: 'ubuntu-16.04'
steps:
- template: release-prepare-source.yml
- script: |
set -ex
# Rename the folder to openttd-NNN-source
mkdir openttd-$(Build.BuildNumber)
find . -maxdepth 1 -not -name . -not -name openttd-$(Build.BuildNumber) -exec mv {} openttd-$(Build.BuildNumber)/ \;
# Copy back release_date, as it is needed for the template 'release-bundles'
cp openttd-$(Build.BuildNumber)/.release_date .release_date
mkdir bundles
tar --xz -cf bundles/openttd-$(Build.BuildNumber)-source.tar.xz openttd-$(Build.BuildNumber)
zip -9 -r -q bundles/openttd-$(Build.BuildNumber)-source.zip openttd-$(Build.BuildNumber)
displayName: 'Create bundle'
- template: release-bundles.yml
- job: meta
displayName: 'Metadata'
pool:
vmImage: 'ubuntu-16.04'
dependsOn: source
steps:
- template: release-fetch-source.yml
- script: |
set -ex
mkdir -p bundles
cp .changelog bundles/changelog.txt
cp .release_date bundles/released.txt
cp README.md bundles/README.md
displayName: 'Copy meta files'
- template: release-bundles.yml
parameters:
CalculateChecksums: false
- job: docs
displayName: 'Docs'
pool:
vmImage: 'ubuntu-16.04'
dependsOn: source
steps:
- template: release-fetch-source.yml
- template: linux-build.yml
parameters:
Image: compile-farm
ContainerCommand: '$(Build.BuildNumber)'
Tag: docs
- template: linux-claim-bundles.yml
- template: release-bundles.yml
- job: windows
displayName: 'Windows'
pool:
vmImage: 'VS2017-Win2016'
dependsOn: source
strategy:
matrix:
Win32:
BuildPlatform: 'Win32'
BundlePlatform: 'win32'
Win64:
BuildPlatform: 'x64'
BundlePlatform: 'win64'
steps:
- template: release-fetch-source.yml
- template: windows-dependencies.yml
- template: windows-dependency-zip.yml
- ${{ if eq(parameters.IsStableRelease, true) }}:
- template: windows-dependency-nsis.yml
- template: windows-build.yml
parameters:
BuildPlatform: $(BuildPlatform)
- bash: |
set -ex
make -f Makefile.msvc bundle_pdb bundle_zip PLATFORM=$(BundlePlatform) BUNDLE_NAME=openttd-$(Build.BuildNumber)-windows-$(BundlePlatform)
displayName: 'Create bundles'
- ${{ if eq(parameters.IsStableRelease, true) }}:
- bash: |
set -ex
# NSIS will be part of the Hosted image in the next update. Till then, we set the PATH ourself
export PATH="${PATH}:/c/Program Files (x86)/NSIS"
make -f Makefile.msvc bundle_exe PLATFORM=$(BundlePlatform) BUNDLE_NAME=openttd-$(Build.BuildNumber)-windows-$(BundlePlatform)
displayName: 'Create installer bundle'
- template: release-bundles.yml
- ${{ if eq(parameters.IsStableRelease, true) }}:
- job: linux_stable
displayName: 'Linux'
pool:
vmImage: 'ubuntu-16.04'
dependsOn: source
strategy:
matrix:
linux-ubuntu-xenial-i386-gcc: {}
linux-ubuntu-xenial-amd64-gcc: {}
linux-ubuntu-bionic-i386-gcc: {}
linux-ubuntu-bionic-amd64-gcc: {}
linux-debian-jessie-i386-gcc: {}
linux-debian-jessie-amd64-gcc: {}
linux-debian-stretch-i386-gcc: {}
linux-debian-stretch-amd64-gcc: {}
steps:
- template: release-fetch-source.yml
- template: linux-build.yml
parameters:
Image: compile-farm
ContainerCommand: '$(Build.BuildNumber)'
Tag: $(Agent.JobName)
- template: linux-claim-bundles.yml
- template: release-bundles.yml
- job: macos
displayName: 'MacOS'
pool:
vmImage: 'macOS-10.13'
dependsOn: source
steps:
- template: release-fetch-source.yml
- template: osx-dependencies.yml
- template: osx-build.yml
- script: 'make bundle_zip bundle_dmg BUNDLE_NAME=openttd-$(Build.BuildNumber)-macosx'
displayName: 'Create bundles'
- template: release-bundles.yml
- job: manifest
displayName: 'Manifest'
pool:
vmImage: 'ubuntu-16.04'
dependsOn:
- source
- docs
- windows
- ${{ if eq(parameters.IsStableRelease, true) }}:
- linux_stable
- macos
# "Skipped" is not a status, and is not succeeded. So it seems to be
# considered failed. So we trigger if all the earlier jobs are done (which
# might be succeeded, failed, or skipped), and run this job. This is not
# optimal, but given the rules, it is the only way to get this to work (as
# some jobs might be skipped).
condition: succeededOrFailed()
steps:
- template: release-fetch-source.yml
- template: release-manifest.yml
- template: release-bundles.yml
parameters:
CalculateChecksums: false

View File

@ -0,0 +1,11 @@
parameters:
BuildPlatform: ''
steps:
- task: VSBuild@1
displayName: 'Build'
inputs:
solution: 'projects/openttd_vs141.sln'
platform: ${{ parameters.BuildPlatform }}
configuration: Release
maximumCpuCount: true

View File

@ -0,0 +1,10 @@
steps:
- bash: |
set -ex
curl -L https://github.com/OpenTTD/OpenTTD-CF/releases/download/latest/windows-dependencies.zip > windows-dependencies.zip
unzip windows-dependencies.zip
rm -f windows-dependencies.zip
displayName: 'Download dependencies'
workingDirectory: $(Build.ArtifactStagingDirectory)
- script: $(Build.ArtifactStagingDirectory)\windows-dependencies\vcpkg.exe integrate install
displayName: 'Install dependencies'

View File

@ -0,0 +1,28 @@
parameters:
condition: true
steps:
- bash: |
set -ex
# NSIS will be part of the Hosted image in the next update. Till then, we install it ourself
choco install nsis -y
mkdir nsis-plugin; cd nsis-plugin
curl -L https://devs.openttd.org/~truebrain/nsis-plugins/Nsis7z.zip > Nsis7z.zip
unzip Nsis7z.zip
cp -R Plugins/* "/c/Program Files (x86)/NSIS/Plugins/"
cd ..; rm -rf nsis-plugin
mkdir nsis-plugin; cd nsis-plugin
curl -L https://devs.openttd.org/~truebrain/nsis-plugins/NsisGetVersion.zip > NsisGetVersion.zip
unzip NsisGetVersion.zip
cp -R Plugins/* "/c/Program Files (x86)/NSIS/Plugins/x86-ansi/"
cd ..; rm -rf nsis-plugin
mkdir nsis-plugin; cd nsis-plugin
curl -L https://devs.openttd.org/~truebrain/nsis-plugins/NsisFindProc.zip > NsisFindProc.zip
unzip NsisFindProc.zip
cp -R *.dll "/c/Program Files (x86)/NSIS/Plugins/x86-ansi/"
cd ..; rm -rf nsis-plugin
displayName: 'Install NSIS with the 7z, GetVersion, and FindProc plugins'
condition: and(succeeded(), ${{ parameters.condition }})

View File

@ -0,0 +1,5 @@
steps:
- bash: |
set -ex
choco install zip
displayName: 'Install zip'

View File

@ -74,8 +74,6 @@ if [ -d "$ROOT_DIR/.git" ]; then
if [ -n "$TAG" ]; then
VERSION="${TAG}"
elif [ "${BRANCH}" = "master" ]; then
VERSION="${ISODATE}-g${SHORTHASH}"
else
VERSION="${ISODATE}-${BRANCH}-g${SHORTHASH}"
fi