From 2f96ccc18a8303d7f2dfd7cc14aeea146c3092ae Mon Sep 17 00:00:00 2001 From: Patric Stout Date: Wed, 30 Aug 2023 20:27:15 +0200 Subject: [PATCH] Add: [CI] Create and store breakpad symbols for releases (#11247) Additionally, also store PDB and exe files on the Symbol Server for easier debugging with MSVC. --- .github/workflows/release-linux.yml | 29 +++++++++++++++++++ .github/workflows/release-macos.yml | 31 +++++++++++++++++++++ .github/workflows/release-windows.yml | 36 +++++++++++++++++++++--- .github/workflows/upload-cdn.yml | 40 +++++++++++++++++++++++++-- docs/symbol_server.md | 34 +++++++++++++++++++++++ 5 files changed, 163 insertions(+), 7 deletions(-) create mode 100644 docs/symbol_server.md diff --git a/.github/workflows/release-linux.yml b/.github/workflows/release-linux.yml index 247cd0b0c3..fad44dec9a 100644 --- a/.github/workflows/release-linux.yml +++ b/.github/workflows/release-linux.yml @@ -28,6 +28,19 @@ jobs: run: | tar -xf source.tar.gz --strip-components=1 + # dtolnay/rust-toolchain uses a curl argument (--retry-connrefused) that the curl shipped with our container doesn't support. + # So we need to do one part ourselves; the action takes over the rest and prepares the setup further. + - name: Prepare Rust toolchain + run: | + curl --proto '=https' --tlsv1.2 --retry 10 --location --silent --show-error --fail "https://sh.rustup.rs" | sh -s -- --default-toolchain none -y + echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> $GITHUB_PATH + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Enable Rust cache + uses: Swatinem/rust-cache@v2 + - name: Enable vcpkg cache uses: actions/cache@v3 with: @@ -124,6 +137,10 @@ jobs: ) echo "::endgroup::" + echo "::group::Install breakpad dependencies" + cargo install dump_syms + echo "::endgroup::" + - name: Install GCC problem matcher uses: ammaraskar/gcc-problem-matcher@master @@ -146,6 +163,11 @@ jobs: cmake --build . -j $(nproc) --target openttd echo "::endgroup::" + - name: Create breakpad symbols + run: | + cd build + dump_syms ./openttd --inlines --store symbols + - name: Create bundles run: | cd ${GITHUB_WORKSPACE}/build @@ -165,3 +187,10 @@ jobs: name: openttd-linux-generic path: build/bundles retention-days: 5 + + - name: Store symbols + uses: actions/upload-artifact@v3 + with: + name: symbols-linux-generic + path: build/symbols + retention-days: 5 diff --git a/.github/workflows/release-macos.yml b/.github/workflows/release-macos.yml index 7741064bf7..165bb6b6ab 100644 --- a/.github/workflows/release-macos.yml +++ b/.github/workflows/release-macos.yml @@ -26,15 +26,27 @@ jobs: run: | tar -xf source.tar.gz --strip-components=1 + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Enable Rust cache + uses: Swatinem/rust-cache@v2 + - name: Install dependencies env: HOMEBREW_NO_AUTO_UPDATE: 1 HOMEBREW_NO_INSTALL_CLEANUP: 1 run: | + echo "::group::Install brew dependencies" brew install \ pandoc \ pkg-config \ # EOF + echo "::endgroup::" + + echo "::group::Install breakpad dependencies" + cargo install dump_syms + echo "::endgroup::" - name: Prepare cache key id: key @@ -144,6 +156,18 @@ jobs: cmake --build . -j $(sysctl -n hw.logicalcpu) --target openttd echo "::endgroup::" + - name: Create breakpad symbols + run: | + cd build-x64 + mkdir dSYM + dsymutil ./openttd -o dSYM/openttd + dump_syms ./dSYM/openttd --inlines --store symbols + + cd ../build-arm64 + mkdir dSYM + dsymutil ./openttd -o dSYM/openttd + dump_syms ./dSYM/openttd --inlines --store ../build-x64/symbols + - name: Create bundles run: | cd build-x64 @@ -209,3 +233,10 @@ jobs: name: openttd-macos-universal path: build-x64/bundles retention-days: 5 + + - name: Store symbols + uses: actions/upload-artifact@v3 + with: + name: symbols-macos-universal + path: build-x64/symbols + retention-days: 5 diff --git a/.github/workflows/release-windows.yml b/.github/workflows/release-windows.yml index ea1d4c8ef2..7122a57f0b 100644 --- a/.github/workflows/release-windows.yml +++ b/.github/workflows/release-windows.yml @@ -39,10 +39,22 @@ jobs: run: | tar -xf source.tar.gz --strip-components=1 + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Enable Rust cache + uses: Swatinem/rust-cache@v2 + - name: Install dependencies shell: bash run: | + echo "::group::Install choco dependencies" choco install pandoc + echo "::endgroup::" + + echo "::group::Install breakpad dependencies" + cargo install dump_syms + echo "::endgroup::" - name: Prepare cache key id: key @@ -176,6 +188,12 @@ jobs: env: WINDOWS_CERTIFICATE_COMMON_NAME: ${{ secrets.WINDOWS_CERTIFICATE_COMMON_NAME }} + - name: Create breakpad symbols + shell: bash + run: | + cd build + dump_syms openttd.pdb --inlines --store symbols + - name: Create bundles shell: bash run: | @@ -184,10 +202,13 @@ jobs: cpack echo "::endgroup::" - echo "::group::Prepare PDB to be bundled" - PDB=$(ls bundles/*.zip | cut -d/ -f2 | sed 's/.zip$/.pdb/') - cp openttd.pdb bundles/${PDB} - xz -9 bundles/${PDB} + echo "::group::Move PDB and exe to symbols" + PDB_FOLDER=$(find symbols -mindepth 2 -type d) + cp openttd.pdb ${PDB_FOLDER}/ + + EXE_FOLDER=symbols/openttd.exe/$(grep "INFO CODE_ID" symbols/*/*/openttd.sym | cut -d\ -f3) + mkdir -p ${EXE_FOLDER} + cp openttd.exe ${EXE_FOLDER}/ echo "::endgroup::" echo "::group::Cleanup" @@ -213,3 +234,10 @@ jobs: name: openttd-windows-${{ matrix.arch }} path: build/bundles retention-days: 5 + + - name: Store symbols + uses: actions/upload-artifact@v3 + with: + name: symbols-windows-${{ matrix.arch }} + path: build/symbols + retention-days: 5 diff --git a/.github/workflows/upload-cdn.yml b/.github/workflows/upload-cdn.yml index f7f925222c..79cce0adde 100644 --- a/.github/workflows/upload-cdn.yml +++ b/.github/workflows/upload-cdn.yml @@ -54,6 +54,21 @@ jobs: done fi + - name: Merge symbols + run: | + mkdir symbols + cp -R symbols-*/* symbols/ + + # Compress all files as gzip, to reduce cost of storage on the CDN. + for i in $(find symbols -mindepth 2 -type f); do + gzip ${i} + done + + # Leave a mark in each folder what version actually created the symbol file. + for i in $(find symbols -mindepth 2 -type d); do + touch ${i}/.${{ inputs.version }}.txt + done + - name: Store bundles uses: actions/upload-artifact@v3 with: @@ -61,11 +76,18 @@ jobs: path: bundles/* retention-days: 5 - publish: + - name: Store breakpad symbols + uses: actions/upload-artifact@v3 + with: + name: cdn-symbols + path: symbols/* + retention-days: 5 + + publish-bundles: needs: - prepare - name: Publish + name: Publish bundles uses: OpenTTD/actions/.github/workflows/rw-cdn-upload.yml@v4 secrets: CDN_SIGNING_KEY: ${{ secrets.CDN_SIGNING_KEY }} @@ -76,10 +98,22 @@ jobs: folder: ${{ inputs.folder }} version: ${{ inputs.version }} + publish-symbols: + needs: + - prepare + + name: Publish symbols + uses: OpenTTD/actions/.github/workflows/rw-symbols-upload.yml@v4 + secrets: + SYMBOLS_SIGNING_KEY: ${{ secrets.SYMBOLS_SIGNING_KEY }} + with: + artifact-name: cdn-symbols + repository: OpenTTD + docs: if: ${{ inputs.trigger_type == 'new-master' }} needs: - - publish + - publish-bundles name: Publish docs diff --git a/docs/symbol_server.md b/docs/symbol_server.md new file mode 100644 index 0000000000..c1c522a1b6 --- /dev/null +++ b/docs/symbol_server.md @@ -0,0 +1,34 @@ +# OpenTTD's Symbol Server + +For all official releases, OpenTTD collects the Breakpad Symbols (SYM-files) and Microsoft's Symbols (PDB-files), and publishes them on our own Symbol Server (https://symbols.openttd.org). + +These symbol files are needed to analyze `crash.dmp` files as attached to issues by users. +A `crash.dmp` is created on Windows, Linux, and MacOS when a crash happens. +This combined with the `crash.log` should give a pretty good indication what was going on at the moment the game crashed. + +## Analyzing a crash.dmp + +### MSVC + +In MSVC you can add the above URL as Symbol Server (and please enable MSVC's for all other libraries), allowing you to analyze `crash.dmp`. + +Now simply open up the `crash.dmp`, and start debugging. + +### All other platforms + +The best tool to use is `minidump-stackwalk` as published in the Rust's cargo index: + +```bash +cargo install minidump-stackwalk +``` + +For how to install Rust, please see [here](https://doc.rust-lang.org/cargo/getting-started/installation.html). + +Now run the tool like: + +```bash +minidump-stackwalk --symbols-url https://symbols.openttd.org +``` + +For convenience, the above Symbol Server also check with Mozilla's Symbol Server in case any other library but OpenTTD is requested. +This means files like `libc`, `kernel32.dll`, etc are all available on the above mentioned Symbol Server.