Add: [CI] Create and store breakpad symbols for releases (#11247)

Additionally, also store PDB and exe files on the Symbol Server
for easier debugging with MSVC.

.github/workflows/release-linux.yml

@@ -28,6 +28,19 @@ jobs:
       run: |
         tar -xf source.tar.gz --strip-components=1
 
+    # dtolnay/rust-toolchain uses a curl argument (--retry-connrefused) that the curl shipped with our container doesn't support.
+    # So we need to do one part ourselves; the action takes over the rest and prepares the setup further.
+    - name: Prepare Rust toolchain
+      run: |
+        curl --proto '=https' --tlsv1.2 --retry 10 --location --silent --show-error --fail "https://sh.rustup.rs" | sh -s -- --default-toolchain none -y
+        echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> $GITHUB_PATH
+
+    - name: Install Rust toolchain
+      uses: dtolnay/rust-toolchain@stable
+
+    - name: Enable Rust cache
+      uses: Swatinem/rust-cache@v2
+
     - name: Enable vcpkg cache
       uses: actions/cache@v3
       with:
@@ -124,6 +137,10 @@ jobs:
         )
         echo "::endgroup::"
 
+        echo "::group::Install breakpad dependencies"
+        cargo install dump_syms
+        echo "::endgroup::"
+
     - name: Install GCC problem matcher
       uses: ammaraskar/gcc-problem-matcher@master
 
@@ -146,6 +163,11 @@ jobs:
         cmake --build . -j $(nproc) --target openttd
         echo "::endgroup::"
 
+    - name: Create breakpad symbols
+      run: |
+        cd build
+        dump_syms ./openttd --inlines --store symbols
+
     - name: Create bundles
       run: |
         cd ${GITHUB_WORKSPACE}/build
@@ -165,3 +187,10 @@ jobs:
         name: openttd-linux-generic
         path: build/bundles
         retention-days: 5
+
+    - name: Store symbols
+      uses: actions/upload-artifact@v3
+      with:
+        name: symbols-linux-generic
+        path: build/symbols
+        retention-days: 5

.github/workflows/release-macos.yml

@@ -26,15 +26,27 @@ jobs:
       run: |
         tar -xf source.tar.gz --strip-components=1
 
+    - name: Install Rust toolchain
+      uses: dtolnay/rust-toolchain@stable
+
+    - name: Enable Rust cache
+      uses: Swatinem/rust-cache@v2
+
     - name: Install dependencies
       env:
         HOMEBREW_NO_AUTO_UPDATE: 1
         HOMEBREW_NO_INSTALL_CLEANUP: 1
       run: |
+        echo "::group::Install brew dependencies"
         brew install \
           pandoc \
           pkg-config \
           # EOF
+        echo "::endgroup::"
+
+        echo "::group::Install breakpad dependencies"
+        cargo install dump_syms
+        echo "::endgroup::"
 
     - name: Prepare cache key
       id: key
@@ -144,6 +156,18 @@ jobs:
         cmake --build . -j $(sysctl -n hw.logicalcpu) --target openttd
         echo "::endgroup::"
 
+    - name: Create breakpad symbols
+      run: |
+        cd build-x64
+        mkdir dSYM
+        dsymutil ./openttd -o dSYM/openttd
+        dump_syms ./dSYM/openttd --inlines --store symbols
+
+        cd ../build-arm64
+        mkdir dSYM
+        dsymutil ./openttd -o dSYM/openttd
+        dump_syms ./dSYM/openttd --inlines --store ../build-x64/symbols
+
     - name: Create bundles
       run: |
         cd build-x64
@@ -209,3 +233,10 @@ jobs:
         name: openttd-macos-universal
         path: build-x64/bundles
         retention-days: 5
+
+    - name: Store symbols
+      uses: actions/upload-artifact@v3
+      with:
+        name: symbols-macos-universal
+        path: build-x64/symbols
+        retention-days: 5

.github/workflows/release-windows.yml

@@ -39,10 +39,22 @@ jobs:
       run: |
         tar -xf source.tar.gz --strip-components=1
 
+    - name: Install Rust toolchain
+      uses: dtolnay/rust-toolchain@stable
+
+    - name: Enable Rust cache
+      uses: Swatinem/rust-cache@v2
+
     - name: Install dependencies
       shell: bash
       run: |
+        echo "::group::Install choco dependencies"
         choco install pandoc
+        echo "::endgroup::"
+
+        echo "::group::Install breakpad dependencies"
+        cargo install dump_syms
+        echo "::endgroup::"
 
     - name: Prepare cache key
       id: key
@@ -176,6 +188,12 @@ jobs:
       env:
         WINDOWS_CERTIFICATE_COMMON_NAME: ${{ secrets.WINDOWS_CERTIFICATE_COMMON_NAME }}
 
+    - name: Create breakpad symbols
+      shell: bash
+      run: |
+        cd build
+        dump_syms openttd.pdb --inlines --store symbols
+
     - name: Create bundles
       shell: bash
       run: |
@@ -184,10 +202,13 @@ jobs:
         cpack
         echo "::endgroup::"
 
-        echo "::group::Prepare PDB to be bundled"
-        PDB=$(ls bundles/*.zip | cut -d/ -f2 | sed 's/.zip$/.pdb/')
-        cp openttd.pdb bundles/${PDB}
-        xz -9 bundles/${PDB}
+        echo "::group::Move PDB and exe to symbols"
+        PDB_FOLDER=$(find symbols -mindepth 2 -type d)
+        cp openttd.pdb ${PDB_FOLDER}/
+
+        EXE_FOLDER=symbols/openttd.exe/$(grep "INFO CODE_ID" symbols/*/*/openttd.sym | cut -d\  -f3)
+        mkdir -p ${EXE_FOLDER}
+        cp openttd.exe ${EXE_FOLDER}/
         echo "::endgroup::"
 
         echo "::group::Cleanup"
@@ -213,3 +234,10 @@ jobs:
         name: openttd-windows-${{ matrix.arch }}
         path: build/bundles
         retention-days: 5
+
+    - name: Store symbols
+      uses: actions/upload-artifact@v3
+      with:
+        name: symbols-windows-${{ matrix.arch }}
+        path: build/symbols
+        retention-days: 5

.github/workflows/upload-cdn.yml

@@ -54,6 +54,21 @@ jobs:
           done
         fi
 
+    - name: Merge symbols
+      run: |
+        mkdir symbols
+        cp -R symbols-*/* symbols/
+
+        # Compress all files as gzip, to reduce cost of storage on the CDN.
+        for i in $(find symbols -mindepth 2 -type f); do
+          gzip ${i}
+        done
+
+        # Leave a mark in each folder what version actually created the symbol file.
+        for i in $(find symbols -mindepth 2 -type d); do
+          touch ${i}/.${{ inputs.version }}.txt
+        done
+
     - name: Store bundles
       uses: actions/upload-artifact@v3
       with:
@@ -61,11 +76,18 @@ jobs:
         path: bundles/*
         retention-days: 5
 
-  publish:
+    - name: Store breakpad symbols
+      uses: actions/upload-artifact@v3
+      with:
+        name: cdn-symbols
+        path: symbols/*
+        retention-days: 5
+
+  publish-bundles:
     needs:
     - prepare
 
-    name: Publish
+    name: Publish bundles
     uses: OpenTTD/actions/.github/workflows/rw-cdn-upload.yml@v4
     secrets:
       CDN_SIGNING_KEY: ${{ secrets.CDN_SIGNING_KEY }}
@@ -76,10 +98,22 @@ jobs:
       folder: ${{ inputs.folder }}
       version: ${{ inputs.version }}
 
+  publish-symbols:
+    needs:
+    - prepare
+
+    name: Publish symbols
+    uses: OpenTTD/actions/.github/workflows/rw-symbols-upload.yml@v4
+    secrets:
+      SYMBOLS_SIGNING_KEY: ${{ secrets.SYMBOLS_SIGNING_KEY }}
+    with:
+      artifact-name: cdn-symbols
+      repository: OpenTTD
+
   docs:
     if: ${{ inputs.trigger_type == 'new-master' }}
     needs:
-    - publish
+    - publish-bundles
 
     name: Publish docs
 

docs/symbol_server.md

@@ -0,0 +1,34 @@
+# OpenTTD's Symbol Server
+
+For all official releases, OpenTTD collects the Breakpad Symbols (SYM-files) and Microsoft's Symbols (PDB-files), and publishes them on our own Symbol Server (https://symbols.openttd.org).
+
+These symbol files are needed to analyze `crash.dmp` files as attached to issues by users.
+A `crash.dmp` is created on Windows, Linux, and MacOS when a crash happens.
+This combined with the `crash.log` should give a pretty good indication what was going on at the moment the game crashed.
+
+## Analyzing a crash.dmp
+
+### MSVC
+
+In MSVC you can add the above URL as Symbol Server (and please enable MSVC's for all other libraries), allowing you to analyze `crash.dmp`.
+
+Now simply open up the `crash.dmp`, and start debugging.
+
+### All other platforms
+
+The best tool to use is `minidump-stackwalk` as published in the Rust's cargo index:
+
+```bash
+cargo install minidump-stackwalk
+```
+
+For how to install Rust, please see [here](https://doc.rust-lang.org/cargo/getting-started/installation.html).
+
+Now run the tool like:
+
+```bash
+minidump-stackwalk <crash.dmp> --symbols-url https://symbols.openttd.org
+```
+
+For convenience, the above Symbol Server also check with Mozilla's Symbol Server in case any other library but OpenTTD is requested.
+This means files like `libc`, `kernel32.dll`, etc are all available on the above mentioned Symbol Server.