Update: emsdk to 3.1.28 and lzma (for emsdk) to 5.4.0 (#10234)

Co-authored-by: Patric Stout <truebrain@openttd.org>

.github/workflows/ci-build.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-20.04
     container:
       # If you change this version, change the number in the cache step too.
-      image: emscripten/emsdk:2.0.31
+      image: emscripten/emsdk:3.1.28
 
     steps:
     - name: Checkout
@@ -26,7 +26,7 @@ jobs:
       uses: actions/cache@v3
       with:
         path: /emsdk/upstream/emscripten/cache
-        key: 2.0.31-${{ runner.os }}
+        key: 3.1.28-${{ runner.os }}
 
     - name: Patch Emscripten to support LZMA
       run: |

.github/workflows/preview_build.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-20.04
     container:
       # If you change this version, change the number in the cache step too.
-      image: emscripten/emsdk:2.0.31
+      image: emscripten/emsdk:3.1.28
 
     steps:
     - name: Update deployment status to in progress
@@ -44,7 +44,7 @@ jobs:
       uses: actions/cache@v3
       with:
         path: /emsdk/upstream/emscripten/cache
-        key: 2.0.31-${{ runner.os }}
+        key: 3.1.28-${{ runner.os }}
 
     - name: Patch Emscripten to support LZMA
       run: |

os/emscripten/Dockerfile

@@ -1,4 +1,4 @@
-FROM emscripten/emsdk:2.0.34
+FROM emscripten/emsdk:3.1.28
 
 COPY emsdk-liblzma.patch /
 RUN cd /emsdk/upstream/emscripten && patch -p1 < /emsdk-liblzma.patch

os/emscripten/README.md

@@ -1,40 +1,38 @@
 ## How to build with Emscripten
 
-Building with Emscripten works with emsdk 2.0.31 and above.
+Please use docker with the supplied `Dockerfile` to build for emscripten.
+It takes care of a few things:
+- Use a version of emscripten we know works
+- Patch in LibLZMA support (as this is not supported by upstream)
 
-Currently there is no LibLZMA support upstream; for this we suggest to apply
+First, build the docker image by navigating in the folder this `README.md` is in, and executing:
-the provided patch in this folder to your emsdk installation.
-
-For convenience, a Dockerfile is supplied that does this patches for you
-against upstream emsdk docker. Best way to use it:
-
-Build the docker image:
 ```
   docker build -t emsdk-lzma .
 ```
 
-Build the host tools first:
+Next, navigate back to the root folder of this project.
+
+Now we build the host tools first:
 ```
   mkdir build-host
   docker run -it --rm -v $(pwd):$(pwd) -u $(id -u):$(id -g) --workdir $(pwd)/build-host emsdk-lzma cmake .. -DOPTION_TOOLS_ONLY=ON
-  docker run -it --rm -v $(pwd):$(pwd) -u $(id -u):$(id -g) --workdir $(pwd)/build-host emsdk-lzma make -j5 tools
+  docker run -it --rm -v $(pwd):$(pwd) -u $(id -u):$(id -g) --workdir $(pwd)/build-host emsdk-lzma make -j$(nproc) tools
 ```
 
-Next, build the game with emscripten:
+Finally, we build the actual game:
-
 ```
   mkdir build
   docker run -it --rm -v $(pwd):$(pwd) -u $(id -u):$(id -g) --workdir $(pwd)/build emsdk-lzma emcmake cmake .. -DHOST_BINARY_DIR=../build-host -DCMAKE_BUILD_TYPE=Release -DOPTION_USE_ASSERTS=OFF
-  docker run -it --rm -v $(pwd):$(pwd) -u $(id -u):$(id -g) --workdir $(pwd)/build emsdk-lzma emmake make -j5
+  docker run -it --rm -v $(pwd):$(pwd) -u $(id -u):$(id -g) --workdir $(pwd)/build emsdk-lzma emmake make -j$(nproc)
 ```
 
-And now you have in your build folder files like "openttd.html".
+In the `build` folder you will now see `openttd.html`.
 
-To run it locally, you would have to start a local webserver, like:
+To run it locally, you would have to start a local webserver; something like:
 
 ```
   cd build
   python3 -m http.server
 ````
 
-Now you can play the game via http://127.0.0.1:8000/openttd.html .
+You can now play the game via http://127.0.0.1:8000/openttd.html .

os/emscripten/emsdk-liblzma.patch

@@ -13,7 +13,7 @@ of emsdk.
 diff --git a/tools/settings.py b/tools/settings.py
 --- a/tools/settings.py
 +++ b/tools/settings.py
-@@ -38,6 +38,7 @@
+@@ -40,6 +40,7 @@ PORTS_SETTINGS = {
      'USE_SDL_NET',
      'USE_SDL_GFX',
      'USE_LIBJPEG',
@@ -24,24 +24,23 @@ diff --git a/tools/settings.py b/tools/settings.py
 diff --git a/src/settings.js b/src/settings.js
 --- a/src/settings.js
 +++ b/src/settings.js
-@@ -1382,8 +1382,12 @@ var USE_BZIP2 = 0;
+@@ -1450,6 +1450,10 @@ var USE_GIFLIB = false;
- // 1 = use libjpeg from emscripten-ports
+ // [compile+link]
- // [link]
+ var USE_LIBJPEG = false;
- var USE_LIBJPEG = 0;
 
 +// 1 = use liblzma from emscripten-ports
-+// [link]
++// [compile+link]
-+var USE_LIBLZMA = 0;
++var USE_LIBLZMA = false;
 +
  // 1 = use libpng from emscripten-ports
- // [link]
+ // [compile+link]
- var USE_LIBPNG = 0;
+ var USE_LIBPNG = false;
 
 diff --git a/tools/ports/liblzma.py b/tools/ports/liblzma.py
 new file mode 100644
 --- /dev/null
 +++ b/tools/ports/liblzma.py
-@@ -0,0 +1,160 @@
+@@ -0,0 +1,151 @@
 +# Copyright 2020 The Emscripten Authors.  All rights reserved.
 +# Emscripten is available under two separate licenses, the MIT license and the
 +# University of Illinois/NCSA Open Source License.  Both these licenses can be
@@ -52,8 +51,8 @@ new file mode 100644
 +import logging
 +from pathlib import Path
 +
-+VERSION = '5.2.5'
++VERSION = '5.4.0'
-+HASH = '7443674247deda2935220fbc4dfc7665e5bb5a260be8ad858c8bd7d7b9f0f868f04ea45e62eb17c0a5e6a2de7c7500ad2d201e2d668c48ca29bd9eea5a73a3ce'
++HASH = '29b2cd25bb5b234b329ffe9547692d2c29be393db9d8d4ce70a66dfdaebd54433e79a89d80c57e58cd4559c3c68b9845507d5fedf3eec1c528a81e3d9ddbd811'
 +
 +
 +def needed(settings):
@@ -61,40 +60,31 @@ new file mode 100644
 +
 +
 +def get(ports, settings, shared):
-+  ports.fetch_project('liblzma', 'https://tukaani.org/xz/xz-' + VERSION + '.tar.gz', 'xz-' + VERSION, sha512hash=HASH)
++  ports.fetch_project('liblzma', f'https://tukaani.org/xz/xz-{VERSION}.tar.gz', sha512hash=HASH)
 +
 +  def create(final):
 +    logging.info('building port: liblzma')
 +
 +    ports.clear_project_build('liblzma')
 +
-+    source_path = os.path.join(ports.get_dir(), 'liblzma', 'xz-' + VERSION)
++    source_path = os.path.join(ports.get_dir(), 'liblzma', f'xz-{VERSION}', 'src', 'liblzma')
-+    dest_path = os.path.join(ports.get_build_dir(), 'liblzma')
++    ports.write_file(os.path.join(source_path, 'config.h'), config_h)
-+
++    ports.install_headers(os.path.join(source_path, 'api'), pattern='lzma.h')
-+    shared.try_delete(dest_path)
++    ports.install_headers(os.path.join(source_path, 'api', 'lzma'), pattern='*.h', target='lzma')
-+    os.makedirs(dest_path)
-+    shutil.rmtree(dest_path, ignore_errors=True)
-+    shutil.copytree(source_path, dest_path)
 +
 +    build_flags = ['-DHAVE_CONFIG_H', '-DTUKLIB_SYMBOL_PREFIX=lzma_', '-fvisibility=hidden']
-+    exclude_dirs = ['xzdec', 'xz', 'lzmainfo']
++    exclude_files = ['crc32_small.c', 'crc64_small.c', 'crc32_tablegen.c', 'crc64_tablegen.c', 'price_tablegen.c', 'fastpos_tablegen.c',
-+    exclude_files = ['crc32_small.c', 'crc64_small.c', 'crc32_tablegen.c', 'crc64_tablegen.c', 'price_tablegen.c', 'fastpos_tablegen.c'
 +                     'tuklib_exit.c', 'tuklib_mbstr_fw.c', 'tuklib_mbstr_width.c', 'tuklib_open_stdxxx.c', 'tuklib_progname.c']
-+    include_dirs_rel = ['../common', 'api', 'common', 'check', 'lz', 'rangecoder', 'lzma', 'delta', 'simple']
++    include_dirs_rel = ['../common', 'api', 'check', 'common', 'delta', 'lz', 'lzma', 'rangecoder', 'simple']
 +
-+    Path(dest_path, os.path.join('src', 'config.h')).write_text(config_h)
++    include_dirs = [os.path.join(source_path, p) for p in include_dirs_rel]
++    ports.build_port(source_path, final, 'liblzma', flags=build_flags, exclude_files=exclude_files, includes=include_dirs)
 +
-+    include_dirs = [os.path.join(dest_path, 'src', 'liblzma', p) for p in include_dirs_rel]
++  return [shared.cache.get_lib('liblzma.a', create, what='port')]
-+    ports.build_port(os.path.join(dest_path, 'src'), final, flags=build_flags, exclude_dirs=exclude_dirs, exclude_files=exclude_files, includes=include_dirs)
-+
-+    ports.install_headers(os.path.join(dest_path, 'src', 'liblzma', 'api'), 'lzma.h')
-+    ports.install_headers(os.path.join(dest_path, 'src', 'liblzma', 'api', 'lzma'), '*.h', 'lzma')
-+
-+  return [shared.Cache.get_lib('liblzma.a', create, what='port')]
 +
 +
 +def clear(ports, settings, shared):
-+  shared.Cache.erase_lib('liblzma.a')
++  shared.cache.erase_lib('liblzma.a')
 +
 +
 +def process_args(ports):
@@ -105,7 +95,7 @@ new file mode 100644
 +  return 'liblzma (USE_LIBLZMA=1; public domain)'
 +
 +
-+config_h = r'''
++config_h = '''
 +#define ASSUME_RAM 128
 +#define ENABLE_NLS 1
 +#define HAVE_CHECK_CRC32 1
@@ -177,9 +167,9 @@ new file mode 100644
 +#define PACKAGE "xz"
 +#define PACKAGE_BUGREPORT "lasse.collin@tukaani.org"
 +#define PACKAGE_NAME "XZ Utils"
-+#define PACKAGE_STRING "XZ Utils 5.2.5"
++#define PACKAGE_STRING "XZ Utils 5.4.0"
 +#define PACKAGE_TARNAME "xz"
-+#define PACKAGE_VERSION "5.2.5"
++#define PACKAGE_VERSION "5.4.0"
 +#define SIZEOF_SIZE_T 4
 +#define STDC_HEADERS 1
 +#define TUKLIB_CPUCORES_SYSCONF 1
@@ -200,5 +190,5 @@ new file mode 100644
 +#ifndef __EXTENSIONS__
 +# define __EXTENSIONS__ 1
 +#endif
-+#define VERSION "5.2.5"
++#define VERSION "5.4.0"
 +'''