From 04a3bf76e840cdf4b1dc79479b62b4d626c4bee1 Mon Sep 17 00:00:00 2001 From: Patric Stout Date: Thu, 18 Apr 2024 19:37:33 +0200 Subject: [PATCH] Codechange: upgrade Emscripten to 3.1.57 (#12526) This also upgrades liblzma to 5.4.6, and uses the new ports.contrib system Emscripten 3.1.56 introduced. --- .github/workflows/ci-emscripten.yml | 9 +- .github/workflows/preview-build.yml | 9 +- os/emscripten/Dockerfile | 5 +- os/emscripten/README.md | 4 +- os/emscripten/cmake/FindLibLZMA.cmake | 8 +- os/emscripten/emsdk-liblzma.patch | 198 -------------------------- os/emscripten/ports/liblzma.py | 139 ++++++++++++++++++ 7 files changed, 155 insertions(+), 217 deletions(-) delete mode 100644 os/emscripten/emsdk-liblzma.patch create mode 100644 os/emscripten/ports/liblzma.py diff --git a/.github/workflows/ci-emscripten.yml b/.github/workflows/ci-emscripten.yml index e1814ed79b..8c50713b9e 100644 --- a/.github/workflows/ci-emscripten.yml +++ b/.github/workflows/ci-emscripten.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-20.04 container: # If you change this version, change the number in the cache step too. - image: emscripten/emsdk:3.1.42 + image: emscripten/emsdk:3.1.57 steps: - name: Checkout @@ -23,12 +23,11 @@ jobs: uses: actions/cache@v4 with: path: /emsdk/upstream/emscripten/cache - key: 3.1.42-${{ runner.os }} + key: 3.1.57-${{ runner.os }} - - name: Patch Emscripten to support LZMA + - name: Add liblzma support run: | - cd /emsdk/upstream/emscripten - patch -p1 < ${GITHUB_WORKSPACE}/os/emscripten/emsdk-liblzma.patch + cp ${GITHUB_WORKSPACE}/os/emscripten/ports/liblzma.py /emsdk/upstream/emscripten/tools/ports/contrib/ - name: Build (host tools) run: | diff --git a/.github/workflows/preview-build.yml b/.github/workflows/preview-build.yml index 0bb79ed4b9..9cf6e9573c 100644 --- a/.github/workflows/preview-build.yml +++ b/.github/workflows/preview-build.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-latest container: # If you change this version, change the number in the cache step too. - image: emscripten/emsdk:3.1.42 + image: emscripten/emsdk:3.1.57 steps: - name: Checkout @@ -38,12 +38,11 @@ jobs: uses: actions/cache@v4 with: path: /emsdk/upstream/emscripten/cache - key: 3.1.42-${{ runner.os }} + key: 3.1.57-${{ runner.os }} - - name: Patch Emscripten to support LZMA + - name: Add liblzma support run: | - cd /emsdk/upstream/emscripten - patch -p1 < ${GITHUB_WORKSPACE}/os/emscripten/emsdk-liblzma.patch + cp ${GITHUB_WORKSPACE}/os/emscripten/ports/liblzma.py /emsdk/upstream/emscripten/tools/ports/contrib/ - name: Build (host tools) run: | diff --git a/os/emscripten/Dockerfile b/os/emscripten/Dockerfile index 178f56500b..543bf6b2de 100644 --- a/os/emscripten/Dockerfile +++ b/os/emscripten/Dockerfile @@ -1,4 +1,3 @@ -FROM emscripten/emsdk:3.1.42 +FROM emscripten/emsdk:3.1.57 -COPY emsdk-liblzma.patch / -RUN cd /emsdk/upstream/emscripten && patch -p1 < /emsdk-liblzma.patch +COPY ports/liblzma.py /emsdk/upstream/emscripten/tools/ports/contrib/liblzma.py diff --git a/os/emscripten/README.md b/os/emscripten/README.md index 9184b0144e..cdc038ade3 100644 --- a/os/emscripten/README.md +++ b/os/emscripten/README.md @@ -2,8 +2,8 @@ Please use docker with the supplied `Dockerfile` to build for emscripten. It takes care of a few things: -- Use a version of emscripten we know works -- Patch in LibLZMA support (as this is not supported by upstream) +- Use a version of emscripten we know works. +- Add LibLZMA library under contrib ports. First, build the docker image by navigating in the folder this `README.md` is in, and executing: ``` diff --git a/os/emscripten/cmake/FindLibLZMA.cmake b/os/emscripten/cmake/FindLibLZMA.cmake index cd6b44ad3d..0880d62f70 100644 --- a/os/emscripten/cmake/FindLibLZMA.cmake +++ b/os/emscripten/cmake/FindLibLZMA.cmake @@ -1,7 +1,7 @@ # LibLZMA is a custom addition to the emscripten SDK, so it is possible # someone patched their SDK. Test out if the SDK supports LibLZMA. include(CheckCXXSourceCompiles) -set(CMAKE_REQUIRED_FLAGS "-sUSE_LIBLZMA=1") +set(CMAKE_REQUIRED_FLAGS "--use-port=contrib.liblzma") check_cxx_source_compiles(" #include @@ -12,9 +12,9 @@ check_cxx_source_compiles(" if (LIBLZMA_FOUND) add_library(LibLZMA::LibLZMA INTERFACE IMPORTED) set_target_properties(LibLZMA::LibLZMA PROPERTIES - INTERFACE_COMPILE_OPTIONS "-sUSE_LIBLZMA=1" - INTERFACE_LINK_LIBRARIES "-sUSE_LIBLZMA=1" + INTERFACE_COMPILE_OPTIONS "--use-port=contrib.liblzma" + INTERFACE_LINK_LIBRARIES "--use-port=contrib.liblzma" ) else() - message(WARNING "You are using an emscripten SDK without LibLZMA support. Many savegames won't be able to load in OpenTTD. Please apply 'emsdk-liblzma.patch' to your local emsdk installation.") + message(WARNING "You are using an emscripten SDK without LibLZMA support. Many savegames won't be able to load in OpenTTD. Please copy liblzma.py to your ports/contrib folder in your local emsdk installation.") endif() diff --git a/os/emscripten/emsdk-liblzma.patch b/os/emscripten/emsdk-liblzma.patch deleted file mode 100644 index 7bfdd47de2..0000000000 --- a/os/emscripten/emsdk-liblzma.patch +++ /dev/null @@ -1,198 +0,0 @@ -From 84d0e9112d5c87a714abd21ec8547921f46f37b5 Mon Sep 17 00:00:00 2001 -From: milek7 -Date: Tue, 8 Dec 2020 01:03:31 +0100 -Subject: [PATCH] Add liblzma port - ---- - src/settings.js | 4 ++ - tools/ports/liblzma.py | 151 +++++++++++++++++++++++++++++++++++++++++ - tools/settings.py | 1 + - 3 files changed, 156 insertions(+) - create mode 100644 tools/ports/liblzma.py - -diff --git a/src/settings.js b/src/settings.js -index f93140d..7b6bec9 100644 ---- a/src/settings.js -+++ b/src/settings.js -@@ -1451,6 +1451,10 @@ var USE_GIFLIB = false; - // [compile+link] - var USE_LIBJPEG = false; - -+// 1 = use liblzma from emscripten-ports -+// [compile+link] -+var USE_LIBLZMA = false; -+ - // 1 = use libpng from emscripten-ports - // [compile+link] - var USE_LIBPNG = false; -diff --git a/tools/ports/liblzma.py b/tools/ports/liblzma.py -new file mode 100644 -index 0000000..6872a8b ---- /dev/null -+++ b/tools/ports/liblzma.py -@@ -0,0 +1,151 @@ -+# Copyright 2020 The Emscripten Authors. All rights reserved. -+# Emscripten is available under two separate licenses, the MIT license and the -+# University of Illinois/NCSA Open Source License. Both these licenses can be -+# found in the LICENSE file. -+ -+import os -+import shutil -+import logging -+from pathlib import Path -+ -+VERSION = '5.4.2' -+HASH = '149f980338bea3d66de1ff5994b2b236ae1773135eda68b62b009df0c9dcdf5467f8cb2c06da95a71b6556d60bd3d21f475feced34d5dfdb80ee95416a2f9737' -+ -+ -+def needed(settings): -+ return settings.USE_LIBLZMA -+ -+ -+def get(ports, settings, shared): -+ ports.fetch_project('liblzma', f'https://tukaani.org/xz/xz-{VERSION}.tar.gz', sha512hash=HASH) -+ -+ def create(final): -+ logging.info('building port: liblzma') -+ -+ ports.clear_project_build('liblzma') -+ -+ source_path = os.path.join(ports.get_dir(), 'liblzma', f'xz-{VERSION}', 'src', 'liblzma') -+ ports.write_file(os.path.join(source_path, 'config.h'), config_h) -+ ports.install_headers(os.path.join(source_path, 'api'), pattern='lzma.h') -+ ports.install_headers(os.path.join(source_path, 'api', 'lzma'), pattern='*.h', target='lzma') -+ -+ build_flags = ['-DHAVE_CONFIG_H', '-DTUKLIB_SYMBOL_PREFIX=lzma_', '-fvisibility=hidden'] -+ exclude_files = ['crc32_small.c', 'crc64_small.c', 'crc32_tablegen.c', 'crc64_tablegen.c', 'price_tablegen.c', 'fastpos_tablegen.c', -+ 'tuklib_exit.c', 'tuklib_mbstr_fw.c', 'tuklib_mbstr_width.c', 'tuklib_open_stdxxx.c', 'tuklib_progname.c'] -+ include_dirs_rel = ['../common', 'api', 'check', 'common', 'delta', 'lz', 'lzma', 'rangecoder', 'simple'] -+ -+ include_dirs = [os.path.join(source_path, p) for p in include_dirs_rel] -+ ports.build_port(source_path, final, 'liblzma', flags=build_flags, exclude_files=exclude_files, includes=include_dirs) -+ -+ return [shared.cache.get_lib('liblzma.a', create, what='port')] -+ -+ -+def clear(ports, settings, shared): -+ shared.cache.erase_lib('liblzma.a') -+ -+ -+def process_args(ports): -+ return [] -+ -+ -+def show(): -+ return 'liblzma (USE_LIBLZMA=1; public domain)' -+ -+ -+config_h = ''' -+#define ASSUME_RAM 128 -+#define ENABLE_NLS 1 -+#define HAVE_CHECK_CRC32 1 -+#define HAVE_CHECK_CRC64 1 -+#define HAVE_CHECK_SHA256 1 -+#define HAVE_CLOCK_GETTIME 1 -+#define HAVE_DCGETTEXT 1 -+#define HAVE_DECL_CLOCK_MONOTONIC 1 -+#define HAVE_DECL_PROGRAM_INVOCATION_NAME 1 -+#define HAVE_DECODERS 1 -+#define HAVE_DECODER_ARM 1 -+#define HAVE_DECODER_ARMTHUMB 1 -+#define HAVE_DECODER_DELTA 1 -+#define HAVE_DECODER_IA64 1 -+#define HAVE_DECODER_LZMA1 1 -+#define HAVE_DECODER_LZMA2 1 -+#define HAVE_DECODER_POWERPC 1 -+#define HAVE_DECODER_SPARC 1 -+#define HAVE_DECODER_X86 1 -+#define HAVE_DLFCN_H 1 -+#define HAVE_ENCODERS 1 -+#define HAVE_ENCODER_ARM 1 -+#define HAVE_ENCODER_ARMTHUMB 1 -+#define HAVE_ENCODER_DELTA 1 -+#define HAVE_ENCODER_IA64 1 -+#define HAVE_ENCODER_LZMA1 1 -+#define HAVE_ENCODER_LZMA2 1 -+#define HAVE_ENCODER_POWERPC 1 -+#define HAVE_ENCODER_SPARC 1 -+#define HAVE_ENCODER_X86 1 -+#define HAVE_FCNTL_H 1 -+#define HAVE_FUTIMENS 1 -+#define HAVE_GETOPT_H 1 -+#define HAVE_GETOPT_LONG 1 -+#define HAVE_GETTEXT 1 -+#define HAVE_IMMINTRIN_H 1 -+#define HAVE_INTTYPES_H 1 -+#define HAVE_LIMITS_H 1 -+#define HAVE_MBRTOWC 1 -+#define HAVE_MEMORY_H 1 -+#define HAVE_MF_BT2 1 -+#define HAVE_MF_BT3 1 -+#define HAVE_MF_BT4 1 -+#define HAVE_MF_HC3 1 -+#define HAVE_MF_HC4 1 -+#define HAVE_OPTRESET 1 -+#define HAVE_POSIX_FADVISE 1 -+#define HAVE_PTHREAD_CONDATTR_SETCLOCK 1 -+#define HAVE_PTHREAD_PRIO_INHERIT 1 -+#define HAVE_STDBOOL_H 1 -+#define HAVE_STDINT_H 1 -+#define HAVE_STDLIB_H 1 -+#define HAVE_STRINGS_H 1 -+#define HAVE_STRING_H 1 -+#define HAVE_STRUCT_STAT_ST_ATIM_TV_NSEC 1 -+#define HAVE_SYS_PARAM_H 1 -+#define HAVE_SYS_STAT_H 1 -+#define HAVE_SYS_TIME_H 1 -+#define HAVE_SYS_TYPES_H 1 -+#define HAVE_UINTPTR_T 1 -+#define HAVE_UNISTD_H 1 -+#define HAVE_VISIBILITY 1 -+#define HAVE_WCWIDTH 1 -+#define HAVE__BOOL 1 -+#define HAVE___BUILTIN_ASSUME_ALIGNED 1 -+#define HAVE___BUILTIN_BSWAPXX 1 -+#define MYTHREAD_POSIX 1 -+#define NDEBUG 1 -+#define PACKAGE "xz" -+#define PACKAGE_BUGREPORT "lasse.collin@tukaani.org" -+#define PACKAGE_NAME "XZ Utils" -+#define PACKAGE_STRING "XZ Utils 5.4.0" -+#define PACKAGE_TARNAME "xz" -+#define PACKAGE_VERSION "5.4.0" -+#define SIZEOF_SIZE_T 4 -+#define STDC_HEADERS 1 -+#define TUKLIB_CPUCORES_SYSCONF 1 -+#define TUKLIB_FAST_UNALIGNED_ACCESS 1 -+#define TUKLIB_PHYSMEM_SYSCONF 1 -+#ifndef _ALL_SOURCE -+# define _ALL_SOURCE 1 -+#endif -+#ifndef _GNU_SOURCE -+# define _GNU_SOURCE 1 -+#endif -+#ifndef _POSIX_PTHREAD_SEMANTICS -+# define _POSIX_PTHREAD_SEMANTICS 1 -+#endif -+#ifndef _TANDEM_SOURCE -+# define _TANDEM_SOURCE 1 -+#endif -+#ifndef __EXTENSIONS__ -+# define __EXTENSIONS__ 1 -+#endif -+#define VERSION "5.4.0" -+''' -diff --git a/tools/settings.py b/tools/settings.py -index 10d6ca0..827e4a9 100644 ---- a/tools/settings.py -+++ b/tools/settings.py -@@ -40,6 +40,7 @@ PORTS_SETTINGS = { - 'USE_SDL_NET', - 'USE_SDL_GFX', - 'USE_LIBJPEG', -+ 'USE_LIBLZMA', - 'USE_OGG', - 'USE_REGAL', - 'USE_BOOST_HEADERS', --- -2.34.1 diff --git a/os/emscripten/ports/liblzma.py b/os/emscripten/ports/liblzma.py new file mode 100644 index 0000000000..0adf0e5697 --- /dev/null +++ b/os/emscripten/ports/liblzma.py @@ -0,0 +1,139 @@ +import os +import logging + +VERSION = '5.4.6' +HASH = '495cc890d25c075c927c907b77e60d86dd8a4c377cea5b1172c8e916984149a7bb5fb32db25091f7219346b83155b47e4bc0404cc8529d992014cd7ed0c278b7' + +URL = 'https://github.com/tukaani-project/xz' +DESCRIPTION = 'liblzma provides a general-purpose data-compression library.' +LICENSE = 'LGPL-2.1' + +def get(ports, settings, shared): + ports.fetch_project('contrib.liblzma', f'https://github.com/tukaani-project/xz/releases/download/v{VERSION}/xz-{VERSION}.tar.xz', sha512hash=HASH) + + def create(final): + logging.info('building port: contrib.liblzma') + + ports.clear_project_build('contrib.liblzma') + + source_path = os.path.join(ports.get_dir(), 'contrib.liblzma', f'xz-{VERSION}', 'src', 'liblzma') + ports.write_file(os.path.join(source_path, 'config.h'), config_h) + ports.install_headers(os.path.join(source_path, 'api'), pattern='lzma.h') + ports.install_headers(os.path.join(source_path, 'api', 'lzma'), pattern='*.h', target='lzma') + + build_flags = ['-DHAVE_CONFIG_H', '-DTUKLIB_SYMBOL_PREFIX=lzma_', '-fvisibility=hidden'] + exclude_files = ['crc32_small.c', 'crc64_small.c', 'crc32_tablegen.c', 'crc64_tablegen.c', 'price_tablegen.c', 'fastpos_tablegen.c', + 'tuklib_exit.c', 'tuklib_mbstr_fw.c', 'tuklib_mbstr_width.c', 'tuklib_open_stdxxx.c', 'tuklib_progname.c'] + include_dirs_rel = ['../common', 'api', 'check', 'common', 'delta', 'lz', 'lzma', 'rangecoder', 'simple'] + + include_dirs = [os.path.join(source_path, p) for p in include_dirs_rel] + ports.build_port(source_path, final, 'contrib.liblzma', flags=build_flags, exclude_files=exclude_files, includes=include_dirs) + + return [shared.cache.get_lib('liblzma.a', create, what='port')] + + +def clear(ports, settings, shared): + shared.cache.erase_lib('liblzma.a') + + +def process_args(ports): + return [] + + +config_h = ''' +#define ASSUME_RAM 128 +#define ENABLE_NLS 1 +#define HAVE_CHECK_CRC32 1 +#define HAVE_CHECK_CRC64 1 +#define HAVE_CHECK_SHA256 1 +#define HAVE_CLOCK_GETTIME 1 +#define HAVE_DCGETTEXT 1 +#define HAVE_DECL_CLOCK_MONOTONIC 1 +#define HAVE_DECL_PROGRAM_INVOCATION_NAME 1 +#define HAVE_DECODERS 1 +#define HAVE_DECODER_ARM 1 +#define HAVE_DECODER_ARMTHUMB 1 +#define HAVE_DECODER_DELTA 1 +#define HAVE_DECODER_IA64 1 +#define HAVE_DECODER_LZMA1 1 +#define HAVE_DECODER_LZMA2 1 +#define HAVE_DECODER_POWERPC 1 +#define HAVE_DECODER_SPARC 1 +#define HAVE_DECODER_X86 1 +#define HAVE_DLFCN_H 1 +#define HAVE_ENCODERS 1 +#define HAVE_ENCODER_ARM 1 +#define HAVE_ENCODER_ARMTHUMB 1 +#define HAVE_ENCODER_DELTA 1 +#define HAVE_ENCODER_IA64 1 +#define HAVE_ENCODER_LZMA1 1 +#define HAVE_ENCODER_LZMA2 1 +#define HAVE_ENCODER_POWERPC 1 +#define HAVE_ENCODER_SPARC 1 +#define HAVE_ENCODER_X86 1 +#define HAVE_FCNTL_H 1 +#define HAVE_FUTIMENS 1 +#define HAVE_GETOPT_H 1 +#define HAVE_GETOPT_LONG 1 +#define HAVE_GETTEXT 1 +#define HAVE_IMMINTRIN_H 1 +#define HAVE_INTTYPES_H 1 +#define HAVE_LIMITS_H 1 +#define HAVE_MBRTOWC 1 +#define HAVE_MEMORY_H 1 +#define HAVE_MF_BT2 1 +#define HAVE_MF_BT3 1 +#define HAVE_MF_BT4 1 +#define HAVE_MF_HC3 1 +#define HAVE_MF_HC4 1 +#define HAVE_OPTRESET 1 +#define HAVE_POSIX_FADVISE 1 +#define HAVE_PTHREAD_CONDATTR_SETCLOCK 1 +#define HAVE_PTHREAD_PRIO_INHERIT 1 +#define HAVE_STDBOOL_H 1 +#define HAVE_STDINT_H 1 +#define HAVE_STDLIB_H 1 +#define HAVE_STRINGS_H 1 +#define HAVE_STRING_H 1 +#define HAVE_STRUCT_STAT_ST_ATIM_TV_NSEC 1 +#define HAVE_SYS_PARAM_H 1 +#define HAVE_SYS_STAT_H 1 +#define HAVE_SYS_TIME_H 1 +#define HAVE_SYS_TYPES_H 1 +#define HAVE_UINTPTR_T 1 +#define HAVE_UNISTD_H 1 +#define HAVE_VISIBILITY 1 +#define HAVE_WCWIDTH 1 +#define HAVE__BOOL 1 +#define HAVE___BUILTIN_ASSUME_ALIGNED 1 +#define HAVE___BUILTIN_BSWAPXX 1 +#define MYTHREAD_POSIX 1 +#define NDEBUG 1 +#define PACKAGE "xz" +#define PACKAGE_BUGREPORT "lasse.collin@tukaani.org" +#define PACKAGE_NAME "XZ Utils" +#define PACKAGE_STRING "XZ Utils 5.4.0" +#define PACKAGE_TARNAME "xz" +#define PACKAGE_VERSION "5.4.0" +#define SIZEOF_SIZE_T 4 +#define STDC_HEADERS 1 +#define TUKLIB_CPUCORES_SYSCONF 1 +#define TUKLIB_FAST_UNALIGNED_ACCESS 1 +#define TUKLIB_PHYSMEM_SYSCONF 1 +#ifndef _ALL_SOURCE +# define _ALL_SOURCE 1 +#endif +#ifndef _GNU_SOURCE +# define _GNU_SOURCE 1 +#endif +#ifndef _POSIX_PTHREAD_SEMANTICS +# define _POSIX_PTHREAD_SEMANTICS 1 +#endif +#ifndef _TANDEM_SOURCE +# define _TANDEM_SOURCE 1 +#endif +#ifndef __EXTENSIONS__ +# define __EXTENSIONS__ 1 +#endif +#define VERSION "5.4.0" +'''