From efd71e9a2e9eacf4999a40d542f8b846ae81612e Mon Sep 17 00:00:00 2001 From: juj Date: Wed, 27 Aug 2025 16:24:20 +0300 Subject: [PATCH] Nightly Node.js and Big Endian cross compilation Node.js (#1590) This PR adds support for `./emsdk install node-nightly-64bit` which finds the latest nightly node.js version and installs that. It is a moving target, so the next day when a new nightly is produced, reissuing `./emsdk install node-nightly-64bit` will install the newer published version. Also, this PR adds a fixed 22.18.0 version for the cross compilation s390x Node.js target as well. On an x64 Linux system, this allows running `./emsdk install node-big-endian-crosscompile-22.16.0-64bit` to install the big endian Node.js. This greatly simplifies the steps at https://github.com/emscripten-core/emscripten/compare/main...juj:emscripten:bigendian_test_suite#diff-c36b90121be240017fa490a1c00e63e47fa3235f5c1be0593e2b7502d017c778R9985-R10000 and enables a trivial way to switch between LE and BE Node.js versions for testing. CC @slavek-kucera --- emsdk.py | 100 ++++++++++++++++++++++++++++++++++++++++---- emsdk_manifest.json | 23 ++++++++++ 2 files changed, 114 insertions(+), 9 deletions(-) diff --git a/emsdk.py b/emsdk.py index 1a553d4..3ac1ab5 100644 --- a/emsdk.py +++ b/emsdk.py @@ -1477,6 +1477,77 @@ def find_emscripten_root(active_tools): return root +def fetch_nightly_node_versions(): + url = "https://nodejs.org/download/nightly/" + with urlopen(url) as response: + html = response.read().decode("utf-8") + + # Regex to capture href values like v7.0.0-nightly2016080175c6d9dd95/ + pattern = re.compile(r'') + matches = pattern.findall(html) + return matches + + +def dir_installed_nightly_node_versions(): + path = os.path.abspath('node') + return [name for name in os.listdir(path) if os.path.isdir(os.path.join(path, name)) and name.startswith("nightly-")] + + +def extract_newest_node_nightly_version(versions): + def parse(v): + # example: v7.0.0-nightly2016080175c6d9dd95 + m = re.match(r'v(\d+)\.(\d+)\.(\d+)-nightly(\d+)', v) + if m: + major, minor, patch, nightly = m.groups() + return [int(major), int(minor), int(patch), int(nightly)] + else: + return [] + + try: + return max(versions, key=lambda v: parse(v)) + except Exception: + return None + + +def download_node_nightly(tool): + nightly_versions = fetch_nightly_node_versions() + latest_nightly = extract_newest_node_nightly_version(nightly_versions) + print('Latest Node.js Nightly download available is "' + latest_nightly + '"') + + output_dir = os.path.abspath('node/nightly-' + latest_nightly) + # Node.js zip structure quirk: Linux and macOS archives have a /bin, + # Windows does not. Unify the file structures. + if WINDOWS: + output_dir += '/bin' + + if os.path.isdir(output_dir): + return True + + url = tool.url.replace('%version%', latest_nightly) + if WINDOWS: + os_ = 'win' + elif LINUX: + os_ = 'linux' + elif MACOS: + os_ = 'darwin' + else: + os_ = '' + if platform.machine().lower() in ["x86_64", "amd64"]: + arch = 'x64' + elif platform.machine().lower() in ["arm64", "aarch64"]: + arch = 'arm64' + if WINDOWS: + zip_suffix = 'zip' + else: + zip_suffix = 'tar.gz' + url = url.replace('%os%', os_) + url = url.replace('%arch%', arch) + url = url.replace('%zip_suffix%', zip_suffix) + download_and_extract(url, output_dir) + open(tool.get_version_file_path(), 'w').write('node-nightly-64bit') + return True + + # returns a tuple (string,string) of config files paths that need to used # to activate emsdk env depending on $SHELL, defaults to bash. def get_emsdk_shell_env_configs(): @@ -1511,7 +1582,10 @@ def generate_em_config(active_tools, permanently_activate, system): activated_config['NODE_JS'] = node_fallback for name, value in activated_config.items(): - cfg += name + " = '" + value + "'\n" + if value.startswith('['): + cfg += name + " = " + value + "\n" + else: + cfg += name + " = '" + value + "'\n" emroot = find_emscripten_root(active_tools) if emroot: @@ -1606,6 +1680,11 @@ class Tool(object): str = str.replace('%.exe%', '.exe' if WINDOWS else '') if '%llvm_build_bin_dir%' in str: str = str.replace('%llvm_build_bin_dir%', llvm_build_bin_dir(self)) + if '%latest_downloaded_node_nightly_dir%' in str: + installed_node_nightlys = dir_installed_nightly_node_versions() + latest_node_nightly = extract_newest_node_nightly_version(installed_node_nightlys) + if latest_node_nightly: + str = str.replace('%latest_downloaded_node_nightly_dir%', latest_node_nightly) return str @@ -1727,10 +1806,11 @@ class Tool(object): return False if self.download_url() is None: - # This tool does not contain downloadable elements, so it is installed by default. + debug_print(str(self) + ' has no files to download, so is installed by default.') return True content_exists = is_nonempty_directory(self.installation_path()) + debug_print(str(self) + ' installation path is ' + self.installation_path() + ', exists: ' + str(content_exists) + '.') # For e.g. fastcomp clang from git repo, the activated PATH is the # directory where the compiler is built to, and installation_path is @@ -1876,12 +1956,14 @@ class Tool(object): print("Installing tool '" + str(self) + "'..") url = self.download_url() - if hasattr(self, 'custom_install_script') and self.custom_install_script == 'build_llvm': - success = build_llvm(self) - elif hasattr(self, 'custom_install_script') and self.custom_install_script == 'build_ninja': - success = build_ninja(self) - elif hasattr(self, 'custom_install_script') and self.custom_install_script == 'build_ccache': - success = build_ccache(self) + custom_install_scripts = { + 'build_llvm': build_llvm, + 'build_ninja': build_ninja, + 'build_ccache': build_ccache, + 'download_node_nightly': download_node_nightly + } + if hasattr(self, 'custom_install_script') and self.custom_install_script in custom_install_scripts: + success = custom_install_scripts[self.custom_install_script](self) elif hasattr(self, 'git_branch'): success = git_clone_checkout_and_pull(url, self.installation_path(), self.git_branch, getattr(self, 'remote_name', 'origin')) elif url.endswith(ARCHIVE_SUFFIXES): @@ -1896,7 +1978,7 @@ class Tool(object): if hasattr(self, 'custom_install_script'): if self.custom_install_script == 'emscripten_npm_install': success = emscripten_npm_install(self, self.installation_path()) - elif self.custom_install_script in ('build_llvm', 'build_ninja', 'build_ccache'): + elif self.custom_install_script in ('build_llvm', 'build_ninja', 'build_ccache', 'download_node_nightly'): # 'build_llvm' is a special one that does the download on its # own, others do the download manually. pass diff --git a/emsdk_manifest.json b/emsdk_manifest.json index 1d05304..3b2c21b 100644 --- a/emsdk_manifest.json +++ b/emsdk_manifest.json @@ -204,6 +204,29 @@ "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%" }, + { + "id": "node-big-endian-crosscompile", + "version": "22.16.0", + "arch": "x86_64", + "bitness": 64, + "linux_url": "https://nodejs.org/download/release/v22.16.0/node-v22.16.0-linux-s390x.tar.gz", + "activated_path": "%installation_dir%/bin", + "activated_path_skip": "node", + "activated_cfg": "NODE_JS_TEST=['qemu-s390x', '-L', '/usr/s390x-linux-gnu/', '%installation_dir%/bin/node']" + }, + { + "id": "node", + "version": "nightly", + "bitness": 64, + "install_path": "node/%latest_downloaded_node_nightly_dir%", + "custom_install_script": "download_node_nightly", + "activated_path": "node/%latest_downloaded_node_nightly_dir%/bin", + "activated_path_skip": "node", + "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'", + "activated_env": "EMSDK_NODE=%installation_dir%/bin/node%.exe%", + "url": "https://nodejs.org/download/nightly/%version%/node-%version%-%os%-%arch%.%zip_suffix%", + "git_branch": "nightly" + }, {