From 3e6d8b395f63cee2b9eddc354939082490e1bee6 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 25 Jun 2019 12:41:28 -0700 Subject: [PATCH] Use prebuilt cache contents (#282) See WebAssembly/waterfall#542 The builds now contain lib/wasm-obj/ or lib/asmjs/ which have some cache contents. This places those in the cache so the user doesn't need to build them on first run, which for libc at least can be quite slow. The mechanism here is to run emcc a first time in the emsdk. That clears the cache (since the emsdk just updated the .emscripten file). We can then safely place the files in the cache. Note that FROZEN_CACHE is not used, since we do want to leave the user the option to build other things to the cache - we'll never ship all possible system and ports builds in the emsdk downloads, probably. This contains a test, which passes on tot-upstream. The last tagged release doesn't have this yet. --- emsdk | 45 ++++++++++++++++++++++++++++++++++++++++++++- emsdk_manifest.json | 6 ++++-- test.py | 8 ++++++++ 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/emsdk b/emsdk index 0623dfa..62c495a 100755 --- a/emsdk +++ b/emsdk @@ -2053,6 +2053,40 @@ def process_tool_list(tools_to_activate, log_errors=True): return tools_to_activate +def run_emcc(tools_to_activate): + for tool in tools_to_activate: + activated_path = getattr(tool, 'activated_path', None) + if activated_path and activated_path.endswith('/emscripten'): + activated_path = to_native_path(tool.expand_vars(tool.activated_path)) + emcc_path = os.path.join(activated_path, 'emcc.py') + if os.path.exists(emcc_path): + if VERBOSE: print('Calling emcc to initialize it') + subprocess.call([sys.executable, emcc_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + return + + +def emscripten_cache_directory(): + return os.path.join(emscripten_config_directory, ".emscripten_cache") + + +# Copy over any emscripten cache contents that were pregenerated. This avoids the user +# needing to immediately build libc etc. on first run. +def copy_pregenerated_cache(tools_to_activate): + for tool in tools_to_activate: + pregenerated_cache = getattr(tool, 'pregenerated_cache', None) + if pregenerated_cache: + # Finish the install of an emscripten-releases build. + install_path = to_native_path(sdk_path(tool.expand_vars(tool.install_path))) + in_cache = os.path.join(install_path, 'lib', pregenerated_cache) + if os.path.exists(in_cache): + out_cache = os.path.join(emscripten_cache_directory(), pregenerated_cache) + os.makedirs(out_cache) + for filename in os.listdir(in_cache): + if VERBOSE: print('Copying ' + filename + ' to cache dir') + shutil.copy2(os.path.join(in_cache, filename), + os.path.join(out_cache, filename)) + + # Reconfigure .emscripten to choose the currently activated toolset, set PATH and other environment variables. # Returns the full list of deduced tools that are now active. def set_active_tools(tools_to_activate, permanently_activate): @@ -2060,6 +2094,15 @@ def set_active_tools(tools_to_activate, permanently_activate): generate_dot_emscripten(tools_to_activate) + # Generating .emscripten will cause emcc to clear the cache on first run (emcc sees that the file has + # changed, since we write it here in the emsdk, and it never saw it before; so it clears the cache + # as it assumes a new config file means system libraries may need rebuilding). To avoid emcc's clearing + # wiping out the pregenerated cache contents we want to copy in, run emcc here, then copy the cache + # contents. + run_emcc(tools_to_activate) + + copy_pregenerated_cache(tools_to_activate) + # Construct a .bat script that will be invoked to set env. vars and PATH if WINDOWS: env_string = construct_env(tools_to_activate, False) @@ -2078,7 +2121,7 @@ def set_active_tools(tools_to_activate, permanently_activate): if len(tools_to_activate) > 0: tools = filter(lambda x: not x.is_sdk, tools_to_activate) - print('Set the following tools as active:\n ' + '\n '.join(map(lambda x: str(x), tools))) + print('\nSet the following tools as active:\n ' + '\n '.join(map(lambda x: str(x), tools))) print('') return tools_to_activate diff --git a/emsdk_manifest.json b/emsdk_manifest.json index 4886f3b..376415a 100644 --- a/emsdk_manifest.json +++ b/emsdk_manifest.json @@ -170,7 +170,8 @@ "install_path": "upstream", "activated_path": "%installation_dir%/emscripten", "activated_cfg": "LLVM_ROOT='%installation_dir%/bin';BINARYEN_ROOT='%installation_dir%'", - "emscripten_releases_hash": "%releases-tag%" + "emscripten_releases_hash": "%releases-tag%", + "pregenerated_cache": "wasm-obj" }, { "id": "releases", @@ -183,7 +184,8 @@ "install_path": "fastcomp", "activated_path": "%installation_dir%/emscripten", "activated_cfg": "LLVM_ROOT='%installation_dir%/fastcomp/bin';BINARYEN_ROOT='%installation_dir%'", - "emscripten_releases_hash": "%releases-tag%" + "emscripten_releases_hash": "%releases-tag%", + "pregenerated_cache": "asmjs" }, { diff --git a/test.py b/test.py index 88d9acc..672f964 100644 --- a/test.py +++ b/test.py @@ -34,6 +34,8 @@ open('hello_world.cpp', 'w').write('int main() {}') TAGS = json.loads(open('emscripten-releases-tags.txt').read()) +LIBC = os.path.expanduser('~/.emscripten_cache/wasm-obj/libc.a') + # Tests print('update') @@ -54,9 +56,15 @@ assert 'fastcomp' not in open(os.path.expanduser('~/.emscripten')).read() print('verify version') checked_call_with_output('upstream/emscripten/emcc -v', TAGS['latest'], stderr=subprocess.STDOUT) +print('clear cache') +check_call('upstream/emscripten/emcc --clear-cache') +assert not os.path.exists(LIBC) + print('test tot-upstream') check_call('./emsdk install tot-upstream') +assert not os.path.exists(LIBC) check_call('./emsdk activate tot-upstream') +assert os.path.exists(LIBC), 'activation supplies prebuilt libc' # TODO; test on latest as well check_call('upstream/emscripten/emcc hello_world.cpp') print('test tot-fastcomp')