[Bazel] Support generating a secondary cache (#1405)

This is a working solution for generating a separate Emscripten cache.
Note that this requires an additional entry in the workspace as follows:
```starlark
load("@emsdk//:emscripten_cache.bzl", emsdk_emscripten_cache = "emscripten_cache")
emsdk_emscripten_cache()
```
When used like this, the default Emscripten cache will be used. However,
if the entry is as follows:
```starlark
load("@emsdk//:emscripten_cache.bzl", emsdk_emscripten_cache = "emscripten_cache")
emsdk_emscripten_cache(flags = ["--lto"])
```
Then embuilder will be called to build all system libraries and ports
(i.e., the `ALL` option to embuilder) with the LTO option enabled. This
can take awhile, so I have also made possible to specify which libraries
you want to build explicitly:
```starlark
load("@emsdk//:emscripten_cache.bzl", emsdk_emscripten_cache = "emscripten_cache")
emsdk_emscripten_cache(
    flags = ["--lto"],
    libraries = [
        "crtbegin",
        "libprintf_long_double-debug",
        "libstubs-debug",
        "libnoexit",
        "libc-debug",
        "libdlmalloc",
        "libcompiler_rt",
        "libc++-noexcept",
        "libc++abi-debug-noexcept",
        "libsockets"
    ]
)
```

Resolves #807, resolves #971, resolves #1099, resolves #1362, resolves
#1401

bazel/README.md

@@ -66,4 +66,30 @@ rules.
 and all of its dependencies, and does not require amending `.bazelrc`. This
 is the preferred way, since it also unpacks the resulting tarball.
 
+The Emscripten cache shipped by default does not include LTO, 64-bit or PIC
+builds of the system libraries and ports. If you wish to use these features you
+will need to declare the cache when you register the toolchain as follows. Note
+that the configuration consists of the same flags that can be passed to
+embuilder. If `targets` is not provided, all system libraries and ports will be
+built, i.e., the `ALL` option to embuilder.
+
+```starlark
+load("@emsdk//:toolchains.bzl", "register_emscripten_toolchains")
+register_emscripten_toolchains(cache = {
+    "configuration": ["--lto"],
+    "targets": [
+        "crtbegin",
+        "libprintf_long_double-debug",
+        "libstubs-debug",
+        "libnoexit",
+        "libc-debug",
+        "libdlmalloc",
+        "libcompiler_rt",
+        "libc++-noexcept",
+        "libc++abi-debug-noexcept",
+        "libsockets"
+    ]
+})
+```
+
 See `test_external/` for an example using [embind](https://emscripten.org/docs/porting/connecting_cpp_and_javascript/embind.html).

bazel/emscripten_deps.bzl

@@ -27,6 +27,7 @@ filegroup(
     name = "emcc_common",
     srcs = [
         "emscripten/emcc.py",
+        "emscripten/embuilder.py",
         "emscripten/emscripten-version.txt",
         "emscripten/cache/sysroot_install.stamp",
         "emscripten/src/settings.js",

bazel/emscripten_toolchain/BUILD.bazel

@@ -5,7 +5,7 @@ package(default_visibility = ["//visibility:public"])
 filegroup(
     name = "common_files",
     srcs = [
-        "emscripten_config",
+        "@emscripten_cache//:emscripten_config",
         "env.sh",
         "env.bat",
         "@nodejs//:node_files",
@@ -60,7 +60,7 @@ cc_library(name = "malloc")
 emscripten_cc_toolchain_config_rule(
     name = "wasm",
     cpu = "wasm",
-    em_config = "emscripten_config",
+    em_config = "@emscripten_cache//:emscripten_config",
     emscripten_binaries = "@emsdk//:compiler_files",
     script_extension = select({
         "@bazel_tools//src/conditions:host_windows": "bat",

bazel/test_secondary_lto_cache/.bazelrc

@@ -0,0 +1 @@
+build --incompatible_enable_cc_toolchain_resolution

bazel/test_secondary_lto_cache/.gitignore

@@ -0,0 +1,4 @@
+bazel-bin
+bazel-out
+bazel-test_secondary_lto_cache
+bazel-testlogs

bazel/test_secondary_lto_cache/BUILD

@@ -0,0 +1,19 @@
+load("@emsdk//emscripten_toolchain:wasm_rules.bzl", "wasm_cc_binary")
+
+cc_binary(
+    name = "hello-world",
+    srcs = ["hello-world.cc"],
+    linkopts = [
+        "-sAUTO_NATIVE_LIBRARIES=0",
+        "-flto",
+    ],
+)
+
+wasm_cc_binary(
+    name = "hello-world-wasm",
+    cc_target = ":hello-world",
+    outputs = [
+        "hello-world.js",
+        "hello-world.wasm",
+    ],
+)

bazel/test_secondary_lto_cache/MODULE.bazel

@@ -0,0 +1 @@
+bazel_dep(name = "platforms", version = "0.0.9")

bazel/test_secondary_lto_cache/WORKSPACE

@@ -0,0 +1,30 @@
+local_repository(
+    name = "emsdk",
+    path = "..",
+)
+
+load("@emsdk//:deps.bzl", "deps")
+
+deps()
+
+load("@emsdk//:emscripten_deps.bzl", "emscripten_deps")
+
+emscripten_deps()
+
+load("@emsdk//:toolchains.bzl", "register_emscripten_toolchains")
+
+register_emscripten_toolchains(cache = {
+    "configuration": ["--lto"],
+    "targets": [
+        "crtbegin",
+        "libprintf_long_double-debug",
+        "libstubs-debug",
+        "libnoexit",
+        "libc-debug",
+        "libdlmalloc",
+        "libcompiler_rt",
+        "libc++-noexcept",
+        "libc++abi-debug-noexcept",
+        "libsockets"
+    ]
+})

bazel/test_secondary_lto_cache/hello-world.cc

@@ -0,0 +1,6 @@
+#include <iostream>
+
+int main(int argc, char** argv) {
+  std::cout << "hello world!" << std::endl;
+  return 0;
+}

bazel/toolchains.bzl

@@ -1,2 +1,96 @@
-def register_emscripten_toolchains():
+BUILD_FILE_CONTENT_TEMPLATE = """
+package(default_visibility = ['//visibility:public'])
+exports_files(['emscripten_config'])
+"""
+
+EMBUILDER_CONFIG_TEMPLATE = """
+CACHE = '{cache}'
+BINARYEN_ROOT = '{binaryen_root}'
+LLVM_ROOT = '{llvm_root}'
+"""
+
+def get_root_and_script_ext(repository_ctx):
+    if repository_ctx.os.name.startswith('linux'):
+        if 'amd64' in repository_ctx.os.arch or 'x86_64' in repository_ctx.os.arch:
+            return (repository_ctx.path(Label("@emscripten_bin_linux//:BUILD.bazel")).dirname, '')
+        elif 'aarch64' in repository_ctx.os.arch:
+            return (repository_ctx.path(Label("@emscripten_bin_linux_arm64//:BUILD.bazel")).dirname, '')
+        else:
+            fail('Unsupported architecture for Linux')
+    elif repository_ctx.os.name.startswith('mac'):
+        if 'amd64' in repository_ctx.os.arch or 'x86_64' in repository_ctx.os.arch:
+            return (repository_ctx.path(Label("@emscripten_bin_mac//:BUILD.bazel")).dirname, '')
+        elif 'aarch64' in repository_ctx.os.arch:
+            return (repository_ctx.path(Label("@emscripten_bin_mac_arm64//:BUILD.bazel")).dirname, '')
+        else:
+            fail('Unsupported architecture for MacOS')
+    elif repository_ctx.os.name.startswith('windows'):
+        return (repository_ctx.path(Label("@emscripten_bin_win//:BUILD.bazel")).dirname, '.bat')
+    else:
+        fail('Unsupported operating system')
+
+def _emscripten_cache_impl(repository_ctx):
+    # Read the default emscripten configuration file
+    default_config = repository_ctx.read(
+        repository_ctx.path(
+            Label("@emsdk//emscripten_toolchain:default_config")
+        )
+    )
+
+    if repository_ctx.attr.targets or repository_ctx.attr.configuration:
+        root, script_ext = get_root_and_script_ext(repository_ctx)
+        llvm_root = root.get_child("bin")
+        cache = repository_ctx.path("cache")
+        # Create configuration file
+        embuilder_config_content = EMBUILDER_CONFIG_TEMPLATE.format(
+            cache=cache,
+            binaryen_root=root,
+            llvm_root=llvm_root,
+        )
+        repository_ctx.file("embuilder_config", embuilder_config_content)
+        embuilder_config_path = repository_ctx.path("embuilder_config")
+        embuilder_path = "{}{}".format(root.get_child("emscripten").get_child("embuilder"), script_ext)
+        # Prepare the command line
+        if repository_ctx.attr.targets:
+            targets = repository_ctx.attr.targets
+        else:
+            # If no targets are requested, build everything
+            targets = ["ALL"]
+        flags = ["--em-config", embuilder_config_path] + repository_ctx.attr.configuration
+        embuilder_args = [embuilder_path] + flags + ["build"] + targets
+        # Run embuilder
+        repository_ctx.report_progress("Building secondary cache")
+        result = repository_ctx.execute(
+            embuilder_args,
+            quiet=True,
+            environment = {
+                "EM_IGNORE_SANITY": "1",
+                "EM_NODE_JS": "empty",
+            }
+        )
+        if result.return_code != 0:
+            fail("Embuilder exited with a non-zero return code")
+        # Override Emscripten's cache with the secondary cache
+        default_config += "CACHE = '{}'\n".format(cache)
+
+    # Create the configuration file for the toolchain and export
+    repository_ctx.file('emscripten_config', default_config)
+    repository_ctx.file('BUILD.bazel', BUILD_FILE_CONTENT_TEMPLATE)
+
+_emscripten_cache = repository_rule(
+    implementation = _emscripten_cache_impl,
+    attrs = {
+        "configuration": attr.string_list(),
+        "targets": attr.string_list(),
+    },
+    local = True
+)
+
+def register_emscripten_toolchains(cache = {}):
+    _emscripten_cache(
+        name = "emscripten_cache",
+        configuration = cache["configuration"] if "configuration" in cache else [],
+        targets = cache["targets"] if "targets" in cache else [],
+    )
+
     native.register_toolchains(str(Label("//emscripten_toolchain:cc-toolchain-wasm")))

test/test_bazel.ps1

@@ -22,3 +22,9 @@ if (-not $?) { Exit $LastExitCode }
 # Test use of the closure compiler
 bazel build //:hello-embind-wasm --compilation_mode opt # release
 if (-not $?) { Exit $LastExitCode }
+
+Set-Location ..\test_secondary_lto_cache
+
+bazel build //:hello-world-wasm
+if (-not $?) { Exit $LastExitCode }
+

test/test_bazel.sh

@@ -30,3 +30,7 @@ bazel build //:hello-embind-wasm --compilation_mode dbg # debug
 bazel build //:hello-embind-wasm --compilation_mode opt # release
 # This function should not be minified if the externs file is loaded correctly.
 grep "customJSFunctionToTestClosure" bazel-bin/hello-embind-wasm/hello-embind.js
+
+cd ../test_secondary_lto_cache
+bazel build //:hello-world-wasm
+

test/test_bazel_mac.sh

@@ -24,3 +24,7 @@ bazel build //hello-world:hello-world-wasm-simd
 cd test_external
 bazel build //long_command_line:long_command_line_wasm
 bazel build //:hello-world-wasm
+
+cd ../test_secondary_lto_cache
+bazel build //:hello-world-wasm
+