add a wasm bazel toolchain (#603)

bazel/emscripten_toolchain/BUILD.bazel

@@ -0,0 +1,80 @@
+load(":crosstool.bzl", "emscripten_cc_toolchain_config_rule")
+
+package(default_visibility = ["//visibility:public"])
+
+filegroup(
+    name = "common-script-includes",
+    srcs = [
+        "emar.sh",
+        "emcc.sh",
+        "emscripten_config",
+        "env.sh",
+        "@emscripten//:all",
+        "@nodejs//:node_files",
+        "@npm//:node_modules",
+    ],
+)
+
+filegroup(
+    name = "compile-emscripten",
+    srcs = [":common-script-includes"],
+)
+
+filegroup(
+    name = "link-emscripten",
+    srcs = [
+        "emcc_link.sh",
+        "link_wrapper.py",
+        ":common-script-includes",
+        "@emscripten//:all",
+        "@nodejs//:node_files",
+    ],
+)
+
+filegroup(
+    name = "every-file",
+    srcs = [
+        ":compile-emscripten",
+        ":link-emscripten",
+        "@emscripten//:all",
+        "@nodejs//:node_files",
+    ],
+)
+
+filegroup(name = "empty")
+
+# dlmalloc.bc is implictly added by the emscripten toolchain
+cc_library(name = "malloc")
+
+emscripten_cc_toolchain_config_rule(
+    name = "wasm",
+    cpu = "wasm",
+    emscripten_version = "emscripten",
+)
+
+cc_toolchain(
+    name = "cc-compiler-wasm",
+    all_files = ":every-file",
+    ar_files = ":common-script-includes",
+    as_files = ":empty",
+    compiler_files = ":compile-emscripten",
+    dwp_files = ":empty",
+    linker_files = ":link-emscripten",
+    objcopy_files = ":empty",
+    strip_files = ":empty",
+    toolchain_config = "wasm",
+    toolchain_identifier = "emscripten-wasm",
+)
+
+cc_toolchain_suite(
+    name = "everything",
+    toolchains = {
+        "wasm": ":cc-compiler-wasm",
+        "wasm|emscripten": ":cc-compiler-wasm",
+    },
+)
+
+py_binary(
+    name = "wasm_binary",
+    srcs = ["wasm_binary.py"],
+)

bazel/emscripten_toolchain/emar.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+source emscripten_toolchain/env.sh
+
+exec python3 $EMSCRIPTEN/emar.py "$@"

bazel/emscripten_toolchain/emcc.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+source emscripten_toolchain/env.sh
+
+exec python3 external/emscripten/emscripten/emcc.py "$@"

bazel/emscripten_toolchain/emcc_link.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+source emscripten_toolchain/env.sh
+
+exec python3 emscripten_toolchain/link_wrapper.py "$@"

bazel/emscripten_toolchain/emscripten.BUILD

@@ -0,0 +1,6 @@
+package(default_visibility = ['//visibility:public'])
+
+filegroup(
+    name = "all",
+    srcs = glob(["**"]),
+)

bazel/emscripten_toolchain/emscripten_config

@@ -0,0 +1,9 @@
+import os
+
+ROOT_DIR = os.environ["ROOT_DIR"]
+EMSCRIPTEN_ROOT = os.environ["EMSCRIPTEN"]
+
+LLVM_ROOT = ROOT_DIR + "/external/emscripten/bin"
+EMSCRIPTEN_NATIVE_OPTIMIZER = LLVM_ROOT + "/optimizer"
+NODE_JS = ROOT_DIR + "/external/nodejs_linux_amd64/bin/node"
+BINARYEN_ROOT = ROOT_DIR + "/external/emscripten"

bazel/emscripten_toolchain/env.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+export ROOT_DIR=`(pwd -P)`
+export EMSCRIPTEN=${ROOT_DIR}/external/emscripten/emscripten
+
+export EM_CONFIG=${ROOT_DIR}/emscripten_toolchain/emscripten_config
+export EM_CACHE=${ROOT_DIR}/emscripten_toolchain/cache

bazel/emscripten_toolchain/link_wrapper.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+"""wrapper around emcc link step.
+
+This wrapper currently serves the following purposes.
+
+1. Ensures we always link to file with .js extension. The upstream default
+   it to link to an llvm bitcode file which is never (AFAICT) want to do that.
+
+2. When building with --config=wasm the final output is multiple files, usually
+   at least one .js and one .wasm file. Since the cc_binary link step only
+   allows a single output, we must tar up the outputs into a single file.
+
+3. Add quotes around arguments that need them in the response file to work
+   around a bazel quirk.
+"""
+
+from __future__ import print_function
+
+import os
+import subprocess
+import sys
+
+# Only argument should be @path/to/parameter/file
+assert sys.argv[1][0] == '@'
+param_filename = sys.argv[1][1:]
+param_file_args = [l.strip() for l in open(param_filename, 'r').readlines()]
+
+output_index = param_file_args.index('-o') + 1
+orig_output = js_output = param_file_args[output_index]
+outdir = os.path.dirname(orig_output)
+
+# google3-only(TODO(b/139440956): Default to False once the bug is fixed)
+replace_response_file = any(' ' in a for a in param_file_args)
+
+if not os.path.splitext(orig_output)[1]:
+  js_output = orig_output + '.js'
+  param_file_args[output_index] = js_output
+  replace_response_file = True
+
+# Re-write response file if needed.
+if replace_response_file:
+  new_param_filename = param_filename + '.modified'
+  with open(new_param_filename, 'w') as f:
+    for param in param_file_args:
+      if ' ' in param:
+        f.write('"%s"' % param)
+      else:
+        f.write(param)
+      f.write('\n')
+  sys.argv[1] = '@' + new_param_filename
+
+emcc_py = os.path.join(os.environ['EMSCRIPTEN'], 'emcc.py')
+rtn = subprocess.call(['python3', emcc_py] + sys.argv[1:])
+if rtn != 0:
+  sys.exit(1)
+
+js_name = os.path.basename(js_output)
+base_name = os.path.splitext(js_name)[0]
+
+files = []
+extensions = [
+    '.js',
+    '.wasm',
+    '.wasm.map',
+    '.js.mem',
+    '.fetch.js',
+    '.worker.js',
+    '.data',
+    '.js.symbols',
+    '.wasm.debug.wasm'
+]
+
+for ext in extensions:
+  filename = base_name + ext
+  if os.path.exists(os.path.join(outdir, filename)):
+    files.append(filename)
+
+wasm_base = os.path.join(outdir, base_name + '.wasm')
+if os.path.exists(wasm_base + '.debug.wasm') and os.path.exists(wasm_base):
+  # If we have a .wasm.debug.wasm file and a .wasm file, we need to rewrite the
+  # section in the .wasm file that refers to it. The path that's in there
+  # is the blaze output path; we want it to be just the filename.
+
+  llvm_objcopy = os.path.join(
+      os.environ['EMSCRIPTEN'], 'llvm-bin/llvm-objcopy')
+  # First, check to make sure the .wasm file has the header that needs to be
+  # rewritten.
+  rtn = subprocess.call([
+      llvm_objcopy,
+      '--dump-section=external_debug_info=/dev/null',
+      wasm_base], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+  if rtn == 0:
+    # If llvm-objcopy did not return an error, the external_debug_info section
+    # must exist, so we're good to continue.
+
+    # Next we need to convert length of the filename to LEB128.
+    # Start by converting the length of the filename to a bit string.
+    bit_string = '{0:b}'.format(len(base_name + '.wasm.debug.wasm'))
+
+    # Pad the bit string with 0s so that its length is a multiple of 7.
+    while len(bit_string) % 7 != 0:
+      bit_string = '0' + bit_string
+
+    # Break up our bit string into chunks of 7.
+    # We do this backwards because the final format is little-endian.
+    final_bytes = bytearray()
+    for i in reversed(range(0, len(bit_string), 7)):
+      binary_part = bit_string[i:i + 7]
+      if i != 0:
+        # Every chunk except the last one needs to be prepended with '1'.
+        # The length of each chunk is 7, so that one has an implicit '0'.
+        binary_part = '1' + binary_part
+      final_bytes.append(int(binary_part, 2))
+    # Finally, add the actual filename.
+    final_bytes.extend(base_name + '.wasm.debug.wasm')
+
+    # Write our length + filename bytes to a temp file.
+    with open('debugsection.tmp', 'wb+') as f:
+      f.write(final_bytes)
+      f.close()
+
+    # First delete the old section.
+    subprocess.check_call([
+        llvm_objcopy,
+        wasm_base,
+        '--remove-section=external_debug_info'])
+    # Rewrite section with the new size and filename from the temp file.
+    subprocess.check_call([
+        llvm_objcopy,
+        wasm_base,
+        '--add-section=external_debug_info=debugsection.tmp'])
+
+# If we have more than one output file then create tarball
+if len(files) > 1:
+  cmd = ['tar', 'cf', 'tmp.tar'] + files
+  subprocess.check_call(cmd, cwd=outdir)
+  os.rename(os.path.join(outdir, 'tmp.tar'), orig_output)
+elif len(files) == 1:
+  # Otherwise, if only have a single output than move it to the expected name
+  if files[0] != os.path.basename(orig_output):
+    os.rename(os.path.join(outdir, files[0]), orig_output)
+else:
+  print('emcc.py did not appear to output any known files!')
+  sys.exit(1)
+
+sys.exit(0)

bazel/emscripten_toolchain/wasm_binary.py

@@ -0,0 +1,84 @@
+"""Unpackages a bazel emscripten archive for use in a bazel BUILD rule.
+
+This script will take a tar archive containing the output of the emscripten
+toolchain. This file contains any output files produced by a wasm_cc_binary or a
+cc_binary built with --config=wasm. The files are extracted into the given
+output path.
+
+The name of archive is expected to be of the format `foo` or `foo.XXX` and
+the contents are expected to be foo.js and foo.wasm.
+
+Several optional files may also be in the archive, including but not limited to
+foo.js.mem, pthread-main.js, and foo.wasm.map.
+
+If the file is not a tar archive, the passed file will simply be copied to its
+destination.
+
+This script and its accompanying Bazel rule should allow you to extract a
+WebAssembly binary into a larger web application.
+"""
+
+import os
+import subprocess
+import sys
+
+from absl import app
+from absl import flags
+
+FLAGS = flags.FLAGS
+flags.DEFINE_string('archive', None, 'The the archive to extract from.')
+flags.DEFINE_string('output_path', None, 'The path to extract into.')
+
+
+def ensure(f):
+  if not os.path.exists(f):
+    with open(f, 'w'):
+      pass
+
+
+def check(f):
+  if not os.path.exists(f):
+    raise Exception('Expected file in archive: %s' % f)
+
+
+def main(argv):
+  basename = os.path.basename(FLAGS.archive)
+  stem = basename.split('.')[0]
+
+  # Check the type of the input file
+  mimetype_bytes = subprocess.check_output(['file', '-Lib', FLAGS.archive])
+  mimetype = mimetype_bytes.decode(sys.stdout.encoding)
+
+  # If we have a tar, extract all files. If we have just a single file, copy it.
+  if 'tar' in mimetype:
+    subprocess.check_call(
+        ['tar', 'xf', FLAGS.archive, '-C', FLAGS.output_path])
+  elif 'binary' in mimetype:
+    subprocess.check_call([
+        'cp',
+        FLAGS.archive,
+        os.path.join(FLAGS.output_path, stem + '.wasm')])
+  elif 'text' in mimetype:
+    subprocess.check_call([
+        'cp',
+        FLAGS.archive,
+        os.path.join(FLAGS.output_path, stem + '.js')])
+  else:
+    subprocess.check_call(['cp', FLAGS.archive, FLAGS.output_path])
+
+  # At least one of these two files should exist at this point.
+  ensure(os.path.join(FLAGS.output_path, stem + '.js'))
+  ensure(os.path.join(FLAGS.output_path, stem + '.wasm'))
+
+  # And can optionally contain these extra files.
+  ensure(os.path.join(FLAGS.output_path, stem + '.wasm.map'))
+  ensure(os.path.join(FLAGS.output_path, stem + '.worker.js'))
+  ensure(os.path.join(FLAGS.output_path, stem + '.js.mem'))
+  ensure(os.path.join(FLAGS.output_path, stem + '.data'))
+  ensure(os.path.join(FLAGS.output_path, stem + '.fetch.js'))
+  ensure(os.path.join(FLAGS.output_path, stem + '.js.symbols'))
+  ensure(os.path.join(FLAGS.output_path, stem + '.wasm.debug.wasm'))
+
+
+if __name__ == '__main__':
+  app.run(main)

bazel/emscripten_toolchain/wasm_cc_binary.bzl

@@ -0,0 +1,150 @@
+"""wasm_cc_binary rule for compiling C++ targets to WebAssembly.
+"""
+
+def _wasm_transition_impl(settings, attr):
+    _ignore = (settings, attr)
+
+    features = list(settings["//command_line_option:features"])
+    linkopts = list(settings["//command_line_option:linkopt"])
+
+    if attr.threads == "emscripten":
+        # threads enabled
+        features.append("use_pthreads")
+    elif attr.threads == "off":
+        # threads disabled
+        features.append("-use_pthreads")
+
+    if attr.exit_runtime == True:
+        features.append("exit_runtime")
+
+    if attr.backend == "llvm":
+        features.append("llvm_backend")
+    elif attr.backend == "emscripten":
+        features.append("-llvm_backend")
+
+    if attr.simd:
+        features.append("wasm_simd")
+
+    return {
+        "//command_line_option:compiler": "emscripten",
+        "//command_line_option:crosstool_top": "//emscripten_toolchain:everything",
+        "//command_line_option:cpu": "wasm",
+        "//command_line_option:features": features,
+        "//command_line_option:dynamic_mode": "off",
+        "//command_line_option:linkopt": linkopts,
+        "//command_line_option:platforms": [],
+        "//command_line_option:custom_malloc": "//emscripten_toolchain:malloc",
+    }
+
+_wasm_transition = transition(
+    implementation = _wasm_transition_impl,
+    inputs = [
+        "//command_line_option:features",
+        "//command_line_option:linkopt",
+    ],
+    outputs = [
+        "//command_line_option:compiler",
+        "//command_line_option:cpu",
+        "//command_line_option:crosstool_top",
+        "//command_line_option:features",
+        "//command_line_option:dynamic_mode",
+        "//command_line_option:linkopt",
+        "//command_line_option:platforms",
+        "//command_line_option:custom_malloc",
+    ],
+)
+
+def _wasm_binary_impl(ctx):
+    cc_target = ctx.attr.cc_target[0]
+
+    args = [
+        "--output_path={}".format(ctx.outputs.loader.dirname),
+    ] + [
+        ctx.expand_location("--archive=$(location {})".format(
+            cc_target.label,
+        ), [cc_target]),
+    ]
+    outputs = [
+        ctx.outputs.loader,
+        ctx.outputs.wasm,
+        ctx.outputs.map,
+        ctx.outputs.mem,
+        ctx.outputs.fetch,
+        ctx.outputs.worker,
+        ctx.outputs.data,
+        ctx.outputs.symbols,
+        ctx.outputs.dwarf,
+    ]
+
+    ctx.actions.run(
+        inputs = ctx.files.cc_target,
+        outputs = outputs,
+        arguments = args,
+        executable = ctx.executable._wasm_binary_extractor,
+    )
+
+    return DefaultInfo(
+        files = depset(outputs),
+        # This is needed since rules like web_test usually have a data
+        # dependency on this target.
+        data_runfiles = ctx.runfiles(transitive_files = depset(outputs)),
+    )
+
+def _wasm_binary_outputs(name, cc_target):
+    basename = cc_target.name
+    basename = basename.split(".")[0]
+    outputs = {
+        "loader": "{}/{}.js".format(name, basename),
+        "wasm": "{}/{}.wasm".format(name, basename),
+        "map": "{}/{}.wasm.map".format(name, basename),
+        "mem": "{}/{}.js.mem".format(name, basename),
+        "fetch": "{}/{}.fetch.js".format(name, basename),
+        "worker": "{}/{}.worker.js".format(name, basename),
+        "data": "{}/{}.data".format(name, basename),
+        "symbols": "{}/{}.js.symbols".format(name, basename),
+        "dwarf": "{}/{}.wasm.debug.wasm".format(name, basename),
+    }
+
+    return outputs
+
+# Wraps a C++ Blaze target, extracting the appropriate files.
+#
+# This rule will transition to the emscripten toolchain in order
+# to build the the cc_target as a WebAssembly binary.
+#
+# Args:
+#   name: The name of the rule.
+#   cc_target: The cc_binary or cc_library to extract files from.
+wasm_cc_binary = rule(
+    implementation = _wasm_binary_impl,
+    attrs = {
+        "backend": attr.string(
+            default = "_default",
+            values = ["_default", "emscripten", "llvm"],
+        ),
+        "cc_target": attr.label(
+            cfg = _wasm_transition,
+            mandatory = True,
+        ),
+        "exit_runtime": attr.bool(
+            default = False,
+        ),
+        "threads": attr.string(
+            default = "_default",
+            values = ["_default", "emscripten", "off"],
+        ),
+        "simd": attr.bool(
+            default = False,
+        ),
+        "_allowlist_function_transition": attr.label(
+            default = "@bazel_tools//tools/allowlists/function_transition_allowlist",
+        ),
+        "_wasm_binary_extractor": attr.label(
+            executable = True,
+            allow_files = True,
+            cfg = "exec",
+            default = Label("//emscripten_toolchain:wasm_binary"),
+        ),
+    },
+    outputs = _wasm_binary_outputs,
+)

bazel/emscripten_toolchain/wasm_rules.bzl

@@ -0,0 +1,6 @@
+"""Rules related to C++ and WebAssembly.
+"""
+
+load("//emscripten_toolchain:wasm_cc_binary.bzl", _wasm_cc_binary = "wasm_cc_binary")
+
+wasm_cc_binary = _wasm_cc_binary