From 774b871eca047688fff1ccdf4b2f7e784e4092ad Mon Sep 17 00:00:00 2001 From: Ezekiel Warren Date: Mon, 20 Dec 2021 11:24:56 -0800 Subject: [PATCH] Bazel Windows Support (#929) --- .circleci/config.yml | 25 +++++++++++++++++++ bazel/.gitignore | 1 + bazel/emscripten_deps.bzl | 10 ++++---- bazel/emscripten_toolchain/BUILD.bazel | 8 ++++++ bazel/emscripten_toolchain/crosstool.bzl | 17 ++++++++----- bazel/emscripten_toolchain/emar.bat | 5 ++++ bazel/emscripten_toolchain/emcc.bat | 5 ++++ bazel/emscripten_toolchain/emcc_link.bat | 5 ++++ bazel/emscripten_toolchain/emscripten_config | 2 +- bazel/emscripten_toolchain/env.bat | 5 ++++ bazel/emscripten_toolchain/link_wrapper.py | 2 +- bazel/emscripten_toolchain/wasm_binary.py | 8 +++--- bazel/emscripten_toolchain/wasm_cc_binary.bzl | 13 +++------- bazel/revisions.bzl | 4 +-- scripts/update_bazel_workspace.sh | 12 ++++----- test/test_bazel.ps1 | 21 ++++++++++++++++ 16 files changed, 110 insertions(+), 33 deletions(-) create mode 100644 bazel/.gitignore create mode 100644 bazel/emscripten_toolchain/emar.bat create mode 100644 bazel/emscripten_toolchain/emcc.bat create mode 100644 bazel/emscripten_toolchain/emcc_link.bat create mode 100644 bazel/emscripten_toolchain/env.bat create mode 100644 test/test_bazel.ps1 diff --git a/.circleci/config.yml b/.circleci/config.yml index 38b81e7..e21abe9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -206,6 +206,28 @@ jobs: brew install bazel - run: test/test_bazel_mac.sh + test-bazel-windows: + executor: + name: win/vs2019 + shell: powershell.exe -ExecutionPolicy Bypass + environment: + PYTHONUNBUFFERED: "1" + EMSDK_NOTTY: "1" + steps: + - checkout + - run: + name: Download Bazelisk + shell: powershell.exe + command: | + $ProgressPreference = "SilentlyContinue" + Invoke-WebRequest -Uri https://github.com/bazelbuild/bazelisk/releases/download/v1.10.1/bazelisk-windows-amd64.exe -OutFile ( New-Item -Path "temp\bazel\bazel.exe" -Force ) + - run: + name: Run Tests + shell: powershell.exe + command: | + $env:Path += ";C:\Python27amd64;$pwd\temp\bazel" + .\test\test_bazel.ps1 + workflows: flake8: jobs: @@ -234,3 +256,6 @@ workflows: test-bazel-mac: jobs: - test-bazel-mac + test-bazel-windows: + jobs: + - test-bazel-windows diff --git a/bazel/.gitignore b/bazel/.gitignore new file mode 100644 index 0000000..a6ef824 --- /dev/null +++ b/bazel/.gitignore @@ -0,0 +1 @@ +/bazel-* diff --git a/bazel/emscripten_deps.bzl b/bazel/emscripten_deps.bzl index 1a2b844..2189360 100644 --- a/bazel/emscripten_deps.bzl +++ b/bazel/emscripten_deps.bzl @@ -19,7 +19,7 @@ def emscripten_deps(emscripten_version = "latest"): revision = EMSCRIPTEN_TAGS[version] - emscripten_url = "https://storage.googleapis.com/webassembly/emscripten-releases-builds/{}/{}/wasm-binaries.tbz2" + emscripten_url = "https://storage.googleapis.com/webassembly/emscripten-releases-builds/{}/{}/wasm-binaries.{}" # This could potentially backfire for projects with multiple emscripten # dependencies that use different emscripten versions @@ -28,7 +28,7 @@ def emscripten_deps(emscripten_version = "latest"): http_archive( name = "emscripten_bin_linux", strip_prefix = "install", - url = emscripten_url.format("linux", revision.hash), + url = emscripten_url.format("linux", revision.hash, "tbz2"), sha256 = revision.sha_linux, build_file = "@emsdk//emscripten_toolchain:emscripten.BUILD", type = "tar.bz2", @@ -38,7 +38,7 @@ def emscripten_deps(emscripten_version = "latest"): http_archive( name = "emscripten_bin_mac", strip_prefix = "install", - url = emscripten_url.format("mac", revision.hash), + url = emscripten_url.format("mac", revision.hash, "tbz2"), sha256 = revision.sha_mac, build_file = "@emsdk//emscripten_toolchain:emscripten.BUILD", type = "tar.bz2", @@ -48,10 +48,10 @@ def emscripten_deps(emscripten_version = "latest"): http_archive( name = "emscripten_bin_win", strip_prefix = "install", - url = emscripten_url.format("win", revision.hash), + url = emscripten_url.format("win", revision.hash, "zip"), sha256 = revision.sha_win, build_file = "@emsdk//emscripten_toolchain:emscripten.BUILD", - type = "tar.bz2", + type = "zip", ) if "emscripten_npm_linux" not in excludes: diff --git a/bazel/emscripten_toolchain/BUILD.bazel b/bazel/emscripten_toolchain/BUILD.bazel index 8b2ae7f..eb36959 100644 --- a/bazel/emscripten_toolchain/BUILD.bazel +++ b/bazel/emscripten_toolchain/BUILD.bazel @@ -6,9 +6,12 @@ filegroup( name = "common-script-includes", srcs = [ "emar.sh", + "emar.bat", "emcc.sh", + "emcc.bat", "emscripten_config", "env.sh", + "env.bat", "@emsdk//:binaries", "@emsdk//:node_modules", "@nodejs//:node_files", @@ -24,6 +27,7 @@ filegroup( name = "link-emscripten", srcs = [ "emcc_link.sh", + "emcc_link.bat", "link_wrapper.py", ":common-script-includes", "@emsdk//:binaries", @@ -51,6 +55,10 @@ emscripten_cc_toolchain_config_rule( cpu = "wasm", em_config = "emscripten_config", emscripten_binaries = "@emsdk//:binaries", + script_extension = select({ + "@bazel_tools//src/conditions:host_windows": "bat", + "//conditions:default": "sh", + }), ) cc_toolchain( diff --git a/bazel/emscripten_toolchain/crosstool.bzl b/bazel/emscripten_toolchain/crosstool.bzl index b4883ea..ce87ebb 100644 --- a/bazel/emscripten_toolchain/crosstool.bzl +++ b/bazel/emscripten_toolchain/crosstool.bzl @@ -74,23 +74,27 @@ def _impl(ctx): builtin_sysroot = emscripten_dir + "/emscripten/cache/sysroot" + emcc_script = "emcc.%s" % ctx.attr.script_extension + emcc_link_script = "emcc_link.%s" % ctx.attr.script_extension + emar_script = "emar.%s" % ctx.attr.script_extension + ################################################################ # Tools ################################################################ - clang_tool = tool(path = "emcc.sh") + clang_tool = tool(path = emcc_script) clif_match_tool = tool(path = "dummy_clif_matcher") - link_tool = tool(path = "emcc_link.sh") - archive_tool = tool(path = "emar.sh") + link_tool = tool(path = emcc_link_script) + archive_tool = tool(path = emar_script) strip_tool = tool(path = "NOT_USED_STRIP_TOOL") #### Legacy tool paths (much of this is redundant with action_configs, but #### these are still used for some things) tool_paths = [ - tool_path(name = "ar", path = "emar.sh"), + tool_path(name = "ar", path = emar_script), tool_path(name = "cpp", path = "/bin/false"), - tool_path(name = "gcc", path = "emcc.sh"), + tool_path(name = "gcc", path = emcc_script), tool_path(name = "gcov", path = "/bin/false"), - tool_path(name = "ld", path = "emcc_link.sh"), + tool_path(name = "ld", path = emcc_link_script), tool_path(name = "nm", path = "NOT_USED"), tool_path(name = "objdump", path = "/bin/false"), tool_path(name = "strip", path = "NOT_USED"), @@ -1106,6 +1110,7 @@ emscripten_cc_toolchain_config_rule = rule( "cpu": attr.string(mandatory = True, values = ["asmjs", "wasm"]), "em_config": attr.label(mandatory = True, allow_single_file = True), "emscripten_binaries": attr.label(mandatory = True), + "script_extension": attr.string(mandatory = True, values = ["sh", "bat"]), }, provides = [CcToolchainConfigInfo], ) diff --git a/bazel/emscripten_toolchain/emar.bat b/bazel/emscripten_toolchain/emar.bat new file mode 100644 index 0000000..b8e9125 --- /dev/null +++ b/bazel/emscripten_toolchain/emar.bat @@ -0,0 +1,5 @@ +@ECHO OFF + +call external\emsdk\emscripten_toolchain\env.bat + +py -3 %EMSCRIPTEN%\emar.py %* diff --git a/bazel/emscripten_toolchain/emcc.bat b/bazel/emscripten_toolchain/emcc.bat new file mode 100644 index 0000000..aba66f4 --- /dev/null +++ b/bazel/emscripten_toolchain/emcc.bat @@ -0,0 +1,5 @@ +@ECHO OFF + +call external\emsdk\emscripten_toolchain\env.bat + +py -3 %EMSCRIPTEN%\emcc.py %* diff --git a/bazel/emscripten_toolchain/emcc_link.bat b/bazel/emscripten_toolchain/emcc_link.bat new file mode 100644 index 0000000..8e5a6eb --- /dev/null +++ b/bazel/emscripten_toolchain/emcc_link.bat @@ -0,0 +1,5 @@ +@ECHO OFF + +call external\emsdk\emscripten_toolchain\env.bat + +py -3 external\emsdk\emscripten_toolchain\link_wrapper.py %* diff --git a/bazel/emscripten_toolchain/emscripten_config b/bazel/emscripten_toolchain/emscripten_config index d1275ff..85af7b2 100644 --- a/bazel/emscripten_toolchain/emscripten_config +++ b/bazel/emscripten_toolchain/emscripten_config @@ -8,5 +8,5 @@ LLVM_ROOT = BINARYEN_ROOT + "/bin" FROZEN_CACHE = True system = platform.system() -nodejs_binary = "node.exe" if(system =="Windows") else "bin/node" +nodejs_binary = "bin/nodejs/node.exe" if(system =="Windows") else "bin/node" NODE_JS = ROOT_DIR + "/external/nodejs_{}_amd64/{}".format(system.lower(), nodejs_binary) diff --git a/bazel/emscripten_toolchain/env.bat b/bazel/emscripten_toolchain/env.bat new file mode 100644 index 0000000..644519e --- /dev/null +++ b/bazel/emscripten_toolchain/env.bat @@ -0,0 +1,5 @@ +@ECHO OFF + +set ROOT_DIR=%CD% +set EMSCRIPTEN=%ROOT_DIR%\%EM_BIN_PATH%\emscripten +set EM_CONFIG=%ROOT_DIR%\%EM_CONFIG_PATH% diff --git a/bazel/emscripten_toolchain/link_wrapper.py b/bazel/emscripten_toolchain/link_wrapper.py index 3134c8d..d6b715d 100644 --- a/bazel/emscripten_toolchain/link_wrapper.py +++ b/bazel/emscripten_toolchain/link_wrapper.py @@ -39,7 +39,7 @@ if any(' ' in a for a in param_file_args): sys.argv[1] = '@' + new_param_filename emcc_py = os.path.join(os.environ['EMSCRIPTEN'], 'emcc.py') -rtn = subprocess.call(['python3', emcc_py] + sys.argv[1:]) +rtn = subprocess.call([sys.executable, emcc_py] + sys.argv[1:]) if rtn != 0: sys.exit(1) diff --git a/bazel/emscripten_toolchain/wasm_binary.py b/bazel/emscripten_toolchain/wasm_binary.py index 12aa870..8c67c7c 100644 --- a/bazel/emscripten_toolchain/wasm_binary.py +++ b/bazel/emscripten_toolchain/wasm_binary.py @@ -20,7 +20,7 @@ WebAssembly binary into a larger web application. import argparse import os -import subprocess +import tarfile def ensure(f): @@ -40,12 +40,14 @@ def main(): parser.add_argument('--output_path', help='The path to extract into.') args = parser.parse_args() + args.archive = os.path.normpath(args.archive) + basename = os.path.basename(args.archive) stem = basename.split('.')[0] # Extract all files from the tarball. - subprocess.check_call( - ['tar', 'xf', args.archive, '-C', args.output_path]) + tar = tarfile.open(args.archive) + tar.extractall(args.output_path) # At least one of these two files should exist at this point. ensure(os.path.join(args.output_path, stem + '.js')) diff --git a/bazel/emscripten_toolchain/wasm_cc_binary.bzl b/bazel/emscripten_toolchain/wasm_cc_binary.bzl index 9d68041..96101a5 100644 --- a/bazel/emscripten_toolchain/wasm_cc_binary.bzl +++ b/bazel/emscripten_toolchain/wasm_cc_binary.bzl @@ -55,15 +55,10 @@ _wasm_transition = transition( ) def _wasm_binary_impl(ctx): - cc_target = ctx.attr.cc_target[0] + args = ctx.actions.args() + args.add("--output_path", ctx.outputs.loader.dirname) + args.add_all("--archive", ctx.files.cc_target) - args = [ - "--output_path={}".format(ctx.outputs.loader.dirname), - ] + [ - ctx.expand_location("--archive=$(location {})".format( - cc_target.label, - ), [cc_target]), - ] outputs = [ ctx.outputs.loader, ctx.outputs.wasm, @@ -80,7 +75,7 @@ def _wasm_binary_impl(ctx): ctx.actions.run( inputs = ctx.files.cc_target, outputs = outputs, - arguments = args, + arguments = [args], executable = ctx.executable._wasm_binary_extractor, ) diff --git a/bazel/revisions.bzl b/bazel/revisions.bzl index 4eb36f5..910b9e0 100644 --- a/bazel/revisions.bzl +++ b/bazel/revisions.bzl @@ -12,13 +12,13 @@ EMSCRIPTEN_TAGS = { hash = "7fbe748230f2ce99abbf975d9ad997699efb3153", sha_linux = "10646b64daea15354f14f89f7e79937f420b77f31bda7c4b174de2474835950f", sha_mac = "ebb17bc91c6a72ca06d17337d27aa1a2be4c9af4c68644c221712123f663b8ab", - sha_win = "28a0a5983d8cdbe6b61439a111ef8a5f0f93ccb444d08e0540682b8d3c8c0f2e", + sha_win = "0d4f2ff5d88a8eef5ed769ee4ffc5d5574143911d2e0079325cdc5206c9e9bb1", ), "2.0.34": struct( hash = "d8fc1b92dbc0ce8d740a7adb937c5137ba4755e0", sha_linux = "a6304e3a52c172eb178c6f9817d74aa3ee411e97ef00bcae0884377799c49954", sha_mac = "975ae11000100362baf19d161fec04d82e1f7c9fb7d43c43864ddd65a47f1780", - sha_win = "30f64dbc6425490157360439d40141feae359dbf586f59fa23185d7e009f3209", + sha_win = "8167a44bb895a0fdc153836bed91bf387be57f2dc1b8f103bf70e68923b61d39", ), "2.0.33": struct( hash = "cef8850d57278271766fb2163eebcb07354018e7", diff --git a/scripts/update_bazel_workspace.sh b/scripts/update_bazel_workspace.sh index ad2fd5f..6bc903f 100755 --- a/scripts/update_bazel_workspace.sh +++ b/scripts/update_bazel_workspace.sh @@ -34,16 +34,16 @@ if [[ $ERR = 1 ]]; then fi URL1=https://storage.googleapis.com/webassembly/emscripten-releases-builds/ -URL2=/wasm-binaries.tbz2 +URL2=/wasm-binaries. # Get commit hash for $1 version get_hash () { echo $(grep "$1" emscripten-releases-tags.json | grep -v latest | cut -f4 -d\") } -# Get sha256 for $1 os $2 hash +# Get sha256 for $1 os $2 extname $3 hash get_sha () { - echo $(curl "${URL1}$1/$2${URL2}" 2>/dev/null | sha256sum | awk '{print $1}') + echo $(curl "${URL1}$1/$3${URL2}$2" 2>/dev/null | sha256sum | awk '{print $1}') } # Assemble dictionary line @@ -52,9 +52,9 @@ revisions_item () { echo \ "\ \"$1\": struct(\n" \ "\ hash = \"$(get_hash ${hash})\",\n" \ - "\ sha_linux = \"$(get_sha linux ${hash})\",\n" \ - "\ sha_mac = \"$(get_sha mac ${hash})\",\n" \ - "\ sha_win = \"$(get_sha win ${hash})\",\n" \ + "\ sha_linux = \"$(get_sha linux tbz2 ${hash})\",\n" \ + "\ sha_mac = \"$(get_sha mac tbz2 ${hash})\",\n" \ + "\ sha_win = \"$(get_sha win zip ${hash})\",\n" \ "\ )," } diff --git a/test/test_bazel.ps1 b/test/test_bazel.ps1 new file mode 100644 index 0000000..3fd269c --- /dev/null +++ b/test/test_bazel.ps1 @@ -0,0 +1,21 @@ +$ErrorActionPreference = 'Stop' + +Set-Location bazel + +bazel build //hello-world:hello-world-wasm +if (-not $?) { Exit $LastExitCode } + +bazel build //hello-world:hello-world-wasm-simd +if (-not $?) { Exit $LastExitCode } + +Set-Location test_external + +bazel build //:hello-world-wasm +if (-not $?) { Exit $LastExitCode } + +bazel build //:hello-embind-wasm --compilation_mode dbg # debug +if (-not $?) { Exit $LastExitCode } + +# Test use of the closure compiler +bazel build //:hello-embind-wasm --compilation_mode opt # release +if (-not $?) { Exit $LastExitCode }