Build wasm master (#401)

* Build wasm upstream * Fix lld build for wasm-ld. Add note about compiler-rt not working. * Address review * Address review * Address review * Address review * If targeting fastcomp Clang, apply EMCC_WASM_BACKEND=0 so that previous leftover environment with EMCC_WASM_BACKEND=1 will not cause a conflict * Add docs about setting up Emscripten
2019-12-16 23:59:15 +02:00
parent e10be710da
commit 2546d5ab51
3 changed files with 150 additions and 23 deletions
--- a/README.md
+++ b/README.md
@@ -118,11 +118,37 @@ Emsdk contains a history of old compiler versions that you can use to maintain
 your migration path. Type `emsdk list --old` to get a list of archived tool and
 SDK versions, and `emsdk install <name_of_tool>` to install it.

+### I want to build from source/I want to download a precompiled build!
+
+Some Emsdk Tool and SDK targets refer to packages that are precompiled, and
+no compilation is needed when installing them. Other Emsdk Tools and SDK
+targets come "from source", meaning that they will fetch the source repositories
+using git, and compile the package on demand.
+
+When you run `emsdk list`, it will group the Tools and SDKs under these two
+categories.
+
+To obtain and build latest upstream wasm SDK from source, run
+
+```
+emsdk install sdk-upstream-incoming-64bit
+```
+
+You can use this target for example to bootstrap developing patches to LLVM,
+Binaryen or Emscripten. (After initial installation, use `git remote add`
+in the cloned tree to add your own fork to push changes as patches)
+
+If you only intend to contribute to Emscripten repository, and not to LLVM
+or Binaryen, you can also use precompiled versions of them, and only git
+clone the Emscripten repository. For more details, see
+
+https://emscripten.org/docs/contributing/developers_guide.html?highlight=developer#setting-up
+
 ### When working on git branches compiled from source, how do I update to a newer compiler version?

 Unlike tags and precompiled versions, a few of the SDK packages are based on
-"moving" git branches and compiled from source (sdk-incoming, sdk-master,
-emscripten-incoming, emscripten-master, binaryen-master). Because of that, the
+"moving" git branches and compiled from source (e.g. sdk-upstream-incoming,
+sdk-incoming, emscripten-incoming, binaryen-master). Because of that, the
 compiled versions will eventually go out of date as new commits are introduced
 to the development branches. To update an old compiled installation of one of
 this branches, simply reissue the "emsdk install" command on that tool/SDK. This
--- a/emsdk.py
+++ b/emsdk.py
@@ -805,7 +805,7 @@ def decide_cmake_build_type(tool):


 # The root directory of the build.
-def fastcomp_build_dir(tool):
+def llvm_build_dir(tool):
  generator_suffix = ''
  if CMAKE_GENERATOR == 'Visual Studio 10':
    generator_suffix = '_vs2010'
@@ -838,7 +838,7 @@ def exe_suffix(filename):
 # The directory where the binaries are produced. (relative to the installation
 # root directory of the tool)
 def fastcomp_build_bin_dir(tool):
-  build_dir = fastcomp_build_dir(tool)
+  build_dir = llvm_build_dir(tool)
  if WINDOWS and 'Visual Studio' in CMAKE_GENERATOR:
    old_llvm_bin_dir = os.path.join(build_dir, 'bin', decide_cmake_build_type(tool))

@@ -1052,8 +1052,8 @@ def xcode_sdk_version():
    return subprocess.checkplatform.mac_ver()[0].split('.')


-def build_llvm_tool(tool):
-  debug_print('build_llvm_tool(' + str(tool) + ')')
+def build_llvm_fastcomp(tool):
+  debug_print('build_llvm_fastcomp(' + str(tool) + ')')
  fastcomp_root = tool.installation_path()
  fastcomp_src_root = os.path.join(fastcomp_root, 'src')
  # Does this tool want to be git cloned from github?
@@ -1061,10 +1061,11 @@ def build_llvm_tool(tool):
    success = git_clone_checkout_and_pull(tool.download_url(), fastcomp_src_root, tool.git_branch)
    if not success:
      return False
-    clang_root = os.path.join(fastcomp_src_root, 'tools/clang')
-    success = git_clone_checkout_and_pull(tool.clang_url, clang_root, tool.git_branch)
-    if not success:
-      return False
+    if hasattr(tool, 'clang_url'):
+      clang_root = os.path.join(fastcomp_src_root, 'tools/clang')
+      success = git_clone_checkout_and_pull(tool.clang_url, clang_root, tool.git_branch)
+      if not success:
+        return False
    if hasattr(tool, 'lld_url'):
      lld_root = os.path.join(fastcomp_src_root, 'tools/lld')
      success = git_clone_checkout_and_pull(tool.lld_url, lld_root, tool.git_branch)
@@ -1083,7 +1084,7 @@ def build_llvm_tool(tool):
  if 'Visual Studio' in CMAKE_GENERATOR and tool.bitness == 64:
    cmake_generator += ' Win64'

-  build_dir = fastcomp_build_dir(tool)
+  build_dir = llvm_build_dir(tool)
  build_root = os.path.join(fastcomp_root, build_dir)

  build_type = decide_cmake_build_type(tool)
@@ -1129,6 +1130,69 @@ def build_llvm_tool(tool):
  return success


+# LLVM git source tree migrated to a single repository instead of multiple ones, build_llvm_monorepo() builds via that repository structure
+def build_llvm_monorepo(tool):
+  debug_print('build_llvm_monorepo(' + str(tool) + ')')
+  llvm_root = tool.installation_path()
+  llvm_src_root = os.path.join(llvm_root, 'src')
+  success = git_clone_checkout_and_pull(tool.download_url(), llvm_src_root, tool.git_branch)
+  if not success:
+    return False
+
+  build_dir = llvm_build_dir(tool)
+  build_root = os.path.join(llvm_root, build_dir)
+
+  build_type = decide_cmake_build_type(tool)
+
+  # Configure
+  global BUILD_FOR_TESTING, ENABLE_LLVM_ASSERTIONS
+  tests_arg = 'ON' if BUILD_FOR_TESTING else 'OFF'
+
+  enable_assertions = ENABLE_LLVM_ASSERTIONS.lower() == 'on' or (ENABLE_LLVM_ASSERTIONS == 'auto' and build_type.lower() != 'release' and build_type.lower() != 'minsizerel')
+
+  if ARCH == 'x86' or ARCH == 'x86_64':
+    targets_to_build = 'WebAssembly;X86'
+  elif ARCH == 'arm':
+    targets_to_build = 'WebAssembly;ARM'
+  elif ARCH == 'aarch64':
+    targets_to_build = 'WebAssembly;AArch64'
+  else:
+    targets_to_build = 'WebAssembly'
+  args = ['-DLLVM_TARGETS_TO_BUILD=' + targets_to_build,
+          '-DLLVM_INCLUDE_EXAMPLES=OFF',
+          '-DCLANG_INCLUDE_EXAMPLES=OFF',
+          '-DLLVM_INCLUDE_TESTS=' + tests_arg,
+          '-DCLANG_INCLUDE_TESTS=' + tests_arg,
+          '-DLLVM_ENABLE_ASSERTIONS=' + ('ON' if enable_assertions else 'OFF')]
+  # LLVM build system bug: looks like everything needs to be passed to LLVM_ENABLE_PROJECTS twice. (or every second field is ignored?)
+
+  # LLVM build system bug #2: compiler-rt does not build on Windows. It insists on performing a CMake install step that writes to C:\Program Files. Attempting
+  # to reroute that to build_root directory then fails on an error
+  #  file INSTALL cannot find
+  #  "C:/code/emsdk/llvm/git/build_master_vs2017_64/$(Configuration)/lib/clang/10.0.0/lib/windows/clang_rt.ubsan_standalone-x86_64.lib".
+  # (there instead of $(Configuration), one would need ${CMAKE_BUILD_TYPE} ?)
+  # It looks like compiler-rt is not compatible to build on Windows?
+  args += ['-DLLVM_ENABLE_PROJECTS="clang;clang;lld;lld"']
+  cmake_generator = CMAKE_GENERATOR
+  if 'Visual Studio' in CMAKE_GENERATOR and tool.bitness == 64:
+    cmake_generator += ' Win64'
+    args += ['-Thost=x64']
+
+  if os.environ.get('LLVM_CMAKE_ARGS'):
+    extra_args = os.environ['LLVM_CMAKE_ARGS'].split(',')
+    print('Passing the following extra arguments to LLVM CMake configuration: ' + str(extra_args))
+    args += extra_args
+
+  cmakelists_dir = os.path.join(llvm_src_root, 'llvm')
+  success = cmake_configure(cmake_generator, build_root, cmakelists_dir, build_type, args)
+  if not success:
+    return False
+
+  # Make
+  success = make_build(build_root, build_type, 'x64' if tool.bitness == 64 else 'Win32')
+  return success
+
+
 # Emscripten asm.js optimizer build scripts:
 def optimizer_build_root(tool):
  build_root = tool.installation_path().strip()
@@ -1454,7 +1518,7 @@ class Tool(object):
      str = str.replace('%generator_prefix%', cmake_generator_prefix())
    str = str.replace('%.exe%', '.exe' if WINDOWS else '')
    if '%fastcomp_build_dir%' in str:
-      str = str.replace('%fastcomp_build_dir%', fastcomp_build_dir(self))
+      str = str.replace('%fastcomp_build_dir%', llvm_build_dir(self))
    if '%fastcomp_build_bin_dir%' in str:
      str = str.replace('%fastcomp_build_bin_dir%', fastcomp_build_bin_dir(self))
    return str
@@ -1718,7 +1782,9 @@ class Tool(object):
      url = self.download_url()

      if hasattr(self, 'custom_install_script') and self.custom_install_script == 'build_fastcomp':
-        success = build_llvm_tool(self)
+        success = build_llvm_fastcomp(self)
+      elif hasattr(self, 'custom_install_script') and self.custom_install_script == 'build_llvm_monorepo':
+        success = build_llvm_monorepo(self)
      elif hasattr(self, 'git_branch'):
        success = git_clone_checkout_and_pull(url, self.installation_path(), self.git_branch)
      elif url.endswith(ARCHIVE_SUFFIXES):
@@ -1742,7 +1808,7 @@ class Tool(object):
        if hasattr(self, 'custom_install_script'):
          if self.custom_install_script == 'build_optimizer':
            success = build_optimizer_tool(self)
-          elif self.custom_install_script == 'build_fastcomp':
+          elif self.custom_install_script in ('build_fastcomp', 'build_llvm_monorepo'):
            # 'build_fastcomp' is a special one that does the download on its
            # own, others do the download manually.
            pass
--- a/emsdk_manifest.json
+++ b/emsdk_manifest.json
@@ -1,5 +1,34 @@
 {
  "tools": [
+  {
+    "id": "llvm-git",
+    "version": "master",
+    "bitness": 32,
+    "install_path": "llvm/git",
+    "git_branch": "master",
+    "url": "https://github.com/llvm/llvm-project.git",
+    "custom_install_script": "build_llvm_monorepo",
+    "only_supports_wasm": true,
+    "activated_path": "%installation_dir%/%fastcomp_build_bin_dir%",
+    "activated_cfg": "LLVM_ROOT='%installation_dir%/%fastcomp_build_bin_dir%'",
+    "activated_env": "LLVM_ROOT=%installation_dir%/%fastcomp_build_bin_dir%;EMCC_WASM_BACKEND=1",
+    "cmake_build_type": "Release"
+  },
+  {
+    "id": "llvm-git",
+    "version": "master",
+    "bitness": 64,
+    "install_path": "llvm/git",
+    "git_branch": "master",
+    "url": "https://github.com/llvm/llvm-project.git",
+    "custom_install_script": "build_llvm_monorepo",
+    "only_supports_wasm": true,
+    "activated_path": "%installation_dir%/%fastcomp_build_bin_dir%",
+    "activated_cfg": "LLVM_ROOT='%installation_dir%/%fastcomp_build_bin_dir%'",
+    "activated_env": "LLVM_ROOT=%installation_dir%/%fastcomp_build_bin_dir%;EMCC_WASM_BACKEND=1",
+    "cmake_build_type": "Release"
+  },
+
  {
    "id": "clang",
    "version": "tag-e%tag%",
@@ -12,7 +41,7 @@
    "custom_install_script": "build_fastcomp",
    "activated_path": "%installation_dir%/%fastcomp_build_bin_dir%",
    "activated_cfg": "LLVM_ROOT='%installation_dir%/%fastcomp_build_bin_dir%'",
-    "activated_env": "LLVM_ROOT=%installation_dir%/%fastcomp_build_bin_dir%",
+    "activated_env": "LLVM_ROOT=%installation_dir%/%fastcomp_build_bin_dir%;EMCC_WASM_BACKEND=0",
    "cmake_build_type": "Release"
  },
  {
@@ -27,7 +56,7 @@
    "custom_install_script": "build_fastcomp",
    "activated_path": "%installation_dir%/%fastcomp_build_bin_dir%",
    "activated_cfg": "LLVM_ROOT='%installation_dir%/%fastcomp_build_bin_dir%'",
-    "activated_env": "LLVM_ROOT=%installation_dir%/%fastcomp_build_bin_dir%",
+    "activated_env": "LLVM_ROOT=%installation_dir%/%fastcomp_build_bin_dir%;EMCC_WASM_BACKEND=0",
    "cmake_build_type": "Release"
  },
  {
@@ -41,7 +70,7 @@
    "linux_url": "llvm/nightly/linux_64bit/emscripten-llvm-e%nightly-llvm-64bit%.tar.gz",
    "activated_path": "%installation_dir%",
    "activated_cfg": "LLVM_ROOT='%installation_dir%'",
-    "activated_env": "LLVM_ROOT=%installation_dir%"
+    "activated_env": "LLVM_ROOT=%installation_dir%;EMCC_WASM_BACKEND=0"
  },
  {
    "id": "fastcomp-clang",
@@ -54,7 +83,7 @@
    "custom_install_script": "build_fastcomp",
    "activated_path": "%installation_dir%/%fastcomp_build_bin_dir%",
    "activated_cfg": "LLVM_ROOT='%installation_dir%/%fastcomp_build_bin_dir%'",
-    "activated_env": "LLVM_ROOT=%installation_dir%/%fastcomp_build_bin_dir%",
+    "activated_env": "LLVM_ROOT=%installation_dir%/%fastcomp_build_bin_dir%;EMCC_WASM_BACKEND=0",
    "cmake_build_type": "Release"
  },
  {
@@ -68,7 +97,7 @@
    "custom_install_script": "build_fastcomp",
    "activated_path": "%installation_dir%/%fastcomp_build_bin_dir%",
    "activated_cfg": "LLVM_ROOT='%installation_dir%/%fastcomp_build_bin_dir%'",
-    "activated_env": "LLVM_ROOT=%installation_dir%/%fastcomp_build_bin_dir%",
+    "activated_env": "LLVM_ROOT=%installation_dir%/%fastcomp_build_bin_dir%;EMCC_WASM_BACKEND=0",
    "cmake_build_type": "Release"
  },
  {
@@ -82,7 +111,7 @@
    "custom_install_script": "build_fastcomp",
    "activated_path": "%installation_dir%/%fastcomp_build_bin_dir%",
    "activated_cfg": "LLVM_ROOT='%installation_dir%/%fastcomp_build_bin_dir%'",
-    "activated_env": "LLVM_ROOT=%installation_dir%/%fastcomp_build_bin_dir%",
+    "activated_env": "LLVM_ROOT=%installation_dir%/%fastcomp_build_bin_dir%;EMCC_WASM_BACKEND=0",
    "cmake_build_type": "Release"
  },
  {
@@ -96,7 +125,7 @@
    "custom_install_script": "build_fastcomp",
    "activated_path": "%installation_dir%/%fastcomp_build_bin_dir%",
    "activated_cfg": "LLVM_ROOT='%installation_dir%/%fastcomp_build_bin_dir%'",
-    "activated_env": "LLVM_ROOT=%installation_dir%/%fastcomp_build_bin_dir%",
+    "activated_env": "LLVM_ROOT=%installation_dir%/%fastcomp_build_bin_dir%;EMCC_WASM_BACKEND=0",
    "cmake_build_type": "Release"
  },

@@ -141,7 +170,7 @@
    "linux_url": "llvm/tag/linux_64bit/emscripten-llvm-e%precompiled_tag32%.tar.gz",
    "activated_path": "%installation_dir%",
    "activated_cfg": "LLVM_ROOT='%installation_dir%';EMSCRIPTEN_NATIVE_OPTIMIZER='%installation_dir%/optimizer%.exe%';BINARYEN_ROOT='%installation_dir%/binaryen'",
-    "activated_env": "LLVM_ROOT=%installation_dir%;EMSCRIPTEN_NATIVE_OPTIMIZER=%installation_dir%/optimizer%.exe%;BINARYEN_ROOT=%installation_dir%/binaryen"
+    "activated_env": "LLVM_ROOT=%installation_dir%;EMSCRIPTEN_NATIVE_OPTIMIZER=%installation_dir%/optimizer%.exe%;BINARYEN_ROOT=%installation_dir%/binaryen;EMCC_WASM_BACKEND=0"
  },
  {
    "id": "fastcomp-clang",
@@ -153,7 +182,7 @@
    "linux_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/old/linux/emscripten-llvm-e%precompiled_tag64%.tar.gz",
    "activated_path": "%installation_dir%",
    "activated_cfg": "LLVM_ROOT='%installation_dir%';EMSCRIPTEN_NATIVE_OPTIMIZER='%installation_dir%/optimizer%.exe%';BINARYEN_ROOT='%installation_dir%/binaryen'",
-    "activated_env": "LLVM_ROOT=%installation_dir%;EMSCRIPTEN_NATIVE_OPTIMIZER=%installation_dir%/optimizer%.exe%;BINARYEN_ROOT=%installation_dir%/binaryen"
+    "activated_env": "LLVM_ROOT=%installation_dir%;EMSCRIPTEN_NATIVE_OPTIMIZER=%installation_dir%/optimizer%.exe%;BINARYEN_ROOT=%installation_dir%/binaryen;EMCC_WASM_BACKEND=0"
  },
  {
    "id": "node",
@@ -540,6 +569,12 @@
  ],

  "sdks": [
+  {
+    "version": "upstream-incoming",
+    "bitness": 64,
+    "uses": ["llvm-git-master-64bit", "node-12.9.1-64bit", "python-2.7.13.1-64bit", "emscripten-incoming-64bit"],
+    "os": "win"
+  },
  {
    "version": "fastcomp-incoming",
    "bitness": 32,