Enable "emsdk [install|activate] latest-upstream" (#195)

This makes it possible to tell the emsdk to get "latest-upstream", which fetches the latest lkgr from there. This will probably be a common use pattern, I expect we may want to recommend users start trying out the wasm backend that way soon. This will also let us simplify this code: https://github.com/kripken/emscripten/blob/incoming/.circleci/config.yml#L334

Aside from adding the "latest-upstream" alias, this PR has

* A few minor cleanups in the emsdk code.
( Minor restructuring of how we define the upstream stuff in the manifest: It seemed odd to have 3 things (clang, emscripten, binaryen) that are all coming from a single archive from the waterfall. Simpler to have just one - the archive is one big lump, there's no way to download just part of it.
* Also add node 8.9.1 as a dependency of the upstream sdk, which makes things usable out of the box (node.js is the one thing not provided by the waterfall archive).

Dockerfile

@@ -6,11 +6,16 @@ RUN mkdir -p /root/emsdk/
 COPY . /root/emsdk/
 
 RUN cd /root/ \
+ && echo "int main() {}" > hello_world.cpp \
  && apt-get update \
  && apt-get install -y python cmake build-essential openjdk-9-jre-headless \
  && /root/emsdk/emsdk install latest \
  && /root/emsdk/emsdk activate latest \
  && source /root/emsdk/emsdk_env.sh --build=Release \
- && echo "int main() {}" > hello_world.cpp \
+ && emcc hello_world.cpp \
- && emcc hello_world.cpp
+ && /root/emsdk/emsdk update-tags \
+ && /root/emsdk/emsdk install latest-upstream \
+ && /root/emsdk/emsdk activate latest-upstream \
+ && source /root/emsdk/emsdk_env.sh --build=Release \
+ && emcc hello_world.cpp -s WASM_OBJECT_FILES=1
 

emsdk

@@ -1142,6 +1142,9 @@ class Tool:
   def __str__(self):
     return self.name
 
+  def __repr__(self):
+    return self.name
+
   def expand_vars(self, str):
     if WINDOWS and '%MSBuildPlatformsDir%' in str:
       str = str.replace('%MSBuildPlatformsDir%', find_msbuild_dir())
@@ -1524,6 +1527,13 @@ def find_latest_nightly_sdk():
   else:
     return find_latest_nightly_32bit_sdk()
 
+def find_latest_upstream_sdk():
+  waterfall_lkgr = load_waterfall_lkgr()
+  if not waterfall_lkgr:
+    print('Failed to find an upstream lkgr')
+    sys.exit(1)
+  return 'sdk-upstream-%s-64bit' % waterfall_lkgr[0]
+
 # Finds the best-matching python tool for use.
 def find_used_python():
   for t in reversed(tools): # Find newest tool first - those are always at the end of the list.
@@ -1697,10 +1707,13 @@ def download_waterfall_lkgr():
 
 def load_waterfall_lkgr():
   try:
-    data = json.loads(open(os.path.join('upstream', 'lkgr.json'), 'r').read())
+    text = open(sdk_path(os.path.join('upstream', 'lkgr.json')), 'r').read()
+    data = json.loads(text)
     lkgr = data['build']
     return [lkgr]
-  except:
+  except Exception as e:
+    print('Error parsing lkgr.json!')
+    print(str(e))
     return []
 
 def is_string(s):
@@ -1992,7 +2005,7 @@ def construct_env(tools_to_activate, permanent):
   # A core variable EMSDK points to the root of Emscripten SDK directory.
   env_vars_to_add += [('EMSDK', to_unix_path(emsdk_path()))]
 
-  em_config_path = os.path.normpath(os.path.join(emscripten_config_directory, '.emscripten'))
+  em_config_path = os.path.normpath(dot_emscripten_path())
   if not 'EM_CONFIG' in os.environ or to_unix_path(os.environ['EM_CONFIG']) != to_unix_path(em_config_path):
     env_vars_to_add += [('EM_CONFIG', em_config_path)]
   if emscripten_config_directory == emsdk_path():
@@ -2210,6 +2223,8 @@ def main():
         sys.argv[i] = str(find_latest_nightly_32bit_sdk())
       elif sys.argv[i] == 'sdk-nightly-latest-64bit':
         sys.argv[i] = str(find_latest_nightly_64bit_sdk())
+      elif sys.argv[i] == 'latest-upstream' or sys.argv[i] == 'latest-clang-upstream':
+        sys.argv[i] = str(find_latest_upstream_sdk())
 
   if cmd == 'list':
     print('')

emsdk_manifest.json

@@ -191,32 +191,13 @@
     "cmake_build_type": "Release"
   },
   {
-    "id": "clang",
+    "id": "waterfall",
     "version": "upstream-%waterfall-lkgr%",
     "bitness": 64,
     "linux_url": "https://storage.googleapis.com/wasm-llvm/builds/linux/%waterfall-lkgr%/wasm-binaries.tbz2",
     "install_path": "upstream/%waterfall-lkgr%",
-    "activated_path": "%installation_dir%",
+    "activated_path": "%installation_dir%/emscripten",
-    "activated_cfg": "LLVM_ROOT='%installation_dir%/bin'"
+    "activated_cfg": "LLVM_ROOT='%installation_dir%/bin';BINARYEN_ROOT='%installation_dir%'"
-  },
-  {
-    "id": "emscripten",
-    "version": "upstream-%waterfall-lkgr%",
-    "bitness": 64,
-    "linux_url": "https://storage.googleapis.com/wasm-llvm/builds/linux/%waterfall-lkgr%/wasm-binaries.tbz2",
-    "install_path": "upstream/%waterfall-lkgr%",
-    "activated_path": "%installation_dir%",
-    "activated_cfg": "EMSCRIPTEN_ROOT='%installation_dir%/emscripten'"
-  },
-  {
-    "id": "binaryen",
-    "version": "upstream-%waterfall-lkgr%",
-    "bitness": 64,
-    "linux_url": "https://storage.googleapis.com/wasm-llvm/builds/linux/%waterfall-lkgr%/wasm-binaries.tbz2",
-    "install_path": "upstream/%waterfall-lkgr%",
-    "activated_path": "%installation_dir%",
-    "activated_cfg": "BINARYEN_ROOT='%installation_dir%'",
-    "activated_env": "BINARYEN_ROOT=%installation_dir%"
   },
   {
     "id": "clang",
@@ -1565,7 +1546,7 @@
   {
     "version": "upstream-%waterfall-lkgr%",
     "bitness": 64,
-    "uses": ["clang-upstream-%waterfall-lkgr%-64bit", "emscripten-upstream-%waterfall-lkgr%-64bit", "binaryen-upstream-%waterfall-lkgr%-64bit"],
+    "uses": ["waterfall-upstream-%waterfall-lkgr%-64bit", "node-8.9.1-64bit"],
     "os": "linux"
   },
   {