Prepare for emscripten-releases (#237)

Updated to our current initial plan here:

* Add emscripten-releases-tags.txt which is a JSON file with a map of tag name to git hash in the emscripten-releases repo. Right now this file is just checked in here; in the future we may make the emsdk update it from git tags with update-tags etc.
* That file contains a "latest" tag, which is fetched for when getting latest-releases-[upstream|fastcomp]. We may want that to eventually be updated based on the latest green stuff on the bots perhaps.
* Rename the current sdk-[upstream|fastcomp]-* etc. to sdk-waterfall-[upstream\|fastcomp]-*. Then we'll have sdk-releases-[upstream|fastcomp]-* etc. This is just an internal name change - emsdk install latest-upstream still works as before, and still uses the waterfall (so we don't break our github CI).
* Add support for emsdk install latest-releases-[upstream\|fastcomp]
* Add a test using the actual emscripten-releases builds

Dockerfile

@@ -19,7 +19,7 @@ RUN cd /root/ \
  && /root/emsdk/emsdk install latest-upstream \
  && /root/emsdk/emsdk activate latest-upstream \
  && source /root/emsdk/emsdk_env.sh --build=Release \
- && emcc hello_world.cpp -s WASM_OBJECT_FILES=1 \
+ && emcc hello_world.cpp \
  && python -c "import os ; assert open(os.path.expanduser('~/.emscripten')).read().count('LLVM_ROOT') == 1" \
  && python -c "import os ; assert 'upstream' in open(os.path.expanduser('~/.emscripten')).read()" \
  && echo "test fastcomp (waterfall)" \
@@ -28,5 +28,16 @@ RUN cd /root/ \
  && source /root/emsdk/emsdk_env.sh --build=Release \
  && emcc hello_world.cpp \
  && emcc hello_world.cpp -s WASM=0 \
+ && emcc --clear-cache \
+ && echo "test latest-releases-upstream" \
+ && /root/emsdk/emsdk install latest-releases-upstream \
+ && /root/emsdk/emsdk activate latest-releases-upstream \
+ && source /root/emsdk/emsdk_env.sh --build=Release \
+ && emcc hello_world.cpp \
+ && echo "test latest-releases-fastcomp" \
+ && /root/emsdk/emsdk install latest-releases-fastcomp \
+ && /root/emsdk/emsdk activate latest-releases-fastcomp \
+ && source /root/emsdk/emsdk_env.sh --build=Release \
+ && emcc hello_world.cpp \
  && echo "test binaryen source build" \
  && /root/emsdk/emsdk install --build=Release binaryen-master-64bit

emscripten-releases-tags.txt

@@ -0,0 +1,4 @@
+{
+  "latest": "0ea08ee8a5f0524b4c9ac94de4f161ceeacb3e74"
+}
+

emsdk

@@ -88,6 +88,14 @@ ENABLE_WASM = False
 ENABLE_LLVM_ASSERTIONS = 'auto'
 
 
+def os_name():
+  if WINDOWS: return 'win'
+  elif LINUX: return 'linux'
+  elif OSX: return 'osx'
+  else:
+    raise Exception('unknown OS')
+
+
 def to_unix_path(p):
   return p.replace('\\', '/')
 
@@ -1653,15 +1661,12 @@ def find_latest_waterfall_sdk(which):
   if not waterfall_lkgr:
     print('Failed to find an upstream lkgr')
     sys.exit(1)
-  return 'sdk-%s-%s-64bit' % (which, waterfall_lkgr[0])
+  return 'sdk-waterfall-%s-%s-64bit' % (which, waterfall_lkgr[0])
 
 
-def find_latest_upstream_sdk():
+def find_latest_releases_sdk(which):
-  return find_latest_waterfall_sdk('upstream')
+  releases_info = load_releases_info()
-
+  return 'sdk-releases-%s-%s-64bit' % (which, releases_info['latest'])
-
-def find_latest_fastcomp_sdk():
-  return find_latest_waterfall_sdk('fastcomp')
 
 
 # Finds the best-matching python tool for use.
@@ -1732,26 +1737,17 @@ def fetch_emscripten_tags():
     else:
       print('Done. No tagged Binaryen releases available.')
 
-  def os_name_for_llvm_location():
-    if WINDOWS: return 'win'
-    if LINUX: return 'linux'
-    if OSX: return 'osx'
-
-  def os_name_for_emscripten_location():
-    if WINDOWS: return 'win'
-    else: return 'linux'
-
 #  Emscripten Nightlies support has been removed, clear the list of known Nightlies locally.
 #  print('Fetching all precompiled Nightly versions..')
-#  download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/nightly/' + os_name_for_llvm_location() + '_32bit/index.txt', 'llvm-nightlies-32bit.txt', download_even_if_exists=True)
+#  download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/nightly/' + os_name() + '_32bit/index.txt', 'llvm-nightlies-32bit.txt', download_even_if_exists=True)
-#  download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/nightly/' + os_name_for_llvm_location() + '_64bit/index.txt', 'llvm-nightlies-64bit.txt', download_even_if_exists=True)
+#  download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/nightly/' + os_name() + '_64bit/index.txt', 'llvm-nightlies-64bit.txt', download_even_if_exists=True)
-#  download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/emscripten/nightly/' + os_name_for_emscripten_location() + '/index.txt', 'emscripten-nightlies.txt', download_even_if_exists=True)
+#  download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/emscripten/nightly/' + os_name() + '/index.txt', 'emscripten-nightlies.txt', download_even_if_exists=True)
   for f in ['llvm-nightlies-32bit.txt', 'llvm-nightlies-64bit.txt', 'emscripten-nightlies.txt']:
     if os.path.isfile(f): os.remove(f)
 
   print('Fetching all precompiled tagged releases..')
-  download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/tag/' + os_name_for_llvm_location() + '_32bit/index.txt', 'llvm-tags-32bit.txt', download_even_if_exists=True)
+  download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/tag/' + os_name() + '_32bit/index.txt', 'llvm-tags-32bit.txt', download_even_if_exists=True)
-  download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/tag/' + os_name_for_llvm_location() + '_64bit/index.txt', 'llvm-tags-64bit.txt', download_even_if_exists=True)
+  download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/tag/' + os_name() + '_64bit/index.txt', 'llvm-tags-64bit.txt', download_even_if_exists=True)
   download_waterfall_lkgr()
 
   if not git:
@@ -1849,7 +1845,7 @@ def load_llvm_precompiled_tags_64bit():
 
 
 def download_waterfall_lkgr():
-  lkgr_url = 'https://storage.googleapis.com/wasm-llvm/builds/linux/lkgr.json'
+  lkgr_url = 'https://storage.googleapis.com/wasm-llvm/builds/%s/lkgr.json' % os_name()
   download_file(lkgr_url, 'upstream', download_even_if_exists=True)
 
 
@@ -1865,6 +1861,23 @@ def load_waterfall_lkgr():
     return []
 
 
+# Load the json info for emscripten-releases.
+def load_releases_info():
+  try:
+    text = open(sdk_path('emscripten-releases-tags.txt'), 'r').read()
+    return json.loads(text)
+  except Exception as e:
+    print('Error parsing emscripten-releases-tags.txt!')
+    print(str(e))
+    sys.exit(1)
+
+
+# Get a list of tags for emscripten-releases.
+def load_releases_tags():
+  info = load_releases_info()
+  return info.values()
+
+
 def is_string(s):
   if sys.version_info[0] >= 3:
     return isinstance(s, str)
@@ -1889,6 +1902,7 @@ def load_sdk_manifest():
   llvm_64bit_nightlies = list(reversed(load_llvm_64bit_nightlies()))
   emscripten_nightlies = list(reversed(load_emscripten_nightlies()))
   waterfall_lkgr = load_waterfall_lkgr()
+  releases_tags = load_releases_tags()
 
   def dependencies_exist(sdk):
     for tool_name in sdk.uses:
@@ -1960,6 +1974,7 @@ def load_sdk_manifest():
       elif '%nightly-llvm-32bit%' in t.version: expand_category_param('%nightly-llvm-32bit%', llvm_32bit_nightlies, t, is_sdk=False)
       elif '%nightly-emscripten%' in t.version: expand_category_param('%nightly-emscripten%', emscripten_nightlies, t, is_sdk=False)
       elif '%waterfall-lkgr%' in t.version: expand_category_param('%waterfall-lkgr%', waterfall_lkgr, t, is_sdk=False)
+      elif '%releases-tag%' in t.version: expand_category_param('%releases-tag%', releases_tags, t, is_sdk=False)
       else:
         add_tool(t)
 
@@ -1978,6 +1993,7 @@ def load_sdk_manifest():
       elif '%nightly-llvm-32bit%' in sdk.version: expand_category_param('%nightly-llvm-32bit%', llvm_32bit_nightlies, sdk, is_sdk=True)
       elif '%nightly-emscripten%' in sdk.version: expand_category_param('%nightly-emscripten%', emscripten_nightlies, sdk, is_sdk=True)
       elif '%waterfall-lkgr%' in sdk.version: expand_category_param('%waterfall-lkgr%', waterfall_lkgr, sdk, is_sdk=True)
+      elif '%releases-tag%' in sdk.version: expand_category_param('%releases-tag%', releases_tags, sdk, is_sdk=True)
       else:
         add_sdk(sdk)
 
@@ -2388,9 +2404,13 @@ def main():
       elif sys.argv[i] == 'sdk-nightly-latest-64bit':
         sys.argv[i] = str(find_latest_nightly_64bit_sdk())
       elif sys.argv[i] == 'latest-upstream' or sys.argv[i] == 'latest-clang-upstream':
-        sys.argv[i] = str(find_latest_upstream_sdk())
+        sys.argv[i] = str(find_latest_waterfall_sdk('upstream'))
       elif sys.argv[i] == 'latest-fastcomp':
-        sys.argv[i] = str(find_latest_fastcomp_sdk())
+        sys.argv[i] = str(find_latest_waterfall_sdk('fastcomp'))
+      elif sys.argv[i] == 'latest-releases-upstream':
+        sys.argv[i] = str(find_latest_releases_sdk('upstream'))
+      elif sys.argv[i] == 'latest-releases-fastcomp':
+        sys.argv[i] = str(find_latest_releases_sdk('fastcomp'))
 
   if cmd == 'list':
     print('')

emsdk_manifest.json

@@ -210,6 +210,32 @@
     "activated_path": "%installation_dir%/emscripten",
     "activated_cfg": "LLVM_ROOT='%installation_dir%/fastcomp/bin';BINARYEN_ROOT='%installation_dir%'"
   },
+
+  {
+    "id": "releases",
+    "version": "upstream-%releases-tag%",
+    "bitness": 64,
+    "linux_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/%releases-tag%/wasm-binaries.tbz2",
+    "osx_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/%releases-tag%/wasm-binaries.tbz2",
+    "windows_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/windows/%releases-tag%/wasm-binaries.tbz2",
+    "zipfile_prefix": "%releases-tag%-",
+    "install_path": "upstream/%releases-tag%",
+    "activated_path": "%installation_dir%/emscripten",
+    "activated_cfg": "LLVM_ROOT='%installation_dir%/bin';BINARYEN_ROOT='%installation_dir%'"
+  },
+  {
+    "id": "releases",
+    "version": "fastcomp-%releases-tag%",
+    "bitness": 64,
+    "linux_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/%releases-tag%/wasm-binaries.tbz2",
+    "osx_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/%releases-tag%/wasm-binaries.tbz2",
+    "windows_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/windows/%releases-tag%/wasm-binaries.zip",
+    "zipfile_prefix": "%releases-tag%-",
+    "install_path": "fastcomp/%releases-tag%",
+    "activated_path": "%installation_dir%/emscripten",
+    "activated_cfg": "LLVM_ROOT='%installation_dir%/fastcomp/bin';BINARYEN_ROOT='%installation_dir%'"
+  },
+
   {
     "id": "clang",
     "version": "e1.13.0",
@@ -1585,17 +1611,53 @@
     "os": "win"
   },
   {
-    "version": "upstream-%waterfall-lkgr%",
+    "version": "waterfall-upstream-%waterfall-lkgr%",
     "bitness": 64,
     "uses": ["waterfall-upstream-%waterfall-lkgr%-64bit", "node-8.9.1-64bit"],
     "os": "linux"
   },
   {
-    "version": "fastcomp-%waterfall-lkgr%",
+    "version": "waterfall-fastcomp-%waterfall-lkgr%",
     "bitness": 64,
     "uses": ["waterfall-fastcomp-%waterfall-lkgr%-64bit", "node-8.9.1-64bit"],
     "os": "linux"
   },
+  {
+    "version": "releases-upstream-%releases-tag%",
+    "bitness": 64,
+    "uses": ["releases-upstream-%releases-tag%-64bit", "node-8.9.1-64bit"],
+    "os": "linux"
+  },
+  {
+    "version": "releases-upstream-%releases-tag%",
+    "bitness": 64,
+    "uses": ["releases-upstream-%releases-tag%-64bit", "node-8.9.1-64bit"],
+    "os": "osx"
+  },
+  {
+    "version": "releases-upstream-%releases-tag%",
+    "bitness": 64,
+    "uses": ["releases-upstream-%releases-tag%-64bit", "node-8.9.1-64bit"],
+    "os": "win"
+  },
+  {
+    "version": "releases-fastcomp-%releases-tag%",
+    "bitness": 64,
+    "uses": ["releases-fastcomp-%releases-tag%-64bit", "node-8.9.1-64bit"],
+    "os": "linux"
+  },
+  {
+    "version": "releases-fastcomp-%releases-tag%",
+    "bitness": 64,
+    "uses": ["releases-fastcomp-%releases-tag%-64bit", "node-8.9.1-64bit"],
+    "os": "osx"
+  },
+  {
+    "version": "releases-fastcomp-%releases-tag%",
+    "bitness": 64,
+    "uses": ["releases-fastcomp-%releases-tag%-64bit", "node-8.9.1-64bit"],
+    "os": "win"
+  },
   {
     "version": "%precompiled_tag32%",
     "bitness": 32,