Prepare for emscripten-releases (#237)

Updated to our current initial plan here: * Add emscripten-releases-tags.txt which is a JSON file with a map of tag name to git hash in the emscripten-releases repo. Right now this file is just checked in here; in the future we may make the emsdk update it from git tags with update-tags etc. * That file contains a "latest" tag, which is fetched for when getting latest-releases-[upstream|fastcomp]. We may want that to eventually be updated based on the latest green stuff on the bots perhaps. * Rename the current sdk-[upstream|fastcomp]-* etc. to sdk-waterfall-[upstream\|fastcomp]-*. Then we'll have sdk-releases-[upstream|fastcomp]-* etc. This is just an internal name change - emsdk install latest-upstream still works as before, and still uses the waterfall (so we don't break our github CI). * Add support for emsdk install latest-releases-[upstream\|fastcomp] * Add a test using the actual emscripten-releases builds
2019-05-22 17:23:04 -07:00
parent 751355a267
commit e2434062a8
4 changed files with 124 additions and 27 deletions
--- a/13
+++ b/13
@@ -19,7 +19,7 @@ RUN cd /root/ \
 && /root/emsdk/emsdk install latest-upstream \
 && /root/emsdk/emsdk activate latest-upstream \
 && source /root/emsdk/emsdk_env.sh --build=Release \
- && emcc hello_world.cpp -s WASM_OBJECT_FILES=1 \
+ && emcc hello_world.cpp \
 && python -c "import os ; assert open(os.path.expanduser('~/.emscripten')).read().count('LLVM_ROOT') == 1" \
 && python -c "import os ; assert 'upstream' in open(os.path.expanduser('~/.emscripten')).read()" \
 && echo "test fastcomp (waterfall)" \
@@ -28,5 +28,16 @@ RUN cd /root/ \
 && source /root/emsdk/emsdk_env.sh --build=Release \
 && emcc hello_world.cpp \
 && emcc hello_world.cpp -s WASM=0 \
+ && emcc --clear-cache \
+ && echo "test latest-releases-upstream" \
+ && /root/emsdk/emsdk install latest-releases-upstream \
+ && /root/emsdk/emsdk activate latest-releases-upstream \
+ && source /root/emsdk/emsdk_env.sh --build=Release \
+ && emcc hello_world.cpp \
+ && echo "test latest-releases-fastcomp" \
+ && /root/emsdk/emsdk install latest-releases-fastcomp \
+ && /root/emsdk/emsdk activate latest-releases-fastcomp \
+ && source /root/emsdk/emsdk_env.sh --build=Release \
+ && emcc hello_world.cpp \
 && echo "test binaryen source build" \
 && /root/emsdk/emsdk install --build=Release binaryen-master-64bit
--- a/emscripten-releases-tags.txt
+++ b/emscripten-releases-tags.txt
@@ -0,0 +1,4 @@
+{
+  "latest": "0ea08ee8a5f0524b4c9ac94de4f161ceeacb3e74"
+}
+
--- a/68
+++ b/68
@@ -88,6 +88,14 @@ ENABLE_WASM = False
 ENABLE_LLVM_ASSERTIONS = 'auto'


+def os_name():
+  if WINDOWS: return 'win'
+  elif LINUX: return 'linux'
+  elif OSX: return 'osx'
+  else:
+    raise Exception('unknown OS')
+
+
 def to_unix_path(p):
  return p.replace('\\', '/')

@@ -1653,15 +1661,12 @@ def find_latest_waterfall_sdk(which):
  if not waterfall_lkgr:
    print('Failed to find an upstream lkgr')
    sys.exit(1)
-  return 'sdk-%s-%s-64bit' % (which, waterfall_lkgr[0])
+  return 'sdk-waterfall-%s-%s-64bit' % (which, waterfall_lkgr[0])


-def find_latest_upstream_sdk():
-  return find_latest_waterfall_sdk('upstream')
-
-
-def find_latest_fastcomp_sdk():
-  return find_latest_waterfall_sdk('fastcomp')
+def find_latest_releases_sdk(which):
+  releases_info = load_releases_info()
+  return 'sdk-releases-%s-%s-64bit' % (which, releases_info['latest'])


 # Finds the best-matching python tool for use.
@@ -1732,26 +1737,17 @@ def fetch_emscripten_tags():
    else:
      print('Done. No tagged Binaryen releases available.')

-  def os_name_for_llvm_location():
-    if WINDOWS: return 'win'
-    if LINUX: return 'linux'
-    if OSX: return 'osx'
-
-  def os_name_for_emscripten_location():
-    if WINDOWS: return 'win'
-    else: return 'linux'
-
 #  Emscripten Nightlies support has been removed, clear the list of known Nightlies locally.
 #  print('Fetching all precompiled Nightly versions..')
-#  download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/nightly/' + os_name_for_llvm_location() + '_32bit/index.txt', 'llvm-nightlies-32bit.txt', download_even_if_exists=True)
-#  download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/nightly/' + os_name_for_llvm_location() + '_64bit/index.txt', 'llvm-nightlies-64bit.txt', download_even_if_exists=True)
-#  download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/emscripten/nightly/' + os_name_for_emscripten_location() + '/index.txt', 'emscripten-nightlies.txt', download_even_if_exists=True)
+#  download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/nightly/' + os_name() + '_32bit/index.txt', 'llvm-nightlies-32bit.txt', download_even_if_exists=True)
+#  download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/nightly/' + os_name() + '_64bit/index.txt', 'llvm-nightlies-64bit.txt', download_even_if_exists=True)
+#  download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/emscripten/nightly/' + os_name() + '/index.txt', 'emscripten-nightlies.txt', download_even_if_exists=True)
  for f in ['llvm-nightlies-32bit.txt', 'llvm-nightlies-64bit.txt', 'emscripten-nightlies.txt']:
    if os.path.isfile(f): os.remove(f)

  print('Fetching all precompiled tagged releases..')
-  download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/tag/' + os_name_for_llvm_location() + '_32bit/index.txt', 'llvm-tags-32bit.txt', download_even_if_exists=True)
-  download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/tag/' + os_name_for_llvm_location() + '_64bit/index.txt', 'llvm-tags-64bit.txt', download_even_if_exists=True)
+  download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/tag/' + os_name() + '_32bit/index.txt', 'llvm-tags-32bit.txt', download_even_if_exists=True)
+  download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/tag/' + os_name() + '_64bit/index.txt', 'llvm-tags-64bit.txt', download_even_if_exists=True)
  download_waterfall_lkgr()

  if not git:
@@ -1849,7 +1845,7 @@ def load_llvm_precompiled_tags_64bit():


 def download_waterfall_lkgr():
-  lkgr_url = 'https://storage.googleapis.com/wasm-llvm/builds/linux/lkgr.json'
+  lkgr_url = 'https://storage.googleapis.com/wasm-llvm/builds/%s/lkgr.json' % os_name()
  download_file(lkgr_url, 'upstream', download_even_if_exists=True)


@@ -1865,6 +1861,23 @@ def load_waterfall_lkgr():
    return []


+# Load the json info for emscripten-releases.
+def load_releases_info():
+  try:
+    text = open(sdk_path('emscripten-releases-tags.txt'), 'r').read()
+    return json.loads(text)
+  except Exception as e:
+    print('Error parsing emscripten-releases-tags.txt!')
+    print(str(e))
+    sys.exit(1)
+
+
+# Get a list of tags for emscripten-releases.
+def load_releases_tags():
+  info = load_releases_info()
+  return info.values()
+
+
 def is_string(s):
  if sys.version_info[0] >= 3:
    return isinstance(s, str)
@@ -1889,6 +1902,7 @@ def load_sdk_manifest():
  llvm_64bit_nightlies = list(reversed(load_llvm_64bit_nightlies()))
  emscripten_nightlies = list(reversed(load_emscripten_nightlies()))
  waterfall_lkgr = load_waterfall_lkgr()
+  releases_tags = load_releases_tags()

  def dependencies_exist(sdk):
    for tool_name in sdk.uses:
@@ -1960,6 +1974,7 @@ def load_sdk_manifest():
      elif '%nightly-llvm-32bit%' in t.version: expand_category_param('%nightly-llvm-32bit%', llvm_32bit_nightlies, t, is_sdk=False)
      elif '%nightly-emscripten%' in t.version: expand_category_param('%nightly-emscripten%', emscripten_nightlies, t, is_sdk=False)
      elif '%waterfall-lkgr%' in t.version: expand_category_param('%waterfall-lkgr%', waterfall_lkgr, t, is_sdk=False)
+      elif '%releases-tag%' in t.version: expand_category_param('%releases-tag%', releases_tags, t, is_sdk=False)
      else:
        add_tool(t)

@@ -1978,6 +1993,7 @@ def load_sdk_manifest():
      elif '%nightly-llvm-32bit%' in sdk.version: expand_category_param('%nightly-llvm-32bit%', llvm_32bit_nightlies, sdk, is_sdk=True)
      elif '%nightly-emscripten%' in sdk.version: expand_category_param('%nightly-emscripten%', emscripten_nightlies, sdk, is_sdk=True)
      elif '%waterfall-lkgr%' in sdk.version: expand_category_param('%waterfall-lkgr%', waterfall_lkgr, sdk, is_sdk=True)
+      elif '%releases-tag%' in sdk.version: expand_category_param('%releases-tag%', releases_tags, sdk, is_sdk=True)
      else:
        add_sdk(sdk)

@@ -2388,9 +2404,13 @@ def main():
      elif sys.argv[i] == 'sdk-nightly-latest-64bit':
        sys.argv[i] = str(find_latest_nightly_64bit_sdk())
      elif sys.argv[i] == 'latest-upstream' or sys.argv[i] == 'latest-clang-upstream':
-        sys.argv[i] = str(find_latest_upstream_sdk())
+        sys.argv[i] = str(find_latest_waterfall_sdk('upstream'))
      elif sys.argv[i] == 'latest-fastcomp':
-        sys.argv[i] = str(find_latest_fastcomp_sdk())
+        sys.argv[i] = str(find_latest_waterfall_sdk('fastcomp'))
+      elif sys.argv[i] == 'latest-releases-upstream':
+        sys.argv[i] = str(find_latest_releases_sdk('upstream'))
+      elif sys.argv[i] == 'latest-releases-fastcomp':
+        sys.argv[i] = str(find_latest_releases_sdk('fastcomp'))

  if cmd == 'list':
    print('')
--- a/emsdk_manifest.json
+++ b/emsdk_manifest.json
@@ -210,6 +210,32 @@
    "activated_path": "%installation_dir%/emscripten",
    "activated_cfg": "LLVM_ROOT='%installation_dir%/fastcomp/bin';BINARYEN_ROOT='%installation_dir%'"
  },
+
+  {
+    "id": "releases",
+    "version": "upstream-%releases-tag%",
+    "bitness": 64,
+    "linux_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/%releases-tag%/wasm-binaries.tbz2",
+    "osx_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/%releases-tag%/wasm-binaries.tbz2",
+    "windows_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/windows/%releases-tag%/wasm-binaries.tbz2",
+    "zipfile_prefix": "%releases-tag%-",
+    "install_path": "upstream/%releases-tag%",
+    "activated_path": "%installation_dir%/emscripten",
+    "activated_cfg": "LLVM_ROOT='%installation_dir%/bin';BINARYEN_ROOT='%installation_dir%'"
+  },
+  {
+    "id": "releases",
+    "version": "fastcomp-%releases-tag%",
+    "bitness": 64,
+    "linux_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/%releases-tag%/wasm-binaries.tbz2",
+    "osx_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/%releases-tag%/wasm-binaries.tbz2",
+    "windows_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/windows/%releases-tag%/wasm-binaries.zip",
+    "zipfile_prefix": "%releases-tag%-",
+    "install_path": "fastcomp/%releases-tag%",
+    "activated_path": "%installation_dir%/emscripten",
+    "activated_cfg": "LLVM_ROOT='%installation_dir%/fastcomp/bin';BINARYEN_ROOT='%installation_dir%'"
+  },
+
  {
    "id": "clang",
    "version": "e1.13.0",
@@ -1585,17 +1611,53 @@
    "os": "win"
  },
  {
-    "version": "upstream-%waterfall-lkgr%",
+    "version": "waterfall-upstream-%waterfall-lkgr%",
    "bitness": 64,
    "uses": ["waterfall-upstream-%waterfall-lkgr%-64bit", "node-8.9.1-64bit"],
    "os": "linux"
  },
  {
-    "version": "fastcomp-%waterfall-lkgr%",
+    "version": "waterfall-fastcomp-%waterfall-lkgr%",
    "bitness": 64,
    "uses": ["waterfall-fastcomp-%waterfall-lkgr%-64bit", "node-8.9.1-64bit"],
    "os": "linux"
  },
+  {
+    "version": "releases-upstream-%releases-tag%",
+    "bitness": 64,
+    "uses": ["releases-upstream-%releases-tag%-64bit", "node-8.9.1-64bit"],
+    "os": "linux"
+  },
+  {
+    "version": "releases-upstream-%releases-tag%",
+    "bitness": 64,
+    "uses": ["releases-upstream-%releases-tag%-64bit", "node-8.9.1-64bit"],
+    "os": "osx"
+  },
+  {
+    "version": "releases-upstream-%releases-tag%",
+    "bitness": 64,
+    "uses": ["releases-upstream-%releases-tag%-64bit", "node-8.9.1-64bit"],
+    "os": "win"
+  },
+  {
+    "version": "releases-fastcomp-%releases-tag%",
+    "bitness": 64,
+    "uses": ["releases-fastcomp-%releases-tag%-64bit", "node-8.9.1-64bit"],
+    "os": "linux"
+  },
+  {
+    "version": "releases-fastcomp-%releases-tag%",
+    "bitness": 64,
+    "uses": ["releases-fastcomp-%releases-tag%-64bit", "node-8.9.1-64bit"],
+    "os": "osx"
+  },
+  {
+    "version": "releases-fastcomp-%releases-tag%",
+    "bitness": 64,
+    "uses": ["releases-fastcomp-%releases-tag%-64bit", "node-8.9.1-64bit"],
+    "os": "win"
+  },
  {
    "version": "%precompiled_tag32%",
    "bitness": 32,