Add support for installing precompiled nightly LLVM/Emscripten/SDK versions.

emscripten-tags.txt

@@ -77,4 +77,7 @@
 1.36.5
 1.36.6
 1.36.7
-1.36.8
+1.36.8
+1.36.9
+1.36.10
+1.36.11

emsdk

@@ -373,7 +373,7 @@ def path_points_to_directory(path):
   if no_suffix:
     return True
   suffix = path[last_dot:]
-  if suffix == '.exe' or suffix == '.zip': # Very simple logic for the only file suffixes used by emsdk downloader. Other suffixes, like 'clang-3.2' are treated as dirs.
+  if suffix == '.exe' or suffix == '.zip' or suffix == '.txt': # Very simple logic for the only file suffixes used by emsdk downloader. Other suffixes, like 'clang-3.2' are treated as dirs.
     return False
   else:
     return True
@@ -947,6 +947,9 @@ class Tool:
     if hasattr(self, 'uses'):
       for tool_name in self.uses:
         tool = find_tool(tool_name)
+        if not tool:
+          if VERBOSE: print('Tool ' + str(self) + ' depends on ' + tool_name + ' which does not exist!')
+          continue
         if tool.needs_compilation(): return True
 
     return False
@@ -1326,6 +1329,19 @@ def fetch_emscripten_tags():
   else:
     print('Done. No tagged releases available.')
 
+  def os_name_for_llvm_location():
+    if WINDOWS: return 'win'
+    if LINUX: return 'linux'
+    if OSX: return 'osx'
+  def os_name_for_emscripten_location():
+    if WINDOWS: return 'win'
+    else: return 'linux'
+
+  print('Fetching all precompiled Nightly versions..')
+  download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/nightly/' + os_name_for_llvm_location() + '_32bit/index.txt', 'llvm-nightlies-32bit.txt', download_even_if_exists=True)
+  download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/nightly/' + os_name_for_llvm_location() + '_64bit/index.txt', 'llvm-nightlies-64bit.txt', download_even_if_exists=True)
+  download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/emscripten/nightly/' + os_name_for_emscripten_location() + '/index.txt', 'emscripten-nightlies.txt', download_even_if_exists=True)
+
 def update_emsdk():
   if WINDOWS:
     download_and_unzip(urljoin(emsdk_packages_url, 'emsdk_windows_update.zip'), emsdk_path(), download_even_if_exists=True)
@@ -1343,8 +1359,35 @@ def load_emscripten_tags():
   except:
     return []
 
+def remove_prefix(s, prefix):
+  if s.startswith(prefix): return s[len(prefix):]
+  else: return s
+
+def remove_suffix(s, suffix):
+  if s.endswith(suffix): return s[:len(s)-len(suffix)]
+  else: return s
+
+# Kind should be one of: 'llvm-nightlies-32bit.txt', 'llvm-nightlies-64bit.txt', 'emscripten-nightlies.txt'
+def load_nightlies_list(kind):
+  try:
+    items = open(sdk_path(kind), 'r').read().split('\n')
+    items = map(lambda x: remove_suffix(remove_suffix(remove_prefix(remove_prefix(x, 'emscripten-llvm-e'), 'emscripten-nightly-'), '.tar.gz'), '.zip').strip(), items)
+    items = filter(lambda x: 'latest' not in x and len(x) > 0, items)
+    return items
+  except:
+    return []
+
+def load_llvm_32bit_nightlies():
+  return load_nightlies_list('llvm-nightlies-32bit.txt')
+
+def load_llvm_64bit_nightlies():
+  return load_nightlies_list('llvm-nightlies-64bit.txt')
+
+def load_emscripten_nightlies():
+  return load_nightlies_list('emscripten-nightlies.txt')
+
 def load_sdk_manifest():
-  global tools
+  global tools, sdks
   try:
     manifest = json.loads(open(sdk_path("emsdk_manifest.json"), "r").read())
   except Exception as e:
@@ -1353,6 +1396,34 @@ def load_sdk_manifest():
     return
 
   emscripten_tags = load_emscripten_tags()
+  llvm_32bit_nightlies = load_llvm_32bit_nightlies()
+  llvm_64bit_nightlies = load_llvm_64bit_nightlies()
+  emscripten_nightlies = load_emscripten_nightlies()
+
+  def dependencies_exist(sdk):
+    for tool_name in sdk.uses:
+      tool = find_tool(tool_name)
+      if not tool: return False
+    return True
+
+  # A 'category parameter' is a %foo%-encoded identifier that specifies
+  # a class of tools instead of just one tool, e.g. %tag% or %nightly..%
+  def expand_category_param(param, category_list, t, is_sdk):
+    global tools, sdks
+    for i in range(len(category_list)):
+      ver = category_list[i]
+      t2 = copy.copy(t)
+      for p, v in vars(t2).iteritems():
+        if isinstance(v, basestring) and param in v:
+          t2.__dict__[p] = v.replace(param, ver)
+      t2.is_old = i < len(category_list) - 2
+      if hasattr(t2, 'uses'):
+        t2.uses = map((lambda x: x.replace(param, ver)), t2.uses)
+      if is_sdk:
+        if dependencies_exist(t2):
+          sdks.append(t2)
+      else:
+        tools.append(t2)
 
   for tool in manifest['tools']:
     t = Tool(tool)
@@ -1360,16 +1431,15 @@ def load_sdk_manifest():
       if not hasattr(t, 'is_old'):
         t.is_old = False
 
+      # Expand the metapackages that refer to tags or nightlies.
       if '%tag%' in t.version:
-        # Expand the metapackage that refers to all tags.
-        for i in range(len(emscripten_tags)):
-          tag = emscripten_tags[i]
-          t2 = copy.copy(t)
-          for p, v in vars(t2).iteritems():
-            if isinstance(v, basestring) and '%tag%' in v:
-              t2.__dict__[p] = v.replace('%tag%', tag)
-          t2.is_old = i < len(emscripten_tags) - 2
-          tools.append(t2)
+        expand_category_param('%tag%', emscripten_tags, t, is_sdk=False)
+      elif '%nightly-llvm-64bit%' in t.version:
+        expand_category_param('%nightly-llvm-64bit%', llvm_64bit_nightlies, t, is_sdk=False)
+      elif '%nightly-llvm-32bit%' in t.version:
+        expand_category_param('%nightly-llvm-32bit%', llvm_32bit_nightlies, t, is_sdk=False)
+      elif '%nightly-emscripten%' in t.version:
+        expand_category_param('%nightly-emscripten%', emscripten_nightlies, t, is_sdk=False)
       else:
         tools.append(t)
 
@@ -1381,16 +1451,13 @@ def load_sdk_manifest():
         sdk.is_old = False
 
       if '%tag%' in sdk.version:
-        # Expand the metapackage that refers to all tags.
-        for i in range(len(emscripten_tags)):
-          tag = emscripten_tags[i]
-          sdk2 = copy.copy(sdk)
-          for p, v in vars(sdk2).iteritems():
-            if isinstance(v, basestring) and '%tag%' in v:
-              sdk2.__dict__[p] = v.replace('%tag%', tag)
-          sdk2.is_old = i < len(emscripten_tags) - 2
-          sdk2.uses = map((lambda x: x.replace('%tag%', tag)), sdk2.uses)
-          sdks.append(sdk2)
+        expand_category_param('%tag%', emscripten_tags, sdk, is_sdk=True)
+      elif '%nightly-llvm-64bit%' in sdk.version:
+        expand_category_param('%nightly-llvm-64bit%', llvm_64bit_nightlies, sdk, is_sdk=True)
+      elif '%nightly-llvm-32bit%' in sdk.version:
+        expand_category_param('%nightly-llvm-32bit%', llvm_32bit_nightlies, sdk, is_sdk=True)
+      elif '%nightly-emscripten%' in sdk.version:
+        expand_category_param('%nightly-emscripten%', emscripten_nightlies, sdk, is_sdk=True)
       else:
         sdks.append(sdk)
 
@@ -1777,7 +1844,7 @@ def main():
           active = '*' if sdk.is_active() else ' '
           print('    ' + active + '    {0: <25}'.format(str(sdk)) + installed)
         print('')
-      print('The following precompiled SDKs are available for download:')
+      print('The following precompiled SDKs are available for download: (Run "./emsdk update" to pull in the latest list)')
       print_sdks(find_sdks(False))
       print('The following SDKs can be compiled from source:')
       print_sdks(find_sdks(True))

emsdk_manifest.json

@@ -82,6 +82,17 @@
     "activated_cfg": "LLVM_ROOT='%installation_dir%/%fastcomp_build_bin_dir%'",
     "cmake_build_type": "RelWithDebInfo"
   },
+  {
+    "id": "clang",
+    "version": "nightly-e%nightly-llvm-64bit%",
+    "bitness": 64,
+    "append_bitness": false,
+    "windows_url": "llvm/nightly/win_64bit/emscripten-llvm-e%nightly-llvm-64bit%.zip",
+    "osx_url": "llvm/nightly/osx_64bit/emscripten-llvm-e%nightly-llvm-64bit%.tar.gz",
+    "linux_url": "llvm/nightly/linux_64bit/emscripten-llvm-e%nightly-llvm-64bit%.tar.gz",
+    "activated_path": "%installation_dir%",
+    "activated_cfg": "LLVM_ROOT='%installation_dir%'"
+  },
   {
     "id": "clang",
     "version": "incoming",
@@ -641,6 +652,15 @@
     "custom_is_installed_script": "is_optimizer_installed",
     "custom_uninstall_script": "uninstall_optimizer"
   },
+  {
+    "id": "emscripten",
+    "version": "nightly-%nightly-emscripten%",
+    "windows_url": "emscripten/nightly/win/emscripten-nightly-%nightly-emscripten%.zip",
+    "unix_url": "emscripten/nightly/linux/emscripten-nightly-%nightly-emscripten%.tar.gz",
+    "activated_cfg": "EMSCRIPTEN_ROOT='%installation_dir%'",
+    "activated_path": "%installation_dir%",
+    "activated_env": "EMSCRIPTEN=%installation_dir%"
+  },
   {
     "id": "emscripten",
     "version": "incoming",
@@ -850,6 +870,12 @@
     "uses": ["clang-tag-e%tag%-64bit", "node-4.1.1-64bit", "emscripten-tag-%tag%-64bit"],
     "os": "unix"
   },  
+  {
+    "version": "nightly-%nightly-llvm-64bit%",
+    "bitness": 64,
+    "uses": ["clang-nightly-e%nightly-llvm-64bit%-64bit", "node-4.1.1-64bit", "emscripten-nightly-%nightly-llvm-64bit%"],
+    "os": "unix"
+  },
   {
     "version": "1.5.6",
     "bitness": 32,