From f95446505533e176036c76e4020d88449a30707c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Wed, 28 Sep 2016 19:06:56 +0300 Subject: [PATCH] Add support for installing precompiled nightly LLVM/Emscripten/SDK versions. --- emscripten-tags.txt | 5 +- emsdk | 111 +++++++++++++++++++++++++++++++++++--------- emsdk_manifest.json | 26 +++++++++++ 3 files changed, 119 insertions(+), 23 deletions(-) diff --git a/emscripten-tags.txt b/emscripten-tags.txt index b546188..3ab38b2 100644 --- a/emscripten-tags.txt +++ b/emscripten-tags.txt @@ -77,4 +77,7 @@ 1.36.5 1.36.6 1.36.7 -1.36.8 \ No newline at end of file +1.36.8 +1.36.9 +1.36.10 +1.36.11 \ No newline at end of file diff --git a/emsdk b/emsdk index 9f2e250..ad84650 100755 --- a/emsdk +++ b/emsdk @@ -373,7 +373,7 @@ def path_points_to_directory(path): if no_suffix: return True suffix = path[last_dot:] - if suffix == '.exe' or suffix == '.zip': # Very simple logic for the only file suffixes used by emsdk downloader. Other suffixes, like 'clang-3.2' are treated as dirs. + if suffix == '.exe' or suffix == '.zip' or suffix == '.txt': # Very simple logic for the only file suffixes used by emsdk downloader. Other suffixes, like 'clang-3.2' are treated as dirs. return False else: return True @@ -947,6 +947,9 @@ class Tool: if hasattr(self, 'uses'): for tool_name in self.uses: tool = find_tool(tool_name) + if not tool: + if VERBOSE: print('Tool ' + str(self) + ' depends on ' + tool_name + ' which does not exist!') + continue if tool.needs_compilation(): return True return False @@ -1326,6 +1329,19 @@ def fetch_emscripten_tags(): else: print('Done. No tagged releases available.') + def os_name_for_llvm_location(): + if WINDOWS: return 'win' + if LINUX: return 'linux' + if OSX: return 'osx' + def os_name_for_emscripten_location(): + if WINDOWS: return 'win' + else: return 'linux' + + print('Fetching all precompiled Nightly versions..') + download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/nightly/' + os_name_for_llvm_location() + '_32bit/index.txt', 'llvm-nightlies-32bit.txt', download_even_if_exists=True) + download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/nightly/' + os_name_for_llvm_location() + '_64bit/index.txt', 'llvm-nightlies-64bit.txt', download_even_if_exists=True) + download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/emscripten/nightly/' + os_name_for_emscripten_location() + '/index.txt', 'emscripten-nightlies.txt', download_even_if_exists=True) + def update_emsdk(): if WINDOWS: download_and_unzip(urljoin(emsdk_packages_url, 'emsdk_windows_update.zip'), emsdk_path(), download_even_if_exists=True) @@ -1343,8 +1359,35 @@ def load_emscripten_tags(): except: return [] +def remove_prefix(s, prefix): + if s.startswith(prefix): return s[len(prefix):] + else: return s + +def remove_suffix(s, suffix): + if s.endswith(suffix): return s[:len(s)-len(suffix)] + else: return s + +# Kind should be one of: 'llvm-nightlies-32bit.txt', 'llvm-nightlies-64bit.txt', 'emscripten-nightlies.txt' +def load_nightlies_list(kind): + try: + items = open(sdk_path(kind), 'r').read().split('\n') + items = map(lambda x: remove_suffix(remove_suffix(remove_prefix(remove_prefix(x, 'emscripten-llvm-e'), 'emscripten-nightly-'), '.tar.gz'), '.zip').strip(), items) + items = filter(lambda x: 'latest' not in x and len(x) > 0, items) + return items + except: + return [] + +def load_llvm_32bit_nightlies(): + return load_nightlies_list('llvm-nightlies-32bit.txt') + +def load_llvm_64bit_nightlies(): + return load_nightlies_list('llvm-nightlies-64bit.txt') + +def load_emscripten_nightlies(): + return load_nightlies_list('emscripten-nightlies.txt') + def load_sdk_manifest(): - global tools + global tools, sdks try: manifest = json.loads(open(sdk_path("emsdk_manifest.json"), "r").read()) except Exception as e: @@ -1353,6 +1396,34 @@ def load_sdk_manifest(): return emscripten_tags = load_emscripten_tags() + llvm_32bit_nightlies = load_llvm_32bit_nightlies() + llvm_64bit_nightlies = load_llvm_64bit_nightlies() + emscripten_nightlies = load_emscripten_nightlies() + + def dependencies_exist(sdk): + for tool_name in sdk.uses: + tool = find_tool(tool_name) + if not tool: return False + return True + + # A 'category parameter' is a %foo%-encoded identifier that specifies + # a class of tools instead of just one tool, e.g. %tag% or %nightly..% + def expand_category_param(param, category_list, t, is_sdk): + global tools, sdks + for i in range(len(category_list)): + ver = category_list[i] + t2 = copy.copy(t) + for p, v in vars(t2).iteritems(): + if isinstance(v, basestring) and param in v: + t2.__dict__[p] = v.replace(param, ver) + t2.is_old = i < len(category_list) - 2 + if hasattr(t2, 'uses'): + t2.uses = map((lambda x: x.replace(param, ver)), t2.uses) + if is_sdk: + if dependencies_exist(t2): + sdks.append(t2) + else: + tools.append(t2) for tool in manifest['tools']: t = Tool(tool) @@ -1360,16 +1431,15 @@ def load_sdk_manifest(): if not hasattr(t, 'is_old'): t.is_old = False + # Expand the metapackages that refer to tags or nightlies. if '%tag%' in t.version: - # Expand the metapackage that refers to all tags. - for i in range(len(emscripten_tags)): - tag = emscripten_tags[i] - t2 = copy.copy(t) - for p, v in vars(t2).iteritems(): - if isinstance(v, basestring) and '%tag%' in v: - t2.__dict__[p] = v.replace('%tag%', tag) - t2.is_old = i < len(emscripten_tags) - 2 - tools.append(t2) + expand_category_param('%tag%', emscripten_tags, t, is_sdk=False) + elif '%nightly-llvm-64bit%' in t.version: + expand_category_param('%nightly-llvm-64bit%', llvm_64bit_nightlies, t, is_sdk=False) + elif '%nightly-llvm-32bit%' in t.version: + expand_category_param('%nightly-llvm-32bit%', llvm_32bit_nightlies, t, is_sdk=False) + elif '%nightly-emscripten%' in t.version: + expand_category_param('%nightly-emscripten%', emscripten_nightlies, t, is_sdk=False) else: tools.append(t) @@ -1381,16 +1451,13 @@ def load_sdk_manifest(): sdk.is_old = False if '%tag%' in sdk.version: - # Expand the metapackage that refers to all tags. - for i in range(len(emscripten_tags)): - tag = emscripten_tags[i] - sdk2 = copy.copy(sdk) - for p, v in vars(sdk2).iteritems(): - if isinstance(v, basestring) and '%tag%' in v: - sdk2.__dict__[p] = v.replace('%tag%', tag) - sdk2.is_old = i < len(emscripten_tags) - 2 - sdk2.uses = map((lambda x: x.replace('%tag%', tag)), sdk2.uses) - sdks.append(sdk2) + expand_category_param('%tag%', emscripten_tags, sdk, is_sdk=True) + elif '%nightly-llvm-64bit%' in sdk.version: + expand_category_param('%nightly-llvm-64bit%', llvm_64bit_nightlies, sdk, is_sdk=True) + elif '%nightly-llvm-32bit%' in sdk.version: + expand_category_param('%nightly-llvm-32bit%', llvm_32bit_nightlies, sdk, is_sdk=True) + elif '%nightly-emscripten%' in sdk.version: + expand_category_param('%nightly-emscripten%', emscripten_nightlies, sdk, is_sdk=True) else: sdks.append(sdk) @@ -1777,7 +1844,7 @@ def main(): active = '*' if sdk.is_active() else ' ' print(' ' + active + ' {0: <25}'.format(str(sdk)) + installed) print('') - print('The following precompiled SDKs are available for download:') + print('The following precompiled SDKs are available for download: (Run "./emsdk update" to pull in the latest list)') print_sdks(find_sdks(False)) print('The following SDKs can be compiled from source:') print_sdks(find_sdks(True)) diff --git a/emsdk_manifest.json b/emsdk_manifest.json index 32ce3c2..120d766 100644 --- a/emsdk_manifest.json +++ b/emsdk_manifest.json @@ -82,6 +82,17 @@ "activated_cfg": "LLVM_ROOT='%installation_dir%/%fastcomp_build_bin_dir%'", "cmake_build_type": "RelWithDebInfo" }, + { + "id": "clang", + "version": "nightly-e%nightly-llvm-64bit%", + "bitness": 64, + "append_bitness": false, + "windows_url": "llvm/nightly/win_64bit/emscripten-llvm-e%nightly-llvm-64bit%.zip", + "osx_url": "llvm/nightly/osx_64bit/emscripten-llvm-e%nightly-llvm-64bit%.tar.gz", + "linux_url": "llvm/nightly/linux_64bit/emscripten-llvm-e%nightly-llvm-64bit%.tar.gz", + "activated_path": "%installation_dir%", + "activated_cfg": "LLVM_ROOT='%installation_dir%'" + }, { "id": "clang", "version": "incoming", @@ -641,6 +652,15 @@ "custom_is_installed_script": "is_optimizer_installed", "custom_uninstall_script": "uninstall_optimizer" }, + { + "id": "emscripten", + "version": "nightly-%nightly-emscripten%", + "windows_url": "emscripten/nightly/win/emscripten-nightly-%nightly-emscripten%.zip", + "unix_url": "emscripten/nightly/linux/emscripten-nightly-%nightly-emscripten%.tar.gz", + "activated_cfg": "EMSCRIPTEN_ROOT='%installation_dir%'", + "activated_path": "%installation_dir%", + "activated_env": "EMSCRIPTEN=%installation_dir%" + }, { "id": "emscripten", "version": "incoming", @@ -850,6 +870,12 @@ "uses": ["clang-tag-e%tag%-64bit", "node-4.1.1-64bit", "emscripten-tag-%tag%-64bit"], "os": "unix" }, + { + "version": "nightly-%nightly-llvm-64bit%", + "bitness": 64, + "uses": ["clang-nightly-e%nightly-llvm-64bit%-64bit", "node-4.1.1-64bit", "emscripten-nightly-%nightly-llvm-64bit%"], + "os": "unix" + }, { "version": "1.5.6", "bitness": 32,