Prefer curl for downloading files over urllib (#1355)

This is especially important on macOS where `urllib` can fail to
verify certificates.

See https://stackoverflow.com/questions/40684543/how-to-make-python-use-ca-certificates-from-mac-os-truststore

Fixes: #1207, #85, #1356, #1357, #1358

emsdk.py

@@ -670,19 +670,18 @@ def get_download_target(url, dstpath, filename_prefix=''):
   return file_name
 
 
-# On success, returns the filename on the disk pointing to the destination file that was produced
+def download_with_curl(url, file_name):
-# On failure, returns None.
+  print("Downloading: %s from %s" % (file_name, url))
-def download_file(url, dstpath, download_even_if_exists=False,
+  if not which('curl'):
-                  filename_prefix='', silent=False):
+    exit_with_error('curl not found in PATH')
-  debug_print('download_file(url=' + url + ', dstpath=' + dstpath + ')')
+  # -#: show progress bar
-  file_name = get_download_target(url, dstpath, filename_prefix)
+  # -L: Follow HTTP 3XX redirections
+  # -f: Fail on HTTP errors
+  subprocess.check_call(['curl', '-#', '-f', '-L', '-o', file_name, url])
 
-  if os.path.exists(file_name) and not download_even_if_exists:
+
-    print("File '" + file_name + "' already downloaded, skipping.")
+def download_with_urllib(url, file_name):
-    return file_name
-  try:
   u = urlopen(url)
-    mkdir_p(os.path.dirname(file_name))
   with open(file_name, 'wb') as f:
     file_size = get_content_length(u)
     if file_size > 0:
@@ -717,16 +716,37 @@ def download_file(url, dstpath, download_even_if_exists=False,
     if not TTY_OUTPUT:
       print(']')
       sys.stdout.flush()
+
+
+# On success, returns the filename on the disk pointing to the destination file that was produced
+# On failure, returns None.
+def download_file(url, dstpath, download_even_if_exists=False,
+                  filename_prefix='', silent=False):
+  debug_print('download_file(url=' + url + ', dstpath=' + dstpath + ')')
+  file_name = get_download_target(url, dstpath, filename_prefix)
+
+  if os.path.exists(file_name) and not download_even_if_exists:
+    print("File '" + file_name + "' already downloaded, skipping.")
+    return file_name
+
+  mkdir_p(os.path.dirname(file_name))
+
+  try:
+    # Use curl on macOS to avoid CERTIFICATE_VERIFY_FAILED issue with
+    # python's urllib:
+    # https://stackoverflow.com/questions/40684543/how-to-make-python-use-ca-certificates-from-mac-os-truststore
+    # Unlike on linux or windows, curl is always available on macOS systems.
+    if MACOS:
+      download_with_curl(url, file_name)
+    else:
+      download_with_urllib(url, file_name)
   except Exception as e:
-    if not silent:
     errlog("Error: Downloading URL '" + url + "': " + str(e))
-      if "SSL: CERTIFICATE_VERIFY_FAILED" in str(e) or "urlopen error unknown url type: https" in str(e):
-        errlog("Warning: Possibly SSL/TLS issue. Update or install Python SSL root certificates (2048-bit or greater) supplied in Python folder or https://pypi.org/project/certifi/ and try again.")
-    rmfile(file_name)
     return None
   except KeyboardInterrupt:
     rmfile(file_name)
-    exit_with_error("aborted by user, exiting")
+    raise
+
   return file_name
 
 
@@ -3093,4 +3113,8 @@ def main(args):
 
 
 if __name__ == '__main__':
+  try:
     sys.exit(main(sys.argv[1:]))
+  except KeyboardInterrupt:
+    exit_with_error('aborted by user, exiting')
+    sys.exit(1)

test/test.py

@@ -270,9 +270,9 @@ int main() {
     # install of 2.0.28, and again when we install 2.0.29, but not on the
     # second install of 2.0.28 because the zip should already be local.
     shutil.rmtree('downloads')
-    checked_call_with_output(emsdk + ' install 2.0.28', expected='Downloading:', env=env)
+    checked_call_with_output(emsdk + ' install 3.1.54', expected='Downloading:', env=env)
-    checked_call_with_output(emsdk + ' install 2.0.29', expected='Downloading:', env=env)
+    checked_call_with_output(emsdk + ' install 3.1.55', expected='Downloading:', env=env)
-    checked_call_with_output(emsdk + ' install 2.0.28', expected='already downloaded, skipping', unexpected='Downloading:', env=env)
+    checked_call_with_output(emsdk + ' install 3.1.54', expected='already downloaded, skipping', unexpected='Downloading:', env=env)
 
 
 if __name__ == '__main__':