Fix support for Apple M1 (#753)

* Fix support for Apple M1. Node.js will still run via Rosetta 2 emulation since they do not yet have M1 support, but Python, LLVM, Emscripten and Binaryen will be native.

* Update M1 python version and URL

* Remove .gitignore additions

* Move python first in the manifest (#441)

* Use macosx-version-min when building python

* Update Intel macOS python package name

emsdk

@@ -6,10 +6,15 @@
 
 # Wrapper script that runs emsdk.py
 
-# First look for pre-built python (macos)
+# First look for python bundled in Emsdk
 if [ -z "$EMSDK_PYTHON" ]; then
-  PYTHON3=$(dirname $0)/python/3.7.4-2_64bit/bin/python3
+  PYTHON3=$(dirname $0)/python/3.9.2-1_64bit/bin/python3
-  if [ -e $PYTHON3 ]; then
+  PYTHON3_CERT_FILE=$(dirname $0)/python/3.9.2-1_64bit/lib/python3.9/site-packages/certifi/cacert.pem
+  if [ ! -f $PYTHON3 ]; then
+    PYTHON3=$(dirname $0)/python/3.7.4-2_64bit/bin/python3
+    PYTHON3_CERT_FILE=$(dirname $0)/python/3.7.4-2_64bit/lib/python3.7/site-packages/certifi/cacert.pem
+  fi
+  if [ -f $PYTHON3 ]; then
     EMSDK_PYTHON=$PYTHON3
 
     # When using our bundled python we never want the users
@@ -21,11 +26,11 @@ if [ -z "$EMSDK_PYTHON" ]; then
     # This is needed for MacOS.  Without this, the urlopen
     # code will try to use /usr/local/etc/openssl/cert.pem
     # which may or may not exist on the system.
-    export SSL_CERT_FILE=$(dirname $0)/python/3.7.4-2_64bit/lib/python3.7/site-packages/certifi/cacert.pem
+    export SSL_CERT_FILE=$PYTHON3_CERT_FILE
   fi
 fi
 
-# Look for `python3` first.  This is especially important on macOS (See:
+# If bundled python is not found, look for `python3` in PATH.  This is especially important on macOS (See:
 # https://github.com/emscripten-core/emsdk/pull/273)
 if [ -z "$EMSDK_PYTHON" ]; then
   PYTHON3=$(which python3 2> /dev/null)

emsdk_manifest.json

@@ -221,6 +221,7 @@
     "version": "14.15.5",
     "arch": "aarch64",
     "bitness": 64,
+    "macos_url": "node-v14.15.5-darwin-x64.tar.gz",
     "linux_url": "node-v14.15.5-linux-arm64.tar.xz",
     "activated_path": "%installation_dir%/bin",
     "activated_cfg": "NODE_JS='%installation_dir%/bin/node%.exe%'",
@@ -289,6 +290,24 @@
     "activated_cfg": "PYTHON='%installation_dir%/bin/python3'",
     "activated_env": "EMSDK_PYTHON=%installation_dir%/bin/python3;SSL_CERT_FILE=%installation_dir%/lib/python3.7/site-packages/certifi/cacert.pem"
   },
+  {
+    "id": "python",
+    "version": "3.9.2-1",
+    "bitness": 64,
+    "arch": "x86_64",
+    "macos_url": "python-3.9.2-1-macos-x86_64.tar.gz",
+    "activated_cfg": "PYTHON='%installation_dir%/bin/python3'",
+    "activated_env": "EMSDK_PYTHON=%installation_dir%/bin/python3;SSL_CERT_FILE=%installation_dir%/lib/python3.9/site-packages/certifi/cacert.pem"
+  },
+  {
+    "id": "python",
+    "version": "3.9.2-1",
+    "bitness": 64,
+    "arch": "aarch64",
+    "macos_url": "python-3.9.2-1-macos-arm64.tar.gz",
+    "activated_cfg": "PYTHON='%installation_dir%/bin/python3'",
+    "activated_env": "EMSDK_PYTHON=%installation_dir%/bin/python3;SSL_CERT_FILE=%installation_dir%/lib/python3.9/site-packages/certifi/cacert.pem"
+  },
   {
     "id": "java",
     "version": "8.152",
@@ -464,13 +483,13 @@
   {
     "version": "upstream-main",
     "bitness": 64,
-    "uses": ["llvm-git-main-64bit", "node-14.15.5-64bit", "python-3.7.4-pywin32-64bit", "emscripten-main-64bit", "binaryen-main-64bit"],
+    "uses": ["python-3.7.4-pywin32-64bit", "llvm-git-main-64bit", "node-14.15.5-64bit", "emscripten-main-64bit", "binaryen-main-64bit"],
     "os": "win"
   },
   {
     "version": "upstream-main",
     "bitness": 64,
-    "uses": ["llvm-git-main-64bit", "node-14.15.5-64bit", "python-3.7.4-2-64bit", "emscripten-main-64bit", "binaryen-main-64bit"],
+    "uses": ["python-3.9.2-1-64bit", "llvm-git-main-64bit", "node-14.15.5-64bit", "emscripten-main-64bit", "binaryen-main-64bit"],
     "os": "macos"
   },
   {
@@ -533,6 +552,7 @@
     "bitness": 64,
     "uses": ["node-14.15.5-64bit", "python-3.7.4-2-64bit", "releases-upstream-%releases-tag%-64bit"],
     "os": "macos",
+    "arch": "x86_64",
     "custom_install_script": "emscripten_npm_install"
   },
   {
@@ -554,6 +574,7 @@
     "bitness": 64,
     "uses": ["node-14.15.5-64bit", "python-3.7.4-2-64bit", "releases-fastcomp-%releases-tag%-64bit"],
     "os": "macos",
+    "arch": "x86_64",
     "custom_install_script": "emscripten_npm_install"
   },
   {
@@ -604,6 +625,7 @@
     "bitness": 32,
     "uses": ["fastcomp-clang-e%precompiled_tag32%-32bit", "node-8.9.1-32bit", "python-3.7.4-2-64bit", "emscripten-%precompiled_tag32%"],
     "os": "macos",
+    "arch": "x86_64",
     "version_filter": [
       ["%precompiled_tag32%", ">", "1.37.22"]
     ]
@@ -613,6 +635,7 @@
     "bitness": 64,
     "uses": ["fastcomp-clang-e%precompiled_tag64%-64bit", "node-8.9.1-64bit", "python-3.7.4-2-64bit", "emscripten-%precompiled_tag64%"],
     "os": "macos",
+    "arch": "x86_64",
     "version_filter": [
       ["%precompiled_tag64%", ">", "1.37.22"]
     ]

scripts/update_python.py

@@ -24,15 +24,17 @@ macOS recipe:
 import glob
 import multiprocessing
 import os
+import platform
 import urllib.request
 import shutil
 import subprocess
 import sys
 from subprocess import check_call
 
-version = '3.7.4'
+version = '3.9.2'
+major_minor_version = '.'.join(version.split('.')[:2])  # e.g. '3.9.2' -> '3.9'
 base = 'https://www.python.org/ftp/python/%s/' % version
-revision = '2'
+revision = '1'
 
 pywin32_version = '227'
 pywin32_base = 'https://github.com/mhammond/pywin32/releases/download/b%s/' % pywin32_version
@@ -42,9 +44,9 @@ upload_base = 'gs://webassembly/emscripten-releases-builds/deps/'
 
 def make_python_patch(arch):
     if arch == 'amd64':
-      pywin32_filename = 'pywin32-%s.win-%s-py3.7.exe' % (pywin32_version, arch)
+      pywin32_filename = 'pywin32-%s.win-%s-py%s.exe' % (pywin32_version, arch, major_minor_version)
     else:
-      pywin32_filename = 'pywin32-%s.%s-py3.7.exe' % (pywin32_version, arch)
+      pywin32_filename = 'pywin32-%s.%s-py%s.exe' % (pywin32_version, arch, major_minor_version)
     filename = 'python-%s-embed-%s.zip' % (version, arch)
     out_filename = 'python-%s-embed-%s+pywin32.zip' % (version, arch)
     if not os.path.exists(pywin32_filename):
@@ -86,9 +88,21 @@ def build_python():
         osname = 'macos'
         # Take some rather drastic steps to link openssl statically
         check_call(['brew', 'install', 'openssl', 'pkg-config'])
-        os.remove('/usr/local/opt/openssl/lib/libssl.dylib')
+        if platform.machine() == 'x86_64':
-        os.remove('/usr/local/opt/openssl/lib/libcrypto.dylib')
+            prefix = '/usr/local'
-        os.environ['PKG_CONFIG_PATH'] = '/usr/local/opt/openssl/lib/pkgconfig/'
+            min_macos_version = '10.13'
+        elif platform.machine() == 'arm64':
+            prefix = '/opt/homebrew'
+            min_macos_version = '11.0'
+
+        osname += '-' + platform.machine()  # Append '-x86_64' or '-arm64' depending on current arch. (TODO: Do this for Linux too, move this below?)
+
+        try:
+            os.remove(os.path.join(prefix, 'opt', 'openssl', 'lib', 'libssl.dylib'))
+            os.remove(os.path.join(prefix, 'opt', 'openssl', 'lib', 'libcrypto.dylib'))
+        except Exception:
+            pass
+        os.environ['PKG_CONFIG_PATH'] = os.path.join(prefix, 'opt', 'openssl', 'lib', 'pkgconfig')
     else:
         osname = 'linux'
 
@@ -96,9 +110,14 @@ def build_python():
     if not os.path.exists(src_dir):
       check_call(['git', 'clone', 'https://github.com/python/cpython'])
     check_call(['git', 'checkout', 'v' + version], cwd=src_dir)
-    check_call(['./configure'], cwd=src_dir)
+
-    check_call(['make', '-j', str(multiprocessing.cpu_count())], cwd=src_dir)
+    min_macos_version_line = '-mmacosx-version-min=' + min_macos_version  # Specify the min OS version we want the build to work on
-    check_call(['make', 'install', 'DESTDIR=install'], cwd=src_dir)
+    build_flags = min_macos_version_line + ' -Werror=partial-availability'  # Build against latest SDK, but issue an error if using any API that would not work on the min OS version
+    env = os.environ.copy()
+    env['MACOSX_DEPLOYMENT_TARGET'] = min_macos_version
+    check_call(['./configure', 'CFLAGS=' + build_flags, 'CXXFLAGS=' + build_flags, 'LDFLAGS=' + min_macos_version_line], cwd=src_dir, env=env)
+    check_call(['make', '-j', str(multiprocessing.cpu_count())], cwd=src_dir, env=env)
+    check_call(['make', 'install', 'DESTDIR=install'], cwd=src_dir, env=env)
 
     install_dir = os.path.join(src_dir, 'install')
 
@@ -109,9 +128,12 @@ def build_python():
     check_call([pybin, pip, 'install', 'requests'])
 
     dirname = 'python-%s-%s' % (version, revision)
+    if os.path.isdir(dirname):
+        print('Erasing old build directory ' + dirname)
+        shutil.rmtree(dirname)
     os.rename(os.path.join(install_dir, 'usr', 'local'), dirname)
     tarball = 'python-%s-%s-%s.tar.gz' % (version, revision, osname)
-    shutil.rmtree(os.path.join(dirname, 'lib', 'python3.7', 'test'))
+    shutil.rmtree(os.path.join(dirname, 'lib', 'python' + major_minor_version, 'test'))
     shutil.rmtree(os.path.join(dirname, 'include'))
     for lib in glob.glob(os.path.join(dirname, 'lib', 'lib*.a')):
       os.remove(lib)