Use bundled/embedded python3 binary on OSX (#561)

See https://github.com/emscripten-core/emscripten/issues/7198

.circleci/config.yml

@@ -54,9 +54,6 @@ jobs:
       - run:
           name: Install cmake
           command: brew install cmake
-      - run:
-          name: Install python 3
-          command: brew install python3
       - run: scripts/test.sh
       - run:
           name: test.py

emsdk

@@ -6,13 +6,13 @@
 
 # Wrapper script that runs emsdk.py
 
-base_dir=$(dirname "$0")
-
-# Look for python3 first.  This is especially important on macOS (See:
-# https://github.com/emscripten-core/emsdk/pull/273)
-python=$(which python3 2> /dev/null)
-if [ $? != 0 ]; then
-  python=python
+if [ -z "$EMSDK_PYTHON" ]; then
+  # Look for python3 first.  This is especially important on macOS (See:
+  # https://github.com/emscripten-core/emsdk/pull/273)
+  EMSDK_PYTHON=$(which python3 2> /dev/null)
+  if [ $? != 0 ]; then
+    EMSDK_PYTHON=python
+  fi
 fi
 
-exec "$python" "$0.py" "$@"
+exec "$EMSDK_PYTHON" "$0.py" "$@"

emsdk_manifest.json

@@ -288,6 +288,16 @@
     "activated_cfg": "PYTHON='%installation_dir%/python.exe'",
     "activated_env": "EMSDK_PYTHON=%installation_dir%/python.exe"
   },
+  {
+    "id": "python",
+    "version": "3.7.4",
+    "bitness": 64,
+    "arch": "x86_64",
+    "osx_url": "python-3.7.4-macos.tar.gz",
+    "activated_path": "%installation_dir%/bin",
+    "activated_cfg": "PYTHON='%installation_dir%/bin/python3'",
+    "activated_env": "EMSDK_PYTHON=%installation_dir%/bin/python3"
+  },
   {
     "id": "java",
     "version": "8.152",
@@ -475,7 +485,7 @@
   {
     "version": "upstream-master",
     "bitness": 64,
-    "uses": ["llvm-git-master-64bit", "node-12.18.1-64bit", "emscripten-master-64bit", "binaryen-master-64bit"],
+    "uses": ["llvm-git-master-64bit", "node-12.18.1-64bit", "python-3.7.4-64bit", "emscripten-master-64bit", "binaryen-master-64bit"],
     "os": "osx"
   },
   {
@@ -505,7 +515,7 @@
   {
     "version": "fastcomp-master",
     "bitness": 64,
-    "uses": ["fastcomp-clang-master-64bit", "node-12.18.1-64bit", "emscripten-master-64bit", "binaryen-master-64bit"],
+    "uses": ["fastcomp-clang-master-64bit", "node-12.18.1-64bit", "python-3.7.4-64bit", "emscripten-master-64bit", "binaryen-master-64bit"],
     "os": "osx"
   },
   {
@@ -566,7 +576,7 @@
   {
     "version": "releases-upstream-%releases-tag%",
     "bitness": 64,
-    "uses": ["node-12.18.1-64bit", "releases-upstream-%releases-tag%-64bit"],
+    "uses": ["node-12.18.1-64bit", "python-3.7.4-64bit", "releases-upstream-%releases-tag%-64bit"],
     "os": "osx",
     "custom_install_script": "emscripten_npm_install"
   },
@@ -587,7 +597,7 @@
   {
     "version": "releases-fastcomp-%releases-tag%",
     "bitness": 64,
-    "uses": ["node-12.18.1-64bit", "releases-fastcomp-%releases-tag%-64bit"],
+    "uses": ["node-12.18.1-64bit", "python-3.7.4-64bit", "releases-fastcomp-%releases-tag%-64bit"],
     "os": "osx",
     "custom_install_script": "emscripten_npm_install"
   },
@@ -637,7 +647,7 @@
   {
     "version": "fastcomp-%precompiled_tag32%",
     "bitness": 32,
-    "uses": ["fastcomp-clang-e%precompiled_tag32%-32bit", "node-8.9.1-32bit", "emscripten-%precompiled_tag32%"],
+    "uses": ["fastcomp-clang-e%precompiled_tag32%-32bit", "node-8.9.1-32bit", "python-3.7.4-64bit", "emscripten-%precompiled_tag32%"],
     "os": "osx",
     "version_filter": [
       ["%precompiled_tag32%", ">", "1.37.22"]
@@ -646,7 +656,7 @@
   {
     "version": "fastcomp-%precompiled_tag64%",
     "bitness": 64,
-    "uses": ["fastcomp-clang-e%precompiled_tag64%-64bit", "node-8.9.1-64bit", "emscripten-%precompiled_tag64%"],
+    "uses": ["fastcomp-clang-e%precompiled_tag64%-64bit", "node-8.9.1-64bit", "python-3.7.4-64bit", "emscripten-%precompiled_tag64%"],
     "os": "osx",
     "version_filter": [
       ["%precompiled_tag64%", ">", "1.37.22"]

scripts/test.sh

@@ -8,4 +8,7 @@ set -e
 ./emsdk install latest
 ./emsdk activate latest
 source ./emsdk_env.sh --build=Release
+# On mac and windows python3 should be in the path and point to the
+# bundled version.
+which python3
 emcc -v

scripts/update_python.py

@@ -7,22 +7,28 @@
 """Updates the python binaries that we cache store at
 http://storage.google.com/webassembly.
 
-Currently this is windows only and we rely on the system python on other
-platforms.
+We only supply binaries for windows and macOS, but we do it very different ways for those two OSes.
 
-We currently bundle a version of python for windows use the following
-recipe:
+Windows recipe:
   1. Download the "embeddable zip file" version of python from python.org
   2. Remove .pth file to work around https://bugs.python.org/issue34841
   3. Download and install pywin32 in the `site-packages` directory
   4. Re-zip and upload to storage.google.com
+
+macOS recipe:
+  1. Clone cpython
+  2. Use homebrew to install and configure openssl (for static linking!)
+  3. Build cpython from source and use `make install` to create archive.
 """
 
-import urllib.request
-import subprocess
+import glob
+import multiprocessing
 import os
+import urllib.request
 import shutil
+import subprocess
 import sys
+from subprocess import check_call
 
 version = '3.7.4'
 base = 'https://www.python.org/ftp/python/%s/' % version
@@ -51,7 +57,7 @@ def make_python_patch(arch):
         urllib.request.urlretrieve(download_url, filename)
 
     os.mkdir('python-embed')
-    subprocess.check_call(['unzip', '-q', os.path.abspath(filename)], cwd='python-embed')
+    check_call(['unzip', '-q', os.path.abspath(filename)], cwd='python-embed')
     os.remove(os.path.join('python-embed', 'python37._pth'))
 
     os.mkdir('pywin32')
@@ -61,23 +67,56 @@ def make_python_patch(arch):
     os.mkdir(os.path.join('python-embed', 'lib'))
     shutil.move(os.path.join('pywin32', 'PLATLIB'), os.path.join('python-embed', 'lib', 'site-packages'))
 
-    subprocess.check_call(['zip', '-rq', os.path.join('..', out_filename), '.'], cwd='python-embed')
+    check_call(['zip', '-rq', os.path.join('..', out_filename), '.'], cwd='python-embed')
 
     upload_url = upload_base + out_filename
     print('Uploading: ' + upload_url)
     cmd = ['gsutil', 'cp', '-n', out_filename, upload_url]
     print(' '.join(cmd))
-    subprocess.check_call(cmd)
+    check_call(cmd)
 
     # cleanup if everything went fine
     shutil.rmtree('python-embed')
     shutil.rmtree('pywin32')
 
 
-def main():
-    for arch in ('amd64', 'win32'):
-        make_python_patch(arch)
+def build_python():
+    if sys.platform.startswith('darwin'):
+        osname = 'macos'
+        # Take some rather drastic steps to link openssl statically
+        check_call(['brew', 'install', 'openssl', 'pkg-config'])
+        os.remove('/usr/local/opt/openssl/lib/libssl.dylib')
+        os.remove('/usr/local/opt/openssl/lib/libcrypto.dylib')
+        os.environ['PKG_CONFIG_PATH'] = '/usr/local/opt/openssl/lib/pkgconfig/'
+    else:
+        osname = 'linux'
 
+    src_dir = 'cpython'
+    if not os.path.exists(src_dir):
+      check_call(['git', 'clone', 'https://github.com/python/cpython'])
+    check_call(['git', 'checkout', 'v' + version], cwd=src_dir)
+    check_call(['./configure'], cwd=src_dir)
+    check_call(['make', '-j', str(multiprocessing.cpu_count())], cwd=src_dir)
+    check_call(['make', 'install', 'DESTDIR=install'], cwd=src_dir)
+
+    install_dir = os.path.join(src_dir, 'install')
+    os.rename(os.path.join(install_dir, 'usr', 'local'), 'python-%s' % version)
+    tarball = 'python-%s-%s.tar.gz' % (version, osname)
+    shutil.rmtree(os.path.join('python-%s' % version, 'lib', 'python3.7', 'test'))
+    shutil.rmtree(os.path.join('python-%s' % version, 'include'))
+    for lib in glob.glob(os.path.join('python-%s' % version, 'lib', 'lib*.a')):
+      os.remove(lib)
+    check_call(['tar', 'zcvf', tarball, 'python-%s' % version])
+    print('Uploading: ' + upload_base + tarball)
+    check_call(['gsutil', 'cp', '-n', tarball, upload_base + tarball])
+
+
+def main():
+    if sys.platform.startswith('win'):
+        for arch in ('amd64', 'win32'):
+            make_python_patch(arch)
+    else:
+        build_python()
     return 0