diff --git a/.circleci/config.yml b/.circleci/config.yml index 47653df..f4a2db2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,9 +54,6 @@ jobs: - run: name: Install cmake command: brew install cmake - - run: - name: Install python 3 - command: brew install python3 - run: scripts/test.sh - run: name: test.py diff --git a/emsdk b/emsdk index 6859b54..eb24599 100755 --- a/emsdk +++ b/emsdk @@ -6,13 +6,13 @@ # Wrapper script that runs emsdk.py -base_dir=$(dirname "$0") - -# Look for python3 first. This is especially important on macOS (See: -# https://github.com/emscripten-core/emsdk/pull/273) -python=$(which python3 2> /dev/null) -if [ $? != 0 ]; then - python=python +if [ -z "$EMSDK_PYTHON" ]; then + # Look for python3 first. This is especially important on macOS (See: + # https://github.com/emscripten-core/emsdk/pull/273) + EMSDK_PYTHON=$(which python3 2> /dev/null) + if [ $? != 0 ]; then + EMSDK_PYTHON=python + fi fi -exec "$python" "$0.py" "$@" +exec "$EMSDK_PYTHON" "$0.py" "$@" diff --git a/emsdk_manifest.json b/emsdk_manifest.json index 970c0cb..50e524e 100644 --- a/emsdk_manifest.json +++ b/emsdk_manifest.json @@ -288,6 +288,16 @@ "activated_cfg": "PYTHON='%installation_dir%/python.exe'", "activated_env": "EMSDK_PYTHON=%installation_dir%/python.exe" }, + { + "id": "python", + "version": "3.7.4", + "bitness": 64, + "arch": "x86_64", + "osx_url": "python-3.7.4-macos.tar.gz", + "activated_path": "%installation_dir%/bin", + "activated_cfg": "PYTHON='%installation_dir%/bin/python3'", + "activated_env": "EMSDK_PYTHON=%installation_dir%/bin/python3" + }, { "id": "java", "version": "8.152", @@ -475,7 +485,7 @@ { "version": "upstream-master", "bitness": 64, - "uses": ["llvm-git-master-64bit", "node-12.18.1-64bit", "emscripten-master-64bit", "binaryen-master-64bit"], + "uses": ["llvm-git-master-64bit", "node-12.18.1-64bit", "python-3.7.4-64bit", "emscripten-master-64bit", "binaryen-master-64bit"], "os": "osx" }, { @@ -505,7 +515,7 @@ { "version": "fastcomp-master", "bitness": 64, - "uses": ["fastcomp-clang-master-64bit", "node-12.18.1-64bit", "emscripten-master-64bit", "binaryen-master-64bit"], + "uses": ["fastcomp-clang-master-64bit", "node-12.18.1-64bit", "python-3.7.4-64bit", "emscripten-master-64bit", "binaryen-master-64bit"], "os": "osx" }, { @@ -566,7 +576,7 @@ { "version": "releases-upstream-%releases-tag%", "bitness": 64, - "uses": ["node-12.18.1-64bit", "releases-upstream-%releases-tag%-64bit"], + "uses": ["node-12.18.1-64bit", "python-3.7.4-64bit", "releases-upstream-%releases-tag%-64bit"], "os": "osx", "custom_install_script": "emscripten_npm_install" }, @@ -587,7 +597,7 @@ { "version": "releases-fastcomp-%releases-tag%", "bitness": 64, - "uses": ["node-12.18.1-64bit", "releases-fastcomp-%releases-tag%-64bit"], + "uses": ["node-12.18.1-64bit", "python-3.7.4-64bit", "releases-fastcomp-%releases-tag%-64bit"], "os": "osx", "custom_install_script": "emscripten_npm_install" }, @@ -637,7 +647,7 @@ { "version": "fastcomp-%precompiled_tag32%", "bitness": 32, - "uses": ["fastcomp-clang-e%precompiled_tag32%-32bit", "node-8.9.1-32bit", "emscripten-%precompiled_tag32%"], + "uses": ["fastcomp-clang-e%precompiled_tag32%-32bit", "node-8.9.1-32bit", "python-3.7.4-64bit", "emscripten-%precompiled_tag32%"], "os": "osx", "version_filter": [ ["%precompiled_tag32%", ">", "1.37.22"] @@ -646,7 +656,7 @@ { "version": "fastcomp-%precompiled_tag64%", "bitness": 64, - "uses": ["fastcomp-clang-e%precompiled_tag64%-64bit", "node-8.9.1-64bit", "emscripten-%precompiled_tag64%"], + "uses": ["fastcomp-clang-e%precompiled_tag64%-64bit", "node-8.9.1-64bit", "python-3.7.4-64bit", "emscripten-%precompiled_tag64%"], "os": "osx", "version_filter": [ ["%precompiled_tag64%", ">", "1.37.22"] diff --git a/scripts/test.sh b/scripts/test.sh index 1a4ae2f..73c0d42 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -8,4 +8,7 @@ set -e ./emsdk install latest ./emsdk activate latest source ./emsdk_env.sh --build=Release +# On mac and windows python3 should be in the path and point to the +# bundled version. +which python3 emcc -v diff --git a/scripts/update_python.py b/scripts/update_python.py index eed07ef..e1f81e3 100755 --- a/scripts/update_python.py +++ b/scripts/update_python.py @@ -7,22 +7,28 @@ """Updates the python binaries that we cache store at http://storage.google.com/webassembly. -Currently this is windows only and we rely on the system python on other -platforms. +We only supply binaries for windows and macOS, but we do it very different ways for those two OSes. -We currently bundle a version of python for windows use the following -recipe: +Windows recipe: 1. Download the "embeddable zip file" version of python from python.org 2. Remove .pth file to work around https://bugs.python.org/issue34841 3. Download and install pywin32 in the `site-packages` directory 4. Re-zip and upload to storage.google.com + +macOS recipe: + 1. Clone cpython + 2. Use homebrew to install and configure openssl (for static linking!) + 3. Build cpython from source and use `make install` to create archive. """ -import urllib.request -import subprocess +import glob +import multiprocessing import os +import urllib.request import shutil +import subprocess import sys +from subprocess import check_call version = '3.7.4' base = 'https://www.python.org/ftp/python/%s/' % version @@ -51,7 +57,7 @@ def make_python_patch(arch): urllib.request.urlretrieve(download_url, filename) os.mkdir('python-embed') - subprocess.check_call(['unzip', '-q', os.path.abspath(filename)], cwd='python-embed') + check_call(['unzip', '-q', os.path.abspath(filename)], cwd='python-embed') os.remove(os.path.join('python-embed', 'python37._pth')) os.mkdir('pywin32') @@ -61,23 +67,56 @@ def make_python_patch(arch): os.mkdir(os.path.join('python-embed', 'lib')) shutil.move(os.path.join('pywin32', 'PLATLIB'), os.path.join('python-embed', 'lib', 'site-packages')) - subprocess.check_call(['zip', '-rq', os.path.join('..', out_filename), '.'], cwd='python-embed') + check_call(['zip', '-rq', os.path.join('..', out_filename), '.'], cwd='python-embed') upload_url = upload_base + out_filename print('Uploading: ' + upload_url) cmd = ['gsutil', 'cp', '-n', out_filename, upload_url] print(' '.join(cmd)) - subprocess.check_call(cmd) + check_call(cmd) # cleanup if everything went fine shutil.rmtree('python-embed') shutil.rmtree('pywin32') -def main(): - for arch in ('amd64', 'win32'): - make_python_patch(arch) +def build_python(): + if sys.platform.startswith('darwin'): + osname = 'macos' + # Take some rather drastic steps to link openssl statically + check_call(['brew', 'install', 'openssl', 'pkg-config']) + os.remove('/usr/local/opt/openssl/lib/libssl.dylib') + os.remove('/usr/local/opt/openssl/lib/libcrypto.dylib') + os.environ['PKG_CONFIG_PATH'] = '/usr/local/opt/openssl/lib/pkgconfig/' + else: + osname = 'linux' + src_dir = 'cpython' + if not os.path.exists(src_dir): + check_call(['git', 'clone', 'https://github.com/python/cpython']) + check_call(['git', 'checkout', 'v' + version], cwd=src_dir) + check_call(['./configure'], cwd=src_dir) + check_call(['make', '-j', str(multiprocessing.cpu_count())], cwd=src_dir) + check_call(['make', 'install', 'DESTDIR=install'], cwd=src_dir) + + install_dir = os.path.join(src_dir, 'install') + os.rename(os.path.join(install_dir, 'usr', 'local'), 'python-%s' % version) + tarball = 'python-%s-%s.tar.gz' % (version, osname) + shutil.rmtree(os.path.join('python-%s' % version, 'lib', 'python3.7', 'test')) + shutil.rmtree(os.path.join('python-%s' % version, 'include')) + for lib in glob.glob(os.path.join('python-%s' % version, 'lib', 'lib*.a')): + os.remove(lib) + check_call(['tar', 'zcvf', tarball, 'python-%s' % version]) + print('Uploading: ' + upload_base + tarball) + check_call(['gsutil', 'cp', '-n', tarball, upload_base + tarball]) + + +def main(): + if sys.platform.startswith('win'): + for arch in ('amd64', 'win32'): + make_python_patch(arch) + else: + build_python() return 0