Add circleci config with mac, windows and linux testing (#338)

- Add EMSDK_NOTTY variable which we can use on CI to prevent tty output. - Add test.bat for windows which mirrors test.sh. - Remove travis support
2019-08-29 00:35:36 -07:00
parent 80f490f749
commit 880be59788
8 changed files with 179 additions and 87 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -0,0 +1,92 @@
+version: 2.1
+
+orbs:
+  win: circleci/windows@1.0.0
+
+executors:
+  bionic:
+    docker:
+      - image: buildpack-deps:bionic
+
+jobs:
+  flake8:
+    executor: bionic
+    steps:
+      - checkout
+      - run:
+          name: install pip
+          command: |
+            apt-get update -q
+            apt-get install -q -y python-pip python3-pip
+      - run: python2 -m pip install --upgrade pip
+      - run: python3 -m pip install --upgrade pip
+      - run: python2 -m pip install flake8==3.7.8
+      - run: python3 -m pip install flake8==3.7.8
+      - run: python2 -m flake8 --show-source --statistics
+      - run: python3 -m flake8 --show-source --statistics
+  test-linux:
+    executor: bionic
+    environment:
+      EMSDK_NOTTY: "1"
+      # I don't know why circleci VMs pretent to have 36 cores but its a lie.
+      EMSDK_NUM_CORES: "4"
+    steps:
+      - checkout
+      - run:
+          name: Install debian packages
+          command: apt-get update -q && apt-get install -q -y cmake build-essential openjdk-8-jre-headless
+      - run: ./test.sh
+      - run: ./test.py
+  test-mac:
+    macos:
+      xcode: "9.0"
+    environment:
+      EMSDK_NOTTY: "1"
+      # Without this, any `brew installl` command will result in self-update of
+      # brew itself which takes more than 4 minutes.
+      HOMEBREW_NO_AUTO_UPDATE: "1"
+    steps:
+      - checkout
+      - run:
+          name: Install cmake
+          command: brew install cmake
+      - run: ./test.sh
+      # TODO(sbc): Re-enable once the upstream emscripten fix makes it into
+      # a relase: https://github.com/emscripten-core/emscripten/pull/9347
+      #- run: ./test.py
+  test-windows:
+    executor:
+      name: win/vs2019
+      shell: bash.exe
+    environment:
+      # We need python installed before we can test anytyhing.
+      # There seems to be undocument copy of python installed here. Hopefully
+      # if this disappears there will be another way of getting a re-installed
+      # version.
+      PYTHON_BIN: "C:\\Python27amd64"
+      PYTHONUNBUFFERED: "1"
+      EMSDK_NOTTY: "1"
+    steps:
+      - checkout
+      - run:
+          name: Add python to bash path
+          command: echo "export PATH=\"$PATH:/c/python27amd64/\"" >> $BASH_ENV
+      - run:
+          name: Install latest
+          shell: cmd.exe
+          command: test.bat
+      - run: python test.py
+
+workflows:
+  flake8:
+    jobs:
+      - flake8
+  test-linux:
+    jobs:
+      - test-linux
+  test-mac:
+    jobs:
+      - test-mac
+  test-windows:
+    jobs:
+      - test-windows
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,19 +0,0 @@
-language: python
-
-services:
-  - docker
-
-before_install:
-  - docker pull ubuntu:16.04
-
-install:
-  - pip install flake8==3.7.8
-
-script:
-  - flake8
-  - set -o errexit
-  - echo "running..."
-  - docker build .
-
-notifications:
-  email: false
--- a/15
+++ b/15
@@ -1,15 +0,0 @@
-# For travis
-FROM buildpack-deps:xenial
-SHELL ["/bin/bash", "-c"]
-ENV DEBIAN_FRONTEND=noninteractive LANG=C.UTF-8
-RUN mkdir -p /root/emsdk/
-COPY . /root/emsdk/
-
-RUN cd /root/ \
- && echo "int main() {}" > hello_world.cpp \
- && apt-get update \
- && apt-get install -y python python3 cmake build-essential openjdk-9-jre-headless \
- && /root/emsdk/emsdk update-tags \
- && cd /root/emsdk/ \
- && bash test.sh \
- && python test.py
--- a/2
+++ b/2
@@ -10,7 +10,7 @@ base_dir=$(dirname "$0")

 # Look for python3 first.  This is especially important on macOS (See:
 # https://github.com/emscripten-core/emsdk/pull/273)
-python=$(which python3)
+python=$(which python3 2> /dev/null)
 if [ $? != 0 ]; then
  python=python
 fi
--- a/emsdk.py
+++ b/emsdk.py
@@ -43,8 +43,6 @@ else:

 emsdk_packages_url = emsdk_master_server

-emscripten_git_repo = 'https://github.com/emscripten-core/emscripten.git'
-binaryen_git_repo = 'https://github.com/WebAssembly/binaryen.git'
 emscripten_releases_repo = 'https://chromium.googlesource.com/emscripten-releases'

 emscripten_releases_download_url_template = "https://storage.googleapis.com/webassembly/emscripten-releases-builds/%s/%s/wasm-binaries.%s"
@@ -54,8 +52,8 @@ emsdk_zip_download_url = 'https://github.com/emscripten-core/emsdk/archive/maste
 zips_subdir = 'zips/'

 # Enable this to do very verbose printing about the different steps that are being run. Useful for debugging.
-VERBOSE = bool(os.getenv('EMSDK_VERBOSE')) if os.getenv('EMSDK_VERBOSE') is not None else False
-TTY_OUTPUT = sys.stdout.isatty()
+VERBOSE = int(os.getenv('EMSDK_VERBOSE', '0'))
+TTY_OUTPUT = not os.getenv('EMSDK_NOTTY', not sys.stdout.isatty())

 POWERSHELL = bool(os.getenv('EMSDK_POWERSHELL'))

@@ -98,7 +96,7 @@ else:
  print()

 # Don't saturate all cores to not steal the whole system, but be aggressive.
-CPU_CORES = max(multiprocessing.cpu_count() - 1, 1)
+CPU_CORES = int(os.environ.get('EMSDK_NUM_CORES', max(multiprocessing.cpu_count() - 1, 1)))

 CMAKE_BUILD_TYPE_OVERRIDE = None

@@ -627,7 +625,8 @@ def download_file(url, dstpath, download_even_if_exists=False, filename_prefix='
                      sys.stdout.flush()
                      progress_shown += 1
      if not TTY_OUTPUT:
-          print(']')
+        print(']')
+        sys.stdout.flush()
  except Exception as e:
    print("Error downloading URL '" + url + "': " + str(e))
    rmfile(file_name)
@@ -918,7 +917,8 @@ def make_build(build_root, build_type, build_target_platform='x64'):
 def cmake_configure(generator, build_root, src_root, build_type, extra_cmake_args=[]):
  if VERBOSE: print('cmake_configure(generator=' + str(generator) + ', build_root=' + str(build_root) + ', src_root=' + str(src_root) + ', build_type=' + str(build_type) + ', extra_cmake_args=' + str(extra_cmake_args) + ')')
  # Configure
-  if not os.path.isdir(build_root): os.mkdir(build_root) # Create build output directory if it doesn't yet exist.
+  if not os.path.isdir(build_root):
+    os.mkdir(build_root) # Create build output directory if it doesn't yet exist.
  try:
    if generator: generator = ['-G', generator]
    else: generator = []
@@ -1463,7 +1463,8 @@ class Tool(object):

      def each_path_exists(pathlist):
        for path in pathlist:
-          if not os.path.exists(path): return False
+          if not os.path.exists(path):
+            return False
        return True

      content_exists = os.path.exists(self.installation_path()) and each_path_exists(activated_path) and (os.path.isfile(self.installation_path()) or num_files_in_directory(self.installation_path()) > 0)
@@ -1953,12 +1954,9 @@ def load_releases_versions():


 def is_string(s):
-  # Python3 compat
-  try:
-    basestring
-  except NameError:
-    basestring = str
-  return isinstance(s, basestring)
+  if sys.version_info[0] >= 3:
+    return isinstance(s, str)
+  return isinstance(s, basestring)  # noqa


 def load_sdk_manifest():
@@ -2759,6 +2757,7 @@ def main():
      success = tool.install()
      if not success:
        return 1
+    return 0
  elif cmd == 'uninstall':
    if len(sys.argv) <= 2:
      print("Syntax error. Call 'emsdk uninstall <tool name>'. Call 'emsdk list' to obtain a list of available tools.")
@@ -2768,9 +2767,10 @@ def main():
      print("Error: Tool by name '" + sys.argv[2] + "' was not found.")
      return 1
    tool.uninstall()
-  else:
-    print("Unknown command '" + cmd + "' given! Type 'emsdk help' to get a list of commands.")
-    return 1
+    return 0
+
+  print("Unknown command '" + cmd + "' given! Type 'emsdk help' to get a list of commands.")
+  return 1


 if __name__ == '__main__':
--- a/test.bat
+++ b/test.bat
@@ -0,0 +1,6 @@
+:: equivilent of test.sh as windows bat file
+set PATH=%PATH%;%PYTHON_BIN%
+@CALL emsdk install latest
+@CALL emsdk activate latest
+@CALL emsdk_env.bat --build=Release
+@CALL emcc.bat -v
--- a/test.py
+++ b/test.py
@@ -1,9 +1,9 @@
 #!/usr/bin/env python
 import json
 import os
-import shlex
 import shutil
 import subprocess
+import sys
 import tempfile

 # Utilities
@@ -15,12 +15,17 @@ def listify(x):
  return [x]


-def check_call(cmd):
-  subprocess.check_call(shlex.split(cmd))
+def check_call(cmd, **args):
+  if type(cmd) != list:
+    cmd = cmd.split()
+  print('running: %s' % cmd)
+  subprocess.check_call(cmd, **args)


 def checked_call_with_output(cmd, expected=None, unexpected=None, stderr=None):
-  stdout = subprocess.check_output(cmd.split(' '), stderr=stderr)
+  cmd = cmd.split(' ')
+  print('running: %s' % cmd)
+  stdout = subprocess.check_output(cmd, stderr=stderr)
  if expected:
    for x in listify(expected):
      assert x in stdout, 'call had the right output: ' + stdout + '\n[[[' + x + ']]]'
@@ -62,9 +67,9 @@ assert 'upstream' not in open(os.path.expanduser('~/.emscripten')).read()
 print('building proper system libraries')


-def test_lib_building(prefix, use_asmjs_optimizer):
+def test_lib_building(emcc, use_asmjs_optimizer):
  def test_build(args, expected=None, unexpected=None):
-    checked_call_with_output(prefix + '/emscripten/emcc hello_world.cpp' + args,
+    checked_call_with_output(emcc + ' hello_world.cpp' + args,
                             expected=expected,
                             unexpected=unexpected,
                             stderr=subprocess.STDOUT)
@@ -90,62 +95,81 @@ def test_lib_building(prefix, use_asmjs_optimizer):
  test_build(' -O2 -s WASM=0', unexpected=unexpected_system_libs + first_time_system_libs)


-test_lib_building('fastcomp', use_asmjs_optimizer=True)
+def run_emsdk(cmd):
+  if type(cmd) != list:
+    cmd = cmd.split()
+  check_call([emsdk] + cmd)
+
+
+WINDOWS = sys.platform.startswith('win')
+
+upstream_emcc = os.path.join('upstream', 'emscripten', 'emcc')
+fastcomp_emcc = os.path.join('fastcomp', 'emscripten', 'emcc')
+emsdk = './emsdk'
+if WINDOWS:
+  upstream_emcc += '.bat'
+  fastcomp_emcc += '.bat'
+  emsdk = 'emsdk.bat'
+else:
+  emsdk = './emsdk'
+
+test_lib_building(fastcomp_emcc, use_asmjs_optimizer=True)

 print('update')
-check_call('./emsdk update-tags')
+run_emsdk('update-tags')

 print('test latest-releases-upstream')
-check_call('python2 ./emsdk.py install latest-upstream')
-check_call('./emsdk activate latest-upstream')
-test_lib_building('upstream', use_asmjs_optimizer=False)
+run_emsdk('install latest-upstream')
+run_emsdk('activate latest-upstream')
+test_lib_building(upstream_emcc, use_asmjs_optimizer=False)
 assert open(os.path.expanduser('~/.emscripten')).read().count('LLVM_ROOT') == 1
 assert 'upstream' in open(os.path.expanduser('~/.emscripten')).read()
 assert 'fastcomp' not in open(os.path.expanduser('~/.emscripten')).read()

+
 print('verify version')
-checked_call_with_output('upstream/emscripten/emcc -v', TAGS['latest'], stderr=subprocess.STDOUT)
+checked_call_with_output(upstream_emcc + ' -v', TAGS['latest'], stderr=subprocess.STDOUT)

 print('clear cache')
-check_call('upstream/emscripten/emcc --clear-cache')
+check_call(upstream_emcc + ' --clear-cache')
 assert not os.path.exists(LIBC)

 print('test tot-upstream')
-check_call('./emsdk install tot-upstream')
+run_emsdk('install tot-upstream')
 assert not os.path.exists(LIBC)
 old_config = open(os.path.expanduser('~/.emscripten')).read()
-check_call('./emsdk activate tot-upstream')
+run_emsdk('activate tot-upstream')
 assert old_config == open(os.path.expanduser('~/.emscripten.old')).read()
 assert os.path.exists(LIBC), 'activation supplies prebuilt libc' # TODO; test on latest as well
-check_call('upstream/emscripten/emcc hello_world.cpp')
+check_call(upstream_emcc + ' hello_world.cpp')

 print('test tot-fastcomp')
-check_call('./emsdk install tot-fastcomp')
-check_call('./emsdk activate tot-fastcomp')
-check_call('fastcomp/emscripten/emcc hello_world.cpp')
+run_emsdk('install tot-fastcomp')
+run_emsdk('activate tot-fastcomp')
+check_call(fastcomp_emcc + ' hello_world.cpp')

 print('test specific release (old)')
-check_call('./emsdk install sdk-1.38.31-64bit')
-check_call('./emsdk activate sdk-1.38.31-64bit')
+run_emsdk('install sdk-1.38.31-64bit')
+run_emsdk('activate sdk-1.38.31-64bit')

 print('test specific release (new, short name)')
-check_call('./emsdk install 1.38.33')
+run_emsdk('install 1.38.33')
 print('another install must re-download')
-checked_call_with_output('./emsdk install 1.38.33', expected='Downloading:', unexpected='already exist in destination')
-check_call('./emsdk activate 1.38.33')
+checked_call_with_output(emsdk + ' install 1.38.33', expected='Downloading:', unexpected='already exist in destination')
+run_emsdk('activate 1.38.33')
 assert 'fastcomp' in open(os.path.expanduser('~/.emscripten')).read()
 assert 'upstream' not in open(os.path.expanduser('~/.emscripten')).read()

 print('test specific release (new, full name)')
-check_call('./emsdk install sdk-1.38.33-upstream-64bit')
-check_call('./emsdk activate sdk-1.38.33-upstream-64bit')
+run_emsdk('install sdk-1.38.33-upstream-64bit')
+run_emsdk('activate sdk-1.38.33-upstream-64bit')

 print('test specific release (new, full name)')
-check_call('./emsdk install sdk-tag-1.38.33-64bit')
-check_call('./emsdk activate sdk-tag-1.38.33-64bit')
+run_emsdk('install sdk-tag-1.38.33-64bit')
+run_emsdk('activate sdk-tag-1.38.33-64bit')

 print('test binaryen source build')
-check_call('./emsdk install --build=Release --generator="Unix Makefiles" binaryen-master-64bit')
+run_emsdk(['install', '--build=Release', '--generator=Unix Makefiles', 'binaryen-master-64bit'])

 print('test 32-bit error')

@@ -161,9 +185,9 @@ for filename in os.listdir('.'):

 os.chdir(temp_dir)

-check_call('./emsdk update')
+run_emsdk('update')
 print('second time')
-check_call('./emsdk update')
+run_emsdk('update')

 print('verify downloads exist for all OSes')
 latest_hash = TAGS['releases'][TAGS['latest']]
@@ -173,5 +197,5 @@ for osname, suffix in [
  ('win', 'zip')
 ]:
  url = 'https://storage.googleapis.com/webassembly/emscripten-releases-builds/%s/%s/wasm-binaries.%s' % (osname, latest_hash, suffix)
-  print('  url: ' + url),
-  check_call('wget ' + url)
+  print('  checking url: ' + url),
+  check_call('curl --fail --head --silent ' + url, stdout=subprocess.PIPE)
--- a/test.sh
+++ b/test.sh
@@ -1,7 +1,11 @@
-#!/usr/bin/env sh
+#!/usr/bin/env bash
+
 echo "test the standard workflow (as close as possible to how a user would do it, in the shell)"
+
+set -x
+set -e
+
 ./emsdk install latest
 ./emsdk activate latest
 source ./emsdk_env.sh --build=Release
 emcc -v
-