Improved clarity on tests

2024-12-05 02:20:19 +00:00 · 2019-07-05 18:36:34 -07:00 · 2019-07-05 18:36:34 -07:00 · 39a7b70aa9
commit 39a7b70aa9
parent 7ba2a4c7cc
3 changed files with 146 additions and 187 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -60,17 +60,16 @@ jobs:
          keys:
            - v8-test-cargo-cache-linux-stable-{{ arch }}-{{ checksum "Cargo.lock" }}
      - <<: *run_install_dependencies
+      - run: rustup default nightly-2019-05-20
      - run:
          name: Tests
          command: make test
-      - run:
-          name: Emscripten Tests
-          command: |
-            make test-emscripten-clif
-            make test-emscripten-llvm
      - run:
          name: Integration Tests
          command: make integration-tests
+      - run:
+          name: Release
+          command: make fast-release
      - save_cache:
          paths:
            - /usr/local/cargo/registry
@ -100,9 +99,9 @@ jobs:
      - run:
          name: Install Rust
          command: |
-            curl -sSf https://sh.rustup.rs | sh -s -- -y
+            curl -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2019-05-20
            export PATH="$HOME/.cargo/bin:$PATH"
-            cargo --version
+            cargo +nightly --version
      - run:
          name: Tests
          command: |
@ -113,17 +112,6 @@ jobs:
            ulimit -n 8000
            sudo sysctl -w kern.maxfiles=655360 kern.maxfilesperproc=327680
            make test
-      - run:
-          name: Emscripten Tests
-          command: |
-            export PATH="$HOME/.cargo/bin:$PATH"
-            export PATH="`pwd`/cmake-3.4.1-Darwin-x86_64/CMake.app/Contents/bin:$PATH"
-            export LLVM_SYS_70_PREFIX="`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/"
-            # We increase the ulimit for fixing cargo unclosed files in mac
-            ulimit -n 8000
-            sudo sysctl -w kern.maxfiles=655360 kern.maxfilesperproc=327680
-            make test-emscripten-clif
-            make test-emscripten-llvm
      - run:
          name: Integration Tests
          command: |
@ -131,12 +119,14 @@ jobs:
            export PATH="`pwd`/cmake-3.4.1-Darwin-x86_64/CMake.app/Contents/bin:$PATH"
            export LLVM_SYS_70_PREFIX="`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/"
            make integration-tests
+      - run:
+          name: Release
+          command: |
+            export PATH="$HOME/.cargo/bin:$PATH"
+            make fast-release
      - save_cache:
          paths:
            - ~/.cargo/registry/
-            - target/debug/.fingerprint
-            - target/debug/build
-            - target/debug/deps
            - target/release/.fingerprint
            - target/release/build
            - target/release/deps
@ -161,29 +151,22 @@ jobs:
            sudo apt-get install -y cmake
            curl -O https://releases.llvm.org/7.0.0/clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
            tar xf clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
-      # Use rust nightly (for singlepass, for now)
-      - run: rustup default nightly-2019-04-11
+      - run: rustup default nightly-2019-05-20
+      - run:
+          name: Debug flag checked
+          command: |
+            cargo +nightly check --features "debug" --release
      - run:
          name: Tests
          command: |
            export LLVM_SYS_70_PREFIX="`pwd`/clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04/"
            make test
-      - run:
-          name: Emscripten Tests
-          command: |
-            export LLVM_SYS_70_PREFIX="`pwd`/clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04/"
-            make test-emscripten-clif
-            make test-emscripten-llvm
-      - run:
-          name: Debug flag checked
-          command: |
-            cargo check --features "debug"
      - run:
          name: Release Build
          command: |
            export LLVM_SYS_70_PREFIX="`pwd`/clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04/"
-            make production-release
-            cargo build --release --manifest-path wapm-cli/Cargo.toml --features telemetry
+            make release
+            cargo +nightly build --release --manifest-path wapm-cli/Cargo.toml --features telemetry
            mkdir -p artifacts
            VERSION=$(cargo pkgid | cut -d# -f2 | cut -d: -f2)
            # GIT_VERSION=$(git describe --exact-match --tags)
@ -194,7 +177,7 @@ jobs:
      - run:
          name: Dynamic library
          command: |
-            cargo build --release --manifest-path lib/runtime-c-api/Cargo.toml
+            cargo +nightly build --release --manifest-path lib/runtime-c-api/Cargo.toml
            cp target/release/libwasmer_runtime_c_api.so ./artifacts
      - persist_to_workspace:
          root: .
@ -203,9 +186,6 @@ jobs:
      - save_cache:
          paths:
            - /usr/local/cargo/registry
-            - target/debug/.fingerprint
-            - target/debug/build
-            - target/debug/deps
            - target/release/.fingerprint
            - target/release/build
            - target/release/deps
@ -240,15 +220,9 @@ jobs:
      - run:
          name: Install Rust
          command: |
-            curl -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly
+            curl -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly-2019-05-20
            export PATH="$HOME/.cargo/bin:$PATH"
-            cargo --version
-      # Use rust nightly (for singlepass, for now)
-      # - run:
-      #     name: Install Rust nightly
-      #     command: |
-      #       export PATH="$HOME/.rustup/bin:$PATH"
-      #       rustup default nightly-2019-04-11
+            cargo +nightly --version
      - run:
          name: Tests
          command: |
@ -258,26 +232,16 @@ jobs:
            # We increase the ulimit for fixing cargo unclosed files in mac
            ulimit -n 8000
            sudo sysctl -w kern.maxfiles=655360 kern.maxfilesperproc=327680
+
            make test
-      - run:
-          name: Emscripten Tests
-          command: |
-            export PATH="`pwd`/cmake-3.4.1-Darwin-x86_64/CMake.app/Contents/bin:$PATH"
-            export PATH="$HOME/.cargo/bin:$PATH"
-            export LLVM_SYS_70_PREFIX="`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/"
-            # We increase the ulimit for fixing cargo unclosed files in mac
-            ulimit -n 8000
-            sudo sysctl -w kern.maxfiles=655360 kern.maxfilesperproc=327680
-            make test-emscripten-clif
-            make test-emscripten-singlepass
      - run:
          name: Release Build
          command: |
            export PATH="`pwd`/cmake-3.4.1-Darwin-x86_64/CMake.app/Contents/bin:$PATH"
            export PATH="$HOME/.cargo/bin:$PATH"
            export LLVM_SYS_70_PREFIX="`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/"
-            make production-release
-            cargo build --release --manifest-path wapm-cli/Cargo.toml --features telemetry
+            make release
+            cargo +nightly build --release --manifest-path wapm-cli/Cargo.toml --features telemetry
            mkdir -p artifacts
            make build-install
            cp ./wasmer.tar.gz ./artifacts/$(./binary-name.sh)
@ -287,7 +251,7 @@ jobs:
          name: Generate dynamic library for the runtime C API
          command: |
            export PATH="$HOME/.cargo/bin:$PATH"
-            cargo build --release --manifest-path lib/runtime-c-api/Cargo.toml
+            cargo +nightly build --release --manifest-path lib/runtime-c-api/Cargo.toml
            install_name_tool -id "@rpath/libwasmer_runtime_c_api.dylib" target/release/libwasmer_runtime_c_api.dylib
            cp target/release/libwasmer_runtime_c_api.dylib ./artifacts
      - persist_to_workspace:
@ -297,9 +261,6 @@ jobs:
      - save_cache:
          paths:
            - ~/.cargo/registry/
-            - target/debug/.fingerprint
-            - target/debug/build
-            - target/debug/deps
            - target/release/.fingerprint
            - target/release/build
            - target/release/deps
@ -308,41 +269,6 @@ jobs:
            - wapm-cli/target/release/deps
          key: v8-cargo-cache-darwin-nightly-{ arch }}-{{ checksum "Cargo.lock" }}

-  test-rust-nightly:
-    docker:
-      - image: circleci/rust:latest
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-            - v8-cargo-cache-linux-nightly-{{ arch }}-{{ checksum "Cargo.lock" }}
-      - run:
-          name: Install dependencies
-          command: |
-            sudo apt-get install -y cmake
-            curl -O https://releases.llvm.org/7.0.0/clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
-            tar xf clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
-            rustup toolchain install nightly
-            rustup target add wasm32-wasi --toolchain nightly
-      - run: |
-          rustup default nightly
-          make test-wasi-singlepass
-          make test-wasi-clif
-      - run: rustup default nightly-2019-04-11
-      - run: |
-          export LLVM_SYS_70_PREFIX="`pwd`/clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04/"
-          make test
-          make test-singlepass
-          make test-emscripten-clif
-          make test-emscripten-singlepass
-      - save_cache:
-          paths:
-            - /usr/local/cargo/registry
-            - target/debug/.fingerprint
-            - target/debug/build
-            - target/debug/deps
-          key: v8-cargo-cache-linux-nightly-{{ arch }}-{{ checksum "Cargo.lock" }}-nightly
-
  publish-github-release:
    docker:
      - image: cibuilds/github
--- a/180
+++ b/180
@ -1,41 +1,125 @@
-ifeq (test, $(firstword $(MAKECMDGOALS)))
-  runargs := $(wordlist 2, $(words $(MAKECMDGOALS)), $(MAKECMDGOALS))
-  $(eval $(runargs):;@true)
-endif
-
 .PHONY: spectests emtests clean build install lint precommit

-# This will re-generate the Rust test files based on spectests/*.wast
-spectests:
-	WASMER_RUNTIME_GENERATE_SPECTESTS=1 cargo build -p wasmer-runtime-core
+# Generate files
+generate-spectests:
+	WASMER_RUNTIME_GENERATE_SPECTESTS=1 cargo +nightly build -p wasmer-runtime-core --release

-emtests:
-	WASM_EMSCRIPTEN_GENERATE_EMTESTS=1 cargo build -p wasmer-emscripten
+generate-emtests:
+	WASM_EMSCRIPTEN_GENERATE_EMTESTS=1 cargo +nightly build -p wasmer-emscripten --release

-wasitests:
-	WASM_WASI_GENERATE_WASITESTS=1 cargo build -p wasmer-wasi
+generate-wasitests:
+	WASM_WASI_GENERATE_WASITESTS=1 cargo +nightly build -p wasmer-wasi --release

-# clean:
-#     rm -rf artifacts
+generate: generate-spectests generate-emtests generate-wasitests

-build:
-	cargo build --features debug

-install:
-	cargo install --path .
+# Spectests
+spectests-singlepass:
+	cargo +nightly test --manifest-path lib/spectests/Cargo.toml --release --features singlepass

+spectests-cranelift:
+	cargo +nightly test --manifest-path lib/spectests/Cargo.toml --release --features clif
+
+spectests-llvm:
+	cargo +nightly test --manifest-path lib/spectests/Cargo.toml --release --features llvm
+
+spectests: spectests-singlepass spectests-cranelift spectests-llvm
+
+
+# Emscripten tests
+emtests-singlepass:
+	cargo +nightly test --manifest-path lib/emscripten/Cargo.toml --release --features singlepass -- --test-threads=1
+
+emtests-cranelift:
+	cargo +nightly test --manifest-path lib/emscripten/Cargo.toml --release --features clif -- --test-threads=1
+
+emtests-llvm:
+	cargo +nightly test --manifest-path lib/emscripten/Cargo.toml --release --features llvm -- --test-threads=1
+
+emtests: emtests-singlepass emtests-cranelift emtests-llvm
+
+
+# Middleware tests
+middleware-singlepass:
+	cargo +nightly test --manifest-path lib/middleware-common/Cargo.toml --release --features singlepass
+
+middleware-cranelift:
+	cargo +nightly test --manifest-path lib/middleware-common/Cargo.toml --release --features clif
+
+middleware-llvm:
+	cargo +nightly test --manifest-path lib/middleware-common/Cargo.toml --release --features llvm
+
+middleware: middleware-singlepass middleware-cranelift middleware-llvm
+
+
+# Wasitests
+wasitests-singlepass:
+	cargo +nightly test --manifest-path lib/wasi/Cargo.toml --release --features singlepass -- --test-threads=1
+
+wasitests-cranelift:
+	cargo +nightly test --manifest-path lib/wasi/Cargo.toml --release --features clif -- --test-threads=1
+
+wasitests-llvm:
+	cargo +nightly test --manifest-path lib/wasi/Cargo.toml --release --features llvm -- --test-threads=1
+
+wasitests: wasitests-singlepass wasitests-cranelift wasitests-llvm
+
+
+# Backends
+singlepass: spectests-singlepass emtests-singlepass middleware-singlepass wasitests-singlepass
+	cargo +nightly test -p wasmer-singlepass-backend --release
+
+cranelift: spectests-cranelift emtests-cranelift middleware-cranelift wasitests-cranelift
+	cargo +nightly test -p wasmer-clif-backend --release
+
+llvm: spectests-llvm emtests-llvm middleware-llvm wasitests-llvm
+	cargo +nightly test -p wasmer-llvm-backend --release
+
+
+# All tests
+test-rest:
+	cargo +nightly test --release --all --exclude wasmer-emscripten --exclude wasmer-spectests --exclude wasmer-wasi --exclude wasmer-middleware-common --exclude wasmer-singlepass-backend --exclude wasmer-clif-backend --exclude wasmer-llvm-backend
+
+circleci-clean:
+	@if [ ! -z "${CIRCLE_JOB}" ]; then rm -f /home/circleci/project/target/debug/deps/libcranelift_wasm* && rm -f /Users/distiller/project/target/debug/deps/libcranelift_wasm*; fi;
+
+test: spectests emtests middleware wasitests circleci-clean test-rest
+
+
+# Integration tests
 integration-tests: release
 	echo "Running Integration Tests"
 	./integration_tests/lua/test.sh
 	./integration_tests/nginx/test.sh
 	./integration_tests/cowsay/test.sh

+
+# Utils
 lint:
-	cargo fmt --all -- --check
-	cargo +nightly-2019-05-20 clippy --all
+	cargo +nightly fmt --all -- --check

 precommit: lint test

+build:
+	cargo +nightly build --release --features debug
+
+install:
+	cargo +nightly install --release --path .
+
+release:
+	cargo +nightly build --release --features backend:singlepass,backend:llvm,loader:kernel
+
+# Only one backend (cranelift)
+release-fast:
+	# If you are in OS-X, you will need mingw-w64 for cross compiling to windows
+	# brew install mingw-w64
+	cargo +nightly build --release
+
+bench:
+	cargo +nightly bench --all
+
+
+# Build utils
 build-install:
 	mkdir -p ./install/bin
 	cp ./wapm-cli/target/release/wapm ./install/bin/
@ -46,62 +130,6 @@ build-install:
 do-install:
 	tar -C ~/.wasmer -zxvf wasmer.tar.gz

-test:
-	# We use one thread so the emscripten stdouts doesn't collide
-	cargo test --all --exclude wasmer-runtime-c-api --exclude wasmer-emscripten --exclude wasmer-spectests --exclude wasmer-singlepass-backend --exclude wasmer-wasi --exclude wasmer-middleware-common -- $(runargs)
-	# cargo test --all --exclude wasmer-emscripten -- --test-threads=1 $(runargs)
-	cargo test --manifest-path lib/spectests/Cargo.toml --features clif
-	cargo test --manifest-path lib/middleware-common/Cargo.toml --features clif
-	@if [ ! -z "${CIRCLE_JOB}" ]; then rm -f /home/circleci/project/target/debug/deps/libcranelift_wasm* && rm -f /Users/distiller/project/target/debug/deps/libcranelift_wasm*; fi;
-	cargo test --manifest-path lib/spectests/Cargo.toml --features llvm
-	cargo test --manifest-path lib/runtime/Cargo.toml --features llvm
-	cargo test --manifest-path lib/middleware-common/Cargo.toml --features llvm
-	cargo build -p wasmer-runtime-c-api
-	cargo test -p wasmer-runtime-c-api -- --nocapture
-
-test-singlepass:
-	cargo test --manifest-path lib/spectests/Cargo.toml --features singlepass
-	cargo test --manifest-path lib/runtime/Cargo.toml --features singlepass
-	cargo test --manifest-path lib/middleware-common/Cargo.toml --features singlepass
-
-test-emscripten-llvm:
-	cargo test --manifest-path lib/emscripten/Cargo.toml --features llvm -- --test-threads=1 $(runargs)
-
-test-emscripten-clif:
-	cargo test --manifest-path lib/emscripten/Cargo.toml --features clif -- --test-threads=1 $(runargs)
-
-test-emscripten-singlepass:
-	cargo test --manifest-path lib/emscripten/Cargo.toml --features singlepass -- --test-threads=1 $(runargs)
-
-test-wasi-clif:
-	cargo test --manifest-path lib/wasi/Cargo.toml --features "clif" -- --test-threads=1 $(runargs)
-
-test-wasi-singlepass:
-	cargo test --manifest-path lib/wasi/Cargo.toml --features "singlepass" -- --test-threads=1 $(runargs)
-
-singlepass-debug-release:
-	cargo +nightly build --features backend:singlepass,debug --release
-
-singlepass-release:
-	cargo +nightly build --features backend:singlepass --release
-
-singlepass-build:
-	cargo +nightly build --features backend:singlepass,debug
-
-release:
-	# If you are in OS-X, you will need mingw-w64 for cross compiling to windows
-	# brew install mingw-w64
-	cargo build --release
-
-production-release:
-	cargo build --release --features backend:singlepass,backend:llvm,loader:kernel
-
-debug-release:
-	cargo build --release --features debug
-
-extra-debug-release:
-	cargo build --release --features extra-debug
-
 publish-release:
 	ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -delete ${VERSION} ./artifacts/

--- a/README.md
+++ b/README.md
@ -160,38 +160,43 @@ cd wasmer

 # install tools
 # make sure that `python` is accessible.
-cargo install --path .
+make install
 ```

 ## Testing

 Thanks to [spec tests](https://github.com/wasmerio/wasmer/tree/master/lib/spectests/spectests) we can ensure 100% compatibility with the WebAssembly spec test suite.

-Tests can be run with:
+You can run all the tests with:

 ```sh
 make test
 ```

-If you need to regenerate the Rust tests from the spec tests
-you can run:
+### Testing backends

-```sh
-make spectests
-```
+Each backend can be tested separately:

-You can also run integration tests with:
+* Singlepass: `make singlepass`
+* Cranelift: `make cranelift`
+* LLVM: `make llvm`
+
+### Testing integrations
+
+Each integration can be tested separately:
+
+* Spec tests: `make spectests`
+* Emscripten: `make emtests`
+* WASI: `make wasi`
+* Middleware: `make middleware`

-```sh
-make integration-tests
-```

 ## Benchmarking

 Benchmarks can be run with:

 ```sh
-cargo bench --all
+make bench
 ```

 ## Roadmap