From a3089a4501939af3a6df928953a4338c9b2d6676 Mon Sep 17 00:00:00 2001 From: Mark McCaskey Date: Wed, 24 Jul 2019 17:31:59 -0700 Subject: [PATCH 1/7] fix caching for other backends, improve default compiler code --- Cargo.toml | 7 ++--- lib/runtime-core/Cargo.toml | 1 + lib/runtime-core/src/backend.rs | 1 + lib/runtime-core/src/cache.rs | 1 + lib/runtime/Cargo.toml | 9 ++++--- lib/runtime/src/cache.rs | 7 ++++- lib/runtime/src/lib.rs | 46 ++++++++++++++++++++++++++++++--- 7 files changed, 62 insertions(+), 10 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1a3432803..fd7d5d10f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,15 +66,16 @@ glob = "0.2.11" rustc_version = "0.2.3" [features] -default = ["fast-tests", "wasi"] +default = ["fast-tests", "wasi", "backend-cranelift"] "loader-kernel" = ["wasmer-kernel-loader"] debug = ["wasmer-runtime-core/debug"] trace = ["wasmer-runtime-core/trace"] extra-debug = ["wasmer-clif-backend/debug", "wasmer-runtime-core/debug"] # This feature will allow cargo test to run much faster fast-tests = [] -"backend-llvm" = ["wasmer-llvm-backend", "wasmer-runtime-core/backend-llvm"] -"backend-singlepass" = ["wasmer-singlepass-backend", "wasmer-runtime-core/backend-singlepass"] +backend-cranelift = ["wasmer-runtime-core/backend-cranelift", "wasmer-runtime/cranelift"] +backend-llvm = ["wasmer-llvm-backend", "wasmer-runtime-core/backend-llvm", "wasmer-runtime/llvm"] +backend-singlepass = ["wasmer-singlepass-backend", "wasmer-runtime-core/backend-singlepass", "wasmer-runtime/singlepass"] wasi = ["wasmer-wasi"] # vfs = ["wasmer-runtime-abi"] diff --git a/lib/runtime-core/Cargo.toml b/lib/runtime-core/Cargo.toml index 364b82262..ed5c54592 100644 --- a/lib/runtime-core/Cargo.toml +++ b/lib/runtime-core/Cargo.toml @@ -55,5 +55,6 @@ cc = "1.0" debug = [] trace = ["debug"] # backend flags used in conditional compilation of Backend::variants +"backend-cranelift" = [] "backend-singlepass" = [] "backend-llvm" = [] diff --git a/lib/runtime-core/src/backend.rs b/lib/runtime-core/src/backend.rs index 1eb8ba108..91f737b69 100644 --- a/lib/runtime-core/src/backend.rs +++ b/lib/runtime-core/src/backend.rs @@ -32,6 +32,7 @@ pub enum Backend { impl Backend { pub fn variants() -> &'static [&'static str] { &[ + #[cfg(feature = "backend-cranelift")] "cranelift", #[cfg(feature = "backend-singlepass")] "singlepass", diff --git a/lib/runtime-core/src/cache.rs b/lib/runtime-core/src/cache.rs index 18c571b00..57f36e7ea 100644 --- a/lib/runtime-core/src/cache.rs +++ b/lib/runtime-core/src/cache.rs @@ -20,6 +20,7 @@ pub enum Error { Unknown(String), InvalidFile(InvalidFileType), InvalidatedCache, + UnsupportedBackend(Backend), } impl From for Error { diff --git a/lib/runtime/Cargo.toml b/lib/runtime/Cargo.toml index d613ded7d..e8e02afd0 100644 --- a/lib/runtime/Cargo.toml +++ b/lib/runtime/Cargo.toml @@ -32,12 +32,15 @@ path = "../llvm-backend" optional = true [features] -default = ["default-compiler"] -default-compiler = ["wasmer-clif-backend"] -cache = ["default-compiler"] +default = ["cranelift", "default-backend-cranelift"] +cranelift = ["wasmer-clif-backend"] +cache = ["cranelift"] debug = ["wasmer-clif-backend/debug", "wasmer-runtime-core/debug"] llvm = ["wasmer-llvm-backend"] singlepass = ["wasmer-singlepass-backend"] +default-backend-singlepass = ["singlepass"] +default-backend-llvm = ["llvm"] +default-backend-cranelift = ["cranelift"] [[bench]] name = "nginx" diff --git a/lib/runtime/src/cache.rs b/lib/runtime/src/cache.rs index fdb0d58c8..015212e1b 100644 --- a/lib/runtime/src/cache.rs +++ b/lib/runtime/src/cache.rs @@ -103,7 +103,12 @@ impl Cache for FileSystemCache { let serialized_cache = Artifact::deserialize(&mmap[..])?; unsafe { - wasmer_runtime_core::load_cache_with(serialized_cache, &super::default_compiler()) + wasmer_runtime_core::load_cache_with( + serialized_cache, + super::compiler_for_backend(backend) + .ok_or_else(|| CacheError::UnsupportedBackend(backend))? + .as_ref(), + ) } } diff --git a/lib/runtime/src/lib.rs b/lib/runtime/src/lib.rs index efe72ea10..0ea37a0d3 100644 --- a/lib/runtime/src/lib.rs +++ b/lib/runtime/src/lib.rs @@ -76,6 +76,7 @@ //! [`wasmer-clif-backend`]: https://crates.io/crates/wasmer-clif-backend //! [`compile_with`]: fn.compile_with.html +pub use wasmer_runtime_core::backend::Backend; pub use wasmer_runtime_core::codegen::{MiddlewareChain, StreamingCompiler}; pub use wasmer_runtime_core::export::Export; pub use wasmer_runtime_core::global::Global; @@ -179,17 +180,56 @@ pub fn instantiate(wasm: &[u8], import_object: &ImportObject) -> error::Result impl Compiler { - #[cfg(feature = "llvm")] + #[cfg(any( + all( + feature = "default-backend-llvm", + any( + feature = "default-backend-cranelift", + feature = "default-backend-singlepass" + ) + ), + all( + feature = "default-backend-cranelift", + feature = "default-backend-singlepass" + ) + ))] + compile_error!( + "The `default-backend-X` features are mutually exclusive. Please choose just one" + ); + + #[cfg(feature = "default-backend-llvm")] use wasmer_llvm_backend::LLVMCompiler as DefaultCompiler; - #[cfg(feature = "singlepass")] + #[cfg(feature = "default-backend-singlepass")] use wasmer_singlepass_backend::SinglePassCompiler as DefaultCompiler; - #[cfg(not(any(feature = "llvm", feature = "singlepass")))] + #[cfg(feature = "default-backend-cranelift")] use wasmer_clif_backend::CraneliftCompiler as DefaultCompiler; DefaultCompiler::new() } +pub fn compiler_for_backend(backend: Backend) -> Option> { + match backend { + #[cfg(feature = "cranelift")] + Backend::Cranelift => Some(Box::new(wasmer_clif_backend::CraneliftCompiler::new())), + + #[cfg(feature = "singlepass")] + Backend::Singlepass => Some(Box::new( + wasmer_singlepass_backend::SinglePassCompiler::new(), + )), + + #[cfg(feature = "llvm")] + Backend::LLVM => Some(Box::new(wasmer_llvm_backend::LLVMCompiler::new())), + + #[cfg(any( + not(feature = "llvm"), + not(feature = "singlepass"), + not(feature = "cranelift") + ))] + _ => None, + } +} + /// The current version of this crate pub const VERSION: &str = env!("CARGO_PKG_VERSION"); From 99b75c4deaae31cfe0c94a6e89524b70e1f5f077 Mon Sep 17 00:00:00 2001 From: Mark McCaskey Date: Wed, 24 Jul 2019 17:46:32 -0700 Subject: [PATCH 2/7] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e097611c..c81eae7bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ All PRs to the Wasmer repository must add to this file. Blocks of changes will separated by version increments. ## **[Unreleased]** +- [#579](https://github.com/wasmerio/wasmer/pull/579) Fix bug in caching with LLVM and Singlepass backends. Add `default-backend-singlepass`, `default-backend-llvm`, and `default-backend-cranelift` features to `wasmer-runtime` to control the `default_compiler()` function (this is a breaking change). Add `compiler_for_backend` function in `wasmer-runtime` - [#542](https://github.com/wasmerio/wasmer/pull/542) Add SIMD support to wasmer and implement it in the LLVM backend only. ## 0.5.7 From 8f417f3d595585b18dd5cdcf5edc0443a57c1c05 Mon Sep 17 00:00:00 2001 From: Nick Lewycky Date: Thu, 25 Jul 2019 23:33:30 -0700 Subject: [PATCH 3/7] Add --enable-simd flag to wasmer run and wasmer validate. --- lib/runtime-core/src/backend.rs | 6 +++++ lib/runtime-core/src/lib.rs | 20 +++++++++++++++- src/bin/kwasmd.rs | 1 + src/bin/wasmer.rs | 42 +++++++++++++++++++++++++++++---- 4 files changed, 63 insertions(+), 6 deletions(-) diff --git a/lib/runtime-core/src/backend.rs b/lib/runtime-core/src/backend.rs index 1eb8ba108..572fe7fb9 100644 --- a/lib/runtime-core/src/backend.rs +++ b/lib/runtime-core/src/backend.rs @@ -110,6 +110,11 @@ impl Default for MemoryBoundCheckMode { } } +#[derive(Default)] +pub struct Features { + pub simd: bool, +} + /// Configuration data for the compiler #[derive(Default)] pub struct CompilerConfig { @@ -118,6 +123,7 @@ pub struct CompilerConfig { pub memory_bound_check_mode: MemoryBoundCheckMode, pub enforce_stack_check: bool, pub track_state: bool, + pub features: Features, } pub trait Compiler { diff --git a/lib/runtime-core/src/lib.rs b/lib/runtime-core/src/lib.rs index 52b7cf82a..2eb26cb89 100644 --- a/lib/runtime-core/src/lib.rs +++ b/lib/runtime-core/src/lib.rs @@ -117,8 +117,26 @@ pub fn validate(wasm: &[u8]) -> bool { /// The same as `validate` but with an Error message on failure pub fn validate_and_report_errors(wasm: &[u8]) -> ::std::result::Result<(), String> { + validate_and_report_errors_with_features(wasm, Default::default()) +} + +/// The same as `validate_and_report_errors` but with a Features. +pub fn validate_and_report_errors_with_features( + wasm: &[u8], + features: backend::Features, +) -> ::std::result::Result<(), String> { use wasmparser::WasmDecoder; - let mut parser = wasmparser::ValidatingParser::new(wasm, None); + let config = wasmparser::ValidatingParserConfig { + operator_config: wasmparser::OperatorValidatorConfig { + enable_simd: features.simd, + enable_bulk_memory: false, + enable_multi_value: false, + enable_reference_types: false, + enable_threads: false, + }, + mutable_global_imports: false, + }; + let mut parser = wasmparser::ValidatingParser::new(wasm, Some(config)); loop { let state = parser.read(); match *state { diff --git a/src/bin/kwasmd.rs b/src/bin/kwasmd.rs index bee998471..d0e0723cd 100644 --- a/src/bin/kwasmd.rs +++ b/src/bin/kwasmd.rs @@ -55,6 +55,7 @@ fn handle_client(mut stream: UnixStream) { memory_bound_check_mode: MemoryBoundCheckMode::Disable, enforce_stack_check: true, track_state: false, + features: Default::default(), }, &SinglePassCompiler::new(), ) diff --git a/src/bin/wasmer.rs b/src/bin/wasmer.rs index 2f24548e3..0ac460dfc 100644 --- a/src/bin/wasmer.rs +++ b/src/bin/wasmer.rs @@ -23,7 +23,7 @@ use wasmer_runtime::{ }; use wasmer_runtime_core::{ self, - backend::{Backend, Compiler, CompilerConfig, MemoryBoundCheckMode}, + backend::{Backend, Compiler, CompilerConfig, Features, MemoryBoundCheckMode}, debug, loader::{Instance as LoadedInstance, LocalLoader}, }; @@ -67,6 +67,17 @@ enum CLIOptions { SelfUpdate, } +#[derive(Debug, StructOpt)] +struct PrestandardFeatures { + /// Enable support for the SIMD proposal. + #[structopt(long = "enable-simd")] + simd: bool, + + /// Enable support for all pre-standard proposals. + #[structopt(long = "enable-all")] + all: bool, +} + #[derive(Debug, StructOpt)] struct Run { // Disable the cache @@ -134,6 +145,9 @@ struct Run { #[structopt(long = "cache-key", hidden = true)] cache_key: Option, + #[structopt(flatten)] + features: PrestandardFeatures, + /// Application arguments #[structopt(name = "--", raw(multiple = "true"))] args: Vec, @@ -185,6 +199,9 @@ struct Validate { /// Input file #[structopt(parse(from_os_str))] path: PathBuf, + + #[structopt(flatten)] + features: PrestandardFeatures, } /// Read the contents of a file @@ -315,10 +332,14 @@ fn execute_wasm(options: &Run) -> Result<(), String> { None }; + // Don't error on --enable-all for other backends. + if options.features.simd && options.backend != Backend::LLVM { + return Err("SIMD is only supported in the LLVM backend for now".to_string()); + } + if !utils::is_wasm_binary(&wasm_binary) { let mut features = wabt::Features::new(); - if options.backend == Backend::LLVM { - // SIMD is only supported in the LLVM backend for now + if options.features.simd || options.features.all { features.enable_simd(); } wasm_binary = wabt::wat2wasm_with_features(wasm_binary, features) @@ -357,6 +378,9 @@ fn execute_wasm(options: &Run) -> Result<(), String> { memory_bound_check_mode: MemoryBoundCheckMode::Disable, enforce_stack_check: true, track_state, + features: Features { + simd: options.features.simd || options.features.all, + }, }, &*compiler, ) @@ -367,6 +391,9 @@ fn execute_wasm(options: &Run) -> Result<(), String> { CompilerConfig { symbol_map: em_symbol_map, track_state, + features: Features { + simd: options.features.simd || options.features.all, + }, ..Default::default() }, &*compiler, @@ -737,8 +764,13 @@ fn validate_wasm(validate: Validate) -> Result<(), String> { )); } - wasmer_runtime_core::validate_and_report_errors(&wasm_binary) - .map_err(|err| format!("Validation failed: {}", err))?; + wasmer_runtime_core::validate_and_report_errors_with_features( + &wasm_binary, + Features { + simd: validate.features.simd || validate.features.all, + }, + ) + .map_err(|err| format!("Validation failed: {}", err))?; Ok(()) } From 6cddc25ddde5bb1d6c8f34e744d2679ff4f98128 Mon Sep 17 00:00:00 2001 From: Nick Lewycky Date: Fri, 26 Jul 2019 11:12:13 -0700 Subject: [PATCH 4/7] Derive Debug on these small structs. --- lib/runtime-core/src/backend.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/runtime-core/src/backend.rs b/lib/runtime-core/src/backend.rs index 572fe7fb9..5253a59f5 100644 --- a/lib/runtime-core/src/backend.rs +++ b/lib/runtime-core/src/backend.rs @@ -110,13 +110,13 @@ impl Default for MemoryBoundCheckMode { } } -#[derive(Default)] +#[derive(Debug, Default)] pub struct Features { pub simd: bool, } /// Configuration data for the compiler -#[derive(Default)] +#[derive(Debug, Default)] pub struct CompilerConfig { /// Symbol information generated from emscripten; used for more detailed debug messages pub symbol_map: Option>, From a666fb3e4af83694e9f540e161e111fef292dad8 Mon Sep 17 00:00:00 2001 From: Mark McCaskey Date: Fri, 26 Jul 2019 11:24:00 -0700 Subject: [PATCH 5/7] Add Rust plugin example example to CI --- .circleci/config.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2461e71bf..e6fc97ec7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -129,6 +129,26 @@ jobs: - target/release/deps key: v8-test-cargo-cache-linux-nightly-{{ arch }}-{{ checksum "Cargo.lock" }} + test-rust-example: + docker: + - image: circleci/rust:latest + <<: *run_with_build_env_vars + steps: + - checkout + - run: + name: "Check Wasmer Rust example" + command: | + git clone https://github.com/wasmerio/wasmer-rust-example + rustup default stable + rustup target add wasm32-unknown-unknown + cd wasmer-rust-example + cd wasm-sample-app + cargo build --release + cd .. + sed -i 's/wasmer-runtime.*/wasmer-runtime = \{ path = "..\/lib\/runtime" \}/g' Cargo.toml + cargo run + cargo test + test-macos: macos: xcode: "9.0" @@ -383,6 +403,12 @@ workflows: only: - trying - staging + - test-rust-example: + filters: + branches: + only: + - trying + - staging - test-macos: filters: branches: From ce64f7ad0f8484873b11461de146a04c7c0e577c Mon Sep 17 00:00:00 2001 From: Mark McCaskey Date: Fri, 26 Jul 2019 15:00:52 -0700 Subject: [PATCH 6/7] Add wasm plugin rust example test to bors.toml --- bors.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/bors.toml b/bors.toml index d54fd2222..5fadbdfa0 100644 --- a/bors.toml +++ b/bors.toml @@ -3,6 +3,7 @@ status = [ "ci/circleci: test", "ci/circleci: test-macos", "ci/circleci: test-stable", + "ci/circleci: test-rust-example", "continuous-integration/appveyor/branch" ] required_approvals = 1 From d7c11bcf5fdeb5c95a4ee8d2f10b392ae69cff78 Mon Sep 17 00:00:00 2001 From: Brandon Fish Date: Fri, 26 Jul 2019 17:58:57 -0500 Subject: [PATCH 7/7] Fixes case with empty module, handle case with no signatures --- lib/clif-backend/src/code.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/clif-backend/src/code.rs b/lib/clif-backend/src/code.rs index 576f513e1..947ea88e2 100644 --- a/lib/clif-backend/src/code.rs +++ b/lib/clif-backend/src/code.rs @@ -304,8 +304,15 @@ impl ModuleCodeGenerator let trampolines = Arc::new(Trampolines::new(&*self.isa, module_info)); + let signatures_empty = Map::new(); + let signatures = if self.signatures.is_some() { + &self.signatures.as_ref().unwrap() + } else { + &signatures_empty + }; + let (func_resolver, backend_cache) = func_resolver_builder.finalize( - &self.signatures.as_ref().unwrap(), + signatures, Arc::clone(&trampolines), handler_data.clone(), )?;