diff --git a/Cargo.lock b/Cargo.lock index 1f63d5ed7..e1b93e1c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16,6 +16,11 @@ dependencies = [ "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "approx" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "arrayref" version = "0.3.5" @@ -65,6 +70,16 @@ dependencies = [ "libc 0.2.57 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "bincode" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "bindgen" version = "0.46.0" @@ -163,6 +178,16 @@ name = "cfg-if" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "cgmath" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "approx 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "clang-sys" version = "0.26.4" @@ -203,6 +228,15 @@ dependencies = [ "cc 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "colored" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winconsole 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "constant_time_eq" version = "0.1.3" @@ -706,6 +740,14 @@ dependencies = [ "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "num-traits" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "num-traits" version = "0.2.8" @@ -814,6 +856,18 @@ dependencies = [ "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.57 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rand" version = "0.6.5" @@ -999,6 +1053,11 @@ dependencies = [ "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rgb" +version = "0.8.13" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "rustc-demangle" version = "0.1.15" @@ -1496,7 +1555,10 @@ dependencies = [ name = "wasmer-runtime-core" version = "0.5.3" dependencies = [ + "bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "blake2b_simd 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)", + "colored 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "digest 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "errno 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", "field-offset 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1523,6 +1585,7 @@ name = "wasmer-singlepass-backend" version = "0.5.3" dependencies = [ "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "colored 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "dynasm 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "dynasmrt 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "hashbrown 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1635,15 +1698,28 @@ dependencies = [ "winapi-util 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "winconsole" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cgmath 0.16.1 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rgb 0.8.13 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + [metadata] "checksum aho-corasick 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e6f484ae0c99fec2e858eb6134949117399f222608d84cadb3f58c1f97c2364c" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +"checksum approx 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "08abcc3b4e9339e33a3d0a5ed15d84a687350c05689d825e0f6655eef9e76a94" "checksum arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0d382e583f07208808f6b1249e60848879ba3543f57c32277bf52d69c2f0f0ee" "checksum arrayvec 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "92c7fb76bc8826a8b33b4ee5bb07a247a81e76764ab4d55e8f73e3a4d8808c71" "checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" "checksum autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "0e49efa51329a5fd37e7c79db4621af617cd4e3e5bc224939808d076077077bf" "checksum backtrace 0.3.26 (registry+https://github.com/rust-lang/crates.io-index)" = "1a13fc43f04daf08ab4f71e3d27e1fc27fc437d3e95ac0063a796d92fb40f39b" "checksum backtrace-sys 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)" = "797c830ac25ccc92a7f8a7b9862bde440715531514594a6154e3d4a54dd769b6" +"checksum bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "9f04a5e50dc80b3d5d35320889053637d15011aed5e66b66b37ae798c65da6f7" "checksum bindgen 0.46.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8f7f7f0701772b17de73e4f5cbcb1dd6926f4706cba4c1ab62c5367f8bdc94e1" "checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" "checksum blake2b_simd 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ce2571a6cd634670daa2977cc894c1cc2ba57c563c498e5a82c35446f34d056e" @@ -1655,10 +1731,12 @@ dependencies = [ "checksum cc 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)" = "39f75544d7bbaf57560d2168f28fd649ff9c76153874db88bdbdfd839b1a7e7d" "checksum cexpr 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "a7fa24eb00d5ffab90eaeaf1092ac85c04c64aaf358ea6f84505b8116d24c6af" "checksum cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b486ce3ccf7ffd79fdeb678eac06a9e6c09fc88d33836340becb8fffe87c5e33" +"checksum cgmath 0.16.1 (registry+https://github.com/rust-lang/crates.io-index)" = "64a4b57c8f4e3a2e9ac07e0f6abc9c24b6fc9e1b54c3478cfb598f3d0023e51c" "checksum clang-sys 0.26.4 (registry+https://github.com/rust-lang/crates.io-index)" = "6ef0c1bcf2e99c649104bd7a7012d8f8802684400e03db0ec0af48583c6fa0e4" "checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum cmake 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "2ca4386c8954b76a8415b63959337d940d724b336cabd3afe189c2b51a7e1ff0" +"checksum colored 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6cdb90b60f2927f8d76139c72dbde7e10c3a2bc47c8594c9c7a66529f2687c03" "checksum constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8ff012e225ce166d4422e0e78419d901719760f62ae2b7969ca6b564d1b54a9e" "checksum cranelift-bforest 0.31.0 (git+https://github.com/wasmerio/cranelift.git?rev=2ada531d79b34a9e6c94c81f2615677e22d68780)" = "" "checksum cranelift-codegen 0.31.0 (git+https://github.com/wasmerio/cranelift.git?rev=2ada531d79b34a9e6c94c81f2615677e22d68780)" = "" @@ -1715,6 +1793,7 @@ dependencies = [ "checksum nix 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "46f0f3210768d796e8fa79ec70ee6af172dacbe7147f5e69be5240a47778302b" "checksum nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945" "checksum nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" +"checksum num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" "checksum num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "6ba9a427cfca2be13aa6f6403b0b7e7368fe982bfa16fccc450ce74c46cd9b32" "checksum num_cpus 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1a23f0ed30a54abaa0c7e83b1d2d87ada7c3c23078d1d87815af3e3b6385fbba" "checksum numtoa 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b8f8bdf33df195859076e54ab11ee78a1b208382d3a26ec40d142ffc1ecc49ef" @@ -1729,6 +1808,7 @@ dependencies = [ "checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" "checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" "checksum quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "faf4799c5d274f3868a4aae320a0a182cbd2baee377b378f080e16a23e9d80db" +"checksum rand 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" "checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" "checksum rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" "checksum rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" @@ -1749,6 +1829,7 @@ dependencies = [ "checksum regex 1.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "8f0a0bcab2fd7d1d7c54fa9eae6f43eddeb9ce2e7352f8518a814a4f65d60c58" "checksum regex-syntax 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "dcfd8681eebe297b81d98498869d4aae052137651ad7b96822f09ceb690d0a96" "checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" +"checksum rgb 0.8.13 (registry+https://github.com/rust-lang/crates.io-index)" = "4f089652ca87f5a82a62935ec6172a534066c7b97be003cc8f702ee9a7a59c92" "checksum rustc-demangle 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "a7f4dccf6f4891ebcc0c39f9b6eb1a83b9bf5d747cb439ec6fba4f3b977038af" "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" "checksum ryu 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "b96a9549dc8d48f2c283938303c4b5a77aa29bfbc5b54b084fb1630408899a8f" @@ -1804,3 +1885,4 @@ dependencies = [ "checksum winapi-util 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7168bab6e1daee33b4557efd0e95d5ca70a03706d39fa5f3fe7a236f584b03c9" "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" "checksum wincolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "561ed901ae465d6185fa7864d63fbd5720d0ef718366c9a4dc83cf6170d7e9ba" +"checksum winconsole 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ef84b96d10db72dd980056666d7f1e7663ce93d82fa33b63e71c966f4cf5032" diff --git a/examples/iterative_hash/Cargo.lock b/examples/iterative_hash/Cargo.lock new file mode 100644 index 000000000..7c055be14 --- /dev/null +++ b/examples/iterative_hash/Cargo.lock @@ -0,0 +1,74 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "blake2" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "crypto-mac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "opaque-debug 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "crypto-mac" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", + "subtle 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "digest" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "generic-array" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "iterative_hash" +version = "0.1.0" +dependencies = [ + "blake2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "opaque-debug" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "subtle" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "typenum" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[metadata] +"checksum blake2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "91721a6330935673395a0607df4d49a9cb90ae12d259f1b3e0a3f6e1d486872e" +"checksum byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" +"checksum crypto-mac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5" +"checksum digest 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05f47366984d3ad862010e22c7ce81a7dbcaebbdfb37241a620f8b6596ee135c" +"checksum generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +"checksum opaque-debug 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "93f5bb2e8e8dec81642920ccff6b61f1eb94fa3020c5a325c9851ff604152409" +"checksum subtle 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee" +"checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169" diff --git a/examples/iterative_hash/Cargo.toml b/examples/iterative_hash/Cargo.toml new file mode 100644 index 000000000..5cc908684 --- /dev/null +++ b/examples/iterative_hash/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "iterative_hash" +version = "0.1.0" +authors = ["losfair "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[workspace] + +[dependencies] +blake2 = "0.8" diff --git a/examples/iterative_hash/src/main.rs b/examples/iterative_hash/src/main.rs new file mode 100644 index 000000000..f25672047 --- /dev/null +++ b/examples/iterative_hash/src/main.rs @@ -0,0 +1,16 @@ +use blake2::{Blake2b, Digest}; + +fn main() { + let mut data: Vec = b"test".to_vec(); + + for i in 0.. { + let mut hasher = Blake2b::new(); + hasher.input(&data); + let out = hasher.result(); + data = out.to_vec(); + + if i % 1000000 == 0 { + println!("Round {}: {:?}", i, data); + } + } +} diff --git a/examples/trie_traversal/trie_traversal.rs b/examples/trie_traversal/trie_traversal.rs new file mode 100644 index 000000000..368128405 --- /dev/null +++ b/examples/trie_traversal/trie_traversal.rs @@ -0,0 +1,67 @@ +#[link(wasm_import_module = "wasmer_suspend")] +extern "C" { + fn suspend(); +} + +use std::collections::BTreeMap; + +#[derive(Default)] +struct Node { + count: usize, + children: BTreeMap, +} + +impl Node { + fn insert(&mut self, mut s: impl Iterator) { + match s.next() { + Some(x) => { + self.children.entry(x).or_default().insert(s); + } + None => { + self.count += 1; + } + } + } + + fn for_each_dyn(&self, cb: &dyn Fn(&str, usize), prefix: &mut String) { + if self.count > 0 { + cb(&prefix, self.count); + } + + for (k, v) in self.children.iter() { + prefix.push(*k); + v.for_each_dyn(cb, prefix); + prefix.pop().unwrap(); + } + } +} + +fn main() { + let mut root = Node::default(); + root.insert("Ava".chars()); + root.insert("Alexander".chars()); + root.insert("Aiden".chars()); + root.insert("Bella".chars()); + root.insert("Brianna".chars()); + root.insert("Brielle".chars()); + root.insert("Charlotte".chars()); + root.insert("Chloe".chars()); + root.insert("Camila".chars()); + + println!("Tree ready, suspending."); + unsafe { + suspend(); + } + + root.for_each_dyn( + &|seq, count| { + println!("{}: {}", seq, count); + unsafe { + suspend(); + } + }, + &mut "".into(), + ); + + println!("[END]"); +} diff --git a/lib/llvm-backend/src/intrinsics.rs b/lib/llvm-backend/src/intrinsics.rs index d34ccb2e4..c3a95f775 100644 --- a/lib/llvm-backend/src/intrinsics.rs +++ b/lib/llvm-backend/src/intrinsics.rs @@ -164,6 +164,7 @@ impl Intrinsics { let memory_base_ty = i8_ty; let memory_bound_ty = i8_ty; let internals_ty = i64_ty; + let interrupt_signal_mem_ty = i8_ty; let local_function_ty = i8_ptr_ty; let anyfunc_ty = context.struct_type( @@ -222,6 +223,9 @@ impl Intrinsics { internals_ty .ptr_type(AddressSpace::Generic) .as_basic_type_enum(), + interrupt_signal_mem_ty + .ptr_type(AddressSpace::Generic) + .as_basic_type_enum(), local_function_ty .ptr_type(AddressSpace::Generic) .as_basic_type_enum(), diff --git a/lib/middleware-common/src/call_trace.rs b/lib/middleware-common/src/call_trace.rs index 9c47d7d7b..04a763abc 100644 --- a/lib/middleware-common/src/call_trace.rs +++ b/lib/middleware-common/src/call_trace.rs @@ -17,6 +17,7 @@ impl FunctionMiddleware for CallTrace { Event::Internal(InternalEvent::FunctionBegin(id)) => sink.push(Event::Internal( InternalEvent::Breakpoint(Box::new(move |_| { eprintln!("func ({})", id); + Ok(()) })), )), _ => {} diff --git a/lib/middleware-common/src/metering.rs b/lib/middleware-common/src/metering.rs index 12d4ab240..024a1c7cd 100644 --- a/lib/middleware-common/src/metering.rs +++ b/lib/middleware-common/src/metering.rs @@ -94,11 +94,9 @@ impl FunctionMiddleware for Metering { sink.push(Event::WasmOwned(Operator::If { ty: WpTypeOrFuncType::Type(WpType::EmptyBlockType), })); - sink.push(Event::Internal(InternalEvent::Breakpoint(Box::new( - move |ctx| unsafe { - (ctx.throw)(Box::new(ExecutionLimitExceededError)); - }, - )))); + sink.push(Event::Internal(InternalEvent::Breakpoint(Box::new(|_| { + Err(Box::new(ExecutionLimitExceededError)) + })))); sink.push(Event::WasmOwned(Operator::End)); } _ => {} diff --git a/lib/runtime-core/Cargo.toml b/lib/runtime-core/Cargo.toml index e0d12a45d..b8e266e39 100644 --- a/lib/runtime-core/Cargo.toml +++ b/lib/runtime-core/Cargo.toml @@ -18,6 +18,8 @@ errno = "0.2.4" libc = "0.2.49" hex = "0.3.2" smallvec = "0.6.9" +bincode = "1.1" +colored = "1.8" # Dependencies for caching. [dependencies.serde] @@ -47,6 +49,7 @@ field-offset = "0.1.1" [build-dependencies] blake2b_simd = "0.4.1" rustc_version = "0.2.3" +cc = "1.0" [features] debug = [] diff --git a/lib/runtime-core/build.rs b/lib/runtime-core/build.rs index 38071e31c..35aafc835 100644 --- a/lib/runtime-core/build.rs +++ b/lib/runtime-core/build.rs @@ -28,4 +28,16 @@ fn main() { if rustc_version::version_meta().unwrap().channel == rustc_version::Channel::Nightly { println!("cargo:rustc-cfg=nightly"); } + + if cfg!(all(target_os = "linux", target_arch = "x86_64")) { + cc::Build::new() + .file("image-loading-linux-x86-64.s") + .compile("image-loading"); + } else if cfg!(all(target_os = "macos", target_arch = "x86_64")) { + cc::Build::new() + .file("image-loading-macos-x86-64.s") + .compile("image-loading"); + } else { + + } } diff --git a/lib/runtime-core/image-loading-linux-x86-64.s b/lib/runtime-core/image-loading-linux-x86-64.s new file mode 100644 index 000000000..37ed0f986 --- /dev/null +++ b/lib/runtime-core/image-loading-linux-x86-64.s @@ -0,0 +1,69 @@ +# NOTE: Keep this consistent with `fault.rs`. + +.globl run_on_alternative_stack +run_on_alternative_stack: +# (stack_end, stack_begin) +# We need to ensure 16-byte alignment here. +pushq %r15 +pushq %r14 +pushq %r13 +pushq %r12 +pushq %rbx +pushq %rbp +movq %rsp, -16(%rdi) + +leaq run_on_alternative_stack.returning(%rip), %rax +movq %rax, -24(%rdi) + +movq %rsi, %rsp + +movq (%rsp), %xmm0 +add $8, %rsp + +movq (%rsp), %xmm1 +add $8, %rsp + +movq (%rsp), %xmm2 +add $8, %rsp + +movq (%rsp), %xmm3 +add $8, %rsp + +movq (%rsp), %xmm4 +add $8, %rsp + +movq (%rsp), %xmm5 +add $8, %rsp + +movq (%rsp), %xmm6 +add $8, %rsp + +movq (%rsp), %xmm7 +add $8, %rsp + +popq %rbp +popq %rax +popq %rbx +popq %rcx +popq %rdx +popq %rdi +popq %rsi +popq %r8 +popq %r9 +popq %r10 +popq %r11 +popq %r12 +popq %r13 +popq %r14 +popq %r15 +retq + +run_on_alternative_stack.returning: +movq (%rsp), %rsp +popq %rbp +popq %rbx +popq %r12 +popq %r13 +popq %r14 +popq %r15 +retq diff --git a/lib/runtime-core/image-loading-macos-x86-64.s b/lib/runtime-core/image-loading-macos-x86-64.s new file mode 100644 index 000000000..a6a307f1f --- /dev/null +++ b/lib/runtime-core/image-loading-macos-x86-64.s @@ -0,0 +1,69 @@ +# NOTE: Keep this consistent with `fault.rs`. + +.globl _run_on_alternative_stack +_run_on_alternative_stack: +# (stack_end, stack_begin) +# We need to ensure 16-byte alignment here. +pushq %r15 +pushq %r14 +pushq %r13 +pushq %r12 +pushq %rbx +pushq %rbp +movq %rsp, -16(%rdi) + +leaq _run_on_alternative_stack.returning(%rip), %rax +movq %rax, -24(%rdi) + +movq %rsi, %rsp + +movq (%rsp), %xmm0 +add $8, %rsp + +movq (%rsp), %xmm1 +add $8, %rsp + +movq (%rsp), %xmm2 +add $8, %rsp + +movq (%rsp), %xmm3 +add $8, %rsp + +movq (%rsp), %xmm4 +add $8, %rsp + +movq (%rsp), %xmm5 +add $8, %rsp + +movq (%rsp), %xmm6 +add $8, %rsp + +movq (%rsp), %xmm7 +add $8, %rsp + +popq %rbp +popq %rax +popq %rbx +popq %rcx +popq %rdx +popq %rdi +popq %rsi +popq %r8 +popq %r9 +popq %r10 +popq %r11 +popq %r12 +popq %r13 +popq %r14 +popq %r15 +retq + +_run_on_alternative_stack.returning: +movq (%rsp), %rsp +popq %rbp +popq %rbx +popq %r12 +popq %r13 +popq %r14 +popq %r15 +retq diff --git a/lib/runtime-core/src/backend.rs b/lib/runtime-core/src/backend.rs index c03b4a2a2..ed8733bde 100644 --- a/lib/runtime-core/src/backend.rs +++ b/lib/runtime-core/src/backend.rs @@ -1,6 +1,7 @@ use crate::{ error::CompileResult, module::ModuleInner, + state::ModuleStateMap, typed_func::Wasm, types::{LocalFuncIndex, SigIndex}, vm, @@ -8,6 +9,7 @@ use crate::{ use crate::{ cache::{Artifact, Error as CacheError}, + codegen::BreakpointMap, module::ModuleInfo, sys::Memory, }; @@ -84,6 +86,14 @@ pub trait RunnableModule: Send + Sync { local_func_index: LocalFuncIndex, ) -> Option>; + fn get_module_state_map(&self) -> Option { + None + } + + fn get_breakpoints(&self) -> Option { + None + } + /// A wasm trampoline contains the necessary data to dynamically call an exported wasm function. /// Given a particular signature index, we are returned a trampoline that is matched with that /// signature and an invoke function that can call the trampoline. diff --git a/lib/runtime-core/src/codegen.rs b/lib/runtime-core/src/codegen.rs index 4ff0c25cb..25f42612c 100644 --- a/lib/runtime-core/src/codegen.rs +++ b/lib/runtime-core/src/codegen.rs @@ -9,6 +9,7 @@ use crate::{ }; use smallvec::SmallVec; use std::any::Any; +use std::collections::HashMap; use std::fmt; use std::fmt::Debug; use std::marker::PhantomData; @@ -16,6 +17,10 @@ use std::sync::{Arc, RwLock}; use wasmparser::{self, WasmDecoder}; use wasmparser::{Operator, Type as WpType}; +pub type BreakpointHandler = + Box Result<(), Box> + Send + Sync + 'static>; +pub type BreakpointMap = Arc>; + #[derive(Debug)] pub enum Event<'a, 'b> { Internal(InternalEvent), @@ -26,7 +31,7 @@ pub enum Event<'a, 'b> { pub enum InternalEvent { FunctionBegin(u32), FunctionEnd, - Breakpoint(Box), + Breakpoint(BreakpointHandler), SetInternal(u32), GetInternal(u32), } @@ -43,8 +48,8 @@ impl fmt::Debug for InternalEvent { } } -pub struct BkptInfo { - pub throw: unsafe fn(Box) -> !, +pub struct BreakpointInfo<'a> { + pub fault: Option<&'a dyn Any>, } pub trait ModuleCodeGenerator, RM: RunnableModule, E: Debug> { diff --git a/lib/runtime-core/src/fault.rs b/lib/runtime-core/src/fault.rs new file mode 100644 index 000000000..119aeae04 --- /dev/null +++ b/lib/runtime-core/src/fault.rs @@ -0,0 +1,464 @@ +mod raw { + use std::ffi::c_void; + + extern "C" { + pub fn run_on_alternative_stack(stack_end: *mut u64, stack_begin: *mut u64) -> u64; + pub fn setjmp(env: *mut c_void) -> i32; + pub fn longjmp(env: *mut c_void, val: i32) -> !; + } +} + +use crate::codegen::{BreakpointInfo, BreakpointMap}; +use crate::state::x64::{build_instance_image, read_stack, X64Register, GPR, XMM}; +use crate::vm; +use libc::{mmap, mprotect, siginfo_t, MAP_ANON, MAP_PRIVATE, PROT_NONE, PROT_READ, PROT_WRITE}; +use nix::sys::signal::{ + sigaction, SaFlags, SigAction, SigHandler, SigSet, Signal, SIGBUS, SIGFPE, SIGILL, SIGINT, + SIGSEGV, SIGTRAP, +}; +use std::any::Any; +use std::cell::UnsafeCell; +use std::ffi::c_void; +use std::process; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Once; + +pub(crate) unsafe fn run_on_alternative_stack(stack_end: *mut u64, stack_begin: *mut u64) -> u64 { + raw::run_on_alternative_stack(stack_end, stack_begin) +} + +const TRAP_STACK_SIZE: usize = 1048576; // 1MB + +const SETJMP_BUFFER_LEN: usize = 27; +type SetJmpBuffer = [i32; SETJMP_BUFFER_LEN]; + +struct UnwindInfo { + jmpbuf: SetJmpBuffer, // in + breakpoints: Option, + payload: Option>, // out +} + +thread_local! { + static UNWIND: UnsafeCell> = UnsafeCell::new(None); +} + +struct InterruptSignalMem(*mut u8); +unsafe impl Send for InterruptSignalMem {} +unsafe impl Sync for InterruptSignalMem {} + +const INTERRUPT_SIGNAL_MEM_SIZE: usize = 4096; + +lazy_static! { + static ref INTERRUPT_SIGNAL_MEM: InterruptSignalMem = { + let ptr = unsafe { + mmap( + ::std::ptr::null_mut(), + INTERRUPT_SIGNAL_MEM_SIZE, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, + -1, + 0, + ) + }; + if ptr as isize == -1 { + panic!("cannot allocate code memory"); + } + InterruptSignalMem(ptr as _) + }; +} +static INTERRUPT_SIGNAL_DELIVERED: AtomicBool = AtomicBool::new(false); + +pub unsafe fn get_wasm_interrupt_signal_mem() -> *mut u8 { + INTERRUPT_SIGNAL_MEM.0 +} + +pub unsafe fn set_wasm_interrupt() { + let mem: *mut u8 = INTERRUPT_SIGNAL_MEM.0; + if mprotect(mem as _, INTERRUPT_SIGNAL_MEM_SIZE, PROT_NONE) < 0 { + panic!("cannot set PROT_NONE on signal mem"); + } +} + +pub unsafe fn clear_wasm_interrupt() { + let mem: *mut u8 = INTERRUPT_SIGNAL_MEM.0; + if mprotect(mem as _, INTERRUPT_SIGNAL_MEM_SIZE, PROT_READ | PROT_WRITE) < 0 { + panic!("cannot set PROT_READ | PROT_WRITE on signal mem"); + } +} + +pub unsafe fn catch_unsafe_unwind R>( + f: F, + breakpoints: Option, +) -> Result> { + let unwind = UNWIND.with(|x| x.get()); + let old = (*unwind).take(); + *unwind = Some(UnwindInfo { + jmpbuf: [0; SETJMP_BUFFER_LEN], + breakpoints: breakpoints, + payload: None, + }); + + if raw::setjmp(&mut (*unwind).as_mut().unwrap().jmpbuf as *mut SetJmpBuffer as *mut _) != 0 { + // error + let ret = (*unwind).as_mut().unwrap().payload.take().unwrap(); + *unwind = old; + Err(ret) + } else { + let ret = f(); + // implicit control flow to the error case... + *unwind = old; + Ok(ret) + } +} + +pub unsafe fn begin_unsafe_unwind(e: Box) -> ! { + let unwind = UNWIND.with(|x| x.get()); + let inner = (*unwind) + .as_mut() + .expect("not within a catch_unsafe_unwind scope"); + inner.payload = Some(e); + raw::longjmp(&mut inner.jmpbuf as *mut SetJmpBuffer as *mut _, 0xffff); +} + +unsafe fn with_breakpoint_map) -> R>(f: F) -> R { + let unwind = UNWIND.with(|x| x.get()); + let inner = (*unwind) + .as_mut() + .expect("not within a catch_unsafe_unwind scope"); + f(inner.breakpoints.as_ref()) +} + +pub fn allocate_and_run R>(size: usize, f: F) -> R { + struct Context R, R> { + f: Option, + ret: Option, + } + + extern "C" fn invoke R, R>(ctx: &mut Context) { + let f = ctx.f.take().unwrap(); + ctx.ret = Some(f()); + } + + unsafe { + let mut ctx = Context { + f: Some(f), + ret: None, + }; + assert!(size % 16 == 0); + assert!(size >= 4096); + + let mut stack: Vec = vec![0; size / 8]; + let end_offset = stack.len(); + + stack[end_offset - 4] = invoke:: as usize as u64; + + // NOTE: Keep this consistent with `image-loading-*.s`. + stack[end_offset - 4 - 10] = &mut ctx as *mut Context as usize as u64; // rdi + const NUM_SAVED_REGISTERS: usize = 23; + let stack_begin = stack + .as_mut_ptr() + .offset((end_offset - 4 - NUM_SAVED_REGISTERS) as isize); + let stack_end = stack.as_mut_ptr().offset(end_offset as isize); + + raw::run_on_alternative_stack(stack_end, stack_begin); + ctx.ret.take().unwrap() + } +} + +extern "C" fn signal_trap_handler( + signum: ::nix::libc::c_int, + siginfo: *mut siginfo_t, + ucontext: *mut c_void, +) { + unsafe { + let fault = get_fault_info(siginfo as _, ucontext); + + let mut unwind_result: Box = Box::new(()); + + let should_unwind = allocate_and_run(TRAP_STACK_SIZE, || { + let mut is_suspend_signal = false; + + match Signal::from_c_int(signum) { + Ok(SIGTRAP) => { + // breakpoint + let out: Option>> = with_breakpoint_map(|bkpt_map| { + bkpt_map.and_then(|x| x.get(&(fault.ip as usize))).map(|x| { + x(BreakpointInfo { + fault: Some(&fault), + }) + }) + }); + match out { + Some(Ok(())) => { + return false; + } + Some(Err(e)) => { + unwind_result = e; + return true; + } + None => {} + } + } + Ok(SIGSEGV) | Ok(SIGBUS) => { + if fault.faulting_addr as usize == get_wasm_interrupt_signal_mem() as usize { + is_suspend_signal = true; + clear_wasm_interrupt(); + INTERRUPT_SIGNAL_DELIVERED.store(false, Ordering::SeqCst); + } + } + _ => {} + } + + // TODO: make this safer + let ctx = &mut *(fault.known_registers[X64Register::GPR(GPR::R15).to_index().0].unwrap() + as *mut vm::Ctx); + let rsp = fault.known_registers[X64Register::GPR(GPR::RSP).to_index().0].unwrap(); + + let msm = (*ctx.module) + .runnable_module + .get_module_state_map() + .unwrap(); + let code_base = (*ctx.module).runnable_module.get_code().unwrap().as_ptr() as usize; + let es_image = read_stack( + &msm, + code_base, + rsp as usize as *const u64, + fault.known_registers, + Some(fault.ip as usize as u64), + ); + + if is_suspend_signal { + let image = build_instance_image(ctx, es_image); + unwind_result = Box::new(image); + } else { + use colored::*; + if es_image.frames.len() > 0 { + eprintln!( + "\n{}", + "Wasmer encountered an error while running your WebAssembly program." + .bold() + .red() + ); + es_image.print_backtrace_if_needed(); + } + // Just let the error propagate otherrwise + } + + true + }); + + if should_unwind { + begin_unsafe_unwind(unwind_result); + } + } +} + +extern "C" fn sigint_handler( + _signum: ::nix::libc::c_int, + _siginfo: *mut siginfo_t, + _ucontext: *mut c_void, +) { + if INTERRUPT_SIGNAL_DELIVERED.swap(true, Ordering::SeqCst) { + eprintln!("Got another SIGINT before trap is triggered on WebAssembly side, aborting"); + process::abort(); + } + unsafe { + set_wasm_interrupt(); + } +} + +pub fn ensure_sighandler() { + INSTALL_SIGHANDLER.call_once(|| unsafe { + install_sighandler(); + }); +} + +static INSTALL_SIGHANDLER: Once = Once::new(); + +unsafe fn install_sighandler() { + let sa_trap = SigAction::new( + SigHandler::SigAction(signal_trap_handler), + SaFlags::SA_ONSTACK, + SigSet::empty(), + ); + sigaction(SIGFPE, &sa_trap).unwrap(); + sigaction(SIGILL, &sa_trap).unwrap(); + sigaction(SIGSEGV, &sa_trap).unwrap(); + sigaction(SIGBUS, &sa_trap).unwrap(); + sigaction(SIGTRAP, &sa_trap).unwrap(); + + let sa_interrupt = SigAction::new( + SigHandler::SigAction(sigint_handler), + SaFlags::SA_ONSTACK, + SigSet::empty(), + ); + sigaction(SIGINT, &sa_interrupt).unwrap(); +} + +pub struct FaultInfo { + pub faulting_addr: *const c_void, + pub ip: *const c_void, + pub known_registers: [Option; 24], +} + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +pub unsafe fn get_fault_info(siginfo: *const c_void, ucontext: *const c_void) -> FaultInfo { + use libc::{ + _libc_xmmreg, ucontext_t, REG_R10, REG_R11, REG_R12, REG_R13, REG_R14, REG_R15, REG_R8, + REG_R9, REG_RAX, REG_RBP, REG_RBX, REG_RCX, REG_RDI, REG_RDX, REG_RIP, REG_RSI, REG_RSP, + }; + + fn read_xmm(reg: &_libc_xmmreg) -> u64 { + (reg.element[0] as u64) | ((reg.element[1] as u64) << 32) + } + + #[allow(dead_code)] + #[repr(C)] + struct siginfo_t { + si_signo: i32, + si_errno: i32, + si_code: i32, + si_addr: u64, + // ... + } + + let siginfo = siginfo as *const siginfo_t; + let si_addr = (*siginfo).si_addr; + + let ucontext = ucontext as *const ucontext_t; + let gregs = &(*ucontext).uc_mcontext.gregs; + let fpregs = &*(*ucontext).uc_mcontext.fpregs; + + let mut known_registers: [Option; 24] = [None; 24]; + known_registers[X64Register::GPR(GPR::R15).to_index().0] = Some(gregs[REG_R15 as usize] as _); + known_registers[X64Register::GPR(GPR::R14).to_index().0] = Some(gregs[REG_R14 as usize] as _); + known_registers[X64Register::GPR(GPR::R13).to_index().0] = Some(gregs[REG_R13 as usize] as _); + known_registers[X64Register::GPR(GPR::R12).to_index().0] = Some(gregs[REG_R12 as usize] as _); + known_registers[X64Register::GPR(GPR::R11).to_index().0] = Some(gregs[REG_R11 as usize] as _); + known_registers[X64Register::GPR(GPR::R10).to_index().0] = Some(gregs[REG_R10 as usize] as _); + known_registers[X64Register::GPR(GPR::R9).to_index().0] = Some(gregs[REG_R9 as usize] as _); + known_registers[X64Register::GPR(GPR::R8).to_index().0] = Some(gregs[REG_R8 as usize] as _); + known_registers[X64Register::GPR(GPR::RSI).to_index().0] = Some(gregs[REG_RSI as usize] as _); + known_registers[X64Register::GPR(GPR::RDI).to_index().0] = Some(gregs[REG_RDI as usize] as _); + known_registers[X64Register::GPR(GPR::RDX).to_index().0] = Some(gregs[REG_RDX as usize] as _); + known_registers[X64Register::GPR(GPR::RCX).to_index().0] = Some(gregs[REG_RCX as usize] as _); + known_registers[X64Register::GPR(GPR::RBX).to_index().0] = Some(gregs[REG_RBX as usize] as _); + known_registers[X64Register::GPR(GPR::RAX).to_index().0] = Some(gregs[REG_RAX as usize] as _); + + known_registers[X64Register::GPR(GPR::RBP).to_index().0] = Some(gregs[REG_RBP as usize] as _); + known_registers[X64Register::GPR(GPR::RSP).to_index().0] = Some(gregs[REG_RSP as usize] as _); + + known_registers[X64Register::XMM(XMM::XMM0).to_index().0] = Some(read_xmm(&fpregs._xmm[0])); + known_registers[X64Register::XMM(XMM::XMM1).to_index().0] = Some(read_xmm(&fpregs._xmm[1])); + known_registers[X64Register::XMM(XMM::XMM2).to_index().0] = Some(read_xmm(&fpregs._xmm[2])); + known_registers[X64Register::XMM(XMM::XMM3).to_index().0] = Some(read_xmm(&fpregs._xmm[3])); + known_registers[X64Register::XMM(XMM::XMM4).to_index().0] = Some(read_xmm(&fpregs._xmm[4])); + known_registers[X64Register::XMM(XMM::XMM5).to_index().0] = Some(read_xmm(&fpregs._xmm[5])); + known_registers[X64Register::XMM(XMM::XMM6).to_index().0] = Some(read_xmm(&fpregs._xmm[6])); + known_registers[X64Register::XMM(XMM::XMM7).to_index().0] = Some(read_xmm(&fpregs._xmm[7])); + + FaultInfo { + faulting_addr: si_addr as usize as _, + ip: gregs[REG_RIP as usize] as _, + known_registers, + } +} + +#[cfg(all(target_os = "macos", target_arch = "x86_64"))] +pub unsafe fn get_fault_info(siginfo: *const c_void, ucontext: *const c_void) -> FaultInfo { + #[allow(dead_code)] + #[repr(C)] + struct ucontext_t { + uc_onstack: u32, + uc_sigmask: u32, + uc_stack: libc::stack_t, + uc_link: *const ucontext_t, + uc_mcsize: u64, + uc_mcontext: *const mcontext_t, + } + #[repr(C)] + struct exception_state { + trapno: u16, + cpu: u16, + err: u32, + faultvaddr: u64, + } + #[repr(C)] + struct regs { + rax: u64, + rbx: u64, + rcx: u64, + rdx: u64, + rdi: u64, + rsi: u64, + rbp: u64, + rsp: u64, + r8: u64, + r9: u64, + r10: u64, + r11: u64, + r12: u64, + r13: u64, + r14: u64, + r15: u64, + rip: u64, + rflags: u64, + cs: u64, + fs: u64, + gs: u64, + } + #[repr(C)] + struct fpstate { + _unused: [u8; 168], + xmm: [[u64; 2]; 8], + } + #[allow(dead_code)] + #[repr(C)] + struct mcontext_t { + es: exception_state, + ss: regs, + fs: fpstate, + } + + let siginfo = siginfo as *const siginfo_t; + let si_addr = (*siginfo).si_addr; + + let ucontext = ucontext as *const ucontext_t; + let ss = &(*(*ucontext).uc_mcontext).ss; + let fs = &(*(*ucontext).uc_mcontext).fs; + + let mut known_registers: [Option; 24] = [None; 24]; + + known_registers[X64Register::GPR(GPR::R15).to_index().0] = Some(ss.r15); + known_registers[X64Register::GPR(GPR::R14).to_index().0] = Some(ss.r14); + known_registers[X64Register::GPR(GPR::R13).to_index().0] = Some(ss.r13); + known_registers[X64Register::GPR(GPR::R12).to_index().0] = Some(ss.r12); + known_registers[X64Register::GPR(GPR::R11).to_index().0] = Some(ss.r11); + known_registers[X64Register::GPR(GPR::R10).to_index().0] = Some(ss.r10); + known_registers[X64Register::GPR(GPR::R9).to_index().0] = Some(ss.r9); + known_registers[X64Register::GPR(GPR::R8).to_index().0] = Some(ss.r8); + known_registers[X64Register::GPR(GPR::RSI).to_index().0] = Some(ss.rsi); + known_registers[X64Register::GPR(GPR::RDI).to_index().0] = Some(ss.rdi); + known_registers[X64Register::GPR(GPR::RDX).to_index().0] = Some(ss.rdx); + known_registers[X64Register::GPR(GPR::RCX).to_index().0] = Some(ss.rcx); + known_registers[X64Register::GPR(GPR::RBX).to_index().0] = Some(ss.rbx); + known_registers[X64Register::GPR(GPR::RAX).to_index().0] = Some(ss.rax); + + known_registers[X64Register::GPR(GPR::RBP).to_index().0] = Some(ss.rbp); + known_registers[X64Register::GPR(GPR::RSP).to_index().0] = Some(ss.rsp); + + known_registers[X64Register::XMM(XMM::XMM0).to_index().0] = Some(fs.xmm[0][0]); + known_registers[X64Register::XMM(XMM::XMM1).to_index().0] = Some(fs.xmm[1][0]); + known_registers[X64Register::XMM(XMM::XMM2).to_index().0] = Some(fs.xmm[2][0]); + known_registers[X64Register::XMM(XMM::XMM3).to_index().0] = Some(fs.xmm[3][0]); + known_registers[X64Register::XMM(XMM::XMM4).to_index().0] = Some(fs.xmm[4][0]); + known_registers[X64Register::XMM(XMM::XMM5).to_index().0] = Some(fs.xmm[5][0]); + known_registers[X64Register::XMM(XMM::XMM6).to_index().0] = Some(fs.xmm[6][0]); + known_registers[X64Register::XMM(XMM::XMM7).to_index().0] = Some(fs.xmm[7][0]); + + FaultInfo { + faulting_addr: si_addr, + ip: ss.rip as _, + known_registers, + } +} diff --git a/lib/runtime-core/src/import.rs b/lib/runtime-core/src/import.rs index 9522b0829..7a606f6c5 100644 --- a/lib/runtime-core/src/import.rs +++ b/lib/runtime-core/src/import.rs @@ -46,7 +46,7 @@ impl IsExport for Export { /// ``` pub struct ImportObject { map: Rc>>>, - state_creator: Option (*mut c_void, fn(*mut c_void))>>, + pub(crate) state_creator: Option (*mut c_void, fn(*mut c_void))>>, pub allow_missing_functions: bool, } diff --git a/lib/runtime-core/src/lib.rs b/lib/runtime-core/src/lib.rs index d7e56e7df..52b7cf82a 100644 --- a/lib/runtime-core/src/lib.rs +++ b/lib/runtime-core/src/lib.rs @@ -43,6 +43,9 @@ pub mod vm; pub mod vmcalls; #[cfg(all(unix, target_arch = "x86_64"))] pub use trampoline_x64 as trampoline; +#[cfg(all(unix, target_arch = "x86_64"))] +pub mod fault; +pub mod state; use self::error::CompileResult; #[doc(inline)] diff --git a/lib/runtime-core/src/parse.rs b/lib/runtime-core/src/parse.rs index b3bf141ae..93f8d24d4 100644 --- a/lib/runtime-core/src/parse.rs +++ b/lib/runtime-core/src/parse.rs @@ -223,13 +223,6 @@ pub fn read_module< let fcg = mcg .next_function(Arc::clone(&info)) .map_err(|x| LoadError::Codegen(format!("{:?}", x)))?; - middlewares - .run( - Some(fcg), - Event::Internal(InternalEvent::FunctionBegin(id as u32)), - &info.read().unwrap(), - ) - .map_err(|x| LoadError::Codegen(x))?; let info_read = info.read().unwrap(); let sig = info_read @@ -271,6 +264,13 @@ pub fn read_module< body_begun = true; fcg.begin_body(&info.read().unwrap()) .map_err(|x| LoadError::Codegen(format!("{:?}", x)))?; + middlewares + .run( + Some(fcg), + Event::Internal(InternalEvent::FunctionBegin(id as u32)), + &info.read().unwrap(), + ) + .map_err(|x| LoadError::Codegen(x))?; } middlewares .run(Some(fcg), Event::Wasm(op), &info.read().unwrap()) diff --git a/lib/runtime-core/src/state.rs b/lib/runtime-core/src/state.rs new file mode 100644 index 000000000..72046bf6f --- /dev/null +++ b/lib/runtime-core/src/state.rs @@ -0,0 +1,840 @@ +use std::collections::BTreeMap; +use std::ops::Bound::{Included, Unbounded}; + +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] +pub struct RegisterIndex(pub usize); + +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] +pub enum WasmAbstractValue { + Runtime, + Const(u64), +} + +#[derive(Clone, Debug)] +pub struct MachineState { + pub stack_values: Vec, + pub register_values: Vec, + + pub wasm_stack: Vec, + pub wasm_stack_private_depth: usize, + + pub wasm_inst_offset: usize, +} + +#[derive(Clone, Debug, Default)] +pub struct MachineStateDiff { + pub last: Option, + pub stack_push: Vec, + pub stack_pop: usize, + pub reg_diff: Vec<(RegisterIndex, MachineValue)>, + + pub wasm_stack_push: Vec, + pub wasm_stack_pop: usize, + pub wasm_stack_private_depth: usize, // absolute value; not a diff. + + pub wasm_inst_offset: usize, // absolute value; not a diff. +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] +pub enum MachineValue { + Undefined, + Vmctx, + PreserveRegister(RegisterIndex), + CopyStackBPRelative(i32), // relative to Base Pointer, in byte offset + ExplicitShadow, // indicates that all values above this are above the shadow region + WasmStack(usize), + WasmLocal(usize), +} + +#[derive(Clone, Debug)] +pub struct FunctionStateMap { + pub initial: MachineState, + pub local_function_id: usize, + pub locals: Vec, + pub shadow_size: usize, // for single-pass backend, 32 bytes on x86-64 + pub diffs: Vec, + pub wasm_function_header_target_offset: Option, + pub wasm_offset_to_target_offset: BTreeMap, + pub loop_offsets: BTreeMap, /* suspend_offset -> info */ + pub call_offsets: BTreeMap, /* suspend_offset -> info */ + pub trappable_offsets: BTreeMap, /* suspend_offset -> info */ +} + +#[derive(Clone, Copy, Debug)] +pub enum SuspendOffset { + Loop(usize), + Call(usize), + Trappable(usize), +} + +#[derive(Clone, Debug)] +pub struct OffsetInfo { + pub diff_id: usize, + pub activate_offset: usize, +} + +#[derive(Clone, Debug)] +pub struct ModuleStateMap { + pub local_functions: BTreeMap, + pub total_size: usize, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct WasmFunctionStateDump { + pub local_function_id: usize, + pub wasm_inst_offset: usize, + pub stack: Vec>, + pub locals: Vec>, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ExecutionStateImage { + pub frames: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InstanceImage { + pub memory: Option>, + pub globals: Vec, + pub execution_state: ExecutionStateImage, +} + +impl ModuleStateMap { + fn lookup_call_ip(&self, ip: usize, base: usize) -> Option<(&FunctionStateMap, MachineState)> { + if ip < base || ip - base >= self.total_size { + None + } else { + let (_, fsm) = self + .local_functions + .range((Unbounded, Included(&(ip - base)))) + .last() + .unwrap(); + + match fsm.call_offsets.get(&(ip - base)) { + Some(x) => Some((fsm, fsm.diffs[x.diff_id].build_state(fsm))), + None => None, + } + } + } + + fn lookup_trappable_ip( + &self, + ip: usize, + base: usize, + ) -> Option<(&FunctionStateMap, MachineState)> { + if ip < base || ip - base >= self.total_size { + None + } else { + let (_, fsm) = self + .local_functions + .range((Unbounded, Included(&(ip - base)))) + .last() + .unwrap(); + + match fsm.trappable_offsets.get(&(ip - base)) { + Some(x) => Some((fsm, fsm.diffs[x.diff_id].build_state(fsm))), + None => None, + } + } + } + + fn lookup_loop_ip(&self, ip: usize, base: usize) -> Option<(&FunctionStateMap, MachineState)> { + if ip < base || ip - base >= self.total_size { + None + } else { + let (_, fsm) = self + .local_functions + .range((Unbounded, Included(&(ip - base)))) + .last() + .unwrap(); + + match fsm.loop_offsets.get(&(ip - base)) { + Some(x) => Some((fsm, fsm.diffs[x.diff_id].build_state(fsm))), + None => None, + } + } + } +} + +impl FunctionStateMap { + pub fn new( + initial: MachineState, + local_function_id: usize, + shadow_size: usize, + locals: Vec, + ) -> FunctionStateMap { + FunctionStateMap { + initial, + local_function_id, + shadow_size, + locals, + diffs: vec![], + wasm_function_header_target_offset: None, + wasm_offset_to_target_offset: BTreeMap::new(), + loop_offsets: BTreeMap::new(), + call_offsets: BTreeMap::new(), + trappable_offsets: BTreeMap::new(), + } + } +} + +impl MachineState { + pub fn diff(&self, old: &MachineState) -> MachineStateDiff { + let first_diff_stack_depth: usize = self + .stack_values + .iter() + .zip(old.stack_values.iter()) + .enumerate() + .find(|&(_, (&a, &b))| a != b) + .map(|x| x.0) + .unwrap_or(old.stack_values.len().min(self.stack_values.len())); + assert_eq!(self.register_values.len(), old.register_values.len()); + let reg_diff: Vec<_> = self + .register_values + .iter() + .zip(old.register_values.iter()) + .enumerate() + .filter(|&(_, (&a, &b))| a != b) + .map(|(i, (&a, _))| (RegisterIndex(i), a)) + .collect(); + let first_diff_wasm_stack_depth: usize = self + .wasm_stack + .iter() + .zip(old.wasm_stack.iter()) + .enumerate() + .find(|&(_, (&a, &b))| a != b) + .map(|x| x.0) + .unwrap_or(old.wasm_stack.len().min(self.wasm_stack.len())); + MachineStateDiff { + last: None, + stack_push: self.stack_values[first_diff_stack_depth..].to_vec(), + stack_pop: old.stack_values.len() - first_diff_stack_depth, + reg_diff: reg_diff, + + wasm_stack_push: self.wasm_stack[first_diff_wasm_stack_depth..].to_vec(), + wasm_stack_pop: old.wasm_stack.len() - first_diff_wasm_stack_depth, + wasm_stack_private_depth: self.wasm_stack_private_depth, + + wasm_inst_offset: self.wasm_inst_offset, + } + } +} + +impl MachineStateDiff { + pub fn build_state(&self, m: &FunctionStateMap) -> MachineState { + let mut chain: Vec<&MachineStateDiff> = vec![]; + chain.push(self); + let mut current = self.last; + while let Some(x) = current { + let that = &m.diffs[x]; + current = that.last; + chain.push(that); + } + chain.reverse(); + let mut state = m.initial.clone(); + for x in chain { + for _ in 0..x.stack_pop { + state.stack_values.pop().unwrap(); + } + for v in &x.stack_push { + state.stack_values.push(*v); + } + for &(index, v) in &x.reg_diff { + state.register_values[index.0] = v; + } + for _ in 0..x.wasm_stack_pop { + state.wasm_stack.pop().unwrap(); + } + for v in &x.wasm_stack_push { + state.wasm_stack.push(*v); + } + } + state.wasm_stack_private_depth = self.wasm_stack_private_depth; + state.wasm_inst_offset = self.wasm_inst_offset; + state + } +} + +impl ExecutionStateImage { + pub fn print_backtrace_if_needed(&self) { + use std::env; + + if let Ok(x) = env::var("WASMER_BACKTRACE") { + if x == "1" { + eprintln!("{}", self.colored_output()); + return; + } + } + + eprintln!("Run with `WASMER_BACKTRACE=1` environment variable to display a backtrace."); + } + + pub fn colored_output(&self) -> String { + use colored::*; + + fn join_strings(x: impl Iterator, sep: &str) -> String { + let mut ret = String::new(); + let mut first = true; + + for s in x { + if first { + first = false; + } else { + ret += sep; + } + ret += &s; + } + + ret + } + + fn format_optional_u64_sequence(x: &[Option]) -> String { + if x.len() == 0 { + "(empty)".into() + } else { + join_strings( + x.iter().enumerate().map(|(i, x)| { + format!( + "[{}] = {}", + i, + x.map(|x| format!("{}", x)) + .unwrap_or_else(|| "?".to_string()) + .bold() + .cyan() + ) + }), + ", ", + ) + } + } + + let mut ret = String::new(); + + if self.frames.len() == 0 { + ret += &"Unknown fault address, cannot read stack.".yellow(); + ret += "\n"; + } else { + ret += &"Backtrace:".bold(); + ret += "\n"; + for (i, f) in self.frames.iter().enumerate() { + ret += &format!("* Frame {} @ Local function {}", i, f.local_function_id).bold(); + ret += "\n"; + ret += &format!( + " {} {}\n", + "Offset:".bold().yellow(), + format!("{}", f.wasm_inst_offset).bold().cyan(), + ); + ret += &format!( + " {} {}\n", + "Locals:".bold().yellow(), + format_optional_u64_sequence(&f.locals) + ); + ret += &format!( + " {} {}\n\n", + "Stack:".bold().yellow(), + format_optional_u64_sequence(&f.stack) + ); + } + } + + ret + } +} + +impl InstanceImage { + pub fn from_bytes(input: &[u8]) -> Option { + use bincode::deserialize; + match deserialize(input) { + Ok(x) => Some(x), + Err(_) => None, + } + } + + pub fn to_bytes(&self) -> Vec { + use bincode::serialize; + serialize(self).unwrap() + } +} + +#[cfg(all(unix, target_arch = "x86_64"))] +pub mod x64 { + use super::*; + use crate::codegen::BreakpointMap; + use crate::fault::{catch_unsafe_unwind, run_on_alternative_stack}; + use crate::structures::TypedIndex; + use crate::types::LocalGlobalIndex; + use crate::vm::Ctx; + use std::any::Any; + + pub fn new_machine_state() -> MachineState { + MachineState { + stack_values: vec![], + register_values: vec![MachineValue::Undefined; 16 + 8], + wasm_stack: vec![], + wasm_stack_private_depth: 0, + wasm_inst_offset: ::std::usize::MAX, + } + } + + #[warn(unused_variables)] + pub unsafe fn invoke_call_return_on_stack( + msm: &ModuleStateMap, + code_base: usize, + image: InstanceImage, + vmctx: &mut Ctx, + breakpoints: Option, + ) -> Result> { + let mut stack: Vec = vec![0; 1048576 * 8 / 8]; // 8MB stack + let mut stack_offset: usize = stack.len(); + + stack_offset -= 3; // placeholder for call return + + let mut last_stack_offset: u64 = 0; // rbp + + let mut known_registers: [Option; 24] = [None; 24]; + + let local_functions_vec: Vec<&FunctionStateMap> = + msm.local_functions.iter().map(|(_, v)| v).collect(); + + // Bottom to top + for f in image.execution_state.frames.iter().rev() { + let fsm = local_functions_vec[f.local_function_id]; + let suspend_offset = if f.wasm_inst_offset == ::std::usize::MAX { + fsm.wasm_function_header_target_offset + } else { + fsm.wasm_offset_to_target_offset + .get(&f.wasm_inst_offset) + .map(|x| *x) + } + .expect("instruction is not a critical point"); + + let (activate_offset, diff_id) = match suspend_offset { + SuspendOffset::Loop(x) => fsm.loop_offsets.get(&x), + SuspendOffset::Call(x) => fsm.call_offsets.get(&x), + SuspendOffset::Trappable(x) => fsm.trappable_offsets.get(&x), + } + .map(|x| (x.activate_offset, x.diff_id)) + .expect("offset cannot be found in table"); + + let diff = &fsm.diffs[diff_id]; + let state = diff.build_state(fsm); + + stack_offset -= 1; + stack[stack_offset] = stack.as_ptr().offset(last_stack_offset as isize) as usize as u64; // push rbp + last_stack_offset = stack_offset as _; + + let mut got_explicit_shadow = false; + + for v in state.stack_values.iter() { + match *v { + MachineValue::Undefined => stack_offset -= 1, + MachineValue::Vmctx => { + stack_offset -= 1; + stack[stack_offset] = vmctx as *mut Ctx as usize as u64; + } + MachineValue::PreserveRegister(index) => { + stack_offset -= 1; + stack[stack_offset] = known_registers[index.0].unwrap_or(0); + } + MachineValue::CopyStackBPRelative(byte_offset) => { + assert!(byte_offset % 8 == 0); + let target_offset = (byte_offset / 8) as isize; + let v = stack[(last_stack_offset as isize + target_offset) as usize]; + stack_offset -= 1; + stack[stack_offset] = v; + } + MachineValue::ExplicitShadow => { + assert!(fsm.shadow_size % 8 == 0); + stack_offset -= fsm.shadow_size / 8; + got_explicit_shadow = true; + } + MachineValue::WasmStack(x) => { + stack_offset -= 1; + match state.wasm_stack[x] { + WasmAbstractValue::Const(x) => { + stack[stack_offset] = x; + } + WasmAbstractValue::Runtime => { + stack[stack_offset] = f.stack[x].unwrap(); + } + } + } + MachineValue::WasmLocal(x) => { + stack_offset -= 1; + match fsm.locals[x] { + WasmAbstractValue::Const(x) => { + stack[stack_offset] = x; + } + WasmAbstractValue::Runtime => { + stack[stack_offset] = f.locals[x].unwrap(); + } + } + } + } + } + if !got_explicit_shadow { + assert!(fsm.shadow_size % 8 == 0); + stack_offset -= fsm.shadow_size / 8; + } + for (i, v) in state.register_values.iter().enumerate() { + match *v { + MachineValue::Undefined => {} + MachineValue::Vmctx => { + known_registers[i] = Some(vmctx as *mut Ctx as usize as u64); + } + MachineValue::WasmStack(x) => match state.wasm_stack[x] { + WasmAbstractValue::Const(x) => { + known_registers[i] = Some(x); + } + WasmAbstractValue::Runtime => { + known_registers[i] = Some(f.stack[x].unwrap()); + } + }, + MachineValue::WasmLocal(x) => match fsm.locals[x] { + WasmAbstractValue::Const(x) => { + known_registers[i] = Some(x); + } + WasmAbstractValue::Runtime => { + known_registers[i] = Some(f.locals[x].unwrap()); + } + }, + _ => unreachable!(), + } + } + + // no need to check 16-byte alignment here because it's possible that we're not at a call entry. + + stack_offset -= 1; + stack[stack_offset] = (code_base + activate_offset) as u64; // return address + } + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::R15).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::R14).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::R13).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::R12).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::R11).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::R10).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::R9).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::R8).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::RSI).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::RDI).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::RDX).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::RCX).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::RBX).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::RAX).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = stack.as_ptr().offset(last_stack_offset as isize) as usize as u64; // rbp + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM7).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM6).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM5).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM4).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM3).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM2).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM1).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM0).to_index().0].unwrap_or(0); + + if let Some(ref memory) = image.memory { + assert!(vmctx.internal.memory_bound <= memory.len()); + + if vmctx.internal.memory_bound < memory.len() { + let grow: unsafe extern "C" fn(ctx: &mut Ctx, memory_index: usize, delta: usize) = + ::std::mem::transmute((*vmctx.internal.intrinsics).memory_grow); + grow( + vmctx, + 0, + (memory.len() - vmctx.internal.memory_bound) / 65536, + ); + assert_eq!(vmctx.internal.memory_bound, memory.len()); + } + + ::std::slice::from_raw_parts_mut( + vmctx.internal.memory_base, + vmctx.internal.memory_bound, + ) + .copy_from_slice(memory); + } + + let globals_len = (*vmctx.module).info.globals.len(); + for i in 0..globals_len { + (*(*vmctx.local_backing).globals[LocalGlobalIndex::new(i)].vm_local_global()).data = + image.globals[i]; + } + + drop(image); // free up host memory + + catch_unsafe_unwind( + || { + run_on_alternative_stack( + stack.as_mut_ptr().offset(stack.len() as isize), + stack.as_mut_ptr().offset(stack_offset as isize), + ) + }, + breakpoints, + ) + } + + pub fn build_instance_image( + vmctx: &mut Ctx, + execution_state: ExecutionStateImage, + ) -> InstanceImage { + unsafe { + let memory = if vmctx.internal.memory_base.is_null() { + None + } else { + Some( + ::std::slice::from_raw_parts( + vmctx.internal.memory_base, + vmctx.internal.memory_bound, + ) + .to_vec(), + ) + }; + + // FIXME: Imported globals + let globals_len = (*vmctx.module).info.globals.len(); + let globals: Vec = (0..globals_len) + .map(|i| { + (*vmctx.local_backing).globals[LocalGlobalIndex::new(i)] + .get() + .to_u64() + }) + .collect(); + + InstanceImage { + memory: memory, + globals: globals, + execution_state: execution_state, + } + } + } + + #[warn(unused_variables)] + pub unsafe fn read_stack( + msm: &ModuleStateMap, + code_base: usize, + mut stack: *const u64, + initially_known_registers: [Option; 24], + mut initial_address: Option, + ) -> ExecutionStateImage { + let mut known_registers: [Option; 24] = initially_known_registers; + let mut results: Vec = vec![]; + + for _ in 0.. { + let ret_addr = initial_address.take().unwrap_or_else(|| { + let x = *stack; + stack = stack.offset(1); + x + }); + let (fsm, state) = match msm + .lookup_call_ip(ret_addr as usize, code_base) + .or_else(|| msm.lookup_trappable_ip(ret_addr as usize, code_base)) + .or_else(|| msm.lookup_loop_ip(ret_addr as usize, code_base)) + { + Some(x) => x, + _ => return ExecutionStateImage { frames: results }, + }; + + let mut wasm_stack: Vec> = state + .wasm_stack + .iter() + .map(|x| match *x { + WasmAbstractValue::Const(x) => Some(x), + WasmAbstractValue::Runtime => None, + }) + .collect(); + let mut wasm_locals: Vec> = fsm + .locals + .iter() + .map(|x| match *x { + WasmAbstractValue::Const(x) => Some(x), + WasmAbstractValue::Runtime => None, + }) + .collect(); + + // This must be before the next loop because that modifies `known_registers`. + for (i, v) in state.register_values.iter().enumerate() { + match *v { + MachineValue::Undefined => {} + MachineValue::Vmctx => {} + MachineValue::WasmStack(idx) => { + if let Some(v) = known_registers[i] { + wasm_stack[idx] = Some(v); + } else { + eprintln!( + "BUG: Register {} for WebAssembly stack slot {} has unknown value.", + i, idx + ); + } + } + MachineValue::WasmLocal(idx) => { + if let Some(v) = known_registers[i] { + wasm_locals[idx] = Some(v); + } + } + _ => unreachable!(), + } + } + + let mut found_shadow = false; + for v in state.stack_values.iter() { + match *v { + MachineValue::ExplicitShadow => { + found_shadow = true; + break; + } + _ => {} + } + } + if !found_shadow { + stack = stack.offset((fsm.shadow_size / 8) as isize); + } + + for v in state.stack_values.iter().rev() { + match *v { + MachineValue::ExplicitShadow => { + stack = stack.offset((fsm.shadow_size / 8) as isize); + } + MachineValue::Undefined => { + stack = stack.offset(1); + } + MachineValue::Vmctx => { + stack = stack.offset(1); + } + MachineValue::PreserveRegister(idx) => { + known_registers[idx.0] = Some(*stack); + stack = stack.offset(1); + } + MachineValue::CopyStackBPRelative(_) => { + stack = stack.offset(1); + } + MachineValue::WasmStack(idx) => { + wasm_stack[idx] = Some(*stack); + stack = stack.offset(1); + } + MachineValue::WasmLocal(idx) => { + wasm_locals[idx] = Some(*stack); + stack = stack.offset(1); + } + } + } + stack = stack.offset(1); // RBP + + wasm_stack.truncate( + wasm_stack + .len() + .checked_sub(state.wasm_stack_private_depth) + .unwrap(), + ); + + let wfs = WasmFunctionStateDump { + local_function_id: fsm.local_function_id, + wasm_inst_offset: state.wasm_inst_offset, + stack: wasm_stack, + locals: wasm_locals, + }; + results.push(wfs); + } + + unreachable!(); + } + + #[repr(u8)] + #[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] + pub enum GPR { + RAX, + RCX, + RDX, + RBX, + RSP, + RBP, + RSI, + RDI, + R8, + R9, + R10, + R11, + R12, + R13, + R14, + R15, + } + + #[repr(u8)] + #[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] + pub enum XMM { + XMM0, + XMM1, + XMM2, + XMM3, + XMM4, + XMM5, + XMM6, + XMM7, + } + + pub enum X64Register { + GPR(GPR), + XMM(XMM), + } + + impl X64Register { + pub fn to_index(&self) -> RegisterIndex { + match *self { + X64Register::GPR(x) => RegisterIndex(x as usize), + X64Register::XMM(x) => RegisterIndex(x as usize + 16), + } + } + } +} diff --git a/lib/runtime-core/src/trampoline_x64.rs b/lib/runtime-core/src/trampoline_x64.rs index 4defd1fa7..80de3e0d6 100644 --- a/lib/runtime-core/src/trampoline_x64.rs +++ b/lib/runtime-core/src/trampoline_x64.rs @@ -7,6 +7,8 @@ //! Variadic functions are not supported because `rax` is used by the trampoline code. use crate::loader::CodeMemory; +use crate::vm::Ctx; +use std::fmt; use std::{mem, slice}; lazy_static! { @@ -98,6 +100,46 @@ impl TrampolineBufferBuilder { idx } + pub fn add_context_rsp_state_preserving_trampoline( + &mut self, + target: unsafe extern "C" fn(&mut Ctx, *const CallContext, *const u64), + context: *const CallContext, + ) -> usize { + let idx = self.offsets.len(); + self.offsets.push(self.code.len()); + + self.code.extend_from_slice(&[ + 0x53, // push %rbx + 0x41, 0x54, // push %r12 + 0x41, 0x55, // push %r13 + 0x41, 0x56, // push %r14 + 0x41, 0x57, // push %r15 + ]); + self.code.extend_from_slice(&[ + 0x48, 0xbe, // movabsq ?, %rsi + ]); + self.code.extend_from_slice(value_to_bytes(&context)); + self.code.extend_from_slice(&[ + 0x48, 0x89, 0xe2, // mov %rsp, %rdx + ]); + + self.code.extend_from_slice(&[ + 0x48, 0xb8, // movabsq ?, %rax + ]); + self.code.extend_from_slice(value_to_bytes(&target)); + self.code.extend_from_slice(&[ + 0xff, 0xd0, // callq *%rax + ]); + self.code.extend_from_slice(&[ + 0x48, 0x81, 0xc4, // add ?, %rsp + ]); + self.code.extend_from_slice(value_to_bytes(&40i32)); // 5 * 8 + self.code.extend_from_slice(&[ + 0xc3, //retq + ]); + idx + } + /// Adds a callinfo trampoline. /// /// This generates a trampoline function that collects `num_params` parameters into an array @@ -196,6 +238,12 @@ impl TrampolineBuffer { } } +impl fmt::Debug for TrampolineBuffer { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "TrampolineBuffer {{}}") + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/lib/runtime-core/src/typed_func.rs b/lib/runtime-core/src/typed_func.rs index 0a9bc8c2f..76cf452d8 100644 --- a/lib/runtime-core/src/typed_func.rs +++ b/lib/runtime-core/src/typed_func.rs @@ -180,6 +180,10 @@ where _phantom: PhantomData, } } + + pub fn get_vm_func(&self) -> NonNull { + self.f + } } impl<'a, Args, Rets> Func<'a, Args, Rets, Host> @@ -364,30 +368,35 @@ macro_rules! impl_traits { impl< $( $x: WasmExternType, )* Rets: WasmTypeList, Trap: TrapEarly, FN: Fn( &mut Ctx $( ,$x )* ) -> Trap> ExternalFunction<($( $x ),*), Rets> for FN { #[allow(non_snake_case)] fn to_raw(&self) -> NonNull { - assert_eq!(mem::size_of::(), 0, "you cannot use a closure that captures state for `Func`."); + if mem::size_of::() == 0 { + /// This is required for the llvm backend to be able to unwind through this function. + #[cfg_attr(nightly, unwind(allowed))] + extern fn wrap<$( $x: WasmExternType, )* Rets: WasmTypeList, Trap: TrapEarly, FN: Fn( &mut Ctx $( ,$x )* ) -> Trap>( ctx: &mut Ctx $( ,$x: <$x as WasmExternType>::Native )* ) -> Rets::CStruct { + let f: FN = unsafe { mem::transmute_copy(&()) }; - /// This is required for the llvm backend to be able to unwind through this function. - #[cfg_attr(nightly, unwind(allowed))] - extern fn wrap<$( $x: WasmExternType, )* Rets: WasmTypeList, Trap: TrapEarly, FN: Fn( &mut Ctx $( ,$x )* ) -> Trap>( ctx: &mut Ctx $( ,$x: <$x as WasmExternType>::Native )* ) -> Rets::CStruct { - let f: FN = unsafe { mem::transmute_copy(&()) }; + let err = match panic::catch_unwind(panic::AssertUnwindSafe(|| { + f( ctx $( ,WasmExternType::from_native($x) )* ).report() + })) { + Ok(Ok(returns)) => return returns.into_c_struct(), + Ok(Err(err)) => { + let b: Box<_> = err.into(); + b as Box + }, + Err(err) => err, + }; - let err = match panic::catch_unwind(panic::AssertUnwindSafe(|| { - f( ctx $( ,WasmExternType::from_native($x) )* ).report() - })) { - Ok(Ok(returns)) => return returns.into_c_struct(), - Ok(Err(err)) => { - let b: Box<_> = err.into(); - b as Box - }, - Err(err) => err, - }; - - unsafe { - (&*ctx.module).runnable_module.do_early_trap(err) + unsafe { + (&*ctx.module).runnable_module.do_early_trap(err) + } } - } - NonNull::new(wrap::<$( $x, )* Rets, Trap, Self> as *mut vm::Func).unwrap() + NonNull::new(wrap::<$( $x, )* Rets, Trap, Self> as *mut vm::Func).unwrap() + } else { + assert_eq!(mem::size_of::(), mem::size_of::(), "you cannot use a closure that captures state for `Func`."); + NonNull::new(unsafe { + ::std::mem::transmute_copy::<_, *mut vm::Func>(self) + }).unwrap() + } } } diff --git a/lib/runtime-core/src/vm.rs b/lib/runtime-core/src/vm.rs index a84551449..ffc517910 100644 --- a/lib/runtime-core/src/vm.rs +++ b/lib/runtime-core/src/vm.rs @@ -38,8 +38,8 @@ pub struct Ctx { /// These are pointers to things that are known to be owned /// by the owning `Instance`. - local_backing: *mut LocalBacking, - import_backing: *mut ImportBacking, + pub local_backing: *mut LocalBacking, + pub import_backing: *mut ImportBacking, pub module: *const ModuleInner, //// This is intended to be user-supplied, per-instance @@ -100,6 +100,8 @@ pub struct InternalCtx { pub memory_bound: usize, pub internals: *mut [u64; INTERNALS_SIZE], // TODO: Make this dynamic? + + pub interrupt_signal_mem: *mut u8, } static INTERNAL_FIELDS: AtomicUsize = AtomicUsize::new(0); @@ -207,6 +209,17 @@ fn get_intrinsics_for_module(m: &ModuleInfo) -> *const Intrinsics { } } +#[cfg(all(unix, target_arch = "x86_64"))] +fn get_interrupt_signal_mem() -> *mut u8 { + unsafe { crate::fault::get_wasm_interrupt_signal_mem() } +} + +#[cfg(not(all(unix, target_arch = "x86_64")))] +fn get_interrupt_signal_mem() -> *mut u8 { + static mut REGION: u64 = 0; + unsafe { &mut REGION as *mut u64 as *mut u8 } +} + impl Ctx { #[doc(hidden)] pub unsafe fn new( @@ -245,6 +258,8 @@ impl Ctx { memory_bound: mem_bound, internals: &mut local_backing.internals.0, + + interrupt_signal_mem: get_interrupt_signal_mem(), }, local_functions: local_backing.local_functions.as_ptr(), @@ -296,6 +311,8 @@ impl Ctx { memory_bound: mem_bound, internals: &mut local_backing.internals.0, + + interrupt_signal_mem: get_interrupt_signal_mem(), }, local_functions: local_backing.local_functions.as_ptr(), @@ -419,9 +436,13 @@ impl Ctx { 12 * (mem::size_of::() as u8) } - pub fn offset_local_functions() -> u8 { + pub fn offset_interrupt_signal_mem() -> u8 { 13 * (mem::size_of::() as u8) } + + pub fn offset_local_functions() -> u8 { + 14 * (mem::size_of::() as u8) + } } enum InnerFunc {} @@ -640,6 +661,11 @@ mod vm_offset_tests { offset_of!(InternalCtx => internals).get_byte_offset(), ); + assert_eq!( + Ctx::offset_interrupt_signal_mem() as usize, + offset_of!(InternalCtx => interrupt_signal_mem).get_byte_offset(), + ); + assert_eq!( Ctx::offset_local_functions() as usize, offset_of!(Ctx => local_functions).get_byte_offset(), diff --git a/lib/singlepass-backend/Cargo.toml b/lib/singlepass-backend/Cargo.toml index 17f9cce0f..8e30cc644 100644 --- a/lib/singlepass-backend/Cargo.toml +++ b/lib/singlepass-backend/Cargo.toml @@ -18,3 +18,4 @@ nix = "0.13.0" libc = "0.2.49" smallvec = "0.6.9" hashbrown = "0.1" +colored = "1.8" diff --git a/lib/singlepass-backend/src/codegen_x64.rs b/lib/singlepass-backend/src/codegen_x64.rs index 23131b643..dde93b57b 100644 --- a/lib/singlepass-backend/src/codegen_x64.rs +++ b/lib/singlepass-backend/src/codegen_x64.rs @@ -1,4 +1,5 @@ #![allow(clippy::forget_copy)] // Used by dynasm. +#![warn(unused_imports)] use crate::emitter_x64::*; use crate::machine::*; @@ -10,7 +11,7 @@ use smallvec::SmallVec; use std::ptr::NonNull; use std::{ any::Any, - collections::HashMap, + collections::{BTreeMap, HashMap}, sync::{Arc, RwLock}, }; use wasmer_runtime_core::{ @@ -21,6 +22,10 @@ use wasmer_runtime_core::{ codegen::*, memory::MemoryType, module::{ModuleInfo, ModuleInner}, + state::{ + x64::new_machine_state, x64::X64Register, FunctionStateMap, MachineState, MachineValue, + ModuleStateMap, OffsetInfo, SuspendOffset, WasmAbstractValue, + }, structures::{Map, TypedIndex}, typed_func::Wasm, types::{ @@ -130,24 +135,27 @@ pub struct X64ModuleCodeGenerator { config: Option>, } -#[derive(Copy, Clone, Debug, Eq, PartialEq)] -enum LocalOrTemp { - Local, - Temp, -} - pub struct X64FunctionCode { + local_function_id: usize, + signatures: Arc>, function_signatures: Arc>, + fsm: FunctionStateMap, + offset: usize, assembler: Option, function_labels: Option)>>, - breakpoints: Option>>, + breakpoints: Option< + HashMap< + AssemblyOffset, + Box Result<(), Box> + Send + Sync + 'static>, + >, + >, returns: SmallVec<[WpType; 1]>, locals: Vec, num_params: usize, num_locals: usize, - value_stack: Vec<(Location, LocalOrTemp)>, + value_stack: Vec, control_stack: Vec, machine: Machine, unreachable_depth: usize, @@ -170,8 +178,9 @@ pub struct X64ExecutionContext { function_pointers: Vec, function_offsets: Vec, signatures: Arc>, - breakpoints: Arc>>, + breakpoints: BreakpointMap, func_import_count: usize, + msm: ModuleStateMap, } #[derive(Debug)] @@ -181,6 +190,8 @@ pub struct ControlFrame { pub if_else: IfElseState, pub returns: SmallVec<[WpType; 1]>, pub value_stack_depth: usize, + pub state: MachineState, + pub state_diff_id: usize, } #[derive(Debug, Copy, Clone)] @@ -201,6 +212,14 @@ impl RunnableModule for X64ExecutionContext { .and_then(|ptr| NonNull::new(ptr.0 as *mut vm::Func)) } + fn get_module_state_map(&self) -> Option { + Some(self.msm.clone()) + } + + fn get_breakpoints(&self) -> Option { + Some(self.breakpoints.clone()) + } + fn get_trampoline(&self, _: &ModuleInfo, sig_index: SigIndex) -> Option { use std::ffi::c_void; use wasmer_runtime_core::typed_func::WasmTrapInfo; @@ -229,16 +248,17 @@ impl RunnableModule for X64ExecutionContext { num_params_plus_one.unwrap().as_ptr() as usize - 1, ); let args_reverse: SmallVec<[u64; 8]> = args.iter().cloned().rev().collect(); - protect_unix::BKPT_MAP - .with(|x| x.borrow_mut().push(execution_context.breakpoints.clone())); - let ret = match protect_unix::call_protected(|| { - CONSTRUCT_STACK_AND_CALL_WASM( - args_reverse.as_ptr(), - args_reverse.as_ptr().offset(args_reverse.len() as isize), - ctx, - func.as_ptr(), - ) - }) { + let ret = match protect_unix::call_protected( + || { + CONSTRUCT_STACK_AND_CALL_WASM( + args_reverse.as_ptr(), + args_reverse.as_ptr().offset(args_reverse.len() as isize), + ctx, + func.as_ptr(), + ) + }, + Some(execution_context.breakpoints.clone()), + ) { Ok(x) => { if !rets.is_null() { *rets = x; @@ -253,7 +273,6 @@ impl RunnableModule for X64ExecutionContext { false } }; - protect_unix::BKPT_MAP.with(|x| x.borrow_mut().pop().unwrap()); ret } @@ -347,6 +366,7 @@ impl ModuleCodeGenerator begin_label_info.1 = Some(begin_offset); let begin_label = begin_label_info.0; + let machine = Machine::new(); dynasm!( assembler @@ -354,8 +374,12 @@ impl ModuleCodeGenerator //; int 3 ); let code = X64FunctionCode { + local_function_id: self.functions.len(), + signatures: self.signatures.as_ref().unwrap().clone(), function_signatures: self.function_signatures.as_ref().unwrap().clone(), + fsm: FunctionStateMap::new(new_machine_state(), self.functions.len(), 32, vec![]), // only a placeholder; this is initialized later in `begin_body` + offset: begin_offset.0, assembler: Some(assembler), function_labels: Some(function_labels), @@ -366,7 +390,7 @@ impl ModuleCodeGenerator num_locals: 0, value_stack: vec![], control_stack: vec![], - machine: Machine::new(), + machine, unreachable_depth: 0, config: self.config.as_ref().unwrap().clone(), }; @@ -386,6 +410,7 @@ impl ModuleCodeGenerator }); } }; + let total_size = assembler.get_offset().0; let output = assembler.finalize().unwrap(); let function_labels = if let Some(x) = self.functions.last() { @@ -424,6 +449,12 @@ impl ModuleCodeGenerator .collect(), ); + let local_function_maps: BTreeMap = self + .functions + .iter() + .map(|x| (x.offset, x.fsm.clone())) + .collect(); + struct Placeholder; impl CacheGen for Placeholder { fn generate_cache(&self) -> Result<(Box<[u8]>, Memory), CacheError> { @@ -441,6 +472,10 @@ impl ModuleCodeGenerator func_import_count: self.func_import_count, function_pointers: out_labels, function_offsets: out_offsets, + msm: ModuleStateMap { + local_functions: local_function_maps, + total_size, + }, }, Box::new(Placeholder), )) @@ -508,23 +543,48 @@ impl ModuleCodeGenerator } impl X64FunctionCode { + fn mark_trappable( + a: &mut Assembler, + m: &Machine, + fsm: &mut FunctionStateMap, + control_stack: &mut [ControlFrame], + ) { + let state_diff_id = Self::get_state_diff(m, fsm, control_stack); + let offset = a.get_offset().0; + fsm.trappable_offsets.insert( + offset, + OffsetInfo { + activate_offset: offset, + diff_id: state_diff_id, + }, + ); + fsm.wasm_offset_to_target_offset + .insert(m.state.wasm_inst_offset, SuspendOffset::Trappable(offset)); + } + /// Moves `loc` to a valid location for `div`/`idiv`. fn emit_relaxed_xdiv( a: &mut Assembler, - _m: &mut Machine, + m: &mut Machine, op: fn(&mut Assembler, Size, Location), sz: Size, loc: Location, + fsm: &mut FunctionStateMap, + control_stack: &mut [ControlFrame], ) { + m.state.wasm_stack_private_depth += 1; match loc { Location::Imm64(_) | Location::Imm32(_) => { a.emit_mov(sz, loc, Location::GPR(GPR::RCX)); // must not be used during div (rax, rdx) + Self::mark_trappable(a, m, fsm, control_stack); op(a, sz, Location::GPR(GPR::RCX)); } _ => { + Self::mark_trappable(a, m, fsm, control_stack); op(a, sz, loc); } } + m.state.wasm_stack_private_depth -= 1; } /// Moves `src` and `dst` to valid locations for `movzx`/`movsx`. @@ -724,13 +784,17 @@ impl X64FunctionCode { fn emit_binop_i32( a: &mut Assembler, m: &mut Machine, - value_stack: &mut Vec<(Location, LocalOrTemp)>, + value_stack: &mut Vec, f: fn(&mut Assembler, Size, Location, Location), ) { // Using Red Zone here. let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); - let ret = m.acquire_locations(a, &[WpType::I32], false)[0]; + let ret = m.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; if loc_a != ret { let tmp = m.acquire_temp_gpr().unwrap(); @@ -756,20 +820,24 @@ impl X64FunctionCode { Self::emit_relaxed_binop(a, m, f, Size::S32, loc_b, ret); } - value_stack.push((ret, LocalOrTemp::Temp)); + value_stack.push(ret); } /// I64 binary operation with both operands popped from the virtual stack. fn emit_binop_i64( a: &mut Assembler, m: &mut Machine, - value_stack: &mut Vec<(Location, LocalOrTemp)>, + value_stack: &mut Vec, f: fn(&mut Assembler, Size, Location, Location), ) { // Using Red Zone here. let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); - let ret = m.acquire_locations(a, &[WpType::I64], false)[0]; + let ret = m.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; if loc_a != ret { let tmp = m.acquire_temp_gpr().unwrap(); @@ -795,21 +863,25 @@ impl X64FunctionCode { Self::emit_relaxed_binop(a, m, f, Size::S64, loc_b, ret); } - value_stack.push((ret, LocalOrTemp::Temp)); + value_stack.push(ret); } /// I32 comparison with `loc_b` from input. fn emit_cmpop_i32_dynamic_b( a: &mut Assembler, m: &mut Machine, - value_stack: &mut Vec<(Location, LocalOrTemp)>, + value_stack: &mut Vec, c: Condition, loc_b: Location, ) { // Using Red Zone here. let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); - let ret = m.acquire_locations(a, &[WpType::I32], false)[0]; + let ret = m.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; match ret { Location::GPR(x) => { Self::emit_relaxed_binop(a, m, Assembler::emit_cmp, Size::S32, loc_b, loc_a); @@ -826,14 +898,14 @@ impl X64FunctionCode { } _ => unreachable!(), } - value_stack.push((ret, LocalOrTemp::Temp)); + value_stack.push(ret); } /// I32 comparison with both operands popped from the virtual stack. fn emit_cmpop_i32( a: &mut Assembler, m: &mut Machine, - value_stack: &mut Vec<(Location, LocalOrTemp)>, + value_stack: &mut Vec, c: Condition, ) { let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); @@ -844,14 +916,18 @@ impl X64FunctionCode { fn emit_cmpop_i64_dynamic_b( a: &mut Assembler, m: &mut Machine, - value_stack: &mut Vec<(Location, LocalOrTemp)>, + value_stack: &mut Vec, c: Condition, loc_b: Location, ) { // Using Red Zone here. let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); - let ret = m.acquire_locations(a, &[WpType::I32], false)[0]; + let ret = m.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; match ret { Location::GPR(x) => { Self::emit_relaxed_binop(a, m, Assembler::emit_cmp, Size::S64, loc_b, loc_a); @@ -868,14 +944,14 @@ impl X64FunctionCode { } _ => unreachable!(), } - value_stack.push((ret, LocalOrTemp::Temp)); + value_stack.push(ret); } /// I64 comparison with both operands popped from the virtual stack. fn emit_cmpop_i64( a: &mut Assembler, m: &mut Machine, - value_stack: &mut Vec<(Location, LocalOrTemp)>, + value_stack: &mut Vec, c: Condition, ) { let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); @@ -886,11 +962,15 @@ impl X64FunctionCode { fn emit_xcnt_i32( a: &mut Assembler, m: &mut Machine, - value_stack: &mut Vec<(Location, LocalOrTemp)>, + value_stack: &mut Vec, f: fn(&mut Assembler, Size, Location, Location), ) { let loc = get_location_released(a, m, value_stack.pop().unwrap()); - let ret = m.acquire_locations(a, &[WpType::I32], false)[0]; + let ret = m.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; match loc { Location::Imm32(_) => { @@ -918,18 +998,22 @@ impl X64FunctionCode { } _ => unreachable!(), } - value_stack.push((ret, LocalOrTemp::Temp)); + value_stack.push(ret); } /// I64 `lzcnt`/`tzcnt`/`popcnt` with operand popped from the virtual stack. fn emit_xcnt_i64( a: &mut Assembler, m: &mut Machine, - value_stack: &mut Vec<(Location, LocalOrTemp)>, + value_stack: &mut Vec, f: fn(&mut Assembler, Size, Location, Location), ) { let loc = get_location_released(a, m, value_stack.pop().unwrap()); - let ret = m.acquire_locations(a, &[WpType::I64], false)[0]; + let ret = m.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; match loc { Location::Imm64(_) | Location::Imm32(_) => { @@ -957,19 +1041,23 @@ impl X64FunctionCode { } _ => unreachable!(), } - value_stack.push((ret, LocalOrTemp::Temp)); + value_stack.push(ret); } /// I32 shift with both operands popped from the virtual stack. fn emit_shift_i32( a: &mut Assembler, m: &mut Machine, - value_stack: &mut Vec<(Location, LocalOrTemp)>, + value_stack: &mut Vec, f: fn(&mut Assembler, Size, Location, Location), ) { let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); - let ret = m.acquire_locations(a, &[WpType::I32], false)[0]; + let ret = m.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; a.emit_mov(Size::S32, loc_b, Location::GPR(GPR::RCX)); @@ -978,19 +1066,23 @@ impl X64FunctionCode { } f(a, Size::S32, Location::GPR(GPR::RCX), ret); - value_stack.push((ret, LocalOrTemp::Temp)); + value_stack.push(ret); } /// I64 shift with both operands popped from the virtual stack. fn emit_shift_i64( a: &mut Assembler, m: &mut Machine, - value_stack: &mut Vec<(Location, LocalOrTemp)>, + value_stack: &mut Vec, f: fn(&mut Assembler, Size, Location, Location), ) { let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); - let ret = m.acquire_locations(a, &[WpType::I64], false)[0]; + let ret = m.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; a.emit_mov(Size::S64, loc_b, Location::GPR(GPR::RCX)); @@ -999,20 +1091,24 @@ impl X64FunctionCode { } f(a, Size::S64, Location::GPR(GPR::RCX), ret); - value_stack.push((ret, LocalOrTemp::Temp)); + value_stack.push(ret); } /// Floating point (AVX) binary operation with both operands popped from the virtual stack. fn emit_fp_binop_avx( a: &mut Assembler, m: &mut Machine, - value_stack: &mut Vec<(Location, LocalOrTemp)>, + value_stack: &mut Vec, f: fn(&mut Assembler, XMM, XMMOrMemory, XMM), ) { let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); - let ret = m.acquire_locations(a, &[WpType::F64], false)[0]; - value_stack.push((ret, LocalOrTemp::Temp)); + let ret = m.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + value_stack.push(ret); Self::emit_relaxed_avx(a, m, f, loc_a, loc_b, ret); } @@ -1021,13 +1117,17 @@ impl X64FunctionCode { fn emit_fp_cmpop_avx( a: &mut Assembler, m: &mut Machine, - value_stack: &mut Vec<(Location, LocalOrTemp)>, + value_stack: &mut Vec, f: fn(&mut Assembler, XMM, XMMOrMemory, XMM), ) { let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); - let ret = m.acquire_locations(a, &[WpType::I32], false)[0]; - value_stack.push((ret, LocalOrTemp::Temp)); + let ret = m.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + value_stack.push(ret); Self::emit_relaxed_avx(a, m, f, loc_a, loc_b, ret); a.emit_and(Size::S32, Location::Imm32(1), ret); // FIXME: Why? @@ -1037,12 +1137,16 @@ impl X64FunctionCode { fn emit_fp_unop_avx( a: &mut Assembler, m: &mut Machine, - value_stack: &mut Vec<(Location, LocalOrTemp)>, + value_stack: &mut Vec, f: fn(&mut Assembler, XMM, XMMOrMemory, XMM), ) { let loc = get_location_released(a, m, value_stack.pop().unwrap()); - let ret = m.acquire_locations(a, &[WpType::F64], false)[0]; - value_stack.push((ret, LocalOrTemp::Temp)); + let ret = m.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; + value_stack.push(ret); Self::emit_relaxed_avx(a, m, f, loc, loc, ret); } @@ -1055,13 +1159,20 @@ impl X64FunctionCode { m: &mut Machine, cb: F, params: I, + state_context: Option<(&mut FunctionStateMap, &mut [ControlFrame])>, ) { + // Values pushed in this function are above the shadow region. + m.state.stack_values.push(MachineValue::ExplicitShadow); + let params: Vec<_> = params.collect(); // Save used GPRs. let used_gprs = m.get_used_gprs(); for r in used_gprs.iter() { a.emit_push(Size::S64, Location::GPR(*r)); + let content = m.state.register_values[X64Register::GPR(*r).to_index().0]; + assert!(content != MachineValue::Undefined); + m.state.stack_values.push(content); } // Save used XMM registers. @@ -1083,6 +1194,11 @@ impl X64FunctionCode { Location::Memory(GPR::RCX, (i * 8) as i32), ); } + for r in used_xmms.iter().rev() { + let content = m.state.register_values[X64Register::XMM(*r).to_index().0]; + assert!(content != MachineValue::Undefined); + m.state.stack_values.push(content); + } } let mut stack_offset: usize = 0; @@ -1099,9 +1215,12 @@ impl X64FunctionCode { } // Align stack to 16 bytes. - if (m.get_stack_offset() + used_gprs.len() * 8 + stack_offset) % 16 != 0 { + if (m.get_stack_offset() + used_gprs.len() * 8 + used_xmms.len() * 8 + stack_offset) % 16 + != 0 + { a.emit_sub(Size::S64, Location::Imm32(8), Location::GPR(GPR::RSP)); stack_offset += 8; + m.state.stack_values.push(MachineValue::Undefined); } let mut call_movs: Vec<(Location, GPR)> = vec![]; @@ -1114,6 +1233,32 @@ impl X64FunctionCode { call_movs.push((*param, x)); } Location::Memory(_, _) => { + match *param { + Location::GPR(x) => { + let content = m.state.register_values[X64Register::GPR(x).to_index().0]; + // FIXME: There might be some corner cases (release -> emit_call_sysv -> acquire?) that cause this assertion to fail. + // Hopefully nothing would be incorrect at runtime. + + //assert!(content != MachineValue::Undefined); + m.state.stack_values.push(content); + } + Location::XMM(x) => { + let content = m.state.register_values[X64Register::XMM(x).to_index().0]; + //assert!(content != MachineValue::Undefined); + m.state.stack_values.push(content); + } + Location::Memory(reg, offset) => { + if reg != GPR::RBP { + unreachable!(); + } + m.state + .stack_values + .push(MachineValue::CopyStackBPRelative(offset)); // TODO: Read value at this offset + } + _ => { + m.state.stack_values.push(MachineValue::Undefined); + } + } match *param { // Dynasm bug: RSP in memory operand does not work Location::Imm64(_) | Location::XMM(_) => { @@ -1170,8 +1315,25 @@ impl X64FunctionCode { Machine::get_param_location(0), ); // vmctx + assert!(m.state.stack_values.len() % 2 == 1); // explicit shadow takes one slot + cb(a); + // Offset needs to be after the 'call' instruction. + if let Some((fsm, control_stack)) = state_context { + let state_diff_id = Self::get_state_diff(m, fsm, control_stack); + let offset = a.get_offset().0; + fsm.call_offsets.insert( + offset, + OffsetInfo { + activate_offset: offset, + diff_id: state_diff_id, + }, + ); + fsm.wasm_offset_to_target_offset + .insert(m.state.wasm_inst_offset, SuspendOffset::Call(offset)); + } + // Restore stack. if stack_offset > 0 { a.emit_add( @@ -1179,6 +1341,10 @@ impl X64FunctionCode { Location::Imm32(stack_offset as u32), Location::GPR(GPR::RSP), ); + assert!(stack_offset % 8 == 0); + for _ in 0..stack_offset / 8 { + m.state.stack_values.pop().unwrap(); + } } // Restore XMMs. @@ -1197,12 +1363,21 @@ impl X64FunctionCode { Location::Imm32((used_xmms.len() * 8) as u32), Location::GPR(GPR::RSP), ); + for _ in 0..used_xmms.len() { + m.state.stack_values.pop().unwrap(); + } } // Restore GPRs. for r in used_gprs.iter().rev() { a.emit_pop(Size::S64, Location::GPR(*r)); + m.state.stack_values.pop().unwrap(); } + + assert_eq!( + m.state.stack_values.pop().unwrap(), + MachineValue::ExplicitShadow + ); } /// Emits a System V call sequence, specialized for labels as the call target. @@ -1211,8 +1386,9 @@ impl X64FunctionCode { m: &mut Machine, label: DynamicLabel, params: I, + state_context: Option<(&mut FunctionStateMap, &mut [ControlFrame])>, ) { - Self::emit_call_sysv(a, m, |a| a.emit_call_label(label), params) + Self::emit_call_sysv(a, m, |a| a.emit_call_label(label), params, state_context) } /// Emits a memory operation. @@ -1406,6 +1582,21 @@ impl X64FunctionCode { m.release_temp_xmm(tmp_x); m.release_temp_gpr(tmp); } + + pub fn get_state_diff( + m: &Machine, + fsm: &mut FunctionStateMap, + control_stack: &mut [ControlFrame], + ) -> usize { + let last_frame = control_stack.last_mut().unwrap(); + let mut diff = m.state.diff(&last_frame.state); + diff.last = Some(last_frame.state_diff_id); + let id = fsm.diffs.len(); + last_frame.state = m.state.clone(); + last_frame.state_diff_id = id; + fsm.diffs.push(diff); + id + } } impl FunctionCodeGenerator for X64FunctionCode { @@ -1447,6 +1638,24 @@ impl FunctionCodeGenerator for X64FunctionCode { .machine .init_locals(a, self.num_locals, self.num_params); + self.machine.state.register_values + [X64Register::GPR(Machine::get_vmctx_reg()).to_index().0] = MachineValue::Vmctx; + + self.fsm = FunctionStateMap::new( + new_machine_state(), + self.local_function_id, + 32, + (0..self.locals.len()) + .map(|_| WasmAbstractValue::Runtime) + .collect(), + ); + + let diff = self.machine.state.diff(&new_machine_state()); + let state_diff_id = self.fsm.diffs.len(); + self.fsm.diffs.push(diff); + + //println!("initial state = {:?}", self.machine.state); + a.emit_sub(Size::S64, Location::Imm32(32), Location::GPR(GPR::RSP)); // simulate "red zone" if not supported by the platform self.control_stack.push(ControlFrame { @@ -1455,7 +1664,37 @@ impl FunctionCodeGenerator for X64FunctionCode { if_else: IfElseState::None, returns: self.returns.clone(), value_stack_depth: 0, + state: self.machine.state.clone(), + state_diff_id, }); + + // Check interrupt signal without branching + let activate_offset = a.get_offset().0; + + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_interrupt_signal_mem() as i32, + ), + Location::GPR(GPR::RAX), + ); + self.fsm.loop_offsets.insert( + a.get_offset().0, + OffsetInfo { + activate_offset, + diff_id: state_diff_id, + }, + ); + self.fsm.wasm_function_header_target_offset = Some(SuspendOffset::Loop(a.get_offset().0)); + a.emit_mov( + Size::S64, + Location::Memory(GPR::RAX, 0), + Location::GPR(GPR::RAX), + ); + + assert_eq!(self.machine.state.wasm_inst_offset, ::std::usize::MAX); + Ok(()) } @@ -1466,6 +1705,18 @@ impl FunctionCodeGenerator for X64FunctionCode { } fn feed_event(&mut self, ev: Event, module_info: &ModuleInfo) -> Result<(), CodegenError> { + let a = self.assembler.as_mut().unwrap(); + + match ev { + Event::Internal(InternalEvent::FunctionBegin(_)) + | Event::Internal(InternalEvent::FunctionEnd) => { + return Ok(()); + } + _ => {} + } + + self.machine.state.wasm_inst_offset = self.machine.state.wasm_inst_offset.wrapping_add(1); + //println!("{:?} {}", op, self.value_stack.len()); let was_unreachable; @@ -1500,8 +1751,6 @@ impl FunctionCodeGenerator for X64FunctionCode { was_unreachable = false; } - let a = self.assembler.as_mut().unwrap(); - let op = match ev { Event::Wasm(x) => x, Event::WasmOwned(ref x) => x, @@ -1533,10 +1782,10 @@ impl FunctionCodeGenerator for X64FunctionCode { let loc = self.machine.acquire_locations( a, - &[WpType::I64], + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false, )[0]; - self.value_stack.push((loc, LocalOrTemp::Temp)); + self.value_stack.push(loc); // Move internal into the result location. Self::emit_relaxed_binop( @@ -1583,6 +1832,7 @@ impl FunctionCodeGenerator for X64FunctionCode { return Ok(()); } }; + match *op { Operator::GetGlobal { global_index } => { let global_index = global_index as usize; @@ -1606,7 +1856,10 @@ impl FunctionCodeGenerator for X64FunctionCode { ); self.machine.acquire_locations( a, - &[type_to_wp_type(module_info.globals[local_index].desc.ty)], + &[( + type_to_wp_type(module_info.globals[local_index].desc.ty), + MachineValue::WasmStack(self.value_stack.len()), + )], false, )[0] } @@ -1626,14 +1879,15 @@ impl FunctionCodeGenerator for X64FunctionCode { ); self.machine.acquire_locations( a, - &[type_to_wp_type( - module_info.imported_globals[import_index].1.ty, + &[( + type_to_wp_type(module_info.imported_globals[import_index].1.ty), + MachineValue::WasmStack(self.value_stack.len()), )], false, )[0] } }; - self.value_stack.push((loc, LocalOrTemp::Temp)); + self.value_stack.push(loc); Self::emit_relaxed_binop( a, @@ -1692,7 +1946,11 @@ impl FunctionCodeGenerator for X64FunctionCode { } Operator::GetLocal { local_index } => { let local_index = local_index as usize; - let ret = self.machine.acquire_locations(a, &[WpType::I64], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; Self::emit_relaxed_binop( a, &mut self.machine, @@ -1701,7 +1959,7 @@ impl FunctionCodeGenerator for X64FunctionCode { self.locals[local_index], ret, ); - self.value_stack.push((ret, LocalOrTemp::Temp)); + self.value_stack.push(ret); } Operator::SetLocal { local_index } => { let local_index = local_index as usize; @@ -1719,7 +1977,7 @@ impl FunctionCodeGenerator for X64FunctionCode { } Operator::TeeLocal { local_index } => { let local_index = local_index as usize; - let (loc, _) = *self.value_stack.last().unwrap(); + let loc = *self.value_stack.last().unwrap(); Self::emit_relaxed_binop( a, @@ -1730,9 +1988,13 @@ impl FunctionCodeGenerator for X64FunctionCode { self.locals[local_index], ); } - Operator::I32Const { value } => self - .value_stack - .push((Location::Imm32(value as u32), LocalOrTemp::Temp)), + Operator::I32Const { value } => { + self.value_stack.push(Location::Imm32(value as u32)); + self.machine + .state + .wasm_stack + .push(WasmAbstractValue::Const(value as u32 as u64)); + } Operator::I32Add => Self::emit_binop_i32( a, &mut self.machine, @@ -1757,7 +2019,11 @@ impl FunctionCodeGenerator for X64FunctionCode { get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let loc_a = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I32], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; a.emit_mov(Size::S32, loc_a, Location::GPR(GPR::RAX)); a.emit_xor(Size::S32, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX)); Self::emit_relaxed_xdiv( @@ -1766,9 +2032,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Assembler::emit_div, Size::S32, loc_b, + &mut self.fsm, + &mut self.control_stack, ); a.emit_mov(Size::S32, Location::GPR(GPR::RAX), ret); - self.value_stack.push((ret, LocalOrTemp::Temp)); + self.value_stack.push(ret); } Operator::I32DivS => { // We assume that RAX and RDX are temporary registers here. @@ -1776,7 +2044,11 @@ impl FunctionCodeGenerator for X64FunctionCode { get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let loc_a = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I32], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; a.emit_mov(Size::S32, loc_a, Location::GPR(GPR::RAX)); a.emit_cdq(); Self::emit_relaxed_xdiv( @@ -1785,9 +2057,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Assembler::emit_idiv, Size::S32, loc_b, + &mut self.fsm, + &mut self.control_stack, ); a.emit_mov(Size::S32, Location::GPR(GPR::RAX), ret); - self.value_stack.push((ret, LocalOrTemp::Temp)); + self.value_stack.push(ret); } Operator::I32RemU => { // We assume that RAX and RDX are temporary registers here. @@ -1795,7 +2069,11 @@ impl FunctionCodeGenerator for X64FunctionCode { get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let loc_a = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I32], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; a.emit_mov(Size::S32, loc_a, Location::GPR(GPR::RAX)); a.emit_xor(Size::S32, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX)); Self::emit_relaxed_xdiv( @@ -1804,9 +2082,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Assembler::emit_div, Size::S32, loc_b, + &mut self.fsm, + &mut self.control_stack, ); a.emit_mov(Size::S32, Location::GPR(GPR::RDX), ret); - self.value_stack.push((ret, LocalOrTemp::Temp)); + self.value_stack.push(ret); } Operator::I32RemS => { // We assume that RAX and RDX are temporary registers here. @@ -1814,7 +2094,11 @@ impl FunctionCodeGenerator for X64FunctionCode { get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let loc_a = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I32], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; let normal_path = a.get_label(); let end = a.get_label(); @@ -1849,9 +2133,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Assembler::emit_idiv, Size::S32, loc_b, + &mut self.fsm, + &mut self.control_stack, ); a.emit_mov(Size::S32, Location::GPR(GPR::RDX), ret); - self.value_stack.push((ret, LocalOrTemp::Temp)); + self.value_stack.push(ret); a.emit_label(end); } @@ -1987,8 +2273,11 @@ impl FunctionCodeGenerator for X64FunctionCode { ), Operator::I64Const { value } => { let value = value as u64; - self.value_stack - .push((Location::Imm64(value), LocalOrTemp::Temp)); + self.value_stack.push(Location::Imm64(value)); + self.machine + .state + .wasm_stack + .push(WasmAbstractValue::Const(value)); } Operator::I64Add => Self::emit_binop_i64( a, @@ -2014,7 +2303,11 @@ impl FunctionCodeGenerator for X64FunctionCode { get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let loc_a = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I64], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; a.emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX)); a.emit_xor(Size::S64, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX)); Self::emit_relaxed_xdiv( @@ -2023,9 +2316,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Assembler::emit_div, Size::S64, loc_b, + &mut self.fsm, + &mut self.control_stack, ); a.emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); - self.value_stack.push((ret, LocalOrTemp::Temp)); + self.value_stack.push(ret); } Operator::I64DivS => { // We assume that RAX and RDX are temporary registers here. @@ -2033,7 +2328,11 @@ impl FunctionCodeGenerator for X64FunctionCode { get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let loc_a = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I64], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; a.emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX)); a.emit_cqo(); Self::emit_relaxed_xdiv( @@ -2042,9 +2341,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Assembler::emit_idiv, Size::S64, loc_b, + &mut self.fsm, + &mut self.control_stack, ); a.emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); - self.value_stack.push((ret, LocalOrTemp::Temp)); + self.value_stack.push(ret); } Operator::I64RemU => { // We assume that RAX and RDX are temporary registers here. @@ -2052,7 +2353,11 @@ impl FunctionCodeGenerator for X64FunctionCode { get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let loc_a = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I64], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; a.emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX)); a.emit_xor(Size::S64, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX)); Self::emit_relaxed_xdiv( @@ -2061,9 +2366,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Assembler::emit_div, Size::S64, loc_b, + &mut self.fsm, + &mut self.control_stack, ); a.emit_mov(Size::S64, Location::GPR(GPR::RDX), ret); - self.value_stack.push((ret, LocalOrTemp::Temp)); + self.value_stack.push(ret); } Operator::I64RemS => { // We assume that RAX and RDX are temporary registers here. @@ -2071,7 +2378,11 @@ impl FunctionCodeGenerator for X64FunctionCode { get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let loc_a = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I64], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; let normal_path = a.get_label(); let end = a.get_label(); @@ -2107,9 +2418,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Assembler::emit_idiv, Size::S64, loc_b, + &mut self.fsm, + &mut self.control_stack, ); a.emit_mov(Size::S64, Location::GPR(GPR::RDX), ret); - self.value_stack.push((ret, LocalOrTemp::Temp)); + self.value_stack.push(ret); a.emit_label(end); } Operator::I64And => Self::emit_binop_i64( @@ -2245,8 +2558,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64ExtendUI32 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); Self::emit_relaxed_binop( a, &mut self.machine, @@ -2259,8 +2576,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64ExtendSI32 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); Self::emit_relaxed_zx_sx( a, &mut self.machine, @@ -2274,8 +2595,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I32WrapI64 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I32], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); Self::emit_relaxed_binop( a, &mut self.machine, @@ -2286,9 +2611,13 @@ impl FunctionCodeGenerator for X64FunctionCode { ); } - Operator::F32Const { value } => self - .value_stack - .push((Location::Imm32(value.bits()), LocalOrTemp::Temp)), + Operator::F32Const { value } => { + self.value_stack.push(Location::Imm32(value.bits())); + self.machine + .state + .wasm_stack + .push(WasmAbstractValue::Const(value.bits() as u64)); + } Operator::F32Add => Self::emit_fp_binop_avx( a, &mut self.machine, @@ -2397,8 +2726,12 @@ impl FunctionCodeGenerator for X64FunctionCode { get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let loc_a = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::F32], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); let tmp1 = self.machine.acquire_temp_gpr().unwrap(); let tmp2 = self.machine.acquire_temp_gpr().unwrap(); @@ -2423,8 +2756,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F32Abs => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::F32], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); let tmp = self.machine.acquire_temp_gpr().unwrap(); a.emit_mov(Size::S32, loc, Location::GPR(tmp)); a.emit_and( @@ -2439,8 +2776,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F32Neg => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::F32], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); let tmp = self.machine.acquire_temp_gpr().unwrap(); a.emit_mov(Size::S32, loc, Location::GPR(tmp)); a.emit_btc_gpr_imm8_32(31, tmp); @@ -2448,9 +2789,13 @@ impl FunctionCodeGenerator for X64FunctionCode { self.machine.release_temp_gpr(tmp); } - Operator::F64Const { value } => self - .value_stack - .push((Location::Imm64(value.bits()), LocalOrTemp::Temp)), + Operator::F64Const { value } => { + self.value_stack.push(Location::Imm64(value.bits())); + self.machine + .state + .wasm_stack + .push(WasmAbstractValue::Const(value.bits())); + } Operator::F64Add => Self::emit_fp_binop_avx( a, &mut self.machine, @@ -2559,8 +2904,12 @@ impl FunctionCodeGenerator for X64FunctionCode { get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let loc_a = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::F64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); let tmp1 = self.machine.acquire_temp_gpr().unwrap(); let tmp2 = self.machine.acquire_temp_gpr().unwrap(); @@ -2594,8 +2943,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F64Abs => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::F64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); let tmp = self.machine.acquire_temp_gpr().unwrap(); let c = self.machine.acquire_temp_gpr().unwrap(); @@ -2616,8 +2969,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F64Neg => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::F64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); let tmp = self.machine.acquire_temp_gpr().unwrap(); a.emit_mov(Size::S64, loc, Location::GPR(tmp)); a.emit_btc_gpr_imm8_64(63, tmp); @@ -2641,8 +2998,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I32ReinterpretF32 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I32], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); if loc != ret { Self::emit_relaxed_binop( @@ -2658,8 +3019,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F32ReinterpretI32 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::F32], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); if loc != ret { Self::emit_relaxed_binop( @@ -2676,8 +3041,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64ReinterpretF64 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); if loc != ret { Self::emit_relaxed_binop( @@ -2693,8 +3062,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F64ReinterpretI64 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::F64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); if loc != ret { Self::emit_relaxed_binop( @@ -2711,8 +3084,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I32TruncUF32 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I32], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); let tmp_out = self.machine.acquire_temp_gpr().unwrap(); let tmp_in = self.machine.acquire_temp_xmm().unwrap(); @@ -2736,8 +3113,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I32TruncSF32 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I32], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); let tmp_out = self.machine.acquire_temp_gpr().unwrap(); let tmp_in = self.machine.acquire_temp_xmm().unwrap(); @@ -2767,8 +3148,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64TruncSF32 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); let tmp_out = self.machine.acquire_temp_gpr().unwrap(); let tmp_in = self.machine.acquire_temp_xmm().unwrap(); @@ -2812,8 +3197,12 @@ impl FunctionCodeGenerator for X64FunctionCode { */ let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); let tmp_out = self.machine.acquire_temp_gpr().unwrap(); let tmp_in = self.machine.acquire_temp_xmm().unwrap(); // xmm2 @@ -2867,8 +3256,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I32TruncUF64 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I32], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); let tmp_out = self.machine.acquire_temp_gpr().unwrap(); let tmp_in = self.machine.acquire_temp_xmm().unwrap(); @@ -2892,8 +3285,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I32TruncSF64 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I32], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); let tmp_out = self.machine.acquire_temp_gpr().unwrap(); let tmp_in = self.machine.acquire_temp_xmm().unwrap(); @@ -2928,8 +3325,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64TruncSF64 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); let tmp_out = self.machine.acquire_temp_gpr().unwrap(); let tmp_in = self.machine.acquire_temp_xmm().unwrap(); @@ -2959,8 +3360,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64TruncUF64 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); let tmp_out = self.machine.acquire_temp_gpr().unwrap(); let tmp_in = self.machine.acquire_temp_xmm().unwrap(); // xmm2 @@ -3014,8 +3419,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F32ConvertSI32 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::F32], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); let tmp_out = self.machine.acquire_temp_xmm().unwrap(); let tmp_in = self.machine.acquire_temp_gpr().unwrap(); @@ -3029,8 +3438,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F32ConvertUI32 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::F32], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); let tmp_out = self.machine.acquire_temp_xmm().unwrap(); let tmp_in = self.machine.acquire_temp_gpr().unwrap(); @@ -3044,8 +3457,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F32ConvertSI64 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::F32], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); let tmp_out = self.machine.acquire_temp_xmm().unwrap(); let tmp_in = self.machine.acquire_temp_gpr().unwrap(); @@ -3059,8 +3476,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F32ConvertUI64 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::F32], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); let tmp_out = self.machine.acquire_temp_xmm().unwrap(); let tmp_in = self.machine.acquire_temp_gpr().unwrap(); let tmp = self.machine.acquire_temp_gpr().unwrap(); @@ -3091,8 +3512,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F64ConvertSI32 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::F64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); let tmp_out = self.machine.acquire_temp_xmm().unwrap(); let tmp_in = self.machine.acquire_temp_gpr().unwrap(); @@ -3106,8 +3531,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F64ConvertUI32 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::F64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); let tmp_out = self.machine.acquire_temp_xmm().unwrap(); let tmp_in = self.machine.acquire_temp_gpr().unwrap(); @@ -3121,8 +3550,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F64ConvertSI64 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::F64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); let tmp_out = self.machine.acquire_temp_xmm().unwrap(); let tmp_in = self.machine.acquire_temp_gpr().unwrap(); @@ -3136,8 +3569,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F64ConvertUI64 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::F64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); let tmp_out = self.machine.acquire_temp_xmm().unwrap(); let tmp_in = self.machine.acquire_temp_gpr().unwrap(); let tmp = self.machine.acquire_temp_gpr().unwrap(); @@ -3188,25 +3625,30 @@ impl FunctionCodeGenerator for X64FunctionCode { .value_stack .drain(self.value_stack.len() - param_types.len()..) .collect(); - let released: SmallVec<[Location; 8]> = params - .iter() - .filter(|&&(_, lot)| lot == LocalOrTemp::Temp) - .map(|&(x, _)| x) - .collect(); - self.machine.release_locations_only_regs(&released); + self.machine.release_locations_only_regs(¶ms); + + self.machine.release_locations_only_osr_state(params.len()); Self::emit_call_sysv_label( a, &mut self.machine, label, - params.iter().map(|&(x, _)| x), + params.iter().map(|x| *x), + Some((&mut self.fsm, &mut self.control_stack)), ); - self.machine.release_locations_only_stack(a, &released); + self.machine.release_locations_only_stack(a, ¶ms); if return_types.len() > 0 { - let ret = self.machine.acquire_locations(a, &[return_types[0]], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[( + return_types[0], + MachineValue::WasmStack(self.value_stack.len()), + )], + false, + )[0]; + self.value_stack.push(ret); a.emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); } } @@ -3225,12 +3667,7 @@ impl FunctionCodeGenerator for X64FunctionCode { .value_stack .drain(self.value_stack.len() - param_types.len()..) .collect(); - let released: SmallVec<[Location; 8]> = params - .iter() - .filter(|&&(_, lot)| lot == LocalOrTemp::Temp) - .map(|&(x, _)| x) - .collect(); - self.machine.release_locations_only_regs(&released); + self.machine.release_locations_only_regs(¶ms); let table_base = self.machine.acquire_temp_gpr().unwrap(); let table_count = self.machine.acquire_temp_gpr().unwrap(); @@ -3303,6 +3740,8 @@ impl FunctionCodeGenerator for X64FunctionCode { ); } + self.machine.release_locations_only_osr_state(params.len()); + Self::emit_call_sysv( a, &mut self.machine, @@ -3312,14 +3751,22 @@ impl FunctionCodeGenerator for X64FunctionCode { (vm::Anyfunc::offset_func() as usize) as i32, )); }, - params.iter().map(|&(x, _)| x), + params.iter().map(|x| *x), + Some((&mut self.fsm, &mut self.control_stack)), ); - self.machine.release_locations_only_stack(a, &released); + self.machine.release_locations_only_stack(a, ¶ms); if return_types.len() > 0 { - let ret = self.machine.acquire_locations(a, &[return_types[0]], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[( + return_types[0], + MachineValue::WasmStack(self.value_stack.len()), + )], + false, + )[0]; + self.value_stack.push(ret); a.emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); } } @@ -3330,7 +3777,7 @@ impl FunctionCodeGenerator for X64FunctionCode { let cond = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - self.control_stack.push(ControlFrame { + let frame = ControlFrame { label: label_end, loop_like: false, if_else: IfElseState::If(label_else), @@ -3340,7 +3787,14 @@ impl FunctionCodeGenerator for X64FunctionCode { _ => panic!("multi-value returns not yet implemented"), }, value_stack_depth: self.value_stack.len(), - }); + state: self.machine.state.clone(), + state_diff_id: Self::get_state_diff( + &self.machine, + &mut self.fsm, + &mut self.control_stack, + ), + }; + self.control_stack.push(frame); Self::emit_relaxed_binop( a, &mut self.machine, @@ -3355,7 +3809,7 @@ impl FunctionCodeGenerator for X64FunctionCode { let mut frame = self.control_stack.last_mut().unwrap(); if !was_unreachable && frame.returns.len() > 0 { - let (loc, _) = *self.value_stack.last().unwrap(); + let loc = *self.value_stack.last().unwrap(); Self::emit_relaxed_binop( a, &mut self.machine, @@ -3366,13 +3820,9 @@ impl FunctionCodeGenerator for X64FunctionCode { ); } - let released: Vec = self - .value_stack - .drain(frame.value_stack_depth..) - .filter(|&(_, lot)| lot == LocalOrTemp::Temp) - .map(|(x, _)| x) - .collect(); - self.machine.release_locations(a, &released); + let released: &[Location] = &self.value_stack[frame.value_stack_depth..]; + self.machine.release_locations(a, released); + self.value_stack.truncate(frame.value_stack_depth); match frame.if_else { IfElseState::If(label) => { @@ -3390,8 +3840,12 @@ impl FunctionCodeGenerator for X64FunctionCode { get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let v_a = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); let end_label = a.get_label(); let zero_label = a.get_label(); @@ -3430,7 +3884,7 @@ impl FunctionCodeGenerator for X64FunctionCode { a.emit_label(end_label); } Operator::Block { ty } => { - self.control_stack.push(ControlFrame { + let frame = ControlFrame { label: a.get_label(), loop_like: false, if_else: IfElseState::None, @@ -3440,10 +3894,21 @@ impl FunctionCodeGenerator for X64FunctionCode { _ => panic!("multi-value returns not yet implemented"), }, value_stack_depth: self.value_stack.len(), - }); + state: self.machine.state.clone(), + state_diff_id: Self::get_state_diff( + &self.machine, + &mut self.fsm, + &mut self.control_stack, + ), + }; + self.control_stack.push(frame); } Operator::Loop { ty } => { let label = a.get_label(); + let state_diff_id = + Self::get_state_diff(&self.machine, &mut self.fsm, &mut self.control_stack); + let activate_offset = a.get_offset().0; + self.control_stack.push(ControlFrame { label: label, loop_like: true, @@ -3454,8 +3919,36 @@ impl FunctionCodeGenerator for X64FunctionCode { _ => panic!("multi-value returns not yet implemented"), }, value_stack_depth: self.value_stack.len(), + state: self.machine.state.clone(), + state_diff_id, }); a.emit_label(label); + + // Check interrupt signal without branching + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_interrupt_signal_mem() as i32, + ), + Location::GPR(GPR::RAX), + ); + self.fsm.loop_offsets.insert( + a.get_offset().0, + OffsetInfo { + activate_offset, + diff_id: state_diff_id, + }, + ); + self.fsm.wasm_offset_to_target_offset.insert( + self.machine.state.wasm_inst_offset, + SuspendOffset::Loop(a.get_offset().0), + ); + a.emit_mov( + Size::S64, + Location::Memory(GPR::RAX, 0), + Location::GPR(GPR::RAX), + ); } Operator::Nop => {} Operator::MemorySize { reserved } => { @@ -3480,18 +3973,21 @@ impl FunctionCodeGenerator for X64FunctionCode { a.emit_call_location(Location::GPR(GPR::RAX)); }, ::std::iter::once(Location::Imm32(memory_index.index() as u32)), + None, ); - let ret = self.machine.acquire_locations(a, &[WpType::I64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); a.emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); } Operator::MemoryGrow { reserved } => { let memory_index = MemoryIndex::new(reserved as usize); - let (param_pages, param_pages_lot) = self.value_stack.pop().unwrap(); + let param_pages = self.value_stack.pop().unwrap(); - if param_pages_lot == LocalOrTemp::Temp { - self.machine.release_locations_only_regs(&[param_pages]); - } + self.machine.release_locations_only_regs(&[param_pages]); a.emit_mov( Size::S64, @@ -3507,6 +4003,8 @@ impl FunctionCodeGenerator for X64FunctionCode { Location::GPR(GPR::RAX), ); + self.machine.release_locations_only_osr_state(1); + Self::emit_call_sysv( a, &mut self.machine, @@ -3515,21 +4013,28 @@ impl FunctionCodeGenerator for X64FunctionCode { }, ::std::iter::once(Location::Imm32(memory_index.index() as u32)) .chain(::std::iter::once(param_pages)), + None, ); - if param_pages_lot == LocalOrTemp::Temp { - self.machine.release_locations_only_stack(a, &[param_pages]); - } + self.machine.release_locations_only_stack(a, &[param_pages]); - let ret = self.machine.acquire_locations(a, &[WpType::I64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); a.emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); } Operator::I32Load { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I32], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); Self::emit_memory_op( module_info, @@ -3554,8 +4059,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F32Load { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::F32], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); Self::emit_memory_op( module_info, @@ -3580,8 +4089,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I32Load8U { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I32], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); Self::emit_memory_op( module_info, @@ -3607,8 +4120,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I32Load8S { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I32], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); Self::emit_memory_op( module_info, @@ -3634,8 +4151,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I32Load16U { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I32], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); Self::emit_memory_op( module_info, @@ -3661,8 +4182,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I32Load16S { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I32], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); Self::emit_memory_op( module_info, @@ -3792,8 +4317,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64Load { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); Self::emit_memory_op( module_info, @@ -3818,8 +4347,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F64Load { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::F64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); Self::emit_memory_op( module_info, @@ -3844,8 +4377,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64Load8U { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); Self::emit_memory_op( module_info, @@ -3871,8 +4408,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64Load8S { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); Self::emit_memory_op( module_info, @@ -3898,8 +4439,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64Load16U { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); Self::emit_memory_op( module_info, @@ -3925,8 +4470,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64Load16S { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); Self::emit_memory_op( module_info, @@ -3952,8 +4501,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64Load32U { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); Self::emit_memory_op( module_info, @@ -3984,8 +4537,12 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64Load32S { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[WpType::I64], false)[0]; - self.value_stack.push((ret, LocalOrTemp::Temp)); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); Self::emit_memory_op( module_info, @@ -4139,6 +4696,7 @@ impl FunctionCodeGenerator for X64FunctionCode { ); } Operator::Unreachable => { + Self::mark_trappable(a, &self.machine, &mut self.fsm, &mut self.control_stack); a.emit_ud2(); self.unreachable_depth = 1; } @@ -4146,7 +4704,7 @@ impl FunctionCodeGenerator for X64FunctionCode { let frame = &self.control_stack[0]; if frame.returns.len() > 0 { assert_eq!(frame.returns.len(), 1); - let (loc, _) = *self.value_stack.last().unwrap(); + let loc = *self.value_stack.last().unwrap(); Self::emit_relaxed_binop( a, &mut self.machine, @@ -4156,12 +4714,8 @@ impl FunctionCodeGenerator for X64FunctionCode { Location::GPR(GPR::RAX), ); } - let released: Vec = self.value_stack[frame.value_stack_depth..] - .iter() - .filter(|&&(_, lot)| lot == LocalOrTemp::Temp) - .map(|&(x, _)| x) - .collect(); - self.machine.release_locations_keep_state(a, &released); + let released = &self.value_stack[frame.value_stack_depth..]; + self.machine.release_locations_keep_state(a, released); a.emit_jmp(Condition::None, frame.label); self.unreachable_depth = 1; } @@ -4170,15 +4724,11 @@ impl FunctionCodeGenerator for X64FunctionCode { &self.control_stack[self.control_stack.len() - 1 - (relative_depth as usize)]; if !frame.loop_like && frame.returns.len() > 0 { assert_eq!(frame.returns.len(), 1); - let (loc, _) = *self.value_stack.last().unwrap(); + let loc = *self.value_stack.last().unwrap(); a.emit_mov(Size::S64, loc, Location::GPR(GPR::RAX)); } - let released: Vec = self.value_stack[frame.value_stack_depth..] - .iter() - .filter(|&&(_, lot)| lot == LocalOrTemp::Temp) - .map(|&(x, _)| x) - .collect(); - self.machine.release_locations_keep_state(a, &released); + let released = &self.value_stack[frame.value_stack_depth..]; + self.machine.release_locations_keep_state(a, released); a.emit_jmp(Condition::None, frame.label); self.unreachable_depth = 1; } @@ -4200,15 +4750,11 @@ impl FunctionCodeGenerator for X64FunctionCode { &self.control_stack[self.control_stack.len() - 1 - (relative_depth as usize)]; if !frame.loop_like && frame.returns.len() > 0 { assert_eq!(frame.returns.len(), 1); - let (loc, _) = *self.value_stack.last().unwrap(); + let loc = *self.value_stack.last().unwrap(); a.emit_mov(Size::S64, loc, Location::GPR(GPR::RAX)); } - let released: Vec = self.value_stack[frame.value_stack_depth..] - .iter() - .filter(|&&(_, lot)| lot == LocalOrTemp::Temp) - .map(|&(x, _)| x) - .collect(); - self.machine.release_locations_keep_state(a, &released); + let released = &self.value_stack[frame.value_stack_depth..]; + self.machine.release_locations_keep_state(a, released); a.emit_jmp(Condition::None, frame.label); a.emit_label(after); @@ -4244,15 +4790,11 @@ impl FunctionCodeGenerator for X64FunctionCode { &self.control_stack[self.control_stack.len() - 1 - (*target as usize)]; if !frame.loop_like && frame.returns.len() > 0 { assert_eq!(frame.returns.len(), 1); - let (loc, _) = *self.value_stack.last().unwrap(); + let loc = *self.value_stack.last().unwrap(); a.emit_mov(Size::S64, loc, Location::GPR(GPR::RAX)); } - let released: Vec = self.value_stack[frame.value_stack_depth..] - .iter() - .filter(|&&(_, lot)| lot == LocalOrTemp::Temp) - .map(|&(x, _)| x) - .collect(); - self.machine.release_locations_keep_state(a, &released); + let released = &self.value_stack[frame.value_stack_depth..]; + self.machine.release_locations_keep_state(a, released); a.emit_jmp(Condition::None, frame.label); } a.emit_label(default_br); @@ -4262,15 +4804,11 @@ impl FunctionCodeGenerator for X64FunctionCode { [self.control_stack.len() - 1 - (default_target as usize)]; if !frame.loop_like && frame.returns.len() > 0 { assert_eq!(frame.returns.len(), 1); - let (loc, _) = *self.value_stack.last().unwrap(); + let loc = *self.value_stack.last().unwrap(); a.emit_mov(Size::S64, loc, Location::GPR(GPR::RAX)); } - let released: Vec = self.value_stack[frame.value_stack_depth..] - .iter() - .filter(|&&(_, lot)| lot == LocalOrTemp::Temp) - .map(|&(x, _)| x) - .collect(); - self.machine.release_locations_keep_state(a, &released); + let released = &self.value_stack[frame.value_stack_depth..]; + self.machine.release_locations_keep_state(a, released); a.emit_jmp(Condition::None, frame.label); } @@ -4287,7 +4825,7 @@ impl FunctionCodeGenerator for X64FunctionCode { let frame = self.control_stack.pop().unwrap(); if !was_unreachable && frame.returns.len() > 0 { - let (loc, _) = *self.value_stack.last().unwrap(); + let loc = *self.value_stack.last().unwrap(); Self::emit_relaxed_binop( a, &mut self.machine, @@ -4305,13 +4843,9 @@ impl FunctionCodeGenerator for X64FunctionCode { a.emit_pop(Size::S64, Location::GPR(GPR::RBP)); a.emit_ret(); } else { - let released: Vec = self - .value_stack - .drain(frame.value_stack_depth..) - .filter(|&(_, lot)| lot == LocalOrTemp::Temp) - .map(|(x, _)| x) - .collect(); - self.machine.release_locations(a, &released); + let released = &self.value_stack[frame.value_stack_depth..]; + self.machine.release_locations(a, released); + self.value_stack.truncate(frame.value_stack_depth); if !frame.loop_like { a.emit_label(frame.label); @@ -4323,9 +4857,16 @@ impl FunctionCodeGenerator for X64FunctionCode { if frame.returns.len() > 0 { assert_eq!(frame.returns.len(), 1); - let loc = self.machine.acquire_locations(a, &frame.returns, false)[0]; + let loc = self.machine.acquire_locations( + a, + &[( + frame.returns[0], + MachineValue::WasmStack(self.value_stack.len()), + )], + false, + )[0]; a.emit_mov(Size::S64, Location::GPR(GPR::RAX), loc); - self.value_stack.push((loc, LocalOrTemp::Temp)); + self.value_stack.push(loc); } } } @@ -4347,14 +4888,8 @@ fn type_to_wp_type(ty: Type) -> WpType { } } -fn get_location_released( - a: &mut Assembler, - m: &mut Machine, - (loc, lot): (Location, LocalOrTemp), -) -> Location { - if lot == LocalOrTemp::Temp { - m.release_locations(a, &[loc]); - } +fn get_location_released(a: &mut Assembler, m: &mut Machine, loc: Location) -> Location { + m.release_locations(a, &[loc]); loc } @@ -4368,4 +4903,38 @@ fn sort_call_movs(movs: &mut [(Location, GPR)]) { } } } + + /* + { + use std::collections::{HashMap, HashSet, VecDeque}; + let mut mov_map: HashMap> = HashMap::new(); + for mov in movs.iter() { + if let Location::GPR(src_gpr) = mov.0 { + if src_gpr != mov.1 { + mov_map.entry(src_gpr).or_insert_with(|| HashSet::new()).insert(mov.1); + } + } + } + + for (start, _) in mov_map.iter() { + let mut q: VecDeque = VecDeque::new(); + let mut black: HashSet = HashSet::new(); + + q.push_back(*start); + black.insert(*start); + + while q.len() > 0 { + let reg = q.pop_front().unwrap(); + let empty_set = HashSet::new(); + for x in mov_map.get(®).unwrap_or(&empty_set).iter() { + if black.contains(x) { + panic!("cycle detected"); + } + q.push_back(*x); + black.insert(*x); + } + } + } + } + */ } diff --git a/lib/singlepass-backend/src/emitter_x64.rs b/lib/singlepass-backend/src/emitter_x64.rs index f94de7013..0fc7795ff 100644 --- a/lib/singlepass-backend/src/emitter_x64.rs +++ b/lib/singlepass-backend/src/emitter_x64.rs @@ -1,38 +1,5 @@ use dynasmrt::{x64::Assembler, AssemblyOffset, DynamicLabel, DynasmApi, DynasmLabelApi}; - -#[repr(u8)] -#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub enum GPR { - RAX, - RCX, - RDX, - RBX, - RSP, - RBP, - RSI, - RDI, - R8, - R9, - R10, - R11, - R12, - R13, - R14, - R15, -} - -#[repr(u8)] -#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub enum XMM { - XMM0, - XMM1, - XMM2, - XMM3, - XMM4, - XMM5, - XMM6, - XMM7, -} +pub use wasmer_runtime_core::state::x64::{GPR, XMM}; #[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] pub enum Location { @@ -87,7 +54,7 @@ pub trait Emitter { type Offset; fn get_label(&mut self) -> Self::Label; - fn get_offset(&mut self) -> Self::Offset; + fn get_offset(&self) -> Self::Offset; fn emit_u64(&mut self, x: u64); @@ -488,7 +455,7 @@ impl Emitter for Assembler { self.new_dynamic_label() } - fn get_offset(&mut self) -> AssemblyOffset { + fn get_offset(&self) -> AssemblyOffset { self.offset() } diff --git a/lib/singlepass-backend/src/lib.rs b/lib/singlepass-backend/src/lib.rs index f1011591d..27aa5925f 100644 --- a/lib/singlepass-backend/src/lib.rs +++ b/lib/singlepass-backend/src/lib.rs @@ -22,7 +22,7 @@ extern crate smallvec; mod codegen_x64; mod emitter_x64; mod machine; -mod protect_unix; +pub mod protect_unix; pub use codegen_x64::X64FunctionCode as FunctionCodeGenerator; pub use codegen_x64::X64ModuleCodeGenerator as ModuleCodeGenerator; diff --git a/lib/singlepass-backend/src/machine.rs b/lib/singlepass-backend/src/machine.rs index d5258f919..61c3c79ce 100644 --- a/lib/singlepass-backend/src/machine.rs +++ b/lib/singlepass-backend/src/machine.rs @@ -1,6 +1,8 @@ use crate::emitter_x64::*; use smallvec::SmallVec; use std::collections::HashSet; +use wasmer_runtime_core::state::x64::X64Register; +use wasmer_runtime_core::state::*; use wasmparser::Type as WpType; struct MachineStackOffset(usize); @@ -10,6 +12,7 @@ pub struct Machine { used_xmms: HashSet, stack_offset: MachineStackOffset, save_area_offset: Option, + pub state: MachineState, } impl Machine { @@ -19,6 +22,7 @@ impl Machine { used_xmms: HashSet::new(), stack_offset: MachineStackOffset(0), save_area_offset: None, + state: x64::new_machine_state(), } } @@ -129,13 +133,13 @@ impl Machine { pub fn acquire_locations( &mut self, assembler: &mut E, - tys: &[WpType], + tys: &[(WpType, MachineValue)], zeroed: bool, ) -> SmallVec<[Location; 1]> { let mut ret = smallvec![]; let mut delta_stack_offset: usize = 0; - for ty in tys { + for (ty, mv) in tys { let loc = match *ty { WpType::F32 | WpType::F64 => self.pick_xmm().map(Location::XMM), WpType::I32 | WpType::I64 => self.pick_gpr().map(Location::GPR), @@ -151,9 +155,14 @@ impl Machine { }; if let Location::GPR(x) = loc { self.used_gprs.insert(x); + self.state.register_values[X64Register::GPR(x).to_index().0] = *mv; } else if let Location::XMM(x) = loc { self.used_xmms.insert(x); + self.state.register_values[X64Register::XMM(x).to_index().0] = *mv; + } else { + self.state.stack_values.push(*mv); } + self.state.wasm_stack.push(WasmAbstractValue::Runtime); ret.push(loc); } @@ -180,9 +189,13 @@ impl Machine { match *loc { Location::GPR(ref x) => { assert_eq!(self.used_gprs.remove(x), true); + self.state.register_values[X64Register::GPR(*x).to_index().0] = + MachineValue::Undefined; } Location::XMM(ref x) => { assert_eq!(self.used_xmms.remove(x), true); + self.state.register_values[X64Register::XMM(*x).to_index().0] = + MachineValue::Undefined; } Location::Memory(GPR::RBP, x) => { if x >= 0 { @@ -194,9 +207,11 @@ impl Machine { } self.stack_offset.0 -= 8; delta_stack_offset += 8; + self.state.stack_values.pop().unwrap(); } _ => {} } + self.state.wasm_stack.pop().unwrap(); } if delta_stack_offset != 0 { @@ -213,12 +228,17 @@ impl Machine { match *loc { Location::GPR(ref x) => { assert_eq!(self.used_gprs.remove(x), true); + self.state.register_values[X64Register::GPR(*x).to_index().0] = + MachineValue::Undefined; } Location::XMM(ref x) => { assert_eq!(self.used_xmms.remove(x), true); + self.state.register_values[X64Register::XMM(*x).to_index().0] = + MachineValue::Undefined; } _ => {} } + // Wasm state popping is deferred to `release_locations_only_osr_state`. } } @@ -241,9 +261,11 @@ impl Machine { } self.stack_offset.0 -= 8; delta_stack_offset += 8; + self.state.stack_values.pop().unwrap(); } _ => {} } + // Wasm state popping is deferred to `release_locations_only_osr_state`. } if delta_stack_offset != 0 { @@ -255,6 +277,12 @@ impl Machine { } } + pub fn release_locations_only_osr_state(&mut self, n: usize) { + for _ in 0..n { + self.state.wasm_stack.pop().unwrap(); + } + } + pub fn release_locations_keep_state(&self, assembler: &mut E, locs: &[Location]) { let mut delta_stack_offset: usize = 0; let mut stack_offset = self.stack_offset.0; @@ -314,7 +342,11 @@ impl Machine { allocated += 1; get_local_location(old_idx) } - Location::Memory(_, _) => loc, + Location::Memory(_, _) => { + let old_idx = allocated; + allocated += 1; + get_local_location(old_idx) + } _ => unreachable!(), }); } @@ -325,6 +357,19 @@ impl Machine { allocated += 1; } + for (i, loc) in locations.iter().enumerate() { + match *loc { + Location::GPR(x) => { + self.state.register_values[X64Register::GPR(x).to_index().0] = + MachineValue::WasmLocal(i); + } + Location::Memory(_, _) => { + self.state.stack_values.push(MachineValue::WasmLocal(i)); + } + _ => unreachable!(), + } + } + // How many machine stack slots did all the locals use? let num_mem_slots = locations .iter() @@ -346,15 +391,21 @@ impl Machine { // Save callee-saved registers. for loc in locations.iter() { - if let Location::GPR(_) = *loc { + if let Location::GPR(x) = *loc { a.emit_push(Size::S64, *loc); self.stack_offset.0 += 8; + self.state.stack_values.push(MachineValue::PreserveRegister( + X64Register::GPR(x).to_index(), + )); } } // Save R15 for vmctx use. a.emit_push(Size::S64, Location::GPR(GPR::R15)); self.stack_offset.0 += 8; + self.state.stack_values.push(MachineValue::PreserveRegister( + X64Register::GPR(GPR::R15).to_index(), + )); // Save the offset of static area. self.save_area_offset = Some(MachineStackOffset(self.stack_offset.0)); @@ -366,7 +417,17 @@ impl Machine { Location::GPR(_) => { a.emit_mov(Size::S64, loc, locations[i]); } - _ => break, + Location::Memory(_, _) => match locations[i] { + Location::GPR(_) => { + a.emit_mov(Size::S64, loc, locations[i]); + } + Location::Memory(_, _) => { + a.emit_mov(Size::S64, loc, Location::GPR(GPR::RAX)); + a.emit_mov(Size::S64, Location::GPR(GPR::RAX), locations[i]); + } + _ => unreachable!(), + }, + _ => unreachable!(), } } @@ -429,7 +490,11 @@ mod test { fn test_release_locations_keep_state_nopanic() { let mut machine = Machine::new(); let mut assembler = Assembler::new().unwrap(); - let locs = machine.acquire_locations(&mut assembler, &[WpType::I32; 10], false); + let locs = machine.acquire_locations( + &mut assembler, + &[(WpType::I32, MachineValue::Undefined); 10], + false, + ); machine.release_locations_keep_state(&mut assembler, &locs); } diff --git a/lib/singlepass-backend/src/protect_unix.rs b/lib/singlepass-backend/src/protect_unix.rs index f777c2482..136f9bf08 100644 --- a/lib/singlepass-backend/src/protect_unix.rs +++ b/lib/singlepass-backend/src/protect_unix.rs @@ -9,77 +9,18 @@ //! are very special, the async signal unsafety of Rust's TLS implementation generally does not affect the correctness here //! unless you have memory unsafety elsewhere in your code. //! -use libc::{c_int, c_void, siginfo_t}; -use nix::sys::signal::{ - sigaction, SaFlags, SigAction, SigHandler, SigSet, Signal, SIGBUS, SIGFPE, SIGILL, SIGSEGV, - SIGTRAP, -}; use std::any::Any; -use std::cell::{Cell, RefCell, UnsafeCell}; -use std::collections::HashMap; -use std::ptr; -use std::sync::Arc; -use std::sync::Once; -use wasmer_runtime_core::codegen::BkptInfo; +use std::cell::Cell; +use wasmer_runtime_core::codegen::BreakpointMap; +use wasmer_runtime_core::fault::{begin_unsafe_unwind, catch_unsafe_unwind, ensure_sighandler}; use wasmer_runtime_core::typed_func::WasmTrapInfo; -extern "C" fn signal_trap_handler( - signum: ::nix::libc::c_int, - siginfo: *mut siginfo_t, - ucontext: *mut c_void, -) { - unsafe { - match Signal::from_c_int(signum) { - Ok(SIGTRAP) => { - let (_, ip) = get_faulting_addr_and_ip(siginfo as _, ucontext); - let bkpt_map = BKPT_MAP.with(|x| x.borrow().last().map(|x| x.clone())); - if let Some(bkpt_map) = bkpt_map { - if let Some(ref x) = bkpt_map.get(&(ip as usize)) { - (x)(BkptInfo { throw: throw }); - return; - } - } - } - _ => {} - } - - do_unwind(signum, siginfo as _, ucontext); - } -} - -extern "C" { - pub fn setjmp(env: *mut c_void) -> c_int; - fn longjmp(env: *mut c_void, val: c_int) -> !; -} - -pub unsafe fn install_sighandler() { - let sa = SigAction::new( - SigHandler::SigAction(signal_trap_handler), - SaFlags::SA_ONSTACK, - SigSet::empty(), - ); - sigaction(SIGFPE, &sa).unwrap(); - sigaction(SIGILL, &sa).unwrap(); - sigaction(SIGSEGV, &sa).unwrap(); - sigaction(SIGBUS, &sa).unwrap(); - sigaction(SIGTRAP, &sa).unwrap(); -} - -const SETJMP_BUFFER_LEN: usize = 27; -pub static SIGHANDLER_INIT: Once = Once::new(); - thread_local! { - pub static SETJMP_BUFFER: UnsafeCell<[c_int; SETJMP_BUFFER_LEN]> = UnsafeCell::new([0; SETJMP_BUFFER_LEN]); - pub static CAUGHT_ADDRESSES: Cell<(*const c_void, *const c_void)> = Cell::new((ptr::null(), ptr::null())); - pub static CURRENT_EXECUTABLE_BUFFER: Cell<*const c_void> = Cell::new(ptr::null()); pub static TRAP_EARLY_DATA: Cell>> = Cell::new(None); - pub static BKPT_MAP: RefCell>>>> = RefCell::new(Vec::new()); } pub unsafe fn trigger_trap() -> ! { - let jmp_buf = SETJMP_BUFFER.with(|buf| buf.get()); - - longjmp(jmp_buf as *mut c_void, 0) + begin_unsafe_unwind(Box::new(())); } pub enum CallProtError { @@ -87,157 +28,26 @@ pub enum CallProtError { Error(Box), } -pub fn call_protected(f: impl FnOnce() -> T) -> Result { +pub fn call_protected( + f: impl FnOnce() -> T, + breakpoints: Option, +) -> Result { + ensure_sighandler(); unsafe { - let jmp_buf = SETJMP_BUFFER.with(|buf| buf.get()); - let prev_jmp_buf = *jmp_buf; - - SIGHANDLER_INIT.call_once(|| { - install_sighandler(); - }); - - let signum = setjmp(jmp_buf as *mut _); - if signum != 0 { - *jmp_buf = prev_jmp_buf; - - if let Some(data) = TRAP_EARLY_DATA.with(|cell| cell.replace(None)) { - Err(CallProtError::Error(data)) - } else { - // let (faulting_addr, _inst_ptr) = CAUGHT_ADDRESSES.with(|cell| cell.get()); - - // let signal = match Signal::from_c_int(signum) { - // Ok(SIGFPE) => "floating-point exception", - // Ok(SIGILL) => "illegal instruction", - // Ok(SIGSEGV) => "segmentation violation", - // Ok(SIGBUS) => "bus error", - // Err(_) => "error while getting the Signal", - // _ => "unknown trapped signal", - // }; - // // When the trap-handler is fully implemented, this will return more information. - // Err(RuntimeError::Trap { - // msg: format!("unknown trap at {:p} - {}", faulting_addr, signal).into(), - // } - // .into()) - Err(CallProtError::Trap(WasmTrapInfo::Unknown)) + let ret = catch_unsafe_unwind(|| f(), breakpoints); + match ret { + Ok(x) => Ok(x), + Err(e) => { + if let Some(data) = TRAP_EARLY_DATA.with(|cell| cell.replace(None)) { + Err(CallProtError::Error(data)) + } else { + Err(CallProtError::Error(e)) + } } - } else { - let ret = f(); // TODO: Switch stack? - *jmp_buf = prev_jmp_buf; - Ok(ret) } } } pub unsafe fn throw(payload: Box) -> ! { - let jmp_buf = SETJMP_BUFFER.with(|buf| buf.get()); - if *jmp_buf == [0; SETJMP_BUFFER_LEN] { - ::std::process::abort(); - } - TRAP_EARLY_DATA.with(|cell| cell.replace(Some(payload))); - longjmp(jmp_buf as *mut ::nix::libc::c_void, 0xffff); -} - -/// Unwinds to last protected_call. -pub unsafe fn do_unwind(signum: i32, siginfo: *const c_void, ucontext: *const c_void) -> ! { - // Since do_unwind is only expected to get called from WebAssembly code which doesn't hold any host resources (locks etc.) - // itself, accessing TLS here is safe. In case any other code calls this, it often indicates a memory safety bug and you should - // temporarily disable the signal handlers to debug it. - - let jmp_buf = SETJMP_BUFFER.with(|buf| buf.get()); - if *jmp_buf == [0; SETJMP_BUFFER_LEN] { - ::std::process::abort(); - } - - CAUGHT_ADDRESSES.with(|cell| cell.set(get_faulting_addr_and_ip(siginfo, ucontext))); - - longjmp(jmp_buf as *mut ::nix::libc::c_void, signum) -} - -#[cfg(all(target_os = "linux", target_arch = "x86_64"))] -unsafe fn get_faulting_addr_and_ip( - siginfo: *const c_void, - ucontext: *const c_void, -) -> (*const c_void, *const c_void) { - use libc::{ucontext_t, RIP}; - - #[allow(dead_code)] - #[repr(C)] - struct siginfo_t { - si_signo: i32, - si_errno: i32, - si_code: i32, - si_addr: u64, - // ... - } - - let siginfo = siginfo as *const siginfo_t; - let si_addr = (*siginfo).si_addr; - - let ucontext = ucontext as *const ucontext_t; - let rip = (*ucontext).uc_mcontext.gregs[RIP as usize]; - - (si_addr as _, rip as _) -} - -#[cfg(all(target_os = "macos", target_arch = "x86_64"))] -unsafe fn get_faulting_addr_and_ip( - siginfo: *const c_void, - ucontext: *const c_void, -) -> (*const c_void, *const c_void) { - #[allow(dead_code)] - #[repr(C)] - struct ucontext_t { - uc_onstack: u32, - uc_sigmask: u32, - uc_stack: libc::stack_t, - uc_link: *const ucontext_t, - uc_mcsize: u64, - uc_mcontext: *const mcontext_t, - } - #[repr(C)] - struct exception_state { - trapno: u16, - cpu: u16, - err: u32, - faultvaddr: u64, - } - #[repr(C)] - struct regs { - rax: u64, - rbx: u64, - rcx: u64, - rdx: u64, - rdi: u64, - rsi: u64, - rbp: u64, - rsp: u64, - r8: u64, - r9: u64, - r10: u64, - r11: u64, - r12: u64, - r13: u64, - r14: u64, - r15: u64, - rip: u64, - rflags: u64, - cs: u64, - fs: u64, - gs: u64, - } - #[allow(dead_code)] - #[repr(C)] - struct mcontext_t { - es: exception_state, - ss: regs, - // ... - } - - let siginfo = siginfo as *const siginfo_t; - let si_addr = (*siginfo).si_addr; - - let ucontext = ucontext as *const ucontext_t; - let rip = (*(*ucontext).uc_mcontext).ss.rip; - - (si_addr, rip as _) + begin_unsafe_unwind(payload); } diff --git a/src/bin/wasmer.rs b/src/bin/wasmer.rs index 021a7eaf0..cc29747e2 100644 --- a/src/bin/wasmer.rs +++ b/src/bin/wasmer.rs @@ -19,7 +19,6 @@ use wasmer_clif_backend::CraneliftCompiler; use wasmer_llvm_backend::LLVMCompiler; use wasmer_runtime::{ cache::{Cache as BaseCache, FileSystemCache, WasmHash, WASMER_VERSION_HASH}, - error::RuntimeError, Func, Value, }; use wasmer_runtime_core::{ @@ -112,6 +111,10 @@ struct Run { )] loader: Option, + #[cfg(feature = "backend:singlepass")] + #[structopt(long = "resume")] + resume: Option, + #[structopt(long = "command-name", hidden = true)] command_name: Option, @@ -151,7 +154,7 @@ impl FromStr for LoaderName { } #[allow(dead_code)] -#[derive(Debug)] +#[derive(Debug, Eq, PartialEq)] enum Backend { Cranelift, Singlepass, @@ -483,7 +486,7 @@ fn execute_wasm(options: &Run) -> Result<(), String> { .map_err(|e| format!("{:?}", e))?; } else { if cfg!(feature = "wasi") && wasmer_wasi::is_wasi_module(&module) { - let import_object = wasmer_wasi::generate_import_object( + let mut import_object = wasmer_wasi::generate_import_object( if let Some(cn) = &options.command_name { [cn.clone()] } else { @@ -502,24 +505,95 @@ fn execute_wasm(options: &Run) -> Result<(), String> { mapped_dirs, ); - let instance = module + let mut instance = module .instantiate(&import_object) .map_err(|e| format!("Can't instantiate module: {:?}", e))?; let start: Func<(), ()> = instance.func("_start").map_err(|e| format!("{:?}", e))?; - let result = start.call(); + #[cfg(feature = "backend:singlepass")] + unsafe { + if options.backend == Backend::Singlepass { + use wasmer_runtime_core::fault::{catch_unsafe_unwind, ensure_sighandler}; + use wasmer_runtime_core::state::{ + x64::invoke_call_return_on_stack, InstanceImage, + }; + use wasmer_runtime_core::vm::Ctx; - if let Err(ref err) = result { - match err { - RuntimeError::Trap { msg } => panic!("wasm trap occured: {}", msg), - RuntimeError::Error { data } => { - if let Some(error_code) = data.downcast_ref::() { - std::process::exit(error_code.code as i32) + ensure_sighandler(); + + let start_raw: extern "C" fn(&mut Ctx) = + ::std::mem::transmute(start.get_vm_func()); + + let mut image: Option = if let Some(ref path) = options.resume { + let mut f = File::open(path).unwrap(); + let mut out: Vec = vec![]; + f.read_to_end(&mut out).unwrap(); + Some(InstanceImage::from_bytes(&out).expect("failed to decode image")) + } else { + None + }; + let breakpoints = instance.module.runnable_module.get_breakpoints(); + + loop { + let ret = if let Some(image) = image.take() { + let msm = instance + .module + .runnable_module + .get_module_state_map() + .unwrap(); + let code_base = + instance.module.runnable_module.get_code().unwrap().as_ptr() + as usize; + invoke_call_return_on_stack( + &msm, + code_base, + image, + instance.context_mut(), + breakpoints.clone(), + ) + .map(|_| ()) + } else { + catch_unsafe_unwind( + || start_raw(instance.context_mut()), + breakpoints.clone(), + ) + }; + if let Err(e) = ret { + if let Some(new_image) = e.downcast_ref::() { + let op = interactive_shell(InteractiveShellContext { + image: Some(new_image.clone()), + }); + match op { + ShellExitOperation::ContinueWith(new_image) => { + image = Some(new_image); + } + } + } else { + return Err("Error while executing WebAssembly".into()); + } + } else { + return Ok(()); } } } - panic!("error: {:?}", err) + } + + { + use wasmer_runtime::error::RuntimeError; + let result = start.call(); + + if let Err(ref err) = result { + match err { + RuntimeError::Trap { msg } => panic!("wasm trap occured: {}", msg), + RuntimeError::Error { data } => { + if let Some(error_code) = data.downcast_ref::() { + std::process::exit(error_code.code as i32) + } + } + } + panic!("error: {:?}", err) + } } } else { let import_object = wasmer_runtime_core::import::ImportObject::new(); @@ -544,11 +618,95 @@ fn execute_wasm(options: &Run) -> Result<(), String> { Ok(()) } +#[cfg(feature = "backend:singlepass")] +struct InteractiveShellContext { + image: Option, +} + +#[cfg(feature = "backend:singlepass")] +#[derive(Debug)] +enum ShellExitOperation { + ContinueWith(wasmer_runtime_core::state::InstanceImage), +} + +#[cfg(feature = "backend:singlepass")] +fn interactive_shell(mut ctx: InteractiveShellContext) -> ShellExitOperation { + use std::io::Write; + + let mut stdout = ::std::io::stdout(); + let stdin = ::std::io::stdin(); + + loop { + print!("Wasmer> "); + stdout.flush().unwrap(); + let mut line = String::new(); + stdin.read_line(&mut line).unwrap(); + let mut parts = line.split(" ").filter(|x| x.len() > 0).map(|x| x.trim()); + + let cmd = parts.next(); + if cmd.is_none() { + println!("Command required"); + continue; + } + let cmd = cmd.unwrap(); + + match cmd { + "snapshot" => { + let path = parts.next(); + if path.is_none() { + println!("Usage: snapshot [out_path]"); + continue; + } + let path = path.unwrap(); + + if let Some(ref image) = ctx.image { + let buf = image.to_bytes(); + let mut f = match File::create(path) { + Ok(x) => x, + Err(e) => { + println!("Cannot open output file at {}: {:?}", path, e); + continue; + } + }; + if let Err(e) = f.write_all(&buf) { + println!("Cannot write to output file at {}: {:?}", path, e); + continue; + } + println!("Done"); + } else { + println!("Program state not available"); + } + } + "continue" | "c" => { + if let Some(image) = ctx.image.take() { + return ShellExitOperation::ContinueWith(image); + } else { + println!("Program state not available, cannot continue execution"); + } + } + "backtrace" | "bt" => { + if let Some(ref image) = ctx.image { + println!("{}", image.execution_state.colored_output()); + } else { + println!("State not available"); + } + } + "exit" | "quit" => { + exit(0); + } + "" => {} + _ => { + println!("Unknown command: {}", cmd); + } + } + } +} + fn run(options: Run) { match execute_wasm(&options) { Ok(()) => {} Err(message) => { - eprintln!("{:?}", message); + eprintln!("execute_wasm: {:?}", message); exit(1); } }