Merge branch 'master' into feature/map-dir

This commit is contained in:
Mark McCaskey 2019-05-17 16:24:26 -07:00 committed by GitHub
commit 4b03abafcc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
41 changed files with 429 additions and 24 deletions

View File

@ -60,7 +60,6 @@ jobs:
- restore_cache:
keys:
- v8-test-cargo-cache-linux-stable-{{ arch }}-{{ checksum "Cargo.lock" }}
- v8-test-cargo-cache-linux-stable-{{ arch }}
- <<: *run_install_dependencies
- run:
name: Tests
@ -158,7 +157,6 @@ jobs:
- restore_cache:
keys:
- v8-cargo-cache-linux-nightly-{{ arch }}-{{ checksum "Cargo.lock" }}
- v8-cargo-cache-linux-nightly-{{ arch }}
- run:
name: Install dependencies
command: |
@ -226,7 +224,6 @@ jobs:
- restore_cache:
keys:
- v8-cargo-cache-darwin-nightly-{{ arch }}-{{ checksum "Cargo.lock" }}
- v8-cargo-cache-darwin-nightly-{{ arch }}
- run:
name: Install crate dependencies
command: |
@ -309,18 +306,21 @@ jobs:
- restore_cache:
keys:
- v8-cargo-cache-linux-nightly-{{ arch }}-{{ checksum "Cargo.lock" }}
- v8-cargo-cache-linux-nightly-{{ arch }}
- run:
name: Install dependencies
command: |
sudo apt-get install -y cmake
curl -O https://releases.llvm.org/7.0.0/clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
tar xf clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
rustup toolchain install nightly
rustup target add wasm32-wasi --toolchain nightly
- run: rustup default nightly-2019-04-11
- run: |
export LLVM_SYS_70_PREFIX="`pwd`/clang+llvm-7.0.0-x86_64-linux-gnu-ubuntu-16.04/"
make test
make test-singlepass
make test-wasi-singlepass
make test-wasi-clif
make test-emscripten-clif
make test-emscripten-singlepass
- save_cache:

13
Cargo.lock generated
View File

@ -2287,6 +2287,7 @@ dependencies = [
"structopt 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)",
"wabt 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)",
"wasmer-clif-backend 0.4.2",
"wasmer-dev-utils 0.4.2",
"wasmer-emscripten 0.4.2",
"wasmer-kernel-loader 0.1.0",
"wasmer-llvm-backend 0.4.2",
@ -2322,6 +2323,13 @@ dependencies = [
"winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "wasmer-dev-utils"
version = "0.4.2"
dependencies = [
"libc 0.2.54 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "wasmer-emscripten"
version = "0.4.2"
@ -2334,6 +2342,7 @@ dependencies = [
"time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
"wabt 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)",
"wasmer-clif-backend 0.4.2",
"wasmer-dev-utils 0.4.2",
"wasmer-llvm-backend 0.4.2",
"wasmer-runtime-core 0.4.2",
"wasmer-singlepass-backend 0.4.2",
@ -2474,11 +2483,15 @@ version = "0.4.2"
dependencies = [
"byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"generational-arena 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
"hashbrown 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.54 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
"wasmer-clif-backend 0.4.2",
"wasmer-dev-utils 0.4.2",
"wasmer-runtime-core 0.4.2",
"wasmer-singlepass-backend 0.4.2",
]
[[package]]

View File

@ -34,9 +34,10 @@ wasmer-emscripten = { path = "lib/emscripten" }
wasmer-llvm-backend = { path = "lib/llvm-backend", optional = true }
wasmer-wasi = { path = "lib/wasi", optional = true }
wasmer-kernel-loader = { path = "lib/kernel-loader", optional = true }
wasmer-dev-utils = { path = "lib/dev-utils", optional = true }
[workspace]
members = ["lib/clif-backend", "lib/singlepass-backend", "lib/runtime", "lib/runtime-abi", "lib/runtime-core", "lib/emscripten", "lib/spectests", "lib/win-exception-handler", "lib/runtime-c-api", "lib/llvm-backend", "lib/wasi", "lib/middleware-common", "lib/kernel-loader", "lib/kernel-net", "examples/plugin-for-example"]
members = ["lib/clif-backend", "lib/singlepass-backend", "lib/runtime", "lib/runtime-abi", "lib/runtime-core", "lib/emscripten", "lib/spectests", "lib/win-exception-handler", "lib/runtime-c-api", "lib/llvm-backend", "lib/wasi", "lib/middleware-common", "lib/kernel-loader", "lib/kernel-net", "lib/dev-utils", "examples/plugin-for-example"]
[build-dependencies]
wabt = "0.7.2"

View File

@ -12,6 +12,9 @@ spectests:
emtests:
WASM_EMSCRIPTEN_GENERATE_EMTESTS=1 cargo build -p wasmer-emscripten
wasitests:
WASM_WASI_GENERATE_WASITESTS=1 cargo build -p wasmer-wasi
# clean:
# rm -rf artifacts
@ -45,7 +48,7 @@ do-install:
test:
# We use one thread so the emscripten stdouts doesn't collide
cargo test --all --exclude wasmer-runtime-c-api --exclude wasmer-emscripten --exclude wasmer-spectests --exclude wasmer-singlepass-backend -- $(runargs)
cargo test --all --exclude wasmer-runtime-c-api --exclude wasmer-emscripten --exclude wasmer-spectests --exclude wasmer-singlepass-backend --exclude wasmer-wasi -- $(runargs)
# cargo test --all --exclude wasmer-emscripten -- --test-threads=1 $(runargs)
cargo test --manifest-path lib/spectests/Cargo.toml --features clif
cargo test --manifest-path lib/spectests/Cargo.toml --features llvm
@ -66,6 +69,12 @@ test-emscripten-clif:
test-emscripten-singlepass:
cargo test --manifest-path lib/emscripten/Cargo.toml --features singlepass -- --test-threads=1 $(runargs)
test-wasi-clif:
cargo test --manifest-path lib/wasi/Cargo.toml --features "clif" -- $(runargs)
test-wasi-singlepass:
cargo test --manifest-path lib/wasi/Cargo.toml --features "singlepass" -- $(runargs)
singlepass-debug-release:
cargo +nightly build --features backend:singlepass,debug --release

11
lib/dev-utils/Cargo.toml Normal file
View File

@ -0,0 +1,11 @@
[package]
name = "wasmer-dev-utils"
version = "0.4.2"
description = "Wasmer runtime core library"
license = "MIT"
authors = ["The Wasmer Engineering Team <engineering@wasmer.io>"]
edition = "2018"
repository = "https://github.com/wasmerio/wasmer"
[dependencies]
libc = "0.2.49"

3
lib/dev-utils/README.md Normal file
View File

@ -0,0 +1,3 @@
# Dev Utils
This is shared code between the modules for testing and development only. Code in this crate will not be shipped.

2
lib/dev-utils/src/lib.rs Normal file
View File

@ -0,0 +1,2 @@
pub mod file_descriptor;
pub mod stdio;

View File

@ -23,6 +23,7 @@ rand = "0.6"
[dev-dependencies]
wabt = "0.7.2"
wasmer-dev-utils = { path = "../dev-utils", version = "0.4.2"}
[build-dependencies]
glob = "0.2.11"

View File

@ -5,6 +5,7 @@
//! - Generate the test that will compare the output of running the .wasm file
//! with wasmer with the expected output
use glob::glob;
use std::collections::HashSet;
use std::fs;
use std::path::Path;
use std::path::PathBuf;
@ -20,7 +21,7 @@ static BANNER: &str = "// Rust test file autogenerated with cargo build (build/e
const EXTENSIONS: [&str; 2] = ["c", "cpp"];
const EXCLUDES: [&str; 0] = [];
pub fn compile(file: &str, ignores: &Vec<String>) -> Option<String> {
pub fn compile(file: &str, ignores: &HashSet<String>) -> Option<String> {
let mut output_path = PathBuf::from(file);
output_path.set_extension("out");
// let output_str = output_path.to_str().unwrap();
@ -49,11 +50,7 @@ pub fn compile(file: &str, ignores: &Vec<String>) -> Option<String> {
let wasm_file_metadata = {
let mut wasm_file_path = PathBuf::from(file);
wasm_file_path.set_extension("wasm");
if let Ok(wasm_file) = File::open(wasm_file_path) {
Some(wasm_file.metadata().unwrap())
} else {
None
}
File::open(wasm_file_path).and_then(|wf| wf.metadata()).ok()
};
let real_file = File::open(file).unwrap();
@ -111,10 +108,7 @@ pub fn compile(file: &str, ignores: &Vec<String>) -> Option<String> {
"txt"
};
let ignored = if ignores
.iter()
.any(|i| &i.to_lowercase() == &module_name.to_lowercase())
{
let ignored = if ignores.contains(&module_name.to_lowercase()) {
"\n#[ignore]"
} else {
""
@ -193,8 +187,11 @@ pub fn build() {
}
}
fn read_ignore_list() -> Vec<String> {
fn read_ignore_list() -> HashSet<String> {
let f = File::open("emtests/ignores.txt").unwrap();
let f = BufReader::new(f);
f.lines().filter_map(Result::ok).collect()
f.lines()
.filter_map(Result::ok)
.map(|v| v.to_lowercase())
.collect()
}

View File

@ -24,9 +24,6 @@ use wasmer_runtime_core::{
#[macro_use]
mod macros;
//#[cfg(test)]
mod file_descriptor;
pub mod stdio;
// EMSCRIPTEN APIS
mod bitwise;

View File

@ -4,9 +4,11 @@ macro_rules! assert_emscripten_output {
use wasmer_emscripten::{
EmscriptenGlobals,
generate_emscripten_env,
stdio::StdioCapturer
};
use wasmer_runtime_core::backend::Compiler;
use wasmer_runtime_core::{
backend::Compiler,
};
use wasmer_dev_utils::stdio::StdioCapturer;
#[cfg(feature = "clif")]
fn get_compiler() -> impl Compiler {

View File

@ -6,6 +6,7 @@ license = "MIT"
authors = ["The Wasmer Engineering Team <engineering@wasmer.io>"]
repository = "https://github.com/wasmerio/wasmer"
edition = "2018"
build = "build/mod.rs"
[dependencies]
wasmer-runtime-core = { path = "../runtime-core", version = "0.4.2" }
@ -16,3 +17,16 @@ hashbrown = "0.1.8"
generational-arena = "0.2.2"
log = "0.4.6"
byteorder = "1.3.1"
# hack to get tests to work
wasmer-singlepass-backend = { path = "../singlepass-backend", version = "0.4.2", optional = true }
[build-dependencies]
glob = "0.2.11"
[dev-dependencies]
wasmer-clif-backend = { path = "../clif-backend", version = "0.4.2" }
wasmer-dev-utils = { path = "../dev-utils", version = "0.4.2"}
[features]
clif = []
singlepass = ["wasmer-singlepass-backend"]

11
lib/wasi/build/mod.rs Normal file
View File

@ -0,0 +1,11 @@
use std::env;
mod wasitests;
static WASITESTS_ENV_VAR: &str = "WASM_WASI_GENERATE_WASITESTS";
fn main() {
if env::var(WASITESTS_ENV_VAR).unwrap_or("0".to_string()) == "1" {
wasitests::build();
}
}

145
lib/wasi/build/wasitests.rs Normal file
View File

@ -0,0 +1,145 @@
//! This file will run at build time to autogenerate the WASI regression tests
//! It will compile the files indicated in TESTS, to:executable and .wasm
//! - Compile with the native rust target to get the expected output
//! - Compile with the latest WASI target to get the wasm
//! - Generate the test that will compare the output of running the .wasm file
//! with wasmer with the expected output
use glob::glob;
use std::collections::HashSet;
use std::fs;
use std::path::PathBuf;
use std::process::Command;
use std::fs::File;
use std::io::prelude::*;
use std::io::BufReader;
static BANNER: &str = "// !!! THIS IS A GENERATED FILE !!!
// ANY MANUAL EDITS MAY BE OVERWRITTEN AT ANY TIME
// Files autogenerated with cargo build (build/wasitests.rs).\n";
pub fn compile(file: &str, ignores: &HashSet<String>) -> Option<String> {
dbg!(file);
let mut output_path = PathBuf::from(file);
output_path.set_extension("out");
assert!(file.ends_with(".rs"));
let normalized_name = {
let mut nn = file.to_lowercase();
nn.truncate(file.len() - 3);
nn
};
Command::new("rustc")
.arg("+nightly")
.arg(file)
.arg("-o")
.arg(&normalized_name)
.output()
.expect("Failed to compile program to native code");
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let normal_path = PathBuf::from(&normalized_name);
let mut perm = normal_path
.metadata()
.expect("native executable")
.permissions();
perm.set_mode(0o766);
fs::set_permissions(normal_path, perm).expect("set permissions");
}
let rs_module_name = {
let temp = PathBuf::from(&normalized_name);
temp.file_name().unwrap().to_string_lossy().to_string()
};
let result = Command::new(&normalized_name)
.output()
.expect("Failed to execute native program");
let wasm_out_name = format!("{}.wasm", &normalized_name);
Command::new("rustc")
.arg("+nightly")
.arg("--target=wasm32-wasi")
.arg(file)
.arg("-o")
.arg(&wasm_out_name)
.output()
.expect("Failed to compile program to native code");
let ignored = if ignores.contains(&rs_module_name) {
"\n#[ignore]"
} else {
""
};
let contents = format!(
"#[test]{ignore}
fn test_{rs_module_name}() {{
assert_wasi_output!(
\"../../{module_path}\",
\"{rs_module_name}\",
vec![],
\"../../{test_output_path}\"
);
}}
",
ignore = ignored,
module_path = wasm_out_name,
rs_module_name = rs_module_name,
test_output_path = format!("{}.out", normalized_name),
);
let rust_test_filepath = format!(
concat!(env!("CARGO_MANIFEST_DIR"), "/tests/{}.rs"),
normalized_name,
);
fs::write(&rust_test_filepath, contents.as_bytes()).expect("writing test file");
fs::write(&output_path, result.stdout).expect("writing output to file");
Some(rs_module_name)
}
pub fn build() {
let rust_test_modpath = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/wasitests/mod.rs");
let mut modules: Vec<String> = Vec::new();
let ignores = read_ignore_list();
for entry in glob("wasitests/*.rs").unwrap() {
match entry {
Ok(path) => {
let test = path.to_str().unwrap();
if let Some(module_name) = compile(test, &ignores) {
modules.push(module_name);
}
}
Err(e) => println!("{:?}", e),
}
}
modules.sort();
let mut modules: Vec<String> = modules.iter().map(|m| format!("mod {};", m)).collect();
assert!(modules.len() > 0, "Expected > 0 modules found");
modules.insert(0, BANNER.to_string());
modules.insert(1, "// The _common module is not autogenerated. It provides common macros for the wasitests\n#[macro_use]\nmod _common;".to_string());
// We add an empty line
modules.push("".to_string());
let modfile: String = modules.join("\n");
let source = fs::read(&rust_test_modpath).unwrap();
// We only modify the mod file if has changed
if source != modfile.as_bytes() {
fs::write(&rust_test_modpath, modfile.as_bytes()).unwrap();
}
}
fn read_ignore_list() -> HashSet<String> {
let f = File::open("wasitests/ignores.txt").unwrap();
let f = BufReader::new(f);
f.lines()
.filter_map(Result::ok)
.map(|v| v.to_lowercase())
.collect()
}

View File

@ -0,0 +1 @@
mod wasitests;

View File

@ -0,0 +1,61 @@
macro_rules! assert_wasi_output {
($file:expr, $name:expr, $args:expr, $expected:expr) => {{
use wasmer_dev_utils::stdio::StdioCapturer;
use wasmer_runtime_core::{backend::Compiler, Func};
use wasmer_wasi::generate_import_object;
#[cfg(feature = "clif")]
fn get_compiler() -> impl Compiler {
use wasmer_clif_backend::CraneliftCompiler;
CraneliftCompiler::new()
}
#[cfg(feature = "llvm")]
fn get_compiler() -> impl Compiler {
compile_error!("LLVM compiler not supported right now");
}
#[cfg(feature = "singlepass")]
fn get_compiler() -> impl Compiler {
use wasmer_singlepass_backend::SinglePassCompiler;
SinglePassCompiler::new()
}
#[cfg(not(any(feature = "llvm", feature = "clif", feature = "singlepass")))]
fn get_compiler() -> impl Compiler {
compile_error!("compiler not specified, activate a compiler via features");
unreachable!();
}
let wasm_bytes = include_bytes!($file);
let module = wasmer_runtime_core::compile_with(&wasm_bytes[..], &get_compiler())
.expect("WASM can't be compiled");
let import_object = generate_import_object(vec![], vec![], vec![".".to_string()]);
let instance = module
.instantiate(&import_object)
.map_err(|err| format!("Can't instantiate the WebAssembly module: {:?}", err))
.unwrap(); // NOTE: Need to figure what the unwrap is for ??
let capturer = StdioCapturer::new();
let start: Func<(), ()> = instance
.func("_start")
.map_err(|e| format!("{:?}", e))
.expect("start function in wasi module");
let _result = start.call();
let output = capturer.end().unwrap().0;
let expected_output = include_str!($expected);
assert!(
output.contains(expected_output),
"Output: `{}` does not contain expected output: `{}`",
output,
expected_output
);
}};
}

View File

@ -0,0 +1,10 @@
#[test]
#[ignore]
fn test_create_dir() {
assert_wasi_output!(
"../../wasitests/create_dir.wasm",
"create_dir",
vec![],
"../../wasitests/create_dir.out"
);
}

View File

@ -0,0 +1,10 @@
#[test]
#[ignore]
fn test_file_metadata() {
assert_wasi_output!(
"../../wasitests/file_metadata.wasm",
"file_metadata",
vec![],
"../../wasitests/file_metadata.out"
);
}

View File

@ -0,0 +1,9 @@
#[test]
fn test_hello() {
assert_wasi_output!(
"../../wasitests/hello.wasm",
"hello",
vec![],
"../../wasitests/hello.out"
);
}

View File

@ -0,0 +1,11 @@
// !!! THIS IS A GENERATED FILE !!!
// ANY MANUAL EDITS MAY BE OVERWRITTEN AT ANY TIME
// Files autogenerated with cargo build (build/wasitests.rs).
// The _common module is not autogenerated. It provides common macros for the wasitests
#[macro_use]
mod _common;
mod create_dir;
mod file_metadata;
mod hello;
mod quine;

View File

@ -0,0 +1,9 @@
#[test]
fn test_quine() {
assert_wasi_output!(
"../../wasitests/quine.wasm",
"quine",
vec![],
"../../wasitests/quine.out"
);
}

BIN
lib/wasi/wasitests/create_dir Executable file

Binary file not shown.

View File

@ -0,0 +1,5 @@
Test file exists: false
Dir exists: false
Dir exists: false
Dir exists: false
Success

View File

@ -0,0 +1,37 @@
use std::fs;
use std::io::{Read, Seek, SeekFrom, Write};
use std::path::*;
fn main() {
let mut path = PathBuf::from("wasitests/testing/nested/directories");
let test_file = path.join("test.file");
fs::create_dir_all(&path).unwrap();
{
let mut file = fs::OpenOptions::new()
.read(true)
.write(true)
.truncate(true)
.create(true)
.open(&test_file)
.unwrap();
assert_eq!(file.write(b"hello").unwrap(), 5);
file.flush().unwrap();
file.seek(SeekFrom::Start(0)).unwrap();
let mut in_str = String::new();
file.read_to_string(&mut in_str).unwrap();
assert_eq!(&in_str, "hello");
}
fs::remove_file(&test_file).unwrap();
println!("Test file exists: {}", test_file.exists());
assert!(!test_file.exists());
for _ in 0..3 {
fs::remove_dir_all(&path).unwrap();
println!("Dir exists: {}", path.exists());
assert!(!path.exists());
path.pop();
}
println!("Success");
}

Binary file not shown.

BIN
lib/wasi/wasitests/file_metadata Executable file

Binary file not shown.

View File

@ -0,0 +1,2 @@
is dir: false
file info: FileType(FileType { mode: 33188 }) 419 Ok(SystemTime { tv_sec: 1558132188, tv_nsec: 545288295 }) Ok(SystemTime { tv_sec: 1558132188, tv_nsec: 545243056 }) Ok(SystemTime { tv_sec: 1558132191, tv_nsec: 359031112 })

View File

@ -0,0 +1,17 @@
use std::fs;
use std::io::Read;
fn main() {
let mut this_file =
fs::File::open("wasitests/file_metadata.rs").expect("could not find src file");
let md = this_file.metadata().unwrap();
println!("is dir: {}", md.is_dir());
println!(
"file info: {:?} {} {:?} {:?} {:?}",
md.file_type(),
md.len(),
md.modified(),
md.created(),
md.accessed()
);
}

Binary file not shown.

BIN
lib/wasi/wasitests/hello Executable file

Binary file not shown.

View File

@ -0,0 +1 @@
Hello, world!

View File

@ -0,0 +1,3 @@
fn main() {
println!("Hello, world!");
}

BIN
lib/wasi/wasitests/hello.wasm Executable file

Binary file not shown.

View File

@ -0,0 +1,2 @@
file_metadata
create_dir

BIN
lib/wasi/wasitests/quine Executable file

Binary file not shown.

View File

@ -0,0 +1,11 @@
use std::fs;
use std::io::Read;
fn main() {
let mut this_file = fs::File::open("wasitests/quine.rs").expect("could not find src file");
let md = this_file.metadata().unwrap();
let mut in_str = String::new();
this_file.read_to_string(&mut in_str).unwrap();
println!("{}", in_str);
}

View File

@ -0,0 +1,10 @@
use std::fs;
use std::io::Read;
fn main() {
let mut this_file = fs::File::open("wasitests/quine.rs").expect("could not find src file");
let md = this_file.metadata().unwrap();
let mut in_str = String::new();
this_file.read_to_string(&mut in_str).unwrap();
println!("{}", in_str);
}

BIN
lib/wasi/wasitests/quine.wasm Executable file

Binary file not shown.