#![allow(warnings, dead_code)] #[cfg(test)] mod tests { // TODO fix spec failures // TODO Add more assertions // TODO Allow running WAST &str directly (E.g. for use outside of spectests) // TODO Files could be run with multiple threads // TODO implment allowed failures in excludes // TODO fix NaN checking // TODO Fix Readme and remove old spectest // TODO consider submodule for spectests? // TODO cleanup refactor // TODO fix panics and remove panic handlers struct SpecFailure { file: String, line: u64, kind: String, message: String, } struct TestReport { failures: Vec, passed: u32, failed: u32, } impl TestReport { pub fn countPassed(&mut self) { self.passed += 1; } pub fn hasFailures(&self) -> bool { self.failed > 0 } pub fn addFailure(&mut self, failure: SpecFailure) { self.failed += 1; self.failures.push(failure); } } use wasmer_runtime_core::backend::Compiler; #[cfg(feature = "clif")] fn get_compiler() -> impl Compiler { use wasmer_clif_backend::CraneliftCompiler; CraneliftCompiler::new() } #[cfg(feature = "llvm")] fn get_compiler() -> impl Compiler { use wasmer_llvm_backend::LLVMCompiler; LLVMCompiler::new() } #[cfg(feature = "singlepass")] fn get_compiler() -> impl Compiler { use wasmer_singlepass_backend::SinglePassCompiler; SinglePassCompiler::new() } #[cfg(not(any(feature = "llvm", feature = "clif", feature = "singlepass")))] fn get_compiler() -> impl Compiler { panic!("compiler not specified, activate a compiler via features"); use wasmer_clif_backend::CraneliftCompiler; CraneliftCompiler::new() } #[cfg(feature = "clif")] fn get_compiler_name() -> &'static str { "clif" } #[cfg(feature = "llvm")] fn get_compiler_name() -> &'static str { "llvm" } #[cfg(feature = "singlepass")] fn get_compiler_name() -> &'static str { "singlepass" } #[cfg(not(any(feature = "llvm", feature = "clif", feature = "singlepass")))] fn get_compiler_name() -> &'static str { panic!("compiler not specified, activate a compiler via features"); "unknown" } use glob::glob; use std::collections::HashMap; use std::path::PathBuf; use std::{env, fs, io::Write}; use wabt::script::{Action, Command, CommandKind, ScriptParser, Value}; use wasmer_runtime_core::error::CompileError; use wasmer_runtime_core::import::ImportObject; use wasmer_runtime_core::Instance; use wasmer_runtime_core::{func, imports, vm::Ctx}; use wasmer_runtime_core::{ global::Global, memory::Memory, prelude::*, table::Table, types::{ElementType, MemoryDescriptor, TableDescriptor}, units::Pages, }; fn parse_and_run( path: &PathBuf, excludes: &HashMap, ) -> Result { let mut test_report = TestReport { failures: vec![], passed: 0, failed: 0, }; let source = fs::read(&path).unwrap(); let filename = path.file_name().unwrap().to_str().unwrap(); let source = fs::read(&path).unwrap(); let backend = get_compiler_name(); let star_key = format!("{}:{}:*", backend, filename); if excludes.contains_key(&star_key) { return Ok(test_report); } let mut features = wabt::Features::new(); features.enable_simd(); let mut parser: ScriptParser = ScriptParser::from_source_and_name_with_features(&source, filename, features) .expect(&format!("Failed to parse script {}", &filename)); use std::panic; let mut instance: Option = None; while let Some(Command { kind, line }) = parser.next().map_err(|e| format!("Parse err: {:?}", e))? { let test_key = format!("{}:{}:{}", backend, filename, line); // Use this line to debug which test is running println!("Running test: {}", test_key); if excludes.contains_key(&test_key) && *excludes.get(&test_key).unwrap() == Exclude::Skip { println!("Skipping test: {}", test_key); continue; } match kind { CommandKind::Module { module, name } => { // println!("Module"); let result = panic::catch_unwind(|| { let spectest_import_object = get_spectest_import_object(); let module = wasmer_runtime_core::compile_with(&module.into_vec(), &get_compiler()) .expect("WASM can't be compiled"); let i = module .instantiate(&spectest_import_object) .expect("WASM can't be instantiated"); i }); match result { Err(e) => { test_report.addFailure(SpecFailure { file: filename.to_string(), line: line, kind: format!("{:?}", "Module"), message: format!("caught panic {:?}", e), }); instance = None; } Ok(i) => { instance = Some(i); } } } CommandKind::AssertReturn { action, expected } => { match action { Action::Invoke { module, field, args, } => { if (&instance).is_none() { test_report.addFailure(SpecFailure { file: filename.to_string(), line: line, kind: format!("{:?}", "AssertReturn"), message: format!("No instance available"), }); } else { let params: Vec = args.iter().cloned().map(|x| convert_value(x)).collect(); let call_result = instance.as_ref().unwrap().call(&field, ¶ms[..]); match call_result { Err(e) => { test_report.addFailure(SpecFailure { file: filename.to_string(), line, kind: format!("{:?}", "AssertReturn"), message: format!("Call failed {:?}", e), }); } Ok(values) => { for (i, v) in values.iter().enumerate() { let expected_value = convert_value(*expected.get(i).unwrap()); if (*v != expected_value) { test_report.addFailure(SpecFailure { file: filename.to_string(), line, kind: format!("{:?}", "AssertReturn"), message: format!( "result {:?} ({:?}) does not match expected {:?} ({:?})", v, to_hex(v.clone()), expected_value, to_hex(expected_value.clone()) ), }); } else { test_report.countPassed(); } } } } } } _ => println!("unexpected action in assert return"), } // println!("in assert return"); } CommandKind::AssertReturnCanonicalNan { action } => match action { Action::Invoke { module, field, args, } => { if (&instance).is_none() { test_report.addFailure(SpecFailure { file: filename.to_string(), line: line, kind: format!("{:?}", "AssertReturnCanonicalNan"), message: format!("No instance available"), }); } else { let params: Vec = args.iter().cloned().map(|x| convert_value(x)).collect(); let call_result = instance.as_ref().unwrap().call(&field, ¶ms[..]); match call_result { Err(e) => { test_report.addFailure(SpecFailure { file: filename.to_string(), line, kind: format!("{:?}", "AssertReturnCanonicalNan"), message: format!("Call failed {:?}", e), }); } Ok(values) => { for (i, v) in values.iter().enumerate() { if is_canonical_nan(v.clone()) { test_report.countPassed(); } else { test_report.addFailure(SpecFailure { file: filename.to_string(), line, kind: format!("{:?}", "AssertReturnCanonicalNan"), message: format!( "value is not canonical nan {:?}", v ), }); } } } } } } _ => panic!("unexpected action in assert return canonical nan"), }, CommandKind::AssertReturnArithmeticNan { action } => match action { Action::Invoke { module, field, args, } => { if (&instance).is_none() { test_report.addFailure(SpecFailure { file: filename.to_string(), line: line, kind: format!("{:?}", "AssertReturnArithmeticNan"), message: format!("No instance available"), }); } else { let params: Vec = args.iter().cloned().map(|x| convert_value(x)).collect(); let call_result = instance.as_ref().unwrap().call(&field, ¶ms[..]); match call_result { Err(e) => { test_report.addFailure(SpecFailure { file: filename.to_string(), line, kind: format!("{:?}", "AssertReturnArithmeticNan"), message: format!("Call failed {:?}", e), }); } Ok(values) => { for (i, v) in values.iter().enumerate() { if is_arithmetic_nan(v.clone()) { test_report.countPassed(); } else { test_report.addFailure(SpecFailure { file: filename.to_string(), line, kind: format!("{:?}", "AssertReturnArithmeticNan"), message: format!( "value is not arithmetic nan {:?}", v ), }); } } } } } } _ => panic!("unexpected action in assert return arithmetic nan"), }, CommandKind::AssertTrap { action, message } => match action { Action::Invoke { module, field, args, } => { if (&instance).is_none() { test_report.addFailure(SpecFailure { file: filename.to_string(), line: line, kind: format!("{:?}", "AssertTrap"), message: format!("No instance available"), }); } else { let params: Vec = args.iter().cloned().map(|x| convert_value(x)).collect(); let call_result = instance.as_ref().unwrap().call(&field, ¶ms[..]); use wasmer_runtime_core::error::{CallError, RuntimeError}; match call_result { Err(e) => { match e { CallError::Resolve(_) => { test_report.addFailure(SpecFailure { file: filename.to_string(), line, kind: format!("{:?}", "AssertTrap"), message: format!("expected trap, got {:?}", e), }); } CallError::Runtime(r) => { match r { RuntimeError::Trap { .. } => { // TODO assert message? test_report.countPassed() } RuntimeError::Error { .. } => { test_report.addFailure(SpecFailure { file: filename.to_string(), line, kind: format!("{:?}", "AssertTrap"), message: format!( "expected trap, got Runtime:Error {:?}", r ), }); } } } } } Ok(values) => { test_report.addFailure(SpecFailure { file: filename.to_string(), line, kind: format!("{:?}", "AssertTrap"), message: format!("expected trap, got {:?}", values), }); } } } } _ => println!("unexpected action"), }, CommandKind::AssertInvalid { module, message } => { // println!("AssertInvalid"); let result = panic::catch_unwind(|| { wasmer_runtime_core::compile_with(&module.into_vec(), &get_compiler()) }); match result { Ok(module) => { if let Err(CompileError::InternalError { msg }) = module { test_report.countPassed(); // println!("expected: {:?}", message); // println!("actual: {:?}", msg); } else if let Err(CompileError::ValidationError { msg }) = module { test_report.countPassed(); // println!("validation expected: {:?}", message); // println!("validation actual: {:?}", msg); } else { test_report.addFailure(SpecFailure { file: filename.to_string(), line: line, kind: format!("{:?}", "AssertInvalid"), message: "Should be invalid".to_string(), }); } } Err(p) => { test_report.addFailure(SpecFailure { file: filename.to_string(), line: line, kind: format!("{:?}", "AssertInvalid"), message: format!("caught panic {:?}", p), }); } } } CommandKind::AssertMalformed { module, message } => { // println!("AssertMalformed"); let result = panic::catch_unwind(|| { wasmer_runtime_core::compile_with(&module.into_vec(), &get_compiler()) }); match result { Ok(module) => { if let Err(CompileError::InternalError { msg }) = module { test_report.countPassed(); // println!("expected: {:?}", message); // println!("actual: {:?}", msg); } else if let Err(CompileError::ValidationError { msg }) = module { test_report.countPassed(); // println!("validation expected: {:?}", message); // println!("validation actual: {:?}", msg); } else { test_report.addFailure(SpecFailure { file: filename.to_string(), line: line, kind: format!("{:?}", "AssertMalformed"), message: format!("should be malformed"), }); } } Err(p) => { test_report.addFailure(SpecFailure { file: filename.to_string(), line: line, kind: format!("{:?}", "AssertMalformed"), message: format!("caught panic {:?}", p), }); } } } CommandKind::AssertUninstantiable { module, message } => { println!("AssertUninstantiable not yet implmented") } CommandKind::AssertExhaustion { action, message } => { println!("AssertExhaustion not yet implemented") } CommandKind::AssertUnlinkable { module, message } => { println!("AssertUnlinkable {:? }{:?}", filename, line); let result = panic::catch_unwind(|| { let module = wasmer_runtime_core::compile_with(&module.into_vec(), &get_compiler()) .expect("WASM can't be compiled"); module.instantiate(&ImportObject::new()) }); match result { Err(e) => { test_report.addFailure(SpecFailure { file: filename.to_string(), line: line, kind: format!("{:?}", "AssertUnlinkable"), message: format!("caught panic {:?}", e), }); } Ok(result) => match result { Ok(_) => { test_report.addFailure(SpecFailure { file: filename.to_string(), line: line, kind: format!("{:?}", "AssertUnlinkable"), message: format!("instantiate successful, expected unlinkable"), }); } Err(e) => match e { wasmer_runtime_core::error::Error::LinkError(_) => { test_report.countPassed(); } _ => { test_report.addFailure(SpecFailure { file: filename.to_string(), line: line, kind: format!("{:?}", "AssertUnlinkable"), message: format!("expected link error, got {:?}", e), }); } }, }, } println!("AssertUnlinkable Done"); } CommandKind::Register { name, as_name } => { println!("Register not implemented {:?} {:?}", filename, line) } CommandKind::PerformAction(ref action) => match action { Action::Invoke { module, field, args, } => { println!("PerformAction: {:?} {:?}", filename, line); if module.is_some() { panic!("unexpected module in invoke"); } else { if (&instance).is_none() { test_report.addFailure(SpecFailure { file: filename.to_string(), line: line, kind: format!("{:?}", "PerformAction"), message: format!("No instance available"), }); } else { let params: Vec = args.iter().cloned().map(|x| convert_value(x)).collect(); let call_result = instance.as_ref().unwrap().call(&field, ¶ms[..]); match call_result { Err(e) => { test_report.addFailure(SpecFailure { file: filename.to_string(), line, kind: format!("{:?}", "PerformAction"), message: format!("Call failed {:?}", e), }); } Ok(values) => { test_report.countPassed(); } } } } } Action::Get { module, field } => { println!("Action Get not implemented {:?} {:?}", filename, line) } }, _ => panic!("unknown wast command"), } } Ok(test_report) } fn is_canonical_nan(val: wasmer_runtime_core::types::Value) -> bool { match val { wasmer_runtime_core::types::Value::F32(x) => x.is_canonical_nan(), wasmer_runtime_core::types::Value::F64(x) => x.is_canonical_nan(), _ => panic!("value is not a float {:?}", val), } } fn is_arithmetic_nan(val: wasmer_runtime_core::types::Value) -> bool { match val { wasmer_runtime_core::types::Value::F32(x) => x.is_quiet_nan(), wasmer_runtime_core::types::Value::F64(x) => x.is_quiet_nan(), _ => panic!("value is not a float {:?}", val), } } fn convert_value(other: Value) -> wasmer_runtime_core::types::Value { match other { Value::I32(v) => wasmer_runtime_core::types::Value::I32(v), Value::I64(v) => wasmer_runtime_core::types::Value::I64(v), Value::F32(v) => wasmer_runtime_core::types::Value::F32(v), Value::F64(v) => wasmer_runtime_core::types::Value::F64(v), Value::V128(v) => wasmer_runtime_core::types::Value::V128(v), } } fn to_hex(v: wasmer_runtime_core::types::Value) -> String { match v { wasmer_runtime_core::types::Value::I32(v) => format!("{:#x}", v), wasmer_runtime_core::types::Value::I64(v) => format!("{:#x}", v), wasmer_runtime_core::types::Value::F32(v) => format!("{:#x}", v.to_bits()), wasmer_runtime_core::types::Value::F64(v) => format!("{:#x}", v.to_bits()), wasmer_runtime_core::types::Value::V128(v) => format!("{:#x}", v), } } fn print_i32(ctx: &mut Ctx, val: i32) { println!("{}", val); } fn get_spectest_import_object() -> ImportObject { let memory = Memory::new(MemoryDescriptor { minimum: Pages(1), maximum: Some(Pages(1)), shared: false, }) .unwrap(); let global_i32 = Global::new(wasmer_runtime_core::types::Value::I32(666)); let global_f32 = Global::new(wasmer_runtime_core::types::Value::F32(666.0)); let global_f64 = Global::new(wasmer_runtime_core::types::Value::F64(666.0)); let table = Table::new(TableDescriptor { element: ElementType::Anyfunc, minimum: 10, maximum: Some(20), }) .unwrap(); imports! { "spectest" => { "print_i32" => func!(print_i32), "table" => table, "global_i32" => global_i32, "global_f32" => global_f32, "global_f64" => global_f64, }, } } #[derive(Debug, Copy, Clone, PartialEq, Eq)] enum Exclude { Skip, Fail, } use std::fs::File; use std::io::{BufRead, BufReader, Error}; /// Reads the excludes.txt file into a hash map fn read_excludes() -> HashMap { let mut excludes_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); excludes_path.push("tests"); excludes_path.push("excludes.txt"); let input = File::open(excludes_path).unwrap(); let buffered = BufReader::new(input); let mut result = HashMap::new(); for line in buffered.lines() { let mut line = line.unwrap(); if line.trim().is_empty() || line.starts_with("#") { // ignore line } else { if line.contains("#") { // Allow end of line comment let l: Vec<&str> = line.split('#').collect(); line = l.get(0).unwrap().to_string(); } //println!("exclude line {}", line); // ::: let split: Vec<&str> = line.trim().split(':').collect(); assert_eq!(split.len(), 4); let kind = match *split.get(1).unwrap() { "skip" => Exclude::Skip, "fail" => Exclude::Fail, _ => panic!("unknown exclude kind"), }; let backend = split.get(0).unwrap(); let testfile = split.get(2).unwrap(); let line = split.get(3).unwrap(); let key = format!("{}:{}:{}", backend, testfile, line); result.insert(key, kind); } } result } #[test] fn test_run_spectests() { let mut success = true; let mut test_reports = vec![]; let excludes = read_excludes(); let mut glob_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); glob_path.push("spectests"); glob_path.push("*.wast"); let glob_str = glob_path.to_str().unwrap(); for entry in glob(glob_str).expect("Failed to read glob pattern") { match entry { Ok(wast_path) => { let result = parse_and_run(&wast_path, &excludes); match result { Ok(test_report) => { if test_report.hasFailures() { success = false } test_reports.push(test_report); } Err(e) => { success = false; println!("Unexpected test run error: {:?}", e) } } } Err(e) => panic!("glob err: {:?}", e), } } // Print summary let mut failures = vec![]; let mut total_passed = 0; let mut total_failed = 0; for mut test_report in test_reports.into_iter() { total_passed += test_report.passed; total_failed += test_report.failed; failures.append(&mut test_report.failures); } println!(""); println!(""); println!("Spec tests summary report: "); println!("total: {}", total_passed + total_failed); println!("passed: {}", total_passed); println!("failed: {}", total_failed); for failure in failures.iter() { println!( " {:?} {:?} {:?} {:?}", failure.file, failure.line, failure.kind, failure.message ); } println!(""); println!(""); assert!(success, "tests passed") } /// Bit pattern of an f32 value: /// 1-bit sign + 8-bit mantissa + 23-bit exponent = 32 bits /// /// Bit pattern of an f64 value: /// 1-bit sign + 11-bit mantissa + 52-bit exponent = 64 bits /// /// NOTE: On some old platforms (PA-RISC, some MIPS) quiet NaNs (qNaN) have /// their mantissa MSB unset and set for signaling NaNs (sNaN). /// /// Links: /// * https://en.wikipedia.org/wiki/Floating-point_arithmetic /// * https://github.com/WebAssembly/spec/issues/286 /// * https://en.wikipedia.org/wiki/NaN /// pub trait NaNCheck { fn is_quiet_nan(&self) -> bool; fn is_canonical_nan(&self) -> bool; } impl NaNCheck for f32 { /// The MSB of the mantissa must be set for a NaN to be a quiet NaN. fn is_quiet_nan(&self) -> bool { let bit_mask = 0b1 << 22; // Used to check if 23rd bit is set, which is MSB of the mantissa self.is_nan() && (self.to_bits() & bit_mask) == bit_mask } /// For a NaN to be canonical, its mantissa bits must all be unset fn is_canonical_nan(&self) -> bool { let bit_mask: u32 = 0b1____0000_0000____011_1111_1111_1111_1111_1111; let masked_value = self.to_bits() ^ bit_mask; masked_value == 0xFFFF_FFFF || masked_value == 0x7FFF_FFFF } } impl NaNCheck for f64 { /// The MSB of the mantissa must be set for a NaN to be a quiet NaN. fn is_quiet_nan(&self) -> bool { let bit_mask = 0b1 << 51; // Used to check if 52st bit is set, which is MSB of the mantissa self.is_nan() && (self.to_bits() & bit_mask) == bit_mask } /// For a NaN to be canonical, its mantissa bits must all be unset fn is_canonical_nan(&self) -> bool { let bit_mask: u64 = 0b1____000_0000_0000____0111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111; let masked_value = self.to_bits() ^ bit_mask; masked_value == 0x7FFF_FFFF_FFFF_FFFF || masked_value == 0xFFF_FFFF_FFFF_FFFF } } }