fluencelabs/wasmer
1
0
Fork 0
mirror of https://github.com/fluencelabs/wasmer synced 2024-12-14 14:45:40 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #985 from wasmerio/feature/update-spectest-excludes

Browse Source 
Refactor spectests runner
This commit is contained in:
Syrus Akbary 2019-11-20 08:07:13 -08:00 committed by GitHub
commit e466e487dd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 209 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lib/spectests/tests/excludes.txt
View File

@ -12,10 +12,12 @@
# Star line allows skipping an entire wast file # Star line allows skipping an entire wast file
# clif:skip:simd.wast:* # clif:skip:simd.wast:*
# #
# Excludes can also contain platform # Excludes can also contain target family
# clif:skip:data.wast:172:windows # clif:skip:data.wast:172:windows
# clif:skip:data.wast:172:unix # clif:skip:data.wast:172:unix
# #
# Or target arch
# singlepass:skip:atomic.wast:*:*:aarch64
# Cranelift # Cranelift
clif:skip:atomic.wast:* # Threads not implemented clif:skip:atomic.wast:* # Threads not implemented

252
lib/spectests/tests/spectest.rs
View File

@ -18,6 +18,7 @@ mod tests {
// TODO Files could be run with multiple threads // TODO Files could be run with multiple threads
// TODO Allow running WAST &str directly (E.g. for use outside of spectests) // TODO Allow running WAST &str directly (E.g. for use outside of spectests)
use std::collections::HashSet;
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
struct SpecFailure { struct SpecFailure {
@ -46,15 +47,14 @@ mod tests {
pub fn add_failure( pub fn add_failure(
&mut self, &mut self,
failure: SpecFailure, failure: SpecFailure,
testkey: &str, _testkey: &str,
excludes: &HashMap<String, Exclude>, excludes: &Vec<Exclude>,
line: u64,
) { ) {
if excludes.contains_key(testkey) { if excludes
self.allowed_failure += 1; .iter()
return; .any(|e| e.line_matches(line) && e.exclude_kind == ExcludeKind::Fail)
} {
let platform_key = format!("{}:{}", testkey, get_platform());
if excludes.contains_key(&platform_key) {
self.allowed_failure += 1; self.allowed_failure += 1;
return; return;
} }
@ -104,15 +104,113 @@ mod tests {
} }
#[cfg(unix)] #[cfg(unix)]
fn get_platform() -> &'static str { fn get_target_family() -> &'static str {
"unix" "unix"
} }
#[cfg(windows)] #[cfg(windows)]
fn get_platform() -> &'static str { fn get_target_family() -> &'static str {
"windows" "windows"
} }
fn get_target_arch() -> &'static str {
if cfg!(target_arch = "x86_64") {
"x86_64"
} else if cfg!(target_arch = "aarch64") {
"aarch64"
} else if cfg!(target_arch = "x86") {
"x86"
} else if cfg!(target_arch = "mips") {
"mips"
} else if cfg!(target_arch = "powerpc") {
"powerpc"
} else if cfg!(target_arch = "powerpc64") {
"powerpc64"
} else if cfg!(target_arch = "arm") {
"arm"
} else {
panic!("unknown target arch")
}
}
// clif:skip:data.wast:172:unix:x86
#[allow(dead_code)]
struct Exclude {
backend: Option<String>,
exclude_kind: ExcludeKind,
file: String,
line: Option<u64>,
target_family: Option<String>,
target_arch: Option<String>,
}
impl Exclude {
fn line_matches(&self, value: u64) -> bool {
self.line.is_none() || self.line.unwrap() == value
}
fn line_exact_match(&self, value: u64) -> bool {
self.line.is_some() && self.line.unwrap() == value
}
fn matches_backend(&self, value: &str) -> bool {
self.backend.is_none() || self.backend.as_ref().unwrap() == value
}
fn matches_target_family(&self, value: &str) -> bool {
self.target_family.is_none() || self.target_family.as_ref().unwrap() == value
}
fn matches_target_arch(&self, value: &str) -> bool {
self.target_arch.is_none() || self.target_arch.as_ref().unwrap() == value
}
fn from(
backend: &str,
exclude_kind: &str,
file: &str,
line: &str,
target_family: &str,
target_arch: &str,
) -> Exclude {
let backend: Option<String> = match backend {
"*" => None,
"clif" => Some("clif".to_string()),
"singlepass" => Some("singlepass".to_string()),
"llvm" => Some("llvm".to_string()),
_ => panic!("backend {:?} not recognized", backend),
};
let exclude_kind = match exclude_kind {
"skip" => ExcludeKind::Skip,
"fail" => ExcludeKind::Fail,
_ => panic!("exclude kind {:?} not recognized", exclude_kind),
};
let line = match line {
"*" => None,
_ => Some(
line.parse::<u64>()
.expect(&format!("expected * or int: {:?}", line)),
),
};
let target_family = match target_family {
"*" => None,
_ => Some(target_family.to_string()),
};
let target_arch = match target_arch {
"*" => None,
_ => Some(target_arch.to_string()),
};
Exclude {
backend,
exclude_kind,
file: file.to_string(),
line,
target_family,
target_arch,
}
}
}
#[cfg(not(any(feature = "llvm", feature = "clif", feature = "singlepass")))] #[cfg(not(any(feature = "llvm", feature = "clif", feature = "singlepass")))]
fn get_compiler_name() -> &'static str { fn get_compiler_name() -> &'static str {
panic!("compiler not specified, activate a compiler via features"); panic!("compiler not specified, activate a compiler via features");
@ -160,7 +258,8 @@ mod tests {
fn parse_and_run( fn parse_and_run(
path: &PathBuf, path: &PathBuf,
excludes: &HashMap<String, Exclude>, file_excludes: &HashSet<String>,
excludes: &HashMap<String, Vec<Exclude>>,
) -> Result<TestReport, String> { ) -> Result<TestReport, String> {
let mut test_report = TestReport { let mut test_report = TestReport {
failures: vec![], failures: vec![],
@ -171,15 +270,9 @@ mod tests {
let filename = path.file_name().unwrap().to_str().unwrap(); let filename = path.file_name().unwrap().to_str().unwrap();
let source = fs::read(&path).unwrap(); let source = fs::read(&path).unwrap();
let backend = get_compiler_name();
let platform = get_platform(); // Entire file is excluded by line * and skip
let star_key = format!("{}:{}:*", backend, filename); if file_excludes.contains(filename) {
let platform_star_key = format!("{}:{}:*:{}", backend, filename, platform);
if (excludes.contains_key(&star_key) && *excludes.get(&star_key).unwrap() == Exclude::Skip)
|| (excludes.contains_key(&platform_star_key)
&& *excludes.get(&platform_star_key).unwrap() == Exclude::Skip)
{
return Ok(test_report); return Ok(test_report);
} }
@ -198,21 +291,27 @@ mod tests {
let mut registered_modules: HashMap<String, Arc<Mutex<Instance>>> = HashMap::new(); let mut registered_modules: HashMap<String, Arc<Mutex<Instance>>> = HashMap::new();
// //
let empty_excludes = vec![];
let excludes = if excludes.contains_key(filename) {
excludes.get(filename).unwrap()
} else {
&empty_excludes
};
let backend = get_compiler_name();
while let Some(Command { kind, line }) = while let Some(Command { kind, line }) =
parser.next().map_err(|e| format!("Parse err: {:?}", e))? parser.next().map_err(|e| format!("Parse err: {:?}", e))?
{ {
let test_key = format!("{}:{}:{}", backend, filename, line); let test_key = format!("{}:{}:{}", backend, filename, line);
let test_platform_key = format!("{}:{}:{}:{}", backend, filename, line, platform);
// Use this line to debug which test is running // Use this line to debug which test is running
println!("Running test: {}", test_key); println!("Running test: {}", test_key);
if (excludes.contains_key(&test_key) // Skip tests that match this line
&& *excludes.get(&test_key).unwrap() == Exclude::Skip) if excludes
|| (excludes.contains_key(&test_platform_key) .iter()
&& *excludes.get(&test_platform_key).unwrap() == Exclude::Skip) .any(|e| e.line_exact_match(line) && e.exclude_kind == ExcludeKind::Skip)
{ {
// println!("Skipping test: {}", test_key);
continue; continue;
} }
@ -251,6 +350,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
instance = None; instance = None;
} }
@ -290,6 +390,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} else { } else {
let call_result = maybe_call_result.unwrap(); let call_result = maybe_call_result.unwrap();
@ -304,6 +405,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} }
Ok(values) => { Ok(values) => {
@ -320,7 +422,7 @@ mod tests {
"result {:?} ({:?}) does not match expected {:?} ({:?})", "result {:?} ({:?}) does not match expected {:?} ({:?})",
v, to_hex(v.clone()), expected_value, to_hex(expected_value.clone()) v, to_hex(v.clone()), expected_value, to_hex(expected_value.clone())
), ),
}, &test_key, excludes); }, &test_key, excludes, line);
} else { } else {
test_report.count_passed(); test_report.count_passed();
} }
@ -350,6 +452,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} else { } else {
let export: Export = maybe_call_result.unwrap(); let export: Export = maybe_call_result.unwrap();
@ -373,6 +476,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} }
} }
@ -386,6 +490,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} }
} }
@ -416,6 +521,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} else { } else {
let call_result = maybe_call_result.unwrap(); let call_result = maybe_call_result.unwrap();
@ -430,6 +536,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} }
Ok(values) => { Ok(values) => {
@ -453,6 +560,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} }
} }
@ -484,6 +592,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} else { } else {
let call_result = maybe_call_result.unwrap(); let call_result = maybe_call_result.unwrap();
@ -498,6 +607,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} }
Ok(values) => { Ok(values) => {
@ -521,6 +631,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} }
} }
@ -552,6 +663,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} else { } else {
let call_result = maybe_call_result.unwrap(); let call_result = maybe_call_result.unwrap();
@ -569,6 +681,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} }
CallError::Runtime(r) => { CallError::Runtime(r) => {
@ -590,6 +703,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} }
} }
@ -606,6 +720,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} }
} }
@ -649,6 +764,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} }
} }
@ -662,6 +778,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} }
} }
@ -704,6 +821,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} }
} }
@ -717,6 +835,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} }
} }
@ -755,6 +874,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} }
}; };
@ -786,6 +906,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} else { } else {
let call_result = maybe_call_result.unwrap(); let call_result = maybe_call_result.unwrap();
@ -807,6 +928,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} }
} }
@ -845,6 +967,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} }
Ok(result) => match result { Ok(result) => match result {
@ -860,6 +983,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} }
Err(e) => match e { Err(e) => match e {
@ -876,6 +1000,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} }
}, },
@ -909,6 +1034,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} }
} }
@ -934,6 +1060,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} else { } else {
let call_result = maybe_call_result.unwrap(); let call_result = maybe_call_result.unwrap();
@ -948,6 +1075,7 @@ mod tests {
}, },
&test_key, &test_key,
excludes, excludes,
line,
); );
} }
Ok(_values) => { Ok(_values) => {
@ -1105,7 +1233,7 @@ mod tests {
} }
#[derive(Debug, Copy, Clone, PartialEq, Eq)] #[derive(Debug, Copy, Clone, PartialEq, Eq)]
enum Exclude { enum ExcludeKind {
Skip, Skip,
Fail, Fail,
} }
@ -1115,13 +1243,18 @@ mod tests {
use std::io::{BufRead, BufReader}; use std::io::{BufRead, BufReader};
/// Reads the excludes.txt file into a hash map /// Reads the excludes.txt file into a hash map
fn read_excludes() -> HashMap<String, Exclude> { fn read_excludes() -> (HashMap<String, Vec<Exclude>>, HashSet<String>) {
let mut excludes_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); let mut excludes_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
excludes_path.push("tests"); excludes_path.push("tests");
excludes_path.push("excludes.txt"); excludes_path.push("excludes.txt");
let input = File::open(excludes_path).unwrap(); let input = File::open(excludes_path).unwrap();
let buffered = BufReader::new(input); let buffered = BufReader::new(input);
let mut result = HashMap::new(); let mut result = HashMap::new();
let mut file_excludes = HashSet::new();
let current_backend = get_compiler_name();
let current_target_family = get_target_family();
let current_target_arch = get_target_arch();
for line in buffered.lines() { for line in buffered.lines() {
let mut line = line.unwrap(); let mut line = line.unwrap();
if line.trim().is_empty() || line.starts_with("#") { if line.trim().is_empty() || line.starts_with("#") {
@ -1136,26 +1269,53 @@ mod tests {
// <backend>:<exclude-kind>:<test-file-name>:<test-file-line> // <backend>:<exclude-kind>:<test-file-name>:<test-file-line>
let split: Vec<&str> = line.trim().split(':').collect(); let split: Vec<&str> = line.trim().split(':').collect();
let kind = match *split.get(1).unwrap() { let file = *split.get(2).unwrap();
"skip" => Exclude::Skip, let exclude = match split.len() {
"fail" => Exclude::Fail, 0..=3 => panic!("expected at least 4 exclude conditions"),
_ => panic!("unknown exclude kind"), 4 => Exclude::from(
*split.get(0).unwrap(),
*split.get(1).unwrap(),
*split.get(2).unwrap(),
*split.get(3).unwrap(),
"*",
"*",
),
5 => Exclude::from(
*split.get(0).unwrap(),
*split.get(1).unwrap(),
*split.get(2).unwrap(),
*split.get(3).unwrap(),
*split.get(4).unwrap(),
"*",
),
6 => Exclude::from(
*split.get(0).unwrap(),
*split.get(1).unwrap(),
*split.get(2).unwrap(),
*split.get(3).unwrap(),
*split.get(4).unwrap(),
*split.get(5).unwrap(),
),
_ => panic!("too many exclude conditions {}", split.len()),
}; };
let has_platform = split.len() > 4;
let backend = split.get(0).unwrap(); if exclude.matches_backend(current_backend)
let testfile = split.get(2).unwrap(); && exclude.matches_target_family(current_target_family)
let line = split.get(3).unwrap(); && exclude.matches_target_arch(current_target_arch)
let key = if has_platform { {
let platform = split.get(4).unwrap(); // Skip the whole file for line * and skip
format!("{}:{}:{}:{}", backend, testfile, line, platform) if exclude.line.is_none() && exclude.exclude_kind == ExcludeKind::Skip {
} else { file_excludes.insert(file.to_string());
format!("{}:{}:{}", backend, testfile, line) }
};
result.insert(key, kind); if !result.contains_key(file) {
result.insert(file.to_string(), vec![]);
}
result.get_mut(file).unwrap().push(exclude);
}
} }
} }
result (result, file_excludes)
} }
#[test] #[test]
@ -1163,7 +1323,7 @@ mod tests {
let mut success = true; let mut success = true;
let mut test_reports = vec![]; let mut test_reports = vec![];
let excludes = read_excludes(); let (excludes, file_excludes) = read_excludes();
let mut glob_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); let mut glob_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
glob_path.push("spectests"); glob_path.push("spectests");
@ -1173,7 +1333,7 @@ mod tests {
for entry in glob(glob_str).expect("Failed to read glob pattern") { for entry in glob(glob_str).expect("Failed to read glob pattern") {
match entry { match entry {
Ok(wast_path) => { Ok(wast_path) => {
let result = parse_and_run(&wast_path, &excludes); let result = parse_and_run(&wast_path, &file_excludes, &excludes);
match result { match result {
Ok(test_report) => { Ok(test_report) => {
if test_report.has_failures() { if test_report.has_failures() {