From 696fd8fce35173fca4c3a6d4cd829b256cff10fb Mon Sep 17 00:00:00 2001 From: Lachlan Sneff Date: Thu, 21 Feb 2019 17:06:49 -0800 Subject: [PATCH] Update api again and change SerializedCache to Artifact --- Cargo.lock | 2 +- lib/clif-backend/src/cache.rs | 4 +- lib/clif-backend/src/lib.rs | 8 +-- lib/clif-backend/src/module.rs | 4 +- lib/runtime-core/Cargo.toml | 2 - lib/runtime-core/src/backend.rs | 8 +-- lib/runtime-core/src/cache.rs | 122 ++++++++++++++------------------ lib/runtime-core/src/lib.rs | 4 +- lib/runtime-core/src/module.rs | 6 +- lib/runtime/Cargo.toml | 1 + lib/runtime/src/cache.rs | 20 ++++-- 11 files changed, 84 insertions(+), 97 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ebfe7ac0d..aa14e70be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1101,6 +1101,7 @@ name = "wasmer-runtime" version = "0.1.4" dependencies = [ "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memmap 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "wasmer-clif-backend 0.1.2", "wasmer-runtime-core 0.1.2", ] @@ -1126,7 +1127,6 @@ dependencies = [ "indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", - "memmap 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "nix 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", "page_size 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/lib/clif-backend/src/cache.rs b/lib/clif-backend/src/cache.rs index 3c38a5266..5af708703 100644 --- a/lib/clif-backend/src/cache.rs +++ b/lib/clif-backend/src/cache.rs @@ -4,7 +4,7 @@ use hashbrown::HashMap; use std::sync::Arc; use wasmer_runtime_core::{ backend::{sys::Memory, CacheGen}, - cache::{Error, SerializedCache}, + cache::{Artifact, Error}, module::{ModuleInfo, ModuleInner}, structures::Map, types::{LocalFuncIndex, SigIndex}, @@ -61,7 +61,7 @@ pub struct BackendCache { } impl BackendCache { - pub fn from_cache(cache: SerializedCache) -> Result<(ModuleInfo, Memory, Self), Error> { + pub fn from_cache(cache: Artifact) -> Result<(ModuleInfo, Memory, Self), Error> { let (info, backend_data, compiled_code) = cache.consume(); let backend_cache = diff --git a/lib/clif-backend/src/lib.rs b/lib/clif-backend/src/lib.rs index daef34c5d..df2e4e208 100644 --- a/lib/clif-backend/src/lib.rs +++ b/lib/clif-backend/src/lib.rs @@ -14,7 +14,7 @@ use cranelift_codegen::{ }; use target_lexicon::Triple; -use wasmer_runtime_core::cache::{Error as CacheError, SerializedCache}; +use wasmer_runtime_core::cache::{Artifact, Error as CacheError}; use wasmer_runtime_core::{ backend::{Compiler, Token}, error::{CompileError, CompileResult}, @@ -53,11 +53,7 @@ impl Compiler for CraneliftCompiler { /// Create a wasmer Module from an already-compiled cache. - unsafe fn from_cache( - &self, - cache: SerializedCache, - _: Token, - ) -> Result { + unsafe fn from_cache(&self, cache: Artifact, _: Token) -> Result { module::Module::from_cache(cache) } diff --git a/lib/clif-backend/src/module.rs b/lib/clif-backend/src/module.rs index 002d5185c..6f95ea828 100644 --- a/lib/clif-backend/src/module.rs +++ b/lib/clif-backend/src/module.rs @@ -7,7 +7,7 @@ use cranelift_wasm; use hashbrown::HashMap; use std::sync::Arc; -use wasmer_runtime_core::cache::{Error as CacheError, SerializedCache, WasmHash}; +use wasmer_runtime_core::cache::{Artifact, Error as CacheError, WasmHash}; use wasmer_runtime_core::{ backend::Backend, @@ -88,7 +88,7 @@ impl Module { }) } - pub fn from_cache(cache: SerializedCache) -> Result { + pub fn from_cache(cache: Artifact) -> Result { let (info, compiled_code, backend_cache) = BackendCache::from_cache(cache)?; let (func_resolver_builder, trampolines, handler_data) = diff --git a/lib/runtime-core/Cargo.toml b/lib/runtime-core/Cargo.toml index 75e49840c..03d4e5ae6 100644 --- a/lib/runtime-core/Cargo.toml +++ b/lib/runtime-core/Cargo.toml @@ -28,8 +28,6 @@ version = "1.0" version = "0.10" [dependencies.serde-bench] version = "0.0.7" -[dependencies.memmap] -version = "0.7.0" [dependencies.sha2] version = "0.8.0" [dependencies.hashbrown] diff --git a/lib/runtime-core/src/backend.rs b/lib/runtime-core/src/backend.rs index ab7c27d53..e88383800 100644 --- a/lib/runtime-core/src/backend.rs +++ b/lib/runtime-core/src/backend.rs @@ -8,7 +8,7 @@ use crate::{ }; use crate::{ - cache::{Error as CacheError, SerializedCache}, + cache::{Artifact, Error as CacheError}, module::ModuleInfo, sys::Memory, }; @@ -42,11 +42,7 @@ pub trait Compiler { /// be called from inside the runtime. fn compile(&self, wasm: &[u8], _: Token) -> CompileResult; - unsafe fn from_cache( - &self, - cache: SerializedCache, - _: Token, - ) -> Result; + unsafe fn from_cache(&self, cache: Artifact, _: Token) -> Result; } /// The functionality exposed by this trait is expected to be used diff --git a/lib/runtime-core/src/cache.rs b/lib/runtime-core/src/cache.rs index a58cc05f1..3d894bb09 100644 --- a/lib/runtime-core/src/cache.rs +++ b/lib/runtime-core/src/cache.rs @@ -2,16 +2,8 @@ use crate::{ module::{Module, ModuleInfo}, sys::Memory, }; -use memmap::Mmap; -use serde_bench::{deserialize, serialize}; use sha2::{Digest, Sha256}; -use std::{ - fs::File, - io::{self, Seek, SeekFrom, Write}, - mem, - path::Path, - slice, -}; +use std::{io, mem, slice}; #[derive(Debug)] pub enum InvalidFileType { @@ -67,23 +59,43 @@ impl WasmHash { } const CURRENT_CACHE_VERSION: u64 = 0; +static WASMER_CACHE_MAGIC: [u8; 8] = *b"WASMER\0\0"; /// The header of a cache file. #[repr(C, packed)] -struct SerializedCacheHeader { +struct ArtifactHeader { magic: [u8; 8], // [W, A, S, M, E, R, \0, \0] version: u64, data_len: u64, wasm_hash: [u8; 32], // Sha256 of the wasm in binary format. } -impl SerializedCacheHeader { +impl ArtifactHeader { pub fn read_from_slice(buffer: &[u8]) -> Result<(&Self, &[u8]), Error> { - if buffer.len() >= mem::size_of::() { - if &buffer[..8] == "WASMER\0\0".as_bytes() { + if buffer.len() >= mem::size_of::() { + if &buffer[..8] == &WASMER_CACHE_MAGIC { + let (header_slice, body_slice) = buffer.split_at(mem::size_of::()); + let header = unsafe { &*(header_slice.as_ptr() as *const ArtifactHeader) }; + + if header.version == CURRENT_CACHE_VERSION { + Ok((header, body_slice)) + } else { + Err(Error::InvalidatedCache) + } + } else { + Err(Error::InvalidFile(InvalidFileType::InvalidMagic)) + } + } else { + Err(Error::InvalidFile(InvalidFileType::InvalidSize)) + } + } + + pub fn read_from_slice_mut(buffer: &mut [u8]) -> Result<(&mut Self, &mut [u8]), Error> { + if buffer.len() >= mem::size_of::() { + if &buffer[..8] == &WASMER_CACHE_MAGIC { let (header_slice, body_slice) = - buffer.split_at(mem::size_of::()); - let header = unsafe { &*(header_slice.as_ptr() as *const SerializedCacheHeader) }; + buffer.split_at_mut(mem::size_of::()); + let header = unsafe { &mut *(header_slice.as_ptr() as *mut ArtifactHeader) }; if header.version == CURRENT_CACHE_VERSION { Ok((header, body_slice)) @@ -99,31 +111,31 @@ impl SerializedCacheHeader { } pub fn as_slice(&self) -> &[u8] { - let ptr = self as *const SerializedCacheHeader as *const u8; - unsafe { slice::from_raw_parts(ptr, mem::size_of::()) } + let ptr = self as *const ArtifactHeader as *const u8; + unsafe { slice::from_raw_parts(ptr, mem::size_of::()) } } } #[derive(Serialize, Deserialize)] -struct SerializedCacheInner { +struct ArtifactInner { info: Box, #[serde(with = "serde_bytes")] backend_metadata: Box<[u8]>, compiled_code: Memory, } -pub struct SerializedCache { - inner: SerializedCacheInner, +pub struct Artifact { + inner: ArtifactInner, } -impl SerializedCache { +impl Artifact { pub(crate) fn from_parts( info: Box, backend_metadata: Box<[u8]>, compiled_code: Memory, ) -> Self { Self { - inner: SerializedCacheInner { + inner: ArtifactInner { info, backend_metadata, compiled_code, @@ -131,20 +143,13 @@ impl SerializedCache { } } - pub fn open

(path: P) -> Result - where - P: AsRef, - { - let file = File::open(path).map_err(|e| Error::IoError(e))?; + pub fn deserialize(bytes: &[u8]) -> Result { + let (_, body_slice) = ArtifactHeader::read_from_slice(bytes)?; - let mmap = unsafe { Mmap::map(&file).map_err(|e| Error::IoError(e))? }; + let inner = serde_bench::deserialize(body_slice) + .map_err(|e| Error::DeserializeError(format!("{:#?}", e)))?; - let (_header, body_slice) = SerializedCacheHeader::read_from_slice(&mmap[..])?; - - let inner = - deserialize(body_slice).map_err(|e| Error::DeserializeError(format!("{:#?}", e)))?; - - Ok(SerializedCache { inner }) + Ok(Artifact { inner }) } pub fn info(&self) -> &ModuleInfo { @@ -160,44 +165,25 @@ impl SerializedCache { ) } - pub fn store

(&self, path: P) -> Result<(), Error> - where - P: AsRef, - { - let mut file = File::create(path).map_err(|e| Error::IoError(e))?; - - let mut buffer = Vec::new(); - - serialize(&mut buffer, &self.inner).map_err(|e| Error::SerializeError(e.to_string()))?; - - let data_len = buffer.len() as u64; - - file.seek(SeekFrom::Start( - mem::size_of::() as u64 - )) - .map_err(|e| Error::IoError(e))?; - - file.write(buffer.as_slice()) - .map_err(|e| Error::IoError(e))?; - - file.seek(SeekFrom::Start(0)) - .map_err(|e| Error::Unknown(e.to_string()))?; - - let wasm_hash = self.inner.info.wasm_hash.into_array(); - - let cache_header = SerializedCacheHeader { - magic: [ - 'W' as u8, 'A' as u8, 'S' as u8, 'M' as u8, 'E' as u8, 'R' as u8, 0, 0, - ], + pub fn serialize(&self) -> Result, Error> { + let cache_header = ArtifactHeader { + magic: WASMER_CACHE_MAGIC, version: CURRENT_CACHE_VERSION, - data_len, - wasm_hash, + data_len: 0, + wasm_hash: self.inner.info.wasm_hash.into_array(), }; - file.write(cache_header.as_slice()) - .map_err(|e| Error::IoError(e))?; + let mut buffer = cache_header.as_slice().to_vec(); - Ok(()) + serde_bench::serialize(&mut buffer, &self.inner) + .map_err(|e| Error::SerializeError(e.to_string()))?; + + let data_len = (buffer.len() - mem::size_of::()) as u64; + + let (header, _) = ArtifactHeader::read_from_slice_mut(&mut buffer)?; + header.data_len = data_len; + + Ok(buffer) } } diff --git a/lib/runtime-core/src/lib.rs b/lib/runtime-core/src/lib.rs index 0e42531b8..663e17534 100644 --- a/lib/runtime-core/src/lib.rs +++ b/lib/runtime-core/src/lib.rs @@ -43,7 +43,7 @@ pub use self::module::Module; pub use self::typed_func::Func; use std::sync::Arc; -use self::cache::{Error as CacheError, SerializedCache}; +use self::cache::{Artifact, Error as CacheError}; pub mod prelude { pub use crate::import::{ImportObject, Namespace}; @@ -89,7 +89,7 @@ pub fn validate(wasm: &[u8]) -> bool { } pub unsafe fn load_cache_with( - cache: SerializedCache, + cache: Artifact, compiler: &dyn backend::Compiler, ) -> std::result::Result { let token = backend::Token::generate(); diff --git a/lib/runtime-core/src/module.rs b/lib/runtime-core/src/module.rs index 43deb7c4d..130082491 100644 --- a/lib/runtime-core/src/module.rs +++ b/lib/runtime-core/src/module.rs @@ -1,6 +1,6 @@ use crate::{ backend::{Backend, FuncResolver, ProtectedCaller}, - cache::{Error as CacheError, SerializedCache, WasmHash}, + cache::{Artifact, Error as CacheError, WasmHash}, error, import::ImportObject, structures::{Map, TypedIndex}, @@ -106,9 +106,9 @@ impl Module { Instance::new(Arc::clone(&self.inner), import_object) } - pub fn cache(&self) -> Result { + pub fn cache(&self) -> Result { let (info, backend_metadata, code) = self.inner.cache_gen.generate_cache(&self.inner)?; - Ok(SerializedCache::from_parts(info, backend_metadata, code)) + Ok(Artifact::from_parts(info, backend_metadata, code)) } pub fn info(&self) -> &ModuleInfo { diff --git a/lib/runtime/Cargo.toml b/lib/runtime/Cargo.toml index 29af96844..e4169a652 100644 --- a/lib/runtime/Cargo.toml +++ b/lib/runtime/Cargo.toml @@ -10,6 +10,7 @@ readme = "README.md" [dependencies] lazy_static = "1.2.0" +memmap = "0.7.0" [dependencies.wasmer-runtime-core] path = "../runtime-core" diff --git a/lib/runtime/src/cache.rs b/lib/runtime/src/cache.rs index b8f261c95..50507d802 100644 --- a/lib/runtime/src/cache.rs +++ b/lib/runtime/src/cache.rs @@ -1,8 +1,13 @@ use crate::Module; -use std::{fs::create_dir_all, io, path::PathBuf}; +use memmap::Mmap; +use std::{ + fs::{create_dir_all, File}, + io::{self, Write}, + path::PathBuf, +}; -pub use wasmer_runtime_core::cache::{Cache, WasmHash}; -use wasmer_runtime_core::cache::{Error as CacheError, SerializedCache}; +use wasmer_runtime_core::cache::Error as CacheError; +pub use wasmer_runtime_core::cache::{Artifact, Cache, WasmHash}; /// Representation of a directory that contains compiled wasm artifacts. /// @@ -80,8 +85,10 @@ impl Cache for FileSystemCache { let filename = key.encode(); let mut new_path_buf = self.path.clone(); new_path_buf.push(filename); + let file = File::open(new_path_buf)?; + let mmap = unsafe { Mmap::map(&file)? }; - let serialized_cache = SerializedCache::open(new_path_buf)?; + let serialized_cache = Artifact::deserialize(&mmap[..])?; unsafe { wasmer_runtime_core::load_cache_with(serialized_cache, super::default_compiler()) } } @@ -92,7 +99,10 @@ impl Cache for FileSystemCache { new_path_buf.push(filename); let serialized_cache = module.cache()?; - serialized_cache.store(new_path_buf)?; + let buffer = serialized_cache.serialize()?; + + let mut file = File::create(new_path_buf)?; + file.write(&buffer)?; Ok(key) }