diff --git a/Cargo.lock b/Cargo.lock index 6d301f57f..2eaad2339 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -81,7 +81,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "block-buffer" -version = "0.7.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "block-padding 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -360,6 +360,11 @@ dependencies = [ "unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "hex" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "humantime" version = "1.2.0" @@ -791,7 +796,7 @@ name = "sha2" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "block-buffer 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "block-buffer 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "digest 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "opaque-debug 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1060,6 +1065,7 @@ dependencies = [ "errno 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", "field-offset 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "hashbrown 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.48 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1172,7 +1178,7 @@ dependencies = [ "checksum backtrace-sys 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)" = "797c830ac25ccc92a7f8a7b9862bde440715531514594a6154e3d4a54dd769b6" "checksum bindgen 0.46.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8f7f7f0701772b17de73e4f5cbcb1dd6926f4706cba4c1ab62c5367f8bdc94e1" "checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" -"checksum block-buffer 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "509de513cca6d92b6aacf9c61acfe7eaa160837323a81068d690cc1f8e5740da" +"checksum block-buffer 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "49665c62e0e700857531fa5d3763e91b539ff1abeebd56808d378b495870d60d" "checksum block-padding 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d75255892aeb580d3c566f213a2b6fdc1c66667839f45719ee1d30ebf2aea591" "checksum byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" "checksum byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a019b10a2a7cdeb292db131fc8113e57ea2a908f6e7894b0c3c671893b65dbeb" @@ -1206,6 +1212,7 @@ dependencies = [ "checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" "checksum hashbrown 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "3bae29b6653b3412c2e71e9d486db9f9df5d701941d86683005efb9f2d28e3da" "checksum heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" +"checksum hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77" "checksum humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ca7e5f2e110db35f93b837c81797f3714500b81d517bf20c431b16d3ca4f114" "checksum indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7e81a7c05f79578dbc15793d8b619db9ba32b4577003ef3af1a91c416798c58d" "checksum itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b" diff --git a/lib/clif-backend/src/cache.rs b/lib/clif-backend/src/cache.rs index 84db980b0..3c38a5266 100644 --- a/lib/clif-backend/src/cache.rs +++ b/lib/clif-backend/src/cache.rs @@ -4,7 +4,7 @@ use hashbrown::HashMap; use std::sync::Arc; use wasmer_runtime_core::{ backend::{sys::Memory, CacheGen}, - cache::{Cache, Error}, + cache::{Error, SerializedCache}, module::{ModuleInfo, ModuleInner}, structures::Map, types::{LocalFuncIndex, SigIndex}, @@ -61,7 +61,7 @@ pub struct BackendCache { } impl BackendCache { - pub fn from_cache(cache: Cache) -> Result<(ModuleInfo, Memory, Self), Error> { + pub fn from_cache(cache: SerializedCache) -> Result<(ModuleInfo, Memory, Self), Error> { let (info, backend_data, compiled_code) = cache.consume(); let backend_cache = diff --git a/lib/clif-backend/src/lib.rs b/lib/clif-backend/src/lib.rs index 8d756420e..daef34c5d 100644 --- a/lib/clif-backend/src/lib.rs +++ b/lib/clif-backend/src/lib.rs @@ -14,7 +14,7 @@ use cranelift_codegen::{ }; use target_lexicon::Triple; -use wasmer_runtime_core::cache::{Cache, Error as CacheError}; +use wasmer_runtime_core::cache::{Error as CacheError, SerializedCache}; use wasmer_runtime_core::{ backend::{Compiler, Token}, error::{CompileError, CompileResult}, @@ -53,7 +53,11 @@ impl Compiler for CraneliftCompiler { /// Create a wasmer Module from an already-compiled cache. - unsafe fn from_cache(&self, cache: Cache, _: Token) -> Result { + unsafe fn from_cache( + &self, + cache: SerializedCache, + _: Token, + ) -> Result { module::Module::from_cache(cache) } diff --git a/lib/clif-backend/src/module.rs b/lib/clif-backend/src/module.rs index 0d0c66d7c..002d5185c 100644 --- a/lib/clif-backend/src/module.rs +++ b/lib/clif-backend/src/module.rs @@ -7,7 +7,7 @@ use cranelift_wasm; use hashbrown::HashMap; use std::sync::Arc; -use wasmer_runtime_core::cache::{Cache, Error as CacheError}; +use wasmer_runtime_core::cache::{Error as CacheError, SerializedCache, WasmHash}; use wasmer_runtime_core::{ backend::Backend, @@ -19,8 +19,6 @@ use wasmer_runtime_core::{ }, }; -use wasmer_runtime_core::module::WasmHash; - /// This contains all of the items in a `ModuleInner` except the `func_resolver`. pub struct Module { pub info: ModuleInfo, @@ -90,7 +88,7 @@ impl Module { }) } - pub fn from_cache(cache: Cache) -> Result { + pub fn from_cache(cache: SerializedCache) -> Result { let (info, compiled_code, backend_cache) = BackendCache::from_cache(cache)?; let (func_resolver_builder, trampolines, handler_data) = diff --git a/lib/runtime-core/Cargo.toml b/lib/runtime-core/Cargo.toml index 4781751af..75e49840c 100644 --- a/lib/runtime-core/Cargo.toml +++ b/lib/runtime-core/Cargo.toml @@ -16,6 +16,7 @@ lazy_static = "1.2.0" indexmap = "1.0.2" errno = "0.2.4" libc = "0.2.48" +hex = "0.3.2" # Dependencies for caching. [dependencies.serde] diff --git a/lib/runtime-core/src/backend.rs b/lib/runtime-core/src/backend.rs index e490b5e3e..ab7c27d53 100644 --- a/lib/runtime-core/src/backend.rs +++ b/lib/runtime-core/src/backend.rs @@ -8,14 +8,12 @@ use crate::{ }; use crate::{ - cache::{Cache, Error as CacheError}, + cache::{Error as CacheError, SerializedCache}, module::ModuleInfo, sys::Memory, }; use std::ptr::NonNull; -use std::sync::Arc; - pub mod sys { pub use crate::sys::*; } @@ -44,7 +42,11 @@ pub trait Compiler { /// be called from inside the runtime. fn compile(&self, wasm: &[u8], _: Token) -> CompileResult; - unsafe fn from_cache(&self, cache: Cache, _: Token) -> Result; + unsafe fn from_cache( + &self, + cache: SerializedCache, + _: Token, + ) -> Result; } /// The functionality exposed by this trait is expected to be used diff --git a/lib/runtime-core/src/cache.rs b/lib/runtime-core/src/cache.rs index d49dd4215..34c09c3aa 100644 --- a/lib/runtime-core/src/cache.rs +++ b/lib/runtime-core/src/cache.rs @@ -1,4 +1,7 @@ -use crate::{module::ModuleInfo, sys::Memory}; +use crate::{ + module::{Module, ModuleInfo}, + sys::Memory, +}; use memmap::Mmap; use serde_bench::{deserialize, serialize}; use sha2::{Digest, Sha256}; @@ -8,7 +11,6 @@ use std::{ mem, path::Path, slice, - sync::Arc, }; #[derive(Debug)] @@ -27,23 +29,43 @@ pub enum Error { InvalidatedCache, } +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct WasmHash([u8; 32]); + +impl WasmHash { + pub fn generate(wasm: &[u8]) -> Self { + let mut array = [0u8; 32]; + array.copy_from_slice(Sha256::digest(wasm).as_slice()); + WasmHash(array) + } + + pub fn encode(self) -> String { + hex::encode(self.0) + } + + pub(crate) fn into_array(self) -> [u8; 32] { + self.0 + } +} + const CURRENT_CACHE_VERSION: u64 = 0; /// The header of a cache file. #[repr(C, packed)] -struct CacheHeader { +struct SerializedCacheHeader { magic: [u8; 8], // [W, A, S, M, E, R, \0, \0] version: u64, data_len: u64, wasm_hash: [u8; 32], // Sha256 of the wasm in binary format. } -impl CacheHeader { - pub fn read_from_slice(buffer: &[u8]) -> Result<(&CacheHeader, &[u8]), Error> { - if buffer.len() >= mem::size_of::() { +impl SerializedCacheHeader { + pub fn read_from_slice(buffer: &[u8]) -> Result<(&Self, &[u8]), Error> { + if buffer.len() >= mem::size_of::() { if &buffer[..8] == "WASMER\0\0".as_bytes() { - let (header_slice, body_slice) = buffer.split_at(mem::size_of::()); - let header = unsafe { &*(header_slice.as_ptr() as *const CacheHeader) }; + let (header_slice, body_slice) = + buffer.split_at(mem::size_of::()); + let header = unsafe { &*(header_slice.as_ptr() as *const SerializedCacheHeader) }; if header.version == CURRENT_CACHE_VERSION { Ok((header, body_slice)) @@ -59,31 +81,31 @@ impl CacheHeader { } pub fn as_slice(&self) -> &[u8] { - let ptr = self as *const CacheHeader as *const u8; - unsafe { slice::from_raw_parts(ptr, mem::size_of::()) } + let ptr = self as *const SerializedCacheHeader as *const u8; + unsafe { slice::from_raw_parts(ptr, mem::size_of::()) } } } #[derive(Serialize, Deserialize)] -struct CacheInner { +struct SerializedCacheInner { info: Box, #[serde(with = "serde_bytes")] backend_metadata: Box<[u8]>, compiled_code: Memory, } -pub struct Cache { - inner: CacheInner, +pub struct SerializedCache { + inner: SerializedCacheInner, } -impl Cache { +impl SerializedCache { pub(crate) fn from_parts( info: Box, backend_metadata: Box<[u8]>, compiled_code: Memory, ) -> Self { Self { - inner: CacheInner { + inner: SerializedCacheInner { info, backend_metadata, compiled_code, @@ -91,7 +113,7 @@ impl Cache { } } - pub fn open

(path: P) -> Result + pub fn open

(path: P) -> Result where P: AsRef, { @@ -99,12 +121,12 @@ impl Cache { let mmap = unsafe { Mmap::map(&file).map_err(|e| Error::IoError(e))? }; - let (header, body_slice) = CacheHeader::read_from_slice(&mmap[..])?; + let (_header, body_slice) = SerializedCacheHeader::read_from_slice(&mmap[..])?; let inner = deserialize(body_slice).map_err(|e| Error::DeserializeError(format!("{:#?}", e)))?; - Ok(Cache { inner }) + Ok(SerializedCache { inner }) } pub fn info(&self) -> &ModuleInfo { @@ -132,8 +154,10 @@ impl Cache { let data_len = buffer.len() as u64; - file.seek(SeekFrom::Start(mem::size_of::() as u64)) - .map_err(|e| Error::IoError(e))?; + file.seek(SeekFrom::Start( + mem::size_of::() as u64 + )) + .map_err(|e| Error::IoError(e))?; file.write(buffer.as_slice()) .map_err(|e| Error::IoError(e))?; @@ -143,7 +167,7 @@ impl Cache { let wasm_hash = self.inner.info.wasm_hash.into_array(); - let cache_header = CacheHeader { + let cache_header = SerializedCacheHeader { magic: [ 'W' as u8, 'A' as u8, 'S' as u8, 'M' as u8, 'E' as u8, 'R' as u8, 0, 0, ], @@ -159,8 +183,10 @@ impl Cache { } } -pub fn hash_data(data: &[u8]) -> [u8; 32] { - let mut array = [0u8; 32]; - array.copy_from_slice(Sha256::digest(data).as_slice()); - array +pub trait Cache { + type LoadError; + type StoreError; + + unsafe fn load(&self, key: WasmHash) -> Result; + fn store(&mut self, module: Module) -> Result; } diff --git a/lib/runtime-core/src/lib.rs b/lib/runtime-core/src/lib.rs index 7025acf5a..0e42531b8 100644 --- a/lib/runtime-core/src/lib.rs +++ b/lib/runtime-core/src/lib.rs @@ -43,7 +43,7 @@ pub use self::module::Module; pub use self::typed_func::Func; use std::sync::Arc; -use self::cache::{Cache, Error as CacheError}; +use self::cache::{Error as CacheError, SerializedCache}; pub mod prelude { pub use crate::import::{ImportObject, Namespace}; @@ -88,20 +88,8 @@ pub fn validate(wasm: &[u8]) -> bool { } } -// -// pub fn compile_to_cache_with( -// wasm: &[u8], -// compiler: &dyn backend::Compiler, -// ) -> CompileResult { -// let token = backend::Token::generate(); -// let (info, backend_metadata, compiled_code) = -// compiler.compile_to_backend_cache_data(wasm, token)?; - -// Ok(Cache::new(wasm, info, backend_metadata, compiled_code)) -// } - pub unsafe fn load_cache_with( - cache: Cache, + cache: SerializedCache, compiler: &dyn backend::Compiler, ) -> std::result::Result { let token = backend::Token::generate(); diff --git a/lib/runtime-core/src/module.rs b/lib/runtime-core/src/module.rs index 5cb1b5b21..43deb7c4d 100644 --- a/lib/runtime-core/src/module.rs +++ b/lib/runtime-core/src/module.rs @@ -1,6 +1,6 @@ use crate::{ backend::{Backend, FuncResolver, ProtectedCaller}, - cache::{Cache, Error as CacheError}, + cache::{Error as CacheError, SerializedCache, WasmHash}, error, import::ImportObject, structures::{Map, TypedIndex}, @@ -60,19 +60,6 @@ pub struct ModuleInfo { pub wasm_hash: WasmHash, } -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct WasmHash([u8; 32]); - -impl WasmHash { - pub fn generate(wasm: &[u8]) -> Self { - WasmHash(crate::cache::hash_data(wasm)) - } - - pub(crate) fn into_array(self) -> [u8; 32] { - self.0 - } -} - /// A compiled WebAssembly module. /// /// `Module` is returned by the [`compile`] and @@ -119,9 +106,9 @@ impl Module { Instance::new(Arc::clone(&self.inner), import_object) } - pub fn cache(&self) -> Result { + pub fn cache(&self) -> Result { let (info, backend_metadata, code) = self.inner.cache_gen.generate_cache(&self.inner)?; - Ok(Cache::from_parts(info, backend_metadata, code)) + Ok(SerializedCache::from_parts(info, backend_metadata, code)) } pub fn info(&self) -> &ModuleInfo { diff --git a/lib/runtime/src/cache.rs b/lib/runtime/src/cache.rs index 01bf90c25..2fbf7f47b 100644 --- a/lib/runtime/src/cache.rs +++ b/lib/runtime/src/cache.rs @@ -1,46 +1,9 @@ use crate::Module; -use std::path::Path; -use wasmer_runtime_core::cache::{hash_data, Cache as CoreCache}; -use wasmer_runtime_core::module::WasmHash; +use std::{fs::create_dir_all, io, path::PathBuf}; -pub use wasmer_runtime_core::cache::Error; +pub use wasmer_runtime_core::cache::{Cache, WasmHash}; +use wasmer_runtime_core::cache::{Error as CacheError, SerializedCache}; -// /// On-disk storage of compiled WebAssembly. -// /// -// /// A `Cache` can be used to quickly reload already -// /// compiled WebAssembly from a previous execution -// /// during which the wasm was explicitly compiled -// /// as a `Cache`. -// /// -// /// # Usage: -// /// -// /// ``` -// /// use wasmer_runtime::{compile_cache, Cache}; -// /// -// /// # use wasmer_runtime::error::{CompileResult, CacheError}; -// /// # fn make_cache(wasm: &[u8]) -> CompileResult<()> { -// /// // Make a cache. -// /// let cache = compile_cache(wasm)?; -// /// -// /// # Ok(()) -// /// # } -// /// # fn usage_cache(cache: Cache) -> Result<(), CacheError> { -// /// // Store the cache in a file. -// /// cache.store("some_cache_file")?; -// /// -// /// // Load the cache. -// /// let cache = Cache::load("some_cache_file")?; -// /// let module = unsafe { cache.into_module()? }; -// /// # Ok(()) -// /// # } -// /// ``` -// /// -// /// # Performance Characteristics: -// /// -// /// Loading caches from files has been optimized for latency. -// /// There is still more work to do that will reduce -// /// loading time, especially for very large modules, -// /// but it will require signifigant internal work. // /// // /// # Drawbacks: // /// @@ -128,11 +91,66 @@ pub use wasmer_runtime_core::cache::Error; // } // } -pub trait Cache { - type Key; - type LoadError; - type StoreError; - - unsafe fn load(&self, key: Self::Key) -> Result; - fn store(&mut self, module: Module) -> Result; +pub struct FSCache { + path: PathBuf, +} + +impl FSCache { + pub fn open>(path: P) -> io::Result { + let path: PathBuf = path.into(); + + if path.exists() { + let metadata = path.metadata()?; + if metadata.is_dir() { + if !metadata.permissions().readonly() { + Ok(Self { path }) + } else { + // This directory is readonly. + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + format!("the supplied path is readonly: {}", path.display()), + )) + } + } else { + // This path points to a file. + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + format!( + "the supplied path already points to a file: {}", + path.display() + ), + )) + } + } else { + // Create the directory and any parent directories if they don't yet exist. + create_dir_all(&path)?; + Ok(Self { path }) + } + } +} + +impl Cache for FSCache { + type LoadError = CacheError; + type StoreError = CacheError; + + unsafe fn load(&self, key: WasmHash) -> Result { + let filename = key.encode(); + let mut new_path_buf = self.path.clone(); + new_path_buf.push(filename); + + let serialized_cache = SerializedCache::open(new_path_buf)?; + wasmer_runtime_core::load_cache_with(serialized_cache, super::default_compiler()) + } + + fn store(&mut self, module: Module) -> Result { + let key = module.info().wasm_hash; + let filename = key.encode(); + let mut new_path_buf = self.path.clone(); + new_path_buf.push(filename); + + let serialized_cache = module.cache()?; + serialized_cache.store(new_path_buf)?; + + Ok(key) + } } diff --git a/lib/runtime/src/lib.rs b/lib/runtime/src/lib.rs index 48a4c27f5..012bdb593 100644 --- a/lib/runtime/src/lib.rs +++ b/lib/runtime/src/lib.rs @@ -99,7 +99,7 @@ pub mod wasm { } pub mod error { - pub use super::cache::Error as CacheError; + pub use wasmer_runtime_core::cache::Error as CacheError; pub use wasmer_runtime_core::error::*; } @@ -108,12 +108,10 @@ pub mod units { pub use wasmer_runtime_core::units::{Bytes, Pages}; } -mod cache; +pub mod cache; use wasmer_runtime_core::backend::Compiler; -pub use self::cache::Cache; - /// Compile WebAssembly binary code into a [`Module`]. /// This function is useful if it is necessary to /// compile a module before it can be instantiated