From fa3ae391ecc99f14acee57862adc799f88dd3dc9 Mon Sep 17 00:00:00 2001 From: folex <0xdxdy@gmail.com> Date: Mon, 20 Jul 2020 17:13:11 +0300 Subject: [PATCH] Add with_module_names constructor to FluenceFaaS (#11) --- Cargo.lock | 36 ++++++++----- fluence-faas/src/faas.rs | 108 +++++++++++++++++++++++++++++++++------ 2 files changed, 114 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 11780170..de3f866d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -343,9 +343,9 @@ dependencies = [ [[package]] name = "errno" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b480f641ccf0faf324e20c1d3e53d81b7484c698b42ea677f6907ae4db195371" +checksum = "6eab5ee3df98a279d9b316b1af6ac95422127b1290317e6d18c1743c99418b01" dependencies = [ "errno-dragonfly", "libc", @@ -434,7 +434,7 @@ dependencies = [ [[package]] name = "fluence" version = "0.2.0" -source = "git+https://github.com/fluencelabs/rust-sdk#be47b96ce8e067296c37680c4083130c2ce9c859" +source = "git+https://github.com/fluencelabs/rust-sdk#79a5896015659ddb1c20526c4aa142b8afae35ad" dependencies = [ "fluence-sdk-macro", "fluence-sdk-main", @@ -459,7 +459,7 @@ dependencies = [ [[package]] name = "fluence-sdk-macro" version = "0.2.0" -source = "git+https://github.com/fluencelabs/rust-sdk#be47b96ce8e067296c37680c4083130c2ce9c859" +source = "git+https://github.com/fluencelabs/rust-sdk#79a5896015659ddb1c20526c4aa142b8afae35ad" dependencies = [ "fluence-sdk-wit", ] @@ -467,7 +467,7 @@ dependencies = [ [[package]] name = "fluence-sdk-main" version = "0.2.0" -source = "git+https://github.com/fluencelabs/rust-sdk#be47b96ce8e067296c37680c4083130c2ce9c859" +source = "git+https://github.com/fluencelabs/rust-sdk#79a5896015659ddb1c20526c4aa142b8afae35ad" dependencies = [ "log", ] @@ -475,7 +475,7 @@ dependencies = [ [[package]] name = "fluence-sdk-wit" version = "0.2.0" -source = "git+https://github.com/fluencelabs/rust-sdk#be47b96ce8e067296c37680c4083130c2ce9c859" +source = "git+https://github.com/fluencelabs/rust-sdk#79a5896015659ddb1c20526c4aa142b8afae35ad" dependencies = [ "proc-macro2", "quote", @@ -566,6 +566,15 @@ dependencies = [ "fluence-faas", ] +[[package]] +name = "hashbrown" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34f595585f103464d8d2f6e9864682d74c1601fed5e07d62b1c9058dba8246fb" +dependencies = [ + "autocfg", +] + [[package]] name = "heck" version = "0.3.1" @@ -607,11 +616,12 @@ checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" [[package]] name = "indexmap" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c398b2b113b55809ceb9ee3e753fcbac793f1956663f3c36549c1346015c2afe" +checksum = "5b88cd59ee5f71fea89a62248fc8f387d44400cefe05ef548466d61ced9029a7" dependencies = [ "autocfg", + "hashbrown", "serde", ] @@ -680,9 +690,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.72" +version = "0.2.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9f8082297d534141b30c8d39e9b1773713ab50fdbe4ff30f750d063b3bfd701" +checksum = "bd7d4bd64732af4bf3a67f367c27df8520ad7e230c5817b8ff485864d80242b9" [[package]] name = "lock_api" @@ -845,9 +855,9 @@ checksum = "237a5ed80e274dbc66f86bd59c1e25edc039660be53194b5fe0a482e0f2612ea" [[package]] name = "proc-macro2" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" +checksum = "04f5f085b5d71e2188cb8271e5da0161ad52c3f227a661a3c135fdf28e258b12" dependencies = [ "unicode-xid", ] @@ -1374,7 +1384,7 @@ dependencies = [ [[package]] name = "wasmer-interface-types" version = "0.17.0" -source = "git+https://github.com/fluencelabs/interface-types?branch=byte_array#bb9e4dbdfc0da6e6803392664a1c34284e418a58" +source = "git+https://github.com/fluencelabs/interface-types?branch=byte_array#b65b34b0f627b3da5cd5761fe309cefc4980449b" dependencies = [ "nom", "serde", diff --git a/fluence-faas/src/faas.rs b/fluence-faas/src/faas.rs index 607d598f..f22a38a6 100644 --- a/fluence-faas/src/faas.rs +++ b/fluence-faas/src/faas.rs @@ -29,10 +29,51 @@ use std::convert::TryInto; use std::fs; use std::path::PathBuf; use crate::faas_interface::FaaSFunctionSignature; +use std::collections::HashSet; // TODO: remove and use mutex instead unsafe impl Send for FluenceFaaS {} +/// Strategy for module loading: either `All`, or only those specified in `Named` +pub enum ModulesLoadStrategy<'a> { + All, + Named(&'a HashSet), +} + +impl<'a> ModulesLoadStrategy<'a> { + #[inline] + /// Returns true if `module` should be loaded. + pub fn should_load(&self, module: &str) -> bool { + match self { + ModulesLoadStrategy::All => true, + ModulesLoadStrategy::Named(set) => set.contains(module), + } + } + + #[inline] + /// Returns the number of modules that must be loaded. + pub fn required_modules_len(&self) -> usize { + match self { + ModulesLoadStrategy::Named(set) => set.len(), + _ => 0, + } + } + + #[inline] + /// Returns difference between required and loaded modules. + pub fn missing_modules<'s>(&self, loaded: impl Iterator) -> Vec<&'s String> { + match self { + ModulesLoadStrategy::Named(set) => loaded.fold(vec![], |mut vec, module| { + if !set.contains(module) { + vec.push(module) + } + vec + }), + _ => <_>::default(), + } + } +} + pub struct FluenceFaaS { fce: FCE, @@ -50,10 +91,9 @@ impl FluenceFaaS { /// Creates FaaS from config deserialized from TOML. pub fn with_raw_config(config: RawCoreModulesConfig) -> Result { let config = crate::misc::from_raw_config(config)?; - let modules = config - .core_modules_dir - .as_ref() - .map_or(Ok(vec![]), |dir| Self::load_modules(dir))?; + let modules = config.core_modules_dir.as_ref().map_or(Ok(vec![]), |dir| { + Self::load_modules(dir, ModulesLoadStrategy::All) + })?; Self::with_modules(modules, config) } @@ -80,29 +120,63 @@ impl FluenceFaaS { }) } + /// Searches for modules in `config.core_modules_dir`, loads only those in the `names` set + pub fn with_module_names(names: &HashSet, config: C) -> Result + where + C: TryInto, + FaaSError: From, + { + let config = config.try_into()?; + let modules = config.core_modules_dir.as_ref().map_or(Ok(vec![]), |dir| { + Self::load_modules(dir, ModulesLoadStrategy::Named(names)) + })?; + + Self::with_modules::<_, CoreModulesConfig>(modules, config) + } + /// Loads modules from a directory at a given path. Non-recursive, ignores subdirectories. - fn load_modules(core_modules_dir: &str) -> Result)>> { + fn load_modules( + core_modules_dir: &str, + modules: ModulesLoadStrategy, + ) -> Result)>> { use FaaSError::IOError; let mut dir_entries = fs::read_dir(core_modules_dir) .map_err(|e| IOError(format!("{}: {}", core_modules_dir, e)))?; - dir_entries.try_fold(vec![], |mut vec, entry| { + let loaded = dir_entries.try_fold(vec![], |mut vec, entry| { let entry = entry?; let path = entry.path(); - if !path.is_dir() { - let module_name = path - .file_name() - .ok_or_else(|| IOError(format!("No file name in path {:?}", path)))? - .to_os_string() - .into_string() - .map_err(|name| IOError(format!("invalid file name: {:?}", name)))?; - let module_bytes = fs::read(path)?; - vec.push((module_name, module_bytes)) + // Skip directories + if path.is_dir() { + return Ok(vec); } - Ok(vec) - }) + let module_name = path + .file_name() + .ok_or_else(|| IOError(format!("No file name in path {:?}", path)))? + .to_os_string() + .into_string() + .map_err(|name| IOError(format!("invalid file name: {:?}", name)))?; + + if modules.should_load(&module_name) { + let module_bytes = fs::read(path)?; + vec.push((module_name, module_bytes)); + } + + Result::Ok(vec) + })?; + + if modules.required_modules_len() > loaded.len() { + let loaded = loaded.iter().map(|(n, _)| n); + let not_found = modules.missing_modules(loaded); + return Err(FaaSError::ConfigParseError(format!( + "the following modules were not found: {:?}", + not_found + ))); + } + + Ok(loaded) } /// Executes provided Wasm code in the internal environment (with access to module exports).