fluencelabs/marine
1
0
Fork 0
mirror of https://github.com/fluencelabs/marine.git synced 2024-12-12 14:55:32 +00:00
Code Issues Projects Releases Wiki Activity

Add with_module_names constructor to FluenceFaaS (#11)

Browse Source
This commit is contained in:
folex 2020-07-20 17:13:11 +03:00 committed by GitHub
parent ee55c63ff8
commit fa3ae391ec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 114 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
Cargo.lock generated
View File

@ -343,9 +343,9 @@ dependencies = [
[[package]]
name = "errno"
version = "0.2.5"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b480f641ccf0faf324e20c1d3e53d81b7484c698b42ea677f6907ae4db195371"
checksum = "6eab5ee3df98a279d9b316b1af6ac95422127b1290317e6d18c1743c99418b01"
dependencies = [
"errno-dragonfly",
"libc",
@ -434,7 +434,7 @@ dependencies = [
[[package]]
name = "fluence"
version = "0.2.0"
source = "git+https://github.com/fluencelabs/rust-sdk#be47b96ce8e067296c37680c4083130c2ce9c859"
source = "git+https://github.com/fluencelabs/rust-sdk#79a5896015659ddb1c20526c4aa142b8afae35ad"
dependencies = [
"fluence-sdk-macro",
"fluence-sdk-main",
@ -459,7 +459,7 @@ dependencies = [
[[package]]
name = "fluence-sdk-macro"
version = "0.2.0"
source = "git+https://github.com/fluencelabs/rust-sdk#be47b96ce8e067296c37680c4083130c2ce9c859"
source = "git+https://github.com/fluencelabs/rust-sdk#79a5896015659ddb1c20526c4aa142b8afae35ad"
dependencies = [
"fluence-sdk-wit",
]
@ -467,7 +467,7 @@ dependencies = [
[[package]]
name = "fluence-sdk-main"
version = "0.2.0"
source = "git+https://github.com/fluencelabs/rust-sdk#be47b96ce8e067296c37680c4083130c2ce9c859"
source = "git+https://github.com/fluencelabs/rust-sdk#79a5896015659ddb1c20526c4aa142b8afae35ad"
dependencies = [
"log",
]
@ -475,7 +475,7 @@ dependencies = [
[[package]]
name = "fluence-sdk-wit"
version = "0.2.0"
source = "git+https://github.com/fluencelabs/rust-sdk#be47b96ce8e067296c37680c4083130c2ce9c859"
source = "git+https://github.com/fluencelabs/rust-sdk#79a5896015659ddb1c20526c4aa142b8afae35ad"
dependencies = [
"proc-macro2",
"quote",
@ -566,6 +566,15 @@ dependencies = [
"fluence-faas",
]
[[package]]
name = "hashbrown"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34f595585f103464d8d2f6e9864682d74c1601fed5e07d62b1c9058dba8246fb"
dependencies = [
"autocfg",
]
[[package]]
name = "heck"
version = "0.3.1"
@ -607,11 +616,12 @@ checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005"
[[package]]
name = "indexmap"
version = "1.4.0"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c398b2b113b55809ceb9ee3e753fcbac793f1956663f3c36549c1346015c2afe"
checksum = "5b88cd59ee5f71fea89a62248fc8f387d44400cefe05ef548466d61ced9029a7"
dependencies = [
"autocfg",
"hashbrown",
"serde",
]
@ -680,9 +690,9 @@ dependencies = [
[[package]]
name = "libc"
version = "0.2.72"
version = "0.2.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9f8082297d534141b30c8d39e9b1773713ab50fdbe4ff30f750d063b3bfd701"
checksum = "bd7d4bd64732af4bf3a67f367c27df8520ad7e230c5817b8ff485864d80242b9"
[[package]]
name = "lock_api"
@ -845,9 +855,9 @@ checksum = "237a5ed80e274dbc66f86bd59c1e25edc039660be53194b5fe0a482e0f2612ea"
[[package]]
name = "proc-macro2"
version = "1.0.18"
version = "1.0.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa"
checksum = "04f5f085b5d71e2188cb8271e5da0161ad52c3f227a661a3c135fdf28e258b12"
dependencies = [
"unicode-xid",
]
@ -1374,7 +1384,7 @@ dependencies = [
[[package]]
name = "wasmer-interface-types"
version = "0.17.0"
source = "git+https://github.com/fluencelabs/interface-types?branch=byte_array#bb9e4dbdfc0da6e6803392664a1c34284e418a58"
source = "git+https://github.com/fluencelabs/interface-types?branch=byte_array#b65b34b0f627b3da5cd5761fe309cefc4980449b"
dependencies = [
"nom",
"serde",

108
fluence-faas/src/faas.rs
View File

@ -29,10 +29,51 @@ use std::convert::TryInto;
use std::fs;
use std::path::PathBuf;
use crate::faas_interface::FaaSFunctionSignature;
use std::collections::HashSet;
// TODO: remove and use mutex instead
unsafe impl Send for FluenceFaaS {}
/// Strategy for module loading: either `All`, or only those specified in `Named`
pub enum ModulesLoadStrategy<'a> {
All,
Named(&'a HashSet<String>),
}
impl<'a> ModulesLoadStrategy<'a> {
#[inline]
/// Returns true if `module` should be loaded.
pub fn should_load(&self, module: &str) -> bool {
match self {
ModulesLoadStrategy::All => true,
ModulesLoadStrategy::Named(set) => set.contains(module),
}
}
#[inline]
/// Returns the number of modules that must be loaded.
pub fn required_modules_len(&self) -> usize {
match self {
ModulesLoadStrategy::Named(set) => set.len(),
_ => 0,
}
}
#[inline]
/// Returns difference between required and loaded modules.
pub fn missing_modules<'s>(&self, loaded: impl Iterator<Item = &'s String>) -> Vec<&'s String> {
match self {
ModulesLoadStrategy::Named(set) => loaded.fold(vec![], |mut vec, module| {
if !set.contains(module) {
vec.push(module)
}
vec
}),
_ => <_>::default(),
}
}
}
pub struct FluenceFaaS {
fce: FCE,
@ -50,10 +91,9 @@ impl FluenceFaaS {
/// Creates FaaS from config deserialized from TOML.
pub fn with_raw_config(config: RawCoreModulesConfig) -> Result<Self> {
let config = crate::misc::from_raw_config(config)?;
let modules = config
.core_modules_dir
.as_ref()
.map_or(Ok(vec![]), |dir| Self::load_modules(dir))?;
let modules = config.core_modules_dir.as_ref().map_or(Ok(vec![]), |dir| {
Self::load_modules(dir, ModulesLoadStrategy::All)
})?;
Self::with_modules(modules, config)
}
@ -80,29 +120,63 @@ impl FluenceFaaS {
})
}
/// Searches for modules in `config.core_modules_dir`, loads only those in the `names` set
pub fn with_module_names<C>(names: &HashSet<String>, config: C) -> Result<Self>
where
C: TryInto<CoreModulesConfig>,
FaaSError: From<C::Error>,
{
let config = config.try_into()?;
let modules = config.core_modules_dir.as_ref().map_or(Ok(vec![]), |dir| {
Self::load_modules(dir, ModulesLoadStrategy::Named(names))
})?;
Self::with_modules::<_, CoreModulesConfig>(modules, config)
}
/// Loads modules from a directory at a given path. Non-recursive, ignores subdirectories.
fn load_modules(core_modules_dir: &str) -> Result<Vec<(String, Vec<u8>)>> {
fn load_modules(
core_modules_dir: &str,
modules: ModulesLoadStrategy,
) -> Result<Vec<(String, Vec<u8>)>> {
use FaaSError::IOError;
let mut dir_entries = fs::read_dir(core_modules_dir)
.map_err(|e| IOError(format!("{}: {}", core_modules_dir, e)))?;
dir_entries.try_fold(vec![], |mut vec, entry| {
let loaded = dir_entries.try_fold(vec![], |mut vec, entry| {
let entry = entry?;
let path = entry.path();
if !path.is_dir() {
let module_name = path
.file_name()
.ok_or_else(|| IOError(format!("No file name in path {:?}", path)))?
.to_os_string()
.into_string()
.map_err(|name| IOError(format!("invalid file name: {:?}", name)))?;
let module_bytes = fs::read(path)?;
vec.push((module_name, module_bytes))
// Skip directories
if path.is_dir() {
return Ok(vec);
}
Ok(vec)
})
let module_name = path
.file_name()
.ok_or_else(|| IOError(format!("No file name in path {:?}", path)))?
.to_os_string()
.into_string()
.map_err(|name| IOError(format!("invalid file name: {:?}", name)))?;
if modules.should_load(&module_name) {
let module_bytes = fs::read(path)?;
vec.push((module_name, module_bytes));
}
Result::Ok(vec)
})?;
if modules.required_modules_len() > loaded.len() {
let loaded = loaded.iter().map(|(n, _)| n);
let not_found = modules.missing_modules(loaded);
return Err(FaaSError::ConfigParseError(format!(
"the following modules were not found: {:?}",
not_found
)));
}
Ok(loaded)
}
/// Executes provided Wasm code in the internal environment (with access to module exports).