From 77cd79e124440765b924b070a1c9031f06380ebc Mon Sep 17 00:00:00 2001 From: folex <0xdxdy@gmail.com> Date: Fri, 14 Aug 2020 13:28:15 +0300 Subject: [PATCH] Pass envs to AppService, name modules without .wasm extension (#16) --- fluence-app-service/src/service.rs | 42 +++++------------------------- fluence-faas/src/faas.rs | 34 +++++++++++++++++++----- fluence-faas/src/misc/config.rs | 13 +++++++++ tools/repl/src/repl.rs | 21 +++++++-------- 4 files changed, 56 insertions(+), 54 deletions(-) diff --git a/fluence-app-service/src/service.rs b/fluence-app-service/src/service.rs index 8627bc0d..2fbce14a 100644 --- a/fluence-app-service/src/service.rs +++ b/fluence-app-service/src/service.rs @@ -22,7 +22,6 @@ use fluence_faas::FluenceFaaS; use fluence_faas::ModulesConfig; use std::convert::TryInto; -use std::path::PathBuf; const SERVICE_ID_ENV_NAME: &str = "service_id"; const SERVICE_LOCAL_DIR_NAME: &str = "local"; @@ -37,41 +36,15 @@ pub struct AppService { impl AppService { /// Create Service with given modules and service id. - pub fn new(modules: I, config: C, service_id: S) -> Result + pub fn new(config: C, service_id: S, envs: Vec) -> Result where - I: IntoIterator, C: TryInto, S: AsRef, AppServiceError: From, { let config: ModulesConfig = config.try_into()?; let service_id = service_id.as_ref(); - let config = Self::set_env_and_dirs(config, service_id, None)?; - - let modules = modules.into_iter().collect(); - let faas = FluenceFaaS::with_module_names(&modules, config)?; - - Ok(Self { faas }) - } - - /// Create Service with given raw config, service id and service base dir. - pub fn with_raw_config( - config: P, - service_id: SI, - service_base_dir: Option<&str>, - ) -> Result - where - P: Into, - SI: AsRef, - { - let service_id = service_id.as_ref(); - let service_base_dir = service_base_dir; - - let config_content = std::fs::read(config.into())?; - let config: crate::RawModulesConfig = toml::from_slice(&config_content)?; - - let config = config.try_into()?; - let config = Self::set_env_and_dirs(config, service_id, service_base_dir)?; + let config = Self::set_env_and_dirs(config, service_id, envs)?; let faas = FluenceFaaS::with_raw_config(config)?; @@ -106,11 +79,10 @@ impl AppService { fn set_env_and_dirs( mut config: ModulesConfig, service_id: &str, - service_base_dir: Option<&str>, + mut envs: Vec, ) -> Result { - let base_dir = match (&config.service_base_dir, service_base_dir) { - (_, Some(base_dir)) => base_dir, - (Some(ref base_dir), None) => base_dir, + let base_dir = match config.service_base_dir.as_ref() { + Some(base_dir) => base_dir, _ => { return Err(AppServiceError::IOError(String::from( "service_base_dir should be specified", @@ -130,19 +102,19 @@ impl AppService { let local_dir: String = local_dir_path.to_string_lossy().into(); let tmp_dir: String = tmp_dir_path.to_string_lossy().into(); - let service_id_env = vec![format!("{}={}", SERVICE_ID_ENV_NAME, service_id).into_bytes()]; let preopened_files = vec![local_dir.clone(), tmp_dir.clone()]; let mapped_dirs = vec![ (String::from(SERVICE_LOCAL_DIR_NAME), local_dir), (String::from(SERVICE_TMP_DIR_NAME), tmp_dir), ]; + envs.push(format!("{}={}", SERVICE_ID_ENV_NAME, service_id)); config.modules_config = config .modules_config .into_iter() .map(|(name, module_config)| { let module_config = module_config - .extend_wasi_envs(service_id_env.clone()) + .extend_wasi_envs(envs.iter().map(|s| s.clone().into_bytes()).collect()) .extend_wasi_files(preopened_files.clone(), mapped_dirs.clone()); (name, module_config) diff --git a/fluence-faas/src/faas.rs b/fluence-faas/src/faas.rs index d46803d7..da513786 100644 --- a/fluence-faas/src/faas.rs +++ b/fluence-faas/src/faas.rs @@ -27,24 +27,30 @@ use std::convert::TryInto; use std::collections::HashSet; use std::collections::HashMap; use std::fs; -use std::path::PathBuf; +use std::path::{PathBuf, Path}; // TODO: remove and use mutex instead unsafe impl Send for FluenceFaaS {} -/// Strategy for module loading: either `All`, or only those specified in `Named` +/// Strategies for module loading. pub enum ModulesLoadStrategy<'a> { + /// Try to load all files in a given directory + #[allow(dead_code)] All, + /// Try to load only files contained in the set Named(&'a HashSet), + /// In a given directory, try to load all files ending with .wasm + WasmOnly, } impl<'a> ModulesLoadStrategy<'a> { #[inline] /// Returns true if `module` should be loaded. - pub fn should_load(&self, module: &str) -> bool { + pub fn should_load(&self, module: &Path) -> bool { match self { ModulesLoadStrategy::All => true, - ModulesLoadStrategy::Named(set) => set.contains(module), + ModulesLoadStrategy::Named(set) => set.contains(module.to_string_lossy().as_ref()), + ModulesLoadStrategy::WasmOnly => module.extension().map_or(false, |e| e == ".wasm"), } } @@ -70,6 +76,19 @@ impl<'a> ModulesLoadStrategy<'a> { _ => <_>::default(), } } + + #[inline] + pub fn extract_module_name(&self, module: String) -> String { + match self { + ModulesLoadStrategy::WasmOnly => { + let path: &Path = module.as_ref(); + path.file_stem() + .map(|s| s.to_string_lossy().to_string()) + .unwrap_or(module) + } + _ => module, + } + } } pub struct FluenceFaaS { @@ -94,7 +113,7 @@ impl FluenceFaaS { .modules_dir .as_ref() .map_or(Ok(HashMap::new()), |dir| { - Self::load_modules(dir, ModulesLoadStrategy::All) + Self::load_modules(dir, ModulesLoadStrategy::WasmOnly) })?; Self::with_modules::(modules, config) @@ -171,11 +190,12 @@ impl FluenceFaaS { .into_string() .map_err(|name| IOError(format!("invalid file name: {:?}", name)))?; - if modules.should_load(&module_name) { + if modules.should_load(&module_name.as_ref()) { let module_bytes = fs::read(path)?; + let module_name = modules.extract_module_name(module_name); if hash_map.insert(module_name, module_bytes).is_some() { return Err(FaaSError::ConfigParseError(String::from( - "config contains modules with the same name", + "module {} is duplicated in config", ))); } } diff --git a/fluence-faas/src/misc/config.rs b/fluence-faas/src/misc/config.rs index 444daf19..f9c6dcb8 100644 --- a/fluence-faas/src/misc/config.rs +++ b/fluence-faas/src/misc/config.rs @@ -21,6 +21,7 @@ use serde_derive::Serialize; use serde_derive::Deserialize; use std::convert::TryInto; +use std::path::PathBuf; /* An example of the config: @@ -60,10 +61,22 @@ service_base_dir = "/Users/user/tmp" pub struct RawModulesConfig { pub modules_dir: Option, pub service_base_dir: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] pub module: Vec, pub default: Option, } +impl RawModulesConfig { + /// Load config from filesystem + pub fn load>(path: P) -> Result { + let path = path.into(); + let bytes = std::fs::read(&path)?; + toml::from_slice(bytes.as_slice()).map_err(|e| { + FaaSError::ConfigParseError(format!("Error parsing config {:?}: {:?}", path, e)) + }) + } +} + impl TryInto for RawModulesConfig { type Error = FaaSError; diff --git a/tools/repl/src/repl.rs b/tools/repl/src/repl.rs index 08900f78..1e3b9f0d 100644 --- a/tools/repl/src/repl.rs +++ b/tools/repl/src/repl.rs @@ -16,7 +16,7 @@ use crate::Result; -use fluence_app_service::AppService; +use fluence_app_service::{AppService, RawModulesConfig}; use std::path::PathBuf; use std::fs; @@ -162,18 +162,15 @@ impl REPL { let service_id = uuid::Uuid::new_v4().to_string(); let start = Instant::now(); - let app_service = match config_file_path { - Some(config_file_path) => { - let config_file_path = config_file_path.into(); - AppService::with_raw_config(config_file_path, &service_id, Some(&tmp_path)) - } - None => { - let mut config: fluence_app_service::RawModulesConfig = <_>::default(); - config.service_base_dir = Some(tmp_path); - AppService::new(std::iter::empty(), config, &service_id) - } - }?; + let mut config = config_file_path + .map(|p| RawModulesConfig::load(p.into())) + .transpose()? + .unwrap_or_default(); + config.service_base_dir = Some(tmp_path); + + let app_service = AppService::new(config, &service_id, vec![])?; + let duration = start.elapsed(); println!(