fluencelabs/marine
1
0
Fork 0
mirror of https://github.com/fluencelabs/marine.git synced 2024-12-12 06:45:32 +00:00
Code Issues Projects Releases Wiki Activity

Allow to specify module file name in the config (#62)

Browse Source
This commit is contained in:
folex 2021-02-12 13:44:52 +03:00 committed by GitHub
parent 9dc33e99f0
commit 12795cc0cb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 156 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.circleci/config.yml
View File

@ -3,8 +3,9 @@ jobs:
fce:
docker:
- image: circleci/rust:latest
resource_class: xlarge
environment:
RUST_BACKTRACE: 1
RUST_BACKTRACE: full
#RUST_TEST_THREADS: 1
steps:
- checkout

1
Cargo.lock generated
View File

@ -781,6 +781,7 @@ dependencies = [
"serde",
"serde_derive",
"serde_json",
"thiserror",
"toml",
"wasmer-interface-types-fl",
"wasmer-runtime-core-fl",

20
aquamarine-vm/src/aquamarine_stepper_vm.rs
View File

@ -18,7 +18,7 @@ use crate::{Result, IType, CallServiceClosure};
use crate::AquamarineVMError;
use crate::config::AquamarineVMConfig;
use fluence_faas::{FaaSConfig, HostExportedFunc};
use fluence_faas::{FaaSConfig, HostExportedFunc, ModuleDescriptor};
use fluence_faas::FluenceFaaS;
use fluence_faas::HostImportDescriptor;
use fluence_faas::IValue;
@ -180,10 +180,10 @@ fn call_service_descriptor(
}
}
/// Splits given path into its directory and file stem
/// Splits given path into its directory and file name
///
/// # Example
/// For path `/path/to/aquamarine.wasm` result will be `Ok(PathBuf(/path/to), "aquamarine")`
/// For path `/path/to/aquamarine.wasm` result will be `Ok(PathBuf(/path/to), "aquamarine.wasm")`
fn split_dirname(path: PathBuf) -> Result<(PathBuf, String)> {
use AquamarineVMError::InvalidAquamarinePath;
@ -201,16 +201,16 @@ fn split_dirname(path: PathBuf) -> Result<(PathBuf, String)> {
});
}
let file_stem = path
.file_stem()
let file_name = path
.file_name()
.expect("checked to be a file, file name must be defined");
let file_stem = file_stem.to_string_lossy().into_owned();
let file_name = file_name.to_string_lossy().into_owned();
let mut path = path;
// drop file name from path
path.pop();
Ok((path, file_stem))
Ok((path, file_name))
}
fn make_faas_config(
@ -242,7 +242,11 @@ fn make_faas_config(
FaaSConfig {
modules_dir: Some(aquamarine_wasm_dir),
modules_config: vec![(String::from(aquamarine_wasm_file), aquamarine_module_config)],
modules_config: vec![ModuleDescriptor {
file_name: String::from(aquamarine_wasm_file),
import_name: String::from(aquamarine_wasm_file),
config: aquamarine_module_config,
}],
default_modules_config: None,
}
}

4
crates/wit-parser/src/extractor.rs
View File

@ -21,9 +21,9 @@ pub use functions::*;
pub use wit::*;
use crate::Result;
use std::path::PathBuf;
use std::path::Path;
pub fn module_interface(module_path: PathBuf) -> Result<ServiceInterface> {
pub fn module_interface(module_path: &Path) -> Result<ServiceInterface> {
use fce_wit_interfaces::FCEWITInterfaces;
let wit_section_bytes = extract_wit_section_bytes(module_path)?;

10
crates/wit-parser/src/extractor/wit.rs
View File

@ -21,11 +21,11 @@ use walrus::{IdsToIndices, ModuleConfig};
use wasmer_wit::ast::Interfaces;
use wasmer_core::Module as WasmerModule;
use std::path::PathBuf;
use std::path::Path;
/// Extracts WIT section of provided Wasm binary and converts it to a string.
pub fn extract_text_wit(wasm_file_path: PathBuf) -> Result<String, WITParserError> {
let wit_section_bytes = extract_wit_section_bytes(wasm_file_path)?;
pub fn extract_text_wit(wasm_file_path: &Path) -> Result<String, WITParserError> {
let wit_section_bytes = extract_wit_section_bytes(&wasm_file_path)?;
let wit = extract_wit_with_fn(&wit_section_bytes)?;
Ok((&wit).to_string())
}
@ -53,9 +53,7 @@ pub(crate) fn extract_wit_with_fn(
}
}
pub(crate) fn extract_wit_section_bytes(
wasm_file_path: PathBuf,
) -> Result<Vec<u8>, WITParserError> {
pub(crate) fn extract_wit_section_bytes(wasm_file_path: &Path) -> Result<Vec<u8>, WITParserError> {
let module = ModuleConfig::new()
.parse_file(wasm_file_path)
.map_err(WITParserError::CorruptedWasmFile)?;

8
engine/src/engine.rs
View File

@ -65,16 +65,16 @@ impl FCE {
/// Load a new module inside FCE.
pub fn load_module<S: Into<String>>(
&mut self,
name: S,
import_name: S,
wasm_bytes: &[u8],
config: FCEModuleConfig,
) -> Result<()> {
self.load_module_(name.into(), wasm_bytes, config)
self.load_module_(import_name.into(), wasm_bytes, config)
}
fn load_module_(
&mut self,
name: String,
import_name: String,
wasm_bytes: &[u8],
config: FCEModuleConfig,
) -> Result<()> {
@ -82,7 +82,7 @@ impl FCE {
let module = FCEModule::new(&wasm_bytes, config, &self.modules)?;
match self.modules.entry(name) {
match self.modules.entry(import_name) {
Entry::Vacant(entry) => {
entry.insert(module);
Ok(())

2
fluence-app-service/src/lib.rs
View File

@ -48,9 +48,9 @@ pub use fluence_faas::TomlFaaSConfig;
pub use fluence_faas::TomlFaaSModuleConfig;
pub use fluence_faas::TomlFaaSNamedModuleConfig;
pub use fluence_faas::TomlWASIConfig;
pub use fluence_faas::ModuleDescriptor;
pub use fluence_faas::from_toml_faas_config;
pub use fluence_faas::from_toml_module_config;
pub use fluence_faas::from_toml_named_module_config;
pub use fluence_faas::from_toml_wasi_config;
pub use fluence_faas::FaaSError;

10
fluence-app-service/src/service.rs
View File

@ -55,7 +55,7 @@ impl AppService {
"config should contain at least one module",
))
})?
.0
.import_name
.clone();
let service_id = service_id.into();
@ -150,9 +150,11 @@ impl AppService {
service_id.into_bytes(),
);
for (_, module_config) in &mut config.faas_config.modules_config {
module_config.extend_wasi_envs(envs.clone());
module_config.extend_wasi_files(preopened_files.clone(), mapped_dirs.clone());
for module in &mut config.faas_config.modules_config {
module.config.extend_wasi_envs(envs.clone());
module
.config
.extend_wasi_files(preopened_files.clone(), mapped_dirs.clone());
}
Ok(())

1
fluence-faas/Cargo.toml
View File

@ -25,6 +25,7 @@ itertools = "0.9.0"
cmd_lib = "0.7.8"
log = "0.4.8"
safe-transmute = "0.11.0"
thiserror = "1.0.23"
[dev-dependencies]
once_cell = "1.4.0"

12
fluence-faas/src/config.rs
View File

@ -18,7 +18,15 @@ use fce::HostImportDescriptor;
use std::collections::HashMap;
use std::collections::HashSet;
use std::path::PathBuf;
use std::path::{PathBuf};
/// Info to load a module from filesystem into runtime.
#[derive(Default)]
pub struct ModuleDescriptor {
pub file_name: String,
pub import_name: String,
pub config: FaaSModuleConfig,
}
/// Describes the behaviour of FluenceFaaS.
#[derive(Default)]
@ -27,7 +35,7 @@ pub struct FaaSConfig {
pub modules_dir: Option<PathBuf>,
/// Settings for a module with particular name (not HashMap because the order is matter).
pub modules_config: Vec<(String, FaaSModuleConfig)>,
pub modules_config: Vec<ModuleDescriptor>,
/// Settings for a module that name's not been found in modules_config.
pub default_modules_config: Option<FaaSModuleConfig>,

55
fluence-faas/src/errors.rs
View File

@ -17,66 +17,59 @@
use fce::FCEError;
use std::io::Error as IOError;
use std::error::Error;
use thiserror::Error;
use std::path::PathBuf;
#[derive(Debug)]
#[derive(Debug, Error)]
pub enum FaaSError {
/// An error related to config parsing.
ConfigParseError(String),
/// Errors that happened due to invalid config content
#[error("InvalidConfig: {0}")]
InvalidConfig(String),
/// An error occurred at the instantiation step.
InstantiationError(String),
#[error(
"module with name {module_import_name} is specified in config (dir: {modules_dir:?}), \
but not found in provided modules: {provided_modules:?}"
)]
InstantiationError {
module_import_name: String,
modules_dir: Option<PathBuf>,
provided_modules: Vec<String>,
},
/// Various errors related to file i/o.
#[error("IOError: {0}")]
IOError(String),
/// A function with specified name is missing.
#[error("function with name `{0}` is missing")]
MissingFunctionError(String),
/// An argument with specified name is missing.
#[error(r#"argument with name "{0}" is missing"#)]
MissingArgumentError(String),
/// Returns when there is no module with such name.
#[error(r#"module with name "{0}" is missing"#)]
NoSuchModule(String),
/// Provided arguments aren't compatible with a called function signature.
#[error("JsonArgumentsDeserializationError: {0}")]
JsonArgumentsDeserializationError(String),
/// Returned outputs aren't compatible with a called function signature.
#[error("JsonOutputSerializationError: {0}")]
JsonOutputSerializationError(String),
/// Errors related to invalid config.
#[error("ParseConfigError: {0}")]
ParseConfigError(toml::de::Error),
/// FCE errors.
#[error("EngineError: {0}")]
EngineError(FCEError),
}
impl Error for FaaSError {}
impl std::fmt::Display for FaaSError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
match self {
FaaSError::ConfigParseError(err_msg) => write!(f, "{}", err_msg),
FaaSError::InstantiationError(err_msg) => write!(f, "{}", err_msg),
FaaSError::MissingFunctionError(func_name) => {
write!(f, "function with name `{}` is missing", func_name)
}
FaaSError::MissingArgumentError(arg_name) => {
write!(f, r#"argument with name "{}" is missing"#, arg_name)
}
FaaSError::NoSuchModule(module_name) => {
write!(f, r#"module with name "{}" is missing"#, module_name)
}
FaaSError::JsonArgumentsDeserializationError(args) => write!(f, "{}", args),
FaaSError::JsonOutputSerializationError(args) => write!(f, "{}", args),
FaaSError::IOError(err_msg) => write!(f, "{}", err_msg),
FaaSError::EngineError(err) => write!(f, "{}", err),
FaaSError::ParseConfigError(err) => write!(f, "{}", err),
}
}
}
impl From<IOError> for FaaSError {
fn from(err: IOError) -> Self {
FaaSError::IOError(format!("{}", err))
@ -91,7 +84,7 @@ impl From<FCEError> for FaaSError {
impl From<toml::de::Error> for FaaSError {
fn from(err: toml::de::Error) -> Self {
FaaSError::ConfigParseError(format!("{}", err))
FaaSError::ParseConfigError(err)
}
}

42
fluence-faas/src/faas.rs
View File

@ -34,10 +34,8 @@ use fluence_sdk_main::CallParameters;
use serde_json::Value as JValue;
use std::cell::RefCell;
use std::convert::TryInto;
use std::collections::HashSet;
use std::collections::HashMap;
use std::rc::Rc;
use std::path::PathBuf;
struct ModuleInterface {
function_signatures: HashMap<SharedString, (Rc<Vec<IFunctionArg>>, Rc<Vec<IType>>)>,
@ -59,12 +57,6 @@ pub struct FluenceFaaS {
}
impl FluenceFaaS {
/// Creates FaaS from config on filesystem.
pub fn with_config_path<P: Into<PathBuf>>(config_file_path: P) -> Result<Self> {
let config = crate::raw_toml_config::TomlFaaSConfig::load(config_file_path.into())?;
Self::with_raw_config(config)
}
/// Creates FaaS from config deserialized from TOML.
pub fn with_raw_config<C>(config: C) -> Result<Self>
where
@ -73,13 +65,11 @@ impl FluenceFaaS {
{
let config = config.try_into()?;
let modules = config
.modules_dir
.as_ref()
.map_or(Ok(HashMap::new()), |dir| {
load_modules_from_fs(dir, ModulesLoadStrategy::WasmOnly)
})?;
Self::with_modules::<FaaSConfig>(modules, config)
.modules_config
.iter()
.map(|m| (m.file_name.clone(), m.import_name.clone()))
.collect();
Self::with_module_names::<FaaSConfig>(&modules, config)
}
/// Creates FaaS with given modules.
@ -98,22 +88,22 @@ impl FluenceFaaS {
let wasm_log_env = std::env::var(WASM_LOG_ENV_NAME).unwrap_or_default();
let logger_filter = LoggerFilter::from_env_string(&wasm_log_env);
for (module_name, module_config) in config.modules_config {
let module_bytes =
modules.remove(&module_name).ok_or_else(|| {
FaaSError::InstantiationError(format!(
"module with name {} is specified in config (dir: {:?}), but not found in provided modules: {:?}",
module_name, modules_dir, modules.keys().collect::<Vec<_>>()
))
for module in config.modules_config {
let module_bytes = modules.remove(&module.import_name).ok_or_else(|| {
FaaSError::InstantiationError {
module_import_name: module.import_name.clone(),
modules_dir: modules_dir.clone(),
provided_modules: modules.keys().cloned().collect::<Vec<_>>(),
}
})?;
let fce_module_config = crate::misc::make_fce_config(
module_name.clone(),
Some(module_config),
module.import_name.clone(),
Some(module.config),
call_parameters.clone(),
&logger_filter,
)?;
fce.load_module(module_name, &module_bytes, fce_module_config)?;
fce.load_module(module.import_name, &module_bytes, fce_module_config)?;
}
Ok(Self {
@ -124,7 +114,7 @@ impl FluenceFaaS {
}
/// Searches for modules in `config.modules_dir`, loads only those in the `names` set
pub fn with_module_names<C>(names: &HashSet<String>, config: C) -> Result<Self>
pub fn with_module_names<C>(names: &HashMap<String, String>, config: C) -> Result<Self>
where
C: TryInto<FaaSConfig>,
FaaSError: From<C::Error>,

2
fluence-faas/src/lib.rs
View File

@ -41,6 +41,7 @@ pub use faas_interface::itype_text_view;
pub use config::FaaSConfig;
pub use config::FaaSModuleConfig;
pub use config::FaaSWASIConfig;
pub use config::ModuleDescriptor;
pub use raw_toml_config::TomlFaaSConfig;
pub use raw_toml_config::TomlFaaSModuleConfig;
@ -48,7 +49,6 @@ pub use raw_toml_config::TomlFaaSNamedModuleConfig;
pub use raw_toml_config::TomlWASIConfig;
pub use raw_toml_config::from_toml_faas_config;
pub use raw_toml_config::from_toml_module_config;
pub use raw_toml_config::from_toml_named_module_config;
pub use raw_toml_config::from_toml_wasi_config;
pub use errors::FaaSError;

63
fluence-faas/src/misc/modules_load_strategy.rs
View File

@ -15,16 +15,25 @@
*/
use std::path::Path;
use std::collections::HashSet;
use std::collections::{HashMap, HashSet};
use crate::FaaSError;
use std::ffi::OsStr;
use std::borrow::Cow;
type ImportName = String;
type FileName = String;
/// Strategies for module loading.
#[derive(Debug, Clone)]
pub enum ModulesLoadStrategy<'a> {
/// Try to load all files in a given directory
/// Load all files in a given directory
#[allow(dead_code)]
All,
/// Try to load only files contained in the set
Named(&'a HashSet<String>),
/// Load only files contained in the set
/// Correspondence between module file name and import name is crucial for `extract_module_name`
Named(&'a HashMap<FileName, ImportName>),
/// In a given directory, try to load all files ending with .wasm
#[allow(dead_code)]
WasmOnly,
}
@ -34,7 +43,7 @@ impl<'a> ModulesLoadStrategy<'a> {
pub fn should_load(&self, module: &Path) -> bool {
match self {
ModulesLoadStrategy::All => true,
ModulesLoadStrategy::Named(set) => set.contains(module.to_string_lossy().as_ref()),
ModulesLoadStrategy::Named(map) => map.contains_key(module.to_string_lossy().as_ref()),
ModulesLoadStrategy::WasmOnly => module.extension().map_or(false, |e| e == "wasm"),
}
}
@ -50,28 +59,46 @@ impl<'a> ModulesLoadStrategy<'a> {
#[inline]
/// Returns difference between required and loaded modules.
pub fn missing_modules<'s>(&self, loaded: impl Iterator<Item = &'s String>) -> Vec<&'s String> {
pub fn missing_modules<'i>(
&self,
loaded: impl Iterator<Item = &'i String>,
) -> HashSet<&String> {
match self {
ModulesLoadStrategy::Named(set) => loaded.fold(vec![], |mut vec, module| {
if !set.contains(module) {
vec.push(module)
ModulesLoadStrategy::Named(map) => {
let set: HashSet<_> = map.keys().collect();
loaded.fold(set, |mut set, module| {
set.remove(module);
set
})
}
vec
}),
_ => <_>::default(),
}
}
#[inline]
pub fn extract_module_name(&self, module: String) -> String {
pub fn extract_module_name(&self, module_path: &Path) -> Result<String, FaaSError> {
use FaaSError::*;
fn as_str<'a>(
os_str: Option<&'a OsStr>,
path: &'a Path,
) -> Result<Cow<'a, str>, FaaSError> {
os_str
.map(|s| s.to_string_lossy())
.ok_or_else(|| IOError(format!("No file name in path {:?}", path)))
}
match self {
ModulesLoadStrategy::WasmOnly => {
let path: &Path = module.as_ref();
path.file_stem()
.map(|s| s.to_string_lossy().to_string())
.unwrap_or(module)
Self::Named(map) => {
let file_name = as_str(module_path.file_name(), module_path)?;
// Take import_name from the mapping and return it
let import_name = map.get(file_name.as_ref());
let import_name = import_name.ok_or_else(|| NoSuchModule(file_name.to_string()))?;
Ok(import_name.clone())
}
_ => module,
// for other strategies, simply use file name without extension
_ => Ok(as_str(module_path.file_stem(), module_path)?.to_string()),
}
}
}

22
fluence-faas/src/misc/utils.rs
View File

@ -32,7 +32,7 @@ use wasmer_wit::IType;
use std::collections::HashMap;
use std::cell::RefCell;
use std::path::PathBuf;
use std::path::{PathBuf, Path};
use std::rc::Rc;
use std::ops::Deref;
@ -176,18 +176,16 @@ pub(crate) fn load_modules_from_fs(
return Ok(hash_map);
}
let module_name = path
.file_name()
.ok_or_else(|| IOError(format!("No file name in path {:?}", path)))?
.to_os_string()
.into_string()
.map_err(|name| IOError(format!("invalid file name: {:?}", name)))?;
let file_name = Path::new(
path.file_name()
.ok_or_else(|| IOError(format!("No file name in path {:?}", path)))?,
);
if modules.should_load(&module_name.as_ref()) {
let module_bytes = std::fs::read(path)?;
let module_name = modules.extract_module_name(module_name);
if modules.should_load(&file_name) {
let module_bytes = std::fs::read(&path)?;
let module_name = modules.extract_module_name(&path)?;
if hash_map.insert(module_name, module_bytes).is_some() {
return Err(FaaSError::ConfigParseError(String::from(
return Err(FaaSError::InvalidConfig(String::from(
"module {} is duplicated in config",
)));
}
@ -199,7 +197,7 @@ pub(crate) fn load_modules_from_fs(
if modules.required_modules_len() > loaded.len() {
let loaded = loaded.iter().map(|(n, _)| n);
let not_found = modules.missing_modules(loaded);
return Err(FaaSError::ConfigParseError(format!(
return Err(FaaSError::InvalidConfig(format!(
"the following modules were not found: {:?}",
not_found
)));

47
fluence-faas/src/raw_toml_config.rs
View File

@ -25,6 +25,7 @@ use std::convert::TryInto;
use std::collections::HashMap;
use std::collections::HashSet;
use std::path::PathBuf;
use serde::export::TryFrom;
/*
An example of the config:
@ -72,9 +73,7 @@ impl TomlFaaSConfig {
pub fn load<P: Into<PathBuf>>(path: P) -> Result<Self> {
let path = path.into();
let file_content = std::fs::read(&path)?;
toml::from_slice(&file_content).map_err(|e| {
FaaSError::ConfigParseError(format!("Error parsing config {:?}: {:?}", path, e))
})
Ok(toml::from_slice(&file_content)?)
}
}
@ -89,23 +88,21 @@ impl TryInto<FaaSConfig> for TomlFaaSConfig {
#[derive(Deserialize, Serialize, Debug, Clone, Default)]
pub struct TomlFaaSNamedModuleConfig {
pub name: String,
#[serde(default)]
pub file_name: Option<String>,
#[serde(flatten)]
pub config: TomlFaaSModuleConfig,
}
impl TryInto<(String, FaaSModuleConfig)> for TomlFaaSNamedModuleConfig {
impl TryFrom<TomlFaaSNamedModuleConfig> for ModuleDescriptor {
type Error = FaaSError;
fn try_into(self) -> Result<(String, FaaSModuleConfig)> {
from_toml_named_module_config(self)
}
}
impl TryInto<FaaSModuleConfig> for TomlFaaSNamedModuleConfig {
type Error = FaaSError;
fn try_into(self) -> Result<FaaSModuleConfig> {
from_toml_named_module_config(self).map(|(_, module_config)| module_config)
fn try_from(config: TomlFaaSNamedModuleConfig) -> Result<Self> {
Ok(ModuleDescriptor {
file_name: config.file_name.unwrap_or(format!("{}.wasm", config.name)),
import_name: config.name,
config: from_toml_module_config(config.config)?,
})
}
}
@ -118,18 +115,6 @@ pub struct TomlFaaSModuleConfig {
pub logging_mask: Option<i32>,
}
impl TomlFaaSNamedModuleConfig {
pub fn new<S>(name: S) -> Self
where
S: Into<String>,
{
Self {
name: name.into(),
config: <_>::default(),
}
}
}
#[derive(Deserialize, Serialize, Debug, Clone, Default)]
pub struct TomlWASIConfig {
pub preopened_files: Option<Vec<String>>,
@ -142,7 +127,7 @@ pub fn from_toml_faas_config(config: TomlFaaSConfig) -> Result<FaaSConfig> {
let modules_config = config
.module
.into_iter()
.map(from_toml_named_module_config)
.map(ModuleDescriptor::try_from)
.collect::<Result<Vec<_>>>()?;
let default_modules_config = config.default.map(from_toml_module_config).transpose()?;
@ -154,13 +139,6 @@ pub fn from_toml_faas_config(config: TomlFaaSConfig) -> Result<FaaSConfig> {
})
}
pub fn from_toml_named_module_config(
config: TomlFaaSNamedModuleConfig,
) -> Result<(String, FaaSModuleConfig)> {
let module_config = from_toml_module_config(config.config)?;
Ok((config.name, module_config))
}
pub fn from_toml_module_config(config: TomlFaaSModuleConfig) -> Result<FaaSModuleConfig> {
let mounted_binaries = config.mounted_binaries.unwrap_or_default();
let mounted_binaries = mounted_binaries
@ -236,6 +214,7 @@ mod tests {
fn serialize_named() {
let config = TomlFaaSNamedModuleConfig {
name: "name".to_string(),
file_name: Some("file_name".to_string()),
config: TomlFaaSModuleConfig {
mem_pages_count: Some(100),
logger_enabled: Some(false),

4
tools/cli/src/main.rs
View File

@ -70,9 +70,9 @@ pub fn main() -> std::result::Result<(), anyhow::Error> {
}
("show", Some(arg)) => {
let wasm_path = arg.value_of(args::IN_WASM_PATH).unwrap();
let wasm_path = std::path::PathBuf::from(wasm_path);
let wasm_path = std::path::Path::new(wasm_path);
let result = fce_wit_parser::extract_text_wit(wasm_path)?;
let result = fce_wit_parser::extract_text_wit(&wasm_path)?;
println!("{}", result);
Ok(())