mirror of
https://github.com/fluencelabs/marine.git
synced 2024-12-12 14:55:32 +00:00
Allow to specify module file name in the config (#62)
This commit is contained in:
parent
9dc33e99f0
commit
12795cc0cb
@ -3,8 +3,9 @@ jobs:
|
|||||||
fce:
|
fce:
|
||||||
docker:
|
docker:
|
||||||
- image: circleci/rust:latest
|
- image: circleci/rust:latest
|
||||||
|
resource_class: xlarge
|
||||||
environment:
|
environment:
|
||||||
RUST_BACKTRACE: 1
|
RUST_BACKTRACE: full
|
||||||
#RUST_TEST_THREADS: 1
|
#RUST_TEST_THREADS: 1
|
||||||
steps:
|
steps:
|
||||||
- checkout
|
- checkout
|
||||||
|
1
Cargo.lock
generated
1
Cargo.lock
generated
@ -781,6 +781,7 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
"serde_derive",
|
"serde_derive",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
"thiserror",
|
||||||
"toml",
|
"toml",
|
||||||
"wasmer-interface-types-fl",
|
"wasmer-interface-types-fl",
|
||||||
"wasmer-runtime-core-fl",
|
"wasmer-runtime-core-fl",
|
||||||
|
@ -18,7 +18,7 @@ use crate::{Result, IType, CallServiceClosure};
|
|||||||
use crate::AquamarineVMError;
|
use crate::AquamarineVMError;
|
||||||
use crate::config::AquamarineVMConfig;
|
use crate::config::AquamarineVMConfig;
|
||||||
|
|
||||||
use fluence_faas::{FaaSConfig, HostExportedFunc};
|
use fluence_faas::{FaaSConfig, HostExportedFunc, ModuleDescriptor};
|
||||||
use fluence_faas::FluenceFaaS;
|
use fluence_faas::FluenceFaaS;
|
||||||
use fluence_faas::HostImportDescriptor;
|
use fluence_faas::HostImportDescriptor;
|
||||||
use fluence_faas::IValue;
|
use fluence_faas::IValue;
|
||||||
@ -180,10 +180,10 @@ fn call_service_descriptor(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Splits given path into its directory and file stem
|
/// Splits given path into its directory and file name
|
||||||
///
|
///
|
||||||
/// # Example
|
/// # Example
|
||||||
/// For path `/path/to/aquamarine.wasm` result will be `Ok(PathBuf(/path/to), "aquamarine")`
|
/// For path `/path/to/aquamarine.wasm` result will be `Ok(PathBuf(/path/to), "aquamarine.wasm")`
|
||||||
fn split_dirname(path: PathBuf) -> Result<(PathBuf, String)> {
|
fn split_dirname(path: PathBuf) -> Result<(PathBuf, String)> {
|
||||||
use AquamarineVMError::InvalidAquamarinePath;
|
use AquamarineVMError::InvalidAquamarinePath;
|
||||||
|
|
||||||
@ -201,16 +201,16 @@ fn split_dirname(path: PathBuf) -> Result<(PathBuf, String)> {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
let file_stem = path
|
let file_name = path
|
||||||
.file_stem()
|
.file_name()
|
||||||
.expect("checked to be a file, file name must be defined");
|
.expect("checked to be a file, file name must be defined");
|
||||||
let file_stem = file_stem.to_string_lossy().into_owned();
|
let file_name = file_name.to_string_lossy().into_owned();
|
||||||
|
|
||||||
let mut path = path;
|
let mut path = path;
|
||||||
// drop file name from path
|
// drop file name from path
|
||||||
path.pop();
|
path.pop();
|
||||||
|
|
||||||
Ok((path, file_stem))
|
Ok((path, file_name))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn make_faas_config(
|
fn make_faas_config(
|
||||||
@ -242,7 +242,11 @@ fn make_faas_config(
|
|||||||
|
|
||||||
FaaSConfig {
|
FaaSConfig {
|
||||||
modules_dir: Some(aquamarine_wasm_dir),
|
modules_dir: Some(aquamarine_wasm_dir),
|
||||||
modules_config: vec![(String::from(aquamarine_wasm_file), aquamarine_module_config)],
|
modules_config: vec![ModuleDescriptor {
|
||||||
|
file_name: String::from(aquamarine_wasm_file),
|
||||||
|
import_name: String::from(aquamarine_wasm_file),
|
||||||
|
config: aquamarine_module_config,
|
||||||
|
}],
|
||||||
default_modules_config: None,
|
default_modules_config: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -21,9 +21,9 @@ pub use functions::*;
|
|||||||
pub use wit::*;
|
pub use wit::*;
|
||||||
|
|
||||||
use crate::Result;
|
use crate::Result;
|
||||||
use std::path::PathBuf;
|
use std::path::Path;
|
||||||
|
|
||||||
pub fn module_interface(module_path: PathBuf) -> Result<ServiceInterface> {
|
pub fn module_interface(module_path: &Path) -> Result<ServiceInterface> {
|
||||||
use fce_wit_interfaces::FCEWITInterfaces;
|
use fce_wit_interfaces::FCEWITInterfaces;
|
||||||
|
|
||||||
let wit_section_bytes = extract_wit_section_bytes(module_path)?;
|
let wit_section_bytes = extract_wit_section_bytes(module_path)?;
|
||||||
|
@ -21,11 +21,11 @@ use walrus::{IdsToIndices, ModuleConfig};
|
|||||||
use wasmer_wit::ast::Interfaces;
|
use wasmer_wit::ast::Interfaces;
|
||||||
use wasmer_core::Module as WasmerModule;
|
use wasmer_core::Module as WasmerModule;
|
||||||
|
|
||||||
use std::path::PathBuf;
|
use std::path::Path;
|
||||||
|
|
||||||
/// Extracts WIT section of provided Wasm binary and converts it to a string.
|
/// Extracts WIT section of provided Wasm binary and converts it to a string.
|
||||||
pub fn extract_text_wit(wasm_file_path: PathBuf) -> Result<String, WITParserError> {
|
pub fn extract_text_wit(wasm_file_path: &Path) -> Result<String, WITParserError> {
|
||||||
let wit_section_bytes = extract_wit_section_bytes(wasm_file_path)?;
|
let wit_section_bytes = extract_wit_section_bytes(&wasm_file_path)?;
|
||||||
let wit = extract_wit_with_fn(&wit_section_bytes)?;
|
let wit = extract_wit_with_fn(&wit_section_bytes)?;
|
||||||
Ok((&wit).to_string())
|
Ok((&wit).to_string())
|
||||||
}
|
}
|
||||||
@ -53,9 +53,7 @@ pub(crate) fn extract_wit_with_fn(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn extract_wit_section_bytes(
|
pub(crate) fn extract_wit_section_bytes(wasm_file_path: &Path) -> Result<Vec<u8>, WITParserError> {
|
||||||
wasm_file_path: PathBuf,
|
|
||||||
) -> Result<Vec<u8>, WITParserError> {
|
|
||||||
let module = ModuleConfig::new()
|
let module = ModuleConfig::new()
|
||||||
.parse_file(wasm_file_path)
|
.parse_file(wasm_file_path)
|
||||||
.map_err(WITParserError::CorruptedWasmFile)?;
|
.map_err(WITParserError::CorruptedWasmFile)?;
|
||||||
|
@ -65,16 +65,16 @@ impl FCE {
|
|||||||
/// Load a new module inside FCE.
|
/// Load a new module inside FCE.
|
||||||
pub fn load_module<S: Into<String>>(
|
pub fn load_module<S: Into<String>>(
|
||||||
&mut self,
|
&mut self,
|
||||||
name: S,
|
import_name: S,
|
||||||
wasm_bytes: &[u8],
|
wasm_bytes: &[u8],
|
||||||
config: FCEModuleConfig,
|
config: FCEModuleConfig,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
self.load_module_(name.into(), wasm_bytes, config)
|
self.load_module_(import_name.into(), wasm_bytes, config)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn load_module_(
|
fn load_module_(
|
||||||
&mut self,
|
&mut self,
|
||||||
name: String,
|
import_name: String,
|
||||||
wasm_bytes: &[u8],
|
wasm_bytes: &[u8],
|
||||||
config: FCEModuleConfig,
|
config: FCEModuleConfig,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
@ -82,7 +82,7 @@ impl FCE {
|
|||||||
|
|
||||||
let module = FCEModule::new(&wasm_bytes, config, &self.modules)?;
|
let module = FCEModule::new(&wasm_bytes, config, &self.modules)?;
|
||||||
|
|
||||||
match self.modules.entry(name) {
|
match self.modules.entry(import_name) {
|
||||||
Entry::Vacant(entry) => {
|
Entry::Vacant(entry) => {
|
||||||
entry.insert(module);
|
entry.insert(module);
|
||||||
Ok(())
|
Ok(())
|
||||||
|
@ -48,9 +48,9 @@ pub use fluence_faas::TomlFaaSConfig;
|
|||||||
pub use fluence_faas::TomlFaaSModuleConfig;
|
pub use fluence_faas::TomlFaaSModuleConfig;
|
||||||
pub use fluence_faas::TomlFaaSNamedModuleConfig;
|
pub use fluence_faas::TomlFaaSNamedModuleConfig;
|
||||||
pub use fluence_faas::TomlWASIConfig;
|
pub use fluence_faas::TomlWASIConfig;
|
||||||
|
pub use fluence_faas::ModuleDescriptor;
|
||||||
pub use fluence_faas::from_toml_faas_config;
|
pub use fluence_faas::from_toml_faas_config;
|
||||||
pub use fluence_faas::from_toml_module_config;
|
pub use fluence_faas::from_toml_module_config;
|
||||||
pub use fluence_faas::from_toml_named_module_config;
|
|
||||||
pub use fluence_faas::from_toml_wasi_config;
|
pub use fluence_faas::from_toml_wasi_config;
|
||||||
|
|
||||||
pub use fluence_faas::FaaSError;
|
pub use fluence_faas::FaaSError;
|
||||||
|
@ -55,7 +55,7 @@ impl AppService {
|
|||||||
"config should contain at least one module",
|
"config should contain at least one module",
|
||||||
))
|
))
|
||||||
})?
|
})?
|
||||||
.0
|
.import_name
|
||||||
.clone();
|
.clone();
|
||||||
|
|
||||||
let service_id = service_id.into();
|
let service_id = service_id.into();
|
||||||
@ -150,9 +150,11 @@ impl AppService {
|
|||||||
service_id.into_bytes(),
|
service_id.into_bytes(),
|
||||||
);
|
);
|
||||||
|
|
||||||
for (_, module_config) in &mut config.faas_config.modules_config {
|
for module in &mut config.faas_config.modules_config {
|
||||||
module_config.extend_wasi_envs(envs.clone());
|
module.config.extend_wasi_envs(envs.clone());
|
||||||
module_config.extend_wasi_files(preopened_files.clone(), mapped_dirs.clone());
|
module
|
||||||
|
.config
|
||||||
|
.extend_wasi_files(preopened_files.clone(), mapped_dirs.clone());
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
|
@ -25,6 +25,7 @@ itertools = "0.9.0"
|
|||||||
cmd_lib = "0.7.8"
|
cmd_lib = "0.7.8"
|
||||||
log = "0.4.8"
|
log = "0.4.8"
|
||||||
safe-transmute = "0.11.0"
|
safe-transmute = "0.11.0"
|
||||||
|
thiserror = "1.0.23"
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
once_cell = "1.4.0"
|
once_cell = "1.4.0"
|
||||||
|
@ -18,7 +18,15 @@ use fce::HostImportDescriptor;
|
|||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
use std::path::PathBuf;
|
use std::path::{PathBuf};
|
||||||
|
|
||||||
|
/// Info to load a module from filesystem into runtime.
|
||||||
|
#[derive(Default)]
|
||||||
|
pub struct ModuleDescriptor {
|
||||||
|
pub file_name: String,
|
||||||
|
pub import_name: String,
|
||||||
|
pub config: FaaSModuleConfig,
|
||||||
|
}
|
||||||
|
|
||||||
/// Describes the behaviour of FluenceFaaS.
|
/// Describes the behaviour of FluenceFaaS.
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
@ -27,7 +35,7 @@ pub struct FaaSConfig {
|
|||||||
pub modules_dir: Option<PathBuf>,
|
pub modules_dir: Option<PathBuf>,
|
||||||
|
|
||||||
/// Settings for a module with particular name (not HashMap because the order is matter).
|
/// Settings for a module with particular name (not HashMap because the order is matter).
|
||||||
pub modules_config: Vec<(String, FaaSModuleConfig)>,
|
pub modules_config: Vec<ModuleDescriptor>,
|
||||||
|
|
||||||
/// Settings for a module that name's not been found in modules_config.
|
/// Settings for a module that name's not been found in modules_config.
|
||||||
pub default_modules_config: Option<FaaSModuleConfig>,
|
pub default_modules_config: Option<FaaSModuleConfig>,
|
||||||
|
@ -17,66 +17,59 @@
|
|||||||
use fce::FCEError;
|
use fce::FCEError;
|
||||||
|
|
||||||
use std::io::Error as IOError;
|
use std::io::Error as IOError;
|
||||||
use std::error::Error;
|
use thiserror::Error;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug, Error)]
|
||||||
pub enum FaaSError {
|
pub enum FaaSError {
|
||||||
/// An error related to config parsing.
|
/// Errors that happened due to invalid config content
|
||||||
ConfigParseError(String),
|
#[error("InvalidConfig: {0}")]
|
||||||
|
InvalidConfig(String),
|
||||||
|
|
||||||
/// An error occurred at the instantiation step.
|
/// An error occurred at the instantiation step.
|
||||||
InstantiationError(String),
|
#[error(
|
||||||
|
"module with name {module_import_name} is specified in config (dir: {modules_dir:?}), \
|
||||||
|
but not found in provided modules: {provided_modules:?}"
|
||||||
|
)]
|
||||||
|
InstantiationError {
|
||||||
|
module_import_name: String,
|
||||||
|
modules_dir: Option<PathBuf>,
|
||||||
|
provided_modules: Vec<String>,
|
||||||
|
},
|
||||||
|
|
||||||
/// Various errors related to file i/o.
|
/// Various errors related to file i/o.
|
||||||
|
#[error("IOError: {0}")]
|
||||||
IOError(String),
|
IOError(String),
|
||||||
|
|
||||||
/// A function with specified name is missing.
|
/// A function with specified name is missing.
|
||||||
|
#[error("function with name `{0}` is missing")]
|
||||||
MissingFunctionError(String),
|
MissingFunctionError(String),
|
||||||
|
|
||||||
/// An argument with specified name is missing.
|
/// An argument with specified name is missing.
|
||||||
|
#[error(r#"argument with name "{0}" is missing"#)]
|
||||||
MissingArgumentError(String),
|
MissingArgumentError(String),
|
||||||
|
|
||||||
/// Returns when there is no module with such name.
|
/// Returns when there is no module with such name.
|
||||||
|
#[error(r#"module with name "{0}" is missing"#)]
|
||||||
NoSuchModule(String),
|
NoSuchModule(String),
|
||||||
|
|
||||||
/// Provided arguments aren't compatible with a called function signature.
|
/// Provided arguments aren't compatible with a called function signature.
|
||||||
|
#[error("JsonArgumentsDeserializationError: {0}")]
|
||||||
JsonArgumentsDeserializationError(String),
|
JsonArgumentsDeserializationError(String),
|
||||||
|
|
||||||
/// Returned outputs aren't compatible with a called function signature.
|
/// Returned outputs aren't compatible with a called function signature.
|
||||||
|
#[error("JsonOutputSerializationError: {0}")]
|
||||||
JsonOutputSerializationError(String),
|
JsonOutputSerializationError(String),
|
||||||
|
|
||||||
/// Errors related to invalid config.
|
/// Errors related to invalid config.
|
||||||
|
#[error("ParseConfigError: {0}")]
|
||||||
ParseConfigError(toml::de::Error),
|
ParseConfigError(toml::de::Error),
|
||||||
|
|
||||||
/// FCE errors.
|
/// FCE errors.
|
||||||
|
#[error("EngineError: {0}")]
|
||||||
EngineError(FCEError),
|
EngineError(FCEError),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Error for FaaSError {}
|
|
||||||
|
|
||||||
impl std::fmt::Display for FaaSError {
|
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
|
|
||||||
match self {
|
|
||||||
FaaSError::ConfigParseError(err_msg) => write!(f, "{}", err_msg),
|
|
||||||
FaaSError::InstantiationError(err_msg) => write!(f, "{}", err_msg),
|
|
||||||
FaaSError::MissingFunctionError(func_name) => {
|
|
||||||
write!(f, "function with name `{}` is missing", func_name)
|
|
||||||
}
|
|
||||||
FaaSError::MissingArgumentError(arg_name) => {
|
|
||||||
write!(f, r#"argument with name "{}" is missing"#, arg_name)
|
|
||||||
}
|
|
||||||
FaaSError::NoSuchModule(module_name) => {
|
|
||||||
write!(f, r#"module with name "{}" is missing"#, module_name)
|
|
||||||
}
|
|
||||||
FaaSError::JsonArgumentsDeserializationError(args) => write!(f, "{}", args),
|
|
||||||
FaaSError::JsonOutputSerializationError(args) => write!(f, "{}", args),
|
|
||||||
FaaSError::IOError(err_msg) => write!(f, "{}", err_msg),
|
|
||||||
FaaSError::EngineError(err) => write!(f, "{}", err),
|
|
||||||
FaaSError::ParseConfigError(err) => write!(f, "{}", err),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl From<IOError> for FaaSError {
|
impl From<IOError> for FaaSError {
|
||||||
fn from(err: IOError) -> Self {
|
fn from(err: IOError) -> Self {
|
||||||
FaaSError::IOError(format!("{}", err))
|
FaaSError::IOError(format!("{}", err))
|
||||||
@ -91,7 +84,7 @@ impl From<FCEError> for FaaSError {
|
|||||||
|
|
||||||
impl From<toml::de::Error> for FaaSError {
|
impl From<toml::de::Error> for FaaSError {
|
||||||
fn from(err: toml::de::Error) -> Self {
|
fn from(err: toml::de::Error) -> Self {
|
||||||
FaaSError::ConfigParseError(format!("{}", err))
|
FaaSError::ParseConfigError(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -34,10 +34,8 @@ use fluence_sdk_main::CallParameters;
|
|||||||
use serde_json::Value as JValue;
|
use serde_json::Value as JValue;
|
||||||
use std::cell::RefCell;
|
use std::cell::RefCell;
|
||||||
use std::convert::TryInto;
|
use std::convert::TryInto;
|
||||||
use std::collections::HashSet;
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::rc::Rc;
|
use std::rc::Rc;
|
||||||
use std::path::PathBuf;
|
|
||||||
|
|
||||||
struct ModuleInterface {
|
struct ModuleInterface {
|
||||||
function_signatures: HashMap<SharedString, (Rc<Vec<IFunctionArg>>, Rc<Vec<IType>>)>,
|
function_signatures: HashMap<SharedString, (Rc<Vec<IFunctionArg>>, Rc<Vec<IType>>)>,
|
||||||
@ -59,12 +57,6 @@ pub struct FluenceFaaS {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl FluenceFaaS {
|
impl FluenceFaaS {
|
||||||
/// Creates FaaS from config on filesystem.
|
|
||||||
pub fn with_config_path<P: Into<PathBuf>>(config_file_path: P) -> Result<Self> {
|
|
||||||
let config = crate::raw_toml_config::TomlFaaSConfig::load(config_file_path.into())?;
|
|
||||||
Self::with_raw_config(config)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Creates FaaS from config deserialized from TOML.
|
/// Creates FaaS from config deserialized from TOML.
|
||||||
pub fn with_raw_config<C>(config: C) -> Result<Self>
|
pub fn with_raw_config<C>(config: C) -> Result<Self>
|
||||||
where
|
where
|
||||||
@ -73,13 +65,11 @@ impl FluenceFaaS {
|
|||||||
{
|
{
|
||||||
let config = config.try_into()?;
|
let config = config.try_into()?;
|
||||||
let modules = config
|
let modules = config
|
||||||
.modules_dir
|
.modules_config
|
||||||
.as_ref()
|
.iter()
|
||||||
.map_or(Ok(HashMap::new()), |dir| {
|
.map(|m| (m.file_name.clone(), m.import_name.clone()))
|
||||||
load_modules_from_fs(dir, ModulesLoadStrategy::WasmOnly)
|
.collect();
|
||||||
})?;
|
Self::with_module_names::<FaaSConfig>(&modules, config)
|
||||||
|
|
||||||
Self::with_modules::<FaaSConfig>(modules, config)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Creates FaaS with given modules.
|
/// Creates FaaS with given modules.
|
||||||
@ -98,22 +88,22 @@ impl FluenceFaaS {
|
|||||||
let wasm_log_env = std::env::var(WASM_LOG_ENV_NAME).unwrap_or_default();
|
let wasm_log_env = std::env::var(WASM_LOG_ENV_NAME).unwrap_or_default();
|
||||||
let logger_filter = LoggerFilter::from_env_string(&wasm_log_env);
|
let logger_filter = LoggerFilter::from_env_string(&wasm_log_env);
|
||||||
|
|
||||||
for (module_name, module_config) in config.modules_config {
|
for module in config.modules_config {
|
||||||
let module_bytes =
|
let module_bytes = modules.remove(&module.import_name).ok_or_else(|| {
|
||||||
modules.remove(&module_name).ok_or_else(|| {
|
FaaSError::InstantiationError {
|
||||||
FaaSError::InstantiationError(format!(
|
module_import_name: module.import_name.clone(),
|
||||||
"module with name {} is specified in config (dir: {:?}), but not found in provided modules: {:?}",
|
modules_dir: modules_dir.clone(),
|
||||||
module_name, modules_dir, modules.keys().collect::<Vec<_>>()
|
provided_modules: modules.keys().cloned().collect::<Vec<_>>(),
|
||||||
))
|
}
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
let fce_module_config = crate::misc::make_fce_config(
|
let fce_module_config = crate::misc::make_fce_config(
|
||||||
module_name.clone(),
|
module.import_name.clone(),
|
||||||
Some(module_config),
|
Some(module.config),
|
||||||
call_parameters.clone(),
|
call_parameters.clone(),
|
||||||
&logger_filter,
|
&logger_filter,
|
||||||
)?;
|
)?;
|
||||||
fce.load_module(module_name, &module_bytes, fce_module_config)?;
|
fce.load_module(module.import_name, &module_bytes, fce_module_config)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
@ -124,7 +114,7 @@ impl FluenceFaaS {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Searches for modules in `config.modules_dir`, loads only those in the `names` set
|
/// Searches for modules in `config.modules_dir`, loads only those in the `names` set
|
||||||
pub fn with_module_names<C>(names: &HashSet<String>, config: C) -> Result<Self>
|
pub fn with_module_names<C>(names: &HashMap<String, String>, config: C) -> Result<Self>
|
||||||
where
|
where
|
||||||
C: TryInto<FaaSConfig>,
|
C: TryInto<FaaSConfig>,
|
||||||
FaaSError: From<C::Error>,
|
FaaSError: From<C::Error>,
|
||||||
|
@ -41,6 +41,7 @@ pub use faas_interface::itype_text_view;
|
|||||||
pub use config::FaaSConfig;
|
pub use config::FaaSConfig;
|
||||||
pub use config::FaaSModuleConfig;
|
pub use config::FaaSModuleConfig;
|
||||||
pub use config::FaaSWASIConfig;
|
pub use config::FaaSWASIConfig;
|
||||||
|
pub use config::ModuleDescriptor;
|
||||||
|
|
||||||
pub use raw_toml_config::TomlFaaSConfig;
|
pub use raw_toml_config::TomlFaaSConfig;
|
||||||
pub use raw_toml_config::TomlFaaSModuleConfig;
|
pub use raw_toml_config::TomlFaaSModuleConfig;
|
||||||
@ -48,7 +49,6 @@ pub use raw_toml_config::TomlFaaSNamedModuleConfig;
|
|||||||
pub use raw_toml_config::TomlWASIConfig;
|
pub use raw_toml_config::TomlWASIConfig;
|
||||||
pub use raw_toml_config::from_toml_faas_config;
|
pub use raw_toml_config::from_toml_faas_config;
|
||||||
pub use raw_toml_config::from_toml_module_config;
|
pub use raw_toml_config::from_toml_module_config;
|
||||||
pub use raw_toml_config::from_toml_named_module_config;
|
|
||||||
pub use raw_toml_config::from_toml_wasi_config;
|
pub use raw_toml_config::from_toml_wasi_config;
|
||||||
|
|
||||||
pub use errors::FaaSError;
|
pub use errors::FaaSError;
|
||||||
|
@ -15,16 +15,25 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::collections::HashSet;
|
use std::collections::{HashMap, HashSet};
|
||||||
|
use crate::FaaSError;
|
||||||
|
use std::ffi::OsStr;
|
||||||
|
use std::borrow::Cow;
|
||||||
|
|
||||||
|
type ImportName = String;
|
||||||
|
type FileName = String;
|
||||||
|
|
||||||
/// Strategies for module loading.
|
/// Strategies for module loading.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
pub enum ModulesLoadStrategy<'a> {
|
pub enum ModulesLoadStrategy<'a> {
|
||||||
/// Try to load all files in a given directory
|
/// Load all files in a given directory
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
All,
|
All,
|
||||||
/// Try to load only files contained in the set
|
/// Load only files contained in the set
|
||||||
Named(&'a HashSet<String>),
|
/// Correspondence between module file name and import name is crucial for `extract_module_name`
|
||||||
|
Named(&'a HashMap<FileName, ImportName>),
|
||||||
/// In a given directory, try to load all files ending with .wasm
|
/// In a given directory, try to load all files ending with .wasm
|
||||||
|
#[allow(dead_code)]
|
||||||
WasmOnly,
|
WasmOnly,
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -34,7 +43,7 @@ impl<'a> ModulesLoadStrategy<'a> {
|
|||||||
pub fn should_load(&self, module: &Path) -> bool {
|
pub fn should_load(&self, module: &Path) -> bool {
|
||||||
match self {
|
match self {
|
||||||
ModulesLoadStrategy::All => true,
|
ModulesLoadStrategy::All => true,
|
||||||
ModulesLoadStrategy::Named(set) => set.contains(module.to_string_lossy().as_ref()),
|
ModulesLoadStrategy::Named(map) => map.contains_key(module.to_string_lossy().as_ref()),
|
||||||
ModulesLoadStrategy::WasmOnly => module.extension().map_or(false, |e| e == "wasm"),
|
ModulesLoadStrategy::WasmOnly => module.extension().map_or(false, |e| e == "wasm"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -50,28 +59,46 @@ impl<'a> ModulesLoadStrategy<'a> {
|
|||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
/// Returns difference between required and loaded modules.
|
/// Returns difference between required and loaded modules.
|
||||||
pub fn missing_modules<'s>(&self, loaded: impl Iterator<Item = &'s String>) -> Vec<&'s String> {
|
pub fn missing_modules<'i>(
|
||||||
|
&self,
|
||||||
|
loaded: impl Iterator<Item = &'i String>,
|
||||||
|
) -> HashSet<&String> {
|
||||||
match self {
|
match self {
|
||||||
ModulesLoadStrategy::Named(set) => loaded.fold(vec![], |mut vec, module| {
|
ModulesLoadStrategy::Named(map) => {
|
||||||
if !set.contains(module) {
|
let set: HashSet<_> = map.keys().collect();
|
||||||
vec.push(module)
|
loaded.fold(set, |mut set, module| {
|
||||||
|
set.remove(module);
|
||||||
|
set
|
||||||
|
})
|
||||||
}
|
}
|
||||||
vec
|
|
||||||
}),
|
|
||||||
_ => <_>::default(),
|
_ => <_>::default(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn extract_module_name(&self, module: String) -> String {
|
pub fn extract_module_name(&self, module_path: &Path) -> Result<String, FaaSError> {
|
||||||
|
use FaaSError::*;
|
||||||
|
|
||||||
|
fn as_str<'a>(
|
||||||
|
os_str: Option<&'a OsStr>,
|
||||||
|
path: &'a Path,
|
||||||
|
) -> Result<Cow<'a, str>, FaaSError> {
|
||||||
|
os_str
|
||||||
|
.map(|s| s.to_string_lossy())
|
||||||
|
.ok_or_else(|| IOError(format!("No file name in path {:?}", path)))
|
||||||
|
}
|
||||||
|
|
||||||
match self {
|
match self {
|
||||||
ModulesLoadStrategy::WasmOnly => {
|
Self::Named(map) => {
|
||||||
let path: &Path = module.as_ref();
|
let file_name = as_str(module_path.file_name(), module_path)?;
|
||||||
path.file_stem()
|
// Take import_name from the mapping and return it
|
||||||
.map(|s| s.to_string_lossy().to_string())
|
let import_name = map.get(file_name.as_ref());
|
||||||
.unwrap_or(module)
|
let import_name = import_name.ok_or_else(|| NoSuchModule(file_name.to_string()))?;
|
||||||
|
|
||||||
|
Ok(import_name.clone())
|
||||||
}
|
}
|
||||||
_ => module,
|
// for other strategies, simply use file name without extension
|
||||||
|
_ => Ok(as_str(module_path.file_stem(), module_path)?.to_string()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -32,7 +32,7 @@ use wasmer_wit::IType;
|
|||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::cell::RefCell;
|
use std::cell::RefCell;
|
||||||
use std::path::PathBuf;
|
use std::path::{PathBuf, Path};
|
||||||
use std::rc::Rc;
|
use std::rc::Rc;
|
||||||
use std::ops::Deref;
|
use std::ops::Deref;
|
||||||
|
|
||||||
@ -176,18 +176,16 @@ pub(crate) fn load_modules_from_fs(
|
|||||||
return Ok(hash_map);
|
return Ok(hash_map);
|
||||||
}
|
}
|
||||||
|
|
||||||
let module_name = path
|
let file_name = Path::new(
|
||||||
.file_name()
|
path.file_name()
|
||||||
.ok_or_else(|| IOError(format!("No file name in path {:?}", path)))?
|
.ok_or_else(|| IOError(format!("No file name in path {:?}", path)))?,
|
||||||
.to_os_string()
|
);
|
||||||
.into_string()
|
|
||||||
.map_err(|name| IOError(format!("invalid file name: {:?}", name)))?;
|
|
||||||
|
|
||||||
if modules.should_load(&module_name.as_ref()) {
|
if modules.should_load(&file_name) {
|
||||||
let module_bytes = std::fs::read(path)?;
|
let module_bytes = std::fs::read(&path)?;
|
||||||
let module_name = modules.extract_module_name(module_name);
|
let module_name = modules.extract_module_name(&path)?;
|
||||||
if hash_map.insert(module_name, module_bytes).is_some() {
|
if hash_map.insert(module_name, module_bytes).is_some() {
|
||||||
return Err(FaaSError::ConfigParseError(String::from(
|
return Err(FaaSError::InvalidConfig(String::from(
|
||||||
"module {} is duplicated in config",
|
"module {} is duplicated in config",
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
@ -199,7 +197,7 @@ pub(crate) fn load_modules_from_fs(
|
|||||||
if modules.required_modules_len() > loaded.len() {
|
if modules.required_modules_len() > loaded.len() {
|
||||||
let loaded = loaded.iter().map(|(n, _)| n);
|
let loaded = loaded.iter().map(|(n, _)| n);
|
||||||
let not_found = modules.missing_modules(loaded);
|
let not_found = modules.missing_modules(loaded);
|
||||||
return Err(FaaSError::ConfigParseError(format!(
|
return Err(FaaSError::InvalidConfig(format!(
|
||||||
"the following modules were not found: {:?}",
|
"the following modules were not found: {:?}",
|
||||||
not_found
|
not_found
|
||||||
)));
|
)));
|
||||||
|
@ -25,6 +25,7 @@ use std::convert::TryInto;
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
use serde::export::TryFrom;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
An example of the config:
|
An example of the config:
|
||||||
@ -72,9 +73,7 @@ impl TomlFaaSConfig {
|
|||||||
pub fn load<P: Into<PathBuf>>(path: P) -> Result<Self> {
|
pub fn load<P: Into<PathBuf>>(path: P) -> Result<Self> {
|
||||||
let path = path.into();
|
let path = path.into();
|
||||||
let file_content = std::fs::read(&path)?;
|
let file_content = std::fs::read(&path)?;
|
||||||
toml::from_slice(&file_content).map_err(|e| {
|
Ok(toml::from_slice(&file_content)?)
|
||||||
FaaSError::ConfigParseError(format!("Error parsing config {:?}: {:?}", path, e))
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -89,23 +88,21 @@ impl TryInto<FaaSConfig> for TomlFaaSConfig {
|
|||||||
#[derive(Deserialize, Serialize, Debug, Clone, Default)]
|
#[derive(Deserialize, Serialize, Debug, Clone, Default)]
|
||||||
pub struct TomlFaaSNamedModuleConfig {
|
pub struct TomlFaaSNamedModuleConfig {
|
||||||
pub name: String,
|
pub name: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub file_name: Option<String>,
|
||||||
#[serde(flatten)]
|
#[serde(flatten)]
|
||||||
pub config: TomlFaaSModuleConfig,
|
pub config: TomlFaaSModuleConfig,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl TryInto<(String, FaaSModuleConfig)> for TomlFaaSNamedModuleConfig {
|
impl TryFrom<TomlFaaSNamedModuleConfig> for ModuleDescriptor {
|
||||||
type Error = FaaSError;
|
type Error = FaaSError;
|
||||||
|
|
||||||
fn try_into(self) -> Result<(String, FaaSModuleConfig)> {
|
fn try_from(config: TomlFaaSNamedModuleConfig) -> Result<Self> {
|
||||||
from_toml_named_module_config(self)
|
Ok(ModuleDescriptor {
|
||||||
}
|
file_name: config.file_name.unwrap_or(format!("{}.wasm", config.name)),
|
||||||
}
|
import_name: config.name,
|
||||||
|
config: from_toml_module_config(config.config)?,
|
||||||
impl TryInto<FaaSModuleConfig> for TomlFaaSNamedModuleConfig {
|
})
|
||||||
type Error = FaaSError;
|
|
||||||
|
|
||||||
fn try_into(self) -> Result<FaaSModuleConfig> {
|
|
||||||
from_toml_named_module_config(self).map(|(_, module_config)| module_config)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -118,18 +115,6 @@ pub struct TomlFaaSModuleConfig {
|
|||||||
pub logging_mask: Option<i32>,
|
pub logging_mask: Option<i32>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl TomlFaaSNamedModuleConfig {
|
|
||||||
pub fn new<S>(name: S) -> Self
|
|
||||||
where
|
|
||||||
S: Into<String>,
|
|
||||||
{
|
|
||||||
Self {
|
|
||||||
name: name.into(),
|
|
||||||
config: <_>::default(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Deserialize, Serialize, Debug, Clone, Default)]
|
#[derive(Deserialize, Serialize, Debug, Clone, Default)]
|
||||||
pub struct TomlWASIConfig {
|
pub struct TomlWASIConfig {
|
||||||
pub preopened_files: Option<Vec<String>>,
|
pub preopened_files: Option<Vec<String>>,
|
||||||
@ -142,7 +127,7 @@ pub fn from_toml_faas_config(config: TomlFaaSConfig) -> Result<FaaSConfig> {
|
|||||||
let modules_config = config
|
let modules_config = config
|
||||||
.module
|
.module
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(from_toml_named_module_config)
|
.map(ModuleDescriptor::try_from)
|
||||||
.collect::<Result<Vec<_>>>()?;
|
.collect::<Result<Vec<_>>>()?;
|
||||||
|
|
||||||
let default_modules_config = config.default.map(from_toml_module_config).transpose()?;
|
let default_modules_config = config.default.map(from_toml_module_config).transpose()?;
|
||||||
@ -154,13 +139,6 @@ pub fn from_toml_faas_config(config: TomlFaaSConfig) -> Result<FaaSConfig> {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn from_toml_named_module_config(
|
|
||||||
config: TomlFaaSNamedModuleConfig,
|
|
||||||
) -> Result<(String, FaaSModuleConfig)> {
|
|
||||||
let module_config = from_toml_module_config(config.config)?;
|
|
||||||
Ok((config.name, module_config))
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn from_toml_module_config(config: TomlFaaSModuleConfig) -> Result<FaaSModuleConfig> {
|
pub fn from_toml_module_config(config: TomlFaaSModuleConfig) -> Result<FaaSModuleConfig> {
|
||||||
let mounted_binaries = config.mounted_binaries.unwrap_or_default();
|
let mounted_binaries = config.mounted_binaries.unwrap_or_default();
|
||||||
let mounted_binaries = mounted_binaries
|
let mounted_binaries = mounted_binaries
|
||||||
@ -236,6 +214,7 @@ mod tests {
|
|||||||
fn serialize_named() {
|
fn serialize_named() {
|
||||||
let config = TomlFaaSNamedModuleConfig {
|
let config = TomlFaaSNamedModuleConfig {
|
||||||
name: "name".to_string(),
|
name: "name".to_string(),
|
||||||
|
file_name: Some("file_name".to_string()),
|
||||||
config: TomlFaaSModuleConfig {
|
config: TomlFaaSModuleConfig {
|
||||||
mem_pages_count: Some(100),
|
mem_pages_count: Some(100),
|
||||||
logger_enabled: Some(false),
|
logger_enabled: Some(false),
|
||||||
|
@ -70,9 +70,9 @@ pub fn main() -> std::result::Result<(), anyhow::Error> {
|
|||||||
}
|
}
|
||||||
("show", Some(arg)) => {
|
("show", Some(arg)) => {
|
||||||
let wasm_path = arg.value_of(args::IN_WASM_PATH).unwrap();
|
let wasm_path = arg.value_of(args::IN_WASM_PATH).unwrap();
|
||||||
let wasm_path = std::path::PathBuf::from(wasm_path);
|
let wasm_path = std::path::Path::new(wasm_path);
|
||||||
|
|
||||||
let result = fce_wit_parser::extract_text_wit(wasm_path)?;
|
let result = fce_wit_parser::extract_text_wit(&wasm_path)?;
|
||||||
println!("{}", result);
|
println!("{}", result);
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
|
Loading…
Reference in New Issue
Block a user