fluencelabs/aquavm
1
0
Fork 0
mirror of https://github.com/fluencelabs/aquavm synced 2024-12-04 15:20:16 +00:00
Code Issues Projects Releases Wiki Activity

feat(data)!: flexible serialization formats (#757)

Browse Source 
New crate air_interpreter_sede introduces a flexible serialization and deserialization infrastructure.

Low level API: the Format trait and related. It includes simple and universal mechanisms for serializing any possible (e.g. implementing the Serde traits) value.

High level API: Representation trait and related. It allows declaring representation types that can serialize and deserialize only certain types (for example, InterpreterDataRepr can serialize only the InterpreterData, but deserialize InterpreterData and Versions` types), producing newtype values as serialization results.

The serialized representation of CallResults now contains byte arrays, not strings, because the new infrastructure serializes to byte arrays only.

Also, the basic support for multicodecs is added.
This commit is contained in:
Ivan Boldyrev 2023-12-15 18:23:09 +04:00 committed by GitHub
parent 17d7ef6c5a
commit 771d42dec4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
58 changed files with 2881 additions and 1247 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
Cargo.lock generated
View File

@ -112,6 +112,7 @@ name = "air-interpreter-data"
version = "0.15.0"
dependencies = [
"air-interpreter-cid",
"air-interpreter-sede",
"air-interpreter-signatures",
"air-utils",
"aquavm-air-parser",
@ -131,12 +132,28 @@ dependencies = [
name = "air-interpreter-interface"
version = "0.15.2"
dependencies = [
"air-interpreter-sede",
"fluence-it-types",
"marine-call-parameters",
"marine-rs-sdk",
"serde",
"serde_bytes",
"serde_json",
]
[[package]]
name = "air-interpreter-sede"
version = "0.1.0"
dependencies = [
"marine-rs-sdk",
"rmp-serde",
"serde",
"serde_bytes",
"serde_json",
"thiserror",
"unsigned-varint 0.8.0",
]
[[package]]
name = "air-interpreter-signatures"
version = "0.1.5"
@ -183,6 +200,7 @@ dependencies = [
"air-interpreter-cid",
"air-interpreter-data",
"air-interpreter-interface",
"air-interpreter-sede",
"air-interpreter-signatures",
"aquavm-air",
"avm-interface",
@ -337,6 +355,7 @@ dependencies = [
"air-interpreter-cid",
"air-interpreter-data",
"air-interpreter-interface",
"air-interpreter-sede",
"air-interpreter-signatures",
"air-lambda-ast",
"air-lambda-parser",
@ -378,6 +397,7 @@ version = "0.5.4"
dependencies = [
"air-beautifier",
"air-interpreter-interface",
"air-interpreter-sede",
"air-test-utils",
"anyhow",
"aquavm-air",
@ -585,6 +605,7 @@ name = "avm-interface"
version = "0.29.3"
dependencies = [
"air-interpreter-interface",
"air-interpreter-sede",
"air-utils",
"log",
"maplit",
@ -600,6 +621,7 @@ name = "avm-server"
version = "0.33.4"
dependencies = [
"air-interpreter-interface",
"air-interpreter-sede",
"air-utils",
"avm-data-store",
"avm-interface",
@ -4671,6 +4693,28 @@ dependencies = [
"digest 0.10.7",
]
[[package]]
name = "rmp"
version = "0.8.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f9860a6cc38ed1da53456442089b4dfa35e7cedaa326df63017af88385e6b20"
dependencies = [
"byteorder",
"num-traits",
"paste",
]
[[package]]
name = "rmp-serde"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bffea85eea980d8a74453e5d02a8d93028f3c34725de143085a844ebe953258a"
dependencies = [
"byteorder",
"rmp",
"serde",
]
[[package]]
name = "rust-argon2"
version = "0.8.3"

1
Cargo.toml
View File

@ -9,6 +9,7 @@ members = [
"crates/air-lib/interpreter-cid",
"crates/air-lib/interpreter-data",
"crates/air-lib/interpreter-interface",
"crates/air-lib/interpreter-sede",
"crates/air-lib/interpreter-signatures",
"crates/air-lib/lambda/ast",
"crates/air-lib/lambda/parser",

6
air-interpreter/src/marine.rs
View File

@ -49,7 +49,7 @@ pub fn invoke(
params: RunParameters,
call_results: Vec<u8>,
) -> InterpreterOutcome {
execute_air(air, prev_data, data, params, call_results)
execute_air(air, prev_data, data, params, call_results.into())
}
#[allow(clippy::too_many_arguments)]
@ -79,7 +79,9 @@ pub fn invoke_tracing(
let subscriber = builder.finish();
Dispatch::new(subscriber)
};
tracing::dispatcher::with_default(&dispatch, || execute_air(air, prev_data, data, params, call_results))
tracing::dispatcher::with_default(&dispatch, || {
execute_air(air, prev_data, data, params, call_results.into())
})
}
#[marine]

2
air-interpreter/src/wasm_bindgen.rs
View File

@ -67,7 +67,7 @@ pub fn invoke(
let params: RunParameters = serde_json::from_slice(&params).expect("cannot parse RunParameters");
let outcome = execute_air(air, prev_data, data, params, call_results);
let outcome = execute_air(air, prev_data, data, params, call_results.into());
serde_json::to_string(&outcome).expect("Cannot parse InterpreterOutcome")
}

1
air/Cargo.toml
View File

@ -20,6 +20,7 @@ aquavm-air-parser = { version = "0.11.0", path = "../crates/air-lib/air-parser"
air-execution-info-collector = { version = "0.7.11", path = "../crates/air-lib/execution-info-collector" }
air-interpreter-cid = { version = "0.7.0", path = "../crates/air-lib/interpreter-cid" }
air-interpreter-data = { version = "0.15.0", path = "../crates/air-lib/interpreter-data" }
air-interpreter-sede = { version = "0.1.0", path = "../crates/air-lib/interpreter-sede" }
air-interpreter-signatures = { version = "0.1.5", path = "../crates/air-lib/interpreter-signatures" }
air-interpreter-interface = { version = "0.15.2", path = "../crates/air-lib/interpreter-interface", default-features = false }
air-log-targets = { version = "0.1.0", path = "../crates/air-lib/log-targets" }

9
air/src/execution_step/errors/uncatchable_errors.rs
View File

@ -25,6 +25,9 @@ use crate::ToErrorCode;
use air_interpreter_cid::CidCalculationError;
use air_interpreter_cid::CidRef;
use air_interpreter_data::ValueRef;
use air_interpreter_interface::CallArgumentsRepr;
use air_interpreter_interface::TetrapletsRepr;
use air_interpreter_sede::Representation;
use air_trace_handler::GenerationCompactificationError;
use air_trace_handler::IntConversionError;
use air_trace_handler::TraceHandlerError;
@ -131,6 +134,12 @@ pub enum UncatchableError {
#[error("failed to sign data: {0}")]
SigningError(#[from] fluence_keypair::error::SigningError),
#[error("failed to serialize tetraplets {0}")]
TetrapletSerializationFailed(<TetrapletsRepr as Representation>::SerializeError),
#[error("failed to serialize call arguments {0}")]
CallArgumentsSerializationFailed(<CallArgumentsRepr as Representation>::SerializeError),
}
impl ToErrorCode for UncatchableError {

19
air/src/execution_step/instructions/call/resolved_call.rs
View File

@ -25,13 +25,15 @@ use crate::execution_step::RcSecurityTetraplet;
use crate::execution_step::RcSecurityTetraplets;
use crate::execution_step::UncatchableError;
use crate::trace_to_exec_err;
use crate::JValue;
use crate::SecurityTetraplet;
use air_interpreter_cid::value_to_json_cid;
use air_interpreter_cid::CidRef;
use air_interpreter_data::CallResult;
use air_interpreter_interface::CallArgumentsRepr;
use air_interpreter_interface::CallRequestParams;
use air_interpreter_interface::SerializedCallArguments;
use air_interpreter_interface::TetrapletsRepr;
use air_parser::ast;
use air_trace_handler::merger::MergerCallResult;
use air_trace_handler::TraceHandler;
@ -48,7 +50,7 @@ pub(super) struct ResolvedCall<'i> {
#[derive(Debug, Clone, PartialEq)]
struct ResolvedArguments {
call_arguments: String,
call_arguments: SerializedCallArguments,
tetraplets: Vec<RcSecurityTetraplets>,
}
@ -156,12 +158,16 @@ impl<'i> ResolvedCall<'i> {
exec_ctx: &ExecutionCtx<'_>,
tetraplet: &SecurityTetraplet,
) -> ExecutionResult<CallRequestParams> {
use air_interpreter_sede::ToSerialized;
let ResolvedArguments {
call_arguments,
tetraplets,
} = self.resolve_args(exec_ctx)?;
let serialized_tetraplets = serde_json::to_string(&tetraplets).expect("default serializer shouldn't fail");
let serialized_tetraplets = TetrapletsRepr
.serialize(&tetraplets)
.map_err(UncatchableError::TetrapletSerializationFailed)?;
let request_params = CallRequestParams::new(
tetraplet.service_id.to_string(),
@ -197,10 +203,13 @@ impl<'i> ResolvedCall<'i> {
/// Prepare arguments of this call instruction by resolving and preparing their security tetraplets.
fn resolve_args(&self, exec_ctx: &ExecutionCtx<'i>) -> ExecutionResult<ResolvedArguments> {
use air_interpreter_sede::ToSerialized;
let (call_arguments, tetraplets) = self.collect_args(exec_ctx)?;
let call_arguments = JValue::Array(call_arguments);
let call_arguments = call_arguments.to_string();
let call_arguments = CallArgumentsRepr
.serialize(&call_arguments)
.map_err(UncatchableError::CallArgumentsSerializationFailed)?;
let resolved_arguments = ResolvedArguments {
call_arguments,

24
air/src/farewell_step/outcome.rs
View File

@ -24,6 +24,8 @@ use crate::INTERPRETER_SUCCESS;
use air_interpreter_data::InterpreterData;
use air_interpreter_interface::CallRequests;
use air_interpreter_interface::CallRequestsRepr;
use air_interpreter_sede::ToSerialized;
use air_interpreter_signatures::KeyPair;
use air_utils::measure;
use fluence_keypair::error::SigningError;
@ -60,7 +62,9 @@ pub(crate) fn from_uncatchable_error(
) -> InterpreterOutcome {
let ret_code = error.to_error_code();
let data = data.into();
let call_requests = serde_json::to_vec(&CallRequests::new()).expect("default serializer shouldn't fail");
let call_requests = CallRequestsRepr
.serialize(&CallRequests::new())
.expect("default serializer shouldn't fail");
InterpreterOutcome::new(ret_code, error.to_string(), data, vec![], call_requests)
}
@ -109,16 +113,18 @@ fn populate_outcome_from_contexts(
semver::Version::parse(env!("CARGO_PKG_VERSION")).expect("cargo version is valid"),
);
let data = measure!(
serde_json::to_vec(&data).expect("default serializer shouldn't fail"),
tracing::Level::TRACE,
"serde_json::to_vec(data)"
data.serialize().expect("default serializer shouldn't fail"),
tracing::Level::INFO,
"InterpreterData::serialize"
);
let next_peer_pks = dedup(exec_ctx.next_peer_pks);
let call_requests = measure!(
serde_json::to_vec(&exec_ctx.call_requests).expect("default serializer shouldn't fail"),
tracing::Level::TRACE,
"serde_json::to_vec(call_results)",
CallRequestsRepr
.serialize(&exec_ctx.call_requests)
.expect("default serializer shouldn't fail"),
tracing::Level::INFO,
"CallRequestsRepr.serialize",
);
InterpreterOutcome::new(ret_code, error_message, data, next_peer_pks, call_requests)
}
@ -146,11 +152,11 @@ fn sign_result(exec_ctx: &mut ExecutionCtx<'_>, keypair: &KeyPair) -> Result<(),
// these methods are called only if there is an internal error in the interpreter and
// new execution trace was corrupted
fn execution_error_into_outcome(error: ExecutionError) -> InterpreterOutcome {
InterpreterOutcome::new(error.to_error_code(), error.to_string(), vec![], vec![], vec![])
InterpreterOutcome::new(error.to_error_code(), error.to_string(), vec![], vec![], <_>::default())
}
fn signing_error_into_outcome(error: SigningError) -> InterpreterOutcome {
InterpreterOutcome::new(error.to_error_code(), error.to_string(), vec![], vec![], vec![])
InterpreterOutcome::new(error.to_error_code(), error.to_string(), vec![], vec![], <_>::default())
}
/// Deduplicate values in a supplied vector.

25
air/src/preparation_step/errors.rs
View File

@ -18,14 +18,18 @@ use crate::ToErrorCode;
use air_interpreter_data::data_version;
use air_interpreter_data::verification::DataVerifierError;
use air_interpreter_data::CidStoreVerificationError;
use air_interpreter_data::InterpreterDataRepr;
use air_interpreter_data::Versions;
use serde_json::Error as SerdeJsonError;
use air_interpreter_interface::CallResultsDeserializeError;
use air_interpreter_interface::SerializedCallResults;
use air_interpreter_sede::Representation;
use strum::IntoEnumIterator;
use strum_macros::EnumDiscriminants;
use strum_macros::EnumIter;
use thiserror::Error as ThisError;
type SerdeDeserializeError = <InterpreterDataRepr as Representation>::DeserializeError;
/// Errors happened during the interpreter preparation step.
#[derive(Debug, EnumDiscriminants, ThisError)]
#[strum_discriminants(derive(EnumIter))]
@ -43,7 +47,10 @@ pub enum PreparationError {
super::interpreter_version(),
data_version()
)]
DataDeFailed { data: Vec<u8>, error: SerdeJsonError },
DataDeFailed {
data: Vec<u8>,
error: SerdeDeserializeError,
},
/// Errors occurred on executed trace deserialization
/// when it was possible to recover versions.
@ -59,7 +66,7 @@ pub enum PreparationError {
)]
DataDeFailedWithVersions {
data: Vec<u8>,
error: SerdeJsonError,
error: SerdeDeserializeError,
versions: Versions,
},
@ -69,8 +76,8 @@ pub enum PreparationError {
Call results: {call_results:?}"
)]
CallResultsDeFailed {
call_results: Vec<u8>,
error: SerdeJsonError,
call_results: SerializedCallResults,
error: CallResultsDeserializeError,
},
/// Error occurred when a version of interpreter produced supplied data is less then minimal.
@ -101,15 +108,15 @@ impl ToErrorCode for PreparationError {
}
impl PreparationError {
pub fn data_de_failed(data: Vec<u8>, error: SerdeJsonError) -> Self {
pub fn data_de_failed(data: Vec<u8>, error: SerdeDeserializeError) -> Self {
Self::DataDeFailed { data, error }
}
pub fn data_de_failed_with_versions(data: Vec<u8>, error: SerdeJsonError, versions: Versions) -> Self {
pub fn data_de_failed_with_versions(data: Vec<u8>, error: SerdeDeserializeError, versions: Versions) -> Self {
Self::DataDeFailedWithVersions { data, error, versions }
}
pub fn call_results_de_failed(call_results: Vec<u8>, error: SerdeJsonError) -> Self {
pub fn call_results_de_failed(call_results: SerializedCallResults, error: CallResultsDeserializeError) -> Self {
Self::CallResultsDeFailed { call_results, error }
}

24
air/src/preparation_step/preparation.rs
View File

@ -20,11 +20,17 @@ use crate::execution_step::ExecutionCtx;
use crate::execution_step::TraceHandler;
use air_interpreter_data::InterpreterData;
use air_interpreter_data::InterpreterDataRepr;
use air_interpreter_interface::CallResultsRepr;
use air_interpreter_interface::RunParameters;
use air_interpreter_interface::SerializedCallResults;
use air_interpreter_sede::FromSerialized;
use air_interpreter_sede::Representation;
use air_interpreter_signatures::KeyError;
use air_interpreter_signatures::KeyPair;
use air_interpreter_signatures::SignatureStore;
use air_parser::ast::Instruction;
use air_utils::measure;
use fluence_keypair::KeyFormat;
use std::convert::TryFrom;
@ -64,7 +70,7 @@ pub(crate) fn prepare<'i>(
prev_data: InterpreterData,
current_data: InterpreterData,
raw_air: &'i str,
call_results: &[u8],
call_results: &SerializedCallResults,
run_parameters: RunParameters,
signature_store: SignatureStore,
) -> PreparationResult<PreparationDescriptor<'static, 'i>> {
@ -112,7 +118,10 @@ pub(crate) fn try_to_data(raw_data: &[u8]) -> PreparationResult<InterpreterData>
InterpreterData::try_from_slice(raw_data).map_err(|de_error| to_date_de_error(raw_data.to_vec(), de_error))
}
fn to_date_de_error(raw_data: Vec<u8>, de_error: serde_json::Error) -> PreparationError {
fn to_date_de_error(
raw_data: Vec<u8>,
de_error: <InterpreterDataRepr as Representation>::DeserializeError,
) -> PreparationError {
match InterpreterData::try_get_versions(&raw_data) {
Ok(versions) => PreparationError::data_de_failed_with_versions(raw_data, de_error, versions),
Err(_) => PreparationError::data_de_failed(raw_data, de_error),
@ -123,12 +132,17 @@ fn to_date_de_error(raw_data: Vec<u8>, de_error: serde_json::Error) -> Preparati
fn make_exec_ctx(
prev_ingredients: ExecCtxIngredients,
current_ingredients: ExecCtxIngredients,
call_results: &[u8],
call_results: &SerializedCallResults,
signature_store: SignatureStore,
run_parameters: &RunParameters,
) -> PreparationResult<ExecutionCtx<'static>> {
let call_results = serde_json::from_slice(call_results)
.map_err(|e| PreparationError::call_results_de_failed(call_results.to_vec(), e))?;
let call_results = measure!(
CallResultsRepr
.deserialize(call_results)
.map_err(|e| PreparationError::call_results_de_failed(call_results.clone(), e))?,
tracing::Level::INFO,
"CallResultsRepr.deserialize",
);
let ctx = ExecutionCtx::new(
prev_ingredients,

5
air/src/runner.rs
View File

@ -25,6 +25,7 @@ use crate::verification_step::verify;
use air_interpreter_interface::InterpreterOutcome;
use air_interpreter_interface::RunParameters;
use air_interpreter_interface::SerializedCallResults;
use air_log_targets::RUN_PARAMS;
use air_utils::farewell_if_fail;
use air_utils::measure;
@ -35,7 +36,7 @@ pub fn execute_air(
prev_data: Vec<u8>,
data: Vec<u8>,
params: RunParameters,
call_results: Vec<u8>,
call_results: SerializedCallResults,
) -> InterpreterOutcome {
use std::convert::identity;
@ -58,7 +59,7 @@ fn execute_air_impl(
raw_prev_data: Vec<u8>,
raw_current_data: Vec<u8>,
params: RunParameters,
call_results: Vec<u8>,
call_results: SerializedCallResults,
) -> Result<InterpreterOutcome, InterpreterOutcome> {
let ParsedDataPair {
prev_data,

6
air/tests/test_module/features/misc/version_check.rs
View File

@ -26,7 +26,7 @@ fn minimal_version_check() {
let actual_version = semver::Version::new(0, 31, 1);
let current_data = InterpreterData::new(actual_version.clone());
let current_data = serde_json::to_vec(&current_data).expect("default serializer shouldn't fail");
let current_data = current_data.serialize().expect("default serializer shouldn't fail");
let result = call_vm!(vm, <_>::default(), script, "", current_data);
let expected_error = PreparationError::UnsupportedInterpreterVersion {
@ -45,7 +45,7 @@ fn publish_version_check() {
let actual_version =
semver::Version::parse("1.0.1-feat-VM-173-add-interpreter-version-in-data-a2d575b-205-1.0").unwrap();
let current_data = InterpreterData::new(actual_version);
let current_data = serde_json::to_vec(&current_data).expect("default serializer shouldn't fail");
let current_data = current_data.serialize().expect("default serializer shouldn't fail");
let result = call_vm!(vm, <_>::default(), script, "", current_data);
assert_eq!(result.ret_code, INTERPRETER_SUCCESS, "{:?}", result.error_message);
@ -58,7 +58,7 @@ fn publish_unsupported_version_check() {
let actual_version =
semver::Version::parse("0.31.1-feat-VM-173-add-interpreter-version-in-data-a2d575b-205-1.0").unwrap();
let current_data = InterpreterData::new(actual_version.clone());
let current_data = serde_json::to_vec(&current_data).expect("default serializer shouldn't fail");
let current_data = current_data.serialize().expect("default serializer shouldn't fail");
let result = call_vm!(vm, <_>::default(), "", "", current_data);
let expected_error = PreparationError::UnsupportedInterpreterVersion {

5
air/tests/test_module/features/signatures/algorithms.rs
View File

@ -15,7 +15,8 @@
*/
use air::{min_supported_version, PreparationError};
use air_interpreter_data::{verification::DataVerifierError, InterpreterData};
use air_interpreter_data::{verification::DataVerifierError, InterpreterData, InterpreterDataRepr};
use air_interpreter_sede::{Format, Representation};
use air_interpreter_signatures::KeyError;
use air_test_utils::{
assert_error_eq,
@ -56,7 +57,7 @@ fn test_banned_signature() {
data["signatures"] = bad_signature_store;
let current_data = data.to_string();
let current_data = InterpreterDataRepr.get_format().to_vec(&data).unwrap();
let mut avm = create_avm(unit_call_service(), "other_peer_id");
let res = avm

18
air/tests/test_module/features/signatures/attacks.rs
View File

@ -82,8 +82,8 @@ fn test_attack_injection_current_peer_scalar() {
let mut alice_avm = create_avm_with_key::<NativeAirRunner>(alice_keypair, unit_call_service());
let test_run_params = TestRunParameters::from_init_peer_id(alice_peer_id);
let prev_data = serde_json::to_vec(&alice_data).unwrap();
let cur_data = serde_json::to_vec(&mallory_data).unwrap();
let prev_data = alice_data.serialize().unwrap();
let cur_data = mallory_data.serialize().unwrap();
let res = alice_avm
.call(&air_script, prev_data, cur_data, test_run_params)
.unwrap();
@ -153,8 +153,8 @@ fn test_attack_injection_current_peer_stream() {
let mut alice_avm = create_avm_with_key::<NativeAirRunner>(alice_keypair, unit_call_service());
let test_run_params = TestRunParameters::from_init_peer_id(alice_peer_id);
let prev_data = serde_json::to_vec(&alice_data).unwrap();
let cur_data = serde_json::to_vec(&mallory_data).unwrap();
let prev_data = alice_data.serialize().unwrap();
let cur_data = mallory_data.serialize().unwrap();
let res = alice_avm
.call(&air_script, prev_data, cur_data, test_run_params)
.unwrap();
@ -222,8 +222,8 @@ fn test_attack_injection_current_injection_unused() {
let mut alice_avm = create_avm_with_key::<NativeAirRunner>(alice_keypair, unit_call_service());
let test_run_params = TestRunParameters::from_init_peer_id(alice_peer_id);
let prev_data = serde_json::to_vec(&alice_data).unwrap();
let cur_data = serde_json::to_vec(&mallory_data).unwrap();
let prev_data = alice_data.serialize().unwrap();
let cur_data = mallory_data.serialize().unwrap();
let res = alice_avm
.call(&air_script, prev_data, cur_data, test_run_params)
.unwrap();
@ -284,7 +284,7 @@ fn test_attack_injection_other_peer_scalar() {
let mut bob_avm = create_avm_with_key::<NativeAirRunner>(bob_keypair, unit_call_service());
let test_run_params = TestRunParameters::from_init_peer_id(alice_peer_id);
let prev_data = "";
let cur_data = serde_json::to_vec(&mallory_data).unwrap();
let cur_data = mallory_data.serialize().unwrap();
let res = bob_avm.call(&air_script, prev_data, cur_data, test_run_params).unwrap();
assert_ne!(res.ret_code, 0);
}
@ -342,7 +342,7 @@ fn test_attack_injection_other_peer_stream() {
let mut bob_avm = create_avm_with_key::<NativeAirRunner>(bob_keypair, unit_call_service());
let test_run_params = TestRunParameters::from_init_peer_id(alice_peer_id);
let prev_data = "";
let cur_data = serde_json::to_vec(&mallory_data).unwrap();
let cur_data = mallory_data.serialize().unwrap();
let res = bob_avm.call(&air_script, prev_data, cur_data, test_run_params).unwrap();
assert_ne!(res.ret_code, 0, "{}", res.error_message);
}
@ -400,7 +400,7 @@ fn test_attack_injection_other_peer_unused() {
let mut bob_avm = create_avm_with_key::<NativeAirRunner>(bob_keypair, unit_call_service());
let test_run_params = TestRunParameters::from_init_peer_id(alice_peer_id);
let prev_data = "";
let cur_data = serde_json::to_vec(&mallory_data).unwrap();
let cur_data = mallory_data.serialize().unwrap();
let res = bob_avm.call(&air_script, prev_data, cur_data, test_run_params).unwrap();
// please not that such injection is not caught

12
air/tests/test_module/features/signatures/corruption.rs
View File

@ -88,7 +88,7 @@ fn test_attack_replace_value() {
let mut bob_avm = create_avm(unit_call_service(), bob_peer_id);
let test_run_params = TestRunParameters::from_init_peer_id(alice_peer_id);
let prev_data = "";
let cur_data = serde_json::to_vec(&mallory_data).unwrap();
let cur_data = mallory_data.serialize().unwrap();
let res = bob_avm.call(&air_script, prev_data, cur_data, test_run_params).unwrap();
assert_error_eq!(
@ -164,7 +164,7 @@ fn test_attack_replace_tetraplet() {
let mut bob_avm = create_avm(unit_call_service(), bob_peer_id);
let test_run_params = TestRunParameters::from_init_peer_id(alice_peer_id);
let prev_data = "";
let cur_data = serde_json::to_vec(&mallory_data).unwrap();
let cur_data = mallory_data.serialize().unwrap();
let res = bob_avm.call(&air_script, prev_data, cur_data, test_run_params).unwrap();
assert_error_eq!(
@ -247,7 +247,7 @@ fn test_attack_replace_call_result() {
let mut bob_avm = create_avm(unit_call_service(), bob_peer_id);
let test_run_params = TestRunParameters::from_init_peer_id(alice_peer_id);
let prev_data = "";
let cur_data = serde_json::to_vec(&mallory_data).unwrap();
let cur_data = mallory_data.serialize().unwrap();
let res = bob_avm.call(&air_script, prev_data, cur_data, test_run_params).unwrap();
assert_error_eq!(
@ -337,7 +337,7 @@ fn test_attack_replace_canon_value() {
let mut bob_avm = create_avm(unit_call_service(), bob_peer_id);
let test_run_params = TestRunParameters::from_init_peer_id(alice_peer_id);
let prev_data = "";
let cur_data = serde_json::to_vec(&mallory_data).unwrap();
let cur_data = mallory_data.serialize().unwrap();
let res = bob_avm.call(&air_script, prev_data, cur_data, test_run_params).unwrap();
assert_error_eq!(
@ -436,7 +436,7 @@ fn test_attack_replace_canon_result_values() {
let mut bob_avm = create_avm(unit_call_service(), bob_peer_id);
let test_run_params = TestRunParameters::from_init_peer_id(alice_peer_id);
let prev_data = "";
let cur_data = serde_json::to_vec(&mallory_data).unwrap();
let cur_data = mallory_data.serialize().unwrap();
let res = bob_avm.call(&air_script, prev_data, cur_data, test_run_params).unwrap();
assert_error_eq!(
@ -539,7 +539,7 @@ fn test_attack_replace_canon_result_tetraplet() {
let mut bob_avm = create_avm(unit_call_service(), bob_peer_id);
let test_run_params = TestRunParameters::from_init_peer_id(alice_peer_id);
let prev_data = "";
let cur_data = serde_json::to_vec(&mallory_data).unwrap();
let cur_data = mallory_data.serialize().unwrap();
let res = bob_avm.call(&air_script, prev_data, cur_data, test_run_params).unwrap();
assert_error_eq!(

10
air/tests/test_module/features/signatures/runtime.rs
View File

@ -60,7 +60,7 @@ fn test_runtime_executed_call_argument_hash() {
mallory_raw_trace.swap(0, 1);
mallory_data.trace = ExecutionTrace::from(mallory_raw_trace);
let mallory_data = serde_json::to_vec(&mallory_data).unwrap();
let mallory_data = mallory_data.serialize().unwrap();
let bob_res = bob_avm.call(air_script, "", mallory_data, test_run_params).unwrap();
assert_error_eq!(
@ -112,7 +112,7 @@ fn test_runtime_executed_call_tetraplet() {
mallory_raw_trace.swap(0, 1);
mallory_data.trace = ExecutionTrace::from(mallory_raw_trace);
let mallory_data = serde_json::to_vec(&mallory_data).unwrap();
let mallory_data = mallory_data.serialize().unwrap();
let bob_res = bob_avm.call(air_script, "", mallory_data, test_run_params).unwrap();
let expected_value = format!(
@ -182,7 +182,7 @@ fn test_runtime_executed_failed_argument_hash() {
mallory_raw_trace.swap(0, 1);
mallory_data.trace = ExecutionTrace::from(mallory_raw_trace);
let mallory_data = serde_json::to_vec(&mallory_data).unwrap();
let mallory_data = mallory_data.serialize().unwrap();
let bob_res = bob_avm.call(air_script, "", mallory_data, test_run_params).unwrap();
assert_error_eq!(
@ -238,7 +238,7 @@ fn test_runtime_failed_call_tetraplet() {
mallory_raw_trace.swap(0, 1);
mallory_data.trace = ExecutionTrace::from(mallory_raw_trace);
let mallory_data = serde_json::to_vec(&mallory_data).unwrap();
let mallory_data = mallory_data.serialize().unwrap();
let bob_res = bob_avm.call(air_script, "", mallory_data, test_run_params).unwrap();
let expected_value = format!(
@ -306,7 +306,7 @@ fn test_runtime_canon_tetraplet() {
mallory_raw_trace.swap(2, 3);
mallory_data.trace = ExecutionTrace::from(mallory_raw_trace);
let mallory_data = serde_json::to_vec(&mallory_data).unwrap();
let mallory_data = mallory_data.serialize().unwrap();
let bob_res = bob_avm.call(air_script, "", mallory_data, test_run_params).unwrap();
let expected_value = format!(

2
air/tests/test_module/negative_tests/farewell_step.rs
View File

@ -23,7 +23,7 @@ fn unprocessed_call_result() {
let client_peer_id = "some_peer_id";
let mut client_vm = create_avm(unit_call_service(), client_peer_id);
let prev_data = InterpreterData::new(semver::Version::new(1, 1, 1));
let prev_data: Vec<u8> = serde_json::to_vec(&prev_data).unwrap();
let prev_data: Vec<u8> = prev_data.serialize().unwrap();
let call_service_result = air_test_utils::CallServiceResult::ok(json!("null"));
let call_results_4_call = maplit::hashmap!(
70 => call_service_result,

36
air/tests/test_module/negative_tests/preparation_step.rs
View File

@ -15,7 +15,10 @@
*/
use air::PreparationError;
use air_interpreter_interface::{CallResults, RunParameters};
use air_interpreter_interface::CallResultsFormat;
use air_interpreter_interface::CallResultsRepr;
use air_interpreter_interface::RunParameters;
use air_interpreter_sede::FromSerialized;
use air_test_utils::prelude::*;
use serde::Deserialize;
@ -23,6 +26,9 @@ use serde::Serialize;
#[test]
fn invalid_data_without_versions() {
use air_interpreter_sede::Format;
use air_interpreter_sede::Representation;
#[derive(Serialize, Deserialize)]
struct InvalidDataStruct {
pub trace: Vec<u8>,
@ -33,11 +39,12 @@ fn invalid_data_without_versions() {
let script = r#"(null)"#;
let invalid_data = InvalidDataStruct { trace: vec![1, 2, 3] };
let invalid_data = serde_json::to_vec(&invalid_data).unwrap();
let invalid_data = InterpreterDataRepr.get_format().to_vec(&invalid_data).unwrap();
let result = call_vm!(vm, <_>::default(), script, "", invalid_data.clone());
let expected_serde_error = serde_json::from_slice::<InterpreterData>(&invalid_data).err().unwrap();
let expected_serde_error = InterpreterData::try_from_slice(&invalid_data).unwrap_err();
let expected_error = PreparationError::DataDeFailed {
data: invalid_data,
error: expected_serde_error,
@ -47,6 +54,9 @@ fn invalid_data_without_versions() {
#[test]
fn invalid_data_with_versions() {
use air_interpreter_sede::Format;
use air_interpreter_sede::Representation;
#[derive(Serialize, Deserialize)]
struct InvalidDataStruct {
pub trace: Vec<u8>,
@ -63,11 +73,11 @@ fn invalid_data_with_versions() {
trace: vec![1, 2, 3],
versions: versions.clone(),
};
let invalid_data = serde_json::to_vec(&invalid_data).unwrap();
let invalid_data = InterpreterDataRepr.get_format().to_vec(&invalid_data).unwrap();
let result = call_vm!(vm, <_>::default(), script, "", invalid_data.clone());
let expected_serde_error = serde_json::from_slice::<InterpreterData>(&invalid_data).err().unwrap();
let expected_serde_error = InterpreterData::try_from_slice(&invalid_data).unwrap_err();
let expected_error = PreparationError::DataDeFailedWithVersions {
data: invalid_data,
error: expected_serde_error,
@ -78,13 +88,15 @@ fn invalid_data_with_versions() {
#[test]
fn invalid_callresults() {
use air_interpreter_sede::Format;
let air = r#"(null)"#.to_string();
let client_peer_id = "some_peer_id".to_string();
let prev_data = InterpreterData::new(semver::Version::new(1, 1, 1));
let prev_data: Vec<u8> = serde_json::to_vec(&prev_data).unwrap();
let prev_data: Vec<u8> = prev_data.serialize().unwrap();
let data = Vec::<u8>::new();
let wrong_call_results = Vec::<u32>::new();
let wrong_call_results = serde_json::to_vec(&wrong_call_results).unwrap();
let vec = Vec::<u8>::new();
let wrong_call_results = CallResultsFormat::default().to_vec(&vec).unwrap();
let keypair = fluence_keypair::KeyPair::generate_ed25519();
let run_parameters = RunParameters::new(
client_peer_id.clone(),
@ -96,15 +108,13 @@ fn invalid_callresults() {
"".to_owned(),
);
let result = air::execute_air(air, prev_data, data, run_parameters, wrong_call_results.clone());
let result = air::execute_air(air, prev_data, data, run_parameters, wrong_call_results.clone().into());
let result = RawAVMOutcome::from_interpreter_outcome(result).unwrap();
let expected_serde_error = serde_json::from_slice::<CallResults>(&wrong_call_results)
.err()
.unwrap();
let expected_serde_error = CallResultsRepr.deserialize(&wrong_call_results).unwrap_err();
let expected_error = PreparationError::CallResultsDeFailed {
error: expected_serde_error,
call_results: wrong_call_results,
call_results: wrong_call_results.into(),
};
assert!(check_error(&result, expected_error));

5
air/tests/test_module/negative_tests/uncatchable_trace_related.rs
View File

@ -303,6 +303,9 @@ fn different_executed_state_expected() {
#[test]
fn invalid_dst_generations() {
use air_interpreter_sede::Format;
use air_interpreter_sede::Representation;
let vm_peer_id_1 = "vm_peer_id_1";
let mut peer_vm_1 = create_avm(unit_call_service(), vm_peer_id_1);
let script = format!(
@ -321,7 +324,7 @@ fn invalid_dst_generations() {
let mut data_value = serde_json::to_value(&empty_data).unwrap();
data_value["trace"] = json!([{"ap": {"gens": [42, 42]}}]);
let data: Vec<u8> = serde_json::to_vec(&data_value).unwrap();
let data = InterpreterDataRepr.get_format().to_vec(&data_value).unwrap();
// let result = peer_vm_1.call(script, "", data, <_>::default()).unwrap();
let result = call_vm!(peer_vm_1, <_>::default(), &script, "", data);
let expected_error = UncatchableError::TraceError {

10
avm/client/src/avmHelpers.ts
View File

@ -83,15 +83,17 @@ export function deserializeAvmResult(result: any): InterpreterResult {
let arguments_;
let tetraplets;
try {
arguments_ = JSON.parse(callRequest.arguments);
const argumentsStr = decoder.decode(new Uint8Array(callRequest.arguments));
arguments_ = JSON.parse(argumentsStr);
} catch (e) {
throw "Couldn't parse arguments: " + e + '. Original string is: ' + arguments_;
throw "Couldn't parse arguments: " + e + '. Original data is: ' + callRequest.arguments;
}
try {
tetraplets = JSON.parse(callRequest.tetraplets);
const tetrapletsStr = decoder.decode(new Uint8Array(callRequest.tetraplets));
tetraplets = JSON.parse(tetrapletsStr);
} catch (e) {
throw "Couldn't parse tetraplets: " + e + '. Original string is: ' + tetraplets;
throw "Couldn't parse tetraplets: " + e + '. Original data is: ' + callRequest.tetraplets;
}
resultCallRequests.push([

1
avm/interface/Cargo.toml
View File

@ -16,6 +16,7 @@ path = "src/lib.rs"
[dependencies]
air-interpreter-interface = { version = "0.15.2", path = "../../crates/air-lib/interpreter-interface", default-features = false }
air-interpreter-sede = { version = "0.1.0", path = "../../crates/air-lib/interpreter-sede", default-features = false }
air-utils = { version = "0.2.0", path = "../../crates/air-lib/utils" }
polyplets = { version = "0.5.1", path = "../../crates/air-lib/polyplets" }

22
avm/interface/src/call_request_parameters.rs
View File

@ -17,6 +17,7 @@
use super::JValue;
use crate::CallSeDeErrors;
use air_interpreter_interface::SerializedCallRequests;
use polyplets::SecurityTetraplet;
use serde::Deserialize;
use serde::Serialize;
@ -60,15 +61,19 @@ impl CallRequestParams {
pub(crate) fn from_raw(
call_params: air_interpreter_interface::CallRequestParams,
) -> Result<Self, CallSeDeErrors> {
let arguments: Vec<JValue> =
serde_json::from_str(&call_params.arguments).map_err(|de_error| {
CallSeDeErrors::CallParamsArgsDeFailed {
use air_interpreter_interface::CallArgumentsRepr;
use air_interpreter_interface::TetrapletsRepr;
use air_interpreter_sede::FromSerialized;
let arguments: Vec<JValue> = CallArgumentsRepr
.deserialize(&call_params.arguments)
.map_err(|de_error| CallSeDeErrors::CallParamsArgsDeFailed {
call_params: call_params.clone(),
de_error,
}
})?;
let tetraplets: Vec<Vec<SecurityTetraplet>> = serde_json::from_str(&call_params.tetraplets)
let tetraplets: Vec<Vec<SecurityTetraplet>> = TetrapletsRepr
.deserialize(&call_params.tetraplets)
.map_err(|de_error| CallSeDeErrors::CallParamsTetrapletsDeFailed {
call_params: call_params.clone(),
de_error,
@ -86,10 +91,13 @@ impl CallRequestParams {
}
pub(crate) fn from_raw_call_requests(
raw_call_params: Vec<u8>,
raw_call_params: SerializedCallRequests,
) -> Result<CallRequests, CallSeDeErrors> {
use air_interpreter_interface::CallRequestsRepr;
use air_interpreter_sede::FromSerialized;
let call_requests: air_interpreter_interface::CallRequests =
match serde_json::from_slice(&raw_call_params) {
match CallRequestsRepr.deserialize(&raw_call_params) {
Ok(requests) => requests,
Err(error) => {
return Err(CallSeDeErrors::CallRequestsDeError {

1
avm/interface/src/call_service_result.rs
View File

@ -53,6 +53,7 @@ impl CallServiceResult {
air_interpreter_interface::CallServiceResult {
ret_code,
// TODO serializer
result: result.to_string(),
}
}

17
avm/interface/src/lib.rs
View File

@ -32,7 +32,11 @@ mod outcome;
mod particle_parameters;
pub mod raw_outcome;
use serde_json::Error as SerdeError;
use air_interpreter_interface::CallArgumentsDeserializeError;
use air_interpreter_interface::CallRequestsDeserializeError;
use air_interpreter_interface::CallResultsSerializeError;
use air_interpreter_interface::SerializedCallRequests;
use air_interpreter_interface::TetrapletDeserializeError;
use thiserror::Error as ThisError;
#[derive(Debug, ThisError)]
@ -42,14 +46,14 @@ pub enum CallSeDeErrors {
#[error("error occurred while call results `{call_results:?}` deserialization: {se_error}")]
CallResultsSeFailed {
call_results: air_interpreter_interface::CallResults,
se_error: SerdeError,
se_error: CallResultsSerializeError,
},
/// This error is encountered when deserialization pof call requests failed for some reason.
#[error("'{raw_call_request:?}' can't been serialized with error '{error}'")]
CallRequestsDeError {
raw_call_request: Vec<u8>,
error: SerdeError,
raw_call_request: SerializedCallRequests,
error: CallRequestsDeserializeError,
},
/// Errors encountered while trying to deserialize arguments from call parameters returned
@ -58,7 +62,7 @@ pub enum CallSeDeErrors {
#[error("error occurred while deserialization of arguments from call params `{call_params:?}`: {de_error}")]
CallParamsArgsDeFailed {
call_params: air_interpreter_interface::CallRequestParams,
de_error: SerdeError,
de_error: CallArgumentsDeserializeError,
},
/// Errors encountered while trying to deserialize tetraplets from call parameters returned
@ -67,9 +71,10 @@ pub enum CallSeDeErrors {
#[error("error occurred while deserialization of tetraplets from call params `{call_params:?}`: {de_error}")]
CallParamsTetrapletsDeFailed {
call_params: air_interpreter_interface::CallRequestParams,
de_error: SerdeError,
de_error: TetrapletDeserializeError,
},
}
type JValue = serde_json::Value;
pub use call_request_parameters::*;

2
avm/interface/src/raw_outcome.rs
View File

@ -43,7 +43,7 @@ impl RawAVMOutcome {
next_peer_pks,
} = outcome;
let call_requests = crate::from_raw_call_requests(call_requests)?;
let call_requests = crate::from_raw_call_requests(call_requests.into())?;
let raw_avm_outcome = Self {
ret_code,

1
avm/server/Cargo.toml
View File

@ -16,6 +16,7 @@ path = "src/lib.rs"
[dependencies]
air-interpreter-interface = { version = "0.15.2", path = "../../crates/air-lib/interpreter-interface" }
air-interpreter-sede = { version = "0.1.0", path = "../../crates/air-lib/interpreter-sede" }
air-utils = { version = "0.2.0", path = "../../crates/air-lib/utils" }
avm-data-store = { version = "0.7.3", path = "../../crates/data-store" }
marine-runtime = "0.30.0"

10
avm/server/src/runner.rs
View File

@ -17,7 +17,9 @@
use crate::RunnerError;
use crate::RunnerResult;
use air_interpreter_interface::CallResultsRepr;
use air_interpreter_interface::InterpreterOutcome;
use air_interpreter_sede::ToSerialized;
use air_utils::measure;
use avm_interface::raw_outcome::RawAVMOutcome;
use avm_interface::CallResults;
@ -210,9 +212,11 @@ fn prepare_args(
let call_results = avm_interface::into_raw_result(call_results);
let call_results = measure!(
serde_json::to_vec(&call_results).expect("the default serializer shouldn't fail"),
CallResultsRepr
.serialize(&call_results)
.expect("the default serializer shouldn't fail"),
tracing::Level::INFO,
"serde_json::to_vec call_results"
"CallResultsRepr.serialize"
);
vec![
@ -220,7 +224,7 @@ fn prepare_args(
IValue::ByteArray(prev_data.into()),
IValue::ByteArray(data.into()),
run_parameters,
IValue::ByteArray(call_results),
IValue::ByteArray(call_results.to_vec()),
]
}

1650
benches/PERFORMANCE.json
View File

File diff suppressed because it is too large Load Diff

1180
benches/PERFORMANCE.txt
View File

File diff suppressed because it is too large Load Diff

1
crates/air-lib/interpreter-data/Cargo.toml
View File

@ -19,6 +19,7 @@ air-utils = { version = "0.2.0", path = "../utils" }
aquavm-air-parser = { version = "0.11.0", path = "../air-parser" }
air-interpreter-cid = { version = "0.7.0", path = "../interpreter-cid" }
air-interpreter-signatures = { version = "0.1.5", path = "../interpreter-signatures" }
air-interpreter-sede = { version = "0.1.0", path = "../interpreter-sede", features = ["json"] }
polyplets = { version = "0.5.1", path = "../polyplets" }
fluence-keypair = { version = "0.10.3", default-features = false }

26
crates/air-lib/interpreter-data/src/interpreter_data.rs
View File

@ -15,11 +15,17 @@
*/
pub(crate) mod errors;
pub(crate) mod repr;
pub mod verification;
pub use self::repr::InterpreterDataFormat;
pub use self::repr::InterpreterDataRepr;
use crate::CidInfo;
use crate::ExecutionTrace;
use air_interpreter_sede::FromSerialized;
use air_interpreter_sede::Representation;
use air_interpreter_sede::ToSerialized;
use air_interpreter_signatures::SignatureStore;
use air_utils::measure;
@ -97,17 +103,27 @@ impl InterpreterData {
}
/// Tries to de InterpreterData from slice according to the data version.
pub fn try_from_slice(slice: &[u8]) -> Result<Self, serde_json::Error> {
pub fn try_from_slice(
slice: &[u8],
) -> Result<Self, <InterpreterDataRepr as Representation>::DeserializeError> {
measure!(
serde_json::from_slice(slice),
InterpreterDataRepr.deserialize(slice),
tracing::Level::INFO,
"serde_json::from_slice"
"InterpreterData::try_from_slice"
)
}
/// Tries to de only versions part of interpreter data.
pub fn try_get_versions(slice: &[u8]) -> Result<Versions, serde_json::Error> {
serde_json::from_slice(slice)
pub fn try_get_versions(
slice: &[u8],
) -> Result<Versions, <InterpreterDataRepr as Representation>::DeserializeError> {
InterpreterDataRepr.deserialize(slice)
}
pub fn serialize(
&self,
) -> Result<Vec<u8>, <InterpreterDataRepr as Representation>::SerializeError> {
InterpreterDataRepr.serialize(self)
}
}

75
crates/air-lib/interpreter-data/src/interpreter_data/repr.rs Normal file
View File

@ -0,0 +1,75 @@
/*
* Copyright 2023 Fluence Labs Limited
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
use crate::InterpreterData;
use crate::Versions;
use air_interpreter_sede::Format;
use air_interpreter_sede::FromSerialized;
use air_interpreter_sede::JsonFormat;
use air_interpreter_sede::Representation;
use air_interpreter_sede::ToSerialized;
use air_interpreter_sede::ToWriter;
#[derive(Default, Debug)]
pub struct InterpreterDataRepr;
pub type InterpreterDataFormat = JsonFormat;
impl Representation for InterpreterDataRepr {
type SerializeError = <InterpreterDataFormat as Format<InterpreterData>>::SerializationError;
type DeserializeError =
<InterpreterDataFormat as Format<InterpreterData>>::DeserializationError;
type WriteError = <InterpreterDataFormat as Format<InterpreterData>>::WriteError;
type Format = InterpreterDataFormat;
type SerializedValue = Vec<u8>; // TODO a typed wrapper
fn get_format(&self) -> InterpreterDataFormat {
InterpreterDataFormat::default()
}
}
impl ToSerialized<InterpreterData> for InterpreterDataRepr {
#[inline]
fn serialize(&self, value: &InterpreterData) -> Result<Vec<u8>, Self::SerializeError> {
Self::get_format(self).to_vec(value)
}
}
impl FromSerialized<InterpreterData> for InterpreterDataRepr {
#[inline]
fn deserialize(&self, repr: &[u8]) -> Result<InterpreterData, Self::DeserializeError> {
Self::get_format(self).from_slice(repr)
}
}
impl ToWriter<InterpreterData> for InterpreterDataRepr {
#[inline]
fn to_writer<W: std::io::Write>(
&self,
value: &InterpreterData,
writer: &mut W,
) -> Result<(), Self::WriteError> {
Self::get_format(self).to_writer(value, writer)
}
}
impl FromSerialized<Versions> for InterpreterDataRepr {
#[inline]
fn deserialize(&self, repr: &[u8]) -> Result<Versions, Self::DeserializeError> {
Self::get_format(self).from_slice(repr)
}
}

5
crates/air-lib/interpreter-interface/Cargo.toml
View File

@ -17,10 +17,13 @@ path = "src/lib.rs"
[dependencies]
marine-rs-sdk = {version = "0.10.2", optional = true }
fluence-it-types = { version = "0.4.1", optional = true }
air-interpreter-sede = { version = "0.1.0", path = "../interpreter-sede", features = ["json"] }
marine-call-parameters = { version = "0.10.1", default-features = false }
serde = "1.0.190"
serde_json = "1.0.108"
serde_bytes = "0.11.12"
[features]
default = ["marine"]
marine = ["marine-rs-sdk", "fluence-it-types"]
marine = ["marine-rs-sdk", "fluence-it-types", "air-interpreter-sede/marine", "marine-call-parameters/marine-abi"]

64
crates/air-lib/interpreter-interface/src/call_request_parameters.rs
View File

@ -14,14 +14,70 @@
* limitations under the License.
*/
use air_interpreter_sede::define_simple_representation;
use air_interpreter_sede::derive_serialized_type;
use air_interpreter_sede::Format;
use air_interpreter_sede::FromSerialized;
use air_interpreter_sede::JsonFormat;
use air_interpreter_sede::Representation;
use marine_call_parameters::SecurityTetraplet;
#[cfg(feature = "marine")]
use marine_rs_sdk::marine;
use serde::Deserialize;
use serde::Serialize;
use std::collections::HashMap;
use std::rc::Rc;
pub type CallRequests = HashMap<u32, CallRequestParams>;
derive_serialized_type!(SerializedCallArguments);
derive_serialized_type!(SerializedTetraplets);
derive_serialized_type!(SerializedCallRequests);
pub type CallArgumentsFormat = JsonFormat;
pub type TetrapletsFormat = JsonFormat;
pub type CallRequestsFormat = JsonFormat;
define_simple_representation! {
CallArgumentsRepr,
Vec<serde_json::Value>,
CallArgumentsFormat,
SerializedCallArguments
}
pub type CallArgumentsDeserializeError = <CallArgumentsRepr as Representation>::DeserializeError;
define_simple_representation! {
TetrapletsRepr,
// additional implementation for Vec<Vec<SecurityTetraplet>> is defined below
// TODO allow this macro to define implementations for multiple types
Vec<Vec<Rc<SecurityTetraplet>>>,
TetrapletsFormat,
SerializedTetraplets
}
pub type TetrapletDeserializeError = <TetrapletsRepr as Representation>::DeserializeError;
define_simple_representation! {
CallRequestsRepr,
CallRequests,
CallRequestsFormat,
SerializedCallRequests
}
pub type CallRequestsDeserializeError = <CallRequestsRepr as Representation>::DeserializeError;
impl FromSerialized<Vec<Vec<SecurityTetraplet>>> for TetrapletsRepr {
fn deserialize(
&self,
repr: &[u8],
) -> Result<Vec<Vec<SecurityTetraplet>>, Self::DeserializeError> {
CallArgumentsRepr.get_format().from_slice(repr)
}
}
/// Contains arguments of a call instruction and all other necessary information
/// required for calling a service.
#[cfg_attr(feature = "marine", marine)]
@ -34,18 +90,18 @@ pub struct CallRequestParams {
pub function_name: String,
/// Serialized to JSON string Vec<JValue> of arguments that should be passed to a service.
pub arguments: String,
pub arguments: SerializedCallArguments,
/// Serialized to JSON string Vec<Vec<SecurityTetraplet>> that should be passed to a service.
pub tetraplets: String,
pub tetraplets: SerializedTetraplets,
}
impl CallRequestParams {
pub fn new(
service_id: String,
function_name: String,
arguments: String,
tetraplets: String,
arguments: SerializedCallArguments,
tetraplets: SerializedTetraplets,
) -> Self {
Self {
service_id,

18
crates/air-lib/interpreter-interface/src/call_service_result.rs
View File

@ -14,6 +14,10 @@
* limitations under the License.
*/
use air_interpreter_sede::define_simple_representation;
use air_interpreter_sede::derive_serialized_type;
use air_interpreter_sede::JsonFormat;
use air_interpreter_sede::Representation;
use serde::Deserialize;
use serde::Serialize;
use serde_json::Value as JValue;
@ -22,6 +26,20 @@ use std::collections::HashMap;
pub type CallResults = HashMap<u32, CallServiceResult>;
pub const CALL_SERVICE_SUCCESS: i32 = 0;
pub type CallResultsFormat = JsonFormat;
derive_serialized_type!(SerializedCallResults);
define_simple_representation! {
CallResultsRepr,
CallResults,
CallResultsFormat,
SerializedCallResults
}
pub type CallResultsDeserializeError = <CallResultsRepr as Representation>::DeserializeError;
pub type CallResultsSerializeError = <CallResultsRepr as Representation>::SerializeError;
/// Represents an executed host function result.
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct CallServiceResult {

13
crates/air-lib/interpreter-interface/src/interpreter_outcome.rs
View File

@ -51,8 +51,9 @@ impl InterpreterOutcome {
error_message: String,
data: Vec<u8>,
next_peer_pks: Vec<String>,
call_requests: Vec<u8>,
call_requests: SerializedCallRequests,
) -> Self {
let call_requests = call_requests.into();
Self {
ret_code,
error_message,
@ -81,7 +82,13 @@ impl InterpreterOutcome {
let error_message = try_as_string(record_values.pop().unwrap(), "error_message")?;
let ret_code = try_as_i64(record_values.pop().unwrap(), "ret_code")?;
let outcome = Self::new(ret_code, error_message, data, next_peer_pks, call_requests);
let outcome = Self::new(
ret_code,
error_message,
data,
next_peer_pks,
call_requests.into(),
);
Ok(outcome)
}
@ -90,6 +97,8 @@ impl InterpreterOutcome {
#[cfg(feature = "marine")]
use fluence_it_types::ne_vec::NEVec;
use crate::SerializedCallRequests;
#[cfg(feature = "marine")]
fn try_as_record(ivalue: IValue) -> Result<NEVec<IValue>, String> {
match ivalue {

28
crates/air-lib/interpreter-sede/Cargo.toml Normal file
View File

@ -0,0 +1,28 @@
[package]
name = "air-interpreter-sede"
description = "AIR interpreter serialization and deserialization util module"
version = "0.1.0"
authors = ["Fluence Labs"]
edition = "2021"
license = "Apache-2.0"
documentation = "https://docs.rs/air-interpreter-sede"
repository = "https://github.com/fluencelabs/aquavm/tree/master/crates/air-lib/interpreter-sede"
keywords = ["fluence", "air", "programming-language", "TODO"]
categories = ["wasm"]
[dependencies]
rmp-serde = { version = "1.1.2", optional = true }
serde = { version = "1.0.190" }
serde_json = { version = "1.0.108" }
marine-rs-sdk = { version = "0.10.0", optional = true }
unsigned-varint = { version = "0.8.0", default-features = false, features = ["std"] }
thiserror = "1.0.50"
serde_bytes = "0.11.12"
[features]
default = []
serde_json = []
rmp-serde = ["dep:rmp-serde"]
marine = ["dep:marine-rs-sdk"]
json = ["serde_json"]
msgpack = ["rmp-serde"]

49
crates/air-lib/interpreter-sede/src/format.rs Normal file
View File

@ -0,0 +1,49 @@
/*
* Copyright 2023 Fluence Labs Limited
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
use std::fmt::Debug;
pub trait Format<Value> {
type SerializationError: Debug;
type DeserializationError: Debug;
type WriteError: Debug;
fn to_vec(&self, val: &Value) -> Result<Vec<u8>, Self::SerializationError>;
// todo owned_from_slice
#[allow(clippy::wrong_self_convention)]
fn from_slice(&self, slice: &[u8]) -> Result<Value, Self::DeserializationError>;
fn to_writer<W: std::io::Write>(
&self,
value: &Value,
write: &mut W,
) -> Result<(), Self::WriteError>;
}
pub trait BorrowFormat<'data, Value: 'data>: Format<Value> {
fn borrow_from_slice(&self, slice: &'data [u8]) -> Result<Value, Self::DeserializationError>;
}
pub trait ArchivedFormat<Value>: Format<Value> {
type Archived;
type ValidationError;
fn archived_from_slice<'data>(
&self,
slice: &'data [u8],
) -> Result<&'data Self::Archived, Self::ValidationError>;
}

52
crates/air-lib/interpreter-sede/src/lib.rs Normal file
View File

@ -0,0 +1,52 @@
/*
* Copyright 2023 Fluence Labs Limited
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
pub mod multiformat;
pub(crate) mod format;
pub(crate) mod representation;
pub(crate) mod serialized_type;
pub use crate::format::Format;
pub use crate::representation::FromSerialiedBorrow;
pub use crate::representation::FromSerialized;
pub use crate::representation::Representation;
pub use crate::representation::ToSerialized;
pub use crate::representation::ToWriter;
#[cfg(feature = "rmp-serde")]
pub(crate) mod rmp_serde;
#[cfg(feature = "rmp-serde")]
pub use crate::rmp_serde::RmpSerdeFormat;
#[cfg(feature = "rmp-serde")]
pub use crate::rmp_serde::RmpSerdeMultiformat;
#[cfg(feature = "msgpack")]
pub use crate::rmp_serde::RmpSerdeFormat as MsgPackFormat;
#[cfg(feature = "msgpack")]
pub use crate::rmp_serde::RmpSerdeMultiformat as MsgPackMultiformat;
#[cfg(feature = "serde_json")]
pub(crate) mod serde_json;
#[cfg(feature = "serde_json")]
pub use crate::serde_json::SerdeJsonFormat;
#[cfg(feature = "serde_json")]
pub use crate::serde_json::SerdeJsonMultiformat;
#[cfg(feature = "json")]
pub use crate::serde_json::SerdeJsonFormat as JsonFormat;
#[cfg(feature = "json")]
pub use crate::serde_json::SerdeJsonMultiformat as JsonMultiformat;

94
crates/air-lib/interpreter-sede/src/multiformat.rs Normal file
View File

@ -0,0 +1,94 @@
/*
* Copyright 2023 Fluence Labs Limited
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
use std::io::Write;
use crate::Format;
use unsigned_varint::decode as varint_decode;
use unsigned_varint::encode as varint_encode;
pub type SerializationCodec = u32;
const ENCODING_BUFFER_CAPACITY: usize = 1024;
#[derive(thiserror::Error, Debug)]
pub enum DecodeError<FormatError> {
#[error(transparent)]
Format(FormatError),
#[error("unsupported multiformat codec: {0}")]
Codec(SerializationCodec),
#[error("failed to parse multiformat: {0}")]
VarInt(#[from] varint_decode::Error),
}
#[derive(thiserror::Error, Debug)]
pub enum EncodeError<FormatError> {
#[error(transparent)]
Format(FormatError),
#[error("failed to write: {0}")]
Io(#[from] std::io::Error),
}
pub fn parse_multiformat_bytes(
data: &[u8],
) -> Result<(SerializationCodec, &[u8]), varint_decode::Error> {
varint_decode::u32(data)
}
pub fn encode_multiformat<Value, Fmt: Format<Value>>(
data: &Value,
codec: SerializationCodec,
format: &Fmt,
) -> Result<Vec<u8>, EncodeError<<Fmt as Format<Value>>::WriteError>> {
let mut output = Vec::with_capacity(ENCODING_BUFFER_CAPACITY);
write_multiformat(data, codec, format, &mut output)?;
Ok(output)
}
pub fn write_multiformat<Value, Fmt: Format<Value>, W: Write>(
data: &Value,
codec: SerializationCodec,
format: &Fmt,
output: &mut W,
) -> Result<(), EncodeError<<Fmt as Format<Value>>::WriteError>> {
// looks weird, but that's how the API is
let mut buf = varint_encode::u32_buffer();
let codec_bytes = varint_encode::u32(codec, &mut buf);
output.write_all(codec_bytes)?;
format
.to_writer(data, output)
.map_err(EncodeError::Format)?;
Ok(())
}
pub fn decode_multiformat<Value, Fmt: Format<Value>>(
multiformat_data: &[u8],
expected_codec: SerializationCodec,
format: &Fmt,
) -> Result<Value, DecodeError<<Fmt as Format<Value>>::DeserializationError>> {
let (data_codec, data) = parse_multiformat_bytes(multiformat_data)?;
if data_codec != expected_codec {
// TODO we may be more permissive, having kind of registry for the possible incoming formats, akin to
// CID algorithms; but it may be *really* tricky to organize it
return Err(DecodeError::Codec(data_codec));
}
format.from_slice(data).map_err(DecodeError::Format)
}

111
crates/air-lib/interpreter-sede/src/representation.rs Normal file
View File

@ -0,0 +1,111 @@
/*
* Copyright 2023 Fluence Labs Limited
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/// A formatter intended for particular type, a base type that defines generic behavior
/// used by particular implementations.
pub trait Representation {
type SerializeError;
type DeserializeError;
type WriteError;
type Format;
type SerializedValue: std::ops::Deref<Target = [u8]>;
fn get_format(&self) -> Self::Format;
}
/// Serialization trait restricted to for particular type.
pub trait ToSerialized<Value>: Representation {
fn serialize(&self, value: &Value) -> Result<Self::SerializedValue, Self::SerializeError>;
}
/// Owned deserialization trait restricted to for particular type.
pub trait FromSerialized<Value>: Representation {
fn deserialize(&self, repr: &[u8]) -> Result<Value, Self::DeserializeError>;
}
/// Borrow deserialization trait restricted to for particular type.
pub trait FromSerialiedBorrow<'data, Value: 'data>: Representation {
fn deserialize_borrow(&self, repr: &'data [u8]) -> Result<Value, Self::DeserializeError>;
}
/// Writing deserialization trait restricted to for particular type.
pub trait ToWriter<Value>: Representation {
fn to_writer<W: std::io::Write>(
&self,
value: &Value,
writer: &mut W,
) -> Result<(), Self::WriteError>;
}
#[macro_export]
macro_rules! define_simple_representation {
($repr_type:ident, $value_type:ty, $format_type:ty, $serialized_value:ty) => {
#[derive(Default)]
pub struct $repr_type;
impl $crate::Representation for $repr_type {
type SerializeError = <$format_type as $crate::Format<$value_type>>::SerializationError;
type DeserializeError =
<$format_type as $crate::Format<$value_type>>::DeserializationError;
type WriteError = <$format_type as $crate::Format<$value_type>>::WriteError;
type Format = $format_type;
type SerializedValue = $serialized_value;
#[inline]
fn get_format(&self) -> Self::Format {
<$format_type>::default()
}
}
impl $crate::ToSerialized<$value_type> for $repr_type {
#[inline]
fn serialize(
&self,
value: &$value_type,
) -> Result<$serialized_value, Self::SerializeError> {
use $crate::Format;
use $crate::Representation;
Self::get_format(self).to_vec(value).map(Into::into)
}
}
impl $crate::FromSerialized<$value_type> for $repr_type {
#[inline]
fn deserialize(&self, repr: &[u8]) -> Result<$value_type, Self::DeserializeError> {
use $crate::Format;
use $crate::Representation;
Self::get_format(self).from_slice(repr)
}
}
impl $crate::ToWriter<$value_type> for $repr_type {
#[inline]
fn to_writer<W: std::io::Write>(
&self,
value: &$value_type,
writer: &mut W,
) -> Result<(), Self::WriteError> {
use $crate::Format;
use $crate::Representation;
Self::get_format(self).to_writer(value, writer)
}
}
};
}

95
crates/air-lib/interpreter-sede/src/rmp_serde.rs Normal file
View File

@ -0,0 +1,95 @@
/*
* Copyright 2023 Fluence Labs Limited
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
use crate::format::BorrowFormat;
use crate::multiformat::SerializationCodec;
use crate::Format;
// https://github.com/multiformats/multicodec/blob/master/table.csv
const MULTIFORMAT_MSGPCK: SerializationCodec = 0x0201;
// rmp_serde has config with human-readable representation too, but I'm not sure it worth it
#[derive(Copy, Clone, Default)]
pub struct RmpSerdeFormat;
impl<Value> Format<Value> for RmpSerdeFormat
where
Value: serde::Serialize + serde::de::DeserializeOwned,
{
type SerializationError = rmp_serde::encode::Error;
type DeserializationError = rmp_serde::decode::Error;
type WriteError = rmp_serde::encode::Error;
#[inline]
fn to_vec(&self, val: &Value) -> Result<Vec<u8>, Self::SerializationError> {
rmp_serde::to_vec(val)
}
#[inline]
fn from_slice(&self, slice: &[u8]) -> Result<Value, Self::DeserializationError> {
rmp_serde::from_slice(slice)
}
#[inline]
fn to_writer<W: std::io::Write>(
&self,
value: &Value,
write: &mut W,
) -> Result<(), Self::WriteError> {
value.serialize(&mut rmp_serde::Serializer::new(write))
}
}
impl<'data, Value: 'data> BorrowFormat<'data, Value> for RmpSerdeFormat
where
Value: serde::Serialize + for<'de> serde::Deserialize<'de>,
{
#[inline]
fn borrow_from_slice(&self, slice: &'data [u8]) -> Result<Value, Self::DeserializationError> {
rmp_serde::from_slice(slice)
}
}
#[derive(Copy, Clone, Default)]
pub struct RmpSerdeMultiformat;
impl<Value> Format<Value> for RmpSerdeMultiformat
where
Value: serde::Serialize + serde::de::DeserializeOwned,
{
type SerializationError = crate::multiformat::EncodeError<rmp_serde::encode::Error>;
type DeserializationError = crate::multiformat::DecodeError<rmp_serde::decode::Error>;
type WriteError = crate::multiformat::EncodeError<rmp_serde::encode::Error>;
#[inline]
fn to_vec(&self, value: &Value) -> Result<Vec<u8>, Self::SerializationError> {
crate::multiformat::encode_multiformat(value, MULTIFORMAT_MSGPCK, &RmpSerdeFormat)
}
#[inline]
fn from_slice(&self, slice: &[u8]) -> Result<Value, Self::DeserializationError> {
crate::multiformat::decode_multiformat(slice, MULTIFORMAT_MSGPCK, &RmpSerdeFormat)
}
#[inline]
fn to_writer<W: std::io::Write>(
&self,
value: &Value,
write: &mut W,
) -> Result<(), Self::WriteError> {
crate::multiformat::write_multiformat(value, MULTIFORMAT_MSGPCK, &RmpSerdeFormat, write)
}
}

84
crates/air-lib/interpreter-sede/src/serde_json.rs Normal file
View File

@ -0,0 +1,84 @@
/*
* Copyright 2023 Fluence Labs Limited
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
use crate::multiformat::SerializationCodec;
use crate::Format;
// https://github.com/multiformats/multicodec/blob/master/table.csv
const MULTIFORMAT_JSON: SerializationCodec = 0x0200;
// TODO a human-readability flag?
#[derive(Copy, Clone, Default)]
pub struct SerdeJsonFormat;
impl<Value> Format<Value> for SerdeJsonFormat
where
Value: serde::Serialize + serde::de::DeserializeOwned,
{
type SerializationError = serde_json::Error;
type DeserializationError = serde_json::Error;
type WriteError = serde_json::Error;
#[inline]
fn to_vec(&self, value: &Value) -> Result<Vec<u8>, Self::SerializationError> {
serde_json::to_vec(value)
}
#[inline]
fn from_slice(&self, slice: &[u8]) -> Result<Value, Self::DeserializationError> {
serde_json::from_slice(slice)
}
#[inline]
fn to_writer<W: std::io::Write>(
&self,
value: &Value,
write: &mut W,
) -> Result<(), Self::WriteError> {
serde_json::to_writer(write, value)
}
}
#[derive(Copy, Clone, Default)]
pub struct SerdeJsonMultiformat;
impl<Value> Format<Value> for SerdeJsonMultiformat
where
Value: serde::Serialize + serde::de::DeserializeOwned,
{
type SerializationError = crate::multiformat::EncodeError<serde_json::Error>;
type DeserializationError = crate::multiformat::DecodeError<serde_json::Error>;
type WriteError = crate::multiformat::EncodeError<serde_json::Error>;
#[inline]
fn to_vec(&self, value: &Value) -> Result<Vec<u8>, Self::SerializationError> {
crate::multiformat::encode_multiformat(value, MULTIFORMAT_JSON, &SerdeJsonFormat)
}
#[inline]
fn from_slice(&self, slice: &[u8]) -> Result<Value, Self::DeserializationError> {
crate::multiformat::decode_multiformat(slice, MULTIFORMAT_JSON, &SerdeJsonFormat)
}
#[inline]
fn to_writer<W: std::io::Write>(
&self,
value: &Value,
write: &mut W,
) -> Result<(), Self::WriteError> {
crate::multiformat::write_multiformat(value, MULTIFORMAT_JSON, &SerdeJsonFormat, write)
}
}

101
crates/air-lib/interpreter-sede/src/serialized_type.rs Normal file
View File

@ -0,0 +1,101 @@
/*
* Copyright 2023 Fluence Labs Limited
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#[macro_export]
macro_rules! derive_serialized_type {
($type_name:ident) => {
#[derive(
::core::fmt::Debug,
::core::default::Default,
::serde::Serialize,
::serde::Deserialize,
::core::cmp::PartialEq,
::core::cmp::Eq,
::core::hash::Hash,
::core::clone::Clone,
)]
#[serde(transparent)]
#[repr(transparent)]
#[cfg_attr(
any(feature = "marine", feature = "marine-abi"),
::marine_rs_sdk::marine
)]
pub struct $type_name {
// it cannot be implemented as a tuple as marine doesn't support tuple structs
#[serde(with = "serde_bytes")]
value: ::std::vec::Vec<u8>,
}
impl ::core::convert::From<::std::vec::Vec<u8>> for $type_name {
#[inline]
fn from(value: ::std::vec::Vec<u8>) -> Self {
Self { value }
}
}
impl ::core::convert::From<$type_name> for ::std::vec::Vec<u8> {
#[inline]
fn from(value: $type_name) -> Self {
value.value
}
}
impl ::core::ops::Deref for $type_name {
type Target = [u8];
#[inline]
fn deref(&self) -> &Self::Target {
&self.value
}
}
};
($type_name:ident, $decl:meta) => {
#[derive(
::core::fmt::Debug,
::core::default::Default,
::serde::Serialize,
::serde::Deserialize,
::core::cmp::PartialEq,
::core::cmp::Eq,
::core::hash::Hash,
::core::clone::Clone,
)]
#[serde(transparent)]
#[repr(transparent)]
#[$decl]
pub struct $type_name {
// it cannot be implemented as a tuple as marine doesn't support tuple structs
value: ::std::vec::Vec<u8>,
}
impl ::core::convert::From<::std::vec::Vec<u8>> for $type_name {
#[inline]
fn from(value: ::std::vec::Vec<u8>) -> Self {
Self { value }
}
}
impl ::core::ops::Deref for $type_name {
type Target = [u8];
#[inline]
fn deref(&self) -> &Self::Target {
&self.value
}
}
};
}

1
crates/air-lib/test-utils/Cargo.toml
View File

@ -19,6 +19,7 @@ aquavm-air = { version = "0.55.0", path = "../../../air" }
air-interpreter-cid = { version = "0.7.0", path = "../interpreter-cid" }
air-interpreter-data = { version = "0.15.0", path = "../interpreter-data" }
air-interpreter-interface = { version = "0.15.2", path = "../interpreter-interface" }
air-interpreter-sede = { version = "0.1.0", path = "../interpreter-sede" }
air-interpreter-signatures = { version = "0.1.5", path = "../interpreter-signatures" }
avm-interface = { version = "0.29.3", path = "../../../avm/interface" }
avm-server = { version = "0.33.4", path = "../../../avm/server" }

12
crates/air-lib/test-utils/src/lib.rs
View File

@ -35,6 +35,8 @@ pub mod native_test_runner;
pub mod wasm_test_runner;
pub use air::interpreter_data::*;
use air_interpreter_sede::ToSerialized;
use air::ExecutionCidState;
pub use avm_interface::raw_outcome::*;
pub use avm_server::*;
@ -84,7 +86,7 @@ pub fn trace_from_result(result: &RawAVMOutcome) -> ExecutionTrace {
}
pub fn data_from_result(result: &RawAVMOutcome) -> InterpreterData {
serde_json::from_slice(&result.data).expect("default serializer shouldn't fail")
InterpreterData::try_from_slice(&result.data).expect("default serializer shouldn't fail")
}
pub fn raw_data_from_trace(
@ -98,7 +100,9 @@ pub fn raw_data_from_trace(
0,
semver::Version::new(1, 1, 1),
);
serde_json::to_vec(&data).expect("default serializer shouldn't fail")
InterpreterDataRepr
.serialize(&data)
.expect("default serializer shouldn't fail")
}
pub fn raw_data_from_trace_with_canon(
@ -118,7 +122,9 @@ pub fn raw_data_from_trace_with_canon(
0,
semver::Version::new(1, 1, 1),
);
serde_json::to_vec(&data).expect("default serializer shouldn't fail")
InterpreterDataRepr
.serialize(&data)
.expect("default serializer shouldn't fail")
}
#[macro_export]

4
crates/air-lib/test-utils/src/native_test_runner.rs
View File

@ -15,7 +15,9 @@
*/
use crate::test_runner::AirRunner;
use air_interpreter_interface::CallResultsRepr;
use air_interpreter_interface::RunParameters;
use air_interpreter_sede::ToSerialized;
use avm_server::avm_runner::*;
use avm_server::into_raw_result;
use fluence_keypair::KeyPair;
@ -46,7 +48,7 @@ impl AirRunner for NativeAirRunner {
) -> Result<RawAVMOutcome, Box<dyn std::error::Error>> {
// some inner parts transformations
let raw_call_results = into_raw_result(call_results);
let raw_call_results = serde_json::to_vec(&raw_call_results).unwrap();
let raw_call_results = CallResultsRepr.serialize(&raw_call_results).unwrap();
let current_peer_id =
override_current_peer_id.unwrap_or_else(|| self.current_peer_id.clone());

6
crates/air-lib/test-utils/src/test_runner.rs
View File

@ -282,6 +282,12 @@ mod tests {
)
.expect("call should be success");
assert_eq!(
current_result_1.ret_code, 0,
"{:?}",
current_result_1.error_message
);
let expected_current_call_requests = HashMap::new();
let expected_current_next_peers_pks = vec![spell_id.to_owned()];

1
junk/gen-bench-data/Cargo.toml
View File

@ -10,6 +10,7 @@ aquavm-air = { path = "../../air", features = ["gen_signatures", "check_signatur
air-interpreter-cid = { path = "../../crates/air-lib/interpreter-cid" }
air-interpreter-data = { path = "../../crates/air-lib/interpreter-data" }
air-interpreter-interface = { path = "../../crates/air-lib/interpreter-interface" }
air-interpreter-sede = { path = "../../crates/air-lib/interpreter-sede", features = ["serde_json", "rmp-serde"] }
air-interpreter-signatures = { path = "../../crates/air-lib/interpreter-signatures" }
air-test-utils = { path = "../../crates/air-lib/test-utils" }
air-testing-framework = { path = "../../crates/testing-framework" }

5
junk/gen-bench-data/src/cid_benchmarking/mod.rs
View File

@ -61,6 +61,7 @@ fn derive_peer_id(peer_name: &str, peer_id_cache: &mut HashMap<String, String>)
.clone()
}
// input data is always JSON and doesn't depend on InterpreterData representation
fn read_data<T: DeserializeOwned>(path: &str) -> T {
let inp = std::fs::File::open(path).unwrap();
serde_json::from_reader(inp).unwrap()
@ -109,6 +110,8 @@ pub fn cid_benchmarking_data(
peer_id: String,
particle_id: &str,
) -> Vec<u8> {
use air_interpreter_sede::Format;
let mut curr_data: PreCidInterpeterData = read_data(curr_data_filename);
let calls: TraceCalls = read_data("src/cid_benchmarking/simple-calls-info.json");
let mut calls = calls.into_iter();
@ -158,7 +161,7 @@ pub fn cid_benchmarking_data(
.unwrap()
.insert("signatures".to_owned(), json!(ss));
serde_json::to_vec(&curr_data).unwrap()
InterpreterDataFormat::default().to_vec(&curr_data).unwrap().into()
}
pub fn cid_benchmarking_long_data(

77
junk/gen-bench-data/src/main.rs
View File

@ -1,3 +1,9 @@
use air_interpreter_data::InterpreterDataFormat;
use air_interpreter_sede::Format;
use air_interpreter_sede::RmpSerdeFormat;
use air_interpreter_sede::RmpSerdeMultiformat;
use air_interpreter_sede::SerdeJsonFormat;
use air_interpreter_sede::SerdeJsonMultiformat;
use air_test_framework::*;
use air_test_utils::key_utils::derive_dummy_keypair;
use air_test_utils::prelude::*;
@ -6,7 +12,8 @@ use clap::Parser;
use clap::Subcommand;
use itertools::Itertools as _;
use maplit::hashmap;
use serde::Serialize;
use std::borrow::Cow;
use std::collections::HashMap;
use std::path::Path;
use std::path::PathBuf;
@ -103,17 +110,17 @@ fn save_data(dest_dir: &Path, data: Data) -> Result<(), Box<dyn std::error::Erro
save_file(
dest_dir,
"prev_data.json",
&reformat_json_if_possible(&data.prev_data),
reformat_json_if_possible::<InterpreterDataFormat>(&data.prev_data),
)?;
save_file(
dest_dir,
"cur_data.json",
&reformat_json_if_possible(&data.cur_data),
reformat_json_if_possible::<InterpreterDataFormat>(&data.cur_data),
)?;
save_file(
dest_dir,
"params.json",
&serde_json::to_vec_pretty(&data.params_json)?,
serde_json::to_vec_pretty(&data.params_json)?,
)?;
save_file(dest_dir, "keypair.ed25519", &data.keypair)?;
@ -123,7 +130,9 @@ fn save_data(dest_dir: &Path, data: Data) -> Result<(), Box<dyn std::error::Erro
"call_results.json",
// these call results are intended for manual generation too for the AIR CLI, so
// simplier representation from avm_interface::CallResults is used, and JSON is used explicitely
&reformat_json_if_possible(&serde_json::to_vec(&call_results).unwrap()),
reformat_json_if_possible::<SerdeJsonFormat>(
&serde_json::to_vec(&call_results).unwrap(),
),
)
.unwrap();
}
@ -131,29 +140,61 @@ fn save_data(dest_dir: &Path, data: Data) -> Result<(), Box<dyn std::error::Erro
Ok(())
}
trait Reformatter: Format<()> {
fn reformat(data: &[u8]) -> Cow<'_, [u8]>;
}
impl Reformatter for SerdeJsonFormat {
/// make zero-indentation data for more convenient git diffs
fn reformat(data: &[u8]) -> Cow<'_, [u8]> {
use serde::ser::Serialize;
let obj: serde_json::Value = serde_json::from_slice(data).unwrap();
let fmt = serde_json::ser::PrettyFormatter::with_indent(&[]);
let mut out = vec![];
{
let mut ser = serde_json::ser::Serializer::with_formatter(&mut out, fmt);
obj.serialize(&mut ser).unwrap();
}
out.into()
}
}
impl Reformatter for SerdeJsonMultiformat {
/// make zero-indentation data for more convenient git diffs
fn reformat(data: &[u8]) -> Cow<'_, [u8]> {
data.into()
}
}
impl Reformatter for RmpSerdeFormat {
fn reformat(data: &[u8]) -> Cow<'_, [u8]> {
data.into()
}
}
impl Reformatter for RmpSerdeMultiformat {
fn reformat(data: &[u8]) -> Cow<'_, [u8]> {
data.into()
}
}
/// make zero-indentation data for better git diffs
fn reformat_json_if_possible(data: &[u8]) -> Vec<u8> {
fn reformat_json_if_possible<R: Reformatter>(data: &[u8]) -> Cow<'_, [u8]> {
if data.is_empty() {
return data.into();
}
let obj: serde_json::Value = serde_json::from_slice(data).unwrap();
let fmt = serde_json::ser::PrettyFormatter::with_indent(&[]);
let mut out = vec![];
let mut ser = serde_json::ser::Serializer::with_formatter(&mut out, fmt);
obj.serialize(&mut ser).unwrap();
out
R::reformat(data)
}
fn save_file(
dest_dir: &Path,
filename: &str,
data: impl AsRef<[u8]>,
) -> Result<(), Box<dyn std::error::Error>> {
fn save_file(dest_dir: &Path, filename: &str, data: impl AsRef<[u8]>) -> std::io::Result<()> {
let mut dest_dir = dest_dir.to_owned();
dest_dir.push(filename);
Ok(std::fs::write(dest_dir, data)?)
std::fs::write(dest_dir, data)
}
#[derive(Debug, Default)]

1
tools/cli/air/Cargo.toml
View File

@ -15,6 +15,7 @@ air-beautifier = { version = "0.4.0", path = "../../../crates/beautifier" }
avm-data-store = { version = "0.7.3", path = "../../../crates/data-store" }
avm-interface = { version = "0.29.3", path = "../../../avm/interface" }
air-interpreter-interface = { version = "0.15.2", path = "../../../crates/air-lib/interpreter-interface", default-features = false }
air-interpreter-sede = { version = "0.1.0", path = "../../../crates/air-lib/interpreter-sede", default-features = false }
air-test-utils = { version = "0.13.0",path = "../../../crates/air-lib/test-utils", optional = true }
anyhow = "1.0.75"

14
tools/cli/air/src/trace/run.rs
View File

@ -191,8 +191,8 @@ pub(crate) fn run(args: Args) -> anyhow::Result<()> {
let result = runner
.call_tracing(
execution_data.air_script.clone(),
execution_data.prev_data.clone().into(),
execution_data.current_data.clone().into(),
execution_data.prev_data.clone(),
execution_data.current_data.clone(),
particle.init_peer_id.clone().into_owned(),
particle.timestamp,
particle.ttl,
@ -276,7 +276,7 @@ fn read_call_results(call_results_path: Option<&Path>) -> anyhow::Result<CallRes
load_data(call_results_path).context("failed to read call_results")?;
// call resuls are may be manually crafted, so JSON representation
// of avm_interface::CallResults is more user-friendly
Ok(serde_json::from_str(&call_results_json)
Ok(serde_json::from_slice(&call_results_json)
.context("failed to parse call_results data")?)
}
}
@ -284,16 +284,16 @@ fn read_call_results(call_results_path: Option<&Path>) -> anyhow::Result<CallRes
fn load_data_or_default(
data_path: Option<impl AsRef<Path>>,
default: &str,
) -> anyhow::Result<String> {
default: &[u8],
) -> anyhow::Result<Vec<u8>> {
match data_path {
None => Ok(default.to_owned()),
Some(data_path) => load_data(data_path.as_ref()),
}
}
fn load_data(data_path: &Path) -> anyhow::Result<String> {
Ok(std::fs::read_to_string(data_path)?)
fn load_data(data_path: &Path) -> anyhow::Result<Vec<u8>> {
Ok(std::fs::read(data_path)?)
}
fn load_keypair_ed25519(path: &PathBuf) -> Result<KeyPair, anyhow::Error> {

11
tools/cli/air/src/trace/run/data/anomaly.rs
View File

@ -32,15 +32,12 @@ pub(crate) struct AnomalyDataArgs {
pub(crate) fn load(args: &AnomalyDataArgs) -> anyhow::Result<super::ExecutionData<'_>> {
let anomaly_json = load_data(&args.anomaly_data_path).context("Failed to read anomaly data")?;
let anomaly_data: AnomalyData<'_> =
serde_json::from_str(&anomaly_json).context("Failed to parse anomaly data")?;
serde_json::from_slice(&anomaly_json).context("Failed to parse anomaly data")?;
let air_script = anomaly_data.air_script.to_string();
let prev_data = String::from_utf8(anomaly_data.prev_data.to_vec())
.context("Anomaly current_data is not a valid string")?;
let current_data = String::from_utf8(anomaly_data.current_data.to_vec())
.context("Anomaly current_data is not a valid string")?;
let particle: ParticleParameters<'static> =
serde_json::from_reader(&*anomaly_data.particle.to_vec())
let prev_data = anomaly_data.prev_data.to_vec();
let current_data = anomaly_data.current_data.to_vec();
let particle: ParticleParameters<'static> = serde_json::from_slice(&anomaly_data.particle)
.context("Anomaly particle is not a valid JSON")?;
Ok(ExecutionData {

4
tools/cli/air/src/trace/run/data/mod.rs
View File

@ -21,7 +21,7 @@ use avm_interface::ParticleParameters;
pub(crate) struct ExecutionData<'ctx> {
pub(crate) air_script: String,
pub(crate) current_data: String,
pub(crate) prev_data: String,
pub(crate) current_data: Vec<u8>,
pub(crate) prev_data: Vec<u8>,
pub(crate) particle: ParticleParameters<'ctx>,
}

2
tools/cli/air/src/trace/run/data/plain.rs
View File

@ -22,7 +22,7 @@ use anyhow::Context;
use std::path::{Path, PathBuf};
const DEFAULT_DATA: &str = "";
const DEFAULT_DATA: &[u8] = b"";
#[derive(clap::Args, Debug)]
pub(crate) struct PlainDataArgs {

4
tools/cli/air/src/trace/run/native.rs
View File

@ -15,6 +15,7 @@
*/
use super::runner::AirRunner;
use air_interpreter_interface::CallResultsRepr;
use air_interpreter_interface::RunParameters;
use avm_interface::raw_outcome::RawAVMOutcome;
use fluence_keypair::KeyPair;
@ -38,11 +39,12 @@ impl AirRunner for NativeAvmRunner {
keypair: &KeyPair,
particle_id: String,
) -> anyhow::Result<RawAVMOutcome> {
use air_interpreter_sede::ToSerialized;
use avm_interface::into_raw_result;
// some inner parts transformations
let raw_call_results = into_raw_result(call_results);
let raw_call_results = serde_json::to_vec(&raw_call_results).unwrap();
let raw_call_results = CallResultsRepr.serialize(&raw_call_results).unwrap();
let key_format = keypair.key_format().into();
let secret_key_bytes = keypair.secret().expect("Failed to get secret key");

14
tools/cli/performance_metering/performance_metering/bench.py
View File

@ -108,7 +108,19 @@ class Bench:
)
lines = proc.stderr.decode("utf-8").split('\n')
all_output.extend(lines)
return list(map(json.loads, filter(lambda x: x, all_output)))
def try_to_parse_json(s):
try:
return json.loads(s)
except Exception:
logger.error("failed to parse JSON: %r", s)
return None
return list(filter(
lambda x: x is not None,
map(try_to_parse_json,
filter(lambda x: x, all_output))
))
def get_name(self):
"""Return the bench name."""