fluencelabs/aquavm
1
0
Fork 0
mirror of https://github.com/fluencelabs/aquavm synced 2024-12-04 07:10:18 +00:00
Code Issues Projects Releases Wiki Activity

feat(avm)!: improve anomaly detection (#479)

Browse Source
This commit is contained in:
Mike Voronov 2023-02-21 20:28:56 +03:00 committed by GitHub
parent d8c3d70656
commit 5e6863d4d5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 70 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
Cargo.lock generated
View File

@ -65,7 +65,7 @@ version = "0.1.1"
dependencies = [
"air-beautifier",
"anyhow",
"clap 4.1.4",
"clap 4.1.6",
]
[[package]]
@ -233,7 +233,7 @@ dependencies = [
"anyhow",
"avm-data-store",
"avm-interface",
"clap 4.1.4",
"clap 4.1.6",
"itertools",
"serde",
"serde_json",
@ -333,6 +333,7 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
name = "avm-data-store"
version = "0.4.1"
dependencies = [
"avm-interface",
"serde",
"serde_bytes",
"serde_json",
@ -344,7 +345,6 @@ version = "0.28.1"
dependencies = [
"air-interpreter-interface",
"air-utils",
"avm-data-store",
"log",
"maplit",
"polyplets",
@ -452,18 +452,6 @@ version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfa8873f51c92e232f9bac4065cddef41b714152812bfc5f7672ba16d6ef8cd9"
[[package]]
name = "bstr"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"
dependencies = [
"lazy_static",
"memchr",
"regex-automata",
"serde",
]
[[package]]
name = "bumpalo"
version = "3.12.0"
@ -596,9 +584,9 @@ dependencies = [
[[package]]
name = "clap"
version = "4.1.4"
version = "4.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f13b9c79b5d1dd500d20ef541215a6423c75829ef43117e1b4d17fd8af0b5d76"
checksum = "ec0b0588d44d4d63a87dbd75c136c166bbfd9a86a31cb89e09906521c7d3f5e3"
dependencies = [
"bitflags",
"clap_derive",
@ -911,13 +899,12 @@ dependencies = [
[[package]]
name = "csv"
version = "1.1.6"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1"
checksum = "af91f40b7355f82b0a891f50e70399475945bb0b0da4f1700ce60761c9d3e359"
dependencies = [
"bstr",
"csv-core",
"itoa 0.4.8",
"itoa",
"ryu",
"serde",
]
@ -943,9 +930,9 @@ dependencies = [
[[package]]
name = "cxx"
version = "1.0.89"
version = "1.0.91"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc831ee6a32dd495436e317595e639a587aa9907bef96fe6e6abc290ab6204e9"
checksum = "86d3488e7665a7a483b57e25bdd90d0aeb2bc7608c8d0346acf2ad3f1caf1d62"
dependencies = [
"cc",
"cxxbridge-flags",
@ -955,9 +942,9 @@ dependencies = [
[[package]]
name = "cxx-build"
version = "1.0.89"
version = "1.0.91"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94331d54f1b1a8895cd81049f7eaaaef9d05a7dcb4d1fd08bf3ff0806246789d"
checksum = "48fcaf066a053a41a81dfb14d57d99738b767febb8b735c3016e469fac5da690"
dependencies = [
"cc",
"codespan-reporting",
@ -970,15 +957,15 @@ dependencies = [
[[package]]
name = "cxxbridge-flags"
version = "1.0.89"
version = "1.0.91"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48dcd35ba14ca9b40d6e4b4b39961f23d835dbb8eed74565ded361d93e1feb8a"
checksum = "a2ef98b8b717a829ca5603af80e1f9e2e48013ab227b68ef37872ef84ee479bf"
[[package]]
name = "cxxbridge-macro"
version = "1.0.89"
version = "1.0.91"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81bbeb29798b407ccd82a3324ade1a7286e0d29851475990b612670f6f5124d2"
checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892"
dependencies = [
"proc-macro2",
"quote",
@ -1183,9 +1170,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
[[package]]
name = "fluence-app-service"
version = "0.23.0"
version = "0.23.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3264bb11b79b9e3e4e5b722d986efd461e4606477ba2e454e61ddf79bc1d3bf1"
checksum = "942caba094d1c219ad3656d51c827470d12dac834b4325843e9f61b12bf0d0dd"
dependencies = [
"log",
"maplit",
@ -1521,12 +1508,6 @@ dependencies = [
"either",
]
[[package]]
name = "itoa"
version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
[[package]]
name = "itoa"
version = "1.0.5"
@ -1824,9 +1805,9 @@ dependencies = [
[[package]]
name = "marine-runtime"
version = "0.24.0"
version = "0.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba3726e9dccce778d0b848ebc9adb312f33f338128bb182aa5ed8429f3f7aac1"
checksum = "545262bf5b68d2b4f4b9eed2557f09421cc4bb62a10e71c307140f27b6411b9c"
dependencies = [
"bytesize",
"it-json-serde",
@ -2053,9 +2034,9 @@ dependencies = [
[[package]]
name = "once_cell"
version = "1.17.0"
version = "1.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"
checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
[[package]]
name = "oorandom"
@ -2577,7 +2558,7 @@ version = "1.0.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76"
dependencies = [
"itoa 1.0.5",
"itoa",
"ryu",
"serde",
]
@ -2595,7 +2576,7 @@ dependencies = [
"serde",
"serde_json",
"serde_with_macros",
"time 0.3.17",
"time 0.3.19",
]
[[package]]
@ -2781,10 +2762,11 @@ dependencies = [
[[package]]
name = "thread_local"
version = "1.1.4"
version = "1.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180"
checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152"
dependencies = [
"cfg-if 1.0.0",
"once_cell",
]
@ -2801,11 +2783,11 @@ dependencies = [
[[package]]
name = "time"
version = "0.3.17"
version = "0.3.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376"
checksum = "53250a3b3fed8ff8fd988587d8925d26a83ac3845d9e03b220b37f34c2b8d6c2"
dependencies = [
"itoa 1.0.5",
"itoa",
"serde",
"time-core",
"time-macros",
@ -2819,9 +2801,9 @@ checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd"
[[package]]
name = "time-macros"
version = "0.2.6"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2"
checksum = "a460aeb8de6dcb0f381e1ee05f1cd56fcf5a5f6eb8187ff3d8f0b11078d38b7c"
dependencies = [
"time-core",
]
@ -2911,7 +2893,7 @@ dependencies = [
"sharded-slab",
"smallvec",
"thread_local",
"time 0.3.17",
"time 0.3.19",
"tracing",
"tracing-core",
"tracing-serde",

1
avm/interface/Cargo.toml
View File

@ -17,7 +17,6 @@ path = "src/lib.rs"
[dependencies]
air-interpreter-interface = { version = "0.12.1", path = "../../crates/air-lib/interpreter-interface", default-features = false }
air-utils = { version = "0.1.0", path = "../../crates/air-lib/utils" }
avm-data-store = { version = "0.4.1", path = "../../crates/data-store" }
polyplets = { version = "0.3.2", path = "../../crates/air-lib/polyplets" }
thiserror = "1.0.38"

13
avm/server/src/avm.rs
View File

@ -102,16 +102,20 @@ impl<E> AVM<E> {
particle_parameters.timestamp,
particle_parameters.ttl,
particle_parameters.current_peer_id.clone(),
call_results,
call_results.clone(),
)
.map_err(AVMError::RunnerError)?;
let execution_time = execution_start_time.elapsed();
let memory_delta = self.memory_stats().memory_size - memory_size_before;
if self.data_store.detect_anomaly(execution_time, memory_delta) {
if self
.data_store
.detect_anomaly(execution_time, memory_delta, &outcome)
{
self.save_anomaly_data(
&air,
&current_data,
&call_results,
&particle_parameters,
&outcome,
execution_time,
@ -139,11 +143,12 @@ impl<E> AVM<E> {
self.runner.memory_stats()
}
#[allow(clippy::result_large_err)]
#[allow(clippy::result_large_err, clippy::too_many_arguments)]
fn save_anomaly_data(
&mut self,
air_script: &str,
current_data: &[u8],
call_result: &CallResults,
particle_parameters: &ParticleParameters<'_>,
avm_outcome: &RawAVMOutcome,
execution_time: Duration,
@ -152,6 +157,7 @@ impl<E> AVM<E> {
let prev_data = self
.data_store
.read_data(&particle_parameters.particle_id)?;
let call_results = serde_json::to_vec(call_result).map_err(AVMError::AnomalyDataSeError)?;
let ser_particle =
serde_json::to_vec(particle_parameters).map_err(AVMError::AnomalyDataSeError)?;
let ser_avm_outcome =
@ -162,6 +168,7 @@ impl<E> AVM<E> {
&ser_particle,
&prev_data,
current_data,
&call_results,
&ser_avm_outcome,
execution_time,
memory_delta,

2
crates/air-lib/air-parser/Cargo.toml
View File

@ -14,7 +14,7 @@ lalrpop = "0.19.8"
[dependencies]
air-lambda-ast = { version = "0.1.0", path = "../lambda/ast" }
air-lambda-parser = { vesrion = "0.1.0", path = "../lambda/parser" }
air-lambda-parser = { version = "0.1.0", path = "../lambda/parser" }
lalrpop-util = "0.19.8"
regex = "1.7.1"

1
crates/data-store/Cargo.toml
View File

@ -15,6 +15,7 @@ name = "avm_data_store"
path = "src/lib.rs"
[dependencies]
avm-interface = { version = "0.28.1", path = "../../avm/interface"}
serde = { version = "1.0.152", features = ["derive"] }
serde_bytes = "0.11.9"

29
crates/data-store/src/lib.rs
View File

@ -26,6 +26,8 @@
unreachable_patterns
)]
use avm_interface::raw_outcome::RawAVMOutcome;
use serde::Deserialize;
use serde::Serialize;
use std::borrow::Cow;
@ -48,10 +50,16 @@ pub trait DataStore {
/// Returns true if an anomaly happened and it's necessary to save execution data
/// for debugging purposes.
/// execution_time - is time taken by the interpreter to execute provided script
/// memory_delta - is a count of bytes on which an interpreter heap has been extended
/// execution_time - time taken by the interpreter to execute provided script
/// memory_delta - count of bytes on which an interpreter heap has been extended
/// during execution of a particle
fn detect_anomaly(&self, execution_time: Duration, memory_delta: usize) -> bool;
/// outcome - a result of AquaVM invocation
fn detect_anomaly(
&self,
execution_time: Duration,
memory_delta: usize,
outcome: &RawAVMOutcome,
) -> bool;
fn collect_anomaly_data(
&mut self,
@ -71,17 +79,21 @@ pub struct AnomalyData<'data> {
#[serde(borrow, with = "serde_bytes")]
pub current_data: Cow<'data, [u8]>,
#[serde(borrow, with = "serde_bytes")]
pub avm_outcome: Cow<'data, [u8]>, // it's byte because of the restriction on trait objects methods
pub call_results: Cow<'data, [u8]>,
#[serde(borrow, with = "serde_bytes")]
pub avm_outcome: Cow<'data, [u8]>,
pub execution_time: Duration,
pub memory_delta: usize,
}
impl<'data> AnomalyData<'data> {
#[allow(clippy::too_many_arguments)]
pub fn new(
air_script: &'data str,
particle: &'data [u8],
prev_data: &'data [u8],
current_data: &'data [u8],
call_results: &'data [u8],
avm_outcome: &'data [u8],
execution_time: Duration,
memory_delta: usize,
@ -91,6 +103,7 @@ impl<'data> AnomalyData<'data> {
particle: particle.into(),
prev_data: prev_data.into(),
current_data: current_data.into(),
call_results: call_results.into(),
avm_outcome: avm_outcome.into(),
execution_time,
memory_delta,
@ -107,6 +120,7 @@ mod tests {
particle: &[u8],
prev_data: &[u8],
current_data: &[u8],
call_results: &[u8],
avm_outcome: &[u8],
) -> String {
format!(
@ -115,6 +129,7 @@ mod tests {
r#""particle":{particle},"#,
r#""prev_data":{prev_data},"#,
r#""current_data":{current_data},"#,
r#""call_results":{call_results},"#,
r#""avm_outcome":{avm_outcome},"#,
r#""execution_time":{{"secs":42,"nanos":0}},"#,
r#""memory_delta":123"#,
@ -124,6 +139,7 @@ mod tests {
particle = serde_json::to_string(particle).unwrap(),
prev_data = serde_json::to_string(prev_data).unwrap(),
current_data = serde_json::to_string(current_data).unwrap(),
call_results = serde_json::to_string(call_results).unwrap(),
avm_outcome = serde_json::to_string(avm_outcome).unwrap(),
)
}
@ -135,6 +151,7 @@ mod tests {
br#"{"trace":[]}"#, // not real data
br#"{"trace":[1,2,3]}"#, // not real data
b"{}", // not real data
b"{}",
Duration::from_secs(42),
123,
);
@ -145,6 +162,7 @@ mod tests {
&anomaly.particle,
&anomaly.prev_data,
&anomaly.current_data,
&anomaly.call_results,
&anomaly.avm_outcome,
);
assert_eq!(json_data, expected);
@ -156,11 +174,13 @@ mod tests {
let current_data = br#"{"data":"current"}"#;
let prev_data = br#"{"data":"prev"}"#;
let avm_outcome = br#"{"avm":[1,2,3]}"#;
let call_results = br#"{"call_results": "excellent result"}"#;
let json_data = anomaly_json(
"(null)",
&particle[..],
&prev_data[..],
&current_data[..],
&call_results[..],
&avm_outcome[..],
);
@ -174,6 +194,7 @@ mod tests {
&particle[..],
&prev_data[..],
&current_data[..],
&call_results[..],
&avm_outcome[..],
Duration::from_secs(42),
123,