Improve flattening (#69)

Cargo.lock

@@ -11,7 +11,7 @@ dependencies = [
 
 [[package]]
 name = "air-parser"
-version = "0.4.0"
+version = "0.5.0"
 dependencies = [
  "codespan",
  "codespan-reporting",
@@ -79,7 +79,7 @@ dependencies = [
 
 [[package]]
 name = "aquamarine"
-version = "0.6.0"
+version = "0.7.0"
 dependencies = [
  "fluence",
  "interpreter-lib",
@@ -240,9 +240,9 @@ checksum = "cfa8873f51c92e232f9bac4065cddef41b714152812bfc5f7672ba16d6ef8cd9"
 
 [[package]]
 name = "bstr"
-version = "0.2.14"
+version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "473fc6b38233f9af7baa94fb5852dca389e3d95b8e21c8e3719301462c5d9faf"
+checksum = "a40b47ad93e1a5404e6c18dec46b628214fee441c70f4ab5d6942142cc268a3d"
 dependencies = [
  "lazy_static",
  "memchr",
@@ -252,9 +252,9 @@ dependencies = [
 
 [[package]]
 name = "bumpalo"
-version = "3.5.0"
+version = "3.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f07aa6688c702439a1be0307b6a94dffe1168569e45b9500c1372bc580740d59"
+checksum = "099e596ef14349721d9016f6b80dd3419ea1bf289ab9b44df8e4dfd3a005d5d9"
 
 [[package]]
 name = "byte-tools"
@@ -528,9 +528,9 @@ dependencies = [
 
 [[package]]
 name = "ctor"
-version = "0.1.18"
+version = "0.1.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10bcb9d7dcbf7002aaffbb53eac22906b64cdcc127971dcc387d8eb7c95d5560"
+checksum = "e8f45d9ad417bcef4817d614a501ab55cdd96a6fdb24f49aab89a54acfd66b19"
 dependencies = [
  "quote",
  "syn",
@@ -976,7 +976,7 @@ checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8"
 dependencies = [
  "cfg-if 1.0.0",
  "libc",
- "wasi 0.10.1+wasi-snapshot-preview1",
+ "wasi 0.10.2+wasi-snapshot-preview1",
 ]
 
 [[package]]
@@ -1073,7 +1073,7 @@ dependencies = [
 
 [[package]]
 name = "interpreter-lib"
-version = "0.6.0"
+version = "0.7.0"
 dependencies = [
  "air-parser",
  "aqua-interpreter-interface 0.3.1",
@@ -1159,9 +1159,9 @@ dependencies = [
 
 [[package]]
 name = "jsonpath_lib-fl"
-version = "0.2.6"
+version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "243653439f0992adf0bbf6ed5b798966fdbacd417b9dcb025b50200ec20c17ff"
+checksum = "e81233a3c2e1f4579f1fdb856eeec115dcb23817374268212ebad691bd53e664"
 dependencies = [
  "array_tool",
  "env_logger",
@@ -1216,22 +1216,22 @@ checksum = "3576a87f2ba00f6f106fdfcd16db1d698d648a26ad8e0573cad8537c3c362d2a"
 
 [[package]]
 name = "lexical-core"
-version = "0.7.4"
+version = "0.7.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db65c6da02e61f55dae90a0ae427b2a5f6b3e8db09f58d10efab23af92592616"
+checksum = "21f866863575d0e1d654fbeeabdc927292fdf862873dc3c96c6f753357e13374"
 dependencies = [
  "arrayvec",
  "bitflags",
- "cfg-if 0.1.10",
+ "cfg-if 1.0.0",
  "ryu",
  "static_assertions",
 ]
 
 [[package]]
 name = "libc"
-version = "0.2.83"
+version = "0.2.86"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7eb0c4e9c72ee9d69b767adebc5f4788462a3b45624acd919475c92597bcaf4f"
+checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c"
 
 [[package]]
 name = "lock_api"
@@ -1416,7 +1416,7 @@ checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb"
 dependencies = [
  "instant",
  "lock_api 0.4.2",
- "parking_lot_core 0.8.2",
+ "parking_lot_core 0.8.3",
 ]
 
 [[package]]
@@ -1428,21 +1428,21 @@ dependencies = [
  "cfg-if 0.1.10",
  "cloudabi",
  "libc",
- "redox_syscall",
+ "redox_syscall 0.1.57",
  "smallvec",
  "winapi",
 ]
 
 [[package]]
 name = "parking_lot_core"
-version = "0.8.2"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272"
+checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018"
 dependencies = [
  "cfg-if 1.0.0",
  "instant",
  "libc",
- "redox_syscall",
+ "redox_syscall 0.2.5",
  "smallvec",
  "winapi",
 ]
@@ -1554,9 +1554,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
 
 [[package]]
 name = "quote"
-version = "1.0.8"
+version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df"
+checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
 dependencies = [
  "proc-macro2",
 ]
@@ -1603,6 +1603,15 @@ version = "0.1.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"
 
+[[package]]
+name = "redox_syscall"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9"
+dependencies = [
+ "bitflags",
+]
+
 [[package]]
 name = "redox_users"
 version = "0.3.5"
@@ -1610,7 +1619,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d"
 dependencies = [
  "getrandom 0.1.16",
- "redox_syscall",
+ "redox_syscall 0.1.57",
  "rust-argon2",
 ]
 
@@ -1670,9 +1679,9 @@ checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
 
 [[package]]
 name = "safe-transmute"
-version = "0.11.0"
+version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50b8b2cd387f744f69469aaed197954ba4c0ecdb31e02edf99b023e0df11178a"
+checksum = "1d95e7284b4bd97e24af76023904cd0157c9cc9da0310beb4139a1e88a748d47"
 
 [[package]]
 name = "same-file"
@@ -1888,9 +1897,9 @@ dependencies = [
 
 [[package]]
 name = "thread_local"
-version = "1.1.2"
+version = "1.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8208a331e1cb318dd5bd76951d2b8fc48ca38a69f5f4e4af1b6a9f8c6236915"
+checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd"
 dependencies = [
  "once_cell",
 ]
@@ -2038,9 +2047,9 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
 
 [[package]]
 name = "wasi"
-version = "0.10.1+wasi-snapshot-preview1"
+version = "0.10.2+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93c6c3420963c5c64bca373b25e77acb562081b9bb4dd5bb864187742186cea9"
+checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
 
 [[package]]
 name = "wasm-bindgen"

crates/air-parser/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "air-parser"
-version = "0.4.0"
+version = "0.5.0"
 authors = ["Fluence Labs"]
 edition = "2018"
 license = "Apache-2.0"

crates/air-parser/src/parser/air.lalrpop

@@ -1,5 +1,6 @@
 use crate::parser::ast::*;
-use crate::parser::into_variable_and_path;
+use crate::parser::air_parser::into_variable_and_path;
+use crate::parser::air_parser::make_flattened_error;
 use crate::parser::lexer::LexerError;
 use crate::parser::lexer::Token;
 use crate::parser::lexer::Number;
@@ -71,9 +72,14 @@ ServiceId = CallInstrValue;
 CallInstrValue: CallInstrValue<'input> = {
     <s:Literal> => CallInstrValue::Literal(s),
     <s:Alphanumeric> => CallInstrValue::Variable(s),
-    <v:JsonPath> => {
+    <l: @L> <v:JsonPath> <r: @R> => {
-        let (variable, path) = into_variable_and_path(v.0, v.1);
+        let (variable, path) = into_variable_and_path(v.0, v.1, v.2);
-        CallInstrValue::JsonPath { variable, path }
+        let should_flatten = v.2;
+        if !should_flatten {
+            let token = Token::JsonPath(v.0, v.1, v.2);
+            errors.push(make_flattened_error(l, token, r));
+        }
+        CallInstrValue::JsonPath { variable, path, should_flatten }
     },
     InitPeerId => CallInstrValue::InitPeerId,
 }
@@ -84,8 +90,9 @@ CallInstrArgValue: CallInstrArgValue<'input> = {
     <s:Literal> => CallInstrArgValue::Literal(s),
     <s:Alphanumeric> => CallInstrArgValue::Variable(s),
     <v:JsonPath> => {
-        let (variable, path) = into_variable_and_path(v.0, v.1);
+        let (variable, path) = into_variable_and_path(v.0, v.1, v.2);
-        CallInstrArgValue::JsonPath { variable, path }
+        let should_flatten = v.2;
+        CallInstrArgValue::JsonPath { variable, path, should_flatten }
     },
     <n:Number> => CallInstrArgValue::Number(n),
     <b:Boolean> => CallInstrArgValue::Boolean(b),
@@ -96,8 +103,9 @@ CallInstrArgValue: CallInstrArgValue<'input> = {
 Iterable: IterableValue<'input> = {
     <s:Alphanumeric> => IterableValue::Variable(s),
     <v:JsonPath> => {
-        let (variable, path) = into_variable_and_path(v.0, v.1);
+        let (variable, path) = into_variable_and_path(v.0, v.1, v.2);
-        IterableValue::JsonPath { variable, path }
+        let should_flatten = v.2;
+        IterableValue::JsonPath { variable, path, should_flatten }
     },
 }
 
@@ -105,8 +113,9 @@ Matchable: MatchableValue<'input> = {
     <s:Alphanumeric> => MatchableValue::Variable(s),
     <s:Literal> => MatchableValue::Literal(s),
     <v:JsonPath> => {
-        let (variable, path) = into_variable_and_path(v.0, v.1);
+        let (variable, path) = into_variable_and_path(v.0, v.1, v.2);
-        MatchableValue::JsonPath { variable, path }
+        let should_flatten = v.2;
+        MatchableValue::JsonPath { variable, path, should_flatten }
     },
 }
 
@@ -122,7 +131,7 @@ extern {
 
         Alphanumeric => Token::Alphanumeric(<&'input str>),
         Literal => Token::StringLiteral(<&'input str>),
-        JsonPath => Token::JsonPath(<&'input str>, <usize>),
+        JsonPath => Token::JsonPath(<&'input str>, <usize>, <bool>),
         Accumulator => Token::Accumulator(<&'input str>),
         Number => Token::Number(<Number>),
         Boolean => Token::Boolean(<bool>),

crates/air-parser/src/parser/air_parser.rs

@@ -147,5 +147,34 @@ fn lexical_error_to_label(file_id: usize, error: LexerError) -> Label<usize> {
         LeadingDot(start, end) => {
             Label::primary(file_id, start..end).with_message(error.to_string())
         }
+        CallArgsNotFlattened(start, end) => {
+            Label::primary(file_id, start..end).with_message(error.to_string())
+        }
+    }
+}
+
+pub(super) fn into_variable_and_path(str: &str, pos: usize, should_flatten: bool) -> (&str, &str) {
+    let json_path = if should_flatten {
+        &str[pos + 1..str.len() - 1]
+    } else {
+        &str[pos + 1..]
+    };
+
+    (&str[0..pos], json_path)
+}
+
+pub(super) fn make_flattened_error(
+    start_pos: usize,
+    token: Token<'_>,
+    end_pos: usize,
+) -> ErrorRecovery<usize, Token<'_>, LexerError> {
+    let error = LexerError::CallArgsNotFlattened(start_pos, end_pos);
+    let error = ParseError::User { error };
+
+    let dropped_tokens = vec![(start_pos, token, end_pos)];
+
+    ErrorRecovery {
+        error,
+        dropped_tokens,
     }
 }

crates/air-parser/src/parser/ast.rs

@@ -63,7 +63,11 @@ pub enum CallInstrValue<'i> {
     InitPeerId,
     Literal(&'i str),
     Variable(&'i str),
-    JsonPath { variable: &'i str, path: &'i str },
+    JsonPath {
+        variable: &'i str,
+        path: &'i str,
+        should_flatten: bool,
+    },
 }
 
 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
@@ -74,13 +78,21 @@ pub enum CallInstrArgValue<'i> {
     Number(Number),
     Boolean(bool),
     Variable(&'i str),
-    JsonPath { variable: &'i str, path: &'i str },
+    JsonPath {
+        variable: &'i str,
+        path: &'i str,
+        should_flatten: bool,
+    },
 }
 
 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
 pub enum IterableValue<'i> {
     Variable(&'i str),
-    JsonPath { variable: &'i str, path: &'i str },
+    JsonPath {
+        variable: &'i str,
+        path: &'i str,
+        should_flatten: bool,
+    },
 }
 
 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
@@ -89,7 +101,11 @@ pub enum MatchableValue<'i> {
     Number(Number),
     Boolean(bool),
     Variable(&'i str),
-    JsonPath { variable: &'i str, path: &'i str },
+    JsonPath {
+        variable: &'i str,
+        path: &'i str,
+        should_flatten: bool,
+    },
 }
 
 #[derive(Serialize, Debug, PartialEq, Clone)]

crates/air-parser/src/parser/ast/traits.rs

@@ -29,11 +29,26 @@ impl fmt::Display for CallInstrArgValue<'_> {
             Number(number) => write!(f, "{}", number),
             Boolean(bool) => write!(f, "{}", bool),
             Variable(str) => write!(f, "{}", str),
-            JsonPath { variable, path } => write!(f, "{}.{}", variable, path),
+            JsonPath {
+                variable,
+                path,
+                should_flatten,
+            } => print_json_path(variable, path, should_flatten, f),
         }
     }
 }
 
+fn print_json_path(
+    variable: &str,
+    path: &str,
+    should_flatten: &bool,
+    f: &mut fmt::Formatter,
+) -> fmt::Result {
+    let maybe_flatten_char = if *should_flatten { "!" } else { "" };
+
+    write!(f, "{}.{}{}", variable, path, maybe_flatten_char)
+}
+
 impl fmt::Display for CallInstrValue<'_> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         use CallInstrValue::*;
@@ -42,7 +57,11 @@ impl fmt::Display for CallInstrValue<'_> {
             InitPeerId => write!(f, "%init_peer_id%"),
             Literal(str) => write!(f, r#""{}""#, str),
             Variable(str) => write!(f, "{}", str),
-            JsonPath { variable, path } => write!(f, "{}.{}", variable, path),
+            JsonPath {
+                variable,
+                path,
+                should_flatten,
+            } => print_json_path(variable, path, should_flatten, f),
         }
     }
 }
@@ -53,7 +72,11 @@ impl fmt::Display for IterableValue<'_> {
 
         match self {
             Variable(str) => write!(f, "{}", str),
-            JsonPath { variable, path } => write!(f, "{}.{}", variable, path),
+            JsonPath {
+                variable,
+                path,
+                should_flatten,
+            } => print_json_path(variable, path, should_flatten, f),
         }
     }
 }
@@ -67,7 +90,11 @@ impl fmt::Display for MatchableValue<'_> {
             Number(number) => write!(f, "{}", number),
             Boolean(bool) => write!(f, "{}", bool),
             Variable(str) => write!(f, "{}", str),
-            JsonPath { variable, path } => write!(f, "{}.{}", variable, path),
+            JsonPath {
+                variable,
+                path,
+                should_flatten,
+            } => print_json_path(variable, path, should_flatten, f),
         }
     }
 }

crates/air-parser/src/parser/lexer/call_variable_parser.rs

@@ -34,6 +34,7 @@ struct ParserState {
     pub(self) first_dot_met_pos: Option<usize>,
     pub(self) non_numeric_met: bool,
     pub(self) digit_met: bool,
+    pub(self) flattening_met: bool,
     pub(self) is_first_char: bool,
     pub(self) current_char: char,
     pub(self) current_pos: usize,
@@ -58,6 +59,7 @@ impl<'input> CallVariableParser<'input> {
             first_dot_met_pos: None,
             non_numeric_met: false,
             digit_met: false,
+            flattening_met: false,
             is_first_char: true,
             current_char,
             current_pos,
@@ -186,8 +188,8 @@ impl<'input> CallVariableParser<'input> {
         Ok(())
     }
 
-    fn try_parse_as_json_path(&self) -> LexerResult<()> {
+    fn try_parse_as_json_path(&mut self) -> LexerResult<()> {
-        if !self.json_path_allowed_char() {
+        if !self.json_path_allowed_char() && !self.try_parse_as_flattening() {
             let error_pos = self.pos_in_string_to_parse();
             return Err(LexerError::InvalidJsonPath(error_pos, error_pos));
         }
@@ -195,6 +197,15 @@ impl<'input> CallVariableParser<'input> {
         Ok(())
     }
 
+    fn try_parse_as_flattening(&mut self) -> bool {
+        if self.is_last_char() && self.current_char() == '!' {
+            self.state.flattening_met = true;
+            return true;
+        }
+
+        false
+    }
+
     fn try_parse_first_met_dot(&mut self) -> LexerResult<bool> {
         if !self.dot_met() && self.current_char() == '.' {
             if self.current_pos() == 0 {
@@ -238,6 +249,10 @@ impl<'input> CallVariableParser<'input> {
         self.state.current_char
     }
 
+    fn is_last_char(&self) -> bool {
+        self.current_pos() == self.string_to_parse.len() - 1
+    }
+
     fn to_token(&self) -> LexerResult<Token<'input>> {
         use super::token::UnparsedNumber;
 
@@ -256,6 +271,7 @@ impl<'input> CallVariableParser<'input> {
             (false, true) => Ok(Token::JsonPath(
                 self.string_to_parse,
                 self.state.first_dot_met_pos.unwrap(),
+                self.state.flattening_met,
             )),
         }
     }

crates/air-parser/src/parser/lexer/errors.rs

@@ -53,6 +53,9 @@ pub enum LexerError {
 
     #[error("leading dot without any symbols before - please write 0 if it's float or variable name if it's json path")]
     LeadingDot(usize, usize),
+
+    #[error("while using json path in call triplet, result should be flattened, add ! at the end")]
+    CallArgsNotFlattened(usize, usize),
 }
 
 impl From<std::convert::Infallible> for LexerError {

crates/air-parser/src/parser/lexer/tests.rs

@@ -269,24 +269,28 @@ fn too_big_float_number() {
 #[test]
 fn json_path() {
     // this json path contains all allowed in json path charactes
-    const JSON_PATH: &str = r#"value.$[$@[]():?.*,"!]"#;
+    const JSON_PATH: &str = r#"value.$[$@[]():?.*,"]"#;
 
     lexer_test(
         JSON_PATH,
-        Single(Ok((0, Token::JsonPath(JSON_PATH, 5), JSON_PATH.len()))),
+        Single(Ok((
+            0,
+            Token::JsonPath(JSON_PATH, 5, false),
+            JSON_PATH.len(),
+        ))),
     );
 }
 
 #[test]
 fn json_path_numbers() {
-    const JSON_PATH: &str = r#"12345.$[$@[]():?.*,"!]"#;
+    const JSON_PATH: &str = r#"12345.$[$@[]():?.*,"]"#;
 
     lexer_test(
         JSON_PATH,
         Single(Err(LexerError::UnallowedCharInNumber(6, 6))),
     );
 
-    const JSON_PATH1: &str = r#"+12345.$[$@[]():?.*,"!]"#;
+    const JSON_PATH1: &str = r#"+12345.$[$@[]():?.*,"]"#;
 
     lexer_test(
         JSON_PATH1,
@@ -320,13 +324,21 @@ fn unclosed_quote() {
 
 #[test]
 fn bad_value() {
-    // value contains ! that only allowed in json path
+    // value contains ! that only allowed at the end of a json path
-    const INVALID_VALUE: &str = r#"val!ue.$[$@[]():?.*,"\!]"#;
+    const INVALID_VALUE: &str = r#"val!ue.$[$@[]():?.*,"\]"#;
 
     lexer_test(
         INVALID_VALUE,
         Single(Err(LexerError::IsNotAlphanumeric(3, 3))),
     );
+
+    // value contains ! that only allowed at the end of a json path
+    const INVALID_VALUE2: &str = r#"value.$![$@[]():?.*,"\]"#;
+
+    lexer_test(
+        INVALID_VALUE2,
+        Single(Err(LexerError::InvalidJsonPath(7, 7))),
+    );
 }
 
 #[test]

crates/air-parser/src/parser/lexer/token.rs

@@ -26,7 +26,7 @@ pub enum Token<'input> {
 
     StringLiteral(&'input str),
     Alphanumeric(&'input str),
-    JsonPath(&'input str, usize),
+    JsonPath(&'input str, usize, bool),
     Accumulator(&'input str),
     Number(Number),
     Boolean(bool),

crates/air-parser/src/parser/lexer/utils.rs

@@ -37,7 +37,6 @@ pub(super) fn is_json_path_allowed_char(ch: char) -> bool {
         ',' => true,
         '"' => true,
         '\'' => true,
-        '!' => true,
         ch => is_aqua_alphanumeric(ch),
     }
 }

crates/air-parser/src/parser/mod.rs

@@ -30,7 +30,3 @@ pub mod tests;
 pub use self::air_parser::parse;
 pub use air::AIRParser;
 pub use lexer::AIRLexer;
-
-fn into_variable_and_path(str: &str, pos: usize) -> (&str, &str) {
-    (&str[0..pos], &str[pos + 1..])
-}

crates/air-parser/src/parser/tests.rs

@@ -123,13 +123,14 @@ fn parse_json_path() {
     use ast::PeerPart::*;
 
     let source_code = r#"
-        (call id.$.a "f" ["hello" name] void[])
+        (call id.$.a! "f" ["hello" name] void[])
         "#;
     let instruction = parse(source_code);
     let expected = Instruction::Call(Call {
         peer_part: PeerPk(CallInstrValue::JsonPath {
             variable: "id",
             path: "$.a",
+            should_flatten: true,
         }),
         function_part: FuncName(CallInstrValue::Literal("f")),
         args: Rc::new(vec![
@@ -141,6 +142,24 @@ fn parse_json_path() {
     assert_eq!(instruction, expected);
 }
 
+#[test]
+fn parse_json_path_without_flattening() {
+    let source_code = r#"
+        (call id.$.a "f" ["hello" name] void[])
+        "#;
+
+    let lexer = crate::AIRLexer::new(source_code);
+
+    let parser = crate::AIRParser::new();
+    let mut errors = Vec::new();
+    parser
+        .parse(source_code, &mut errors, lexer)
+        .expect("parser shoudn't fail");
+
+    assert_eq!(errors.len(), 1);
+    assert!(matches!(errors[0], lalrpop_util::ErrorRecovery { .. }));
+}
+
 #[test]
 fn parse_json_path_complex() {
     use ast::Call;
@@ -151,8 +170,8 @@ fn parse_json_path_complex() {
 
     let source_code = r#"
         (seq
-            (call m.$.[1] "f" [] void)
+            (call m.$.[1]! "f" [] void)
-            (call m.$.abc["c"].cde[a][0].cde["bcd"] "f" [] void)
+            (call m.$.abc["c"].cde[a][0].cde["bcd"]! "f" [] void)
         )
         "#;
     let instruction = parse(source_code);
@@ -161,6 +180,7 @@ fn parse_json_path_complex() {
             peer_part: PeerPk(CallInstrValue::JsonPath {
                 variable: "m",
                 path: "$.[1]",
+                should_flatten: true,
             }),
             function_part: FuncName(CallInstrValue::Literal("f")),
             args: Rc::new(vec![]),
@@ -170,6 +190,7 @@ fn parse_json_path_complex() {
             peer_part: PeerPk(CallInstrValue::JsonPath {
                 variable: "m",
                 path: r#"$.abc["c"].cde[a][0].cde["bcd"]"#,
+                should_flatten: true,
             }),
             function_part: FuncName(CallInstrValue::Literal("f")),
             args: Rc::new(vec![]),
@@ -189,13 +210,14 @@ fn json_path_square_braces() {
     use ast::PeerPart::*;
 
     let source_code = r#"
-        (call u.$["peer_id"] ("return" "") [u.$["peer_id"].cde[0]["abc"].abc u.$["name"]] void[])
+        (call u.$["peer_id"]! ("return" "") [u.$["peer_id"].cde[0]["abc"].abc u.$["name"]] void[])
         "#;
     let instruction = parse(source_code);
     let expected = Instruction::Call(Call {
         peer_part: PeerPk(CallInstrValue::JsonPath {
             variable: "u",
             path: r#"$["peer_id"]"#,
+            should_flatten: true,
         }),
         function_part: ServiceIdWithFuncName(
             CallInstrValue::Literal("return"),
@@ -205,10 +227,12 @@ fn json_path_square_braces() {
             CallInstrArgValue::JsonPath {
                 variable: "u",
                 path: r#"$["peer_id"].cde[0]["abc"].abc"#,
+                should_flatten: false,
             },
             CallInstrArgValue::JsonPath {
                 variable: "u",
                 path: r#"$["name"]"#,
+                should_flatten: false,
             },
         ]),
         output: Accumulator("void"),
@@ -600,6 +624,7 @@ fn fold_json_path() {
         iterable: JsonPath {
             variable: "members",
             path: "$.[\"users\"]",
+            should_flatten: false,
         },
         iterator: "m",
         instruction: Rc::new(null()),
@@ -622,6 +647,7 @@ fn comments() {
         iterable: JsonPath {
             variable: "members",
             path: "$.[\"users\"]",
+            should_flatten: false,
         },
         iterator: "m",
         instruction: Rc::new(null()),

interpreter-lib/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "interpreter-lib"
-version = "0.6.0"
+version = "0.7.0"
 authors = ["Fluence Labs"]
 edition = "2018"
 
@@ -17,7 +17,7 @@ aqua-interpreter-interface = { path = "../crates/interpreter-interface" }
 serde = { version = "=1.0.118", features = [ "derive", "rc" ] }
 serde_json = "=1.0.61"
 
-jsonpath_lib-fl = "=0.2.6"
+jsonpath_lib-fl = "=0.2.5"
 
 boolinator = "2.4.0"
 log = "0.4.11"

interpreter-lib/src/execution/air/call/triplet.rs

@@ -89,7 +89,14 @@ fn resolve_to_string<'i>(value: &CallInstrValue<'i>, ctx: &ExecutionCtx<'i>) ->
             let jvalue = resolved.into_jvalue();
             jvalue_to_string(jvalue)?
         }
-        CallInstrValue::JsonPath { variable, path } => {
+        CallInstrValue::JsonPath {
+            variable,
+            path,
+            should_flatten,
+        } => {
+            // this is checked on the parsing stage
+            debug_assert!(*should_flatten);
+
             let resolved = resolve_to_jvaluable(variable, ctx)?;
             let resolved = resolved.apply_json_path(path)?;
             vec_to_string(resolved, path)?

interpreter-lib/src/execution/air/compare_matchable/compare_matchable.rs

@@ -50,7 +50,23 @@ pub(crate) fn are_matchable_eq<'ctx>(
 
             Ok(left_value == right_value)
         }
-        (JsonPath { variable: lv, path: lp }, JsonPath { variable: rv, path: rp }) => {
+        (
+            JsonPath {
+                variable: lv,
+                path: lp,
+                should_flatten: lsf,
+            },
+            JsonPath {
+                variable: rv,
+                path: rp,
+                should_flatten: rsf,
+            },
+        ) => {
+            // TODO: improve comparison
+            if lsf != rsf {
+                return Ok(false);
+            }
+
             let left_jvaluable = resolve_to_jvaluable(lv, exec_ctx)?;
             let left_value = left_jvaluable.apply_json_path(lp)?;
 
@@ -91,14 +107,27 @@ fn compare_matchable<'ctx>(
             let jvalue = jvaluable.as_jvalue();
             Ok(comparator(jvalue))
         }
-        JsonPath { variable, path } => {
+        JsonPath {
+            variable,
+            path,
+            should_flatten,
+        } => {
             let jvaluable = resolve_to_jvaluable(variable, exec_ctx)?;
             let jvalues = jvaluable.apply_json_path(path)?;
+
+            let jvalue = if *should_flatten {
                 if jvalues.len() != 1 {
                     return Ok(false);
                 }
+                Cow::Borrowed(jvalues[0])
+            } else {
+                let jvalue = jvalues.into_iter().cloned().collect::<Vec<_>>();
+                let jvalue = JValue::Array(jvalue);
 
-            Ok(comparator(Cow::Borrowed(jvalues[0])))
+                Cow::Owned(jvalue)
+            };
+
+            Ok(comparator(jvalue))
         }
     }
 }

interpreter-lib/src/execution/air/fold/utils.rs

@@ -37,7 +37,11 @@ pub(super) fn construct_iterable_value<'ctx>(
 ) -> ExecutionResult<Option<IterableValue>> {
     match ast_iterable {
         ast::IterableValue::Variable(name) => handle_instruction_variable(exec_ctx, name),
-        ast::IterableValue::JsonPath { variable, path } => handle_instruction_json_path(exec_ctx, variable, path),
+        ast::IterableValue::JsonPath {
+            variable,
+            path,
+            should_flatten,
+        } => handle_instruction_json_path(exec_ctx, variable, path, *should_flatten),
     }
 }
 
@@ -97,13 +101,14 @@ fn handle_instruction_json_path<'ctx>(
     exec_ctx: &ExecutionCtx<'ctx>,
     variable_name: &str,
     json_path: &str,
+    should_flatten: bool,
 ) -> ExecutionResult<Option<IterableValue>> {
     use ExecutionError::JValueAccJsonPathError;
 
-    let iterable: Option<IterableValue> = match exec_ctx.data_cache.get(variable_name) {
+    match exec_ctx.data_cache.get(variable_name) {
         Some(AValue::JValueRef(variable)) => {
             let jvalues = apply_json_path(&variable.result, json_path)?;
-            from_jvalues(jvalues, variable.triplet.clone(), json_path)
+            from_jvalues(jvalues, variable.triplet.clone(), json_path, should_flatten)
         }
         Some(AValue::JValueAccumulatorRef(acc)) => {
             let acc = acc.borrow();
@@ -115,7 +120,7 @@ fn handle_instruction_json_path<'ctx>(
             let (jvalues, tetraplet_indices) = select_with_iter(acc_iter, &json_path)
                 .map_err(|e| JValueAccJsonPathError(acc.clone(), json_path.to_string(), e))?;
 
-            let jvalues = jvalues.into_iter().cloned().collect();
+            let jvalues = construct_iterable_jvalues(jvalues, should_flatten)?;
             let tetraplets = tetraplet_indices
                 .into_iter()
                 .map(|id| SecurityTetraplet {
@@ -125,19 +130,17 @@ fn handle_instruction_json_path<'ctx>(
                 .collect::<Vec<_>>();
 
             let foldable = IterableVecJsonPathResult::init(jvalues, tetraplets);
-            Some(Box::new(foldable))
+            Ok(Some(Box::new(foldable)))
         }
         Some(AValue::JValueFoldCursor(fold_state)) => {
             let iterable_value = fold_state.iterable.peek().unwrap();
             let jvalues = iterable_value.apply_json_path(json_path)?;
             let triplet = as_triplet(&iterable_value);
 
-            from_jvalues(jvalues, triplet, json_path)
+            from_jvalues(jvalues, triplet, json_path, should_flatten)
         }
         _ => return exec_err!(ExecutionError::VariableNotFound(variable_name.to_string())),
-    };
+    }
-
-    Ok(iterable)
 }
 
 fn apply_json_path<'jvalue, 'str>(
@@ -150,12 +153,17 @@ fn apply_json_path<'jvalue, 'str>(
 }
 
 /// Applies json_path to provided jvalues and construct IterableValue from the result and given triplet.
-fn from_jvalues(jvalues: Vec<&JValue>, triplet: Rc<ResolvedTriplet>, json_path: &str) -> Option<IterableValue> {
+fn from_jvalues(
+    jvalues: Vec<&JValue>,
+    triplet: Rc<ResolvedTriplet>,
+    json_path: &str,
+    should_flatten: bool,
+) -> ExecutionResult<Option<IterableValue>> {
     if jvalues.is_empty() {
-        return None;
+        return Ok(None);
     }
 
-    let jvalues = jvalues.into_iter().cloned().collect();
+    let jvalues = construct_iterable_jvalues(jvalues, should_flatten)?;
 
     let tetraplet = SecurityTetraplet {
         triplet,
@@ -163,7 +171,29 @@ fn from_jvalues(jvalues: Vec<&JValue>, triplet: Rc<ResolvedTriplet>, json_path:
     };
 
     let foldable = IterableJsonPathResult::init(jvalues, tetraplet);
-    Some(Box::new(foldable))
+    Ok(Some(Box::new(foldable)))
+}
+
+fn construct_iterable_jvalues(jvalues: Vec<&JValue>, should_flatten: bool) -> ExecutionResult<Vec<JValue>> {
+    if !should_flatten {
+        let jvalues = jvalues.into_iter().cloned().collect();
+        return Ok(jvalues);
+    }
+
+    if jvalues.len() != 1 {
+        let jvalues = jvalues.into_iter().cloned().collect();
+        let jvalue = JValue::Array(jvalues);
+        return exec_err!(ExecutionError::FlatteningError(jvalue));
+    }
+
+    match jvalues[0] {
+        JValue::Array(values) => Ok(values.into_iter().cloned().collect::<Vec<_>>()),
+        _ => {
+            let jvalues = jvalues.into_iter().cloned().collect();
+            let jvalue = JValue::Array(jvalues);
+            exec_err!(ExecutionError::FlatteningError(jvalue))
+        }
+    }
 }
 
 fn as_triplet(iterable: &IterableItem<'_>) -> Rc<ResolvedTriplet> {

interpreter-lib/src/execution/air/mod.rs

@@ -44,7 +44,6 @@ macro_rules! execute {
                 }
 
                 let instruction = format!("{}", $self);
-                println!("set error on {}", instruction);
                 let last_error = LastErrorDescriptor::new(e.clone(), instruction, None);
                 $exec_ctx.last_error = Some(last_error);
                 Err(e)

interpreter-lib/src/execution/errors.rs

@@ -91,6 +91,10 @@ pub(crate) enum ExecutionError {
     /// This error type is produced by a mismatch to notify xor that compared values aren't equal.
     #[error("mismatch is used without corresponding xor")]
     MismatchWithoutXorError,
+
+    /// This error type is produced by a mismatch to notify xor that compared values aren't equal.
+    #[error("jvalue '{0}' can't be flattened, to be flattened a jvalue should have an array type and consist only one value")]
+    FlatteningError(JValue),
 }
 
 impl ExecutionError {
@@ -114,6 +118,7 @@ impl ExecutionError {
             ShadowingError(_) => 14,
             MatchWithoutXorError => 15,
             MismatchWithoutXorError => 16,
+            FlatteningError(_) => 17,
         }
     }
 }

interpreter-lib/src/execution/utils/resolve.rs

@@ -36,7 +36,11 @@ pub(crate) fn resolve_to_args<'i>(
         CallInstrArgValue::Boolean(value) => prepare_consts(*value, ctx),
         CallInstrArgValue::Number(value) => prepare_consts(value, ctx),
         CallInstrArgValue::Variable(name) => prepare_variable(name, ctx),
-        CallInstrArgValue::JsonPath { variable, path } => prepare_json_path(variable, path, ctx),
+        CallInstrArgValue::JsonPath {
+            variable,
+            path,
+            should_flatten,
+        } => prepare_json_path(variable, path, *should_flatten, ctx),
     }
 }
 
@@ -84,12 +88,22 @@ fn prepare_variable<'i>(name: &str, ctx: &ExecutionCtx<'i>) -> ExecutionResult<(
 fn prepare_json_path<'i>(
     name: &str,
     json_path: &str,
+    should_flatten: bool,
     ctx: &ExecutionCtx<'i>,
 ) -> ExecutionResult<(JValue, Vec<SecurityTetraplet>)> {
     let resolved = resolve_to_jvaluable(name, ctx)?;
     let (jvalue, tetraplets) = resolved.apply_json_path_with_tetraplets(json_path)?;
+
+    let jvalue = if should_flatten {
+        if jvalue.len() != 1 {
             let jvalue = jvalue.into_iter().cloned().collect::<Vec<_>>();
-    let jvalue = JValue::Array(jvalue);
+            return crate::exec_err!(ExecutionError::FlatteningError(JValue::Array(jvalue)));
+        }
+        jvalue[0].clone()
+    } else {
+        let jvalue = jvalue.into_iter().cloned().collect::<Vec<_>>();
+        JValue::Array(jvalue)
+    };
 
     Ok((jvalue, tetraplets))
 }

interpreter-lib/tests/flattening.rs

@@ -0,0 +1,209 @@
+/*
+ * Copyright 2020 Fluence Labs Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+use aqua_test_utils::call_vm;
+use aqua_test_utils::create_aqua_vm;
+use aqua_test_utils::set_variable_call_service;
+use aqua_test_utils::CallServiceClosure;
+use aqua_test_utils::IValue;
+use aqua_test_utils::NEVec;
+
+use serde_json::json;
+
+use std::cell::RefCell;
+use std::rc::Rc;
+
+type ClosureSettableVar<T> = Rc<RefCell<T>>;
+
+#[derive(Default, Clone, Debug, PartialEq, Eq)]
+struct ClosureCallArgs {
+    pub(self) service_id_var: Rc<RefCell<String>>,
+    pub(self) function_name_var: ClosureSettableVar<String>,
+    pub(self) args_var: ClosureSettableVar<Vec<i32>>,
+    pub(self) tetraplets: ClosureSettableVar<Vec<Vec<String>>>,
+}
+
+fn create_check_service_closure(closure_call_args: ClosureCallArgs) -> CallServiceClosure {
+    Box::new(move |_, args| -> Option<IValue> {
+        use std::ops::Deref;
+
+        let service_id = match &args[0] {
+            IValue::String(str) => str,
+            _ => unreachable!(),
+        };
+        *closure_call_args.service_id_var.deref().borrow_mut() = service_id.clone();
+
+        let function_name = match &args[1] {
+            IValue::String(str) => str,
+            _ => unreachable!(),
+        };
+        *closure_call_args.function_name_var.deref().borrow_mut() = function_name.clone();
+
+        let call_args = match &args[2] {
+            IValue::String(str) => str,
+            _ => unreachable!(),
+        };
+
+        let call_args: Vec<i32> = serde_json::from_str(call_args).expect("json deserialization shouldn't fail");
+        *closure_call_args.args_var.deref().borrow_mut() = call_args;
+
+        Some(IValue::Record(
+            NEVec::new(vec![IValue::S32(0), IValue::String(r#""""#.to_string())]).unwrap(),
+        ))
+    })
+}
+
+#[test]
+fn flattening_scalar_arrays() {
+    let scalar_array = json!({"iterable": [
+        {"peer_id" : "local_peer_id", "service_id": "local_service_id", "function_name": "local_function_name", "args": [0, 1]},
+        {"peer_id" : "local_peer_id", "service_id": "local_service_id", "function_name": "local_function_name", "args": [2, 3]},
+    ]});
+
+    let scalar_array = serde_json::to_string(&scalar_array).expect("the default serializer shouldn't fail");
+    let set_variable_peer_id = "set_variable";
+    let mut set_variable_vm = create_aqua_vm(set_variable_call_service(scalar_array), set_variable_peer_id);
+
+    let closure_call_args = ClosureCallArgs::default();
+    let local_peer_id = "local_peer_id";
+    let mut local_vm = create_aqua_vm(create_check_service_closure(closure_call_args.clone()), local_peer_id);
+
+    let script = format!(
+        r#"
+        (seq
+            (call "{0}" ("" "") [] scalar_array)
+            (fold scalar_array.$.iterable! v
+                (seq
+                    (call v.$.peer_id! (v.$.service_id! v.$.function_name!) [v.$.args[0]! v.$.args[1]!])
+                    (next v)
+                )
+            )
+        )
+        "#,
+        set_variable_peer_id
+    );
+
+    let res = call_vm!(set_variable_vm, "asd", script.clone(), "", "");
+    let res = call_vm!(local_vm, "asd", script.clone(), "", res.data);
+
+    assert_eq!(res.ret_code, 0);
+    assert_eq!(
+        closure_call_args.service_id_var,
+        Rc::new(RefCell::new("local_service_id".to_string()))
+    );
+    assert_eq!(
+        closure_call_args.function_name_var,
+        Rc::new(RefCell::new("local_function_name".to_string()))
+    );
+    assert_eq!(closure_call_args.args_var, Rc::new(RefCell::new(vec![2, 3])));
+}
+
+#[test]
+fn flattening_streams() {
+    let stream_value = json!(
+        {"peer_id" : "local_peer_id", "service_id": "local_service_id", "function_name": "local_function_name", "args": [0, 1]}
+    );
+
+    let stream_value = serde_json::to_string(&stream_value).expect("the default serializer shouldn't fail");
+    let set_variable_peer_id = "set_variable";
+    let mut set_variable_vm = create_aqua_vm(set_variable_call_service(stream_value), set_variable_peer_id);
+
+    let closure_call_args = ClosureCallArgs::default();
+    let local_peer_id = "local_peer_id";
+    let mut local_vm = create_aqua_vm(create_check_service_closure(closure_call_args.clone()), local_peer_id);
+
+    let script = format!(
+        r#"
+        (seq
+            (seq
+                (seq
+                    (call "{0}" ("" "") [] stream[])
+                    (call "{0}" ("" "") [] stream[])
+                )
+                (call "{0}" ("" "") [] stream[])
+            )
+            (fold stream.$.[0,1,2] v
+                (seq
+                    (call v.$.peer_id! (v.$.service_id! v.$.function_name!) [v.$.args[0]! v.$.args[1]!])
+                    (next v)
+                )
+            )
+        )
+        "#,
+        set_variable_peer_id
+    );
+
+    let res = call_vm!(set_variable_vm, "asd", script.clone(), "", "");
+    let res = call_vm!(local_vm, "asd", script.clone(), "", res.data);
+
+    assert_eq!(res.ret_code, 0);
+    assert_eq!(
+        closure_call_args.service_id_var,
+        Rc::new(RefCell::new("local_service_id".to_string()))
+    );
+    assert_eq!(
+        closure_call_args.function_name_var,
+        Rc::new(RefCell::new("local_function_name".to_string()))
+    );
+    assert_eq!(closure_call_args.args_var, Rc::new(RefCell::new(vec![0, 1])));
+}
+
+#[test]
+fn test_handling_non_flattening_values() {
+    let stream_value = json!(
+        {"peer_id" : "local_peer_id", "service_id": "local_service_id", "function_name": "local_function_name", "args": [0, 1]}
+    );
+
+    let stream_value = serde_json::to_string(&stream_value).expect("the default serializer shouldn't fail");
+    let set_variable_peer_id = "set_variable";
+    let mut set_variable_vm = create_aqua_vm(set_variable_call_service(stream_value), set_variable_peer_id);
+
+    let closure_call_args = ClosureCallArgs::default();
+    let local_peer_id = "local_peer_id";
+    let mut local_vm = create_aqua_vm(create_check_service_closure(closure_call_args.clone()), local_peer_id);
+
+    let script = format!(
+        r#"
+        (seq
+            (seq
+                (seq
+                    (call "{0}" ("" "") [] stream[])
+                    (call "{0}" ("" "") [] stream[])
+                )
+                (call "{0}" ("" "") [] stream[])
+            )
+            (fold stream.$.[0,1,2]! v
+                (seq
+                    (call v.$.peer_id! (v.$.service_id! v.$.function_name!) [v.$.args[0]! v.$.args[1]!])
+                    (next v)
+                )
+            )
+        )
+        "#,
+        set_variable_peer_id
+    );
+
+    let res = call_vm!(set_variable_vm, "asd", script.clone(), "", "");
+    let res = call_vm!(local_vm, "asd", script.clone(), "", res.data);
+
+    assert_eq!(res.ret_code, 1017);
+    assert_eq!(
+        res.error_message,
+        String::from(
+            r#"jvalue '[{"peer_id":"local_peer_id","service_id":"local_service_id","function_name":"local_function_name","args":[0,1]},{"peer_id":"local_peer_id","service_id":"local_service_id","function_name":"local_function_name","args":[0,1]},{"peer_id":"local_peer_id","service_id":"local_service_id","function_name":"local_function_name","args":[0,1]}]' can't be flattened, to be flattened a jvalue should have an array type and consist only one value"#
+        )
+    );
+}

interpreter-lib/tests/join.rs

@@ -57,8 +57,8 @@ fn join_chat() {
                         (fold members m
                             (par 
                                 (seq
-                                    (call m.$.[1] ("identity" "") [] void[])
+                                    (call m.$.[1]! ("identity" "") [] void[])
-                                    (call m.$.[0] ("fgemb3" "add") [] void3[])
+                                    (call m.$.[0]! ("fgemb3" "add") [] void3[])
                                 )
                                 (next m)
                             )

interpreter/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "aquamarine"
-version = "0.6.0"
+version = "0.7.0"
 authors = ["Fluence Labs"]
 edition = "2018"