Introduce canon instruction (#292)

CHANGELOG.md

@@ -1,14 +1,41 @@
-## Version 0.24.0 (2021-04-21)
+## Version 0.27.0 (2022-08-23)
+
+[PR 292](https://github.com/fluencelabs/aquavm/pull/292):
+Introduced a new `canon` instruction
+
+[PR 296](https://github.com/fluencelabs/aquavm/pull/296):  
+A bug with an inappropriate check of states in the `Ap` merger was fixed
+
+## Version 0.26.0 (2022-08-19)
+
+[PR 294](https://github.com/fluencelabs/aquavm/pull/294):  
+Accompanying to air-interpreter update that makes interface more handy for `air-trace`
+
+## Version 0.25.0 (2022-07-27)
+
+[PR 287](https://github.com/fluencelabs/aquavm/pull/287):
+Memory leak introduced by switching to reactor/command model in WASI fixed
+
+[PR 276](https://github.com/fluencelabs/aquavm/pull/276):  
+AquaVM performance was improved by removing excess logging
+
+[PR 273](https://github.com/fluencelabs/aquavm/pull/273):  
+Introduced `TracePos` for `TraceHandler` positions
+
+[PR 270](https://github.com/fluencelabs/aquavm/pull/270):
+A bug with empty subtrace lore in TraceHandler was fixed 
+
+## Version 0.24.0 (2022-04-21)
 
 [PR 253](https://github.com/fluencelabs/aquavm/pull/253):
 Introduced %ttl% keyword
 
-## Version 0.24.0 (2021-04-20)
+## Version 0.24.0 (2022-04-20)
 
 [PR 250](https://github.com/fluencelabs/aquavm/pull/250):
 Introduced %timestamp% keyword
 
-## Version 0.23.0 (2021-04-20)
+## Version 0.23.0 (2022-04-20)
 
 [PR 248](https://github.com/fluencelabs/aquavm/pull/248):
 Introduced new for scalars
@@ -16,7 +43,7 @@ Introduced new for scalars
 [PR 244](https://github.com/fluencelabs/aquavm/pull/244):
 Stack size was increased to 50 MiB
 
-## Version 0.22.0 (2021-04-14)
+## Version 0.22.0 (2022-04-14)
 
 [PR 243](https://github.com/fluencelabs/aquavm/pull/243):
 Clean scalars at the end of scope, only one next in a fold over scalar is possible now
@@ -27,13 +54,13 @@ Test refactoring
 [PR 228](https://github.com/fluencelabs/aquavm/pull/228):
 Improve stream determinism
 
-## Version 0.21.0 (2021-02-26)
+## Version 0.21.0 (2022-02-26)
 
 [PR 225](https://github.com/fluencelabs/aquavm/pull/225):  
 Introduce recursive streams
 
 [PR 224](https://github.com/fluencelabs/aquavm/pull/224) [PR 220](https://github.com/fluencelabs/aquavm/pull/224) [PR 217](https://github.com/fluencelabs/aquavm/pull/217) [PR 215](https://github.com/fluencelabs/aquavm/pull/215) [PR 212](https://github.com/fluencelabs/aquavm/pull/212) [PR 207](https://github.com/fluencelabs/aquavm/pull/207):  
-Various bugs fixed
+Various bugs were fixed
 
 [PR 210](https://github.com/fluencelabs/aquavm/pull/210):  
 Add API for returning AquaVM consumed memory size

Cargo.lock

@@ -13,7 +13,7 @@ dependencies = [
 
 [[package]]
 name = "air"
-version = "0.26.0"
+version = "0.27.0"
 dependencies = [
  "air-execution-info-collector",
  "air-interpreter-data",
@@ -72,7 +72,7 @@ version = "0.1.0"
 
 [[package]]
 name = "air-interpreter"
-version = "0.26.0"
+version = "0.27.0"
 dependencies = [
  "air",
  "air-interpreter-interface",
@@ -88,7 +88,7 @@ dependencies = [
 
 [[package]]
 name = "air-interpreter-data"
-version = "0.2.2"
+version = "0.3.0"
 dependencies = [
  "air-utils",
  "once_cell",
@@ -199,6 +199,7 @@ dependencies = [
  "air-interpreter-data",
  "air-log-targets",
  "air-parser",
+ "bimap",
  "log",
  "serde_json",
  "thiserror",
@@ -307,6 +308,12 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "bimap"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc0455254eb5c6964c4545d8bac815e1a1be4f3afe0ae695ea539c12d728d44b"
+
 [[package]]
 name = "bincode"
 version = "1.3.3"
@@ -1829,7 +1836,7 @@ dependencies = [
 
 [[package]]
 name = "polyplets"
-version = "0.3.0"
+version = "0.3.1"
 dependencies = [
  "marine-macro",
  "marine-rs-sdk-main",

README.md

@@ -86,6 +86,24 @@ Example:
 )
 ```
 
+#### canon
+
+```wasm
+(canon "peer_id" <$stream> <#canon_stream>)
+```
+
+- executes on peer_id, takes $stream as it is on the moment of first canonicalization
+- every next execution #canon_stream will be the same — as first seen by peer_id
+
+Example:
+
+```wasm
+(seq
+    (ap user $users)
+    (canon "peer_id" $stream #canon_stream)
+)
+```
+
 #### match/mismath
 
 ```wasm
@@ -178,3 +196,11 @@ Example
 - versioned
 - could be used only by call and fold instructions (more instructions for streams to come)
 - could be turned to scalar (canonicalized)
+
+#### Canonicalized streams
+
+- contains an array of elements that was in a stream at the moment of canonicalization
+- canonicalized streams are imutable and fully consistent as scalars
+- has the same algebra as a stream for `match`/`mismatch` and `call` argument
+- has the same algebra as a scalar for `new`
+- has mixed behaviour for with other instructions

air-interpreter/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "air-interpreter"
-version = "0.26.0"
+version = "0.27.0"
 description = "Crate-wrapper for air"
 authors = ["Fluence Labs"]
 edition = "2018"

air/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "air"
-version = "0.26.0"
+version = "0.27.0"
 description = "Interpreter of AIR scripts intended to coordinate request flow in the Fluence network"
 authors = ["Fluence Labs"]
 edition = "2018"

air/src/execution_step/air/ap.rs

@@ -30,8 +30,8 @@ use crate::SecurityTetraplet;
 use apply_to_arguments::*;
 use utils::*;
 
-use air_parser::ast;
 use air_parser::ast::Ap;
+use air_parser::ast::ApResult;
 use air_trace_handler::MergerApResult;
 
 use std::rc::Rc;
@@ -57,7 +57,7 @@ impl<'i> super::ExecutableInstruction<'i> for Ap<'i> {
 /// This function is intended to check whether a Ap instruction should produce
 /// a new state in data.
 fn should_touch_trace(ap: &Ap<'_>) -> bool {
-    matches!(ap.result, ast::Variable::Stream(_))
+    matches!(ap.result, ApResult::Stream(_))
 }
 
 fn to_merger_ap_result(
@@ -77,16 +77,14 @@ fn to_merger_ap_result(
 }
 
 fn update_context<'ctx>(
-    ap_result_type: &ast::Variable<'ctx>,
+    ap_result_type: &ApResult<'ctx>,
     merger_ap_result: &MergerApResult,
     result: ValueAggregate,
     exec_ctx: &mut ExecutionCtx<'ctx>,
 ) -> ExecutionResult<Option<u32>> {
-    use ast::Variable::*;
-
     match ap_result_type {
-        Scalar(scalar) => exec_ctx.scalars.set_value(scalar.name, result).map(|_| None),
+        ApResult::Scalar(scalar) => exec_ctx.scalars.set_scalar_value(scalar.name, result).map(|_| None),
-        Stream(stream) => {
+        ApResult::Stream(stream) => {
             let generation = ap_result_to_generation(merger_ap_result);
             exec_ctx
                 .streams

air/src/execution_step/air/ap/apply_to_arguments.rs

@@ -36,6 +36,7 @@ pub(super) fn apply_to_arg(
         Boolean(value) => apply_const(*value, exec_ctx, trace_ctx),
         EmptyArray => apply_const(serde_json::json!([]), exec_ctx, trace_ctx),
         Scalar(scalar) => apply_scalar(scalar, exec_ctx, trace_ctx, should_touch_trace)?,
+        CanonStream(canon_stream) => apply_canon_stream(canon_stream, exec_ctx)?,
     };
 
     Ok(result)
@@ -71,7 +72,7 @@ fn apply_scalar(
 ) -> ExecutionResult<ValueAggregate> {
     // TODO: refactor this code after boxed value
     match &scalar.lambda {
-        Some(lambda) => apply_scalar_wl_impl(scalar.name, scalar.position, lambda, exec_ctx, trace_ctx),
+        Some(lambda) => apply_scalar_wl_impl(scalar.name, lambda, exec_ctx, trace_ctx),
         None => apply_scalar_impl(scalar.name, exec_ctx, trace_ctx, should_touch_trace),
     }
 }
@@ -106,15 +107,53 @@ fn apply_scalar_impl(
 
 fn apply_scalar_wl_impl(
     scalar_name: &str,
-    position: usize,
     lambda: &LambdaAST<'_>,
     exec_ctx: &ExecutionCtx<'_>,
     trace_ctx: &TraceHandler,
 ) -> ExecutionResult<ValueAggregate> {
-    let variable = Variable::scalar(scalar_name, position);
+    let variable = Variable::scalar(scalar_name);
     let (jvalue, tetraplet) = apply_lambda(variable, lambda, exec_ctx)?;
     let tetraplet = Rc::new(tetraplet);
     let result = ValueAggregate::new(Rc::new(jvalue), tetraplet, trace_ctx.trace_pos());
 
     Ok(result)
 }
+
+fn apply_canon_stream(
+    canon_stream: &ast::CanonStreamWithLambda<'_>,
+    exec_ctx: &ExecutionCtx<'_>,
+) -> ExecutionResult<ValueAggregate> {
+    match &canon_stream.lambda {
+        Some(lambda) => apply_canon_stream_with_lambda(canon_stream.name, lambda, exec_ctx),
+        None => apply_canon_stream_without_lambda(canon_stream.name, exec_ctx),
+    }
+}
+
+fn apply_canon_stream_with_lambda(
+    stream_name: &str,
+    lambda: &LambdaAST<'_>,
+    exec_ctx: &ExecutionCtx<'_>,
+) -> ExecutionResult<ValueAggregate> {
+    use crate::execution_step::boxed_value::JValuable;
+
+    let canon_stream = exec_ctx.scalars.get_canon_stream(stream_name)?;
+    let (result, tetraplet) = JValuable::apply_lambda_with_tetraplets(&canon_stream, lambda, exec_ctx)?;
+    // TODO: refactor this code after boxed value
+    let value = ValueAggregate::new(Rc::new(result.clone()), Rc::new(tetraplet), canon_stream.position());
+    Ok(value)
+}
+
+fn apply_canon_stream_without_lambda(
+    stream_name: &str,
+    exec_ctx: &ExecutionCtx<'_>,
+) -> ExecutionResult<ValueAggregate> {
+    use crate::execution_step::boxed_value::JValuable;
+
+    let canon_stream = exec_ctx.scalars.get_canon_stream(stream_name)?;
+    // TODO: refactor this code after boxed value
+    let value = JValuable::as_jvalue(&canon_stream).into_owned();
+
+    let tetraplet = canon_stream.tetraplet().clone();
+    let value = ValueAggregate::new(Rc::new(value), tetraplet, canon_stream.position());
+    Ok(value)
+}

air/src/execution_step/air/ap/utils.rs

@@ -39,12 +39,12 @@ pub(super) fn try_match_trace_to_instr(merger_ap_result: &MergerApResult, instr:
 }
 
 fn match_position_variable(
-    variable: &ast::Variable<'_>,
+    variable: &ast::ApResult<'_>,
     generation: Option<u32>,
     ap_result: &MergerApResult,
 ) -> ExecutionResult<()> {
     use crate::execution_step::UncatchableError::ApResultNotCorrespondToInstr;
-    use ast::Variable::*;
+    use ast::ApResult::*;
 
     match (variable, generation) {
         (Stream(_), Some(_)) => Ok(()),

air/src/execution_step/air/call.rs

@@ -17,7 +17,7 @@
 pub(crate) mod call_result_setter;
 mod prev_result_handler;
 mod resolved_call;
-mod triplet;
+pub(crate) mod triplet;
 
 use resolved_call::ResolvedCall;
 

air/src/execution_step/air/call/call_result_setter.rs

@@ -23,7 +23,6 @@ use air_interpreter_data::CallResult;
 use air_interpreter_data::TracePos;
 use air_interpreter_data::Value;
 use air_parser::ast::CallOutputValue;
-use air_parser::ast::Variable;
 use air_trace_handler::TraceHandler;
 
 /// Writes result of a local `Call` instruction to `ExecutionCtx` at `output`.
@@ -35,11 +34,11 @@ pub(crate) fn set_local_result<'i>(
 ) -> ExecutionResult<CallResult> {
     let result_value = executed_result.result.clone();
     match output {
-        CallOutputValue::Variable(Variable::Scalar(scalar)) => {
+        CallOutputValue::Scalar(scalar) => {
-            exec_ctx.scalars.set_value(scalar.name, executed_result)?;
+            exec_ctx.scalars.set_scalar_value(scalar.name, executed_result)?;
             Ok(CallResult::executed_scalar(result_value))
         }
-        CallOutputValue::Variable(Variable::Stream(stream)) => {
+        CallOutputValue::Stream(stream) => {
             let generation =
                 exec_ctx
                     .streams
@@ -58,11 +57,11 @@ pub(crate) fn set_result_from_value<'i>(
     exec_ctx: &mut ExecutionCtx<'i>,
 ) -> ExecutionResult<()> {
     match (output, value) {
-        (CallOutputValue::Variable(Variable::Scalar(scalar)), Value::Scalar(value)) => {
+        (CallOutputValue::Scalar(scalar), Value::Scalar(value)) => {
             let result = ValueAggregate::new(value, tetraplet, trace_pos);
-            exec_ctx.scalars.set_value(scalar.name, result)?;
+            exec_ctx.scalars.set_scalar_value(scalar.name, result)?;
         }
-        (CallOutputValue::Variable(Variable::Stream(stream)), Value::Stream { value, generation }) => {
+        (CallOutputValue::Stream(stream), Value::Stream { value, generation }) => {
             let result = ValueAggregate::new(value, tetraplet, trace_pos);
             let generation = Generation::Nth(generation);
             let _ = exec_ctx

air/src/execution_step/air/call/resolved_call.rs

@@ -217,7 +217,7 @@ fn check_output_name(output: &ast::CallOutputValue<'_>, exec_ctx: &ExecutionCtx<
     use crate::execution_step::boxed_value::ScalarRef;
 
     let scalar_name = match output {
-        ast::CallOutputValue::Variable(ast::Variable::Scalar(scalar)) => scalar.name,
+        ast::CallOutputValue::Scalar(scalar) => scalar.name,
         _ => return Ok(()),
     };
 

air/src/execution_step/air/call/triplet.rs

@@ -43,7 +43,11 @@ pub(crate) fn resolve<'i>(triplet: &ast::Triplet<'i>, ctx: &ExecutionCtx<'i>) ->
 
 /// Resolve value to string by either resolving variable from `ExecutionCtx`, taking literal value, or etc.
 // TODO: return Rc<String> to avoid excess cloning
-fn resolve_to_string<'i>(value: &ast::CallInstrValue<'i>, ctx: &ExecutionCtx<'i>) -> ExecutionResult<String> {
+// TODO: move this function into resolve in boxed value PR
+pub(crate) fn resolve_to_string<'i>(
+    value: &ast::CallInstrValue<'i>,
+    ctx: &ExecutionCtx<'i>,
+) -> ExecutionResult<String> {
     use crate::execution_step::resolver::resolve_ast_variable_wl;
     use ast::CallInstrValue::*;
 

air/src/execution_step/air/canon.rs

@@ -0,0 +1,170 @@
+/*
+ * Copyright 2022 Fluence Labs Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+use super::ExecutionCtx;
+use super::ExecutionResult;
+use super::TraceHandler;
+use crate::execution_step::boxed_value::CanonStream;
+use crate::execution_step::Generation;
+use crate::log_instruction;
+use crate::trace_to_exec_err;
+use crate::CatchableError;
+use crate::ExecutionError;
+use crate::UncatchableError;
+
+use air_interpreter_data::CanonResult;
+use air_interpreter_data::TracePos;
+use air_parser::ast;
+use air_trace_handler::MergerCanonResult;
+
+use std::rc::Rc;
+
+impl<'i> super::ExecutableInstruction<'i> for ast::Canon<'i> {
+    #[tracing::instrument(level = "debug", skip(exec_ctx, trace_ctx))]
+    fn execute(&self, exec_ctx: &mut ExecutionCtx<'i>, trace_ctx: &mut TraceHandler) -> ExecutionResult<()> {
+        log_instruction!(call, exec_ctx, trace_ctx);
+        let canon_result = trace_to_exec_err!(trace_ctx.meet_canon_start(), self)?;
+
+        match canon_result {
+            MergerCanonResult::CanonResult { stream_elements_pos } => {
+                handle_seen_canon(self, stream_elements_pos, exec_ctx, trace_ctx)
+            }
+            MergerCanonResult::Empty => handle_unseen_canon(self, exec_ctx, trace_ctx),
+        }
+    }
+}
+
+fn handle_seen_canon(
+    ast_canon: &ast::Canon<'_>,
+    stream_elements_pos: Vec<TracePos>,
+    exec_ctx: &mut ExecutionCtx<'_>,
+    trace_ctx: &mut TraceHandler,
+) -> ExecutionResult<()> {
+    let canon_stream = create_canon_stream_from_pos(&stream_elements_pos, ast_canon, exec_ctx)?;
+    let stream_with_positions = StreamWithPositions {
+        canon_stream,
+        stream_elements_pos,
+    };
+
+    epilog(ast_canon.canon_stream.name, stream_with_positions, exec_ctx, trace_ctx)
+}
+
+fn handle_unseen_canon(
+    ast_canon: &ast::Canon<'_>,
+    exec_ctx: &mut ExecutionCtx<'_>,
+    trace_ctx: &mut TraceHandler,
+) -> ExecutionResult<()> {
+    let peer_id = crate::execution_step::air::resolve_to_string(&ast_canon.peer_pk, exec_ctx)?;
+
+    if exec_ctx.run_parameters.current_peer_id.as_str() != peer_id {
+        exec_ctx.subgraph_complete = false;
+        exec_ctx.next_peer_pks.push(peer_id);
+        //this branch is executed only when
+        //  this canon instruction executes for the first time
+        //  a peer is different from one set in peer_id of a this canon instruction
+        //
+        // the former means that there wasn't canon associated state in data, the latter that it
+        // can't be obtained on this peer, so it's intended not to call meet_canon_end here.
+        return Ok(());
+    }
+
+    let stream_with_positions = create_canon_stream_from_name(ast_canon, peer_id, exec_ctx)?;
+    epilog(ast_canon.canon_stream.name, stream_with_positions, exec_ctx, trace_ctx)
+}
+
+fn create_canon_stream_from_pos(
+    stream_elements_pos: &[TracePos],
+    ast_canon: &ast::Canon<'_>,
+    exec_ctx: &ExecutionCtx<'_>,
+) -> ExecutionResult<CanonStream> {
+    let stream = exec_ctx
+        .streams
+        .get(ast_canon.stream.name, ast_canon.stream.position)
+        .ok_or_else(|| {
+            ExecutionError::Catchable(Rc::new(CatchableError::StreamsForCanonNotFound(
+                ast_canon.stream.name.to_string(),
+            )))
+        })?;
+
+    let values = stream_elements_pos
+        .iter()
+        .map(|&position| {
+            stream
+                .get_value_by_pos(position)
+                .ok_or(ExecutionError::Uncatchable(UncatchableError::VariableNotFoundByPos(
+                    position,
+                )))
+                .cloned()
+        })
+        .collect::<Result<Vec<_>, _>>()?;
+
+    let peer_id = crate::execution_step::air::resolve_to_string(&ast_canon.peer_pk, exec_ctx)?;
+    let canon_stream = CanonStream::new(values, peer_id, ast_canon.stream.position.into());
+    Ok(canon_stream)
+}
+
+fn epilog(
+    canon_stream_name: &str,
+    stream_with_positions: StreamWithPositions,
+    exec_ctx: &mut ExecutionCtx<'_>,
+    trace_ctx: &mut TraceHandler,
+) -> ExecutionResult<()> {
+    let StreamWithPositions {
+        canon_stream,
+        stream_elements_pos,
+    } = stream_with_positions;
+
+    exec_ctx
+        .scalars
+        .set_canon_value(canon_stream_name, canon_stream)
+        .map(|_| ())?;
+    trace_ctx.meet_canon_end(CanonResult::new(stream_elements_pos));
+    Ok(())
+}
+
+struct StreamWithPositions {
+    canon_stream: CanonStream,
+    stream_elements_pos: Vec<TracePos>,
+}
+
+fn create_canon_stream_from_name(
+    ast_canon: &ast::Canon<'_>,
+    peer_id: String,
+    exec_ctx: &ExecutionCtx<'_>,
+) -> ExecutionResult<StreamWithPositions> {
+    let stream = exec_ctx
+        .streams
+        .get(ast_canon.stream.name, ast_canon.stream.position)
+        .ok_or_else(|| {
+            ExecutionError::Catchable(Rc::new(CatchableError::StreamsForCanonNotFound(
+                ast_canon.stream.name.to_string(),
+            )))
+        })?;
+    let canon_stream = CanonStream::from_stream(stream, peer_id, ast_canon.canon_stream.position.into());
+    let stream_elements_pos = stream
+        .iter(Generation::Last)
+        // it's always safe to iter over all generations
+        .unwrap()
+        .map(|value| value.trace_pos)
+        .collect::<Vec<_>>();
+
+    let result = StreamWithPositions {
+        canon_stream,
+        stream_elements_pos,
+    };
+
+    Ok(result)
+}

air/src/execution_step/air/fail.rs

@@ -18,7 +18,7 @@ use super::ExecutionCtx;
 use super::ExecutionResult;
 use super::TraceHandler;
 use crate::execution_step::execution_context::check_error_object;
-use crate::execution_step::resolver::resolve_ast_scalar_wl;
+use crate::execution_step::resolver::{apply_lambda, resolve_ast_scalar_wl};
 use crate::execution_step::CatchableError;
 use crate::execution_step::LastError;
 use crate::execution_step::RcSecurityTetraplet;
@@ -30,6 +30,8 @@ use air_parser::ast;
 use air_parser::ast::Fail;
 use polyplets::SecurityTetraplet;
 
+use crate::execution_step::boxed_value::Variable;
+use air_lambda_ast::LambdaAST;
 use std::rc::Rc;
 
 impl<'i> super::ExecutableInstruction<'i> for Fail<'i> {
@@ -42,6 +44,7 @@ impl<'i> super::ExecutableInstruction<'i> for Fail<'i> {
                 ret_code,
                 error_message,
             } => fail_with_literals(ret_code, error_message, self, exec_ctx),
+            Fail::CanonStream { name, lambda } => fail_with_canon_stream(name, lambda, exec_ctx),
             // bubble last error up
             Fail::LastError => fail_with_last_error(exec_ctx),
         }
@@ -76,6 +79,20 @@ fn fail_with_literals<'i>(
     fail_with_error_object(exec_ctx, Rc::new(error_object), Some(literal_tetraplet))
 }
 
+fn fail_with_canon_stream<'i>(
+    name: &'i str,
+    lambda: &LambdaAST<'i>,
+    exec_ctx: &mut ExecutionCtx<'i>,
+) -> ExecutionResult<()> {
+    let variable = Variable::CanonStream { name };
+
+    let (value, tetraplet) = apply_lambda(variable, lambda, exec_ctx)?;
+    // tetraplets always have one element here and it'll be refactored after boxed value
+    check_error_object(&value).map_err(CatchableError::InvalidLastErrorObjectError)?;
+
+    fail_with_error_object(exec_ctx, Rc::new(value), Some(Rc::new(tetraplet)))
+}
+
 fn fail_with_last_error(exec_ctx: &mut ExecutionCtx<'_>) -> ExecutionResult<()> {
     let LastError { error, tetraplet } = exec_ctx.last_error_descriptor.last_error();
 

air/src/execution_step/air/fold/mod.rs

@@ -25,5 +25,4 @@ use super::ExecutionCtx;
 use super::ExecutionResult;
 use super::Instruction;
 use super::ScalarRef;
-use super::ValueAggregate;
 use crate::execution_step::boxed_value::*;

air/src/execution_step/air/fold/utils.rs

@@ -31,7 +31,7 @@ pub(crate) type IterableValue = Box<dyn for<'ctx> Iterable<'ctx, Item = Iterable
 
 pub(crate) enum FoldIterableScalar {
     Empty,
-    Scalar(IterableValue),
+    ScalarBased(IterableValue),
 }
 
 /// Constructs iterable value for given scalar iterable.
@@ -45,6 +45,18 @@ pub(crate) fn construct_scalar_iterable_value<'ctx>(
     }
 }
 
+/// Constructs iterable value for given canon stream.
+pub(crate) fn construct_canon_stream_iterable_value<'ctx>(
+    ast_canon_stream: &ast::CanonStream<'ctx>,
+    exec_ctx: &ExecutionCtx<'ctx>,
+) -> ExecutionResult<FoldIterableScalar> {
+    let canon_stream = exec_ctx.scalars.get_canon_stream(ast_canon_stream.name)?;
+    // TODO: this one is a relatively long operation and will be refactored in Boxed Value
+    let iterable_ingredients = CanonStreamIterableIngredients::init(canon_stream.clone());
+    let iterable = Box::new(iterable_ingredients);
+    Ok(FoldIterableScalar::ScalarBased(iterable))
+}
+
 /// Constructs iterable value for given stream iterable.
 pub(crate) fn construct_stream_iterable_values(
     stream: &Stream,
@@ -72,17 +84,17 @@ fn create_scalar_iterable<'ctx>(
     variable_name: &str,
 ) -> ExecutionResult<FoldIterableScalar> {
     match exec_ctx.scalars.get_value(variable_name)? {
-        ScalarRef::Value(call_result) => from_call_result(call_result.clone(), variable_name),
+        ScalarRef::Value(call_result) => from_value(call_result.clone(), variable_name),
         ScalarRef::IterableValue(fold_state) => {
             let iterable_value = fold_state.iterable.peek().unwrap();
             let call_result = iterable_value.into_resolved_result();
-            from_call_result(call_result, variable_name)
+            from_value(call_result, variable_name)
         }
     }
 }
 
 /// Constructs iterable value from resolved call result.
-fn from_call_result(call_result: ValueAggregate, variable_name: &str) -> ExecutionResult<FoldIterableScalar> {
+fn from_value(call_result: ValueAggregate, variable_name: &str) -> ExecutionResult<FoldIterableScalar> {
     let len = match &call_result.result.deref() {
         JValue::Array(array) => {
             if array.is_empty() {
@@ -103,7 +115,7 @@ fn from_call_result(call_result: ValueAggregate, variable_name: &str) -> Executi
 
     let foldable = IterableResolvedCall::init(call_result, len);
     let foldable = Box::new(foldable);
-    let iterable = FoldIterableScalar::Scalar(foldable);
+    let iterable = FoldIterableScalar::ScalarBased(foldable);
 
     Ok(iterable)
 }
@@ -154,7 +166,7 @@ fn from_jvalue(
 
     let iterable = iterable.to_vec();
     let foldable = IterableLambdaResult::init(iterable, tetraplet);
-    let iterable = FoldIterableScalar::Scalar(Box::new(foldable));
+    let iterable = FoldIterableScalar::ScalarBased(Box::new(foldable));
     Ok(iterable)
 }
 

air/src/execution_step/air/fold_scalar.rs

@@ -33,17 +33,21 @@ impl<'i> ExecutableInstruction<'i> for FoldScalar<'i> {
     fn execute(&self, exec_ctx: &mut ExecutionCtx<'i>, trace_ctx: &mut TraceHandler) -> ExecutionResult<()> {
         log_instruction!(fold, exec_ctx, trace_ctx);
 
-        let scalar = match &self.iterable {
+        let iterable = match &self.iterable {
-            FoldScalarIterable::Scalar(scalar) => scalar,
+            FoldScalarIterable::Scalar(scalar) => {
+                joinable!(construct_scalar_iterable_value(scalar, exec_ctx), exec_ctx)?
+            }
+            FoldScalarIterable::CanonStream(canon_stream) => {
+                construct_canon_stream_iterable_value(canon_stream, exec_ctx)?
+            }
             // just do nothing on an empty array
             FoldScalarIterable::EmptyArray => return Ok(()),
         };
-        let scalar_iterable = joinable!(construct_scalar_iterable_value(scalar, exec_ctx), exec_ctx)?;
 
-        match scalar_iterable {
+        match iterable {
             // just exit on empty iterable
             FoldIterableScalar::Empty => Ok(()),
-            FoldIterableScalar::Scalar(iterable) => fold(
+            FoldIterableScalar::ScalarBased(iterable) => fold(
                 iterable,
                 IterableType::Scalar,
                 self.iterator.name,

air/src/execution_step/air/mod.rs

@@ -16,6 +16,7 @@
 
 mod ap;
 mod call;
+mod canon;
 mod compare_matchable;
 mod fail;
 mod fold;
@@ -30,6 +31,7 @@ mod par;
 mod seq;
 mod xor;
 
+pub(crate) use call::triplet::resolve_to_string;
 pub(crate) use fold::FoldState;
 
 use super::boxed_value::ScalarRef;
@@ -74,6 +76,7 @@ impl<'i> ExecutableInstruction<'i> for Instruction<'i> {
             Instruction::Call(call) => call.execute(exec_ctx, trace_ctx),
 
             Instruction::Ap(ap) => execute!(self, ap, exec_ctx, trace_ctx),
+            Instruction::Canon(canon) => execute!(self, canon, exec_ctx, trace_ctx),
             Instruction::Fail(fail) => execute!(self, fail, exec_ctx, trace_ctx),
             Instruction::FoldScalar(fold) => execute!(self, fold, exec_ctx, trace_ctx),
             Instruction::FoldStream(fold) => execute!(self, fold, exec_ctx, trace_ctx),

air/src/execution_step/air/new.rs

@@ -20,7 +20,7 @@ use super::TraceHandler;
 use crate::log_instruction;
 
 use air_parser::ast::New;
-use air_parser::ast::Variable;
+use air_parser::ast::NewArgument;
 
 impl<'i> super::ExecutableInstruction<'i> for New<'i> {
     fn execute(&self, exec_ctx: &mut ExecutionCtx<'i>, trace_ctx: &mut TraceHandler) -> ExecutionResult<()> {
@@ -48,12 +48,15 @@ impl<'i> super::ExecutableInstruction<'i> for New<'i> {
 
 fn prolog<'i>(new: &New<'i>, exec_ctx: &mut ExecutionCtx<'i>) {
     let position = new.span.left;
-    match &new.variable {
+    match &new.argument {
-        Variable::Stream(stream) => {
+        NewArgument::Stream(stream) => {
             let iteration = exec_ctx.tracker.new_tracker.get_iteration(position);
             exec_ctx.streams.meet_scope_start(stream.name, new.span, iteration);
         }
-        Variable::Scalar(scalar) => exec_ctx.scalars.meet_new_start(scalar.name),
+        NewArgument::Scalar(scalar) => exec_ctx.scalars.meet_new_start_scalar(scalar.name.to_string()),
+        NewArgument::CanonStream(canon_stream) => exec_ctx
+            .scalars
+            .meet_new_start_canon_stream(canon_stream.name.to_string()),
     }
 
     exec_ctx.tracker.meet_new(position);
@@ -61,13 +64,14 @@ fn prolog<'i>(new: &New<'i>, exec_ctx: &mut ExecutionCtx<'i>) {
 
 fn epilog<'i>(new: &New<'i>, exec_ctx: &mut ExecutionCtx<'i>) -> ExecutionResult<()> {
     let position = new.span.left;
-    match &new.variable {
+    match &new.argument {
-        Variable::Stream(stream) => {
+        NewArgument::Stream(stream) => {
             exec_ctx
                 .streams
                 .meet_scope_end(stream.name.to_string(), position as u32);
             Ok(())
         }
-        Variable::Scalar(scalar) => exec_ctx.scalars.meet_new_end(scalar.name),
+        NewArgument::Scalar(scalar) => exec_ctx.scalars.meet_new_end_scalar(scalar.name),
+        NewArgument::CanonStream(canon_stream) => exec_ctx.scalars.meet_new_end_canon_stream(canon_stream.name),
     }
 }

air/src/execution_step/boxed_value/canon_stream.rs

@@ -0,0 +1,102 @@
+/*
+ * Copyright 2022 Fluence Labs Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+use super::Stream;
+use super::ValueAggregate;
+use crate::execution_step::Generation;
+use crate::JValue;
+
+use air_interpreter_data::TracePos;
+use polyplets::SecurityTetraplet;
+
+use std::rc::Rc;
+
+/// Canon stream is a value type lies between a scalar and a stream, it has the same algebra as
+/// scalars, and represent a stream fixed at some execution point.
+#[derive(Debug, Default, Clone)]
+pub struct CanonStream {
+    values: Vec<ValueAggregate>,
+    // tetraplet is needed to handle adding canon streams as a whole to a stream
+    tetraplet: Rc<SecurityTetraplet>,
+    position: TracePos,
+}
+
+impl CanonStream {
+    pub(crate) fn new(values: Vec<ValueAggregate>, peer_pk: String, position: TracePos) -> Self {
+        // tetraplet is comprised only from peer_pk here
+        let tetraplet = SecurityTetraplet::new(peer_pk, "", "", "");
+        Self {
+            values,
+            tetraplet: Rc::new(tetraplet),
+            position,
+        }
+    }
+
+    pub(crate) fn from_stream(stream: &Stream, peer_pk: String, position: TracePos) -> Self {
+        // it's always possible to iter over all generations of a stream
+        let values = stream.iter(Generation::Last).unwrap().cloned().collect::<Vec<_>>();
+        let tetraplet = SecurityTetraplet::new(peer_pk, "", "", "");
+        Self {
+            values,
+            tetraplet: Rc::new(tetraplet),
+            position,
+        }
+    }
+
+    pub(crate) fn len(&self) -> usize {
+        self.values.len()
+    }
+
+    pub(crate) fn is_empty(&self) -> bool {
+        self.values.is_empty()
+    }
+
+    pub(crate) fn as_jvalue(&self) -> JValue {
+        use std::ops::Deref;
+
+        // TODO: this clone will be removed after boxed values
+        let jvalue_array = self.values.iter().map(|r| r.result.deref().clone()).collect::<Vec<_>>();
+        JValue::Array(jvalue_array)
+    }
+
+    pub(crate) fn iter(&self) -> impl ExactSizeIterator<Item = &ValueAggregate> {
+        self.values.iter()
+    }
+
+    pub(crate) fn nth(&self, idx: usize) -> Option<&ValueAggregate> {
+        self.values.get(idx)
+    }
+
+    pub(crate) fn tetraplet(&self) -> &Rc<SecurityTetraplet> {
+        &self.tetraplet
+    }
+
+    pub(crate) fn position(&self) -> TracePos {
+        self.position
+    }
+}
+
+use std::fmt;
+
+impl fmt::Display for CanonStream {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "[")?;
+        for value in self.values.iter() {
+            write!(f, "{}, ", value)?;
+        }
+        write!(f, "]")
+    }
+}

air/src/execution_step/boxed_value/iterable.rs

@@ -14,11 +14,12 @@
  * limitations under the License.
  */
 
+mod canon_stream;
 mod json_path_result;
 mod resolved_call;
-mod vec_json_path_result;
 mod vec_resolved_call;
 
+pub(crate) use canon_stream::CanonStreamIterableIngredients;
 pub(crate) use json_path_result::IterableLambdaResult;
 pub(crate) use resolved_call::IterableResolvedCall;
 pub(crate) use vec_resolved_call::IterableVecResolvedCall;

air/src/execution_step/boxed_value/iterable/canon_stream.rs

@@ -16,36 +16,31 @@
 
 use super::Iterable;
 use super::IterableItem;
-use crate::execution_step::RcSecurityTetraplets;
+use crate::execution_step::boxed_value::CanonStream;
 use crate::foldable_next;
 use crate::foldable_prev;
-use crate::JValue;
 
-/// Used for iterating over a result of applied to a stream lambda.
+const EXPECT_VALUE_IN_STREAM: &str = "value must exist, because length checked before creation and canonicalized stream can't be modified during iteration";
-#[derive(Clone, Debug, Eq, PartialEq)]
+
-pub(crate) struct IterableVecJsonPathResult {
+pub(crate) struct CanonStreamIterableIngredients {
-    pub(crate) jvalues: Vec<JValue>,
+    canon_stream: CanonStream,
-    pub(crate) tetraplets: RcSecurityTetraplets,
+    cursor: usize,
-    pub(crate) cursor: usize,
 }
 
-impl IterableVecJsonPathResult {
+impl CanonStreamIterableIngredients {
-    #[allow(dead_code)]
+    pub(crate) fn init(canon_stream: CanonStream) -> Self {
-    pub(crate) fn init(jvalues: Vec<JValue>, tetraplets: RcSecurityTetraplets) -> Self {
-        // TODO: add assert on length
         Self {
-            jvalues,
+            canon_stream,
-            tetraplets,
             cursor: 0,
         }
     }
 }
 
-impl<'ctx> Iterable<'ctx> for IterableVecJsonPathResult {
+impl<'ctx> Iterable<'ctx> for CanonStreamIterableIngredients {
     type Item = IterableItem<'ctx>;
 
     fn next(&mut self) -> bool {
-        foldable_next!(self, self.jvalues.len())
+        foldable_next!(self, self.len())
     }
 
     fn prev(&mut self) -> bool {
@@ -53,18 +48,16 @@ impl<'ctx> Iterable<'ctx> for IterableVecJsonPathResult {
     }
 
     fn peek(&'ctx self) -> Option<Self::Item> {
-        if self.jvalues.is_empty() {
+        if self.canon_stream.is_empty() {
             return None;
         }
 
-        let jvalue = &self.jvalues[self.cursor];
+        let value = self.canon_stream.nth(self.cursor).expect(EXPECT_VALUE_IN_STREAM);
-        let tetraplet = &self.tetraplets[self.cursor];
+        let result = IterableItem::RefRef((&value.result, &value.tetraplet, value.trace_pos));
-        let result = IterableItem::RefRef((jvalue, tetraplet, 0.into()));
-
         Some(result)
     }
 
     fn len(&self) -> usize {
-        self.jvalues.len()
+        self.canon_stream.len()
     }
 }

air/src/execution_step/boxed_value/jvaluable.rs

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+mod canon_stream;
 mod cell_vec_resolved_call_result;
 mod empty_stream;
 mod iterable_item;

air/src/execution_step/boxed_value/jvaluable/canon_stream.rs

@@ -0,0 +1,68 @@
+/*
+ * Copyright 2022 Fluence Labs Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+use super::select_from_stream;
+use super::ExecutionResult;
+use super::JValuable;
+use crate::execution_step::boxed_value::CanonStream;
+use crate::execution_step::ExecutionCtx;
+use crate::execution_step::RcSecurityTetraplets;
+use crate::JValue;
+use crate::LambdaAST;
+use crate::SecurityTetraplet;
+
+use air_lambda_ast::format_ast;
+
+use std::borrow::Cow;
+use std::ops::Deref;
+
+impl JValuable for &CanonStream {
+    fn apply_lambda<'i>(&self, lambda: &LambdaAST<'_>, exec_ctx: &ExecutionCtx<'i>) -> ExecutionResult<&JValue> {
+        let iter = self.iter().map(|v| v.result.deref());
+        let select_result = select_from_stream(iter, lambda, exec_ctx)?;
+
+        Ok(select_result.result)
+    }
+
+    fn apply_lambda_with_tetraplets<'i>(
+        &self,
+        lambda: &LambdaAST<'_>,
+        exec_ctx: &ExecutionCtx<'i>,
+    ) -> ExecutionResult<(&JValue, SecurityTetraplet)> {
+        let iter = self.iter().map(|v| v.result.deref());
+        let select_result = select_from_stream(iter, lambda, exec_ctx)?;
+
+        // unwrap is safe here because each value has a tetraplet and a lambda always returns a valid index
+        let resolved_call = self.nth(select_result.tetraplet_idx).unwrap();
+        let mut tetraplet = resolved_call.tetraplet.as_ref().clone();
+        tetraplet.add_lambda(&format_ast(lambda));
+
+        Ok((select_result.result, tetraplet))
+    }
+
+    fn as_jvalue(&self) -> Cow<'_, JValue> {
+        let jvalue = CanonStream::as_jvalue(self);
+        Cow::Owned(jvalue)
+    }
+
+    fn into_jvalue(self: Box<Self>) -> JValue {
+        CanonStream::as_jvalue(&self)
+    }
+
+    fn as_tetraplets(&self) -> RcSecurityTetraplets {
+        self.iter().map(|r| r.tetraplet.clone()).collect::<Vec<_>>()
+    }
+}

air/src/execution_step/boxed_value/mod.rs

@@ -14,12 +14,14 @@
  * limitations under the License.
  */
 
+mod canon_stream;
 mod iterable;
 mod jvaluable;
 mod scalar;
 mod stream;
 mod variable;
 
+pub(crate) use canon_stream::*;
 pub(crate) use iterable::*;
 pub(crate) use jvaluable::*;
 pub(crate) use scalar::ScalarRef;

air/src/execution_step/boxed_value/scalar.rs

@@ -23,8 +23,6 @@ use air_interpreter_data::TracePos;
 use serde::Deserialize;
 use serde::Serialize;
 
-use std::fmt::Display;
-use std::fmt::Formatter;
 use std::rc::Rc;
 
 #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
@@ -61,8 +59,20 @@ impl ValueAggregate {
     }
 }
 
-impl<'i> Display for ScalarRef<'i> {
+use std::fmt;
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+
+impl fmt::Display for ValueAggregate {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "value: {}, tetraplet: {}, position: {} ",
+            self.result, self.tetraplet, self.trace_pos
+        )
+    }
+}
+
+impl<'i> fmt::Display for ScalarRef<'i> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
             ScalarRef::Value(value) => write!(f, "{:?}", value)?,
             ScalarRef::IterableValue(cursor) => {

air/src/execution_step/boxed_value/stream.rs

@@ -19,70 +19,90 @@ use super::ValueAggregate;
 use crate::execution_step::CatchableError;
 use crate::JValue;
 
+use air_interpreter_data::TracePos;
+
+use std::collections::HashMap;
 use std::fmt::Formatter;
 
 /// Streams are CRDT-like append only data structures. They are guaranteed to have the same order
 /// of values on each peer.
-///
-/// The first Vec represents generations, the second values in a generation. Generation is a set
-/// of values that interpreter obtained from one particle. It means that number of generation on
-/// a peer is equal to number of the interpreter runs in context of one particle. And each set of
-/// obtained values from a current_data that were not present in prev_data becomes a new generation.
 #[derive(Debug, Default, Clone)]
-pub struct Stream(Vec<Vec<ValueAggregate>>);
+pub struct Stream {
+    /// The first Vec represents generations, the second values in a generation. Generation is a set
+    /// of values that interpreter obtained from one particle. It means that number of generation on
+    /// a peer is equal to number of the interpreter runs in context of one particle. And each set of
+    /// obtained values from a current_data that were not present in prev_data becomes a new generation.
+    values: Vec<Vec<ValueAggregate>>,
+
+    /// This map is intended to support canonicalized stream creation, such streams has
+    /// corresponding value positions in a data and this field are used to create such streams.
+    values_by_pos: HashMap<TracePos, StreamValueLocation>,
+}
 
 impl Stream {
     pub(crate) fn from_generations_count(count: usize) -> Self {
-        Self(vec![vec![]; count + 1])
+        Self {
+            values: vec![vec![]; count + 1],
+            values_by_pos: HashMap::new(),
+        }
     }
 
     pub(crate) fn from_value(value: ValueAggregate) -> Self {
-        Self(vec![vec![value]])
+        let values_by_pos = maplit::hashmap! {
+            value.trace_pos => StreamValueLocation::new(0, 0),
+        };
+        Self {
+            values: vec![vec![value]],
+            values_by_pos,
+        }
     }
 
     // if generation is None, value would be added to the last generation, otherwise it would
     // be added to given generation
     pub(crate) fn add_value(&mut self, value: ValueAggregate, generation: Generation) -> ExecutionResult<u32> {
         let generation = match generation {
-            Generation::Last => self.0.len() - 1,
+            Generation::Last => self.values.len() - 1,
             Generation::Nth(id) => id as usize,
         };
 
-        if generation >= self.0.len() {
+        if generation >= self.values.len() {
             return Err(CatchableError::StreamDontHaveSuchGeneration(self.clone(), generation).into());
         }
 
-        self.0[generation].push(value);
+        let values = &mut self.values[generation];
+        self.values_by_pos
+            .insert(value.trace_pos, StreamValueLocation::new(generation, values.len()));
+        values.push(value);
         Ok(generation as u32)
     }
 
     pub(crate) fn generations_count(&self) -> usize {
         // the last generation could be empty due to the logic of from_generations_count ctor
-        self.0.iter().filter(|gen| !gen.is_empty()).count()
+        self.values.iter().filter(|gen| !gen.is_empty()).count()
     }
 
     /// Add a new empty generation if the latest isn't empty.
     pub(crate) fn add_new_generation_if_non_empty(&mut self) -> bool {
-        let should_add_generation = match self.0.last() {
+        let should_add_generation = match self.values.last() {
             Some(last) => !last.is_empty(),
             None => true,
         };
 
         if should_add_generation {
-            self.0.push(vec![]);
+            self.values.push(vec![]);
         }
         should_add_generation
     }
 
     /// Remove a last generation if it's empty.
     pub(crate) fn remove_last_generation_if_empty(&mut self) -> bool {
-        let should_remove_generation = match self.0.last() {
+        let should_remove_generation = match self.values.last() {
             Some(last) => last.is_empty(),
             None => false,
         };
 
         if should_remove_generation {
-            self.0.pop();
+            self.values.pop();
         }
 
         should_remove_generation
@@ -91,17 +111,17 @@ impl Stream {
     pub(crate) fn elements_count(&self, generation: Generation) -> Option<usize> {
         match generation {
             Generation::Nth(generation) if generation as usize > self.generations_count() => None,
-            Generation::Nth(generation) => Some(self.0.iter().take(generation as usize).map(|v| v.len()).sum()),
+            Generation::Nth(generation) => Some(self.values.iter().take(generation as usize).map(|v| v.len()).sum()),
-            Generation::Last => Some(self.0.iter().map(|v| v.len()).sum()),
+            Generation::Last => Some(self.values.iter().map(|v| v.len()).sum()),
         }
     }
 
     pub(crate) fn is_empty(&self) -> bool {
-        if self.0.is_empty() {
+        if self.values.is_empty() {
             return false;
         }
 
-        self.0.iter().all(|v| v.is_empty())
+        self.values.iter().all(|v| v.is_empty())
     }
 
     pub(crate) fn as_jvalue(&self, generation: Generation) -> Option<JValue> {
@@ -113,11 +133,22 @@ impl Stream {
         Some(JValue::Array(jvalue_array))
     }
 
+    pub(crate) fn get_value_by_pos(&self, position: TracePos) -> Option<&ValueAggregate> {
+        let StreamValueLocation {
+            generation,
+            position_in_generation,
+        } = self.values_by_pos.get(&position)?;
+        let value = &self.values[*generation][*position_in_generation];
+        Some(value)
+    }
+
     pub(crate) fn iter(&self, generation: Generation) -> Option<StreamIter<'_>> {
         let iter: Box<dyn Iterator<Item = &ValueAggregate>> = match generation {
             Generation::Nth(generation) if generation as usize >= self.generations_count() => return None,
-            Generation::Nth(generation) => Box::new(self.0.iter().take(generation as usize + 1).flat_map(|v| v.iter())),
+            Generation::Nth(generation) => {
-            Generation::Last => Box::new(self.0.iter().flat_map(|v| v.iter())),
+                Box::new(self.values.iter().take(generation as usize + 1).flat_map(|v| v.iter()))
+            }
+            Generation::Last => Box::new(self.values.iter().flat_map(|v| v.iter())),
         };
         // unwrap is safe here, because generation's been already checked
         let len = self.elements_count(generation).unwrap();
@@ -146,7 +177,7 @@ impl Stream {
 
         let len = (end - start) as usize + 1;
         let iter: Box<dyn Iterator<Item = &[ValueAggregate]>> =
-            Box::new(self.0.iter().skip(start as usize).take(len).map(|v| v.as_slice()));
+            Box::new(self.values.iter().skip(start as usize).take(len).map(|v| v.as_slice()));
         let iter = StreamSliceIter { iter, len };
 
         Some(iter)
@@ -210,16 +241,31 @@ impl<'slice> Iterator for StreamSliceIter<'slice> {
     }
 }
 
+#[derive(Clone, Copy, Debug, Default)]
+struct StreamValueLocation {
+    pub generation: usize,
+    pub position_in_generation: usize,
+}
+
+impl StreamValueLocation {
+    pub(super) fn new(generation: usize, position_in_generation: usize) -> Self {
+        Self {
+            generation,
+            position_in_generation,
+        }
+    }
+}
+
 use std::fmt;
 
 impl fmt::Display for Stream {
     fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
-        if self.0.is_empty() {
+        if self.values.is_empty() {
             return write!(f, "[]");
         }
 
         writeln!(f, "[")?;
-        for (id, generation) in self.0.iter().enumerate() {
+        for (id, generation) in self.values.iter().enumerate() {
             write!(f, " -- {}: ", id)?;
             for value in generation.iter() {
                 write!(f, "{:?}, ", value)?;

air/src/execution_step/boxed_value/variable.rs

@@ -19,19 +19,22 @@ use air_parser::ast;
 
 #[derive(Clone, Copy, Debug)]
 pub(crate) enum Variable<'i> {
-    #[allow(dead_code)]
+    Scalar {
-    // position will be needed to implement new for operators
+        name: &'i str,
-    Scalar { name: &'i str, position: usize },
+    },
     Stream {
         name: &'i str,
         generation: Generation,
         position: usize,
     },
+    CanonStream {
+        name: &'i str,
+    },
 }
 
 impl<'i> Variable<'i> {
-    pub(crate) fn scalar(name: &'i str, position: usize) -> Self {
+    pub(crate) fn scalar(name: &'i str) -> Self {
-        Self::Scalar { name, position }
+        Self::Scalar { name }
     }
 
     pub(crate) fn stream(name: &'i str, generation: Generation, position: usize) -> Self {
@@ -41,6 +44,10 @@ impl<'i> Variable<'i> {
             position,
         }
     }
+
+    pub(crate) fn canon_stream(name: &'i str) -> Self {
+        Self::CanonStream { name }
+    }
 }
 
 impl<'i> From<&ast::Variable<'i>> for Variable<'i> {
@@ -48,8 +55,9 @@ impl<'i> From<&ast::Variable<'i>> for Variable<'i> {
         use ast::Variable::*;
 
         match ast_variable {
-            Scalar(scalar) => Self::scalar(scalar.name, scalar.position),
+            Scalar(scalar) => Self::scalar(scalar.name),
             Stream(stream) => Self::stream(stream.name, Generation::Last, stream.position),
+            CanonStream(canon_stream) => Self::canon_stream(canon_stream.name),
         }
     }
 }
@@ -59,8 +67,9 @@ impl<'i> From<&ast::VariableWithLambda<'i>> for Variable<'i> {
         use ast::VariableWithLambda::*;
 
         match ast_variable {
-            Scalar(scalar) => Self::scalar(scalar.name, scalar.position),
+            Scalar(scalar) => Self::scalar(scalar.name),
             Stream(stream) => Self::stream(stream.name, Generation::Last, stream.position),
+            CanonStream(canon_stream) => Self::canon_stream(canon_stream.name),
         }
     }
 }

air/src/execution_step/errors/catchable_errors.rs

@@ -87,6 +87,10 @@ pub enum CatchableError {
     /// that is prohibited.
     #[error("variable with name '{0}' was cleared by new and then wasn't set")]
     VariableWasNotInitializedAfterNew(String),
+
+    /// Canon instruction can't canonicalize a stream since it's been found.
+    #[error("stream with name {0} wasn't defined, so canon instruction can't canonicalize it")]
+    StreamsForCanonNotFound(String),
 }
 
 impl From<LambdaError> for Rc<CatchableError> {

air/src/execution_step/errors/uncatchable_errors.rs

@@ -16,6 +16,7 @@
 
 use crate::ToErrorCode;
 
+use air_interpreter_data::TracePos;
 use air_trace_handler::MergerApResult;
 use air_trace_handler::TraceHandlerError;
 use strum::IntoEnumIterator;
@@ -65,6 +66,12 @@ pub enum UncatchableError {
     /// be caught by a xor instruction.
     #[error("new end block tries to pop up a variable '{scalar_name}' that wasn't defined at depth {depth}")]
     ScalarsStateCorrupted { scalar_name: String, depth: usize },
+
+    /// Variable with such a position wasn't defined during AIR script execution.
+    /// Canon instruction requires this value to be present in data, otherwise it's considered
+    /// as a hard error.
+    #[error("variable with position '{0}' wasn't defined during script execution")]
+    VariableNotFoundByPos(TracePos),
 }
 
 impl ToErrorCode for UncatchableError {

air/src/execution_step/execution_context/scalar_variables.rs

@@ -14,20 +14,17 @@
  * limitations under the License.
  */
 
+mod values_sparse_matrix;
+
+use crate::execution_step::boxed_value::CanonStream;
 use crate::execution_step::boxed_value::ScalarRef;
 use crate::execution_step::errors_prelude::*;
 use crate::execution_step::ExecutionResult;
 use crate::execution_step::FoldState;
 use crate::execution_step::ValueAggregate;
-
+use values_sparse_matrix::ValuesSparseMatrix;
-use non_empty_vec::NonEmpty;
 
 use std::collections::HashMap;
-use std::collections::HashSet;
-use std::rc::Rc;
-
-/// Depth of a global scope.
-const GLOBAL_DEPTH: usize = 0;
 
 // TODO: move this code snippet to documentation when it's ready
 
@@ -87,85 +84,32 @@ pub(crate) struct Scalars<'i> {
     ///   - global variables have 0 depth
     ///   - cells in a row are sorted by depth
     ///   - all depths in cell in one row are unique
-    pub(crate) non_iterable_variables: HashMap<String, NonEmpty<SparseCell>>,
+    pub(crate) non_iterable_variables: ValuesSparseMatrix<ValueAggregate>,
 
-    /// This set contains depths were invalidated at the certain moment of script execution.
+    pub(crate) canon_streams: ValuesSparseMatrix<CanonStream>,
-    /// They are needed for careful isolation of scopes produced by iterations in fold blocks,
-    /// precisely to limit access of non iterable variables defined on one depths to ones
-    /// defined on another.
-    pub(crate) allowed_depths: HashSet<usize>,
 
     pub(crate) iterable_variables: HashMap<String, FoldState<'i>>,
-
-    /// Count of met scopes at the particular moment of execution.
-    pub(crate) current_depth: usize,
-}
-
-#[derive(Debug)]
-pub(crate) struct SparseCell {
-    /// Scope depth where the value was set.
-    pub(crate) depth: usize,
-    pub(crate) value: Option<ValueAggregate>,
-}
-
-impl SparseCell {
-    pub(crate) fn from_value(depth: usize, value: ValueAggregate) -> Self {
-        Self {
-            depth,
-            value: Some(value),
-        }
-    }
-
-    pub(crate) fn from_met_new(depth: usize) -> Self {
-        Self { depth, value: None }
-    }
 }
 
 impl<'i> Scalars<'i> {
     pub fn new() -> Self {
-        let allowed_depths = maplit::hashset! { GLOBAL_DEPTH };
-
         Self {
-            non_iterable_variables: HashMap::new(),
+            non_iterable_variables: ValuesSparseMatrix::new(),
-            allowed_depths,
+            canon_streams: ValuesSparseMatrix::new(),
             iterable_variables: HashMap::new(),
-            current_depth: GLOBAL_DEPTH,
         }
     }
 
     /// Returns true if there was a previous value for the provided key on the same
     /// fold block.
-    pub(crate) fn set_value(&mut self, name: impl Into<String>, value: ValueAggregate) -> ExecutionResult<bool> {
+    pub(crate) fn set_scalar_value(&mut self, name: impl Into<String>, value: ValueAggregate) -> ExecutionResult<bool> {
-        use std::collections::hash_map::Entry::{Occupied, Vacant};
+        self.non_iterable_variables.set_value(name, value)
+    }
 
-        let name = name.into();
+    /// Returns true if there was a previous value for the provided key on the same
-        let variable_could_be_set = self.variable_could_be_set(&name);
+    /// fold block.
-        match self.non_iterable_variables.entry(name) {
+    pub(crate) fn set_canon_value(&mut self, name: impl Into<String>, value: CanonStream) -> ExecutionResult<bool> {
-            Vacant(entry) => {
+        self.canon_streams.set_value(name, value)
-                let cell = SparseCell::from_value(self.current_depth, value);
-                let cells = NonEmpty::new(cell);
-                entry.insert(cells);
-
-                Ok(false)
-            }
-            Occupied(entry) => {
-                if !variable_could_be_set {
-                    return Err(UncatchableError::ShadowingIsNotAllowed(entry.key().clone()).into());
-                }
-
-                let values = entry.into_mut();
-                let last_cell = values.last_mut();
-                if last_cell.depth == self.current_depth {
-                    // just rewrite a value if fold level is the same
-                    last_cell.value = Some(value);
-                    Ok(true)
-                } else {
-                    let new_cell = SparseCell::from_value(self.current_depth, value);
-                    values.push(new_cell);
-                    Ok(false)
-                }
-            }
-        }
     }
 
     pub(crate) fn set_iterable_value(
@@ -188,20 +132,8 @@ impl<'i> Scalars<'i> {
         self.iterable_variables.remove(name);
     }
 
-    pub(crate) fn get_non_iterable_value(&'i self, name: &str) -> ExecutionResult<Option<&'i ValueAggregate>> {
+    pub(crate) fn get_non_iterable_scalar(&'i self, name: &str) -> ExecutionResult<Option<&'i ValueAggregate>> {
-        self.non_iterable_variables
+        self.non_iterable_variables.get_value(name)
-            .get(name)
-            .and_then(|values| {
-                let last_cell = values.last();
-                let depth_allowed = self.allowed_depths.contains(&last_cell.depth);
-
-                if depth_allowed {
-                    Some(last_cell.value.as_ref())
-                } else {
-                    None
-                }
-            })
-            .ok_or_else(|| ExecutionError::Catchable(Rc::new(CatchableError::VariableNotFound(name.to_string()))))
     }
 
     pub(crate) fn get_iterable_mut(&mut self, name: &str) -> ExecutionResult<&mut FoldState<'i>> {
@@ -210,8 +142,14 @@ impl<'i> Scalars<'i> {
             .ok_or_else(|| UncatchableError::FoldStateNotFound(name.to_string()).into())
     }
 
+    pub(crate) fn get_canon_stream(&'i self, name: &str) -> ExecutionResult<&'i CanonStream> {
+        self.canon_streams
+            .get_value(name)?
+            .ok_or_else(|| CatchableError::VariableWasNotInitializedAfterNew(name.to_string()).into())
+    }
+
     pub(crate) fn get_value(&'i self, name: &str) -> ExecutionResult<ScalarRef<'i>> {
-        let value = self.get_non_iterable_value(name);
+        let value = self.get_non_iterable_scalar(name);
         let iterable_value = self.iterable_variables.get(name);
 
         match (value, iterable_value) {
@@ -223,141 +161,62 @@ impl<'i> Scalars<'i> {
         }
     }
 
+    pub(crate) fn variable_could_be_set(&self, variable_name: &str) -> bool {
+        self.non_iterable_variables.variable_could_be_set(variable_name)
+            || self.canon_streams.variable_could_be_set(variable_name)
+    }
+
     pub(crate) fn meet_fold_start(&mut self) {
-        self.current_depth += 1;
+        self.non_iterable_variables.meet_fold_start();
-        self.allowed_depths.insert(self.current_depth);
+        self.canon_streams.meet_fold_start();
     }
 
     // meet next before recursion
     pub(crate) fn meet_next_before(&mut self) {
-        self.allowed_depths.remove(&self.current_depth);
+        self.non_iterable_variables.meet_next_before();
-        self.current_depth += 1;
+        self.canon_streams.meet_next_before();
-        self.allowed_depths.insert(self.current_depth);
     }
 
     // meet next after recursion
     pub(crate) fn meet_next_after(&mut self) {
-        self.allowed_depths.remove(&self.current_depth);
+        self.non_iterable_variables.meet_next_after();
-        self.current_depth -= 1;
+        self.canon_streams.meet_next_after();
-        self.allowed_depths.insert(self.current_depth);
-
-        self.cleanup_obsolete_values();
     }
 
     pub(crate) fn meet_fold_end(&mut self) {
-        self.allowed_depths.remove(&self.current_depth);
+        self.non_iterable_variables.meet_fold_end();
-        self.current_depth -= 1;
+        self.canon_streams.meet_fold_end();
-        self.cleanup_obsolete_values();
     }
 
-    pub(crate) fn meet_new_start(&mut self, scalar_name: &str) {
+    pub(crate) fn meet_new_start_scalar(&mut self, scalar_name: String) {
-        use std::collections::hash_map::Entry::{Occupied, Vacant};
+        self.non_iterable_variables.meet_new_start(scalar_name);
-
-        let new_cell = SparseCell::from_met_new(self.current_depth);
-        match self.non_iterable_variables.entry(scalar_name.to_string()) {
-            Vacant(entry) => {
-                let ne_vec = NonEmpty::new(new_cell);
-                entry.insert(ne_vec);
-            }
-            Occupied(entry) => {
-                let entry = entry.into_mut();
-                entry.push(new_cell);
-            }
-        }
     }
 
-    pub(crate) fn meet_new_end(&mut self, scalar_name: &str) -> ExecutionResult<()> {
+    pub(crate) fn meet_new_start_canon_stream(&mut self, canon_stream_name: String) {
-        let current_depth = self.current_depth;
+        self.canon_streams.meet_new_start(canon_stream_name);
-        let should_remove_values = self
-            .non_iterable_variables
-            .get_mut(scalar_name)
-            .and_then(|values| {
-                // carefully check that we're popping up an appropriate value,
-                // returning None means an error here
-                match values.pop() {
-                    Some(value) if value.depth == current_depth => Some(false),
-                    Some(_) => None,
-                    // None means that the value was last in a row
-                    None if values.last().depth == current_depth => Some(true),
-                    None => None,
-                }
-            })
-            .ok_or_else(|| UncatchableError::ScalarsStateCorrupted {
-                scalar_name: scalar_name.to_string(),
-                depth: self.current_depth,
-            })
-            .map_err(Into::<ExecutionError>::into)?;
-
-        if should_remove_values {
-            self.non_iterable_variables.remove(scalar_name);
-        }
-        Ok(())
     }
 
-    pub(crate) fn variable_could_be_set(&self, variable_name: &str) -> bool {
+    pub(crate) fn meet_new_end_scalar(&mut self, scalar_name: &str) -> ExecutionResult<()> {
-        if self.shadowing_allowed() {
+        self.non_iterable_variables.meet_new_end(scalar_name)
-            return true;
-        }
-
-        match self.non_iterable_variables.get(variable_name) {
-            Some(values) => values.last().value.is_none(),
-            None => false,
-        }
     }
 
-    pub(crate) fn shadowing_allowed(&self) -> bool {
+    pub(crate) fn meet_new_end_canon_stream(&mut self, canon_name: &str) -> ExecutionResult<()> {
-        // shadowing is allowed only inside a fold block, 0 here means that execution flow
+        self.canon_streams.meet_new_end(canon_name)
-        // is in a global scope
-        self.current_depth != 0
-    }
-
-    fn cleanup_obsolete_values(&mut self) {
-        // TODO: it takes O(N) where N is a count of all scalars, but it could be optimized
-        // by maintaining array of value indices that should be removed on each depth level
-        let mut values_to_delete = Vec::new();
-        for (name, values) in self.non_iterable_variables.iter_mut() {
-            let value_depth = values.last().depth;
-            if !is_global_value(value_depth) && is_value_obsolete(value_depth, self.current_depth) {
-                // it can't be empty, so it returns None if it contains 1 element
-                if values.pop().is_none() {
-                    // TODO: optimize this cloning in next PR
-                    values_to_delete.push(name.to_string());
-                }
-            }
-        }
-
-        for value_name in values_to_delete {
-            self.non_iterable_variables.remove(&value_name);
-        }
     }
 }
 
-fn is_global_value(value_depth: usize) -> bool {
-    value_depth == GLOBAL_DEPTH
-}
-
-fn is_value_obsolete(value_depth: usize, current_scope_depth: usize) -> bool {
-    value_depth > current_scope_depth
-}
-
-use std::fmt;
-
 impl Default for Scalars<'_> {
     fn default() -> Self {
         Scalars::new()
     }
 }
 
+use std::fmt;
+
 impl<'i> fmt::Display for Scalars<'i> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        writeln!(f, "fold_block_id: {}", self.current_depth)?;
+        writeln!(f, "scalars:\n{}", self.non_iterable_variables)?;
-
+        writeln!(f, "canon_streams:\n{}", self.canon_streams)?;
-        for (name, _) in self.non_iterable_variables.iter() {
-            let value = self.get_non_iterable_value(name);
-            if let Ok(Some(last_value)) = value {
-                writeln!(f, "{} => {}", name, last_value.result)?;
-            }
-        }
 
         for (name, _) in self.iterable_variables.iter() {
             // it's impossible to print an iterable value for now
@@ -367,46 +226,3 @@ impl<'i> fmt::Display for Scalars<'i> {
         Ok(())
     }
 }
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use polyplets::SecurityTetraplet;
-
-    use serde_json::json;
-
-    use std::num::NonZeroUsize;
-    use std::rc::Rc;
-
-    #[test]
-    fn test_local_cleanup() {
-        let mut scalars = Scalars::default();
-
-        let tetraplet = SecurityTetraplet::default();
-        let rc_tetraplet = Rc::new(tetraplet);
-        let value = json!(1u64);
-        let rc_value = Rc::new(value);
-        let value_aggregate = ValueAggregate::new(rc_value, rc_tetraplet, 1.into());
-        let value_1_name = "name_1";
-        scalars.set_value(value_1_name, value_aggregate.clone()).unwrap();
-
-        let value_2_name = "name_2";
-        scalars.meet_fold_start();
-        scalars.set_value(value_2_name, value_aggregate.clone()).unwrap();
-        scalars.meet_fold_start();
-        scalars.set_value(value_2_name, value_aggregate.clone()).unwrap();
-
-        let expected_values_count = scalars.non_iterable_variables.get(value_2_name).unwrap().len();
-        assert_eq!(expected_values_count, NonZeroUsize::new(2).unwrap());
-
-        scalars.meet_fold_end();
-        let expected_values_count = scalars.non_iterable_variables.get(value_2_name).unwrap().len();
-        assert_eq!(expected_values_count, NonZeroUsize::new(1).unwrap());
-
-        scalars.meet_fold_end();
-        assert!(scalars.non_iterable_variables.get(value_2_name).is_none());
-
-        let expected_values_count = scalars.non_iterable_variables.get(value_1_name).unwrap().len();
-        assert_eq!(expected_values_count, NonZeroUsize::new(1).unwrap());
-    }
-}

air/src/execution_step/execution_context/scalar_variables/traits.rs

@@ -1,39 +0,0 @@
-/*
- * Copyright 2021 Fluence Labs Limited
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-use super::Scalars;
-
-use std::fmt;
-
-impl<'i> fmt::Display for Scalars<'i> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        writeln!(f, "fold_block_id: {}", self.fold_blocks_sequence.len())?;
-
-        for (name, _) in self.values.iter() {
-            let descriptor = self.values.get(name);
-            if let Some(descriptor) = descriptor {
-                writeln!(f, "{} => {:?}", name, descriptor.values.last())?;
-            }
-        }
-
-        for (name, _) in self.iterable_values.iter() {
-            // it's impossible to print an iterable value for now
-            writeln!(f, "{} => iterable", name)?;
-        }
-
-        Ok(())
-    }
-}

air/src/execution_step/execution_context/scalar_variables/values_sparse_matrix.rs

@@ -0,0 +1,323 @@
+/*
+ * Copyright 2022 Fluence Labs Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+use crate::execution_step::CatchableError;
+use crate::execution_step::ExecutionError;
+use crate::execution_step::ExecutionResult;
+use crate::execution_step::UncatchableError;
+
+use non_empty_vec::NonEmpty;
+
+use std::collections::HashMap;
+use std::collections::HashSet;
+use std::rc::Rc;
+
+/// Depth of a global scope.
+const GLOBAL_DEPTH: usize = 0;
+
+pub(crate) struct ValuesSparseMatrix<T> {
+    cells: HashMap<String, NonEmpty<SparseCell<T>>>,
+
+    /// This set contains depths were invalidated at the certain moment of script execution.
+    /// They are needed for careful isolation of scopes produced by iterations in fold blocks,
+    /// precisely to limit access of non iterable variables defined on one depths to ones
+    /// defined on another.
+    allowed_depths: HashSet<usize>,
+
+    /// Count of met scopes at the particular moment of execution.
+    current_depth: usize,
+}
+
+impl<T> ValuesSparseMatrix<T> {
+    pub(super) fn new() -> Self {
+        let allowed_depths = maplit::hashset! { GLOBAL_DEPTH };
+
+        Self {
+            cells: HashMap::new(),
+            allowed_depths,
+            current_depth: GLOBAL_DEPTH,
+        }
+    }
+
+    pub(super) fn set_value(&mut self, name: impl Into<String>, value: T) -> ExecutionResult<bool> {
+        use std::collections::hash_map::Entry::{Occupied, Vacant};
+
+        let name = name.into();
+        let variable_could_be_set = self.variable_could_be_set(&name);
+        match self.cells.entry(name) {
+            Vacant(entry) => {
+                let cell = SparseCell::from_value(self.current_depth, value);
+                let cells = NonEmpty::new(cell);
+                entry.insert(cells);
+
+                Ok(false)
+            }
+            Occupied(entry) => {
+                if !variable_could_be_set {
+                    return Err(UncatchableError::ShadowingIsNotAllowed(entry.key().clone()).into());
+                }
+
+                let values = entry.into_mut();
+                let last_cell = values.last_mut();
+                if last_cell.depth == self.current_depth {
+                    // just rewrite a value if fold level is the same
+                    last_cell.value = Some(value);
+                    Ok(true)
+                } else {
+                    let new_cell = SparseCell::from_value(self.current_depth, value);
+                    values.push(new_cell);
+                    Ok(false)
+                }
+            }
+        }
+    }
+
+    pub(super) fn get_value(&self, name: &str) -> ExecutionResult<Option<&T>> {
+        self.cells
+            .get(name)
+            .and_then(|values| {
+                let last_cell = values.last();
+                let depth_allowed = self.allowed_depths.contains(&last_cell.depth);
+
+                if depth_allowed {
+                    Some(last_cell.value.as_ref())
+                } else {
+                    None
+                }
+            })
+            .ok_or_else(|| ExecutionError::Catchable(Rc::new(CatchableError::VariableNotFound(name.to_string()))))
+    }
+
+    pub(super) fn meet_fold_start(&mut self) {
+        self.current_depth += 1;
+        self.allowed_depths.insert(self.current_depth);
+    }
+
+    // meet next before recursion
+    pub(super) fn meet_next_before(&mut self) {
+        self.allowed_depths.remove(&self.current_depth);
+        self.current_depth += 1;
+        self.allowed_depths.insert(self.current_depth);
+    }
+
+    // meet next after recursion
+    pub(super) fn meet_next_after(&mut self) {
+        self.allowed_depths.remove(&self.current_depth);
+        self.current_depth -= 1;
+        self.allowed_depths.insert(self.current_depth);
+
+        self.cleanup_obsolete_values();
+    }
+
+    pub(super) fn meet_fold_end(&mut self) {
+        self.allowed_depths.remove(&self.current_depth);
+        self.current_depth -= 1;
+        self.cleanup_obsolete_values();
+    }
+
+    pub(super) fn meet_new_start(&mut self, scalar_name: String) {
+        use std::collections::hash_map::Entry::{Occupied, Vacant};
+
+        let new_cell = SparseCell::from_met_new(self.current_depth);
+        match self.cells.entry(scalar_name) {
+            Vacant(entry) => {
+                let ne_vec = NonEmpty::new(new_cell);
+                entry.insert(ne_vec);
+            }
+            Occupied(entry) => {
+                let entry = entry.into_mut();
+                entry.push(new_cell);
+            }
+        }
+    }
+
+    pub(super) fn meet_new_end(&mut self, scalar_name: &str) -> ExecutionResult<()> {
+        let current_depth = self.current_depth;
+        let should_remove_values = self
+            .cells
+            .get_mut(scalar_name)
+            .and_then(|values| {
+                // carefully check that we're popping up an appropriate value,
+                // returning None means an error here
+                match values.pop() {
+                    Some(value) if value.depth == current_depth => Some(false),
+                    Some(_) => None,
+                    // None means that the value was last in a row
+                    None if values.last().depth == current_depth => Some(true),
+                    None => None,
+                }
+            })
+            .ok_or_else(|| UncatchableError::ScalarsStateCorrupted {
+                scalar_name: scalar_name.to_string(),
+                depth: self.current_depth,
+            })
+            .map_err(Into::<ExecutionError>::into)?;
+
+        if should_remove_values {
+            self.cells.remove(scalar_name);
+        }
+        Ok(())
+    }
+
+    pub(super) fn variable_could_be_set(&self, variable_name: &str) -> bool {
+        if self.shadowing_allowed() {
+            return true;
+        }
+
+        match self.cells.get(variable_name) {
+            Some(values) => values.last().value.is_none(),
+            None => false,
+        }
+    }
+
+    pub(super) fn shadowing_allowed(&self) -> bool {
+        // shadowing is allowed only inside a fold block, 0 here means that execution flow
+        // is in a global scope
+        self.current_depth != 0
+    }
+
+    fn cleanup_obsolete_values(&mut self) {
+        // TODO: it takes O(N) where N is a count of all scalars, but it could be optimized
+        // by maintaining array of value indices that should be removed on each depth level
+        let mut values_to_delete = Vec::new();
+        for (name, values) in self.cells.iter_mut() {
+            let value_depth = values.last().depth;
+            if !is_global_value(value_depth) && is_value_obsolete(value_depth, self.current_depth) {
+                // it can't be empty, so it returns None if it contains 1 element
+                if values.pop().is_none() {
+                    // TODO: optimize this cloning in next PR
+                    values_to_delete.push(name.to_string());
+                }
+            }
+        }
+
+        for value_name in values_to_delete {
+            self.cells.remove(&value_name);
+        }
+    }
+}
+
+impl<T> Default for ValuesSparseMatrix<T> {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+fn is_global_value(value_depth: usize) -> bool {
+    value_depth == GLOBAL_DEPTH
+}
+
+fn is_value_obsolete(value_depth: usize, current_scope_depth: usize) -> bool {
+    value_depth > current_scope_depth
+}
+
+#[derive(Debug)]
+pub(crate) struct SparseCell<T> {
+    /// Scope depth where the value was set.
+    pub(crate) depth: usize,
+    pub(crate) value: Option<T>,
+}
+
+impl<T> SparseCell<T> {
+    pub(crate) fn from_value(depth: usize, value: T) -> Self {
+        Self {
+            depth,
+            value: Some(value),
+        }
+    }
+
+    pub(crate) fn from_met_new(depth: usize) -> Self {
+        Self { depth, value: None }
+    }
+}
+
+use std::fmt;
+
+impl<T> fmt::Display for SparseCell<T>
+where
+    T: fmt::Display,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match &self.value {
+            Some(value) => write!(f, "{}", value),
+            None => write!(f, "none"),
+        }
+    }
+}
+
+impl<T> fmt::Display for ValuesSparseMatrix<T>
+where
+    T: fmt::Display,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        writeln!(f, "current_depth: {}", self.current_depth)?;
+
+        for (name, values) in self.cells.iter() {
+            write!(f, "{}: ", name)?;
+
+            for value in values.iter() {
+                write!(f, "{} ", value)?;
+            }
+            writeln!(f)?;
+        }
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::execution_step::ValueAggregate;
+    use polyplets::SecurityTetraplet;
+
+    use serde_json::json;
+
+    use std::num::NonZeroUsize;
+    use std::rc::Rc;
+
+    #[test]
+    fn test_local_cleanup() {
+        let mut scalars = ValuesSparseMatrix::new();
+
+        let tetraplet = SecurityTetraplet::default();
+        let rc_tetraplet = Rc::new(tetraplet);
+        let value = json!(1u64);
+        let rc_value = Rc::new(value);
+        let value_aggregate = ValueAggregate::new(rc_value, rc_tetraplet, 1.into());
+        let value_1_name = "name_1";
+        scalars.set_value(value_1_name, value_aggregate.clone()).unwrap();
+
+        let value_2_name = "name_2";
+        scalars.meet_fold_start();
+        scalars.set_value(value_2_name, value_aggregate.clone()).unwrap();
+        scalars.meet_fold_start();
+        scalars.set_value(value_2_name, value_aggregate.clone()).unwrap();
+
+        let expected_values_count = scalars.cells.get(value_2_name).unwrap().len();
+        assert_eq!(expected_values_count, NonZeroUsize::new(2).unwrap());
+
+        scalars.meet_fold_end();
+        let expected_values_count = scalars.cells.get(value_2_name).unwrap().len();
+        assert_eq!(expected_values_count, NonZeroUsize::new(1).unwrap());
+
+        scalars.meet_fold_end();
+        assert!(scalars.cells.get(value_2_name).is_none());
+
+        let expected_values_count = scalars.cells.get(value_1_name).unwrap().len();
+        assert_eq!(expected_values_count, NonZeroUsize::new(1).unwrap());
+    }
+}

air/src/execution_step/resolver/resolve.rs

@@ -111,6 +111,10 @@ pub(crate) fn resolve_variable<'ctx, 'i>(
                 None => Ok(Box::new(())),
             }
         }
+        Variable::CanonStream { name, .. } => {
+            let canon_stream = ctx.scalars.get_canon_stream(name)?;
+            Ok(Box::new(canon_stream))
+        }
     }
 }
 

air/tests/test_module/features/join_behaviour/join_behaviour.rs

@@ -246,3 +246,23 @@ fn fold_with_join_behaviour() {
     let trace = trace_from_result(&result);
     assert_eq!(trace.len(), 2);
 }
+
+#[test]
+fn canon_with_join_behaviour() {
+    let peer_1_id = "peer_1_id";
+    let peer_2_id = "peer_2_id";
+
+    let mut peer_2 = create_avm(unit_call_service(), peer_2_id);
+
+    let script = f!(r#"
+        (par
+            (call "{peer_1_id}" ("" "") [] $stream)
+            (canon "{peer_2_id}" $stream #canon_stream))
+    "#);
+
+    let result = checked_call_vm!(peer_2, <_>::default(), script, "", "");
+    let actual_trace = trace_from_result(&result);
+    let expected_trace = vec![executed_state::par(1, 0), executed_state::request_sent_by(peer_2_id)];
+
+    assert_eq!(actual_trace, expected_trace);
+}

air/tests/test_module/instructions/ap.rs

@@ -16,6 +16,8 @@
 
 use air_test_utils::prelude::*;
 
+use std::cell::RefCell;
+
 #[test]
 fn ap_with_scalars() {
     let vm_1_peer_id = "vm_1_peer_id";
@@ -210,3 +212,80 @@ fn ap_with_dst_stream() {
     assert_eq!(actual_trace, expected_state);
     assert!(result.next_peer_pks.is_empty());
 }
+
+#[test]
+fn ap_canon_stream_with_lambda() {
+    let vm_1_peer_id = "vm_1_peer_id";
+    let (echo_call_service, tetraplet_checker) = tetraplet_host_function(echo_call_service());
+    let mut vm_1 = create_avm(echo_call_service, vm_1_peer_id);
+
+    let service_name = "some_service_name";
+    let function_name = "some_function_name";
+    let script = f!(r#"
+        (seq
+            (seq
+                (call "{vm_1_peer_id}" ("" "") [0] $stream)
+                (call "{vm_1_peer_id}" ("{service_name}" "{function_name}") [1] $stream))
+            (seq
+                (canon "{vm_1_peer_id}" $stream #canon_stream)
+                (seq
+                    (ap #canon_stream.$.[1] $stream_2)
+                    (call "{vm_1_peer_id}" ("" "") [$stream_2]))))
+        "#);
+
+    let result = checked_call_vm!(vm_1, <_>::default(), &script, "", "");
+
+    let actual_trace = trace_from_result(&result);
+    let expected_state = vec![
+        executed_state::stream_number(0, 0),
+        executed_state::stream_number(1, 1),
+        executed_state::canon(vec![0.into(), 1.into()]),
+        executed_state::ap(Some(0)),
+        executed_state::scalar(json!([1])),
+    ];
+    assert_eq!(actual_trace, expected_state);
+
+    let expected_tetraplet = RefCell::new(vec![vec![SecurityTetraplet::new(
+        vm_1_peer_id,
+        service_name,
+        function_name,
+        ".[1]",
+    )]]);
+    assert_eq!(tetraplet_checker.as_ref(), &expected_tetraplet);
+}
+
+#[test]
+fn ap_canon_stream() {
+    let vm_1_peer_id = "vm_1_peer_id";
+    let (echo_call_service, tetraplet_checker) = tetraplet_host_function(echo_call_service());
+    let mut vm_1 = create_avm(echo_call_service, vm_1_peer_id);
+
+    let service_name = "some_service_name";
+    let function_name = "some_function_name";
+    let script = f!(r#"
+        (seq
+            (seq
+                (call "{vm_1_peer_id}" ("" "") [0] $stream)
+                (call "{vm_1_peer_id}" ("{service_name}" "{function_name}") [1] $stream))
+            (seq
+                (canon "{vm_1_peer_id}" $stream #canon_stream)
+                (seq
+                    (ap #canon_stream $stream_2)
+                    (call "{vm_1_peer_id}" ("" "") [$stream_2]))))
+        "#);
+
+    let result = checked_call_vm!(vm_1, <_>::default(), &script, "", "");
+
+    let actual_trace = trace_from_result(&result);
+    let expected_state = vec![
+        executed_state::stream_number(0, 0),
+        executed_state::stream_number(1, 1),
+        executed_state::canon(vec![0.into(), 1.into()]),
+        executed_state::ap(Some(0)),
+        executed_state::scalar(json!([[0, 1]])),
+    ];
+    assert_eq!(actual_trace, expected_state);
+
+    let expected_tetraplet = RefCell::new(vec![vec![SecurityTetraplet::new(vm_1_peer_id, "", "", "")]]);
+    assert_eq!(tetraplet_checker.as_ref(), &expected_tetraplet);
+}

air/tests/test_module/instructions/canon.rs

@@ -0,0 +1,235 @@
+/*
+ * Copyright 2022 Fluence Labs Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+use air_test_utils::prelude::*;
+
+use fstrings::f;
+use fstrings::format_args_f;
+
+#[test]
+fn canon_moves_execution_flow() {
+    let mut vm = create_avm(echo_call_service(), "A");
+    let peer_id_1 = "peer_id_1";
+    let peer_id_2 = "peer_id_2";
+
+    let script = f!(r#"
+            (par
+                (call "{peer_id_1}" ("" "") [] $stream)
+                (canon "{peer_id_2}" $stream #canon_stream)
+            )"#);
+
+    let result = checked_call_vm!(vm, <_>::default(), script, "", "");
+
+    assert_next_pks!(&result.next_peer_pks, &[peer_id_1, peer_id_2]);
+}
+
+#[test]
+fn basic_canon() {
+    let mut vm = create_avm(echo_call_service(), "A");
+    let mut set_variable_vm = create_avm(
+        set_variable_call_service(json!(["1", "2", "3", "4", "5"])),
+        "set_variable",
+    );
+
+    let script = r#"
+            (seq
+                (call "set_variable" ("" "") [] Iterable)
+                (seq
+                    (fold Iterable i
+                        (seq
+                            (call "A" ("" "") [i] $stream)
+                            (next i)))
+                    (canon "A" $stream #canon_stream)))
+                    "#;
+
+    let result = checked_call_vm!(set_variable_vm, <_>::default(), script, "", "");
+    let result = checked_call_vm!(vm, <_>::default(), script, "", result.data);
+    let actual_state = &trace_from_result(&result)[6.into()];
+
+    let expected_state = executed_state::canon(vec![1.into(), 2.into(), 3.into(), 4.into(), 5.into()]);
+    assert_eq!(actual_state, &expected_state);
+}
+
+#[test]
+fn canon_fixes_stream_correct() {
+    let peer_id_1 = "peer_id_1";
+    let mut vm_1 = create_avm(echo_call_service(), peer_id_1);
+    let peer_id_2 = "peer_id_2";
+    let mut vm_2 = create_avm(echo_call_service(), peer_id_2);
+    let peer_id_3 = "peer_id_3";
+    let mut vm_3 = create_avm(echo_call_service(), peer_id_3);
+    let peer_id_4 = "peer_id_4";
+    let mut vm_4 = create_avm(echo_call_service(), peer_id_4);
+
+    let script = f!(r#"
+        (seq
+            (par
+                (call "{peer_id_1}" ("" "") [1] $stream)
+                (par
+                     (call "{peer_id_2}" ("" "") [2] $stream)
+                     (call "{peer_id_3}" ("" "") [3] $stream)))
+            (seq
+                (call "{peer_id_4}" ("" "") [4])
+                (seq
+                     (canon "{peer_id_3}" $stream #canon_stream)
+                     (par
+                         (call "{peer_id_3}" ("" "") [#canon_stream])
+                         (call "{peer_id_1}" ("" "") [#canon_stream])))))
+            "#);
+
+    let vm_1_result_1 = checked_call_vm!(vm_1, <_>::default(), &script, "", "");
+    let vm_2_result = checked_call_vm!(vm_2, <_>::default(), &script, "", "");
+    let vm_3_result_1 = checked_call_vm!(vm_3, <_>::default(), &script, "", vm_2_result.data);
+    let vm_4_result = checked_call_vm!(vm_4, <_>::default(), &script, "", vm_3_result_1.data.clone());
+    let vm_3_result_2 = checked_call_vm!(vm_3, <_>::default(), &script, vm_3_result_1.data, vm_4_result.data);
+    let actual_vm_3_result_2_trace = trace_from_result(&vm_3_result_2);
+    let expected_vm_3_result_2_trace = vec![
+        executed_state::par(1, 3),
+        executed_state::request_sent_by(peer_id_2),
+        executed_state::par(1, 1),
+        executed_state::stream_number(2, 0),
+        executed_state::stream_number(3, 1),
+        executed_state::scalar_number(4),
+        executed_state::canon(vec![3.into(), 4.into()]),
+        executed_state::par(1, 1),
+        executed_state::scalar(json!([2, 3])),
+        executed_state::request_sent_by(peer_id_3),
+    ];
+    assert_eq!(actual_vm_3_result_2_trace, expected_vm_3_result_2_trace);
+
+    let vm_1_result_2 = checked_call_vm!(vm_1, <_>::default(), script, vm_1_result_1.data, vm_3_result_2.data);
+    let vm_1_result_2_trace = trace_from_result(&vm_1_result_2);
+    let expected_vm_1_result_2_trace = vec![
+        executed_state::par(1, 3),
+        executed_state::stream_number(1, 0),
+        executed_state::par(1, 1),
+        executed_state::stream_number(2, 1),
+        executed_state::stream_number(3, 1),
+        executed_state::scalar_number(4),
+        executed_state::canon(vec![3.into(), 4.into()]),
+        executed_state::par(1, 1),
+        executed_state::scalar(json!([2, 3])),
+        executed_state::scalar(json!([2, 3])),
+    ];
+    assert_eq!(vm_1_result_2_trace, expected_vm_1_result_2_trace);
+}
+
+#[test]
+fn canon_stream_can_be_created_from_aps() {
+    let vm_1_peer_id = "vm_1_peer_id";
+    let mut vm_1 = create_avm(echo_call_service(), vm_1_peer_id);
+
+    let vm_2_peer_id = "vm_2_peer_id";
+    let mut vm_2 = create_avm(echo_call_service(), vm_2_peer_id);
+
+    let script = f!(r#"
+        (seq
+            (seq
+                (ap 0 $stream)
+                (ap 1 $stream))
+            (seq
+                (canon "{vm_1_peer_id}" $stream #canon_stream)
+                (seq
+                    (ap #canon_stream $stream_2)
+                    (call "{vm_2_peer_id}" ("" "") [$stream_2]))))
+        "#);
+
+    let result_1 = checked_call_vm!(vm_1, <_>::default(), &script, "", "");
+    let result_2 = checked_call_vm!(vm_2, <_>::default(), &script, "", result_1.data.clone());
+    // it fails on this call if canon merger can't handle ap results
+    let _ = checked_call_vm!(vm_2, <_>::default(), &script, result_1.data, result_2.data);
+}
+
+#[test]
+fn canon_gates() {
+    let peer_id_1 = "peer_id_1";
+    let mut vm_1 = create_avm(set_variable_call_service(json!([1, 2, 3, 4, 5])), peer_id_1);
+
+    let peer_id_2 = "peer_id_2";
+    let mut vm_2 = create_avm(echo_call_service(), peer_id_2);
+
+    let peer_id_3 = "peer_id_3";
+    let stop_len_count = 2;
+    let vm_3_call_service: CallServiceClosure = Box::new(move |params: CallRequestParams| -> CallServiceResult {
+        let value = params.arguments[0].as_array().unwrap().len();
+        if value >= stop_len_count {
+            CallServiceResult::ok(json!(true))
+        } else {
+            CallServiceResult::ok(json!(false))
+        }
+    });
+    let mut vm_3 = create_avm(vm_3_call_service, peer_id_3);
+
+    let script = f!(r#"
+        (seq
+          (seq
+            (call "{peer_id_1}" ("" "") [] iterable)
+            (fold iterable iterator
+              (par
+                (call "{peer_id_2}" ("" "") [iterator] $stream)
+                (next iterator))))
+          (new $tmp
+            (fold $stream s
+              (xor
+                (seq
+                  (ap s $tmp)
+                  (seq
+                    (seq
+                      (canon "{peer_id_3}" $tmp #t)
+                      (call "{peer_id_3}" ("" "") [#t] x))
+                    (match x true
+                      (call "{peer_id_3}" ("" "") [#t]))))
+                (next s)))))
+            "#);
+
+    let vm_1_result = checked_call_vm!(vm_1, <_>::default(), &script, "", "");
+    let vm_2_result = checked_call_vm!(vm_2, <_>::default(), &script, "", vm_1_result.data);
+    let vm_3_result = checked_call_vm!(vm_3, <_>::default(), &script, "", vm_2_result.data);
+
+    let actual_trace = trace_from_result(&vm_3_result);
+    let fold = match &actual_trace[11.into()] {
+        ExecutedState::Fold(fold_result) => fold_result,
+        _ => unreachable!(),
+    };
+
+    // fold should stop at the correspond len
+    assert_eq!(fold.lore.len(), stop_len_count);
+}
+
+#[test]
+fn canon_empty_stream() {
+    let peer_id_1 = "peer_id_1";
+    let mut vm_1 = create_avm(echo_call_service(), peer_id_1);
+    let peer_id_2 = "peer_id_2";
+    let mut vm_2 = create_avm(echo_call_service(), peer_id_2);
+
+    let script = f!(r#"
+            (new $stream
+                (seq
+                    (canon "{peer_id_1}" $stream #canon_stream)
+                    (call "{peer_id_1}" ("" "") [#canon_stream])))
+                    "#);
+
+    let result = checked_call_vm!(vm_1, <_>::default(), &script, "", "");
+    let actual_trace = trace_from_result(&result);
+    let expected_trace = vec![executed_state::canon(vec![]), executed_state::scalar(json!([]))];
+    assert_eq!(actual_trace, expected_trace);
+
+    let result = checked_call_vm!(vm_2, <_>::default(), script, "", result.data);
+    let actual_trace = trace_from_result(&result);
+    let expected_trace = vec![executed_state::canon(vec![]), executed_state::scalar(json!([]))];
+    assert_eq!(actual_trace, expected_trace);
+}

air/tests/test_module/instructions/fail.rs

@@ -115,3 +115,34 @@ fn fail_with_literals_tetraplets() {
         SecurityTetraplet::literal_tetraplet(local_peer_id)
     );
 }
+
+#[test]
+fn fail_with_canon_stream() {
+    let vm_peer_id = "local_peer_id";
+    let error_code = 1337i64;
+    let error_message = "error message";
+    let mut vm = create_avm(
+        set_variable_call_service(json!({"error_code": error_code, "message": error_message})),
+        vm_peer_id,
+    );
+
+    let script = f!(r#"
+            (seq
+                (seq
+                    (call "{vm_peer_id}" ("" "") [] $stream)
+                    (canon "{vm_peer_id}" $stream #canon_stream)
+                )
+                (fail #canon_stream.$.[0])
+            )"#);
+
+    let test_params = TestRunParameters::from_init_peer_id("init_peer_id");
+    let result = call_vm!(vm, test_params.clone(), script, "", "");
+
+    let expected_error = CatchableError::UserError {
+        error: rc!(json!( {
+        "error_code": error_code,
+        "message": error_message,
+        })),
+    };
+    assert!(check_error(&result, expected_error));
+}

air/tests/test_module/instructions/mod.rs

@@ -16,6 +16,7 @@
 
 mod ap;
 mod call;
+mod canon;
 mod fail;
 mod fold;
 mod match_;

crates/air-lib/air-parser/src/ast/instruction_arguments.rs

@@ -14,17 +14,22 @@
  * limitations under the License.
  */
 
+mod impls;
 mod traits;
 
-use super::Variable;
+use super::CanonStream;
+use super::Scalar;
+use super::ScalarWithLambda;
+use super::Stream;
 use super::VariableWithLambda;
-use crate::ast::ScalarWithLambda;
 
 use air_lambda_ast::LambdaAST;
 
+use crate::ast::CanonStreamWithLambda;
 use serde::Deserialize;
 use serde::Serialize;
 
+// TODO: rename CallInstrValue, since it'd used by the canon instruction
 #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
 pub enum CallInstrValue<'i> {
     InitPeerId,
@@ -60,7 +65,10 @@ pub enum Value<'i> {
 
 #[derive(Serialize, Debug, PartialEq, Eq, Clone)]
 pub enum CallOutputValue<'i> {
-    Variable(Variable<'i>),
+    #[serde(borrow)]
+    Scalar(Scalar<'i>),
+    #[serde(borrow)]
+    Stream(Stream<'i>),
     None,
 }
 
@@ -75,6 +83,15 @@ pub enum ApArgument<'i> {
     Boolean(bool),
     EmptyArray,
     Scalar(ScalarWithLambda<'i>),
+    CanonStream(CanonStreamWithLambda<'i>),
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
+pub enum ApResult<'i> {
+    #[serde(borrow)]
+    Scalar(Scalar<'i>),
+    #[serde(borrow)]
+    Stream(Stream<'i>),
 }
 
 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
@@ -87,5 +104,18 @@ pub enum Number {
 pub enum FoldScalarIterable<'i> {
     #[serde(borrow)]
     Scalar(ScalarWithLambda<'i>),
+    // it's important not to have lambda here
+    #[serde(borrow)]
+    CanonStream(CanonStream<'i>),
     EmptyArray,
 }
+
+#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
+pub enum NewArgument<'i> {
+    #[serde(borrow)]
+    Scalar(Scalar<'i>),
+    #[serde(borrow)]
+    Stream(Stream<'i>),
+    #[serde(borrow)]
+    CanonStream(CanonStream<'i>),
+}

crates/air-lib/air-parser/src/ast/instruction_arguments/impls.rs

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2022 Fluence Labs Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+use super::ApResult;
+use super::CallOutputValue;
+use super::NewArgument;
+use super::Scalar;
+use super::Stream;
+
+impl<'i> NewArgument<'i> {
+    pub fn name(&self) -> &'i str {
+        match self {
+            Self::Scalar(scalar) => scalar.name,
+            Self::Stream(stream) => stream.name,
+            Self::CanonStream(canon_stream) => canon_stream.name,
+        }
+    }
+}
+
+impl<'i> ApResult<'i> {
+    pub fn scalar(name: &'i str, position: usize) -> Self {
+        Self::Scalar(Scalar { name, position })
+    }
+
+    pub fn stream(name: &'i str, position: usize) -> Self {
+        Self::Stream(Stream { name, position })
+    }
+
+    pub fn name(&self) -> &'i str {
+        match self {
+            Self::Scalar(scalar) => scalar.name,
+            Self::Stream(stream) => stream.name,
+        }
+    }
+}
+
+impl<'i> CallOutputValue<'i> {
+    pub fn scalar(name: &'i str, position: usize) -> Self {
+        Self::Scalar(Scalar { name, position })
+    }
+
+    pub fn stream(name: &'i str, position: usize) -> Self {
+        Self::Stream(Stream { name, position })
+    }
+}

crates/air-lib/air-parser/src/ast/instruction_arguments/traits.rs

@@ -17,6 +17,17 @@
 use super::*;
 use std::fmt;
 
+impl fmt::Display for ApResult<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        use ApResult::*;
+
+        match self {
+            Scalar(scalar) => write!(f, "{}", scalar),
+            Stream(stream) => write!(f, "{}", stream),
+        }
+    }
+}
+
 impl fmt::Display for Value<'_> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         use Value::*;
@@ -52,7 +63,8 @@ impl fmt::Display for CallOutputValue<'_> {
         use CallOutputValue::*;
 
         match self {
-            Variable(variable) => write!(f, "{}", variable),
+            Scalar(scalar) => write!(f, "{}", scalar),
+            Stream(stream) => write!(f, "{}", stream),
             None => Ok(()),
         }
     }
@@ -72,6 +84,7 @@ impl fmt::Display for ApArgument<'_> {
             Boolean(bool) => write!(f, "{}", bool),
             EmptyArray => write!(f, "[]"),
             Scalar(scalar) => write!(f, "{}", scalar),
+            CanonStream(canon_stream) => write!(f, "{}", canon_stream),
         }
     }
 }
@@ -86,6 +99,16 @@ impl fmt::Display for Triplet<'_> {
     }
 }
 
+impl fmt::Display for NewArgument<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Scalar(scalar) => write!(f, "{}", scalar),
+            Self::Stream(stream) => write!(f, "{}", stream),
+            Self::CanonStream(canon_stream) => write!(f, "{}", canon_stream),
+        }
+    }
+}
+
 impl fmt::Display for Number {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         use Number::*;
@@ -103,6 +126,7 @@ impl fmt::Display for FoldScalarIterable<'_> {
 
         match self {
             Scalar(variable) => write!(f, "{}", variable),
+            CanonStream(canon_stream) => write!(f, "{}", canon_stream),
             EmptyArray => write!(f, "[]"),
         }
     }

crates/air-lib/air-parser/src/ast/instructions.rs

@@ -18,6 +18,7 @@ mod impls;
 mod traits;
 
 use super::*;
+use air_lambda_ast::LambdaAST;
 use serde::Serialize;
 use std::rc::Rc;
 
@@ -27,6 +28,7 @@ use std::rc::Rc;
 pub enum Instruction<'i> {
     Call(Call<'i>),
     Ap(Ap<'i>),
+    Canon(Canon<'i>),
     Seq(Seq<'i>),
     Par(Par<'i>),
     Xor(Xor<'i>),
@@ -53,7 +55,15 @@ pub struct Call<'i> {
 #[derive(Serialize, Debug, PartialEq)]
 pub struct Ap<'i> {
     pub argument: ApArgument<'i>,
-    pub result: Variable<'i>,
+    pub result: ApResult<'i>,
+}
+
+/// (canon peer_id $stream #canon_stream)
+#[derive(Serialize, Debug, PartialEq, Eq)]
+pub struct Canon<'i> {
+    pub peer_pk: CallInstrValue<'i>,
+    pub stream: Stream<'i>,
+    pub canon_stream: CanonStream<'i>,
 }
 
 /// (seq instruction instruction)
@@ -93,6 +103,10 @@ pub enum Fail<'i> {
         ret_code: i64,
         error_message: &'i str,
     },
+    CanonStream {
+        name: &'i str,
+        lambda: LambdaAST<'i>,
+    },
     LastError,
 }
 
@@ -126,7 +140,7 @@ pub struct Next<'i> {
 /// (new variable instruction)
 #[derive(Serialize, Debug, PartialEq)]
 pub struct New<'i> {
-    pub variable: Variable<'i>,
+    pub argument: NewArgument<'i>,
     pub instruction: Box<Instruction<'i>>,
     pub span: Span,
 }

crates/air-lib/air-parser/src/ast/instructions/impls.rs

@@ -17,7 +17,7 @@
 use super::*;
 
 impl<'i> Ap<'i> {
-    pub fn new(argument: ApArgument<'i>, result: Variable<'i>) -> Self {
+    pub fn new(argument: ApArgument<'i>, result: ApResult<'i>) -> Self {
         Self { argument, result }
     }
 }
@@ -36,6 +36,20 @@ impl<'i> Call<'i> {
     }
 }
 
+impl<'i> Canon<'i> {
+    pub fn new(
+        peer_pk: CallInstrValue<'i>,
+        stream: Stream<'i>,
+        canon_stream: CanonStream<'i>,
+    ) -> Self {
+        Self {
+            peer_pk,
+            stream,
+            canon_stream,
+        }
+    }
+}
+
 impl<'i> Seq<'i> {
     pub fn new(
         left_instruction: Box<Instruction<'i>>,
@@ -131,9 +145,9 @@ impl<'i> Next<'i> {
 
 impl<'i> New<'i> {
     #[allow(clippy::self_named_constructors)]
-    pub fn new(variable: Variable<'i>, instruction: Box<Instruction<'i>>, span: Span) -> Self {
+    pub fn new(argument: NewArgument<'i>, instruction: Box<Instruction<'i>>, span: Span) -> Self {
         Self {
-            variable,
+            argument,
             instruction,
             span,
         }

crates/air-lib/air-parser/src/ast/instructions/traits.rs

@@ -16,6 +16,7 @@
 
 use super::*;
 
+use air_lambda_ast::format_ast;
 use std::fmt;
 
 impl fmt::Display for Instruction<'_> {
@@ -24,6 +25,7 @@ impl fmt::Display for Instruction<'_> {
 
         match self {
             Call(call) => write!(f, "{}", call),
+            Canon(canon) => write!(f, "{}", canon),
             Ap(ap) => write!(f, "{}", ap),
             Seq(seq) => write!(f, "{}", seq),
             Par(par) => write!(f, "{}", par),
@@ -50,6 +52,16 @@ impl fmt::Display for Call<'_> {
     }
 }
 
+impl fmt::Display for Canon<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "canon {} {} {}",
+            self.peer_pk, self.stream, self.canon_stream
+        )
+    }
+}
+
 impl fmt::Display for Ap<'_> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "ap {} {}", self.argument, self.result)
@@ -64,6 +76,9 @@ impl fmt::Display for Fail<'_> {
                 ret_code,
                 error_message,
             } => write!(f, r#"fail {} "{}""#, ret_code, error_message),
+            Fail::CanonStream { name, lambda, .. } => {
+                write!(f, "fail {}.$.{}", name, format_ast(lambda))
+            }
             Fail::LastError => write!(f, "fail %last_error%"),
         }
     }
@@ -125,6 +140,6 @@ impl fmt::Display for Next<'_> {
 
 impl fmt::Display for New<'_> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "new {}", self.variable)
+        write!(f, "new {}", self.argument)
     }
 }

crates/air-lib/air-parser/src/ast/values.rs

@@ -54,6 +54,22 @@ pub struct StreamWithLambda<'i> {
     pub position: usize,
 }
 
+/// A canonicalized stream without lambda.
+#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
+pub struct CanonStream<'i> {
+    pub name: &'i str,
+    pub position: usize,
+}
+
+/// A canonicalized stream with lambda.
+#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
+pub struct CanonStreamWithLambda<'i> {
+    pub name: &'i str,
+    #[serde(borrow)]
+    pub lambda: Option<LambdaAST<'i>>,
+    pub position: usize,
+}
+
 /// A variable that could be either scalar or stream without lambda.
 #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
 pub enum Variable<'i> {
@@ -61,6 +77,8 @@ pub enum Variable<'i> {
     Scalar(Scalar<'i>),
     #[serde(borrow)]
     Stream(Stream<'i>),
+    #[serde(borrow)]
+    CanonStream(CanonStream<'i>),
 }
 
 /// A variable that could be either scalar or stream with possible lambda expression.
@@ -70,4 +88,6 @@ pub enum VariableWithLambda<'i> {
     Scalar(ScalarWithLambda<'i>),
     #[serde(borrow)]
     Stream(StreamWithLambda<'i>),
+    #[serde(borrow)]
+    CanonStream(CanonStreamWithLambda<'i>),
 }

crates/air-lib/air-parser/src/ast/values/impls.rs

@@ -67,6 +67,22 @@ impl<'i> StreamWithLambda<'i> {
     }
 }
 
+impl<'i> CanonStream<'i> {
+    pub fn new(name: &'i str, position: usize) -> Self {
+        Self { name, position }
+    }
+}
+
+impl<'i> CanonStreamWithLambda<'i> {
+    pub fn new(name: &'i str, lambda: Option<LambdaAST<'i>>, position: usize) -> Self {
+        Self {
+            name,
+            lambda,
+            position,
+        }
+    }
+}
+
 impl<'i> Scalar<'i> {
     pub fn new(name: &'i str, position: usize) -> Self {
         Self { name, position }
@@ -92,6 +108,7 @@ impl<'i> Variable<'i> {
         match self {
             Variable::Scalar(scalar) => scalar.name,
             Variable::Stream(stream) => stream.name,
+            Variable::CanonStream(stream) => stream.name,
         }
     }
 }
@@ -113,10 +130,19 @@ impl<'i> VariableWithLambda<'i> {
         Self::Stream(StreamWithLambda::new(name, Some(lambda), position))
     }
 
+    pub fn canon_stream(name: &'i str, position: usize) -> Self {
+        Self::CanonStream(CanonStreamWithLambda::new(name, None, position))
+    }
+
+    pub fn canon_stream_wl(name: &'i str, lambda: LambdaAST<'i>, position: usize) -> Self {
+        Self::CanonStream(CanonStreamWithLambda::new(name, Some(lambda), position))
+    }
+
     pub fn name(&self) -> &'i str {
         match self {
             VariableWithLambda::Scalar(scalar) => scalar.name,
             VariableWithLambda::Stream(stream) => stream.name,
+            VariableWithLambda::CanonStream(canon_stream) => canon_stream.name,
         }
     }
 
@@ -124,6 +150,7 @@ impl<'i> VariableWithLambda<'i> {
         match self {
             VariableWithLambda::Scalar(scalar) => &scalar.lambda,
             VariableWithLambda::Stream(stream) => &stream.lambda,
+            VariableWithLambda::CanonStream(canon_stream) => &canon_stream.lambda,
         }
     }
 

crates/air-lib/air-parser/src/ast/values/traits.rs

@@ -39,6 +39,12 @@ impl fmt::Display for Stream<'_> {
     }
 }
 
+impl fmt::Display for CanonStream<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.name)
+    }
+}
+
 impl fmt::Display for StreamWithLambda<'_> {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match &self.lambda {
@@ -48,6 +54,15 @@ impl fmt::Display for StreamWithLambda<'_> {
     }
 }
 
+impl fmt::Display for CanonStreamWithLambda<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match &self.lambda {
+            Some(lambda) => write!(f, "{}.${}", self.name, format_ast(lambda)),
+            None => write!(f, "{}", self.name),
+        }
+    }
+}
+
 impl fmt::Display for Variable<'_> {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         use Variable::*;
@@ -55,6 +70,7 @@ impl fmt::Display for Variable<'_> {
         match self {
             Scalar(scalar) => write!(f, "{}", scalar),
             Stream(stream) => write!(f, "{}", stream),
+            CanonStream(canon_stream) => write!(f, "{}", canon_stream),
         }
     }
 }
@@ -66,6 +82,7 @@ impl fmt::Display for VariableWithLambda<'_> {
         match self {
             Scalar(scalar) => write!(f, "{}", scalar),
             Stream(stream) => write!(f, "{}", stream),
+            CanonStream(canon_stream) => write!(f, "{}", canon_stream),
         }
     }
 }

crates/air-lib/air-parser/src/parser/air.lalrpop

@@ -16,7 +16,7 @@ pub AIR = Instr;
 Instr: Box<Instruction<'input>> = {
     <left: @L> "(" call <triplet:Triplet> <args:Args> <output:CallOutput?> ")" <right: @R> => {
         let args = Rc::new(args);
-        let output = output.map(CallOutputValue::Variable).unwrap_or(CallOutputValue::None);
+        let output = output.unwrap_or(CallOutputValue::None);
         let call = Call::new(triplet, args, output);
         let span = Span::new(left, right);
 
@@ -25,6 +25,16 @@ Instr: Box<Instruction<'input>> = {
         Box::new(Instruction::Call(call))
     },
 
+    <left: @L> "(" canon <peer_pk:CallInstrValue> <stream:StreamArgument> <canon_stream:CanonStreamArgument> ")" <right: @R> => {
+        let canon = Canon::new(peer_pk, stream, canon_stream);
+
+        let span = Span::new(left, right);
+        validator.met_canon(&canon, span);
+
+        Box::new(Instruction::Canon(canon))
+    },
+
+
     <left: @L> "(" ap <arg:ApArgument> <result:ApResult> ")" <right: @R> => {
         let apply = Ap::new(arg, result);
 
@@ -38,9 +48,9 @@ Instr: Box<Instruction<'input>> = {
     "(" par <l:Instr> <r:Instr> ")" => Box::new(Instruction::Par(Par::new(l, r))),
     "(" null ")" => Box::new(Instruction::Null(Null)),
 
-    <left: @L> "(" new <variable: ScriptVariable> <instruction:Instr> ")" <right: @R> => {
+    <left: @L> "(" new <argument: NewArgument> <instruction:Instr> ")" <right: @R> => {
         let span = Span::new(left, right);
-        let new = New::new(variable, instruction, span);
+        let new = New::new(argument, instruction, span);
 
         validator.met_new(&new, span);
 
@@ -114,12 +124,14 @@ Triplet: Triplet<'input> = {
     }
 }
 
-ApResult = ScriptVariable;
+ApResult: ApResult<'input> = {
-CallOutput = ScriptVariable;
+    <scalar:Scalar> => ApResult::scalar(scalar.0, scalar.1),
+    <stream:Stream> => ApResult::stream(stream.0, stream.1),
+};
 
-ScriptVariable: Variable<'input> = {
+CallOutput: CallOutputValue<'input> = {
-    <scalar:Scalar> => Variable::scalar(scalar.0, scalar.1),
+    <scalar:Scalar> => CallOutputValue::scalar(scalar.0, scalar.1),
-    <stream:Stream> => Variable::stream(stream.0, stream.1),
+    <stream:Stream> => CallOutputValue::stream(stream.0, stream.1),
 };
 
 FailBody: Fail<'input> = {
@@ -129,6 +141,10 @@ FailBody: Fail<'input> = {
         ret_code,
         error_message,
     },
+    <canon_stream:CanonStreamWithLambda> => Fail::CanonStream {
+        name: canon_stream.0,
+        lambda: canon_stream.1,
+    },
     <left: @L> <l:LastError> <right: @R> => {
         Fail::LastError
     }
@@ -137,6 +153,7 @@ FailBody: Fail<'input> = {
 FoldScalarIterable: FoldScalarIterable<'input> = {
     <scalar:Scalar> => FoldScalarIterable::Scalar(ScalarWithLambda::new(scalar.0, None, scalar.1)),
     <scalar:ScalarWithLambda> => FoldScalarIterable::Scalar(ScalarWithLambda::new(scalar.0, Some(scalar.1), scalar.2)),
+    <canon_stream:CanonStream> => FoldScalarIterable::CanonStream(CanonStream::new(canon_stream.0, canon_stream.1)),
     "[" "]" => FoldScalarIterable::EmptyArray,
 };
 
@@ -144,6 +161,7 @@ Function = CallInstrValue;
 PeerId = CallInstrValue;
 ServiceId = CallInstrValue;
 
+// TODO: call triplet should receive only streams with lambdas
 CallInstrValue: CallInstrValue<'input> = {
     InitPeerId => CallInstrValue::InitPeerId,
     <l:Literal> => CallInstrValue::Literal(l),
@@ -151,6 +169,14 @@ CallInstrValue: CallInstrValue<'input> = {
     <scalar:ScalarWithLambda> => CallInstrValue::Variable(VariableWithLambda::scalar_wl(scalar.0, scalar.1, scalar.2)),
     <stream:Stream> => CallInstrValue::Variable(VariableWithLambda::stream(stream.0, stream.1)),
     <stream:StreamWithLambda> => CallInstrValue::Variable(VariableWithLambda::stream_wl(stream.0, stream.1, stream.2)),
+    <canon_stream:CanonStream> => CallInstrValue::Variable(VariableWithLambda::canon_stream(canon_stream.0, canon_stream.1)),
+    <canon_stream:CanonStreamWithLambda> => CallInstrValue::Variable(VariableWithLambda::canon_stream_wl(canon_stream.0, canon_stream.1, canon_stream.2)),
+}
+
+NewArgument: NewArgument<'input> = {
+    <scalar:Scalar> => NewArgument::Scalar(Scalar::new(scalar.0, scalar.1)),
+    <stream:Stream> => NewArgument::Stream(Stream::new(stream.0, stream.1)),
+    <canon_stream:CanonStream> => NewArgument::CanonStream(CanonStream::new(canon_stream.0, canon_stream.1)),
 }
 
 Number: Number = {
@@ -174,6 +200,8 @@ Value: Value<'input> = {
     <scalar:ScalarWithLambda> => Value::Variable(VariableWithLambda::scalar_wl(scalar.0, scalar.1, scalar.2)),
     <stream:Stream> => Value::Variable(VariableWithLambda::stream(stream.0, stream.1)),
     <stream:StreamWithLambda> => Value::Variable(VariableWithLambda::stream_wl(stream.0, stream.1, stream.2)),
+    <canon_stream:CanonStream> => Value::Variable(VariableWithLambda::canon_stream(canon_stream.0, canon_stream.1)),
+    <canon_stream:CanonStreamWithLambda> => Value::Variable(VariableWithLambda::canon_stream_wl(canon_stream.0, canon_stream.1, canon_stream.2)),
 }
 
 ApArgument: ApArgument<'input> = {
@@ -188,6 +216,16 @@ ApArgument: ApArgument<'input> = {
     "[" "]" => ApArgument::EmptyArray,
     <scalar:Scalar> => ApArgument::Scalar(ScalarWithLambda::new(scalar.0, None, scalar.1)),
     <scalar:ScalarWithLambda> => ApArgument::Scalar(ScalarWithLambda::new(scalar.0, Some(scalar.1), scalar.2)),
+    <canon_stream:CanonStream> => ApArgument::CanonStream(CanonStreamWithLambda::new(canon_stream.0, None, canon_stream.1)),
+    <canon_stream:CanonStreamWithLambda> => ApArgument::CanonStream(CanonStreamWithLambda::new(canon_stream.0, Some(canon_stream.1), canon_stream.2)),
+}
+
+StreamArgument: Stream<'input> = {
+    <stream:Stream> => Stream::new(stream.0, stream.1),
+}
+
+CanonStreamArgument: CanonStream<'input> = {
+    <canon_stream:CanonStream> => CanonStream::new(canon_stream.0, canon_stream.1),
 }
 
 extern {
@@ -204,6 +242,8 @@ extern {
         ScalarWithLambda => Token::ScalarWithLambda { name: <&'input str>, lambda: <LambdaAST<'input>>, position: <usize> },
         Stream => Token::Stream { name: <&'input str>, position: <usize> },
         StreamWithLambda => Token::StreamWithLambda {name: <&'input str>, lambda:<LambdaAST<'input>>, position: <usize>},
+        CanonStream => Token::CanonStream { name: <&'input str>, position: <usize> },
+        CanonStreamWithLambda => Token::CanonStreamWithLambda {name: <&'input str>, lambda:<LambdaAST<'input>>, position: <usize>},
 
         Literal => Token::StringLiteral(<&'input str>),
         I64 => Token::I64(<i64>),
@@ -217,6 +257,7 @@ extern {
         TTL => Token::TTL,
 
         call => Token::Call,
+        canon => Token::Canon,
         ap => Token::Ap,
         seq => Token::Seq,
         par => Token::Par,

crates/air-lib/air-parser/src/parser/lexer/air_lexer.rs

@@ -176,6 +176,7 @@ fn string_to_token(input: &str, start_pos: usize) -> LexerResult<Token> {
         "" => Err(LexerError::empty_string(start_pos..start_pos)),
 
         CALL_INSTR => Ok(Token::Call),
+        CANON_INSTR => Ok(Token::Canon),
         AP_INSTR => Ok(Token::Ap),
         SEQ_INSTR => Ok(Token::Seq),
         PAR_INSTR => Ok(Token::Par),
@@ -226,6 +227,7 @@ fn parse_last_error(input: &str, start_pos: usize) -> LexerResult<Token<'_>> {
 }
 
 const CALL_INSTR: &str = "call";
+const CANON_INSTR: &str = "canon";
 const AP_INSTR: &str = "ap";
 const SEQ_INSTR: &str = "seq";
 const PAR_INSTR: &str = "par";

crates/air-lib/air-parser/src/parser/lexer/call_variable_parser.rs

@@ -22,8 +22,6 @@ use crate::LambdaAST;
 use std::iter::Peekable;
 use std::str::CharIndices;
 
-const STREAM_START_TAG: char = '$';
-
 pub(super) fn try_parse_call_variable(
     string_to_parse: &str,
     start_pos: usize,
@@ -31,14 +29,21 @@ pub(super) fn try_parse_call_variable(
     CallVariableParser::try_parse(string_to_parse, start_pos)
 }
 
+#[derive(Debug)]
+enum MetTag {
+    None,
+    Stream,
+    CanonStream,
+}
+
 #[derive(Debug)]
 struct ParserState {
     pub(self) first_dot_met_pos: Option<usize>,
     pub(self) non_numeric_met: bool,
     pub(self) digit_met: bool,
     pub(self) flattening_met: bool,
+    pub(self) met_tag: MetTag,
     pub(self) is_first_char: bool,
-    pub(self) is_first_stream_tag: bool,
     pub(self) current_char: char,
     pub(self) current_pos: usize,
 }
@@ -64,7 +69,7 @@ impl<'input> CallVariableParser<'input> {
             digit_met: false,
             flattening_met: false,
             is_first_char: true,
-            is_first_stream_tag: false,
+            met_tag: MetTag::None,
             current_char,
             current_pos,
         };
@@ -180,13 +185,14 @@ impl<'input> CallVariableParser<'input> {
     }
 
     fn try_parse_as_stream_start(&mut self) -> LexerResult<bool> {
-        if self.current_pos() == 0 && self.current_char() == STREAM_START_TAG {
+        let stream_tag = MetTag::from_tag(self.current_char());
+        if self.current_pos() == 0 && stream_tag.is_tag() {
             if self.string_to_parse.len() == 1 {
                 let error_pos = self.pos_in_string_to_parse();
                 return Err(LexerError::empty_stream_name(error_pos..error_pos));
             }
 
-            self.state.is_first_stream_tag = true;
+            self.state.met_tag = stream_tag;
             return Ok(true);
         }
 
@@ -271,32 +277,39 @@ impl<'input> CallVariableParser<'input> {
     }
 
     fn to_variable_token<'v>(&self, name: &'v str) -> Token<'v> {
-        if self.state.is_first_stream_tag {
+        match self.state.met_tag {
-            Token::Stream {
+            MetTag::None => Token::Scalar {
                 name,
                 position: self.start_pos,
-            }
+            },
-        } else {
+            MetTag::Stream => Token::Stream {
-            Token::Scalar {
                 name,
                 position: self.start_pos,
-            }
+            },
+            MetTag::CanonStream => Token::CanonStream {
+                name,
+                position: self.start_pos,
+            },
         }
     }
 
     fn to_variable_token_with_lambda<'v>(&self, name: &'v str, lambda: LambdaAST<'v>) -> Token<'v> {
-        if self.state.is_first_stream_tag {
+        match self.state.met_tag {
-            Token::StreamWithLambda {
+            MetTag::None => Token::ScalarWithLambda {
                 name,
                 lambda,
                 position: self.start_pos,
-            }
+            },
-        } else {
+            MetTag::Stream => Token::StreamWithLambda {
-            Token::ScalarWithLambda {
                 name,
                 lambda,
                 position: self.start_pos,
-            }
+            },
+            MetTag::CanonStream => Token::CanonStreamWithLambda {
+                name,
+                lambda,
+                position: self.start_pos,
+            },
         }
     }
 
@@ -356,3 +369,17 @@ impl<'input> CallVariableParser<'input> {
         }
     }
 }
+
+impl MetTag {
+    fn from_tag(tag: char) -> Self {
+        match tag {
+            '$' => Self::Stream,
+            '#' => Self::CanonStream,
+            _ => Self::None,
+        }
+    }
+
+    fn is_tag(&self) -> bool {
+        !matches!(self, Self::None)
+    }
+}

crates/air-lib/air-parser/src/parser/lexer/token.rs

@@ -45,6 +45,15 @@ pub enum Token<'input> {
         lambda: LambdaAST<'input>,
         position: usize,
     },
+    CanonStream {
+        name: &'input str,
+        position: usize,
+    },
+    CanonStreamWithLambda {
+        name: &'input str,
+        lambda: LambdaAST<'input>,
+        position: usize,
+    },
 
     StringLiteral(&'input str),
     I64(i64),
@@ -58,6 +67,7 @@ pub enum Token<'input> {
     TTL,
 
     Call,
+    Canon,
     Ap,
     Seq,
     Par,

crates/air-lib/air-parser/src/parser/tests/ap.rs

@@ -19,6 +19,8 @@ use super::parse;
 use crate::ast::*;
 
 use air_lambda_ast::{LambdaAST, ValueAccessor};
+use fstrings::f;
+use fstrings::format_args_f;
 
 #[test]
 fn ap_with_literal() {
@@ -29,7 +31,7 @@ fn ap_with_literal() {
     let actual = parse(source_code);
     let expected = ap(
         ApArgument::Literal("some_string"),
-        Variable::stream("$stream", 27),
+        ApResult::Stream(Stream::new("$stream", 27)),
     );
 
     assert_eq!(actual, expected);
@@ -44,7 +46,7 @@ fn ap_with_number() {
     let actual = parse(source_code);
     let expected = ap(
         ApArgument::Number(Number::Int(-100)),
-        Variable::stream("$stream", 18),
+        ApResult::Stream(Stream::new("$stream", 18)),
     );
 
     assert_eq!(actual, expected);
@@ -57,7 +59,10 @@ fn ap_with_bool() {
     "#;
 
     let actual = parse(source_code);
-    let expected = ap(ApArgument::Boolean(true), Variable::stream("$stream", 18));
+    let expected = ap(
+        ApArgument::Boolean(true),
+        ApResult::Stream(Stream::new("$stream", 18)),
+    );
 
     assert_eq!(actual, expected);
 }
@@ -75,7 +80,7 @@ fn ap_with_last_error() {
                 field_name: "message",
             }])
         })),
-        Variable::stream("$stream", 37),
+        ApResult::Stream(Stream::new("$stream", 37)),
     );
 
     assert_eq!(actual, expected);
@@ -88,7 +93,10 @@ fn ap_with_empty_array() {
     "#;
 
     let actual = parse(source_code);
-    let expected = ap(ApArgument::EmptyArray, Variable::stream("$stream", 16));
+    let expected = ap(
+        ApArgument::EmptyArray,
+        ApResult::Stream(Stream::new("$stream", 16)),
+    );
 
     assert_eq!(actual, expected);
 }
@@ -100,7 +108,10 @@ fn ap_with_init_peer_id() {
     "#;
 
     let actual = parse(source_code);
-    let expected = ap(ApArgument::InitPeerId, Variable::stream("$stream", 28));
+    let expected = ap(
+        ApArgument::InitPeerId,
+        ApResult::Stream(Stream::new("$stream", 28)),
+    );
 
     assert_eq!(actual, expected);
 }
@@ -112,7 +123,10 @@ fn ap_with_timestamp() {
     "#;
 
     let actual = parse(source_code);
-    let expected = ap(ApArgument::Timestamp, Variable::stream("$stream", 25));
+    let expected = ap(
+        ApArgument::Timestamp,
+        ApResult::Stream(Stream::new("$stream", 25)),
+    );
 
     assert_eq!(actual, expected);
 }
@@ -124,7 +138,48 @@ fn ap_with_ttl() {
     "#;
 
     let actual = parse(source_code);
-    let expected = ap(ApArgument::TTL, Variable::stream("$stream", 19));
+    let expected = ap(
+        ApArgument::TTL,
+        ApResult::Stream(Stream::new("$stream", 19)),
+    );
+
+    assert_eq!(actual, expected);
+}
+
+#[test]
+fn ap_with_canon_stream() {
+    let canon_stream = "#canon_stream";
+    let scalar = "scalar";
+    let source_code = f!(r#"
+        (ap {canon_stream} {scalar})
+    "#);
+
+    let actual = parse(&source_code);
+    let expected = ap(
+        ApArgument::CanonStream(CanonStreamWithLambda::new(canon_stream, None, 13)),
+        ApResult::Scalar(Scalar::new(scalar, 27)),
+    );
+
+    assert_eq!(actual, expected);
+}
+
+#[test]
+fn ap_with_canon_stream_with_lambda() {
+    let canon_stream = "#canon_stream";
+    let scalar = "scalar";
+    let source_code = f!(r#"
+        (ap {canon_stream}.$.[0] {scalar})
+    "#);
+
+    let actual = parse(&source_code);
+    let expected = ap(
+        ApArgument::CanonStream(CanonStreamWithLambda::new(
+            canon_stream,
+            Some(unsafe { LambdaAST::new_unchecked(vec![ValueAccessor::ArrayAccess { idx: 0 }]) }),
+            13,
+        )),
+        ApResult::Scalar(Scalar::new(scalar, 33)),
+    );
 
     assert_eq!(actual, expected);
 }

crates/air-lib/air-parser/src/parser/tests/call.rs

@@ -19,7 +19,9 @@ use super::parse;
 use crate::ast::*;
 use crate::parser::ParserError;
 
-use air_lambda_ast::ValueAccessor;
+use air_lambda_ast::{LambdaAST, ValueAccessor};
+use fstrings::f;
+use fstrings::format_args_f;
 use lalrpop_util::ParseError;
 
 use std::rc::Rc;
@@ -43,7 +45,7 @@ fn parse_json_path() {
             Value::Literal("hello"),
             Value::Variable(VariableWithLambda::scalar("name", 68)),
         ]),
-        CallOutputValue::Variable(Variable::stream("$void", 74)),
+        CallOutputValue::Stream(Stream::new("$void", 74)),
     );
     assert_eq!(instruction, expected);
 }
@@ -193,7 +195,7 @@ fn parse_lambda_complex() {
             CallInstrValue::Literal("service_id"),
             CallInstrValue::Literal("function_name"),
             Rc::new(vec![]),
-            CallOutputValue::Variable(Variable::scalar("void", 75)),
+            CallOutputValue::Scalar(Scalar::new("void", 75)),
         ),
         call(
             CallInstrValue::Variable(VariableWithLambda::from_raw_lambda_scalar(
@@ -212,7 +214,7 @@ fn parse_lambda_complex() {
             CallInstrValue::Literal("service_id"),
             CallInstrValue::Literal("function_name"),
             Rc::new(vec![]),
-            CallOutputValue::Variable(Variable::scalar("void", 162)),
+            CallOutputValue::Scalar(Scalar::new("void", 162)),
         ),
     );
     assert_eq!(instruction, expected);
@@ -245,7 +247,7 @@ fn parse_lambda_with_scalars_complex() {
             CallInstrValue::Literal("service_id"),
             CallInstrValue::Literal("function_name"),
             Rc::new(vec![]),
-            CallOutputValue::Variable(Variable::scalar("void", 97)),
+            CallOutputValue::Scalar(Scalar::new("void", 97)),
         ),
         call(
             CallInstrValue::Variable(VariableWithLambda::from_raw_lambda_scalar(
@@ -270,7 +272,7 @@ fn parse_lambda_with_scalars_complex() {
             CallInstrValue::Literal("service_id"),
             CallInstrValue::Literal("function_name"),
             Rc::new(vec![]),
-            CallOutputValue::Variable(Variable::scalar("void", 205)),
+            CallOutputValue::Scalar(Scalar::new("void", 205)),
         ),
     );
     assert_eq!(instruction, expected);
@@ -310,7 +312,7 @@ fn json_path_square_braces() {
                 64,
             )),
         ]),
-        CallOutputValue::Variable(Variable::stream("$void", 74)),
+        CallOutputValue::Stream(Stream::new("$void", 74)),
     );
 
     assert_eq!(instruction, expected);
@@ -410,6 +412,82 @@ fn parse_last_error() {
     assert_eq!(instruction, expected);
 }
 
+#[test]
+fn canon_stream_in_args() {
+    let service_id = "service_id";
+    let function_name = "function_name";
+    let canon_stream = "#canon_stream";
+    let source_code = f!(r#"
+            (call %init_peer_id% ("{service_id}" "{function_name}") [{canon_stream}])
+        "#);
+
+    let instruction = parse(&source_code);
+    let expected = call(
+        CallInstrValue::InitPeerId,
+        CallInstrValue::Literal(service_id),
+        CallInstrValue::Literal(function_name),
+        Rc::new(vec![Value::Variable(VariableWithLambda::canon_stream(
+            canon_stream,
+            66,
+        ))]),
+        CallOutputValue::None,
+    );
+
+    assert_eq!(instruction, expected);
+}
+
+#[test]
+fn canon_stream_in_triplet() {
+    let service_id = "service_id";
+    let function_name = "function_name";
+    let canon_stream = "#canon_stream";
+    let source_code = f!(r#"
+            (call {canon_stream} ("{service_id}" "{function_name}") [])
+        "#);
+
+    let instruction = parse(&source_code);
+    let expected = call(
+        CallInstrValue::Variable(VariableWithLambda::canon_stream(canon_stream, 19)),
+        CallInstrValue::Literal(service_id),
+        CallInstrValue::Literal(function_name),
+        Rc::new(vec![]),
+        CallOutputValue::None,
+    );
+
+    assert_eq!(instruction, expected);
+}
+
+#[test]
+fn canon_stream_with_lambda_in_triplet() {
+    let service_id = "service_id";
+    let function_name = "function_name";
+    let canon_stream = "#canon_stream";
+    let canon_stream_lambda = ".$.[0].path!";
+    let source_code = f!(r#"
+            (call {canon_stream}{canon_stream_lambda} ("{service_id}" "{function_name}") [])
+        "#);
+
+    let instruction = parse(&source_code);
+    let expected = call(
+        CallInstrValue::Variable(VariableWithLambda::canon_stream_wl(
+            canon_stream,
+            unsafe {
+                LambdaAST::new_unchecked(vec![
+                    ValueAccessor::ArrayAccess { idx: 0 },
+                    ValueAccessor::FieldAccessByName { field_name: "path" },
+                ])
+            },
+            19,
+        )),
+        CallInstrValue::Literal(service_id),
+        CallInstrValue::Literal(function_name),
+        Rc::new(vec![]),
+        CallOutputValue::None,
+    );
+
+    assert_eq!(instruction, expected);
+}
+
 #[test]
 fn seq_par_call() {
     let peer_id = "some_peer_id";
@@ -430,14 +508,14 @@ fn seq_par_call() {
                 CallInstrValue::Literal("local_service_id"),
                 CallInstrValue::Literal("local_fn_name"),
                 Rc::new(vec![]),
-                CallOutputValue::Variable(Variable::scalar("result_1", 108)),
+                CallOutputValue::Scalar(Scalar::new("result_1", 108)),
             ),
             call(
                 CallInstrValue::Literal(peer_id),
                 CallInstrValue::Literal("service_id"),
                 CallInstrValue::Literal("fn_name"),
                 Rc::new(vec![]),
-                CallOutputValue::Variable(Variable::scalar("g", 183)),
+                CallOutputValue::Scalar(Scalar::new("g", 183)),
             ),
         ),
         call(
@@ -445,7 +523,7 @@ fn seq_par_call() {
             CallInstrValue::Literal("local_service_id"),
             CallInstrValue::Literal("local_fn_name"),
             Rc::new(vec![]),
-            CallOutputValue::Variable(Variable::scalar("result_2", 273)),
+            CallOutputValue::Scalar(Scalar::new("result_2", 273)),
         ),
     );
 
@@ -485,14 +563,14 @@ fn seq_with_empty_and_dash() {
                     CallInstrValue::Literal(""),
                     CallInstrValue::Literal(""),
                     Rc::new(vec![Value::Literal("module-bytes")]),
-                    CallOutputValue::Variable(Variable::scalar("module-bytes", 119)),
+                    CallOutputValue::Scalar(Scalar::new("module-bytes", 119)),
                 ),
                 call(
                     CallInstrValue::Literal("set_variables"),
                     CallInstrValue::Literal(""),
                     CallInstrValue::Literal(""),
                     Rc::new(vec![Value::Literal("module_config")]),
-                    CallOutputValue::Variable(Variable::scalar("module_config", 201)),
+                    CallOutputValue::Scalar(Scalar::new("module_config", 201)),
                 ),
             ),
             call(
@@ -500,7 +578,7 @@ fn seq_with_empty_and_dash() {
                 CallInstrValue::Literal(""),
                 CallInstrValue::Literal(""),
                 Rc::new(vec![Value::Literal("blueprint")]),
-                CallOutputValue::Variable(Variable::scalar("blueprint", 294)),
+                CallOutputValue::Scalar(Scalar::new("blueprint", 294)),
             ),
         ),
         seq(
@@ -512,7 +590,7 @@ fn seq_with_empty_and_dash() {
                     Value::Variable(VariableWithLambda::scalar("module-bytes", 381)),
                     Value::Variable(VariableWithLambda::scalar("module_config", 394)),
                 ]),
-                CallOutputValue::Variable(Variable::scalar("module", 409)),
+                CallOutputValue::Scalar(Scalar::new("module", 409)),
             ),
             seq(
                 Instruction::Call(Call {
@@ -525,7 +603,7 @@ fn seq_with_empty_and_dash() {
                         "blueprint",
                         490,
                     ))]),
-                    output: CallOutputValue::Variable(Variable::scalar("blueprint_id", 501)),
+                    output: CallOutputValue::Scalar(Scalar::new("blueprint_id", 501)),
                 }),
                 seq(
                     call(
@@ -536,7 +614,7 @@ fn seq_with_empty_and_dash() {
                             "blueprint_id",
                             589,
                         ))]),
-                        CallOutputValue::Variable(Variable::scalar("service_id", 603)),
+                        CallOutputValue::Scalar(Scalar::new("service_id", 603)),
                     ),
                     call(
                         CallInstrValue::Literal("remote_peer_id"),
@@ -546,7 +624,7 @@ fn seq_with_empty_and_dash() {
                             "service_id",
                             671,
                         ))]),
-                        CallOutputValue::Variable(Variable::scalar("client_result", 683)),
+                        CallOutputValue::Scalar(Scalar::new("client_result", 683)),
                     ),
                 ),
             ),

crates/air-lib/air-parser/src/parser/tests/canon.rs

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2022 Fluence Labs Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+use super::dsl::*;
+use super::parse;
+use crate::ast::*;
+
+use fstrings::f;
+use fstrings::format_args_f;
+
+#[test]
+fn canon_with_literal_peer_id() {
+    let peer_id = "peer_id";
+    let stream = "$stream";
+    let canon_stream = "#canon_stream";
+    let source_code = f!(r#"
+        (canon "{peer_id}" {stream} {canon_stream})
+    "#);
+
+    let actual = parse(&source_code);
+    let expected = canon(
+        CallInstrValue::Literal(peer_id),
+        Stream::new(stream, 26),
+        CanonStream::new(canon_stream, 34),
+    );
+
+    assert_eq!(actual, expected);
+}
+
+#[test]
+fn canon_with_variable_peer_id() {
+    let peer_id = "peer_id";
+    let stream = "$stream";
+    let canon_stream = "#canon_stream";
+    let source_code = f!(r#"
+        (canon {peer_id} {stream} {canon_stream})
+    "#);
+
+    let actual = parse(&source_code);
+    let expected = canon(
+        CallInstrValue::Variable(VariableWithLambda::scalar(peer_id, 16)),
+        Stream::new(stream, 24),
+        CanonStream::new(canon_stream, 32),
+    );
+
+    assert_eq!(actual, expected);
+}

crates/air-lib/air-parser/src/parser/tests/dsl.rs

@@ -54,12 +54,12 @@ pub(super) fn seqnn() -> Instruction<'static> {
 }
 
 pub(super) fn new<'i>(
-    variable: Variable<'i>,
+    argument: NewArgument<'i>,
     instruction: Instruction<'i>,
     span: Span,
 ) -> Instruction<'i> {
     Instruction::New(New {
-        variable,
+        argument,
         instruction: Box::new(instruction),
         span,
     })
@@ -98,6 +98,20 @@ pub(super) fn fold_scalar_variable<'i>(
     })
 }
 
+pub(super) fn fold_scalar_canon_stream<'i>(
+    canon_stream: CanonStream<'i>,
+    iterator: Scalar<'i>,
+    instruction: Instruction<'i>,
+    span: Span,
+) -> Instruction<'i> {
+    Instruction::FoldScalar(FoldScalar {
+        iterable: FoldScalarIterable::CanonStream(canon_stream),
+        iterator,
+        instruction: Rc::new(instruction),
+        span,
+    })
+}
+
 pub(super) fn fold_scalar_empty_array<'i>(
     iterator: Scalar<'i>,
     instruction: Instruction<'i>,
@@ -149,10 +163,22 @@ pub(super) fn mismatch<'i>(
     })
 }
 
-pub(super) fn ap<'i>(argument: ApArgument<'i>, result: Variable<'i>) -> Instruction<'i> {
+pub(super) fn ap<'i>(argument: ApArgument<'i>, result: ApResult<'i>) -> Instruction<'i> {
     Instruction::Ap(Ap { argument, result })
 }
 
+pub(super) fn canon<'i>(
+    peer_pk: CallInstrValue<'i>,
+    stream: Stream<'i>,
+    canon_stream: CanonStream<'i>,
+) -> Instruction<'i> {
+    Instruction::Canon(Canon {
+        peer_pk,
+        stream,
+        canon_stream,
+    })
+}
+
 pub(super) fn binary_instruction<'i, 'b>(
     name: &'i str,
 ) -> impl Fn(Instruction<'b>, Instruction<'b>) -> Instruction<'b> {

crates/air-lib/air-parser/src/parser/tests/fold.rs

@@ -20,6 +20,8 @@ use crate::ast::*;
 use crate::parser::ParserError;
 
 use air_lambda_ast::ValueAccessor;
+use fstrings::f;
+use fstrings::format_args_f;
 use lalrpop_util::ParseError;
 
 #[test]
@@ -329,6 +331,24 @@ fn fold_on_stream() {
     assert_eq!(instruction, expected);
 }
 
+#[test]
+fn fold_on_canon_stream() {
+    let canon_stream = "#canon_stream";
+    let iterator = "iterator";
+    let source_code = f!(r#"
+        (fold {canon_stream} {iterator} (null))
+    "#);
+
+    let instruction = parse(&source_code);
+    let expected = fold_scalar_canon_stream(
+        CanonStream::new(canon_stream, 15),
+        Scalar::new(iterator, 29),
+        null(),
+        Span::new(9, 45),
+    );
+    assert_eq!(instruction, expected);
+}
+
 #[test]
 fn comments() {
     let source_code = r#"

crates/air-lib/air-parser/src/parser/tests/match_.rs

@@ -18,6 +18,10 @@ use super::dsl::*;
 use super::parse;
 use crate::ast::*;
 
+use air_lambda_ast::{LambdaAST, ValueAccessor};
+use fstrings::f;
+use fstrings::format_args_f;
+
 #[test]
 fn parse_match() {
     let source_code = r#"
@@ -34,6 +38,29 @@ fn parse_match() {
     assert_eq!(instruction, expected);
 }
 
+#[test]
+fn parse_match_with_canon_stream() {
+    let canon_stream = "#canon_stream";
+    let canon_stream_lambda = ".$.[0]";
+    let source_code = f!(r#"
+        (match {canon_stream}{canon_stream_lambda} v2
+            (null)
+        )
+        "#);
+
+    let instruction = parse(&source_code);
+    let expected = match_(
+        Value::Variable(VariableWithLambda::canon_stream_wl(
+            canon_stream,
+            unsafe { LambdaAST::new_unchecked(vec![ValueAccessor::ArrayAccess { idx: 0 }]) },
+            16,
+        )),
+        Value::Variable(VariableWithLambda::scalar("v2", 36)),
+        null(),
+    );
+    assert_eq!(instruction, expected);
+}
+
 #[test]
 fn parse_match_with_init_peer_id() {
     let source_code = r#"

crates/air-lib/air-parser/src/parser/tests/mod.rs

@@ -16,6 +16,7 @@
 
 mod ap;
 mod call;
+mod canon;
 mod dsl;
 mod fail;
 mod fold;
@@ -35,6 +36,7 @@ fn parse(source_code: &str) -> Instruction {
         let mut errors = Vec::new();
         let lexer = crate::parser::AIRLexer::new(source_code);
         let mut validator = crate::parser::VariableValidator::new();
+
         parser
             .parse(source_code, &mut errors, &mut validator, lexer)
             .expect("parsing should be successful")

crates/air-lib/air-parser/src/parser/tests/new.rs

@@ -29,7 +29,11 @@ fn parse_new_with_scalar() {
         "#;
 
     let instruction = parse(source_code);
-    let expected = new(Variable::scalar("scalar", 5), null(), Span::new(0, 40));
+    let expected = new(
+        NewArgument::Scalar(Scalar::new("scalar", 5)),
+        null(),
+        Span::new(0, 40),
+    );
     assert_eq!(instruction, expected);
 }
 
@@ -41,7 +45,27 @@ fn parse_new_with_stream() {
         "#;
 
     let instruction = parse(source_code);
-    let expected = new(Variable::stream("$stream", 5), null(), Span::new(0, 41));
+    let expected = new(
+        NewArgument::Stream(Stream::new("$stream", 5)),
+        null(),
+        Span::new(0, 41),
+    );
+    assert_eq!(instruction, expected);
+}
+
+#[test]
+fn parse_new_with_canon_stream() {
+    let source_code = r#"(new #canon_stream
+            (null)
+        )
+        "#;
+
+    let instruction = parse(source_code);
+    let expected = new(
+        NewArgument::CanonStream(CanonStream::new("#canon_stream", 5)),
+        null(),
+        Span::new(0, 47),
+    );
     assert_eq!(instruction, expected);
 }
 

crates/air-lib/air-parser/src/parser/tests/seq.rs

@@ -36,7 +36,7 @@ fn parse_seq() {
             CallInstrValue::Variable(VariableWithLambda::scalar("service_id", 41)),
             CallInstrValue::Variable(VariableWithLambda::scalar("function_name", 52)),
             Rc::new(vec![Value::EmptyArray, Value::EmptyArray]),
-            CallOutputValue::Variable(Variable::scalar("output", 75)),
+            CallOutputValue::Scalar(Scalar::new("output", 75)),
         ),
         call(
             CallInstrValue::Literal("peer_id"),
@@ -90,7 +90,7 @@ fn parse_seq_seq() {
                 Value::Literal("hello"),
                 Value::Variable(VariableWithLambda::scalar("name", 236)),
             ]),
-            CallOutputValue::Variable(Variable::stream("$output", 242)),
+            CallOutputValue::Stream(Stream::new("$output", 242)),
         ),
     );
     assert_eq!(instruction, expected);

crates/air-lib/air-parser/src/parser/validator.rs

@@ -20,6 +20,7 @@ use crate::parser::lexer::Token;
 use crate::parser::ParserError;
 use crate::parser::Span;
 
+use air_lambda_ast::LambdaAST;
 use air_lambda_ast::ValueAccessor;
 use lalrpop_util::ErrorRecovery;
 use lalrpop_util::ParseError;
@@ -71,11 +72,17 @@ impl<'i> VariableValidator<'i> {
         self.met_args(call.args.deref(), span);
 
         match &call.output {
-            CallOutputValue::Variable(variable) => self.met_variable_definition(variable, span),
+            CallOutputValue::Scalar(scalar) => self.met_variable_name_definition(scalar.name, span),
+            CallOutputValue::Stream(stream) => self.met_variable_name_definition(stream.name, span),
             CallOutputValue::None => {}
         };
     }
 
+    pub(super) fn met_canon(&mut self, canon: &Canon<'i>, span: Span) {
+        self.met_variable_name(canon.stream.name, span);
+        self.met_variable_name_definition(canon.canon_stream.name, span);
+    }
+
     pub(super) fn met_match(&mut self, match_: &Match<'i>, span: Span) {
         self.met_matchable(&match_.left_value, span);
         self.met_matchable(&match_.right_value, span);
@@ -90,7 +97,13 @@ impl<'i> VariableValidator<'i> {
         use FoldScalarIterable::*;
 
         match &fold.iterable {
-            Scalar(variable) => self.met_variable_name(variable.name, span),
+            Scalar(variable) => {
+                self.met_variable_name(variable.name, span);
+                self.met_maybe_lambda(&variable.lambda, span);
+            }
+            CanonStream(canon_stream) => {
+                self.met_variable_name(canon_stream.name, span);
+            }
             EmptyArray => {}
         };
         self.met_iterator_definition(&fold.iterator, span);
@@ -103,9 +116,9 @@ impl<'i> VariableValidator<'i> {
 
     pub(super) fn met_new(&mut self, new: &New<'i>, span: Span) {
         self.not_iterators_candidates
-            .push((new.variable.name(), span));
+            .push((new.argument.name(), span));
         // new defines a new variable
-        self.met_variable_definition(&new.variable, span);
+        self.met_variable_name_definition(new.argument.name(), span);
     }
 
     pub(super) fn met_next(&mut self, next: &Next<'i>, span: Span) {
@@ -128,10 +141,15 @@ impl<'i> VariableValidator<'i> {
             | ApArgument::EmptyArray
             | ApArgument::LastError(_) => {}
             ApArgument::Scalar(scalar) => {
-                self.met_variable_wl(&VariableWithLambda::Scalar(scalar.clone()), span)
+                self.met_variable_name(scalar.name, span);
+                self.met_maybe_lambda(&scalar.lambda, span);
+            }
+            ApArgument::CanonStream(canon_stream) => {
+                self.met_variable_name(canon_stream.name, span);
+                self.met_maybe_lambda(&canon_stream.lambda, span);
             }
         }
-        self.met_variable_definition(&ap.result, span);
+        self.met_variable_name_definition(ap.result.name(), span);
     }
 
     pub(super) fn finalize(self) -> Vec<ErrorRecovery<usize, Token<'i>, ParserError>> {
@@ -171,11 +189,24 @@ impl<'i> VariableValidator<'i> {
 
     fn met_variable_wl(&mut self, variable: &VariableWithLambda<'i>, span: Span) {
         self.met_variable_name(variable.name(), span);
-        let lambda = match variable.lambda() {
+        self.met_maybe_lambda(variable.lambda(), span);
+    }
+
+    fn met_variable_name(&mut self, name: &'i str, span: Span) {
+        if !self.contains_variable(name, span) {
+            self.unresolved_variables.insert(name, span);
+        }
+    }
+
+    fn met_maybe_lambda(&mut self, lambda: &Option<LambdaAST<'i>>, span: Span) {
+        let lambda = match lambda {
             Some(lambda) => lambda,
             None => return,
         };
+        self.met_lambda(lambda, span)
+    }
 
+    fn met_lambda(&mut self, lambda: &LambdaAST<'i>, span: Span) {
         for accessor in lambda.iter() {
             match accessor {
                 &ValueAccessor::FieldAccessByScalar { scalar_name } => {
@@ -188,12 +219,6 @@ impl<'i> VariableValidator<'i> {
         }
     }
 
-    fn met_variable_name(&mut self, name: &'i str, span: Span) {
-        if !self.contains_variable(name, span) {
-            self.unresolved_variables.insert(name, span);
-        }
-    }
-
     fn contains_variable(&self, key: &str, key_span: Span) -> bool {
         if let Some(found_span) = self.met_variable_definitions.get(key) {
             if found_span < &key_span {
@@ -209,10 +234,6 @@ impl<'i> VariableValidator<'i> {
         found_spans.iter().any(|s| s < &key_span)
     }
 
-    fn met_variable_definition(&mut self, variable: &Variable<'i>, span: Span) {
-        self.met_variable_name_definition(variable.name(), span);
-    }
-
     fn met_variable_name_definition(&mut self, name: &'i str, span: Span) {
         use std::collections::hash_map::Entry;
 

crates/air-lib/interpreter-data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## Version 0.3.0
+
+[PR 292](https://github.com/fluencelabs/aquavm/pull/292):  
+- added a new state in data for a canon instruction result
+
 ## Version 0.2.2
 
 [PR 169](https://github.com/fluencelabs/aquavm/pull/169):  

crates/air-lib/interpreter-data/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "air-interpreter-data"
 description = "Data format of the AIR interpreter"
-version = "0.2.2"
+version = "0.3.0"
 authors = ["Fluence Labs"]
 edition = "2018"
 license = "Apache-2.0"

crates/air-lib/interpreter-data/src/executed_state.rs

@@ -124,6 +124,14 @@ pub struct ApResult {
     pub res_generations: Vec<u32>,
 }
 
+/// Contains ids of element that were on a stream at the moment of an appropriate canon call.
+#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub struct CanonResult {
+    #[serde(rename = "ids")]
+    pub stream_elements_pos: Vec<TracePos>,
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(rename_all = "snake_case")]
 pub enum ExecutedState {
@@ -132,4 +140,5 @@ pub enum ExecutedState {
     Call(CallResult),
     Fold(FoldResult),
     Ap(ApResult),
+    Canon(CanonResult),
 }

crates/air-lib/interpreter-data/src/executed_state/impls.rs

@@ -86,6 +86,14 @@ impl ApResult {
     }
 }
 
+impl CanonResult {
+    pub fn new(stream_elements_pos: Vec<TracePos>) -> Self {
+        Self {
+            stream_elements_pos,
+        }
+    }
+}
+
 impl std::fmt::Display for ExecutedState {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         use CallResult::*;
@@ -121,6 +129,9 @@ impl std::fmt::Display for ExecutedState {
             Ap(ap) => {
                 write!(f, "ap: _ -> {:?}", ap.res_generations)
             }
+            Canon(canon) => {
+                write!(f, "canon {:?}", canon.stream_elements_pos)
+            }
         }
     }
 }

crates/air-lib/polyplets/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "polyplets"
-version = "0.3.0"
+version = "0.3.1"
 description = "Security primitives to verify origin of service calls in Fluence network"
 authors = ["Fluence Labs"]
 edition = "2018"

crates/air-lib/polyplets/src/tetraplet.rs

@@ -79,3 +79,15 @@ impl SecurityTetraplet {
         self.json_path.push_str(json_path)
     }
 }
+
+use std::fmt;
+
+impl fmt::Display for SecurityTetraplet {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "peer_pk: {}, service_id: {}, function_name: {}, json_path: {}",
+            self.peer_pk, self.service_id, self.function_name, self.json_path
+        )
+    }
+}

crates/air-lib/test-utils/src/executed_state.rs

@@ -16,6 +16,7 @@
 
 use super::ApResult;
 use super::CallResult;
+use super::CanonResult;
 use super::ExecutedState;
 use super::JValue;
 use super::ParResult;
@@ -138,6 +139,11 @@ pub fn ap(dst: Option<u32>) -> ExecutedState {
     ExecutedState::Ap(ap_result)
 }
 
+pub fn canon(stream_elements_pos: Vec<TracePos>) -> ExecutedState {
+    let canon_result = CanonResult::new(stream_elements_pos);
+    ExecutedState::Canon(canon_result)
+}
+
 fn option_to_vec(maybe_value: Option<u32>) -> Vec<u32> {
     match maybe_value {
         Some(value) => vec![value],

crates/air-lib/trace-handler/Cargo.toml

@@ -18,6 +18,7 @@ air-interpreter-data = { path = "../interpreter-data" }
 air-log-targets = { path = "../log-targets" }
 air-parser = { path = "../air-parser" }
 
+bimap = "0.6.2"
 serde_json = "1.0.68"
 log = "0.4.14"
 thiserror = "1.0.29"

crates/air-lib/trace-handler/src/data_keeper/keeper.rs

@@ -21,14 +21,15 @@ use crate::TracePos;
 
 use air_interpreter_data::InterpreterData;
 
-use std::collections::HashMap;
+use bimap::BiHashMap;
 
 /// Keeps all necessary data for merging.
 #[derive(Debug, Default, PartialEq)]
 pub(crate) struct DataKeeper {
     pub(crate) prev_ctx: MergeCtx,
     pub(crate) current_ctx: MergeCtx,
-    pub(crate) new_to_old_pos: HashMap<TracePos, DataPositions>,
+    pub(crate) new_to_prev_pos: BiHashMap<TracePos, TracePos>,
+    pub(crate) new_to_current_pos: BiHashMap<TracePos, TracePos>,
     pub(crate) result_trace: ExecutionTrace,
 }
 
@@ -40,7 +41,8 @@ impl DataKeeper {
         Self {
             prev_ctx,
             current_ctx,
-            new_to_old_pos: <_>::default(),
+            new_to_prev_pos: <_>::default(),
+            new_to_current_pos: <_>::default(),
             result_trace: <_>::default(),
         }
     }
@@ -69,9 +71,3 @@ impl DataKeeper {
         &mut self.current_ctx.slider
     }
 }
-
-#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
-pub(crate) struct DataPositions {
-    pub(crate) prev_pos: Option<TracePos>,
-    pub(crate) current_pos: Option<TracePos>,
-}

crates/air-lib/trace-handler/src/data_keeper/mod.rs

@@ -24,7 +24,6 @@ pub use merge_ctx::MergeCtx;
 pub use trace_slider::TraceSlider;
 
 pub(crate) use keeper::DataKeeper;
-pub(crate) use keeper::DataPositions;
 
 pub(self) type KeeperResult<T> = std::result::Result<T, KeeperError>;
 

crates/air-lib/trace-handler/src/data_keeper/trace_slider.rs

@@ -106,7 +106,7 @@ impl TraceSlider {
         self.subtrace_len - self.seen_elements
     }
 
-    pub(super) fn state_at_position(&self, position: TracePos) -> Option<&ExecutedState> {
+    pub(crate) fn state_at_position(&self, position: TracePos) -> Option<&ExecutedState> {
         // it would be nice to have the `impl SliceIndex for TracePos`, but it is unstable
         self.trace.get(position)
     }

crates/air-lib/trace-handler/src/handler.rs

@@ -86,6 +86,16 @@ impl TraceHandler {
     }
 }
 
+impl TraceHandler {
+    pub fn meet_canon_start(&mut self) -> TraceHandlerResult<MergerCanonResult> {
+        try_merge_next_state_as_canon(&mut self.data_keeper).map_err(Into::into)
+    }
+
+    pub fn meet_canon_end(&mut self, canon_result: CanonResult) {
+        self.data_keeper.result_trace.push(ExecutedState::Canon(canon_result));
+    }
+}
+
 impl TraceHandler {
     pub fn meet_par_start(&mut self) -> TraceHandlerResult<()> {
         let ingredients = merger::try_merge_next_state_as_par(&mut self.data_keeper)?;

crates/air-lib/trace-handler/src/lib.rs

@@ -14,6 +14,17 @@
  * limitations under the License.
  */
 
+#![warn(rust_2018_idioms)]
+#![deny(
+    dead_code,
+    nonstandard_style,
+    unused_imports,
+    unused_mut,
+    unused_variables,
+    unused_unsafe,
+    unreachable_patterns
+)]
+
 mod data_keeper;
 mod errors;
 mod handler;
@@ -29,6 +40,7 @@ pub use merger::MergeCtxType;
 pub use merger::MergeError;
 pub use merger::MergerApResult;
 pub use merger::MergerCallResult;
+pub use merger::MergerCanonResult;
 pub use state_automata::SubgraphType;
 
 pub type TraceHandlerResult<T> = std::result::Result<T, TraceHandlerError>;

crates/air-lib/trace-handler/src/merger/call_merger.rs

@@ -117,10 +117,8 @@ pub(crate) enum ValueType<'i> {
 
 impl<'i> ValueType<'i> {
     pub(self) fn from_output_value(output_value: &'i CallOutputValue<'_>) -> Self {
-        use air_parser::ast::Variable;
-
         match output_value {
-            CallOutputValue::Variable(Variable::Stream(stream)) => ValueType::Stream(stream.name),
+            CallOutputValue::Stream(stream) => ValueType::Stream(stream.name),
             _ => ValueType::Scalar,
         }
     }

crates/air-lib/trace-handler/src/merger/canon_merger.rs

@@ -0,0 +1,135 @@
+/*
+ * Copyright 2022 Fluence Labs Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+use super::*;
+use crate::merger::errors::CanonResultError;
+
+use bimap::BiHashMap;
+
+const EXPECTED_STATE_NAME: &str = "canon";
+
+#[derive(Debug, Clone)]
+pub enum MergerCanonResult {
+    /// There is no corresponding state in a trace for this call.
+    Empty,
+
+    /// There was a state in at least one of the contexts. If there were two states in
+    /// both contexts, they were successfully merged.
+    /// Positions correspond to a new data trace.
+    CanonResult { stream_elements_pos: Vec<TracePos> },
+}
+
+pub(crate) fn try_merge_next_state_as_canon(data_keeper: &mut DataKeeper) -> MergeResult<MergerCanonResult> {
+    use ExecutedState::Canon;
+
+    let prev_state = data_keeper.prev_slider_mut().next_state();
+    let current_state = data_keeper.current_slider_mut().next_state();
+
+    match (prev_state, current_state) {
+        (Some(Canon(prev_canon)), Some(Canon(current_canon))) => {
+            prepare_both_canon_result(&prev_canon, &current_canon, data_keeper)
+        }
+        (Some(Canon(prev_canon)), None) => prepare_single_canon_result(&prev_canon, &data_keeper.new_to_prev_pos),
+        (None, Some(Canon(current_canon))) => {
+            prepare_single_canon_result(&current_canon, &data_keeper.new_to_current_pos)
+        }
+        (None, None) => Ok(MergerCanonResult::Empty),
+        (prev_state, current_state) => Err(MergeError::incompatible_states(
+            prev_state,
+            current_state,
+            EXPECTED_STATE_NAME,
+        )),
+    }
+}
+
+fn prepare_both_canon_result(
+    prev_canon_result: &CanonResult,
+    current_canon_result: &CanonResult,
+    data_keeper: &DataKeeper,
+) -> MergeResult<MergerCanonResult> {
+    check_canon_results(prev_canon_result, current_canon_result, data_keeper)
+        .map_err(MergeError::IncorrectCanonResult)?;
+    prepare_single_canon_result(prev_canon_result, &data_keeper.new_to_prev_pos)
+}
+
+fn prepare_single_canon_result(
+    canon_result: &CanonResult,
+    new_to_other_pos: &BiHashMap<TracePos, TracePos>,
+) -> MergeResult<MergerCanonResult> {
+    let new_positions = canon_result
+        .stream_elements_pos
+        .iter()
+        .map(|pos| {
+            new_to_other_pos
+                .get_by_right(pos)
+                .cloned()
+                .ok_or_else(|| CanonResultError::not_met_position(canon_result.clone(), *pos))
+        })
+        .collect::<Result<Vec<_>, _>>()?;
+
+    Ok(MergerCanonResult::CanonResult {
+        stream_elements_pos: new_positions,
+    })
+}
+
+fn check_canon_results(
+    prev_canon_result: &CanonResult,
+    current_canon_result: &CanonResult,
+    data_keeper: &DataKeeper,
+) -> Result<(), CanonResultError> {
+    if prev_canon_result.stream_elements_pos.len() != current_canon_result.stream_elements_pos.len() {
+        return Err(CanonResultError::different_lens(
+            prev_canon_result.clone(),
+            current_canon_result.clone(),
+        ));
+    }
+
+    let prev_slider = data_keeper.prev_slider();
+    let current_slider = data_keeper.current_slider();
+    for (position, (prev_idx, current_idx)) in prev_canon_result
+        .stream_elements_pos
+        .iter()
+        .zip(current_canon_result.stream_elements_pos.iter())
+        .enumerate()
+    {
+        let prev_state = prev_slider.state_at_position(*prev_idx);
+        let current_state = current_slider.state_at_position(*current_idx);
+
+        match (prev_state, current_state) {
+            (Some(ExecutedState::Call(prev_call_result)), Some(ExecutedState::Call(current_call_result)))
+                if prev_call_result == current_call_result =>
+            {
+                continue;
+            }
+            (Some(ExecutedState::Ap(prev_ap_result)), Some(ExecutedState::Ap(current_ap_result)))
+                if prev_ap_result == current_ap_result =>
+            {
+                continue;
+            }
+            _ => {
+                return Err(CanonResultError::incompatible_state(
+                    prev_canon_result.clone(),
+                    current_canon_result.clone(),
+                    prev_state.cloned(),
+                    current_state.cloned(),
+                    position,
+                ))
+            }
+        }
+    }
+
+    Ok(())
+}

crates/air-lib/trace-handler/src/merger/errors.rs

@@ -22,6 +22,7 @@ use super::FoldResult;
 use super::KeeperError;
 use super::Value;
 
+use air_interpreter_data::CanonResult;
 use air_interpreter_data::TracePos;
 use thiserror::Error as ThisError;
 
@@ -46,6 +47,9 @@ pub enum MergeError {
     #[error(transparent)]
     IncorrectCallResult(#[from] CallResultError),
 
+    #[error(transparent)]
+    IncorrectCanonResult(#[from] CanonResultError),
+
     #[error(transparent)]
     IncorrectFoldResult(#[from] FoldResultError),
 }
@@ -73,6 +77,30 @@ pub enum CallResultError {
     DataNotMatchAIR { air_type: String, data_value: Value },
 }
 
+#[derive(ThisError, Debug)]
+pub enum CanonResultError {
+    #[error("canon results have different length: {prev_canon_result:?} != {current_canon_result:?}")]
+    LensNotEqual {
+        prev_canon_result: CanonResult,
+        current_canon_result: CanonResult,
+    },
+
+    #[error("canon results {prev_canon_result:?} {current_canon_result:?} at position {position} points to incompatible execution states: {prev_state:?} {current_state:?}")]
+    IncompatibleState {
+        prev_canon_result: CanonResult,
+        current_canon_result: CanonResult,
+        prev_state: Option<ExecutedState>,
+        current_state: Option<ExecutedState>,
+        position: usize,
+    },
+
+    #[error("position {position} from canon result {canon_result:?} hasn't been met yet")]
+    NotMetPosition {
+        canon_result: CanonResult,
+        position: TracePos,
+    },
+}
+
 #[derive(ThisError, Debug)]
 pub enum FoldResultError {
     #[error("the first {count} subtrace descriptors lens of fold {fold_result:?} overflows")]
@@ -138,6 +166,35 @@ impl CallResultError {
     }
 }
 
+impl CanonResultError {
+    pub(crate) fn different_lens(prev_canon_result: CanonResult, current_canon_result: CanonResult) -> Self {
+        Self::LensNotEqual {
+            prev_canon_result,
+            current_canon_result,
+        }
+    }
+
+    pub(crate) fn incompatible_state(
+        prev_canon_result: CanonResult,
+        current_canon_result: CanonResult,
+        prev_state: Option<ExecutedState>,
+        current_state: Option<ExecutedState>,
+        position: usize,
+    ) -> Self {
+        Self::IncompatibleState {
+            prev_canon_result,
+            current_canon_result,
+            prev_state,
+            current_state,
+            position,
+        }
+    }
+
+    pub(crate) fn not_met_position(canon_result: CanonResult, position: TracePos) -> Self {
+        Self::NotMetPosition { canon_result, position }
+    }
+}
+
 #[derive(Clone, Copy, Debug)]
 pub enum DataType {
     Previous,

crates/air-lib/trace-handler/src/merger/mod.rs

@@ -16,6 +16,7 @@
 
 mod ap_merger;
 mod call_merger;
+mod canon_merger;
 mod errors;
 mod fold_merger;
 mod par_merger;
@@ -23,6 +24,7 @@ mod position_mapping;
 
 pub use ap_merger::MergerApResult;
 pub use call_merger::MergerCallResult;
+pub use canon_merger::MergerCanonResult;
 pub use fold_merger::MergerFoldResult;
 pub use par_merger::MergerParResult;
 
@@ -36,6 +38,7 @@ pub use fold_merger::ResolvedSubTraceDescs;
 
 pub(super) use ap_merger::try_merge_next_state_as_ap;
 pub(super) use call_merger::try_merge_next_state_as_call;
+pub(super) use canon_merger::try_merge_next_state_as_canon;
 pub(crate) use fold_merger::try_merge_next_state_as_fold;
 pub(crate) use par_merger::try_merge_next_state_as_par;
 
@@ -44,7 +47,6 @@ use position_mapping::PreparationScheme;
 
 type MergeResult<T> = std::result::Result<T, MergeError>;
 
-use super::data_keeper::DataPositions;
 use super::data_keeper::KeeperError;
 use super::DataKeeper;
 

crates/air-lib/trace-handler/src/merger/position_mapping.rs

@@ -15,7 +15,6 @@
  */
 
 use super::DataKeeper;
-use super::DataPositions;
 
 pub(super) enum PreparationScheme {
     Previous,
@@ -27,19 +26,23 @@ pub(super) enum PreparationScheme {
 pub(super) fn prepare_positions_mapping(scheme: PreparationScheme, data_keeper: &mut DataKeeper) {
     use PreparationScheme::*;
 
-    // it's safe to sub 1 from positions iff scheme was set correctly
+    let new_pos = data_keeper.result_trace_next_pos();
-    let prev_pos = match scheme {
+
-        Previous | Both => Some(data_keeper.prev_slider().position() - 1),
+    // it's safe to sub 1 from positions here iff scheme was set correctly
-        Current => None,
+    match scheme {
+        Previous => {
+            let prev_pos = data_keeper.prev_slider().position() - 1;
+            data_keeper.new_to_prev_pos.insert(new_pos, prev_pos);
+        }
+        Current => {
+            let current_pos = data_keeper.current_slider().position() - 1;
+            data_keeper.new_to_current_pos.insert(new_pos, current_pos);
+        }
+        Both => {
+            let prev_pos = data_keeper.prev_slider().position() - 1;
+            let current_pos = data_keeper.current_slider().position() - 1;
+            data_keeper.new_to_prev_pos.insert(new_pos, prev_pos);
+            data_keeper.new_to_current_pos.insert(new_pos, current_pos);
+        }
     };
-
-    let current_pos = match scheme {
-        Current | Both => Some(data_keeper.current_slider().position() - 1),
-        Previous => None,
-    };
-
-    let data_positions = DataPositions { prev_pos, current_pos };
-
-    let trace_pos = data_keeper.result_trace_next_pos();
-    data_keeper.new_to_old_pos.insert(trace_pos, data_positions);
 }

crates/air-lib/trace-handler/src/state_automata/fold_fsm.rs

@@ -66,13 +66,11 @@ impl FoldFSM {
     }
 
     pub(crate) fn meet_iteration_start(&mut self, value_pos: TracePos, data_keeper: &mut DataKeeper) -> FSMResult<()> {
-        let (prev_pos, current_pos) = match data_keeper.new_to_old_pos.get(&value_pos) {
+        let prev_pos = data_keeper.new_to_prev_pos.get_by_left(&value_pos);
-            Some(DataPositions { prev_pos, current_pos }) => (prev_pos, current_pos),
+        let current_pos = data_keeper.new_to_current_pos.get_by_left(&value_pos);
-            None => return self.prepare(None, None, value_pos, data_keeper),
-        };
 
-        let prev_lore = prev_pos.map(|pos| self.prev_fold.lore.remove(&pos)).flatten();
+        let prev_lore = prev_pos.and_then(|pos| self.prev_fold.lore.remove(pos));
-        let current_lore = current_pos.map(|pos| self.current_fold.lore.remove(&pos)).flatten();
+        let current_lore = current_pos.and_then(|pos| self.current_fold.lore.remove(pos));
 
         self.prepare(prev_lore, current_lore, value_pos, data_keeper)
     }

crates/air-lib/trace-handler/src/state_automata/mod.rs

@@ -30,7 +30,6 @@ pub(super) use fold_fsm::FoldFSM;
 pub(super) use fsm_queue::FSMKeeper;
 pub(super) use par_fsm::ParFSM;
 
-use super::data_keeper::DataPositions;
 use super::data_keeper::KeeperError;
 use super::merger::MergerParResult;
 use super::DataKeeper;

crates/beautifier/src/beautifier.rs

@@ -135,6 +135,7 @@ impl<W: io::Write> Beautifier<W> {
         match node {
             ast::Instruction::Call(call) => self.beautify_call(call, indent),
             ast::Instruction::Ap(ap) => self.beautify_simple(ap, indent),
+            ast::Instruction::Canon(canon) => self.beautify_simple(canon, indent),
             ast::Instruction::Seq(seq) => self.beautify_seq(seq, indent),
             ast::Instruction::Par(par) => self.beautify_par(par, indent),
             ast::Instruction::Xor(xor) => self.beautify_xor(xor, indent),
@@ -157,7 +158,8 @@ impl<W: io::Write> Beautifier<W> {
     fn beautify_call(&mut self, call: &ast::Call, indent: usize) -> io::Result<()> {
         fmt_indent(&mut self.output, indent)?;
         match &call.output {
-            ast::CallOutputValue::Variable(v) => write!(&mut self.output, "{} <- ", v)?,
+            ast::CallOutputValue::Scalar(v) => write!(&mut self.output, "{} <- ", v)?,
+            ast::CallOutputValue::Stream(v) => write!(&mut self.output, "{} <- ", v)?,
             ast::CallOutputValue::None => {}
         }
         writeln!(