feat(interface)!: Use MessagePack for calls (#780)

Top-level structs use multicodec-prefixed MessagePack, the nested
structures (arguments, tetraplets) use just ordinary MessagePack.

* JS-compatible `CallResults`

Binary format implementations like `rmp-serde` do not bother converting keys from strings, unlike `serde_json`.
So, we use `CallResults` with string keys, as JS client cannot produce anything else.

---------

Co-authored-by: Mike Voronov <michail.vms@gmail.com>

air/src/execution_step/instructions/call/prev_result_handler.rs

@@ -80,7 +80,9 @@ pub(super) fn handle_prev_state<'i>(
         RequestSentBy(Sender::PeerIdWithCallId { ref peer_id, call_id })
             if peer_id.as_str() == exec_ctx.run_parameters.current_peer_id.as_str() =>
         {
-            // call results are identified by call_id that is saved in data
+            // call results are identified by call_id that is saved in data;
+            // for compatiblity with JavaScript with binary formats, string IDs are used
+            let call_id = call_id.to_string();
             match exec_ctx.call_results.remove(&call_id) {
                 Some(call_result) => {
                     update_state_with_service_result(

air/tests/test_module/negative_tests/farewell_step.rs

@@ -35,7 +35,7 @@ fn unprocessed_call_result() {
 
     let expected_call_service_result = air_interpreter_interface::CallServiceResult::ok(&json!("null"));
     let expected_call_results = maplit::hashmap!(
-        70 => expected_call_service_result,
+        "70".to_owned() => expected_call_service_result,
     );
     let expected_error = FarewellError::UnprocessedCallResult(expected_call_results);
     assert!(check_error(&result, expected_error));

avm/client/src/avmHelpers.ts

@@ -15,16 +15,16 @@
  */
 
 import { CallResultsArray, InterpreterResult, CallRequest, RunParameters, JSONArray, JSONObject } from './types';
-import { JsonRepr } from './formats'
+import { MulticodecRepr, MsgPackRepr } from './formats'
 
 // Have to match the air-interpreter-interface.
-const callRequestsRepr = new JsonRepr();
+const callRequestsRepr = new MulticodecRepr(new MsgPackRepr());
 // Have to match the air-interpreter-interface.
-const argumentRepr = new JsonRepr();
+const argumentRepr = new MsgPackRepr();
 // Have to match the air-interpreter-interface.
-const tetrapletRepr = new JsonRepr();
+const tetrapletRepr = new MsgPackRepr();
 // Have to match the air-interpreter-interface.
-const callResultsRepr = new JsonRepr();
+const callResultsRepr = new MulticodecRepr(new MsgPackRepr());
 
 /**
  * Encodes arguments into JSON array suitable for marine-js

avm/client/src/formats.ts

@@ -93,7 +93,7 @@ export class MulticodecRepr implements Representation {
 
     toBinary(obj: object): Uint8Array {
         let bareData = this.serializer.toBinary(obj);
-        let varintCode = multicodec.getVarintFromCode(this.serializer.get_code());
-        return multicodec.addPrefix(varintCode, bareData)
+        let codeName = multicodec.getNameFromCode(this.serializer.get_code());
+        return multicodec.addPrefix(codeName, bareData)
     }
 }

avm/interface/src/call_service_result.rs

@@ -63,7 +63,7 @@ impl CallServiceResult {
 pub fn into_raw_result(call_results: CallResults) -> air_interpreter_interface::CallResults {
     call_results
         .into_iter()
-        .map(|(call_id, call_result)| (call_id, call_result.into_raw()))
+        .map(|(call_id, call_result)| (call_id.to_string(), call_result.into_raw()))
         .collect::<_>()
 }
 

crates/air-lib/interpreter-interface/Cargo.toml

@@ -17,7 +17,7 @@ path = "src/lib.rs"
 [dependencies]
 marine-rs-sdk = {version = "0.10.2", optional = true }
 fluence-it-types = { version = "0.4.1", optional = true }
-air-interpreter-sede = { version = "0.1.0", path = "../interpreter-sede", features = ["json"] }
+air-interpreter-sede = { version = "0.1.0", path = "../interpreter-sede", features = ["msgpack"] }
 
 marine-call-parameters = { version = "0.10.3", default-features = false }
 serde = "1.0.190"

crates/air-lib/interpreter-interface/src/call_request_parameters.rs

@@ -18,7 +18,8 @@ use air_interpreter_sede::define_simple_representation;
 use air_interpreter_sede::derive_serialized_type;
 use air_interpreter_sede::Format;
 use air_interpreter_sede::FromSerialized;
-use air_interpreter_sede::JsonFormat;
+use air_interpreter_sede::MsgPackFormat;
+use air_interpreter_sede::MsgPackMultiformat;
 use air_interpreter_sede::Representation;
 
 use marine_call_parameters::SecurityTetraplet;
@@ -36,9 +37,9 @@ derive_serialized_type!(SerializedCallArguments);
 derive_serialized_type!(SerializedTetraplets);
 derive_serialized_type!(SerializedCallRequests);
 
-pub type CallArgumentsFormat = JsonFormat;
-pub type TetrapletsFormat = JsonFormat;
-pub type CallRequestsFormat = JsonFormat;
+pub type CallArgumentsFormat = MsgPackFormat;
+pub type TetrapletsFormat = MsgPackFormat;
+pub type CallRequestsFormat = MsgPackMultiformat;
 
 define_simple_representation! {
     CallArgumentsRepr,
@@ -74,7 +75,7 @@ impl FromSerialized<Vec<Vec<SecurityTetraplet>>> for TetrapletsRepr {
         &self,
         repr: &[u8],
     ) -> Result<Vec<Vec<SecurityTetraplet>>, Self::DeserializeError> {
-        CallArgumentsRepr.get_format().from_slice(repr)
+        Self.get_format().from_slice(repr)
     }
 }
 

crates/air-lib/interpreter-interface/src/call_service_result.rs

@@ -16,17 +16,19 @@
 
 use air_interpreter_sede::define_simple_representation;
 use air_interpreter_sede::derive_serialized_type;
-use air_interpreter_sede::JsonFormat;
+use air_interpreter_sede::MsgPackMultiformat;
 use air_interpreter_sede::Representation;
 use serde::Deserialize;
 use serde::Serialize;
 use serde_json::Value as JValue;
 use std::collections::HashMap;
 
-pub type CallResults = HashMap<u32, CallServiceResult>;
+/// This is a map from a String to a service result for compatibility with JavaScript.
+/// Binary format implementations like `rmp-serde` do not convert keys from strings, unlike `serde_json`.
+pub type CallResults = HashMap<String, CallServiceResult>;
 pub const CALL_SERVICE_SUCCESS: i32 = 0;
 
-pub type CallResultsFormat = JsonFormat;
+pub type CallResultsFormat = MsgPackMultiformat;
 
 derive_serialized_type!(SerializedCallResults);
 
@@ -55,6 +57,7 @@ impl CallServiceResult {
     pub fn ok(result: &JValue) -> Self {
         Self {
             ret_code: CALL_SERVICE_SUCCESS,
+            // for compatiblity with JavaScript with binary formats, string IDs are used
             result: result.to_string(),
         }
     }

crates/air-lib/interpreter-sede/src/rmp_serde.rs

@@ -35,7 +35,9 @@ where
 
     #[inline]
     fn to_vec(&self, val: &Value) -> Result<Vec<u8>, Self::SerializationError> {
-        rmp_serde::to_vec(val)
+        // named representation (i.e. structs are serialized as maps, not tuples) is important
+        // for JS interop and data compatibility detection
+        rmp_serde::to_vec_named(val)
     }
 
     #[inline]
@@ -49,7 +51,9 @@ where
         value: &Value,
         write: &mut W,
     ) -> Result<(), Self::WriteError> {
-        value.serialize(&mut rmp_serde::Serializer::new(write))
+        // named representation (i.e. structs are serialized as maps, not tuples) is important
+        // for JS interop and data compatibility detection
+        rmp_serde::encode::write_named(write, value)
     }
 }