feat(js-client)!: Multiformat MsgPack for particle data (#422)

* feat(particle)!: Multiformat MsgPack for particle data

* Fix types

* fix(ci): use nox with msgpack protocol

* fix(avm): avm 0.59.0

* Fix uint64

* Fix

* fix(ci): enable nox debug logs

* Fix commonJS import

* Revert "fix(ci): enable nox debug logs"

This reverts commit ce5bc2e263.

---------

Co-authored-by: Akim Mamedov <akim99999999@gmail.com>
Co-authored-by: folex <0xdxdy@gmail.com>
Co-authored-by: Akim <59872966+akim-bow@users.noreply.github.com>

.github/workflows/e2e.yml

@@ -43,7 +43,7 @@ jobs:
     uses: fluencelabs/aqua/.github/workflows/tests.yml@main
     with:
       js-client-snapshots: "${{ needs.js-client.outputs.js-client-snapshots }}"
-      nox-image: "fluencelabs/nox:unstable"
+      nox-image: "docker.fluence.dev/nox:feat-VM-407-msgpack-particle"
   flox:
     needs:
       - js-client
@@ -51,4 +51,4 @@ jobs:
     uses: fluencelabs/flox/.github/workflows/tests.yml@main
     with:
       js-client-snapshots: "${{ needs.js-client.outputs.js-client-snapshots }}"
-      nox-image: "fluencelabs/nox:unstable"
+      nox-image: "docker.fluence.dev/nox:feat-VM-407-msgpack-particle"

.github/workflows/run-tests.yml

@@ -28,4 +28,4 @@ jobs:
     uses: ./.github/workflows/tests.yml
     with:
       ref: ${{ github.ref }}
-      nox-image: "fluencelabs/nox:unstable"
+      nox-image: "docker.fluence.dev/nox:feat-VM-407-msgpack-particle"

packages/core/js-client/package.json

@@ -30,7 +30,6 @@
   "author": "Fluence Labs",
   "license": "Apache-2.0",
   "dependencies": {
-    "@libp2p/utils": "5.2.2",
     "@chainsafe/libp2p-noise": "14.0.0",
     "@chainsafe/libp2p-yamux": "6.0.1",
     "@fluencelabs/avm": "0.59.0",
@@ -44,10 +43,12 @@
     "@libp2p/peer-id": "4.0.5",
     "@libp2p/peer-id-factory": "4.0.5",
     "@libp2p/ping": "1.0.10",
+    "@libp2p/utils": "5.2.2",
     "@libp2p/websockets": "8.0.12",
     "@multiformats/multiaddr": "12.1.12",
     "bs58": "5.0.0",
     "debug": "4.3.4",
+    "int64-buffer": "1.0.1",
     "it-length-prefixed": "9.0.3",
     "it-map": "3.0.5",
     "it-pipe": "3.0.1",

packages/core/js-client/src/connection/RelayConnection.ts

@@ -29,15 +29,13 @@ import map from "it-map";
 import { pipe } from "it-pipe";
 import { createLibp2p, Libp2p } from "libp2p";
 import { Subject } from "rxjs";
-import { fromString } from "uint8arrays/from-string";
-import { toString } from "uint8arrays/to-string";
 
 import { KeyPair } from "../keypair/index.js";
 import { IParticle } from "../particle/interfaces.js";
 import {
   buildParticleMessage,
   Particle,
-  serializeToString,
+  serializeParticle,
 } from "../particle/Particle.js";
 import { throwHasNoPeerId } from "../util/libp2pUtils.js";
 import { logger } from "../util/logger.js";
@@ -216,7 +214,7 @@ export class RelayConnection implements IConnection {
 
     const sink = stream.sink;
 
-    await pipe([fromString(serializeToString(particle))], encode, sink);
+    await pipe([serializeParticle(particle)], encode, sink);
 
     log.trace(
       "particle %s sent to %s",
@@ -225,11 +223,11 @@ export class RelayConnection implements IConnection {
     );
   }
 
-  private async processIncomingMessage(msg: string) {
+  private async processIncomingMessage(msg: Uint8Array) {
     let particle: Particle | undefined;
 
     try {
-      particle = Particle.fromString(msg);
+      particle = Particle.deserialize(msg);
 
       log.trace(
         "received particle %s from %s",
@@ -290,7 +288,7 @@ export class RelayConnection implements IConnection {
           decode,
           (source) => {
             return map(source, (buf) => {
-              return toString(buf.subarray());
+              return buf.subarray();
             });
           },
           async (source) => {

packages/core/js-client/src/particle/Particle.ts

@@ -14,9 +14,13 @@
  * limitations under the License.
  */
 
-import { CallResultsArray } from "@fluencelabs/avm";
+import {
+  CallResultsArray,
+  MulticodecRepr,
+  MsgPackRepr,
+} from "@fluencelabs/avm";
 import { JSONValue } from "@fluencelabs/interfaces";
-import { fromUint8Array, toUint8Array } from "js-base64";
+import int64Buffer from "int64-buffer";
 import { concat } from "uint8arrays/concat";
 import { v4 as uuidv4 } from "uuid";
 import { z } from "zod";
@@ -27,14 +31,16 @@ import { numberToLittleEndianBytes } from "../util/bytes.js";
 
 import { IParticle } from "./interfaces.js";
 
+const particleRepr = new MulticodecRepr(new MsgPackRepr());
+
 const particleSchema = z.object({
   id: z.string(),
   timestamp: z.number().positive(),
   script: z.string(),
-  data: z.string(),
+  data: z.instanceof(Uint8Array),
   ttl: z.number().positive(),
   init_peer_id: z.string(),
-  signature: z.array(z.number()),
+  signature: z.instanceof(Uint8Array),
 });
 
 export class Particle implements IParticle {
@@ -73,10 +79,10 @@ export class Particle implements IParticle {
     );
   }
 
-  static fromString(str: string): Particle {
+  static deserialize(bytes: Uint8Array): Particle {
-    const json = JSON.parse(str);
+    const obj = particleRepr.fromBinary(bytes);
 
-    const res = particleSchema.safeParse(json);
+    const res = particleSchema.safeParse(obj);
 
     if (!res.success) {
       throw new Error(
@@ -92,10 +98,10 @@ export class Particle implements IParticle {
       data.id,
       data.timestamp,
       data.script,
-      toUint8Array(data.data),
+      data.data,
       data.ttl,
       data.init_peer_id,
-      new Uint8Array(data.signature),
+      data.signature,
     );
   }
 }
@@ -154,16 +160,16 @@ export const cloneWithNewData = (
 /**
  * Serializes particle into string suitable for sending through network
  */
-export const serializeToString = (particle: IParticle): string => {
+export const serializeParticle = (particle: IParticle): Uint8Array => {
-  return JSON.stringify({
+  return particleRepr.toBinary({
     action: "Particle",
     id: particle.id,
     init_peer_id: particle.initPeerId,
-    timestamp: particle.timestamp,
+    timestamp: new int64Buffer.Uint64BE(particle.timestamp),
     ttl: particle.ttl,
     script: particle.script,
     signature: Array.from(particle.signature),
-    data: fromUint8Array(particle.data),
+    data: Array.from(particle.data),
   });
 };
 

pnpm-lock.yaml

@@ -262,6 +262,9 @@ importers:
       debug:
         specifier: 4.3.4
         version: 4.3.4
+      int64-buffer:
+        specifier: 1.0.1
+        version: 1.0.1
       it-length-prefixed:
         specifier: 9.0.3
         version: 9.0.3
@@ -1898,6 +1901,8 @@ packages:
       uint8arraylist: 2.4.8
       uint8arrays: 4.0.10
       wherearewe: 2.0.1
+    transitivePeerDependencies:
+      - supports-color
     dev: false
 
   /@chainsafe/libp2p-yamux@6.0.1:
@@ -3144,6 +3149,8 @@ packages:
       protons-runtime: 5.2.2
       uint8arraylist: 2.4.8
       uint8arrays: 4.0.10
+    transitivePeerDependencies:
+      - supports-color
     dev: false
 
   /@libp2p/crypto@4.0.1:
@@ -8765,6 +8772,11 @@ packages:
   /int64-buffer@0.1.10:
     resolution: {integrity: sha512-v7cSY1J8ydZ0GyjUHqF+1bshJ6cnEVLo9EnjB8p+4HDRPZc9N5jjmvUV7NvEsqQOKyH0pmIBFWXVQbiS0+OBbA==}
 
+  /int64-buffer@1.0.1:
+    resolution: {integrity: sha512-+3azY4pXrjAupJHU1V9uGERWlhoqNswJNji6aD/02xac7oxol508AsMC5lxKhEqyZeDFy3enq5OGWXF4u75hiw==}
+    engines: {node: '>= 4.5.0'}
+    dev: false
+    
   /interface-datastore@8.2.10:
     resolution: {integrity: sha512-D8RuxMdjOPB+j6WMDJ+I2aXTDzUT6DIVjgzo1E+ODL7w8WrSFl9FXD2SYmgj6vVzdb7Kb5qmAI9pEnDZJz7ifg==}
     dependencies: