diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f266cf0a9..1e35cd068 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -43,7 +43,7 @@ jobs: uses: fluencelabs/aqua/.github/workflows/tests.yml@main with: js-client-snapshots: "${{ needs.js-client.outputs.js-client-snapshots }}" - nox-image: "fluencelabs/nox:unstable" + nox-image: "docker.fluence.dev/nox:feat-VM-407-msgpack-particle" flox: needs: - js-client @@ -51,4 +51,4 @@ jobs: uses: fluencelabs/flox/.github/workflows/tests.yml@main with: js-client-snapshots: "${{ needs.js-client.outputs.js-client-snapshots }}" - nox-image: "fluencelabs/nox:unstable" \ No newline at end of file + nox-image: "docker.fluence.dev/nox:feat-VM-407-msgpack-particle" \ No newline at end of file diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 7cfd8bf95..f2c4691ba 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -28,4 +28,4 @@ jobs: uses: ./.github/workflows/tests.yml with: ref: ${{ github.ref }} - nox-image: "fluencelabs/nox:unstable" + nox-image: "docker.fluence.dev/nox:feat-VM-407-msgpack-particle" diff --git a/packages/core/js-client/package.json b/packages/core/js-client/package.json index a20a96622..18cdd5446 100644 --- a/packages/core/js-client/package.json +++ b/packages/core/js-client/package.json @@ -30,7 +30,6 @@ "author": "Fluence Labs", "license": "Apache-2.0", "dependencies": { - "@libp2p/utils": "5.2.2", "@chainsafe/libp2p-noise": "14.0.0", "@chainsafe/libp2p-yamux": "6.0.1", "@fluencelabs/avm": "0.59.0", @@ -44,10 +43,12 @@ "@libp2p/peer-id": "4.0.5", "@libp2p/peer-id-factory": "4.0.5", "@libp2p/ping": "1.0.10", + "@libp2p/utils": "5.2.2", "@libp2p/websockets": "8.0.12", "@multiformats/multiaddr": "12.1.12", "bs58": "5.0.0", "debug": "4.3.4", + "int64-buffer": "1.0.1", "it-length-prefixed": "9.0.3", "it-map": "3.0.5", "it-pipe": "3.0.1", diff --git a/packages/core/js-client/src/connection/RelayConnection.ts b/packages/core/js-client/src/connection/RelayConnection.ts index 2e5e96be0..315d710d4 100644 --- a/packages/core/js-client/src/connection/RelayConnection.ts +++ b/packages/core/js-client/src/connection/RelayConnection.ts @@ -29,15 +29,13 @@ import map from "it-map"; import { pipe } from "it-pipe"; import { createLibp2p, Libp2p } from "libp2p"; import { Subject } from "rxjs"; -import { fromString } from "uint8arrays/from-string"; -import { toString } from "uint8arrays/to-string"; import { KeyPair } from "../keypair/index.js"; import { IParticle } from "../particle/interfaces.js"; import { buildParticleMessage, Particle, - serializeToString, + serializeParticle, } from "../particle/Particle.js"; import { throwHasNoPeerId } from "../util/libp2pUtils.js"; import { logger } from "../util/logger.js"; @@ -216,7 +214,7 @@ export class RelayConnection implements IConnection { const sink = stream.sink; - await pipe([fromString(serializeToString(particle))], encode, sink); + await pipe([serializeParticle(particle)], encode, sink); log.trace( "particle %s sent to %s", @@ -225,11 +223,11 @@ export class RelayConnection implements IConnection { ); } - private async processIncomingMessage(msg: string) { + private async processIncomingMessage(msg: Uint8Array) { let particle: Particle | undefined; try { - particle = Particle.fromString(msg); + particle = Particle.deserialize(msg); log.trace( "received particle %s from %s", @@ -290,7 +288,7 @@ export class RelayConnection implements IConnection { decode, (source) => { return map(source, (buf) => { - return toString(buf.subarray()); + return buf.subarray(); }); }, async (source) => { diff --git a/packages/core/js-client/src/particle/Particle.ts b/packages/core/js-client/src/particle/Particle.ts index cf3db44a9..9d8c7025c 100644 --- a/packages/core/js-client/src/particle/Particle.ts +++ b/packages/core/js-client/src/particle/Particle.ts @@ -14,9 +14,13 @@ * limitations under the License. */ -import { CallResultsArray } from "@fluencelabs/avm"; +import { + CallResultsArray, + MulticodecRepr, + MsgPackRepr, +} from "@fluencelabs/avm"; import { JSONValue } from "@fluencelabs/interfaces"; -import { fromUint8Array, toUint8Array } from "js-base64"; +import int64Buffer from "int64-buffer"; import { concat } from "uint8arrays/concat"; import { v4 as uuidv4 } from "uuid"; import { z } from "zod"; @@ -27,14 +31,16 @@ import { numberToLittleEndianBytes } from "../util/bytes.js"; import { IParticle } from "./interfaces.js"; +const particleRepr = new MulticodecRepr(new MsgPackRepr()); + const particleSchema = z.object({ id: z.string(), timestamp: z.number().positive(), script: z.string(), - data: z.string(), + data: z.instanceof(Uint8Array), ttl: z.number().positive(), init_peer_id: z.string(), - signature: z.array(z.number()), + signature: z.instanceof(Uint8Array), }); export class Particle implements IParticle { @@ -73,10 +79,10 @@ export class Particle implements IParticle { ); } - static fromString(str: string): Particle { - const json = JSON.parse(str); + static deserialize(bytes: Uint8Array): Particle { + const obj = particleRepr.fromBinary(bytes); - const res = particleSchema.safeParse(json); + const res = particleSchema.safeParse(obj); if (!res.success) { throw new Error( @@ -92,10 +98,10 @@ export class Particle implements IParticle { data.id, data.timestamp, data.script, - toUint8Array(data.data), + data.data, data.ttl, data.init_peer_id, - new Uint8Array(data.signature), + data.signature, ); } } @@ -154,16 +160,16 @@ export const cloneWithNewData = ( /** * Serializes particle into string suitable for sending through network */ -export const serializeToString = (particle: IParticle): string => { - return JSON.stringify({ +export const serializeParticle = (particle: IParticle): Uint8Array => { + return particleRepr.toBinary({ action: "Particle", id: particle.id, init_peer_id: particle.initPeerId, - timestamp: particle.timestamp, + timestamp: new int64Buffer.Uint64BE(particle.timestamp), ttl: particle.ttl, script: particle.script, signature: Array.from(particle.signature), - data: fromUint8Array(particle.data), + data: Array.from(particle.data), }); }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 70cdf660c..bea1ec934 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -262,6 +262,9 @@ importers: debug: specifier: 4.3.4 version: 4.3.4 + int64-buffer: + specifier: 1.0.1 + version: 1.0.1 it-length-prefixed: specifier: 9.0.3 version: 9.0.3 @@ -1898,6 +1901,8 @@ packages: uint8arraylist: 2.4.8 uint8arrays: 4.0.10 wherearewe: 2.0.1 + transitivePeerDependencies: + - supports-color dev: false /@chainsafe/libp2p-yamux@6.0.1: @@ -3144,6 +3149,8 @@ packages: protons-runtime: 5.2.2 uint8arraylist: 2.4.8 uint8arrays: 4.0.10 + transitivePeerDependencies: + - supports-color dev: false /@libp2p/crypto@4.0.1: @@ -8765,6 +8772,11 @@ packages: /int64-buffer@0.1.10: resolution: {integrity: sha512-v7cSY1J8ydZ0GyjUHqF+1bshJ6cnEVLo9EnjB8p+4HDRPZc9N5jjmvUV7NvEsqQOKyH0pmIBFWXVQbiS0+OBbA==} + /int64-buffer@1.0.1: + resolution: {integrity: sha512-+3azY4pXrjAupJHU1V9uGERWlhoqNswJNji6aD/02xac7oxol508AsMC5lxKhEqyZeDFy3enq5OGWXF4u75hiw==} + engines: {node: '>= 4.5.0'} + dev: false + /interface-datastore@8.2.10: resolution: {integrity: sha512-D8RuxMdjOPB+j6WMDJ+I2aXTDzUT6DIVjgzo1E+ODL7w8WrSFl9FXD2SYmgj6vVzdb7Kb5qmAI9pEnDZJz7ifg==} dependencies: