fix: JS-client bugs and tech debt [fixes DXJ-520] (#374)

Fix various bugs and a pair of TODO's
This commit is contained in:
Akim 2023-11-06 17:42:24 +07:00 committed by GitHub
parent 2589e39113
commit b460491fbd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 121 additions and 210 deletions

View File

@ -90,7 +90,12 @@ jobs:
- run: pnpm -r i
- run: pnpm -r build
- run: pnpm lint-check
- name: Lint code and fix all fixable errors
run: pnpm lint-fix
- name: Auto-commit
uses: stefanzweifel/git-auto-commit-action@v4
- name: Override dependencies
uses: fluencelabs/github-actions/pnpm-set-dependency@main

4
.gitignore vendored
View File

@ -6,6 +6,6 @@
node_modules/
# Build directory
**/dist
**/public
dist/
public/
.DS_Store

View File

@ -9,4 +9,4 @@ pnpm-lock.yaml
**/CHANGELOG.md
packages/core/js-client/src/versions.ts
packages/core/js-client-isomorphic/src/versions.ts

View File

@ -1,21 +0,0 @@
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
bundle/
dist
esm
types
# Dependency directories
node_modules/
jspm_packages/
.idea

View File

@ -14,8 +14,7 @@
* limitations under the License.
*/
import { Fluence } from "@fluencelabs/js-client";
import type { ClientConfig } from "@fluencelabs/js-client";
import { Fluence, type ClientConfig } from "@fluencelabs/js-client";
import { fromByteArray } from "base64-js";
import { test as particleTest } from "./_aqua/finalize_particle.js";

View File

@ -1,21 +0,0 @@
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
bundle/
dist
esm
types
# Dependency directories
node_modules/
jspm_packages/
.idea

View File

@ -14,7 +14,6 @@
* limitations under the License.
*/
import "@fluencelabs/js-client";
import { runTest } from "@test/aqua_for_test";
await runTest();

View File

@ -1,25 +1 @@
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
# dependencies
/node_modules
/.pnp
.pnp.js
# testing
/coverage
# production
/build
# misc
.DS_Store
.env.local
.env.development.local
.env.test.local
.env.production.local
npm-debug.log*
yarn-debug.log*
yarn-error.log*
public/js-client.min.js
build/

View File

@ -1,21 +0,0 @@
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
dist
# Dependency directories
node_modules/
jspm_packages/
.idea
public/js-client.min.js

View File

@ -0,0 +1 @@
src/versions.ts

View File

@ -8,7 +8,8 @@
],
"main": "index.js",
"scripts": {
"build": "tsc"
"build": "tsc",
"prepare": "node createVersionFile.js"
},
"exports": {
".": "./dist/types.js",
@ -24,7 +25,7 @@
"dependencies": {
"@fluencelabs/avm": "0.54.0",
"@fluencelabs/marine-js": "0.7.2",
"@fluencelabs/marine-worker": "workspace:*",
"@fluencelabs/marine-worker": "0.4.1",
"@fluencelabs/threads": "^2.0.0"
},
"keywords": [],

View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
import type { VersionedPackage } from "../types.js";
import { FetchedPackages, getVersionedPackage } from "../types.js";
/**
* @param pkg name of package with version
@ -22,7 +22,7 @@ import type { VersionedPackage } from "../types.js";
* @param root CDN domain in browser or file system root in node
*/
export async function fetchResource(
pkg: VersionedPackage,
pkg: FetchedPackages,
assetPath: string,
root: string,
) {
@ -30,7 +30,8 @@ export async function fetchResource(
? assetPath.slice(1)
: assetPath;
const url = new URL(`${pkg.name}@${pkg.version}/` + refinedAssetPath, root);
const { name, version } = getVersionedPackage(pkg);
const url = new URL(`${name}@${version}/` + refinedAssetPath, root);
return fetch(url).catch(() => {
throw new Error(`Cannot fetch from ${url.toString()}`);

View File

@ -18,7 +18,7 @@ import { readFile } from "fs/promises";
import { createRequire } from "module";
import { sep, posix, join } from "path";
import type { VersionedPackage } from "../types.js";
import { FetchedPackages, getVersionedPackage } from "../types.js";
/**
* @param pkg name of package with version
@ -26,24 +26,25 @@ import type { VersionedPackage } from "../types.js";
* @param root CDN domain in browser or js-client itself in node
*/
export async function fetchResource(
pkg: VersionedPackage,
pkg: FetchedPackages,
assetPath: string,
root: string,
) {
const { name } = getVersionedPackage(pkg);
// TODO: `root` will be handled somehow in the future. For now, we use filesystem root where js-client is running;
root = "/";
const require = createRequire(import.meta.url);
const packagePathIndex = require.resolve(pkg.name);
const packagePathIndex = require.resolve(name);
// Ensure that windows path is converted to posix path. So we can find a package
const posixPath = packagePathIndex.split(sep).join(posix.sep);
const matches = new RegExp(`(.+${pkg.name})`).exec(posixPath);
const matches = new RegExp(`(.+${name})`).exec(posixPath);
const packagePath = matches?.[0];
if (packagePath == null) {
throw new Error(`Cannot find dependency ${pkg.name} in path ${posixPath}`);
throw new Error(`Cannot find dependency ${name} in path ${posixPath}`);
}
const pathToResource = join(root, packagePath, assetPath);

View File

@ -16,8 +16,18 @@
import { Worker } from "@fluencelabs/threads/master";
export type VersionedPackage = { name: string; version: string };
import versions from "./versions.js";
export type FetchedPackages = keyof typeof versions;
type VersionedPackage = { name: string; version: string };
export type GetWorker = (
pkg: VersionedPackage,
pkg: FetchedPackages,
CDNUrl: string,
) => Promise<Worker>;
export const getVersionedPackage = (pkg: FetchedPackages): VersionedPackage => {
return {
name: pkg,
version: versions[pkg],
};
};

View File

@ -17,10 +17,10 @@
import { BlobWorker } from "@fluencelabs/threads/master";
import { fetchResource } from "../fetchers/browser.js";
import type { GetWorker, VersionedPackage } from "../types.js";
import type { FetchedPackages, GetWorker } from "../types.js";
export const getWorker: GetWorker = async (
pkg: VersionedPackage,
pkg: FetchedPackages,
CDNUrl: string,
) => {
const fetchWorkerCode = async () => {

View File

@ -20,14 +20,16 @@ import { fileURLToPath } from "url";
import { Worker } from "@fluencelabs/threads/master";
import type { GetWorker, VersionedPackage } from "../types.js";
import type { FetchedPackages, GetWorker } from "../types.js";
import { getVersionedPackage } from "../types.js";
export const getWorker: GetWorker = (pkg: VersionedPackage) => {
export const getWorker: GetWorker = (pkg: FetchedPackages) => {
const require = createRequire(import.meta.url);
const pathToThisFile = dirname(fileURLToPath(import.meta.url));
const pathToWorker = require.resolve(pkg.name);
const { name } = getVersionedPackage(pkg);
const pathToWorker = require.resolve(name);
const relativePathToWorker = relative(pathToThisFile, pathToWorker);

View File

@ -1,26 +0,0 @@
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
bundle/
dist
esm
types
# Dependency directories
node_modules/
jspm_packages/
.idea
# workaround to make integration tests work
src/marine/worker-script/index.js
src/versions.ts

View File

@ -24,8 +24,7 @@
"type": "module",
"scripts": {
"build": "tsc && vite build",
"test": "vitest --threads false run",
"prepare": "node createVersionFile.js"
"test": "vitest --threads false run"
},
"repository": "https://github.com/fluencelabs/fluence-js",
"author": "Fluence Labs",
@ -37,6 +36,7 @@
"@fluencelabs/interfaces": "workspace:*",
"@fluencelabs/js-client-isomorphic": "workspace:*",
"@fluencelabs/marine-worker": "0.4.1",
"@fluencelabs/threads": "^2.0.0",
"@libp2p/crypto": "2.0.3",
"@libp2p/interface": "0.1.2",
"@libp2p/peer-id": "3.0.2",
@ -55,7 +55,6 @@
"libp2p": "0.46.6",
"multiformats": "11.0.1",
"rxjs": "7.5.5",
"@fluencelabs/threads": "^2.0.0",
"ts-pattern": "3.3.3",
"uint8arrays": "4.0.3",
"uuid": "8.3.2",

View File

@ -14,7 +14,6 @@
* limitations under the License.
*/
import type { VersionedPackage } from "@fluencelabs/js-client-isomorphic";
import { fetchResource } from "@fluencelabs/js-client-isomorphic/fetcher";
import { getWorker } from "@fluencelabs/js-client-isomorphic/worker-resolver";
@ -27,17 +26,9 @@ import {
import { callAquaFunction } from "./compilerSupport/callFunction.js";
import { registerService } from "./compilerSupport/registerService.js";
import { MarineBackgroundRunner } from "./marine/worker/index.js";
import versions from "./versions.js";
const DEFAULT_CDN_URL = "https://unpkg.com";
const getVersionedPackage = (pkg: keyof typeof versions): VersionedPackage => {
return {
name: pkg,
version: versions[pkg],
};
};
const createClient = async (
relay: RelayOptions,
config: ClientConfig = {},
@ -46,7 +37,7 @@ const createClient = async (
const fetchMarineJsWasm = async () => {
const resource = await fetchResource(
getVersionedPackage("@fluencelabs/marine-js"),
"@fluencelabs/marine-js",
"/dist/marine-js.wasm",
CDNUrl,
);
@ -56,7 +47,7 @@ const createClient = async (
const fetchAvmWasm = async () => {
const resource = await fetchResource(
getVersionedPackage("@fluencelabs/avm"),
"@fluencelabs/avm",
"/dist/avm.wasm",
CDNUrl,
);
@ -70,10 +61,7 @@ const createClient = async (
const marine = new MarineBackgroundRunner(
{
async getValue() {
return getWorker(
getVersionedPackage("@fluencelabs/marine-worker"),
CDNUrl,
);
return getWorker("@fluencelabs/marine-worker", CDNUrl);
},
start() {
return Promise.resolve(undefined);
@ -178,7 +166,13 @@ export const Fluence: FluencePublicApi = {
},
};
export type { ClientConfig, IFluenceClient } from "./clientPeer/types.js";
export type {
ClientConfig,
IFluenceClient,
ConnectionState,
RelayOptions,
KeyPairOptions,
} from "./clientPeer/types.js";
export { v5_callFunction, v5_registerService } from "./api.js";

View File

@ -22,7 +22,6 @@ import {
KeyPairFormat,
serializeAvmArgs,
} from "@fluencelabs/avm";
import { defaultCallParameters } from "@fluencelabs/marine-js/dist/types";
import { fromUint8Array } from "js-base64";
import {
concatMap,
@ -201,12 +200,7 @@ export abstract class FluencePeer {
new Error("Can't use avm: peer is not initialized");
}
const res = await this.marineHost.callService(
"avm",
"ast",
[air],
defaultCallParameters,
);
const res = await this.marineHost.callService("avm", "ast", [air]);
if (!isString(res)) {
throw new Error(
@ -435,7 +429,6 @@ export abstract class FluencePeer {
"avm",
"invoke",
args,
defaultCallParameters,
);
avmCallResult = deserializeAvmResult(res);
@ -654,7 +647,6 @@ export abstract class FluencePeer {
req.serviceId,
req.fnName,
req.args,
defaultCallParameters,
);
return {

View File

@ -99,5 +99,5 @@ describe("FluencePeer flow tests", () => {
expect(res).toEqual(expect.arrayContaining(["test1", "test1"]));
});
}, 1500);
}, 1800);
});

View File

@ -20,9 +20,10 @@ import {
RunParameters,
} from "@fluencelabs/avm";
import { JSONObject, JSONValue, JSONArray } from "@fluencelabs/interfaces";
import { CallParameters } from "@fluencelabs/marine-worker";
import type { Worker as WorkerImplementation } from "@fluencelabs/threads/master";
import { IStartable, CallParameters } from "../util/commonTypes.js";
import { IStartable } from "../util/commonTypes.js";
/**
* Contract for marine host implementations. Marine host is responsible for creating calling and removing marine services
@ -53,7 +54,7 @@ export interface IMarineHost extends IStartable {
serviceId: string,
functionName: string,
args: JSONArray | JSONObject,
callParams: CallParameters,
callParams?: CallParameters,
): Promise<JSONValue>;
}

View File

@ -16,12 +16,11 @@
import { JSONValue } from "@fluencelabs/interfaces";
import type {
JSONArray,
JSONObject,
MarineBackgroundInterface,
LogFunction,
JSONValueNonNullable,
CallParameters,
} from "@fluencelabs/marine-js/dist/types";
import { LogFunction, logLevelToEnv } from "@fluencelabs/marine-js/dist/types";
import type { MarineBackgroundInterface } from "@fluencelabs/marine-worker";
} from "@fluencelabs/marine-worker";
import { ModuleThread, Thread, spawn } from "@fluencelabs/threads/master";
import { MarineLogger, marineLogger } from "../../util/logger.js";
@ -94,18 +93,15 @@ export class MarineBackgroundRunner implements IMarineHost {
throw new Error("Worker is not initialized");
}
// The logging level is controlled by the environment variable passed to enable debug logs.
// We enable all possible log levels passing the control for exact printouts to the logger
const env = logLevelToEnv("info");
this.loggers.set(serviceId, marineLogger(serviceId));
await this.workerThread.createService(serviceModule, serviceId, env);
await this.workerThread.createService(serviceModule, serviceId);
}
async callService(
serviceId: string,
functionName: string,
args: JSONArray | JSONObject,
callParams: CallParameters,
args: Array<JSONValueNonNullable> | Record<string, JSONValueNonNullable>,
callParams?: CallParameters,
): Promise<JSONValue> {
if (this.workerThread == null) {
throw new Error("Worker is not initialized");

View File

@ -14,8 +14,6 @@
* limitations under the License.
*/
export type { CallParameters } from "@fluencelabs/marine-js/dist/types";
export interface IStartable {
start(): Promise<void>;
stop(): Promise<void>;

View File

@ -16,7 +16,7 @@
import { promises as fs } from "fs";
import * as api from "@fluencelabs/aqua-api/aqua-api.js";
import { Path, Aqua } from "@fluencelabs/aqua-api/aqua-api.js";
import {
FunctionCallDef,
JSONArray,
@ -68,11 +68,16 @@ export type CompiledFile = {
services: { [key: string]: ServiceDef };
};
interface FunctionInfo {
script: string;
funcDef: FunctionCallDef;
}
export const compileAqua = async (aquaFile: string): Promise<CompiledFile> => {
await fs.access(aquaFile);
const compilationResult = await api.Aqua.compile(
new api.Path(aquaFile),
const compilationResult = await Aqua.compile(
new Path(aquaFile),
[],
undefined,
);
@ -84,12 +89,10 @@ export const compileAqua = async (aquaFile: string): Promise<CompiledFile> => {
}
const functions = Object.entries(compilationResult.functions)
.map(([name, fnInfo]) => {
.map(([name, fnInfo]: [string, FunctionInfo]) => {
const callFn = (peer: FluencePeer, args: PassedArgs) => {
return callAquaFunction({
// TODO: Set our compiler here and fix this
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
def: fnInfo.funcDef as FunctionCallDef,
def: fnInfo.funcDef,
script: fnInfo.script,
config: {},
peer: peer,
@ -105,9 +108,7 @@ export const compileAqua = async (aquaFile: string): Promise<CompiledFile> => {
return {
functions,
// TODO: set our compiler here and fix this
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
services: compilationResult.services as Record<string, ServiceDef>,
services: compilationResult.services,
};
};

View File

@ -26,7 +26,11 @@ import type {
JSONObject,
LogMessage,
} from "@fluencelabs/marine-js/dist/types";
import { JSONValue } from "@fluencelabs/marine-js/dist/types";
import {
defaultCallParameters,
JSONValue,
logLevelToEnv,
} from "@fluencelabs/marine-js/dist/types";
import { expose } from "@fluencelabs/threads/worker";
import { Observable, Subject } from "observable-fns";
@ -81,7 +85,12 @@ const toExpose = {
throw new Error(`Service with name ${serviceId} already registered`);
}
const marineConfig = createSimpleMarineService(serviceId, envs);
const marineConfig = createSimpleMarineService(serviceId, {
// We enable all possible log levels passing the control for exact printouts to the logger
...logLevelToEnv("info"),
...envs,
});
const modules = { [serviceId]: new Uint8Array(wasm) };
const srv = new MarineService(
@ -123,7 +132,7 @@ const toExpose = {
serviceId: string,
functionName: string,
args: JSONArray | JSONObject,
callParams: CallParameters,
callParams: CallParameters = defaultCallParameters,
) => {
const srv = marineServices.get(serviceId);
@ -142,5 +151,11 @@ const toExpose = {
};
export type MarineBackgroundInterface = typeof toExpose;
export type {
LogFunction,
LogMessage,
JSONValue as JSONValueNonNullable,
CallParameters,
} from "@fluencelabs/marine-js/dist/types";
expose(toExpose);

39
pnpm-lock.yaml generated
View File

@ -249,7 +249,7 @@ importers:
version: 6.0.3
debug:
specifier: 4.3.4
version: 4.3.4(supports-color@8.1.1)
version: 4.3.4
it-length-prefixed:
specifier: 8.0.4
version: 8.0.4
@ -313,7 +313,7 @@ importers:
version: 4.4.11(@types/node@20.7.0)
vite-tsconfig-paths:
specifier: 4.0.3
version: 4.0.3(typescript@5.1.6)(vite@4.4.11)
version: 4.0.3(vite@4.4.11)
vitest:
specifier: 0.34.6
version: 0.34.6
@ -327,7 +327,7 @@ importers:
specifier: 0.7.2
version: 0.7.2
'@fluencelabs/marine-worker':
specifier: workspace:*
specifier: 0.4.1
version: link:../marine-worker
'@fluencelabs/threads':
specifier: ^2.0.0
@ -4047,7 +4047,7 @@ packages:
resolution: {integrity: sha512-dgYpZg55OcEmop1U3G2bFKEJXg2avjXWYfWsdPlkSbHOHguaRifvr5bgwIYTg1wxoPGcn0jegcjKKwrY0qrV+g==}
dependencies:
callsites: 3.1.0
debug: 4.3.4(supports-color@8.1.1)
debug: 4.3.4
is-observable: 2.1.0
observable-fns: 0.6.1
optionalDependencies:
@ -4581,7 +4581,7 @@ packages:
dependencies:
'@libp2p/interface-peer-id': 2.0.2
'@multiformats/multiaddr': 12.1.7
debug: 4.3.4(supports-color@8.1.1)
debug: 4.3.4
interface-datastore: 8.2.3
multiformats: 11.0.2
transitivePeerDependencies:
@ -4592,7 +4592,7 @@ packages:
dependencies:
'@libp2p/interface': 0.1.2
'@multiformats/multiaddr': 12.1.7
debug: 4.3.4(supports-color@8.1.1)
debug: 4.3.4
interface-datastore: 8.2.3
multiformats: 12.0.1
transitivePeerDependencies:
@ -9492,6 +9492,17 @@ packages:
dependencies:
ms: 2.1.3
/debug@4.3.4:
resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==}
engines: {node: '>=6.0'}
peerDependencies:
supports-color: '*'
peerDependenciesMeta:
supports-color:
optional: true
dependencies:
ms: 2.1.2
/debug@4.3.4(supports-color@8.1.1):
resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==}
engines: {node: '>=6.0'}
@ -9808,7 +9819,7 @@ packages:
resolution: {integrity: sha512-Lm/eXB7yAQLJ5WxlBGwYfBY7utduXPZykcSmcG6K7ozM0wrZFvxZavhT6PqI0kd/5CUTfev/RrEFQqyU4CGPew==}
engines: {node: '>=16.0.0', npm: '>=7.0.0'}
dependencies:
debug: 4.3.4(supports-color@8.1.1)
debug: 4.3.4
native-fetch: 4.0.2(undici@5.22.1)
receptacle: 1.3.2
undici: 5.22.1
@ -19003,7 +19014,7 @@ packages:
resolution: {integrity: sha512-tL0w8U/pgaacOmkb9fRlYzWEUDCfVjjv9dD4wHTgZ61MjhuMt46VNWTG747NqW6vRzoWIKABVhFSOJ82FvXrfA==}
dev: false
/tsconfck@2.1.1(typescript@5.1.6):
/tsconfck@2.1.1:
resolution: {integrity: sha512-ZPCkJBKASZBmBUNqGHmRhdhM8pJYDdOXp4nRgj/O0JwUwsMq50lCDRQP/M5GBNAA0elPrq4gAeu4dkaVCuKWww==}
engines: {node: ^14.13.1 || ^16 || >=18}
hasBin: true
@ -19012,8 +19023,6 @@ packages:
peerDependenciesMeta:
typescript:
optional: true
dependencies:
typescript: 5.1.6
dev: true
/tsconfig-paths@3.14.2:
@ -19501,7 +19510,7 @@ packages:
hasBin: true
dependencies:
cac: 6.7.14
debug: 4.3.4(supports-color@8.1.1)
debug: 4.3.4
mlly: 1.4.2
pathe: 1.1.1
picocolors: 1.0.0
@ -19517,14 +19526,14 @@ packages:
- terser
dev: true
/vite-tsconfig-paths@4.0.3(typescript@5.1.6)(vite@4.4.11):
/vite-tsconfig-paths@4.0.3(vite@4.4.11):
resolution: {integrity: sha512-gRO2Q/tOkV+9kMht5tz90+IaEKvW2zCnvwJV3tp2ruPNZOTM5rF+yXorJT4ggmAMYEaJ3nyXjx5P5jY5FwiZ+A==}
peerDependencies:
vite: '>2.0.0-0'
dependencies:
debug: 4.3.4(supports-color@8.1.1)
debug: 4.3.4
globrex: 0.1.2
tsconfck: 2.1.1(typescript@5.1.6)
tsconfck: 2.1.1
vite: 4.4.11(@types/node@20.7.0)
transitivePeerDependencies:
- supports-color
@ -19646,7 +19655,7 @@ packages:
acorn-walk: 8.2.0
cac: 6.7.14
chai: 4.3.10
debug: 4.3.4(supports-color@8.1.1)
debug: 4.3.4
local-pkg: 0.4.3
magic-string: 0.30.5
pathe: 1.1.1