fluencelabs/fluence-js
1
0
Fork 0
mirror of https://github.com/fluencelabs/fluence-js.git synced 2024-12-04 18:00:18 +00:00
Code Issues Projects Releases Wiki Activity

chore: fix build [fixes DXJ-482] (#352)

Browse Source 
* Remove additional node targeted build

* Fix test errors

* Typo fix

* Remove headless

* Prevent JSON error

* Cache json parsing

* Remove resource test as it's not working in new nox

* Fix test output

* enable smoke tests

* add puppeteer to deps

* Remove headless option
This commit is contained in:
Akim 2023-09-29 16:23:06 +07:00 committed by GitHub
parent 63e4ce3f84
commit 15a2c91917
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 414 additions and 706 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/tests.yml vendored
View File

@ -6,7 +6,7 @@ on:
nox-image:
description: "nox image tag"
type: string
default: "fluencelabs/nox:minimal_0.2.5"
default: "fluencelabs/nox:minimal_0.2.9"
avm-version:
description: "@fluencelabs/avm version"
type: string

12
packages/@tests/aqua/src/index.ts
View File

@ -53,15 +53,6 @@ export const runTest = async (): Promise<TestResult> => {
console.log('my peer id: ', client.getPeerId());
console.log('my sk id: ', fromByteArray(client.getPeerSecretKey()));
console.log('running resource test...');
const [res, errors] = await resourceTest('my_resource');
if (res === null) {
console.log('resource test failed, errors', errors);
return { type: 'failure', error: errors.join(', ') };
} else {
console.log('resource test finished, result', res);
}
console.log('running hello test...');
const hello = await helloTest();
console.log('hello test finished, result: ', hello);
@ -75,9 +66,8 @@ export const runTest = async (): Promise<TestResult> => {
console.log('marine test finished, result: ', marine);
const returnVal = {
res,
hello,
// marine,
marine,
};
return { type: 'success', data: JSON.stringify(returnVal) };
} finally {

2
packages/@tests/smoke/node/package.json
View File

@ -11,7 +11,7 @@
"type": "module",
"scripts": {
"build": "tsc",
"test_commented_out": "node --loader ts-node/esm ./src/index.ts"
"test": "node --loader ts-node/esm ./src/index.ts"
},
"repository": "https://github.com/fluencelabs/fluence-js",
"author": "Fluence Labs",

5
packages/@tests/smoke/web-cra-ts/package.json
View File

@ -19,10 +19,11 @@
"web-vitals": "2.1.4"
},
"devDependencies": {
"@test/test-utils": "workspace:^"
"@test/test-utils": "workspace:^",
"puppeteer": "19.7.2"
},
"scripts": {
"test_commented_out": "node --loader ts-node/esm ./test/index.ts",
"test": "node --loader ts-node/esm ./test/index.ts",
"simulate-cdn": "http-server -p 8766 ../../../client/js-client.web.standalone/dist",
"start": "react-scripts start",
"build": "react-scripts build",

2
packages/@tests/smoke/web-cra-ts/test/index.ts
View File

@ -19,7 +19,7 @@ const test = async () => {
}
console.log('starting puppeteer...');
const browser = await puppeteer.launch({ headless: false });
const browser = await puppeteer.launch();
const page = (await browser.pages())[0];
// uncomment to debug what's happening inside the browser

6
packages/@tests/smoke/web/package.json
View File

@ -12,7 +12,7 @@
"scripts": {
"build": "tsc",
"simulate-cdn": "http-server -p 8765 ../../../client/js-client.web.standalone/dist",
"test_commented_out": "node --loader ts-node/esm ./src/index.ts",
"test": "node --loader ts-node/esm ./src/index.ts",
"serve": "http-server public"
},
"repository": "https://github.com/fluencelabs/fluence-js",
@ -22,5 +22,7 @@
"@fluencelabs/js-client": "workspace:^",
"@test/test-utils": "workspace:../../test-utils"
},
"devDependencies": {}
"devDependencies": {
"puppeteer": "19.7.2"
}
}

102
packages/core/js-client/build.ts
View File

@ -1,102 +0,0 @@
import path, { dirname } from 'path';
import type { InlineConfig, PluginOption } from 'vite';
import { build } from 'vite';
import { builtinModules, createRequire } from 'module';
import tsconfigPaths from 'vite-tsconfig-paths';
import inject from '@rollup/plugin-inject';
import stdLibBrowser from 'node-stdlib-browser';
import { fileURLToPath } from 'url';
import { rm, rename } from 'fs/promises';
import { replaceCodePlugin } from 'vite-plugin-replace';
import pkg from './package.json' assert { type: 'json' };
import libAssetsPlugin from '@laynezh/vite-plugin-lib-assets';
const require = createRequire(import.meta.url);
const commonConfig = (isNode: boolean): InlineConfig & Required<Pick<InlineConfig, 'build'>> => {
const esbuildShim = require.resolve('node-stdlib-browser/helpers/esbuild/shim');
return {
build: {
target: 'modules',
minify: 'esbuild',
lib: {
entry: './src/index.ts',
name: 'js-client',
fileName: `${isNode ? 'node' : 'browser'}/index`,
},
outDir: './dist',
emptyOutDir: false,
...(isNode ? {
rollupOptions: {
external: [...builtinModules, ...builtinModules.map(bm => `node:${bm}`)],
plugins: [
// @ts-ignore
inject({
self: 'global',
'WorkerScope': ['worker_threads', '*'],
'Worker': ['worker_threads', 'Worker'],
'isMainThread': ['worker_threads', 'isMainThread'],
})
]
}
} : {
rollupOptions: {
plugins: [
{
// @ts-ignore
...inject({
global: [esbuildShim, 'global'],
process: [esbuildShim, 'process'],
Buffer: [esbuildShim, 'Buffer']
}), enforce: 'post'
}
],
}
})
},
plugins: [tsconfigPaths(), libAssetsPlugin({
include: ['**/*.wasm*', '**/marine-worker.umd.cjs*'],
publicUrl: '/',
}), ...(isNode ? [replaceCodePlugin({
replacements: [
// After 'threads' package is built, it produces wrong output, which throws runtime errors.
// This code aims to fix such places.
// Should remove this after we move from threads to other package.
{ from: 'eval("require")("worker_threads")', to: 'WorkerScope' },
{ from: 'eval("require")("worker_threads")', to: 'WorkerScope' },
]
})] : [])] as PluginOption[],
optimizeDeps: {
esbuildOptions: {
define: {
global: 'globalThis',
},
},
},
resolve: {
browserField: !isNode,
conditions: isNode ? ['node'] : ['browser']
},
// Used only by browser
define: {
__JS_CLIENT_VERSION__: pkg.version,
__ENV__: isNode ? 'node' : 'browser'
},
};
};
const buildClient = async () => {
const nodeConfig = commonConfig(true);
const browserConfig = commonConfig(false);
try {
await rm('./dist', { recursive: true });
} catch {}
await build(nodeConfig);
await build(browserConfig);
};
buildClient()
.then(() => console.log('Built successfully'))
.catch((err) => console.error('failed', err));

18
packages/core/js-client/package.json
View File

@ -9,17 +9,17 @@
"files": [
"dist"
],
"main": "./dist/browser/index.js",
"unpkg": "./dist/browser/index.js",
"types": "./dist/types/index.d.ts",
"main": "./dist/index.js",
"unpkg": "./dist/browser/index.umd.js",
"types": "./dist/index.d.ts",
"exports": {
"types": "./dist/types/index.d.ts",
"node": "./dist/node/index.js",
"types": "./dist/index.d.ts",
"node": "./dist/index.js",
"default": "./dist/browser/index.js"
},
"type": "module",
"scripts": {
"build": "node --loader ts-node/esm build.ts && tsc --emitDeclarationOnly",
"build": "tsc && vite build",
"test": "vitest --threads false run"
},
"repository": "https://github.com/fluencelabs/fluence-js",
@ -29,6 +29,7 @@
"@chainsafe/libp2p-noise": "13.0.0",
"@chainsafe/libp2p-yamux": "5.0.0",
"@fluencelabs/interfaces": "workspace:*",
"@fluencelabs/marine-worker": "0.3.3",
"@libp2p/crypto": "2.0.3",
"@libp2p/interface": "0.1.2",
"@libp2p/peer-id": "3.0.2",
@ -55,15 +56,12 @@
"@fluencelabs/aqua-api": "0.9.3",
"@fluencelabs/avm": "0.48.0",
"@fluencelabs/marine-js": "0.7.2",
"@fluencelabs/marine-worker": "workspace:*",
"@laynezh/vite-plugin-lib-assets": "0.5.2",
"@rollup/plugin-inject": "5.0.3",
"@types/bs58": "4.0.1",
"@types/debug": "4.1.7",
"@types/node": "20.7.0",
"@types/uuid": "8.3.2",
"node-stdlib-browser": "1.2.0",
"vite": "4.0.4",
"vite-plugin-replace": "0.1.1",
"vite-tsconfig-paths": "4.0.3",
"vitest": "0.29.7"
}

25
packages/core/js-client/src/fetchers/browser.ts
View File

@ -14,6 +14,27 @@
* limitations under the License.
*/
export async function fetchResource(assetPath: string, version: string) {
return fetch(new globalThis.URL(`@fluencelabs/js-client@${version}/dist` + assetPath, `https://unpkg.com/`));
interface PackageJsonContent {
dependencies: Record<string, string | undefined>;
devDependencies: Record<string, string | undefined>;
}
// This will be substituted in build phase
const packageJsonContentString = `__PACKAGE_JSON_CONTENT__`;
let parsedPackageJsonContent: PackageJsonContent;
const PRIMARY_CDN = "https://unpkg.com/";
export async function fetchResource(pkg: string, assetPath: string) {
const packageJsonContent = parsedPackageJsonContent || (parsedPackageJsonContent = JSON.parse(packageJsonContentString));
const version = packageJsonContent.dependencies[pkg] || packageJsonContent.devDependencies[pkg];
if (version === undefined) {
const availableDeps = [...Object.keys(packageJsonContent.dependencies), ...Object.keys(packageJsonContent.devDependencies)];
throw new Error(`Cannot find version of ${pkg} in package.json. Available versions: ${availableDeps.join(',')}`);
}
const refinedAssetPath = assetPath.startsWith('/') ? assetPath.slice(1) : assetPath;
return fetch(new globalThis.URL(`${pkg}@${version}/` + refinedAssetPath, PRIMARY_CDN));
}

6
packages/core/js-client/src/fetchers/index.ts
View File

@ -20,11 +20,11 @@ import process from 'process';
const isNode = typeof process !== 'undefined' && process?.release?.name === 'node';
export async function fetchResource(assetPath: string, version: string) {
export async function fetchResource(pkg: string, path: string) {
switch (true) {
case isNode:
return fetchResourceNode(assetPath, version);
return fetchResourceNode(pkg, path);
default:
return fetchResourceBrowser(assetPath, version);
return fetchResourceBrowser(pkg, path);
}
}

25
packages/core/js-client/src/fetchers/node.ts
View File

@ -15,15 +15,29 @@
*/
import fs from 'fs';
import url from 'url';
import path from 'path';
import module from 'module';
export async function fetchResource(pkg: string, assetPath: string) {
const require = module.createRequire(import.meta.url);
const packagePathIndex = require.resolve(pkg);
// Ensure that windows path is converted to posix path. So we can find a package
const posixPath = packagePathIndex.split(path.sep).join(path.posix.sep);
const matches = new RegExp(`(.+${pkg})`).exec(posixPath);
const packagePath = matches?.[0];
if (!packagePath) {
throw new Error(`Cannot find dependency ${pkg} in path ${posixPath}`);
}
const pathToResource = path.join(packagePath, assetPath);
export async function fetchResource(assetPath: string, version: string) {
const file = await new Promise<ArrayBuffer>((resolve, reject) => {
// Cannot use 'fs/promises' with current vite config. This module is not polyfilled by default.
const root = path.dirname(url.fileURLToPath(import.meta.url));
const workerFilePath = path.join(root, '..', assetPath);
fs.readFile(workerFilePath, (err, data) => {
fs.readFile(pathToResource, (err, data) => {
if (err) {
reject(err);
return;
@ -31,6 +45,7 @@ export async function fetchResource(assetPath: string, version: string) {
resolve(data);
});
});
return new Response(file, {
headers: {
'Content-type':

28
packages/core/js-client/src/index.ts
View File

@ -23,28 +23,32 @@ import { BlobWorker, Worker } from 'threads';
import { doRegisterNodeUtils } from './services/NodeUtils.js';
import { fetchResource } from './fetchers/index.js';
import process from 'process';
import avmWasmUrl from '../node_modules/@fluencelabs/avm/dist/avm.wasm?url';
import marineJsWasmUrl from '../node_modules/@fluencelabs/marine-js/dist/marine-js.wasm?url';
import workerCodeUrl from '../node_modules/@fluencelabs/marine-worker/dist/__ENV__/marine-worker.umd.cjs?url';
const JS_CLIENT_VERSION = '__JS_CLIENT_VERSION__';
import path from 'path';
import url from 'url';
import module from 'module';
const isNode = typeof process !== 'undefined' && process?.release?.name === 'node';
const fetchWorkerCode = () => fetchResource(workerCodeUrl, JS_CLIENT_VERSION).then(res => res.text());
const fetchMarineJsWasm = () => fetchResource(marineJsWasmUrl, JS_CLIENT_VERSION).then(res => res.arrayBuffer());
const fetchAvmWasm = () => fetchResource(avmWasmUrl, JS_CLIENT_VERSION).then(res => res.arrayBuffer());
const fetchWorkerCode = () => fetchResource('@fluencelabs/marine-worker', '/dist/browser/marine-worker.umd.cjs').then(res => res.text());
const fetchMarineJsWasm = () => fetchResource('@fluencelabs/marine-js', '/dist/marine-js.wasm').then(res => res.arrayBuffer());
const fetchAvmWasm = () => fetchResource('@fluencelabs/avm', '/dist/avm.wasm').then(res => res.arrayBuffer());
const createClient = async (relay: RelayOptions, config: ClientConfig): Promise<IFluenceClient> => {
const workerCode = await fetchWorkerCode();
const marineJsWasm = await fetchMarineJsWasm();
const avmWasm = await fetchAvmWasm();
const marine = new MarineBackgroundRunner({
getValue() {
async getValue() {
if (isNode) {
const require = module.createRequire(import.meta.url);
const pathToThisFile = path.dirname(url.fileURLToPath(import.meta.url));
const pathToWorker = require.resolve('@fluencelabs/marine-worker');
const relativePathToWorker = path.relative(pathToThisFile, pathToWorker);
return new Worker(relativePathToWorker);
} else {
const workerCode = await fetchWorkerCode();
return BlobWorker.fromText(workerCode)
}
},
start() {
return Promise.resolve(undefined);

2
packages/core/js-client/src/marine/worker-script/workerLoader.ts
View File

@ -21,6 +21,6 @@ import { LazyLoader } from '../interfaces.js';
export class WorkerLoader extends LazyLoader<WorkerImplementation> {
constructor() {
super(() => new Worker('../../../node_modules/@fluencelabs/marine-worker/dist/node/marine-worker.umd.cjs'));
super(() => new Worker('../../../node_modules/@fluencelabs/marine-worker/dist/index.js'));
}
}

2
packages/core/js-client/src/marine/worker/index.ts
View File

@ -57,7 +57,7 @@ export class MarineBackgroundRunner implements IMarineHost {
await this.avmWasmLoader.start();
await this.workerLoader.start();
const worker = this.workerLoader.getValue();
const worker = await this.workerLoader.getValue();
const workerThread = await spawn<MarineBackgroundInterface>(worker);
const logfn: LogFunction = (message) => {

4
packages/core/js-client/tsconfig.json
View File

@ -1,8 +1,8 @@
{
"extends": "../../../tsconfig.json",
"compilerOptions": {
"types": ["vite/client"],
"outDir": "dist/types",
"types": ["vite/client", "node"],
"outDir": "./dist",
"esModuleInterop": true,
"resolveJsonModule": true
},

59
packages/core/js-client/vite.config.ts Normal file
View File

@ -0,0 +1,59 @@
/*
* Copyright 2023 Fluence Labs Limited
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import inject from '@rollup/plugin-inject';
import tsconfigPaths from 'vite-tsconfig-paths';
import { createRequire } from 'module';
import { readFileSync } from 'fs';
const require = createRequire(import.meta.url);
const esbuildShim = require.resolve('node-stdlib-browser/helpers/esbuild/shim');
export default {
build: {
target: 'modules',
minify: 'esbuild',
lib: {
entry: './src/index.ts',
name: 'js-client',
fileName: 'index',
},
outDir: './dist/browser',
rollupOptions: {
plugins: [
{
// @ts-ignore
...inject({
global: [esbuildShim, 'global'],
process: [esbuildShim, 'process'],
Buffer: [esbuildShim, 'Buffer']
}), enforce: 'post'
}
],
}
},
plugins: [tsconfigPaths()],
optimizeDeps: {
esbuildOptions: {
define: {
global: 'globalThis',
},
},
},
define: {
__PACKAGE_JSON_CONTENT__: readFileSync('./package.json', 'utf-8')
},
}

68
packages/core/marine-worker/build.ts
View File

@ -1,68 +0,0 @@
import { build, defineConfig, InlineConfig, PluginOption, UserConfig, UserConfigExport } from 'vite'
import { dirname, resolve } from 'path';
import { builtinModules, createRequire } from 'module';
import inject from '@rollup/plugin-inject';
// @ts-ignore
import merge from 'deepmerge';
import { fileURLToPath } from 'url';
import { replaceCodePlugin } from 'vite-plugin-replace';
const require = createRequire(import.meta.url);
const esbuildShim = require.resolve('node-stdlib-browser/helpers/esbuild/shim');
const commonConfig = defineConfig({
build: {
lib: {
entry: resolve(dirname(fileURLToPath(import.meta.url)), 'src/index.ts'),
name: 'MarineWorker'
},
}
}) as UserConfig;
const browserConfig: InlineConfig = await merge(commonConfig, defineConfig({
build: {
outDir: 'dist/browser',
},
plugins: [{
// @ts-ignore
...inject({
global: [esbuildShim, 'global'],
process: [esbuildShim, 'process'],
Buffer: [esbuildShim, 'Buffer']
}), enforce: 'post'
} as PluginOption],
}) as UserConfig);
const nodeConfig: InlineConfig = await merge(commonConfig, defineConfig({
build: {
target: 'es2022',
outDir: 'dist/node',
rollupOptions: {
external: [...builtinModules],
plugins: [
// @ts-ignore
inject({
self: 'global',
'WorkerScope': ['worker_threads', '*'],
'Worker': ['worker_threads', 'Worker'],
'isMainThread': ['worker_threads', 'isMainThread'],
})
]
}
},
plugins: [
replaceCodePlugin({
replacements: [
{ from: 'eval("require")("worker_threads")', to: 'WorkerScope' },
{ from: 'eval("require")("worker_threads")', to: 'WorkerScope' },
]
})
],
resolve: {
browserField: false,
}
}) as UserConfig);
await build(browserConfig!);
await build(nodeConfig!);

26
packages/core/marine-worker/package.json
View File

@ -1,31 +1,16 @@
{
"type": "module",
"name": "@fluencelabs/marine-worker",
"version": "0.3.3",
"description": "Marine worker",
"files": [
"dist"
],
"main": "./dist/node/marine-worker.umd.cjs",
"main": "./dist/index.js",
"unpkg": "./dist/browser/marine-worker.umd.cjs",
"types": "./dist/types/index.d.ts",
"exports": {
".": {
"types": "./dist/types/index.d.ts",
"node": "./dist/node/marine-worker.umd.cjs",
"default": "./dist/browser/marine-worker.umd.cjs"
},
"./dist/marine-worker.js": {
"node": "./dist/node/marine-worker.js",
"default": "./dist/browser/marine-worker.js"
},
"./dist/marine-worker.umd.cjs": {
"node": "./dist/node/marine-worker.umd.cjs",
"default": "./dist/browser/marine-worker.umd.cjs"
}
},
"type": "module",
"types": "./dist/index.d.ts",
"scripts": {
"build": "tsc --emitDeclarationOnly && node --loader ts-node/esm build.ts"
"build": "tsc && vite build"
},
"repository": "https://github.com/fluencelabs/fluence-js",
"author": "Fluence Labs",
@ -34,12 +19,9 @@
"devDependencies": {
"@rollup/plugin-inject": "5.0.3",
"@types/node": "20.4.5",
"deepmerge": "4.3.1",
"node-stdlib-browser": "1.2.0",
"typescript": "5.1.6",
"vite": "4.0.4",
"vite-plugin-dts": "3.4.0",
"vite-plugin-replace": "0.1.1",
"vitest": "0.29.7"
},
"dependencies": {

2
packages/core/marine-worker/src/index.ts
View File

@ -19,7 +19,7 @@ import type { Env, MarineModuleConfig, MarineServiceConfig, ModuleDescriptor } f
import type { JSONArray, JSONObject, LogMessage, CallParameters } from '@fluencelabs/marine-js/dist/types';
import { Observable, Subject } from 'observable-fns';
// @ts-ignore no types provided for package
import { expose } from 'threads';
import { expose } from 'threads/worker';
const createSimpleModuleDescriptor = (name: string, envs?: Env): ModuleDescriptor => {
return {

2
packages/core/marine-worker/tsconfig.json
View File

@ -1,7 +1,7 @@
{
"extends": "../../../tsconfig.json",
"compilerOptions": {
"outDir": "./dist/types"
"outDir": "./dist"
},
"include": ["src/**/*"],
"exclude": ["node_modules", "dist"]

42
packages/core/marine-worker/vite.config.ts Normal file
View File

@ -0,0 +1,42 @@
/*
* Copyright 2023 Fluence Labs Limited
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { PluginOption } from 'vite'
import { dirname, resolve } from 'path';
import { createRequire } from 'module';
import inject from '@rollup/plugin-inject';
import { fileURLToPath } from 'url';
const require = createRequire(import.meta.url);
const esbuildShim = require.resolve('node-stdlib-browser/helpers/esbuild/shim');
export default {
build: {
lib: {
entry: resolve(dirname(fileURLToPath(import.meta.url)), 'src/index.ts'),
name: 'MarineWorker'
},
outDir: 'dist/browser',
},
plugins: [{
// @ts-ignore
...inject({
global: [esbuildShim, 'global'],
process: [esbuildShim, 'process'],
Buffer: [esbuildShim, 'Buffer']
}), enforce: 'post'
} as PluginOption],
};

676
pnpm-lock.yaml generated
View File

File diff suppressed because it is too large Load Diff