fix(tests): Repair integration tests [fixes DXJ-506] (#364)

* Add test server

* remove headless

* Support for web-cra

* Review fixes and bug fixes

* Add error message

.eslintrc.json

@@ -18,7 +18,7 @@
     "license-header",
     "unused-imports"
   ],
-  "ignorePatterns": ["**/node_modules/**/*", "**/dist/**/*"],
+  "ignorePatterns": ["**/node_modules/", "**/dist/", "**/build/", "**/public/"],
   "rules": {
     "eqeqeq": [
       "error",

.gitignore

@@ -7,3 +7,4 @@ node_modules/
 
 # Build directory
 **/dist
+**/public

packages/@tests/smoke/web-cra-ts/test/index.ts

@@ -4,10 +4,11 @@ import { fileURLToPath } from "url";
 
 import {
   CDN_PUBLIC_PATH,
+  createSymlinkIfNotExists,
+  JS_CLIENT_DEPS_PATH,
   startContentServer,
   stopServer,
 } from "@test/test-utils";
-import { access, symlink } from "fs/promises";
 
 const port = 3001;
 const uri = `http://localhost:${port}/`;
@@ -16,11 +17,11 @@ const publicPath = join(__dirname, "../build/");
 
 const test = async () => {
   const localServer = await startContentServer(port, publicPath);
-  try {
-    await access(join(publicPath, "source"));
-  } catch {
-    await symlink(CDN_PUBLIC_PATH, join(publicPath, "source"));
-  }
+  await createSymlinkIfNotExists(CDN_PUBLIC_PATH, join(publicPath, "source"));
+  await createSymlinkIfNotExists(
+    JS_CLIENT_DEPS_PATH,
+    join(publicPath, "node_modules"),
+  );
 
   console.log("starting puppeteer...");
   const browser = await puppeteer.launch();

packages/@tests/smoke/web/public/index.js

@@ -85,7 +85,9 @@ const getRelayTime = () => {
 
 const main = async () => {
   console.log("starting fluence...");
-  fluence.defaultClient = await fluence.clientFactory(relay, {});
+  fluence.defaultClient = await fluence.clientFactory(relay, {
+    CDNUrl: "http://localhost:3000",
+  });
   console.log("started fluence");
 
   console.log("getting relay time...");

packages/@tests/smoke/web/src/index.ts

@@ -14,12 +14,13 @@
  * limitations under the License.
  */
 
-import { symlink, access } from "fs/promises";
 import { dirname, join } from "path";
 import { fileURLToPath } from "url";
 
 import {
   CDN_PUBLIC_PATH,
+  createSymlinkIfNotExists,
+  JS_CLIENT_DEPS_PATH,
   startContentServer,
   stopServer,
 } from "@test/test-utils";
@@ -33,11 +34,12 @@ const publicPath = join(__dirname, "../public/");
 const test = async () => {
   const localServer = await startContentServer(port, publicPath);
 
-  try {
-    await access(join(publicPath, "source"));
-  } catch {
-    await symlink(CDN_PUBLIC_PATH, join(publicPath, "source"));
-  }
+  await createSymlinkIfNotExists(CDN_PUBLIC_PATH, join(publicPath, "source"));
+
+  await createSymlinkIfNotExists(
+    JS_CLIENT_DEPS_PATH,
+    join(publicPath, "node_modules"),
+  );
 
   console.log("starting puppeteer...");
   const browser = await puppeteer.launch();

packages/@tests/test-utils/src/index.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import { access, symlink } from "fs/promises";
 import { createServer } from "http";
 import type { Server } from "http";
 import { dirname, join } from "path";
@@ -28,10 +29,26 @@ export const CDN_PUBLIC_PATH = join(
   "../../../core/js-client/dist/browser",
 );
 
+export const JS_CLIENT_DEPS_PATH = join(
+  __dirname,
+  "../../../core/js-client/node_modules",
+);
+
 export const startCdn = (port: number) => {
   return startContentServer(port, CDN_PUBLIC_PATH);
 };
 
+export const createSymlinkIfNotExists = async (
+  target: string,
+  path: string,
+) => {
+  try {
+    await access(path);
+  } catch {
+    await symlink(target, path);
+  }
+};
+
 export const startContentServer = (
   port: number,
   publicDir: string,
@@ -44,6 +61,22 @@ export const startContentServer = (
           source: "/js-client.min.js",
           destination: "/source/index.umd.cjs",
         },
+        // TODO:
+        // something like this
+        // {
+        //   source: "/@fluencelabs/:name(\\w+)@:version([\\d.]+)/:path*",
+        //   destination: "/deps/@fluencelabs/:name/:path",
+        // }
+        // not supported for some reason. Need to manually iterate over all possible paths
+        {
+          source: "/@fluencelabs/:name([\\w-]+)@:version([\\d.]+)/dist/:asset",
+          destination: "/node_modules/@fluencelabs/:name/dist/:asset",
+        },
+        {
+          source:
+            "/@fluencelabs/:name([\\w-]+)@:version([\\d.]+)/dist/:prefix/:asset",
+          destination: "/node_modules/@fluencelabs/:name/dist/:prefix/:asset",
+        },
       ],
       headers: [
         {

packages/core/interfaces/src/commonTypes.ts

@@ -23,14 +23,6 @@ import { InterfaceToType, MaybePromise } from "./utils.js";
  */
 export type PeerIdB58 = string;
 
-/**
- * Node of the Fluence network specified as a pair of node's multiaddr and it's peer id
- */
-export type Node = {
-  peerId: PeerIdB58;
-  multiaddr: string;
-};
-
 /**
  * Additional information about a service call
  * @typeparam ArgName

packages/core/interfaces/src/compilerSupport/compilerSupportInterface.ts

@@ -15,7 +15,6 @@
  */
 
 import { JSONValue } from "../commonTypes.js";
-import { IFluenceInternalApi } from "../fluenceClient.js";
 
 import {
   FnConfig,
@@ -38,11 +37,12 @@ export type PassedArgs = { [key: string]: JSONValue | ArgCallbackFunction };
 /**
  * Arguments for callAquaFunction function
  */
+// TODO: move to js-client side
 export interface CallAquaFunctionArgs {
   /**
    * Peer to call the function on
    */
-  peer: IFluenceInternalApi;
+  peer: unknown;
 
   /**
    * Function definition
@@ -79,7 +79,7 @@ export interface RegisterServiceArgs {
   /**
    * Peer to register the service on
    */
-  peer: IFluenceInternalApi;
+  peer: unknown;
 
   /**
    * Service definition

packages/core/interfaces/src/index.ts

@@ -17,5 +17,4 @@
 export * from "./compilerSupport/aquaTypeDefinitions.js";
 export * from "./compilerSupport/compilerSupportInterface.js";
 export * from "./commonTypes.js";
-export * from "./fluenceClient.js";
 export * from "./future.js";

packages/core/js-client/src/clientPeer/ClientPeer.ts

@@ -14,13 +14,6 @@
  * limitations under the License.
  */
 
-import {
-  ClientConfig,
-  ConnectionState,
-  IFluenceClient,
-  RelayOptions,
-} from "@fluencelabs/interfaces";
-
 import {
   RelayConnection,
   RelayConnectionConfig,
@@ -32,6 +25,13 @@ import { IMarineHost } from "../marine/interfaces.js";
 import { relayOptionToMultiaddr } from "../util/libp2pUtils.js";
 import { logger } from "../util/logger.js";
 
+import {
+  ClientConfig,
+  IFluenceClient,
+  ConnectionState,
+  RelayOptions,
+} from "./types.js";
+
 const log = logger("client");
 
 const DEFAULT_TTL_MS = 7000;

packages/core/js-client/src/clientPeer/types.ts

@@ -14,7 +14,18 @@
  * limitations under the License.
  */
 
-import type { Node } from "./commonTypes.js";
+/**
+ * Peer ID's id as a base58 string (multihash/CIDv0).
+ */
+export type PeerIdB58 = string;
+
+/**
+ * Node of the Fluence network specified as a pair of node's multiaddr and it's peer id
+ */
+export type Node = {
+  peerId: PeerIdB58;
+  multiaddr: string;
+};
 
 /**
  * A node in Fluence network a client can connect to.
@@ -37,64 +48,6 @@ export type KeyPairOptions = {
   source: "random" | Uint8Array;
 };
 
-/**
- * Configuration used when initiating Fluence Client
- */
-export interface ClientConfig {
-  /**
-   * Specify the KeyPair to be used to identify the Fluence Peer.
-   * Will be generated randomly if not specified
-   */
-  keyPair?: KeyPairOptions;
-
-  /**
-   * Options to configure the connection to the Fluence network
-   */
-  connectionOptions?: {
-    /**
-     * When the peer established the connection to the network it sends a ping-like message to check if it works correctly.
-     * The options allows to specify the timeout for that message in milliseconds.
-     * If not specified the default timeout will be used
-     */
-    skipCheckConnection?: boolean;
-
-    /**
-     * The dialing timeout in milliseconds
-     */
-    dialTimeoutMs?: number;
-
-    /**
-     * The maximum number of inbound streams for the libp2p node.
-     * Default: 1024
-     */
-    maxInboundStreams?: number;
-
-    /**
-     * The maximum number of outbound streams for the libp2p node.
-     * Default: 1024
-     */
-    maxOutboundStreams?: number;
-  };
-
-  /**
-   * Sets the default TTL for all particles originating from the peer with no TTL specified.
-   * If the originating particle's TTL is defined then that value will be used
-   * If the option is not set default TTL will be 7000
-   */
-  defaultTtlMs?: number;
-
-  /**
-   * Enables\disabled various debugging features
-   */
-  debug?: {
-    /**
-     * If set to true, newly initiated particle ids will be printed to console.
-     * Useful to see what particle id is responsible for aqua function
-     */
-    printParticleId?: boolean;
-  };
-}
-
 /**
  * Fluence JS Client connection states as string literals
  */
@@ -153,3 +106,66 @@ export interface IFluenceClient extends IFluenceInternalApi {
    */
   getRelayPeerId(): string;
 }
+
+/**
+ * Configuration used when initiating Fluence Client
+ */
+export interface ClientConfig {
+  /**
+   * Specify the KeyPair to be used to identify the Fluence Peer.
+   * Will be generated randomly if not specified
+   */
+  keyPair?: KeyPairOptions;
+
+  /**
+   * Options to configure the connection to the Fluence network
+   */
+  connectionOptions?: {
+    /**
+     * When the peer established the connection to the network it sends a ping-like message to check if it works correctly.
+     * The options allows to specify the timeout for that message in milliseconds.
+     * If not specified the default timeout will be used
+     */
+    skipCheckConnection?: boolean;
+
+    /**
+     * The dialing timeout in milliseconds
+     */
+    dialTimeoutMs?: number;
+
+    /**
+     * The maximum number of inbound streams for the libp2p node.
+     * Default: 1024
+     */
+    maxInboundStreams?: number;
+
+    /**
+     * The maximum number of outbound streams for the libp2p node.
+     * Default: 1024
+     */
+    maxOutboundStreams?: number;
+  };
+
+  /**
+   * Sets the default TTL for all particles originating from the peer with no TTL specified.
+   * If the originating particle's TTL is defined then that value will be used
+   * If the option is not set default TTL will be 7000
+   */
+  defaultTtlMs?: number;
+
+  /**
+   * Property for passing custom CDN Url to load dependencies from browser. https://unpkg.com used by default
+   */
+  CDNUrl?: string;
+
+  /**
+   * Enables\disabled various debugging features
+   */
+  debug?: {
+    /**
+     * If set to true, newly initiated particle ids will be printed to console.
+     * Useful to see what particle id is responsible for aqua function
+     */
+    printParticleId?: boolean;
+  };
+}

packages/core/js-client/src/fetchers/browser.ts

@@ -23,9 +23,16 @@ interface PackageJsonContent {
 const packageJsonContentString = `__PACKAGE_JSON_CONTENT__`;
 let parsedPackageJsonContent: PackageJsonContent | undefined;
 
-const PRIMARY_CDN = "https://unpkg.com/";
-
-export async function fetchResource(pkg: string, assetPath: string) {
+/**
+ * @param pkg name of package
+ * @param assetPath path of required asset in given package
+ * @param root CDN domain in browser or file system root in node
+ */
+export async function fetchResource(
+  pkg: string,
+  assetPath: string,
+  root: string,
+) {
   const packageJsonContent =
     parsedPackageJsonContent ??
     // TODO: Should be validated
@@ -55,7 +62,8 @@ export async function fetchResource(pkg: string, assetPath: string) {
     ? assetPath.slice(1)
     : assetPath;
 
-  return fetch(
-    new globalThis.URL(`${pkg}@${version}/` + refinedAssetPath, PRIMARY_CDN),
-  );
+  const url = new globalThis.URL(`${pkg}@${version}/` + refinedAssetPath, root);
+  return fetch(url).catch(() => {
+    throw new Error(`Cannot fetch from ${url.toString()}`);
+  });
 }

packages/core/js-client/src/fetchers/node.ts

@@ -18,7 +18,18 @@ import fs from "fs";
 import module from "module";
 import path from "path";
 
-export async function fetchResource(pkg: string, assetPath: string) {
+/**
+ * @param pkg name of package
+ * @param assetPath path of required asset in given package
+ * @param root CDN domain in browser or file system root in node
+ */
+export async function fetchResource(
+  pkg: string,
+  assetPath: string,
+  root: string,
+) {
+  // TODO: `root` will be handled somehow in the future. For now, we use filesystem root where js-client is running;
+  root = "/";
   const require = module.createRequire(import.meta.url);
   const packagePathIndex = require.resolve(pkg);
 
@@ -33,7 +44,7 @@ export async function fetchResource(pkg: string, assetPath: string) {
     throw new Error(`Cannot find dependency ${pkg} in path ${posixPath}`);
   }
 
-  const pathToResource = path.join(packagePath, assetPath);
+  const pathToResource = path.join(root, packagePath, assetPath);
 
   const file = await new Promise<ArrayBuffer>((resolve, reject) => {
     // Cannot use 'fs/promises' with current vite config. This module is not polyfilled by default.

packages/core/js-client/src/index.ts

@@ -19,14 +19,14 @@ import path from "path";
 import process from "process";
 import url from "url";
 
-import type {
-  ClientConfig,
-  ConnectionState,
-  RelayOptions,
-} from "@fluencelabs/interfaces";
 import { BlobWorker, Worker } from "threads/master";
 
 import { ClientPeer, makeClientPeerConfig } from "./clientPeer/ClientPeer.js";
+import {
+  ClientConfig,
+  ConnectionState,
+  RelayOptions,
+} from "./clientPeer/types.js";
 import { callAquaFunction } from "./compilerSupport/callFunction.js";
 import { registerService } from "./compilerSupport/registerService.js";
 import { MarineBackgroundRunner } from "./marine/worker/index.js";
@@ -34,38 +34,49 @@ import { doRegisterNodeUtils } from "./services/NodeUtils.js";
 
 import { fetchResource } from "#fetcher";
 
+const DEFAULT_CDN_URL = "https://unpkg.com";
+
 const isNode =
   // process.release is undefined in browser env
   // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
   typeof process !== "undefined" && process.release?.name === "node";
 
-const fetchWorkerCode = async () => {
-  const resource = await fetchResource(
-    "@fluencelabs/marine-worker",
-    "/dist/browser/marine-worker.umd.cjs",
-  );
-
-  return resource.text();
-};
-
-const fetchMarineJsWasm = async () => {
-  const resource = await fetchResource(
-    "@fluencelabs/marine-js",
-    "/dist/marine-js.wasm",
-  );
-
-  return resource.arrayBuffer();
-};
-
-const fetchAvmWasm = async () => {
-  const resource = await fetchResource("@fluencelabs/avm", "/dist/avm.wasm");
-  return resource.arrayBuffer();
-};
-
 const createClient = async (
   relay: RelayOptions,
   config: ClientConfig,
 ): Promise<ClientPeer> => {
+  const CDNUrl = config.CDNUrl ?? DEFAULT_CDN_URL;
+
+  const fetchWorkerCode = async () => {
+    const resource = await fetchResource(
+      "@fluencelabs/marine-worker",
+      "/dist/browser/marine-worker.umd.cjs",
+      CDNUrl,
+    );
+
+    return resource.text();
+  };
+
+  const fetchMarineJsWasm = async () => {
+    const resource = await fetchResource(
+      "@fluencelabs/marine-js",
+      "/dist/marine-js.wasm",
+      CDNUrl,
+    );
+
+    return resource.arrayBuffer();
+  };
+
+  const fetchAvmWasm = async () => {
+    const resource = await fetchResource(
+      "@fluencelabs/avm",
+      "/dist/avm.wasm",
+      CDNUrl,
+    );
+
+    return resource.arrayBuffer();
+  };
+
   const marineJsWasm = await fetchMarineJsWasm();
   const avmWasm = await fetchAvmWasm();
 
@@ -194,11 +205,9 @@ export const Fluence: FluencePublicApi = {
   },
 };
 
-export type {
-  IFluenceClient,
-  ClientConfig,
-  CallParams,
-} from "@fluencelabs/interfaces";
+export type { CallParams } from "@fluencelabs/interfaces";
+
+export type { ClientConfig, IFluenceClient } from "./clientPeer/types.js";
 
 export type {
   ArrayType,

packages/core/js-client/src/keypair/index.ts

@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-import { KeyPairOptions } from "@fluencelabs/interfaces";
 import {
   generateKeyPairFromSeed,
   generateKeyPair,
@@ -26,6 +25,8 @@ import { createFromPrivKey } from "@libp2p/peer-id-factory";
 import bs58 from "bs58";
 import { toUint8Array } from "js-base64";
 
+import { KeyPairOptions } from "../clientPeer/types.js";
+
 export class KeyPair {
   private publicKey: PublicKey;
 

packages/core/js-client/src/util/libp2pUtils.ts

@@ -14,9 +14,10 @@
  * limitations under the License.
  */
 
-import { RelayOptions } from "@fluencelabs/interfaces";
 import { multiaddr, Multiaddr } from "@multiformats/multiaddr";
 
+import { RelayOptions } from "../clientPeer/types.js";
+
 import { isString } from "./utils.js";
 
 export function relayOptionToMultiaddr(relay: RelayOptions): Multiaddr {

packages/core/js-client/src/util/testUtils.ts

@@ -18,16 +18,15 @@ import { promises as fs } from "fs";
 
 import * as api from "@fluencelabs/aqua-api/aqua-api.js";
 import {
-  ClientConfig,
   FunctionCallDef,
   JSONArray,
   PassedArgs,
-  RelayOptions,
   ServiceDef,
 } from "@fluencelabs/interfaces";
 import { Subject, Subscribable } from "rxjs";
 
 import { ClientPeer, makeClientPeerConfig } from "../clientPeer/ClientPeer.js";
+import { ClientConfig, RelayOptions } from "../clientPeer/types.js";
 import { callAquaFunction } from "../compilerSupport/callFunction.js";
 import { IConnection } from "../connection/interfaces.js";
 import { DEFAULT_CONFIG, FluencePeer } from "../jsPeer/FluencePeer.js";

pnpm-workspace.yaml

@@ -1,2 +1,5 @@
 packages:
-  - "packages/**/*"
+  - "packages/core/*"
+  - "packages/@tests/aqua"
+  - "packages/@tests/smoke/*"
+  - "packages/@tests/test-utils"