update url-downloader

build/config.gypi

@@ -0,0 +1,347 @@
+# Do not edit. File was generated by node-gyp's "configure" step
+{
+  "target_defaults": {
+    "cflags": [],
+    "default_configuration": "Release",
+    "defines": [],
+    "include_dirs": [],
+    "libraries": []
+  },
+  "variables": {
+    "asan": 0,
+    "coverage": "false",
+    "dcheck_always_on": 0,
+    "debug_nghttp2": "false",
+    "debug_node": "false",
+    "enable_lto": "false",
+    "enable_pgo_generate": "false",
+    "enable_pgo_use": "false",
+    "error_on_warn": "false",
+    "force_dynamic_crt": 0,
+    "host_arch": "x64",
+    "icu_data_in": "../../deps/icu-tmp/icudt69l.dat",
+    "icu_endianness": "l",
+    "icu_gyp_path": "tools/icu/icu-generic.gyp",
+    "icu_path": "deps/icu-small",
+    "icu_small": "false",
+    "icu_ver_major": "69",
+    "is_debug": 0,
+    "llvm_version": "11.0",
+    "napi_build_version": "8",
+    "node_byteorder": "little",
+    "node_debug_lib": "false",
+    "node_enable_d8": "false",
+    "node_install_npm": "true",
+    "node_library_files": [
+      "lib/constants.js",
+      "lib/net.js",
+      "lib/trace_events.js",
+      "lib/events.js",
+      "lib/repl.js",
+      "lib/util.js",
+      "lib/dgram.js",
+      "lib/vm.js",
+      "lib/stream.js",
+      "lib/child_process.js",
+      "lib/assert.js",
+      "lib/_tls_wrap.js",
+      "lib/http2.js",
+      "lib/inspector.js",
+      "lib/os.js",
+      "lib/_http_server.js",
+      "lib/console.js",
+      "lib/perf_hooks.js",
+      "lib/readline.js",
+      "lib/punycode.js",
+      "lib/_http_incoming.js",
+      "lib/https.js",
+      "lib/_stream_wrap.js",
+      "lib/domain.js",
+      "lib/dns.js",
+      "lib/_http_client.js",
+      "lib/diagnostics_channel.js",
+      "lib/tty.js",
+      "lib/_http_agent.js",
+      "lib/timers.js",
+      "lib/_http_outgoing.js",
+      "lib/querystring.js",
+      "lib/_tls_common.js",
+      "lib/module.js",
+      "lib/_stream_passthrough.js",
+      "lib/_stream_transform.js",
+      "lib/worker_threads.js",
+      "lib/sys.js",
+      "lib/_stream_duplex.js",
+      "lib/path.js",
+      "lib/_http_common.js",
+      "lib/string_decoder.js",
+      "lib/cluster.js",
+      "lib/v8.js",
+      "lib/crypto.js",
+      "lib/wasi.js",
+      "lib/_stream_readable.js",
+      "lib/zlib.js",
+      "lib/url.js",
+      "lib/tls.js",
+      "lib/_stream_writable.js",
+      "lib/async_hooks.js",
+      "lib/process.js",
+      "lib/http.js",
+      "lib/buffer.js",
+      "lib/fs.js",
+      "lib/util/types.js",
+      "lib/timers/promises.js",
+      "lib/path/win32.js",
+      "lib/path/posix.js",
+      "lib/stream/consumers.js",
+      "lib/stream/promises.js",
+      "lib/stream/web.js",
+      "lib/internal/constants.js",
+      "lib/internal/abort_controller.js",
+      "lib/internal/net.js",
+      "lib/internal/v8_prof_processor.js",
+      "lib/internal/event_target.js",
+      "lib/internal/inspector_async_hook.js",
+      "lib/internal/validators.js",
+      "lib/internal/linkedlist.js",
+      "lib/internal/cli_table.js",
+      "lib/internal/repl.js",
+      "lib/internal/util.js",
+      "lib/internal/histogram.js",
+      "lib/internal/error_serdes.js",
+      "lib/internal/dgram.js",
+      "lib/internal/child_process.js",
+      "lib/internal/assert.js",
+      "lib/internal/fixed_queue.js",
+      "lib/internal/blocklist.js",
+      "lib/internal/v8_prof_polyfill.js",
+      "lib/internal/options.js",
+      "lib/internal/worker.js",
+      "lib/internal/dtrace.js",
+      "lib/internal/idna.js",
+      "lib/internal/watchdog.js",
+      "lib/internal/encoding.js",
+      "lib/internal/tty.js",
+      "lib/internal/freeze_intrinsics.js",
+      "lib/internal/timers.js",
+      "lib/internal/heap_utils.js",
+      "lib/internal/querystring.js",
+      "lib/internal/js_stream_socket.js",
+      "lib/internal/errors.js",
+      "lib/internal/priority_queue.js",
+      "lib/internal/freelist.js",
+      "lib/internal/blob.js",
+      "lib/internal/socket_list.js",
+      "lib/internal/socketaddress.js",
+      "lib/internal/stream_base_commons.js",
+      "lib/internal/url.js",
+      "lib/internal/async_hooks.js",
+      "lib/internal/http.js",
+      "lib/internal/buffer.js",
+      "lib/internal/trace_events_async_hooks.js",
+      "lib/internal/crypto/sig.js",
+      "lib/internal/crypto/rsa.js",
+      "lib/internal/crypto/aes.js",
+      "lib/internal/crypto/util.js",
+      "lib/internal/crypto/scrypt.js",
+      "lib/internal/crypto/random.js",
+      "lib/internal/crypto/keys.js",
+      "lib/internal/crypto/x509.js",
+      "lib/internal/crypto/certificate.js",
+      "lib/internal/crypto/ec.js",
+      "lib/internal/crypto/keygen.js",
+      "lib/internal/crypto/mac.js",
+      "lib/internal/crypto/diffiehellman.js",
+      "lib/internal/crypto/hkdf.js",
+      "lib/internal/crypto/cipher.js",
+      "lib/internal/crypto/hash.js",
+      "lib/internal/crypto/pbkdf2.js",
+      "lib/internal/crypto/webcrypto.js",
+      "lib/internal/crypto/dsa.js",
+      "lib/internal/crypto/hashnames.js",
+      "lib/internal/cluster/shared_handle.js",
+      "lib/internal/cluster/round_robin_handle.js",
+      "lib/internal/cluster/worker.js",
+      "lib/internal/cluster/primary.js",
+      "lib/internal/cluster/utils.js",
+      "lib/internal/cluster/child.js",
+      "lib/internal/webstreams/util.js",
+      "lib/internal/webstreams/writablestream.js",
+      "lib/internal/webstreams/readablestream.js",
+      "lib/internal/webstreams/queuingstrategies.js",
+      "lib/internal/webstreams/encoding.js",
+      "lib/internal/webstreams/transformstream.js",
+      "lib/internal/webstreams/transfer.js",
+      "lib/internal/bootstrap/loaders.js",
+      "lib/internal/bootstrap/pre_execution.js",
+      "lib/internal/bootstrap/node.js",
+      "lib/internal/bootstrap/environment.js",
+      "lib/internal/bootstrap/switches/does_not_own_process_state.js",
+      "lib/internal/bootstrap/switches/is_not_main_thread.js",
+      "lib/internal/bootstrap/switches/does_own_process_state.js",
+      "lib/internal/bootstrap/switches/is_main_thread.js",
+      "lib/internal/test/binding.js",
+      "lib/internal/test/transfer.js",
+      "lib/internal/util/types.js",
+      "lib/internal/util/inspector.js",
+      "lib/internal/util/comparisons.js",
+      "lib/internal/util/debuglog.js",
+      "lib/internal/util/inspect.js",
+      "lib/internal/util/iterable_weak_map.js",
+      "lib/internal/streams/add-abort-signal.js",
+      "lib/internal/streams/compose.js",
+      "lib/internal/streams/duplexify.js",
+      "lib/internal/streams/destroy.js",
+      "lib/internal/streams/legacy.js",
+      "lib/internal/streams/passthrough.js",
+      "lib/internal/streams/readable.js",
+      "lib/internal/streams/from.js",
+      "lib/internal/streams/writable.js",
+      "lib/internal/streams/state.js",
+      "lib/internal/streams/buffer_list.js",
+      "lib/internal/streams/end-of-stream.js",
+      "lib/internal/streams/utils.js",
+      "lib/internal/streams/transform.js",
+      "lib/internal/streams/lazy_transform.js",
+      "lib/internal/streams/duplex.js",
+      "lib/internal/streams/pipeline.js",
+      "lib/internal/readline/utils.js",
+      "lib/internal/readline/emitKeypressEvents.js",
+      "lib/internal/readline/callbacks.js",
+      "lib/internal/repl/history.js",
+      "lib/internal/repl/utils.js",
+      "lib/internal/repl/await.js",
+      "lib/internal/legacy/processbinding.js",
+      "lib/internal/assert/calltracker.js",
+      "lib/internal/assert/assertion_error.js",
+      "lib/internal/http2/util.js",
+      "lib/internal/http2/core.js",
+      "lib/internal/http2/compat.js",
+      "lib/internal/per_context/messageport.js",
+      "lib/internal/per_context/primordials.js",
+      "lib/internal/per_context/domexception.js",
+      "lib/internal/vm/module.js",
+      "lib/internal/tls/secure-pair.js",
+      "lib/internal/tls/parse-cert-string.js",
+      "lib/internal/tls/secure-context.js",
+      "lib/internal/child_process/serialization.js",
+      "lib/internal/debugger/inspect_repl.js",
+      "lib/internal/debugger/inspect_client.js",
+      "lib/internal/debugger/inspect.js",
+      "lib/internal/worker/io.js",
+      "lib/internal/worker/js_transferable.js",
+      "lib/internal/main/repl.js",
+      "lib/internal/main/print_help.js",
+      "lib/internal/main/eval_string.js",
+      "lib/internal/main/check_syntax.js",
+      "lib/internal/main/prof_process.js",
+      "lib/internal/main/worker_thread.js",
+      "lib/internal/main/inspect.js",
+      "lib/internal/main/eval_stdin.js",
+      "lib/internal/main/run_main_module.js",
+      "lib/internal/modules/run_main.js",
+      "lib/internal/modules/package_json_reader.js",
+      "lib/internal/modules/esm/module_job.js",
+      "lib/internal/modules/esm/get_source.js",
+      "lib/internal/modules/esm/translators.js",
+      "lib/internal/modules/esm/resolve.js",
+      "lib/internal/modules/esm/create_dynamic_module.js",
+      "lib/internal/modules/esm/load.js",
+      "lib/internal/modules/esm/module_map.js",
+      "lib/internal/modules/esm/get_format.js",
+      "lib/internal/modules/esm/loader.js",
+      "lib/internal/modules/cjs/helpers.js",
+      "lib/internal/modules/cjs/loader.js",
+      "lib/internal/source_map/source_map.js",
+      "lib/internal/source_map/prepare_stack_trace.js",
+      "lib/internal/source_map/source_map_cache.js",
+      "lib/internal/dns/promises.js",
+      "lib/internal/dns/utils.js",
+      "lib/internal/fs/watchers.js",
+      "lib/internal/fs/promises.js",
+      "lib/internal/fs/read_file_context.js",
+      "lib/internal/fs/rimraf.js",
+      "lib/internal/fs/sync_write_stream.js",
+      "lib/internal/fs/dir.js",
+      "lib/internal/fs/streams.js",
+      "lib/internal/fs/utils.js",
+      "lib/internal/fs/cp/cp.js",
+      "lib/internal/fs/cp/cp-sync.js",
+      "lib/internal/perf/nodetiming.js",
+      "lib/internal/perf/usertiming.js",
+      "lib/internal/perf/performance_entry.js",
+      "lib/internal/perf/performance.js",
+      "lib/internal/perf/timerify.js",
+      "lib/internal/perf/utils.js",
+      "lib/internal/perf/observe.js",
+      "lib/internal/perf/event_loop_delay.js",
+      "lib/internal/perf/event_loop_utilization.js",
+      "lib/internal/policy/manifest.js",
+      "lib/internal/policy/sri.js",
+      "lib/internal/process/task_queues.js",
+      "lib/internal/process/per_thread.js",
+      "lib/internal/process/warning.js",
+      "lib/internal/process/policy.js",
+      "lib/internal/process/promises.js",
+      "lib/internal/process/signal.js",
+      "lib/internal/process/execution.js",
+      "lib/internal/process/esm_loader.js",
+      "lib/internal/process/report.js",
+      "lib/internal/process/worker_thread_only.js",
+      "lib/internal/console/constructor.js",
+      "lib/internal/console/global.js",
+      "lib/assert/strict.js",
+      "lib/dns/promises.js",
+      "lib/fs/promises.js"
+    ],
+    "node_module_version": 93,
+    "node_no_browser_globals": "false",
+    "node_prefix": "/",
+    "node_release_urlbase": "https://nodejs.org/download/release/",
+    "node_shared": "false",
+    "node_shared_brotli": "false",
+    "node_shared_cares": "false",
+    "node_shared_http_parser": "false",
+    "node_shared_libuv": "false",
+    "node_shared_nghttp2": "false",
+    "node_shared_nghttp3": "false",
+    "node_shared_ngtcp2": "false",
+    "node_shared_openssl": "false",
+    "node_shared_zlib": "false",
+    "node_tag": "",
+    "node_target_type": "executable",
+    "node_use_bundled_v8": "true",
+    "node_use_dtrace": "true",
+    "node_use_etw": "false",
+    "node_use_node_code_cache": "true",
+    "node_use_node_snapshot": "true",
+    "node_use_openssl": "true",
+    "node_use_v8_platform": "true",
+    "node_with_ltcg": "false",
+    "node_without_node_options": "false",
+    "openssl_fips": "",
+    "openssl_is_fips": "false",
+    "openssl_quic": "true",
+    "ossfuzz": "false",
+    "shlib_suffix": "93.dylib",
+    "target_arch": "x64",
+    "v8_enable_31bit_smis_on_64bit_arch": 0,
+    "v8_enable_gdbjit": 0,
+    "v8_enable_i18n_support": 1,
+    "v8_enable_inspector": 1,
+    "v8_enable_lite_mode": 0,
+    "v8_enable_object_print": 1,
+    "v8_enable_pointer_compression": 0,
+    "v8_enable_webassembly": 1,
+    "v8_no_strict_aliasing": 1,
+    "v8_optimized_debug": 1,
+    "v8_promise_internal_field_count": 1,
+    "v8_random_seed": 0,
+    "v8_trace_maps": 0,
+    "v8_use_siphash": 1,
+    "want_separate_host_toolset": 0,
+    "xcode_version": "11.0",
+    "nodedir": "/Users/mike/Library/Caches/node-gyp/16.13.2",
+    "standalone_static_library": 1
+  }
+}

greeting/artifacts/greeting.json

@@ -0,0 +1 @@
+{"name": "greeting"}

url-downloader/README.md

@@ -0,0 +1,15 @@
+# Download file to disk
+
+Example that shows how to work with disk access and how to link several Wasm modules into a service.
+
+# Build & deploy it
+
+```shell
+./deploy.sh
+```
+
+# Call it
+
+```shell
+fldist run_air -p download.air -d '{"service": "f7db6966-2d75-4424-812e-85e2ec5cb61b"}'
+```

url-downloader/artifacts/curl_adapter.json

@@ -0,0 +1,7 @@
+{
+  "name": "curl_adapter",
+  "mountedBinaries":
+    {
+      "curl": "/usr/bin/curl"
+    }
+}

url-downloader/artifacts/facade.json

@@ -0,0 +1 @@
+{"name": "facade"}

url-downloader/artifacts/local_storage.json

@@ -0,0 +1,12 @@
+{
+  "name": "local_storage",
+  "preopenedFiles": [
+    "/tmp"
+  ],
+  "mappedDirs": {
+    "sites": "/tmp"
+  },
+  "mountedBinaries": {
+    "curl": "/usr/bin/curl"
+  }
+}

url-downloader/build.sh

@@ -1,26 +1,26 @@
-#!/bin/sh
+#!/usr/bin/env bash
+set -o errexit -o nounset -o pipefail
 
+# This script builds all subprojects and puts all created Wasm modules in one dir
 (
-  cd local_storage || exit;
+  cd local_storage
-  cargo update --aggressive;
+  cargo update --aggressive
-  marine build --release;
+  marine build --release
+)
+(
+  cd curl_adapter
+  cargo update --aggressive
+  marine build --release
+)
+(
+  cd facade
+  cargo update --aggressive
+  marine build --release
 )
 
-(
+mkdir -p sites
-  cd curl_adapter || exit;
-  cargo update --aggressive;
-  marine build --release;
-)
-
-(
-  cd facade || exit;
-  cargo update --aggressive;
-  marine build --release;
-)
-
-rm -f artifacts/* || true
 mkdir -p artifacts
-
+rm -f artifacts/*.wasm
-cp ./local_storage/target/wasm32-wasi/release/local_storage.wasm artifacts/
+cp local_storage/target/wasm32-wasi/release/local_storage.wasm artifacts/
-cp ./curl_adapter/target/wasm32-wasi/release/curl_adapter.wasm artifacts/
+cp curl_adapter/target/wasm32-wasi/release/curl_adapter.wasm artifacts/
-cp ./facade/target/wasm32-wasi/release/facade.wasm artifacts/
+cp facade/target/wasm32-wasi/release/facade.wasm artifacts/

url-downloader/compose.air

@@ -0,0 +1,13 @@
+(xor
+    (seq
+        (seq
+           (seq
+               (call relay (curl "download") ["https://fluence.network/img/svg/logo_new.svg"] contents)
+               (call relay (storage "put") ["logo.svg" contents.$.stdout!] ret_code)
+           )
+           (call relay (storage "get") ["logo.svg"] bytes)
+        )
+        (call %init_peer_id% (returnService "run") [ret_code bytes])
+    )
+    (call %init_peer_id% (returnService "run") [%last_error%])
+)

url-downloader/curl_adapter/Cargo.toml

@@ -10,5 +10,5 @@ path = "src/main.rs"
 name = "curl_adapter"
 
 [dependencies]
-marine-rs-sdk ={ version = "0.6.13", features = ["logger"] }
+marine-rs-sdk = { version = "0.6.10", features = ["logger"] }
 log = "0.4.8"

url-downloader/curl_adapter/src/main.rs

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020 Fluence Labs Limited
+ * Copyright 2021 Fluence Labs Limited
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,8 +19,8 @@
 use marine_rs_sdk::marine;
 use marine_rs_sdk::module_manifest;
 
-use marine_rs_sdk::WasmLoggerBuilder;
 use marine_rs_sdk::MountedBinaryResult;
+use marine_rs_sdk::WasmLoggerBuilder;
 
 module_manifest!();
 

url-downloader/deploy.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -o errexit -o nounset -o pipefail
+
+# build wasms
+./build.sh
+
+(
+  cd artifacts
+  fldist new_service --name "url_downloader" --modules \
+    curl_adapter.wasm:curl_adapter.json \
+    local_storage.wasm:local_storage.json \
+    facade.wasm:facade.json
+)

url-downloader/deploy_separate.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -o errexit -o nounset -o pipefail
+
+./build.sh
+
+echo "Deploying storage"
+(
+  cd artifacts
+  fldist new_service --name "local_storage" --modules local_storage.wasm:local_storage.json
+)
+
+echo "\n\nDeploying curl"
+(
+  cd artifacts
+  fldist new_service --name "curl_adapter" --modules curl_adapter.wasm:curl_adapter.json
+)

url-downloader/download.air

@@ -0,0 +1,10 @@
+(xor
+    (seq
+        (seq
+           (call relay (service "get_n_save") ["https://fluence.network/img/svg/logo_new.svg" "logo.svg"] ret_code)
+           (call relay (service "load_file") ["logo.svg"] bytes)
+        )
+        (call %init_peer_id% (returnService "run") [ret_code bytes])
+    )
+    (call %init_peer_id% (returnService "run") [%last_error%])
+)

url-downloader/facade/Cargo.toml

@@ -10,6 +10,6 @@ name = "facade"
 path = "src/main.rs"
 
 [dependencies]
-marine-rs-sdk ={ version = "0.6.13", features = ["logger"] }
+marine-rs-sdk = { version = "0.6.10", features = ["logger"] }
 anyhow = "1.0.31"
-log = "0.4.8"
+log = "0.4.8"

url-downloader/facade/src/main.rs

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020 Fluence Labs Limited
+ * Copyright 2021 Fluence Labs Limited
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

url-downloader/local_storage/Cargo.toml

@@ -10,7 +10,6 @@ name = "local_storage"
 path = "src/main.rs"
 
 [dependencies]
-marine-rs-sdk ={ version = "0.6.13", features = ["logger"] }
+marine-rs-sdk = { version = "0.6.10", features = ["logger"] }
-wasm-tracing-allocator = "0.1.0"
+wasm-tracing-allocator = "0.1.1"
-
 log = "0.4.8"

url-downloader/local_storage/src/main.rs

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020 Fluence Labs Limited
+ * Copyright 2021 Fluence Labs Limited
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -36,7 +36,7 @@ pub fn put(name: String, file_content: Vec<u8>) -> String {
     log::info!("put called with file name {}\n", name);
     let rpc_tmp_filepath = format!("{}{}", SITES_DIR, name);
 
-    let result = fs::write(PathBuf::from(rpc_tmp_filepath), file_content);
+    let result = fs::write(PathBuf::from(rpc_tmp_filepath.clone()), file_content);
     if let Err(e) = result {
         return format!("file can't be written: {}", e);
     }

url-downloader/repl_config.toml

@@ -0,0 +1,21 @@
+modules_dir = "artifacts/"
+
+[[module]]
+name = "local_storage"
+logger_enabled = true
+
+[module.wasi]
+preopened_files = ["."]
+# this is where files will be stored
+mapped_dirs = { "sites" = "." }
+
+[[module]]
+name = "curl_adapter"
+logger_enabled = true
+
+[module.mounted_binaries]
+curl = "/usr/bin/curl"
+
+[[module]]
+name = "facade"
+logger_enabled = true

url-downloader/sites/asd

@@ -1,6 +0,0 @@
-<HTML><HEAD><meta http-equiv="content-type" content="text/html;charset=utf-8">
-<TITLE>301 Moved</TITLE></HEAD><BODY>
-<H1>301 Moved</H1>
-The document has moved
-<A HREF="http://www.google.com/">here</A>.
-</BODY></HTML>