From 020e0c73191069a0f2dbdb995e44a8bb5ad5494d Mon Sep 17 00:00:00 2001 From: folex <0xdxdy@gmail.com> Date: Wed, 12 Apr 2023 13:45:58 -0400 Subject: [PATCH] deploy url-downloader via workers --- .../url-downloader/.fluence/aqua/workers.aqua | 28 +++++++------ .../.fluence/schemas/fluence.yaml.json | 22 +--------- .../.fluence/schemas/module.yaml.json | 11 +---- .../.fluence/schemas/service.yaml.json | 22 +--------- .../.fluence/schemas/workers.yaml.json | 40 ++++--------------- .../url-downloader/.fluence/tmp/Config.toml | 23 ----------- .../url-downloader/.fluence/workers.yaml | 16 ++++---- .../url-downloader/.fluence/workers.yaml.bsk | 16 -------- .../url-downloader/fluence-lock.yaml | 2 + marine-examples/url-downloader/fluence.yaml | 14 ++----- .../service/modules/local_storage/src/main.rs | 6 +-- .../url-downloader/src/aqua/main.aqua | 19 ++++++--- 12 files changed, 58 insertions(+), 161 deletions(-) delete mode 100644 marine-examples/url-downloader/.fluence/tmp/Config.toml delete mode 100644 marine-examples/url-downloader/.fluence/workers.yaml.bsk diff --git a/marine-examples/url-downloader/.fluence/aqua/workers.aqua b/marine-examples/url-downloader/.fluence/aqua/workers.aqua index 942fa4c..5004405 100644 --- a/marine-examples/url-downloader/.fluence/aqua/workers.aqua +++ b/marine-examples/url-downloader/.fluence/aqua/workers.aqua @@ -1,18 +1,20 @@ -data GetWorkersInfoDealsDefaultWorker: - installationSpells: ?u8 - definition: string - timestamp: string - dealIdOriginal: string - dealId: string - chainNetwork: string - chainNetworkId: f64 +data GetWorkersInfoHostsDefaultWorkerInstallationSpells: + hostId: string + spellId: string + workerId: string -data GetWorkersInfoDeals: - defaultWorker: GetWorkersInfoDealsDefaultWorker +data GetWorkersInfoHostsDefaultWorker: + definition: string + installationSpells: []GetWorkersInfoHostsDefaultWorkerInstallationSpells + timestamp: string + relayId: string + +data GetWorkersInfoHosts: + defaultWorker: GetWorkersInfoHostsDefaultWorker data GetWorkersInfo: - deals: GetWorkersInfoDeals - hosts: ?u8 + deals: ?u8 + hosts: GetWorkersInfoHosts func getWorkersInfo() -> GetWorkersInfo: - <- GetWorkersInfo(deals=GetWorkersInfoDeals(defaultWorker=GetWorkersInfoDealsDefaultWorker(installationSpells=nil,definition="bafkreihndikxxxvl6r2alrfzh7awwywiolyklkwxzmzoyfcolvlxho2zuy",timestamp="2023-04-04T00:29:31.854Z",dealIdOriginal="0xd0c75aEEA62bc0ABf16F5f01361277CC8287AB0E",dealId="d0c75aeea62bc0abf16f5f01361277cc8287ab0e",chainNetwork="testnet",chainNetworkId=80001.0)),hosts=nil) + <- GetWorkersInfo(deals=nil,hosts=GetWorkersInfoHosts(defaultWorker=GetWorkersInfoHostsDefaultWorker(definition="bafkreic2hb3mqpqoapy6cb3ozzyvknds3jk5rvov5bty6zgdtegwg7j37i",installationSpells=[GetWorkersInfoHostsDefaultWorkerInstallationSpells(hostId="12D3KooWDcpWuyrMTDinqNgmXAuRdfd2mTdY9VoXZSAet2pDzh6r",spellId="05759fa5-5c23-4d1e-9a8f-512e866b363e",workerId="12D3KooWC99oo2UvyJU8b5oKQsRxBCSs9tbgeCvH9zP4Pt35Twtr")],timestamp="2023-04-12T17:15:00.700Z",relayId="12D3KooWDcpWuyrMTDinqNgmXAuRdfd2mTdY9VoXZSAet2pDzh6r"))) diff --git a/marine-examples/url-downloader/.fluence/schemas/fluence.yaml.json b/marine-examples/url-downloader/.fluence/schemas/fluence.yaml.json index ad9c9b2..499acbf 100644 --- a/marine-examples/url-downloader/.fluence/schemas/fluence.yaml.json +++ b/marine-examples/url-downloader/.fluence/schemas/fluence.yaml.json @@ -48,16 +48,7 @@ "nullable": true, "required": [], "title": "Volumes", - "description": "A map of accessible files and their aliases. Aliases should be used in Marine module development because it's hard to know the full path to a file. (This property replaces the legacy \"mapped_dirs\" property so there is no need to duplicate the same paths in \"preopenedFiles\" dir)" - }, - "preopenedFiles": { - "type": "array", - "title": "Preopened files", - "description": "A list of files and directories that this module could access with WASI", - "items": { - "type": "string" - }, - "nullable": true + "description": "A map of accessible files and their aliases. Aliases should be used in Marine module development because it's hard to know the full path to a file" }, "envs": { "type": "object", @@ -153,16 +144,7 @@ "nullable": true, "required": [], "title": "Volumes", - "description": "A map of accessible files and their aliases. Aliases should be used in Marine module development because it's hard to know the full path to a file. (This property replaces the legacy \"mapped_dirs\" property so there is no need to duplicate the same paths in \"preopenedFiles\" dir)" - }, - "preopenedFiles": { - "type": "array", - "title": "Preopened files", - "description": "A list of files and directories that this module could access with WASI", - "items": { - "type": "string" - }, - "nullable": true + "description": "A map of accessible files and their aliases. Aliases should be used in Marine module development because it's hard to know the full path to a file" }, "envs": { "type": "object", diff --git a/marine-examples/url-downloader/.fluence/schemas/module.yaml.json b/marine-examples/url-downloader/.fluence/schemas/module.yaml.json index 910e6e2..1ac8895 100644 --- a/marine-examples/url-downloader/.fluence/schemas/module.yaml.json +++ b/marine-examples/url-downloader/.fluence/schemas/module.yaml.json @@ -38,16 +38,7 @@ "nullable": true, "required": [], "title": "Volumes", - "description": "A map of accessible files and their aliases. Aliases should be used in Marine module development because it's hard to know the full path to a file. (This property replaces the legacy \"mapped_dirs\" property so there is no need to duplicate the same paths in \"preopenedFiles\" dir)" - }, - "preopenedFiles": { - "type": "array", - "title": "Preopened files", - "description": "A list of files and directories that this module could access with WASI", - "items": { - "type": "string" - }, - "nullable": true + "description": "A map of accessible files and their aliases. Aliases should be used in Marine module development because it's hard to know the full path to a file" }, "envs": { "type": "object", diff --git a/marine-examples/url-downloader/.fluence/schemas/service.yaml.json b/marine-examples/url-downloader/.fluence/schemas/service.yaml.json index dc51e68..7f24ba2 100644 --- a/marine-examples/url-downloader/.fluence/schemas/service.yaml.json +++ b/marine-examples/url-downloader/.fluence/schemas/service.yaml.json @@ -40,16 +40,7 @@ "nullable": true, "required": [], "title": "Volumes", - "description": "A map of accessible files and their aliases. Aliases should be used in Marine module development because it's hard to know the full path to a file. (This property replaces the legacy \"mapped_dirs\" property so there is no need to duplicate the same paths in \"preopenedFiles\" dir)" - }, - "preopenedFiles": { - "type": "array", - "title": "Preopened files", - "description": "A list of files and directories that this module could access with WASI", - "items": { - "type": "string" - }, - "nullable": true + "description": "A map of accessible files and their aliases. Aliases should be used in Marine module development because it's hard to know the full path to a file" }, "envs": { "type": "object", @@ -99,16 +90,7 @@ "nullable": true, "required": [], "title": "Volumes", - "description": "A map of accessible files and their aliases. Aliases should be used in Marine module development because it's hard to know the full path to a file. (This property replaces the legacy \"mapped_dirs\" property so there is no need to duplicate the same paths in \"preopenedFiles\" dir)" - }, - "preopenedFiles": { - "type": "array", - "title": "Preopened files", - "description": "A list of files and directories that this module could access with WASI", - "items": { - "type": "string" - }, - "nullable": true + "description": "A map of accessible files and their aliases. Aliases should be used in Marine module development because it's hard to know the full path to a file" }, "envs": { "type": "object", diff --git a/marine-examples/url-downloader/.fluence/schemas/workers.yaml.json b/marine-examples/url-downloader/.fluence/schemas/workers.yaml.json index eb35275..5451c70 100644 --- a/marine-examples/url-downloader/.fluence/schemas/workers.yaml.json +++ b/marine-examples/url-downloader/.fluence/schemas/workers.yaml.json @@ -14,29 +14,6 @@ "additionalProperties": { "type": "object", "properties": { - "installation_spells": { - "type": "array", - "description": "A list of installation spells", - "items": { - "type": "object", - "properties": { - "host_id": { - "type": "string" - }, - "spell_id": { - "type": "string" - }, - "worker_id": { - "type": "string" - } - }, - "required": [ - "host_id", - "spell_id", - "worker_id" - ] - } - }, "definition": { "type": "string" }, @@ -62,7 +39,6 @@ } }, "required": [ - "installation_spells", "timestamp", "definition", "dealId", @@ -80,6 +56,13 @@ "additionalProperties": { "type": "object", "properties": { + "definition": { + "type": "string" + }, + "timestamp": { + "type": "string", + "description": "ISO timestamp of the time when the worker was deployed" + }, "installation_spells": { "type": "array", "description": "A list of installation spells", @@ -103,21 +86,14 @@ ] } }, - "definition": { - "type": "string" - }, - "timestamp": { - "type": "string", - "description": "ISO timestamp of the time when the worker was deployed" - }, "relayId": { "type": "string" } }, "required": [ - "installation_spells", "timestamp", "definition", + "installation_spells", "relayId" ] }, diff --git a/marine-examples/url-downloader/.fluence/tmp/Config.toml b/marine-examples/url-downloader/.fluence/tmp/Config.toml deleted file mode 100644 index 3f0eb62..0000000 --- a/marine-examples/url-downloader/.fluence/tmp/Config.toml +++ /dev/null @@ -1,23 +0,0 @@ -[[module]] -name = "local_storage" -load_from = "/Users/bebo/localdev/examples/marine-examples/url-downloader/target/wasm32-wasi/release/local_storage.wasm" -logger_enabled = true - - [module.wasi] - preopened_files = [ "./tmp" ] - - [module.wasi.mapped_dirs] - sites = "./tmp" - -[[module]] -name = "curl_adapter" -load_from = "/Users/bebo/localdev/examples/marine-examples/url-downloader/target/wasm32-wasi/release/curl_adapter.wasm" -logger_enabled = true - - [module.mounted_binaries] - curl = "/usr/bin/curl" - -[[module]] -name = "url_downloader" -load_from = "/Users/bebo/localdev/examples/marine-examples/url-downloader/target/wasm32-wasi/release/url_downloader.wasm" -logger_enabled = true diff --git a/marine-examples/url-downloader/.fluence/workers.yaml b/marine-examples/url-downloader/.fluence/workers.yaml index bffc3f5..9cc434c 100644 --- a/marine-examples/url-downloader/.fluence/workers.yaml +++ b/marine-examples/url-downloader/.fluence/workers.yaml @@ -5,12 +5,12 @@ # Documentation: https://github.com/fluencelabs/fluence-cli/tree/main/docs/configs/workers.md version: 0 -deals: +hosts: defaultWorker: - installation_spells: [] - definition: bafkreihndikxxxvl6r2alrfzh7awwywiolyklkwxzmzoyfcolvlxho2zuy - timestamp: 2023-04-04T00:29:31.854Z - dealIdOriginal: "0xd0c75aEEA62bc0ABf16F5f01361277CC8287AB0E" - dealId: d0c75aeea62bc0abf16f5f01361277cc8287ab0e - chainNetwork: testnet - chainNetworkId: 80001 + definition: bafkreic2hb3mqpqoapy6cb3ozzyvknds3jk5rvov5bty6zgdtegwg7j37i + installation_spells: + - host_id: 12D3KooWDcpWuyrMTDinqNgmXAuRdfd2mTdY9VoXZSAet2pDzh6r + spell_id: 05759fa5-5c23-4d1e-9a8f-512e866b363e + worker_id: 12D3KooWC99oo2UvyJU8b5oKQsRxBCSs9tbgeCvH9zP4Pt35Twtr + timestamp: 2023-04-12T17:15:00.700Z + relayId: 12D3KooWDcpWuyrMTDinqNgmXAuRdfd2mTdY9VoXZSAet2pDzh6r diff --git a/marine-examples/url-downloader/.fluence/workers.yaml.bsk b/marine-examples/url-downloader/.fluence/workers.yaml.bsk deleted file mode 100644 index a2288b9..0000000 --- a/marine-examples/url-downloader/.fluence/workers.yaml.bsk +++ /dev/null @@ -1,16 +0,0 @@ -# yaml-language-server: $schema=schemas/workers.yaml.json - -# A result of app deployment. This file is created automatically after successful deployment using `fluence workers deploy` command - -# Documentation: https://github.com/fluencelabs/fluence-cli/tree/main/docs/configs/workers.md - -version: 0 -deals: - defaultWorker: - installation_spells: [] - definition: bafkreigvh7cxlf5nkisqqxq6alpoj32l44ttxkfxafa6agp2u344kl3k4m - timestamp: 2023-04-03T22:37:49.594Z - dealIdOriginal: "0xEA6f3D3177c40123C7B8B1d89440F1DdcfA6014B" - dealId: ea6f3d3177c40123c7b8b1d89440f1ddcfa6014b - chainNetwork: testnet - chainNetworkId: 80001 diff --git a/marine-examples/url-downloader/fluence-lock.yaml b/marine-examples/url-downloader/fluence-lock.yaml index 69232f2..ed9f095 100644 --- a/marine-examples/url-downloader/fluence-lock.yaml +++ b/marine-examples/url-downloader/fluence-lock.yaml @@ -10,3 +10,5 @@ npm: "@fluencelabs/aqua-lib": 0.6.0 "@fluencelabs/spell": 0.5.7 "@fluencelabs/registry": 0.8.2 +cargo: + marine: 0.14.1 diff --git a/marine-examples/url-downloader/fluence.yaml b/marine-examples/url-downloader/fluence.yaml index 877e387..7f35c27 100644 --- a/marine-examples/url-downloader/fluence.yaml +++ b/marine-examples/url-downloader/fluence.yaml @@ -6,15 +6,6 @@ version: 2 aquaInputPath: src/aqua/main.aqua -dependencies: - npm: - "@fluencelabs/aqua": 0.10.3 - "@fluencelabs/aqua-lib": 0.6.0 - "@fluencelabs/spell": 0.5.7 - "@fluencelabs/registry": 0.8.2 - cargo: - marine: 0.14.0 - mrepl: 0.21.2 workers: defaultWorker: services: [ url_downloader ] @@ -25,8 +16,9 @@ deals: hosts: defaultWorker: peerIds: - - 12D3KooWDUszU2NeWyUVjCXhGEt1MoZrhvdmaQQwtZUriuGN1jTr -relays: kras + - 12D3KooWDcpWuyrMTDinqNgmXAuRdfd2mTdY9VoXZSAet2pDzh6r +relays: + - /dns4/stage.fluence.dev/tcp/19990/wss/p2p/12D3KooWDcpWuyrMTDinqNgmXAuRdfd2mTdY9VoXZSAet2pDzh6r services: url_downloader: get: service diff --git a/marine-examples/url-downloader/service/modules/local_storage/src/main.rs b/marine-examples/url-downloader/service/modules/local_storage/src/main.rs index 5e5a1c4..b493474 100644 --- a/marine-examples/url-downloader/service/modules/local_storage/src/main.rs +++ b/marine-examples/url-downloader/service/modules/local_storage/src/main.rs @@ -7,7 +7,7 @@ use std::path::PathBuf; module_manifest!(); -const SITES_DIR: &str = "/sites/"; +const SITES_DIR: &str = "sites/"; /// Log level can be changed by `RUST_LOG` env as well. pub fn main() { @@ -20,9 +20,9 @@ pub fn put(name: String, file_content: Vec) -> String { log::info!("put called with file name {}\n", name); let rpc_tmp_filepath = format!("{}{}", SITES_DIR, name); - let result = fs::write(PathBuf::from(rpc_tmp_filepath.clone()), file_content); + let result = fs::write(PathBuf::from(rpc_tmp_filepath), file_content); if let Err(e) = result { - return format!("file can't be written: {}", e); + return format!("WTF WTF WTF: {}", e); } String::from("Ok") diff --git a/marine-examples/url-downloader/src/aqua/main.aqua b/marine-examples/url-downloader/src/aqua/main.aqua index 4a5551d..d6e699f 100644 --- a/marine-examples/url-downloader/src/aqua/main.aqua +++ b/marine-examples/url-downloader/src/aqua/main.aqua @@ -9,11 +9,12 @@ export get_and_save, put, put_all, get, worker_n func getWorkers() -> []Record: - workersInfo <- getWorkersInfo() - dealId = workersInfo.deals.defaultWorker.dealId - on HOST_PEER_ID: - workersOp <- resolveSubnetwork(dealId) - <- workersOp! + -- workersInfo <- getWorkersInfo() + -- dealId = workersInfo.deals.defaultWorker.dealId + -- on HOST_PEER_ID: + -- workersOp <- resolveSubnetwork(dealId) + -- <- workersOp! + <- [] func worker_n() -> u32: workers <- getWorkers() @@ -44,6 +45,14 @@ func put(file_name: string, file_content: []u8) -> string: res <- UrlDownloader.put(file_name, file_content) <- res +func put_workers(file_name: string, file_content: []u8) -> string, []u8: + info <- getWorkersInfo() + spell = info.hosts.defaultWorker.installationSpells[0] + on spell.workerId via spell.hostId: + res <- UrlDownloader.put(file_name, file_content) + get <- UrlDownloader.get(file_name) + <- res, get + func get(file_name: string) -> []u8: workers <- getWorkers() w = workers[5]