update url-downloader to fluence cli (#442)

* update to fluence cli

* update aqua

* add config toml, update aqua script
This commit is contained in:
boneyard93501 2023-04-23 14:17:21 -05:00 committed by GitHub
parent d3a5daab30
commit cdec8be7ae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 95 additions and 165 deletions

View File

@ -1,18 +1,31 @@
data GetWorkersInfoDealsDefaultWorker: data GetWorkersInfoDealsDefaultWorker:
installationSpells: ?u8
definition: string definition: string
timestamp: string timestamp: string
dealIdOriginal: string dealIdOriginal: string
dealId: string dealId: string
chainNetwork: string chainNetwork: string
chainNetworkId: f64 chainNetworkId: u64
data GetWorkersInfoDeals: data GetWorkersInfoDeals:
defaultWorker: GetWorkersInfoDealsDefaultWorker defaultWorker: GetWorkersInfoDealsDefaultWorker
data GetWorkersInfoHostsDefaultWorkerInstallationSpells:
hostId: string
spellId: string
workerId: string
data GetWorkersInfoHostsDefaultWorker:
definition: string
installationSpells: []GetWorkersInfoHostsDefaultWorkerInstallationSpells
timestamp: string
relayId: string
data GetWorkersInfoHosts:
defaultWorker: GetWorkersInfoHostsDefaultWorker
data GetWorkersInfo: data GetWorkersInfo:
deals: GetWorkersInfoDeals deals: GetWorkersInfoDeals
hosts: ?u8 hosts: GetWorkersInfoHosts
func getWorkersInfo() -> GetWorkersInfo: func getWorkersInfo() -> GetWorkersInfo:
<- GetWorkersInfo(deals=GetWorkersInfoDeals(defaultWorker=GetWorkersInfoDealsDefaultWorker(installationSpells=nil,definition="bafkreihndikxxxvl6r2alrfzh7awwywiolyklkwxzmzoyfcolvlxho2zuy",timestamp="2023-04-04T00:29:31.854Z",dealIdOriginal="0xd0c75aEEA62bc0ABf16F5f01361277CC8287AB0E",dealId="d0c75aeea62bc0abf16f5f01361277cc8287ab0e",chainNetwork="testnet",chainNetworkId=80001.0)),hosts=nil) <- GetWorkersInfo(deals=GetWorkersInfoDeals(defaultWorker=GetWorkersInfoDealsDefaultWorker(definition="bafkreicdordcic2qmd2pnacl5adffsuhwg4eleu4prcgaldjvzr6ds2f2m",timestamp="2023-04-23T07:01:18.682Z",dealIdOriginal="0x9d7503E13Bc2d3430536e3d5D635D8172141d884",dealId="9d7503e13bc2d3430536e3d5d635d8172141d884",chainNetwork="testnet",chainNetworkId=1313161555)),hosts=GetWorkersInfoHosts(defaultWorker=GetWorkersInfoHostsDefaultWorker(definition="bafkreicdordcic2qmd2pnacl5adffsuhwg4eleu4prcgaldjvzr6ds2f2m",installationSpells=[GetWorkersInfoHostsDefaultWorkerInstallationSpells(hostId="12D3KooWJ4bTHirdTFNZpCS72TAzwtdmavTBkkEXtzo6wHL25CtE",spellId="039f4e06-923e-4a21-8814-8164fa9b137c",workerId="12D3KooWKsxVXgnpL4wrTts9d49jiNSEN1auaxotdMYa4HpFS66g")],timestamp="2023-04-23T07:00:05.549Z",relayId="12D3KooWAKNos2KogexTXhrkMZzFYpLHuWJ4PgoAhurSAv7o5CWA")))

View File

@ -48,16 +48,7 @@
"nullable": true, "nullable": true,
"required": [], "required": [],
"title": "Volumes", "title": "Volumes",
"description": "A map of accessible files and their aliases. Aliases should be used in Marine module development because it's hard to know the full path to a file. (This property replaces the legacy \"mapped_dirs\" property so there is no need to duplicate the same paths in \"preopenedFiles\" dir)" "description": "A map of accessible files and their aliases. Aliases should be used in Marine module development because it's hard to know the full path to a file"
},
"preopenedFiles": {
"type": "array",
"title": "Preopened files",
"description": "A list of files and directories that this module could access with WASI",
"items": {
"type": "string"
},
"nullable": true
}, },
"envs": { "envs": {
"type": "object", "type": "object",
@ -153,16 +144,7 @@
"nullable": true, "nullable": true,
"required": [], "required": [],
"title": "Volumes", "title": "Volumes",
"description": "A map of accessible files and their aliases. Aliases should be used in Marine module development because it's hard to know the full path to a file. (This property replaces the legacy \"mapped_dirs\" property so there is no need to duplicate the same paths in \"preopenedFiles\" dir)" "description": "A map of accessible files and their aliases. Aliases should be used in Marine module development because it's hard to know the full path to a file"
},
"preopenedFiles": {
"type": "array",
"title": "Preopened files",
"description": "A list of files and directories that this module could access with WASI",
"items": {
"type": "string"
},
"nullable": true
}, },
"envs": { "envs": {
"type": "object", "type": "object",
@ -257,7 +239,7 @@
"type": "object", "type": "object",
"title": "Dependencies", "title": "Dependencies",
"nullable": true, "nullable": true,
"description": "A map of dependency versions", "description": "(For advanced users) Overrides for the project dependencies",
"properties": { "properties": {
"npm": { "npm": {
"type": "object", "type": "object",
@ -463,6 +445,14 @@
] ]
}, },
"required": [] "required": []
},
"aquaImports": {
"type": "array",
"description": "A list of path to be considered by aqua compiler to be used as imports. First dependency in the list has the highest priority. Priority of imports is considered in the following order: imports from --import flags, imports from aquaImports property in fluence.yaml, project's .fluence/aqua dir, npm dependencies from fluence.yaml, npm dependencies from user's .fluence/config.yaml, npm dependencies recommended by fluence",
"items": {
"type": "string"
},
"nullable": true
} }
}, },
"required": [ "required": [

View File

@ -38,16 +38,7 @@
"nullable": true, "nullable": true,
"required": [], "required": [],
"title": "Volumes", "title": "Volumes",
"description": "A map of accessible files and their aliases. Aliases should be used in Marine module development because it's hard to know the full path to a file. (This property replaces the legacy \"mapped_dirs\" property so there is no need to duplicate the same paths in \"preopenedFiles\" dir)" "description": "A map of accessible files and their aliases. Aliases should be used in Marine module development because it's hard to know the full path to a file"
},
"preopenedFiles": {
"type": "array",
"title": "Preopened files",
"description": "A list of files and directories that this module could access with WASI",
"items": {
"type": "string"
},
"nullable": true
}, },
"envs": { "envs": {
"type": "object", "type": "object",

View File

@ -40,16 +40,7 @@
"nullable": true, "nullable": true,
"required": [], "required": [],
"title": "Volumes", "title": "Volumes",
"description": "A map of accessible files and their aliases. Aliases should be used in Marine module development because it's hard to know the full path to a file. (This property replaces the legacy \"mapped_dirs\" property so there is no need to duplicate the same paths in \"preopenedFiles\" dir)" "description": "A map of accessible files and their aliases. Aliases should be used in Marine module development because it's hard to know the full path to a file"
},
"preopenedFiles": {
"type": "array",
"title": "Preopened files",
"description": "A list of files and directories that this module could access with WASI",
"items": {
"type": "string"
},
"nullable": true
}, },
"envs": { "envs": {
"type": "object", "type": "object",
@ -99,16 +90,7 @@
"nullable": true, "nullable": true,
"required": [], "required": [],
"title": "Volumes", "title": "Volumes",
"description": "A map of accessible files and their aliases. Aliases should be used in Marine module development because it's hard to know the full path to a file. (This property replaces the legacy \"mapped_dirs\" property so there is no need to duplicate the same paths in \"preopenedFiles\" dir)" "description": "A map of accessible files and their aliases. Aliases should be used in Marine module development because it's hard to know the full path to a file"
},
"preopenedFiles": {
"type": "array",
"title": "Preopened files",
"description": "A list of files and directories that this module could access with WASI",
"items": {
"type": "string"
},
"nullable": true
}, },
"envs": { "envs": {
"type": "object", "type": "object",

View File

@ -14,29 +14,6 @@
"additionalProperties": { "additionalProperties": {
"type": "object", "type": "object",
"properties": { "properties": {
"installation_spells": {
"type": "array",
"description": "A list of installation spells",
"items": {
"type": "object",
"properties": {
"host_id": {
"type": "string"
},
"spell_id": {
"type": "string"
},
"worker_id": {
"type": "string"
}
},
"required": [
"host_id",
"spell_id",
"worker_id"
]
}
},
"definition": { "definition": {
"type": "string" "type": "string"
}, },
@ -62,7 +39,6 @@
} }
}, },
"required": [ "required": [
"installation_spells",
"timestamp", "timestamp",
"definition", "definition",
"dealId", "dealId",
@ -80,6 +56,13 @@
"additionalProperties": { "additionalProperties": {
"type": "object", "type": "object",
"properties": { "properties": {
"definition": {
"type": "string"
},
"timestamp": {
"type": "string",
"description": "ISO timestamp of the time when the worker was deployed"
},
"installation_spells": { "installation_spells": {
"type": "array", "type": "array",
"description": "A list of installation spells", "description": "A list of installation spells",
@ -103,21 +86,14 @@
] ]
} }
}, },
"definition": {
"type": "string"
},
"timestamp": {
"type": "string",
"description": "ISO timestamp of the time when the worker was deployed"
},
"relayId": { "relayId": {
"type": "string" "type": "string"
} }
}, },
"required": [ "required": [
"installation_spells",
"timestamp", "timestamp",
"definition", "definition",
"installation_spells",
"relayId" "relayId"
] ]
}, },

View File

@ -3,11 +3,8 @@ name = "local_storage"
load_from = "/Users/bebo/localdev/examples/marine-examples/url-downloader/target/wasm32-wasi/release/local_storage.wasm" load_from = "/Users/bebo/localdev/examples/marine-examples/url-downloader/target/wasm32-wasi/release/local_storage.wasm"
logger_enabled = true logger_enabled = true
[module.wasi] [module.wasi.mapped_dirs]
preopened_files = [ "./tmp" ] sites = "./tmp"
[module.wasi.mapped_dirs]
sites = "./tmp"
[[module]] [[module]]
name = "curl_adapter" name = "curl_adapter"

View File

@ -5,12 +5,20 @@
# Documentation: https://github.com/fluencelabs/fluence-cli/tree/main/docs/configs/workers.md # Documentation: https://github.com/fluencelabs/fluence-cli/tree/main/docs/configs/workers.md
version: 0 version: 0
hosts:
defaultWorker:
definition: bafkreicdordcic2qmd2pnacl5adffsuhwg4eleu4prcgaldjvzr6ds2f2m
installation_spells:
- host_id: 12D3KooWJ4bTHirdTFNZpCS72TAzwtdmavTBkkEXtzo6wHL25CtE
spell_id: 039f4e06-923e-4a21-8814-8164fa9b137c
worker_id: 12D3KooWKsxVXgnpL4wrTts9d49jiNSEN1auaxotdMYa4HpFS66g
timestamp: 2023-04-23T07:00:05.549Z
relayId: 12D3KooWAKNos2KogexTXhrkMZzFYpLHuWJ4PgoAhurSAv7o5CWA
deals: deals:
defaultWorker: defaultWorker:
installation_spells: [] definition: bafkreicdordcic2qmd2pnacl5adffsuhwg4eleu4prcgaldjvzr6ds2f2m
definition: bafkreihndikxxxvl6r2alrfzh7awwywiolyklkwxzmzoyfcolvlxho2zuy timestamp: 2023-04-23T07:01:18.682Z
timestamp: 2023-04-04T00:29:31.854Z dealIdOriginal: "0x9d7503E13Bc2d3430536e3d5D635D8172141d884"
dealIdOriginal: "0xd0c75aEEA62bc0ABf16F5f01361277CC8287AB0E" dealId: 9d7503e13bc2d3430536e3d5d635d8172141d884
dealId: d0c75aeea62bc0abf16f5f01361277cc8287ab0e
chainNetwork: testnet chainNetwork: testnet
chainNetworkId: 80001 chainNetworkId: 1313161555

View File

@ -1,16 +0,0 @@
# yaml-language-server: $schema=schemas/workers.yaml.json
# A result of app deployment. This file is created automatically after successful deployment using `fluence workers deploy` command
# Documentation: https://github.com/fluencelabs/fluence-cli/tree/main/docs/configs/workers.md
version: 0
deals:
defaultWorker:
installation_spells: []
definition: bafkreigvh7cxlf5nkisqqxq6alpoj32l44ttxkfxafa6agp2u344kl3k4m
timestamp: 2023-04-03T22:37:49.594Z
dealIdOriginal: "0xEA6f3D3177c40123C7B8B1d89440F1DdcfA6014B"
dealId: ea6f3d3177c40123c7b8b1d89440f1ddcfa6014b
chainNetwork: testnet
chainNetworkId: 80001

View File

@ -1,12 +0,0 @@
# yaml-language-server: $schema=.fluence/schemas/fluence-lock.yaml.json
# Defines a lock file for Fluence Project dependencies. When dependencies are installed - their exact versions are saved here.
# Documentation: https://github.com/fluencelabs/fluence-cli/tree/main/docs/configs/fluence-lock.md
version: 0
npm:
"@fluencelabs/aqua": 0.10.3
"@fluencelabs/aqua-lib": 0.6.0
"@fluencelabs/spell": 0.5.7
"@fluencelabs/registry": 0.8.2

View File

@ -7,14 +7,8 @@
version: 2 version: 2
aquaInputPath: src/aqua/main.aqua aquaInputPath: src/aqua/main.aqua
dependencies: dependencies:
npm:
"@fluencelabs/aqua": 0.10.3
"@fluencelabs/aqua-lib": 0.6.0
"@fluencelabs/spell": 0.5.7
"@fluencelabs/registry": 0.8.2
cargo: cargo:
marine: 0.14.0 mrepl: 0.21.3
mrepl: 0.21.2
workers: workers:
defaultWorker: defaultWorker:
services: [ url_downloader ] services: [ url_downloader ]
@ -25,8 +19,8 @@ deals:
hosts: hosts:
defaultWorker: defaultWorker:
peerIds: peerIds:
- 12D3KooWDUszU2NeWyUVjCXhGEt1MoZrhvdmaQQwtZUriuGN1jTr - 12D3KooWJ4bTHirdTFNZpCS72TAzwtdmavTBkkEXtzo6wHL25CtE #stage
relays: kras relays: stage
services: services:
url_downloader: url_downloader:
get: service get: service

View File

@ -1,52 +1,59 @@
import "workers.aqua"
import "services.aqua"
import "@fluencelabs/aqua-lib/builtin.aqua" import "@fluencelabs/aqua-lib/builtin.aqua"
import "@fluencelabs/registry/subnetwork.aqua" import "@fluencelabs/registry/subnetwork.aqua"
import Registry, Record from "@fluencelabs/registry/registry-service.aqua" import Registry, Record from "@fluencelabs/registry/registry-service.aqua"
import "@fluencelabs/spell/spell_service.aqua" import "@fluencelabs/spell/spell_service.aqua"
export get_and_save, put, put_all, get, worker_n import "workers.aqua"
import "services.aqua"
export put, get
export put_workers, get_workers, get_and_save_workers
func getWorkers() -> []Record: func getWorkers() -> []Record:
workersInfo <- getWorkersInfo() workersInfo <- getWorkersInfo()
dealId = workersInfo.deals.defaultWorker.dealId dealId = workersInfo.deals.defaultWorker.dealId
on HOST_PEER_ID:
workersOp <- resolveSubnetwork(dealId) workersOp <- resolveSubnetwork(dealId)
<- workersOp! <- workersOp!
func worker_n() -> u32: func put(file_name: string, file_content: []u8) -> string:
workers <- getWorkers() workers <- getWorkers()
<- workers.length w = workers[0]
on w.metadata.peer_id via w.metadata.relay_id:
res <- UrlDownloader.put(file_name, file_content)
<- res
func get(file_name: string) -> []u8:
workers <- getWorkers()
w = workers[0]
on w.metadata.peer_id via w.metadata.relay_id:
res <- UrlDownloader.get(file_name)
<- res
func get_and_save(url: string, fname: string) -> string: func get_and_save(url: string, fname: string) -> string:
workers <- getWorkers() workers <- getWorkers()
w = workers[8] w = workers[0]
on w.metadata.peer_id via w.metadata.relay_id: on w.metadata.peer_id via w.metadata.relay_id:
res = UrlDownloader.get_n_save(url, fname) res = UrlDownloader.get_n_save(url, fname)
<- res <- res
func put_all(file_name: string, file_content: []u8) -> []string: func put_workers(file_name: string, file_content: []u8) -> string:
res: *string info <- getWorkersInfo()
workers <- getWorkers() spell = info.hosts.defaultWorker.installationSpells[0]
for w <- workers: on spell.workerId via spell.hostId:
on w.metadata.peer_id via w.metadata.relay_id:
try:
res <- UrlDownloader.put(file_name, file_content)
catch e:
res <<- "failed to execute"
<- res
func put(file_name: string, file_content: []u8) -> string:
workers <- getWorkers()
w = workers[1]
on w.metadata.peer_id via w.metadata.relay_id:
res <- UrlDownloader.put(file_name, file_content) res <- UrlDownloader.put(file_name, file_content)
<- res <- res
func get(file_name: string) -> []u8: func get_workers(file_name: string) -> []u8:
workers <- getWorkers() info <- getWorkersInfo()
w = workers[5] spell = info.hosts.defaultWorker.installationSpells[0]
on w.metadata.peer_id via w.metadata.relay_id: on spell.workerId via spell.hostId:
res <- UrlDownloader.get(file_name) res <- UrlDownloader.get(file_name)
<- res <- res
func get_and_save_workers(url: string, fname: string) -> string:
info <- getWorkersInfo()
spell = info.hosts.defaultWorker.installationSpells[0]
on spell.workerId via spell.hostId:
res = UrlDownloader.get_n_save(url, fname)
<- res