From 2814b0e190fa68c7099a55aac7bcfa64b50bcb43 Mon Sep 17 00:00:00 2001 From: folex <0xdxdy@gmail.com> Date: Fri, 26 Feb 2021 01:13:05 +0300 Subject: [PATCH] replace url-downloader with a new one --- url-downloader-new/Config.toml | 21 ------- url-downloader-new/build.sh | 24 ------- url-downloader-new/curl_adapter/Cargo.toml | 14 ----- url-downloader-new/curl_adapter/src/main.rs | 48 -------------- url-downloader-new/facade/Cargo.toml | 15 ----- url-downloader-new/facade/src/main.rs | 63 ------------------- url-downloader-new/local_storage/Cargo.toml | 14 ----- url-downloader-new/local_storage/src/main.rs | 51 --------------- .../artifacts/curl_adapter.json | 0 .../artifacts/facade.json | 0 .../artifacts/local_storage.json | 0 url-downloader/build.sh | 36 ++++++----- .../curl_adapter/Cargo.lock | 0 url-downloader/curl_adapter/Cargo.toml | 2 +- url-downloader/curl_adapter/src/main.rs | 20 +++--- .../deploy.sh | 0 .../download.clj | 0 .../facade/Cargo.lock | 0 url-downloader/facade/Cargo.toml | 4 +- url-downloader/facade/src/main.rs | 24 +++++-- .../local_storage/Cargo.lock | 0 url-downloader/local_storage/Cargo.toml | 2 +- 22 files changed, 56 insertions(+), 282 deletions(-) delete mode 100644 url-downloader-new/Config.toml delete mode 100755 url-downloader-new/build.sh delete mode 100644 url-downloader-new/curl_adapter/Cargo.toml delete mode 100644 url-downloader-new/curl_adapter/src/main.rs delete mode 100644 url-downloader-new/facade/Cargo.toml delete mode 100644 url-downloader-new/facade/src/main.rs delete mode 100644 url-downloader-new/local_storage/Cargo.toml delete mode 100644 url-downloader-new/local_storage/src/main.rs rename {url-downloader-new => url-downloader}/artifacts/curl_adapter.json (100%) rename {url-downloader-new => url-downloader}/artifacts/facade.json (100%) rename {url-downloader-new => url-downloader}/artifacts/local_storage.json (100%) rename {url-downloader-new => url-downloader}/curl_adapter/Cargo.lock (100%) rename {url-downloader-new => url-downloader}/deploy.sh (100%) rename {url-downloader-new => url-downloader}/download.clj (100%) rename {url-downloader-new => url-downloader}/facade/Cargo.lock (100%) rename {url-downloader-new => url-downloader}/local_storage/Cargo.lock (100%) diff --git a/url-downloader-new/Config.toml b/url-downloader-new/Config.toml deleted file mode 100644 index d22508b..0000000 --- a/url-downloader-new/Config.toml +++ /dev/null @@ -1,21 +0,0 @@ -modules_dir = "artifacts/" - -[[module]] - name = "local_storage" - logger_enabled = true - - [module.wasi] - preopened_files = ["./sites"] - # this is where files will be stored - mapped_dirs = { "sites" = "./sites" } - -[[module]] - name = "curl_adapter" - logger_enabled = true - - [module.mounted_binaries] - curl = "/usr/bin/curl" - -[[module]] - name = "facade" - logger_enabled = true diff --git a/url-downloader-new/build.sh b/url-downloader-new/build.sh deleted file mode 100755 index fba8560..0000000 --- a/url-downloader-new/build.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh -euo pipefail - -# This script builds all subprojects and puts all created Wasm modules in one dir -( - cd local_storage - cargo update - fce build --release -) -( - cd curl_adapter - cargo update - fce build --release -) -( - cd facade - cargo update - fce build --release -) - -mkdir -p artifacts -rm -f artifacts/*.wasm -cp local_storage/target/wasm32-wasi/release/local_storage.wasm artifacts/ -cp curl_adapter/target/wasm32-wasi/release/curl_adapter.wasm artifacts/ -cp facade/target/wasm32-wasi/release/facade.wasm artifacts/ diff --git a/url-downloader-new/curl_adapter/Cargo.toml b/url-downloader-new/curl_adapter/Cargo.toml deleted file mode 100644 index 14f91b9..0000000 --- a/url-downloader-new/curl_adapter/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -name = "curl_adapter" -version = "0.1.0" -authors = ["Fluence Labs"] -edition = "2018" -publish = false - -[[bin]] -path = "src/main.rs" -name = "curl_adapter" - -[dependencies] -fluence = { version = "=0.3.3", features = ["logger"] } -log = "0.4.8" diff --git a/url-downloader-new/curl_adapter/src/main.rs b/url-downloader-new/curl_adapter/src/main.rs deleted file mode 100644 index 5903153..0000000 --- a/url-downloader-new/curl_adapter/src/main.rs +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2020 Fluence Labs Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#![allow(improper_ctypes)] - -use fluence::fce; - -use fluence::WasmLoggerBuilder; -use fluence::MountedBinaryResult as Result; -use fluence::MountedBinaryStringResult as StringResult; - -/// Log level can be changed by `RUST_LOG` env as well. -pub fn main() { - WasmLoggerBuilder::new().build().unwrap(); -} - -#[fce] -pub fn request(url: String) -> StringResult { - unsafe { curl(vec![url]) }.stringify().unwrap() -} - -#[fce] -pub fn download(url: String) -> Result { - log::info!("download called with url {}", url); - - unsafe { curl(vec![url]) } -} - - -/// Permissions in `Config.toml` should exist to use host functions. -#[fce] -#[link(wasm_import_module = "host")] -extern "C" { - fn curl(cmd: Vec) -> Result; -} diff --git a/url-downloader-new/facade/Cargo.toml b/url-downloader-new/facade/Cargo.toml deleted file mode 100644 index 3f10a66..0000000 --- a/url-downloader-new/facade/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "facade" -version = "0.1.0" -authors = ["Fluence Labs"] -edition = "2018" -publish = false - -[[bin]] -name = "facade" -path = "src/main.rs" - -[dependencies] -fluence = { version = "=0.3.3", features = ["logger"]} -log = "0.4.8" -base64 = "0.13.0" diff --git a/url-downloader-new/facade/src/main.rs b/url-downloader-new/facade/src/main.rs deleted file mode 100644 index 1a5e04c..0000000 --- a/url-downloader-new/facade/src/main.rs +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2020 Fluence Labs Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#![allow(improper_ctypes)] - -use fluence::fce; -use fluence::MountedBinaryResult as Result; -use fluence::WasmLoggerBuilder; - -pub fn main() { - WasmLoggerBuilder::new().build().unwrap(); -} - -/// Combining of modules: `curl` and `local_storage`. -/// Calls `curl` and stores returned result into a file. -#[fce] -pub fn get_n_save(url: String, file_name: String) -> i32 { - let result = unsafe { download(url) }; - if result.is_success() { - log::info!("saving file {}", file_name); - unsafe { file_put(file_name, result.stdout) }; - } else { - log::error!("download failed: {:#?}", result.as_std()) - } - - result.ret_code -} - -#[fce] -pub fn load_file(file_name: String) -> String { - let bytes = unsafe { file_get(file_name) }; - base64::encode(bytes) -} - -/// Import `curl_adapter` module -#[fce] -#[link(wasm_import_module = "curl_adapter")] -extern "C" { - pub fn download(url: String) -> Result; -} - -/// Import `local_storage` module -#[fce] -#[link(wasm_import_module = "local_storage")] -extern "C" { - #[link_name = "get"] - pub fn file_get(file_name: String) -> Vec; - - #[link_name = "put"] - pub fn file_put(name: String, file_content: Vec) -> String; -} diff --git a/url-downloader-new/local_storage/Cargo.toml b/url-downloader-new/local_storage/Cargo.toml deleted file mode 100644 index 0a0a5ea..0000000 --- a/url-downloader-new/local_storage/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -name = "local_storage" -version = "0.1.0" -authors = ["Fluence Labs"] -edition = "2018" -publish = false - -[[bin]] -name = "local_storage" -path = "src/main.rs" - -[dependencies] -fluence = { version = "=0.3.3", features = ["logger"]} -log = "0.4.8" diff --git a/url-downloader-new/local_storage/src/main.rs b/url-downloader-new/local_storage/src/main.rs deleted file mode 100644 index 015ddc9..0000000 --- a/url-downloader-new/local_storage/src/main.rs +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2020 Fluence Labs Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -use std::fs; -use fluence::fce; -use fluence::WasmLoggerBuilder; -use std::path::PathBuf; - -const SITES_DIR: &str = "/sites/"; - -/// Log level can be changed by `RUST_LOG` env as well. -pub fn main() { - WasmLoggerBuilder::new().build().unwrap(); -} - -/// You can read or write files from the file system if there is permission to use directories described in `Config.toml`. -#[fce] -pub fn put(name: String, file_content: Vec) -> String { - log::info!("put called with file name {}", name); - - let rpc_tmp_filepath = format!("{}{}", SITES_DIR, name); - - let result = fs::write(PathBuf::from(rpc_tmp_filepath.clone()), file_content); - if let Err(e) = result { - return format!("file can't be written: {}", e); - } - - String::from("Ok") -} - -#[fce] -pub fn get(file_name: String) -> Vec { - log::info!("get called with file name: {}", file_name); - - let tmp_filepath = format!("{}{}", SITES_DIR, file_name); - - fs::read(tmp_filepath).unwrap_or_else(|_| b"error while reading file".to_vec()) -} diff --git a/url-downloader-new/artifacts/curl_adapter.json b/url-downloader/artifacts/curl_adapter.json similarity index 100% rename from url-downloader-new/artifacts/curl_adapter.json rename to url-downloader/artifacts/curl_adapter.json diff --git a/url-downloader-new/artifacts/facade.json b/url-downloader/artifacts/facade.json similarity index 100% rename from url-downloader-new/artifacts/facade.json rename to url-downloader/artifacts/facade.json diff --git a/url-downloader-new/artifacts/local_storage.json b/url-downloader/artifacts/local_storage.json similarity index 100% rename from url-downloader-new/artifacts/local_storage.json rename to url-downloader/artifacts/local_storage.json diff --git a/url-downloader/build.sh b/url-downloader/build.sh index bacc9ad..fba8560 100755 --- a/url-downloader/build.sh +++ b/url-downloader/build.sh @@ -1,18 +1,24 @@ -#!/bin/sh +#!/bin/sh -euo pipefail # This script builds all subprojects and puts all created Wasm modules in one dir -cd local_storage -cargo update -fce build --release -cd ../curl_adapter -cargo update -fce build --release -cd ../facade -cargo update -fce build --release +( + cd local_storage + cargo update + fce build --release +) +( + cd curl_adapter + cargo update + fce build --release +) +( + cd facade + cargo update + fce build --release +) -cd .. -rm -f artifacts/* -cp ../../target/wasm32-wasi/release/local_storage.wasm artifacts/ -cp ../../target/wasm32-wasi/release/curl_adapter.wasm artifacts/ -cp ../../target/wasm32-wasi/release/facade.wasm artifacts/ +mkdir -p artifacts +rm -f artifacts/*.wasm +cp local_storage/target/wasm32-wasi/release/local_storage.wasm artifacts/ +cp curl_adapter/target/wasm32-wasi/release/curl_adapter.wasm artifacts/ +cp facade/target/wasm32-wasi/release/facade.wasm artifacts/ diff --git a/url-downloader-new/curl_adapter/Cargo.lock b/url-downloader/curl_adapter/Cargo.lock similarity index 100% rename from url-downloader-new/curl_adapter/Cargo.lock rename to url-downloader/curl_adapter/Cargo.lock diff --git a/url-downloader/curl_adapter/Cargo.toml b/url-downloader/curl_adapter/Cargo.toml index bb85e9b..14f91b9 100644 --- a/url-downloader/curl_adapter/Cargo.toml +++ b/url-downloader/curl_adapter/Cargo.toml @@ -10,5 +10,5 @@ path = "src/main.rs" name = "curl_adapter" [dependencies] -fluence = { version = "=0.3.2", features = ["logger"] } +fluence = { version = "=0.3.3", features = ["logger"] } log = "0.4.8" diff --git a/url-downloader/curl_adapter/src/main.rs b/url-downloader/curl_adapter/src/main.rs index 66df238..5903153 100644 --- a/url-downloader/curl_adapter/src/main.rs +++ b/url-downloader/curl_adapter/src/main.rs @@ -19,7 +19,8 @@ use fluence::fce; use fluence::WasmLoggerBuilder; -use fluence::MountedBinaryResult; +use fluence::MountedBinaryResult as Result; +use fluence::MountedBinaryStringResult as StringResult; /// Log level can be changed by `RUST_LOG` env as well. pub fn main() { @@ -27,16 +28,21 @@ pub fn main() { } #[fce] -pub fn download(url: String) -> String { - log::info!("get called with url {}", url); - - let result = unsafe { curl(vec![url]) }; - String::from_utf8(result.stdout).unwrap() +pub fn request(url: String) -> StringResult { + unsafe { curl(vec![url]) }.stringify().unwrap() } +#[fce] +pub fn download(url: String) -> Result { + log::info!("download called with url {}", url); + + unsafe { curl(vec![url]) } +} + + /// Permissions in `Config.toml` should exist to use host functions. #[fce] #[link(wasm_import_module = "host")] extern "C" { - fn curl(cmd: Vec) -> MountedBinaryResult; + fn curl(cmd: Vec) -> Result; } diff --git a/url-downloader-new/deploy.sh b/url-downloader/deploy.sh similarity index 100% rename from url-downloader-new/deploy.sh rename to url-downloader/deploy.sh diff --git a/url-downloader-new/download.clj b/url-downloader/download.clj similarity index 100% rename from url-downloader-new/download.clj rename to url-downloader/download.clj diff --git a/url-downloader-new/facade/Cargo.lock b/url-downloader/facade/Cargo.lock similarity index 100% rename from url-downloader-new/facade/Cargo.lock rename to url-downloader/facade/Cargo.lock diff --git a/url-downloader/facade/Cargo.toml b/url-downloader/facade/Cargo.toml index e6de140..3f10a66 100644 --- a/url-downloader/facade/Cargo.toml +++ b/url-downloader/facade/Cargo.toml @@ -10,6 +10,6 @@ name = "facade" path = "src/main.rs" [dependencies] -fluence = { version = "=0.3.2", features = ["logger"]} -anyhow = "1.0.31" +fluence = { version = "=0.3.3", features = ["logger"]} log = "0.4.8" +base64 = "0.13.0" diff --git a/url-downloader/facade/src/main.rs b/url-downloader/facade/src/main.rs index 3b2fedc..1a5e04c 100644 --- a/url-downloader/facade/src/main.rs +++ b/url-downloader/facade/src/main.rs @@ -16,6 +16,7 @@ #![allow(improper_ctypes)] use fluence::fce; +use fluence::MountedBinaryResult as Result; use fluence::WasmLoggerBuilder; pub fn main() { @@ -25,21 +26,32 @@ pub fn main() { /// Combining of modules: `curl` and `local_storage`. /// Calls `curl` and stores returned result into a file. #[fce] -pub fn get_n_save(url: String, file_name: String) -> String { +pub fn get_n_save(url: String, file_name: String) -> i32 { let result = unsafe { download(url) }; - unsafe { file_put(file_name, result.into_bytes()) }; + if result.is_success() { + log::info!("saving file {}", file_name); + unsafe { file_put(file_name, result.stdout) }; + } else { + log::error!("download failed: {:#?}", result.as_std()) + } - String::from("Ok") + result.ret_code } -/// Importing `curl` module +#[fce] +pub fn load_file(file_name: String) -> String { + let bytes = unsafe { file_get(file_name) }; + base64::encode(bytes) +} + +/// Import `curl_adapter` module #[fce] #[link(wasm_import_module = "curl_adapter")] extern "C" { - pub fn download(url: String) -> String; + pub fn download(url: String) -> Result; } -/// Importing `local_storage` module +/// Import `local_storage` module #[fce] #[link(wasm_import_module = "local_storage")] extern "C" { diff --git a/url-downloader-new/local_storage/Cargo.lock b/url-downloader/local_storage/Cargo.lock similarity index 100% rename from url-downloader-new/local_storage/Cargo.lock rename to url-downloader/local_storage/Cargo.lock diff --git a/url-downloader/local_storage/Cargo.toml b/url-downloader/local_storage/Cargo.toml index 95e5307..0a0a5ea 100644 --- a/url-downloader/local_storage/Cargo.toml +++ b/url-downloader/local_storage/Cargo.toml @@ -10,5 +10,5 @@ name = "local_storage" path = "src/main.rs" [dependencies] -fluence = { version = "=0.3.2", features = ["logger"]} +fluence = { version = "=0.3.3", features = ["logger"]} log = "0.4.8"