From a44f6f17486623ee3f6e712ca29d4e5e5f53b7c1 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Thu, 2 Feb 2023 21:54:43 +0530 Subject: [PATCH 1/5] feat: add archive base_path in settings --- config/default.toml | 3 +++ src/settings.rs | 28 ++++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/config/default.toml b/config/default.toml index 4a39d18..493d211 100644 --- a/config/default.toml +++ b/config/default.toml @@ -34,6 +34,9 @@ password = "password" name = "postgres" pool = 4 +[archive] +base_path = "/tmp/mcaptcha-survey" + [footer] about = "https://mcapthca.org/about" donate = "https://mcapthca.org/donate" diff --git a/src/settings.rs b/src/settings.rs index b147f18..306fe80 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -15,6 +15,7 @@ * along with this program. If not, see . */ use std::env; +use std::fs; use std::path::Path; use config::{Config, ConfigError, Environment, File}; @@ -84,6 +85,25 @@ pub struct Footer { pub thanks: Url, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Archive { + pub base_path: String, +} + +impl Archive { + fn create_archive_base_path(&self) { + let base_path = Path::new(&self.base_path); + if base_path.exists() { + if !base_path.is_dir() { + fs::remove_file(&base_path).unwrap(); + fs::create_dir_all(&base_path).unwrap(); + } + } else { + fs::create_dir_all(&base_path).unwrap(); + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Settings { pub debug: bool, @@ -94,6 +114,7 @@ pub struct Settings { pub support_email: String, pub default_campaign: String, pub footer: Footer, + pub archive: Archive, } #[cfg(not(tarpaulin_include))] @@ -143,8 +164,11 @@ impl Settings { set_database_url(&mut s); - match s.try_into() { - Ok(val) => Ok(val), + match s.try_into::() { + Ok(val) => { + val.archive.create_archive_base_path(); + Ok(val) + }, Err(e) => Err(ConfigError::Message(format!("\n\nError: {}. If it says missing fields, then please refer to https://github.com/mCaptcha/mcaptcha#configuration to learn more about how mcaptcha reads configuration\n\n", e))), } } From 604fca0a622f160b5ad50533ccc269b2d6639d10 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Thu, 2 Feb 2023 21:55:13 +0530 Subject: [PATCH 2/5] feat: archive campaign and benchmark data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DESCRIPTION FORMATS - Campaign configuration is stored in JSON format - Benchmark data is stored in CSV format DIRECTORY STRUCTURE Each campaign gets a separate directory. A campaign can have multiple archives. Archives are stored in directories whose names would be the same as the UNIX timestamp of when they were recorded. EXAMPLE The example below shows three campaigns with one archive each. Each archive is stored in a directory denoting the moment in which the archive was generated. Each archive includes campaign configuration and benchmark. ```bash 14:53 atm@lab archive → tree . ├── 4e951e01-71ee-4a18-9b97-782965495ae3 │   └── 1675329764 │   ├── benchmark.csv │   └── challenge.json ├── 9d16df08-dffc-484e-bbe2-10c00b431c7e │   └── 1675329764 │   ├── benchmark.csv │   └── challenge.json └── fa9f7c24-afec-4505-adb9-8e0c3ce54d37 └── 1675329764 ├── benchmark.csv └── challenge.json 7 directories, 6 files ``` --- Cargo.lock | 36 +++++++ Cargo.toml | 2 + src/archive.rs | 249 +++++++++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 6 +- 4 files changed, 292 insertions(+), 1 deletion(-) create mode 100644 src/archive.rs diff --git a/Cargo.lock b/Cargo.lock index fa4e85f..4596e49 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -502,6 +502,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f0778972c64420fdedc63f09919c8a88bda7b25135357fd25a5d9f3257e832" dependencies = [ "memchr", + "once_cell", + "regex-automata", "serde 1.0.152", ] @@ -737,6 +739,32 @@ dependencies = [ "typenum", ] +[[package]] +name = "csv-async" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c652d4c48e4dc80b26fadd169c02fb6053d9f57507ddd3e6b8706e7d0242235e" +dependencies = [ + "bstr", + "cfg-if", + "csv-core", + "futures", + "itoa", + "ryu", + "serde 1.0.152", + "tokio", + "tokio-stream", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + [[package]] name = "ctr" version = "0.9.2" @@ -2021,6 +2049,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" + [[package]] name = "regex-syntax" version = "0.6.28" @@ -2484,6 +2518,7 @@ dependencies = [ "argon2-creds", "cache-buster", "config", + "csv-async", "derive_builder", "derive_more", "futures", @@ -2498,6 +2533,7 @@ dependencies = [ "serde_json", "sqlx", "tera", + "tokio", "tracing", "url", "urlencoding", diff --git a/Cargo.toml b/Cargo.toml index 91e6d0e..134edd4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,8 @@ mime = "0.3.16" #sailfish = "0.3.2" tracing = { version = "0.1.37", features = ["log"] } tera = { version="1.17.1", features=["builtins"]} +tokio = { version = "1.25.0", features = ["fs"] } +csv-async = { version = "1.2.5", features = ["serde", "tokio"] } #tokio = "1.11.0" diff --git a/src/archive.rs b/src/archive.rs new file mode 100644 index 0000000..8bdfc88 --- /dev/null +++ b/src/archive.rs @@ -0,0 +1,249 @@ +use std::collections::HashMap; +/* + * Copyright (C) 2023 Aravinth Manivannan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use sqlx::types::time::OffsetDateTime; +use tokio::fs; +use tokio::io::AsyncWriteExt; +use uuid::Uuid; + +use crate::api::v1::admin::campaigns::runners::get_results; +use crate::api::v1::admin::campaigns::SurveyResponse; +use crate::{errors::ServiceResult, AppData, Settings}; + +const CAMPAIGN_INFO_FILE: &str = "campaign.json"; + +const BENCHMARK_FILE: &str = "benchmark.csv"; + +pub struct Archiver { + base_path: String, +} + +impl Archiver { + pub fn new(s: &Settings) -> Self { + Archiver { + base_path: s.archive.base_path.clone(), + } + } + fn campaign_path(&self, id: &Uuid) -> PathBuf { + Path::new(&self.base_path).join(&id.to_string()) + } + + async fn create_dir_util(p: &PathBuf) -> ServiceResult<()> { + if p.exists() { + if !p.is_dir() { + fs::remove_file(&p).await.unwrap(); + fs::create_dir_all(&p).await.unwrap(); + } + } else { + fs::create_dir_all(&p).await.unwrap(); + } + Ok(()) + } + + fn archive_path_now(&self, id: &Uuid) -> PathBuf { + let unix_time = OffsetDateTime::now_utc().unix_timestamp(); + self.campaign_path(id).join(unix_time.to_string()) + } + + fn campaign_file_path(&self, id: &Uuid) -> PathBuf { + self.archive_path_now(id).join(CAMPAIGN_INFO_FILE) + } + + fn benchmark_file_path(&self, id: &Uuid) -> PathBuf { + self.archive_path_now(id).join(BENCHMARK_FILE) + } + + async fn write_campaign_file(&self, c: &Campaign) -> ServiceResult<()> { + let archive_path = self.archive_path_now(&c.id); + Self::create_dir_util(&archive_path).await?; + let campaign_file_path = self.campaign_file_path(&c.id); + let contents = serde_json::to_string(c).unwrap(); + // fs::write(campaign_file_path, contents).await.unwrap(); + let mut file = fs::File::create(&campaign_file_path).await.unwrap(); + file.write(contents.as_bytes()).await.unwrap(); + file.flush().await.unwrap(); + + Ok(()) + } + + async fn write_benchmark_file( + &self, + c: &Campaign, + data: &AppData, + ) -> ServiceResult<()> { + let archive_path = self.archive_path_now(&c.id); + Self::create_dir_util(&archive_path).await?; + + let benchmark_file_path = self.benchmark_file_path(&c.id); + struct Username { + name: String, + } + let owner = sqlx::query_as!( + Username, + "SELECT + survey_admins.name + FROM + survey_admins + INNER JOIN survey_campaigns ON + survey_admins.ID = survey_campaigns.user_id + WHERE + survey_campaigns.ID = $1 + ", + &c.id + ) + .fetch_one(&data.db) + .await?; + + let mut page = 0; + let limit = 50; + let file = fs::OpenOptions::new() + .read(true) + .append(true) + .create(true) + .open(&benchmark_file_path) + .await + .unwrap(); + let mut wri = csv_async::AsyncWriter::from_writer(file); + + loop { + let mut resp = + get_results(&owner.name, &c.id, data, page, limit, None).await?; + + for r in resp.drain(0..) { + let csv_resp = to_hashmap(r, c); + let keys: Vec<&str> = csv_resp.keys().map(|k| k.as_str()).collect(); + wri.write_record(&keys).await.unwrap(); + + let values: Vec<&str> = csv_resp.values().map(|v| v.as_str()).collect(); + wri.write_record(&values).await.unwrap(); + wri.flush().await.unwrap(); + + //wri.serialize(csv_resp).await.unwrap(); + wri.flush().await.unwrap(); + } + page += 1; + + wri.flush().await.unwrap(); + if resp.len() < limit { + break; + } + } + + Ok(()) + } + + pub async fn archive(&self, data: &AppData) -> ServiceResult<()> { + let mut db_campaigns = sqlx::query_as!( + InnerCampaign, + "SELECT ID, name, difficulties, created_at FROM survey_campaigns" + ) + .fetch_all(&data.db) + .await?; + for c in db_campaigns.drain(0..) { + let campaign: Campaign = c.into(); + self.write_campaign_file(&campaign).await?; + self.write_benchmark_file(&campaign, data).await?; + } + Ok(()) + } +} + +pub fn to_hashmap(s: SurveyResponse, c: &Campaign) -> HashMap { + let mut map = HashMap::with_capacity(7 + c.difficulties.len()); + map.insert("user".into(), s.user.id.to_string()); + map.insert("device_user_provided".into(), s.device_user_provided); + map.insert( + "device_software_recognised".into(), + s.device_software_recognised, + ); + map.insert( + "threads".into(), + s.threads.map_or_else(|| "-".into(), |v| v.to_string()), + ); + map.insert("submitted_at".into(), s.submitted_at.to_string()); + map.insert("submission_type".into(), s.submission_type.to_string()); + for d in c.difficulties.iter() { + let bench = s + .benches + .iter() + .find(|b| b.difficulty == *d as i32) + .map_or_else(|| "-".into(), |v| v.duration.to_string()); + map.insert(format!("Difficulty: {d}"), bench); + } + map +} + +//#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +//pub struct CSVSurveyResp { +// pub user: Uuid, +// pub device_user_provided: String, +// pub device_software_recognised: String, +// pub id: usize, +// pub threads: Option, +// pub submitted_at: i64, +// pub submission_type: SubmissionType, +// pub benches: String, +//} +// +//impl From for CSVSurveyResp { +// fn from(s: SurveyResponse) -> Self { +// let mut benches = String::default(); +// for b in s.benches.iter() { +// benches = format!("{benches} ({})", b.to_csv_resp()); +// } +// Self { +// user: s.user.id, +// device_software_recognised: s.device_software_recognised, +// device_user_provided: s.device_user_provided, +// id: s.id, +// threads: s.threads, +// submission_type: s.submission_type, +// benches, +// submitted_at: s.submitted_at, +// } +// } +//} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct InnerCampaign { + id: Uuid, + name: String, + difficulties: Vec, + created_at: OffsetDateTime, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct Campaign { + pub id: Uuid, + pub name: String, + pub difficulties: Vec, + pub created_at: i64, +} + +impl From for Campaign { + fn from(i: InnerCampaign) -> Self { + Self { + id: i.id, + name: i.name, + difficulties: i.difficulties.iter().map(|d| *d as u32).collect(), + created_at: i.created_at.unix_timestamp(), + } + } +} diff --git a/src/main.rs b/src/main.rs index beaf39e..6822b59 100644 --- a/src/main.rs +++ b/src/main.rs @@ -27,6 +27,7 @@ use lazy_static::lazy_static; use log::info; mod api; +mod archive; mod data; mod errors; mod pages; @@ -70,7 +71,7 @@ pub type AppData = actix_web::web::Data>; #[cfg(not(tarpaulin_include))] #[actix_web::main] async fn main() -> std::io::Result<()> { - env::set_var("RUST_LOG", "info"); + //env::set_var("RUST_LOG", "info"); pretty_env_logger::init(); @@ -84,6 +85,9 @@ async fn main() -> std::io::Result<()> { sqlx::migrate!("./migrations/").run(&data.db).await.unwrap(); let data = actix_web::web::Data::new(data); + let arch = archive::Archiver::new(&data.settings); + arch.archive(&data).await.unwrap(); + let ip = settings.server.get_ip(); println!("Starting server on: http://{}", ip); From d2c52cc62caef70b8f24b7bda3fc079158f9ce58 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Tue, 7 Feb 2023 18:28:53 +0530 Subject: [PATCH 3/5] feat: cleanup archiver and include tests --- Cargo.lock | 268 +++++++++++++++++++++------------------- Cargo.toml | 3 + src/archive.rs | 328 +++++++++++++++++++++++++++++++++++-------------- src/tests.rs | 3 + 4 files changed, 379 insertions(+), 223 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4596e49..51a4811 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -48,15 +48,15 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.3.0" +version = "3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0070905b2c4a98d184c4e81025253cb192aa8a73827553f38e9410801ceb35bb" +checksum = "c2079246596c18b4a33e274ae10c0e50613f4d32a4198e09c7b93771013fed74" dependencies = [ "actix-codec", "actix-rt", "actix-service", "actix-utils", - "ahash", + "ahash 0.8.3", "base64 0.21.0", "bitflags", "brotli", @@ -95,7 +95,7 @@ dependencies = [ "actix-utils", "actix-web", "futures-util", - "serde 1.0.152", + "serde 1.0.153", "serde_json", "time", ] @@ -119,7 +119,7 @@ dependencies = [ "bytestring", "http", "regex", - "serde 1.0.152", + "serde 1.0.153", "tracing", ] @@ -175,7 +175,7 @@ dependencies = [ "anyhow", "async-trait", "derive_more", - "serde 1.0.152", + "serde 1.0.153", "serde_json", "time", "tracing", @@ -193,9 +193,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.3.0" +version = "4.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464e0fddc668ede5f26ec1f9557a8d44eda948732f40c6b0ad79126930eb775f" +checksum = "cd3cb42f9566ab176e1ef0b8b3a896529062b4efc6be0123046095914c4c1c96" dependencies = [ "actix-codec", "actix-http", @@ -206,7 +206,7 @@ dependencies = [ "actix-service", "actix-utils", "actix-web-codegen", - "ahash", + "ahash 0.7.6", "bytes", "bytestring", "cfg-if", @@ -223,7 +223,7 @@ dependencies = [ "once_cell", "pin-project-lite", "regex", - "serde 1.0.152", + "serde 1.0.153", "serde_json", "serde_urlencoded", "smallvec", @@ -234,9 +234,9 @@ dependencies = [ [[package]] name = "actix-web-codegen" -version = "4.1.0" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa9362663c8643d67b2d5eafba49e4cb2c8a053a29ed00a0bea121f17c76b13" +checksum = "2262160a7ae29e3415554a3f1fc04c764b1540c116aa524683208078b7a75bc9" dependencies = [ "actix-router", "proc-macro2", @@ -307,6 +307,18 @@ dependencies = [ "version_check", ] +[[package]] +name = "ahash" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "0.7.20" @@ -395,9 +407,9 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "async-trait" -version = "0.1.64" +version = "0.1.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" +checksum = "b84f9ebcc6c1f5b8cb160f6990096a5c127f423fcb6e1ccc46c370cbdfb75dfc" dependencies = [ "proc-macro2", "quote", @@ -497,14 +509,14 @@ dependencies = [ [[package]] name = "bstr" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7f0778972c64420fdedc63f09919c8a88bda7b25135357fd25a5d9f3257e832" +checksum = "5ffdb39cb703212f3c11973452c2861b972f757b021158f3516ba10f2fa8b2c1" dependencies = [ "memchr", "once_cell", "regex-automata", - "serde 1.0.152", + "serde 1.0.153", ] [[package]] @@ -527,9 +539,9 @@ checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "bytestring" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7f83e57d9154148e355404702e2694463241880b939570d7c97c014da7a69a1" +checksum = "238e4886760d98c4f899360c834fa93e62cf7f721ac3c2da375cbdf4b8679aae" dependencies = [ "bytes", ] @@ -543,7 +555,7 @@ dependencies = [ "derive_builder", "mime", "mime_guess", - "serde 1.0.152", + "serde 1.0.153", "serde_json", "sha2", "walkdir", @@ -627,7 +639,7 @@ dependencies = [ "lazy_static", "nom 5.1.2", "rust-ini", - "serde 1.0.152", + "serde 1.0.153", "serde-hjson", "serde_json", "toml", @@ -721,9 +733,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.14" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" dependencies = [ "cfg-if", ] @@ -741,9 +753,9 @@ dependencies = [ [[package]] name = "csv-async" -version = "1.2.5" +version = "1.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c652d4c48e4dc80b26fadd169c02fb6053d9f57507ddd3e6b8706e7d0242235e" +checksum = "71933d3f2d0481d5111cb2817b15b6961961458ec58adf8008194e6c850046f4" dependencies = [ "bstr", "cfg-if", @@ -751,7 +763,7 @@ dependencies = [ "futures", "itoa", "ryu", - "serde 1.0.152", + "serde 1.0.153", "tokio", "tokio-stream", ] @@ -776,9 +788,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.90" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90d59d9acd2a682b4e40605a242f6670eaa58c5957471cbf85e8aa6a0b97a5e8" +checksum = "9a140f260e6f3f79013b8bfc65e7ce630c9ab4388c6a89c71e07226f49487b72" dependencies = [ "cc", "cxxbridge-flags", @@ -788,9 +800,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.90" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebfa40bda659dd5c864e65f4c9a2b0aff19bea56b017b9b77c73d3766a453a38" +checksum = "da6383f459341ea689374bf0a42979739dc421874f112ff26f829b8040b8e613" dependencies = [ "cc", "codespan-reporting", @@ -803,15 +815,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.90" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "457ce6757c5c70dc6ecdbda6925b958aae7f959bda7d8fb9bde889e34a09dc03" +checksum = "90201c1a650e95ccff1c8c0bb5a343213bdd317c6e600a93075bca2eff54ec97" [[package]] name = "cxxbridge-macro" -version = "1.0.90" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebf883b7aacd7b2aeb2a7b338648ee19f57c140d4ee8e52c68979c6b2f7f2263" +checksum = "0b75aed41bb2e6367cae39e6326ef817a851db13c13e4f3263714ca3cfb8de56" dependencies = [ "proc-macro2", "quote", @@ -952,7 +964,7 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" dependencies = [ - "serde 1.0.152", + "serde 1.0.153", ] [[package]] @@ -1175,9 +1187,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" dependencies = [ "bytes", "fnv", @@ -1198,7 +1210,7 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" dependencies = [ - "ahash", + "ahash 0.7.6", ] [[package]] @@ -1277,9 +1289,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ "bytes", "fnv", @@ -1426,15 +1438,15 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "jobserver" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" +checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" dependencies = [ "libc", ] @@ -1618,7 +1630,16 @@ dependencies = [ "libc", "log", "wasi", - "windows-sys 0.45.0", + "windows-sys", +] + +[[package]] +name = "mktemp" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bdc1f74dd7bb717d39f784f844e490d935b3aa7e383008006dbbf29c1f7820a" +dependencies = [ + "uuid 1.2.2", ] [[package]] @@ -1688,9 +1709,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.17.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "opaque-debug" @@ -1743,7 +1764,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] @@ -1757,9 +1778,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" +checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" [[package]] name = "percent-encoding" @@ -1769,9 +1790,9 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "pest" -version = "2.5.5" +version = "2.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "028accff104c4e513bad663bbcd2ad7cfd5304144404c31ed0a77ac103d00660" +checksum = "8cbd939b234e95d72bc393d51788aec68aeeb5d51e748ca08ff3aad58cb722f7" dependencies = [ "thiserror", "ucd-trie", @@ -1779,9 +1800,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.5.5" +version = "2.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ac3922aac69a40733080f53c1ce7f91dcf57e1a5f6c52f421fadec7fbdc4b69" +checksum = "a81186863f3d0a27340815be8f2078dd8050b14cd71913db9fbda795e5f707d7" dependencies = [ "pest", "pest_generator", @@ -1789,9 +1810,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.5.5" +version = "2.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d06646e185566b5961b4058dd107e0a7f56e77c3f484549fb119867773c0f202" +checksum = "75a1ef20bf3193c15ac345acb32e26b3dc3223aff4d77ae4fc5359567683796b" dependencies = [ "pest", "pest_meta", @@ -1802,9 +1823,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.5.5" +version = "2.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6f60b2ba541577e2a0c307c8f39d1439108120eb7903adeb6497fa880c59616" +checksum = "5e3b284b1f13a20dc5ebc90aff59a51b8d7137c221131b52a7260c08cbc1cc80" dependencies = [ "once_cell", "pest", @@ -2090,9 +2111,9 @@ dependencies = [ [[package]] name = "rust-embed" -version = "6.4.2" +version = "6.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "283ffe2f866869428c92e0d61c2f35dfb4355293cdfdc48f49e895c15f1333d1" +checksum = "cb133b9a38b5543fad3807fb2028ea47c5f2b566f4f5e28a11902f1a358348b6" dependencies = [ "rust-embed-impl", "rust-embed-utils", @@ -2101,9 +2122,9 @@ dependencies = [ [[package]] name = "rust-embed-impl" -version = "6.3.1" +version = "6.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31ab23d42d71fb9be1b643fe6765d292c5e14d46912d13f3ae2815ca048ea04d" +checksum = "4d4e0f0ced47ded9a68374ac145edd65a6c1fa13a96447b873660b2a568a0fd7" dependencies = [ "proc-macro2", "quote", @@ -2114,9 +2135,9 @@ dependencies = [ [[package]] name = "rust-embed-utils" -version = "7.3.0" +version = "7.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1669d81dfabd1b5f8e2856b8bbe146c6192b0ba22162edc738ac0a5de18f054" +checksum = "512b0ab6853f7e14e3c8754acb43d6f748bb9ced66aa5915a6553ac8213f7731" dependencies = [ "sha2", "walkdir", @@ -2160,9 +2181,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" [[package]] name = "same-file" @@ -2181,9 +2202,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "scratch" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" +checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" [[package]] name = "sct" @@ -2209,9 +2230,9 @@ checksum = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8" [[package]] name = "serde" -version = "1.0.152" +version = "1.0.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "3a382c72b4ba118526e187430bb4963cd6d55051ebf13d9b25574d379cc98d20" dependencies = [ "serde_derive", ] @@ -2230,9 +2251,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "1ef476a5790f0f6decbc66726b6e5d63680ed518283e64c7df415989d880954f" dependencies = [ "proc-macro2", "quote", @@ -2241,13 +2262,13 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.93" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" +checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea" dependencies = [ "itoa", "ryu", - "serde 1.0.152", + "serde 1.0.153", ] [[package]] @@ -2259,7 +2280,7 @@ dependencies = [ "form_urlencoded", "itoa", "ryu", - "serde 1.0.152", + "serde 1.0.153", ] [[package]] @@ -2301,9 +2322,9 @@ checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" [[package]] name = "slab" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" dependencies = [ "autocfg", ] @@ -2325,9 +2346,9 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", @@ -2366,7 +2387,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcbc16ddba161afc99e14d1713a453747a2b07fc097d2009f4c300ec99286105" dependencies = [ - "ahash", + "ahash 0.7.6", "atoi", "base64 0.13.1", "bitflags", @@ -2398,7 +2419,7 @@ dependencies = [ "rand", "rustls", "rustls-pemfile", - "serde 1.0.152", + "serde 1.0.153", "serde_json", "sha1", "sha2", @@ -2410,7 +2431,7 @@ dependencies = [ "time", "tokio-stream", "url", - "uuid 1.3.0", + "uuid 1.2.2", "webpki-roots", "whoami", ] @@ -2428,7 +2449,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "serde 1.0.152", + "serde 1.0.153", "serde_json", "sha2", "sqlx-core", @@ -2456,16 +2477,16 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "string_cache" -version = "0.8.4" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213494b7a2b503146286049378ce02b482200519accc31872ee8be91fa820a08" +checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" dependencies = [ "new_debug_unreachable", "once_cell", "parking_lot 0.12.1", "phf_shared 0.10.0", "precomputed-hash", - "serde 1.0.152", + "serde 1.0.153", ] [[package]] @@ -2526,10 +2547,11 @@ dependencies = [ "log", "mime", "mime_guess", + "mktemp", "pretty_env_logger", "rand", "rust-embed", - "serde 1.0.152", + "serde 1.0.153", "serde_json", "sqlx", "tera", @@ -2543,9 +2565,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.107" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", "quote", @@ -2579,7 +2601,7 @@ dependencies = [ "pest_derive", "rand", "regex", - "serde 1.0.152", + "serde 1.0.153", "serde_json", "slug", "unic-segment", @@ -2596,18 +2618,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.38" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" +checksum = "a5ab016db510546d856297882807df8da66a16fb8c4101cb8b30054b0d5b2d9c" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.38" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" +checksum = "5420d42e90af0c38c3290abcca25b9b3bdf379fc9f55c528f53a269d9c9a267e" dependencies = [ "proc-macro2", "quote", @@ -2626,12 +2648,12 @@ dependencies = [ [[package]] name = "time" -version = "0.3.17" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" +checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" dependencies = [ "itoa", - "serde 1.0.152", + "serde 1.0.153", "time-core", "time-macros", ] @@ -2644,9 +2666,9 @@ checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" [[package]] name = "time-macros" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" +checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" dependencies = [ "time-core", ] @@ -2668,9 +2690,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.25.0" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" +checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" dependencies = [ "autocfg", "bytes", @@ -2682,7 +2704,7 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "socket2", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -2698,9 +2720,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" +checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" dependencies = [ "futures-core", "pin-project-lite", @@ -2727,7 +2749,7 @@ version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" dependencies = [ - "serde 1.0.152", + "serde 1.0.153", ] [[package]] @@ -2851,9 +2873,9 @@ checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" [[package]] name = "unicode-normalization" @@ -2907,7 +2929,7 @@ dependencies = [ "form_urlencoded", "idna 0.3.0", "percent-encoding", - "serde 1.0.152", + "serde 1.0.153", ] [[package]] @@ -2929,14 +2951,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ "getrandom", - "serde 1.0.152", + "serde 1.0.153", ] [[package]] name = "uuid" -version = "1.3.0" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1674845326ee10d37ca60470760d4288a6f80f304007d92e5c53bab78c9cfd79" +checksum = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c" +dependencies = [ + "getrandom", +] [[package]] name = "validator" @@ -2947,7 +2972,7 @@ dependencies = [ "idna 0.2.3", "lazy_static", "regex", - "serde 1.0.152", + "serde 1.0.153", "serde_derive", "serde_json", "url", @@ -2964,7 +2989,7 @@ dependencies = [ "idna 0.2.3", "lazy_static", "regex", - "serde 1.0.152", + "serde 1.0.153", "serde_derive", "serde_json", "url", @@ -3170,21 +3195,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - [[package]] name = "windows-sys" version = "0.45.0" diff --git a/Cargo.toml b/Cargo.toml index 134edd4..6f55fff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -86,3 +86,6 @@ serde_json = "1" #yaml-rust = "0.4.5" cache-buster = { version = "0.2.0", git = "https://github.com/realaravinth/cache-buster" } mime = "0.3.16" + +[dev-dependencies] +mktemp = "0.5.0" diff --git a/src/archive.rs b/src/archive.rs index 8bdfc88..8e5866b 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -1,4 +1,3 @@ -use std::collections::HashMap; /* * Copyright (C) 2023 Aravinth Manivannan * @@ -28,22 +27,50 @@ use crate::api::v1::admin::campaigns::SurveyResponse; use crate::{errors::ServiceResult, AppData, Settings}; const CAMPAIGN_INFO_FILE: &str = "campaign.json"; - const BENCHMARK_FILE: &str = "benchmark.csv"; pub struct Archiver { base_path: String, } +pub struct Archive { + now: i64, + base_path: String, + campaign: Uuid, +} + +impl Archive { + pub fn new(campaign: Uuid, base_path: String) -> Self { + let now = OffsetDateTime::now_utc().unix_timestamp(); + Self { + now, + campaign, + base_path, + } + } + + fn campaign_path(&self) -> PathBuf { + Path::new(&self.base_path).join(&self.campaign.to_string()) + } + fn archive_path_now(&self) -> PathBuf { + self.campaign_path().join(self.now.to_string()) + } + + fn campaign_file_path(&self) -> PathBuf { + self.archive_path_now().join(CAMPAIGN_INFO_FILE) + } + + fn benchmark_file_path(&self) -> PathBuf { + self.archive_path_now().join(BENCHMARK_FILE) + } +} + impl Archiver { pub fn new(s: &Settings) -> Self { Archiver { base_path: s.archive.base_path.clone(), } } - fn campaign_path(&self, id: &Uuid) -> PathBuf { - Path::new(&self.base_path).join(&id.to_string()) - } async fn create_dir_util(p: &PathBuf) -> ServiceResult<()> { if p.exists() { @@ -57,41 +84,71 @@ impl Archiver { Ok(()) } - fn archive_path_now(&self, id: &Uuid) -> PathBuf { - let unix_time = OffsetDateTime::now_utc().unix_timestamp(); - self.campaign_path(id).join(unix_time.to_string()) - } - - fn campaign_file_path(&self, id: &Uuid) -> PathBuf { - self.archive_path_now(id).join(CAMPAIGN_INFO_FILE) - } - - fn benchmark_file_path(&self, id: &Uuid) -> PathBuf { - self.archive_path_now(id).join(BENCHMARK_FILE) - } - - async fn write_campaign_file(&self, c: &Campaign) -> ServiceResult<()> { - let archive_path = self.archive_path_now(&c.id); + async fn write_campaign_file(&self, c: &Campaign, a: &Archive) -> ServiceResult<()> { + let archive_path = a.archive_path_now(); Self::create_dir_util(&archive_path).await?; - let campaign_file_path = self.campaign_file_path(&c.id); + let campaign_file_path = a.campaign_file_path(); let contents = serde_json::to_string(c).unwrap(); // fs::write(campaign_file_path, contents).await.unwrap(); let mut file = fs::File::create(&campaign_file_path).await.unwrap(); - file.write(contents.as_bytes()).await.unwrap(); + file.write_all(contents.as_bytes()).await.unwrap(); file.flush().await.unwrap(); Ok(()) } + fn get_headers(c: &Campaign) -> Vec { + let mut keys = vec![ + "ID".to_string(), + "user".to_string(), + "device_user_provided".to_string(), + "device_software_recognised".to_string(), + "threads".to_string(), + "submitted_at".to_string(), + "submission_type".to_string(), + ]; + + let mut diff_order = Vec::with_capacity(c.difficulties.len()); + + for d in c.difficulties.iter() { + diff_order.push(d); + keys.push(format!("Difficulty {}", d)); + } + + keys + } + + fn extract_record(c: &Campaign, r: SurveyResponse) -> Vec { + let mut rec = vec![ + r.id.to_string(), + r.user.id.to_string(), + r.device_user_provided, + r.device_software_recognised, + r.threads.map_or_else(|| "-".into(), |v| v.to_string()), + r.submitted_at.to_string(), + r.submission_type.to_string(), + ]; + for d in c.difficulties.iter() { + let bench = r + .benches + .iter() + .find(|b| b.difficulty == *d as i32) + .map_or_else(|| "-".into(), |v| v.duration.to_string()); + rec.push(bench); + } + rec + } + async fn write_benchmark_file( &self, c: &Campaign, + archive: &Archive, data: &AppData, ) -> ServiceResult<()> { - let archive_path = self.archive_path_now(&c.id); + let archive_path = archive.archive_path_now(); Self::create_dir_util(&archive_path).await?; - let benchmark_file_path = self.benchmark_file_path(&c.id); + let benchmark_file_path = archive.benchmark_file_path(); struct Username { name: String, } @@ -122,30 +179,25 @@ impl Archiver { .unwrap(); let mut wri = csv_async::AsyncWriter::from_writer(file); + let keys = Self::get_headers(c); + wri.write_record(&keys).await.unwrap(); + loop { let mut resp = get_results(&owner.name, &c.id, data, page, limit, None).await?; for r in resp.drain(0..) { - let csv_resp = to_hashmap(r, c); - let keys: Vec<&str> = csv_resp.keys().map(|k| k.as_str()).collect(); - wri.write_record(&keys).await.unwrap(); - - let values: Vec<&str> = csv_resp.values().map(|v| v.as_str()).collect(); - wri.write_record(&values).await.unwrap(); - wri.flush().await.unwrap(); - - //wri.serialize(csv_resp).await.unwrap(); + let rec = Self::extract_record(c, r); + wri.write_record(&rec).await.unwrap(); wri.flush().await.unwrap(); } - page += 1; - wri.flush().await.unwrap(); if resp.len() < limit { break; + } else { + page += 1 } } - Ok(()) } @@ -158,69 +210,14 @@ impl Archiver { .await?; for c in db_campaigns.drain(0..) { let campaign: Campaign = c.into(); - self.write_campaign_file(&campaign).await?; - self.write_benchmark_file(&campaign, data).await?; + let archive = Archive::new(campaign.id.clone(), self.base_path.clone()); + self.write_campaign_file(&campaign, &archive).await?; + self.write_benchmark_file(&campaign, &archive, data).await?; } Ok(()) } } -pub fn to_hashmap(s: SurveyResponse, c: &Campaign) -> HashMap { - let mut map = HashMap::with_capacity(7 + c.difficulties.len()); - map.insert("user".into(), s.user.id.to_string()); - map.insert("device_user_provided".into(), s.device_user_provided); - map.insert( - "device_software_recognised".into(), - s.device_software_recognised, - ); - map.insert( - "threads".into(), - s.threads.map_or_else(|| "-".into(), |v| v.to_string()), - ); - map.insert("submitted_at".into(), s.submitted_at.to_string()); - map.insert("submission_type".into(), s.submission_type.to_string()); - for d in c.difficulties.iter() { - let bench = s - .benches - .iter() - .find(|b| b.difficulty == *d as i32) - .map_or_else(|| "-".into(), |v| v.duration.to_string()); - map.insert(format!("Difficulty: {d}"), bench); - } - map -} - -//#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -//pub struct CSVSurveyResp { -// pub user: Uuid, -// pub device_user_provided: String, -// pub device_software_recognised: String, -// pub id: usize, -// pub threads: Option, -// pub submitted_at: i64, -// pub submission_type: SubmissionType, -// pub benches: String, -//} -// -//impl From for CSVSurveyResp { -// fn from(s: SurveyResponse) -> Self { -// let mut benches = String::default(); -// for b in s.benches.iter() { -// benches = format!("{benches} ({})", b.to_csv_resp()); -// } -// Self { -// user: s.user.id, -// device_software_recognised: s.device_software_recognised, -// device_user_provided: s.device_user_provided, -// id: s.id, -// threads: s.threads, -// submission_type: s.submission_type, -// benches, -// submitted_at: s.submitted_at, -// } -// } -//} - #[derive(Clone, Debug, Eq, PartialEq)] struct InnerCampaign { id: Uuid, @@ -247,3 +244,146 @@ impl From for Campaign { } } } + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use csv_async::StringRecord; + use futures::stream::StreamExt; + + use crate::api::v1::bench::Submission; + use crate::api::v1::bench::SubmissionType; + use crate::*; + + use super::*; + use mktemp::Temp; + + #[test] + fn archive_path_works() { + let mut settings = Settings::new().unwrap(); + let tmp_dir = Temp::new_dir().unwrap(); + settings.archive.base_path = tmp_dir.join("base_path").to_str().unwrap().into(); + + let uuid = Uuid::new_v4(); + let archive = Archive::new(uuid.clone(), settings.archive.base_path.clone()); + let archive_path = archive.archive_path_now(); + assert_eq!( + archive_path, + Path::new(&settings.archive.base_path) + .join(&uuid.to_string()) + .join(&archive.now.to_string()) + ); + + let campaign_file_path = archive.campaign_file_path(); + assert_eq!( + campaign_file_path, + Path::new(&settings.archive.base_path) + .join(&uuid.to_string()) + .join(&archive.now.to_string()) + .join(CAMPAIGN_INFO_FILE) + ); + + let benchmark_file_path = archive.benchmark_file_path(); + assert_eq!( + benchmark_file_path, + Path::new(&settings.archive.base_path) + .join(&uuid.to_string()) + .join(&archive.now.to_string()) + .join(BENCHMARK_FILE) + ); + } + + #[actix_rt::test] + async fn archive_is_correct_test() { + use crate::tests::*; + + const NAME: &str = "arciscorrecttesuser"; + const EMAIL: &str = "archive_is_correct_testuser@testadminuser.com"; + const PASSWORD: &str = "longpassword2"; + + const DEVICE_USER_PROVIDED: &str = "foo"; + const DEVICE_SOFTWARE_RECOGNISED: &str = "Foobar.v2"; + const THREADS: i32 = 4; + + { + let data = get_test_data().await; + delete_user(NAME, &data).await; + } + + //let campaign: Campaign = c.into(); + //let archive = Archive::new(campaign.id.clone(), self.base_path.clone()); + //self.write_campaign_file(&campaign, &archive).await?; + //self.write_benchmark_file(&campaign, &archive, data).await?; + + let (data, _creds, signin_resp) = + register_and_signin(NAME, EMAIL, PASSWORD).await; + let cookies = get_cookie!(signin_resp); + let survey = get_survey_user(data.clone()).await; + let survey_cookie = get_cookie!(survey); + let campaign = create_new_campaign(NAME, data.clone(), cookies.clone()).await; + let campaign_config = + get_campaign_config(&campaign, data.clone(), survey_cookie.clone()).await; + + assert_eq!(DIFFICULTIES.to_vec(), campaign_config.difficulties); + + let submit_payload = Submission { + device_user_provided: DEVICE_USER_PROVIDED.into(), + device_software_recognised: DEVICE_SOFTWARE_RECOGNISED.into(), + threads: THREADS, + benches: BENCHES.clone(), + submission_type: SubmissionType::Wasm, + }; + + let _proof = + submit_bench(&submit_payload, &campaign, survey_cookie, data.clone()).await; + + let campaign_id = Uuid::from_str(&campaign.campaign_id).unwrap(); + let db_campaign = sqlx::query_as!( + InnerCampaign, + "SELECT ID, name, difficulties, created_at FROM survey_campaigns WHERE ID = $1", + campaign_id, + ) + .fetch_one(&data.db) + .await.unwrap(); + let campaign: Campaign = db_campaign.into(); + + let archive = + Archive::new(campaign.id.clone(), data.settings.archive.base_path.clone()); + let archiver = Archiver::new(&data.settings); + archiver.archive(&AppData::new(data.clone())).await.unwrap(); + let contents: Campaign = serde_json::from_str( + &fs::read_to_string(&archive.campaign_file_path()) + .await + .unwrap(), + ) + .unwrap(); + assert_eq!(contents, campaign); + + let page = 0; + let limit = 10; + let mut responses = get_results( + NAME, + &campaign_id, + &AppData::new(data.clone()), + page, + limit, + None, + ) + .await + .unwrap(); + assert_eq!(responses.len(), 1); + let r = responses.pop().unwrap(); + let rec = Archiver::extract_record(&campaign, r); + + let mut rdr = csv_async::AsyncReader::from_reader( + fs::File::open(archive.benchmark_file_path()).await.unwrap(), + ); + + let mut records = rdr.records(); + assert_eq!( + records.next().await.unwrap().unwrap(), + StringRecord::from(rec) + ); + } +} diff --git a/src/tests.rs b/src/tests.rs index 41043c6..b5aa11b 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -25,6 +25,7 @@ use actix_web::{ error::ResponseError, http::StatusCode, }; +use mktemp::Temp; use lazy_static::lazy_static; use serde::Serialize; @@ -42,6 +43,8 @@ use crate::V1_API_ROUTES; pub async fn get_test_data() -> Arc { let mut settings = Settings::new().unwrap(); + let tmp_dir = Temp::new_dir().unwrap(); + settings.archive.base_path = tmp_dir.join("base_path").to_str().unwrap().into(); settings.allow_registration = true; Data::new(settings).await } From 9411c2ba9f5c4eb219425c98d57de6defe284310 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Sun, 12 Mar 2023 20:10:13 +0530 Subject: [PATCH 4/5] feat: read publication dir config and serve it --- Cargo.lock | 36 ++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + config/default.toml | 4 ++-- src/main.rs | 2 ++ src/settings.rs | 25 ++++++++++++------------- src/tests.rs | 3 ++- 6 files changed, 55 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 51a4811..3c7a366 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -46,6 +46,29 @@ dependencies = [ "smallvec", ] +[[package]] +name = "actix-files" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d832782fac6ca7369a70c9ee9a20554623c5e51c76e190ad151780ebea1cf689" +dependencies = [ + "actix-http", + "actix-service", + "actix-utils", + "actix-web", + "askama_escape", + "bitflags", + "bytes", + "derive_more", + "futures-core", + "http-range", + "log", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite", +] + [[package]] name = "actix-http" version = "3.3.1" @@ -405,6 +428,12 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" +[[package]] +name = "askama_escape" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341" + [[package]] name = "async-trait" version = "0.1.66" @@ -1298,6 +1327,12 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-range" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21dec9db110f5f872ed9699c3ecf50cf16f423502706ba5c72462e28d3157573" + [[package]] name = "httparse" version = "1.8.0" @@ -2529,6 +2564,7 @@ version = "0.1.0" dependencies = [ "actix-auth-middleware", "actix-cors", + "actix-files", "actix-http", "actix-identity", "actix-rt", diff --git a/Cargo.toml b/Cargo.toml index 6f55fff..c8e7adf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ actix-session = { version = "0.6.1", features = ["cookie-session"]} actix-http = "3.0.4" actix-rt = "2" actix-cors = "0.6.1" +actix-files = "0.6.0" actix-service = "2.0.0" #actix = "0.12" actix-web-codegen-const-routes = { version = "0.1.0", tag = "0.1.0", git = "https://github.com/realaravinth/actix-web-codegen-const-routes" } diff --git a/config/default.toml b/config/default.toml index 493d211..4fe75cd 100644 --- a/config/default.toml +++ b/config/default.toml @@ -34,8 +34,8 @@ password = "password" name = "postgres" pool = 4 -[archive] -base_path = "/tmp/mcaptcha-survey" +[publish] +dir = "/tmp/mcaptcha-survey" [footer] about = "https://mcapthca.org/about" diff --git a/src/main.rs b/src/main.rs index 6822b59..39f2b92 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,6 +17,7 @@ use std::env; use std::sync::Arc; +use actix_files::Files; use actix_identity::{CookieIdentityPolicy, IdentityService}; use actix_session::{storage::CookieSessionStore, SessionMiddleware}; use actix_web::{ @@ -106,6 +107,7 @@ async fn main() -> std::io::Result<()> { actix_middleware::TrailingSlash::Trim, )) .configure(services) + .service(Files::new("/download", &settings.publish.dir).show_files_listing()) .app_data(data.clone()) }) .bind(ip) diff --git a/src/settings.rs b/src/settings.rs index 306fe80..9976e24 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -85,21 +85,20 @@ pub struct Footer { pub thanks: Url, } -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Archive { - pub base_path: String, +pub struct Publish { + pub dir: String, } -impl Archive { - fn create_archive_base_path(&self) { - let base_path = Path::new(&self.base_path); - if base_path.exists() { - if !base_path.is_dir() { - fs::remove_file(&base_path).unwrap(); - fs::create_dir_all(&base_path).unwrap(); +impl Publish { + fn create_root_dir(&self) { + let root = Path::new(&self.dir); + if root.exists() { + if !root.is_dir() { + std::fs::remove_file(&root).unwrap(); + std::fs::create_dir_all(&root).unwrap(); } } else { - fs::create_dir_all(&base_path).unwrap(); + std::fs::create_dir_all(&root).unwrap(); } } } @@ -114,7 +113,7 @@ pub struct Settings { pub support_email: String, pub default_campaign: String, pub footer: Footer, - pub archive: Archive, + pub publish: Publish, } #[cfg(not(tarpaulin_include))] @@ -166,7 +165,7 @@ impl Settings { match s.try_into::() { Ok(val) => { - val.archive.create_archive_base_path(); + val.publish.create_root_dir(); Ok(val) }, Err(e) => Err(ConfigError::Message(format!("\n\nError: {}. If it says missing fields, then please refer to https://github.com/mCaptcha/mcaptcha#configuration to learn more about how mcaptcha reads configuration\n\n", e))), diff --git a/src/tests.rs b/src/tests.rs index b5aa11b..ae6920e 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -44,7 +44,7 @@ use crate::V1_API_ROUTES; pub async fn get_test_data() -> Arc { let mut settings = Settings::new().unwrap(); let tmp_dir = Temp::new_dir().unwrap(); - settings.archive.base_path = tmp_dir.join("base_path").to_str().unwrap().into(); + settings.publish.dir = tmp_dir.join("base_path").to_str().unwrap().into(); settings.allow_registration = true; Data::new(settings).await } @@ -126,6 +126,7 @@ macro_rules! get_app { .wrap(actix_web::middleware::NormalizePath::new( actix_web::middleware::TrailingSlash::Trim, )) + .service(Files::new("/download", &$settings.publish.dir).show_files_listing()) .configure($crate::services) }; From a3f2c3632e9de0a50af7e4bca7a2ede2e12aabf8 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Sun, 12 Mar 2023 20:11:06 +0530 Subject: [PATCH 5/5] feat: publish benchmark data periodically (configurable) --- config/default.toml | 1 + src/archive.rs | 75 ++++++++++++++++++++++++++++++++++++--------- src/main.rs | 14 +++++++-- src/settings.rs | 2 ++ 4 files changed, 74 insertions(+), 18 deletions(-) diff --git a/config/default.toml b/config/default.toml index 4fe75cd..a034271 100644 --- a/config/default.toml +++ b/config/default.toml @@ -36,6 +36,7 @@ pool = 4 [publish] dir = "/tmp/mcaptcha-survey" +duration = 3600 [footer] about = "https://mcapthca.org/about" diff --git a/src/archive.rs b/src/archive.rs index 8e5866b..e152435 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -14,13 +14,15 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +use std::future::Future; use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; use sqlx::types::time::OffsetDateTime; +use sqlx::types::Uuid; use tokio::fs; use tokio::io::AsyncWriteExt; -use uuid::Uuid; +use tokio::sync::oneshot::{self, error::TryRecvError, Sender}; use crate::api::v1::admin::campaigns::runners::get_results; use crate::api::v1::admin::campaigns::SurveyResponse; @@ -68,7 +70,7 @@ impl Archive { impl Archiver { pub fn new(s: &Settings) -> Self { Archiver { - base_path: s.archive.base_path.clone(), + base_path: s.publish.dir.clone(), } } @@ -163,7 +165,7 @@ impl Archiver { WHERE survey_campaigns.ID = $1 ", - &c.id + &Uuid::parse_str(&c.id.to_string()).unwrap() ) .fetch_one(&data.db) .await?; @@ -183,8 +185,15 @@ impl Archiver { wri.write_record(&keys).await.unwrap(); loop { - let mut resp = - get_results(&owner.name, &c.id, data, page, limit, None).await?; + let mut resp = get_results( + &owner.name, + &Uuid::parse_str(&c.id.to_string()).unwrap(), + data, + page, + limit, + None, + ) + .await?; for r in resp.drain(0..) { let rec = Self::extract_record(c, r); @@ -201,6 +210,40 @@ impl Archiver { Ok(()) } + pub async fn init_archive_job( + self, + data: AppData, + ) -> ServiceResult<(Sender, impl Future)> { + let (tx, mut rx) = oneshot::channel(); + + let job = async move { + loop { + // let rx = self.rx.as_mut().unwrap(); + match rx.try_recv() { + // The channel is currently empty + Ok(_) => { + log::info!("Killing archive loop: received signal"); + break; + } + Err(TryRecvError::Empty) => { + let _ = self.archive(&data).await; + + tokio::time::sleep(std::time::Duration::new( + data.settings.publish.duration, + 0, + )) + .await; + } + Err(TryRecvError::Closed) => break, + } + + let _ = self.archive(&data).await; + } + }; + let job_fut = tokio::spawn(job); + Ok((tx, job_fut)) + } + pub async fn archive(&self, data: &AppData) -> ServiceResult<()> { let mut db_campaigns = sqlx::query_as!( InnerCampaign, @@ -209,8 +252,8 @@ impl Archiver { .fetch_all(&data.db) .await?; for c in db_campaigns.drain(0..) { + let archive = Archive::new(c.id.clone(), self.base_path.clone()); let campaign: Campaign = c.into(); - let archive = Archive::new(campaign.id.clone(), self.base_path.clone()); self.write_campaign_file(&campaign, &archive).await?; self.write_benchmark_file(&campaign, &archive, data).await?; } @@ -228,7 +271,7 @@ struct InnerCampaign { #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct Campaign { - pub id: Uuid, + pub id: uuid::Uuid, pub name: String, pub difficulties: Vec, pub created_at: i64, @@ -237,7 +280,7 @@ pub struct Campaign { impl From for Campaign { fn from(i: InnerCampaign) -> Self { Self { - id: i.id, + id: uuid::Uuid::parse_str(&i.id.to_string()).unwrap(), name: i.name, difficulties: i.difficulties.iter().map(|d| *d as u32).collect(), created_at: i.created_at.unix_timestamp(), @@ -263,14 +306,14 @@ mod tests { fn archive_path_works() { let mut settings = Settings::new().unwrap(); let tmp_dir = Temp::new_dir().unwrap(); - settings.archive.base_path = tmp_dir.join("base_path").to_str().unwrap().into(); + settings.publish.dir = tmp_dir.join("base_path").to_str().unwrap().into(); let uuid = Uuid::new_v4(); - let archive = Archive::new(uuid.clone(), settings.archive.base_path.clone()); + let archive = Archive::new(uuid.clone(), settings.publish.dir.clone()); let archive_path = archive.archive_path_now(); assert_eq!( archive_path, - Path::new(&settings.archive.base_path) + Path::new(&settings.publish.dir) .join(&uuid.to_string()) .join(&archive.now.to_string()) ); @@ -278,7 +321,7 @@ mod tests { let campaign_file_path = archive.campaign_file_path(); assert_eq!( campaign_file_path, - Path::new(&settings.archive.base_path) + Path::new(&settings.publish.dir) .join(&uuid.to_string()) .join(&archive.now.to_string()) .join(CAMPAIGN_INFO_FILE) @@ -287,7 +330,7 @@ mod tests { let benchmark_file_path = archive.benchmark_file_path(); assert_eq!( benchmark_file_path, - Path::new(&settings.archive.base_path) + Path::new(&settings.publish.dir) .join(&uuid.to_string()) .join(&archive.now.to_string()) .join(BENCHMARK_FILE) @@ -348,8 +391,10 @@ mod tests { .await.unwrap(); let campaign: Campaign = db_campaign.into(); - let archive = - Archive::new(campaign.id.clone(), data.settings.archive.base_path.clone()); + let archive = Archive::new( + Uuid::parse_str(&campaign.id.to_string()).unwrap(), + data.settings.publish.dir.clone(), + ); let archiver = Archiver::new(&data.settings); archiver.archive(&AppData::new(data.clone())).await.unwrap(); let contents: Campaign = serde_json::from_str( diff --git a/src/main.rs b/src/main.rs index 39f2b92..b70c44d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -72,7 +72,9 @@ pub type AppData = actix_web::web::Data>; #[cfg(not(tarpaulin_include))] #[actix_web::main] async fn main() -> std::io::Result<()> { - //env::set_var("RUST_LOG", "info"); + if env::var("RUST_LOG").is_err() { + env::set_var("RUST_LOG", "info"); + } pretty_env_logger::init(); @@ -87,7 +89,8 @@ async fn main() -> std::io::Result<()> { let data = actix_web::web::Data::new(data); let arch = archive::Archiver::new(&data.settings); - arch.archive(&data).await.unwrap(); + let (archive_kiler, archive_job) = + arch.init_archive_job(data.clone()).await.unwrap(); let ip = settings.server.get_ip(); println!("Starting server on: http://{}", ip); @@ -106,14 +109,19 @@ async fn main() -> std::io::Result<()> { .wrap(actix_middleware::NormalizePath::new( actix_middleware::TrailingSlash::Trim, )) - .configure(services) .service(Files::new("/download", &settings.publish.dir).show_files_listing()) + .configure(services) .app_data(data.clone()) }) .bind(ip) .unwrap() .run() .await + .unwrap(); + + archive_kiler.send(true).unwrap(); + archive_job.await; + Ok(()) } #[cfg(not(tarpaulin_include))] diff --git a/src/settings.rs b/src/settings.rs index 9976e24..e820f03 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -85,8 +85,10 @@ pub struct Footer { pub thanks: Url, } +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Publish { pub dir: String, + pub duration: u64, } impl Publish {