From 8e0e94f98bded9692fc242c89751a0d405bd3d29 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Sun, 2 Jul 2023 00:17:29 +0530 Subject: [PATCH 01/14] feat: DB methods to save analytics from mCaptcha/mCaptcha --- Cargo.lock | 314 ++++++++- Cargo.toml | 4 +- .../20230701082846_survey_mcaptcha_upload.sql | 22 + sqlx-data.json | 615 +----------------- src/api/v1/mcaptcha/db.rs | 509 +++++++++++++++ 5 files changed, 843 insertions(+), 621 deletions(-) create mode 100644 migrations/20230701082846_survey_mcaptcha_upload.sql create mode 100644 src/api/v1/mcaptcha/db.rs diff --git a/Cargo.lock b/Cargo.lock index 9e02df6..d8f0e51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -430,14 +430,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341" [[package]] -name = "async-trait" -version = "0.1.66" +name = "async-compression" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84f9ebcc6c1f5b8cb160f6990096a5c127f423fcb6e1ccc46c370cbdfb75dfc" +checksum = "f658e2baef915ba0f26f1f7c42bfb8e12f532a01f449a090ded75ae7a07e9ba2" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-trait" +version = "0.1.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.38", ] [[package]] @@ -719,6 +732,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.3" @@ -1090,6 +1113,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.1.0" @@ -1266,9 +1304,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.16" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" +checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" dependencies = [ "bytes", "fnv", @@ -1389,6 +1427,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + [[package]] name = "http-range" version = "0.1.5" @@ -1419,6 +1468,43 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hyper" +version = "0.14.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "iana-time-zone" version = "0.1.53" @@ -1522,6 +1608,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "ipnet" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" + [[package]] name = "is-terminal" version = "0.4.9" @@ -1772,6 +1864,24 @@ dependencies = [ "uuid", ] +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "new_debug_unreachable" version = "1.0.4" @@ -1858,6 +1968,50 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "openssl" +version = "0.10.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" +dependencies = [ + "bitflags 2.4.1", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.38", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "ordered-multimap" version = "0.4.3" @@ -2234,6 +2388,46 @@ version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" +[[package]] +name = "reqwest" +version = "0.11.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +dependencies = [ + "async-compression", + "base64 0.21.0", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "system-configuration", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "ring" version = "0.16.20" @@ -2405,6 +2599,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +dependencies = [ + "windows-sys 0.48.0", +] + [[package]] name = "scopeguard" version = "1.1.0" @@ -2427,6 +2630,29 @@ dependencies = [ "untrusted", ] +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.16" @@ -2862,6 +3088,7 @@ dependencies = [ "actix-web", "actix-web-codegen-const-routes", "argon2-creds", + "async-trait", "cache-buster", "config", "csv-async", @@ -2875,6 +3102,7 @@ dependencies = [ "mktemp", "pretty_env_logger", "rand", + "reqwest", "rust-embed", "serde", "serde_json", @@ -2910,6 +3138,27 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tempfile" version = "3.8.0" @@ -3055,6 +3304,16 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.12" @@ -3089,6 +3348,12 @@ dependencies = [ "serde", ] +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + [[package]] name = "tracing" version = "0.1.37" @@ -3122,6 +3387,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + [[package]] name = "typenum" version = "1.16.0" @@ -3399,6 +3670,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -3430,6 +3710,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.84" @@ -3647,6 +3939,16 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "yaml-rust" version = "0.4.5" diff --git a/Cargo.toml b/Cargo.toml index a99259a..db88d76 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ name = "tests-migrate" path = "./src/tests-migrate.rs" [dependencies] -actix-web = "4.0.1" +actix-web = "4.3" actix-identity = "0.4.0" actix-session = { version = "0.6.1", features = ["cookie-session"]} actix-http = "3.0.4" @@ -72,6 +72,8 @@ tracing = { version = "0.1.37", features = ["log"] } tera = { version="1.17.1", features=["builtins"]} tokio = { version = "1.25.0", features = ["fs"] } csv-async = { version = "1.2.5", features = ["serde", "tokio"] } +async-trait = "0.1.68" +reqwest = { version = "0.11.18", features = ["json", "gzip"] } #tokio = "1.11.0" diff --git a/migrations/20230701082846_survey_mcaptcha_upload.sql b/migrations/20230701082846_survey_mcaptcha_upload.sql new file mode 100644 index 0000000..9d9287d --- /dev/null +++ b/migrations/20230701082846_survey_mcaptcha_upload.sql @@ -0,0 +1,22 @@ +CREATE TABLE IF NOT EXISTS survey_mcaptcha_hostname ( + url VARCHAR(3000) UNIQUE NOT NULL, + secret VARCHAR(100) UNIQUE NOT NULL, + ID SERIAL PRIMARY KEY NOT NULL +); + +CREATE TABLE IF NOT EXISTS survey_mcaptcha_campaign ( + campaign_id VARCHAR(100) NOT NULL, + public_id VARCHAR(100) NOT NULL, + url_id INTEGER NOT NULL references survey_mcaptcha_hostname(ID) ON DELETE CASCADE, + synced_till INTEGER NOT NULL DEFAULT 0, + ID SERIAL PRIMARY KEY NOT NULL +); + + +CREATE TABLE IF NOT EXISTS survey_mcaptcha_analytics ( + campaign_id INTEGER references survey_mcaptcha_campaign(ID) ON DELETE CASCADE, + time INTEGER NOT NULL, + difficulty_factor INTEGER NOT NULL, + worker_type VARCHAR(100) NOT NULL, + ID SERIAL PRIMARY KEY NOT NULL +); diff --git a/sqlx-data.json b/sqlx-data.json index dc5f264..95c8c85 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -1,616 +1,3 @@ { - "db": "PostgreSQL", - "0d22134cc5076304b7895827f006ee8269cc500f400114a7472b83f0f1c568b5": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Varchar" - ] - } - }, - "query": "INSERT INTO survey_admins \n (name , password, secret) VALUES ($1, $2, $3)" - }, - "10924f3726a45c3bc709118375d691f2867bbcd50dc47a000ac9bf3ff878c97c": { - "describe": { - "columns": [ - { - "name": "name", - "ordinal": 0, - "type_info": "Varchar" - }, - { - "name": "id", - "ordinal": 1, - "type_info": "Uuid" - } - ], - "nullable": [ - false, - false - ], - "parameters": { - "Left": [] - } - }, - "query": "SELECT name, id FROM survey_campaigns ORDER BY id;" - }, - "117f1ae18f6a3936f27446b75b555951fe217d3a3cefe40a006fdd3cb31f0ac4": { - "describe": { - "columns": [ - { - "name": "id", - "ordinal": 0, - "type_info": "Int4" - } - ], - "nullable": [ - false - ], - "parameters": { - "Left": [ - "Uuid", - "Uuid", - "Varchar", - "Varchar", - "Int4", - "Timestamptz", - "Text" - ] - } - }, - "query": "INSERT INTO survey_responses (\n user_id,\n campaign_id,\n device_user_provided,\n device_software_recognised,\n threads,\n submitted_at,\n submission_bench_type_id\n ) VALUES (\n $1, $2, $3, $4, $5, $6,\n (SELECT ID FROM survey_bench_type WHERE name = $7)\n )\n RETURNING ID;" - }, - "1373df097fa0e58b23a374753318ae53a44559aa0e7eb64680185baf1c481723": { - "describe": { - "columns": [ - { - "name": "password", - "ordinal": 0, - "type_info": "Text" - } - ], - "nullable": [ - false - ], - "parameters": { - "Left": [ - "Text" - ] - } - }, - "query": "SELECT password FROM survey_admins WHERE name = ($1)" - }, - "15a8484de6f035e56c34ce3f6979eadea81f125933f76261c8b3c8319d43bbe0": { - "describe": { - "columns": [ - { - "name": "name", - "ordinal": 0, - "type_info": "Varchar" - } - ], - "nullable": [ - false - ], - "parameters": { - "Left": [ - "Uuid" - ] - } - }, - "query": "SELECT\n survey_admins.name\n FROM\n survey_admins\n INNER JOIN survey_campaigns ON\n survey_admins.ID = survey_campaigns.user_id\n WHERE\n survey_campaigns.ID = $1\n " - }, - "19686bfe8772cbc6831d46d18994e2b9aa40c7181eae9a31e51451cce95f04e8": { - "describe": { - "columns": [ - { - "name": "name", - "ordinal": 0, - "type_info": "Varchar" - }, - { - "name": "password", - "ordinal": 1, - "type_info": "Text" - } - ], - "nullable": [ - false, - false - ], - "parameters": { - "Left": [ - "Text" - ] - } - }, - "query": "SELECT name, password FROM survey_admins WHERE email = ($1)" - }, - "1972be28a6bda2c3a3764a836e95c8cb0c5db277fc4c8a9b19951a03166c6492": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Text", - "Uuid" - ] - } - }, - "query": "DELETE \n FROM survey_campaigns \n WHERE \n user_id = (\n SELECT \n ID \n FROM \n survey_admins \n WHERE \n name = $1\n )\n AND\n id = ($2)" - }, - "1b7e17bfc949fa97e8dec1f95e35a02bcf3aa1aa72a1f6f6c8884e885fc3b953": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Varchar", - "Varchar" - ] - } - }, - "query": "insert into survey_admins \n (name , password, email, secret) values ($1, $2, $3, $4)" - }, - "2ccaecfee4d2f29ef5278188b304017719720aa986d680d4727a1facbb869c7a": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Text" - ] - } - }, - "query": "DELETE FROM survey_admins WHERE name = ($1)" - }, - "43b3e771f38bf8059832169227705be06a28925af1b3799ffef5371d511fd138": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Timestamptz", - "Uuid" - ] - } - }, - "query": "\n INSERT INTO survey_users (created_at, id) VALUES($1, $2)" - }, - "536541ecf2e1c0403c74b6e2e09b42b73a7741ae4a348ff539ac410022e03ace": { - "describe": { - "columns": [ - { - "name": "exists", - "ordinal": 0, - "type_info": "Bool" - } - ], - "nullable": [ - null - ], - "parameters": { - "Left": [ - "Text" - ] - } - }, - "query": "SELECT EXISTS (SELECT 1 from survey_admins WHERE name = $1)" - }, - "55dde28998a6d12744806035f0a648494a403c7d09ea3caf91bf54869a81aa73": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - } - }, - "query": "UPDATE survey_admins set password = $1\n WHERE name = $2" - }, - "57c673ad8529371d77aa305917cf680dd2273ead74c3583ef0322f472b1d33fd": { - "describe": { - "columns": [ - { - "name": "id", - "ordinal": 0, - "type_info": "Int4" - }, - { - "name": "device_software_recognised", - "ordinal": 1, - "type_info": "Varchar" - }, - { - "name": "threads", - "ordinal": 2, - "type_info": "Int4" - }, - { - "name": "user_id", - "ordinal": 3, - "type_info": "Uuid" - }, - { - "name": "submitted_at", - "ordinal": 4, - "type_info": "Timestamptz" - }, - { - "name": "device_user_provided", - "ordinal": 5, - "type_info": "Varchar" - }, - { - "name": "name", - "ordinal": 6, - "type_info": "Varchar" - } - ], - "nullable": [ - false, - false, - true, - false, - false, - false, - false - ], - "parameters": { - "Left": [ - "Uuid", - "Text", - "Int8", - "Int8" - ] - } - }, - "query": "SELECT\n survey_responses.ID,\n survey_responses.device_software_recognised,\n survey_responses.threads,\n survey_responses.user_id,\n survey_responses.submitted_at,\n survey_responses.device_user_provided,\n survey_bench_type.name\n FROM\n survey_responses\n INNER JOIN survey_bench_type ON\n survey_responses.submission_bench_type_id = survey_bench_type.ID\n WHERE\n survey_responses.campaign_id = (\n SELECT ID FROM survey_campaigns\n WHERE\n ID = $1\n AND\n user_id = (SELECT ID FROM survey_admins WHERE name = $2)\n )\n LIMIT $3 OFFSET $4" - }, - "58ec3b8f98c27e13ec2732f8ee23f6eb9845ac5d9fd97b1e5c9f2eed4b1f5693": { - "describe": { - "columns": [ - { - "name": "name", - "ordinal": 0, - "type_info": "Varchar" - } - ], - "nullable": [ - false - ], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - } - }, - "query": "SELECT name \n FROM survey_campaigns\n WHERE \n id = $1\n AND\n user_id = (SELECT ID from survey_admins WHERE name = $2)" - }, - "683707dbc847b37c58c29aaad0d1a978c9fe0657da13af99796e4461134b5a43": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Varchar", - "Text" - ] - } - }, - "query": "UPDATE survey_admins set email = $1\n WHERE name = $2" - }, - "6a26daa84578aed2b2085697cb8358ed7c0a50ba9597fd387b4b09b0a8a154db": { - "describe": { - "columns": [ - { - "name": "exists", - "ordinal": 0, - "type_info": "Bool" - } - ], - "nullable": [ - null - ], - "parameters": { - "Left": [ - "Text" - ] - } - }, - "query": "SELECT EXISTS (SELECT 1 from survey_admins WHERE email = $1)" - }, - "70cc7bfc9b6ff5b68db70c069c0947d51bfc4a53cedc020016ee25ff98586c93": { - "describe": { - "columns": [ - { - "name": "name", - "ordinal": 0, - "type_info": "Varchar" - }, - { - "name": "id", - "ordinal": 1, - "type_info": "Uuid" - } - ], - "nullable": [ - false, - false - ], - "parameters": { - "Left": [ - "Text" - ] - } - }, - "query": "SELECT \n name, id\n FROM \n survey_campaigns \n WHERE\n user_id = (\n SELECT \n ID\n FROM \n survey_admins\n WHERE\n name = $1\n )" - }, - "74c41e33f91cf31ea13582c8b3ca464544374842450d580517ca2bd01d67402e": { - "describe": { - "columns": [ - { - "name": "id", - "ordinal": 0, - "type_info": "Int4" - }, - { - "name": "device_software_recognised", - "ordinal": 1, - "type_info": "Varchar" - }, - { - "name": "threads", - "ordinal": 2, - "type_info": "Int4" - }, - { - "name": "user_id", - "ordinal": 3, - "type_info": "Uuid" - }, - { - "name": "submitted_at", - "ordinal": 4, - "type_info": "Timestamptz" - }, - { - "name": "device_user_provided", - "ordinal": 5, - "type_info": "Varchar" - }, - { - "name": "name", - "ordinal": 6, - "type_info": "Varchar" - } - ], - "nullable": [ - false, - false, - true, - false, - false, - false, - false - ], - "parameters": { - "Left": [ - "Uuid", - "Text", - "Text", - "Int8", - "Int8" - ] - } - }, - "query": "SELECT\n survey_responses.ID,\n survey_responses.device_software_recognised,\n survey_responses.threads,\n survey_responses.user_id,\n survey_responses.submitted_at,\n survey_responses.device_user_provided,\n survey_bench_type.name\n FROM\n survey_responses\n INNER JOIN survey_bench_type ON\n survey_responses.submission_bench_type_id = survey_bench_type.ID\n WHERE\n survey_bench_type.name = $3\n AND\n survey_responses.campaign_id = (\n SELECT ID FROM survey_campaigns\n WHERE\n ID = $1\n AND\n user_id = (SELECT ID FROM survey_admins WHERE name = $2)\n )\n LIMIT $4 OFFSET $5" - }, - "82feafc36533144e49ba374c8c47ca4aa0d6558a9803778ad28cfa7b62382c3e": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Text", - "Uuid", - "Varchar", - "Int4Array", - "Timestamptz" - ] - } - }, - "query": "\n INSERT INTO survey_campaigns (\n user_id, ID, name, difficulties, created_at\n ) VALUES(\n (SELECT id FROM survey_admins WHERE name = $1),\n $2, $3, $4, $5\n );" - }, - "858a4c06a5c1ba7adb79bcac7d42d106d09d0cbff10c197f2242dcb5c437a1df": { - "describe": { - "columns": [ - { - "name": "created_at", - "ordinal": 0, - "type_info": "Timestamptz" - }, - { - "name": "id", - "ordinal": 1, - "type_info": "Uuid" - } - ], - "nullable": [ - false, - false - ], - "parameters": { - "Left": [ - "Uuid" - ] - } - }, - "query": "SELECT\n created_at,\n ID\n FROM\n survey_users\n WHERE\n ID = $1\n " - }, - "9cdade613ce724631cc3f187510758ee0929e93ff3f8ce81fe35594756644246": { - "describe": { - "columns": [ - { - "name": "difficulties", - "ordinal": 0, - "type_info": "Int4Array" - } - ], - "nullable": [ - false - ], - "parameters": { - "Left": [ - "Uuid" - ] - } - }, - "query": "SELECT difficulties FROM survey_campaigns WHERE id = $1;" - }, - "a721cfa249acf328c2f29c4cf8c2aeba1a635bcf49d18ced5474caa10b7cae4f": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Int4", - "Int4", - "Float4" - ] - } - }, - "query": "INSERT INTO survey_benches \n (resp_id, difficulty, duration) \n VALUES ($1, $2, $3);" - }, - "ab951c5c318174c6538037947c2f52c61bcfe5e5be1901379b715e77f5214dd2": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Varchar", - "Text" - ] - } - }, - "query": "UPDATE survey_admins set secret = $1\n WHERE name = $2" - }, - "b2619292aa6bd1ac38dca152cbe607b795a151ddc212361a3c6d8c70ea1c93eb": { - "describe": { - "columns": [ - { - "name": "duration", - "ordinal": 0, - "type_info": "Float4" - }, - { - "name": "difficulty", - "ordinal": 1, - "type_info": "Int4" - } - ], - "nullable": [ - false, - false - ], - "parameters": { - "Left": [ - "Int4" - ] - } - }, - "query": "SELECT\n duration,\n difficulty\n FROM\n survey_benches\n WHERE\n resp_id = $1\n " - }, - "c757589ef26a005e3285e7ab20d8a44c4f2e1cb125f8db061dd198cc380bf807": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Varchar", - "Text" - ] - } - }, - "query": "UPDATE survey_admins set name = $1\n WHERE name = $2" - }, - "e9cf5d6d8c9e8327d5c809d47a14a933f324e267f1e7dbb48e1caf1c021adc3f": { - "describe": { - "columns": [ - { - "name": "secret", - "ordinal": 0, - "type_info": "Varchar" - } - ], - "nullable": [ - false - ], - "parameters": { - "Left": [ - "Text" - ] - } - }, - "query": "SELECT secret FROM survey_admins WHERE name = ($1)" - }, - "efa0e41910fa5bcb187ba9e2fc8f37bee5b25ffe9a2d175f39a69899bc559965": { - "describe": { - "columns": [ - { - "name": "id", - "ordinal": 0, - "type_info": "Uuid" - }, - { - "name": "name", - "ordinal": 1, - "type_info": "Varchar" - }, - { - "name": "difficulties", - "ordinal": 2, - "type_info": "Int4Array" - }, - { - "name": "created_at", - "ordinal": 3, - "type_info": "Timestamptz" - } - ], - "nullable": [ - false, - false, - false, - false - ], - "parameters": { - "Left": [] - } - }, - "query": "SELECT ID, name, difficulties, created_at FROM survey_campaigns" - }, - "fcdc5fe5d496eb516c805e64ec96d9626b74ab33cd6e75e5a08ae88967403b72": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Int4", - "Uuid", - "Uuid" - ] - } - }, - "query": "INSERT INTO survey_response_tokens \n (resp_id, user_id, id)\n VALUES ($1, $2, $3);" - } + "db": "PostgreSQL" } \ No newline at end of file diff --git a/src/api/v1/mcaptcha/db.rs b/src/api/v1/mcaptcha/db.rs new file mode 100644 index 0000000..a6315bc --- /dev/null +++ b/src/api/v1/mcaptcha/db.rs @@ -0,0 +1,509 @@ +/* + * Copyright (C) 2023 Aravinth Manivannan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +use url::Url; + +use crate::api::v1::get_random; +use crate::errors::*; +use crate::mcaptcha::PerformanceAnalytics; +use crate::Data; + +impl Data { + /// Check if an mCaptcha instance is registered on the database + pub async fn mcaptcha_url_exists(&self, url: &str) -> ServiceResult { + let res = sqlx::query!( + "SELECT EXISTS (SELECT 1 from survey_mcaptcha_hostname WHERE url = $1)", + url + ) + .fetch_one(&self.db) + .await?; + + let mut resp = false; + if let Some(x) = res.exists { + if x { + resp = true; + } + } + + Ok(resp) + } + + /// Register an mCaptcha instance + pub async fn mcaptcha_register_instance(&self, url: &str) -> ServiceResult { + let secret = get_random(32); + sqlx::query!( + "INSERT INTO survey_mcaptcha_hostname (url, secret) VALUES ($1, $2)", + url, + &secret, + ) + .execute(&self.db) + .await?; + Ok(secret) + } + + /// Update the secret of an mCaptcha instance + pub async fn mcaptcha_update_secret(&self, url: &str) -> ServiceResult { + let secret = get_random(32); + sqlx::query!( + "UPDATE survey_mcaptcha_hostname set secret = $1 WHERE url = $2", + &secret, + url + ) + .execute(&self.db) + .await?; + Ok(secret) + } + + /// Authenticate an mCaptcha instance and return its URL + pub async fn mcaptcha_authenticate_and_get_url( + &self, + secret: &str, + ) -> ServiceResult { + struct U { + url: String, + } + + let res = sqlx::query!( + "SELECT EXISTS ( + SELECT + url + FROM + survey_mcaptcha_hostname + WHERE secret = $1 + )", + secret + ) + .fetch_one(&self.db) + .await?; + + if !match res.exists { + Some(true) => true, + _ => false, + } { + return Err(ServiceError::WrongPassword); + } + + let url = sqlx::query_as!( + U, + "SELECT + url + FROM + survey_mcaptcha_hostname + WHERE + secret = $1; ", + secret + ) + .fetch_one(&self.db) + .await?; + + Ok(Url::parse(&url.url).unwrap()) + } + + /// Delete mCaptcha instance from database + pub async fn mcaptcha_delete_mcaptcha_instance( + &self, + url: &str, + secret: &str, + ) -> ServiceResult<()> { + sqlx::query!( + "DELETE FROM survey_mcaptcha_hostname WHERE secret = $1 AND url =$2", + secret, + url + ) + .execute(&self.db) + .await?; + Ok(()) + } + /// Delete mCaptcha camapign from database + pub async fn mcaptcha_delete_mcaptcha_campaign( + &self, + campaign_id: &uuid::Uuid, + secret: &str, + ) -> ServiceResult<()> { + let campaign_str = campaign_id.to_string(); + let res = sqlx::query!( + "DELETE FROM + survey_mcaptcha_campaign + WHERE + campaign_id = $1 + AND + url_id = ( + SELECT + ID + FROM + survey_mcaptcha_hostname + WHERE + secret = $2 + )", + &campaign_str, + secret + ) + .execute(&self.db) + .await?; + + Ok(()) + } + + /// Check if an mCaptcha instance campaign is registered on DB + pub async fn mcaptcha_campaign_is_registered( + &self, + campaign_id: &uuid::Uuid, + secret: &str, + ) -> ServiceResult { + let campaign_str = campaign_id.to_string(); + + let res = sqlx::query!( + "SELECT EXISTS ( + SELECT + ID + FROM + survey_mcaptcha_campaign + WHERE + campaign_id = $1 + AND + url_id = ( + SELECT + ID + FROM + survey_mcaptcha_hostname + WHERE + secret = $2 + ) + )", + &campaign_str, + secret + ) + .fetch_one(&self.db) + .await?; + + let mut resp = false; + if let Some(x) = res.exists { + if x { + resp = true; + } + } + Ok(resp) + } + + /// Register an mCaptcha instance campaign on DB + pub async fn mcaptcha_register_campaign( + &self, + campaign_id: &uuid::Uuid, + secret: &str, + ) -> ServiceResult<()> { + let campaign_str = campaign_id.to_string(); + let public_id = uuid::Uuid::new_v4(); + + sqlx::query!( + "INSERT INTO + survey_mcaptcha_campaign (campaign_id, public_id, url_id) + VALUES ($1, $2, (SELECT ID FROM survey_mcaptcha_hostname WHERE secret = $3));", + &campaign_str, + &public_id.to_string(), + secret, + ) + .execute(&self.db) + .await?; + Ok(()) + } + + /// Register an mCaptcha instance campaign on DB + pub async fn mcaptcha_get_campaign_public_id( + &self, + campaign_id: &uuid::Uuid, + secret: &str, + ) -> ServiceResult { + let campaign_str = campaign_id.to_string(); + struct S { + public_id: String, + } + + let res = sqlx::query_as!( + S, + "SELECT + public_id + FROM + survey_mcaptcha_campaign + WHERE + campaign_id = $1 + AND + url_id = (SELECT ID FROM survey_mcaptcha_hostname WHERE secret = $2);", + &campaign_str, + secret, + ) + .fetch_one(&self.db) + .await?; + + Ok(uuid::Uuid::parse_str(&res.public_id).unwrap()) + } + + /// Get an mCaptcha instance campaign checkpoint + pub async fn mcaptcha_get_checkpoint( + &self, + campaign_id: &uuid::Uuid, + secret: &str, + ) -> ServiceResult { + let campaign_str = campaign_id.to_string(); + + struct CheckPoint { + synced_till: i32, + } + + let checkpoint = sqlx::query_as!( + CheckPoint, + "SELECT + synced_till + FROM + survey_mcaptcha_campaign + WHERE + campaign_id = $1 + AND + url_id = ( + SELECT ID FROM survey_mcaptcha_hostname WHERE secret = $2 + );", + &campaign_str, + secret + ) + .fetch_one(&self.db) + .await?; + let checkpoint = checkpoint.synced_till as usize; + Ok(checkpoint) + } + + /// Set an mCaptcha instance campaign checkpoint + pub async fn mcaptcha_set_checkpoint( + &self, + campaign_id: &uuid::Uuid, + secret: &str, + checkpoint: usize, + ) -> ServiceResult<()> { + let campaign_str = campaign_id.to_string(); + sqlx::query!( + "UPDATE + survey_mcaptcha_campaign + SET + synced_till = $1 + WHERE + campaign_id = $2 + AND + url_id = ( + SELECT ID FROM survey_mcaptcha_hostname WHERE secret = $3 + ) + ", + checkpoint as i32, + &campaign_str, + secret + ) + .execute(&self.db) + .await?; + + Ok(()) + } + + /// Store mCaptcha instance campaign analytics + pub async fn mcaptcha_insert_analytics( + &self, + campaign_id: &uuid::Uuid, + secret: &str, + r: &PerformanceAnalytics, + ) -> ServiceResult<()> { + let campaign_str = campaign_id.to_string(); + sqlx::query!( + "INSERT INTO + survey_mcaptcha_analytics ( + campaign_id, time, difficulty_factor, worker_type + ) + VALUES (( + SELECT + ID + FROM + survey_mcaptcha_campaign + WHERE + campaign_id = $1 + AND + url_id = ( + SELECT ID FROM survey_mcaptcha_hostname WHERE secret = $2 + ) + ), $3, $4, $5 + );", + &campaign_str, + secret, + r.time as i32, + r.difficulty_factor as i32, + &r.worker_type, + ) + .execute(&self.db) + .await?; + Ok(()) + } + + /// fetch PoW analytics + pub async fn mcaptcha_analytics_fetch( + &self, + public_id: &uuid::Uuid, + limit: usize, + offset: usize, + ) -> ServiceResult> { + let public_id_str = public_id.to_string(); + struct P { + id: i32, + time: i32, + difficulty_factor: i32, + worker_type: String, + } + + impl From

for PerformanceAnalytics { + fn from(v: P) -> Self { + Self { + time: v.time as u32, + difficulty_factor: v.difficulty_factor as u32, + worker_type: v.worker_type, + id: v.id as usize, + } + } + } + + let mut c = sqlx::query_as!( + P, + "SELECT id, time, difficulty_factor, worker_type FROM survey_mcaptcha_analytics + WHERE + campaign_id = ( + SELECT + ID FROM survey_mcaptcha_campaign + WHERE + public_id = $1 + ) + ORDER BY ID + OFFSET $2 LIMIT $3 + ", + &public_id_str, + offset as i32, + limit as i32 + ) + .fetch_all(&self.db) + .await?; + let mut res = Vec::with_capacity(c.len()); + for i in c.drain(0..) { + res.push(i.into()) + } + + Ok(res) + } +} + +#[cfg(test)] +mod tests { + use crate::{mcaptcha::PerformanceAnalytics, tests::*}; + + use url::Url; + + #[actix_rt::test] + async fn test_db_mcaptcha_works() { + let url = Url::parse("http://test_add_campaign.example").unwrap(); + let data = get_test_data().await; + let url_str = url.to_string(); + if data.mcaptcha_url_exists(&url_str).await.unwrap() { + let secret = data.mcaptcha_update_secret(&url_str).await.unwrap(); + data.mcaptcha_delete_mcaptcha_instance(&url_str, &secret) + .await + .unwrap(); + } + assert!(!data.mcaptcha_url_exists(&url_str).await.unwrap()); + + let secret = data.mcaptcha_register_instance(&url_str).await.unwrap(); + assert!(data.mcaptcha_url_exists(&url_str).await.unwrap()); + let secret2 = data.mcaptcha_update_secret(&url_str).await.unwrap(); + assert_ne!(secret2, secret); + let secret = secret2; + + assert_eq!( + data.mcaptcha_authenticate_and_get_url(&secret) + .await + .unwrap(), + url + ); + + let uuid = uuid::Uuid::new_v4(); + + if data + .mcaptcha_campaign_is_registered(&uuid, &secret) + .await + .unwrap() + { + data.mcaptcha_delete_mcaptcha_campaign(&uuid, &secret) + .await + .unwrap(); + } + + assert!(!data + .mcaptcha_campaign_is_registered(&uuid, &secret) + .await + .unwrap()); + data.mcaptcha_register_campaign(&uuid, &secret) + .await + .unwrap(); + assert!(data + .mcaptcha_campaign_is_registered(&uuid, &secret) + .await + .unwrap()); + + assert_eq!( + data.mcaptcha_get_checkpoint(&uuid, &secret).await.unwrap(), + 0 + ); + data.mcaptcha_set_checkpoint(&uuid, &secret, 1) + .await + .unwrap(); + assert_eq!( + data.mcaptcha_get_checkpoint(&uuid, &secret).await.unwrap(), + 1 + ); + + let analytics = PerformanceAnalytics { + id: 1, + time: 1, + difficulty_factor: 1, + worker_type: "foo".to_string(), + }; + data.mcaptcha_insert_analytics(&uuid, &secret, &analytics) + .await + .unwrap(); + + let public_id = data + .mcaptcha_get_campaign_public_id(&uuid, &secret) + .await + .unwrap(); + let db_analytics = data + .mcaptcha_analytics_fetch(&public_id, 50, 0) + .await + .unwrap(); + + assert_eq!(db_analytics.len(), 1); + assert_eq!(db_analytics[0].time, analytics.time); + assert_eq!( + db_analytics[0].difficulty_factor, + analytics.difficulty_factor + ); + assert_eq!(db_analytics[0].worker_type, analytics.worker_type); + + assert_eq!( + data.mcaptcha_analytics_fetch(&public_id, 50, 1) + .await + .unwrap(), + vec![] + ); + } +} From 0dc74c1c0591ad7515f2db9c478405b567fb9cc1 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Sun, 2 Jul 2023 00:18:24 +0530 Subject: [PATCH 02/14] feat: helper subroutines to spin up docker DB container in make --- Makefile | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 75e41d1..af67ab7 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,15 @@ +define deploy_dependencies + @-docker create --name ${db} \ + -e POSTGRES_PASSWORD=password \ + -p 5433:5432 \ + postgres + docker start ${db} +endef + +define run_migrations + cargo run --bin tests-migrate +endef + default: frontend ## Debug build cargo build @@ -19,6 +31,20 @@ dev-env: ## Download development dependencies cargo fetch yarn install + +env.db.recreate: ## Deploy dependencies + @-docker rm -f ${db} + $(call deploy_dependencies) + sleep 5 + $(call run_migrations) + +env.db: ## Deploy dependencies + $(call deploy_dependencies) + sleep 5 + $(call run_migrations) + + + doc: ## Prepare documentation cargo doc --no-deps --workspace --all-features @@ -43,7 +69,7 @@ lint: ## Lint codebase yarn lint migrate: ## Run database migrations - cargo run --bin tests-migrate + $(call run_migrations) release: frontend ## Release build cargo build --release From cdbf6788f009c9f6e391e1d93dc635e27b0b6a1a Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Sun, 2 Jul 2023 00:19:04 +0530 Subject: [PATCH 03/14] feat: HTTP client to talk to mCaptcha/mCaptcha --- src/data.rs | 9 +- src/main.rs | 6 +- src/mcaptcha.rs | 201 +++++++++++++++++++++++++++++++++++++++++ src/pages/auth/join.rs | 3 +- src/tests.rs | 16 +++- 5 files changed, 231 insertions(+), 4 deletions(-) create mode 100644 src/mcaptcha.rs diff --git a/src/data.rs b/src/data.rs index 7bb9231..0444525 100644 --- a/src/data.rs +++ b/src/data.rs @@ -22,6 +22,7 @@ use argon2_creds::{Config, ConfigBuilder, PasswordPolicy}; use sqlx::postgres::PgPoolOptions; use sqlx::PgPool; +use crate::mcaptcha::*; use crate::settings::Settings; /// App data @@ -30,6 +31,8 @@ pub struct Data { pub db: PgPool, pub creds: Config, pub settings: Settings, + + pub mcaptcha: Box, } impl Data { @@ -45,7 +48,10 @@ impl Data { #[cfg(not(tarpaulin_include))] /// create new instance of app data - pub async fn new(settings: Settings) -> Arc { + pub async fn new( + settings: Settings, + mcaptcha: Box, + ) -> Arc { let creds = Self::get_creds(); let c = creds.clone(); #[allow(unused_variables)] @@ -67,6 +73,7 @@ impl Data { db, creds, settings, + mcaptcha, }; Arc::new(data) diff --git a/src/main.rs b/src/main.rs index 47ce40c..3a6ee72 100644 --- a/src/main.rs +++ b/src/main.rs @@ -31,6 +31,7 @@ mod api; mod archive; mod data; mod errors; +mod mcaptcha; mod pages; mod settings; mod static_assets; @@ -86,7 +87,10 @@ async fn main() -> std::io::Result<()> { ); let settings = Settings::new().unwrap(); - let data = Data::new(settings.clone()).await; + let mcaptcha: Box = + Box::new(mcaptcha::MCaptchaClientReqwest::default()); + + let data = Data::new(settings.clone(), mcaptcha).await; sqlx::migrate!("./migrations/").run(&data.db).await.unwrap(); let data = actix_web::web::Data::new(data); diff --git a/src/mcaptcha.rs b/src/mcaptcha.rs new file mode 100644 index 0000000..d0a61e0 --- /dev/null +++ b/src/mcaptcha.rs @@ -0,0 +1,201 @@ +/* + * Copyright (C) 2023 Aravinth Manivannan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +use async_trait::async_trait; +use reqwest::Client; +use serde::{Deserialize, Serialize}; +use url::Url; + +use crate::errors::*; + +/* TODO: + * 1. Define traits to interact with mCaptcha + * 2. Implement trait with request 3. Implement mocking for testing + * 4. Load to crate::data::Data + */ + +#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq)] +/// Proof-of-Work CAPTCHA performance analytics +pub struct PerformanceAnalytics { + /// log ID + pub id: usize, + /// time taken to generate proof + pub time: u32, + /// difficulty factor for which the proof was generated + pub difficulty_factor: u32, + /// worker/client type: wasm, javascript, python, etc. + pub worker_type: String, +} + +#[async_trait] +pub trait MCaptchaClient: + std::marker::Send + std::marker::Sync + CloneMCaptchaClient +{ + async fn share_secret( + &self, + mut mcaptcha: Url, + secret: &Secret, + ) -> ServiceResult<()>; + async fn download_benchmarks( + &self, + mut mcaptcha: Url, + campaign_id: &str, + page: usize, + ) -> ServiceResult>; +} + +/// Trait to clone MCaptchaClient +pub trait CloneMCaptchaClient { + /// clone client + fn clone_client(&self) -> Box; +} + +impl CloneMCaptchaClient for T +where + T: MCaptchaClient + Clone + 'static, +{ + fn clone_client(&self) -> Box { + Box::new(self.clone()) + } +} + +impl Clone for Box { + fn clone(&self) -> Self { + (**self).clone_client() + } +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq)] +pub struct Secret { + pub secret: String, +} + +#[derive(Clone)] +pub struct MCaptchaClientReqwest { + client: Client, +} + +impl Default for MCaptchaClientReqwest { + fn default() -> Self { + Self { + client: Client::new(), + } + } +} + +#[async_trait] +impl MCaptchaClient for MCaptchaClientReqwest { + async fn share_secret( + &self, + mut mcaptcha: Url, + secret: &Secret, + ) -> ServiceResult<()> { + mcaptcha.set_path("/api/v1/survey/secret"); + self.client + .post(mcaptcha) + .json(secret) + .send() + .await + .unwrap(); + Ok(()) + } + async fn download_benchmarks( + &self, + mut mcaptcha: Url, + campaign_id: &str, + page: usize, + ) -> ServiceResult> { + mcaptcha.set_path(&format!("/api/v1/survey/{campaign_id}/get?page={page}")); + let res = self + .client + .get(mcaptcha) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + Ok(res) + } +} + +#[cfg(test)] +pub mod tests { + use super::*; + use std::collections::HashMap; + use std::sync::{Arc, RwLock}; + + use lazy_static::lazy_static; + + lazy_static! { + pub static ref BENCHMARK: Vec = vec![ + PerformanceAnalytics { + id: 1, + time: 2, + difficulty_factor: 3, + worker_type: "foo".to_string(), + }, + PerformanceAnalytics { + id: 4, + time: 5, + difficulty_factor: 6, + worker_type: "bar".to_string(), + }, + ]; + } + + #[derive(Clone)] + pub struct TestClient { + pub client: Arc>>, + } + + impl Default for TestClient { + fn default() -> Self { + Self { + client: Arc::new(RwLock::new(HashMap::default())), + } + } + } + + #[async_trait] + impl MCaptchaClient for TestClient { + async fn share_secret( + &self, + mut mcaptcha: Url, + secret: &Secret, + ) -> ServiceResult<()> { + mcaptcha.set_path("/api/v1/survey/secret"); + let mut x = self.client.write().unwrap(); + x.insert(mcaptcha.to_string(), secret.secret.to_owned()); + drop(x); + Ok(()) + } + async fn download_benchmarks( + &self, + mcaptcha: Url, + campaign_id: &str, + page: usize, + ) -> ServiceResult> { + println!( + "mcaptcha_url {}, campaign_id {}, page: {page}", + mcaptcha.to_string(), + campaign_id + ); + let res = BENCHMARK.clone(); + Ok(res) + } + } +} diff --git a/src/pages/auth/join.rs b/src/pages/auth/join.rs index 21baf06..ff58c8e 100644 --- a/src/pages/auth/join.rs +++ b/src/pages/auth/join.rs @@ -109,7 +109,8 @@ mod tests { #[actix_rt::test] async fn auth_join_form_works() { let settings = Settings::new().unwrap(); - let data = Data::new(settings).await; + // let data = Data::new(settings).await; + let data = get_test_data().await; const NAME: &str = "testuserformjoin"; const NAME2: &str = "testuserformjoin2"; const EMAIL: &str = "testuserformjoin@a.com"; diff --git a/src/tests.rs b/src/tests.rs index 7721e44..286b8d0 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -41,12 +41,26 @@ use crate::data::Data; use crate::errors::*; use crate::V1_API_ROUTES; +pub async fn get_test_data_with_mcaptcha_client( +) -> (Arc, crate::mcaptcha::tests::TestClient) { + let mut settings = Settings::new().unwrap(); + let tmp_dir = Temp::new_dir().unwrap(); + settings.publish.dir = tmp_dir.join("base_path").to_str().unwrap().into(); + settings.allow_registration = true; + let test_mcaptcha = crate::mcaptcha::tests::TestClient::default(); + ( + Data::new(settings, Box::new(test_mcaptcha.clone())).await, + test_mcaptcha, + ) +} + pub async fn get_test_data() -> Arc { let mut settings = Settings::new().unwrap(); let tmp_dir = Temp::new_dir().unwrap(); settings.publish.dir = tmp_dir.join("base_path").to_str().unwrap().into(); settings.allow_registration = true; - Data::new(settings).await + let test_mcaptcha = Box::new(crate::mcaptcha::tests::TestClient::default()); + Data::new(settings, test_mcaptcha).await } #[macro_export] From 786a9afe221e1d1086c35f8eacdd2bcd051367a0 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Sun, 2 Jul 2023 00:19:18 +0530 Subject: [PATCH 04/14] feat: routes to accept analytics from mCaptcha/mCaptcha --- src/api/v1/mcaptcha/hooks.rs | 272 +++++++++++++++++++++++++++++++++++ src/api/v1/mcaptcha/mod.rs | 57 ++++++++ src/api/v1/mod.rs | 2 + src/api/v1/routes.rs | 3 + 4 files changed, 334 insertions(+) create mode 100644 src/api/v1/mcaptcha/hooks.rs create mode 100644 src/api/v1/mcaptcha/mod.rs diff --git a/src/api/v1/mcaptcha/hooks.rs b/src/api/v1/mcaptcha/hooks.rs new file mode 100644 index 0000000..7d5f1cf --- /dev/null +++ b/src/api/v1/mcaptcha/hooks.rs @@ -0,0 +1,272 @@ +/* + * Copyright (C) 2023 Aravinth Manivannan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +use actix_web::web::ServiceConfig; +use actix_web::{web, HttpResponse, Responder}; +use serde::{Deserialize, Serialize}; +use url::Url; + +use crate::api::v1::ROUTES; +use crate::errors::*; +use crate::mcaptcha::Secret; +use crate::AppData; + +pub fn services(cfg: &mut ServiceConfig) { + cfg.service(register); + cfg.service(upload); + cfg.service(download); +} + +#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq)] +pub struct MCaptchaInstance { + pub url: Url, +} + +#[actix_web_codegen_const_routes::post(path = "ROUTES.mcaptcha.register")] +async fn register( + data: AppData, + payload: web::Json, +) -> ServiceResult { + /* Summary + * 1. Check if secret exists + * 2. If not, add hostname and create secret + * 3. Post to mCaptcha + */ + + let url_str = payload.url.to_string(); + let secret = if data.mcaptcha_url_exists(&url_str).await? { + data.mcaptcha_update_secret(&url_str).await? + } else { + data.mcaptcha_register_instance(&url_str).await? + }; + + let secret = Secret { secret }; + data.mcaptcha + .share_secret(payload.into_inner().url, &secret) + .await?; + + Ok(HttpResponse::Ok()) +} + +#[actix_web_codegen_const_routes::post(path = "ROUTES.mcaptcha.upload")] +async fn upload( + data: AppData, + campaign: web::Path, + payload: web::Json, +) -> ServiceResult { + /* TODO + * 1. Authenticate: Get URL from secret + * 2. Check if campaign exists + * 3. If not: create campaign + * 4. Get last known sync point + * 5. Download results + * 6. Update sync point + */ + let url = data + .mcaptcha_authenticate_and_get_url(&payload.secret) + .await?; + let campaign_str = campaign.to_string(); + + if !data + .mcaptcha_campaign_is_registered(&campaign, &payload.secret) + .await? + { + data.mcaptcha_register_campaign(&campaign, &payload.secret) + .await?; + } + + let checkpoint = data + .mcaptcha_get_checkpoint(&campaign, &payload.secret) + .await?; + const LIMIT: usize = 50; + let mut page = 1 + (checkpoint / LIMIT); + loop { + let mut res = data + .mcaptcha + .download_benchmarks(url.clone(), &campaign_str, page) + .await?; + let skip = checkpoint - ((page - 1) * LIMIT); + let new_records = res.len() - skip as usize; + let mut skip = skip as isize; + for r in res.drain(0..) { + if skip > 0 { + skip -= 1; + continue; + } + data.mcaptcha_insert_analytics(&campaign, &payload.secret, &r) + .await?; + } + data.mcaptcha_set_checkpoint(&campaign, &payload.secret, new_records) + .await?; + + page += 1; + if res.len() < LIMIT { + break; + } + } + + Ok(HttpResponse::Ok()) +} + +#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq)] +pub struct Page { + pub page: usize, +} + +#[actix_web_codegen_const_routes::get(path = "ROUTES.mcaptcha.download")] +async fn download( + data: AppData, + page: web::Query, + public_id: web::Path, +) -> ServiceResult { + const LIMIT: usize = 50; + let offset = LIMIT as isize * ((page.page as isize) - 1); + let offset = if offset < 0 { 0 } else { offset }; + let public_id = public_id.into_inner(); + let resp = data + .mcaptcha_analytics_fetch(&public_id, LIMIT, offset as usize) + .await?; + Ok(HttpResponse::Ok().json(resp)) +} + +#[cfg(test)] +mod tests { + use crate::api::v1::bench::Submission; + use crate::api::v1::bench::SubmissionType; + use crate::errors::*; + use crate::mcaptcha::PerformanceAnalytics; + use crate::mcaptcha::Secret; + use crate::tests::*; + use crate::*; + + use actix_web::{http::header, test}; + + #[actix_rt::test] + async fn mcaptcha_hooks_work() { + let mcaptcha_instance = + url::Url::parse("http://mcaptcha_hooks_work.example.org").unwrap(); + let mcaptcha_instance_str = mcaptcha_instance.to_string(); + let campaign_id = uuid::Uuid::new_v4(); + + let (data, client) = get_test_data_with_mcaptcha_client().await; + let app = get_app!(data).await; + + if data + .mcaptcha_url_exists(&mcaptcha_instance_str) + .await + .unwrap() + { + let secret = data + .mcaptcha_update_secret(&mcaptcha_instance_str) + .await + .unwrap(); + data.mcaptcha_delete_mcaptcha_instance(&mcaptcha_instance_str, &secret) + .await + .unwrap(); + } + + let payload = super::MCaptchaInstance { + url: mcaptcha_instance.clone(), + }; + + let resp = test::call_service( + &app, + post_request!(&payload, V1_API_ROUTES.mcaptcha.register).to_request(), + ) + .await; + assert_eq!(resp.status(), StatusCode::OK); + + let secret = { + let mut mcaptcha = payload.url.clone(); + mcaptcha.set_path("/api/v1/survey/secret"); + let mut x = client.client.write().unwrap(); + x.remove(&mcaptcha.to_string()).unwrap() + }; + + let resp2 = test::call_service( + &app, + post_request!(&payload, V1_API_ROUTES.mcaptcha.register).to_request(), + ) + .await; + assert_eq!(resp2.status(), StatusCode::OK); + + let secret2 = { + let mut mcaptcha = payload.url.clone(); + mcaptcha.set_path("/api/v1/survey/secret"); + let mut x = client.client.write().unwrap(); + x.remove(&mcaptcha.to_string()).unwrap() + }; + + assert_ne!(secret, secret2); + let secret = secret2; + + let payload = Secret { + secret: secret.clone(), + }; + + if data + .mcaptcha_campaign_is_registered(&campaign_id, &secret) + .await + .unwrap() + { + data.mcaptcha_delete_mcaptcha_campaign(&campaign_id, &secret) + .await + .unwrap(); + } + + let resp = test::call_service( + &app, + post_request!( + &payload, + &V1_API_ROUTES + .mcaptcha + .get_upload_route(&campaign_id.to_string()) + ) + .to_request(), + ) + .await; + assert_eq!(resp.status(), StatusCode::OK); + + let public_id = data + .mcaptcha_get_campaign_public_id(&campaign_id, &secret) + .await + .unwrap(); + + let expected = crate::mcaptcha::tests::BENCHMARK.clone(); + + let got = data + .mcaptcha_analytics_fetch(&public_id, 50, 0) + .await + .unwrap(); + + for i in 0..2 { + assert_eq!(got[i].time, expected[i].time); + assert_eq!(got[i].difficulty_factor, expected[i].difficulty_factor); + assert_eq!(got[i].worker_type, expected[i].worker_type); + } + + let resp = get_request!( + &app, + &V1_API_ROUTES + .mcaptcha + .get_download_route(&public_id.to_string(), 0) + ); + assert_eq!(resp.status(), StatusCode::OK); + let resp: Vec = test::read_body_json(resp).await; + assert_eq!(resp.len(), 2); + assert_eq!(resp, got); + } +} diff --git a/src/api/v1/mcaptcha/mod.rs b/src/api/v1/mcaptcha/mod.rs new file mode 100644 index 0000000..87e79ee --- /dev/null +++ b/src/api/v1/mcaptcha/mod.rs @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2023 Aravinth Manivannan + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +use actix_web::web::ServiceConfig; + +pub mod db; +pub mod hooks; + +pub fn services(cfg: &mut ServiceConfig) { + hooks::services(cfg); +} + +pub mod routes { + use serde::{Deserialize, Serialize}; + + #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] + pub struct Mcaptcha { + pub upload: &'static str, + pub download: &'static str, + pub register: &'static str, + } + + impl Mcaptcha { + pub const fn new() -> Self { + Self { + register: "/mcaptcha/api/v1/register", + upload: "/mcaptcha/api/v1/{campaign_id}/upload", + download: "/mcapthca/api/v1/{campaign_id}/download", + } + } + + pub fn get_download_route(&self, campaign_id: &str, page: usize) -> String { + format!( + "{}?page={}", + self.download.replace("{campaign_id}", campaign_id), + page + ) + } + + pub fn get_upload_route(&self, campaign_id: &str) -> String { + self.upload.replace("{campaign_id}", campaign_id) + } + } +} diff --git a/src/api/v1/mod.rs b/src/api/v1/mod.rs index 4288ac8..39554fb 100644 --- a/src/api/v1/mod.rs +++ b/src/api/v1/mod.rs @@ -20,6 +20,7 @@ use sqlx::types::Uuid; pub mod admin; pub mod bench; +pub mod mcaptcha; mod meta; pub mod routes; pub use routes::ROUTES; @@ -28,6 +29,7 @@ pub fn services(cfg: &mut ServiceConfig) { meta::services(cfg); bench::services(cfg); admin::services(cfg); + mcaptcha::services(cfg); } pub fn get_random(len: usize) -> String { diff --git a/src/api/v1/routes.rs b/src/api/v1/routes.rs index 05a0037..eaf7425 100644 --- a/src/api/v1/routes.rs +++ b/src/api/v1/routes.rs @@ -18,6 +18,7 @@ use serde::Serialize; use super::admin::routes::Admin; use super::bench::routes::Benches; +use super::mcaptcha::routes::Mcaptcha; use super::meta::routes::Meta; pub const ROUTES: Routes = Routes::new(); @@ -27,6 +28,7 @@ pub struct Routes { pub admin: Admin, pub meta: Meta, pub benches: Benches, + pub mcaptcha: Mcaptcha, } impl Routes { @@ -35,6 +37,7 @@ impl Routes { admin: Admin::new(), meta: Meta::new(), benches: Benches::new(), + mcaptcha: Mcaptcha::new(), } } } From 3c445411e9c4adc3c3f5c4c86d655e27513d544f Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Thu, 19 Oct 2023 09:55:00 +0530 Subject: [PATCH 05/14] fix: update parameters to mCaptcha/mCaptcha --- ...9b860a74b07b15256f3764434901471ff820b.json | 23 ++++++++++ ...3e7aed1f25290a5f3e8b19c5aa781da362ae3.json | 16 +++++++ ...bd30a5c75215dbc3179fae08a106388341c3f.json | 16 +++++++ ...5d06906e56018798cba2b4672c393327131aa.json | 22 ++++++++++ ...efc5062a25eb0ada991f04197189aa6dcb2c4.json | 22 ++++++++++ ...ce9546106e8bdd788c27ee579b278e671bcb0.json | 42 +++++++++++++++++++ ...e4bd83fdf431e3510f9afb0ef5e9b10f35181.json | 22 ++++++++++ ...cc64119dc08e5565925bb2a3f5fccf663b5ab.json | 15 +++++++ ...4f0d8eed367916b28c240551b97aebcc9eb36.json | 18 ++++++++ ...50b5055de47a7e89f99ffc589b99d22b8ac9d.json | 23 ++++++++++ ...18c7fe6d7824bbfcbae8248905e927049528b.json | 15 +++++++ ...1ecbf294d4bfddbce04209269644f4a7511ed.json | 15 +++++++ ...b4e6307547120b10762140d250f163b584a23.json | 15 +++++++ ...95b43076b0dd18fe5137d7ba87bc8e56102fe.json | 23 ++++++++++ src/api/v1/mcaptcha/hooks.rs | 5 ++- src/mcaptcha.rs | 21 ++++++++-- 16 files changed, 307 insertions(+), 6 deletions(-) create mode 100644 .sqlx/query-2904486838bed381aa00f6a1b1e9b860a74b07b15256f3764434901471ff820b.json create mode 100644 .sqlx/query-38a517b011519ec80d35d12ea463e7aed1f25290a5f3e8b19c5aa781da362ae3.json create mode 100644 .sqlx/query-3eb4799231e726cf566f81d6ec7bd30a5c75215dbc3179fae08a106388341c3f.json create mode 100644 .sqlx/query-5c1ad3208ece06ba7a503d650e15d06906e56018798cba2b4672c393327131aa.json create mode 100644 .sqlx/query-677376147c1346f9136c5d60914efc5062a25eb0ada991f04197189aa6dcb2c4.json create mode 100644 .sqlx/query-714925a5209400a17bcafe23c34ce9546106e8bdd788c27ee579b278e671bcb0.json create mode 100644 .sqlx/query-7d764a7b1c2991dda7498f243c6e4bd83fdf431e3510f9afb0ef5e9b10f35181.json create mode 100644 .sqlx/query-94205e3e65a8f6bf315a282ec8fcc64119dc08e5565925bb2a3f5fccf663b5ab.json create mode 100644 .sqlx/query-997a3d0ce396d48a1804b6272124f0d8eed367916b28c240551b97aebcc9eb36.json create mode 100644 .sqlx/query-9da39f618b9dea08360d4c1625650b5055de47a7e89f99ffc589b99d22b8ac9d.json create mode 100644 .sqlx/query-a3cddc0ace32cfb7df70e171b2618c7fe6d7824bbfcbae8248905e927049528b.json create mode 100644 .sqlx/query-baabef729999fe63426b3b2373f1ecbf294d4bfddbce04209269644f4a7511ed.json create mode 100644 .sqlx/query-dbe5d5c450a50bb829a39e6149eb4e6307547120b10762140d250f163b584a23.json create mode 100644 .sqlx/query-f5c0480ed2829c3a3dbe7a8da2d95b43076b0dd18fe5137d7ba87bc8e56102fe.json diff --git a/.sqlx/query-2904486838bed381aa00f6a1b1e9b860a74b07b15256f3764434901471ff820b.json b/.sqlx/query-2904486838bed381aa00f6a1b1e9b860a74b07b15256f3764434901471ff820b.json new file mode 100644 index 0000000..2e92fb6 --- /dev/null +++ b/.sqlx/query-2904486838bed381aa00f6a1b1e9b860a74b07b15256f3764434901471ff820b.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n public_id\n FROM\n survey_mcaptcha_campaign\n WHERE\n campaign_id = $1\n AND\n url_id = (SELECT ID FROM survey_mcaptcha_hostname WHERE secret = $2);", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "public_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "2904486838bed381aa00f6a1b1e9b860a74b07b15256f3764434901471ff820b" +} diff --git a/.sqlx/query-38a517b011519ec80d35d12ea463e7aed1f25290a5f3e8b19c5aa781da362ae3.json b/.sqlx/query-38a517b011519ec80d35d12ea463e7aed1f25290a5f3e8b19c5aa781da362ae3.json new file mode 100644 index 0000000..070f419 --- /dev/null +++ b/.sqlx/query-38a517b011519ec80d35d12ea463e7aed1f25290a5f3e8b19c5aa781da362ae3.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO\n survey_mcaptcha_campaign (campaign_id, public_id, url_id)\n VALUES ($1, $2, (SELECT ID FROM survey_mcaptcha_hostname WHERE secret = $3));", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "38a517b011519ec80d35d12ea463e7aed1f25290a5f3e8b19c5aa781da362ae3" +} diff --git a/.sqlx/query-3eb4799231e726cf566f81d6ec7bd30a5c75215dbc3179fae08a106388341c3f.json b/.sqlx/query-3eb4799231e726cf566f81d6ec7bd30a5c75215dbc3179fae08a106388341c3f.json new file mode 100644 index 0000000..c3fa03f --- /dev/null +++ b/.sqlx/query-3eb4799231e726cf566f81d6ec7bd30a5c75215dbc3179fae08a106388341c3f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE\n survey_mcaptcha_campaign\n SET\n synced_till = $1\n WHERE \n campaign_id = $2\n AND\n url_id = (\n SELECT ID FROM survey_mcaptcha_hostname WHERE secret = $3\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "3eb4799231e726cf566f81d6ec7bd30a5c75215dbc3179fae08a106388341c3f" +} diff --git a/.sqlx/query-5c1ad3208ece06ba7a503d650e15d06906e56018798cba2b4672c393327131aa.json b/.sqlx/query-5c1ad3208ece06ba7a503d650e15d06906e56018798cba2b4672c393327131aa.json new file mode 100644 index 0000000..145a0e0 --- /dev/null +++ b/.sqlx/query-5c1ad3208ece06ba7a503d650e15d06906e56018798cba2b4672c393327131aa.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (SELECT 1 from survey_mcaptcha_hostname WHERE url = $1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "5c1ad3208ece06ba7a503d650e15d06906e56018798cba2b4672c393327131aa" +} diff --git a/.sqlx/query-677376147c1346f9136c5d60914efc5062a25eb0ada991f04197189aa6dcb2c4.json b/.sqlx/query-677376147c1346f9136c5d60914efc5062a25eb0ada991f04197189aa6dcb2c4.json new file mode 100644 index 0000000..1058331 --- /dev/null +++ b/.sqlx/query-677376147c1346f9136c5d60914efc5062a25eb0ada991f04197189aa6dcb2c4.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n url\n FROM\n survey_mcaptcha_hostname\n WHERE\n secret = $1; ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "url", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "677376147c1346f9136c5d60914efc5062a25eb0ada991f04197189aa6dcb2c4" +} diff --git a/.sqlx/query-714925a5209400a17bcafe23c34ce9546106e8bdd788c27ee579b278e671bcb0.json b/.sqlx/query-714925a5209400a17bcafe23c34ce9546106e8bdd788c27ee579b278e671bcb0.json new file mode 100644 index 0000000..a2a8a25 --- /dev/null +++ b/.sqlx/query-714925a5209400a17bcafe23c34ce9546106e8bdd788c27ee579b278e671bcb0.json @@ -0,0 +1,42 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, time, difficulty_factor, worker_type FROM survey_mcaptcha_analytics\n WHERE \n campaign_id = (\n SELECT \n ID FROM survey_mcaptcha_campaign \n WHERE \n public_id = $1\n )\n ORDER BY ID\n OFFSET $2 LIMIT $3\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "time", + "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "difficulty_factor", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "worker_type", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "714925a5209400a17bcafe23c34ce9546106e8bdd788c27ee579b278e671bcb0" +} diff --git a/.sqlx/query-7d764a7b1c2991dda7498f243c6e4bd83fdf431e3510f9afb0ef5e9b10f35181.json b/.sqlx/query-7d764a7b1c2991dda7498f243c6e4bd83fdf431e3510f9afb0ef5e9b10f35181.json new file mode 100644 index 0000000..8e1cf4f --- /dev/null +++ b/.sqlx/query-7d764a7b1c2991dda7498f243c6e4bd83fdf431e3510f9afb0ef5e9b10f35181.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (\n SELECT\n url\n FROM\n survey_mcaptcha_hostname\n WHERE secret = $1\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "7d764a7b1c2991dda7498f243c6e4bd83fdf431e3510f9afb0ef5e9b10f35181" +} diff --git a/.sqlx/query-94205e3e65a8f6bf315a282ec8fcc64119dc08e5565925bb2a3f5fccf663b5ab.json b/.sqlx/query-94205e3e65a8f6bf315a282ec8fcc64119dc08e5565925bb2a3f5fccf663b5ab.json new file mode 100644 index 0000000..ec6639b --- /dev/null +++ b/.sqlx/query-94205e3e65a8f6bf315a282ec8fcc64119dc08e5565925bb2a3f5fccf663b5ab.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO survey_mcaptcha_hostname (url, secret) VALUES ($1, $2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "94205e3e65a8f6bf315a282ec8fcc64119dc08e5565925bb2a3f5fccf663b5ab" +} diff --git a/.sqlx/query-997a3d0ce396d48a1804b6272124f0d8eed367916b28c240551b97aebcc9eb36.json b/.sqlx/query-997a3d0ce396d48a1804b6272124f0d8eed367916b28c240551b97aebcc9eb36.json new file mode 100644 index 0000000..c3320c6 --- /dev/null +++ b/.sqlx/query-997a3d0ce396d48a1804b6272124f0d8eed367916b28c240551b97aebcc9eb36.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO\n survey_mcaptcha_analytics (\n campaign_id, time, difficulty_factor, worker_type\n )\n VALUES ((\n SELECT\n ID\n FROM\n survey_mcaptcha_campaign\n WHERE \n campaign_id = $1\n AND\n url_id = (\n SELECT ID FROM survey_mcaptcha_hostname WHERE secret = $2\n )\n ), $3, $4, $5\n );", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int4", + "Int4", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "997a3d0ce396d48a1804b6272124f0d8eed367916b28c240551b97aebcc9eb36" +} diff --git a/.sqlx/query-9da39f618b9dea08360d4c1625650b5055de47a7e89f99ffc589b99d22b8ac9d.json b/.sqlx/query-9da39f618b9dea08360d4c1625650b5055de47a7e89f99ffc589b99d22b8ac9d.json new file mode 100644 index 0000000..b418066 --- /dev/null +++ b/.sqlx/query-9da39f618b9dea08360d4c1625650b5055de47a7e89f99ffc589b99d22b8ac9d.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (\n SELECT\n ID\n FROM\n survey_mcaptcha_campaign\n WHERE\n campaign_id = $1\n AND\n url_id = (\n SELECT\n ID\n FROM\n survey_mcaptcha_hostname\n WHERE\n secret = $2\n )\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "9da39f618b9dea08360d4c1625650b5055de47a7e89f99ffc589b99d22b8ac9d" +} diff --git a/.sqlx/query-a3cddc0ace32cfb7df70e171b2618c7fe6d7824bbfcbae8248905e927049528b.json b/.sqlx/query-a3cddc0ace32cfb7df70e171b2618c7fe6d7824bbfcbae8248905e927049528b.json new file mode 100644 index 0000000..29bf5f3 --- /dev/null +++ b/.sqlx/query-a3cddc0ace32cfb7df70e171b2618c7fe6d7824bbfcbae8248905e927049528b.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM survey_mcaptcha_hostname WHERE secret = $1 AND url =$2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "a3cddc0ace32cfb7df70e171b2618c7fe6d7824bbfcbae8248905e927049528b" +} diff --git a/.sqlx/query-baabef729999fe63426b3b2373f1ecbf294d4bfddbce04209269644f4a7511ed.json b/.sqlx/query-baabef729999fe63426b3b2373f1ecbf294d4bfddbce04209269644f4a7511ed.json new file mode 100644 index 0000000..2a6eabb --- /dev/null +++ b/.sqlx/query-baabef729999fe63426b3b2373f1ecbf294d4bfddbce04209269644f4a7511ed.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE survey_mcaptcha_hostname set secret = $1 WHERE url = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "baabef729999fe63426b3b2373f1ecbf294d4bfddbce04209269644f4a7511ed" +} diff --git a/.sqlx/query-dbe5d5c450a50bb829a39e6149eb4e6307547120b10762140d250f163b584a23.json b/.sqlx/query-dbe5d5c450a50bb829a39e6149eb4e6307547120b10762140d250f163b584a23.json new file mode 100644 index 0000000..a793b87 --- /dev/null +++ b/.sqlx/query-dbe5d5c450a50bb829a39e6149eb4e6307547120b10762140d250f163b584a23.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM\n survey_mcaptcha_campaign\n WHERE\n campaign_id = $1\n AND\n url_id = (\n SELECT\n ID\n FROM\n survey_mcaptcha_hostname\n WHERE\n secret = $2\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "dbe5d5c450a50bb829a39e6149eb4e6307547120b10762140d250f163b584a23" +} diff --git a/.sqlx/query-f5c0480ed2829c3a3dbe7a8da2d95b43076b0dd18fe5137d7ba87bc8e56102fe.json b/.sqlx/query-f5c0480ed2829c3a3dbe7a8da2d95b43076b0dd18fe5137d7ba87bc8e56102fe.json new file mode 100644 index 0000000..dc44005 --- /dev/null +++ b/.sqlx/query-f5c0480ed2829c3a3dbe7a8da2d95b43076b0dd18fe5137d7ba87bc8e56102fe.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n synced_till\n FROM\n survey_mcaptcha_campaign\n WHERE \n campaign_id = $1\n AND\n url_id = (\n SELECT ID FROM survey_mcaptcha_hostname WHERE secret = $2\n );", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "synced_till", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "f5c0480ed2829c3a3dbe7a8da2d95b43076b0dd18fe5137d7ba87bc8e56102fe" +} diff --git a/src/api/v1/mcaptcha/hooks.rs b/src/api/v1/mcaptcha/hooks.rs index 7d5f1cf..c8985eb 100644 --- a/src/api/v1/mcaptcha/hooks.rs +++ b/src/api/v1/mcaptcha/hooks.rs @@ -33,6 +33,7 @@ pub fn services(cfg: &mut ServiceConfig) { #[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq)] pub struct MCaptchaInstance { pub url: Url, + pub auth_token: String, } #[actix_web_codegen_const_routes::post(path = "ROUTES.mcaptcha.register")] @@ -53,9 +54,9 @@ async fn register( data.mcaptcha_register_instance(&url_str).await? }; - let secret = Secret { secret }; + let payload = payload.into_inner(); data.mcaptcha - .share_secret(payload.into_inner().url, &secret) + .share_secret(payload.url, secret, payload.auth_token) .await?; Ok(HttpResponse::Ok()) diff --git a/src/mcaptcha.rs b/src/mcaptcha.rs index d0a61e0..0135bb5 100644 --- a/src/mcaptcha.rs +++ b/src/mcaptcha.rs @@ -47,7 +47,8 @@ pub trait MCaptchaClient: async fn share_secret( &self, mut mcaptcha: Url, - secret: &Secret, + secret: String, + auth_token: String, ) -> ServiceResult<()>; async fn download_benchmarks( &self, @@ -101,12 +102,23 @@ impl MCaptchaClient for MCaptchaClientReqwest { async fn share_secret( &self, mut mcaptcha: Url, - secret: &Secret, + secret: String, + auth_token: String, ) -> ServiceResult<()> { + #[derive(Serialize)] + struct S { + secret: String, + auth_token: String, + } + + let msg = S { + secret, + auth_token, + }; mcaptcha.set_path("/api/v1/survey/secret"); self.client .post(mcaptcha) - .json(secret) + .json(&msg) .send() .await .unwrap(); @@ -175,7 +187,8 @@ pub mod tests { async fn share_secret( &self, mut mcaptcha: Url, - secret: &Secret, + secret: String, + auth_token: String, ) -> ServiceResult<()> { mcaptcha.set_path("/api/v1/survey/secret"); let mut x = self.client.write().unwrap(); From b5b83b955ae18c5ab3b19fc13e1e300fa526e121 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Thu, 19 Oct 2023 10:10:33 +0530 Subject: [PATCH 06/14] fix: make archive shutdown responsive --- src/api/v1/mcaptcha/hooks.rs | 2 ++ src/archive.rs | 31 +++++++++++++++---------------- src/mcaptcha.rs | 14 +++----------- 3 files changed, 20 insertions(+), 27 deletions(-) diff --git a/src/api/v1/mcaptcha/hooks.rs b/src/api/v1/mcaptcha/hooks.rs index c8985eb..5474bce 100644 --- a/src/api/v1/mcaptcha/hooks.rs +++ b/src/api/v1/mcaptcha/hooks.rs @@ -147,6 +147,7 @@ async fn download( mod tests { use crate::api::v1::bench::Submission; use crate::api::v1::bench::SubmissionType; + use crate::api::v1::get_random; use crate::errors::*; use crate::mcaptcha::PerformanceAnalytics; use crate::mcaptcha::Secret; @@ -181,6 +182,7 @@ mod tests { let payload = super::MCaptchaInstance { url: mcaptcha_instance.clone(), + auth_token: get_random(23), }; let resp = test::call_service( diff --git a/src/archive.rs b/src/archive.rs index ad4b190..33c6103 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -17,7 +17,6 @@ use std::future::Future; use std::path::{Path, PathBuf}; - use serde::{Deserialize, Serialize}; use sqlx::types::time::OffsetDateTime; use sqlx::types::Uuid; @@ -218,27 +217,27 @@ impl Archiver { ) -> ServiceResult<(Sender, JoinHandle<()>)> { let (tx, mut rx) = oneshot::channel(); + fn can_run(rx: &mut oneshot::Receiver) -> bool { + match rx.try_recv() { + Err(TryRecvError::Empty) => true, + _ => false, + } + } + let job = async move { loop { - // let rx = self.rx.as_mut().unwrap(); - match rx.try_recv() { - // The channel is currently empty - Ok(_) => { + if !can_run(&mut rx) { + log::info!("Killing archive loop: received signal"); + break; + } + + for _ in 0..data.settings.publish.duration { + if !can_run(&mut rx) { log::info!("Killing archive loop: received signal"); break; } - Err(TryRecvError::Empty) => { - let _ = self.archive(&data).await; - - tokio::time::sleep(std::time::Duration::new( - data.settings.publish.duration, - 0, - )) - .await; - } - Err(TryRecvError::Closed) => break, + tokio::time::sleep(std::time::Duration::new(1, 0)).await; } - let _ = self.archive(&data).await; } }; diff --git a/src/mcaptcha.rs b/src/mcaptcha.rs index 0135bb5..c57e695 100644 --- a/src/mcaptcha.rs +++ b/src/mcaptcha.rs @@ -111,17 +111,9 @@ impl MCaptchaClient for MCaptchaClientReqwest { auth_token: String, } - let msg = S { - secret, - auth_token, - }; + let msg = S { secret, auth_token }; mcaptcha.set_path("/api/v1/survey/secret"); - self.client - .post(mcaptcha) - .json(&msg) - .send() - .await - .unwrap(); + self.client.post(mcaptcha).json(&msg).send().await.unwrap(); Ok(()) } async fn download_benchmarks( @@ -192,7 +184,7 @@ pub mod tests { ) -> ServiceResult<()> { mcaptcha.set_path("/api/v1/survey/secret"); let mut x = self.client.write().unwrap(); - x.insert(mcaptcha.to_string(), secret.secret.to_owned()); + x.insert(mcaptcha.to_string(), secret); drop(x); Ok(()) } From 6b935240277cbb48052b86eb16b90c0fa2747afd Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Thu, 19 Oct 2023 11:21:14 +0530 Subject: [PATCH 07/14] feat: schedule and record job states. Create job states during migration --- ...fe4e90c2ba897bb4889426031ffacc2ae06e4.json | 14 +++ ...0cdfca35a710f34a54ac115d23435762a3038.json | 22 +++++ ...80ce5a4a8dff6d0795f277fb18b7b40dc69ef.json | 22 +++++ .../20230701082846_survey_mcaptcha_upload.sql | 16 ++++ src/db.rs | 87 +++++++++++++++++++ src/main.rs | 3 +- src/tests-migrate.rs | 5 +- 7 files changed, 167 insertions(+), 2 deletions(-) create mode 100644 .sqlx/query-11ff04344412d1a2e5fdb1ab654fe4e90c2ba897bb4889426031ffacc2ae06e4.json create mode 100644 .sqlx/query-2d18e0fad79c6df26465f82eca20cdfca35a710f34a54ac115d23435762a3038.json create mode 100644 .sqlx/query-4237df28c12e17ca68a1e1b33ae80ce5a4a8dff6d0795f277fb18b7b40dc69ef.json create mode 100644 src/db.rs diff --git a/.sqlx/query-11ff04344412d1a2e5fdb1ab654fe4e90c2ba897bb4889426031ffacc2ae06e4.json b/.sqlx/query-11ff04344412d1a2e5fdb1ab654fe4e90c2ba897bb4889426031ffacc2ae06e4.json new file mode 100644 index 0000000..07173f2 --- /dev/null +++ b/.sqlx/query-11ff04344412d1a2e5fdb1ab654fe4e90c2ba897bb4889426031ffacc2ae06e4.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO survey_mcaptcha_upload_job_states \n (name) VALUES ($1) ON CONFLICT (name) DO NOTHING;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "11ff04344412d1a2e5fdb1ab654fe4e90c2ba897bb4889426031ffacc2ae06e4" +} diff --git a/.sqlx/query-2d18e0fad79c6df26465f82eca20cdfca35a710f34a54ac115d23435762a3038.json b/.sqlx/query-2d18e0fad79c6df26465f82eca20cdfca35a710f34a54ac115d23435762a3038.json new file mode 100644 index 0000000..9bad777 --- /dev/null +++ b/.sqlx/query-2d18e0fad79c6df26465f82eca20cdfca35a710f34a54ac115d23435762a3038.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (SELECT 1 from survey_mcaptcha_upload_job_states WHERE name = $1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "2d18e0fad79c6df26465f82eca20cdfca35a710f34a54ac115d23435762a3038" +} diff --git a/.sqlx/query-4237df28c12e17ca68a1e1b33ae80ce5a4a8dff6d0795f277fb18b7b40dc69ef.json b/.sqlx/query-4237df28c12e17ca68a1e1b33ae80ce5a4a8dff6d0795f277fb18b7b40dc69ef.json new file mode 100644 index 0000000..cf0ca95 --- /dev/null +++ b/.sqlx/query-4237df28c12e17ca68a1e1b33ae80ce5a4a8dff6d0795f277fb18b7b40dc69ef.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n campaign_id\n FROM\n survey_mcaptcha_campaign\n WHERE ID = (\n SELECT\n campaign_id\n FROM\n survey_mcaptcha_upload_jobs\n WHERE\n job_state = (SELECT ID FROM survey_mcaptcha_upload_job_states WHERE name = $1)\n AND\n finished_at is NULL\n AND\n scheduled_at is NULL\n ORDER BY created_at ASC\n );", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "campaign_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "4237df28c12e17ca68a1e1b33ae80ce5a4a8dff6d0795f277fb18b7b40dc69ef" +} diff --git a/migrations/20230701082846_survey_mcaptcha_upload.sql b/migrations/20230701082846_survey_mcaptcha_upload.sql index 9d9287d..65eb41f 100644 --- a/migrations/20230701082846_survey_mcaptcha_upload.sql +++ b/migrations/20230701082846_survey_mcaptcha_upload.sql @@ -20,3 +20,19 @@ CREATE TABLE IF NOT EXISTS survey_mcaptcha_analytics ( worker_type VARCHAR(100) NOT NULL, ID SERIAL PRIMARY KEY NOT NULL ); + + +CREATE TABLE IF NOT EXISTS survey_mcaptcha_upload_job_states ( + name VARCHAR(20) NOT NULL UNIQUE, + ID SERIAL PRIMARY KEY NOT NULL +); + +CREATE TABLE IF NOT EXISTS survey_mcaptcha_upload_jobs ( + campaign_id INTEGER references survey_mcaptcha_campaign(ID) ON DELETE CASCADE, + public_id varchar(100) NOT NULL UNIQUE, + created_at timestamptz NOT NULL DEFAULT now(), + scheduled_at timestamptz DEFAULT NULL, + finished_at timestamptz DEFAULT NULL, + job_state INTEGER references survey_mcaptcha_upload_job_states(ID) ON DELETE CASCADE, + ID SERIAL PRIMARY KEY NOT NULL +); diff --git a/src/db.rs b/src/db.rs new file mode 100644 index 0000000..ab90125 --- /dev/null +++ b/src/db.rs @@ -0,0 +1,87 @@ +use lazy_static::lazy_static; +use serde::{Deserialize, Serialize}; +use sqlx::PgPool; + +#[derive(Deserialize, Serialize, Clone, Debug, Eq, PartialEq)] +pub struct JobState { + pub name: String, +} + +impl JobState { + pub fn new(name: String) -> Self { + Self { name } + } +} + +lazy_static! { + pub static ref JOB_STATE_CREATE: JobState = JobState::new("job.state.create".into()); + pub static ref JOB_STATE_FINISH: JobState = JobState::new("job.state.finish".into()); + pub static ref JOB_STATE_RUNNING: JobState = + JobState::new("job.state.running".into()); + pub static ref JOB_STATES: [&'static JobState; 3] = + [&*JOB_STATE_CREATE, &*JOB_STATE_FINISH, &*JOB_STATE_RUNNING]; +} + +async fn job_state_exists( + db: &PgPool, + job_state: &JobState, +) -> sqlx::error::Result { + let res = sqlx::query!( + "SELECT EXISTS (SELECT 1 from survey_mcaptcha_upload_job_states WHERE name = $1)", + job_state.name, + ) + .fetch_one(db) + .await?; + + let mut resp = false; + if let Some(x) = res.exists { + resp = x; + } + + Ok(resp) +} + +async fn create_job_states(db: &PgPool) -> sqlx::error::Result<()> { + for j in &*JOB_STATES { + if !job_state_exists(db, j).await? { + sqlx::query!( + "INSERT INTO survey_mcaptcha_upload_job_states + (name) VALUES ($1) ON CONFLICT (name) DO NOTHING;", + j.name + ) + .execute(db) + .await?; + } + } + Ok(()) +} + +pub async fn migrate_db(db: &PgPool) -> sqlx::error::Result<()> { + sqlx::migrate!("./migrations/").run(db).await?; + create_job_states(db).await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[actix_rt::test] + async fn test_mcaptcha_job_states_exist() { + // can't use crate::tests::get_test_data because this module is used by + // ./src/tests-migrate.rs too, which doesn't load tests module + let settings = crate::settings::Settings::new().unwrap(); + let db = sqlx::postgres::PgPoolOptions::new() + .max_connections(2) + .connect(&settings.database.url) + .await + .expect("Unable to form database pool"); + + migrate_db(&db).await.unwrap(); + + for e in (*JOB_STATES).iter() { + println!("checking job state {}", e.name); + assert!(job_state_exists(&db, e).await.unwrap()); + } + } +} diff --git a/src/main.rs b/src/main.rs index 3a6ee72..8666d5c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -30,6 +30,7 @@ use log::info; mod api; mod archive; mod data; +mod db; mod errors; mod mcaptcha; mod pages; @@ -91,7 +92,7 @@ async fn main() -> std::io::Result<()> { Box::new(mcaptcha::MCaptchaClientReqwest::default()); let data = Data::new(settings.clone(), mcaptcha).await; - sqlx::migrate!("./migrations/").run(&data.db).await.unwrap(); + db::migrate_db(&data.db).await.unwrap(); let data = actix_web::web::Data::new(data); let arch = archive::Archiver::new(&data.settings); diff --git a/src/tests-migrate.rs b/src/tests-migrate.rs index b21b85c..338f6e9 100644 --- a/src/tests-migrate.rs +++ b/src/tests-migrate.rs @@ -18,6 +18,7 @@ use std::env; use sqlx::postgres::PgPoolOptions; +mod db; mod settings; pub use settings::Settings; @@ -40,7 +41,9 @@ async fn main() { } } - sqlx::migrate!("./migrations/").run(&db).await.unwrap(); + db::migrate_db(&db).await.unwrap(); + + // sqlx::migrate!("./migrations/").run(&db).await.unwrap(); } fn build() { From c0a125d5f1823a0c7a64ce20bbc268c3f0ef1b95 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Thu, 19 Oct 2023 22:11:31 +0530 Subject: [PATCH 08/14] feat: add db methods to manage scheduled jobs and their states --- ...aa0a1abee44a8b568ce74fa275ab936e8362f.json | 16 + ...54db2741b325727c01852cbc68ea8442d15ef.json | 64 +++ ...80ce5a4a8dff6d0795f277fb18b7b40dc69ef.json | 22 - ...d069a315b45c0c85c0b344d34cd8928b22c9c.json | 64 +++ ...fe551bc57096826119ad914cc1ef2d9ccb79d.json | 28 ++ ...a92214b644194443ae165957d9659d30dc3f9.json | 65 +++ ...3ad6aa4712c782222f2015b92916827f81079.json | 17 + ...e38148bf0dd2d79e1a4f97b8cbf04015ceff0.json | 16 + src/api/v1/mcaptcha/db.rs | 376 +++++++++++++++++- src/archive.rs | 1 - 10 files changed, 632 insertions(+), 37 deletions(-) create mode 100644 .sqlx/query-18495d6198079fdb8e4806d8a59aa0a1abee44a8b568ce74fa275ab936e8362f.json create mode 100644 .sqlx/query-1e41c42d89762ff4dc4b60a534a54db2741b325727c01852cbc68ea8442d15ef.json delete mode 100644 .sqlx/query-4237df28c12e17ca68a1e1b33ae80ce5a4a8dff6d0795f277fb18b7b40dc69ef.json create mode 100644 .sqlx/query-722f2d297a318f9804c1388d427d069a315b45c0c85c0b344d34cd8928b22c9c.json create mode 100644 .sqlx/query-8be51483900058d0bcc4c121440fe551bc57096826119ad914cc1ef2d9ccb79d.json create mode 100644 .sqlx/query-ca41f4e15fa5c5657a525ed9385a92214b644194443ae165957d9659d30dc3f9.json create mode 100644 .sqlx/query-ebfc456dd76b3fb2e5484f935703ad6aa4712c782222f2015b92916827f81079.json create mode 100644 .sqlx/query-fade9f99846165c34486f6492ece38148bf0dd2d79e1a4f97b8cbf04015ceff0.json diff --git a/.sqlx/query-18495d6198079fdb8e4806d8a59aa0a1abee44a8b568ce74fa275ab936e8362f.json b/.sqlx/query-18495d6198079fdb8e4806d8a59aa0a1abee44a8b568ce74fa275ab936e8362f.json new file mode 100644 index 0000000..1125f1f --- /dev/null +++ b/.sqlx/query-18495d6198079fdb8e4806d8a59aa0a1abee44a8b568ce74fa275ab936e8362f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n survey_mcaptcha_upload_jobs\n SET\n job_state = (SELECT ID FROM survey_mcaptcha_upload_job_states WHERE name = $1),\n scheduled_at = $2\n WHERE public_id = $3;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Timestamptz", + "Text" + ] + }, + "nullable": [] + }, + "hash": "18495d6198079fdb8e4806d8a59aa0a1abee44a8b568ce74fa275ab936e8362f" +} diff --git a/.sqlx/query-1e41c42d89762ff4dc4b60a534a54db2741b325727c01852cbc68ea8442d15ef.json b/.sqlx/query-1e41c42d89762ff4dc4b60a534a54db2741b325727c01852cbc68ea8442d15ef.json new file mode 100644 index 0000000..a285802 --- /dev/null +++ b/.sqlx/query-1e41c42d89762ff4dc4b60a534a54db2741b325727c01852cbc68ea8442d15ef.json @@ -0,0 +1,64 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n survey_mcaptcha_upload_jobs.ID,\n survey_mcaptcha_upload_jobs.public_id,\n survey_mcaptcha_campaign.campaign_id,\n survey_mcaptcha_campaign.public_id as campaign_public_id,\n survey_mcaptcha_upload_job_states.name,\n survey_mcaptcha_upload_jobs.created_at,\n survey_mcaptcha_upload_jobs.scheduled_at,\n survey_mcaptcha_upload_jobs.finished_at\n\n FROM survey_mcaptcha_upload_jobs\n INNER JOIN\n survey_mcaptcha_upload_job_states\n ON\n survey_mcaptcha_upload_job_states.ID = survey_mcaptcha_upload_jobs.job_state\n INNER JOIN\n survey_mcaptcha_campaign\n ON\n survey_mcaptcha_campaign.ID = survey_mcaptcha_upload_jobs.campaign_id\n WHERE\n survey_mcaptcha_upload_job_states.name = $1;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "public_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "campaign_id", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "campaign_public_id", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "scheduled_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 7, + "name": "finished_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + true, + true + ] + }, + "hash": "1e41c42d89762ff4dc4b60a534a54db2741b325727c01852cbc68ea8442d15ef" +} diff --git a/.sqlx/query-4237df28c12e17ca68a1e1b33ae80ce5a4a8dff6d0795f277fb18b7b40dc69ef.json b/.sqlx/query-4237df28c12e17ca68a1e1b33ae80ce5a4a8dff6d0795f277fb18b7b40dc69ef.json deleted file mode 100644 index cf0ca95..0000000 --- a/.sqlx/query-4237df28c12e17ca68a1e1b33ae80ce5a4a8dff6d0795f277fb18b7b40dc69ef.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n campaign_id\n FROM\n survey_mcaptcha_campaign\n WHERE ID = (\n SELECT\n campaign_id\n FROM\n survey_mcaptcha_upload_jobs\n WHERE\n job_state = (SELECT ID FROM survey_mcaptcha_upload_job_states WHERE name = $1)\n AND\n finished_at is NULL\n AND\n scheduled_at is NULL\n ORDER BY created_at ASC\n );", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "campaign_id", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false - ] - }, - "hash": "4237df28c12e17ca68a1e1b33ae80ce5a4a8dff6d0795f277fb18b7b40dc69ef" -} diff --git a/.sqlx/query-722f2d297a318f9804c1388d427d069a315b45c0c85c0b344d34cd8928b22c9c.json b/.sqlx/query-722f2d297a318f9804c1388d427d069a315b45c0c85c0b344d34cd8928b22c9c.json new file mode 100644 index 0000000..b7a9262 --- /dev/null +++ b/.sqlx/query-722f2d297a318f9804c1388d427d069a315b45c0c85c0b344d34cd8928b22c9c.json @@ -0,0 +1,64 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n survey_mcaptcha_upload_jobs.ID,\n survey_mcaptcha_upload_jobs.public_id,\n survey_mcaptcha_campaign.campaign_id,\n survey_mcaptcha_campaign.public_id as campaign_public_id,\n survey_mcaptcha_upload_job_states.name,\n survey_mcaptcha_upload_jobs.created_at,\n survey_mcaptcha_upload_jobs.scheduled_at,\n survey_mcaptcha_upload_jobs.finished_at\n\n FROM survey_mcaptcha_upload_jobs\n INNER JOIN\n survey_mcaptcha_upload_job_states\n ON\n survey_mcaptcha_upload_job_states.ID = survey_mcaptcha_upload_jobs.job_state\n INNER JOIN\n survey_mcaptcha_campaign\n ON\n survey_mcaptcha_campaign.ID = survey_mcaptcha_upload_jobs.campaign_id\n WHERE\n survey_mcaptcha_upload_jobs.public_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "public_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "campaign_id", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "campaign_public_id", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "scheduled_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 7, + "name": "finished_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + true, + true + ] + }, + "hash": "722f2d297a318f9804c1388d427d069a315b45c0c85c0b344d34cd8928b22c9c" +} diff --git a/.sqlx/query-8be51483900058d0bcc4c121440fe551bc57096826119ad914cc1ef2d9ccb79d.json b/.sqlx/query-8be51483900058d0bcc4c121440fe551bc57096826119ad914cc1ef2d9ccb79d.json new file mode 100644 index 0000000..3987ac3 --- /dev/null +++ b/.sqlx/query-8be51483900058d0bcc4c121440fe551bc57096826119ad914cc1ef2d9ccb79d.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n survey_mcaptcha_campaign.campaign_id,\n survey_mcaptcha_upload_jobs.public_id\n FROM\n survey_mcaptcha_campaign\n INNER JOIN\n survey_mcaptcha_upload_jobs\n ON\n survey_mcaptcha_upload_jobs.campaign_id = survey_mcaptcha_campaign.ID\n WHERE\n survey_mcaptcha_upload_jobs.job_state = (\n SELECT ID FROM survey_mcaptcha_upload_job_states WHERE name = $1\n )\n AND\n survey_mcaptcha_upload_jobs.finished_at is NULL\n AND\n survey_mcaptcha_upload_jobs.scheduled_at is NULL\n ORDER BY created_at ASC;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "campaign_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "public_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "8be51483900058d0bcc4c121440fe551bc57096826119ad914cc1ef2d9ccb79d" +} diff --git a/.sqlx/query-ca41f4e15fa5c5657a525ed9385a92214b644194443ae165957d9659d30dc3f9.json b/.sqlx/query-ca41f4e15fa5c5657a525ed9385a92214b644194443ae165957d9659d30dc3f9.json new file mode 100644 index 0000000..92e330d --- /dev/null +++ b/.sqlx/query-ca41f4e15fa5c5657a525ed9385a92214b644194443ae165957d9659d30dc3f9.json @@ -0,0 +1,65 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n survey_mcaptcha_upload_jobs.ID,\n survey_mcaptcha_upload_jobs.public_id,\n survey_mcaptcha_campaign.campaign_id,\n survey_mcaptcha_campaign.public_id as campaign_public_id,\n survey_mcaptcha_upload_job_states.name,\n survey_mcaptcha_upload_jobs.created_at,\n survey_mcaptcha_upload_jobs.scheduled_at,\n survey_mcaptcha_upload_jobs.finished_at\n\n FROM survey_mcaptcha_upload_jobs\n INNER JOIN\n survey_mcaptcha_upload_job_states\n ON\n survey_mcaptcha_upload_job_states.ID = survey_mcaptcha_upload_jobs.job_state\n INNER JOIN\n survey_mcaptcha_campaign\n ON\n survey_mcaptcha_campaign.ID = survey_mcaptcha_upload_jobs.campaign_id\n WHERE\n survey_mcaptcha_campaign.campaign_id = $1\n AND\n survey_mcaptcha_upload_job_states.name = $2;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "public_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "campaign_id", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "campaign_public_id", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "scheduled_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 7, + "name": "finished_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + true, + true + ] + }, + "hash": "ca41f4e15fa5c5657a525ed9385a92214b644194443ae165957d9659d30dc3f9" +} diff --git a/.sqlx/query-ebfc456dd76b3fb2e5484f935703ad6aa4712c782222f2015b92916827f81079.json b/.sqlx/query-ebfc456dd76b3fb2e5484f935703ad6aa4712c782222f2015b92916827f81079.json new file mode 100644 index 0000000..af67acf --- /dev/null +++ b/.sqlx/query-ebfc456dd76b3fb2e5484f935703ad6aa4712c782222f2015b92916827f81079.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO survey_mcaptcha_upload_jobs\n (campaign_id, job_state, created_at, public_id)\n VALUES (\n (SELECT ID FROM survey_mcaptcha_campaign WHERE campaign_id = $1),\n (SELECT ID FROM survey_mcaptcha_upload_job_states WHERE name = $2),\n $3, $4)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Timestamptz", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "ebfc456dd76b3fb2e5484f935703ad6aa4712c782222f2015b92916827f81079" +} diff --git a/.sqlx/query-fade9f99846165c34486f6492ece38148bf0dd2d79e1a4f97b8cbf04015ceff0.json b/.sqlx/query-fade9f99846165c34486f6492ece38148bf0dd2d79e1a4f97b8cbf04015ceff0.json new file mode 100644 index 0000000..068f3db --- /dev/null +++ b/.sqlx/query-fade9f99846165c34486f6492ece38148bf0dd2d79e1a4f97b8cbf04015ceff0.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n survey_mcaptcha_upload_jobs\n SET\n job_state = (SELECT ID FROM survey_mcaptcha_upload_job_states WHERE name = $1),\n finished_at = $2\n WHERE public_id = $3;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Timestamptz", + "Text" + ] + }, + "nullable": [] + }, + "hash": "fade9f99846165c34486f6492ece38148bf0dd2d79e1a4f97b8cbf04015ceff0" +} diff --git a/src/api/v1/mcaptcha/db.rs b/src/api/v1/mcaptcha/db.rs index a6315bc..9de3643 100644 --- a/src/api/v1/mcaptcha/db.rs +++ b/src/api/v1/mcaptcha/db.rs @@ -15,12 +15,22 @@ * along with this program. If not, see . */ use url::Url; +use uuid::Uuid; use crate::api::v1::get_random; +use crate::db::{ + JobState, JOB_STATES, JOB_STATE_CREATE, JOB_STATE_FINISH, JOB_STATE_RUNNING, +}; use crate::errors::*; use crate::mcaptcha::PerformanceAnalytics; use crate::Data; +use sqlx::types::time::OffsetDateTime; + +fn now_unix_time_stamp() -> OffsetDateTime { + OffsetDateTime::now_utc() +} + impl Data { /// Check if an mCaptcha instance is registered on the database pub async fn mcaptcha_url_exists(&self, url: &str) -> ServiceResult { @@ -127,14 +137,14 @@ impl Data { .await?; Ok(()) } - /// Delete mCaptcha camapign from database + /// Delete mCaptcha campaign from database pub async fn mcaptcha_delete_mcaptcha_campaign( &self, - campaign_id: &uuid::Uuid, + campaign_id: &Uuid, secret: &str, ) -> ServiceResult<()> { let campaign_str = campaign_id.to_string(); - let res = sqlx::query!( + sqlx::query!( "DELETE FROM survey_mcaptcha_campaign WHERE @@ -160,7 +170,7 @@ impl Data { /// Check if an mCaptcha instance campaign is registered on DB pub async fn mcaptcha_campaign_is_registered( &self, - campaign_id: &uuid::Uuid, + campaign_id: &Uuid, secret: &str, ) -> ServiceResult { let campaign_str = campaign_id.to_string(); @@ -201,11 +211,11 @@ impl Data { /// Register an mCaptcha instance campaign on DB pub async fn mcaptcha_register_campaign( &self, - campaign_id: &uuid::Uuid, + campaign_id: &Uuid, secret: &str, ) -> ServiceResult<()> { let campaign_str = campaign_id.to_string(); - let public_id = uuid::Uuid::new_v4(); + let public_id = Uuid::new_v4(); sqlx::query!( "INSERT INTO @@ -223,9 +233,9 @@ impl Data { /// Register an mCaptcha instance campaign on DB pub async fn mcaptcha_get_campaign_public_id( &self, - campaign_id: &uuid::Uuid, + campaign_id: &Uuid, secret: &str, - ) -> ServiceResult { + ) -> ServiceResult { let campaign_str = campaign_id.to_string(); struct S { public_id: String, @@ -247,13 +257,13 @@ impl Data { .fetch_one(&self.db) .await?; - Ok(uuid::Uuid::parse_str(&res.public_id).unwrap()) + Ok(Uuid::parse_str(&res.public_id).unwrap()) } /// Get an mCaptcha instance campaign checkpoint pub async fn mcaptcha_get_checkpoint( &self, - campaign_id: &uuid::Uuid, + campaign_id: &Uuid, secret: &str, ) -> ServiceResult { let campaign_str = campaign_id.to_string(); @@ -286,7 +296,7 @@ impl Data { /// Set an mCaptcha instance campaign checkpoint pub async fn mcaptcha_set_checkpoint( &self, - campaign_id: &uuid::Uuid, + campaign_id: &Uuid, secret: &str, checkpoint: usize, ) -> ServiceResult<()> { @@ -316,7 +326,7 @@ impl Data { /// Store mCaptcha instance campaign analytics pub async fn mcaptcha_insert_analytics( &self, - campaign_id: &uuid::Uuid, + campaign_id: &Uuid, secret: &str, r: &PerformanceAnalytics, ) -> ServiceResult<()> { @@ -353,7 +363,7 @@ impl Data { /// fetch PoW analytics pub async fn mcaptcha_analytics_fetch( &self, - public_id: &uuid::Uuid, + public_id: &Uuid, limit: usize, offset: usize, ) -> ServiceResult> { @@ -402,12 +412,294 @@ impl Data { Ok(res) } + + pub async fn get_next_job_to_run(&self) -> ServiceResult { + let res = sqlx::query_as!( + InnerSchedulerJob, + "SELECT + survey_mcaptcha_campaign.campaign_id, + survey_mcaptcha_upload_jobs.public_id + FROM + survey_mcaptcha_campaign + INNER JOIN + survey_mcaptcha_upload_jobs + ON + survey_mcaptcha_upload_jobs.campaign_id = survey_mcaptcha_campaign.ID + WHERE + survey_mcaptcha_upload_jobs.job_state = ( + SELECT ID FROM survey_mcaptcha_upload_job_states WHERE name = $1 + ) + AND + survey_mcaptcha_upload_jobs.finished_at is NULL + AND + survey_mcaptcha_upload_jobs.scheduled_at is NULL + ORDER BY created_at ASC;", + &JOB_STATE_CREATE.name + ) + .fetch_one(&self.db) + .await?; + Ok(res.into()) + } + + pub async fn add_job(&self, campaign_id: &Uuid) -> ServiceResult { + let now = now_unix_time_stamp(); + + if let Some(unfinished_job) = + self.get_unfinished_job_for_campaign(campaign_id).await? + { + return Ok(unfinished_job.public_job_id); + } + + let public_id = Uuid::new_v4(); + let public_id_str = public_id.to_string(); + + let campaign_str = campaign_id.to_string(); + sqlx::query!( + "INSERT INTO survey_mcaptcha_upload_jobs + (campaign_id, job_state, created_at, public_id) + VALUES ( + (SELECT ID FROM survey_mcaptcha_campaign WHERE campaign_id = $1), + (SELECT ID FROM survey_mcaptcha_upload_job_states WHERE name = $2), + $3, $4)", + &campaign_str, + &JOB_STATE_CREATE.name, + now, + public_id_str + ) + .execute(&self.db) + .await?; + Ok(public_id) + } + + pub async fn get_unfinished_job_for_campaign( + &self, + campaign_id: &Uuid, + ) -> ServiceResult> { + let res = match sqlx::query_as!( + InnerJob, + " + SELECT + survey_mcaptcha_upload_jobs.ID, + survey_mcaptcha_upload_jobs.public_id, + survey_mcaptcha_campaign.campaign_id, + survey_mcaptcha_campaign.public_id as campaign_public_id, + survey_mcaptcha_upload_job_states.name, + survey_mcaptcha_upload_jobs.created_at, + survey_mcaptcha_upload_jobs.scheduled_at, + survey_mcaptcha_upload_jobs.finished_at + + FROM survey_mcaptcha_upload_jobs + INNER JOIN + survey_mcaptcha_upload_job_states + ON + survey_mcaptcha_upload_job_states.ID = survey_mcaptcha_upload_jobs.job_state + INNER JOIN + survey_mcaptcha_campaign + ON + survey_mcaptcha_campaign.ID = survey_mcaptcha_upload_jobs.campaign_id + WHERE + survey_mcaptcha_campaign.campaign_id = $1 + AND + survey_mcaptcha_upload_job_states.name = $2;", + &campaign_id.to_string(), + &JOB_STATE_CREATE.name + ) + .fetch_one(&self.db) + .await { + Ok(res) => Ok(Some(res.into())), + Err(sqlx::Error::RowNotFound) => Ok(None), + Err(e) => Err(e), + }?; + Ok(res) + } + + pub async fn get_job(&self, public_id: &uuid::Uuid) -> ServiceResult> { + let res = match sqlx::query_as!( + InnerJob, + " + SELECT + survey_mcaptcha_upload_jobs.ID, + survey_mcaptcha_upload_jobs.public_id, + survey_mcaptcha_campaign.campaign_id, + survey_mcaptcha_campaign.public_id as campaign_public_id, + survey_mcaptcha_upload_job_states.name, + survey_mcaptcha_upload_jobs.created_at, + survey_mcaptcha_upload_jobs.scheduled_at, + survey_mcaptcha_upload_jobs.finished_at + + FROM survey_mcaptcha_upload_jobs + INNER JOIN + survey_mcaptcha_upload_job_states + ON + survey_mcaptcha_upload_job_states.ID = survey_mcaptcha_upload_jobs.job_state + INNER JOIN + survey_mcaptcha_campaign + ON + survey_mcaptcha_campaign.ID = survey_mcaptcha_upload_jobs.campaign_id + WHERE + survey_mcaptcha_upload_jobs.public_id = $1", + &public_id.to_string() + ) + .fetch_one(&self.db) + .await { + Ok(res) => Ok(Some(res.into())), + Err(sqlx::Error::RowNotFound) => Ok(None), + Err(e) => Err(e), + }?; + + Ok(res.into()) + } + + pub async fn get_all_jobs_of_state( + &self, + state: &JobState, + ) -> ServiceResult> { + let mut res = sqlx::query_as!( + InnerJob, + " + SELECT + survey_mcaptcha_upload_jobs.ID, + survey_mcaptcha_upload_jobs.public_id, + survey_mcaptcha_campaign.campaign_id, + survey_mcaptcha_campaign.public_id as campaign_public_id, + survey_mcaptcha_upload_job_states.name, + survey_mcaptcha_upload_jobs.created_at, + survey_mcaptcha_upload_jobs.scheduled_at, + survey_mcaptcha_upload_jobs.finished_at + + FROM survey_mcaptcha_upload_jobs + INNER JOIN + survey_mcaptcha_upload_job_states + ON + survey_mcaptcha_upload_job_states.ID = survey_mcaptcha_upload_jobs.job_state + INNER JOIN + survey_mcaptcha_campaign + ON + survey_mcaptcha_campaign.ID = survey_mcaptcha_upload_jobs.campaign_id + WHERE + survey_mcaptcha_upload_job_states.name = $1;", + &state.name + ) + .fetch_all(&self.db) + .await?; + + let res = res.drain(0..).map(|r| r.into()).collect(); + + Ok(res) + } + + pub async fn mark_job_scheduled(&self, public_id: &Uuid) -> ServiceResult<()> { + let now = now_unix_time_stamp(); + sqlx::query!( + " + UPDATE + survey_mcaptcha_upload_jobs + SET + job_state = (SELECT ID FROM survey_mcaptcha_upload_job_states WHERE name = $1), + scheduled_at = $2 + WHERE public_id = $3;", + &JOB_STATE_RUNNING.name, + now, + &public_id.to_string(), + ) + .execute(&self.db) + .await + ?; + + Ok(()) + } + + pub async fn mark_job_finished(&self, public_id: &Uuid) -> ServiceResult<()> { + let now = now_unix_time_stamp(); + sqlx::query!( + " + UPDATE + survey_mcaptcha_upload_jobs + SET + job_state = (SELECT ID FROM survey_mcaptcha_upload_job_states WHERE name = $1), + finished_at = $2 + WHERE public_id = $3;", + &JOB_STATE_FINISH.name, + now, + &public_id.to_string(), + ) + .execute(&self.db) + .await + ?; + + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SchedulerJob { + pub campaign_id: Uuid, + pub public_job_id: Uuid, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct InnerSchedulerJob { + campaign_id: String, + public_id: String, +} +impl From for SchedulerJob { + fn from(j: InnerSchedulerJob) -> Self { + SchedulerJob { + campaign_id: Uuid::parse_str(&j.campaign_id).unwrap(), + public_job_id: Uuid::parse_str(&j.public_id).unwrap(), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Job { + pub state: JobState, + pub campaign_id: Uuid, + pub campaign_public_id: Uuid, + pub public_job_id: Uuid, + pub id: u32, + pub created_at: OffsetDateTime, + pub scheduled_at: Option, + pub finished_at: Option, +} + +struct InnerJob { + name: String, + campaign_id: String, + public_id: String, + campaign_public_id: String, + id: i32, + created_at: OffsetDateTime, + scheduled_at: Option, + finished_at: Option, +} + +impl From for Job { + fn from(j: InnerJob) -> Self { + Job { + state: (JOB_STATES) + .iter() + .find(|d| d.name == j.name) + .unwrap() + .to_owned() + .to_owned(), + id: j.id as u32, + created_at: j.created_at, + scheduled_at: j.scheduled_at, + finished_at: j.finished_at, + + campaign_id: Uuid::parse_str(&j.campaign_id).unwrap(), + campaign_public_id: Uuid::parse_str(&j.campaign_public_id).unwrap(), + public_job_id: Uuid::parse_str(&j.public_id).unwrap(), + } + } } #[cfg(test)] mod tests { use crate::{mcaptcha::PerformanceAnalytics, tests::*}; + use super::*; use url::Url; #[actix_rt::test] @@ -436,7 +728,7 @@ mod tests { url ); - let uuid = uuid::Uuid::new_v4(); + let uuid = Uuid::new_v4(); if data .mcaptcha_campaign_is_registered(&uuid, &secret) @@ -505,5 +797,61 @@ mod tests { .unwrap(), vec![] ); + + // job related stuff + + let job1_public_id = data.add_job(&uuid).await.unwrap(); + let job = data.get_job(&job1_public_id).await.unwrap().unwrap(); + assert_eq!(public_id, job.campaign_public_id); + assert_eq!( + data.get_unfinished_job_for_campaign(&uuid) + .await + .unwrap() + .unwrap(), + job + ); + let job2_public_id = data.add_job(&uuid).await.unwrap(); + let job2 = data.get_job(&job2_public_id).await.unwrap().unwrap(); + assert_eq!(job2, job); + + assert_eq!( + data.get_next_job_to_run().await.unwrap().public_job_id, + job.public_job_id + ); + + assert!(job.created_at < now_unix_time_stamp()); + assert!(job.scheduled_at.is_none()); + assert!(job.finished_at.is_none()); + assert_eq!( + data.get_all_jobs_of_state(&JOB_STATE_CREATE).await.unwrap(), + vec![job.clone()] + ); + + data.mark_job_scheduled(&job.public_job_id).await.unwrap(); + let job = data.get_job(&job.public_job_id).await.unwrap().unwrap(); + assert!(job.scheduled_at.is_some()); + assert_eq!( + data.get_all_jobs_of_state(&JOB_STATE_RUNNING) + .await + .unwrap(), + vec![job.clone()] + ); + + data.mark_job_finished(&job.public_job_id).await.unwrap(); + let job = data.get_job(&job.public_job_id).await.unwrap().unwrap(); + assert!(job.finished_at.is_some()); + assert_eq!( + data.get_all_jobs_of_state(&JOB_STATE_FINISH).await.unwrap(), + vec![job.clone()] + ); + + let job2_public_id = data.add_job(&uuid).await.unwrap(); + let job2 = data.get_job(&job2_public_id).await.unwrap().unwrap(); + assert_ne!(job2.public_job_id, job.public_job_id); + assert_eq!( + data.get_next_job_to_run().await.unwrap().public_job_id, + job2.public_job_id + ); + assert_eq!(public_id, job2.campaign_public_id); } } diff --git a/src/archive.rs b/src/archive.rs index 33c6103..6dc8c46 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -14,7 +14,6 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -use std::future::Future; use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; From c8ecd29e9478c7b43e87a5e4809abace656ce80f Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Fri, 20 Oct 2023 01:40:32 +0530 Subject: [PATCH 09/14] feat: job runner to execute download requests from mCaptcha/mCaptcha --- Cargo.lock | 12 +++++ Cargo.toml | 2 +- src/mcaptcha.rs | 126 ++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 130 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d8f0e51..8ed4986 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3301,9 +3301,21 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "socket2", + "tokio-macros", "windows-sys 0.45.0", ] +[[package]] +name = "tokio-macros" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "tokio-native-tls" version = "0.3.1" diff --git a/Cargo.toml b/Cargo.toml index db88d76..67b6feb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,7 +70,7 @@ mime = "0.3.16" #sailfish = "0.3.2" tracing = { version = "0.1.37", features = ["log"] } tera = { version="1.17.1", features=["builtins"]} -tokio = { version = "1.25.0", features = ["fs"] } +tokio = { version = "1.25.0", features = ["fs", "macros"] } csv-async = { version = "1.2.5", features = ["serde", "tokio"] } async-trait = "0.1.68" reqwest = { version = "0.11.18", features = ["json", "gzip"] } diff --git a/src/mcaptcha.rs b/src/mcaptcha.rs index c57e695..f7c69c5 100644 --- a/src/mcaptcha.rs +++ b/src/mcaptcha.rs @@ -14,12 +14,17 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +use std::time::Duration; + use async_trait::async_trait; use reqwest::Client; use serde::{Deserialize, Serialize}; +use tokio::sync::oneshot; +use tokio::task::JoinHandle; +use tokio::time::sleep; use url::Url; -use crate::errors::*; +use crate::{api::v1::mcaptcha::db::SchedulerJob, errors::*, AppData}; /* TODO: * 1. Define traits to interact with mCaptcha @@ -79,11 +84,6 @@ impl Clone for Box { } } -#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq)] -pub struct Secret { - pub secret: String, -} - #[derive(Clone)] pub struct MCaptchaClientReqwest { client: Client, @@ -122,7 +122,8 @@ impl MCaptchaClient for MCaptchaClientReqwest { campaign_id: &str, page: usize, ) -> ServiceResult> { - mcaptcha.set_path(&format!("/api/v1/survey/{campaign_id}/get?page={page}")); + mcaptcha.set_path(&format!("/api/v1/survey/takeout/{campaign_id}/get")); + mcaptcha.set_query(Some(&format!("page={page}"))); let res = self .client .get(mcaptcha) @@ -136,6 +137,114 @@ impl MCaptchaClient for MCaptchaClientReqwest { } } +#[derive(Clone)] +pub struct MCaptchaDownloader { + data: AppData, +} + +impl MCaptchaDownloader { + pub fn new(data: AppData) -> Self { + Self { data } + } + + fn can_run(rx: &mut oneshot::Receiver<()>) -> bool { + matches!(rx.try_recv(), Err(oneshot::error::TryRecvError::Empty)) + } + + pub async fn start_job( + &self, + ) -> ServiceResult<(oneshot::Sender<()>, JoinHandle<()>)> { + let (tx, mut rx) = oneshot::channel(); + let this = self.clone(); + let fut = async move { + loop { + if !Self::can_run(&mut rx) { + log::info!("stopping survey uploads"); + break; + } + + let task = this.data.get_next_job_to_run().await.unwrap(); + if task.is_none() { + for _ in 0..5 { + if !Self::can_run(&mut rx) { + log::info!("Stopping survey uploads"); + break; + } + sleep(Duration::new(1, 0)).await; + } + + continue; + } + + let task = task.unwrap(); + this.data.mark_job_scheduled(&task).await.unwrap(); + this.exec_job(&task, &mut rx).await.unwrap(); + } + }; + let handle = tokio::spawn(fut); + Ok((tx, handle)) + } + + async fn exec_job( + &self, + job: &SchedulerJob, + rx: &mut oneshot::Receiver<()>, + ) -> ServiceResult<()> { + let checkpoint = self.data.mcaptcha_get_checkpoint(&job.campaign_id).await?; + const LIMIT: usize = 50; + let mut page = 1 + (checkpoint / LIMIT); + let campaign_str = job.campaign_id.to_string(); + log::info!("getting page {page} from {campaign_str}"); + loop { + if !Self::can_run(rx) { + log::info!("Stopping survey downloads"); + break; + } + + let mut res = self + .data + .mcaptcha + .download_benchmarks(job.url.clone(), &campaign_str, page) + .await?; + + if !Self::can_run(rx) { + log::info!("Stopping survey downloads"); + break; + } + + let skip = checkpoint - ((page - 1) * LIMIT); + let new_records = res.len() - skip as usize; + let mut skip = skip as isize; + for r in res.drain(0..) { + if skip > 0 { + skip -= 1; + continue; + } + self.data + .mcaptcha_insert_analytics(&job.campaign_id, &r) + .await?; + } + self.data + .mcaptcha_set_checkpoint(&job.campaign_id, new_records) + .await?; + + if !Self::can_run(rx) { + log::info!("Stopping survey downloads"); + break; + } + + page += 1; + if res.len() < LIMIT { + break; + } + } + + self.data.mark_job_finished(job).await.unwrap(); + + Ok(()) + } +} + #[cfg(test)] pub mod tests { use super::*; @@ -196,8 +305,7 @@ pub mod tests { ) -> ServiceResult> { println!( "mcaptcha_url {}, campaign_id {}, page: {page}", - mcaptcha.to_string(), - campaign_id + mcaptcha, campaign_id ); let res = BENCHMARK.clone(); Ok(res) From 3e5dca9069901b1cdbc2eda7c6af35c620975cc7 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Fri, 20 Oct 2023 01:41:46 +0530 Subject: [PATCH 10/14] feat: clean up auth method and include hostname in scheduler ctx --- src/api/v1/mcaptcha/db.rs | 129 +++++++++++++++----------------------- 1 file changed, 51 insertions(+), 78 deletions(-) diff --git a/src/api/v1/mcaptcha/db.rs b/src/api/v1/mcaptcha/db.rs index 9de3643..8b2eef9 100644 --- a/src/api/v1/mcaptcha/db.rs +++ b/src/api/v1/mcaptcha/db.rs @@ -78,14 +78,7 @@ impl Data { } /// Authenticate an mCaptcha instance and return its URL - pub async fn mcaptcha_authenticate_and_get_url( - &self, - secret: &str, - ) -> ServiceResult { - struct U { - url: String, - } - + pub async fn mcaptcha_authenticate(&self, secret: &str) -> ServiceResult<()> { let res = sqlx::query!( "SELECT EXISTS ( SELECT @@ -99,27 +92,11 @@ impl Data { .fetch_one(&self.db) .await?; - if !match res.exists { - Some(true) => true, - _ => false, - } { + if !matches!(res.exists, Some(true)) { return Err(ServiceError::WrongPassword); } - let url = sqlx::query_as!( - U, - "SELECT - url - FROM - survey_mcaptcha_hostname - WHERE - secret = $1; ", - secret - ) - .fetch_one(&self.db) - .await?; - - Ok(Url::parse(&url.url).unwrap()) + Ok(()) } /// Delete mCaptcha instance from database @@ -264,7 +241,6 @@ impl Data { pub async fn mcaptcha_get_checkpoint( &self, campaign_id: &Uuid, - secret: &str, ) -> ServiceResult { let campaign_str = campaign_id.to_string(); @@ -279,13 +255,8 @@ impl Data { FROM survey_mcaptcha_campaign WHERE - campaign_id = $1 - AND - url_id = ( - SELECT ID FROM survey_mcaptcha_hostname WHERE secret = $2 - );", + campaign_id = $1;", &campaign_str, - secret ) .fetch_one(&self.db) .await?; @@ -297,7 +268,6 @@ impl Data { pub async fn mcaptcha_set_checkpoint( &self, campaign_id: &Uuid, - secret: &str, checkpoint: usize, ) -> ServiceResult<()> { let campaign_str = campaign_id.to_string(); @@ -307,15 +277,9 @@ impl Data { SET synced_till = $1 WHERE - campaign_id = $2 - AND - url_id = ( - SELECT ID FROM survey_mcaptcha_hostname WHERE secret = $3 - ) - ", + campaign_id = $2; ", checkpoint as i32, &campaign_str, - secret ) .execute(&self.db) .await?; @@ -327,7 +291,6 @@ impl Data { pub async fn mcaptcha_insert_analytics( &self, campaign_id: &Uuid, - secret: &str, r: &PerformanceAnalytics, ) -> ServiceResult<()> { let campaign_str = campaign_id.to_string(); @@ -343,14 +306,9 @@ impl Data { survey_mcaptcha_campaign WHERE campaign_id = $1 - AND - url_id = ( - SELECT ID FROM survey_mcaptcha_hostname WHERE secret = $2 - ) - ), $3, $4, $5 + ), $2, $3, $4 );", &campaign_str, - secret, r.time as i32, r.difficulty_factor as i32, &r.worker_type, @@ -413,18 +371,23 @@ impl Data { Ok(res) } - pub async fn get_next_job_to_run(&self) -> ServiceResult { - let res = sqlx::query_as!( + pub async fn get_next_job_to_run(&self) -> ServiceResult> { + let res = match sqlx::query_as!( InnerSchedulerJob, "SELECT survey_mcaptcha_campaign.campaign_id, - survey_mcaptcha_upload_jobs.public_id + survey_mcaptcha_upload_jobs.public_id, + survey_mcaptcha_hostname.url FROM survey_mcaptcha_campaign INNER JOIN survey_mcaptcha_upload_jobs ON survey_mcaptcha_upload_jobs.campaign_id = survey_mcaptcha_campaign.ID + INNER JOIN + survey_mcaptcha_hostname + ON + survey_mcaptcha_hostname.ID = survey_mcaptcha_campaign.url_id WHERE survey_mcaptcha_upload_jobs.job_state = ( SELECT ID FROM survey_mcaptcha_upload_job_states WHERE name = $1 @@ -437,8 +400,13 @@ impl Data { &JOB_STATE_CREATE.name ) .fetch_one(&self.db) - .await?; - Ok(res.into()) + .await + { + Ok(res) => Ok(Some(res.into())), + Err(sqlx::Error::RowNotFound) => Ok(None), + Err(e) => Err(e), + }?; + Ok(res) } pub async fn add_job(&self, campaign_id: &Uuid) -> ServiceResult { @@ -547,7 +515,7 @@ impl Data { Err(e) => Err(e), }?; - Ok(res.into()) + Ok(res) } pub async fn get_all_jobs_of_state( @@ -588,7 +556,7 @@ impl Data { Ok(res) } - pub async fn mark_job_scheduled(&self, public_id: &Uuid) -> ServiceResult<()> { + pub async fn mark_job_scheduled(&self, job: &SchedulerJob) -> ServiceResult<()> { let now = now_unix_time_stamp(); sqlx::query!( " @@ -600,7 +568,7 @@ impl Data { WHERE public_id = $3;", &JOB_STATE_RUNNING.name, now, - &public_id.to_string(), + &job.public_job_id.to_string(), ) .execute(&self.db) .await @@ -609,7 +577,7 @@ impl Data { Ok(()) } - pub async fn mark_job_finished(&self, public_id: &Uuid) -> ServiceResult<()> { + pub async fn mark_job_finished(&self, job: &SchedulerJob) -> ServiceResult<()> { let now = now_unix_time_stamp(); sqlx::query!( " @@ -621,7 +589,7 @@ impl Data { WHERE public_id = $3;", &JOB_STATE_FINISH.name, now, - &public_id.to_string(), + &job.public_job_id.to_string(), ) .execute(&self.db) .await @@ -635,18 +603,21 @@ impl Data { pub struct SchedulerJob { pub campaign_id: Uuid, pub public_job_id: Uuid, + pub url: Url, } #[derive(Clone, Debug, PartialEq, Eq)] struct InnerSchedulerJob { campaign_id: String, public_id: String, + url: String, } impl From for SchedulerJob { fn from(j: InnerSchedulerJob) -> Self { SchedulerJob { campaign_id: Uuid::parse_str(&j.campaign_id).unwrap(), public_job_id: Uuid::parse_str(&j.public_id).unwrap(), + url: Url::parse(&j.url).unwrap(), } } } @@ -721,11 +692,10 @@ mod tests { assert_ne!(secret2, secret); let secret = secret2; + assert!(data.mcaptcha_authenticate(&secret).await.is_ok()); assert_eq!( - data.mcaptcha_authenticate_and_get_url(&secret) - .await - .unwrap(), - url + data.mcaptcha_authenticate("foo").await.err(), + Some(ServiceError::WrongPassword) ); let uuid = Uuid::new_v4(); @@ -752,17 +722,9 @@ mod tests { .await .unwrap()); - assert_eq!( - data.mcaptcha_get_checkpoint(&uuid, &secret).await.unwrap(), - 0 - ); - data.mcaptcha_set_checkpoint(&uuid, &secret, 1) - .await - .unwrap(); - assert_eq!( - data.mcaptcha_get_checkpoint(&uuid, &secret).await.unwrap(), - 1 - ); + assert_eq!(data.mcaptcha_get_checkpoint(&uuid).await.unwrap(), 0); + data.mcaptcha_set_checkpoint(&uuid, 1).await.unwrap(); + assert_eq!(data.mcaptcha_get_checkpoint(&uuid).await.unwrap(), 1); let analytics = PerformanceAnalytics { id: 1, @@ -770,7 +732,7 @@ mod tests { difficulty_factor: 1, worker_type: "foo".to_string(), }; - data.mcaptcha_insert_analytics(&uuid, &secret, &analytics) + data.mcaptcha_insert_analytics(&uuid, &analytics) .await .unwrap(); @@ -813,9 +775,15 @@ mod tests { let job2_public_id = data.add_job(&uuid).await.unwrap(); let job2 = data.get_job(&job2_public_id).await.unwrap().unwrap(); assert_eq!(job2, job); + let scheduler_job = data.get_next_job_to_run().await.unwrap().unwrap(); + assert_eq!(scheduler_job.url, url); assert_eq!( - data.get_next_job_to_run().await.unwrap().public_job_id, + data.get_next_job_to_run() + .await + .unwrap() + .unwrap() + .public_job_id, job.public_job_id ); @@ -827,7 +795,8 @@ mod tests { vec![job.clone()] ); - data.mark_job_scheduled(&job.public_job_id).await.unwrap(); + data.mark_job_scheduled(&scheduler_job).await.unwrap(); + assert!(data.get_next_job_to_run().await.unwrap().is_none(),); let job = data.get_job(&job.public_job_id).await.unwrap().unwrap(); assert!(job.scheduled_at.is_some()); assert_eq!( @@ -837,7 +806,7 @@ mod tests { vec![job.clone()] ); - data.mark_job_finished(&job.public_job_id).await.unwrap(); + data.mark_job_finished(&scheduler_job).await.unwrap(); let job = data.get_job(&job.public_job_id).await.unwrap().unwrap(); assert!(job.finished_at.is_some()); assert_eq!( @@ -849,7 +818,11 @@ mod tests { let job2 = data.get_job(&job2_public_id).await.unwrap().unwrap(); assert_ne!(job2.public_job_id, job.public_job_id); assert_eq!( - data.get_next_job_to_run().await.unwrap().public_job_id, + data.get_next_job_to_run() + .await + .unwrap() + .unwrap() + .public_job_id, job2.public_job_id ); assert_eq!(public_id, job2.campaign_public_id); From 5e7d1cae65d06f07df6fe1ca8dee1930425af3d4 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Fri, 20 Oct 2023 01:42:54 +0530 Subject: [PATCH 11/14] feat: schedule download jobs on reqs from mCaptcha/mCaptcha --- src/api/v1/mcaptcha/hooks.rs | 75 +++++++++++++++++------------------- src/main.rs | 9 ++++- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/src/api/v1/mcaptcha/hooks.rs b/src/api/v1/mcaptcha/hooks.rs index 5474bce..4f123de 100644 --- a/src/api/v1/mcaptcha/hooks.rs +++ b/src/api/v1/mcaptcha/hooks.rs @@ -18,10 +18,10 @@ use actix_web::web::ServiceConfig; use actix_web::{web, HttpResponse, Responder}; use serde::{Deserialize, Serialize}; use url::Url; +use uuid::Uuid; use crate::api::v1::ROUTES; use crate::errors::*; -use crate::mcaptcha::Secret; use crate::AppData; pub fn services(cfg: &mut ServiceConfig) { @@ -62,6 +62,16 @@ async fn register( Ok(HttpResponse::Ok()) } +#[derive(Deserialize, Serialize, Debug, Eq, PartialEq, Clone)] +pub struct UploadJobCreated { + id: Uuid, +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq)] +pub struct Secret { + pub secret: String, +} + #[actix_web_codegen_const_routes::post(path = "ROUTES.mcaptcha.upload")] async fn upload( data: AppData, @@ -76,10 +86,8 @@ async fn upload( * 5. Download results * 6. Update sync point */ - let url = data - .mcaptcha_authenticate_and_get_url(&payload.secret) - .await?; - let campaign_str = campaign.to_string(); + data.mcaptcha_authenticate(&payload.secret).await?; + // let campaign_str = campaign.to_string(); if !data .mcaptcha_campaign_is_registered(&campaign, &payload.secret) @@ -89,37 +97,11 @@ async fn upload( .await?; } - let checkpoint = data - .mcaptcha_get_checkpoint(&campaign, &payload.secret) - .await?; - const LIMIT: usize = 50; - let mut page = 1 + (checkpoint / LIMIT); - loop { - let mut res = data - .mcaptcha - .download_benchmarks(url.clone(), &campaign_str, page) - .await?; - let skip = checkpoint - ((page - 1) * LIMIT); - let new_records = res.len() - skip as usize; - let mut skip = skip as isize; - for r in res.drain(0..) { - if skip > 0 { - skip -= 1; - continue; - } - data.mcaptcha_insert_analytics(&campaign, &payload.secret, &r) - .await?; - } - data.mcaptcha_set_checkpoint(&campaign, &payload.secret, new_records) - .await?; + let res = UploadJobCreated { + id: data.add_job(&campaign).await?, + }; - page += 1; - if res.len() < LIMIT { - break; - } - } - - Ok(HttpResponse::Ok()) + Ok(HttpResponse::Created().json(res)) } #[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq)] @@ -145,16 +127,13 @@ async fn download( #[cfg(test)] mod tests { - use crate::api::v1::bench::Submission; - use crate::api::v1::bench::SubmissionType; use crate::api::v1::get_random; - use crate::errors::*; use crate::mcaptcha::PerformanceAnalytics; use crate::mcaptcha::Secret; use crate::tests::*; use crate::*; - use actix_web::{http::header, test}; + use actix_web::test; #[actix_rt::test] async fn mcaptcha_hooks_work() { @@ -166,6 +145,11 @@ mod tests { let (data, client) = get_test_data_with_mcaptcha_client().await; let app = get_app!(data).await; + let mcaptcha_downloader = + crate::mcaptcha::MCaptchaDownloader::new(AppData::new(data.clone())); + let (mcaptcha_downloader_killer, mcaptcha_downloader_job) = + mcaptcha_downloader.start_job().await.unwrap(); + if data .mcaptcha_url_exists(&mcaptcha_instance_str) .await @@ -241,7 +225,16 @@ mod tests { .to_request(), ) .await; - assert_eq!(resp.status(), StatusCode::OK); + assert_eq!(resp.status(), StatusCode::CREATED); + let job: super::UploadJobCreated = test::read_body_json(resp).await; + loop { + if data.get_job(&job.id).await.unwrap().unwrap().state + == *crate::db::JOB_STATE_FINISH + { + break; + } + tokio::time::sleep(std::time::Duration::new(1, 0)).await; + } let public_id = data .mcaptcha_get_campaign_public_id(&campaign_id, &secret) @@ -271,5 +264,7 @@ mod tests { let resp: Vec = test::read_body_json(resp).await; assert_eq!(resp.len(), 2); assert_eq!(resp, got); + mcaptcha_downloader_killer.send(()).unwrap(); + mcaptcha_downloader_job.await.unwrap(); } } diff --git a/src/main.rs b/src/main.rs index 8666d5c..09575ed 100644 --- a/src/main.rs +++ b/src/main.rs @@ -99,6 +99,10 @@ async fn main() -> std::io::Result<()> { let (archive_kiler, archive_job) = arch.init_archive_job(data.clone()).await.unwrap(); + let mcaptcha_downloader = mcaptcha::MCaptchaDownloader::new(data.clone()); + let (mcaptcha_downloader_killer, mcaptcha_downloader_job) = + mcaptcha_downloader.start_job().await.unwrap(); + let ip = settings.server.get_ip(); println!("Starting server on: http://{}", ip); @@ -128,8 +132,9 @@ async fn main() -> std::io::Result<()> { .await .unwrap(); - archive_kiler.send(true).unwrap(); - archive_job.await; + let _ = mcaptcha_downloader_killer.send(()); + let _ = archive_kiler.send(true); + let _ = tokio::join!(archive_job, mcaptcha_downloader_job); Ok(()) } From dfa83b1031e379239b65559efc8de62f77e88d32 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Fri, 20 Oct 2023 01:46:58 +0530 Subject: [PATCH 12/14] feat: offline sqlx data --- ...e369f9ddc703b8fd3251c6baa52b60c98a1d.json} | 5 ++- ...c0d37bbbb539146f93c6fba24ffd80ad1485.json} | 5 ++- ...efc5062a25eb0ada991f04197189aa6dcb2c4.json | 22 ------------ ...3773274ebfbf9dc204d89609cdff1ebc0335.json} | 5 ++- ...fe551bc57096826119ad914cc1ef2d9ccb79d.json | 28 --------------- ...6e4e52f7886f0164d05ccd3f1818a2a70cf67.json | 34 +++++++++++++++++++ src/api/v1/mcaptcha/hooks.rs | 2 +- 7 files changed, 41 insertions(+), 60 deletions(-) rename .sqlx/{query-f5c0480ed2829c3a3dbe7a8da2d95b43076b0dd18fe5137d7ba87bc8e56102fe.json => query-05b7fe6d93a4c988e9eae32f4a57e369f9ddc703b8fd3251c6baa52b60c98a1d.json} (62%) rename .sqlx/{query-3eb4799231e726cf566f81d6ec7bd30a5c75215dbc3179fae08a106388341c3f.json => query-163a1ab861234bbf52b1b1c03bbac0d37bbbb539146f93c6fba24ffd80ad1485.json} (50%) delete mode 100644 .sqlx/query-677376147c1346f9136c5d60914efc5062a25eb0ada991f04197189aa6dcb2c4.json rename .sqlx/{query-997a3d0ce396d48a1804b6272124f0d8eed367916b28c240551b97aebcc9eb36.json => query-6c8fda20aa4a9174a5b008032d493773274ebfbf9dc204d89609cdff1ebc0335.json} (64%) delete mode 100644 .sqlx/query-8be51483900058d0bcc4c121440fe551bc57096826119ad914cc1ef2d9ccb79d.json create mode 100644 .sqlx/query-d7a099c6f381fd02ad6a114b0146e4e52f7886f0164d05ccd3f1818a2a70cf67.json diff --git a/.sqlx/query-f5c0480ed2829c3a3dbe7a8da2d95b43076b0dd18fe5137d7ba87bc8e56102fe.json b/.sqlx/query-05b7fe6d93a4c988e9eae32f4a57e369f9ddc703b8fd3251c6baa52b60c98a1d.json similarity index 62% rename from .sqlx/query-f5c0480ed2829c3a3dbe7a8da2d95b43076b0dd18fe5137d7ba87bc8e56102fe.json rename to .sqlx/query-05b7fe6d93a4c988e9eae32f4a57e369f9ddc703b8fd3251c6baa52b60c98a1d.json index dc44005..b043c11 100644 --- a/.sqlx/query-f5c0480ed2829c3a3dbe7a8da2d95b43076b0dd18fe5137d7ba87bc8e56102fe.json +++ b/.sqlx/query-05b7fe6d93a4c988e9eae32f4a57e369f9ddc703b8fd3251c6baa52b60c98a1d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n synced_till\n FROM\n survey_mcaptcha_campaign\n WHERE \n campaign_id = $1\n AND\n url_id = (\n SELECT ID FROM survey_mcaptcha_hostname WHERE secret = $2\n );", + "query": "SELECT\n synced_till\n FROM\n survey_mcaptcha_campaign\n WHERE \n campaign_id = $1;", "describe": { "columns": [ { @@ -11,7 +11,6 @@ ], "parameters": { "Left": [ - "Text", "Text" ] }, @@ -19,5 +18,5 @@ false ] }, - "hash": "f5c0480ed2829c3a3dbe7a8da2d95b43076b0dd18fe5137d7ba87bc8e56102fe" + "hash": "05b7fe6d93a4c988e9eae32f4a57e369f9ddc703b8fd3251c6baa52b60c98a1d" } diff --git a/.sqlx/query-3eb4799231e726cf566f81d6ec7bd30a5c75215dbc3179fae08a106388341c3f.json b/.sqlx/query-163a1ab861234bbf52b1b1c03bbac0d37bbbb539146f93c6fba24ffd80ad1485.json similarity index 50% rename from .sqlx/query-3eb4799231e726cf566f81d6ec7bd30a5c75215dbc3179fae08a106388341c3f.json rename to .sqlx/query-163a1ab861234bbf52b1b1c03bbac0d37bbbb539146f93c6fba24ffd80ad1485.json index c3fa03f..b94a89b 100644 --- a/.sqlx/query-3eb4799231e726cf566f81d6ec7bd30a5c75215dbc3179fae08a106388341c3f.json +++ b/.sqlx/query-163a1ab861234bbf52b1b1c03bbac0d37bbbb539146f93c6fba24ffd80ad1485.json @@ -1,16 +1,15 @@ { "db_name": "PostgreSQL", - "query": "UPDATE\n survey_mcaptcha_campaign\n SET\n synced_till = $1\n WHERE \n campaign_id = $2\n AND\n url_id = (\n SELECT ID FROM survey_mcaptcha_hostname WHERE secret = $3\n )\n ", + "query": "UPDATE\n survey_mcaptcha_campaign\n SET\n synced_till = $1\n WHERE \n campaign_id = $2; ", "describe": { "columns": [], "parameters": { "Left": [ "Int4", - "Text", "Text" ] }, "nullable": [] }, - "hash": "3eb4799231e726cf566f81d6ec7bd30a5c75215dbc3179fae08a106388341c3f" + "hash": "163a1ab861234bbf52b1b1c03bbac0d37bbbb539146f93c6fba24ffd80ad1485" } diff --git a/.sqlx/query-677376147c1346f9136c5d60914efc5062a25eb0ada991f04197189aa6dcb2c4.json b/.sqlx/query-677376147c1346f9136c5d60914efc5062a25eb0ada991f04197189aa6dcb2c4.json deleted file mode 100644 index 1058331..0000000 --- a/.sqlx/query-677376147c1346f9136c5d60914efc5062a25eb0ada991f04197189aa6dcb2c4.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n url\n FROM\n survey_mcaptcha_hostname\n WHERE\n secret = $1; ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "url", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false - ] - }, - "hash": "677376147c1346f9136c5d60914efc5062a25eb0ada991f04197189aa6dcb2c4" -} diff --git a/.sqlx/query-997a3d0ce396d48a1804b6272124f0d8eed367916b28c240551b97aebcc9eb36.json b/.sqlx/query-6c8fda20aa4a9174a5b008032d493773274ebfbf9dc204d89609cdff1ebc0335.json similarity index 64% rename from .sqlx/query-997a3d0ce396d48a1804b6272124f0d8eed367916b28c240551b97aebcc9eb36.json rename to .sqlx/query-6c8fda20aa4a9174a5b008032d493773274ebfbf9dc204d89609cdff1ebc0335.json index c3320c6..148d6d8 100644 --- a/.sqlx/query-997a3d0ce396d48a1804b6272124f0d8eed367916b28c240551b97aebcc9eb36.json +++ b/.sqlx/query-6c8fda20aa4a9174a5b008032d493773274ebfbf9dc204d89609cdff1ebc0335.json @@ -1,11 +1,10 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO\n survey_mcaptcha_analytics (\n campaign_id, time, difficulty_factor, worker_type\n )\n VALUES ((\n SELECT\n ID\n FROM\n survey_mcaptcha_campaign\n WHERE \n campaign_id = $1\n AND\n url_id = (\n SELECT ID FROM survey_mcaptcha_hostname WHERE secret = $2\n )\n ), $3, $4, $5\n );", + "query": "INSERT INTO\n survey_mcaptcha_analytics (\n campaign_id, time, difficulty_factor, worker_type\n )\n VALUES ((\n SELECT\n ID\n FROM\n survey_mcaptcha_campaign\n WHERE \n campaign_id = $1\n ), $2, $3, $4\n );", "describe": { "columns": [], "parameters": { "Left": [ - "Text", "Text", "Int4", "Int4", @@ -14,5 +13,5 @@ }, "nullable": [] }, - "hash": "997a3d0ce396d48a1804b6272124f0d8eed367916b28c240551b97aebcc9eb36" + "hash": "6c8fda20aa4a9174a5b008032d493773274ebfbf9dc204d89609cdff1ebc0335" } diff --git a/.sqlx/query-8be51483900058d0bcc4c121440fe551bc57096826119ad914cc1ef2d9ccb79d.json b/.sqlx/query-8be51483900058d0bcc4c121440fe551bc57096826119ad914cc1ef2d9ccb79d.json deleted file mode 100644 index 3987ac3..0000000 --- a/.sqlx/query-8be51483900058d0bcc4c121440fe551bc57096826119ad914cc1ef2d9ccb79d.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n survey_mcaptcha_campaign.campaign_id,\n survey_mcaptcha_upload_jobs.public_id\n FROM\n survey_mcaptcha_campaign\n INNER JOIN\n survey_mcaptcha_upload_jobs\n ON\n survey_mcaptcha_upload_jobs.campaign_id = survey_mcaptcha_campaign.ID\n WHERE\n survey_mcaptcha_upload_jobs.job_state = (\n SELECT ID FROM survey_mcaptcha_upload_job_states WHERE name = $1\n )\n AND\n survey_mcaptcha_upload_jobs.finished_at is NULL\n AND\n survey_mcaptcha_upload_jobs.scheduled_at is NULL\n ORDER BY created_at ASC;", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "campaign_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "public_id", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false - ] - }, - "hash": "8be51483900058d0bcc4c121440fe551bc57096826119ad914cc1ef2d9ccb79d" -} diff --git a/.sqlx/query-d7a099c6f381fd02ad6a114b0146e4e52f7886f0164d05ccd3f1818a2a70cf67.json b/.sqlx/query-d7a099c6f381fd02ad6a114b0146e4e52f7886f0164d05ccd3f1818a2a70cf67.json new file mode 100644 index 0000000..9200ace --- /dev/null +++ b/.sqlx/query-d7a099c6f381fd02ad6a114b0146e4e52f7886f0164d05ccd3f1818a2a70cf67.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n survey_mcaptcha_campaign.campaign_id,\n survey_mcaptcha_upload_jobs.public_id,\n survey_mcaptcha_hostname.url\n FROM\n survey_mcaptcha_campaign\n INNER JOIN\n survey_mcaptcha_upload_jobs\n ON\n survey_mcaptcha_upload_jobs.campaign_id = survey_mcaptcha_campaign.ID\n INNER JOIN\n survey_mcaptcha_hostname\n ON\n survey_mcaptcha_hostname.ID = survey_mcaptcha_campaign.url_id\n WHERE\n survey_mcaptcha_upload_jobs.job_state = (\n SELECT ID FROM survey_mcaptcha_upload_job_states WHERE name = $1\n )\n AND\n survey_mcaptcha_upload_jobs.finished_at is NULL\n AND\n survey_mcaptcha_upload_jobs.scheduled_at is NULL\n ORDER BY created_at ASC;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "campaign_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "public_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "url", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "d7a099c6f381fd02ad6a114b0146e4e52f7886f0164d05ccd3f1818a2a70cf67" +} diff --git a/src/api/v1/mcaptcha/hooks.rs b/src/api/v1/mcaptcha/hooks.rs index 4f123de..6c40d5a 100644 --- a/src/api/v1/mcaptcha/hooks.rs +++ b/src/api/v1/mcaptcha/hooks.rs @@ -129,7 +129,7 @@ async fn download( mod tests { use crate::api::v1::get_random; use crate::mcaptcha::PerformanceAnalytics; - use crate::mcaptcha::Secret; + use super::Secret; use crate::tests::*; use crate::*; From f17e38c53184b098ca7758ba6cc62b86d20f0557 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Fri, 20 Oct 2023 02:38:14 +0530 Subject: [PATCH 13/14] fix: dont use mod db while migrating with tests-migrate --- src/api/v1/mcaptcha/hooks.rs | 11 ++++++----- src/tests-migrate.rs | 5 +---- src/tests.rs | 11 ++++++----- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/api/v1/mcaptcha/hooks.rs b/src/api/v1/mcaptcha/hooks.rs index 6c40d5a..42a3aa4 100644 --- a/src/api/v1/mcaptcha/hooks.rs +++ b/src/api/v1/mcaptcha/hooks.rs @@ -127,9 +127,9 @@ async fn download( #[cfg(test)] mod tests { + use super::Secret; use crate::api::v1::get_random; use crate::mcaptcha::PerformanceAnalytics; - use super::Secret; use crate::tests::*; use crate::*; @@ -228,11 +228,12 @@ mod tests { assert_eq!(resp.status(), StatusCode::CREATED); let job: super::UploadJobCreated = test::read_body_json(resp).await; loop { - if data.get_job(&job.id).await.unwrap().unwrap().state - == *crate::db::JOB_STATE_FINISH - { - break; + if let Some(job) = data.get_job(&job.id).await.unwrap() { + if job.state == *crate::db::JOB_STATE_FINISH { + break; + } } + tokio::time::sleep(std::time::Duration::new(1, 0)).await; } diff --git a/src/tests-migrate.rs b/src/tests-migrate.rs index 338f6e9..b21b85c 100644 --- a/src/tests-migrate.rs +++ b/src/tests-migrate.rs @@ -18,7 +18,6 @@ use std::env; use sqlx::postgres::PgPoolOptions; -mod db; mod settings; pub use settings::Settings; @@ -41,9 +40,7 @@ async fn main() { } } - db::migrate_db(&db).await.unwrap(); - - // sqlx::migrate!("./migrations/").run(&db).await.unwrap(); + sqlx::migrate!("./migrations/").run(&db).await.unwrap(); } fn build() { diff --git a/src/tests.rs b/src/tests.rs index 286b8d0..9c168b0 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -48,10 +48,9 @@ pub async fn get_test_data_with_mcaptcha_client( settings.publish.dir = tmp_dir.join("base_path").to_str().unwrap().into(); settings.allow_registration = true; let test_mcaptcha = crate::mcaptcha::tests::TestClient::default(); - ( - Data::new(settings, Box::new(test_mcaptcha.clone())).await, - test_mcaptcha, - ) + let data = Data::new(settings, Box::new(test_mcaptcha.clone())).await; + db::migrate_db(&data.db).await.unwrap(); + (data, test_mcaptcha) } pub async fn get_test_data() -> Arc { @@ -60,7 +59,9 @@ pub async fn get_test_data() -> Arc { settings.publish.dir = tmp_dir.join("base_path").to_str().unwrap().into(); settings.allow_registration = true; let test_mcaptcha = Box::new(crate::mcaptcha::tests::TestClient::default()); - Data::new(settings, test_mcaptcha).await + let data = Data::new(settings, test_mcaptcha).await; + db::migrate_db(&data.db).await.unwrap(); + data } #[macro_export] From be03da096e483f78e309a94d402d7c9baca4381a Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Fri, 20 Oct 2023 03:07:44 +0530 Subject: [PATCH 14/14] fix: run tests sequentially to avoid race cond --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index af67ab7..17d3a6e 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ sqlx-offline-data: ## prepare sqlx offline data test: frontend ## Run tests echo 'static/' && tree static || true echo 'tree/' && tree assets || true - cargo test --all-features --no-fail-fast + cargo test --all-features --no-fail-fast -j 1 xml-test-coverage: migrate ## Generate cobertura.xml test coverage cargo tarpaulin -t 1200 --out Xml