From b4c93185c73f1a729be75d5900a1d1f23d17596d Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Sat, 27 May 2023 19:39:04 +0530 Subject: [PATCH] feat: load libmcaptcha and add state endpoint --- .gitignore | 157 +++++++++++++++++++++++++++++++ Cargo.lock | 125 +++++++++++++++++++----- Cargo.toml | 4 +- src/lib.rs | 3 +- src/network/api.rs | 54 ++++------- src/network/management.rs | 10 -- src/network/raft_network_impl.rs | 2 +- src/store/mod.rs | 21 ----- src/store/system.rs | 6 +- 9 files changed, 281 insertions(+), 101 deletions(-) diff --git a/.gitignore b/.gitignore index ea8c4bf..06d46ba 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,158 @@ /target +.env +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ +keys +htmlcov/ +tmp/ +static/ diff --git a/Cargo.lock b/Cargo.lock index 07e3bcb..f475e12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -810,14 +810,14 @@ dependencies = [ "byteorder", "clap 4.3.0", "config", - "derive_builder", + "derive_builder 0.11.2", "derive_more", "futures-util", "lazy_static", "libmcaptcha", "maplit", "openraft", - "pretty_env_logger", + "pretty_env_logger 0.4.0", "reqwest", "serde 1.0.163", "serde_json", @@ -835,7 +835,16 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d07adf7be193b71cc36b193d0f5fe60b918a3a9db4dad0449f57bcfd519704a3" dependencies = [ - "derive_builder_macro", + "derive_builder_macro 0.11.2", +] + +[[package]] +name = "derive_builder" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" +dependencies = [ + "derive_builder_macro 0.12.0", ] [[package]] @@ -850,13 +859,35 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_builder_core" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "derive_builder_macro" version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f0314b72bed045f3a68671b3c86328386762c93f82d98c65c3cb5e5f573dd68" dependencies = [ - "derive_builder_core", + "derive_builder_core 0.11.2", + "syn 1.0.109", +] + +[[package]] +name = "derive_builder_macro" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" +dependencies = [ + "derive_builder_core 0.12.0", "syn 1.0.109", ] @@ -941,7 +972,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" dependencies = [ "atty", - "humantime", + "humantime 1.3.0", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "env_logger" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece" +dependencies = [ + "humantime 2.1.0", + "is-terminal", "log", "regex", "termcolor", @@ -1287,6 +1331,12 @@ dependencies = [ "quick-error", ] +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.26" @@ -1454,16 +1504,17 @@ checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" [[package]] name = "libmcaptcha" -version = "0.2.2" +version = "0.2.4" +source = "git+https://github.com/mCaptcha/libmcaptcha?branch=master#e3f456f35b2c9e55e0475b01b3e05d48b21fd51f" dependencies = [ "actix", "crossbeam-channel", - "derive_builder", + "derive_builder 0.12.0", "derive_more", "log", + "mcaptcha_pow_sha256", "num_cpus", - "pow_sha256", - "pretty_env_logger", + "pretty_env_logger 0.5.0", "rand", "redis", "serde 1.0.163", @@ -1535,6 +1586,19 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "mcaptcha_pow_sha256" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da77f893cceca2fb8f47064749c8a12013a71a56a1c97adc975bf6b053d4bd51" +dependencies = [ + "bincode", + "derive_builder 0.12.0", + "num", + "serde 1.0.163", + "sha2", +] + [[package]] name = "md-5" version = "0.10.5" @@ -1911,18 +1975,6 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" -[[package]] -name = "pow_sha256" -version = "0.3.1" -source = "git+https://github.com/mcaptcha/pow_sha256#148f1cb70d19114d1340661a77b2b679e95715f6" -dependencies = [ - "bincode", - "derive_builder", - "num", - "serde 1.0.163", - "sha2", -] - [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1935,7 +1987,17 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "926d36b9553851b8b0005f1275891b392ee4d2d833852c417ed025477350fb9d" dependencies = [ - "env_logger", + "env_logger 0.7.1", + "log", +] + +[[package]] +name = "pretty_env_logger" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "865724d4dbe39d9f3dd3b52b88d859d66bcb2d6a0acfd5ea68a65fb66d4bdc1c" +dependencies = [ + "env_logger 0.10.1", "log", ] @@ -2036,9 +2098,9 @@ dependencies = [ [[package]] name = "redis" -version = "0.21.7" +version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "152f3863635cbb76b73bc247845781098302c6c9ad2060e1a9a7de56840346b6" +checksum = "4f49cdc0bb3f412bf8e7d1bd90fe1d9eb10bc5c399ba90973c14662a27b3f8ba" dependencies = [ "arc-swap", "async-trait", @@ -2053,8 +2115,10 @@ dependencies = [ "r2d2", "rand", "ryu", - "sha1 0.6.1", + "sha1_smol", + "socket2", "tokio", + "tokio-retry", "tokio-util", "url", ] @@ -2859,6 +2923,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-retry" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" +dependencies = [ + "pin-project", + "rand", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.22.0" diff --git a/Cargo.toml b/Cargo.toml index bfb9cd8..58c7ec9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,8 +7,8 @@ edition = "2021" [dependencies] openraft = { version = "0.8.3", features = ["serde"]} -#libmcaptcha = { branch = "master", git = "https://github.com/mCaptcha/libmcaptcha", features = ["full"]} -libmcaptcha = { path="../libmcaptcha/", features = ["full"]} +libmcaptcha = { branch = "master", git = "https://github.com/mCaptcha/libmcaptcha", features = ["full"]} +#libmcaptcha = { path="../libmcaptcha/", features = ["full"]} tracing = { version = "0.1.37", features = ["log"] } serde_json = "1.0.96" serde = { version = "1.0.163", features = ["derive"] } diff --git a/src/lib.rs b/src/lib.rs index 46e1c3a..96e021e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -38,7 +38,7 @@ use crate::store::DcacheResponse; use crate::store::DcacheStore; pub mod app; -pub mod client; +//pub mod client; pub mod network; pub mod store; @@ -128,6 +128,7 @@ pub async fn start_example_raft_node( .service(management::metrics) // application API .service(api::write) + .service(api::state) // .service(api::read) // .service(api::consistent_read) }); diff --git a/src/network/api.rs b/src/network/api.rs index b2e9797..d4d0d2e 100644 --- a/src/network/api.rs +++ b/src/network/api.rs @@ -15,29 +15,17 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +use actix_web::get; use actix_web::post; use actix_web::web; use actix_web::web::Data; use actix_web::Responder; -use openraft::error::CheckIsLeaderError; -use openraft::error::Infallible; -use openraft::error::RaftError; -use openraft::BasicNode; +use libmcaptcha::master::messages::GetInternalData; use web::Json; use crate::app::DcacheApp; use crate::store::DcacheRequest; -use crate::DcacheNodeId; -/** - * Application API - * - * This is where you place your application, you can use the example below to create your - * API. The current implementation: - * - * - `POST - /write` saves a value in a key and sync the nodes. - * - `POST - /read` attempt to find a value from a given key. - */ #[post("/write")] pub async fn write( app: Data, @@ -46,26 +34,18 @@ pub async fn write( let response = app.raft.client_write(req.0).await; Ok(Json(response)) } -// AddVisitor(AddVisitor), -// AddCaptcha(AddCaptcha), -// RenameCaptcha(RenameCaptcha), -// RemoveCaptcha(RemoveCaptcha), -//#[post("/post")] -//pub async fn read(app: Data, req: Json) -> actix_web::Result { -// let state_machine = app.store.state_machine.read().await; -// let key = req.0; -// let value = state_machine.data.get(&key).cloned(); -// -// let res: Result = Ok(value.unwrap_or_default()); -// Ok(Json(res)) -//} -// -//#[post("/visitor/add")] -//pub async fn add_visitor(app: Data, req: Json) -> actix_web::Result { -// let state_machine = app.store.state_machine.read().await; -// let key = req.0; -// let value = state_machine.data.get(&key).cloned(); -// -// let res: Result = Ok(value.unwrap_or_default()); -// Ok(Json(res)) -//} + +#[get("/state")] +pub async fn state(app: Data) -> actix_web::Result { + let sm = app.store.state_machine.read().await; + let data = sm + .data + .master + .send(GetInternalData) + .await + .unwrap() + .await + .unwrap() + .unwrap(); + Ok(Json(data)) +} diff --git a/src/network/management.rs b/src/network/management.rs index cd77ac8..9ff4205 100644 --- a/src/network/management.rs +++ b/src/network/management.rs @@ -31,13 +31,6 @@ use web::Json; use crate::app::DcacheApp; use crate::DcacheNodeId; -// --- Cluster management - -/// Add a node as **Learner**. -/// -/// A Learner receives log replication from the leader but does not vote. -/// This should be done before adding a node as a member into the cluster -/// (by calling `change-membership`) #[post("/add-learner")] pub async fn add_learner( app: Data, @@ -51,7 +44,6 @@ pub async fn add_learner( Ok(Json(res)) } -/// Changes specified learners to members, or remove members. #[post("/change-membership")] pub async fn change_membership( app: Data, @@ -61,7 +53,6 @@ pub async fn change_membership( Ok(Json(res)) } -/// Initialize a single-node cluster. #[post("/init")] pub async fn init(app: Data) -> actix_web::Result { let mut nodes = BTreeMap::new(); @@ -75,7 +66,6 @@ pub async fn init(app: Data) -> actix_web::Result { Ok(Json(res)) } -/// Get the latest metrics of the cluster #[get("/metrics")] pub async fn metrics(app: Data) -> actix_web::Result { let metrics = app.raft.metrics().borrow().clone(); diff --git a/src/network/raft_network_impl.rs b/src/network/raft_network_impl.rs index 3482f6f..11a16ae 100644 --- a/src/network/raft_network_impl.rs +++ b/src/network/raft_network_impl.rs @@ -1,6 +1,6 @@ /* * mCaptcha - A proof of work based DoS protection system - * Copyright © 2021 Aravinth Manivannan + * Copyright © 2023 Aravinth Manivannan * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as diff --git a/src/store/mod.rs b/src/store/mod.rs index de048c9..e799f79 100644 --- a/src/store/mod.rs +++ b/src/store/mod.rs @@ -61,12 +61,6 @@ use libmcaptcha::{master::embedded::master::Master as EmbeddedMaster, system::Sy pub mod system; -/** - * Here you will set the types of request that will interact with the raft nodes. - * For example the `Set` will be used to write data (key and value) to the raft database. - * The `AddNode` will append a new node to the current existing shared list of nodes. - * You will want to add any request that can write data in all nodes here. - */ #[derive(Serialize, Deserialize, Debug, Clone)] pub enum DcacheRequest { //Set { key: String, value: String }, @@ -76,14 +70,6 @@ pub enum DcacheRequest { RemoveCaptcha(RemoveCaptcha), } -/** - * Here you will defined what type of answer you expect from reading the data of a node. - * In this example it will return a optional value from a given key in - * the `DcacheRequest.Set`. - * - * TODO: Should we explain how to create multiple `AppDataResponse`? - * - */ #[derive(Serialize, Deserialize, Debug, Clone)] pub enum DcacheResponse { AddVisitorResult(Option), @@ -97,16 +83,9 @@ pub enum DcacheResponse { pub struct DcacheSnapshot { pub meta: SnapshotMeta, - /// The data of the state machine at the time of this snapshot. pub data: Vec, } -/** - * Here defines a state machine of the raft, this state represents a copy of the data - * between each node. Note that we are using `serde` to serialize the `data`, which has - * a implementation to be serialized. Note that for this test we set both the key and - * value as String, but you could set any type of value that has the serialization impl. - */ pub struct DcacheStateMachine { pub last_applied_log: Option>, diff --git a/src/store/system.rs b/src/store/system.rs index 30d223c..2eb579d 100644 --- a/src/store/system.rs +++ b/src/store/system.rs @@ -19,12 +19,10 @@ use std::sync::Arc; use actix::prelude::*; use libmcaptcha::{ - cache::{hashcache::HashCache, messages::VerifyCaptchaResult}, + cache::hashcache::HashCache, master::embedded::master::Master, - master::messages::AddSiteBuilder, - pow::{ConfigBuilder, Work}, + pow::ConfigBuilder, system::{System, SystemBuilder}, - DefenseBuilder, LevelBuilder, MCaptchaBuilder, }; pub fn init_system(salt: String) -> Arc> {