feat: load libmcaptcha and add state endpoint

.gitignore

@@ -1 +1,158 @@
 /target
+.env
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+keys
+htmlcov/
+tmp/
+static/

Cargo.lock

@@ -810,14 +810,14 @@ dependencies = [
  "byteorder",
  "clap 4.3.0",
  "config",
- "derive_builder",
+ "derive_builder 0.11.2",
  "derive_more",
  "futures-util",
  "lazy_static",
  "libmcaptcha",
  "maplit",
  "openraft",
- "pretty_env_logger",
+ "pretty_env_logger 0.4.0",
  "reqwest",
  "serde 1.0.163",
  "serde_json",
@@ -835,7 +835,16 @@ version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d07adf7be193b71cc36b193d0f5fe60b918a3a9db4dad0449f57bcfd519704a3"
 dependencies = [
- "derive_builder_macro",
+ "derive_builder_macro 0.11.2",
+]
+
+[[package]]
+name = "derive_builder"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8"
+dependencies = [
+ "derive_builder_macro 0.12.0",
 ]
 
 [[package]]
@@ -850,13 +859,35 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "derive_builder_core"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f"
+dependencies = [
+ "darling",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "derive_builder_macro"
 version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f0314b72bed045f3a68671b3c86328386762c93f82d98c65c3cb5e5f573dd68"
 dependencies = [
- "derive_builder_core",
+ "derive_builder_core 0.11.2",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "derive_builder_macro"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e"
+dependencies = [
+ "derive_builder_core 0.12.0",
  "syn 1.0.109",
 ]
 
@@ -941,7 +972,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36"
 dependencies = [
  "atty",
- "humantime",
+ "humantime 1.3.0",
+ "log",
+ "regex",
+ "termcolor",
+]
+
+[[package]]
+name = "env_logger"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece"
+dependencies = [
+ "humantime 2.1.0",
+ "is-terminal",
  "log",
  "regex",
  "termcolor",
@@ -1287,6 +1331,12 @@ dependencies = [
  "quick-error",
 ]
 
+[[package]]
+name = "humantime"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
+
 [[package]]
 name = "hyper"
 version = "0.14.26"
@@ -1454,16 +1504,17 @@ checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1"
 
 [[package]]
 name = "libmcaptcha"
-version = "0.2.2"
+version = "0.2.4"
+source = "git+https://github.com/mCaptcha/libmcaptcha?branch=master#e3f456f35b2c9e55e0475b01b3e05d48b21fd51f"
 dependencies = [
  "actix",
  "crossbeam-channel",
- "derive_builder",
+ "derive_builder 0.12.0",
  "derive_more",
  "log",
+ "mcaptcha_pow_sha256",
  "num_cpus",
- "pow_sha256",
+ "pretty_env_logger 0.5.0",
- "pretty_env_logger",
  "rand",
  "redis",
  "serde 1.0.163",
@@ -1535,6 +1586,19 @@ dependencies = [
  "regex-automata",
 ]
 
+[[package]]
+name = "mcaptcha_pow_sha256"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da77f893cceca2fb8f47064749c8a12013a71a56a1c97adc975bf6b053d4bd51"
+dependencies = [
+ "bincode",
+ "derive_builder 0.12.0",
+ "num",
+ "serde 1.0.163",
+ "sha2",
+]
+
 [[package]]
 name = "md-5"
 version = "0.10.5"
@@ -1911,18 +1975,6 @@ version = "0.3.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
 
-[[package]]
-name = "pow_sha256"
-version = "0.3.1"
-source = "git+https://github.com/mcaptcha/pow_sha256#148f1cb70d19114d1340661a77b2b679e95715f6"
-dependencies = [
- "bincode",
- "derive_builder",
- "num",
- "serde 1.0.163",
- "sha2",
-]
-
 [[package]]
 name = "ppv-lite86"
 version = "0.2.17"
@@ -1935,7 +1987,17 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "926d36b9553851b8b0005f1275891b392ee4d2d833852c417ed025477350fb9d"
 dependencies = [
- "env_logger",
+ "env_logger 0.7.1",
+ "log",
+]
+
+[[package]]
+name = "pretty_env_logger"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "865724d4dbe39d9f3dd3b52b88d859d66bcb2d6a0acfd5ea68a65fb66d4bdc1c"
+dependencies = [
+ "env_logger 0.10.1",
  "log",
 ]
 
@@ -2036,9 +2098,9 @@ dependencies = [
 
 [[package]]
 name = "redis"
-version = "0.21.7"
+version = "0.23.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "152f3863635cbb76b73bc247845781098302c6c9ad2060e1a9a7de56840346b6"
+checksum = "4f49cdc0bb3f412bf8e7d1bd90fe1d9eb10bc5c399ba90973c14662a27b3f8ba"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -2053,8 +2115,10 @@ dependencies = [
  "r2d2",
  "rand",
  "ryu",
- "sha1 0.6.1",
+ "sha1_smol",
+ "socket2",
  "tokio",
+ "tokio-retry",
  "tokio-util",
  "url",
 ]
@@ -2859,6 +2923,17 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tokio-retry"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f"
+dependencies = [
+ "pin-project",
+ "rand",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-rustls"
 version = "0.22.0"

Cargo.toml

@@ -7,8 +7,8 @@ edition = "2021"
 
 [dependencies]
 openraft = { version = "0.8.3", features = ["serde"]}
-#libmcaptcha = { branch = "master", git = "https://github.com/mCaptcha/libmcaptcha", features = ["full"]}
+libmcaptcha = { branch = "master", git = "https://github.com/mCaptcha/libmcaptcha", features = ["full"]}
-libmcaptcha = { path="../libmcaptcha/", features = ["full"]}
+#libmcaptcha = { path="../libmcaptcha/", features = ["full"]}
 tracing = { version = "0.1.37", features = ["log"] }
 serde_json = "1.0.96"
 serde = { version = "1.0.163", features = ["derive"] }

src/lib.rs

@@ -38,7 +38,7 @@ use crate::store::DcacheResponse;
 use crate::store::DcacheStore;
 
 pub mod app;
-pub mod client;
+//pub mod client;
 pub mod network;
 pub mod store;
 
@@ -128,6 +128,7 @@ pub async fn start_example_raft_node(
             .service(management::metrics)
             // application API
             .service(api::write)
+            .service(api::state)
         //            .service(api::read)
         //            .service(api::consistent_read)
     });

src/network/api.rs

@@ -15,29 +15,17 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
+use actix_web::get;
 use actix_web::post;
 use actix_web::web;
 use actix_web::web::Data;
 use actix_web::Responder;
-use openraft::error::CheckIsLeaderError;
+use libmcaptcha::master::messages::GetInternalData;
-use openraft::error::Infallible;
-use openraft::error::RaftError;
-use openraft::BasicNode;
 use web::Json;
 
 use crate::app::DcacheApp;
 use crate::store::DcacheRequest;
-use crate::DcacheNodeId;
 
-/**
- * Application API
- *
- * This is where you place your application, you can use the example below to create your
- * API. The current implementation:
- *
- *  - `POST - /write` saves a value in a key and sync the nodes.
- *  - `POST - /read` attempt to find a value from a given key.
- */
 #[post("/write")]
 pub async fn write(
     app: Data<DcacheApp>,
@@ -46,26 +34,18 @@ pub async fn write(
     let response = app.raft.client_write(req.0).await;
     Ok(Json(response))
 }
-//    AddVisitor(AddVisitor),
+
-//    AddCaptcha(AddCaptcha),
+#[get("/state")]
-//    RenameCaptcha(RenameCaptcha),
+pub async fn state(app: Data<DcacheApp>) -> actix_web::Result<impl Responder> {
-//    RemoveCaptcha(RemoveCaptcha),
+    let sm = app.store.state_machine.read().await;
-//#[post("/post")]
+    let data = sm
-//pub async fn read(app: Data<DcacheApp>, req: Json<String>) -> actix_web::Result<impl Responder> {
+        .data
-//    let state_machine = app.store.state_machine.read().await;
+        .master
-//    let key = req.0;
+        .send(GetInternalData)
-//    let value = state_machine.data.get(&key).cloned();
+        .await
-//
+        .unwrap()
-//    let res: Result<String, Infallible> = Ok(value.unwrap_or_default());
+        .await
-//    Ok(Json(res))
+        .unwrap()
-//}
+        .unwrap();
-//
+    Ok(Json(data))
-//#[post("/visitor/add")]
+}
-//pub async fn add_visitor(app: Data<DcacheApp>, req: Json<String>) -> actix_web::Result<impl Responder> {
-//    let state_machine = app.store.state_machine.read().await;
-//    let key = req.0;
-//    let value = state_machine.data.get(&key).cloned();
-//
-//    let res: Result<String, Infallible> = Ok(value.unwrap_or_default());
-//    Ok(Json(res))
-//}

src/network/management.rs

@@ -31,13 +31,6 @@ use web::Json;
 use crate::app::DcacheApp;
 use crate::DcacheNodeId;
 
-// --- Cluster management
-
-/// Add a node as **Learner**.
-///
-/// A Learner receives log replication from the leader but does not vote.
-/// This should be done before adding a node as a member into the cluster
-/// (by calling `change-membership`)
 #[post("/add-learner")]
 pub async fn add_learner(
     app: Data<DcacheApp>,
@@ -51,7 +44,6 @@ pub async fn add_learner(
     Ok(Json(res))
 }
 
-/// Changes specified learners to members, or remove members.
 #[post("/change-membership")]
 pub async fn change_membership(
     app: Data<DcacheApp>,
@@ -61,7 +53,6 @@ pub async fn change_membership(
     Ok(Json(res))
 }
 
-/// Initialize a single-node cluster.
 #[post("/init")]
 pub async fn init(app: Data<DcacheApp>) -> actix_web::Result<impl Responder> {
     let mut nodes = BTreeMap::new();
@@ -75,7 +66,6 @@ pub async fn init(app: Data<DcacheApp>) -> actix_web::Result<impl Responder> {
     Ok(Json(res))
 }
 
-/// Get the latest metrics of the cluster
 #[get("/metrics")]
 pub async fn metrics(app: Data<DcacheApp>) -> actix_web::Result<impl Responder> {
     let metrics = app.raft.metrics().borrow().clone();

src/network/raft_network_impl.rs

@@ -1,6 +1,6 @@
 /*
  * mCaptcha - A proof of work based DoS protection system
- * Copyright © 2021 Aravinth Manivannan <realravinth@batsense.net>
+ * Copyright © 2023 Aravinth Manivannan <realravinth@batsense.net>
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as

src/store/mod.rs

@@ -61,12 +61,6 @@ use libmcaptcha::{master::embedded::master::Master as EmbeddedMaster, system::Sy
 
 pub mod system;
 
-/**
- * Here you will set the types of request that will interact with the raft nodes.
- * For example the `Set` will be used to write data (key and value) to the raft database.
- * The `AddNode` will append a new node to the current existing shared list of nodes.
- * You will want to add any request that can write data in all nodes here.
- */
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub enum DcacheRequest {
     //Set { key: String, value: String },
@@ -76,14 +70,6 @@ pub enum DcacheRequest {
     RemoveCaptcha(RemoveCaptcha),
 }
 
-/**
- * Here you will defined what type of answer you expect from reading the data of a node.
- * In this example it will return a optional value from a given key in
- * the `DcacheRequest.Set`.
- *
- * TODO: Should we explain how to create multiple `AppDataResponse`?
- *
- */
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub enum DcacheResponse {
     AddVisitorResult(Option<AddVisitorResult>),
@@ -97,16 +83,9 @@ pub enum DcacheResponse {
 pub struct DcacheSnapshot {
     pub meta: SnapshotMeta<DcacheNodeId, BasicNode>,
 
-    /// The data of the state machine at the time of this snapshot.
     pub data: Vec<u8>,
 }
 
-/**
- * Here defines a state machine of the raft, this state represents a copy of the data
- * between each node. Note that we are using `serde` to serialize the `data`, which has
- * a implementation to be serialized. Note that for this test we set both the key and
- * value as String, but you could set any type of value that has the serialization impl.
- */
 pub struct DcacheStateMachine {
     pub last_applied_log: Option<LogId<DcacheNodeId>>,
 

src/store/system.rs

@@ -19,12 +19,10 @@ use std::sync::Arc;
 
 use actix::prelude::*;
 use libmcaptcha::{
-    cache::{hashcache::HashCache, messages::VerifyCaptchaResult},
+    cache::hashcache::HashCache,
     master::embedded::master::Master,
-    master::messages::AddSiteBuilder,
+    pow::ConfigBuilder,
-    pow::{ConfigBuilder, Work},
     system::{System, SystemBuilder},
-    DefenseBuilder, LevelBuilder, MCaptchaBuilder,
 };
 
 pub fn init_system(salt: String) -> Arc<System<HashCache, Master>> {