feat: load libmcaptcha and add state endpoint

This commit is contained in:
Aravinth Manivannan 2023-05-27 19:39:04 +05:30
parent efeb6f3945
commit b4c93185c7
Signed by: realaravinth
GPG key ID: F8F50389936984FF
9 changed files with 281 additions and 101 deletions

157
.gitignore vendored
View file

@ -1 +1,158 @@
/target
.env
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
keys
htmlcov/
tmp/
static/

125
Cargo.lock generated
View file

@ -810,14 +810,14 @@ dependencies = [
"byteorder",
"clap 4.3.0",
"config",
"derive_builder",
"derive_builder 0.11.2",
"derive_more",
"futures-util",
"lazy_static",
"libmcaptcha",
"maplit",
"openraft",
"pretty_env_logger",
"pretty_env_logger 0.4.0",
"reqwest",
"serde 1.0.163",
"serde_json",
@ -835,7 +835,16 @@ version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d07adf7be193b71cc36b193d0f5fe60b918a3a9db4dad0449f57bcfd519704a3"
dependencies = [
"derive_builder_macro",
"derive_builder_macro 0.11.2",
]
[[package]]
name = "derive_builder"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8"
dependencies = [
"derive_builder_macro 0.12.0",
]
[[package]]
@ -850,13 +859,35 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "derive_builder_core"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f"
dependencies = [
"darling",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "derive_builder_macro"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f0314b72bed045f3a68671b3c86328386762c93f82d98c65c3cb5e5f573dd68"
dependencies = [
"derive_builder_core",
"derive_builder_core 0.11.2",
"syn 1.0.109",
]
[[package]]
name = "derive_builder_macro"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e"
dependencies = [
"derive_builder_core 0.12.0",
"syn 1.0.109",
]
@ -941,7 +972,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36"
dependencies = [
"atty",
"humantime",
"humantime 1.3.0",
"log",
"regex",
"termcolor",
]
[[package]]
name = "env_logger"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece"
dependencies = [
"humantime 2.1.0",
"is-terminal",
"log",
"regex",
"termcolor",
@ -1287,6 +1331,12 @@ dependencies = [
"quick-error",
]
[[package]]
name = "humantime"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "hyper"
version = "0.14.26"
@ -1454,16 +1504,17 @@ checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1"
[[package]]
name = "libmcaptcha"
version = "0.2.2"
version = "0.2.4"
source = "git+https://github.com/mCaptcha/libmcaptcha?branch=master#e3f456f35b2c9e55e0475b01b3e05d48b21fd51f"
dependencies = [
"actix",
"crossbeam-channel",
"derive_builder",
"derive_builder 0.12.0",
"derive_more",
"log",
"mcaptcha_pow_sha256",
"num_cpus",
"pow_sha256",
"pretty_env_logger",
"pretty_env_logger 0.5.0",
"rand",
"redis",
"serde 1.0.163",
@ -1535,6 +1586,19 @@ dependencies = [
"regex-automata",
]
[[package]]
name = "mcaptcha_pow_sha256"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da77f893cceca2fb8f47064749c8a12013a71a56a1c97adc975bf6b053d4bd51"
dependencies = [
"bincode",
"derive_builder 0.12.0",
"num",
"serde 1.0.163",
"sha2",
]
[[package]]
name = "md-5"
version = "0.10.5"
@ -1911,18 +1975,6 @@ version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
[[package]]
name = "pow_sha256"
version = "0.3.1"
source = "git+https://github.com/mcaptcha/pow_sha256#148f1cb70d19114d1340661a77b2b679e95715f6"
dependencies = [
"bincode",
"derive_builder",
"num",
"serde 1.0.163",
"sha2",
]
[[package]]
name = "ppv-lite86"
version = "0.2.17"
@ -1935,7 +1987,17 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "926d36b9553851b8b0005f1275891b392ee4d2d833852c417ed025477350fb9d"
dependencies = [
"env_logger",
"env_logger 0.7.1",
"log",
]
[[package]]
name = "pretty_env_logger"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "865724d4dbe39d9f3dd3b52b88d859d66bcb2d6a0acfd5ea68a65fb66d4bdc1c"
dependencies = [
"env_logger 0.10.1",
"log",
]
@ -2036,9 +2098,9 @@ dependencies = [
[[package]]
name = "redis"
version = "0.21.7"
version = "0.23.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "152f3863635cbb76b73bc247845781098302c6c9ad2060e1a9a7de56840346b6"
checksum = "4f49cdc0bb3f412bf8e7d1bd90fe1d9eb10bc5c399ba90973c14662a27b3f8ba"
dependencies = [
"arc-swap",
"async-trait",
@ -2053,8 +2115,10 @@ dependencies = [
"r2d2",
"rand",
"ryu",
"sha1 0.6.1",
"sha1_smol",
"socket2",
"tokio",
"tokio-retry",
"tokio-util",
"url",
]
@ -2859,6 +2923,17 @@ dependencies = [
"tokio",
]
[[package]]
name = "tokio-retry"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f"
dependencies = [
"pin-project",
"rand",
"tokio",
]
[[package]]
name = "tokio-rustls"
version = "0.22.0"

View file

@ -7,8 +7,8 @@ edition = "2021"
[dependencies]
openraft = { version = "0.8.3", features = ["serde"]}
#libmcaptcha = { branch = "master", git = "https://github.com/mCaptcha/libmcaptcha", features = ["full"]}
libmcaptcha = { path="../libmcaptcha/", features = ["full"]}
libmcaptcha = { branch = "master", git = "https://github.com/mCaptcha/libmcaptcha", features = ["full"]}
#libmcaptcha = { path="../libmcaptcha/", features = ["full"]}
tracing = { version = "0.1.37", features = ["log"] }
serde_json = "1.0.96"
serde = { version = "1.0.163", features = ["derive"] }

View file

@ -38,7 +38,7 @@ use crate::store::DcacheResponse;
use crate::store::DcacheStore;
pub mod app;
pub mod client;
//pub mod client;
pub mod network;
pub mod store;
@ -128,6 +128,7 @@ pub async fn start_example_raft_node(
.service(management::metrics)
// application API
.service(api::write)
.service(api::state)
// .service(api::read)
// .service(api::consistent_read)
});

View file

@ -15,29 +15,17 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
use actix_web::get;
use actix_web::post;
use actix_web::web;
use actix_web::web::Data;
use actix_web::Responder;
use openraft::error::CheckIsLeaderError;
use openraft::error::Infallible;
use openraft::error::RaftError;
use openraft::BasicNode;
use libmcaptcha::master::messages::GetInternalData;
use web::Json;
use crate::app::DcacheApp;
use crate::store::DcacheRequest;
use crate::DcacheNodeId;
/**
* Application API
*
* This is where you place your application, you can use the example below to create your
* API. The current implementation:
*
* - `POST - /write` saves a value in a key and sync the nodes.
* - `POST - /read` attempt to find a value from a given key.
*/
#[post("/write")]
pub async fn write(
app: Data<DcacheApp>,
@ -46,26 +34,18 @@ pub async fn write(
let response = app.raft.client_write(req.0).await;
Ok(Json(response))
}
// AddVisitor(AddVisitor),
// AddCaptcha(AddCaptcha),
// RenameCaptcha(RenameCaptcha),
// RemoveCaptcha(RemoveCaptcha),
//#[post("/post")]
//pub async fn read(app: Data<DcacheApp>, req: Json<String>) -> actix_web::Result<impl Responder> {
// let state_machine = app.store.state_machine.read().await;
// let key = req.0;
// let value = state_machine.data.get(&key).cloned();
//
// let res: Result<String, Infallible> = Ok(value.unwrap_or_default());
// Ok(Json(res))
//}
//
//#[post("/visitor/add")]
//pub async fn add_visitor(app: Data<DcacheApp>, req: Json<String>) -> actix_web::Result<impl Responder> {
// let state_machine = app.store.state_machine.read().await;
// let key = req.0;
// let value = state_machine.data.get(&key).cloned();
//
// let res: Result<String, Infallible> = Ok(value.unwrap_or_default());
// Ok(Json(res))
//}
#[get("/state")]
pub async fn state(app: Data<DcacheApp>) -> actix_web::Result<impl Responder> {
let sm = app.store.state_machine.read().await;
let data = sm
.data
.master
.send(GetInternalData)
.await
.unwrap()
.await
.unwrap()
.unwrap();
Ok(Json(data))
}

View file

@ -31,13 +31,6 @@ use web::Json;
use crate::app::DcacheApp;
use crate::DcacheNodeId;
// --- Cluster management
/// Add a node as **Learner**.
///
/// A Learner receives log replication from the leader but does not vote.
/// This should be done before adding a node as a member into the cluster
/// (by calling `change-membership`)
#[post("/add-learner")]
pub async fn add_learner(
app: Data<DcacheApp>,
@ -51,7 +44,6 @@ pub async fn add_learner(
Ok(Json(res))
}
/// Changes specified learners to members, or remove members.
#[post("/change-membership")]
pub async fn change_membership(
app: Data<DcacheApp>,
@ -61,7 +53,6 @@ pub async fn change_membership(
Ok(Json(res))
}
/// Initialize a single-node cluster.
#[post("/init")]
pub async fn init(app: Data<DcacheApp>) -> actix_web::Result<impl Responder> {
let mut nodes = BTreeMap::new();
@ -75,7 +66,6 @@ pub async fn init(app: Data<DcacheApp>) -> actix_web::Result<impl Responder> {
Ok(Json(res))
}
/// Get the latest metrics of the cluster
#[get("/metrics")]
pub async fn metrics(app: Data<DcacheApp>) -> actix_web::Result<impl Responder> {
let metrics = app.raft.metrics().borrow().clone();

View file

@ -1,6 +1,6 @@
/*
* mCaptcha - A proof of work based DoS protection system
* Copyright © 2021 Aravinth Manivannan <realravinth@batsense.net>
* Copyright © 2023 Aravinth Manivannan <realravinth@batsense.net>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as

View file

@ -61,12 +61,6 @@ use libmcaptcha::{master::embedded::master::Master as EmbeddedMaster, system::Sy
pub mod system;
/**
* Here you will set the types of request that will interact with the raft nodes.
* For example the `Set` will be used to write data (key and value) to the raft database.
* The `AddNode` will append a new node to the current existing shared list of nodes.
* You will want to add any request that can write data in all nodes here.
*/
#[derive(Serialize, Deserialize, Debug, Clone)]
pub enum DcacheRequest {
//Set { key: String, value: String },
@ -76,14 +70,6 @@ pub enum DcacheRequest {
RemoveCaptcha(RemoveCaptcha),
}
/**
* Here you will defined what type of answer you expect from reading the data of a node.
* In this example it will return a optional value from a given key in
* the `DcacheRequest.Set`.
*
* TODO: Should we explain how to create multiple `AppDataResponse`?
*
*/
#[derive(Serialize, Deserialize, Debug, Clone)]
pub enum DcacheResponse {
AddVisitorResult(Option<AddVisitorResult>),
@ -97,16 +83,9 @@ pub enum DcacheResponse {
pub struct DcacheSnapshot {
pub meta: SnapshotMeta<DcacheNodeId, BasicNode>,
/// The data of the state machine at the time of this snapshot.
pub data: Vec<u8>,
}
/**
* Here defines a state machine of the raft, this state represents a copy of the data
* between each node. Note that we are using `serde` to serialize the `data`, which has
* a implementation to be serialized. Note that for this test we set both the key and
* value as String, but you could set any type of value that has the serialization impl.
*/
pub struct DcacheStateMachine {
pub last_applied_log: Option<LogId<DcacheNodeId>>,

View file

@ -19,12 +19,10 @@ use std::sync::Arc;
use actix::prelude::*;
use libmcaptcha::{
cache::{hashcache::HashCache, messages::VerifyCaptchaResult},
cache::hashcache::HashCache,
master::embedded::master::Master,
master::messages::AddSiteBuilder,
pow::{ConfigBuilder, Work},
pow::ConfigBuilder,
system::{System, SystemBuilder},
DefenseBuilder, LevelBuilder, MCaptchaBuilder,
};
pub fn init_system(salt: String) -> Arc<System<HashCache, Master>> {