This commit is contained in:
Aravinth Manivannan 2023-05-24 21:22:14 +05:30
commit 2aa4be8248
Signed by: realaravinth
GPG key ID: AD9F0F08E855ED88
15 changed files with 4891 additions and 0 deletions

1
.gitignore vendored Normal file
View file

@ -0,0 +1 @@
/target

3433
Cargo.lock generated Normal file

File diff suppressed because it is too large Load diff

44
Cargo.toml Normal file
View file

@ -0,0 +1,44 @@
[package]
name = "dcache"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
openraft = { version = "0.8.3", features = ["serde"]}
#libmcaptcha = { branch = "master", git = "https://github.com/mCaptcha/libmcaptcha", features = ["full"]}
libmcaptcha = { path="../libmcaptcha/", features = ["full"]}
tracing = { version = "0.1.37", features = ["log"] }
serde_json = "1.0.96"
serde = { version = "1.0.163", features = ["derive"] }
byteorder = "1.4.3"
actix-web = "4"
actix-web-httpauth = "0.8.0"
futures-util = { version = "0.3.17", default-features = false, features = ["std"] }
lazy_static = "1.4.0"
pretty_env_logger = "0.4.0"
uuid = { version = "1", features = ["v4"] }
sqlx = { version = "0.5.13", features = [ "runtime-actix-rustls", "postgres", "time", "offline" ] }
actix-web-codegen-const-routes = { version = "0.1.0", tag = "0.1.0", git = "https://github.com/realaravinth/actix-web-codegen-const-routes" }
derive_builder = "0.11.2"
config = "0.11"
derive_more = "0.99.17"
url = { version = "2.2.2", features = ["serde"]}
async-trait = "0.1.36"
clap = { version = "4.1.11", features = ["derive", "env"] }
reqwest = { version = "0.11.9", features = ["json"] }
tokio = { version = "1.0", default-features = false, features = ["sync"] }
tracing-subscriber = { version = "0.3.0", features = ["env-filter"] }
actix = "0.13.0"
[build-dependencies]
serde_json = "1"
sqlx = { version = "0.5.13", features = [ "runtime-actix-rustls", "postgres", "time", "offline"] }
[dev-dependencies]
actix-rt = "2.7.0"
base64 = "0.13.0"
anyhow = "1.0.63"
maplit = "1.0.2"

17
src/app.rs Normal file
View file

@ -0,0 +1,17 @@
use std::sync::Arc;
use openraft::Config;
use crate::ExampleNodeId;
use crate::ExampleRaft;
use crate::ExampleStore;
// Representation of an application state. This struct can be shared around to share
// instances of raft, store and more.
pub struct ExampleApp {
pub id: ExampleNodeId,
pub addr: String,
pub raft: ExampleRaft,
pub store: Arc<ExampleStore>,
pub config: Arc<Config>,
}

36
src/bin/main.rs Normal file
View file

@ -0,0 +1,36 @@
use clap::Parser;
use dcache::network::raft_network_impl::ExampleNetwork;
use dcache::start_example_raft_node;
use dcache::store::ExampleStore;
use dcache::ExampleTypeConfig;
use openraft::Raft;
use tracing_subscriber::EnvFilter;
pub type ExampleRaft = Raft<ExampleTypeConfig, ExampleNetwork, ExampleStore>;
#[derive(Parser, Clone, Debug)]
#[clap(author, version, about, long_about = None)]
pub struct Opt {
#[clap(long)]
pub id: u64,
#[clap(long)]
pub http_addr: String,
}
#[actix_web::main]
async fn main() -> std::io::Result<()> {
// Setup the logger
tracing_subscriber::fmt()
.with_target(true)
.with_thread_ids(true)
.with_level(true)
.with_ansi(false)
.with_env_filter(EnvFilter::from_default_env())
.init();
// Parse the parameters passed by arguments.
let options = Opt::parse();
start_example_raft_node(options.id, options.http_addr).await
}

227
src/client.rs Normal file
View file

@ -0,0 +1,227 @@
use std::collections::BTreeSet;
use std::sync::Arc;
use std::sync::Mutex;
use std::time::Duration;
use openraft::error::ForwardToLeader;
use openraft::error::NetworkError;
use openraft::error::RPCError;
use openraft::error::RemoteError;
use openraft::BasicNode;
use openraft::RaftMetrics;
use openraft::TryAsRef;
use reqwest::Client;
use serde::de::DeserializeOwned;
use serde::Deserialize;
use serde::Serialize;
use tokio::time::timeout;
use crate::typ;
use crate::ExampleNodeId;
use crate::ExampleRequest;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Empty {}
pub struct ExampleClient {
/// The leader node to send request to.
///
/// All traffic should be sent to the leader in a cluster.
pub leader: Arc<Mutex<(ExampleNodeId, String)>>,
pub inner: Client,
}
impl ExampleClient {
/// Create a client with a leader node id and a node manager to get node address by node id.
pub fn new(leader_id: ExampleNodeId, leader_addr: String) -> Self {
Self {
leader: Arc::new(Mutex::new((leader_id, leader_addr))),
inner: reqwest::Client::new(),
}
}
// --- Application API
/// Submit a write request to the raft cluster.
///
/// The request will be processed by raft protocol: it will be replicated to a quorum and then
/// will be applied to state machine.
///
/// The result of applying the request will be returned.
pub async fn write(
&self,
req: &ExampleRequest,
) -> Result<typ::ClientWriteResponse, typ::RPCError<typ::ClientWriteError>> {
self.send_rpc_to_leader("write", Some(req)).await
}
/// Read value by key, in an inconsistent mode.
///
/// This method may return stale value because it does not force to read on a legal leader.
pub async fn read(&self, req: &String) -> Result<String, typ::RPCError> {
self.do_send_rpc_to_leader("read", Some(req)).await
}
/// Consistent Read value by key, in an inconsistent mode.
///
/// This method MUST return consistent value or CheckIsLeaderError.
pub async fn consistent_read(
&self,
req: &String,
) -> Result<String, typ::RPCError<typ::CheckIsLeaderError>> {
self.do_send_rpc_to_leader("consistent_read", Some(req))
.await
}
// --- Cluster management API
/// Initialize a cluster of only the node that receives this request.
///
/// This is the first step to initialize a cluster.
/// With a initialized cluster, new node can be added with [`write`].
/// Then setup replication with [`add_learner`].
/// Then make the new node a member with [`change_membership`].
pub async fn init(&self) -> Result<(), typ::RPCError<typ::InitializeError>> {
self.do_send_rpc_to_leader("init", Some(&Empty {})).await
}
/// Add a node as learner.
///
/// The node to add has to exist, i.e., being added with `write(ExampleRequest::AddNode{})`
pub async fn add_learner(
&self,
req: (ExampleNodeId, String),
) -> Result<typ::ClientWriteResponse, typ::RPCError<typ::ClientWriteError>> {
self.send_rpc_to_leader("add-learner", Some(&req)).await
}
/// Change membership to the specified set of nodes.
///
/// All nodes in `req` have to be already added as learner with [`add_learner`],
/// or an error [`LearnerNotFound`] will be returned.
pub async fn change_membership(
&self,
req: &BTreeSet<ExampleNodeId>,
) -> Result<typ::ClientWriteResponse, typ::RPCError<typ::ClientWriteError>> {
self.send_rpc_to_leader("change-membership", Some(req))
.await
}
/// Get the metrics about the cluster.
///
/// Metrics contains various information about the cluster, such as current leader,
/// membership config, replication status etc.
/// See [`RaftMetrics`].
pub async fn metrics(&self) -> Result<RaftMetrics<ExampleNodeId, BasicNode>, typ::RPCError> {
self.do_send_rpc_to_leader("metrics", None::<&()>).await
}
// --- Internal methods
/// Send RPC to specified node.
///
/// It sends out a POST request if `req` is Some. Otherwise a GET request.
/// The remote endpoint must respond a reply in form of `Result<T, E>`.
/// An `Err` happened on remote will be wrapped in an [`RPCError::RemoteError`].
async fn do_send_rpc_to_leader<Req, Resp, Err>(
&self,
uri: &str,
req: Option<&Req>,
) -> Result<Resp, typ::RPCError<Err>>
where
Req: Serialize + 'static,
Resp: Serialize + DeserializeOwned,
Err: std::error::Error + Serialize + DeserializeOwned,
{
let (leader_id, url) = {
let t = self.leader.lock().unwrap();
let target_addr = &t.1;
(t.0, format!("http://{}/{}", target_addr, uri))
};
let fu = if let Some(r) = req {
tracing::debug!(
">>> client send request to {}: {}",
url,
serde_json::to_string_pretty(&r).unwrap()
);
self.inner.post(url.clone()).json(r)
} else {
tracing::debug!(">>> client send request to {}", url,);
self.inner.get(url.clone())
}
.send();
let res = timeout(Duration::from_millis(3_000), fu).await;
let resp = match res {
Ok(x) => x.map_err(|e| RPCError::Network(NetworkError::new(&e)))?,
Err(timeout_err) => {
tracing::error!("timeout {} to url: {}", timeout_err, url);
return Err(RPCError::Network(NetworkError::new(&timeout_err)));
}
};
let res: Result<Resp, typ::RaftError<Err>> = resp
.json()
.await
.map_err(|e| RPCError::Network(NetworkError::new(&e)))?;
tracing::debug!(
"<<< client recv reply from {}: {}",
url,
serde_json::to_string_pretty(&res).unwrap()
);
res.map_err(|e| RPCError::RemoteError(RemoteError::new(leader_id, e)))
}
/// Try the best to send a request to the leader.
///
/// If the target node is not a leader, a `ForwardToLeader` error will be
/// returned and this client will retry at most 3 times to contact the updated leader.
async fn send_rpc_to_leader<Req, Resp, Err>(
&self,
uri: &str,
req: Option<&Req>,
) -> Result<Resp, typ::RPCError<Err>>
where
Req: Serialize + 'static,
Resp: Serialize + DeserializeOwned,
Err: std::error::Error
+ Serialize
+ DeserializeOwned
+ TryAsRef<typ::ForwardToLeader>
+ Clone,
{
// Retry at most 3 times to find a valid leader.
let mut n_retry = 3;
loop {
let res: Result<Resp, typ::RPCError<Err>> = self.do_send_rpc_to_leader(uri, req).await;
let rpc_err = match res {
Ok(x) => return Ok(x),
Err(rpc_err) => rpc_err,
};
if let Some(ForwardToLeader {
leader_id: Some(leader_id),
leader_node: Some(leader_node),
}) = rpc_err.forward_to_leader()
{
// Update target to the new leader.
{
let mut t = self.leader.lock().unwrap();
*t = (*leader_id, leader_node.addr.clone());
}
n_retry -= 1;
if n_retry > 0 {
continue;
}
}
return Err(rpc_err);
}
}
}

121
src/lib.rs Normal file
View file

@ -0,0 +1,121 @@
#![allow(clippy::uninlined_format_args)]
use std::sync::Arc;
use actix_web::middleware;
use actix_web::middleware::Logger;
use actix_web::web::Data;
use actix_web::App;
use actix_web::HttpServer;
use openraft::BasicNode;
use openraft::Config;
use openraft::Raft;
use crate::app::ExampleApp;
use crate::network::api;
use crate::network::management;
use crate::network::raft;
use crate::network::raft_network_impl::ExampleNetwork;
use crate::store::ExampleRequest;
use crate::store::ExampleResponse;
use crate::store::ExampleStore;
pub mod app;
pub mod client;
pub mod network;
pub mod store;
pub type ExampleNodeId = u64;
openraft::declare_raft_types!(
/// Declare the type configuration for example K/V store.
pub ExampleTypeConfig: D = ExampleRequest, R = ExampleResponse, NodeId = ExampleNodeId, Node = BasicNode
);
pub type ExampleRaft = Raft<ExampleTypeConfig, ExampleNetwork, Arc<ExampleStore>>;
pub mod typ {
use openraft::BasicNode;
use crate::ExampleNodeId;
use crate::ExampleTypeConfig;
pub type RaftError<E = openraft::error::Infallible> =
openraft::error::RaftError<ExampleNodeId, E>;
pub type RPCError<E = openraft::error::Infallible> =
openraft::error::RPCError<ExampleNodeId, BasicNode, RaftError<E>>;
pub type ClientWriteError = openraft::error::ClientWriteError<ExampleNodeId, BasicNode>;
pub type CheckIsLeaderError = openraft::error::CheckIsLeaderError<ExampleNodeId, BasicNode>;
pub type ForwardToLeader = openraft::error::ForwardToLeader<ExampleNodeId, BasicNode>;
pub type InitializeError = openraft::error::InitializeError<ExampleNodeId, BasicNode>;
pub type ClientWriteResponse = openraft::raft::ClientWriteResponse<ExampleTypeConfig>;
}
pub async fn start_example_raft_node(
node_id: ExampleNodeId,
http_addr: String,
) -> std::io::Result<()> {
// Create a configuration for the raft instance.
let config = Config {
heartbeat_interval: 500,
election_timeout_min: 1500,
election_timeout_max: 3000,
..Default::default()
};
let salt = "foobar12".into();
#[cfg(release)]
todo!("set salt");
let config = Arc::new(config.validate().unwrap());
// Create a instance of where the Raft data will be stored.
let store = Arc::new(ExampleStore::new(salt));
// Create the network layer that will connect and communicate the raft instances and
// will be used in conjunction with the store created above.
let network = ExampleNetwork {};
// Create a local raft instance.
let raft = Raft::new(node_id, config.clone(), network, store.clone())
.await
.unwrap();
// Create an application that will store all the instances created above, this will
// be later used on the actix-web services.
let app = Data::new(ExampleApp {
id: node_id,
addr: http_addr.clone(),
raft,
store,
config,
});
// Start the actix-web server.
let server = HttpServer::new(move || {
App::new()
.wrap(Logger::default())
.wrap(Logger::new("%a %{User-Agent}i"))
.wrap(middleware::Compress::default())
.app_data(app.clone())
// raft internal RPC
.service(raft::append)
.service(raft::snapshot)
.service(raft::vote)
// admin API
.service(management::init)
.service(management::add_learner)
.service(management::change_membership)
.service(management::metrics)
// application API
.service(api::write)
// .service(api::read)
// .service(api::consistent_read)
});
let x = server.bind(http_addr)?;
x.run().await
}

54
src/network/api.rs Normal file
View file

@ -0,0 +1,54 @@
use actix_web::post;
use actix_web::web;
use actix_web::web::Data;
use actix_web::Responder;
use openraft::error::CheckIsLeaderError;
use openraft::error::Infallible;
use openraft::error::RaftError;
use openraft::BasicNode;
use web::Json;
use crate::app::ExampleApp;
use crate::store::ExampleRequest;
use crate::ExampleNodeId;
/**
* Application API
*
* This is where you place your application, you can use the example below to create your
* API. The current implementation:
*
* - `POST - /write` saves a value in a key and sync the nodes.
* - `POST - /read` attempt to find a value from a given key.
*/
#[post("/write")]
pub async fn write(
app: Data<ExampleApp>,
req: Json<ExampleRequest>,
) -> actix_web::Result<impl Responder> {
let response = app.raft.client_write(req.0).await;
Ok(Json(response))
}
// AddVisitor(AddVisitor),
// AddCaptcha(AddCaptcha),
// RenameCaptcha(RenameCaptcha),
// RemoveCaptcha(RemoveCaptcha),
//#[post("/post")]
//pub async fn read(app: Data<ExampleApp>, req: Json<String>) -> actix_web::Result<impl Responder> {
// let state_machine = app.store.state_machine.read().await;
// let key = req.0;
// let value = state_machine.data.get(&key).cloned();
//
// let res: Result<String, Infallible> = Ok(value.unwrap_or_default());
// Ok(Json(res))
//}
//
//#[post("/visitor/add")]
//pub async fn add_visitor(app: Data<ExampleApp>, req: Json<String>) -> actix_web::Result<impl Responder> {
// let state_machine = app.store.state_machine.read().await;
// let key = req.0;
// let value = state_machine.data.get(&key).cloned();
//
// let res: Result<String, Infallible> = Ok(value.unwrap_or_default());
// Ok(Json(res))
//}

68
src/network/management.rs Normal file
View file

@ -0,0 +1,68 @@
use std::collections::BTreeMap;
use std::collections::BTreeSet;
use actix_web::get;
use actix_web::post;
use actix_web::web;
use actix_web::web::Data;
use actix_web::Responder;
use openraft::error::Infallible;
use openraft::BasicNode;
use openraft::RaftMetrics;
use web::Json;
use crate::app::ExampleApp;
use crate::ExampleNodeId;
// --- Cluster management
/// Add a node as **Learner**.
///
/// A Learner receives log replication from the leader but does not vote.
/// This should be done before adding a node as a member into the cluster
/// (by calling `change-membership`)
#[post("/add-learner")]
pub async fn add_learner(
app: Data<ExampleApp>,
req: Json<(ExampleNodeId, String)>,
) -> actix_web::Result<impl Responder> {
let node_id = req.0 .0;
let node = BasicNode {
addr: req.0 .1.clone(),
};
let res = app.raft.add_learner(node_id, node, true).await;
Ok(Json(res))
}
/// Changes specified learners to members, or remove members.
#[post("/change-membership")]
pub async fn change_membership(
app: Data<ExampleApp>,
req: Json<BTreeSet<ExampleNodeId>>,
) -> actix_web::Result<impl Responder> {
let res = app.raft.change_membership(req.0, false).await;
Ok(Json(res))
}
/// Initialize a single-node cluster.
#[post("/init")]
pub async fn init(app: Data<ExampleApp>) -> actix_web::Result<impl Responder> {
let mut nodes = BTreeMap::new();
nodes.insert(
app.id,
BasicNode {
addr: app.addr.clone(),
},
);
let res = app.raft.initialize(nodes).await;
Ok(Json(res))
}
/// Get the latest metrics of the cluster
#[get("/metrics")]
pub async fn metrics(app: Data<ExampleApp>) -> actix_web::Result<impl Responder> {
let metrics = app.raft.metrics().borrow().clone();
let res: Result<RaftMetrics<ExampleNodeId, BasicNode>, Infallible> = Ok(metrics);
Ok(Json(res))
}

4
src/network/mod.rs Normal file
View file

@ -0,0 +1,4 @@
pub mod api;
pub mod management;
pub mod raft;
pub mod raft_network_impl;

41
src/network/raft.rs Normal file
View file

@ -0,0 +1,41 @@
use actix_web::post;
use actix_web::web;
use actix_web::web::Data;
use actix_web::Responder;
use openraft::raft::AppendEntriesRequest;
use openraft::raft::InstallSnapshotRequest;
use openraft::raft::VoteRequest;
use web::Json;
use crate::app::ExampleApp;
use crate::ExampleNodeId;
use crate::ExampleTypeConfig;
// --- Raft communication
#[post("/raft-vote")]
pub async fn vote(
app: Data<ExampleApp>,
req: Json<VoteRequest<ExampleNodeId>>,
) -> actix_web::Result<impl Responder> {
let res = app.raft.vote(req.0).await;
Ok(Json(res))
}
#[post("/raft-append")]
pub async fn append(
app: Data<ExampleApp>,
req: Json<AppendEntriesRequest<ExampleTypeConfig>>,
) -> actix_web::Result<impl Responder> {
let res = app.raft.append_entries(req.0).await;
Ok(Json(res))
}
#[post("/raft-snapshot")]
pub async fn snapshot(
app: Data<ExampleApp>,
req: Json<InstallSnapshotRequest<ExampleTypeConfig>>,
) -> actix_web::Result<impl Responder> {
let res = app.raft.install_snapshot(req.0).await;
Ok(Json(res))
}

View file

@ -0,0 +1,123 @@
use async_trait::async_trait;
use openraft::error::InstallSnapshotError;
use openraft::error::NetworkError;
use openraft::error::RPCError;
use openraft::error::RaftError;
use openraft::error::RemoteError;
use openraft::raft::AppendEntriesRequest;
use openraft::raft::AppendEntriesResponse;
use openraft::raft::InstallSnapshotRequest;
use openraft::raft::InstallSnapshotResponse;
use openraft::raft::VoteRequest;
use openraft::raft::VoteResponse;
use openraft::BasicNode;
use openraft::RaftNetwork;
use openraft::RaftNetworkFactory;
use serde::de::DeserializeOwned;
use serde::Serialize;
use crate::ExampleNodeId;
use crate::ExampleTypeConfig;
pub struct ExampleNetwork {}
impl ExampleNetwork {
pub async fn send_rpc<Req, Resp, Err>(
&self,
target: ExampleNodeId,
target_node: &BasicNode,
uri: &str,
req: Req,
) -> Result<Resp, RPCError<ExampleNodeId, BasicNode, Err>>
where
Req: Serialize,
Err: std::error::Error + DeserializeOwned,
Resp: DeserializeOwned,
{
let addr = &target_node.addr;
let url = format!("http://{}/{}", addr, uri);
tracing::debug!("send_rpc to url: {}", url);
let client = reqwest::Client::new();
tracing::debug!("client is created for: {}", url);
let resp = client
.post(url)
.json(&req)
.send()
.await
.map_err(|e| RPCError::Network(NetworkError::new(&e)))?;
tracing::debug!("client.post() is sent");
let res: Result<Resp, Err> = resp
.json()
.await
.map_err(|e| RPCError::Network(NetworkError::new(&e)))?;
res.map_err(|e| RPCError::RemoteError(RemoteError::new(target, e)))
}
}
// NOTE: This could be implemented also on `Arc<ExampleNetwork>`, but since it's empty, implemented
// directly.
#[async_trait]
impl RaftNetworkFactory<ExampleTypeConfig> for ExampleNetwork {
type Network = ExampleNetworkConnection;
async fn new_client(&mut self, target: ExampleNodeId, node: &BasicNode) -> Self::Network {
ExampleNetworkConnection {
owner: ExampleNetwork {},
target,
target_node: node.clone(),
}
}
}
pub struct ExampleNetworkConnection {
owner: ExampleNetwork,
target: ExampleNodeId,
target_node: BasicNode,
}
#[async_trait]
impl RaftNetwork<ExampleTypeConfig> for ExampleNetworkConnection {
async fn send_append_entries(
&mut self,
req: AppendEntriesRequest<ExampleTypeConfig>,
) -> Result<
AppendEntriesResponse<ExampleNodeId>,
RPCError<ExampleNodeId, BasicNode, RaftError<ExampleNodeId>>,
> {
self.owner
.send_rpc(self.target, &self.target_node, "raft-append", req)
.await
}
async fn send_install_snapshot(
&mut self,
req: InstallSnapshotRequest<ExampleTypeConfig>,
) -> Result<
InstallSnapshotResponse<ExampleNodeId>,
RPCError<ExampleNodeId, BasicNode, RaftError<ExampleNodeId, InstallSnapshotError>>,
> {
self.owner
.send_rpc(self.target, &self.target_node, "raft-snapshot", req)
.await
}
async fn send_vote(
&mut self,
req: VoteRequest<ExampleNodeId>,
) -> Result<
VoteResponse<ExampleNodeId>,
RPCError<ExampleNodeId, BasicNode, RaftError<ExampleNodeId>>,
> {
self.owner
.send_rpc(self.target, &self.target_node, "raft-vote", req)
.await
}
}

528
src/store/mod.rs Normal file
View file

@ -0,0 +1,528 @@
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::fmt::Debug;
use std::io::Cursor;
use std::ops::RangeBounds;
use std::sync::Arc;
use std::sync::Mutex;
use libmcaptcha::AddVisitorResult;
use libmcaptcha::MCaptcha;
use openraft::async_trait::async_trait;
use openraft::storage::LogState;
use openraft::storage::Snapshot;
use openraft::AnyError;
use openraft::BasicNode;
use openraft::Entry;
use openraft::EntryPayload;
use openraft::ErrorSubject;
use openraft::ErrorVerb;
use openraft::LogId;
use openraft::RaftLogReader;
use openraft::RaftSnapshotBuilder;
use openraft::RaftStorage;
use openraft::SnapshotMeta;
use openraft::StorageError;
use openraft::StorageIOError;
use openraft::StoredMembership;
use openraft::Vote;
use serde::Deserialize;
use serde::Serialize;
use sqlx::Statement;
use tokio::sync::RwLock;
use url::quirks::set_pathname;
use crate::ExampleNodeId;
use crate::ExampleTypeConfig;
use actix::prelude::*;
use libmcaptcha::master::messages::{
AddSite as AddCaptcha, AddVisitor, GetInternalData, RemoveCaptcha, Rename as RenameCaptcha,
SetInternalData,
};
use libmcaptcha::{master::embedded::master::Master as EmbeddedMaster, system::System, HashCache};
pub mod system;
/**
* Here you will set the types of request that will interact with the raft nodes.
* For example the `Set` will be used to write data (key and value) to the raft database.
* The `AddNode` will append a new node to the current existing shared list of nodes.
* You will want to add any request that can write data in all nodes here.
*/
#[derive(Serialize, Deserialize, Debug, Clone)]
pub enum ExampleRequest {
//Set { key: String, value: String },
AddVisitor(AddVisitor),
AddCaptcha(AddCaptcha),
RenameCaptcha(RenameCaptcha),
RemoveCaptcha(RemoveCaptcha),
}
/**
* Here you will defined what type of answer you expect from reading the data of a node.
* In this example it will return a optional value from a given key in
* the `ExampleRequest.Set`.
*
* TODO: Should we explain how to create multiple `AppDataResponse`?
*
*/
#[derive(Serialize, Deserialize, Debug, Clone)]
pub enum ExampleResponse {
AddVisitorResult(Option<AddVisitorResult>),
Empty,
// AddCaptchaResult, All returns ()
// RenameCaptchaResult,
// RemoveCaptchaResult,
}
#[derive(Debug)]
pub struct ExampleSnapshot {
pub meta: SnapshotMeta<ExampleNodeId, BasicNode>,
/// The data of the state machine at the time of this snapshot.
pub data: Vec<u8>,
}
/**
* Here defines a state machine of the raft, this state represents a copy of the data
* between each node. Note that we are using `serde` to serialize the `data`, which has
* a implementation to be serialized. Note that for this test we set both the key and
* value as String, but you could set any type of value that has the serialization impl.
*/
pub struct ExampleStateMachine {
pub last_applied_log: Option<LogId<ExampleNodeId>>,
pub last_membership: StoredMembership<ExampleNodeId, BasicNode>,
/// Application data.
pub data: Arc<System<HashCache, EmbeddedMaster>>,
}
#[derive(Serialize, Deserialize, Clone)]
struct PersistableStateMachine {
last_applied_log: Option<LogId<ExampleNodeId>>,
last_membership: StoredMembership<ExampleNodeId, BasicNode>,
/// Application data.
data: HashMap<String, MCaptcha>,
}
impl PersistableStateMachine {
async fn from_statemachine(m: &ExampleStateMachine) -> Self {
let internal_data = m
.data
.master
.send(GetInternalData)
.await
.unwrap()
.await
.unwrap()
.unwrap();
Self {
last_applied_log: m.last_applied_log.clone(),
last_membership: m.last_membership.clone(),
data: internal_data,
}
}
async fn to_statemachine(
self,
data: Arc<System<HashCache, EmbeddedMaster>>,
) -> ExampleStateMachine {
data.master
.send(SetInternalData {
mcaptcha: self.data,
})
.await
.unwrap();
ExampleStateMachine {
last_applied_log: self.last_applied_log,
last_membership: self.last_membership,
data,
}
}
}
pub struct ExampleStore {
last_purged_log_id: RwLock<Option<LogId<ExampleNodeId>>>,
/// The Raft log.
log: RwLock<BTreeMap<u64, Entry<ExampleTypeConfig>>>,
/// The Raft state machine.
pub state_machine: RwLock<ExampleStateMachine>,
/// The current granted vote.
vote: RwLock<Option<Vote<ExampleNodeId>>>,
snapshot_idx: Arc<Mutex<u64>>,
current_snapshot: RwLock<Option<ExampleSnapshot>>,
}
impl ExampleStore {
pub fn new(salt: String) -> Self {
let state_machine = RwLock::new(ExampleStateMachine {
last_applied_log: Default::default(),
last_membership: Default::default(),
data: system::init_system(salt),
});
Self {
last_purged_log_id: Default::default(),
log: Default::default(),
state_machine,
vote: Default::default(),
snapshot_idx: Default::default(),
current_snapshot: Default::default(),
}
}
}
#[async_trait]
impl RaftLogReader<ExampleTypeConfig> for Arc<ExampleStore> {
async fn get_log_state(
&mut self,
) -> Result<LogState<ExampleTypeConfig>, StorageError<ExampleNodeId>> {
let log = self.log.read().await;
let last = log.iter().rev().next().map(|(_, ent)| ent.log_id);
let last_purged = *self.last_purged_log_id.read().await;
let last = match last {
None => last_purged,
Some(x) => Some(x),
};
Ok(LogState {
last_purged_log_id: last_purged,
last_log_id: last,
})
}
async fn try_get_log_entries<RB: RangeBounds<u64> + Clone + Debug + Send + Sync>(
&mut self,
range: RB,
) -> Result<Vec<Entry<ExampleTypeConfig>>, StorageError<ExampleNodeId>> {
let log = self.log.read().await;
let response = log
.range(range.clone())
.map(|(_, val)| val.clone())
.collect::<Vec<_>>();
Ok(response)
}
}
#[async_trait]
impl RaftSnapshotBuilder<ExampleTypeConfig, Cursor<Vec<u8>>> for Arc<ExampleStore> {
#[tracing::instrument(level = "trace", skip(self))]
async fn build_snapshot(
&mut self,
) -> Result<Snapshot<ExampleNodeId, BasicNode, Cursor<Vec<u8>>>, StorageError<ExampleNodeId>>
{
let data;
let last_applied_log;
let last_membership;
{
// Serialize the data of the state machine.
let state_machine = self.state_machine.read().await;
let persistable_state_machine =
PersistableStateMachine::from_statemachine(&state_machine).await;
data = serde_json::to_vec(&persistable_state_machine).map_err(|e| {
StorageIOError::new(
ErrorSubject::StateMachine,
ErrorVerb::Read,
AnyError::new(&e),
)
})?;
last_applied_log = state_machine.last_applied_log;
last_membership = state_machine.last_membership.clone();
}
let snapshot_idx = {
let mut l = self.snapshot_idx.lock().unwrap();
*l += 1;
*l
};
let snapshot_id = if let Some(last) = last_applied_log {
format!("{}-{}-{}", last.leader_id, last.index, snapshot_idx)
} else {
format!("--{}", snapshot_idx)
};
let meta = SnapshotMeta {
last_log_id: last_applied_log,
last_membership,
snapshot_id,
};
let snapshot = ExampleSnapshot {
meta: meta.clone(),
data: data.clone(),
};
{
let mut current_snapshot = self.current_snapshot.write().await;
*current_snapshot = Some(snapshot);
}
Ok(Snapshot {
meta,
snapshot: Box::new(Cursor::new(data)),
})
}
}
#[async_trait]
impl RaftStorage<ExampleTypeConfig> for Arc<ExampleStore> {
type SnapshotData = Cursor<Vec<u8>>;
type LogReader = Self;
type SnapshotBuilder = Self;
#[tracing::instrument(level = "trace", skip(self))]
async fn save_vote(
&mut self,
vote: &Vote<ExampleNodeId>,
) -> Result<(), StorageError<ExampleNodeId>> {
let mut v = self.vote.write().await;
*v = Some(*vote);
Ok(())
}
async fn read_vote(
&mut self,
) -> Result<Option<Vote<ExampleNodeId>>, StorageError<ExampleNodeId>> {
Ok(*self.vote.read().await)
}
#[tracing::instrument(level = "trace", skip(self, entries))]
async fn append_to_log(
&mut self,
entries: &[&Entry<ExampleTypeConfig>],
) -> Result<(), StorageError<ExampleNodeId>> {
let mut log = self.log.write().await;
for entry in entries {
log.insert(entry.log_id.index, (*entry).clone());
}
Ok(())
}
#[tracing::instrument(level = "debug", skip(self))]
async fn delete_conflict_logs_since(
&mut self,
log_id: LogId<ExampleNodeId>,
) -> Result<(), StorageError<ExampleNodeId>> {
tracing::debug!("delete_log: [{:?}, +oo)", log_id);
let mut log = self.log.write().await;
let keys = log
.range(log_id.index..)
.map(|(k, _v)| *k)
.collect::<Vec<_>>();
for key in keys {
log.remove(&key);
}
Ok(())
}
#[tracing::instrument(level = "debug", skip(self))]
async fn purge_logs_upto(
&mut self,
log_id: LogId<ExampleNodeId>,
) -> Result<(), StorageError<ExampleNodeId>> {
tracing::debug!("delete_log: [{:?}, +oo)", log_id);
{
let mut ld = self.last_purged_log_id.write().await;
assert!(*ld <= Some(log_id));
*ld = Some(log_id);
}
{
let mut log = self.log.write().await;
let keys = log
.range(..=log_id.index)
.map(|(k, _v)| *k)
.collect::<Vec<_>>();
for key in keys {
log.remove(&key);
}
}
Ok(())
}
async fn last_applied_state(
&mut self,
) -> Result<
(
Option<LogId<ExampleNodeId>>,
StoredMembership<ExampleNodeId, BasicNode>,
),
StorageError<ExampleNodeId>,
> {
let state_machine = self.state_machine.read().await;
Ok((
state_machine.last_applied_log,
state_machine.last_membership.clone(),
))
}
#[tracing::instrument(level = "trace", skip(self, entries))]
async fn apply_to_state_machine(
&mut self,
entries: &[&Entry<ExampleTypeConfig>],
) -> Result<Vec<ExampleResponse>, StorageError<ExampleNodeId>> {
let mut res = Vec::with_capacity(entries.len());
let mut sm = self.state_machine.write().await;
for entry in entries {
tracing::debug!(%entry.log_id, "replicate to sm");
sm.last_applied_log = Some(entry.log_id);
match entry.payload {
EntryPayload::Blank => res.push(ExampleResponse::Empty),
EntryPayload::Normal(ref req) => match req {
ExampleRequest::AddVisitor(msg) => {
let sm = self.state_machine.read().await;
res.push(ExampleResponse::AddVisitorResult(
sm.data
.master
.send(msg.clone())
.await
.unwrap()
.await
.unwrap()
.unwrap(),
));
}
ExampleRequest::AddCaptcha(msg) => {
let sm = self.state_machine.read().await;
sm.data
.master
.send(msg.clone())
.await
.unwrap()
.await
.unwrap()
.unwrap();
res.push(ExampleResponse::Empty);
}
ExampleRequest::RenameCaptcha(msg) => {
let sm = self.state_machine.read().await;
sm.data
.master
.send(msg.clone())
.await
.unwrap()
.await
.unwrap()
.unwrap();
res.push(ExampleResponse::Empty);
}
ExampleRequest::RemoveCaptcha(msg) => {
let sm = self.state_machine.read().await;
sm.data
.master
.send(msg.clone())
.await
.unwrap()
.await
.unwrap()
.unwrap();
res.push(ExampleResponse::Empty);
}
},
EntryPayload::Membership(ref mem) => {
sm.last_membership = StoredMembership::new(Some(entry.log_id), mem.clone());
res.push(ExampleResponse::Empty)
}
};
}
Ok(res)
}
#[tracing::instrument(level = "trace", skip(self))]
async fn begin_receiving_snapshot(
&mut self,
) -> Result<Box<Self::SnapshotData>, StorageError<ExampleNodeId>> {
Ok(Box::new(Cursor::new(Vec::new())))
}
#[tracing::instrument(level = "trace", skip(self, snapshot))]
async fn install_snapshot(
&mut self,
meta: &SnapshotMeta<ExampleNodeId, BasicNode>,
snapshot: Box<Self::SnapshotData>,
) -> Result<(), StorageError<ExampleNodeId>> {
tracing::info!(
{ snapshot_size = snapshot.get_ref().len() },
"decoding snapshot for installation"
);
let new_snapshot = ExampleSnapshot {
meta: meta.clone(),
data: snapshot.into_inner(),
};
// Update the state machine.
{
let updated_persistable_state_machine: PersistableStateMachine =
serde_json::from_slice(&new_snapshot.data).map_err(|e| {
StorageIOError::new(
ErrorSubject::Snapshot(new_snapshot.meta.signature()),
ErrorVerb::Read,
AnyError::new(&e),
)
})?;
let mut state_machine = self.state_machine.write().await;
let updated_state_machine = updated_persistable_state_machine
.to_statemachine(state_machine.data.clone())
.await;
*state_machine = updated_state_machine;
}
// Update current snapshot.
let mut current_snapshot = self.current_snapshot.write().await;
*current_snapshot = Some(new_snapshot);
Ok(())
}
#[tracing::instrument(level = "trace", skip(self))]
async fn get_current_snapshot(
&mut self,
) -> Result<
Option<Snapshot<ExampleNodeId, BasicNode, Self::SnapshotData>>,
StorageError<ExampleNodeId>,
> {
match &*self.current_snapshot.read().await {
Some(snapshot) => {
let data = snapshot.data.clone();
Ok(Some(Snapshot {
meta: snapshot.meta.clone(),
snapshot: Box::new(Cursor::new(data)),
}))
}
None => Ok(None),
}
}
async fn get_log_reader(&mut self) -> Self::LogReader {
self.clone()
}
async fn get_snapshot_builder(&mut self) -> Self::SnapshotBuilder {
self.clone()
}
}

28
src/store/system.rs Normal file
View file

@ -0,0 +1,28 @@
use std::sync::Arc;
use actix::prelude::*;
use libmcaptcha::{
cache::{hashcache::HashCache, messages::VerifyCaptchaResult},
master::embedded::master::Master,
master::messages::AddSiteBuilder,
pow::{ConfigBuilder, Work},
system::{System, SystemBuilder},
DefenseBuilder, LevelBuilder, MCaptchaBuilder,
};
pub fn init_system(salt: String) -> Arc<System<HashCache, Master>> {
let cache = HashCache::default().start();
let pow = ConfigBuilder::default().salt(salt).build().unwrap();
let master = Master::new(5).start();
let system = SystemBuilder::default()
.master(master)
.cache(cache)
.pow(pow.clone())
.runners(4)
.queue_length(2000)
.build();
Arc::new(system)
}

166
test-cluster.sh Executable file
View file

@ -0,0 +1,166 @@
#!/bin/sh
set -o errexit
cargo build
kill() {
if [ "$(uname)" = "Darwin" ]; then
SERVICE='raft-key-value'
if pgrep -xq -- "${SERVICE}"; then
pkill -f "${SERVICE}"
fi
else
set +e # killall will error if finds no process to kill
killall raft-key-value
set -e
fi
}
rpc() {
local uri=$1
local body="$2"
echo '---'" rpc(:$uri, $body)"
{
if [ ".$body" = "." ]; then
time curl --silent "127.0.0.1:$uri"
else
time curl --silent "127.0.0.1:$uri" -H "Content-Type: application/json" -d "$body"
fi
} | {
if type jq > /dev/null 2>&1; then
jq
else
cat
fi
}
echo
echo
}
export RUST_LOG=trace
echo "Killing all running raft-key-value"
kill
sleep 1
echo "Start 3 uninitialized raft-key-value servers..."
nohup ./target/debug/raft-key-value --id 1 --http-addr 127.0.0.1:21001 > n1.log &
sleep 1
echo "Server 1 started"
nohup ./target/debug/raft-key-value --id 2 --http-addr 127.0.0.1:21002 > n2.log &
sleep 1
echo "Server 2 started"
nohup ./target/debug/raft-key-value --id 3 --http-addr 127.0.0.1:21003 > n3.log &
sleep 1
echo "Server 3 started"
sleep 1
echo "Initialize server 1 as a single-node cluster"
sleep 2
echo
rpc 21001/init '{}'
echo "Server 1 is a leader now"
sleep 2
echo "Get metrics from the leader"
sleep 2
echo
rpc 21001/metrics
sleep 1
echo "Adding node 2 and node 3 as learners, to receive log from leader node 1"
sleep 1
echo
rpc 21001/add-learner '[2, "127.0.0.1:21002"]'
echo "Node 2 added as learner"
sleep 1
echo
rpc 21001/add-learner '[3, "127.0.0.1:21003"]'
echo "Node 3 added as learner"
sleep 1
echo "Get metrics from the leader, after adding 2 learners"
sleep 2
echo
rpc 21001/metrics
sleep 1
echo "Changing membership from [1] to 3 nodes cluster: [1, 2, 3]"
echo
rpc 21001/change-membership '[1, 2, 3]'
sleep 1
echo 'Membership changed to [1, 2, 3]'
sleep 1
echo "Get metrics from the leader again"
sleep 1
echo
rpc 21001/metrics
sleep 1
echo "Write data on leader"
sleep 1
echo
rpc 21001/write '{"Set":{"key":"foo","value":"bar"}}'
sleep 1
echo "Data written"
sleep 1
echo "Read on every node, including the leader"
sleep 1
echo "Read from node 1"
echo
rpc 21001/read '"foo"'
echo "Read from node 2"
echo
rpc 21002/read '"foo"'
echo "Read from node 3"
echo
rpc 21003/read '"foo"'
echo "Changing membership from [1,2,3] to [3]"
echo
rpc 21001/change-membership '[3]'
sleep 1
echo 'Membership changed to [3]'
sleep 1
echo "Get metrics from the node-3"
sleep 1
echo
rpc 21003/metrics
sleep 1
echo "Write foo=zoo on node-3"
sleep 1
echo
rpc 21003/write '{"Set":{"key":"foo","value":"zoo"}}'
sleep 1
echo "Data written"
sleep 1
echo "Read foo=zoo from node-3"
sleep 1
echo "Read from node 3"
echo
rpc 21003/read '"foo"'
echo
echo "Killing all nodes..."
kill