init

.gitignore

@@ -0,0 +1 @@
+/target

Cargo.toml

@@ -0,0 +1,44 @@
+[package]
+name = "dcache"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+openraft = { version = "0.8.3", features = ["serde"]}
+#libmcaptcha = { branch = "master", git = "https://github.com/mCaptcha/libmcaptcha", features = ["full"]}
+libmcaptcha = { path="../libmcaptcha/", features = ["full"]}
+tracing = { version = "0.1.37", features = ["log"] }
+serde_json = "1.0.96"
+serde = { version = "1.0.163", features = ["derive"] }
+byteorder = "1.4.3"
+actix-web = "4"
+actix-web-httpauth = "0.8.0"
+futures-util = { version = "0.3.17", default-features = false, features = ["std"] }
+lazy_static = "1.4.0"
+pretty_env_logger = "0.4.0"
+uuid = { version = "1", features = ["v4"] }
+sqlx = { version = "0.5.13", features = [ "runtime-actix-rustls", "postgres", "time", "offline" ] }
+actix-web-codegen-const-routes = { version = "0.1.0", tag = "0.1.0", git = "https://github.com/realaravinth/actix-web-codegen-const-routes" }
+derive_builder = "0.11.2"
+config = "0.11"
+derive_more = "0.99.17"
+url = { version = "2.2.2", features = ["serde"]}
+async-trait = "0.1.36"
+clap = { version = "4.1.11", features = ["derive", "env"] }
+reqwest = { version = "0.11.9", features = ["json"] }
+tokio = { version = "1.0", default-features = false, features = ["sync"] }
+tracing-subscriber = { version = "0.3.0", features = ["env-filter"] }
+actix = "0.13.0"
+
+
+[build-dependencies]
+serde_json = "1"
+sqlx = { version = "0.5.13", features = [ "runtime-actix-rustls", "postgres", "time", "offline"] }
+
+[dev-dependencies]
+actix-rt = "2.7.0"
+base64 = "0.13.0"
+anyhow = "1.0.63"
+maplit = "1.0.2"

src/app.rs

@@ -0,0 +1,17 @@
+use std::sync::Arc;
+
+use openraft::Config;
+
+use crate::ExampleNodeId;
+use crate::ExampleRaft;
+use crate::ExampleStore;
+
+// Representation of an application state. This struct can be shared around to share
+// instances of raft, store and more.
+pub struct ExampleApp {
+    pub id: ExampleNodeId,
+    pub addr: String,
+    pub raft: ExampleRaft,
+    pub store: Arc<ExampleStore>,
+    pub config: Arc<Config>,
+}

src/bin/main.rs

@@ -0,0 +1,36 @@
+use clap::Parser;
+use dcache::network::raft_network_impl::ExampleNetwork;
+use dcache::start_example_raft_node;
+use dcache::store::ExampleStore;
+use dcache::ExampleTypeConfig;
+use openraft::Raft;
+use tracing_subscriber::EnvFilter;
+
+pub type ExampleRaft = Raft<ExampleTypeConfig, ExampleNetwork, ExampleStore>;
+
+#[derive(Parser, Clone, Debug)]
+#[clap(author, version, about, long_about = None)]
+pub struct Opt {
+    #[clap(long)]
+    pub id: u64,
+
+    #[clap(long)]
+    pub http_addr: String,
+}
+
+#[actix_web::main]
+async fn main() -> std::io::Result<()> {
+    // Setup the logger
+    tracing_subscriber::fmt()
+        .with_target(true)
+        .with_thread_ids(true)
+        .with_level(true)
+        .with_ansi(false)
+        .with_env_filter(EnvFilter::from_default_env())
+        .init();
+
+    // Parse the parameters passed by arguments.
+    let options = Opt::parse();
+
+    start_example_raft_node(options.id, options.http_addr).await
+}

src/client.rs

@@ -0,0 +1,227 @@
+use std::collections::BTreeSet;
+use std::sync::Arc;
+use std::sync::Mutex;
+use std::time::Duration;
+
+use openraft::error::ForwardToLeader;
+use openraft::error::NetworkError;
+use openraft::error::RPCError;
+use openraft::error::RemoteError;
+use openraft::BasicNode;
+use openraft::RaftMetrics;
+use openraft::TryAsRef;
+use reqwest::Client;
+use serde::de::DeserializeOwned;
+use serde::Deserialize;
+use serde::Serialize;
+use tokio::time::timeout;
+
+use crate::typ;
+use crate::ExampleNodeId;
+use crate::ExampleRequest;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Empty {}
+
+pub struct ExampleClient {
+    /// The leader node to send request to.
+    ///
+    /// All traffic should be sent to the leader in a cluster.
+    pub leader: Arc<Mutex<(ExampleNodeId, String)>>,
+
+    pub inner: Client,
+}
+
+impl ExampleClient {
+    /// Create a client with a leader node id and a node manager to get node address by node id.
+    pub fn new(leader_id: ExampleNodeId, leader_addr: String) -> Self {
+        Self {
+            leader: Arc::new(Mutex::new((leader_id, leader_addr))),
+            inner: reqwest::Client::new(),
+        }
+    }
+
+    // --- Application API
+
+    /// Submit a write request to the raft cluster.
+    ///
+    /// The request will be processed by raft protocol: it will be replicated to a quorum and then
+    /// will be applied to state machine.
+    ///
+    /// The result of applying the request will be returned.
+    pub async fn write(
+        &self,
+        req: &ExampleRequest,
+    ) -> Result<typ::ClientWriteResponse, typ::RPCError<typ::ClientWriteError>> {
+        self.send_rpc_to_leader("write", Some(req)).await
+    }
+
+    /// Read value by key, in an inconsistent mode.
+    ///
+    /// This method may return stale value because it does not force to read on a legal leader.
+    pub async fn read(&self, req: &String) -> Result<String, typ::RPCError> {
+        self.do_send_rpc_to_leader("read", Some(req)).await
+    }
+
+    /// Consistent Read value by key, in an inconsistent mode.
+    ///
+    /// This method MUST return consistent value or CheckIsLeaderError.
+    pub async fn consistent_read(
+        &self,
+        req: &String,
+    ) -> Result<String, typ::RPCError<typ::CheckIsLeaderError>> {
+        self.do_send_rpc_to_leader("consistent_read", Some(req))
+            .await
+    }
+
+    // --- Cluster management API
+
+    /// Initialize a cluster of only the node that receives this request.
+    ///
+    /// This is the first step to initialize a cluster.
+    /// With a initialized cluster, new node can be added with [`write`].
+    /// Then setup replication with [`add_learner`].
+    /// Then make the new node a member with [`change_membership`].
+    pub async fn init(&self) -> Result<(), typ::RPCError<typ::InitializeError>> {
+        self.do_send_rpc_to_leader("init", Some(&Empty {})).await
+    }
+
+    /// Add a node as learner.
+    ///
+    /// The node to add has to exist, i.e., being added with `write(ExampleRequest::AddNode{})`
+    pub async fn add_learner(
+        &self,
+        req: (ExampleNodeId, String),
+    ) -> Result<typ::ClientWriteResponse, typ::RPCError<typ::ClientWriteError>> {
+        self.send_rpc_to_leader("add-learner", Some(&req)).await
+    }
+
+    /// Change membership to the specified set of nodes.
+    ///
+    /// All nodes in `req` have to be already added as learner with [`add_learner`],
+    /// or an error [`LearnerNotFound`] will be returned.
+    pub async fn change_membership(
+        &self,
+        req: &BTreeSet<ExampleNodeId>,
+    ) -> Result<typ::ClientWriteResponse, typ::RPCError<typ::ClientWriteError>> {
+        self.send_rpc_to_leader("change-membership", Some(req))
+            .await
+    }
+
+    /// Get the metrics about the cluster.
+    ///
+    /// Metrics contains various information about the cluster, such as current leader,
+    /// membership config, replication status etc.
+    /// See [`RaftMetrics`].
+    pub async fn metrics(&self) -> Result<RaftMetrics<ExampleNodeId, BasicNode>, typ::RPCError> {
+        self.do_send_rpc_to_leader("metrics", None::<&()>).await
+    }
+
+    // --- Internal methods
+
+    /// Send RPC to specified node.
+    ///
+    /// It sends out a POST request if `req` is Some. Otherwise a GET request.
+    /// The remote endpoint must respond a reply in form of `Result<T, E>`.
+    /// An `Err` happened on remote will be wrapped in an [`RPCError::RemoteError`].
+    async fn do_send_rpc_to_leader<Req, Resp, Err>(
+        &self,
+        uri: &str,
+        req: Option<&Req>,
+    ) -> Result<Resp, typ::RPCError<Err>>
+    where
+        Req: Serialize + 'static,
+        Resp: Serialize + DeserializeOwned,
+        Err: std::error::Error + Serialize + DeserializeOwned,
+    {
+        let (leader_id, url) = {
+            let t = self.leader.lock().unwrap();
+            let target_addr = &t.1;
+            (t.0, format!("http://{}/{}", target_addr, uri))
+        };
+
+        let fu = if let Some(r) = req {
+            tracing::debug!(
+                ">>> client send request to {}: {}",
+                url,
+                serde_json::to_string_pretty(&r).unwrap()
+            );
+            self.inner.post(url.clone()).json(r)
+        } else {
+            tracing::debug!(">>> client send request to {}", url,);
+            self.inner.get(url.clone())
+        }
+        .send();
+
+        let res = timeout(Duration::from_millis(3_000), fu).await;
+        let resp = match res {
+            Ok(x) => x.map_err(|e| RPCError::Network(NetworkError::new(&e)))?,
+            Err(timeout_err) => {
+                tracing::error!("timeout {} to url: {}", timeout_err, url);
+                return Err(RPCError::Network(NetworkError::new(&timeout_err)));
+            }
+        };
+
+        let res: Result<Resp, typ::RaftError<Err>> = resp
+            .json()
+            .await
+            .map_err(|e| RPCError::Network(NetworkError::new(&e)))?;
+        tracing::debug!(
+            "<<< client recv reply from {}: {}",
+            url,
+            serde_json::to_string_pretty(&res).unwrap()
+        );
+
+        res.map_err(|e| RPCError::RemoteError(RemoteError::new(leader_id, e)))
+    }
+
+    /// Try the best to send a request to the leader.
+    ///
+    /// If the target node is not a leader, a `ForwardToLeader` error will be
+    /// returned and this client will retry at most 3 times to contact the updated leader.
+    async fn send_rpc_to_leader<Req, Resp, Err>(
+        &self,
+        uri: &str,
+        req: Option<&Req>,
+    ) -> Result<Resp, typ::RPCError<Err>>
+    where
+        Req: Serialize + 'static,
+        Resp: Serialize + DeserializeOwned,
+        Err: std::error::Error
+            + Serialize
+            + DeserializeOwned
+            + TryAsRef<typ::ForwardToLeader>
+            + Clone,
+    {
+        // Retry at most 3 times to find a valid leader.
+        let mut n_retry = 3;
+
+        loop {
+            let res: Result<Resp, typ::RPCError<Err>> = self.do_send_rpc_to_leader(uri, req).await;
+
+            let rpc_err = match res {
+                Ok(x) => return Ok(x),
+                Err(rpc_err) => rpc_err,
+            };
+
+            if let Some(ForwardToLeader {
+                leader_id: Some(leader_id),
+                leader_node: Some(leader_node),
+            }) = rpc_err.forward_to_leader()
+            {
+                // Update target to the new leader.
+                {
+                    let mut t = self.leader.lock().unwrap();
+                    *t = (*leader_id, leader_node.addr.clone());
+                }
+
+                n_retry -= 1;
+                if n_retry > 0 {
+                    continue;
+                }
+            }
+
+            return Err(rpc_err);
+        }
+    }
+}

src/lib.rs

@@ -0,0 +1,121 @@
+#![allow(clippy::uninlined_format_args)]
+
+use std::sync::Arc;
+
+use actix_web::middleware;
+use actix_web::middleware::Logger;
+use actix_web::web::Data;
+use actix_web::App;
+use actix_web::HttpServer;
+use openraft::BasicNode;
+use openraft::Config;
+use openraft::Raft;
+
+use crate::app::ExampleApp;
+use crate::network::api;
+use crate::network::management;
+use crate::network::raft;
+use crate::network::raft_network_impl::ExampleNetwork;
+use crate::store::ExampleRequest;
+use crate::store::ExampleResponse;
+use crate::store::ExampleStore;
+
+pub mod app;
+pub mod client;
+pub mod network;
+pub mod store;
+
+pub type ExampleNodeId = u64;
+
+openraft::declare_raft_types!(
+    /// Declare the type configuration for example K/V store.
+    pub ExampleTypeConfig: D = ExampleRequest, R = ExampleResponse, NodeId = ExampleNodeId, Node = BasicNode
+);
+
+pub type ExampleRaft = Raft<ExampleTypeConfig, ExampleNetwork, Arc<ExampleStore>>;
+
+pub mod typ {
+    use openraft::BasicNode;
+
+    use crate::ExampleNodeId;
+    use crate::ExampleTypeConfig;
+
+    pub type RaftError<E = openraft::error::Infallible> =
+        openraft::error::RaftError<ExampleNodeId, E>;
+    pub type RPCError<E = openraft::error::Infallible> =
+        openraft::error::RPCError<ExampleNodeId, BasicNode, RaftError<E>>;
+
+    pub type ClientWriteError = openraft::error::ClientWriteError<ExampleNodeId, BasicNode>;
+    pub type CheckIsLeaderError = openraft::error::CheckIsLeaderError<ExampleNodeId, BasicNode>;
+    pub type ForwardToLeader = openraft::error::ForwardToLeader<ExampleNodeId, BasicNode>;
+    pub type InitializeError = openraft::error::InitializeError<ExampleNodeId, BasicNode>;
+
+    pub type ClientWriteResponse = openraft::raft::ClientWriteResponse<ExampleTypeConfig>;
+}
+
+pub async fn start_example_raft_node(
+    node_id: ExampleNodeId,
+    http_addr: String,
+) -> std::io::Result<()> {
+    // Create a configuration for the raft instance.
+    let config = Config {
+        heartbeat_interval: 500,
+        election_timeout_min: 1500,
+        election_timeout_max: 3000,
+        ..Default::default()
+    };
+
+    let salt = "foobar12".into();
+    #[cfg(release)]
+    todo!("set salt");
+
+    let config = Arc::new(config.validate().unwrap());
+
+    // Create a instance of where the Raft data will be stored.
+    let store = Arc::new(ExampleStore::new(salt));
+
+    // Create the network layer that will connect and communicate the raft instances and
+    // will be used in conjunction with the store created above.
+    let network = ExampleNetwork {};
+
+    // Create a local raft instance.
+    let raft = Raft::new(node_id, config.clone(), network, store.clone())
+        .await
+        .unwrap();
+
+    // Create an application that will store all the instances created above, this will
+    // be later used on the actix-web services.
+    let app = Data::new(ExampleApp {
+        id: node_id,
+        addr: http_addr.clone(),
+        raft,
+        store,
+        config,
+    });
+
+    // Start the actix-web server.
+    let server = HttpServer::new(move || {
+        App::new()
+            .wrap(Logger::default())
+            .wrap(Logger::new("%a %{User-Agent}i"))
+            .wrap(middleware::Compress::default())
+            .app_data(app.clone())
+            // raft internal RPC
+            .service(raft::append)
+            .service(raft::snapshot)
+            .service(raft::vote)
+            // admin API
+            .service(management::init)
+            .service(management::add_learner)
+            .service(management::change_membership)
+            .service(management::metrics)
+            // application API
+            .service(api::write)
+        //            .service(api::read)
+        //            .service(api::consistent_read)
+    });
+
+    let x = server.bind(http_addr)?;
+
+    x.run().await
+}

src/network/api.rs

@@ -0,0 +1,54 @@
+use actix_web::post;
+use actix_web::web;
+use actix_web::web::Data;
+use actix_web::Responder;
+use openraft::error::CheckIsLeaderError;
+use openraft::error::Infallible;
+use openraft::error::RaftError;
+use openraft::BasicNode;
+use web::Json;
+
+use crate::app::ExampleApp;
+use crate::store::ExampleRequest;
+use crate::ExampleNodeId;
+
+/**
+ * Application API
+ *
+ * This is where you place your application, you can use the example below to create your
+ * API. The current implementation:
+ *
+ *  - `POST - /write` saves a value in a key and sync the nodes.
+ *  - `POST - /read` attempt to find a value from a given key.
+ */
+#[post("/write")]
+pub async fn write(
+    app: Data<ExampleApp>,
+    req: Json<ExampleRequest>,
+) -> actix_web::Result<impl Responder> {
+    let response = app.raft.client_write(req.0).await;
+    Ok(Json(response))
+}
+//    AddVisitor(AddVisitor),
+//    AddCaptcha(AddCaptcha),
+//    RenameCaptcha(RenameCaptcha),
+//    RemoveCaptcha(RemoveCaptcha),
+//#[post("/post")]
+//pub async fn read(app: Data<ExampleApp>, req: Json<String>) -> actix_web::Result<impl Responder> {
+//    let state_machine = app.store.state_machine.read().await;
+//    let key = req.0;
+//    let value = state_machine.data.get(&key).cloned();
+//
+//    let res: Result<String, Infallible> = Ok(value.unwrap_or_default());
+//    Ok(Json(res))
+//}
+//
+//#[post("/visitor/add")]
+//pub async fn add_visitor(app: Data<ExampleApp>, req: Json<String>) -> actix_web::Result<impl Responder> {
+//    let state_machine = app.store.state_machine.read().await;
+//    let key = req.0;
+//    let value = state_machine.data.get(&key).cloned();
+//
+//    let res: Result<String, Infallible> = Ok(value.unwrap_or_default());
+//    Ok(Json(res))
+//}

src/network/management.rs

@@ -0,0 +1,68 @@
+use std::collections::BTreeMap;
+use std::collections::BTreeSet;
+
+use actix_web::get;
+use actix_web::post;
+use actix_web::web;
+use actix_web::web::Data;
+use actix_web::Responder;
+use openraft::error::Infallible;
+use openraft::BasicNode;
+use openraft::RaftMetrics;
+use web::Json;
+
+use crate::app::ExampleApp;
+use crate::ExampleNodeId;
+
+// --- Cluster management
+
+/// Add a node as **Learner**.
+///
+/// A Learner receives log replication from the leader but does not vote.
+/// This should be done before adding a node as a member into the cluster
+/// (by calling `change-membership`)
+#[post("/add-learner")]
+pub async fn add_learner(
+    app: Data<ExampleApp>,
+    req: Json<(ExampleNodeId, String)>,
+) -> actix_web::Result<impl Responder> {
+    let node_id = req.0 .0;
+    let node = BasicNode {
+        addr: req.0 .1.clone(),
+    };
+    let res = app.raft.add_learner(node_id, node, true).await;
+    Ok(Json(res))
+}
+
+/// Changes specified learners to members, or remove members.
+#[post("/change-membership")]
+pub async fn change_membership(
+    app: Data<ExampleApp>,
+    req: Json<BTreeSet<ExampleNodeId>>,
+) -> actix_web::Result<impl Responder> {
+    let res = app.raft.change_membership(req.0, false).await;
+    Ok(Json(res))
+}
+
+/// Initialize a single-node cluster.
+#[post("/init")]
+pub async fn init(app: Data<ExampleApp>) -> actix_web::Result<impl Responder> {
+    let mut nodes = BTreeMap::new();
+    nodes.insert(
+        app.id,
+        BasicNode {
+            addr: app.addr.clone(),
+        },
+    );
+    let res = app.raft.initialize(nodes).await;
+    Ok(Json(res))
+}
+
+/// Get the latest metrics of the cluster
+#[get("/metrics")]
+pub async fn metrics(app: Data<ExampleApp>) -> actix_web::Result<impl Responder> {
+    let metrics = app.raft.metrics().borrow().clone();
+
+    let res: Result<RaftMetrics<ExampleNodeId, BasicNode>, Infallible> = Ok(metrics);
+    Ok(Json(res))
+}

src/network/mod.rs

@@ -0,0 +1,4 @@
+pub mod api;
+pub mod management;
+pub mod raft;
+pub mod raft_network_impl;

src/network/raft.rs

@@ -0,0 +1,41 @@
+use actix_web::post;
+use actix_web::web;
+use actix_web::web::Data;
+use actix_web::Responder;
+use openraft::raft::AppendEntriesRequest;
+use openraft::raft::InstallSnapshotRequest;
+use openraft::raft::VoteRequest;
+use web::Json;
+
+use crate::app::ExampleApp;
+use crate::ExampleNodeId;
+use crate::ExampleTypeConfig;
+
+// --- Raft communication
+
+#[post("/raft-vote")]
+pub async fn vote(
+    app: Data<ExampleApp>,
+    req: Json<VoteRequest<ExampleNodeId>>,
+) -> actix_web::Result<impl Responder> {
+    let res = app.raft.vote(req.0).await;
+    Ok(Json(res))
+}
+
+#[post("/raft-append")]
+pub async fn append(
+    app: Data<ExampleApp>,
+    req: Json<AppendEntriesRequest<ExampleTypeConfig>>,
+) -> actix_web::Result<impl Responder> {
+    let res = app.raft.append_entries(req.0).await;
+    Ok(Json(res))
+}
+
+#[post("/raft-snapshot")]
+pub async fn snapshot(
+    app: Data<ExampleApp>,
+    req: Json<InstallSnapshotRequest<ExampleTypeConfig>>,
+) -> actix_web::Result<impl Responder> {
+    let res = app.raft.install_snapshot(req.0).await;
+    Ok(Json(res))
+}

src/network/raft_network_impl.rs

@@ -0,0 +1,123 @@
+use async_trait::async_trait;
+use openraft::error::InstallSnapshotError;
+use openraft::error::NetworkError;
+use openraft::error::RPCError;
+use openraft::error::RaftError;
+use openraft::error::RemoteError;
+use openraft::raft::AppendEntriesRequest;
+use openraft::raft::AppendEntriesResponse;
+use openraft::raft::InstallSnapshotRequest;
+use openraft::raft::InstallSnapshotResponse;
+use openraft::raft::VoteRequest;
+use openraft::raft::VoteResponse;
+use openraft::BasicNode;
+use openraft::RaftNetwork;
+use openraft::RaftNetworkFactory;
+use serde::de::DeserializeOwned;
+use serde::Serialize;
+
+use crate::ExampleNodeId;
+use crate::ExampleTypeConfig;
+
+pub struct ExampleNetwork {}
+
+impl ExampleNetwork {
+    pub async fn send_rpc<Req, Resp, Err>(
+        &self,
+        target: ExampleNodeId,
+        target_node: &BasicNode,
+        uri: &str,
+        req: Req,
+    ) -> Result<Resp, RPCError<ExampleNodeId, BasicNode, Err>>
+    where
+        Req: Serialize,
+        Err: std::error::Error + DeserializeOwned,
+        Resp: DeserializeOwned,
+    {
+        let addr = &target_node.addr;
+
+        let url = format!("http://{}/{}", addr, uri);
+
+        tracing::debug!("send_rpc to url: {}", url);
+
+        let client = reqwest::Client::new();
+
+        tracing::debug!("client is created for: {}", url);
+
+        let resp = client
+            .post(url)
+            .json(&req)
+            .send()
+            .await
+            .map_err(|e| RPCError::Network(NetworkError::new(&e)))?;
+
+        tracing::debug!("client.post() is sent");
+
+        let res: Result<Resp, Err> = resp
+            .json()
+            .await
+            .map_err(|e| RPCError::Network(NetworkError::new(&e)))?;
+
+        res.map_err(|e| RPCError::RemoteError(RemoteError::new(target, e)))
+    }
+}
+
+// NOTE: This could be implemented also on `Arc<ExampleNetwork>`, but since it's empty, implemented
+// directly.
+#[async_trait]
+impl RaftNetworkFactory<ExampleTypeConfig> for ExampleNetwork {
+    type Network = ExampleNetworkConnection;
+
+    async fn new_client(&mut self, target: ExampleNodeId, node: &BasicNode) -> Self::Network {
+        ExampleNetworkConnection {
+            owner: ExampleNetwork {},
+            target,
+            target_node: node.clone(),
+        }
+    }
+}
+
+pub struct ExampleNetworkConnection {
+    owner: ExampleNetwork,
+    target: ExampleNodeId,
+    target_node: BasicNode,
+}
+
+#[async_trait]
+impl RaftNetwork<ExampleTypeConfig> for ExampleNetworkConnection {
+    async fn send_append_entries(
+        &mut self,
+        req: AppendEntriesRequest<ExampleTypeConfig>,
+    ) -> Result<
+        AppendEntriesResponse<ExampleNodeId>,
+        RPCError<ExampleNodeId, BasicNode, RaftError<ExampleNodeId>>,
+    > {
+        self.owner
+            .send_rpc(self.target, &self.target_node, "raft-append", req)
+            .await
+    }
+
+    async fn send_install_snapshot(
+        &mut self,
+        req: InstallSnapshotRequest<ExampleTypeConfig>,
+    ) -> Result<
+        InstallSnapshotResponse<ExampleNodeId>,
+        RPCError<ExampleNodeId, BasicNode, RaftError<ExampleNodeId, InstallSnapshotError>>,
+    > {
+        self.owner
+            .send_rpc(self.target, &self.target_node, "raft-snapshot", req)
+            .await
+    }
+
+    async fn send_vote(
+        &mut self,
+        req: VoteRequest<ExampleNodeId>,
+    ) -> Result<
+        VoteResponse<ExampleNodeId>,
+        RPCError<ExampleNodeId, BasicNode, RaftError<ExampleNodeId>>,
+    > {
+        self.owner
+            .send_rpc(self.target, &self.target_node, "raft-vote", req)
+            .await
+    }
+}

src/store/mod.rs

@@ -0,0 +1,528 @@
+use std::collections::BTreeMap;
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::io::Cursor;
+use std::ops::RangeBounds;
+use std::sync::Arc;
+use std::sync::Mutex;
+
+use libmcaptcha::AddVisitorResult;
+use libmcaptcha::MCaptcha;
+use openraft::async_trait::async_trait;
+use openraft::storage::LogState;
+use openraft::storage::Snapshot;
+use openraft::AnyError;
+use openraft::BasicNode;
+use openraft::Entry;
+use openraft::EntryPayload;
+use openraft::ErrorSubject;
+use openraft::ErrorVerb;
+use openraft::LogId;
+use openraft::RaftLogReader;
+use openraft::RaftSnapshotBuilder;
+use openraft::RaftStorage;
+use openraft::SnapshotMeta;
+use openraft::StorageError;
+use openraft::StorageIOError;
+use openraft::StoredMembership;
+use openraft::Vote;
+use serde::Deserialize;
+use serde::Serialize;
+use sqlx::Statement;
+use tokio::sync::RwLock;
+use url::quirks::set_pathname;
+
+use crate::ExampleNodeId;
+use crate::ExampleTypeConfig;
+
+use actix::prelude::*;
+use libmcaptcha::master::messages::{
+    AddSite as AddCaptcha, AddVisitor, GetInternalData, RemoveCaptcha, Rename as RenameCaptcha,
+    SetInternalData,
+};
+use libmcaptcha::{master::embedded::master::Master as EmbeddedMaster, system::System, HashCache};
+
+pub mod system;
+
+/**
+ * Here you will set the types of request that will interact with the raft nodes.
+ * For example the `Set` will be used to write data (key and value) to the raft database.
+ * The `AddNode` will append a new node to the current existing shared list of nodes.
+ * You will want to add any request that can write data in all nodes here.
+ */
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub enum ExampleRequest {
+    //Set { key: String, value: String },
+    AddVisitor(AddVisitor),
+    AddCaptcha(AddCaptcha),
+    RenameCaptcha(RenameCaptcha),
+    RemoveCaptcha(RemoveCaptcha),
+}
+
+/**
+ * Here you will defined what type of answer you expect from reading the data of a node.
+ * In this example it will return a optional value from a given key in
+ * the `ExampleRequest.Set`.
+ *
+ * TODO: Should we explain how to create multiple `AppDataResponse`?
+ *
+ */
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub enum ExampleResponse {
+    AddVisitorResult(Option<AddVisitorResult>),
+    Empty,
+    //    AddCaptchaResult, All returns ()
+    //    RenameCaptchaResult,
+    //    RemoveCaptchaResult,
+}
+
+#[derive(Debug)]
+pub struct ExampleSnapshot {
+    pub meta: SnapshotMeta<ExampleNodeId, BasicNode>,
+
+    /// The data of the state machine at the time of this snapshot.
+    pub data: Vec<u8>,
+}
+
+/**
+ * Here defines a state machine of the raft, this state represents a copy of the data
+ * between each node. Note that we are using `serde` to serialize the `data`, which has
+ * a implementation to be serialized. Note that for this test we set both the key and
+ * value as String, but you could set any type of value that has the serialization impl.
+ */
+pub struct ExampleStateMachine {
+    pub last_applied_log: Option<LogId<ExampleNodeId>>,
+
+    pub last_membership: StoredMembership<ExampleNodeId, BasicNode>,
+
+    /// Application data.
+    pub data: Arc<System<HashCache, EmbeddedMaster>>,
+}
+
+#[derive(Serialize, Deserialize, Clone)]
+struct PersistableStateMachine {
+    last_applied_log: Option<LogId<ExampleNodeId>>,
+
+    last_membership: StoredMembership<ExampleNodeId, BasicNode>,
+
+    /// Application data.
+    data: HashMap<String, MCaptcha>,
+}
+
+impl PersistableStateMachine {
+    async fn from_statemachine(m: &ExampleStateMachine) -> Self {
+        let internal_data = m
+            .data
+            .master
+            .send(GetInternalData)
+            .await
+            .unwrap()
+            .await
+            .unwrap()
+            .unwrap();
+        Self {
+            last_applied_log: m.last_applied_log.clone(),
+            last_membership: m.last_membership.clone(),
+            data: internal_data,
+        }
+    }
+
+    async fn to_statemachine(
+        self,
+        data: Arc<System<HashCache, EmbeddedMaster>>,
+    ) -> ExampleStateMachine {
+        data.master
+            .send(SetInternalData {
+                mcaptcha: self.data,
+            })
+            .await
+            .unwrap();
+        ExampleStateMachine {
+            last_applied_log: self.last_applied_log,
+            last_membership: self.last_membership,
+            data,
+        }
+    }
+}
+
+pub struct ExampleStore {
+    last_purged_log_id: RwLock<Option<LogId<ExampleNodeId>>>,
+
+    /// The Raft log.
+    log: RwLock<BTreeMap<u64, Entry<ExampleTypeConfig>>>,
+
+    /// The Raft state machine.
+    pub state_machine: RwLock<ExampleStateMachine>,
+
+    /// The current granted vote.
+    vote: RwLock<Option<Vote<ExampleNodeId>>>,
+
+    snapshot_idx: Arc<Mutex<u64>>,
+
+    current_snapshot: RwLock<Option<ExampleSnapshot>>,
+}
+
+impl ExampleStore {
+    pub fn new(salt: String) -> Self {
+        let state_machine = RwLock::new(ExampleStateMachine {
+            last_applied_log: Default::default(),
+            last_membership: Default::default(),
+            data: system::init_system(salt),
+        });
+
+        Self {
+            last_purged_log_id: Default::default(),
+            log: Default::default(),
+            state_machine,
+            vote: Default::default(),
+            snapshot_idx: Default::default(),
+            current_snapshot: Default::default(),
+        }
+    }
+}
+
+#[async_trait]
+impl RaftLogReader<ExampleTypeConfig> for Arc<ExampleStore> {
+    async fn get_log_state(
+        &mut self,
+    ) -> Result<LogState<ExampleTypeConfig>, StorageError<ExampleNodeId>> {
+        let log = self.log.read().await;
+        let last = log.iter().rev().next().map(|(_, ent)| ent.log_id);
+
+        let last_purged = *self.last_purged_log_id.read().await;
+
+        let last = match last {
+            None => last_purged,
+            Some(x) => Some(x),
+        };
+
+        Ok(LogState {
+            last_purged_log_id: last_purged,
+            last_log_id: last,
+        })
+    }
+
+    async fn try_get_log_entries<RB: RangeBounds<u64> + Clone + Debug + Send + Sync>(
+        &mut self,
+        range: RB,
+    ) -> Result<Vec<Entry<ExampleTypeConfig>>, StorageError<ExampleNodeId>> {
+        let log = self.log.read().await;
+        let response = log
+            .range(range.clone())
+            .map(|(_, val)| val.clone())
+            .collect::<Vec<_>>();
+        Ok(response)
+    }
+}
+
+#[async_trait]
+impl RaftSnapshotBuilder<ExampleTypeConfig, Cursor<Vec<u8>>> for Arc<ExampleStore> {
+    #[tracing::instrument(level = "trace", skip(self))]
+    async fn build_snapshot(
+        &mut self,
+    ) -> Result<Snapshot<ExampleNodeId, BasicNode, Cursor<Vec<u8>>>, StorageError<ExampleNodeId>>
+    {
+        let data;
+        let last_applied_log;
+        let last_membership;
+
+        {
+            // Serialize the data of the state machine.
+            let state_machine = self.state_machine.read().await;
+            let persistable_state_machine =
+                PersistableStateMachine::from_statemachine(&state_machine).await;
+
+            data = serde_json::to_vec(&persistable_state_machine).map_err(|e| {
+                StorageIOError::new(
+                    ErrorSubject::StateMachine,
+                    ErrorVerb::Read,
+                    AnyError::new(&e),
+                )
+            })?;
+
+            last_applied_log = state_machine.last_applied_log;
+            last_membership = state_machine.last_membership.clone();
+        }
+
+        let snapshot_idx = {
+            let mut l = self.snapshot_idx.lock().unwrap();
+            *l += 1;
+            *l
+        };
+
+        let snapshot_id = if let Some(last) = last_applied_log {
+            format!("{}-{}-{}", last.leader_id, last.index, snapshot_idx)
+        } else {
+            format!("--{}", snapshot_idx)
+        };
+
+        let meta = SnapshotMeta {
+            last_log_id: last_applied_log,
+            last_membership,
+            snapshot_id,
+        };
+
+        let snapshot = ExampleSnapshot {
+            meta: meta.clone(),
+            data: data.clone(),
+        };
+
+        {
+            let mut current_snapshot = self.current_snapshot.write().await;
+            *current_snapshot = Some(snapshot);
+        }
+
+        Ok(Snapshot {
+            meta,
+            snapshot: Box::new(Cursor::new(data)),
+        })
+    }
+}
+
+#[async_trait]
+impl RaftStorage<ExampleTypeConfig> for Arc<ExampleStore> {
+    type SnapshotData = Cursor<Vec<u8>>;
+    type LogReader = Self;
+    type SnapshotBuilder = Self;
+
+    #[tracing::instrument(level = "trace", skip(self))]
+    async fn save_vote(
+        &mut self,
+        vote: &Vote<ExampleNodeId>,
+    ) -> Result<(), StorageError<ExampleNodeId>> {
+        let mut v = self.vote.write().await;
+        *v = Some(*vote);
+        Ok(())
+    }
+
+    async fn read_vote(
+        &mut self,
+    ) -> Result<Option<Vote<ExampleNodeId>>, StorageError<ExampleNodeId>> {
+        Ok(*self.vote.read().await)
+    }
+
+    #[tracing::instrument(level = "trace", skip(self, entries))]
+    async fn append_to_log(
+        &mut self,
+        entries: &[&Entry<ExampleTypeConfig>],
+    ) -> Result<(), StorageError<ExampleNodeId>> {
+        let mut log = self.log.write().await;
+        for entry in entries {
+            log.insert(entry.log_id.index, (*entry).clone());
+        }
+        Ok(())
+    }
+
+    #[tracing::instrument(level = "debug", skip(self))]
+    async fn delete_conflict_logs_since(
+        &mut self,
+        log_id: LogId<ExampleNodeId>,
+    ) -> Result<(), StorageError<ExampleNodeId>> {
+        tracing::debug!("delete_log: [{:?}, +oo)", log_id);
+
+        let mut log = self.log.write().await;
+        let keys = log
+            .range(log_id.index..)
+            .map(|(k, _v)| *k)
+            .collect::<Vec<_>>();
+        for key in keys {
+            log.remove(&key);
+        }
+
+        Ok(())
+    }
+
+    #[tracing::instrument(level = "debug", skip(self))]
+    async fn purge_logs_upto(
+        &mut self,
+        log_id: LogId<ExampleNodeId>,
+    ) -> Result<(), StorageError<ExampleNodeId>> {
+        tracing::debug!("delete_log: [{:?}, +oo)", log_id);
+
+        {
+            let mut ld = self.last_purged_log_id.write().await;
+            assert!(*ld <= Some(log_id));
+            *ld = Some(log_id);
+        }
+
+        {
+            let mut log = self.log.write().await;
+
+            let keys = log
+                .range(..=log_id.index)
+                .map(|(k, _v)| *k)
+                .collect::<Vec<_>>();
+            for key in keys {
+                log.remove(&key);
+            }
+        }
+
+        Ok(())
+    }
+
+    async fn last_applied_state(
+        &mut self,
+    ) -> Result<
+        (
+            Option<LogId<ExampleNodeId>>,
+            StoredMembership<ExampleNodeId, BasicNode>,
+        ),
+        StorageError<ExampleNodeId>,
+    > {
+        let state_machine = self.state_machine.read().await;
+        Ok((
+            state_machine.last_applied_log,
+            state_machine.last_membership.clone(),
+        ))
+    }
+
+    #[tracing::instrument(level = "trace", skip(self, entries))]
+    async fn apply_to_state_machine(
+        &mut self,
+        entries: &[&Entry<ExampleTypeConfig>],
+    ) -> Result<Vec<ExampleResponse>, StorageError<ExampleNodeId>> {
+        let mut res = Vec::with_capacity(entries.len());
+
+        let mut sm = self.state_machine.write().await;
+
+        for entry in entries {
+            tracing::debug!(%entry.log_id, "replicate to sm");
+
+            sm.last_applied_log = Some(entry.log_id);
+
+            match entry.payload {
+                EntryPayload::Blank => res.push(ExampleResponse::Empty),
+                EntryPayload::Normal(ref req) => match req {
+                    ExampleRequest::AddVisitor(msg) => {
+                        let sm = self.state_machine.read().await;
+                        res.push(ExampleResponse::AddVisitorResult(
+                            sm.data
+                                .master
+                                .send(msg.clone())
+                                .await
+                                .unwrap()
+                                .await
+                                .unwrap()
+                                .unwrap(),
+                        ));
+                    }
+                    ExampleRequest::AddCaptcha(msg) => {
+                        let sm = self.state_machine.read().await;
+                        sm.data
+                            .master
+                            .send(msg.clone())
+                            .await
+                            .unwrap()
+                            .await
+                            .unwrap()
+                            .unwrap();
+                        res.push(ExampleResponse::Empty);
+                    }
+                    ExampleRequest::RenameCaptcha(msg) => {
+                        let sm = self.state_machine.read().await;
+                        sm.data
+                            .master
+                            .send(msg.clone())
+                            .await
+                            .unwrap()
+                            .await
+                            .unwrap()
+                            .unwrap();
+                        res.push(ExampleResponse::Empty);
+                    }
+                    ExampleRequest::RemoveCaptcha(msg) => {
+                        let sm = self.state_machine.read().await;
+                        sm.data
+                            .master
+                            .send(msg.clone())
+                            .await
+                            .unwrap()
+                            .await
+                            .unwrap()
+                            .unwrap();
+                        res.push(ExampleResponse::Empty);
+                    }
+                },
+                EntryPayload::Membership(ref mem) => {
+                    sm.last_membership = StoredMembership::new(Some(entry.log_id), mem.clone());
+                    res.push(ExampleResponse::Empty)
+                }
+            };
+        }
+        Ok(res)
+    }
+
+    #[tracing::instrument(level = "trace", skip(self))]
+    async fn begin_receiving_snapshot(
+        &mut self,
+    ) -> Result<Box<Self::SnapshotData>, StorageError<ExampleNodeId>> {
+        Ok(Box::new(Cursor::new(Vec::new())))
+    }
+
+    #[tracing::instrument(level = "trace", skip(self, snapshot))]
+    async fn install_snapshot(
+        &mut self,
+        meta: &SnapshotMeta<ExampleNodeId, BasicNode>,
+        snapshot: Box<Self::SnapshotData>,
+    ) -> Result<(), StorageError<ExampleNodeId>> {
+        tracing::info!(
+            { snapshot_size = snapshot.get_ref().len() },
+            "decoding snapshot for installation"
+        );
+
+        let new_snapshot = ExampleSnapshot {
+            meta: meta.clone(),
+            data: snapshot.into_inner(),
+        };
+
+        // Update the state machine.
+        {
+            let updated_persistable_state_machine: PersistableStateMachine =
+                serde_json::from_slice(&new_snapshot.data).map_err(|e| {
+                    StorageIOError::new(
+                        ErrorSubject::Snapshot(new_snapshot.meta.signature()),
+                        ErrorVerb::Read,
+                        AnyError::new(&e),
+                    )
+                })?;
+
+            let mut state_machine = self.state_machine.write().await;
+            let updated_state_machine = updated_persistable_state_machine
+                .to_statemachine(state_machine.data.clone())
+                .await;
+            *state_machine = updated_state_machine;
+        }
+
+        // Update current snapshot.
+        let mut current_snapshot = self.current_snapshot.write().await;
+        *current_snapshot = Some(new_snapshot);
+        Ok(())
+    }
+
+    #[tracing::instrument(level = "trace", skip(self))]
+    async fn get_current_snapshot(
+        &mut self,
+    ) -> Result<
+        Option<Snapshot<ExampleNodeId, BasicNode, Self::SnapshotData>>,
+        StorageError<ExampleNodeId>,
+    > {
+        match &*self.current_snapshot.read().await {
+            Some(snapshot) => {
+                let data = snapshot.data.clone();
+                Ok(Some(Snapshot {
+                    meta: snapshot.meta.clone(),
+                    snapshot: Box::new(Cursor::new(data)),
+                }))
+            }
+            None => Ok(None),
+        }
+    }
+
+    async fn get_log_reader(&mut self) -> Self::LogReader {
+        self.clone()
+    }
+
+    async fn get_snapshot_builder(&mut self) -> Self::SnapshotBuilder {
+        self.clone()
+    }
+}

src/store/system.rs

@@ -0,0 +1,28 @@
+use std::sync::Arc;
+
+use actix::prelude::*;
+use libmcaptcha::{
+    cache::{hashcache::HashCache, messages::VerifyCaptchaResult},
+    master::embedded::master::Master,
+    master::messages::AddSiteBuilder,
+    pow::{ConfigBuilder, Work},
+    system::{System, SystemBuilder},
+    DefenseBuilder, LevelBuilder, MCaptchaBuilder,
+};
+
+pub fn init_system(salt: String) -> Arc<System<HashCache, Master>> {
+    let cache = HashCache::default().start();
+
+    let pow = ConfigBuilder::default().salt(salt).build().unwrap();
+
+    let master = Master::new(5).start();
+
+    let system = SystemBuilder::default()
+        .master(master)
+        .cache(cache)
+        .pow(pow.clone())
+        .runners(4)
+        .queue_length(2000)
+        .build();
+    Arc::new(system)
+}

test-cluster.sh

@@ -0,0 +1,166 @@
+#!/bin/sh
+
+set -o errexit
+
+cargo build
+
+kill() {
+    if [ "$(uname)" = "Darwin" ]; then
+        SERVICE='raft-key-value'
+        if pgrep -xq -- "${SERVICE}"; then
+            pkill -f "${SERVICE}"
+        fi
+    else
+        set +e # killall will error if finds no process to kill
+        killall raft-key-value
+        set -e
+    fi
+}
+
+rpc() {
+    local uri=$1
+    local body="$2"
+
+    echo '---'" rpc(:$uri, $body)"
+
+    {
+        if [ ".$body" = "." ]; then
+            time curl --silent "127.0.0.1:$uri"
+        else
+            time curl --silent "127.0.0.1:$uri" -H "Content-Type: application/json" -d "$body"
+        fi
+    } | {
+        if type jq > /dev/null 2>&1; then
+            jq
+        else
+            cat
+        fi
+    }
+
+    echo
+    echo
+}
+
+export RUST_LOG=trace
+
+echo "Killing all running raft-key-value"
+
+kill
+
+sleep 1
+
+echo "Start 3 uninitialized raft-key-value servers..."
+
+nohup ./target/debug/raft-key-value  --id 1 --http-addr 127.0.0.1:21001 > n1.log &
+sleep 1
+echo "Server 1 started"
+
+nohup ./target/debug/raft-key-value  --id 2 --http-addr 127.0.0.1:21002 > n2.log &
+sleep 1
+echo "Server 2 started"
+
+nohup ./target/debug/raft-key-value  --id 3 --http-addr 127.0.0.1:21003 > n3.log &
+sleep 1
+echo "Server 3 started"
+sleep 1
+
+echo "Initialize server 1 as a single-node cluster"
+sleep 2
+echo
+rpc 21001/init '{}'
+
+echo "Server 1 is a leader now"
+
+sleep 2
+
+echo "Get metrics from the leader"
+sleep 2
+echo
+rpc 21001/metrics
+sleep 1
+
+
+echo "Adding node 2 and node 3 as learners, to receive log from leader node 1"
+
+sleep 1
+echo
+rpc 21001/add-learner       '[2, "127.0.0.1:21002"]'
+echo "Node 2 added as learner"
+sleep 1
+echo
+rpc 21001/add-learner       '[3, "127.0.0.1:21003"]'
+echo "Node 3 added as learner"
+sleep 1
+
+echo "Get metrics from the leader, after adding 2 learners"
+sleep 2
+echo
+rpc 21001/metrics
+sleep 1
+
+echo "Changing membership from [1] to 3 nodes cluster: [1, 2, 3]"
+echo
+rpc 21001/change-membership '[1, 2, 3]'
+sleep 1
+echo 'Membership changed to [1, 2, 3]'
+sleep 1
+
+echo "Get metrics from the leader again"
+sleep 1
+echo
+rpc 21001/metrics
+sleep 1
+
+echo "Write data on leader"
+sleep 1
+echo
+rpc 21001/write '{"Set":{"key":"foo","value":"bar"}}'
+sleep 1
+echo "Data written"
+sleep 1
+
+echo "Read on every node, including the leader"
+sleep 1
+echo "Read from node 1"
+echo
+rpc 21001/read  '"foo"'
+echo "Read from node 2"
+echo
+rpc 21002/read  '"foo"'
+echo "Read from node 3"
+echo
+rpc 21003/read  '"foo"'
+
+
+echo "Changing membership from [1,2,3] to [3]"
+echo
+rpc 21001/change-membership '[3]'
+sleep 1
+echo 'Membership changed to [3]'
+sleep 1
+
+echo "Get metrics from the node-3"
+sleep 1
+echo
+rpc 21003/metrics
+sleep 1
+
+
+echo "Write foo=zoo on node-3"
+sleep 1
+echo
+rpc 21003/write '{"Set":{"key":"foo","value":"zoo"}}'
+sleep 1
+echo "Data written"
+sleep 1
+
+echo "Read foo=zoo from node-3"
+sleep 1
+echo "Read from node 3"
+echo
+rpc 21003/read  '"foo"'
+echo
+
+
+echo "Killing all nodes..."
+kill