dcache/src/lib.rs

/*
 * mCaptcha - A proof of work based DoS protection system
 * Copyright © 2023 Aravinth Manivannan <realravinth@batsense.net>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#![allow(clippy::uninlined_format_args)]

use std::io::Cursor;
use std::sync::Arc;

use openraft::storage::Adaptor;
use openraft::BasicNode;
use openraft::Config;
use openraft::Raft;
use tonic::transport::Server;

use crate::app::DcacheApp;
use crate::network::raft_network_impl::DcacheNetwork;
use crate::protobuf::dcache::dcache_service_client::DcacheServiceClient;
use crate::protobuf::dcache::dcache_service_server::DcacheServiceServer;
use crate::protobuf::dcache::Learner;
use crate::store::DcacheRequest;
use crate::store::DcacheResponse;
use crate::store::DcacheStore;

pub mod app;
mod mcaptcha;
pub mod network;
mod pool;
mod protobuf;
pub mod store;

pub type DcacheNodeId = u64;

openraft::declare_raft_types!(
    /// Declare the type configuration for example K/V store.
    pub DcacheTypeConfig: D = DcacheRequest, R = DcacheResponse, NodeId = DcacheNodeId, Node = BasicNode,
    Entry = openraft::Entry<DcacheTypeConfig>, SnapshotData = Cursor<Vec<u8>>
);

pub type LogStore = Adaptor<DcacheTypeConfig, Arc<DcacheStore>>;
pub type StateMachineStore = Adaptor<DcacheTypeConfig, Arc<DcacheStore>>;
pub type DcacheRaft = Raft<DcacheTypeConfig, Arc<DcacheNetwork>, LogStore, StateMachineStore>;

pub mod typ {
    use openraft::BasicNode;

    use crate::DcacheNodeId;
    use crate::DcacheTypeConfig;

    pub type RaftError<E = openraft::error::Infallible> =
        openraft::error::RaftError<DcacheNodeId, E>;
    pub type RPCError<E = openraft::error::Infallible> =
        openraft::error::RPCError<DcacheNodeId, BasicNode, RaftError<E>>;

    pub type ClientWriteError = openraft::error::ClientWriteError<DcacheNodeId, BasicNode>;
    pub type CheckIsLeaderError = openraft::error::CheckIsLeaderError<DcacheNodeId, BasicNode>;
    pub type ForwardToLeader = openraft::error::ForwardToLeader<DcacheNodeId, BasicNode>;
    pub type InitializeError = openraft::error::InitializeError<DcacheNodeId, BasicNode>;

    pub type ClientWriteResponse = openraft::raft::ClientWriteResponse<DcacheTypeConfig>;
}

pub async fn start_example_raft_node(
    node_id: DcacheNodeId,
    http_addr: String,
    introducer_addr: String,
    introducer_id: DcacheNodeId,
) -> std::io::Result<()> {
    // Create a configuration for the raft instance.
    let config = Config {
        heartbeat_interval: 500,
        election_timeout_min: 1500,
        election_timeout_max: 3000,
        ..Default::default()
    };

    let salt = "foobar12".into();
    #[cfg(release)]
    todo!("set salt");

    let config = Arc::new(config.validate().unwrap());

    // Create a instance of where the Raft data will be stored.
    let store = Arc::new(DcacheStore::new(salt));

    let (log_store, state_machine) = Adaptor::new(store.clone());

    // Create the network layer that will connect and communicate the raft instances and
    // will be used in conjunction with the store created above.

    let (manager_tx, manager_rx) = tokio::sync::mpsc::channel(1000);
    // let health = Arc::new(crate::network::raft_network_impl::HealthLedger::new(manager_tx));
    // let network = Arc::new(DcacheNetwork::new(health));
    let network = Arc::new(DcacheNetwork::new(manager_tx));

    // Create a local raft instance.
    let raft = Raft::new(
        node_id,
        config.clone(),
        network.clone(),
        log_store,
        state_machine,
    )
    .await
    .unwrap();
    raft.enable_heartbeat(true);
    raft.enable_elect(true);

    //    raft.enable_tick(true);

    // Create an application that will store all the instances created above, this will
    // be later used on the actix-web services.
    let app = DcacheApp {
        id: node_id,
        addr: http_addr.clone(),
        raft,
        store,
        config,
        network,
    };
    let app = Arc::new(app);
    let dcache_service = protobuf::MyDcacheImpl::new(app.clone());

    if introducer_addr == http_addr {
        app.init().await.unwrap();
    }

    let app_copy = app.clone();

    let svc = DcacheServiceServer::new(dcache_service);

    let x = Server::builder()
        .add_service(svc)
        .serve(http_addr.clone().parse().unwrap());
    let server_fut = tokio::spawn(x);

    tokio::time::sleep(std::time::Duration::new(3, 0)).await;

    let url = format!("http://{}", introducer_addr);
    let mut client = DcacheServiceClient::connect(url).await.unwrap();
    client
        .add_learner(Learner {
            id: node_id,
            addr: http_addr,
        })
        .await
        .unwrap();

    let health_metrics_handle =
        crate::network::management::HealthMetrics::spawn(app_copy, 5, manager_rx).await;
    server_fut.await?.unwrap();
    health_metrics_handle.abort();
    Ok(())
}