feat: read from db and plot difficulty factor vs time

with random string and random salt for difficulty factors

.env.example

@@ -0,0 +1,3 @@
+#export DATABASE_URL=sqlite:///src/atm/code/mcaptcha/powd/db.sqlite
+export DATABASE_URL="postgres://postgres:password@localhost:5420/postgres"
+export db=powd-postgres

Cargo.toml

@@ -6,6 +6,7 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+actix-rt = "2.8.0"
 bincode = "1.3.3"
 clap = { version = "4.3.11", features = ["derive"] }
 log = "0.4.19"
@@ -16,3 +17,5 @@ rayon = "1.7.0"
 serde = { version = "1.0.169", features = ["derive"] }
 serde_json = "1.0.100"
 sled = "0.34.7"
+sqlx = { version = "0.7", features = [ "runtime-tokio", "postgres" ] }
+url = { version = "2.4.0", features = ["serde"] }

migrations/20230802225710_logs.sql

@@ -0,0 +1,7 @@
+CREATE TABLE IF NOT EXISTS logs (
+    string VARCHAR(100) NOT NULL,
+    salt VARCHAR(100) NOT NULL,
+    time INTEGER NOT NULL,
+	difficulty INTEGER NOT NULL UNIQUE,
+	ID SERIAL PRIMARY KEY NOT NULL
+);

plot.py

@@ -0,0 +1,23 @@
+import psycopg2
+import matplotlib.pyplot as plt
+from mplcursors import cursor  # separate package must be installed
+
+# Connect to your postgres DB
+conn = psycopg2.connect("postgres://postgres:password@localhost:5420/postgres")
+
+# Open a cursor to perform database operations
+cur = conn.cursor()
+
+# Execute a query
+cur.execute("SELECT difficulty FROM logs ORDER BY difficulty")# LIMIT 100000")
+# Retrieve query results
+difficulty = cur.fetchall()
+
+cur.execute("SELECT time FROM logs ORDER BY difficulty")# LIMIT 100000")
+time = cur.fetchall()
+
+# reproducible sample data as a pandas dataframe
+#plt.figure(figsize=(4096, 4096))
+plt.scatter(difficulty, time)
+#cursor(hover=True)
+plt.show()

requirements.txt

@@ -0,0 +1,13 @@
+contourpy==1.1.0
+cycler==0.11.0
+fonttools==4.42.0
+kiwisolver==1.4.4
+matplotlib==3.7.2
+mplcursors==0.5.2
+numpy==1.25.2
+packaging==23.1
+Pillow==10.0.0
+psycopg2==2.9.6
+pyparsing==3.0.9
+python-dateutil==2.8.2
+six==1.16.0

sliqte.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+cargo run --release -- sqlite --sqlite $DATABASE_URL --sled db2/1m3kto1m4k
+cargo run --release -- sqlite --sqlite $DATABASE_URL --sled db2/1m4kto1m5k
+cargo run --release -- sqlite --sqlite $DATABASE_URL --sled db2/1m5kto1m6k
+cargo run --release -- sqlite --sqlite $DATABASE_URL --sled db2/1m6kto1m7k
+cargo run --release -- sqlite --sqlite $DATABASE_URL --sled db2/1m7kto1m8k
+cargo run --release -- sqlite --sqlite $DATABASE_URL --sled db2/1m8kto1m9k
+cargo run --release -- sqlite --sqlite $DATABASE_URL --sled db2/1m9kto2m
+cargo run --release -- sqlite --sqlite $DATABASE_URL --sled db2/5kto50k
+cargo run --release -- sqlite --sqlite $DATABASE_URL --sled db2/50kto100k
+cargo run --release -- sqlite --sqlite $DATABASE_URL --sled db2/200kto300k
+cargo run --release -- sqlite --sqlite $DATABASE_URL --sled db2/300kto500k
+cargo run --release -- sqlite --sqlite $DATABASE_URL --sled db2/500kto600k
+cargo run --release -- sqlite --sqlite $DATABASE_URL --sled db2/600kto700k
+cargo run --release -- sqlite --sqlite $DATABASE_URL --sled db2/700kto800k
+cargo run --release -- sqlite --sqlite $DATABASE_URL --sled db2/800kto900k
+cargo run --release -- sqlite --sqlite $DATABASE_URL --sled db2/900kto1m

src/main.rs

@@ -6,6 +6,8 @@ use clap::Parser;
 use pow_sha256::ConfigBuilder;
 use rayon::prelude::*;
 use serde::{Deserialize, Serialize};
+use sqlx::postgres::PgPoolOptions;
+use std::time::Duration;
 use std::time::Instant;
 
 #[derive(Parser)] // requires `derive` feature
@@ -14,6 +16,59 @@ use std::time::Instant;
 enum PoWdCli {
     Generate(GenerateDeriveArgs),
     PrintDB(PrintRes),
+    Sqlite(Sqlite),
+    ForDiff(ForDiff),
+}
+
+#[derive(clap::Args, Debug, Clone)]
+#[command(author, version, about, long_about = None)]
+struct ForDiff {
+    #[arg(long)]
+    factors: String,
+}
+
+impl ForDiff {
+    fn run(&self) {
+        let factors: Vec<u32> = self
+            .factors
+            .split(',')
+            .map(|d| d.parse::<u32>().unwrap())
+            .collect();
+        let mut summary = String::new();
+        for difficulty in factors.iter() {
+            let trials = 100;
+            let mut times: Vec<u128> = Vec::with_capacity(trials);
+
+            for _ in 0..trials {
+                let salt = get_random(32);
+                let string = get_random(32);
+                let pow_config = ConfigBuilder::default().salt(salt.clone()).build().unwrap();
+
+                let start = Instant::now();
+                pow_config.prove_work(&string, *difficulty).unwrap();
+                let finish = Instant::now();
+                let time_elapsed = finish.duration_since(start);
+
+                let time = time_elapsed.as_micros();
+                log::info!("Difficulty factor {difficulty} generated in {time}");
+                times.push(time);
+            }
+            let mean = (times.iter().sum::<u128>()/ trials as u128) as u32 ;
+            let simple_variance: u32 = (times
+                .iter()
+                .map(|d| {
+                    let x = (*d as u32 - mean ) as u32;
+                    x * x
+                })
+                .sum::<u32>())
+                / (trials as u32 - 1);
+            let simple_variance: usize = ((simple_variance as f32).sqrt() as usize);
+            let min = times.iter().min().unwrap();
+            let max = times.iter().max().unwrap();
+            summary = format!("{summary}\ndifficulty: {difficulty} min: {min} max: {max} mean: {mean} variance {simple_variance}");
+        }
+        println!("{summary}");
+    }
 }
 
 #[derive(clap::Args, Debug, Clone)]
@@ -50,7 +105,9 @@ impl GenerateDeriveArgs {
                 let time_elapsed = finish.duration_since(start);
 
                 let time = time_elapsed.as_micros();
-                log::info!("Difficulty factor {difficulty} generated in {time}");
+                if difficulty % 10000 == 0 {
+                    log::info!("Difficulty factor {difficulty} generated in {time}");
+                }
 
                 let log = Log {
                     salt: salt.clone(),
@@ -74,7 +131,26 @@ struct PrintRes {
     db: std::path::PathBuf,
 }
 
-fn main() {
+#[derive(clap::Args, Debug, Clone)]
+#[command(author, version, about, long_about = None)]
+struct Sqlite {
+    #[arg(long)]
+    sled: std::path::PathBuf,
+    #[arg(long)]
+    sqlite: url::Url,
+}
+
+#[derive(clap::Args, Debug, Clone)]
+#[command(author, version, about, long_about = None)]
+struct Plot {
+    #[arg(long)]
+    graph: std::path::PathBuf,
+    #[arg(long)]
+    sqlite: url::Url,
+}
+
+#[actix_rt::main]
+async fn main() {
     std::env::set_var("RUST_LOG", "INFO");
     pretty_env_logger::init();
     match PoWdCli::parse() {
@@ -92,6 +168,37 @@ fn main() {
                 println!("{difficulty}: {}", log.time);
             }
         }
+
+        PoWdCli::Sqlite(args) => {
+            let sqlite_db = PgPoolOptions::new()
+                .max_connections(1)
+                .acquire_timeout(Duration::new(1000, 0))
+                .idle_timeout(Duration::new(1000, 0))
+                .connect(&args.sqlite.to_string())
+                .await
+                .unwrap();
+
+            println!("{:?}", args);
+            let sled_db = sled::open(args.sled).unwrap();
+            for entry in sled_db.iter() {
+                let (difficulty, entry) = entry.unwrap();
+                let log: Log = bincode::deserialize::<Log>(&entry[..]).unwrap();
+                let difficulty: u32 = bincode::deserialize::<u32>(&difficulty).unwrap();
+
+                let time = log.time as i32;
+                sqlx::query!(
+                    "INSERT INTO logs (string, salt, time, difficulty) VALUES ($1, $2, $3, $4) ON CONFLICT(difficulty) DO NOTHING;",
+                log.string,
+                log.salt,
+                time,
+                difficulty as i32,
+                ).execute(&sqlite_db).await.unwrap();
+            }
+        }
+        PoWdCli::ForDiff(args) => {
+            println!("{:?}", args);
+            args.run();
+        }
     }
 }