feat: implement SCDatabase for sqlite

db/db-sqlx-sqlite/.gitignore

@@ -0,0 +1,2 @@
+/target
+.env

db/db-sqlx-sqlite/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "db-sqlx-sqlite"
+version = "0.1.0"
+edition = "2021"
+homepage = "https://github.com/forgeflux-org/starchart"
+repository = "https://github.com/forgeflux-org/starchart"
+documentation = "https://github.con/forgeflux-org/starchart"
+readme = "https://github.com/forgeflux-org/starchart/blob/master/README.md"
+license = "AGPLv3 or later version"
+authors = ["realaravinth <realaravinth@batsense.net>"]
+include = ["./mgrations/"]
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+sqlx = { version = "0.5.11", features = [ "sqlite", "time", "offline", "runtime-actix-rustls" ] }
+db-core = {path = "../db-core"}
+async-trait = "0.1.51"
+
+[dev-dependencies]
+actix-rt = "2"
+sqlx = { version = "0.5.11", features = [ "runtime-actix-rustls", "postgres", "time", "offline" ] }
+db-core = {path = "../db-core", features = ["test"]}

db/db-sqlx-sqlite/migrations/20220405113942_world_forges.sql

@@ -0,0 +1,49 @@
+CREATE TABLE IF NOT EXISTS starchart_forge_type (
+    name VARCHAR(30) NOT NULL UNIQUE,
+    ID INTEGER PRIMARY KEY NOT NULL
+);
+
+INSERT OR IGNORE INTO starchart_forge_type (name) VALUES('gitea');
+
+CREATE TABLE IF NOT EXISTS starchart_forges (
+	forge_type INTEGER NOT NULL REFERENCES starchart_forge_type(ID) ON DELETE CASCADE,
+	hostname TEXT NOT NULL UNIQUE,
+	verified_on INTEGER  NOT NULL,
+	last_crawl_on INTEGER  DEFAULT NULL,
+	ID INTEGER PRIMARY KEY NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS starchart_dns_challenges (
+	hostname TEXT NOT NULL UNIQUE,
+	challenge TEXT NOT NULL UNIQUE,
+	created INTEGER  NOT NULL,
+	ID INTEGER PRIMARY KEY NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS starchart_users (
+	hostname_id INTEGER NOT NULL REFERENCES starchart_forges(ID) ON DELETE CASCADE,
+	username TEXT NOT NULL,
+	html_url TEXT NOT NULL UNIQUE,
+	ID INTEGER PRIMARY KEY NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS starchart_project_topics (
+	name VARCHAR(50) NOT NULL UNIQUE,
+	ID INTEGER PRIMARY KEY NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS starchart_repositories (
+	ID INTEGER PRIMARY KEY NOT NULL,
+	hostname_id INTEGER NOT NULL REFERENCES starchart_forges(ID) ON DELETE CASCADE,
+	owner_id INTEGER NOT NULL REFERENCES starchart_users(ID) ON DELETE CASCADE,
+	name TEXT NOT NULL,
+	description TEXT NOT NULL,
+	html_url TEXT NOT NULL UNIQUE,
+	created INTEGER NOT NULL,
+	last_crawl INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS starchart_repository_topic_mapping (
+	repository_id INTEGER NOT NULL REFERENCES starchart_repositories(ID) ON DELETE CASCADE,
+	topic_id INTEGER NOT NULL REFERENCES starchart_project_topics(ID) ON DELETE CASCADE
+);

db/db-sqlx-sqlite/sqlx-data.json

@@ -0,0 +1,59 @@
+{
+  "db": "SQLite",
+  "0bb37cc79d5ef803285d05d06e6ef93b62c0b532c0298148fe436178761fd70a": {
+    "describe": {
+      "columns": [
+        {
+          "name": "ID",
+          "ordinal": 0,
+          "type_info": "Int64"
+        }
+      ],
+      "nullable": [
+        false
+      ],
+      "parameters": {
+        "Right": 1
+      }
+    },
+    "query": "SELECT ID FROM starchart_forges WHERE hostname = $1"
+  },
+  "30de2d37dd1bd602249cd2adfab499e41105249c20dc58cb360f539d6a782fa1": {
+    "describe": {
+      "columns": [
+        {
+          "name": "ID",
+          "ordinal": 0,
+          "type_info": "Int64"
+        }
+      ],
+      "nullable": [
+        false
+      ],
+      "parameters": {
+        "Right": 1
+      }
+    },
+    "query": "SELECT ID FROM starchart_forge_type WHERE name = $1"
+  },
+  "8c78e074d78291f9d3c4ef3526bae00cb356591edad79a7fb1f20aa7bb681216": {
+    "describe": {
+      "columns": [],
+      "nullable": [],
+      "parameters": {
+        "Right": 3
+      }
+    },
+    "query": "INSERT INTO\n            starchart_forges (hostname, verified_on, forge_type ) \n        VALUES ($1, $2, (SELECT ID FROM starchart_forge_type WHERE name = $3))"
+  },
+  "f52cde89ec10d5ca2151c9df6ae273ee0d52af9f79bb776765cfa716aad6af53": {
+    "describe": {
+      "columns": [],
+      "nullable": [],
+      "parameters": {
+        "Right": 1
+      }
+    },
+    "query": "DELETE FROM starchart_forges WHERE hostname = ($1)"
+  }
+}

db/db-sqlx-sqlite/src/errors.rs

@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2022  Aravinth Manivannan <realaravinth@batsense.net>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+use std::borrow::Cow;
+
+use db_core::dev::*;
+use sqlx::Error;
+
+pub fn map_register_err(e: Error) -> DBError {
+    if let Error::Database(err) = e {
+        if err.code() == Some(Cow::from("2067")) {
+            let msg = err.message();
+            unimplemented!("deal with errors upon insertaion of duplicate values");
+            println!("{}", msg);
+            if msg.contains("starchart_dns_challenges.hostname") {
+                unimplemented!()
+            } else if msg.contains("starchart_dns_challenges.challenge") {
+                unimplemented!()
+            } else if msg.contains("starchart_users.html_url") {
+                unimplemented!()
+            } else if msg.contains("starchart_project_topics.name") {
+                unimplemented!()
+            } else if msg.contains("starchart_repositories.html_url") {
+                unimplemented!()
+            } else if msg.contains("starchart_forge_type.name") {
+                unimplemented!()
+            } else {
+                DBError::DBError(Box::new(Error::Database(err)).into())
+            }
+        } else {
+            DBError::DBError(Box::new(Error::Database(err)).into())
+        }
+    } else {
+        DBError::DBError(Box::new(e).into())
+    }
+}

db/db-sqlx-sqlite/src/lib.rs

@@ -0,0 +1,191 @@
+/*
+ * ForgeFlux StarChart - A federated software forge spider
+ * Copyright (C) 2022  Aravinth Manivannan <realaravinth@batsense.net>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+use db_core::dev::*;
+use std::str::FromStr;
+
+use sqlx::sqlite::SqlitePool;
+use sqlx::sqlite::SqlitePoolOptions;
+use sqlx::types::time::OffsetDateTime;
+
+pub mod errors;
+#[cfg(test)]
+pub mod tests;
+
+#[derive(Clone)]
+pub struct Database {
+    pub pool: SqlitePool,
+}
+
+/// Use an existing database pool
+pub struct Conn(pub SqlitePool);
+
+/// Connect to databse
+pub enum ConnectionOptions {
+    /// fresh connection
+    Fresh(Fresh),
+    /// existing connection
+    Existing(Conn),
+}
+
+pub struct Fresh {
+    pub pool_options: SqlitePoolOptions,
+    pub url: String,
+}
+
+pub mod dev {
+    pub use super::errors::*;
+    pub use super::Database;
+    pub use db_core::dev::*;
+    pub use prelude::*;
+    pub use sqlx::Error;
+}
+
+pub mod prelude {
+    pub use super::*;
+    pub use db_core::prelude::*;
+}
+
+#[async_trait]
+impl Connect for ConnectionOptions {
+    type Pool = Database;
+    async fn connect(self) -> DBResult<Self::Pool> {
+        let pool = match self {
+            Self::Fresh(fresh) => fresh
+                .pool_options
+                .connect(&fresh.url)
+                .await
+                .map_err(|e| DBError::DBError(Box::new(e)))?,
+            Self::Existing(conn) => conn.0,
+        };
+        Ok(Database { pool })
+    }
+}
+
+use dev::*;
+
+#[async_trait]
+impl Migrate for Database {
+    async fn migrate(&self) -> DBResult<()> {
+        sqlx::migrate!("./migrations/")
+            .run(&self.pool)
+            .await
+            .map_err(|e| DBError::DBError(Box::new(e)))?;
+        Ok(())
+    }
+}
+
+#[async_trait]
+impl SCDatabase for Database {
+    /// ping DB
+    async fn ping(&self) -> bool {
+        use sqlx::Connection;
+
+        if let Ok(mut con) = self.pool.acquire().await {
+            con.ping().await.is_ok()
+        } else {
+            false
+        }
+    }
+
+    /// delete forge isntance
+    async fn delete_forge_instance(&self, hostname: &str) -> DBResult<()> {
+        sqlx::query!(
+            "DELETE FROM starchart_forges WHERE hostname = ($1)",
+            hostname,
+        )
+        .execute(&self.pool)
+        .await
+        .map_err(|e| DBError::DBError(Box::new(e)))?;
+        Ok(())
+    }
+
+    /// create forge isntance DB
+    async fn create_forge_isntance(&self, f: &CreateForge) -> DBResult<()> {
+        let now = now_unix_time_stamp();
+        let forge_type = f.forge_type.to_str();
+        sqlx::query!(
+            "INSERT INTO
+            starchart_forges (hostname, verified_on, forge_type ) 
+        VALUES ($1, $2, (SELECT ID FROM starchart_forge_type WHERE name = $3))",
+            f.hostname,
+            now,
+            forge_type,
+        )
+        .execute(&self.pool)
+        .await
+        .map_err(map_register_err)?;
+
+        Ok(())
+    }
+
+    /// check if a forge instance exists
+    async fn forge_exists(&self, hostname: &str) -> DBResult<bool> {
+        match sqlx::query!(
+            "SELECT ID FROM starchart_forges WHERE hostname = $1",
+            hostname
+        )
+        .fetch_one(&self.pool)
+        .await
+        {
+            Ok(_) => Ok(true),
+            Err(Error::RowNotFound) => Ok(false),
+            Err(e) => Err(DBError::DBError(Box::new(e).into())),
+        }
+    }
+
+    async fn forge_type_exists(&self, forge_type: &ForgeImplementation) -> DBResult<bool> {
+        let forge_type = forge_type.to_str();
+        match sqlx::query!(
+            "SELECT ID FROM starchart_forge_type WHERE name = $1",
+            forge_type
+        )
+        .fetch_one(&self.pool)
+        .await
+        {
+            Ok(_) => Ok(true),
+            Err(Error::RowNotFound) => Ok(false),
+            Err(e) => Err(DBError::DBError(Box::new(e)).into()),
+        }
+    }
+}
+
+fn now_unix_time_stamp() -> i64 {
+    OffsetDateTime::now_utc().unix_timestamp()
+}
+
+//
+//#[allow(non_snake_case)]
+//struct InnerGistComment {
+//    ID: i64,
+//    owner: String,
+//    comment: Option<String>,
+//    gist_public_id: String,
+//    created: i64,
+//}
+//
+//impl From<InnerGistComment> for GistComment {
+//    fn from(g: InnerGistComment) -> Self {
+//        Self {
+//            id: g.ID,
+//            owner: g.owner,
+//            comment: g.comment.unwrap(),
+//            gist_public_id: g.gist_public_id,
+//            created: g.created,
+//        }
+//    }
+//}

db/db-sqlx-sqlite/src/tests.rs

@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2022  Aravinth Manivannan <realaravinth@batsense.net>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+use sqlx::sqlite::SqlitePoolOptions;
+use std::env;
+
+use crate::*;
+
+use db_core::tests::*;
+
+#[actix_rt::test]
+async fn everything_works() {
+    const HOSTNAME: &str = "test-gitea.example.com";
+    let msg = CreateForge {
+        hostname: HOSTNAME,
+        forge_type: ForgeImplementation::Gitea,
+    };
+    let url = env::var("SQLITE_DATABASE_URL").expect("Set SQLITE_DATABASE_URL env var");
+    let pool_options = SqlitePoolOptions::new().max_connections(2);
+    let connection_options = ConnectionOptions::Fresh(Fresh { pool_options, url });
+    let db = connection_options.connect().await.unwrap();
+
+    adding_forge_works(&db, msg).await;
+}
+
+#[actix_rt::test]
+async fn forge_type_exists() {
+    let url = env::var("SQLITE_DATABASE_URL").expect("Set SQLITE_DATABASE_URL env var");
+    let pool_options = SqlitePoolOptions::new().max_connections(2);
+    let connection_options = ConnectionOptions::Fresh(Fresh { pool_options, url });
+    let db = connection_options.connect().await.unwrap();
+
+    db.migrate().await.unwrap();
+    forge_type_exists_helper(&db).await;
+}