Merge branch 'forgeflux-org:master' into master

db/db-core/src/lib.rs

@@ -197,6 +197,9 @@ pub trait SCDatabase: std::marker::Send + std::marker::Sync + CloneSPDatabase {
     /// check if forge type exists
     async fn forge_type_exists(&self, forge_type: &ForgeImplementation) -> DBResult<bool>;
 
+    /// Get all forges
+    async fn get_all_forges(&self, offset: u32, limit: u32) -> DBResult<Vec<Forge>>;
+
     /// add new user to database
     async fn add_user(&self, u: &AddUser) -> DBResult<()>;
 
@@ -217,7 +220,7 @@ pub trait SCDatabase: std::marker::Send + std::marker::Sync + CloneSPDatabase {
     async fn repository_exists(&self, name: &str, owner: &str, hostname: &str) -> DBResult<bool>;
 
     /// Get all repositories
-    async fn get_all_repositories(&self, page: u32, limit: u32) -> DBResult<Vec<Repository>>;
+    async fn get_all_repositories(&self, offset: u32, limit: u32) -> DBResult<Vec<Repository>>;
 
     /// add new repository to database.
     async fn create_repository(&self, r: &AddRepository) -> DBResult<()>;

db/db-core/src/tests.rs

@@ -35,6 +35,18 @@ pub async fn adding_forge_works<'a, T: SCDatabase>(
 
     {
         let forge = db.get_forge(create_forge_msg.hostname).await.unwrap();
+        let forges = db.get_all_forges(0, 10).await.unwrap();
+        assert_eq!(forges.len(), 1);
+
+        assert_eq!(
+            forges.get(0).as_ref().unwrap().forge_type,
+            create_forge_msg.forge_type
+        );
+        assert_eq!(
+            forges.get(0).as_ref().unwrap().hostname,
+            create_forge_msg.hostname
+        );
+
         assert_eq!(forge.hostname, create_forge_msg.hostname);
         assert_eq!(forge.forge_type, create_forge_msg.forge_type);
     }

db/db-sqlx-sqlite/sqlx-data.json

@@ -282,6 +282,36 @@
     },
     "query": "INSERT INTO \n                    starchart_users (\n                        hostname_id, username, html_url,\n                        profile_photo_html_url, added_on, last_crawl_on\n                    ) \n            VALUES (\n                    (SELECT ID FROM starchart_forges WHERE hostname = $1), $2, $3, $4, $5, $6)"
   },
+  "c0439c4b2d683c516bd29780cd1e39a7bc75adaebdb450b864eb0b424f401b0c": {
+    "describe": {
+      "columns": [
+        {
+          "name": "hostname",
+          "ordinal": 0,
+          "type_info": "Text"
+        },
+        {
+          "name": "last_crawl_on",
+          "ordinal": 1,
+          "type_info": "Int64"
+        },
+        {
+          "name": "name",
+          "ordinal": 2,
+          "type_info": "Text"
+        }
+      ],
+      "nullable": [
+        false,
+        true,
+        false
+      ],
+      "parameters": {
+        "Right": 2
+      }
+    },
+    "query": "SELECT\n\t\thostname,\n\t\tlast_crawl_on,\n\t\tstarchart_forge_type.name\n        FROM\n            starchart_forges\n        INNER JOIN\n            starchart_forge_type\n        ON\n            starchart_forges.forge_type = starchart_forge_type.id\n    ORDER BY\n        starchart_forges.ID\n    LIMIT $1 OFFSET $2;\n        "
+  },
   "e00c8a8b0dbeb4a89a673864055c137365c2ae7bc5daf677bdacb20f21d0fcb2": {
     "describe": {
       "columns": [],

db/db-sqlx-sqlite/src/lib.rs

@@ -144,11 +144,6 @@ impl SCDatabase for Database {
 
     /// get forge instance data
     async fn get_forge(&self, hostname: &str) -> DBResult<Forge> {
-        struct InnerForge {
-            hostname: String,
-            last_crawl_on: Option<i64>,
-            name: String,
-        }
         let f = sqlx::query_as!(
             InnerForge,
             "SELECT 
@@ -170,13 +165,38 @@ impl SCDatabase for Database {
         .await
         .map_err(|e| DBError::DBError(Box::new(e)))?;
 
-        let f = Forge {
+        Ok(f.into())
-            hostname: f.hostname,
+    }
-            last_crawl_on: f.last_crawl_on,
-            forge_type: ForgeImplementation::from_str(&f.name).unwrap(),
-        };
 
-        Ok(f)
+    /// Get all forges
+    async fn get_all_forges(&self, offset: u32, limit: u32) -> DBResult<Vec<Forge>> {
+        let mut inter_forges = sqlx::query_as!(
+            InnerForge,
+            "SELECT
+		hostname,
+		last_crawl_on,
+		starchart_forge_type.name
+        FROM
+            starchart_forges
+        INNER JOIN
+            starchart_forge_type
+        ON
+            starchart_forges.forge_type = starchart_forge_type.id
+    ORDER BY
+        starchart_forges.ID
+    LIMIT $1 OFFSET $2;
+        ",
+            limit,
+            offset
+        )
+        .fetch_all(&self.pool)
+        .await
+        .map_err(|e| DBError::DBError(Box::new(e)))?;
+
+        let mut forges: Vec<Forge> = Vec::with_capacity(inter_forges.len());
+        inter_forges.drain(0..).for_each(|f| forges.push(f.into()));
+
+        Ok(forges)
     }
 
     /// check if a forge instance exists
@@ -456,7 +476,7 @@ impl SCDatabase for Database {
     }
 
     /// Get all repositories
-    async fn get_all_repositories(&self, page: u32, limit: u32) -> DBResult<Vec<Repository>> {
+    async fn get_all_repositories(&self, offset: u32, limit: u32) -> DBResult<Vec<Repository>> {
         struct InnerRepository {
             /// html link to the repository
             pub html_url: String,
@@ -499,7 +519,7 @@ ORDER BY
 LIMIT $1 OFFSET $2
 ;",
             limit,
-            page,
+            offset,
         )
         .fetch_all(&self.pool)
         .await
@@ -550,3 +570,19 @@ LIMIT $1 OFFSET $2
 fn now_unix_time_stamp() -> i64 {
     OffsetDateTime::now_utc().unix_timestamp()
 }
+
+struct InnerForge {
+    hostname: String,
+    last_crawl_on: Option<i64>,
+    name: String,
+}
+
+impl From<InnerForge> for Forge {
+    fn from(f: InnerForge) -> Self {
+        Self {
+            hostname: f.hostname,
+            last_crawl_on: f.last_crawl_on,
+            forge_type: ForgeImplementation::from_str(&f.name).unwrap(),
+        }
+    }
+}

src/forge.rs

@@ -1,46 +0,0 @@
-/*
- * ForgeFlux StarChart - A federated software forge spider
- * Copyright © 2022 Aravinth Manivannan <realaravinth@batsense.net>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-use async_trait::async_trait;
-use db_core::prelude::*;
-
-#[async_trait]
-pub trait SCForge: std::marker::Send + std::marker::Sync + CloneSPForge {
-    async fn is_forge(&self) -> bool;
-    async fn get_repositories(&self, limit: usize, page: usize) -> Vec<AddRepository>;
-}
-
-/// Trait to clone SCForge
-pub trait CloneSPForge {
-    /// clone DB
-    fn clone_db(&self) -> Box<dyn SCForge>;
-}
-
-impl<T> CloneSPForge for T
-where
-    T: SCForge + Clone + 'static,
-{
-    fn clone_db(&self) -> Box<dyn SCForge> {
-        Box::new(self.clone())
-    }
-}
-
-impl Clone for Box<dyn SCForge> {
-    fn clone(&self) -> Self {
-        (**self).clone_db()
-    }
-}

src/main.rs

@@ -26,7 +26,6 @@ pub mod db;
 pub mod dns;
 pub mod errors;
 pub mod federate;
-pub mod forge;
 pub mod pages;
 pub mod routes;
 pub mod settings;

src/pages/chart/home.rs

@@ -81,6 +81,20 @@ pub struct Page {
     pub page: u32,
 }
 
+impl Page {
+    pub fn next(&self) -> u32 {
+        self.page + 2
+    }
+
+    pub fn prev(&self) -> u32 {
+        if self.page == 0 {
+            1
+        } else {
+            self.page
+        }
+    }
+}
+
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct OptionalPage {
     pub page: Option<u32>,
@@ -89,8 +103,8 @@ pub struct OptionalPage {
 impl From<OptionalPage> for Page {
     fn from(o: OptionalPage) -> Self {
         match o.page {
-            Some(page) => Self { page: page + 1 },
+            Some(page) => Self { page: page - 1 },
-            None => Page { page: 2 },
+            None => Page { page: 0 },
         }
     }
 }
@@ -104,7 +118,8 @@ pub async fn home(
     let q = q.into_inner();
     async fn _home(_ctx: &ArcCtx, db: &BoxDB, p: &Page) -> ServiceResult<Vec<db_core::Repository>> {
         const LIMIT: u32 = 10;
-        let responses = db.get_all_repositories(p.page, LIMIT).await?;
+        let offset = p.page * LIMIT;
+        let responses = db.get_all_repositories(offset, LIMIT).await?;
         Ok(responses)
     }
     let q: Page = q.into();
@@ -114,12 +129,10 @@ pub async fn home(
         PageError::new(HomePage::new(&ctx.settings, &x), e)
     })?;
 
-    let prev = if q.page == 2 { 1 } else { q.page - 1 };
-
     let payload = HomePagePayload {
         repos,
-        next_page: PAGES.home_next(q.page),
+        next_page: PAGES.home_next(q.next()),
-        prev_page: PAGES.home_next(prev),
+        prev_page: PAGES.home_next(q.prev()),
     };
     let page = HomePage::page(&ctx.settings, &payload);
 
@@ -134,44 +147,17 @@ mod tests {
     fn page_counter_increases() {
         use super::*;
 
-        #[derive(Debug)]
+        let mut page = Page { page: 0 };
-        struct TestPage {
-            // input
-            current: u32,
-            expected_next: u32,
-        }
 
-        impl TestPage {
+        assert_eq!(page.next(), 2);
-            fn new(current: u32) -> Self {
+        assert_eq!(page.prev(), 1);
-                Self {
-                    current,
-                    expected_next: current + 1,
-                }
-            }
-        }
 
-        impl From<&TestPage> for OptionalPage {
+        page.page = 1;
-            fn from(p: &TestPage) -> Self {
+        assert_eq!(page.next(), 3);
-                Self {
+        assert_eq!(page.prev(), 1);
-                    page: Some(p.current),
-                }
-            }
-        }
-
-        let mut res = Vec::with_capacity(100);
-        for i in 0..100 {
-            res.push(TestPage::new(i));
-        }
 
         let op = OptionalPage { page: None };
         let p: Page = op.into();
-        assert_eq!(p.page, 2);
+        assert_eq!(p.page, 0);
-
-        for i in res.iter() {
-            let op: OptionalPage = i.into();
-            let p: Page = op.into();
-            println!("Checking test case {:?}", i);
-            assert_eq!(p.page, i.expected_next);
-        }
     }
 }

src/spider.rs

@@ -28,14 +28,21 @@ use crate::federate::ArcFederate;
 
 impl Ctx {
     pub async fn crawl(&self, instance_url: &str, db: &BoxDB, federate: &ArcFederate) {
-        let gitea = Gitea::new(Url::parse(instance_url).unwrap(), self.client.clone());
+        let forge: Box<dyn SCForge> = Box::new(Gitea::new(
+            Url::parse(instance_url).unwrap(),
+            self.client.clone(),
+        ));
+        if !forge.is_forge().await {
+            unimplemented!("Forge type unimplemented");
+        }
+
         let mut page = 1;
-        let hostname = gitea.get_hostname();
+        let hostname = forge.get_hostname();
         if !db.forge_exists(hostname).await.unwrap() {
             info!("[crawl][{hostname}] Creating forge");
             let msg = CreateForge {
                 hostname,
-                forge_type: gitea.forge_type(),
+                forge_type: forge.forge_type(),
             };
             db.create_forge_instance(&msg).await.unwrap();
         } else {
@@ -51,7 +58,7 @@ impl Ctx {
 
         loop {
             info!("[crawl][{hostname}] Crawling. page: {page}");
-            let res = gitea
+            let res = forge
                 .crawl(
                     self.settings.crawler.items_per_api_call,
                     page,
@@ -65,7 +72,7 @@ impl Ctx {
 
             for (username, u) in res.users.iter() {
                 if !db
-                    .user_exists(username, Some(gitea.get_hostname()))
+                    .user_exists(username, Some(forge.get_hostname()))
                     .await
                     .unwrap()
                 {
@@ -75,7 +82,7 @@ impl Ctx {
                     federate.create_user(&msg).await.unwrap();
                 } else {
                     if !federate
-                        .user_exists(username, gitea.get_hostname())
+                        .user_exists(username, forge.get_hostname())
                         .await
                         .unwrap()
                     {