From 1a3206b57aa8d1efcb71a27780cec774016ccb26 Mon Sep 17 00:00:00 2001 From: realaravinth Date: Sat, 4 Jun 2022 21:33:13 +0530 Subject: [PATCH 1/5] chore: rm forge dead code --- src/forge.rs | 46 ---------------------------------------------- src/main.rs | 1 - 2 files changed, 47 deletions(-) delete mode 100644 src/forge.rs diff --git a/src/forge.rs b/src/forge.rs deleted file mode 100644 index 5544071..0000000 --- a/src/forge.rs +++ /dev/null @@ -1,46 +0,0 @@ -/* - * ForgeFlux StarChart - A federated software forge spider - * Copyright © 2022 Aravinth Manivannan - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -use async_trait::async_trait; -use db_core::prelude::*; - -#[async_trait] -pub trait SCForge: std::marker::Send + std::marker::Sync + CloneSPForge { - async fn is_forge(&self) -> bool; - async fn get_repositories(&self, limit: usize, page: usize) -> Vec; -} - -/// Trait to clone SCForge -pub trait CloneSPForge { - /// clone DB - fn clone_db(&self) -> Box; -} - -impl CloneSPForge for T -where - T: SCForge + Clone + 'static, -{ - fn clone_db(&self) -> Box { - Box::new(self.clone()) - } -} - -impl Clone for Box { - fn clone(&self) -> Self { - (**self).clone_db() - } -} diff --git a/src/main.rs b/src/main.rs index 202ff0b..d2a8a29 100644 --- a/src/main.rs +++ b/src/main.rs @@ -26,7 +26,6 @@ pub mod db; pub mod dns; pub mod errors; pub mod federate; -pub mod forge; pub mod pages; pub mod routes; pub mod settings; From ef2707e4ee70fd842e2bcf17374e65fc59fa1296 Mon Sep 17 00:00:00 2001 From: realaravinth Date: Thu, 9 Jun 2022 01:22:23 +0530 Subject: [PATCH 2/5] feat: crawl generic over SCForge implementaiton and add forge type checks --- src/spider.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/spider.rs b/src/spider.rs index 9d0d97f..ff9a925 100644 --- a/src/spider.rs +++ b/src/spider.rs @@ -28,14 +28,21 @@ use crate::federate::ArcFederate; impl Ctx { pub async fn crawl(&self, instance_url: &str, db: &BoxDB, federate: &ArcFederate) { - let gitea = Gitea::new(Url::parse(instance_url).unwrap(), self.client.clone()); + let forge: Box = Box::new(Gitea::new( + Url::parse(instance_url).unwrap(), + self.client.clone(), + )); + if !forge.is_forge().await { + unimplemented!("Forge type unimplemented"); + } + let mut page = 1; - let hostname = gitea.get_hostname(); + let hostname = forge.get_hostname(); if !db.forge_exists(hostname).await.unwrap() { info!("[crawl][{hostname}] Creating forge"); let msg = CreateForge { hostname, - forge_type: gitea.forge_type(), + forge_type: forge.forge_type(), }; db.create_forge_isntance(&msg).await.unwrap(); } else { @@ -51,7 +58,7 @@ impl Ctx { loop { info!("[crawl][{hostname}] Crawling. page: {page}"); - let res = gitea + let res = forge .crawl( self.settings.crawler.items_per_api_call, page, @@ -65,7 +72,7 @@ impl Ctx { for (username, u) in res.users.iter() { if !db - .user_exists(username, Some(gitea.get_hostname())) + .user_exists(username, Some(forge.get_hostname())) .await .unwrap() { @@ -75,7 +82,7 @@ impl Ctx { federate.create_user(&msg).await.unwrap(); } else { if !federate - .user_exists(username, gitea.get_hostname()) + .user_exists(username, forge.get_hostname()) .await .unwrap() { From 1ec46670ec3fd42f907283f49539899197d3fd67 Mon Sep 17 00:00:00 2001 From: realaravinth Date: Thu, 9 Jun 2022 14:36:22 +0530 Subject: [PATCH 3/5] feat: def interface to get all forges --- db/db-core/src/lib.rs | 5 ++++- db/db-core/src/tests.rs | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/db/db-core/src/lib.rs b/db/db-core/src/lib.rs index bdf79dd..457189e 100644 --- a/db/db-core/src/lib.rs +++ b/db/db-core/src/lib.rs @@ -197,6 +197,9 @@ pub trait SCDatabase: std::marker::Send + std::marker::Sync + CloneSPDatabase { /// check if forge type exists async fn forge_type_exists(&self, forge_type: &ForgeImplementation) -> DBResult; + /// Get all forges + async fn get_all_forges(&self, offset: u32, limit: u32) -> DBResult>; + /// add new user to database async fn add_user(&self, u: &AddUser) -> DBResult<()>; @@ -217,7 +220,7 @@ pub trait SCDatabase: std::marker::Send + std::marker::Sync + CloneSPDatabase { async fn repository_exists(&self, name: &str, owner: &str, hostname: &str) -> DBResult; /// Get all repositories - async fn get_all_repositories(&self, page: u32, limit: u32) -> DBResult>; + async fn get_all_repositories(&self, offset: u32, limit: u32) -> DBResult>; /// add new repository to database. async fn create_repository(&self, r: &AddRepository) -> DBResult<()>; diff --git a/db/db-core/src/tests.rs b/db/db-core/src/tests.rs index c347374..eb3ee3a 100644 --- a/db/db-core/src/tests.rs +++ b/db/db-core/src/tests.rs @@ -35,6 +35,18 @@ pub async fn adding_forge_works<'a, T: SCDatabase>( { let forge = db.get_forge(create_forge_msg.hostname).await.unwrap(); + let forges = db.get_all_forges(0, 10).await.unwrap(); + assert_eq!(forges.len(), 1); + + assert_eq!( + forges.get(0).as_ref().unwrap().forge_type, + create_forge_msg.forge_type + ); + assert_eq!( + forges.get(0).as_ref().unwrap().hostname, + create_forge_msg.hostname + ); + assert_eq!(forge.hostname, create_forge_msg.hostname); assert_eq!(forge.forge_type, create_forge_msg.forge_type); } From 7e1903f80784c7abb7bd5f2176fcd60b1faecb74 Mon Sep 17 00:00:00 2001 From: realaravinth Date: Thu, 9 Jun 2022 14:38:52 +0530 Subject: [PATCH 4/5] feat: sqlx postgres: impl interface to get all forges --- db/db-sqlx-sqlite/sqlx-data.json | 30 ++++++++++++++++ db/db-sqlx-sqlite/src/lib.rs | 62 +++++++++++++++++++++++++------- 2 files changed, 79 insertions(+), 13 deletions(-) diff --git a/db/db-sqlx-sqlite/sqlx-data.json b/db/db-sqlx-sqlite/sqlx-data.json index 76a09ad..6f38c5a 100644 --- a/db/db-sqlx-sqlite/sqlx-data.json +++ b/db/db-sqlx-sqlite/sqlx-data.json @@ -282,6 +282,36 @@ }, "query": "INSERT INTO \n starchart_users (\n hostname_id, username, html_url,\n profile_photo_html_url, added_on, last_crawl_on\n ) \n VALUES (\n (SELECT ID FROM starchart_forges WHERE hostname = $1), $2, $3, $4, $5, $6)" }, + "c0439c4b2d683c516bd29780cd1e39a7bc75adaebdb450b864eb0b424f401b0c": { + "describe": { + "columns": [ + { + "name": "hostname", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "last_crawl_on", + "ordinal": 1, + "type_info": "Int64" + }, + { + "name": "name", + "ordinal": 2, + "type_info": "Text" + } + ], + "nullable": [ + false, + true, + false + ], + "parameters": { + "Right": 2 + } + }, + "query": "SELECT\n\t\thostname,\n\t\tlast_crawl_on,\n\t\tstarchart_forge_type.name\n FROM\n starchart_forges\n INNER JOIN\n starchart_forge_type\n ON\n starchart_forges.forge_type = starchart_forge_type.id\n ORDER BY\n starchart_forges.ID\n LIMIT $1 OFFSET $2;\n " + }, "e00c8a8b0dbeb4a89a673864055c137365c2ae7bc5daf677bdacb20f21d0fcb2": { "describe": { "columns": [], diff --git a/db/db-sqlx-sqlite/src/lib.rs b/db/db-sqlx-sqlite/src/lib.rs index 0460d03..8e6ec68 100644 --- a/db/db-sqlx-sqlite/src/lib.rs +++ b/db/db-sqlx-sqlite/src/lib.rs @@ -144,11 +144,6 @@ impl SCDatabase for Database { /// get forge isntance data async fn get_forge(&self, hostname: &str) -> DBResult { - struct InnerForge { - hostname: String, - last_crawl_on: Option, - name: String, - } let f = sqlx::query_as!( InnerForge, "SELECT @@ -170,13 +165,38 @@ impl SCDatabase for Database { .await .map_err(|e| DBError::DBError(Box::new(e)))?; - let f = Forge { - hostname: f.hostname, - last_crawl_on: f.last_crawl_on, - forge_type: ForgeImplementation::from_str(&f.name).unwrap(), - }; + Ok(f.into()) + } - Ok(f) + /// Get all forges + async fn get_all_forges(&self, offset: u32, limit: u32) -> DBResult> { + let mut inter_forges = sqlx::query_as!( + InnerForge, + "SELECT + hostname, + last_crawl_on, + starchart_forge_type.name + FROM + starchart_forges + INNER JOIN + starchart_forge_type + ON + starchart_forges.forge_type = starchart_forge_type.id + ORDER BY + starchart_forges.ID + LIMIT $1 OFFSET $2; + ", + limit, + offset + ) + .fetch_all(&self.pool) + .await + .map_err(|e| DBError::DBError(Box::new(e)))?; + + let mut forges: Vec = Vec::with_capacity(inter_forges.len()); + inter_forges.drain(0..).for_each(|f| forges.push(f.into())); + + Ok(forges) } /// check if a forge instance exists @@ -456,7 +476,7 @@ impl SCDatabase for Database { } /// Get all repositories - async fn get_all_repositories(&self, page: u32, limit: u32) -> DBResult> { + async fn get_all_repositories(&self, offset: u32, limit: u32) -> DBResult> { struct InnerRepository { /// html link to the repository pub html_url: String, @@ -499,7 +519,7 @@ ORDER BY LIMIT $1 OFFSET $2 ;", limit, - page, + offset, ) .fetch_all(&self.pool) .await @@ -550,3 +570,19 @@ LIMIT $1 OFFSET $2 fn now_unix_time_stamp() -> i64 { OffsetDateTime::now_utc().unix_timestamp() } + +struct InnerForge { + hostname: String, + last_crawl_on: Option, + name: String, +} + +impl From for Forge { + fn from(f: InnerForge) -> Self { + Self { + hostname: f.hostname, + last_crawl_on: f.last_crawl_on, + forge_type: ForgeImplementation::from_str(&f.name).unwrap(), + } + } +} From aba71115a725f48f6a5044edfbe1471a9a53c3cc Mon Sep 17 00:00:00 2001 From: realaravinth Date: Thu, 9 Jun 2022 15:03:14 +0530 Subject: [PATCH 5/5] fix: page counter increment --- src/pages/chart/home.rs | 68 ++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 41 deletions(-) diff --git a/src/pages/chart/home.rs b/src/pages/chart/home.rs index c3741fc..823fb46 100644 --- a/src/pages/chart/home.rs +++ b/src/pages/chart/home.rs @@ -81,6 +81,20 @@ pub struct Page { pub page: u32, } +impl Page { + pub fn next(&self) -> u32 { + self.page + 2 + } + + pub fn prev(&self) -> u32 { + if self.page == 0 { + 1 + } else { + self.page + } + } +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct OptionalPage { pub page: Option, @@ -89,8 +103,8 @@ pub struct OptionalPage { impl From for Page { fn from(o: OptionalPage) -> Self { match o.page { - Some(page) => Self { page: page + 1 }, - None => Page { page: 2 }, + Some(page) => Self { page: page - 1 }, + None => Page { page: 0 }, } } } @@ -104,7 +118,8 @@ pub async fn home( let q = q.into_inner(); async fn _home(_ctx: &ArcCtx, db: &BoxDB, p: &Page) -> ServiceResult> { const LIMIT: u32 = 10; - let responses = db.get_all_repositories(p.page, LIMIT).await?; + let offset = p.page * LIMIT; + let responses = db.get_all_repositories(offset, LIMIT).await?; Ok(responses) } let q: Page = q.into(); @@ -114,12 +129,10 @@ pub async fn home( PageError::new(HomePage::new(&ctx.settings, &x), e) })?; - let prev = if q.page == 2 { 1 } else { q.page - 1 }; - let payload = HomePagePayload { repos, - next_page: PAGES.home_next(q.page), - prev_page: PAGES.home_next(prev), + next_page: PAGES.home_next(q.next()), + prev_page: PAGES.home_next(q.prev()), }; let page = HomePage::page(&ctx.settings, &payload); @@ -134,44 +147,17 @@ mod tests { fn page_counter_increases() { use super::*; - #[derive(Debug)] - struct TestPage { - // input - current: u32, - expected_next: u32, - } + let mut page = Page { page: 0 }; - impl TestPage { - fn new(current: u32) -> Self { - Self { - current, - expected_next: current + 1, - } - } - } + assert_eq!(page.next(), 2); + assert_eq!(page.prev(), 1); - impl From<&TestPage> for OptionalPage { - fn from(p: &TestPage) -> Self { - Self { - page: Some(p.current), - } - } - } - - let mut res = Vec::with_capacity(100); - for i in 0..100 { - res.push(TestPage::new(i)); - } + page.page = 1; + assert_eq!(page.next(), 3); + assert_eq!(page.prev(), 1); let op = OptionalPage { page: None }; let p: Page = op.into(); - assert_eq!(p.page, 2); - - for i in res.iter() { - let op: OptionalPage = i.into(); - let p: Page = op.into(); - println!("Checking test case {:?}", i); - assert_eq!(p.page, i.expected_next); - } + assert_eq!(p.page, 0); } }