ForgeFlux/starchart
2
0
Fork 0
Code Issues Pull requests 2 Projects Releases Packages Wiki Activity Actions

Merge branch 'forgeflux-org:master' into master

Browse source
This commit is contained in:
Dat Adithya 2022-07-05 14:26:55 +05:30 committed by GitHub
commit eb97ded32c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 135 additions and 108 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
db/db-core/src/lib.rs
View file

@ -197,6 +197,9 @@ pub trait SCDatabase: std::marker::Send + std::marker::Sync + CloneSPDatabase {
/// check if forge type exists
async fn forge_type_exists(&self, forge_type: &ForgeImplementation) -> DBResult<bool>;
/// Get all forges
async fn get_all_forges(&self, offset: u32, limit: u32) -> DBResult<Vec<Forge>>;
/// add new user to database
async fn add_user(&self, u: &AddUser) -> DBResult<()>;
@ -217,7 +220,7 @@ pub trait SCDatabase: std::marker::Send + std::marker::Sync + CloneSPDatabase {
async fn repository_exists(&self, name: &str, owner: &str, hostname: &str) -> DBResult<bool>;
/// Get all repositories
async fn get_all_repositories(&self, page: u32, limit: u32) -> DBResult<Vec<Repository>>;
async fn get_all_repositories(&self, offset: u32, limit: u32) -> DBResult<Vec<Repository>>;
/// add new repository to database.
async fn create_repository(&self, r: &AddRepository) -> DBResult<()>;

12
db/db-core/src/tests.rs
View file

@ -35,6 +35,18 @@ pub async fn adding_forge_works<'a, T: SCDatabase>(
{
let forge = db.get_forge(create_forge_msg.hostname).await.unwrap();
let forges = db.get_all_forges(0, 10).await.unwrap();
assert_eq!(forges.len(), 1);
assert_eq!(
forges.get(0).as_ref().unwrap().forge_type,
create_forge_msg.forge_type
);
assert_eq!(
forges.get(0).as_ref().unwrap().hostname,
create_forge_msg.hostname
);
assert_eq!(forge.hostname, create_forge_msg.hostname);
assert_eq!(forge.forge_type, create_forge_msg.forge_type);
}

30
db/db-sqlx-sqlite/sqlx-data.json
View file

@ -282,6 +282,36 @@
},
"query": "INSERT INTO \n starchart_users (\n hostname_id, username, html_url,\n profile_photo_html_url, added_on, last_crawl_on\n ) \n VALUES (\n (SELECT ID FROM starchart_forges WHERE hostname = $1), $2, $3, $4, $5, $6)"
},
"c0439c4b2d683c516bd29780cd1e39a7bc75adaebdb450b864eb0b424f401b0c": {
"describe": {
"columns": [
{
"name": "hostname",
"ordinal": 0,
"type_info": "Text"
},
{
"name": "last_crawl_on",
"ordinal": 1,
"type_info": "Int64"
},
{
"name": "name",
"ordinal": 2,
"type_info": "Text"
}
],
"nullable": [
false,
true,
false
],
"parameters": {
"Right": 2
}
},
"query": "SELECT\n\t\thostname,\n\t\tlast_crawl_on,\n\t\tstarchart_forge_type.name\n FROM\n starchart_forges\n INNER JOIN\n starchart_forge_type\n ON\n starchart_forges.forge_type = starchart_forge_type.id\n ORDER BY\n starchart_forges.ID\n LIMIT $1 OFFSET $2;\n "
},
"e00c8a8b0dbeb4a89a673864055c137365c2ae7bc5daf677bdacb20f21d0fcb2": {
"describe": {
"columns": [],

62
db/db-sqlx-sqlite/src/lib.rs
View file

@ -144,11 +144,6 @@ impl SCDatabase for Database {
/// get forge instance data
async fn get_forge(&self, hostname: &str) -> DBResult<Forge> {
struct InnerForge {
hostname: String,
last_crawl_on: Option<i64>,
name: String,
}
let f = sqlx::query_as!(
InnerForge,
"SELECT
@ -170,13 +165,38 @@ impl SCDatabase for Database {
.await
.map_err(|e| DBError::DBError(Box::new(e)))?;
let f = Forge {
hostname: f.hostname,
last_crawl_on: f.last_crawl_on,
forge_type: ForgeImplementation::from_str(&f.name).unwrap(),
};
Ok(f.into())
}
Ok(f)
/// Get all forges
async fn get_all_forges(&self, offset: u32, limit: u32) -> DBResult<Vec<Forge>> {
let mut inter_forges = sqlx::query_as!(
InnerForge,
"SELECT
hostname,
last_crawl_on,
starchart_forge_type.name
FROM
starchart_forges
INNER JOIN
starchart_forge_type
ON
starchart_forges.forge_type = starchart_forge_type.id
ORDER BY
starchart_forges.ID
LIMIT $1 OFFSET $2;
",
limit,
offset
)
.fetch_all(&self.pool)
.await
.map_err(|e| DBError::DBError(Box::new(e)))?;
let mut forges: Vec<Forge> = Vec::with_capacity(inter_forges.len());
inter_forges.drain(0..).for_each(|f| forges.push(f.into()));
Ok(forges)
}
/// check if a forge instance exists
@ -456,7 +476,7 @@ impl SCDatabase for Database {
}
/// Get all repositories
async fn get_all_repositories(&self, page: u32, limit: u32) -> DBResult<Vec<Repository>> {
async fn get_all_repositories(&self, offset: u32, limit: u32) -> DBResult<Vec<Repository>> {
struct InnerRepository {
/// html link to the repository
pub html_url: String,
@ -499,7 +519,7 @@ ORDER BY
LIMIT $1 OFFSET $2
;",
limit,
page,
offset,
)
.fetch_all(&self.pool)
.await
@ -550,3 +570,19 @@ LIMIT $1 OFFSET $2
fn now_unix_time_stamp() -> i64 {
OffsetDateTime::now_utc().unix_timestamp()
}
struct InnerForge {
hostname: String,
last_crawl_on: Option<i64>,
name: String,
}
impl From<InnerForge> for Forge {
fn from(f: InnerForge) -> Self {
Self {
hostname: f.hostname,
last_crawl_on: f.last_crawl_on,
forge_type: ForgeImplementation::from_str(&f.name).unwrap(),
}
}
}

46
src/forge.rs
View file

@ -1,46 +0,0 @@
/*
* ForgeFlux StarChart - A federated software forge spider
* Copyright © 2022 Aravinth Manivannan <realaravinth@batsense.net>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
use async_trait::async_trait;
use db_core::prelude::*;
#[async_trait]
pub trait SCForge: std::marker::Send + std::marker::Sync + CloneSPForge {
async fn is_forge(&self) -> bool;
async fn get_repositories(&self, limit: usize, page: usize) -> Vec<AddRepository>;
}
/// Trait to clone SCForge
pub trait CloneSPForge {
/// clone DB
fn clone_db(&self) -> Box<dyn SCForge>;
}
impl<T> CloneSPForge for T
where
T: SCForge + Clone + 'static,
{
fn clone_db(&self) -> Box<dyn SCForge> {
Box::new(self.clone())
}
}
impl Clone for Box<dyn SCForge> {
fn clone(&self) -> Self {
(**self).clone_db()
}
}

1
src/main.rs
View file

@ -26,7 +26,6 @@ pub mod db;
pub mod dns;
pub mod errors;
pub mod federate;
pub mod forge;
pub mod pages;
pub mod routes;
pub mod settings;

68
src/pages/chart/home.rs
View file

@ -81,6 +81,20 @@ pub struct Page {
pub page: u32,
}
impl Page {
pub fn next(&self) -> u32 {
self.page + 2
}
pub fn prev(&self) -> u32 {
if self.page == 0 {
1
} else {
self.page
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct OptionalPage {
pub page: Option<u32>,
@ -89,8 +103,8 @@ pub struct OptionalPage {
impl From<OptionalPage> for Page {
fn from(o: OptionalPage) -> Self {
match o.page {
Some(page) => Self { page: page + 1 },
None => Page { page: 2 },
Some(page) => Self { page: page - 1 },
None => Page { page: 0 },
}
}
}
@ -104,7 +118,8 @@ pub async fn home(
let q = q.into_inner();
async fn _home(_ctx: &ArcCtx, db: &BoxDB, p: &Page) -> ServiceResult<Vec<db_core::Repository>> {
const LIMIT: u32 = 10;
let responses = db.get_all_repositories(p.page, LIMIT).await?;
let offset = p.page * LIMIT;
let responses = db.get_all_repositories(offset, LIMIT).await?;
Ok(responses)
}
let q: Page = q.into();
@ -114,12 +129,10 @@ pub async fn home(
PageError::new(HomePage::new(&ctx.settings, &x), e)
})?;
let prev = if q.page == 2 { 1 } else { q.page - 1 };
let payload = HomePagePayload {
repos,
next_page: PAGES.home_next(q.page),
prev_page: PAGES.home_next(prev),
next_page: PAGES.home_next(q.next()),
prev_page: PAGES.home_next(q.prev()),
};
let page = HomePage::page(&ctx.settings, &payload);
@ -134,44 +147,17 @@ mod tests {
fn page_counter_increases() {
use super::*;
#[derive(Debug)]
struct TestPage {
// input
current: u32,
expected_next: u32,
}
let mut page = Page { page: 0 };
impl TestPage {
fn new(current: u32) -> Self {
Self {
current,
expected_next: current + 1,
}
}
}
assert_eq!(page.next(), 2);
assert_eq!(page.prev(), 1);
impl From<&TestPage> for OptionalPage {
fn from(p: &TestPage) -> Self {
Self {
page: Some(p.current),
}
}
}
let mut res = Vec::with_capacity(100);
for i in 0..100 {
res.push(TestPage::new(i));
}
page.page = 1;
assert_eq!(page.next(), 3);
assert_eq!(page.prev(), 1);
let op = OptionalPage { page: None };
let p: Page = op.into();
assert_eq!(p.page, 2);
for i in res.iter() {
let op: OptionalPage = i.into();
let p: Page = op.into();
println!("Checking test case {:?}", i);
assert_eq!(p.page, i.expected_next);
}
assert_eq!(p.page, 0);
}
}

19
src/spider.rs
View file

@ -28,14 +28,21 @@ use crate::federate::ArcFederate;
impl Ctx {
pub async fn crawl(&self, instance_url: &str, db: &BoxDB, federate: &ArcFederate) {
let gitea = Gitea::new(Url::parse(instance_url).unwrap(), self.client.clone());
let forge: Box<dyn SCForge> = Box::new(Gitea::new(
Url::parse(instance_url).unwrap(),
self.client.clone(),
));
if !forge.is_forge().await {
unimplemented!("Forge type unimplemented");
}
let mut page = 1;
let hostname = gitea.get_hostname();
let hostname = forge.get_hostname();
if !db.forge_exists(hostname).await.unwrap() {
info!("[crawl][{hostname}] Creating forge");
let msg = CreateForge {
hostname,
forge_type: gitea.forge_type(),
forge_type: forge.forge_type(),
};
db.create_forge_instance(&msg).await.unwrap();
} else {
@ -51,7 +58,7 @@ impl Ctx {
loop {
info!("[crawl][{hostname}] Crawling. page: {page}");
let res = gitea
let res = forge
.crawl(
self.settings.crawler.items_per_api_call,
page,
@ -65,7 +72,7 @@ impl Ctx {
for (username, u) in res.users.iter() {
if !db
.user_exists(username, Some(gitea.get_hostname()))
.user_exists(username, Some(forge.get_hostname()))
.await
.unwrap()
{
@ -75,7 +82,7 @@ impl Ctx {
federate.create_user(&msg).await.unwrap();
} else {
if !federate
.user_exists(username, gitea.get_hostname())
.user_exists(username, forge.get_hostname())
.await
.unwrap()
{