ForgeFlux/starchart
2
0
Fork 0
Code Issues Pull requests 2 Projects Releases Packages Wiki Activity Actions

feat: crawl generic over SCForge implementaiton and add forge type checks

Browse source
This commit is contained in:
Aravinth Manivannan 2022-06-09 01:22:23 +05:30
parent 1a3206b57a
commit ef2707e4ee
Signed by: realaravinth
GPG key ID: AD9F0F08E855ED88
1 changed files with 13 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
src/spider.rs
View file

@ -28,14 +28,21 @@ use crate::federate::ArcFederate;
impl Ctx {
pub async fn crawl(&self, instance_url: &str, db: &BoxDB, federate: &ArcFederate) {
let gitea = Gitea::new(Url::parse(instance_url).unwrap(), self.client.clone());
let forge: Box<dyn SCForge> = Box::new(Gitea::new(
Url::parse(instance_url).unwrap(),
self.client.clone(),
));
if !forge.is_forge().await {
unimplemented!("Forge type unimplemented");
}
let mut page = 1;
let hostname = gitea.get_hostname();
let hostname = forge.get_hostname();
if !db.forge_exists(hostname).await.unwrap() {
info!("[crawl][{hostname}] Creating forge");
let msg = CreateForge {
hostname,
forge_type: gitea.forge_type(),
forge_type: forge.forge_type(),
};
db.create_forge_isntance(&msg).await.unwrap();
} else {
@ -51,7 +58,7 @@ impl Ctx {
loop {
info!("[crawl][{hostname}] Crawling. page: {page}");
let res = gitea
let res = forge
.crawl(
self.settings.crawler.items_per_api_call,
page,
@ -65,7 +72,7 @@ impl Ctx {
for (username, u) in res.users.iter() {
if !db
.user_exists(username, Some(gitea.get_hostname()))
.user_exists(username, Some(forge.get_hostname()))
.await
.unwrap()
{
@ -75,7 +82,7 @@ impl Ctx {
federate.create_user(&msg).await.unwrap();
} else {
if !federate
.user_exists(username, gitea.get_hostname())
.user_exists(username, forge.get_hostname())
.await
.unwrap()
{