ForgeFlux/starchart
2
0
Fork 0
Code Issues Pull requests 2 Projects Releases Packages Wiki Activity Actions

feat: crawl generic over SCForge implementaiton and add forge type checks

Browse source
This commit is contained in:
Aravinth Manivannan 2022-06-09 01:22:23 +05:30
parent 1a3206b57a
commit ef2707e4ee
Signed by: realaravinth
GPG key ID: AD9F0F08E855ED88
1 changed files with 13 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
src/spider.rs
View file

@ -28,14 +28,21 @@ use crate::federate::ArcFederate;
impl Ctx { impl Ctx {
pub async fn crawl(&self, instance_url: &str, db: &BoxDB, federate: &ArcFederate) { pub async fn crawl(&self, instance_url: &str, db: &BoxDB, federate: &ArcFederate) {
let gitea = Gitea::new(Url::parse(instance_url).unwrap(), self.client.clone()); let forge: Box<dyn SCForge> = Box::new(Gitea::new(
Url::parse(instance_url).unwrap(),
self.client.clone(),
));
if !forge.is_forge().await {
unimplemented!("Forge type unimplemented");
}
let mut page = 1; let mut page = 1;
let hostname = gitea.get_hostname(); let hostname = forge.get_hostname();
if !db.forge_exists(hostname).await.unwrap() { if !db.forge_exists(hostname).await.unwrap() {
info!("[crawl][{hostname}] Creating forge"); info!("[crawl][{hostname}] Creating forge");
let msg = CreateForge { let msg = CreateForge {
hostname, hostname,
forge_type: gitea.forge_type(), forge_type: forge.forge_type(),
}; };
db.create_forge_isntance(&msg).await.unwrap(); db.create_forge_isntance(&msg).await.unwrap();
} else { } else {
@ -51,7 +58,7 @@ impl Ctx {
loop { loop {
info!("[crawl][{hostname}] Crawling. page: {page}"); info!("[crawl][{hostname}] Crawling. page: {page}");
let res = gitea let res = forge
.crawl( .crawl(
self.settings.crawler.items_per_api_call, self.settings.crawler.items_per_api_call,
page, page,
@ -65,7 +72,7 @@ impl Ctx {
for (username, u) in res.users.iter() { for (username, u) in res.users.iter() {
if !db if !db
.user_exists(username, Some(gitea.get_hostname())) .user_exists(username, Some(forge.get_hostname()))
.await .await
.unwrap() .unwrap()
{ {
@ -75,7 +82,7 @@ impl Ctx {
federate.create_user(&msg).await.unwrap(); federate.create_user(&msg).await.unwrap();
} else { } else {
if !federate if !federate
.user_exists(username, gitea.get_hostname()) .user_exists(username, forge.get_hostname())
.await .await
.unwrap() .unwrap()
{ {