ForgeFlux/starchart
2
0
Fork 0
Code Issues Pull requests 2 Projects Releases Packages Wiki Activity Actions

feat: add federation file writes in crawl method

Browse source
This commit is contained in:
Aravinth Manivannan 2022-05-17 20:12:30 +05:30
parent f03bd5aaae
commit d0fd8166e3
Signed by: realaravinth
GPG key ID: AD9F0F08E855ED88
2 changed files with 16 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
src/spider.rs
View file

@ -26,9 +26,10 @@ use gitea::Gitea;
use crate::ctx::Ctx;
use crate::db::BoxDB;
use crate::federate::BoxFederate;
impl Ctx {
pub async fn crawl(&self, instance_url: &str, db: &BoxDB) {
pub async fn crawl(&self, instance_url: &str, db: &BoxDB, federate: &BoxFederate) {
let gitea = Gitea::new(Url::parse(instance_url).unwrap(), self.client.clone());
let mut page = 1;
let hostname = gitea.get_hostname();
@ -38,6 +39,7 @@ impl Ctx {
forge_type: gitea.forge_type(),
};
db.create_forge_isntance(&msg).await.unwrap();
federate.create_forge_isntance(&msg).await.unwrap();
}
loop {
@ -62,6 +64,7 @@ impl Ctx {
{
let msg = u.as_ref().into();
db.add_user(&msg).await.unwrap();
federate.create_user(&msg).await.unwrap();
}
}
@ -73,6 +76,7 @@ impl Ctx {
{
let msg = r.into();
db.create_repository(&msg).await.unwrap();
federate.create_repository(&msg).await.unwrap();
}
}
@ -94,8 +98,8 @@ mod tests {
#[actix_rt::test]
async fn crawl_gitea() {
let (db, ctx) = sqlx_sqlite::get_ctx().await;
ctx.crawl(GITEA_HOST, &db).await;
let (db, ctx, federate, _tmp_dir) = sqlx_sqlite::get_ctx().await;
ctx.crawl(GITEA_HOST, &db, &federate).await;
let hostname = get_hostname(&Url::parse(GITEA_HOST).unwrap());
assert!(db.forge_exists(&hostname).await.unwrap());
assert!(db

11
src/tests.rs
View file

@ -21,6 +21,7 @@ pub use std::sync::Arc;
use crate::ctx::Ctx;
pub use crate::db::BoxDB;
pub use crate::federate::{get_federate, BoxFederate};
use crate::settings::{DBType, Settings};
//pub mod sqlx_postgres {
@ -38,8 +39,9 @@ use crate::settings::{DBType, Settings};
pub mod sqlx_sqlite {
use super::*;
use crate::db::sqlite;
use mktemp::Temp;
pub async fn get_ctx() -> (BoxDB, Arc<Ctx>) {
pub async fn get_ctx() -> (BoxDB, Arc<Ctx>, BoxFederate, Temp) {
let url = env::var("SQLITE_DATABASE_URL").unwrap();
env::set_var("DATABASE_URL", &url);
println!("found db url: {url}");
@ -47,6 +49,11 @@ pub mod sqlx_sqlite {
settings.database.url = url.clone();
settings.database.database_type = DBType::Sqlite;
let db = sqlite::get_data(Some(settings.clone())).await;
(db, Ctx::new(settings).await)
let tmp_dir = Temp::new_dir().unwrap();
settings.repository.root = tmp_dir.to_str().unwrap().to_string();
let federate = get_federate(Some(settings.clone())).await;
(db, Ctx::new(settings).await, federate, tmp_dir)
}
}