ForgeFlux/starchart
2
0
Fork 0
Code Issues Pull requests 2 Projects Releases Packages Wiki Activity Actions

feat: create instance the first time it is encountered and load DB from

Browse source 
test env
This commit is contained in:
Aravinth Manivannan 2022-04-13 14:12:29 +05:30
parent f1ea436080
commit c0303ba8c5
Signed by: realaravinth
GPG key ID: AD9F0F08E855ED88
1 changed files with 19 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
src/spider.rs
View file

@ -19,16 +19,28 @@ use std::time::Duration;
use tokio::time; use tokio::time;
use url::Url; use url::Url;
use db_core::prelude::*;
use crate::data::Data; use crate::data::Data;
use crate::db::BoxDB;
use crate::gitea::SearchResults; use crate::gitea::SearchResults;
const REPO_SEARCH_PATH: &str = "/api/v1/repos/search"; const REPO_SEARCH_PATH: &str = "/api/v1/repos/search";
const GITEA_NODEINFO: &str = "/api/v1/nodeinfo"; const GITEA_NODEINFO: &str = "/api/v1/nodeinfo";
impl Data { impl Data {
pub async fn crawl(&self, hostname: &str) -> Vec<SearchResults> { pub async fn crawl(&self, hostname: &str, db: &BoxDB) -> Vec<SearchResults> {
let mut page = 1; let mut page = 1;
let mut url = Url::parse(hostname).unwrap(); let mut url = Url::parse(hostname).unwrap();
let hostname = url.host().as_ref().unwrap().to_string();
if !db.forge_exists(&hostname).await.unwrap() {
let msg = CreateForge {
hostname: &hostname,
forge_type: ForgeImplementation::Gitea,
};
db.create_forge_isntance(&msg).await.unwrap();
}
url.set_path(REPO_SEARCH_PATH); url.set_path(REPO_SEARCH_PATH);
let mut repos = Vec::new(); let mut repos = Vec::new();
loop { loop {
@ -46,7 +58,7 @@ impl Data {
.json() .json()
.await .await
.unwrap(); .unwrap();
// TODO implement save
time::sleep(Duration::new( time::sleep(Duration::new(
self.settings.crawler.wait_before_next_api_call, self.settings.crawler.wait_before_next_api_call,
0, 0,
@ -88,20 +100,20 @@ impl Data {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use crate::tests::sqlx_sqlite;
use crate::settings::Settings;
pub const GITEA_HOST: &str = "http://localhost:8080"; pub const GITEA_HOST: &str = "http://localhost:8080";
#[actix_rt::test] #[actix_rt::test]
async fn is_gitea_works() { async fn is_gitea_works() {
let data = Data::new(Settings::new().unwrap()).await; let (_db, data) = sqlx_sqlite::get_data().await;
assert!(data.is_gitea(GITEA_HOST).await); assert!(data.is_gitea(GITEA_HOST).await);
} }
#[actix_rt::test] #[actix_rt::test]
async fn crawl_gitea() { async fn crawl_gitea() {
let data = Data::new(Settings::new().unwrap()).await; let (db, data) = sqlx_sqlite::get_data().await;
let res = data.crawl(GITEA_HOST).await; let res = data.crawl(GITEA_HOST, &db).await;
let mut elements = 0; let mut elements = 0;
res.iter().for_each(|r| elements += r.data.len()); res.iter().for_each(|r| elements += r.data.len());
assert_eq!(res.len(), 5); assert_eq!(res.len(), 5);