ForgeFlux/starchart
2
0
Fork 0
Code Issues Pull requests 2 Projects Releases Packages Wiki Activity Actions

feat: add logging to crawl method

Browse source
This commit is contained in:
Aravinth Manivannan 2022-05-19 17:35:47 +05:30
parent 0ca420d1ff
commit 593f3d5c17
1 changed files with 6 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/spider.rs
View file

@ -17,6 +17,7 @@
*/ */
use std::time::Duration; use std::time::Duration;
use log::info;
use tokio::time; use tokio::time;
use url::Url; use url::Url;
@ -34,6 +35,7 @@ impl Ctx {
let mut page = 1; let mut page = 1;
let hostname = gitea.get_hostname(); let hostname = gitea.get_hostname();
if !db.forge_exists(hostname).await.unwrap() { if !db.forge_exists(hostname).await.unwrap() {
info!("[crawl][{hostname}] Creating forge");
let msg = CreateForge { let msg = CreateForge {
hostname, hostname,
forge_type: gitea.forge_type(), forge_type: gitea.forge_type(),
@ -43,10 +45,12 @@ impl Ctx {
} }
loop { loop {
info!("[crawl][{hostname}] Crawling. page: {page}");
let res = gitea let res = gitea
.crawl(self.settings.crawler.items_per_api_call, page) .crawl(self.settings.crawler.items_per_api_call, page)
.await; .await;
if res.repos.is_empty() { if res.repos.is_empty() {
info!("[crawl][{hostname}] Finished crawling. pages: {}", page - 1);
break; break;
} }
@ -62,6 +66,7 @@ impl Ctx {
.await .await
.unwrap() .unwrap()
{ {
info!("[crawl][{hostname}] Creating user: {username}");
let msg = u.as_ref().into(); let msg = u.as_ref().into();
db.add_user(&msg).await.unwrap(); db.add_user(&msg).await.unwrap();
federate.create_user(&msg).await.unwrap(); federate.create_user(&msg).await.unwrap();
@ -74,6 +79,7 @@ impl Ctx {
.await .await
.unwrap() .unwrap()
{ {
info!("[crawl][{hostname}] Creating repository: {}", r.name);
let msg = r.into(); let msg = r.into();
db.create_repository(&msg).await.unwrap(); db.create_repository(&msg).await.unwrap();
federate.create_repository(&msg).await.unwrap(); federate.create_repository(&msg).await.unwrap();