ForgeFlux/starchart
2
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

fix: archive after each crawl

Browse source
This commit is contained in:
Aravinth Manivannan 2023-02-22 18:10:34 +05:30
parent bc2e1870f5
commit d3fde704ff
Signed by: realaravinth
GPG key ID: AD9F0F08E855ED88
1 changed files with 5 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/spider.rs
View file

@ -48,6 +48,7 @@ impl Ctx {
let msg = CreateForge { let msg = CreateForge {
url: url.clone(), url: url.clone(),
forge_type: forge.forge_type(), forge_type: forge.forge_type(),
import: false,
}; };
db.create_forge_instance(&msg).await.unwrap(); db.create_forge_instance(&msg).await.unwrap();
@ -56,6 +57,7 @@ impl Ctx {
let msg = CreateForge { let msg = CreateForge {
url: url.clone(), url: url.clone(),
forge_type: forge.forge_type, forge_type: forge.forge_type,
import: false,
}; };
federate.create_forge_instance(&msg).await.unwrap(); federate.create_forge_instance(&msg).await.unwrap();
} }
@ -189,9 +191,10 @@ impl Crawler {
let forges = c.db.get_all_forges(offset, LIMIT).await.unwrap(); let forges = c.db.get_all_forges(offset, LIMIT).await.unwrap();
if forges.is_empty() { if forges.is_empty() {
c.federate.tar().await.unwrap();
page = 0;
tokio::time::sleep(std::time::Duration::new(c.ctx.settings.crawler.ttl, 0)) tokio::time::sleep(std::time::Duration::new(c.ctx.settings.crawler.ttl, 0))
.await; .await;
c.federate.tar().await.unwrap();
if c.shutdown() { if c.shutdown() {
info!("Stopping crawling job"); info!("Stopping crawling job");
break; break;
@ -209,6 +212,7 @@ impl Crawler {
.await; .await;
page += 1; page += 1;
} }
if c.shutdown() { if c.shutdown() {
info!("Stopping crawling job"); info!("Stopping crawling job");
break; break;