ForgeFlux/starchart
2
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

fix: archive after each crawl

Browse source
This commit is contained in:
Aravinth Manivannan 2023-02-22 18:10:34 +05:30
parent bc2e1870f5
commit d3fde704ff
Signed by: realaravinth
GPG key ID: AD9F0F08E855ED88
1 changed files with 5 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/spider.rs
View file

@ -48,6 +48,7 @@ impl Ctx {
let msg = CreateForge {
url: url.clone(),
forge_type: forge.forge_type(),
import: false,
};
db.create_forge_instance(&msg).await.unwrap();
@ -56,6 +57,7 @@ impl Ctx {
let msg = CreateForge {
url: url.clone(),
forge_type: forge.forge_type,
import: false,
};
federate.create_forge_instance(&msg).await.unwrap();
}
@ -189,9 +191,10 @@ impl Crawler {
let forges = c.db.get_all_forges(offset, LIMIT).await.unwrap();
if forges.is_empty() {
c.federate.tar().await.unwrap();
page = 0;
tokio::time::sleep(std::time::Duration::new(c.ctx.settings.crawler.ttl, 0))
.await;
c.federate.tar().await.unwrap();
if c.shutdown() {
info!("Stopping crawling job");
break;
@ -209,6 +212,7 @@ impl Crawler {
.await;
page += 1;
}
if c.shutdown() {
info!("Stopping crawling job");
break;