ForgeFlux/starchart
2
0
Fork 0
Code Issues Pull requests 2 Projects Releases Packages Wiki Activity Actions

feat: migrate crawl accepts rate-limit configuratoin

Browse source
This commit is contained in:
Aravinth Manivannan 2022-06-04 20:35:24 +05:30
parent b3d9dc38d9
commit 399ef34f08
Signed by: realaravinth
GPG key ID: AD9F0F08E855ED88
1 changed files with 6 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
src/spider.rs
View file

@ -15,10 +15,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
use std::time::Duration;
use log::info;
use tokio::time;
use url::Url;
use db_core::prelude::*;
@ -55,19 +52,17 @@ impl Ctx {
loop {
info!("[crawl][{hostname}] Crawling. page: {page}");
let res = gitea
.crawl(self.settings.crawler.items_per_api_call, page)
.crawl(
self.settings.crawler.items_per_api_call,
page,
self.settings.crawler.wait_before_next_api_call,
)
.await;
if res.repos.is_empty() {
info!("[crawl][{hostname}] Finished crawling. pages: {}", page - 1);
break;
}
let sleep_fut = time::sleep(Duration::new(
self.settings.crawler.wait_before_next_api_call,
0,
));
let sleep_fut = tokio::spawn(sleep_fut);
for (username, u) in res.users.iter() {
if !db
.user_exists(username, Some(gitea.get_hostname()))
@ -94,7 +89,7 @@ impl Ctx {
}
}
sleep_fut.await.unwrap();
// sleep_fut.await.unwrap();
page += 1;
}
federate.tar().await.unwrap();