ForgeFlux/starchart
2
0
Fork 0
Code Issues Pull requests 2 Projects Releases Packages Wiki Activity Actions

feat: migrate crawl accepts rate-limit configuratoin

Browse source
This commit is contained in:
Aravinth Manivannan 2022-06-04 20:35:24 +05:30
parent b3d9dc38d9
commit 399ef34f08
Signed by: realaravinth
GPG key ID: AD9F0F08E855ED88
1 changed files with 6 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
src/spider.rs
View file

@ -15,10 +15,7 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
use std::time::Duration;
use log::info; use log::info;
use tokio::time;
use url::Url; use url::Url;
use db_core::prelude::*; use db_core::prelude::*;
@ -55,19 +52,17 @@ impl Ctx {
loop { loop {
info!("[crawl][{hostname}] Crawling. page: {page}"); info!("[crawl][{hostname}] Crawling. page: {page}");
let res = gitea let res = gitea
.crawl(self.settings.crawler.items_per_api_call, page) .crawl(
self.settings.crawler.items_per_api_call,
page,
self.settings.crawler.wait_before_next_api_call,
)
.await; .await;
if res.repos.is_empty() { if res.repos.is_empty() {
info!("[crawl][{hostname}] Finished crawling. pages: {}", page - 1); info!("[crawl][{hostname}] Finished crawling. pages: {}", page - 1);
break; break;
} }
let sleep_fut = time::sleep(Duration::new(
self.settings.crawler.wait_before_next_api_call,
0,
));
let sleep_fut = tokio::spawn(sleep_fut);
for (username, u) in res.users.iter() { for (username, u) in res.users.iter() {
if !db if !db
.user_exists(username, Some(gitea.get_hostname())) .user_exists(username, Some(gitea.get_hostname()))
@ -94,7 +89,7 @@ impl Ctx {
} }
} }
sleep_fut.await.unwrap(); // sleep_fut.await.unwrap();
page += 1; page += 1;
} }
federate.tar().await.unwrap(); federate.tar().await.unwrap();