From 399ef34f081613442f52a06cce4656e760e6d0b0 Mon Sep 17 00:00:00 2001
From: realaravinth <realaravinth@batsense.net>
Date: Sat, 4 Jun 2022 20:35:24 +0530
Subject: [PATCH] feat: migrate crawl accepts rate-limit configuratoin

---
 src/spider.rs | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/src/spider.rs b/src/spider.rs
index f5cfdc4..b2536ce 100644
--- a/src/spider.rs
+++ b/src/spider.rs
@@ -15,10 +15,7 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
-use std::time::Duration;
-
 use log::info;
-use tokio::time;
 use url::Url;
 
 use db_core::prelude::*;
@@ -55,19 +52,17 @@ impl Ctx {
         loop {
             info!("[crawl][{hostname}] Crawling. page: {page}");
             let res = gitea
-                .crawl(self.settings.crawler.items_per_api_call, page)
+                .crawl(
+                    self.settings.crawler.items_per_api_call,
+                    page,
+                    self.settings.crawler.wait_before_next_api_call,
+                )
                 .await;
             if res.repos.is_empty() {
                 info!("[crawl][{hostname}] Finished crawling. pages: {}", page - 1);
                 break;
             }
 
-            let sleep_fut = time::sleep(Duration::new(
-                self.settings.crawler.wait_before_next_api_call,
-                0,
-            ));
-            let sleep_fut = tokio::spawn(sleep_fut);
-
             for (username, u) in res.users.iter() {
                 if !db
                     .user_exists(username, Some(gitea.get_hostname()))
@@ -94,7 +89,7 @@ impl Ctx {
                 }
             }
 
-            sleep_fut.await.unwrap();
+            //            sleep_fut.await.unwrap();
             page += 1;
         }
         federate.tar().await.unwrap();