feat: configuration option for wait time before next API call

Cargo.toml

@@ -35,3 +35,8 @@ version = "1"
 [dependencies.trust-dns-resolver]
 features = ["tokio-runtime", "dns-over-tls", "dns-over-rustls"]
 version = "0.21.1"
+
+#[workspace]
+#exclude = ["db/migrator"]
+#members = [".", "db/db-core", "db/db-sqlx-sqlite"]
+# "db/db-sqlx-postgres"

config/default.toml

@@ -24,7 +24,7 @@ cookie_secret = "f12d9adf4e364648664442b8f50bf478e748e1d77c4797b2ec1f56803278"
 # username = "batman"
 # password = "somereallycomplicatedBatmanpassword"
 hostname = "localhost"
-port = "5432"
+port = 5432
 username = "postgres"
 password = "password"
 name = "postgres"
@@ -35,6 +35,7 @@ database_type = "postgres"
 ttl = 432000 # of crawled records / how often the instance must be polled. In seconds.
 items_per_api_call = 20
 client_timeout = 60 # of HTTP client involved in crawling. In seconds.
+wait_before_next_api_call = 2 # in seconds
 
 [repository]
-root = "/tmp/starchart.batsense.net"
+root = "/tmp/starchart.forgeflux.org"

src/settings.rs

@@ -119,8 +119,16 @@ impl DatabaseBuilder {
         let mut path = url.path().split('/');
         path.next();
         let name = path.next().expect("no database name").to_string();
+
+        let database_type = DBType::from_url(url).unwrap();
+        let port;
+        if database_type == DBType::Sqlite {
+            port = 0;
+        } else {
+            port = url.port().expect("Enter database port").into();
+        }
         DatabaseBuilder {
-            port: url.port().expect("Enter database port").into(),
+            port,
             hostname: url.host().expect("Enter database host").to_string(),
             username: url.username().into(),
             password: url.password().expect("Enter database password").into(),
@@ -142,6 +150,7 @@ pub struct Crawler {
     pub ttl: u64,
     pub client_timeout: u64,
     pub items_per_api_call: u64,
+    pub wait_before_next_api_call: u64,
 }
 
 #[derive(Debug, Validate, Clone, Deserialize)]