feat: add crawler configuration options

Cargo.lock

@@ -1902,6 +1902,7 @@ dependencies = [
  "idna",
  "matches",
  "percent-encoding",
+ "serde 1.0.136",
 ]
 
 [[package]]

Cargo.toml

@@ -16,7 +16,7 @@ lazy_static = "1.4.0"
 rand = "0.8.5"
 tera = "1.15"
 tokio = { version = "1.17", features = ["fs", "time"] }
-url = "2.2.2"
+url = { version  = "2.2.2", features = ["serde"] }
 validator = { version = "0.14", features = ["derive"]}
 derive_more = "0.99.17"
 log = "0.4.16"

config/default.toml

@@ -31,6 +31,10 @@ name = "postgres"
 pool = 4
 database_type = "postgres"
 
+[crawler]
+ttl = 432000 # of crawled records / how often the instance must be polled. In seconds.
+items_per_api_call = 20
+client_timeout = 60 # of HTTP client involved in crawling. In seconds.
 
 [repository]
 root = "/tmp/starchart.batsense.net"

src/settings.rs

@@ -137,6 +137,13 @@ pub struct Database {
     pub database_type: DBType,
 }
 
+#[derive(Debug, Validate, Clone, Deserialize)]
+pub struct Crawler {
+    pub ttl: u64,
+    pub client_timeout: u64,
+    pub items_per_api_call: u64,
+}
+
 #[derive(Debug, Validate, Clone, Deserialize)]
 pub struct Settings {
     pub log: LogLevel,
@@ -148,6 +155,7 @@ pub struct Settings {
     pub repository: Repository,
     #[validate(email)]
     pub admin_email: String,
+    pub crawler: Crawler,
 }
 
 #[cfg(not(tarpaulin_include))]