diff --git a/Cargo.lock b/Cargo.lock index b414955..6d48f7c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1902,6 +1902,7 @@ dependencies = [ "idna", "matches", "percent-encoding", + "serde 1.0.136", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f6b95af..915c277 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ lazy_static = "1.4.0" rand = "0.8.5" tera = "1.15" tokio = { version = "1.17", features = ["fs", "time"] } -url = "2.2.2" +url = { version = "2.2.2", features = ["serde"] } validator = { version = "0.14", features = ["derive"]} derive_more = "0.99.17" log = "0.4.16" diff --git a/config/default.toml b/config/default.toml index a727feb..39537b4 100644 --- a/config/default.toml +++ b/config/default.toml @@ -31,6 +31,10 @@ name = "postgres" pool = 4 database_type = "postgres" +[crawler] +ttl = 432000 # of crawled records / how often the instance must be polled. In seconds. +items_per_api_call = 20 +client_timeout = 60 # of HTTP client involved in crawling. In seconds. [repository] root = "/tmp/starchart.batsense.net" diff --git a/src/settings.rs b/src/settings.rs index c216042..84c57e9 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -137,6 +137,13 @@ pub struct Database { pub database_type: DBType, } +#[derive(Debug, Validate, Clone, Deserialize)] +pub struct Crawler { + pub ttl: u64, + pub client_timeout: u64, + pub items_per_api_call: u64, +} + #[derive(Debug, Validate, Clone, Deserialize)] pub struct Settings { pub log: LogLevel, @@ -148,6 +155,7 @@ pub struct Settings { pub repository: Repository, #[validate(email)] pub admin_email: String, + pub crawler: Crawler, } #[cfg(not(tarpaulin_include))]