Cache post data using sled
Each post fetch was taking 800ms TAT, so I'm using sled to fetch and cache post data. This reduced TAT down to 2ms. However, this could cause storage issues. I must design some sort of resource manager to clean up cache.
This commit is contained in:
parent
133c4a96b3
commit
af3c43dbf5
|
@ -1 +1,2 @@
|
||||||
/target
|
/target
|
||||||
|
posts_cache
|
||||||
|
|
|
@ -304,6 +304,15 @@ version = "0.13.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
|
checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bincode"
|
||||||
|
version = "1.3.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
|
||||||
|
dependencies = [
|
||||||
|
"serde 1.0.130",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bitflags"
|
name = "bitflags"
|
||||||
version = "1.3.2"
|
version = "1.3.2"
|
||||||
|
@ -480,6 +489,29 @@ dependencies = [
|
||||||
"cfg-if",
|
"cfg-if",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "crossbeam-epoch"
|
||||||
|
version = "0.9.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"crossbeam-utils",
|
||||||
|
"lazy_static",
|
||||||
|
"memoffset",
|
||||||
|
"scopeguard",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "crossbeam-utils"
|
||||||
|
version = "0.8.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"lazy_static",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "derive_more"
|
name = "derive_more"
|
||||||
version = "0.99.16"
|
version = "0.99.16"
|
||||||
|
@ -619,6 +651,16 @@ dependencies = [
|
||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fs2"
|
||||||
|
version = "0.4.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"winapi",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "futures-channel"
|
name = "futures-channel"
|
||||||
version = "0.3.17"
|
version = "0.3.17"
|
||||||
|
@ -660,6 +702,15 @@ dependencies = [
|
||||||
"pin-utils",
|
"pin-utils",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fxhash"
|
||||||
|
version = "0.2.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
|
||||||
|
dependencies = [
|
||||||
|
"byteorder",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "generic-array"
|
name = "generic-array"
|
||||||
version = "0.14.4"
|
version = "0.14.4"
|
||||||
|
@ -983,6 +1034,7 @@ dependencies = [
|
||||||
"actix-rt",
|
"actix-rt",
|
||||||
"actix-web",
|
"actix-web",
|
||||||
"actix-web-codegen 0.5.0-beta.4",
|
"actix-web-codegen 0.5.0-beta.4",
|
||||||
|
"bincode",
|
||||||
"chrono",
|
"chrono",
|
||||||
"config",
|
"config",
|
||||||
"derive_more",
|
"derive_more",
|
||||||
|
@ -995,6 +1047,7 @@ dependencies = [
|
||||||
"sailfish",
|
"sailfish",
|
||||||
"serde 1.0.130",
|
"serde 1.0.130",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
"sled",
|
||||||
"url",
|
"url",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
@ -1052,6 +1105,15 @@ version = "2.4.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
|
checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "memoffset"
|
||||||
|
version = "0.6.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9"
|
||||||
|
dependencies = [
|
||||||
|
"autocfg",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mime"
|
name = "mime"
|
||||||
version = "0.3.16"
|
version = "0.3.16"
|
||||||
|
@ -1700,6 +1762,22 @@ version = "0.4.5"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5"
|
checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "sled"
|
||||||
|
version = "0.34.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935"
|
||||||
|
dependencies = [
|
||||||
|
"crc32fast",
|
||||||
|
"crossbeam-epoch",
|
||||||
|
"crossbeam-utils",
|
||||||
|
"fs2",
|
||||||
|
"fxhash",
|
||||||
|
"libc",
|
||||||
|
"log",
|
||||||
|
"parking_lot",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "smallvec"
|
name = "smallvec"
|
||||||
version = "1.7.0"
|
version = "1.7.0"
|
||||||
|
|
|
@ -38,3 +38,6 @@ reqwest = { version = "0.11.6", features = ["json"] }
|
||||||
graphql_client = { version = "0.10.0", features = ["reqwest"]}
|
graphql_client = { version = "0.10.0", features = ["reqwest"]}
|
||||||
|
|
||||||
chrono = "0.4.19"
|
chrono = "0.4.19"
|
||||||
|
|
||||||
|
sled = "0.34.7"
|
||||||
|
bincode = "1.3.3"
|
||||||
|
|
|
@ -1 +1,3 @@
|
||||||
delimiter: "."
|
delimiter: "."
|
||||||
|
optimization:
|
||||||
|
rm_whitespace: true
|
||||||
|
|
37
src/data.rs
37
src/data.rs
|
@ -15,19 +15,56 @@
|
||||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
*/
|
*/
|
||||||
use actix_web::web;
|
use actix_web::web;
|
||||||
|
use graphql_client::{reqwest::post_graphql, GraphQLQuery};
|
||||||
use reqwest::Client;
|
use reqwest::Client;
|
||||||
|
use sled::{Db, Tree};
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct Data {
|
pub struct Data {
|
||||||
pub client: Client,
|
pub client: Client,
|
||||||
|
cache: Db,
|
||||||
|
pub posts: Tree,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(GraphQLQuery)]
|
||||||
|
#[graphql(
|
||||||
|
schema_path = "schemas/schema.graphql",
|
||||||
|
query_path = "schemas/query.graphql",
|
||||||
|
response_derives = "Debug, Serialize, Deserialize, Clone"
|
||||||
|
)]
|
||||||
|
pub struct GetPost;
|
||||||
|
|
||||||
|
pub type PostResp = get_post::GetPostPost;
|
||||||
|
|
||||||
pub type AppData = web::Data<Data>;
|
pub type AppData = web::Data<Data>;
|
||||||
|
|
||||||
impl Data {
|
impl Data {
|
||||||
pub fn new() -> AppData {
|
pub fn new() -> AppData {
|
||||||
|
let cache = sled::open("posts_cache").unwrap();
|
||||||
|
let posts = cache.open_tree("posts").unwrap();
|
||||||
AppData::new(Self {
|
AppData::new(Self {
|
||||||
client: Client::new(),
|
client: Client::new(),
|
||||||
|
cache,
|
||||||
|
posts,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn get_post(&self, id: &str) -> PostResp {
|
||||||
|
match self.posts.get(id) {
|
||||||
|
Ok(Some(v)) => bincode::deserialize(&v[..]).unwrap(),
|
||||||
|
_ => {
|
||||||
|
let vars = get_post::Variables { id: id.to_owned() };
|
||||||
|
const URL: &str = "https://medium.com/_/graphql";
|
||||||
|
|
||||||
|
let res = post_graphql::<GetPost, _>(&self.client, URL, vars)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let res = res.data.expect("missing response data").post.unwrap();
|
||||||
|
self.posts
|
||||||
|
.insert(id, bincode::serialize(&res).unwrap())
|
||||||
|
.unwrap();
|
||||||
|
res
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
28
src/proxy.rs
28
src/proxy.rs
|
@ -17,9 +17,9 @@
|
||||||
use std::ops::{Bound, RangeBounds};
|
use std::ops::{Bound, RangeBounds};
|
||||||
|
|
||||||
use actix_web::{web, HttpResponse, Responder};
|
use actix_web::{web, HttpResponse, Responder};
|
||||||
use graphql_client::{reqwest::post_graphql, GraphQLQuery};
|
|
||||||
use sailfish::TemplateOnce;
|
use sailfish::TemplateOnce;
|
||||||
|
|
||||||
|
use crate::data::PostResp;
|
||||||
use crate::AppData;
|
use crate::AppData;
|
||||||
|
|
||||||
pub mod routes {
|
pub mod routes {
|
||||||
|
@ -95,18 +95,11 @@ impl StringUtils for str {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(GraphQLQuery)]
|
|
||||||
#[graphql(
|
|
||||||
schema_path = "schemas/schema.graphql",
|
|
||||||
query_path = "schemas/query.graphql",
|
|
||||||
response_derives = "Debug"
|
|
||||||
)]
|
|
||||||
struct GetPost;
|
|
||||||
|
|
||||||
#[derive(TemplateOnce)]
|
#[derive(TemplateOnce)]
|
||||||
#[template(path = "post.html")]
|
#[template(path = "post.html")]
|
||||||
|
#[template(rm_whitespace = true)]
|
||||||
pub struct Post {
|
pub struct Post {
|
||||||
pub data: get_post::GetPostPost,
|
pub data: PostResp,
|
||||||
pub id: String,
|
pub id: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -125,20 +118,11 @@ async fn page(path: web::Path<(String, String)>, data: AppData) -> impl Responde
|
||||||
if post_id.is_none() {
|
if post_id.is_none() {
|
||||||
return HttpResponse::BadRequest().finish();
|
return HttpResponse::BadRequest().finish();
|
||||||
}
|
}
|
||||||
let id = post_id.unwrap().to_string();
|
let id = post_id.unwrap();
|
||||||
|
|
||||||
let vars = get_post::Variables { id: id.clone() };
|
|
||||||
|
|
||||||
const URL: &str = "https://medium.com/_/graphql";
|
|
||||||
|
|
||||||
let res = post_graphql::<GetPost, _>(&data.client, URL, vars)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
let response_data: get_post::ResponseData = res.data.expect("missing response data");
|
|
||||||
|
|
||||||
let page = Post {
|
let page = Post {
|
||||||
id,
|
id: id.to_owned(),
|
||||||
data: response_data.post.unwrap(),
|
data: data.get_post(&id).await,
|
||||||
}
|
}
|
||||||
.render_once()
|
.render_once()
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
|
@ -34,11 +34,8 @@
|
||||||
<span><.= text .></span>
|
<span><.= text .></span>
|
||||||
<.}.>
|
<.}.>
|
||||||
<.}.>
|
<.}.>
|
||||||
|
|
||||||
<. if cur < p.text.len() {.>
|
<. if cur < p.text.len() {.>
|
||||||
<.= p.text.slice(cur..) .>
|
<.= p.text.slice(cur..) .>
|
||||||
<.}.>
|
<.}.>
|
||||||
|
|
||||||
<.}.>
|
<.}.>
|
||||||
</p>
|
</p>
|
||||||
<!-- markup.type_ = [ "CODE", "A", "EM", "STRONG" -->
|
|
||||||
|
|
|
@ -10,7 +10,6 @@
|
||||||
<. use chrono::{TimeZone, Utc}; .>
|
<. use chrono::{TimeZone, Utc}; .>
|
||||||
<. let dt = Utc.timestamp_millis(data.created_at); .>
|
<. let dt = Utc.timestamp_millis(data.created_at); .>
|
||||||
<. let date = dt.format("%b %e, %Y").to_string(); .>
|
<. let date = dt.format("%b %e, %Y").to_string(); .>
|
||||||
|
|
||||||
<h1><.= data.title .></h1>
|
<h1><.= data.title .></h1>
|
||||||
<p class="meta">
|
<p class="meta">
|
||||||
<a class="author" href="https://medium.com/u/<.= data.creator.id .>" rel="noreferrer">
|
<a class="author" href="https://medium.com/u/<.= data.creator.id .>" rel="noreferrer">
|
||||||
|
@ -29,8 +28,6 @@
|
||||||
<. if pindex == 0 && p.type_ == "H3" {.>
|
<. if pindex == 0 && p.type_ == "H3" {.>
|
||||||
<. continue; .>
|
<. continue; .>
|
||||||
<.}.>
|
<.}.>
|
||||||
|
|
||||||
|
|
||||||
<. if p.type_ == "IMG" {.>
|
<. if p.type_ == "IMG" {.>
|
||||||
<. include!("./img.html"); .>
|
<. include!("./img.html"); .>
|
||||||
<.} else if p.type_ == "P" {.>
|
<.} else if p.type_ == "P" {.>
|
||||||
|
@ -46,8 +43,6 @@
|
||||||
<.} else if p.type_ == "H6" {.>
|
<.} else if p.type_ == "H6" {.>
|
||||||
<h6><.= p.text .></h6>
|
<h6><.= p.text .></h6>
|
||||||
<.}.>
|
<.}.>
|
||||||
|
|
||||||
|
|
||||||
<.}.>
|
<.}.>
|
||||||
</article>
|
</article>
|
||||||
</main>
|
</main>
|
||||||
|
|
Loading…
Reference in New Issue