Cache post data using sled

Each post fetch was taking 800ms TAT, so I'm using sled to fetch and
cache post data. This reduced TAT down to 2ms.

However, this could cause storage issues. I must design some sort of
resource manager to clean up cache.
This commit is contained in:
Aravinth Manivannan 2021-11-02 15:30:25 +05:30
parent 133c4a96b3
commit af3c43dbf5
Signed by: realaravinth
GPG key ID: AD9F0F08E855ED88
8 changed files with 127 additions and 30 deletions

1
.gitignore vendored
View file

@ -1 +1,2 @@
/target
posts_cache

78
Cargo.lock generated
View file

@ -304,6 +304,15 @@ version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
[[package]]
name = "bincode"
version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
dependencies = [
"serde 1.0.130",
]
[[package]]
name = "bitflags"
version = "1.3.2"
@ -480,6 +489,29 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd"
dependencies = [
"cfg-if",
"crossbeam-utils",
"lazy_static",
"memoffset",
"scopeguard",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db"
dependencies = [
"cfg-if",
"lazy_static",
]
[[package]]
name = "derive_more"
version = "0.99.16"
@ -619,6 +651,16 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "fs2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "futures-channel"
version = "0.3.17"
@ -660,6 +702,15 @@ dependencies = [
"pin-utils",
]
[[package]]
name = "fxhash"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
dependencies = [
"byteorder",
]
[[package]]
name = "generic-array"
version = "0.14.4"
@ -983,6 +1034,7 @@ dependencies = [
"actix-rt",
"actix-web",
"actix-web-codegen 0.5.0-beta.4",
"bincode",
"chrono",
"config",
"derive_more",
@ -995,6 +1047,7 @@ dependencies = [
"sailfish",
"serde 1.0.130",
"serde_json",
"sled",
"url",
]
@ -1052,6 +1105,15 @@ version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
[[package]]
name = "memoffset"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9"
dependencies = [
"autocfg",
]
[[package]]
name = "mime"
version = "0.3.16"
@ -1700,6 +1762,22 @@ version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5"
[[package]]
name = "sled"
version = "0.34.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935"
dependencies = [
"crc32fast",
"crossbeam-epoch",
"crossbeam-utils",
"fs2",
"fxhash",
"libc",
"log",
"parking_lot",
]
[[package]]
name = "smallvec"
version = "1.7.0"

View file

@ -38,3 +38,6 @@ reqwest = { version = "0.11.6", features = ["json"] }
graphql_client = { version = "0.10.0", features = ["reqwest"]}
chrono = "0.4.19"
sled = "0.34.7"
bincode = "1.3.3"

View file

@ -1 +1,3 @@
delimiter: "."
optimization:
rm_whitespace: true

View file

@ -15,19 +15,56 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
use actix_web::web;
use graphql_client::{reqwest::post_graphql, GraphQLQuery};
use reqwest::Client;
use sled::{Db, Tree};
#[derive(Clone)]
pub struct Data {
pub client: Client,
cache: Db,
pub posts: Tree,
}
#[derive(GraphQLQuery)]
#[graphql(
schema_path = "schemas/schema.graphql",
query_path = "schemas/query.graphql",
response_derives = "Debug, Serialize, Deserialize, Clone"
)]
pub struct GetPost;
pub type PostResp = get_post::GetPostPost;
pub type AppData = web::Data<Data>;
impl Data {
pub fn new() -> AppData {
let cache = sled::open("posts_cache").unwrap();
let posts = cache.open_tree("posts").unwrap();
AppData::new(Self {
client: Client::new(),
cache,
posts,
})
}
pub async fn get_post(&self, id: &str) -> PostResp {
match self.posts.get(id) {
Ok(Some(v)) => bincode::deserialize(&v[..]).unwrap(),
_ => {
let vars = get_post::Variables { id: id.to_owned() };
const URL: &str = "https://medium.com/_/graphql";
let res = post_graphql::<GetPost, _>(&self.client, URL, vars)
.await
.unwrap();
let res = res.data.expect("missing response data").post.unwrap();
self.posts
.insert(id, bincode::serialize(&res).unwrap())
.unwrap();
res
}
}
}
}

View file

@ -17,9 +17,9 @@
use std::ops::{Bound, RangeBounds};
use actix_web::{web, HttpResponse, Responder};
use graphql_client::{reqwest::post_graphql, GraphQLQuery};
use sailfish::TemplateOnce;
use crate::data::PostResp;
use crate::AppData;
pub mod routes {
@ -95,18 +95,11 @@ impl StringUtils for str {
}
}
#[derive(GraphQLQuery)]
#[graphql(
schema_path = "schemas/schema.graphql",
query_path = "schemas/query.graphql",
response_derives = "Debug"
)]
struct GetPost;
#[derive(TemplateOnce)]
#[template(path = "post.html")]
#[template(rm_whitespace = true)]
pub struct Post {
pub data: get_post::GetPostPost,
pub data: PostResp,
pub id: String,
}
@ -125,20 +118,11 @@ async fn page(path: web::Path<(String, String)>, data: AppData) -> impl Responde
if post_id.is_none() {
return HttpResponse::BadRequest().finish();
}
let id = post_id.unwrap().to_string();
let vars = get_post::Variables { id: id.clone() };
const URL: &str = "https://medium.com/_/graphql";
let res = post_graphql::<GetPost, _>(&data.client, URL, vars)
.await
.unwrap();
let response_data: get_post::ResponseData = res.data.expect("missing response data");
let id = post_id.unwrap();
let page = Post {
id,
data: response_data.post.unwrap(),
id: id.to_owned(),
data: data.get_post(&id).await,
}
.render_once()
.unwrap();

View file

@ -34,11 +34,8 @@
<span><.= text .></span>
<.}.>
<.}.>
<. if cur < p.text.len() {.>
<.= p.text.slice(cur..) .>
<.}.>
<.}.>
</p>
<!-- markup.type_ = [ "CODE", "A", "EM", "STRONG" -->

View file

@ -10,7 +10,6 @@
<. use chrono::{TimeZone, Utc}; .>
<. let dt = Utc.timestamp_millis(data.created_at); .>
<. let date = dt.format("%b %e, %Y").to_string(); .>
<h1><.= data.title .></h1>
<p class="meta">
<a class="author" href="https://medium.com/u/<.= data.creator.id .>" rel="noreferrer">
@ -29,8 +28,6 @@
<. if pindex == 0 && p.type_ == "H3" {.>
<. continue; .>
<.}.>
<. if p.type_ == "IMG" {.>
<. include!("./img.html"); .>
<.} else if p.type_ == "P" {.>
@ -46,8 +43,6 @@
<.} else if p.type_ == "H6" {.>
<h6><.= p.text .></h6>
<.}.>
<.}.>
</article>
</main>