forked from realaravinth/libmedium
Cache post data using sled
Each post fetch was taking 800ms TAT, so I'm using sled to fetch and cache post data. This reduced TAT down to 2ms. However, this could cause storage issues. I must design some sort of resource manager to clean up cache.
This commit is contained in:
parent
133c4a96b3
commit
af3c43dbf5
GPG key ID:
AD9F0F08E855ED88
8 changed files with 127 additions and 30 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -1 +1,2 @@
|
|||
/target
|
||||
posts_cache
|
||||
|
|
|
|||
78
Cargo.lock
generated
78
Cargo.lock
generated
|
|
@ -304,6 +304,15 @@ version = "0.13.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
|
||||
|
||||
[[package]]
|
||||
name = "bincode"
|
||||
version = "1.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
|
||||
dependencies = [
|
||||
"serde 1.0.130",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.3.2"
|
||||
|
|
@ -480,6 +489,29 @@ dependencies = [
|
|||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
"lazy_static",
|
||||
"memoffset",
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"lazy_static",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_more"
|
||||
version = "0.99.16"
|
||||
|
|
@ -619,6 +651,16 @@ dependencies = [
|
|||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fs2"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-channel"
|
||||
version = "0.3.17"
|
||||
|
|
@ -660,6 +702,15 @@ dependencies = [
|
|||
"pin-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fxhash"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.4"
|
||||
|
|
@ -983,6 +1034,7 @@ dependencies = [
|
|||
"actix-rt",
|
||||
"actix-web",
|
||||
"actix-web-codegen 0.5.0-beta.4",
|
||||
"bincode",
|
||||
"chrono",
|
||||
"config",
|
||||
"derive_more",
|
||||
|
|
@ -995,6 +1047,7 @@ dependencies = [
|
|||
"sailfish",
|
||||
"serde 1.0.130",
|
||||
"serde_json",
|
||||
"sled",
|
||||
"url",
|
||||
]
|
||||
|
||||
|
|
@ -1052,6 +1105,15 @@ version = "2.4.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mime"
|
||||
version = "0.3.16"
|
||||
|
|
@ -1700,6 +1762,22 @@ version = "0.4.5"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5"
|
||||
|
||||
[[package]]
|
||||
name = "sled"
|
||||
version = "0.34.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
"fs2",
|
||||
"fxhash",
|
||||
"libc",
|
||||
"log",
|
||||
"parking_lot",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.7.0"
|
||||
|
|
|
|||
|
|
@ -38,3 +38,6 @@ reqwest = { version = "0.11.6", features = ["json"] }
|
|||
graphql_client = { version = "0.10.0", features = ["reqwest"]}
|
||||
|
||||
chrono = "0.4.19"
|
||||
|
||||
sled = "0.34.7"
|
||||
bincode = "1.3.3"
|
||||
|
|
|
|||
|
|
@ -1 +1,3 @@
|
|||
delimiter: "."
|
||||
optimization:
|
||||
rm_whitespace: true
|
||||
|
|
|
|||
37
src/data.rs
37
src/data.rs
|
|
@ -15,19 +15,56 @@
|
|||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
use actix_web::web;
|
||||
use graphql_client::{reqwest::post_graphql, GraphQLQuery};
|
||||
use reqwest::Client;
|
||||
use sled::{Db, Tree};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Data {
|
||||
pub client: Client,
|
||||
cache: Db,
|
||||
pub posts: Tree,
|
||||
}
|
||||
|
||||
#[derive(GraphQLQuery)]
|
||||
#[graphql(
|
||||
schema_path = "schemas/schema.graphql",
|
||||
query_path = "schemas/query.graphql",
|
||||
response_derives = "Debug, Serialize, Deserialize, Clone"
|
||||
)]
|
||||
pub struct GetPost;
|
||||
|
||||
pub type PostResp = get_post::GetPostPost;
|
||||
|
||||
pub type AppData = web::Data<Data>;
|
||||
|
||||
impl Data {
|
||||
pub fn new() -> AppData {
|
||||
let cache = sled::open("posts_cache").unwrap();
|
||||
let posts = cache.open_tree("posts").unwrap();
|
||||
AppData::new(Self {
|
||||
client: Client::new(),
|
||||
cache,
|
||||
posts,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn get_post(&self, id: &str) -> PostResp {
|
||||
match self.posts.get(id) {
|
||||
Ok(Some(v)) => bincode::deserialize(&v[..]).unwrap(),
|
||||
_ => {
|
||||
let vars = get_post::Variables { id: id.to_owned() };
|
||||
const URL: &str = "https://medium.com/_/graphql";
|
||||
|
||||
let res = post_graphql::<GetPost, _>(&self.client, URL, vars)
|
||||
.await
|
||||
.unwrap();
|
||||
let res = res.data.expect("missing response data").post.unwrap();
|
||||
self.posts
|
||||
.insert(id, bincode::serialize(&res).unwrap())
|
||||
.unwrap();
|
||||
res
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
28
src/proxy.rs
28
src/proxy.rs
|
|
@ -17,9 +17,9 @@
|
|||
use std::ops::{Bound, RangeBounds};
|
||||
|
||||
use actix_web::{web, HttpResponse, Responder};
|
||||
use graphql_client::{reqwest::post_graphql, GraphQLQuery};
|
||||
use sailfish::TemplateOnce;
|
||||
|
||||
use crate::data::PostResp;
|
||||
use crate::AppData;
|
||||
|
||||
pub mod routes {
|
||||
|
|
@ -95,18 +95,11 @@ impl StringUtils for str {
|
|||
}
|
||||
}
|
||||
|
||||
#[derive(GraphQLQuery)]
|
||||
#[graphql(
|
||||
schema_path = "schemas/schema.graphql",
|
||||
query_path = "schemas/query.graphql",
|
||||
response_derives = "Debug"
|
||||
)]
|
||||
struct GetPost;
|
||||
|
||||
#[derive(TemplateOnce)]
|
||||
#[template(path = "post.html")]
|
||||
#[template(rm_whitespace = true)]
|
||||
pub struct Post {
|
||||
pub data: get_post::GetPostPost,
|
||||
pub data: PostResp,
|
||||
pub id: String,
|
||||
}
|
||||
|
||||
|
|
@ -125,20 +118,11 @@ async fn page(path: web::Path<(String, String)>, data: AppData) -> impl Responde
|
|||
if post_id.is_none() {
|
||||
return HttpResponse::BadRequest().finish();
|
||||
}
|
||||
let id = post_id.unwrap().to_string();
|
||||
|
||||
let vars = get_post::Variables { id: id.clone() };
|
||||
|
||||
const URL: &str = "https://medium.com/_/graphql";
|
||||
|
||||
let res = post_graphql::<GetPost, _>(&data.client, URL, vars)
|
||||
.await
|
||||
.unwrap();
|
||||
let response_data: get_post::ResponseData = res.data.expect("missing response data");
|
||||
let id = post_id.unwrap();
|
||||
|
||||
let page = Post {
|
||||
id,
|
||||
data: response_data.post.unwrap(),
|
||||
id: id.to_owned(),
|
||||
data: data.get_post(&id).await,
|
||||
}
|
||||
.render_once()
|
||||
.unwrap();
|
||||
|
|
|
|||
|
|
@ -34,11 +34,8 @@
|
|||
<span><.= text .></span>
|
||||
<.}.>
|
||||
<.}.>
|
||||
|
||||
<. if cur < p.text.len() {.>
|
||||
<.= p.text.slice(cur..) .>
|
||||
<.}.>
|
||||
|
||||
<.}.>
|
||||
</p>
|
||||
<!-- markup.type_ = [ "CODE", "A", "EM", "STRONG" -->
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@
|
|||
<. use chrono::{TimeZone, Utc}; .>
|
||||
<. let dt = Utc.timestamp_millis(data.created_at); .>
|
||||
<. let date = dt.format("%b %e, %Y").to_string(); .>
|
||||
|
||||
<h1><.= data.title .></h1>
|
||||
<p class="meta">
|
||||
<a class="author" href="https://medium.com/u/<.= data.creator.id .>" rel="noreferrer">
|
||||
|
|
@ -29,8 +28,6 @@
|
|||
<. if pindex == 0 && p.type_ == "H3" {.>
|
||||
<. continue; .>
|
||||
<.}.>
|
||||
|
||||
|
||||
<. if p.type_ == "IMG" {.>
|
||||
<. include!("./img.html"); .>
|
||||
<.} else if p.type_ == "P" {.>
|
||||
|
|
@ -46,8 +43,6 @@
|
|||
<.} else if p.type_ == "H6" {.>
|
||||
<h6><.= p.text .></h6>
|
||||
<.}.>
|
||||
|
||||
|
||||
<.}.>
|
||||
</article>
|
||||
</main>
|
||||
|
|
|
|||
Loading…
Reference in a new issue