Cache post data using sled

Each post fetch was taking 800ms TAT, so I'm using sled to fetch and
cache post data. This reduced TAT down to 2ms.

However, this could cause storage issues. I must design some sort of
resource manager to clean up cache.

.gitignore

@@ -1 +1,2 @@
 /target
+posts_cache

Cargo.lock

@@ -304,6 +304,15 @@ version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
 
+[[package]]
+name = "bincode"
+version = "1.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
+dependencies = [
+ "serde 1.0.130",
+]
+
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@@ -480,6 +489,29 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd"
+dependencies = [
+ "cfg-if",
+ "crossbeam-utils",
+ "lazy_static",
+ "memoffset",
+ "scopeguard",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db"
+dependencies = [
+ "cfg-if",
+ "lazy_static",
+]
+
 [[package]]
 name = "derive_more"
 version = "0.99.16"
@@ -619,6 +651,16 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "fs2"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
 [[package]]
 name = "futures-channel"
 version = "0.3.17"
@@ -660,6 +702,15 @@ dependencies = [
  "pin-utils",
 ]
 
+[[package]]
+name = "fxhash"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
+dependencies = [
+ "byteorder",
+]
+
 [[package]]
 name = "generic-array"
 version = "0.14.4"
@@ -983,6 +1034,7 @@ dependencies = [
  "actix-rt",
  "actix-web",
  "actix-web-codegen 0.5.0-beta.4",
+ "bincode",
  "chrono",
  "config",
  "derive_more",
@@ -995,6 +1047,7 @@ dependencies = [
  "sailfish",
  "serde 1.0.130",
  "serde_json",
+ "sled",
  "url",
 ]
 
@@ -1052,6 +1105,15 @@ version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
 
+[[package]]
+name = "memoffset"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.16"
@@ -1700,6 +1762,22 @@ version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5"
 
+[[package]]
+name = "sled"
+version = "0.34.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935"
+dependencies = [
+ "crc32fast",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+ "fs2",
+ "fxhash",
+ "libc",
+ "log",
+ "parking_lot",
+]
+
 [[package]]
 name = "smallvec"
 version = "1.7.0"

Cargo.toml

@@ -38,3 +38,6 @@ reqwest = { version = "0.11.6", features = ["json"] }
 graphql_client = { version = "0.10.0", features = ["reqwest"]}
 
 chrono = "0.4.19"
+
+sled = "0.34.7"
+bincode = "1.3.3"

sailfish.yml

@@ -1 +1,3 @@
 delimiter: "."
+optimization:
+    rm_whitespace: true

src/data.rs

@@ -15,19 +15,56 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 use actix_web::web;
+use graphql_client::{reqwest::post_graphql, GraphQLQuery};
 use reqwest::Client;
+use sled::{Db, Tree};
 
 #[derive(Clone)]
 pub struct Data {
     pub client: Client,
+    cache: Db,
+    pub posts: Tree,
 }
 
+#[derive(GraphQLQuery)]
+#[graphql(
+    schema_path = "schemas/schema.graphql",
+    query_path = "schemas/query.graphql",
+    response_derives = "Debug, Serialize, Deserialize, Clone"
+)]
+pub struct GetPost;
+
+pub type PostResp = get_post::GetPostPost;
+
 pub type AppData = web::Data<Data>;
 
 impl Data {
     pub fn new() -> AppData {
+        let cache = sled::open("posts_cache").unwrap();
+        let posts = cache.open_tree("posts").unwrap();
         AppData::new(Self {
             client: Client::new(),
+            cache,
+            posts,
         })
     }
+
+    pub async fn get_post(&self, id: &str) -> PostResp {
+        match self.posts.get(id) {
+            Ok(Some(v)) => bincode::deserialize(&v[..]).unwrap(),
+            _ => {
+                let vars = get_post::Variables { id: id.to_owned() };
+                const URL: &str = "https://medium.com/_/graphql";
+
+                let res = post_graphql::<GetPost, _>(&self.client, URL, vars)
+                    .await
+                    .unwrap();
+                let res = res.data.expect("missing response data").post.unwrap();
+                self.posts
+                    .insert(id, bincode::serialize(&res).unwrap())
+                    .unwrap();
+                res
+            }
+        }
+    }
 }

src/proxy.rs

@@ -17,9 +17,9 @@
 use std::ops::{Bound, RangeBounds};
 
 use actix_web::{web, HttpResponse, Responder};
-use graphql_client::{reqwest::post_graphql, GraphQLQuery};
 use sailfish::TemplateOnce;
 
+use crate::data::PostResp;
 use crate::AppData;
 
 pub mod routes {
@@ -95,18 +95,11 @@ impl StringUtils for str {
     }
 }
 
-#[derive(GraphQLQuery)]
-#[graphql(
-    schema_path = "schemas/schema.graphql",
-    query_path = "schemas/query.graphql",
-    response_derives = "Debug"
-)]
-struct GetPost;
-
 #[derive(TemplateOnce)]
 #[template(path = "post.html")]
+#[template(rm_whitespace = true)]
 pub struct Post {
-    pub data: get_post::GetPostPost,
+    pub data: PostResp,
     pub id: String,
 }
 
@@ -125,20 +118,11 @@ async fn page(path: web::Path<(String, String)>, data: AppData) -> impl Responde
     if post_id.is_none() {
         return HttpResponse::BadRequest().finish();
     }
-    let id = post_id.unwrap().to_string();
+    let id = post_id.unwrap();
-
-    let vars = get_post::Variables { id: id.clone() };
-
-    const URL: &str = "https://medium.com/_/graphql";
-
-    let res = post_graphql::<GetPost, _>(&data.client, URL, vars)
-        .await
-        .unwrap();
-    let response_data: get_post::ResponseData = res.data.expect("missing response data");
 
     let page = Post {
-        id,
+        id: id.to_owned(),
-        data: response_data.post.unwrap(),
+        data: data.get_post(&id).await,
     }
     .render_once()
     .unwrap();

templates/p.html

@@ -34,11 +34,8 @@
       <span><.= text .></span>
     <.}.>
   <.}.>
-
   <. if cur < p.text.len() {.>
     <.= p.text.slice(cur..) .>
   <.}.>
-
 <.}.>
 </p>
-<!-- markup.type_ = [ "CODE", "A", "EM", "STRONG" -->

templates/post.html

@@ -10,7 +10,6 @@
       <. use chrono::{TimeZone, Utc}; .> 
       <. let dt = Utc.timestamp_millis(data.created_at); .>
       <. let date = dt.format("%b %e, %Y").to_string(); .>
-
       <h1><.= data.title .></h1>
       <p class="meta">
         <a class="author" href="https://medium.com/u/<.= data.creator.id .>" rel="noreferrer">
@@ -29,8 +28,6 @@
           <. if pindex == 0 && p.type_ == "H3" {.>
             <. continue; .>
           <.}.>
-
-
          <. if p.type_ == "IMG" {.>
             <. include!("./img.html"); .> 
           <.} else if p.type_ == "P" {.>
@@ -46,8 +43,6 @@
           <.} else if p.type_ == "H6" {.>
             <h6><.= p.text .></h6>
           <.}.>
-
-
         <.}.>
       </article>
   </main>