Merge pull request 'fix: remove secrets' (#68) from fix-secrets into master

Reviewed-on: realaravinth/libmedium#68

.github/workflows/clippy-fmt.yml

@@ -1,43 +0,0 @@
-name: Lint
-
-on:
-  pull_request:
-    types: [opened, synchronize, reopened]
-  push:
-    branches:
-      - master
-
-jobs:
-  fmt:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Install Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          components: rustfmt
-      - name: Check with rustfmt
-        uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
-
-  clippy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Install Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          components: clippy
-          override: true
-
-      - name: Check with Clippy
-        uses: actions-rs/clippy-check@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          args: --workspace --tests --all-features

.github/workflows/coverage.yml

@@ -1,49 +0,0 @@
-name: Coverage
-
-on:
-  pull_request:
-    types: [opened, synchronize, reopened]
-  push:
-    branches:
-      - master
-
-jobs:
-  build_and_test:
-    strategy:
-      fail-fast: false
-      matrix:
-        version:
-          - nightly
-
-    name: ${{ matrix.version }} - x86_64-unknown-linux-gnu
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: ⚡ Cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            target
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-
-      - name: Install ${{ matrix.version }}
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: ${{ matrix.version }}-x86_64-unknown-linux-gnu
-          profile: minimal
-          override: true
-
-      - name: Generate coverage file
-        if: (github.ref == 'refs/heads/master' || github.event_name == 'pull_request')
-        uses: actions-rs/tarpaulin@v0.1
-        with:
-          args: "-t 1200"
-        env:
-          GIT_HASH: 8e77345f1597e40c2e266cb4e6dee74888918a61
-
-      - name: Upload to Codecov
-        uses: codecov/codecov-action@v2
-        if: github.ref == 'refs/heads/master'

.github/workflows/linux.yml

@@ -1,70 +0,0 @@
-name: Build
-
-on:
-  pull_request:
-    types: [opened, synchronize, reopened]
-  push:
-    branches:
-      - master
-
-jobs:
-  build_and_test:
-    strategy:
-      fail-fast: false
-      matrix:
-        version:
-          #- 1.51.0
-          - stable
-            #  - nightly
-
-    name: ${{ matrix.version }} - x86_64-unknown-linux-gnu
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: ⚡ Cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            /var/lib/docker
-            ~/.cargo/registry
-            ~/.cargo/git
-            target
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-
-      - name: Login to DockerHub
-        if: (github.ref == 'refs/heads/master' || github.event_name == 'push') && github.repository == 'realaravinth/libmedium'
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Install ${{ matrix.version }}
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: ${{ matrix.version }}-x86_64-unknown-linux-gnu
-          profile: minimal
-          override: true
-
-      - name: build
-        run: make
-
-      - name: run tests
-        run: make test
-
-      - name: build and publish docker images
-        run: make docker-publish
-
-      - name: generate documentation
-        if: matrix.version == 'stable' && (github.repository == 'realaravinth/libmedium')
-        run: make doc
-        env:
-          GIT_HASH: 8e77345f1597e40c2e266cb4e6dee74888918a61 # dummy value
-
-      - name: Deploy to GitHub libmedium
-        if: matrix.version == 'stable' && (github.repository == 'realaravinth/libmedium')
-        uses: JamesIves/github-pages-deploy-action@3.7.1
-        with:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          BRANCH: gh-libmedium
-          FOLDER: target/doc

.woodpecker.yml

@@ -1,6 +1,8 @@
-pipeline:
+steps:
   backend:
     image: rust
+    when:
+      event: [push, pull_request, tag, deployment, cron, manual, release]
     environment:
       - GIT_HASH=8e77345f1597e40c2e266cb4e6dee74888918a61 # dummy value
       - COMPILED_DATE=2021-07-21
@@ -12,22 +14,27 @@ pipeline:
       - make
       - make test
 
-  build_docker_img:
-    image: plugins/docker
+  build-docker-img:
+    image: woodpeckerci/plugin-docker-buildx
     when:
       event: [pull_request]
     settings:
       dry_run: true
       repo: realaravinth/libmedium
-      tags: latest
+      username: realaravinth
+      platforms: linux/amd64,linux/arm64/v8
+      tag: latest
 
-  build_and_publish_docker_img:
-    image: plugins/docker
+
+  publish-docker-img:
+    image: woodpeckerci/plugin-docker-buildx
     when:
       event: [push, tag, deployment]
+      branch: master
     settings:
+      repo: realaravinth/libmedium
       username: realaravinth
       password:
         from_secret: DOCKER_TOKEN
-      repo: realaravinth/libmedium
-      tags: latest
+      platforms: linux/amd64,linux/arm64/v8
+      tag: latest

Cargo.toml

@@ -16,7 +16,7 @@ actix-rt = "2"
 actix-web = "4.0.1"
 bincode = "1.3.3"
 chrono = "0.4.23"
-config = "0.13"
+config = "0.14"
 derive_more = "0.99"
 futures = "0.3.17"
 lazy_static = "1.4"
@@ -30,10 +30,11 @@ sled = "0.34.7"
 syntect = "5.0.0"
 url = "2.2"
 actix-web-codegen-const-routes = "0.2.0"
+sha256 = "1.5.0"
 
 [dependencies.graphql_client]
 features = ["reqwest"]
-version = "0.13.0"
+version = "0.14.0"
 
 [dependencies.reqwest]
 features = ["json"]

README.md

@@ -57,8 +57,8 @@ business.
 | http://md.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad.onion/ | N/A     | Hetzner    | [~vern](https://vern.cc)               |
 | http://vernaqj2qr2pijpgvf3od6ssc3ulz3nv52gwr3hba5l6humuzmgq.b32.i2p/      | N/A     | Hetzner    | [~vern](https://vern.cc)               |
 | https://medium.hostux.net                                                 | France  | Gandi      | [hostux](https://hostux.net)           |
-| https://read.sudovanilla.comn                                             | US      | Cloudflare | [SudoVanilla](https://sudovanilla.com) |
-
+| https://r.sudovanilla.org                                                 | US      | Selfhosted | [SudoVanilla](https://sudovanilla.org) |
+| https://libmedium.ducks.party                                             | DE      | Datalix    | [ducks.party](https://ducks.party)     |
 ## Deploy
 
 1. Grab [`./config/default.toml`](./config/default.toml) and make

config/default.toml

@@ -1,5 +1,5 @@
 debug = true
-source_code = "https://github.com/realaravinth/libmedium"
+source_code = "https://git.batsense.net/realaravinth/libmedium"
 #cache = "/var/lib/libmedium"
 
 [server]

renovate.json

@@ -0,0 +1,12 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": ["config:recommended", ":dependencyDashboard"],
+  "labels": ["renovate-bot"],
+  "prHourlyLimit": 0,
+  "timezone": "Asia/kolkata",
+  "prCreation": "immediate",
+  "vulnerabilityAlerts": {
+    "enabled": true,
+    "labels": ["renovate-bot", "renovate-security", "security"]
+  }
+}

src/data.rs

@@ -21,6 +21,7 @@ use graphql_client::{reqwest::post_graphql, GraphQLQuery};
 use reqwest::header::USER_AGENT;
 use reqwest::Client;
 use serde::{Deserialize, Serialize};
+use sha256::digest;
 use sled::{Db, Tree};
 
 use crate::proxy::StringUtils;
@@ -163,14 +164,14 @@ impl Data {
         }
     }
 
-    pub async fn get_post_light(&self, id: &str) -> PostUrl {
+    pub async fn get_post_light(&self, id: &str) -> Option<PostUrl> {
         match self.posts.get(id) {
             Ok(Some(v)) => {
                 let cached: PostResp = bincode::deserialize(&v[..]).unwrap();
-                PostUrl {
+                Some(PostUrl {
                     slug: cached.unique_slug,
                     username: cached.creator.username,
-                }
+                })
             }
             _ => {
                 let vars = get_post_light::Variables { id: id.to_owned() };
@@ -179,10 +180,16 @@ impl Data {
                 let res = post_graphql::<GetPostLight, _>(&self.client, URL, vars)
                     .await
                     .unwrap();
-                let res = res.data.expect("missing response data").post.unwrap();
-                PostUrl {
+                if res.data.is_none() {
+                    None
+                } else {
+                    match res.data.expect("missing response data").post {
+                        None => None,
+                        Some(res) => Some(PostUrl {
                             slug: res.unique_slug,
                             username: res.creator.username,
+                        }),
+                    }
                 }
             }
         }
@@ -287,19 +294,21 @@ impl Data {
                 filepath: &file.file_name,
                 code: &file.content,
             };
-            file.content = highlight.syntax_highlight();
+            file.content = highlight.syntax_highlight(&digest(&file.raw_url));
             files.push(file);
             GistContent {
                 files,
                 html_url: gist_url,
             }
         } else {
+            let mut index = 1;
             gist.files.iter_mut().for_each(|f| {
                 let highlight = render_html::SourcegraphQuery {
                     filepath: &f.file_name,
                     code: &f.content,
                 };
-                f.content = highlight.syntax_highlight();
+                f.content = highlight.syntax_highlight(&digest(&f.raw_url));
+                index += 1;
             });
             gist
         };

src/post.rs

@@ -345,6 +345,7 @@ pub fn apply_markup(
 ) -> Vec<String> {
     let mut paragraphs: Vec<String> = Vec::with_capacity(data.content.body_model.paragraphs.len());
     let mut state = ListState::default();
+    let mut no_render_html = false;
     for (pindex, p) in data.content.body_model.paragraphs.iter().enumerate() {
         let mut pos = PositionMap::default();
         if p.type_ == "H3" && pindex == 0 {
@@ -383,33 +384,42 @@ pub fn apply_markup(
         }
 
         let mut content = String::with_capacity(p.text.len());
-        content += &Markup::start(p, gists, pindex, &mut state);
+        let start = &Markup::start(p, gists, pindex, &mut state);
+        content += start;
+        if start == "<pre>" {
+            no_render_html = true;
+        }
         pos.arr.sort();
+        let mut page = String::default();
         if let Some(first) = pos.arr.first() {
-            //content += p.text.substring(cur, *first as usize);
-            content += p.text.slice(cur..*first as usize);
+            page += p.text.slice(cur..*first as usize);
             cur = incr_cur(cur, *first);
             for point in pos.arr.iter() {
-                //content.push(p.text.substring(start, start + point);
-                //            if *point != 0 {
-
                 if cur != *point as usize {
-                    //           content += p.text.substring(cur, *point as usize);
-                    content += p.text.slice(cur..*point as usize);
+                    page += p.text.slice(cur..*point as usize);
                 }
                 //           }
                 let pos_markups = pos.map.get(point).unwrap();
                 for m in pos_markups.iter() {
-                    content += &m.apply_markup(pindex);
+                    page += &m.apply_markup(pindex);
                 }
                 cur = incr_cur(cur, *point);
             }
             log::debug!("LAST");
-            content += p.text.slice(cur..);
-            content += &Markup::end(p, pindex, &mut state);
+            page += p.text.slice(cur..);
+            let end = &Markup::end(p, pindex, &mut state);
+            if end == "</pre>" {
+                no_render_html = false;
+            }
+            content += &page;
+            content += end;
         } else {
             log::debug!("LAST WITH NO MARKUP");
-            content += p.text.slice(cur..);
+            page += p.text.slice(cur..);
+            if no_render_html {
+                page = page.replace("<", "&lt;").replace(">", "&gt;");
+            }
+            content += &page;
             content += &Markup::end(p, pindex, &mut state);
         }
         paragraphs.push(content);

src/proxy.rs

@@ -133,7 +133,7 @@ const INDEX: &str = include_str!("../templates/index.html");
 async fn index() -> impl Responder {
     HttpResponse::Ok()
         .content_type("text/html; charset=utf-8")
-        .body(INDEX)
+        .body(INDEX.replace("SOURCE_CODE_REPLACE", &crate::SETTINGS.source_code))
 }
 
 #[actix_web_codegen_const_routes::get(path = "crate::V1_API_ROUTES.proxy.asset")]
@@ -158,29 +158,33 @@ async fn assets(path: web::Path<String>, data: AppData) -> impl Responder {
 
 #[actix_web_codegen_const_routes::get(path = "crate::V1_API_ROUTES.proxy.by_post_id")]
 async fn by_post_id(path: web::Path<String>, data: AppData) -> impl Responder {
-    let post_data = data.get_post_light(&path).await;
-    HttpResponse::Found()
+    match data.get_post_light(&path).await {
+        None => HttpResponse::NotFound().body("Post not found"),
+        Some(post_data) => HttpResponse::Found()
             .append_header((
                 header::LOCATION,
                 crate::V1_API_ROUTES
                     .proxy
                     .get_page(&post_data.username, &post_data.slug),
             ))
-        .finish()
+            .finish(),
+    }
 }
 
 #[actix_web_codegen_const_routes::get(path = "crate::V1_API_ROUTES.proxy.top_level_post")]
 async fn by_top_level_post(path: web::Path<String>, data: AppData) -> impl Responder {
     if let Some(post_id) = path.split('-').last() {
-        let post_data = data.get_post_light(post_id).await;
-        HttpResponse::Found()
+        match data.get_post_light(post_id).await {
+            None => HttpResponse::NotFound().body("Post not found"),
+            Some(post_data) => HttpResponse::Found()
                 .append_header((
                     header::LOCATION,
                     crate::V1_API_ROUTES
                         .proxy
                         .get_page(&post_data.username, &post_data.slug),
                 ))
-            .finish()
+                .finish(),
+        }
     } else {
         HttpResponse::NotFound().body("Post not found, please file bug report")
     }

src/render_html.rs

@@ -38,7 +38,7 @@ pub struct SourcegraphQuery<'a> {
 }
 
 impl<'a> SourcegraphQuery<'a> {
-    pub fn syntax_highlight(&self) -> String {
+    pub fn syntax_highlight(&self, gist_name: &str) -> String {
         //    let ss = SYNTAX_SET;
         let ts = ThemeSet::load_defaults();
 
@@ -66,7 +66,8 @@ impl<'a> SourcegraphQuery<'a> {
                 if line_num == 0 || line_num == total_lines - 1 {
                     output.push_str(line);
                 } else {
-                    output.push_str(&format!("<div title='click for more options' id=\"line-{num}\"class=\"line\"><details class='line_links'><summary class='line_top-link'><a href=\"#line-{num}\"<span class=\"line-number\">{num}</span></a>{line}</summary><a href=\"#line-{num}\"<span class=\"line-link\">Permanant link</span></a><a href=\"#line-{num}\"<span class=\"line-link\">Highlight</span></a></details></div>"
+                    let line_id = format!("{gist_name}-{num}");
+                    output.push_str(&format!("<div title='click for more options' id=\"line-{line_id}\"class=\"line\"><details class='line_links'><summary class='line_top-link'><a href=\"#line-{line_id}\"<span class=\"line-number\">{num}</span></a>{line}</summary><a href=\"#line-{line_id}\"<span class=\"line-link\">Permanant link</span></a><a href=\"#line-{line_id}\"<span class=\"line-link\">Highlight</span></a></details></div>"
                     ));
                     num += 1;
                 }
@@ -153,7 +154,7 @@ mod tests {
         };
         let result = query.determine_language(&syntax_set);
         assert_eq!(result.name, "TeX");
-        let _result = query.syntax_highlight();
+        let _result = query.syntax_highlight("foo");
     }
 
     //#[test]

templates/index.html

@@ -15,7 +15,7 @@
             href="/@tylerneely/fear-and-loathing-in-lock-free-programming-7158b1cdd50c"
             >Demo Article</a
           >
-          | <a href="https://github.com/realaravinth/libmedium">Source Code</a>
+          | <a href="SOURCE_CODE_REPLACE">Source Code</a>
         </p>
       </div>
     </main>

templates/main.css

@@ -7,7 +7,6 @@
 body {
   width: 100%;
   display: flex;
-  flex-direction: column;
 }
 
 main {