/* * Copyright (C) 2022 Aravinth Manivannan * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as * published by the Free Software Foundation, either version 3 of the * License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ use std::path::Path; #[cfg(test)] use std::println as info; #[cfg(test)] use std::println as error; #[cfg(test)] use std::println as debug; use futures_util::StreamExt; use reqwest::header::CONTENT_TYPE; use serde::Deserialize; use serde::Serialize; use tokio::fs; use tokio::io::{self, AsyncWriteExt, BufWriter}; #[cfg(not(test))] use tracing::{debug, error, info}; use url::Url; use crate::ctx::Ctx; use crate::db::Site; use crate::errors::*; use crate::settings::Settings; use crate::utils; #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] pub struct Page { pub file_path: String, pub url: Url, } impl Page { pub fn from_site(settings: &Settings, s: Site) -> Self { Self { file_path: utils::get_website_path(settings, &s.owner, &s.url) .to_str() .unwrap() .to_owned(), url: s.url, } } async fn create_parent_dir_all(&self, path: &str) -> ServiceResult<()> { if let Some(parent) = Path::new(path).parent() { fs::create_dir_all(parent).await?; } Ok(()) } pub async fn archive(&self, ctx: &Ctx) -> ServiceResult<()> { self.create_parent_dir_all(&self.file_path).await?; let res = ctx.client.get(self.url.as_str()).send().await?; let mut fetch_res = false; if let Some(content_type) = res.headers().get(CONTENT_TYPE) { if let Ok(content_type) = content_type.to_str() { if content_type.contains("text/html") { fetch_res = true; } } } let mut bytes = res.bytes_stream(); let file = fs::OpenOptions::new() .read(true) .write(true) .truncate(true) .create(true) .open(&self.file_path) .await?; let mut writer = BufWriter::new(file); while let Some(item) = bytes.next().await { let _ = writer.write(&item?).await?; } writer.flush().await?; Ok(()) } }