feat: implement federate_core on a subset of publiccodeyml schema

REFERENCES
    [0]: https://github.com/forgeflux-org/starchart/issues/3
    [1]: https://github.com/publiccodeyml/publiccode.yml/discussions/157

federate/publiccodeyml/.gitignore

@@ -0,0 +1,2 @@
+/target
+/Cargo.lock

federate/publiccodeyml/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+name = "publiccodeyml"
+version = "0.1.0"
+authors = ["realaravinth <realaravinth@batsense.net>"]
+description = "ForgeFlux StarChart - Federated forge spider"
+documentation = "https://forgeflux.org/"
+edition = "2021"
+license = "AGPLv3 or later version"
+
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+async-trait = "0.1.51"
+serde = { version = "1", features = ["derive"]}
+serde_yaml = "0.8.24"
+tokio = { version = "1.18.2", features = ["fs"]}
+thiserror = "1.0.30"
+url = { version  = "2.2.2", features = ["serde"] }
+
+[dependencies.db-core]
+path = "../../db/db-core"
+
+[dependencies.federate-core]
+path = "../federate-core"
+
+[dev-dependencies]
+actix-rt = "2"
+mktemp = "0.4.1"
+federate-core = { path = "../federate-core", features = ["test"] }

federate/publiccodeyml/src/errors.rs

@@ -0,0 +1,64 @@
+/*
+ * ForgeFlux StarChart - A federated software forge spider
+ * Copyright (C) 2022  Aravinth Manivannan <realaravinth@batsense.net>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+//! represents all the ways a trait can fail using this crate
+use std::error::Error as StdError;
+
+use serde_yaml::Error as YamlError;
+use thiserror::Error;
+use tokio::io::Error as IOError;
+
+use db_core::errors::DBError;
+
+/// Error data structure grouping various error subtypes
+#[derive(Debug, Error)]
+pub enum FederateErorr {
+    /// serialization error
+    #[error("Serialization error: {0}")]
+    SerializationError(YamlError),
+    /// database errors
+    #[error("{0}")]
+    DBError(DBError),
+
+    /// IO Error
+    #[error("{0}")]
+    IOError(IOError),
+}
+
+impl From<DBError> for FederateErorr {
+    fn from(e: DBError) -> Self {
+        Self::DBError(e)
+    }
+}
+
+impl From<IOError> for FederateErorr {
+    fn from(e: IOError) -> Self {
+        Self::IOError(e)
+    }
+}
+
+impl From<YamlError> for FederateErorr {
+    fn from(e: YamlError) -> Self {
+        Self::SerializationError(e)
+    }
+}
+
+/// Convenience type alias for grouping driver-specific errors
+pub type BoxDynError = Box<dyn StdError + 'static + Send + Sync>;
+
+/// Generic result data structure
+pub type FResult<V> = std::result::Result<V, FederateErorr>;

federate/publiccodeyml/src/lib.rs

@@ -0,0 +1,169 @@
+/*
+ * ForgeFlux StarChart - A federated software forge spider
+ * Copyright (C) 2022  Aravinth Manivannan <realaravinth@batsense.net>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+use std::path::{Path, PathBuf};
+
+use async_trait::async_trait;
+use serde::Serialize;
+use tokio::fs;
+
+use db_core::prelude::*;
+
+use federate_core::Federate;
+
+pub mod errors;
+pub mod schema;
+#[cfg(test)]
+mod tests;
+
+use errors::*;
+
+pub const INSTANCE_INFO_FILE: &str = "instance.yml";
+pub const USER_INFO_FILE: &str = "user.yml";
+pub const REPO_INFO_FILE: &str = "publiccode.yml";
+
+#[derive(Clone)]
+pub struct PccFederate {
+    pub base_dir: String,
+}
+
+impl PccFederate {
+    pub async fn new(base_dir: String) -> FResult<Self> {
+        let path = Path::new(&base_dir);
+        if !path.exists() {
+            fs::create_dir_all(&path).await?;
+        }
+        Ok(Self { base_dir })
+    }
+
+    pub async fn get_instance_path(&self, hostname: &str, create_dirs: bool) -> FResult<PathBuf> {
+        let path = Path::new(&self.base_dir).join(hostname);
+        if create_dirs {
+            self.create_dir_if_not_exists(&path).await?;
+        }
+        Ok(path)
+    }
+
+    pub async fn get_user_path(
+        &self,
+        username: &str,
+        hostname: &str,
+        create_dirs: bool,
+    ) -> FResult<PathBuf> {
+        let path = self
+            .get_instance_path(hostname, false)
+            .await?
+            .join(username);
+        if create_dirs {
+            self.create_dir_if_not_exists(&path).await?;
+        }
+        Ok(path)
+    }
+
+    pub async fn get_repo_path(
+        &self,
+        name: &str,
+        owner: &str,
+        hostname: &str,
+        create_dirs: bool,
+    ) -> FResult<PathBuf> {
+        let path = self.get_user_path(owner, hostname, false).await?.join(name);
+        if create_dirs {
+            self.create_dir_if_not_exists(&path).await?;
+        }
+        Ok(path)
+    }
+}
+
+#[async_trait]
+impl Federate for PccFederate {
+    type Error = FederateErorr;
+
+    /// utility method to create dir if not exists
+    async fn create_dir_if_not_exists(&self, path: &Path) -> FResult<()> {
+        if !path.exists() {
+            fs::create_dir_all(path).await?;
+        }
+        Ok(())
+    }
+
+    /// utility method to write data
+    async fn write_util<S: Serialize + Send + Sync>(&self, data: &S, path: &Path) -> FResult<()> {
+        let fcontents = serde_yaml::to_string(data)?;
+        fs::write(path, &fcontents).await?;
+        Ok(())
+    }
+
+    /// utility method to remove file/dir
+    async fn rm_util(&self, path: &Path) -> FResult<()> {
+        if path.exists() {
+            if path.is_dir() {
+                fs::remove_dir_all(path).await?;
+            } else {
+                fs::remove_file(&path).await?;
+            }
+        }
+        Ok(())
+    }
+
+    /// create forge isntance
+    async fn create_forge_isntance(&self, f: &CreateForge<'_>) -> FResult<()> {
+        let path = self.get_instance_path(f.hostname, true).await?;
+        self.write_util(f, &path.join(INSTANCE_INFO_FILE)).await?;
+        Ok(())
+    }
+
+    /// delete forge isntance
+    async fn delete_forge_instance(&self, hostname: &str) -> FResult<()> {
+        let path = self.get_instance_path(hostname, false).await?;
+        self.rm_util(&path).await
+    }
+
+    /// create user isntance
+    async fn create_user(&self, f: &AddUser<'_>) -> Result<(), Self::Error> {
+        let path = self.get_user_path(f.username, f.hostname, true).await?;
+        self.write_util(f, &path.join(USER_INFO_FILE)).await
+    }
+
+    /// add repository isntance
+    async fn create_repository(&self, f: &AddRepository<'_>) -> Result<(), Self::Error> {
+        let path = self
+            .get_repo_path(f.name, f.owner, f.hostname, true)
+            .await?
+            .join(REPO_INFO_FILE);
+        let publiccode: schema::Repository = f.into();
+        self.write_util(&publiccode, &path).await
+    }
+
+    /// delete user
+    async fn delete_user(&self, username: &str, hostname: &str) -> Result<(), Self::Error> {
+        let path = self.get_user_path(username, hostname, false).await?;
+        self.rm_util(&path).await?;
+        Ok(())
+    }
+
+    /// delete repository
+    async fn delete_repository(
+        &self,
+        owner: &str,
+        name: &str,
+        hostname: &str,
+    ) -> Result<(), Self::Error> {
+        let path = self.get_repo_path(name, owner, hostname, false).await?;
+        self.rm_util(&path).await
+    }
+}

federate/publiccodeyml/src/schema.rs

@@ -0,0 +1,100 @@
+/*
+ * ForgeFlux StarChart - A federated software forge spider
+ * Copyright (C) 2022  Aravinth Manivannan <realaravinth@batsense.net>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+use std::collections::HashMap;
+
+use serde::{Deserialize, Serialize};
+use url::Url;
+
+const PUBLIC_CODE_VERSION: &str = "0.2";
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Repository {
+    pub publiccode_yml_version: String,
+    pub name: String,
+    pub url: Url,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub landing_url: Option<Url>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub is_based_on: Option<String>,
+    #[serde(skip_serializing_if = "HashMap::is_empty")]
+    pub description: HashMap<String, Description>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub legal: Option<Legal>,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Description {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub short_description: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub long_description: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub documentation: Option<String>,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Legal {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub license: Option<String>,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Maintenance {
+    #[serde(
+        skip_serializing_if = "Option::is_none",
+        rename(serialize = "type", deserialize = "m_type")
+    )]
+    pub m_type: Option<String>,
+    pub contacts: Vec<Contacts>,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Contacts {
+    pub name: String,
+}
+
+impl From<&db_core::AddRepository<'_>> for Repository {
+    fn from(r: &db_core::AddRepository<'_>) -> Self {
+        let mut description = HashMap::with_capacity(1);
+        description.insert(
+            "en".into(),
+            Description {
+                short_description: r.description.map(|d| d.into()),
+                documentation: r.website.map(|d| d.into()),
+                long_description: None,
+            },
+        );
+
+        let legal = Some(Legal { license: None });
+
+        Self {
+            publiccode_yml_version: PUBLIC_CODE_VERSION.into(),
+            url: Url::parse(r.html_link).unwrap(),
+            landing_url: r.website.map(|s| Url::parse(s).unwrap()),
+            name: r.name.into(),
+            is_based_on: None, // TODO collect is_fork information in forge/*
+            description,
+            legal,
+        }
+    }
+}

federate/publiccodeyml/src/tests.rs

@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2022  Aravinth Manivannan <realaravinth@batsense.net>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+use mktemp::Temp;
+use url::Url;
+
+use crate::*;
+use federate_core::tests;
+
+#[actix_rt::test]
+async fn everything_works() {
+    const HOSTNAME: &str = "https://test-gitea.example.com";
+    const HTML_PROFILE_URL: &str = "https://test-gitea.example.com/user1";
+    const USERNAME: &str = "user1";
+
+    const REPO_NAME: &str = "starchart";
+    const HTML_REPO_URL: &str = "https://test-gitea.example.com/user1/starchart";
+    const TAGS: [&str; 3] = ["test", "starchart", "spider"];
+
+    let tmp_dir = Temp::new_dir().unwrap();
+
+    let hostname = Url::parse(HOSTNAME).unwrap();
+    let hostname = get_hostname(&hostname);
+
+    let create_forge_msg = CreateForge {
+        hostname: &hostname,
+        forge_type: ForgeImplementation::Gitea,
+    };
+
+    let add_user_msg = AddUser {
+        hostname: &hostname,
+        html_link: HTML_PROFILE_URL,
+        profile_photo: None,
+        username: USERNAME,
+    };
+
+    let add_repo_msg = AddRepository {
+        html_link: HTML_REPO_URL,
+        name: REPO_NAME,
+        tags: Some(TAGS.into()),
+        owner: USERNAME,
+        website: None,
+        description: None,
+        hostname: &hostname,
+    };
+
+    let pcc = PccFederate::new(tmp_dir.to_str().unwrap().to_string())
+        .await
+        .unwrap();
+    tests::adding_forge_works(&pcc, create_forge_msg, add_user_msg, add_repo_msg).await;
+}