ForgeFlux/starchart
2
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

fix & chore: handle hostname/URL cleaning within trait implementation

Browse source 
SUMMARY
    Renamed function parameters from "hostname" to "url" wherever a
    url::Url is received

    db_core::get_hostname is renamed to db_core::clean_url, better
    describing its new implementation

    forge_core::get_hostname is renamed to forge_core::get_url to better
    describe its new implementation

URL PROCESSING METHODS

    federate/federate-core
	Parses URL and returns only the hostname

    db/db-core
	Parses URL, cleans it by removing path and query parameters and
	returns the end result
This commit is contained in:
Aravinth Manivannan 2022-07-14 23:47:37 +05:30
parent 29f5556586
commit fe8bd2fb26
Signed by: realaravinth
GPG key ID: AD9F0F08E855ED88
17 changed files with 273 additions and 232 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Cargo.lock generated
View file

@ -1018,6 +1018,7 @@ version = "0.1.0"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"db-core", "db-core",
"url",
] ]
[[package]] [[package]]

68
db/db-core/src/lib.rs
View file

@ -60,25 +60,29 @@ pub mod dev {
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
/// create a new forge on the database /// create a new forge on the database
pub struct CreateForge<'a> { pub struct CreateForge {
/// hostname of the forge instance: with scheme but remove trailing slash /// url of the forge instance: with scheme but remove trailing slash
pub hostname: &'a str, pub url: Url,
/// forge type: which software is the instance running? /// forge type: which software is the instance running?
pub forge_type: ForgeImplementation, pub forge_type: ForgeImplementation,
} }
/// Get hostname from URL /// Get url from URL
/// Utility function for uniform hostname format /// Utility function for uniform url format
pub fn get_hostname(url: &Url) -> String { pub fn clean_url(url: &Url) -> String {
url.host().as_ref().unwrap().to_string() let mut url = url.clone();
url.set_path("");
url.set_query(None);
url.set_fragment(None);
url.as_str().to_string()
} }
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
/// user data /// user data
pub struct User { pub struct User {
/// hostname of the forge instance: with scheme but remove trailing slash /// url of the forge instance: with scheme but remove trailing slash
/// hostname can be derived from html_link also, but used to link to user's forge instance /// url can be derived from html_link also, but used to link to user's forge instance
pub hostname: String, pub url: String,
/// username of the user /// username of the user
pub username: String, pub username: String,
/// html link to the user profile /// html link to the user profile
@ -90,9 +94,9 @@ pub struct User {
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
/// add new user to database /// add new user to database
pub struct AddUser<'a> { pub struct AddUser<'a> {
/// hostname of the forge instance: with scheme but remove trailing slash /// url of the forge instance: with scheme but remove trailing slash
/// hostname can be derived from html_link also, but used to link to user's forge instance /// url can be derived from html_link also, but used to link to user's forge instance
pub hostname: &'a str, pub url: Url,
/// username of the user /// username of the user
pub username: &'a str, pub username: &'a str,
/// html link to the user profile /// html link to the user profile
@ -108,9 +112,9 @@ pub struct AddRepository<'a> {
pub html_link: &'a str, pub html_link: &'a str,
/// repository topic tags /// repository topic tags
pub tags: Option<Vec<&'a str>>, pub tags: Option<Vec<&'a str>>,
/// hostname of the forge instance: with scheme but remove trailing slash /// url of the forge instance: with scheme but remove trailing slash
/// hostname can be derived from html_link also, but used to link to user's forge instance /// url can be derived from html_link also, but used to link to user's forge instance
pub hostname: &'a str, pub url: Url,
/// repository name /// repository name
pub name: &'a str, pub name: &'a str,
/// repository owner /// repository owner
@ -124,8 +128,8 @@ pub struct AddRepository<'a> {
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
/// data representing a forge instance /// data representing a forge instance
pub struct Forge { pub struct Forge {
/// hostname of the forge /// url of the forge
pub hostname: String, pub url: String,
/// type of the forge /// type of the forge
pub forge_type: ForgeImplementation, pub forge_type: ForgeImplementation,
/// last crawl /// last crawl
@ -139,9 +143,9 @@ pub struct Repository {
pub html_url: String, pub html_url: String,
/// repository topic tags /// repository topic tags
pub tags: Option<Vec<String>>, pub tags: Option<Vec<String>>,
/// hostname of the forge instance: with scheme but remove trailing slash /// url of the forge instance: with scheme but remove trailing slash
/// hostname can be derived from html_link also, but used to link to user's forge instance /// url can be derived from html_link also, but used to link to user's forge instance
pub hostname: String, pub url: String,
/// repository name /// repository name
pub name: String, pub name: String,
/// repository owner /// repository owner
@ -155,8 +159,8 @@ pub struct Repository {
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
/// represents a DNS challenge /// represents a DNS challenge
pub struct Challenge { pub struct Challenge {
/// hostname of the forge instance /// url of the forge instance
pub hostname: String, pub url: String,
/// key of TXT record /// key of TXT record
pub key: String, pub key: String,
/// value of TXT record /// value of TXT record
@ -186,13 +190,13 @@ pub trait SCDatabase: std::marker::Send + std::marker::Sync + CloneSPDatabase {
async fn create_forge_instance(&self, f: &CreateForge) -> DBResult<()>; async fn create_forge_instance(&self, f: &CreateForge) -> DBResult<()>;
/// get forge instance data /// get forge instance data
async fn get_forge(&self, hostname: &str) -> DBResult<Forge>; async fn get_forge(&self, url: &Url) -> DBResult<Forge>;
/// delete forge instance /// delete forge instance
async fn delete_forge_instance(&self, hostname: &str) -> DBResult<()>; async fn delete_forge_instance(&self, url: &Url) -> DBResult<()>;
/// check if a forge instance exists /// check if a forge instance exists
async fn forge_exists(&self, hostname: &str) -> DBResult<bool>; async fn forge_exists(&self, url: &Url) -> DBResult<bool>;
/// check if forge type exists /// check if forge type exists
async fn forge_type_exists(&self, forge_type: &ForgeImplementation) -> DBResult<bool>; async fn forge_type_exists(&self, forge_type: &ForgeImplementation) -> DBResult<bool>;
@ -204,20 +208,20 @@ pub trait SCDatabase: std::marker::Send + std::marker::Sync + CloneSPDatabase {
async fn add_user(&self, u: &AddUser) -> DBResult<()>; async fn add_user(&self, u: &AddUser) -> DBResult<()>;
/// get user data /// get user data
async fn get_user(&self, username: &str, hostname: &str) -> DBResult<User>; async fn get_user(&self, username: &str, url: &Url) -> DBResult<User>;
/// check if an user exists. When hostname of a forge instace is provided, username search is /// check if an user exists. When url of a forge instace is provided, username search is
/// done only on that forge /// done only on that forge
async fn user_exists(&self, username: &str, hostname: Option<&str>) -> DBResult<bool>; async fn user_exists(&self, username: &str, url: Option<&Url>) -> DBResult<bool>;
/// delete user /// delete user
async fn delete_user(&self, username: &str, hostname: &str) -> DBResult<()>; async fn delete_user(&self, username: &str, url: &Url) -> DBResult<()>;
/// delete repository /// delete repository
async fn delete_repository(&self, owner: &str, name: &str, hostname: &str) -> DBResult<()>; async fn delete_repository(&self, owner: &str, name: &str, url: &Url) -> DBResult<()>;
/// check if a repository exists. /// check if a repository exists.
async fn repository_exists(&self, name: &str, owner: &str, hostname: &str) -> DBResult<bool>; async fn repository_exists(&self, name: &str, owner: &str, url: &Url) -> DBResult<bool>;
/// Get all repositories /// Get all repositories
async fn get_all_repositories(&self, offset: u32, limit: u32) -> DBResult<Vec<Repository>>; async fn get_all_repositories(&self, offset: u32, limit: u32) -> DBResult<Vec<Repository>>;

32
db/db-core/src/tests.rs
View file

@ -21,20 +21,20 @@ use crate::prelude::*;
/// adding forge works /// adding forge works
pub async fn adding_forge_works<'a, T: SCDatabase>( pub async fn adding_forge_works<'a, T: SCDatabase>(
db: &T, db: &T,
create_forge_msg: CreateForge<'a>, create_forge_msg: CreateForge,
add_user_msg: AddUser<'a>, add_user_msg: AddUser<'a>,
add_user_msg2: AddUser<'a>, add_user_msg2: AddUser<'a>,
add_repo_msg: AddRepository<'a>, add_repo_msg: AddRepository<'a>,
) { ) {
let _ = db.delete_forge_instance(create_forge_msg.hostname).await; let _ = db.delete_forge_instance(&create_forge_msg.url).await;
db.create_forge_instance(&create_forge_msg).await.unwrap(); db.create_forge_instance(&create_forge_msg).await.unwrap();
assert!( assert!(
db.forge_exists(create_forge_msg.hostname).await.unwrap(), db.forge_exists(&create_forge_msg.url).await.unwrap(),
"forge creation failed, forge exinstance check failure" "forge creation failed, forge existence check failure"
); );
{ {
let forge = db.get_forge(create_forge_msg.hostname).await.unwrap(); let forge = db.get_forge(&create_forge_msg.url).await.unwrap();
let forges = db.get_all_forges(0, 10).await.unwrap(); let forges = db.get_all_forges(0, 10).await.unwrap();
assert_eq!(forges.len(), 1); assert_eq!(forges.len(), 1);
@ -43,11 +43,11 @@ pub async fn adding_forge_works<'a, T: SCDatabase>(
create_forge_msg.forge_type create_forge_msg.forge_type
); );
assert_eq!( assert_eq!(
forges.get(0).as_ref().unwrap().hostname, forges.get(0).as_ref().unwrap().url,
create_forge_msg.hostname crate::clean_url(&create_forge_msg.url)
); );
assert_eq!(forge.hostname, create_forge_msg.hostname); assert_eq!(forge.url, crate::clean_url(&create_forge_msg.url));
assert_eq!(forge.forge_type, create_forge_msg.forge_type); assert_eq!(forge.forge_type, create_forge_msg.forge_type);
} }
@ -56,10 +56,10 @@ pub async fn adding_forge_works<'a, T: SCDatabase>(
db.add_user(&add_user_msg2).await.unwrap(); db.add_user(&add_user_msg2).await.unwrap();
{ {
let db_user = db let db_user = db
.get_user(add_user_msg.username, add_user_msg.hostname) .get_user(add_user_msg.username, &add_user_msg.url)
.await .await
.unwrap(); .unwrap();
assert_eq!(db_user.hostname, add_user_msg.hostname); assert_eq!(db_user.url, crate::clean_url(&add_user_msg.url));
assert_eq!(db_user.username, add_user_msg.username); assert_eq!(db_user.username, add_user_msg.username);
assert_eq!(db_user.html_link, add_user_msg.html_link); assert_eq!(db_user.html_link, add_user_msg.html_link);
assert_eq!( assert_eq!(
@ -70,7 +70,7 @@ pub async fn adding_forge_works<'a, T: SCDatabase>(
// verify user exists // verify user exists
assert!(db.user_exists(add_user_msg.username, None).await.unwrap()); assert!(db.user_exists(add_user_msg.username, None).await.unwrap());
assert!(db assert!(db
.user_exists(add_user_msg.username, Some(add_user_msg.hostname)) .user_exists(add_user_msg.username, Some(&add_user_msg.url))
.await .await
.unwrap()); .unwrap());
@ -78,24 +78,24 @@ pub async fn adding_forge_works<'a, T: SCDatabase>(
db.create_repository(&add_repo_msg).await.unwrap(); db.create_repository(&add_repo_msg).await.unwrap();
// verify repo exists // verify repo exists
assert!(db assert!(db
.repository_exists(add_repo_msg.name, add_repo_msg.owner, add_repo_msg.hostname) .repository_exists(add_repo_msg.name, add_repo_msg.owner, &add_repo_msg.url)
.await .await
.unwrap()); .unwrap());
// delete repository // delete repository
db.delete_repository(add_repo_msg.owner, add_repo_msg.name, add_repo_msg.hostname) db.delete_repository(add_repo_msg.owner, add_repo_msg.name, &add_repo_msg.url)
.await .await
.unwrap(); .unwrap();
assert!(!db assert!(!db
.repository_exists(add_repo_msg.name, add_repo_msg.owner, add_repo_msg.hostname) .repository_exists(add_repo_msg.name, add_repo_msg.owner, &add_repo_msg.url)
.await .await
.unwrap()); .unwrap());
// delete user // delete user
db.delete_user(add_user_msg.username, add_user_msg.hostname) db.delete_user(add_user_msg.username, &add_user_msg.url)
.await .await
.unwrap(); .unwrap();
assert!(!db assert!(!db
.user_exists(add_user_msg.username, Some(add_user_msg.hostname)) .user_exists(add_user_msg.username, Some(&add_user_msg.url))
.await .await
.unwrap()); .unwrap());
} }

1
db/db-sqlx-sqlite/Cargo.toml
View file

@ -20,6 +20,7 @@ path = "src/lib.rs"
sqlx = { version = "0.5.11", features = [ "sqlite", "time", "offline", "runtime-actix-rustls" ] } sqlx = { version = "0.5.11", features = [ "sqlite", "time", "offline", "runtime-actix-rustls" ] }
db-core = {path = "../db-core"} db-core = {path = "../db-core"}
async-trait = "0.1.51" async-trait = "0.1.51"
url = { version = "2.2.2", features = ["serde"] }
[dev-dependencies] [dev-dependencies]
actix-rt = "2" actix-rt = "2"

91
db/db-sqlx-sqlite/src/lib.rs
View file

@ -22,6 +22,7 @@ use db_core::dev::*;
use sqlx::sqlite::SqlitePool; use sqlx::sqlite::SqlitePool;
use sqlx::sqlite::SqlitePoolOptions; use sqlx::sqlite::SqlitePoolOptions;
use sqlx::types::time::OffsetDateTime; use sqlx::types::time::OffsetDateTime;
use url::Url;
pub mod errors; pub mod errors;
#[cfg(test)] #[cfg(test)]
@ -112,11 +113,9 @@ impl SCDatabase for Database {
} }
/// delete forge instance /// delete forge instance
async fn delete_forge_instance(&self, hostname: &str) -> DBResult<()> { async fn delete_forge_instance(&self, url: &Url) -> DBResult<()> {
sqlx::query!( let url = db_core::clean_url(url);
"DELETE FROM starchart_forges WHERE hostname = ($1)", sqlx::query!("DELETE FROM starchart_forges WHERE hostname = ($1)", url,)
hostname,
)
.execute(&self.pool) .execute(&self.pool)
.await .await
.map_err(|e| DBError::DBError(Box::new(e)))?; .map_err(|e| DBError::DBError(Box::new(e)))?;
@ -126,12 +125,13 @@ impl SCDatabase for Database {
/// create forge instance DB /// create forge instance DB
async fn create_forge_instance(&self, f: &CreateForge) -> DBResult<()> { async fn create_forge_instance(&self, f: &CreateForge) -> DBResult<()> {
let now = now_unix_time_stamp(); let now = now_unix_time_stamp();
let url = db_core::clean_url(&f.url);
let forge_type = f.forge_type.to_str(); let forge_type = f.forge_type.to_str();
sqlx::query!( sqlx::query!(
"INSERT INTO "INSERT INTO
starchart_forges (hostname, verified_on, forge_type ) starchart_forges (hostname, verified_on, forge_type )
VALUES ($1, $2, (SELECT ID FROM starchart_forge_type WHERE name = $3))", VALUES ($1, $2, (SELECT ID FROM starchart_forge_type WHERE name = $3))",
f.hostname, url,
now, now,
forge_type, forge_type,
) )
@ -143,7 +143,8 @@ impl SCDatabase for Database {
} }
/// get forge instance data /// get forge instance data
async fn get_forge(&self, hostname: &str) -> DBResult<Forge> { async fn get_forge(&self, url: &Url) -> DBResult<Forge> {
let url = db_core::clean_url(url);
let f = sqlx::query_as!( let f = sqlx::query_as!(
InnerForge, InnerForge,
"SELECT "SELECT
@ -159,7 +160,7 @@ impl SCDatabase for Database {
WHERE WHERE
hostname = $1; hostname = $1;
", ",
hostname, url,
) )
.fetch_one(&self.pool) .fetch_one(&self.pool)
.await .await
@ -200,11 +201,9 @@ impl SCDatabase for Database {
} }
/// check if a forge instance exists /// check if a forge instance exists
async fn forge_exists(&self, hostname: &str) -> DBResult<bool> { async fn forge_exists(&self, url: &Url) -> DBResult<bool> {
match sqlx::query!( let url = db_core::clean_url(url);
"SELECT ID FROM starchart_forges WHERE hostname = $1", match sqlx::query!("SELECT ID FROM starchart_forges WHERE hostname = $1", url)
hostname
)
.fetch_one(&self.pool) .fetch_one(&self.pool)
.await .await
{ {
@ -232,6 +231,7 @@ impl SCDatabase for Database {
/// add new user to database /// add new user to database
async fn add_user(&self, u: &AddUser) -> DBResult<()> { async fn add_user(&self, u: &AddUser) -> DBResult<()> {
let now = now_unix_time_stamp(); let now = now_unix_time_stamp();
let url = db_core::clean_url(&u.url);
sqlx::query!( sqlx::query!(
"INSERT INTO "INSERT INTO
starchart_users ( starchart_users (
@ -240,7 +240,7 @@ impl SCDatabase for Database {
) )
VALUES ( VALUES (
(SELECT ID FROM starchart_forges WHERE hostname = $1), $2, $3, $4, $5, $6)", (SELECT ID FROM starchart_forges WHERE hostname = $1), $2, $3, $4, $5, $6)",
u.hostname, url,
u.username, u.username,
u.html_link, u.html_link,
u.profile_photo, u.profile_photo,
@ -255,38 +255,42 @@ impl SCDatabase for Database {
} }
/// get user data /// get user data
async fn get_user(&self, username: &str, hostname: &str) -> DBResult<User> { async fn get_user(&self, username: &str, url: &Url) -> DBResult<User> {
struct InnerUser { struct InnerUser {
profile_photo_html_url: Option<String>, profile_photo_html_url: Option<String>,
html_url: String, html_url: String,
} }
let url = db_core::clean_url(url);
let res = sqlx::query_as!( let res = sqlx::query_as!(
InnerUser, InnerUser,
"SELECT html_url, profile_photo_html_url FROM starchart_users WHERE username = $1 AND "SELECT html_url, profile_photo_html_url FROM starchart_users WHERE username = $1 AND
hostname_id = (SELECT ID FROM starchart_forges WHERE hostname = $2)", hostname_id = (SELECT ID FROM starchart_forges WHERE hostname = $2)",
username, username,
hostname, url,
) )
.fetch_one(&self.pool) .fetch_one(&self.pool)
.await .await
.map_err(|e| DBError::DBError(Box::new(e)))?; .map_err(|e| DBError::DBError(Box::new(e)))?;
Ok(User { Ok(User {
username: username.into(), username: username.into(),
hostname: hostname.into(), url: url.into(),
profile_photo: res.profile_photo_html_url, profile_photo: res.profile_photo_html_url,
html_link: res.html_url, html_link: res.html_url,
}) })
} }
/// check if an user exists. When hostname of a forge instace is provided, username search is /// check if an user exists. When url of a forge instace is provided, username search is
/// done only on that forge /// done only on that forge
async fn user_exists(&self, username: &str, hostname: Option<&str>) -> DBResult<bool> { async fn user_exists(&self, username: &str, url: Option<&Url>) -> DBResult<bool> {
match hostname { match url {
Some(hostname) => match sqlx::query!( Some(url) => {
let url = db_core::clean_url(url);
match sqlx::query!(
"SELECT ID FROM starchart_users WHERE username = $1 AND "SELECT ID FROM starchart_users WHERE username = $1 AND
hostname_id = (SELECT ID FROM starchart_forges WHERE hostname = $2)", hostname_id = (SELECT ID FROM starchart_forges WHERE hostname = $2)",
username, username,
hostname, url,
) )
.fetch_one(&self.pool) .fetch_one(&self.pool)
.await .await
@ -294,7 +298,8 @@ impl SCDatabase for Database {
Ok(_) => Ok(true), Ok(_) => Ok(true),
Err(Error::RowNotFound) => Ok(false), Err(Error::RowNotFound) => Ok(false),
Err(e) => Err(DBError::DBError(Box::new(e).into())), Err(e) => Err(DBError::DBError(Box::new(e).into())),
}, }
}
None => match sqlx::query!( None => match sqlx::query!(
"SELECT ID FROM starchart_users WHERE username = $1", "SELECT ID FROM starchart_users WHERE username = $1",
username username
@ -310,7 +315,8 @@ impl SCDatabase for Database {
} }
/// check if a repo exists. /// check if a repo exists.
async fn repository_exists(&self, name: &str, owner: &str, hostname: &str) -> DBResult<bool> { async fn repository_exists(&self, name: &str, owner: &str, url: &Url) -> DBResult<bool> {
let url = db_core::clean_url(url);
match sqlx::query!( match sqlx::query!(
"SELECT ID FROM starchart_repositories "SELECT ID FROM starchart_repositories
WHERE WHERE
@ -321,7 +327,7 @@ impl SCDatabase for Database {
hostname_id = (SELECT ID FROM starchart_forges WHERE hostname = $3)", hostname_id = (SELECT ID FROM starchart_forges WHERE hostname = $3)",
name, name,
owner, owner,
hostname, url,
) )
.fetch_one(&self.pool) .fetch_one(&self.pool)
.await .await
@ -336,6 +342,7 @@ impl SCDatabase for Database {
async fn create_repository(&self, r: &AddRepository) -> DBResult<()> { async fn create_repository(&self, r: &AddRepository) -> DBResult<()> {
// unimplemented!() // unimplemented!()
let now = now_unix_time_stamp(); let now = now_unix_time_stamp();
let url = db_core::clean_url(&r.url);
sqlx::query!( sqlx::query!(
"INSERT INTO "INSERT INTO
starchart_repositories ( starchart_repositories (
@ -346,7 +353,7 @@ impl SCDatabase for Database {
(SELECT ID FROM starchart_users WHERE username = $2), (SELECT ID FROM starchart_users WHERE username = $2),
$3, $4, $5, $6, $7, $8 $3, $4, $5, $6, $7, $8
);", );",
r.hostname, url,
r.owner, r.owner,
r.name, r.name,
r.description, r.description,
@ -389,12 +396,13 @@ impl SCDatabase for Database {
} }
/// delete user /// delete user
async fn delete_user(&self, username: &str, hostname: &str) -> DBResult<()> { async fn delete_user(&self, username: &str, url: &Url) -> DBResult<()> {
let url = db_core::clean_url(url);
sqlx::query!( sqlx::query!(
" DELETE FROM starchart_users WHERE username = $1 AND " DELETE FROM starchart_users WHERE username = $1 AND
hostname_id = (SELECT ID FROM starchart_forges WHERE hostname = $2)", hostname_id = (SELECT ID FROM starchart_forges WHERE hostname = $2)",
username, username,
hostname url
) )
.execute(&self.pool) .execute(&self.pool)
.await .await
@ -403,7 +411,8 @@ impl SCDatabase for Database {
} }
/// delete repository /// delete repository
async fn delete_repository(&self, owner: &str, name: &str, hostname: &str) -> DBResult<()> { async fn delete_repository(&self, owner: &str, name: &str, url: &Url) -> DBResult<()> {
let url = db_core::clean_url(url);
sqlx::query!( sqlx::query!(
" DELETE FROM starchart_repositories " DELETE FROM starchart_repositories
WHERE WHERE
@ -414,7 +423,7 @@ impl SCDatabase for Database {
hostname_id = (SELECT ID FROM starchart_forges WHERE hostname = $3)", hostname_id = (SELECT ID FROM starchart_forges WHERE hostname = $3)",
name, name,
owner, owner,
hostname url
) )
.execute(&self.pool) .execute(&self.pool)
.await .await
@ -437,14 +446,24 @@ impl SCDatabase for Database {
} }
async fn get_dns_challenge(&self, key: &str) -> DBResult<Challenge> { async fn get_dns_challenge(&self, key: &str) -> DBResult<Challenge> {
struct InnerChallenge {
hostname: String,
key: String,
value: String,
}
let res = sqlx::query_as!( let res = sqlx::query_as!(
Challenge, InnerChallenge,
"SELECT key, value, hostname FROM starchart_dns_challenges WHERE key = $1", "SELECT key, value, hostname FROM starchart_dns_challenges WHERE key = $1",
key key
) )
.fetch_one(&self.pool) .fetch_one(&self.pool)
.await .await
.map_err(|e| DBError::DBError(Box::new(e)))?; .map_err(|e| DBError::DBError(Box::new(e)))?;
let res = Challenge {
url: res.hostname,
key: res.key,
value: res.value,
};
Ok(res) Ok(res)
} }
@ -463,7 +482,7 @@ impl SCDatabase for Database {
"INSERT INTO "INSERT INTO
starchart_dns_challenges (hostname, value, key, created ) starchart_dns_challenges (hostname, value, key, created )
VALUES ($1, $2, $3, $4);", VALUES ($1, $2, $3, $4);",
challenge.hostname, challenge.url,
challenge.value, challenge.value,
challenge.key, challenge.key,
now, now,
@ -480,8 +499,8 @@ impl SCDatabase for Database {
struct InnerRepository { struct InnerRepository {
/// html link to the repository /// html link to the repository
pub html_url: String, pub html_url: String,
/// hostname of the forge instance: with scheme but remove trailing slash /// url of the forge instance: with scheme but remove trailing slash
/// hostname can be derived from html_link also, but used to link to user's forge instance /// url can be derived from html_link also, but used to link to user's forge instance
pub hostname: String, pub hostname: String,
/// repository name /// repository name
pub name: String, pub name: String,
@ -554,7 +573,7 @@ LIMIT $1 OFFSET $2
}; };
res.push(Repository { res.push(Repository {
html_url: repo.html_url, html_url: repo.html_url,
hostname: repo.hostname, url: repo.hostname,
name: repo.name, name: repo.name,
username: repo.username, username: repo.username,
description: repo.description, description: repo.description,
@ -580,7 +599,7 @@ struct InnerForge {
impl From<InnerForge> for Forge { impl From<InnerForge> for Forge {
fn from(f: InnerForge) -> Self { fn from(f: InnerForge) -> Self {
Self { Self {
hostname: f.hostname, url: f.hostname,
last_crawl_on: f.last_crawl_on, last_crawl_on: f.last_crawl_on,
forge_type: ForgeImplementation::from_str(&f.name).unwrap(), forge_type: ForgeImplementation::from_str(&f.name).unwrap(),
} }

16
db/db-sqlx-sqlite/src/tests.rs
View file

@ -25,7 +25,7 @@ use db_core::tests::*;
#[actix_rt::test] #[actix_rt::test]
async fn everything_works() { async fn everything_works() {
const HOSTNAME: &str = "https://test-gitea.example.com"; const URL: &str = "https://test-gitea.example.com";
const HTML_PROFILE_URL: &str = "https://test-gitea.example.com/user1"; const HTML_PROFILE_URL: &str = "https://test-gitea.example.com/user1";
const HTML_PROFILE_PHOTO_URL_2: &str = "https://test-gitea.example.com/profile-photo/user2"; const HTML_PROFILE_PHOTO_URL_2: &str = "https://test-gitea.example.com/profile-photo/user2";
const USERNAME: &str = "user1"; const USERNAME: &str = "user1";
@ -35,33 +35,35 @@ async fn everything_works() {
const HTML_REPO_URL: &str = "https://test-gitea.example.com/user1/starchart"; const HTML_REPO_URL: &str = "https://test-gitea.example.com/user1/starchart";
const TAGS: [&str; 3] = ["test", "starchart", "spider"]; const TAGS: [&str; 3] = ["test", "starchart", "spider"];
let hostname = Url::parse(HOSTNAME).unwrap(); let url = Url::parse(URL).unwrap();
let hostname = get_hostname(&hostname);
let create_forge_msg = CreateForge { let create_forge_msg = CreateForge {
hostname: &hostname, url: url.clone(),
forge_type: ForgeImplementation::Gitea, forge_type: ForgeImplementation::Gitea,
}; };
let add_user_msg = AddUser { let add_user_msg = AddUser {
hostname: &hostname, url: url.clone(),
html_link: HTML_PROFILE_URL, html_link: HTML_PROFILE_URL,
profile_photo: None, profile_photo: None,
username: USERNAME, username: USERNAME,
}; };
let add_user_msg_2 = AddUser { let add_user_msg_2 = AddUser {
hostname: &hostname, url: url.clone(),
html_link: HTML_PROFILE_PHOTO_URL_2, html_link: HTML_PROFILE_PHOTO_URL_2,
profile_photo: Some(HTML_PROFILE_PHOTO_URL_2), profile_photo: Some(HTML_PROFILE_PHOTO_URL_2),
username: USERNAME2, username: USERNAME2,
}; };
let db = {
let url = env::var("SQLITE_DATABASE_URL").expect("Set SQLITE_DATABASE_URL env var"); let url = env::var("SQLITE_DATABASE_URL").expect("Set SQLITE_DATABASE_URL env var");
let pool_options = SqlitePoolOptions::new().max_connections(2); let pool_options = SqlitePoolOptions::new().max_connections(2);
let connection_options = ConnectionOptions::Fresh(Fresh { pool_options, url }); let connection_options = ConnectionOptions::Fresh(Fresh { pool_options, url });
let db = connection_options.connect().await.unwrap(); let db = connection_options.connect().await.unwrap();
db.migrate().await.unwrap(); db.migrate().await.unwrap();
db
};
let add_repo_msg = AddRepository { let add_repo_msg = AddRepository {
html_link: HTML_REPO_URL, html_link: HTML_REPO_URL,
@ -70,7 +72,7 @@ async fn everything_works() {
owner: USERNAME, owner: USERNAME,
website: None, website: None,
description: None, description: None,
hostname: &hostname, url,
}; };
adding_forge_works( adding_forge_works(

19
federate/federate-core/src/lib.rs
View file

@ -20,6 +20,7 @@ use std::path::PathBuf;
use std::result::Result; use std::result::Result;
use async_trait::async_trait; use async_trait::async_trait;
use url::Url;
use db_core::prelude::*; use db_core::prelude::*;
@ -37,16 +38,16 @@ pub trait Federate: Sync + Send {
async fn rm_util(&self, path: &Path) -> Result<(), Self::Error>; async fn rm_util(&self, path: &Path) -> Result<(), Self::Error>;
/// create forge instance /// create forge instance
async fn create_forge_instance(&self, f: &CreateForge<'_>) -> Result<(), Self::Error>; async fn create_forge_instance(&self, f: &CreateForge) -> Result<(), Self::Error>;
/// delete forge instance /// delete forge instance
async fn delete_forge_instance(&self, hostname: &str) -> Result<(), Self::Error>; async fn delete_forge_instance(&self, url: &Url) -> Result<(), Self::Error>;
/// check if a forge instance exists /// check if a forge instance exists
async fn forge_exists(&self, hostname: &str) -> Result<bool, Self::Error>; async fn forge_exists(&self, url: &Url) -> Result<bool, Self::Error>;
/// check if an user exists. /// check if an user exists.
async fn user_exists(&self, username: &str, hostname: &str) -> Result<bool, Self::Error>; async fn user_exists(&self, username: &str, url: &Url) -> Result<bool, Self::Error>;
/// create user instance /// create user instance
async fn create_user(&self, f: &AddUser<'_>) -> Result<(), Self::Error>; async fn create_user(&self, f: &AddUser<'_>) -> Result<(), Self::Error>;
@ -59,20 +60,24 @@ pub trait Federate: Sync + Send {
&self, &self,
name: &str, name: &str,
owner: &str, owner: &str,
hostname: &str, url: &Url,
) -> Result<bool, Self::Error>; ) -> Result<bool, Self::Error>;
/// delete user /// delete user
async fn delete_user(&self, username: &str, hostname: &str) -> Result<(), Self::Error>; async fn delete_user(&self, username: &str, url: &Url) -> Result<(), Self::Error>;
/// delete repository /// delete repository
async fn delete_repository( async fn delete_repository(
&self, &self,
owner: &str, owner: &str,
name: &str, name: &str,
hostname: &str, url: &Url,
) -> Result<(), Self::Error>; ) -> Result<(), Self::Error>;
/// publish results in tar ball /// publish results in tar ball
async fn tar(&self) -> Result<PathBuf, Self::Error>; async fn tar(&self) -> Result<PathBuf, Self::Error>;
} }
pub fn get_hostname(url: &Url) -> &str {
url.host_str().unwrap()
}

22
federate/federate-core/src/tests.rs
View file

@ -21,34 +21,34 @@ use crate::*;
/// adding forge works /// adding forge works
pub async fn adding_forge_works<'a, T: Federate>( pub async fn adding_forge_works<'a, T: Federate>(
ff: &T, ff: &T,
create_forge_msg: CreateForge<'a>, create_forge_msg: CreateForge,
create_user_msg: AddUser<'a>, create_user_msg: AddUser<'a>,
add_repo_msg: AddRepository<'a>, add_repo_msg: AddRepository<'a>,
) { ) {
let _ = ff.delete_forge_instance(create_forge_msg.hostname).await; let _ = ff.delete_forge_instance(&create_forge_msg.url).await;
assert!(!ff.forge_exists(&create_forge_msg.hostname).await.unwrap()); assert!(!ff.forge_exists(&create_forge_msg.url).await.unwrap());
ff.create_forge_instance(&create_forge_msg).await.unwrap(); ff.create_forge_instance(&create_forge_msg).await.unwrap();
assert!(ff.forge_exists(&create_forge_msg.hostname).await.unwrap()); assert!(ff.forge_exists(&create_forge_msg.url).await.unwrap());
// add user // add user
assert!(!ff assert!(!ff
.user_exists(&create_user_msg.username, &create_user_msg.hostname) .user_exists(&create_user_msg.username, &create_user_msg.url)
.await .await
.unwrap()); .unwrap());
ff.create_user(&create_user_msg).await.unwrap(); ff.create_user(&create_user_msg).await.unwrap();
assert!(ff assert!(ff
.user_exists(&create_user_msg.username, &create_user_msg.hostname) .user_exists(&create_user_msg.username, &create_user_msg.url)
.await .await
.unwrap()); .unwrap());
// add repository // add repository
assert!(!ff assert!(!ff
.repository_exists(add_repo_msg.name, add_repo_msg.owner, add_repo_msg.hostname) .repository_exists(add_repo_msg.name, add_repo_msg.owner, &add_repo_msg.url)
.await .await
.unwrap()); .unwrap());
ff.create_repository(&add_repo_msg).await.unwrap(); ff.create_repository(&add_repo_msg).await.unwrap();
assert!(ff assert!(ff
.repository_exists(add_repo_msg.name, add_repo_msg.owner, add_repo_msg.hostname) .repository_exists(add_repo_msg.name, add_repo_msg.owner, &add_repo_msg.url)
.await .await
.unwrap()); .unwrap());
@ -56,17 +56,17 @@ pub async fn adding_forge_works<'a, T: Federate>(
ff.tar().await.unwrap(); ff.tar().await.unwrap();
// delete repository // delete repository
ff.delete_repository(add_repo_msg.owner, add_repo_msg.name, add_repo_msg.hostname) ff.delete_repository(add_repo_msg.owner, add_repo_msg.name, &add_repo_msg.url)
.await .await
.unwrap(); .unwrap();
// delete user // delete user
ff.delete_user(create_user_msg.username, create_user_msg.hostname) ff.delete_user(create_user_msg.username, &create_user_msg.url)
.await .await
.unwrap(); .unwrap();
// delete user // delete user
ff.delete_forge_instance(create_forge_msg.hostname) ff.delete_forge_instance(&create_forge_msg.url)
.await .await
.unwrap(); .unwrap();
} }

47
federate/publiccodeyml/src/lib.rs
View file

@ -20,6 +20,7 @@ use std::path::{Path, PathBuf};
use async_trait::async_trait; use async_trait::async_trait;
use serde::Serialize; use serde::Serialize;
use tokio::fs; use tokio::fs;
use url::Url;
use db_core::prelude::*; use db_core::prelude::*;
@ -58,7 +59,8 @@ impl PccFederate {
Ok(path) Ok(path)
} }
pub async fn get_instance_path(&self, hostname: &str, create_dirs: bool) -> FResult<PathBuf> { pub async fn get_instance_path(&self, url: &Url, create_dirs: bool) -> FResult<PathBuf> {
let hostname = federate_core::get_hostname(url);
let path = self.get_content_path(false).await?.join(hostname); let path = self.get_content_path(false).await?.join(hostname);
if create_dirs { if create_dirs {
self.create_dir_if_not_exists(&path).await?; self.create_dir_if_not_exists(&path).await?;
@ -69,13 +71,10 @@ impl PccFederate {
pub async fn get_user_path( pub async fn get_user_path(
&self, &self,
username: &str, username: &str,
hostname: &str, url: &Url,
create_dirs: bool, create_dirs: bool,
) -> FResult<PathBuf> { ) -> FResult<PathBuf> {
let path = self let path = self.get_instance_path(url, false).await?.join(username);
.get_instance_path(hostname, false)
.await?
.join(username);
if create_dirs { if create_dirs {
self.create_dir_if_not_exists(&path).await?; self.create_dir_if_not_exists(&path).await?;
} }
@ -86,10 +85,10 @@ impl PccFederate {
&self, &self,
name: &str, name: &str,
owner: &str, owner: &str,
hostname: &str, url: &Url,
create_dirs: bool, create_dirs: bool,
) -> FResult<PathBuf> { ) -> FResult<PathBuf> {
let path = self.get_user_path(owner, hostname, false).await?.join(name); let path = self.get_user_path(owner, url, false).await?.join(name);
if create_dirs { if create_dirs {
self.create_dir_if_not_exists(&path).await?; self.create_dir_if_not_exists(&path).await?;
} }
@ -129,21 +128,21 @@ impl Federate for PccFederate {
} }
/// create forge instance /// create forge instance
async fn create_forge_instance(&self, f: &CreateForge<'_>) -> FResult<()> { async fn create_forge_instance(&self, f: &CreateForge) -> FResult<()> {
let path = self.get_instance_path(f.hostname, true).await?; let path = self.get_instance_path(&f.url, true).await?;
self.write_util(f, &path.join(INSTANCE_INFO_FILE)).await?; self.write_util(f, &path.join(INSTANCE_INFO_FILE)).await?;
Ok(()) Ok(())
} }
/// delete forge instance /// delete forge instance
async fn delete_forge_instance(&self, hostname: &str) -> FResult<()> { async fn delete_forge_instance(&self, url: &Url) -> FResult<()> {
let path = self.get_instance_path(hostname, false).await?; let path = self.get_instance_path(&url, false).await?;
self.rm_util(&path).await self.rm_util(&path).await
} }
/// check if a forge instance exists /// check if a forge instance exists
async fn forge_exists(&self, hostname: &str) -> Result<bool, Self::Error> { async fn forge_exists(&self, url: &Url) -> Result<bool, Self::Error> {
let path = self.get_instance_path(hostname, false).await?; let path = self.get_instance_path(url, false).await?;
if path.exists() && path.is_dir() { if path.exists() && path.is_dir() {
let instance = path.join(INSTANCE_INFO_FILE); let instance = path.join(INSTANCE_INFO_FILE);
Ok(instance.exists() && instance.is_file()) Ok(instance.exists() && instance.is_file())
@ -153,8 +152,8 @@ impl Federate for PccFederate {
} }
/// check if an user exists. /// check if an user exists.
async fn user_exists(&self, username: &str, hostname: &str) -> Result<bool, Self::Error> { async fn user_exists(&self, username: &str, url: &Url) -> Result<bool, Self::Error> {
let path = self.get_user_path(username, hostname, false).await?; let path = self.get_user_path(username, url, false).await?;
if path.exists() && path.is_dir() { if path.exists() && path.is_dir() {
let user = path.join(USER_INFO_FILE); let user = path.join(USER_INFO_FILE);
Ok(user.exists() && user.is_file()) Ok(user.exists() && user.is_file())
@ -165,14 +164,14 @@ impl Federate for PccFederate {
/// create user instance /// create user instance
async fn create_user(&self, f: &AddUser<'_>) -> Result<(), Self::Error> { async fn create_user(&self, f: &AddUser<'_>) -> Result<(), Self::Error> {
let path = self.get_user_path(f.username, f.hostname, true).await?; let path = self.get_user_path(f.username, &f.url, true).await?;
self.write_util(f, &path.join(USER_INFO_FILE)).await self.write_util(f, &path.join(USER_INFO_FILE)).await
} }
/// add repository instance /// add repository instance
async fn create_repository(&self, f: &AddRepository<'_>) -> Result<(), Self::Error> { async fn create_repository(&self, f: &AddRepository<'_>) -> Result<(), Self::Error> {
let path = self let path = self
.get_repo_path(f.name, f.owner, f.hostname, true) .get_repo_path(f.name, f.owner, &f.url, true)
.await? .await?
.join(REPO_INFO_FILE); .join(REPO_INFO_FILE);
let publiccode: schema::Repository = f.into(); let publiccode: schema::Repository = f.into();
@ -184,9 +183,9 @@ impl Federate for PccFederate {
&self, &self,
name: &str, name: &str,
owner: &str, owner: &str,
hostname: &str, url: &Url,
) -> Result<bool, Self::Error> { ) -> Result<bool, Self::Error> {
let path = self.get_repo_path(name, owner, hostname, false).await?; let path = self.get_repo_path(name, owner, url, false).await?;
if path.exists() && path.is_dir() { if path.exists() && path.is_dir() {
let repo = path.join(REPO_INFO_FILE); let repo = path.join(REPO_INFO_FILE);
Ok(repo.exists() && repo.is_file()) Ok(repo.exists() && repo.is_file())
@ -196,8 +195,8 @@ impl Federate for PccFederate {
} }
/// delete user /// delete user
async fn delete_user(&self, username: &str, hostname: &str) -> Result<(), Self::Error> { async fn delete_user(&self, username: &str, url: &Url) -> Result<(), Self::Error> {
let path = self.get_user_path(username, hostname, false).await?; let path = self.get_user_path(username, url, false).await?;
self.rm_util(&path).await?; self.rm_util(&path).await?;
Ok(()) Ok(())
} }
@ -207,9 +206,9 @@ impl Federate for PccFederate {
&self, &self,
owner: &str, owner: &str,
name: &str, name: &str,
hostname: &str, url: &Url,
) -> Result<(), Self::Error> { ) -> Result<(), Self::Error> {
let path = self.get_repo_path(name, owner, hostname, false).await?; let path = self.get_repo_path(name, owner, url, false).await?;
self.rm_util(&path).await self.rm_util(&path).await
} }

11
federate/publiccodeyml/src/tests.rs
View file

@ -22,7 +22,7 @@ use federate_core::tests;
#[actix_rt::test] #[actix_rt::test]
async fn everything_works() { async fn everything_works() {
const HOSTNAME: &str = "https://test-gitea.example.com"; const URL: &str = "https://test-gitea.example.com";
const HTML_PROFILE_URL: &str = "https://test-gitea.example.com/user1"; const HTML_PROFILE_URL: &str = "https://test-gitea.example.com/user1";
const USERNAME: &str = "user1"; const USERNAME: &str = "user1";
@ -32,16 +32,15 @@ async fn everything_works() {
let tmp_dir = Temp::new_dir().unwrap(); let tmp_dir = Temp::new_dir().unwrap();
let hostname = Url::parse(HOSTNAME).unwrap(); let url = Url::parse(URL).unwrap();
let hostname = get_hostname(&hostname);
let create_forge_msg = CreateForge { let create_forge_msg = CreateForge {
hostname: &hostname, url: url.clone(),
forge_type: ForgeImplementation::Gitea, forge_type: ForgeImplementation::Gitea,
}; };
let add_user_msg = AddUser { let add_user_msg = AddUser {
hostname: &hostname, url: url.clone(),
html_link: HTML_PROFILE_URL, html_link: HTML_PROFILE_URL,
profile_photo: None, profile_photo: None,
username: USERNAME, username: USERNAME,
@ -54,7 +53,7 @@ async fn everything_works() {
owner: USERNAME, owner: USERNAME,
website: None, website: None,
description: None, description: None,
hostname: &hostname, url: url.clone(),
}; };
let pcc = PccFederate::new(tmp_dir.to_str().unwrap().to_string()) let pcc = PccFederate::new(tmp_dir.to_str().unwrap().to_string())

1
forge/forge-core/Cargo.toml
View file

@ -17,6 +17,7 @@ path = "src/lib.rs"
[dependencies] [dependencies]
async-trait = "0.1.51" async-trait = "0.1.51"
url = { version = "2.2.2", features = ["serde"] }
[dependencies.db-core] [dependencies.db-core]
path = "../../db/db-core" path = "../../db/db-core"

39
forge/forge-core/src/lib.rs
View file

@ -20,6 +20,7 @@ use std::sync::Arc;
use async_trait::async_trait; use async_trait::async_trait;
use db_core::prelude::*; use db_core::prelude::*;
use url::Url;
pub mod prelude { pub mod prelude {
pub use super::*; pub use super::*;
@ -33,10 +34,10 @@ pub mod dev {
} }
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct User<'a> { pub struct User {
/// hostname of the forge instance: with scheme but remove trailing slash /// url of the forge instance: with scheme but remove trailing slash
/// hostname can be derived from html_link also, but used to link to user's forge instance /// url can be derived from html_link also, but used to link to user's forge instance
pub hostname: &'a str, pub url: Url,
/// username of the user /// username of the user
pub username: Arc<String>, pub username: Arc<String>,
/// html link to the user profile /// html link to the user profile
@ -45,10 +46,10 @@ pub struct User<'a> {
pub profile_photo: Option<String>, pub profile_photo: Option<String>,
} }
impl<'a> From<&'a User<'a>> for AddUser<'a> { impl<'a> From<&'a User> for AddUser<'a> {
fn from(u: &'a User) -> Self { fn from(u: &'a User) -> Self {
Self { Self {
hostname: u.hostname, url: u.url.clone(),
username: u.username.as_str(), username: u.username.as_str(),
html_link: &u.html_link, html_link: &u.html_link,
profile_photo: u.profile_photo.as_deref(), profile_photo: u.profile_photo.as_deref(),
@ -58,25 +59,25 @@ impl<'a> From<&'a User<'a>> for AddUser<'a> {
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
/// add new repository to database /// add new repository to database
pub struct Repository<'a> { pub struct Repository {
/// html link to the repository /// html link to the repository
pub html_link: String, pub html_link: String,
/// repository topic tags /// repository topic tags
pub tags: Option<Vec<Arc<String>>>, pub tags: Option<Vec<Arc<String>>>,
/// hostname of the forge instance: with scheme but remove trailing slash /// url of the forge instance: with scheme but remove trailing slash
/// hostname can be deras_ref().map(|p| p.as_str()),ived from html_link also, but used to link to user's forge instance /// url can be deras_ref().map(|p| p.as_str()),ived from html_link also, but used to link to user's forge instance
pub hostname: &'a str, pub url: Url,
/// repository name /// repository name
pub name: String, pub name: String,
/// repository owner /// repository owner
pub owner: Arc<User<'a>>, pub owner: Arc<User>,
/// repository description, if any /// repository description, if any
pub description: Option<String>, pub description: Option<String>,
/// repository website, if any /// repository website, if any
pub website: Option<String>, pub website: Option<String>,
} }
impl<'a> From<&'a Repository<'a>> for AddRepository<'a> { impl<'a> From<&'a Repository> for AddRepository<'a> {
fn from(r: &'a Repository) -> Self { fn from(r: &'a Repository) -> Self {
let tags = if let Some(rtags) = &r.tags { let tags = if let Some(rtags) = &r.tags {
let mut tags = Vec::with_capacity(rtags.len()); let mut tags = Vec::with_capacity(rtags.len());
@ -88,7 +89,7 @@ impl<'a> From<&'a Repository<'a>> for AddRepository<'a> {
None None
}; };
Self { Self {
hostname: r.hostname, url: r.url.clone(),
name: &r.name, name: &r.name,
description: r.description.as_deref(), description: r.description.as_deref(),
owner: r.owner.username.as_str(), owner: r.owner.username.as_str(),
@ -99,21 +100,21 @@ impl<'a> From<&'a Repository<'a>> for AddRepository<'a> {
} }
} }
pub type UserMap<'a> = HashMap<Arc<String>, Arc<User<'a>>>; pub type UserMap = HashMap<Arc<String>, Arc<User>>;
pub type Tags = HashSet<Arc<String>>; pub type Tags = HashSet<Arc<String>>;
pub type Repositories<'a> = Vec<Repository<'a>>; pub type Repositories = Vec<Repository>;
pub struct CrawlResp<'a> { pub struct CrawlResp {
pub repos: Repositories<'a>, pub repos: Repositories,
pub tags: Tags, pub tags: Tags,
pub users: UserMap<'a>, pub users: UserMap,
} }
#[async_trait] #[async_trait]
pub trait SCForge: std::marker::Send + std::marker::Sync + CloneSPForge { pub trait SCForge: std::marker::Send + std::marker::Sync + CloneSPForge {
async fn is_forge(&self) -> bool; async fn is_forge(&self) -> bool;
async fn crawl(&self, limit: u64, page: u64, rate_limit: u64) -> CrawlResp; async fn crawl(&self, limit: u64, page: u64, rate_limit: u64) -> CrawlResp;
fn get_hostname(&self) -> &str; fn get_url(&self) -> &Url;
fn forge_type(&self) -> ForgeImplementation; fn forge_type(&self) -> ForgeImplementation;
} }

17
forge/gitea/src/lib.rs
View file

@ -23,7 +23,6 @@ use tokio::task::JoinHandle;
use url::Url; use url::Url;
use db_core::ForgeImplementation; use db_core::ForgeImplementation;
use db_core::SCDatabase;
use forge_core::dev::*; use forge_core::dev::*;
use forge_core::Repository; use forge_core::Repository;
@ -37,24 +36,24 @@ const GITEA_IDENTIFIER: &str = "gitea";
pub struct Gitea { pub struct Gitea {
pub instance_url: Url, pub instance_url: Url,
pub client: Client, pub client: Client,
hostname: String, url: Url,
} }
impl Gitea { impl Gitea {
pub fn new(instance_url: Url, client: Client) -> Self { pub fn new(instance_url: Url, client: Client) -> Self {
let hostname = db_core::get_hostname(&instance_url); let url = Url::parse(&db_core::clean_url(&instance_url)).unwrap();
Self { Self {
instance_url, instance_url,
client, client,
hostname, url,
} }
} }
} }
impl PartialEq for Gitea { impl PartialEq for Gitea {
fn eq(&self, other: &Self) -> bool { fn eq(&self, other: &Self) -> bool {
self.hostname == other.hostname && self.instance_url == other.instance_url self.url == other.url && self.instance_url == other.instance_url
} }
} }
@ -80,8 +79,8 @@ impl SCForge for Gitea {
} }
} }
fn get_hostname(&self) -> &str { fn get_url(&self) -> &Url {
&self.hostname &self.url
} }
fn forge_type(&self) -> ForgeImplementation { fn forge_type(&self) -> ForgeImplementation {
@ -125,7 +124,7 @@ impl SCForge for Gitea {
username, username,
html_link: profile_url.to_string(), html_link: profile_url.to_string(),
profile_photo: Some(u.avatar_url), profile_photo: Some(u.avatar_url),
hostname: &g.hostname, url: g.url.clone(),
}) })
} }
@ -174,7 +173,7 @@ impl SCForge for Gitea {
} }
let frepo = Repository { let frepo = Repository {
hostname: &self.hostname, url: self.url.clone(),
website: empty_is_none(&repo.website), website: empty_is_none(&repo.website),
name: repo.name, name: repo.name,
owner: user, owner: user,

10
src/pages/auth/add.rs
View file

@ -97,18 +97,17 @@ pub async fn add_submit(
db: &BoxDB, db: &BoxDB,
) -> ServiceResult<TXTChallenge> { ) -> ServiceResult<TXTChallenge> {
let url_hostname = Url::parse(&payload.hostname).unwrap(); let url_hostname = Url::parse(&payload.hostname).unwrap();
let hostname = get_hostname(&url_hostname); let key = TXTChallenge::get_challenge_txt_key(&ctx, &url_hostname);
let key = TXTChallenge::get_challenge_txt_key(&ctx, &hostname);
if db.dns_challenge_exists(&key).await? { if db.dns_challenge_exists(&key).await? {
let value = db.get_dns_challenge(&key).await?.value; let value = db.get_dns_challenge(&key).await?.value;
Ok(TXTChallenge { key, value }) Ok(TXTChallenge { key, value })
} else { } else {
let challenge = TXTChallenge::new(ctx, &hostname); let challenge = TXTChallenge::new(ctx, &url_hostname);
let c = Challenge { let c = Challenge {
key: challenge.key, key: challenge.key,
value: challenge.value, value: challenge.value,
hostname: url_hostname.to_string(), url: url_hostname.to_string(),
}; };
db.create_dns_challenge(&c).await?; db.create_dns_challenge(&c).await?;
@ -180,8 +179,7 @@ mod tests {
}; };
println!("{}", payload.hostname); println!("{}", payload.hostname);
let hostname = Url::parse(&payload.hostname).unwrap();
let hostname = get_hostname(&Url::parse(&payload.hostname).unwrap());
let key = TXTChallenge::get_challenge_txt_key(&ctx, &hostname); let key = TXTChallenge::get_challenge_txt_key(&ctx, &hostname);
db.delete_dns_challenge(&key).await.unwrap(); db.delete_dns_challenge(&key).await.unwrap();

7
src/pages/auth/verify.rs
View file

@ -87,7 +87,7 @@ pub async fn get_verify(
let challenge = Challenge { let challenge = Challenge {
key: payload.hostname, key: payload.hostname,
value: "".into(), value: "".into(),
hostname: "".into(), url: "".into(),
}; };
PageError::new(VerifyChallenge::new(&ctx.settings, &challenge), e) PageError::new(VerifyChallenge::new(&ctx.settings, &challenge), e)
@ -124,7 +124,7 @@ pub async fn submit_verify(
let challenge = Challenge { let challenge = Challenge {
key: payload.hostname.clone(), key: payload.hostname.clone(),
value: "".into(), value: "".into(),
hostname: "".into(), url: "".into(),
}; };
PageError::new(VerifyChallenge::new(&ctx.settings, &challenge), e) PageError::new(VerifyChallenge::new(&ctx.settings, &challenge), e)
@ -143,7 +143,8 @@ pub async fn submit_verify(
let federate = federate.clone(); let federate = federate.clone();
let db = db.clone(); let db = db.clone();
let fut = async move { let fut = async move {
ctx.crawl(&value.hostname, &db, &federate).await; ctx.crawl(&Url::parse(&value.url).unwrap(), &db, &federate)
.await;
}; };
tokio::spawn(fut); tokio::spawn(fut);

63
src/spider.rs
View file

@ -15,7 +15,10 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
use std::future::Future;
use log::info; use log::info;
use tokio::sync::oneshot::{error::TryRecvError, Receiver};
use url::Url; use url::Url;
use db_core::prelude::*; use db_core::prelude::*;
@ -25,31 +28,32 @@ use gitea::Gitea;
use crate::ctx::Ctx; use crate::ctx::Ctx;
use crate::db::BoxDB; use crate::db::BoxDB;
use crate::federate::ArcFederate; use crate::federate::ArcFederate;
use crate::ArcCtx;
impl Ctx { impl Ctx {
pub async fn crawl(&self, instance_url: &str, db: &BoxDB, federate: &ArcFederate) { pub async fn crawl(&self, instance_url: &Url, db: &BoxDB, federate: &ArcFederate) {
let forge: Box<dyn SCForge> = Box::new(Gitea::new( info!("[crawl][{instance_url}] Init crawling");
Url::parse(instance_url).unwrap(), let forge: Box<dyn SCForge> =
self.client.clone(), Box::new(Gitea::new(instance_url.clone(), self.client.clone()));
));
if !forge.is_forge().await { if !forge.is_forge().await {
unimplemented!("Forge type unimplemented"); unimplemented!("Forge type unimplemented");
} }
let mut page = 1; let mut page = 1;
let hostname = forge.get_hostname(); let url = forge.get_url();
if !db.forge_exists(hostname).await.unwrap() { if !db.forge_exists(url).await.unwrap() {
info!("[crawl][{hostname}] Creating forge"); info!("[crawl][{url}] Creating forge");
let msg = CreateForge { let mut msg = CreateForge {
hostname, url: url.clone(),
forge_type: forge.forge_type(), forge_type: forge.forge_type(),
}; };
db.create_forge_instance(&msg).await.unwrap(); db.create_forge_instance(&msg).await.unwrap();
} else { } else {
if !federate.forge_exists(hostname).await.unwrap() { if !federate.forge_exists(&url).await.unwrap() {
let forge = db.get_forge(hostname).await.unwrap(); let forge = db.get_forge(&url).await.unwrap();
let msg = CreateForge { let msg = CreateForge {
hostname, url: url.clone(),
forge_type: forge.forge_type, forge_type: forge.forge_type,
}; };
federate.create_forge_instance(&msg).await.unwrap(); federate.create_forge_instance(&msg).await.unwrap();
@ -57,7 +61,7 @@ impl Ctx {
} }
loop { loop {
info!("[crawl][{hostname}] Crawling. page: {page}"); info!("[crawl][{url}] Crawling. page: {page}");
let res = forge let res = forge
.crawl( .crawl(
self.settings.crawler.items_per_api_call, self.settings.crawler.items_per_api_call,
@ -66,23 +70,23 @@ impl Ctx {
) )
.await; .await;
if res.repos.is_empty() { if res.repos.is_empty() {
info!("[crawl][{hostname}] Finished crawling. pages: {}", page - 1); info!("[crawl][{url}] Finished crawling. pages: {}", page - 1);
break; break;
} }
for (username, u) in res.users.iter() { for (username, u) in res.users.iter() {
if !db if !db
.user_exists(username, Some(forge.get_hostname())) .user_exists(username, Some(forge.get_url()))
.await .await
.unwrap() .unwrap()
{ {
info!("[crawl][{hostname}] Creating user: {username}"); info!("[crawl][{url}] Creating user: {username}");
let msg = u.as_ref().into(); let msg = u.as_ref().into();
db.add_user(&msg).await.unwrap(); db.add_user(&msg).await.unwrap();
federate.create_user(&msg).await.unwrap(); federate.create_user(&msg).await.unwrap();
} else { } else {
if !federate if !federate
.user_exists(username, forge.get_hostname()) .user_exists(username, forge.get_url())
.await .await
.unwrap() .unwrap()
{ {
@ -94,17 +98,17 @@ impl Ctx {
for r in res.repos.iter() { for r in res.repos.iter() {
if !db if !db
.repository_exists(&r.name, &r.owner.username, r.hostname) .repository_exists(&r.name, &r.owner.username, &r.url)
.await .await
.unwrap() .unwrap()
{ {
info!("[crawl][{hostname}] Creating repository: {}", r.name); info!("[crawl][{url}] Creating repository: {}", r.name);
let msg = r.into(); let msg = r.into();
db.create_repository(&msg).await.unwrap(); db.create_repository(&msg).await.unwrap();
federate.create_repository(&msg).await.unwrap(); federate.create_repository(&msg).await.unwrap();
} else { } else {
if !federate if !federate
.repository_exists(&r.name, &r.owner.username, r.hostname) .repository_exists(&r.name, &r.owner.username, &r.url)
.await .await
.unwrap() .unwrap()
{ {
@ -116,7 +120,6 @@ impl Ctx {
page += 1; page += 1;
} }
federate.tar().await.unwrap();
} }
} }
@ -133,21 +136,19 @@ mod tests {
#[actix_rt::test] #[actix_rt::test]
async fn crawl_gitea() { async fn crawl_gitea() {
let (db, ctx, federate, _tmp_dir) = sqlx_sqlite::get_ctx().await; let (db, ctx, federate, _tmp_dir) = sqlx_sqlite::get_ctx().await;
ctx.crawl(GITEA_HOST, &db, &federate).await; let url = Url::parse(GITEA_HOST).unwrap();
let hostname = get_hostname(&Url::parse(GITEA_HOST).unwrap()); ctx.crawl(&url, &db, &federate).await;
assert!(db.forge_exists(&hostname).await.unwrap()); // let hostname = get_hostname(&Url::parse(GITEA_HOST).unwrap());
assert!(db assert!(db.forge_exists(&url).await.unwrap());
.user_exists(GITEA_USERNAME, Some(&hostname)) assert!(db.user_exists(GITEA_USERNAME, Some(&url)).await.unwrap());
.await
.unwrap());
assert!(db.user_exists(GITEA_USERNAME, None).await.unwrap()); assert!(db.user_exists(GITEA_USERNAME, None).await.unwrap());
for i in 0..100 { for i in 0..100 {
let repo = format!("repository_{i}"); let repo = format!("repository_{i}");
assert!(db assert!(db
.repository_exists(&repo, GITEA_USERNAME, hostname.as_str()) .repository_exists(&repo, GITEA_USERNAME, &url)
.await .await
.unwrap()) .unwrap())
} }
assert!(db.forge_exists(&hostname).await.unwrap()); assert!(db.forge_exists(&url).await.unwrap());
} }
} }

22
src/verify.rs
View file

@ -20,6 +20,7 @@ use trust_dns_resolver::{
config::{ResolverConfig, ResolverOpts}, config::{ResolverConfig, ResolverOpts},
AsyncResolver, AsyncResolver,
}; };
use url::Url;
use crate::utils::get_random; use crate::utils::get_random;
use crate::ArcCtx; use crate::ArcCtx;
@ -39,11 +40,15 @@ impl TXTChallenge {
format!("starchart-{}", &ctx.settings.server.domain) format!("starchart-{}", &ctx.settings.server.domain)
} }
pub fn get_challenge_txt_key(ctx: &ArcCtx, hostname: &str) -> String { pub fn get_challenge_txt_key(ctx: &ArcCtx, hostname: &Url) -> String {
format!("{}.{}", Self::get_challenge_txt_key_prefix(ctx), hostname) format!(
"{}.{}",
Self::get_challenge_txt_key_prefix(ctx),
hostname.host_str().unwrap()
)
} }
pub fn new(ctx: &ArcCtx, hostname: &str) -> Self { pub fn new(ctx: &ArcCtx, hostname: &Url) -> Self {
let key = Self::get_challenge_txt_key(ctx, hostname); let key = Self::get_challenge_txt_key(ctx, hostname);
let value = get_random(VALUES_LEN); let value = get_random(VALUES_LEN);
Self { key, value } Self { key, value }
@ -62,7 +67,7 @@ impl TXTChallenge {
pub mod tests { pub mod tests {
use super::*; use super::*;
use crate::tests::sqlx_sqlite; use crate::tests::sqlx_sqlite;
pub const BASE_DOMAIN: &str = "forge.forgeflux.org"; pub const BASE_DOMAIN: &str = "https://forge.forgeflux.org";
pub const VALUE: &str = "ifthisvalueisretrievedbyforgefluxstarchartthenthetestshouldpass"; pub const VALUE: &str = "ifthisvalueisretrievedbyforgefluxstarchartthenthetestshouldpass";
#[actix_rt::test] #[actix_rt::test]
@ -71,12 +76,17 @@ pub mod tests {
let (_db, ctx, _federate, _tmp_dir) = sqlx_sqlite::get_ctx().await; let (_db, ctx, _federate, _tmp_dir) = sqlx_sqlite::get_ctx().await;
let key = TXTChallenge::get_challenge_txt_key(&ctx, BASE_DOMAIN); let base_hostname = Url::parse(BASE_DOMAIN).unwrap();
let key = TXTChallenge::get_challenge_txt_key(&ctx, &base_hostname);
let mut txt_challenge = TXTChallenge { let mut txt_challenge = TXTChallenge {
value: VALUE.to_string(), value: VALUE.to_string(),
key: key.clone(), key: key.clone(),
}; };
assert_eq!(TXTChallenge::get_challenge_txt_key(&ctx, BASE_DOMAIN), key,); assert_eq!(
TXTChallenge::get_challenge_txt_key(&ctx, &base_hostname),
key,
);
assert!( assert!(
txt_challenge.verify_txt().await.unwrap(), txt_challenge.verify_txt().await.unwrap(),