ForgeFlux/starchart
2
0
Fork 0
Code Issues Pull requests 2 Projects Releases Packages Wiki Activity Actions

feat: mark entries as imported to differentiate b/w native crawls and

Browse source 
imported records
This commit is contained in:
Aravinth Manivannan 2023-02-11 19:40:04 +05:30
parent 3f088a8cb8
commit f8d33e04e8
Signed by: realaravinth
GPG key ID: AD9F0F08E855ED88
5 changed files with 96 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
db/db-core/src/lib.rs
View file

@ -65,6 +65,8 @@ pub struct CreateForge {
pub url: Url, pub url: Url,
/// forge type: which software is the instance running? /// forge type: which software is the instance running?
pub forge_type: ForgeImplementation, pub forge_type: ForgeImplementation,
/// is this forge an import
pub import: bool,
} }
/// Get url from URL /// Get url from URL
@ -89,6 +91,8 @@ pub struct User {
pub html_link: String, pub html_link: String,
/// OPTIONAL: html link to the user's profile photo /// OPTIONAL: html link to the user's profile photo
pub profile_photo: Option<String>, pub profile_photo: Option<String>,
/// is this user an import
pub import: bool,
} }
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@ -103,6 +107,8 @@ pub struct AddUser<'a> {
pub html_link: &'a str, pub html_link: &'a str,
/// OPTIONAL: html link to the user's profile photo /// OPTIONAL: html link to the user's profile photo
pub profile_photo: Option<&'a str>, pub profile_photo: Option<&'a str>,
/// is this user an import
pub import: bool,
} }
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@ -123,6 +129,8 @@ pub struct AddRepository<'a> {
pub description: Option<&'a str>, pub description: Option<&'a str>,
/// repository website, if any /// repository website, if any
pub website: Option<&'a str>, pub website: Option<&'a str>,
/// is this repository an import
pub import: bool,
} }
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@ -134,6 +142,8 @@ pub struct Forge {
pub forge_type: ForgeImplementation, pub forge_type: ForgeImplementation,
/// last crawl /// last crawl
pub last_crawl_on: Option<i64>, pub last_crawl_on: Option<i64>,
/// is this forge an import
pub import: bool,
} }
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@ -154,6 +164,8 @@ pub struct Repository {
pub description: Option<String>, pub description: Option<String>,
/// repository website, if any /// repository website, if any
pub website: Option<String>, pub website: Option<String>,
/// is this repository an import
pub import: bool,
} }
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]

4
db/db-sqlx-sqlite/Cargo.toml
View file

@ -17,13 +17,13 @@ name = "db_sqlx_sqlite"
path = "src/lib.rs" path = "src/lib.rs"
[dependencies] [dependencies]
sqlx = { version = "0.5.11", features = [ "sqlite", "time", "offline", "runtime-actix-rustls" ] } sqlx = { version = "0.6.2", features = [ "sqlite", "time", "offline", "runtime-actix-rustls" ] }
db-core = {path = "../db-core"} db-core = {path = "../db-core"}
async-trait = "0.1.51" async-trait = "0.1.51"
url = { version = "2.2.2", features = ["serde"] } url = { version = "2.2.2", features = ["serde"] }
[dev-dependencies] [dev-dependencies]
actix-rt = "2" actix-rt = "2"
sqlx = { version = "0.5.11", features = [ "runtime-actix-rustls", "postgres", "time", "offline" ] } sqlx = { version = "0.6.2", features = [ "runtime-actix-rustls", "postgres", "time", "offline" ] }
db-core = {path = "../db-core", features = ["test"]} db-core = {path = "../db-core", features = ["test"]}
url = { version = "2.2.2", features = ["serde"] } url = { version = "2.2.2", features = ["serde"] }

11
db/db-sqlx-sqlite/migrations/20230211075309_starchart_import.sql Normal file
View file

@ -0,0 +1,11 @@
ALTER TABLE starchart_forges ADD COLUMN imported BOOLEAN NOT NULL DEFAULT(0);
ALTER TABLE starchart_users ADD COLUMN imported BOOLEAN NOT NULL DEFAULT(0);
ALTER TABLE starchart_repositories ADD COLUMN imported BOOLEAN NOT NULL DEFAULT(0);
CREATE TABLE IF NOT EXISTS foo (
ID INTEGER PRIMARY KEY NOT NULL
);
DROP TABLE foo;

53
db/db-sqlx-sqlite/src/lib.rs
View file

@ -129,11 +129,12 @@ impl SCDatabase for Database {
let forge_type = f.forge_type.to_str(); let forge_type = f.forge_type.to_str();
sqlx::query!( sqlx::query!(
"INSERT INTO "INSERT INTO
starchart_forges (hostname, verified_on, forge_type ) starchart_forges (hostname, verified_on, forge_type, imported)
VALUES ($1, $2, (SELECT ID FROM starchart_forge_type WHERE name = $3))", VALUES ($1, $2, (SELECT ID FROM starchart_forge_type WHERE name = $3), $4)",
url, url,
now, now,
forge_type, forge_type,
f.import,
) )
.execute(&self.pool) .execute(&self.pool)
.await .await
@ -150,6 +151,7 @@ impl SCDatabase for Database {
"SELECT "SELECT
hostname, hostname,
last_crawl_on, last_crawl_on,
imported,
starchart_forge_type.name starchart_forge_type.name
FROM FROM
starchart_forges starchart_forges
@ -176,7 +178,8 @@ impl SCDatabase for Database {
"SELECT "SELECT
hostname, hostname,
last_crawl_on, last_crawl_on,
starchart_forge_type.name starchart_forge_type.name,
imported
FROM FROM
starchart_forges starchart_forges
INNER JOIN INNER JOIN
@ -236,16 +239,17 @@ impl SCDatabase for Database {
"INSERT INTO "INSERT INTO
starchart_users ( starchart_users (
hostname_id, username, html_url, hostname_id, username, html_url,
profile_photo_html_url, added_on, last_crawl_on profile_photo_html_url, added_on, last_crawl_on, imported
) )
VALUES ( VALUES (
(SELECT ID FROM starchart_forges WHERE hostname = $1), $2, $3, $4, $5, $6)", (SELECT ID FROM starchart_forges WHERE hostname = $1), $2, $3, $4, $5, $6, $7)",
url, url,
u.username, u.username,
u.html_link, u.html_link,
u.profile_photo, u.profile_photo,
now, now,
now now,
u.import
) )
.execute(&self.pool) .execute(&self.pool)
.await .await
@ -259,12 +263,13 @@ impl SCDatabase for Database {
struct InnerUser { struct InnerUser {
profile_photo_html_url: Option<String>, profile_photo_html_url: Option<String>,
html_url: String, html_url: String,
imported: bool,
} }
let url = db_core::clean_url(url); let url = db_core::clean_url(url);
let res = sqlx::query_as!( let res = sqlx::query_as!(
InnerUser, InnerUser,
"SELECT html_url, profile_photo_html_url FROM starchart_users WHERE username = $1 AND "SELECT html_url, profile_photo_html_url, imported FROM starchart_users WHERE username = $1 AND
hostname_id = (SELECT ID FROM starchart_forges WHERE hostname = $2)", hostname_id = (SELECT ID FROM starchart_forges WHERE hostname = $2)",
username, username,
url, url,
@ -277,6 +282,7 @@ impl SCDatabase for Database {
url, url,
profile_photo: res.profile_photo_html_url, profile_photo: res.profile_photo_html_url,
html_link: res.html_url, html_link: res.html_url,
import: res.imported,
}) })
} }
@ -346,12 +352,13 @@ impl SCDatabase for Database {
sqlx::query!( sqlx::query!(
"INSERT INTO "INSERT INTO
starchart_repositories ( starchart_repositories (
hostname_id, owner_id, name, description, html_url, website, created, last_crawl hostname_id, owner_id, name, description, html_url, website, created,
last_crawl, imported
) )
VALUES ( VALUES (
(SELECT ID FROM starchart_forges WHERE hostname = $1), (SELECT ID FROM starchart_forges WHERE hostname = $1),
(SELECT ID FROM starchart_users WHERE username = $2), (SELECT ID FROM starchart_users WHERE username = $2),
$3, $4, $5, $6, $7, $8 $3, $4, $5, $6, $7, $8, $9
);", );",
url, url,
r.owner, r.owner,
@ -360,7 +367,8 @@ impl SCDatabase for Database {
r.html_link, r.html_link,
r.website, r.website,
now, now,
now now,
r.import,
) )
.execute(&self.pool) .execute(&self.pool)
.await .await
@ -512,32 +520,34 @@ impl SCDatabase for Database {
/// repository website, if any /// repository website, if any
pub website: Option<String>, pub website: Option<String>,
pub ID: i64, pub ID: i64,
pub imported: bool,
} }
let mut db_res = sqlx::query_as!( let mut db_res = sqlx::query_as!(
InnerRepository, InnerRepository,
" SELECT "SELECT
starchart_forges.hostname, starchart_forges.hostname,
starchart_users.username, starchart_users.username,
starchart_repositories.name, starchart_repositories.name,
starchart_repositories.description, starchart_repositories.description,
starchart_repositories.html_url, starchart_repositories.html_url,
starchart_repositories.ID, starchart_repositories.ID,
starchart_repositories.website starchart_repositories.website,
FROM starchart_repositories.imported
FROM
starchart_repositories starchart_repositories
INNER JOIN INNER JOIN
starchart_forges starchart_forges
ON ON
starchart_repositories.hostname_id = starchart_forges.id starchart_repositories.hostname_id = starchart_forges.id
INNER JOIN INNER JOIN
starchart_users starchart_users
ON ON
starchart_repositories.owner_id = starchart_users.id starchart_repositories.owner_id = starchart_users.id
ORDER BY ORDER BY
starchart_repositories.ID starchart_repositories.ID
LIMIT $1 OFFSET $2 LIMIT $1 OFFSET $2
;", ;",
limit, limit,
offset, offset,
) )
@ -580,6 +590,7 @@ LIMIT $1 OFFSET $2
description: repo.description, description: repo.description,
website: repo.website, website: repo.website,
tags: topics, tags: topics,
import: repo.imported,
}); });
} }
@ -595,6 +606,7 @@ struct InnerForge {
hostname: String, hostname: String,
last_crawl_on: Option<i64>, last_crawl_on: Option<i64>,
name: String, name: String,
imported: bool,
} }
impl From<InnerForge> for Forge { impl From<InnerForge> for Forge {
@ -603,6 +615,7 @@ impl From<InnerForge> for Forge {
url: f.hostname, url: f.hostname,
last_crawl_on: f.last_crawl_on, last_crawl_on: f.last_crawl_on,
forge_type: ForgeImplementation::from_str(&f.name).unwrap(), forge_type: ForgeImplementation::from_str(&f.name).unwrap(),
import: f.imported,
} }
} }
} }

4
db/db-sqlx-sqlite/src/tests.rs
View file

@ -40,6 +40,7 @@ async fn everything_works() {
let create_forge_msg = CreateForge { let create_forge_msg = CreateForge {
url: url.clone(), url: url.clone(),
forge_type: ForgeImplementation::Gitea, forge_type: ForgeImplementation::Gitea,
import: false,
}; };
let add_user_msg = AddUser { let add_user_msg = AddUser {
@ -47,6 +48,7 @@ async fn everything_works() {
html_link: HTML_PROFILE_URL, html_link: HTML_PROFILE_URL,
profile_photo: None, profile_photo: None,
username: USERNAME, username: USERNAME,
import: false,
}; };
let add_user_msg_2 = AddUser { let add_user_msg_2 = AddUser {
@ -54,6 +56,7 @@ async fn everything_works() {
html_link: HTML_PROFILE_PHOTO_URL_2, html_link: HTML_PROFILE_PHOTO_URL_2,
profile_photo: Some(HTML_PROFILE_PHOTO_URL_2), profile_photo: Some(HTML_PROFILE_PHOTO_URL_2),
username: USERNAME2, username: USERNAME2,
import: false,
}; };
let db = { let db = {
@ -73,6 +76,7 @@ async fn everything_works() {
website: None, website: None,
description: None, description: None,
url, url,
import: false,
}; };
adding_forge_works( adding_forge_works(