ForgeFlux/starchart
2
0
Fork 0
Code Issues Pull requests 2 Projects Releases Packages Wiki Activity Actions

feat: mark entries as imported to differentiate b/w native crawls and

Browse source 
imported records
This commit is contained in:
Aravinth Manivannan 2023-02-11 19:40:04 +05:30
parent 3f088a8cb8
commit f8d33e04e8
Signed by: realaravinth
GPG key ID: AD9F0F08E855ED88
5 changed files with 96 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
db/db-core/src/lib.rs
View file

@ -65,6 +65,8 @@ pub struct CreateForge {
pub url: Url,
/// forge type: which software is the instance running?
pub forge_type: ForgeImplementation,
/// is this forge an import
pub import: bool,
}
/// Get url from URL
@ -89,6 +91,8 @@ pub struct User {
pub html_link: String,
/// OPTIONAL: html link to the user's profile photo
pub profile_photo: Option<String>,
/// is this user an import
pub import: bool,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@ -103,6 +107,8 @@ pub struct AddUser<'a> {
pub html_link: &'a str,
/// OPTIONAL: html link to the user's profile photo
pub profile_photo: Option<&'a str>,
/// is this user an import
pub import: bool,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@ -123,6 +129,8 @@ pub struct AddRepository<'a> {
pub description: Option<&'a str>,
/// repository website, if any
pub website: Option<&'a str>,
/// is this repository an import
pub import: bool,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@ -134,6 +142,8 @@ pub struct Forge {
pub forge_type: ForgeImplementation,
/// last crawl
pub last_crawl_on: Option<i64>,
/// is this forge an import
pub import: bool,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@ -154,6 +164,8 @@ pub struct Repository {
pub description: Option<String>,
/// repository website, if any
pub website: Option<String>,
/// is this repository an import
pub import: bool,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]

4
db/db-sqlx-sqlite/Cargo.toml
View file

@ -17,13 +17,13 @@ name = "db_sqlx_sqlite"
path = "src/lib.rs"
[dependencies]
sqlx = { version = "0.5.11", features = [ "sqlite", "time", "offline", "runtime-actix-rustls" ] }
sqlx = { version = "0.6.2", features = [ "sqlite", "time", "offline", "runtime-actix-rustls" ] }
db-core = {path = "../db-core"}
async-trait = "0.1.51"
url = { version = "2.2.2", features = ["serde"] }
[dev-dependencies]
actix-rt = "2"
sqlx = { version = "0.5.11", features = [ "runtime-actix-rustls", "postgres", "time", "offline" ] }
sqlx = { version = "0.6.2", features = [ "runtime-actix-rustls", "postgres", "time", "offline" ] }
db-core = {path = "../db-core", features = ["test"]}
url = { version = "2.2.2", features = ["serde"] }

11
db/db-sqlx-sqlite/migrations/20230211075309_starchart_import.sql Normal file
View file

@ -0,0 +1,11 @@
ALTER TABLE starchart_forges ADD COLUMN imported BOOLEAN NOT NULL DEFAULT(0);
ALTER TABLE starchart_users ADD COLUMN imported BOOLEAN NOT NULL DEFAULT(0);
ALTER TABLE starchart_repositories ADD COLUMN imported BOOLEAN NOT NULL DEFAULT(0);
CREATE TABLE IF NOT EXISTS foo (
ID INTEGER PRIMARY KEY NOT NULL
);
DROP TABLE foo;

35
db/db-sqlx-sqlite/src/lib.rs
View file

@ -129,11 +129,12 @@ impl SCDatabase for Database {
let forge_type = f.forge_type.to_str();
sqlx::query!(
"INSERT INTO
starchart_forges (hostname, verified_on, forge_type )
VALUES ($1, $2, (SELECT ID FROM starchart_forge_type WHERE name = $3))",
starchart_forges (hostname, verified_on, forge_type, imported)
VALUES ($1, $2, (SELECT ID FROM starchart_forge_type WHERE name = $3), $4)",
url,
now,
forge_type,
f.import,
)
.execute(&self.pool)
.await
@ -150,6 +151,7 @@ impl SCDatabase for Database {
"SELECT
hostname,
last_crawl_on,
imported,
starchart_forge_type.name
FROM
starchart_forges
@ -176,7 +178,8 @@ impl SCDatabase for Database {
"SELECT
hostname,
last_crawl_on,
starchart_forge_type.name
starchart_forge_type.name,
imported
FROM
starchart_forges
INNER JOIN
@ -236,16 +239,17 @@ impl SCDatabase for Database {
"INSERT INTO
starchart_users (
hostname_id, username, html_url,
profile_photo_html_url, added_on, last_crawl_on
profile_photo_html_url, added_on, last_crawl_on, imported
)
VALUES (
(SELECT ID FROM starchart_forges WHERE hostname = $1), $2, $3, $4, $5, $6)",
(SELECT ID FROM starchart_forges WHERE hostname = $1), $2, $3, $4, $5, $6, $7)",
url,
u.username,
u.html_link,
u.profile_photo,
now,
now
now,
u.import
)
.execute(&self.pool)
.await
@ -259,12 +263,13 @@ impl SCDatabase for Database {
struct InnerUser {
profile_photo_html_url: Option<String>,
html_url: String,
imported: bool,
}
let url = db_core::clean_url(url);
let res = sqlx::query_as!(
InnerUser,
"SELECT html_url, profile_photo_html_url FROM starchart_users WHERE username = $1 AND
"SELECT html_url, profile_photo_html_url, imported FROM starchart_users WHERE username = $1 AND
hostname_id = (SELECT ID FROM starchart_forges WHERE hostname = $2)",
username,
url,
@ -277,6 +282,7 @@ impl SCDatabase for Database {
url,
profile_photo: res.profile_photo_html_url,
html_link: res.html_url,
import: res.imported,
})
}
@ -346,12 +352,13 @@ impl SCDatabase for Database {
sqlx::query!(
"INSERT INTO
starchart_repositories (
hostname_id, owner_id, name, description, html_url, website, created, last_crawl
hostname_id, owner_id, name, description, html_url, website, created,
last_crawl, imported
)
VALUES (
(SELECT ID FROM starchart_forges WHERE hostname = $1),
(SELECT ID FROM starchart_users WHERE username = $2),
$3, $4, $5, $6, $7, $8
$3, $4, $5, $6, $7, $8, $9
);",
url,
r.owner,
@ -360,7 +367,8 @@ impl SCDatabase for Database {
r.html_link,
r.website,
now,
now
now,
r.import,
)
.execute(&self.pool)
.await
@ -512,6 +520,7 @@ impl SCDatabase for Database {
/// repository website, if any
pub website: Option<String>,
pub ID: i64,
pub imported: bool,
}
let mut db_res = sqlx::query_as!(
@ -523,7 +532,8 @@ impl SCDatabase for Database {
starchart_repositories.description,
starchart_repositories.html_url,
starchart_repositories.ID,
starchart_repositories.website
starchart_repositories.website,
starchart_repositories.imported
FROM
starchart_repositories
INNER JOIN
@ -580,6 +590,7 @@ LIMIT $1 OFFSET $2
description: repo.description,
website: repo.website,
tags: topics,
import: repo.imported,
});
}
@ -595,6 +606,7 @@ struct InnerForge {
hostname: String,
last_crawl_on: Option<i64>,
name: String,
imported: bool,
}
impl From<InnerForge> for Forge {
@ -603,6 +615,7 @@ impl From<InnerForge> for Forge {
url: f.hostname,
last_crawl_on: f.last_crawl_on,
forge_type: ForgeImplementation::from_str(&f.name).unwrap(),
import: f.imported,
}
}
}

4
db/db-sqlx-sqlite/src/tests.rs
View file

@ -40,6 +40,7 @@ async fn everything_works() {
let create_forge_msg = CreateForge {
url: url.clone(),
forge_type: ForgeImplementation::Gitea,
import: false,
};
let add_user_msg = AddUser {
@ -47,6 +48,7 @@ async fn everything_works() {
html_link: HTML_PROFILE_URL,
profile_photo: None,
username: USERNAME,
import: false,
};
let add_user_msg_2 = AddUser {
@ -54,6 +56,7 @@ async fn everything_works() {
html_link: HTML_PROFILE_PHOTO_URL_2,
profile_photo: Some(HTML_PROFILE_PHOTO_URL_2),
username: USERNAME2,
import: false,
};
let db = {
@ -73,6 +76,7 @@ async fn everything_works() {
website: None,
description: None,
url,
import: false,
};
adding_forge_works(