diff --git a/db/db-core/src/lib.rs b/db/db-core/src/lib.rs index 2603f10..ad9a78b 100644 --- a/db/db-core/src/lib.rs +++ b/db/db-core/src/lib.rs @@ -65,6 +65,8 @@ pub struct CreateForge { pub url: Url, /// forge type: which software is the instance running? pub forge_type: ForgeImplementation, + /// is this forge an import + pub import: bool, } /// Get url from URL @@ -89,6 +91,8 @@ pub struct User { pub html_link: String, /// OPTIONAL: html link to the user's profile photo pub profile_photo: Option, + /// is this user an import + pub import: bool, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -103,6 +107,8 @@ pub struct AddUser<'a> { pub html_link: &'a str, /// OPTIONAL: html link to the user's profile photo pub profile_photo: Option<&'a str>, + /// is this user an import + pub import: bool, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -123,6 +129,8 @@ pub struct AddRepository<'a> { pub description: Option<&'a str>, /// repository website, if any pub website: Option<&'a str>, + /// is this repository an import + pub import: bool, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -134,6 +142,8 @@ pub struct Forge { pub forge_type: ForgeImplementation, /// last crawl pub last_crawl_on: Option, + /// is this forge an import + pub import: bool, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -154,6 +164,8 @@ pub struct Repository { pub description: Option, /// repository website, if any pub website: Option, + /// is this repository an import + pub import: bool, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] diff --git a/db/db-sqlx-sqlite/Cargo.toml b/db/db-sqlx-sqlite/Cargo.toml index 3395dc0..8dd1aaf 100644 --- a/db/db-sqlx-sqlite/Cargo.toml +++ b/db/db-sqlx-sqlite/Cargo.toml @@ -17,13 +17,13 @@ name = "db_sqlx_sqlite" path = "src/lib.rs" [dependencies] -sqlx = { version = "0.5.11", features = [ "sqlite", "time", "offline", "runtime-actix-rustls" ] } +sqlx = { version = "0.6.2", features = [ "sqlite", "time", "offline", "runtime-actix-rustls" ] } db-core = {path = "../db-core"} async-trait = "0.1.51" url = { version = "2.2.2", features = ["serde"] } [dev-dependencies] actix-rt = "2" -sqlx = { version = "0.5.11", features = [ "runtime-actix-rustls", "postgres", "time", "offline" ] } +sqlx = { version = "0.6.2", features = [ "runtime-actix-rustls", "postgres", "time", "offline" ] } db-core = {path = "../db-core", features = ["test"]} url = { version = "2.2.2", features = ["serde"] } diff --git a/db/db-sqlx-sqlite/migrations/20230211075309_starchart_import.sql b/db/db-sqlx-sqlite/migrations/20230211075309_starchart_import.sql new file mode 100644 index 0000000..94a0b0e --- /dev/null +++ b/db/db-sqlx-sqlite/migrations/20230211075309_starchart_import.sql @@ -0,0 +1,11 @@ +ALTER TABLE starchart_forges ADD COLUMN imported BOOLEAN NOT NULL DEFAULT(0); + +ALTER TABLE starchart_users ADD COLUMN imported BOOLEAN NOT NULL DEFAULT(0); + +ALTER TABLE starchart_repositories ADD COLUMN imported BOOLEAN NOT NULL DEFAULT(0); + +CREATE TABLE IF NOT EXISTS foo ( + ID INTEGER PRIMARY KEY NOT NULL +); + +DROP TABLE foo; diff --git a/db/db-sqlx-sqlite/src/lib.rs b/db/db-sqlx-sqlite/src/lib.rs index ea35239..f1aa5e5 100644 --- a/db/db-sqlx-sqlite/src/lib.rs +++ b/db/db-sqlx-sqlite/src/lib.rs @@ -129,11 +129,12 @@ impl SCDatabase for Database { let forge_type = f.forge_type.to_str(); sqlx::query!( "INSERT INTO - starchart_forges (hostname, verified_on, forge_type ) - VALUES ($1, $2, (SELECT ID FROM starchart_forge_type WHERE name = $3))", + starchart_forges (hostname, verified_on, forge_type, imported) + VALUES ($1, $2, (SELECT ID FROM starchart_forge_type WHERE name = $3), $4)", url, now, forge_type, + f.import, ) .execute(&self.pool) .await @@ -148,17 +149,18 @@ impl SCDatabase for Database { let f = sqlx::query_as!( InnerForge, "SELECT - hostname, - last_crawl_on, - starchart_forge_type.name - FROM - starchart_forges - INNER JOIN - starchart_forge_type - ON - starchart_forges.forge_type = starchart_forge_type.id - WHERE - hostname = $1; + hostname, + last_crawl_on, + imported, + starchart_forge_type.name + FROM + starchart_forges + INNER JOIN + starchart_forge_type + ON + starchart_forges.forge_type = starchart_forge_type.id + WHERE + hostname = $1; ", url, ) @@ -174,18 +176,19 @@ impl SCDatabase for Database { let mut inter_forges = sqlx::query_as!( InnerForge, "SELECT - hostname, - last_crawl_on, - starchart_forge_type.name - FROM - starchart_forges - INNER JOIN - starchart_forge_type - ON - starchart_forges.forge_type = starchart_forge_type.id - ORDER BY - starchart_forges.ID - LIMIT $1 OFFSET $2; + hostname, + last_crawl_on, + starchart_forge_type.name, + imported + FROM + starchart_forges + INNER JOIN + starchart_forge_type + ON + starchart_forges.forge_type = starchart_forge_type.id + ORDER BY + starchart_forges.ID + LIMIT $1 OFFSET $2; ", limit, offset @@ -236,16 +239,17 @@ impl SCDatabase for Database { "INSERT INTO starchart_users ( hostname_id, username, html_url, - profile_photo_html_url, added_on, last_crawl_on + profile_photo_html_url, added_on, last_crawl_on, imported ) VALUES ( - (SELECT ID FROM starchart_forges WHERE hostname = $1), $2, $3, $4, $5, $6)", + (SELECT ID FROM starchart_forges WHERE hostname = $1), $2, $3, $4, $5, $6, $7)", url, u.username, u.html_link, u.profile_photo, now, - now + now, + u.import ) .execute(&self.pool) .await @@ -259,12 +263,13 @@ impl SCDatabase for Database { struct InnerUser { profile_photo_html_url: Option, html_url: String, + imported: bool, } let url = db_core::clean_url(url); let res = sqlx::query_as!( InnerUser, - "SELECT html_url, profile_photo_html_url FROM starchart_users WHERE username = $1 AND + "SELECT html_url, profile_photo_html_url, imported FROM starchart_users WHERE username = $1 AND hostname_id = (SELECT ID FROM starchart_forges WHERE hostname = $2)", username, url, @@ -277,6 +282,7 @@ impl SCDatabase for Database { url, profile_photo: res.profile_photo_html_url, html_link: res.html_url, + import: res.imported, }) } @@ -346,12 +352,13 @@ impl SCDatabase for Database { sqlx::query!( "INSERT INTO starchart_repositories ( - hostname_id, owner_id, name, description, html_url, website, created, last_crawl + hostname_id, owner_id, name, description, html_url, website, created, + last_crawl, imported ) VALUES ( (SELECT ID FROM starchart_forges WHERE hostname = $1), (SELECT ID FROM starchart_users WHERE username = $2), - $3, $4, $5, $6, $7, $8 + $3, $4, $5, $6, $7, $8, $9 );", url, r.owner, @@ -360,7 +367,8 @@ impl SCDatabase for Database { r.html_link, r.website, now, - now + now, + r.import, ) .execute(&self.pool) .await @@ -512,32 +520,34 @@ impl SCDatabase for Database { /// repository website, if any pub website: Option, pub ID: i64, + pub imported: bool, } let mut db_res = sqlx::query_as!( InnerRepository, - " SELECT - starchart_forges.hostname, - starchart_users.username, - starchart_repositories.name, - starchart_repositories.description, - starchart_repositories.html_url, - starchart_repositories.ID, - starchart_repositories.website -FROM - starchart_repositories -INNER JOIN - starchart_forges -ON - starchart_repositories.hostname_id = starchart_forges.id -INNER JOIN - starchart_users -ON - starchart_repositories.owner_id = starchart_users.id -ORDER BY - starchart_repositories.ID -LIMIT $1 OFFSET $2 -;", + "SELECT + starchart_forges.hostname, + starchart_users.username, + starchart_repositories.name, + starchart_repositories.description, + starchart_repositories.html_url, + starchart_repositories.ID, + starchart_repositories.website, + starchart_repositories.imported + FROM + starchart_repositories + INNER JOIN + starchart_forges + ON + starchart_repositories.hostname_id = starchart_forges.id + INNER JOIN + starchart_users + ON + starchart_repositories.owner_id = starchart_users.id + ORDER BY + starchart_repositories.ID + LIMIT $1 OFFSET $2 + ;", limit, offset, ) @@ -580,6 +590,7 @@ LIMIT $1 OFFSET $2 description: repo.description, website: repo.website, tags: topics, + import: repo.imported, }); } @@ -595,6 +606,7 @@ struct InnerForge { hostname: String, last_crawl_on: Option, name: String, + imported: bool, } impl From for Forge { @@ -603,6 +615,7 @@ impl From for Forge { url: f.hostname, last_crawl_on: f.last_crawl_on, forge_type: ForgeImplementation::from_str(&f.name).unwrap(), + import: f.imported, } } } diff --git a/db/db-sqlx-sqlite/src/tests.rs b/db/db-sqlx-sqlite/src/tests.rs index bf14552..0997abb 100644 --- a/db/db-sqlx-sqlite/src/tests.rs +++ b/db/db-sqlx-sqlite/src/tests.rs @@ -40,6 +40,7 @@ async fn everything_works() { let create_forge_msg = CreateForge { url: url.clone(), forge_type: ForgeImplementation::Gitea, + import: false, }; let add_user_msg = AddUser { @@ -47,6 +48,7 @@ async fn everything_works() { html_link: HTML_PROFILE_URL, profile_photo: None, username: USERNAME, + import: false, }; let add_user_msg_2 = AddUser { @@ -54,6 +56,7 @@ async fn everything_works() { html_link: HTML_PROFILE_PHOTO_URL_2, profile_photo: Some(HTML_PROFILE_PHOTO_URL_2), username: USERNAME2, + import: false, }; let db = { @@ -73,6 +76,7 @@ async fn everything_works() { website: None, description: None, url, + import: false, }; adding_forge_works(