feat: mark entries as imported to differentiate b/w native crawls and

imported records

db/db-core/src/lib.rs

@@ -65,6 +65,8 @@ pub struct CreateForge {
     pub url: Url,
     /// forge type: which software is the instance running?
     pub forge_type: ForgeImplementation,
+    /// is this forge an import
+    pub import: bool,
 }
 
 /// Get url from URL
@@ -89,6 +91,8 @@ pub struct User {
     pub html_link: String,
     /// OPTIONAL: html link to the user's profile photo
     pub profile_photo: Option<String>,
+    /// is this user an import
+    pub import: bool,
 }
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@@ -103,6 +107,8 @@ pub struct AddUser<'a> {
     pub html_link: &'a str,
     /// OPTIONAL: html link to the user's profile photo
     pub profile_photo: Option<&'a str>,
+    /// is this user an import
+    pub import: bool,
 }
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@@ -123,6 +129,8 @@ pub struct AddRepository<'a> {
     pub description: Option<&'a str>,
     /// repository website, if any
     pub website: Option<&'a str>,
+    /// is this repository an import
+    pub import: bool,
 }
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@@ -134,6 +142,8 @@ pub struct Forge {
     pub forge_type: ForgeImplementation,
     /// last crawl
     pub last_crawl_on: Option<i64>,
+    /// is this forge an import
+    pub import: bool,
 }
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@@ -154,6 +164,8 @@ pub struct Repository {
     pub description: Option<String>,
     /// repository website, if any
     pub website: Option<String>,
+    /// is this repository an import
+    pub import: bool,
 }
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]

db/db-sqlx-sqlite/Cargo.toml

@@ -17,13 +17,13 @@ name = "db_sqlx_sqlite"
 path = "src/lib.rs"
 
 [dependencies]
-sqlx = { version = "0.5.11", features = [ "sqlite", "time", "offline", "runtime-actix-rustls" ] }
+sqlx = { version = "0.6.2", features = [ "sqlite", "time", "offline", "runtime-actix-rustls" ] }
 db-core = {path = "../db-core"}
 async-trait = "0.1.51"
 url = { version  = "2.2.2", features = ["serde"] }
 
 [dev-dependencies]
 actix-rt = "2"
-sqlx = { version = "0.5.11", features = [ "runtime-actix-rustls", "postgres", "time", "offline" ] }
+sqlx = { version = "0.6.2", features = [ "runtime-actix-rustls", "postgres", "time", "offline" ] }
 db-core = {path = "../db-core", features = ["test"]}
 url = { version  = "2.2.2", features = ["serde"] }

db/db-sqlx-sqlite/migrations/20230211075309_starchart_import.sql

@@ -0,0 +1,11 @@
+ALTER TABLE starchart_forges ADD COLUMN imported BOOLEAN NOT NULL DEFAULT(0);
+
+ALTER TABLE starchart_users ADD COLUMN imported BOOLEAN NOT NULL DEFAULT(0);
+
+ALTER TABLE starchart_repositories ADD COLUMN imported BOOLEAN NOT NULL DEFAULT(0);
+
+CREATE TABLE IF NOT EXISTS foo (
+	ID INTEGER PRIMARY KEY NOT NULL
+);
+
+DROP TABLE foo;

db/db-sqlx-sqlite/src/lib.rs

@@ -129,11 +129,12 @@ impl SCDatabase for Database {
         let forge_type = f.forge_type.to_str();
         sqlx::query!(
             "INSERT INTO
-            starchart_forges (hostname, verified_on, forge_type ) 
+            starchart_forges (hostname, verified_on, forge_type, imported) 
-        VALUES ($1, $2, (SELECT ID FROM starchart_forge_type WHERE name = $3))",
+            VALUES ($1, $2, (SELECT ID FROM starchart_forge_type WHERE name = $3), $4)",
             url,
             now,
             forge_type,
+            f.import,
         )
         .execute(&self.pool)
         .await
@@ -150,6 +151,7 @@ impl SCDatabase for Database {
             "SELECT 
                 hostname,
                 last_crawl_on,
+                imported,
                 starchart_forge_type.name
             FROM
                 starchart_forges
@@ -176,7 +178,8 @@ impl SCDatabase for Database {
             "SELECT
                 hostname,
                 last_crawl_on,
-		starchart_forge_type.name
+                starchart_forge_type.name,
+                imported
             FROM
                 starchart_forges
             INNER JOIN
@@ -236,16 +239,17 @@ impl SCDatabase for Database {
             "INSERT INTO 
                     starchart_users (
                         hostname_id, username, html_url,
-                        profile_photo_html_url, added_on, last_crawl_on
+                        profile_photo_html_url, added_on, last_crawl_on, imported
                     ) 
             VALUES (
-                    (SELECT ID FROM starchart_forges WHERE hostname = $1), $2, $3, $4, $5, $6)",
+                    (SELECT ID FROM starchart_forges WHERE hostname = $1), $2, $3, $4, $5, $6, $7)",
             url,
             u.username,
             u.html_link,
             u.profile_photo,
             now,
-            now
+            now,
+            u.import
         )
         .execute(&self.pool)
         .await
@@ -259,12 +263,13 @@ impl SCDatabase for Database {
         struct InnerUser {
             profile_photo_html_url: Option<String>,
             html_url: String,
+            imported: bool,
         }
 
         let url = db_core::clean_url(url);
         let res = sqlx::query_as!(
             InnerUser,
-            "SELECT html_url, profile_photo_html_url FROM starchart_users WHERE username = $1 AND 
+            "SELECT html_url, profile_photo_html_url, imported FROM starchart_users WHERE username = $1 AND 
                 hostname_id = (SELECT ID FROM starchart_forges WHERE hostname = $2)",
             username,
             url,
@@ -277,6 +282,7 @@ impl SCDatabase for Database {
             url,
             profile_photo: res.profile_photo_html_url,
             html_link: res.html_url,
+            import: res.imported,
         })
     }
 
@@ -346,12 +352,13 @@ impl SCDatabase for Database {
         sqlx::query!(
             "INSERT INTO 
                 starchart_repositories (
-                    hostname_id, owner_id, name, description, html_url, website, created, last_crawl
+                    hostname_id, owner_id, name, description, html_url, website, created,
+                    last_crawl, imported
                 )
                 VALUES (
                     (SELECT ID FROM starchart_forges WHERE hostname = $1),
                     (SELECT ID FROM starchart_users WHERE username = $2),
-                    $3, $4, $5, $6, $7, $8
+                    $3, $4, $5, $6, $7, $8, $9
                 );",
             url,
             r.owner,
@@ -360,7 +367,8 @@ impl SCDatabase for Database {
             r.html_link,
             r.website,
             now,
-            now
+            now,
+            r.import,
         )
         .execute(&self.pool)
         .await
@@ -512,6 +520,7 @@ impl SCDatabase for Database {
             /// repository website, if any
             pub website: Option<String>,
             pub ID: i64,
+            pub imported: bool,
         }
 
         let mut db_res = sqlx::query_as!(
@@ -523,7 +532,8 @@ impl SCDatabase for Database {
                 starchart_repositories.description,
                 starchart_repositories.html_url,
                 starchart_repositories.ID,
-		starchart_repositories.website
+                starchart_repositories.website,
+                starchart_repositories.imported
             FROM
                 starchart_repositories
             INNER JOIN
@@ -580,6 +590,7 @@ LIMIT $1 OFFSET $2
                 description: repo.description,
                 website: repo.website,
                 tags: topics,
+                import: repo.imported,
             });
         }
 
@@ -595,6 +606,7 @@ struct InnerForge {
     hostname: String,
     last_crawl_on: Option<i64>,
     name: String,
+    imported: bool,
 }
 
 impl From<InnerForge> for Forge {
@@ -603,6 +615,7 @@ impl From<InnerForge> for Forge {
             url: f.hostname,
             last_crawl_on: f.last_crawl_on,
             forge_type: ForgeImplementation::from_str(&f.name).unwrap(),
+            import: f.imported,
         }
     }
 }

db/db-sqlx-sqlite/src/tests.rs

@@ -40,6 +40,7 @@ async fn everything_works() {
     let create_forge_msg = CreateForge {
         url: url.clone(),
         forge_type: ForgeImplementation::Gitea,
+        import: false,
     };
 
     let add_user_msg = AddUser {
@@ -47,6 +48,7 @@ async fn everything_works() {
         html_link: HTML_PROFILE_URL,
         profile_photo: None,
         username: USERNAME,
+        import: false,
     };
 
     let add_user_msg_2 = AddUser {
@@ -54,6 +56,7 @@ async fn everything_works() {
         html_link: HTML_PROFILE_PHOTO_URL_2,
         profile_photo: Some(HTML_PROFILE_PHOTO_URL_2),
         username: USERNAME2,
+        import: false,
     };
 
     let db = {
@@ -73,6 +76,7 @@ async fn everything_works() {
         website: None,
         description: None,
         url,
+        import: false,
     };
 
     adding_forge_works(