feat: record starchart imports

db/db-core/src/lib.rs

@@ -271,6 +271,15 @@ pub trait SCDatabase: std::marker::Send + std::marker::Sync + CloneSPDatabase {
 
     /// Search mini index
     async fn search_mini_index(&self, query: &str) -> DBResult<Vec<String>>;
+
+    /// Mark a Starchart instance as imported
+    async fn record_starchart_imports(&self, starchart_url: &Url) -> DBResult<()>;
+
+    /// Unmark a Starchart instance as imported
+    async fn rm_starchart_import(&self, starchart_url: &Url) -> DBResult<()>;
+
+    /// Check if Starchart instance is imported
+    async fn is_starchart_imported(&self, starchart_url: &Url) -> DBResult<bool>;
 }
 
 /// Trait to clone SCDatabase

db/db-core/src/tests.rs

@@ -141,6 +141,13 @@ pub async fn instance_introducer_helper<T: SCDatabase>(db: &T, instance_url: &Ur
     let matching_instances = db.search_mini_index("uniquerq2342").await.unwrap();
     assert_eq!(matching_instances.len(), 1);
     assert_eq!(matching_instances.first().unwrap(), instance_url.as_str());
+
+    db.rm_starchart_import(instance_url).await.unwrap();
+    assert!(!db.is_starchart_imported(instance_url).await.unwrap());
+    db.record_starchart_imports(instance_url).await.unwrap();
+    assert!(db.is_starchart_imported(instance_url).await.unwrap());
+    db.rm_starchart_import(instance_url).await.unwrap();
+    assert!(!db.is_starchart_imported(instance_url).await.unwrap());
 }
 
 /// test if all instance introducer methods work

db/db-sqlx-sqlite/migrations/20230302132435_starchart_imported_starcharts.sql

@@ -0,0 +1,4 @@
+CREATE TABLE IF NOT EXISTS starchart_imported_starcharts (
+    starchart_instance INTEGER REFERENCES starchart_introducer(ID) ON DELETE CASCADE,
+	ID INTEGER PRIMARY KEY NOT NULL
+);

db/db-sqlx-sqlite/sqlx-data.json

@@ -1,5 +1,23 @@
 {
   "db": "SQLite",
+  "014aceff8492c04498b55ce10f7f71d376be18967a5eaf637fae1cd5a46dc210": {
+    "describe": {
+      "columns": [
+        {
+          "name": "instance_url",
+          "ordinal": 0,
+          "type_info": "Text"
+        }
+      ],
+      "nullable": [
+        false
+      ],
+      "parameters": {
+        "Right": 1
+      }
+    },
+    "query": "SELECT\n                starchart_introducer.instance_url\n            FROM \n                starchart_federated_mini_index\n            INNER JOIN\n                starchart_introducer\n            ON\n                starchart_introducer.ID = starchart_instance \n            WHERE\n                mini_index MATCH $1"
+  },
   "069a127ffb1062321bba5a915a4ead3e0bbe4dabf6e0f684d8cc6a6d8a68ad5b": {
     "describe": {
       "columns": [],
@@ -142,6 +160,34 @@
     },
     "query": " DELETE FROM starchart_users WHERE username = $1 AND \n                hostname_id = (SELECT ID FROM starchart_forges WHERE hostname = $2)"
   },
+  "3d65424ec78936842b54492893560dc5fc324cc776de7686a52c9b8e382e6a36": {
+    "describe": {
+      "columns": [
+        {
+          "name": "ID",
+          "ordinal": 0,
+          "type_info": "Int64"
+        }
+      ],
+      "nullable": [
+        false
+      ],
+      "parameters": {
+        "Right": 1
+      }
+    },
+    "query": "SELECT\n                ID\n            FROM\n                starchart_imported_starcharts\n            WHERE\n                starchart_instance = (SELECT ID FROM starchart_introducer WHERE instance_url = $1);"
+  },
+  "4bf5f81ac904e1738975f85b202f9b0640374d562730af5318b0119368b356b5": {
+    "describe": {
+      "columns": [],
+      "nullable": [],
+      "parameters": {
+        "Right": 1
+      }
+    },
+    "query": "DELETE FROM\n                starchart_imported_starcharts\n            WHERE starchart_instance =(SELECT ID FROM starchart_introducer WHERE instance_url = $1);"
+  },
   "6f5ca3d71a541eb6f33e37a5889c048536ab6ad7e81a6236d73aa71433c13717": {
     "describe": {
       "columns": [],
@@ -162,6 +208,16 @@
     },
     "query": "INSERT INTO starchart_forges\n                    (hostname, verified_on, forge_type, starchart_instance)\n                VALUES\n                    (\n                        $1, $2,\n                     (SELECT ID FROM starchart_forge_type WHERE name = $3),\n                     $4)"
   },
+  "7590630f5fe7e05014b70ac0047f9b6c724b88e35e1b1306fb89760612929d55": {
+    "describe": {
+      "columns": [],
+      "nullable": [],
+      "parameters": {
+        "Right": 2
+      }
+    },
+    "query": "INSERT OR IGNORE INTO\n                starchart_federated_mini_index\n            (mini_index, starchart_instance)\n            VALUES ($1, (SELECT ID FROM starchart_introducer WHERE instance_url = $2));"
+  },
   "78e53b067f8706f326fe06d184f8d94dd2c1869f89706e88afa9a0b19717229c": {
     "describe": {
       "columns": [
@@ -420,6 +476,16 @@
     },
     "query": "SELECT \n                starchart_forges.hostname,\n                starchart_users.username,\n                starchart_repositories.name,\n                starchart_repositories.description,\n                starchart_repositories.html_url,\n                starchart_repositories.ID,\n                starchart_repositories.website,\n                starchart_repositories.imported\n            FROM\n                starchart_repositories\n            INNER JOIN\n                starchart_forges\n            ON\n                starchart_repositories.hostname_id = starchart_forges.id\n            INNER JOIN\n                starchart_users\n            ON\n                starchart_repositories.owner_id =  starchart_users.id\n            WHERE starchart_repositories.html_url =  $1\n                ;"
   },
+  "aeb26b35a2fc69fe8f6496e1904d6f0408a29359fb141ee67d49269152fd381b": {
+    "describe": {
+      "columns": [],
+      "nullable": [],
+      "parameters": {
+        "Right": 1
+      }
+    },
+    "query": "INSERT OR IGNORE INTO\n                starchart_imported_starcharts (starchart_instance)\n            VALUES ((SELECT ID FROM starchart_introducer WHERE instance_url = $1));"
+  },
   "c6793ce32409465bc6cce5949059f36fcef96e1b6ec1e865506d8e42802403fb": {
     "describe": {
       "columns": [
@@ -623,5 +689,15 @@
       }
     },
     "query": "SELECT word FROM starchart_mini_index"
+  },
+  "fdf87490759150d528139b13eb1a28532b5bebb546ade00bcb6a7d648abcd445": {
+    "describe": {
+      "columns": [],
+      "nullable": [],
+      "parameters": {
+        "Right": 1
+      }
+    },
+    "query": "DELETE FROM starchart_federated_mini_index\n             WHERE \n                starchart_instance = (\n                    SELECT ID FROM starchart_introducer\n                    WHERE instance_url = $1\n                )"
   }
 }

db/db-sqlx-sqlite/src/lib.rs

@@ -1000,8 +1000,7 @@ impl SCDatabase for Database {
     async fn rm_imported_mini_index(&self, starchart_instance_url: &Url) -> DBResult<()> {
         let url = db_core::clean_url(starchart_instance_url);
         sqlx::query!(
-            "
+            "DELETE FROM starchart_federated_mini_index
-                DELETE FROM starchart_federated_mini_index
              WHERE 
                 starchart_instance = (
                     SELECT ID FROM starchart_introducer
@@ -1018,6 +1017,57 @@ impl SCDatabase for Database {
     async fn search_mini_index(&self, query: &str) -> DBResult<Vec<String>> {
         self.get_federated_mini_index_matrches(query).await
     }
+
+    /// Mark a Starchart instance as imported
+    async fn record_starchart_imports(&self, starchart_url: &Url) -> DBResult<()> {
+        let url = db_core::clean_url(starchart_url);
+        sqlx::query!(
+            "INSERT OR IGNORE INTO
+                starchart_imported_starcharts (starchart_instance)
+            VALUES ((SELECT ID FROM starchart_introducer WHERE instance_url = $1));",
+            url
+        )
+        .execute(&self.pool)
+        .await
+        .map_err(map_register_err)?;
+        Ok(())
+    }
+
+    /// Unmark a Starchart instance as imported
+    async fn rm_starchart_import(&self, starchart_url: &Url) -> DBResult<()> {
+        let url = db_core::clean_url(starchart_url);
+        sqlx::query!(
+            "DELETE FROM
+                starchart_imported_starcharts
+            WHERE starchart_instance =(SELECT ID FROM starchart_introducer WHERE instance_url = $1);",
+            url
+        )
+        .execute(&self.pool)
+        .await
+        .map_err(map_register_err)?;
+        Ok(())
+    }
+
+    /// Check if Starchart instance is imported
+    async fn is_starchart_imported(&self, starchart_url: &Url) -> DBResult<bool> {
+        let url = db_core::clean_url(starchart_url);
+
+        match sqlx::query!(
+            "SELECT
+                ID
+            FROM
+                starchart_imported_starcharts
+            WHERE
+                starchart_instance = (SELECT ID FROM starchart_introducer WHERE instance_url = $1);",
+            url)
+            .fetch_one(&self.pool)
+            .await
+        {
+            Ok(_) => Ok(true),
+            Err(Error::RowNotFound) => Ok(false),
+            Err(e) => Err(DBError::DBError(Box::new(e).into())),
+        }
+    }
 }
 
 fn now_unix_time_stamp() -> i64 {

federate/publiccodeyml/src/lib.rs

@@ -418,6 +418,7 @@ impl Federate for PccFederate {
                 }
             }
         }
+        db.record_starchart_imports(&starchart_url).await?;
         Ok(())
     }
 }