feat & fix: insert fts_repositories when repo is not present and implement introducer

db/db-sqlx-sqlite/migrations/20230228083200_starchart_introducer.sql

@@ -0,0 +1,8 @@
+CREATE TABLE IF NOT EXISTS starchart_introducer (
+	ID INTEGER PRIMARY KEY NOT NULL,
+	instance_url TEXT NOT NULL UNIQUE
+);
+
+
+ALTER TABLE starchart_forges ADD COLUMN starchart_instance INTEGER REFERENCES
+starchart_introducer(ID) ON DELETE CASCADE DEFAULT NULL;

db/db-sqlx-sqlite/sqlx-data.json

@@ -68,78 +68,6 @@
     },
     "query": "SELECT html_url, profile_photo_html_url, imported FROM starchart_users WHERE username = $1 AND \n                hostname_id = (SELECT ID FROM starchart_forges WHERE hostname = $2)"
   },
-  "1b88e5b97fd62fe5036ea3c87c90315f8d368542a8f2e12a0d542d1317a612db": {
-    "describe": {
-      "columns": [
-        {
-          "name": "hostname",
-          "ordinal": 0,
-          "type_info": "Text"
-        },
-        {
-          "name": "last_crawl_on",
-          "ordinal": 1,
-          "type_info": "Int64"
-        },
-        {
-          "name": "name",
-          "ordinal": 2,
-          "type_info": "Text"
-        },
-        {
-          "name": "imported",
-          "ordinal": 3,
-          "type_info": "Bool"
-        }
-      ],
-      "nullable": [
-        false,
-        true,
-        false,
-        false
-      ],
-      "parameters": {
-        "Right": 2
-      }
-    },
-    "query": "SELECT\n                hostname,\n                last_crawl_on,\n                starchart_forge_type.name,\n                imported\n            FROM\n                starchart_forges\n            INNER JOIN\n                starchart_forge_type\n            ON\n                starchart_forges.forge_type = starchart_forge_type.id\n            ORDER BY\n                starchart_forges.ID\n            LIMIT $1 OFFSET $2;\n        "
-  },
-  "1d98f7afc772f43fb7131131517f831ec9add25ec3f66881143d6d9baa9e2cc6": {
-    "describe": {
-      "columns": [
-        {
-          "name": "hostname",
-          "ordinal": 0,
-          "type_info": "Text"
-        },
-        {
-          "name": "last_crawl_on",
-          "ordinal": 1,
-          "type_info": "Int64"
-        },
-        {
-          "name": "imported",
-          "ordinal": 2,
-          "type_info": "Bool"
-        },
-        {
-          "name": "name",
-          "ordinal": 3,
-          "type_info": "Text"
-        }
-      ],
-      "nullable": [
-        false,
-        true,
-        false,
-        false
-      ],
-      "parameters": {
-        "Right": 1
-      }
-    },
-    "query": "SELECT \n                hostname,\n                last_crawl_on,\n                imported,\n                starchart_forge_type.name\n            FROM\n                starchart_forges\n            INNER JOIN\n                starchart_forge_type\n            ON\n                starchart_forges.forge_type = starchart_forge_type.id\n            WHERE\n                hostname = $1;\n            "
-  },
   "2afb17ba3753aa440465a836b46b7a1466f25791cfc4d0acdd38bc2755ae3e86": {
     "describe": {
       "columns": [
@@ -206,23 +134,15 @@
     },
     "query": "INSERT OR IGNORE INTO starchart_project_topics ( name ) VALUES ( $1 );"
   },
-  "71079442588dfaece04582acdb14d2c8928c695d4eab5332d09b82cefc880d54": {
+  "74fb3a1ae4f339b5371a6872e6eb4ed7c1f5968dac70de1639454c394a05cb38": {
     "describe": {
-      "columns": [
+      "columns": [],
-        {
+      "nullable": [],
-          "name": "ID",
-          "ordinal": 0,
-          "type_info": "Int64"
-        }
-      ],
-      "nullable": [
-        false
-      ],
       "parameters": {
-        "Right": 3
+        "Right": 4
       }
     },
-    "query": "SELECT ID FROM starchart_repositories\n                    WHERE \n                        name = $1\n                    AND\n                        owner_id = ( SELECT ID FROM starchart_users WHERE username = $2)\n                    AND\n                        hostname_id = (SELECT ID FROM starchart_forges WHERE hostname = $3)"
+    "query": "INSERT INTO starchart_forges\n                    (hostname, verified_on, forge_type, starchart_instance)\n                VALUES\n                    (\n                        $1, $2,\n                     (SELECT ID FROM starchart_forge_type WHERE name = $3),\n                     $4)"
   },
   "78e53b067f8706f326fe06d184f8d94dd2c1869f89706e88afa9a0b19717229c": {
     "describe": {
@@ -284,6 +204,70 @@
     },
     "query": "SELECT \n                starchart_forges.hostname,\n                starchart_users.username,\n                starchart_repositories.name,\n                starchart_repositories.description,\n                starchart_repositories.html_url,\n                starchart_repositories.ID,\n                starchart_repositories.website,\n                starchart_repositories.imported\n            FROM\n                starchart_repositories\n            INNER JOIN\n                starchart_forges\n            ON\n                starchart_repositories.hostname_id = starchart_forges.id\n            INNER JOIN\n                starchart_users\n            ON\n                starchart_repositories.owner_id =  starchart_users.id\n            ORDER BY\n                starchart_repositories.ID\n            LIMIT $1 OFFSET $2\n                ;"
   },
+  "79f5b63171b7884e8bc46fe2754a1e95742b1b7d34fcc3aa9883258a3df4357a": {
+    "describe": {
+      "columns": [
+        {
+          "name": "hostname",
+          "ordinal": 0,
+          "type_info": "Text"
+        },
+        {
+          "name": "last_crawl_on",
+          "ordinal": 1,
+          "type_info": "Int64"
+        },
+        {
+          "name": "name",
+          "ordinal": 2,
+          "type_info": "Text"
+        },
+        {
+          "name": "instance_url",
+          "ordinal": 3,
+          "type_info": "Text"
+        }
+      ],
+      "nullable": [
+        false,
+        true,
+        false,
+        false
+      ],
+      "parameters": {
+        "Right": 2
+      }
+    },
+    "query": "SELECT\n                hostname,\n                last_crawl_on,\n                starchart_forge_type.name,\n                starchart_introducer.instance_url\n            FROM\n                starchart_forges\n            INNER JOIN\n                starchart_forge_type\n            ON\n                starchart_forges.forge_type = starchart_forge_type.id\n            LEFT JOIN\n                starchart_introducer\n            ON\n                starchart_introducer.ID = starchart_forges.starchart_instance\n            ORDER BY\n                starchart_forges.ID\n            LIMIT $1 OFFSET $2;\n        "
+  },
+  "7ee4e3e06dc7dea3b514c0d7632c916ee0d9346fd52af43563d47f3c4deff22d": {
+    "describe": {
+      "columns": [
+        {
+          "name": "ID",
+          "ordinal": 0,
+          "type_info": "Int64"
+        }
+      ],
+      "nullable": [
+        false
+      ],
+      "parameters": {
+        "Right": 3
+      }
+    },
+    "query": "SELECT ID FROM starchart_repositories\n                WHERE\n                    name = $1\n                AND\n                    owner_id = ( SELECT ID FROM starchart_users WHERE username = $2)\n                AND\n                    hostname_id = (SELECT ID FROM starchart_forges WHERE hostname = $3)"
+  },
+  "7f61c8a9befb15076c783601325de9f761bec88e726243a323714ebb645d8f0e": {
+    "describe": {
+      "columns": [],
+      "nullable": [],
+      "parameters": {
+        "Right": 1
+      }
+    },
+    "query": "INSERT INTO starchart_introducer (instance_url) VALUES ($1);"
+  },
   "989fdcfc0088ac19ffbe29bba5349d2dcade134fa2b450769ec617d91f88ffe1": {
     "describe": {
       "columns": [],
@@ -294,7 +278,7 @@
     },
     "query": "INSERT OR IGNORE INTO fts_users ( username ) VALUES ( $1 );"
   },
-  "9978a056397522cf1375900bb00c55bc17685dcc9cb22127b21a24b194a1e536": {
+  "9f95bbc36ece57546b81052765eaf0d6cec38f8cf19cda3d256ba21c46a7bd4f": {
     "describe": {
       "columns": [],
       "nullable": [],
@@ -302,7 +286,7 @@
         "Right": 4
       }
     },
-    "query": "INSERT INTO\n            starchart_forges (hostname, verified_on, forge_type, imported) \n            VALUES ($1, $2, (SELECT ID FROM starchart_forge_type WHERE name = $3), $4)"
+    "query": "INSERT INTO starchart_forges\n                    (hostname, verified_on, forge_type, starchart_instance)\n                VALUES (\n                        $1,\n                        $2,\n                        (SELECT ID FROM starchart_forge_type WHERE name = $3),\n                        (SELECT ID FROM starchart_introducer WHERE instance_url = $4)\n                    )"
   },
   "a60479593819cc7dc7ec87a2d7a25a8960dc2efa1f59c5187d2084c79e26e3f8": {
     "describe": {
@@ -400,6 +384,42 @@
     },
     "query": "SELECT \n                starchart_forges.hostname,\n                starchart_users.username,\n                starchart_repositories.name,\n                starchart_repositories.description,\n                starchart_repositories.html_url,\n                starchart_repositories.ID,\n                starchart_repositories.website,\n                starchart_repositories.imported\n            FROM\n                starchart_repositories\n            INNER JOIN\n                starchart_forges\n            ON\n                starchart_repositories.hostname_id = starchart_forges.id\n            INNER JOIN\n                starchart_users\n            ON\n                starchart_repositories.owner_id =  starchart_users.id\n            WHERE starchart_repositories.html_url =  $1\n                ;"
   },
+  "c6793ce32409465bc6cce5949059f36fcef96e1b6ec1e865506d8e42802403fb": {
+    "describe": {
+      "columns": [
+        {
+          "name": "hostname",
+          "ordinal": 0,
+          "type_info": "Text"
+        },
+        {
+          "name": "last_crawl_on",
+          "ordinal": 1,
+          "type_info": "Int64"
+        },
+        {
+          "name": "instance_url",
+          "ordinal": 2,
+          "type_info": "Text"
+        },
+        {
+          "name": "name",
+          "ordinal": 3,
+          "type_info": "Text"
+        }
+      ],
+      "nullable": [
+        false,
+        true,
+        false,
+        false
+      ],
+      "parameters": {
+        "Right": 2
+      }
+    },
+    "query": "SELECT\n                hostname,\n                last_crawl_on,\n                starchart_introducer.instance_url,\n                starchart_forge_type.name\n            FROM\n                starchart_forges\n            INNER JOIN\n                starchart_forge_type\n            ON\n                starchart_forges.forge_type = starchart_forge_type.id\n            LEFT JOIN\n                starchart_introducer\n            ON\n                starchart_introducer.ID = starchart_forges.starchart_instance\n            WHERE \n                starchart_forges.imported = false\n            ORDER BY\n                starchart_forges.ID\n            LIMIT $1 OFFSET $2;\n        "
+  },
   "ca22e5f6e7065cf2d4ffdbfac0084f9871de8cd9073d470cbf7eac2de2a73c47": {
     "describe": {
       "columns": [],
@@ -428,6 +448,42 @@
     },
     "query": "SELECT username FROM starchart_users ORDER BY ID LIMIT $1 OFFSET $2"
   },
+  "da9ae5c04ed5576195b472e69af94827da8a718bd1e7e7a9b595dd120226288d": {
+    "describe": {
+      "columns": [
+        {
+          "name": "hostname",
+          "ordinal": 0,
+          "type_info": "Text"
+        },
+        {
+          "name": "last_crawl_on",
+          "ordinal": 1,
+          "type_info": "Int64"
+        },
+        {
+          "name": "instance_url",
+          "ordinal": 2,
+          "type_info": "Text"
+        },
+        {
+          "name": "name",
+          "ordinal": 3,
+          "type_info": "Text"
+        }
+      ],
+      "nullable": [
+        false,
+        true,
+        false,
+        false
+      ],
+      "parameters": {
+        "Right": 1
+      }
+    },
+    "query": "SELECT \n                hostname,\n                last_crawl_on,\n                starchart_introducer.instance_url,\n                starchart_forge_type.name\n            FROM\n                starchart_forges\n            INNER JOIN\n                starchart_forge_type\n            ON\n                starchart_forges.forge_type = starchart_forge_type.id\n            LEFT JOIN\n                starchart_introducer\n            ON\n                starchart_introducer.ID = starchart_forges.starchart_instance\n            WHERE\n                hostname = $1;\n            "
+  },
   "e30ccfaa6aeda8cf30a2b3e9134abd0c0420441c5ed05189c3be605b1405c8e9": {
     "describe": {
       "columns": [],

db/db-sqlx-sqlite/src/lib.rs

@@ -103,7 +103,23 @@ impl Migrate for Database {
     }
 }
 
+struct FTSRepository {
+    html_url: String,
+}
+
 impl Database {
+    async fn get_fts_repository(&self, query: &str) -> DBResult<Vec<FTSRepository>> {
+        let fts_repos = sqlx::query_as_unchecked!(
+            FTSRepository,
+            "SELECT html_url FROM fts_repositories WHERE html_url MATCH $1;",
+            query
+        )
+        .fetch_all(&self.pool)
+        .await
+        .map_err(|e| DBError::DBError(Box::new(e)))?;
+        Ok(fts_repos)
+    }
+
     async fn new_fts_repositories(
         &self,
         name: &str,
@@ -111,6 +127,9 @@ impl Database {
         website: Option<&str>,
         html_url: &str,
     ) -> DBResult<()> {
+        if !self.get_fts_repository(html_url).await?.is_empty() {
+            return Ok(());
+        }
         sqlx::query!(
             "INSERT OR IGNORE INTO fts_repositories ( name, description, website, html_url ) 
             VALUES ( $1, $2, $3, $4 );",
@@ -262,18 +281,42 @@ impl SCDatabase for Database {
         let now = now_unix_time_stamp();
         let url = db_core::clean_url(&f.url);
         let forge_type = f.forge_type.to_str();
+        if let Some(instance_url) = f.starchart_url {
             sqlx::query!(
-            "INSERT INTO
+                "INSERT INTO starchart_forges
-            starchart_forges (hostname, verified_on, forge_type, imported) 
+                    (hostname, verified_on, forge_type, starchart_instance)
-            VALUES ($1, $2, (SELECT ID FROM starchart_forge_type WHERE name = $3), $4)",
+                VALUES (
+                        $1,
+                        $2,
+                        (SELECT ID FROM starchart_forge_type WHERE name = $3),
+                        (SELECT ID FROM starchart_introducer WHERE instance_url = $4)
+                    )",
                 url,
                 now,
                 forge_type,
-            f.import,
+                instance_url
             )
             .execute(&self.pool)
             .await
             .map_err(map_register_err)?;
+        } else {
+            sqlx::query!(
+                "INSERT INTO starchart_forges
+                    (hostname, verified_on, forge_type, starchart_instance)
+                VALUES
+                    (
+                        $1, $2,
+                     (SELECT ID FROM starchart_forge_type WHERE name = $3),
+                     $4)",
+                url,
+                now,
+                forge_type,
+                f.starchart_url
+            )
+            .execute(&self.pool)
+            .await
+            .map_err(map_register_err)?;
+        }
 
         Ok(())
     }
@@ -286,7 +329,7 @@ impl SCDatabase for Database {
             "SELECT 
                 hostname,
                 last_crawl_on,
-                imported,
+                starchart_introducer.instance_url,
                 starchart_forge_type.name
             FROM
                 starchart_forges
@@ -294,6 +337,10 @@ impl SCDatabase for Database {
                 starchart_forge_type
             ON
                 starchart_forges.forge_type = starchart_forge_type.id
+            LEFT JOIN
+                starchart_introducer
+            ON
+                starchart_introducer.ID = starchart_forges.starchart_instance
             WHERE
                 hostname = $1;
             ",
@@ -320,13 +367,17 @@ impl SCDatabase for Database {
                 hostname,
                 last_crawl_on,
                 starchart_forge_type.name,
-                imported
+                starchart_introducer.instance_url
             FROM
                 starchart_forges
             INNER JOIN
                 starchart_forge_type
             ON
                 starchart_forges.forge_type = starchart_forge_type.id
+            LEFT JOIN
+                starchart_introducer
+            ON
+                starchart_introducer.ID = starchart_forges.starchart_instance
             ORDER BY
                 starchart_forges.ID
             LIMIT $1 OFFSET $2;
@@ -343,14 +394,18 @@ impl SCDatabase for Database {
                 "SELECT
                 hostname,
                 last_crawl_on,
-                starchart_forge_type.name,
+                starchart_introducer.instance_url,
-                imported
+                starchart_forge_type.name
             FROM
                 starchart_forges
             INNER JOIN
                 starchart_forge_type
             ON
                 starchart_forges.forge_type = starchart_forge_type.id
+            LEFT JOIN
+                starchart_introducer
+            ON
+                starchart_introducer.ID = starchart_forges.starchart_instance
             WHERE 
                 starchart_forges.imported = false
             ORDER BY
@@ -693,18 +748,7 @@ impl SCDatabase for Database {
 
     /// Search all repositories
     async fn search_repository(&self, query: &str) -> DBResult<Vec<Repository>> {
-        struct FTSRepository {
+        let mut fts_repos = self.get_fts_repository(query).await?;
-            html_url: String,
-        }
-        let mut fts_repos = sqlx::query_as_unchecked!(
-            FTSRepository,
-            "SELECT html_url FROM fts_repositories WHERE html_url MATCH $1;",
-            query
-        )
-        .fetch_all(&self.pool)
-        .await
-        .map_err(|e| DBError::DBError(Box::new(e)))?;
-
         let mut res = Vec::with_capacity(fts_repos.len());
         for fts_repo in fts_repos.drain(0..) {
             let repo = sqlx::query_as!(
@@ -774,6 +818,19 @@ impl SCDatabase for Database {
         }
         Ok(res)
     }
+
+    /// Add Starchart instance to introducer
+    async fn add_starchart_to_introducer(&self, url: &Url) -> DBResult<()> {
+        let url = url.as_str();
+        sqlx::query!(
+            "INSERT INTO starchart_introducer (instance_url) VALUES ($1);",
+            url
+        )
+        .execute(&self.pool)
+        .await
+        .map_err(map_register_err)?;
+        Ok(())
+    }
 }
 
 fn now_unix_time_stamp() -> i64 {
@@ -784,7 +841,7 @@ struct InnerForge {
     hostname: String,
     last_crawl_on: Option<i64>,
     name: String,
-    imported: bool,
+    instance_url: String,
 }
 
 impl From<InnerForge> for Forge {
@@ -793,7 +850,7 @@ impl From<InnerForge> for Forge {
             url: f.hostname,
             last_crawl_on: f.last_crawl_on,
             forge_type: ForgeImplementation::from_str(&f.name).unwrap(),
-            import: f.imported,
+            starchart_url: Some(f.instance_url),
         }
     }
 }