feat: filter imported and native forges

db/db-core/src/lib.rs

@@ -191,7 +191,12 @@ pub trait SCDatabase: std::marker::Send + std::marker::Sync + CloneSPDatabase {
     async fn forge_type_exists(&self, forge_type: &ForgeImplementation) -> DBResult<bool>;
 
     /// Get all forges
-    async fn get_all_forges(&self, offset: u32, limit: u32) -> DBResult<Vec<Forge>>;
+    async fn get_all_forges(
+        &self,
+        with_imports: bool,
+        offset: u32,
+        limit: u32,
+    ) -> DBResult<Vec<Forge>>;
 
     /// add new user to database
     async fn add_user(&self, u: &AddUser) -> DBResult<()>;

db/db-core/src/tests.rs

@@ -35,7 +35,7 @@ pub async fn adding_forge_works<'a, T: SCDatabase>(
 
     {
         let forge = db.get_forge(&create_forge_msg.url).await.unwrap();
-        let forges = db.get_all_forges(0, 10).await.unwrap();
+        let forges = db.get_all_forges(true, 0, 10).await.unwrap();
         assert_eq!(forges.len(), 1);
 
         assert_eq!(

db/db-sqlx-sqlite/src/lib.rs

@@ -307,8 +307,14 @@ impl SCDatabase for Database {
     }
 
     /// Get all forges
-    async fn get_all_forges(&self, offset: u32, limit: u32) -> DBResult<Vec<Forge>> {
+    async fn get_all_forges(
-        let mut inter_forges = sqlx::query_as!(
+        &self,
+        with_imports: bool,
+        offset: u32,
+        limit: u32,
+    ) -> DBResult<Vec<Forge>> {
+        let mut inter_forges = if with_imports {
+            sqlx::query_as!(
                 InnerForge,
                 "SELECT
                 hostname,
@@ -330,7 +336,34 @@ impl SCDatabase for Database {
             )
             .fetch_all(&self.pool)
             .await
-        .map_err(|e| DBError::DBError(Box::new(e)))?;
+            .map_err(|e| DBError::DBError(Box::new(e)))?
+        } else {
+            sqlx::query_as!(
+                InnerForge,
+                "SELECT
+                hostname,
+                last_crawl_on,
+                starchart_forge_type.name,
+                imported
+            FROM
+                starchart_forges
+            INNER JOIN
+                starchart_forge_type
+            ON
+                starchart_forges.forge_type = starchart_forge_type.id
+            WHERE 
+                starchart_forges.imported = false
+            ORDER BY
+                starchart_forges.ID
+            LIMIT $1 OFFSET $2;
+        ",
+                limit,
+                offset
+            )
+            .fetch_all(&self.pool)
+            .await
+            .map_err(|e| DBError::DBError(Box::new(e)))?
+        };
 
         let mut forges: Vec<Forge> = Vec::with_capacity(inter_forges.len());
         inter_forges.drain(0..).for_each(|f| forges.push(f.into()));

src/api.rs

@@ -33,7 +33,7 @@ pub async fn forges(db: WebDB, q: web::Query<OptionalPage>) -> ServiceResult<imp
     let q = q.into_inner();
     let q: Page = q.into();
     let offset = q.page * LIMIT;
-    let forges = db.get_all_forges(offset, LIMIT).await?;
+    let forges = db.get_all_forges(false, offset, LIMIT).await?;
 
     Ok(HttpResponse::Ok().json(forges))
 }

src/spider.rs

@@ -189,7 +189,7 @@ impl Crawler {
                     break;
                 }
 
-                let forges = c.db.get_all_forges(offset, LIMIT).await.unwrap();
+                let forges = c.db.get_all_forges(false, offset, LIMIT).await.unwrap();
                 if forges.is_empty() {
                     c.federate.tar().await.unwrap();
                     page = 0;