feat: implement mini-index

db/db-core/src/lib.rs

@@ -246,6 +246,18 @@ pub trait SCDatabase: std::marker::Send + std::marker::Sync + CloneSPDatabase {
         offset: u32,
         limit: u32,
     ) -> DBResult<Vec<Starchart>>;
+
+    /// Add word to mini index
+    async fn add_word_to_mini_index(&self, word: &str) -> DBResult<()>;
+
+    /// Remove word from mini index
+    async fn rm_word_from_mini_index(&self, word: &str) -> DBResult<()>;
+
+    /// Check if word exists in mini index
+    async fn is_word_mini_indexed(&self, word: &str) -> DBResult<bool>;
+
+    /// consolidate and export mini index
+    async fn export_mini_index(&self) -> DBResult<String>;
 }
 
 /// Trait to clone SCDatabase

db/db-core/src/tests.rs

@@ -120,3 +120,22 @@ pub async fn instance_introducer_helper<T: SCDatabase>(db: &T, instance_url: &Ur
         .iter()
         .any(|i| i.instance_url == instance_url.as_str()));
 }
+
+/// test if all instance introducer methods work
+pub async fn mini_index_helper<T: SCDatabase>(db: &T) {
+    // batman is repeated twice but mini-index should contain it only once
+    // Batman is different from Batman; mini-index is case-sensitive
+    const WORDS: [&str; 5] = ["batman", "superman", "aquaman", "Batman", "batman"];
+
+    let expected_mini_index = "superman aquaman Batman batman";
+
+    for w in WORDS.iter() {
+        db.rm_word_from_mini_index(w).await.unwrap();
+        assert!(!db.is_word_mini_indexed(w).await.unwrap());
+        db.add_word_to_mini_index(w).await.unwrap();
+        assert!(db.is_word_mini_indexed(w).await.unwrap());
+    }
+
+    let mini_index = db.export_mini_index().await.unwrap();
+    assert_eq!(mini_index, expected_mini_index);
+}

db/db-sqlx-sqlite/migrations/20230302072543_starchart_mini_index.sql

@@ -0,0 +1,4 @@
+CREATE TABLE IF NOT EXISTS starchart_mini_index (
+    word TEXT NOT NULL UNIQUE,
+    ID INTEGER PRIMARY KEY NOT NULL
+);

db/db-sqlx-sqlite/sqlx-data.json

@@ -10,6 +10,24 @@
     },
     "query": "INSERT OR IGNORE INTO fts_repositories ( name, description, website, html_url ) \n            VALUES ( $1, $2, $3, $4 );"
   },
+  "0b179588df37779f563f0ad8c43e920a8bc22b3eed682778cef9dd05608f9691": {
+    "describe": {
+      "columns": [
+        {
+          "name": "ID",
+          "ordinal": 0,
+          "type_info": "Int64"
+        }
+      ],
+      "nullable": [
+        false
+      ],
+      "parameters": {
+        "Right": 1
+      }
+    },
+    "query": "SELECT ID FROM starchart_mini_index WHERE word = $1"
+  },
   "0bb37cc79d5ef803285d05d06e6ef93b62c0b532c0298148fe436178761fd70a": {
     "describe": {
       "columns": [
@@ -240,6 +258,16 @@
     },
     "query": "SELECT\n                hostname,\n                last_crawl_on,\n                starchart_forge_type.name,\n                starchart_introducer.instance_url\n            FROM\n                starchart_forges\n            INNER JOIN\n                starchart_forge_type\n            ON\n                starchart_forges.forge_type = starchart_forge_type.id\n            LEFT JOIN\n                starchart_introducer\n            ON\n                starchart_introducer.ID = starchart_forges.starchart_instance\n            ORDER BY\n                starchart_forges.ID\n            LIMIT $1 OFFSET $2;\n        "
   },
+  "7a2ad86f18ed9106b21566cfe810adc7f907143939409cadf361aec1572f76e3": {
+    "describe": {
+      "columns": [],
+      "nullable": [],
+      "parameters": {
+        "Right": 1
+      }
+    },
+    "query": "DELETE FROM starchart_mini_index WHERE word = ($1)"
+  },
   "7ee4e3e06dc7dea3b514c0d7632c916ee0d9346fd52af43563d47f3c4deff22d": {
     "describe": {
       "columns": [
@@ -567,5 +595,33 @@
       }
     },
     "query": "DELETE FROM starchart_forges WHERE hostname = ($1)"
+  },
+  "f9aad84b4586954c224ddb0c2a60409c77589d856acd74c14fcec3799aa23407": {
+    "describe": {
+      "columns": [],
+      "nullable": [],
+      "parameters": {
+        "Right": 1
+      }
+    },
+    "query": "INSERT OR IGNORE INTO starchart_mini_index ( word ) \n            VALUES ( $1);"
+  },
+  "f9fb014c5e0a32d7f51b78c08439a06b30f46ebadd28b062b313a7faf0f0aef1": {
+    "describe": {
+      "columns": [
+        {
+          "name": "word",
+          "ordinal": 0,
+          "type_info": "Text"
+        }
+      ],
+      "nullable": [
+        false
+      ],
+      "parameters": {
+        "Right": 0
+      }
+    },
+    "query": "SELECT word FROM starchart_mini_index"
   }
 }

db/db-sqlx-sqlite/src/lib.rs

@@ -856,6 +856,60 @@ impl SCDatabase for Database {
         .map_err(|e| DBError::DBError(Box::new(e)))?;
         Ok(s)
     }
+
+    /// Add word to mini index
+    async fn add_word_to_mini_index(&self, word: &str) -> DBResult<()> {
+        sqlx::query!(
+            "INSERT OR IGNORE INTO starchart_mini_index ( word ) 
+            VALUES ( $1);",
+            word,
+        )
+        .execute(&self.pool)
+        .await
+        .map_err(map_register_err)?;
+        Ok(())
+    }
+
+    /// Remove word from mini index
+    async fn rm_word_from_mini_index(&self, word: &str) -> DBResult<()> {
+        sqlx::query!("DELETE FROM starchart_mini_index WHERE word = ($1)", word)
+            .execute(&self.pool)
+            .await
+            .map_err(|e| DBError::DBError(Box::new(e)))?;
+        Ok(())
+    }
+
+    /// Check if word exists in mini index
+    async fn is_word_mini_indexed(&self, word: &str) -> DBResult<bool> {
+        match sqlx::query!("SELECT ID FROM starchart_mini_index WHERE word = $1", word)
+            .fetch_one(&self.pool)
+            .await
+        {
+            Ok(_) => Ok(true),
+            Err(Error::RowNotFound) => Ok(false),
+            Err(e) => Err(DBError::DBError(Box::new(e).into())),
+        }
+    }
+
+    /// consolidate and export mini index
+    async fn export_mini_index(&self) -> DBResult<String> {
+        struct Words {
+            word: String,
+        }
+        let mut words = sqlx::query_as!(Words, "SELECT word FROM starchart_mini_index")
+            .fetch_all(&self.pool)
+            .await
+            .map_err(|e| DBError::DBError(Box::new(e)))?;
+        let mut mini_index = String::default();
+        words.drain(0..).for_each(|w| {
+            mini_index = if mini_index.is_empty() {
+                w.word
+            } else {
+                format!("{mini_index} {}", w.word)
+            }
+        });
+        Ok(mini_index)
+    }
 }
 
 fn now_unix_time_stamp() -> i64 {

db/db-sqlx-sqlite/src/tests.rs

@@ -110,3 +110,14 @@ async fn forge_type_exists() {
     db.migrate().await.unwrap();
     forge_type_exists_helper(&db).await;
 }
+
+#[actix_rt::test]
+async fn mini_index_test() {
+    let url = env::var("SQLITE_DATABASE_URL").expect("Set SQLITE_DATABASE_URL env var");
+    let pool_options = SqlitePoolOptions::new().max_connections(2);
+    let connection_options = ConnectionOptions::Fresh(Fresh { pool_options, url });
+    let db = connection_options.connect().await.unwrap();
+
+    db.migrate().await.unwrap();
+    mini_index_helper(&db).await;
+}