feat: implement mini-index

This commit is contained in:
Aravinth Manivannan 2023-03-02 13:09:48 +05:30
parent 3f92a9ff33
commit f53a0afada
Signed by: realaravinth
GPG key ID: AD9F0F08E855ED88
6 changed files with 156 additions and 0 deletions

View file

@ -246,6 +246,18 @@ pub trait SCDatabase: std::marker::Send + std::marker::Sync + CloneSPDatabase {
offset: u32,
limit: u32,
) -> DBResult<Vec<Starchart>>;
/// Add word to mini index
async fn add_word_to_mini_index(&self, word: &str) -> DBResult<()>;
/// Remove word from mini index
async fn rm_word_from_mini_index(&self, word: &str) -> DBResult<()>;
/// Check if word exists in mini index
async fn is_word_mini_indexed(&self, word: &str) -> DBResult<bool>;
/// consolidate and export mini index
async fn export_mini_index(&self) -> DBResult<String>;
}
/// Trait to clone SCDatabase

View file

@ -120,3 +120,22 @@ pub async fn instance_introducer_helper<T: SCDatabase>(db: &T, instance_url: &Ur
.iter()
.any(|i| i.instance_url == instance_url.as_str()));
}
/// test if all instance introducer methods work
pub async fn mini_index_helper<T: SCDatabase>(db: &T) {
// batman is repeated twice but mini-index should contain it only once
// Batman is different from Batman; mini-index is case-sensitive
const WORDS: [&str; 5] = ["batman", "superman", "aquaman", "Batman", "batman"];
let expected_mini_index = "superman aquaman Batman batman";
for w in WORDS.iter() {
db.rm_word_from_mini_index(w).await.unwrap();
assert!(!db.is_word_mini_indexed(w).await.unwrap());
db.add_word_to_mini_index(w).await.unwrap();
assert!(db.is_word_mini_indexed(w).await.unwrap());
}
let mini_index = db.export_mini_index().await.unwrap();
assert_eq!(mini_index, expected_mini_index);
}

View file

@ -0,0 +1,4 @@
CREATE TABLE IF NOT EXISTS starchart_mini_index (
word TEXT NOT NULL UNIQUE,
ID INTEGER PRIMARY KEY NOT NULL
);

View file

@ -10,6 +10,24 @@
},
"query": "INSERT OR IGNORE INTO fts_repositories ( name, description, website, html_url ) \n VALUES ( $1, $2, $3, $4 );"
},
"0b179588df37779f563f0ad8c43e920a8bc22b3eed682778cef9dd05608f9691": {
"describe": {
"columns": [
{
"name": "ID",
"ordinal": 0,
"type_info": "Int64"
}
],
"nullable": [
false
],
"parameters": {
"Right": 1
}
},
"query": "SELECT ID FROM starchart_mini_index WHERE word = $1"
},
"0bb37cc79d5ef803285d05d06e6ef93b62c0b532c0298148fe436178761fd70a": {
"describe": {
"columns": [
@ -240,6 +258,16 @@
},
"query": "SELECT\n hostname,\n last_crawl_on,\n starchart_forge_type.name,\n starchart_introducer.instance_url\n FROM\n starchart_forges\n INNER JOIN\n starchart_forge_type\n ON\n starchart_forges.forge_type = starchart_forge_type.id\n LEFT JOIN\n starchart_introducer\n ON\n starchart_introducer.ID = starchart_forges.starchart_instance\n ORDER BY\n starchart_forges.ID\n LIMIT $1 OFFSET $2;\n "
},
"7a2ad86f18ed9106b21566cfe810adc7f907143939409cadf361aec1572f76e3": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Right": 1
}
},
"query": "DELETE FROM starchart_mini_index WHERE word = ($1)"
},
"7ee4e3e06dc7dea3b514c0d7632c916ee0d9346fd52af43563d47f3c4deff22d": {
"describe": {
"columns": [
@ -567,5 +595,33 @@
}
},
"query": "DELETE FROM starchart_forges WHERE hostname = ($1)"
},
"f9aad84b4586954c224ddb0c2a60409c77589d856acd74c14fcec3799aa23407": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Right": 1
}
},
"query": "INSERT OR IGNORE INTO starchart_mini_index ( word ) \n VALUES ( $1);"
},
"f9fb014c5e0a32d7f51b78c08439a06b30f46ebadd28b062b313a7faf0f0aef1": {
"describe": {
"columns": [
{
"name": "word",
"ordinal": 0,
"type_info": "Text"
}
],
"nullable": [
false
],
"parameters": {
"Right": 0
}
},
"query": "SELECT word FROM starchart_mini_index"
}
}

View file

@ -856,6 +856,60 @@ impl SCDatabase for Database {
.map_err(|e| DBError::DBError(Box::new(e)))?;
Ok(s)
}
/// Add word to mini index
async fn add_word_to_mini_index(&self, word: &str) -> DBResult<()> {
sqlx::query!(
"INSERT OR IGNORE INTO starchart_mini_index ( word )
VALUES ( $1);",
word,
)
.execute(&self.pool)
.await
.map_err(map_register_err)?;
Ok(())
}
/// Remove word from mini index
async fn rm_word_from_mini_index(&self, word: &str) -> DBResult<()> {
sqlx::query!("DELETE FROM starchart_mini_index WHERE word = ($1)", word)
.execute(&self.pool)
.await
.map_err(|e| DBError::DBError(Box::new(e)))?;
Ok(())
}
/// Check if word exists in mini index
async fn is_word_mini_indexed(&self, word: &str) -> DBResult<bool> {
match sqlx::query!("SELECT ID FROM starchart_mini_index WHERE word = $1", word)
.fetch_one(&self.pool)
.await
{
Ok(_) => Ok(true),
Err(Error::RowNotFound) => Ok(false),
Err(e) => Err(DBError::DBError(Box::new(e).into())),
}
}
/// consolidate and export mini index
async fn export_mini_index(&self) -> DBResult<String> {
struct Words {
word: String,
}
let mut words = sqlx::query_as!(Words, "SELECT word FROM starchart_mini_index")
.fetch_all(&self.pool)
.await
.map_err(|e| DBError::DBError(Box::new(e)))?;
let mut mini_index = String::default();
words.drain(0..).for_each(|w| {
mini_index = if mini_index.is_empty() {
w.word
} else {
format!("{mini_index} {}", w.word)
}
});
Ok(mini_index)
}
}
fn now_unix_time_stamp() -> i64 {

View file

@ -110,3 +110,14 @@ async fn forge_type_exists() {
db.migrate().await.unwrap();
forge_type_exists_helper(&db).await;
}
#[actix_rt::test]
async fn mini_index_test() {
let url = env::var("SQLITE_DATABASE_URL").expect("Set SQLITE_DATABASE_URL env var");
let pool_options = SqlitePoolOptions::new().max_connections(2);
let connection_options = ConnectionOptions::Fresh(Fresh { pool_options, url });
let db = connection_options.connect().await.unwrap();
db.migrate().await.unwrap();
mini_index_helper(&db).await;
}