From f53a0afada989e980dc45de128176761e1cbcc12 Mon Sep 17 00:00:00 2001 From: Aravinth Manivannan Date: Thu, 2 Mar 2023 13:09:48 +0530 Subject: [PATCH] feat: implement mini-index --- db/db-core/src/lib.rs | 12 ++++ db/db-core/src/tests.rs | 19 +++++++ .../20230302072543_starchart_mini_index.sql | 4 ++ db/db-sqlx-sqlite/sqlx-data.json | 56 +++++++++++++++++++ db/db-sqlx-sqlite/src/lib.rs | 54 ++++++++++++++++++ db/db-sqlx-sqlite/src/tests.rs | 11 ++++ 6 files changed, 156 insertions(+) create mode 100644 db/db-sqlx-sqlite/migrations/20230302072543_starchart_mini_index.sql diff --git a/db/db-core/src/lib.rs b/db/db-core/src/lib.rs index 1f9f557..ba5bc6d 100644 --- a/db/db-core/src/lib.rs +++ b/db/db-core/src/lib.rs @@ -246,6 +246,18 @@ pub trait SCDatabase: std::marker::Send + std::marker::Sync + CloneSPDatabase { offset: u32, limit: u32, ) -> DBResult>; + + /// Add word to mini index + async fn add_word_to_mini_index(&self, word: &str) -> DBResult<()>; + + /// Remove word from mini index + async fn rm_word_from_mini_index(&self, word: &str) -> DBResult<()>; + + /// Check if word exists in mini index + async fn is_word_mini_indexed(&self, word: &str) -> DBResult; + + /// consolidate and export mini index + async fn export_mini_index(&self) -> DBResult; } /// Trait to clone SCDatabase diff --git a/db/db-core/src/tests.rs b/db/db-core/src/tests.rs index ad86b72..bf4b1af 100644 --- a/db/db-core/src/tests.rs +++ b/db/db-core/src/tests.rs @@ -120,3 +120,22 @@ pub async fn instance_introducer_helper(db: &T, instance_url: &Ur .iter() .any(|i| i.instance_url == instance_url.as_str())); } + +/// test if all instance introducer methods work +pub async fn mini_index_helper(db: &T) { + // batman is repeated twice but mini-index should contain it only once + // Batman is different from Batman; mini-index is case-sensitive + const WORDS: [&str; 5] = ["batman", "superman", "aquaman", "Batman", "batman"]; + + let expected_mini_index = "superman aquaman Batman batman"; + + for w in WORDS.iter() { + db.rm_word_from_mini_index(w).await.unwrap(); + assert!(!db.is_word_mini_indexed(w).await.unwrap()); + db.add_word_to_mini_index(w).await.unwrap(); + assert!(db.is_word_mini_indexed(w).await.unwrap()); + } + + let mini_index = db.export_mini_index().await.unwrap(); + assert_eq!(mini_index, expected_mini_index); +} diff --git a/db/db-sqlx-sqlite/migrations/20230302072543_starchart_mini_index.sql b/db/db-sqlx-sqlite/migrations/20230302072543_starchart_mini_index.sql new file mode 100644 index 0000000..14767d0 --- /dev/null +++ b/db/db-sqlx-sqlite/migrations/20230302072543_starchart_mini_index.sql @@ -0,0 +1,4 @@ +CREATE TABLE IF NOT EXISTS starchart_mini_index ( + word TEXT NOT NULL UNIQUE, + ID INTEGER PRIMARY KEY NOT NULL +); diff --git a/db/db-sqlx-sqlite/sqlx-data.json b/db/db-sqlx-sqlite/sqlx-data.json index 2575ac1..26b175e 100644 --- a/db/db-sqlx-sqlite/sqlx-data.json +++ b/db/db-sqlx-sqlite/sqlx-data.json @@ -10,6 +10,24 @@ }, "query": "INSERT OR IGNORE INTO fts_repositories ( name, description, website, html_url ) \n VALUES ( $1, $2, $3, $4 );" }, + "0b179588df37779f563f0ad8c43e920a8bc22b3eed682778cef9dd05608f9691": { + "describe": { + "columns": [ + { + "name": "ID", + "ordinal": 0, + "type_info": "Int64" + } + ], + "nullable": [ + false + ], + "parameters": { + "Right": 1 + } + }, + "query": "SELECT ID FROM starchart_mini_index WHERE word = $1" + }, "0bb37cc79d5ef803285d05d06e6ef93b62c0b532c0298148fe436178761fd70a": { "describe": { "columns": [ @@ -240,6 +258,16 @@ }, "query": "SELECT\n hostname,\n last_crawl_on,\n starchart_forge_type.name,\n starchart_introducer.instance_url\n FROM\n starchart_forges\n INNER JOIN\n starchart_forge_type\n ON\n starchart_forges.forge_type = starchart_forge_type.id\n LEFT JOIN\n starchart_introducer\n ON\n starchart_introducer.ID = starchart_forges.starchart_instance\n ORDER BY\n starchart_forges.ID\n LIMIT $1 OFFSET $2;\n " }, + "7a2ad86f18ed9106b21566cfe810adc7f907143939409cadf361aec1572f76e3": { + "describe": { + "columns": [], + "nullable": [], + "parameters": { + "Right": 1 + } + }, + "query": "DELETE FROM starchart_mini_index WHERE word = ($1)" + }, "7ee4e3e06dc7dea3b514c0d7632c916ee0d9346fd52af43563d47f3c4deff22d": { "describe": { "columns": [ @@ -567,5 +595,33 @@ } }, "query": "DELETE FROM starchart_forges WHERE hostname = ($1)" + }, + "f9aad84b4586954c224ddb0c2a60409c77589d856acd74c14fcec3799aa23407": { + "describe": { + "columns": [], + "nullable": [], + "parameters": { + "Right": 1 + } + }, + "query": "INSERT OR IGNORE INTO starchart_mini_index ( word ) \n VALUES ( $1);" + }, + "f9fb014c5e0a32d7f51b78c08439a06b30f46ebadd28b062b313a7faf0f0aef1": { + "describe": { + "columns": [ + { + "name": "word", + "ordinal": 0, + "type_info": "Text" + } + ], + "nullable": [ + false + ], + "parameters": { + "Right": 0 + } + }, + "query": "SELECT word FROM starchart_mini_index" } } \ No newline at end of file diff --git a/db/db-sqlx-sqlite/src/lib.rs b/db/db-sqlx-sqlite/src/lib.rs index 8ef02af..1ba9043 100644 --- a/db/db-sqlx-sqlite/src/lib.rs +++ b/db/db-sqlx-sqlite/src/lib.rs @@ -856,6 +856,60 @@ impl SCDatabase for Database { .map_err(|e| DBError::DBError(Box::new(e)))?; Ok(s) } + + /// Add word to mini index + async fn add_word_to_mini_index(&self, word: &str) -> DBResult<()> { + sqlx::query!( + "INSERT OR IGNORE INTO starchart_mini_index ( word ) + VALUES ( $1);", + word, + ) + .execute(&self.pool) + .await + .map_err(map_register_err)?; + Ok(()) + } + + /// Remove word from mini index + async fn rm_word_from_mini_index(&self, word: &str) -> DBResult<()> { + sqlx::query!("DELETE FROM starchart_mini_index WHERE word = ($1)", word) + .execute(&self.pool) + .await + .map_err(|e| DBError::DBError(Box::new(e)))?; + Ok(()) + } + + /// Check if word exists in mini index + async fn is_word_mini_indexed(&self, word: &str) -> DBResult { + match sqlx::query!("SELECT ID FROM starchart_mini_index WHERE word = $1", word) + .fetch_one(&self.pool) + .await + { + Ok(_) => Ok(true), + Err(Error::RowNotFound) => Ok(false), + Err(e) => Err(DBError::DBError(Box::new(e).into())), + } + } + + /// consolidate and export mini index + async fn export_mini_index(&self) -> DBResult { + struct Words { + word: String, + } + let mut words = sqlx::query_as!(Words, "SELECT word FROM starchart_mini_index") + .fetch_all(&self.pool) + .await + .map_err(|e| DBError::DBError(Box::new(e)))?; + let mut mini_index = String::default(); + words.drain(0..).for_each(|w| { + mini_index = if mini_index.is_empty() { + w.word + } else { + format!("{mini_index} {}", w.word) + } + }); + Ok(mini_index) + } } fn now_unix_time_stamp() -> i64 { diff --git a/db/db-sqlx-sqlite/src/tests.rs b/db/db-sqlx-sqlite/src/tests.rs index 8e4de3f..cadcef4 100644 --- a/db/db-sqlx-sqlite/src/tests.rs +++ b/db/db-sqlx-sqlite/src/tests.rs @@ -110,3 +110,14 @@ async fn forge_type_exists() { db.migrate().await.unwrap(); forge_type_exists_helper(&db).await; } + +#[actix_rt::test] +async fn mini_index_test() { + let url = env::var("SQLITE_DATABASE_URL").expect("Set SQLITE_DATABASE_URL env var"); + let pool_options = SqlitePoolOptions::new().max_connections(2); + let connection_options = ConnectionOptions::Fresh(Fresh { pool_options, url }); + let db = connection_options.connect().await.unwrap(); + + db.migrate().await.unwrap(); + mini_index_helper(&db).await; +}