feat: save repositories and tags

This commit is contained in:
Aravinth Manivannan 2022-05-04 22:53:38 +05:30
parent 0acfd137be
commit 59aafc037e
Signed by: realaravinth
GPG key ID: AD9F0F08E855ED88
2 changed files with 44 additions and 1 deletions

View file

@ -17,6 +17,7 @@
*/ */
use std::collections::HashMap; use std::collections::HashMap;
use db_core::AddRepository;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
@ -145,6 +146,11 @@ pub struct Team {
pub units_map: HashMap<String, String>, pub units_map: HashMap<String, String>,
} }
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Topics {
pub topics: Vec<String>,
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;

View file

@ -25,14 +25,23 @@ use db_core::prelude::*;
use crate::data::Data; use crate::data::Data;
use crate::db::BoxDB; use crate::db::BoxDB;
use crate::gitea::SearchResults; use crate::gitea::SearchResults;
use crate::gitea::Topics;
const REPO_SEARCH_PATH: &str = "/api/v1/repos/search"; const REPO_SEARCH_PATH: &str = "/api/v1/repos/search";
const GITEA_NODEINFO: &str = "/api/v1/nodeinfo"; const GITEA_NODEINFO: &str = "/api/v1/nodeinfo";
impl Data { impl Data {
pub async fn crawl(&self, hostname: &str, db: &BoxDB) -> Vec<SearchResults> { pub async fn crawl(&self, hostname: &str, db: &BoxDB) -> Vec<SearchResults> {
fn empty_is_none(s: &str) -> Option<&str> {
if s.trim().is_empty() {
None
} else {
Some(s)
}
}
let mut page = 1; let mut page = 1;
let mut instance_url = Url::parse(hostname).unwrap(); let instance_url = Url::parse(hostname).unwrap();
let hostname = get_hostname(&instance_url); let hostname = get_hostname(&instance_url);
if !db.forge_exists(&hostname).await.unwrap() { if !db.forge_exists(&hostname).await.unwrap() {
let msg = CreateForge { let msg = CreateForge {
@ -83,6 +92,34 @@ impl Data {
}; };
db.add_user(&msg).await.unwrap(); db.add_user(&msg).await.unwrap();
} }
let mut url = instance_url.clone();
url.set_path(&format!(
"/api/v1/repos/{}/{}/topics",
repo.owner.username, repo.name
));
let topics: Topics = self
.client
.get(url)
.send()
.await
.unwrap()
.json()
.await
.unwrap();
let add_repo_msg = AddRepository {
tags: Some(topics.topics),
name: &repo.name,
website: empty_is_none(&repo.website),
description: empty_is_none(&repo.description),
owner: &repo.owner.username,
html_link: &repo.html_url,
hostname: &hostname,
};
db.create_repository(&add_repo_msg).await.unwrap();
} }
sleep_fut.await.unwrap(); sleep_fut.await.unwrap();