ForgeFlux/starchart
2
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

Compare commits

base: ForgeFlux:master
Branches Tags
ForgeFlux:master
ForgeFlux:crawl-forgeflux
ForgeFlux:init-woodpecker
ForgeFlux:renovate/configure
..
compare: ForgeFlux:init-woodpecker
Branches Tags
ForgeFlux:master
ForgeFlux:crawl-forgeflux
ForgeFlux:init-woodpecker
ForgeFlux:renovate/configure

No commits in common. "master" and "init-woodpecker" have entirely different histories.
master ... init-woodp

19 changed files with 127 additions and 539 deletions
Show stats Expand all files Collapse all files

1
.env-sample
View file

@ -3,4 +3,3 @@ export SQLITE_TMP="$(pwd)/db/db-sqlx-sqlite/tmp"
export SQLITE_DATABASE_URL="sqlite://$SQLITE_TMP/admin.db" export SQLITE_DATABASE_URL="sqlite://$SQLITE_TMP/admin.db"
export STARCHART__CRAWLER__WAIT_BEFORE_NEXT_API_CALL=0 export STARCHART__CRAWLER__WAIT_BEFORE_NEXT_API_CALL=0
export FORGEJO_HOST=http://localhost:3000 export FORGEJO_HOST=http://localhost:3000
export FORGEFLUX_HOST=http://localhost:7000

64
.woodpecker.yml
View file

@ -7,11 +7,9 @@ steps:
POSTGRES_DATABASE_URL: postgres://postgres:password@database:5432/postgres POSTGRES_DATABASE_URL: postgres://postgres:password@database:5432/postgres
commands: commands:
- pip install requests - pip install requests
- sed -i 's/localhost\:3000/forgejo/' scripts/gitea.py - sed -i 's/localhost/forgejo/' scripts/gitea.py
- python ./scripts/gitea.py - python ./scripts/gitea.py
# - curl -vv http://forgejo/api/v1/repos/bot/repository_58
# - curl -vv http://forge_forgeflux/forgejo/bot/repository_58
# - curl -vv http://forge_forgeflux/forgejo/bot/repository_01
test: test:
image: rust image: rust
@ -20,8 +18,7 @@ steps:
environment: environment:
POSTGRES_DATABASE_URL: postgres://postgres:password@database:5432/postgres POSTGRES_DATABASE_URL: postgres://postgres:password@database:5432/postgres
SQLITE_DATABASE_URL: sqlite:///tmp/admin.db SQLITE_DATABASE_URL: sqlite:///tmp/admin.db
FORGEJO_HOST: http://forgejo FORGEJO_HOST: http://forgejo:3000
FORGEFLUX_HOST: http://forge_forgeflux
commands: commands:
# - curl -fsSL https://deb.nodesource.com/setup_16.x | bash - &&\ # - curl -fsSL https://deb.nodesource.com/setup_16.x | bash - &&\
# - apt update && apt-get -y --no-install-recommends install nodejs tar gpg curl wget # - apt update && apt-get -y --no-install-recommends install nodejs tar gpg curl wget
@ -35,6 +32,7 @@ steps:
# - make lint # - make lint
- make test.workspaces - make test.workspaces
build_docker_img: build_docker_img:
image: plugins/docker image: plugins/docker
when: when:
@ -44,20 +42,6 @@ steps:
repo: forgeflux/starchart repo: forgeflux/starchart
tags: latest tags: latest
build_and_publish_docker_img:
image: plugins/docker
when:
event: [push, tag, deployment]
branch: master
settings:
username: forgeflux
password:
from_secret: DOCKER_TOKEN
repo: forgeflux/starchart
tags:
latest
services: services:
forgejo: forgejo:
image: codeberg.org/forgejo/forgejo:9 image: codeberg.org/forgejo/forgejo:9
@ -65,47 +49,9 @@ services:
FORGEJO__security__INSTALL_LOCK: true FORGEJO__security__INSTALL_LOCK: true
FORGEJO__federation__ENABLED: true FORGEJO__federation__ENABLED: true
FORGEJO__server__ROOT_URL: http://forgejo FORGEJO__server__ROOT_URL: http://forgejo
FORGEJO__server__HTTP_PORT: 80 FORGEJO__server__HTTP_PORT: 3000
database: database:
image: postgres image: postgres
environment: environment:
POSTGRES_PASSWORD: password POSTGRES_PASSWORD: password
forge_forgeflux_database:
image: postgres
environment:
POSTGRES_PASSWORD: password
forge_forgeflux:
image: forgeflux/forgeflux:latest
depends_on:
- forgeflux_postgres
environment:
FORGEFLUX_server_PUBLIC_URL_HAS_HTTPS: false
FORGEFLUX_debug: true
FORGEFLUX_source_code: https://git.batsense.net/ForgeFlux/ForgeFlux
FORGEFLUX_allow_registration: true
FORGEFLUX_database_POOL: 2
FORGEFLUX_forges_FORGEJO_url: http://forgejo
FORGEFLUX_forges_FORGEJO_client_id: foo
FORGEFLUX_forges_FORGEJO_client_secret: foo
FORGEFLUX_forges_FORGEJO_user_USERNAME: foo
FORGEFLUX_forges_FORGEJO_user_API_TOKEN: foo
DATABASE_URL: postgres://postgres:password@forgeflux_postgres:5432/postgres
PORT: 80
FORGEFLUX_server_DOMAIN: forge_forgeflux
FORGEFLUX_server_COOKIE_SECRET: 995cde0721b6e41602dd111438cc7c1b2506dc14bad31d2653fb9a4adce1f84e
FORGEFLUX_server_IP: 0.0.0.0
FORGEFLUX_forges_GITHUB_url: https://github.com
FORGEFLUX_forges_GITHUB_api_url: https://api.github.com
FORGEFLUX_forges_GITHUB_client_id: foo
FORGEFLUX_forges_GITHUB_client_secret: foo
FORGEFLUX_forges_GITHUB_user_USERNAME: foo
FORGEFLUX_forges_GITHUB_user_PERSONAL_ACCESS_TOKEN: foo
FORGEFLUX_starchart_enable: true
forgeflux_postgres:
image: postgres:17.2
environment:
POSTGRES_PASSWORD: password # change password

17
Cargo.lock generated
View file

@ -1,6 +1,6 @@
# This file is automatically @generated by Cargo. # This file is automatically @generated by Cargo.
# It is not intended for manual editing. # It is not intended for manual editing.
version = 4 version = 3
[[package]] [[package]]
name = "actix" name = "actix"
@ -1147,20 +1147,6 @@ dependencies = [
"url", "url",
] ]
[[package]]
name = "forgeflux"
version = "0.1.0"
dependencies = [
"actix-rt",
"async-trait",
"forge-core",
"reqwest",
"serde",
"serde_json",
"tokio",
"url",
]
[[package]] [[package]]
name = "form_urlencoded" name = "form_urlencoded"
version = "1.0.1" version = "1.0.1"
@ -2936,7 +2922,6 @@ dependencies = [
"derive_more", "derive_more",
"federate-core", "federate-core",
"forge-core", "forge-core",
"forgeflux",
"gitea", "gitea",
"lazy_static", "lazy_static",
"log", "log",

4
Cargo.toml
View file

@ -16,7 +16,6 @@ members = [
"db/db-core", "db/db-core",
"db/db-sqlx-sqlite", "db/db-sqlx-sqlite",
"forge/forge-core", "forge/forge-core",
"forge/forgeflux",
"forge/gitea", "forge/gitea",
"federate/federate-core", "federate/federate-core",
"federate/publiccodeyml" "federate/publiccodeyml"
@ -77,9 +76,6 @@ path = "./db/db-sqlx-sqlite"
[dependencies.gitea] [dependencies.gitea]
path = "./forge/gitea" path = "./forge/gitea"
[dependencies.forgeflux]
path = "./forge/forgeflux"
[dependencies.forge-core] [dependencies.forge-core]
path = "./forge/forge-core" path = "./forge/forge-core"

6
Dockerfile
View file

@ -18,9 +18,9 @@ COPY . .
COPY --from=cacher /src/target target COPY --from=cacher /src/target target
RUN make release RUN make release
FROM debian:latest FROM debian:bullseye-slim
#LABEL org.opencontainers.image.source https://github.com/forgeflux-org/starchart LABEL org.opencontainers.image.source https://github.com/forgeflux-org/starchart
RUN apt-get update && apt-get install -y ca-certificates libssl-dev RUN apt-get update && apt-get install -y ca-certificates
COPY --from=builder /src/target/release/starchart /usr/local/bin/ COPY --from=builder /src/target/release/starchart /usr/local/bin/
COPY --from=builder /src/config/default.toml /etc/starchart/config.toml COPY --from=builder /src/config/default.toml /etc/starchart/config.toml
COPY scripts/entrypoint.sh /usr/local/bin COPY scripts/entrypoint.sh /usr/local/bin

6
Makefile
View file

@ -98,11 +98,11 @@ doc: ## Prepare documentation
cargo doc --no-deps --workspace --all-features cargo doc --no-deps --workspace --all-features
docker: ## Build docker images docker: ## Build docker images
docker build -t forgeflux/starchart:master -t forgeflux/starchart:latest . docker build -t forgedfed/starchart:master -t forgedfed/starchart:latest .
docker-publish: docker ## Build and publish docker images docker-publish: docker ## Build and publish docker images
docker push forgeflux/starchart:master docker push forgedfed/starchart:master
docker push forgeflux/starchart:latest docker push forgedfed/starchart:latest
lint: ## Lint codebase lint: ## Lint codebase
cargo fmt -v --all -- --emit files cargo fmt -v --all -- --emit files

5
db/db-core/src/lib.rs
View file

@ -309,8 +309,6 @@ impl Clone for Box<dyn SCDatabase> {
pub enum ForgeImplementation { pub enum ForgeImplementation {
/// [Gitea](https://gitea.io) softare forge /// [Gitea](https://gitea.io) softare forge
Gitea, Gitea,
/// [ForgeFlux](https://net.forgeflux.net)
ForgeFlux,
} }
impl ForgeImplementation { impl ForgeImplementation {
@ -318,7 +316,6 @@ impl ForgeImplementation {
pub const fn to_str(&self) -> &'static str { pub const fn to_str(&self) -> &'static str {
match self { match self {
ForgeImplementation::Gitea => "gitea", ForgeImplementation::Gitea => "gitea",
ForgeImplementation::ForgeFlux => "forgeflux",
} }
} }
} }
@ -329,11 +326,9 @@ impl FromStr for ForgeImplementation {
/// Convert [str] to [ForgeImplementation] /// Convert [str] to [ForgeImplementation]
fn from_str(s: &str) -> DBResult<Self> { fn from_str(s: &str) -> DBResult<Self> {
const GITEA: &str = ForgeImplementation::Gitea.to_str(); const GITEA: &str = ForgeImplementation::Gitea.to_str();
const FORGEFLUX: &str = ForgeImplementation::ForgeFlux.to_str();
let s = s.trim(); let s = s.trim();
match s { match s {
GITEA => Ok(Self::Gitea), GITEA => Ok(Self::Gitea),
FORGEFLUX => Ok(Self::ForgeFlux),
_ => Err(DBError::UnknownForgeType(s.to_owned())), _ => Err(DBError::UnknownForgeType(s.to_owned())),
} }
} }

1
db/db-sqlx-sqlite/migrations/20250319120857_starchart_forgeflux_forge_type.sql
View file

@ -1 +0,0 @@
INSERT OR IGNORE INTO starchart_forge_type (name) VALUES('forgeflux');

100
db/db-sqlx-sqlite/sqlx-data.json
View file

@ -18,6 +18,16 @@
}, },
"query": "SELECT\n starchart_introducer.instance_url\n FROM \n starchart_federated_mini_index\n INNER JOIN\n starchart_introducer\n ON\n starchart_introducer.ID = starchart_instance \n WHERE\n mini_index MATCH $1" "query": "SELECT\n starchart_introducer.instance_url\n FROM \n starchart_federated_mini_index\n INNER JOIN\n starchart_introducer\n ON\n starchart_introducer.ID = starchart_instance \n WHERE\n mini_index MATCH $1"
}, },
"069a127ffb1062321bba5a915a4ead3e0bbe4dabf6e0f684d8cc6a6d8a68ad5b": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Right": 4
}
},
"query": "INSERT OR IGNORE INTO fts_repositories ( name, description, website, html_url ) \n VALUES ( $1, $2, $3, $4 );"
},
"0b179588df37779f563f0ad8c43e920a8bc22b3eed682778cef9dd05608f9691": { "0b179588df37779f563f0ad8c43e920a8bc22b3eed682778cef9dd05608f9691": {
"describe": { "describe": {
"columns": [ "columns": [
@ -94,26 +104,6 @@
}, },
"query": "SELECT html_url, profile_photo_html_url, imported FROM starchart_users WHERE username = $1 AND \n hostname_id = (SELECT ID FROM starchart_forges WHERE hostname = $2)" "query": "SELECT html_url, profile_photo_html_url, imported FROM starchart_users WHERE username = $1 AND \n hostname_id = (SELECT ID FROM starchart_forges WHERE hostname = $2)"
}, },
"1f47bff0270cfb9d58972ecf6ae36f348d9f605a40a8669ce2908b50a3aac735": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Right": 4
}
},
"query": "INSERT OR REPLACE INTO fts_repositories ( name, description, website, html_url ) \n VALUES ( $1, $2, $3, $4 );"
},
"2ac627ddd905bab19582037ca5e402ebaa268b61d94459e125644028d04e6dc2": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Right": 4
}
},
"query": "INSERT OR REPLACE INTO starchart_forges\n (hostname, verified_on, forge_type, starchart_instance)\n VALUES (\n $1,\n $2,\n (SELECT ID FROM starchart_forge_type WHERE name = $3),\n (SELECT ID FROM starchart_introducer WHERE instance_url = $4)\n )"
},
"2afb17ba3753aa440465a836b46b7a1466f25791cfc4d0acdd38bc2755ae3e86": { "2afb17ba3753aa440465a836b46b7a1466f25791cfc4d0acdd38bc2755ae3e86": {
"describe": { "describe": {
"columns": [ "columns": [
@ -150,6 +140,16 @@
}, },
"query": "SELECT ID FROM starchart_forge_type WHERE name = $1" "query": "SELECT ID FROM starchart_forge_type WHERE name = $1"
}, },
"338fb30307071e6df9efee6a68697c60e579d7b2332630bce401c0e7186a642a": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Right": 7
}
},
"query": "INSERT INTO \n starchart_users (\n hostname_id, username, html_url,\n profile_photo_html_url, added_on, last_crawl_on, imported\n ) \n VALUES (\n (SELECT ID FROM starchart_forges WHERE hostname = $1), $2, $3, $4, $5, $6, $7)"
},
"364c8e3d147318b864fd28ad284f225aaace9479b5cf0428fb97f0e5689e248d": { "364c8e3d147318b864fd28ad284f225aaace9479b5cf0428fb97f0e5689e248d": {
"describe": { "describe": {
"columns": [], "columns": [],
@ -224,16 +224,6 @@
}, },
"query": "SELECT\n hostname,\n last_crawl_on,\n starchart_introducer.instance_url,\n starchart_forge_type.name\n FROM\n starchart_forges\n INNER JOIN\n starchart_forge_type\n ON\n starchart_forges.forge_type = starchart_forge_type.id\n LEFT JOIN\n starchart_introducer\n ON\n starchart_introducer.ID = starchart_forges.starchart_instance\n WHERE \n starchart_forges.imported = 0\n ORDER BY\n starchart_forges.ID\n LIMIT $1 OFFSET $2;\n " "query": "SELECT\n hostname,\n last_crawl_on,\n starchart_introducer.instance_url,\n starchart_forge_type.name\n FROM\n starchart_forges\n INNER JOIN\n starchart_forge_type\n ON\n starchart_forges.forge_type = starchart_forge_type.id\n LEFT JOIN\n starchart_introducer\n ON\n starchart_introducer.ID = starchart_forges.starchart_instance\n WHERE \n starchart_forges.imported = 0\n ORDER BY\n starchart_forges.ID\n LIMIT $1 OFFSET $2;\n "
}, },
"6c6a24873ae0053df5b875a35ed8c605e73c9718951b87140d2cada093ec4cf6": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Right": 9
}
},
"query": "INSERT OR REPLACE INTO \n starchart_repositories (\n hostname_id, owner_id, name, description, html_url, website, created,\n last_crawl, imported\n )\n VALUES (\n (SELECT ID FROM starchart_forges WHERE hostname = $1),\n (SELECT ID FROM starchart_users WHERE username = $2),\n $3, $4, $5, $6, $7, $8, $9\n );"
},
"6f5ca3d71a541eb6f33e37a5889c048536ab6ad7e81a6236d73aa71433c13717": { "6f5ca3d71a541eb6f33e37a5889c048536ab6ad7e81a6236d73aa71433c13717": {
"describe": { "describe": {
"columns": [], "columns": [],
@ -244,6 +234,16 @@
}, },
"query": "INSERT OR IGNORE INTO starchart_project_topics ( name ) VALUES ( $1 );" "query": "INSERT OR IGNORE INTO starchart_project_topics ( name ) VALUES ( $1 );"
}, },
"74fb3a1ae4f339b5371a6872e6eb4ed7c1f5968dac70de1639454c394a05cb38": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Right": 4
}
},
"query": "INSERT INTO starchart_forges\n (hostname, verified_on, forge_type, starchart_instance)\n VALUES\n (\n $1, $2,\n (SELECT ID FROM starchart_forge_type WHERE name = $3),\n $4)"
},
"7590630f5fe7e05014b70ac0047f9b6c724b88e35e1b1306fb89760612929d55": { "7590630f5fe7e05014b70ac0047f9b6c724b88e35e1b1306fb89760612929d55": {
"describe": { "describe": {
"columns": [], "columns": [],
@ -424,16 +424,6 @@
}, },
"query": "SELECT name FROM starchart_project_topics ORDER BY ID LIMIT $1 OFFSET $2;" "query": "SELECT name FROM starchart_project_topics ORDER BY ID LIMIT $1 OFFSET $2;"
}, },
"a754fb4bcdd227f3ab440c5600a534dcabde3e75ea242a530d6aa12c7502c88e": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Right": 7
}
},
"query": "INSERT OR REPLACE INTO \n starchart_users (\n hostname_id, username, html_url,\n profile_photo_html_url, added_on, last_crawl_on, imported\n ) \n VALUES (\n (SELECT ID FROM starchart_forges WHERE hostname = $1), $2, $3, $4, $5, $6, $7)"
},
"a81dd4b5df666e22fac211092e7b8425d838dd9023aa2b17659352f30831944d": { "a81dd4b5df666e22fac211092e7b8425d838dd9023aa2b17659352f30831944d": {
"describe": { "describe": {
"columns": [ "columns": [
@ -452,6 +442,16 @@
}, },
"query": "SELECT ID FROM starchart_users WHERE username = $1 AND \n hostname_id = (SELECT ID FROM starchart_forges WHERE hostname = $2)" "query": "SELECT ID FROM starchart_users WHERE username = $1 AND \n hostname_id = (SELECT ID FROM starchart_forges WHERE hostname = $2)"
}, },
"a912406491a4e9ea1bbf8a3d0003e948a24df7f101c490aca395f5b86ec64d00": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Right": 4
}
},
"query": "INSERT OR IGNORE INTO starchart_forges\n (hostname, verified_on, forge_type, starchart_instance)\n VALUES (\n $1,\n $2,\n (SELECT ID FROM starchart_forge_type WHERE name = $3),\n (SELECT ID FROM starchart_introducer WHERE instance_url = $4)\n )"
},
"ae9295f46da1753fe91a633b5738533084005ad32915c19635f896c454284b6b": { "ae9295f46da1753fe91a633b5738533084005ad32915c19635f896c454284b6b": {
"describe": { "describe": {
"columns": [ "columns": [
@ -522,6 +522,16 @@
}, },
"query": "INSERT OR IGNORE INTO\n starchart_imported_starcharts (starchart_instance)\n VALUES ((SELECT ID FROM starchart_introducer WHERE instance_url = $1));" "query": "INSERT OR IGNORE INTO\n starchart_imported_starcharts (starchart_instance)\n VALUES ((SELECT ID FROM starchart_introducer WHERE instance_url = $1));"
}, },
"ca22e5f6e7065cf2d4ffdbfac0084f9871de8cd9073d470cbf7eac2de2a73c47": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Right": 9
}
},
"query": "INSERT INTO \n starchart_repositories (\n hostname_id, owner_id, name, description, html_url, website, created,\n last_crawl, imported\n )\n VALUES (\n (SELECT ID FROM starchart_forges WHERE hostname = $1),\n (SELECT ID FROM starchart_users WHERE username = $2),\n $3, $4, $5, $6, $7, $8, $9\n );"
},
"d22d18f0c2b49a570ac95e1af5ba398b3d1c7a1a5a6780d6dc646077e466714c": { "d22d18f0c2b49a570ac95e1af5ba398b3d1c7a1a5a6780d6dc646077e466714c": {
"describe": { "describe": {
"columns": [ "columns": [
@ -680,16 +690,6 @@
}, },
"query": "SELECT word FROM starchart_mini_index" "query": "SELECT word FROM starchart_mini_index"
}, },
"fc32b8d0e5599dcc1ceed4127ca860d09754dd6d1fe558fbe5c7f4086c901b8f": {
"describe": {
"columns": [],
"nullable": [],
"parameters": {
"Right": 4
}
},
"query": "INSERT OR REPLACE INTO starchart_forges\n (hostname, verified_on, forge_type, starchart_instance)\n VALUES\n (\n $1, $2,\n (SELECT ID FROM starchart_forge_type WHERE name = $3),\n $4)"
},
"fdf87490759150d528139b13eb1a28532b5bebb546ade00bcb6a7d648abcd445": { "fdf87490759150d528139b13eb1a28532b5bebb546ade00bcb6a7d648abcd445": {
"describe": { "describe": {
"columns": [], "columns": [],

10
db/db-sqlx-sqlite/src/lib.rs
View file

@ -160,7 +160,7 @@ impl Database {
return Ok(()); return Ok(());
} }
sqlx::query!( sqlx::query!(
"INSERT OR REPLACE INTO fts_repositories ( name, description, website, html_url ) "INSERT OR IGNORE INTO fts_repositories ( name, description, website, html_url )
VALUES ( $1, $2, $3, $4 );", VALUES ( $1, $2, $3, $4 );",
name, name,
description, description,
@ -313,7 +313,7 @@ impl SCDatabase for Database {
let forge_type = f.forge_type.to_str(); let forge_type = f.forge_type.to_str();
if let Some(instance_url) = f.starchart_url { if let Some(instance_url) = f.starchart_url {
sqlx::query!( sqlx::query!(
"INSERT OR REPLACE INTO starchart_forges "INSERT OR IGNORE INTO starchart_forges
(hostname, verified_on, forge_type, starchart_instance) (hostname, verified_on, forge_type, starchart_instance)
VALUES ( VALUES (
$1, $1,
@ -331,7 +331,7 @@ impl SCDatabase for Database {
.map_err(map_register_err)?; .map_err(map_register_err)?;
} else { } else {
sqlx::query!( sqlx::query!(
"INSERT OR REPLACE INTO starchart_forges "INSERT INTO starchart_forges
(hostname, verified_on, forge_type, starchart_instance) (hostname, verified_on, forge_type, starchart_instance)
VALUES VALUES
( (
@ -491,7 +491,7 @@ impl SCDatabase for Database {
let now = now_unix_time_stamp(); let now = now_unix_time_stamp();
let url = db_core::clean_url(&u.url); let url = db_core::clean_url(&u.url);
sqlx::query!( sqlx::query!(
"INSERT OR REPLACE INTO "INSERT INTO
starchart_users ( starchart_users (
hostname_id, username, html_url, hostname_id, username, html_url,
profile_photo_html_url, added_on, last_crawl_on, imported profile_photo_html_url, added_on, last_crawl_on, imported
@ -607,7 +607,7 @@ impl SCDatabase for Database {
let now = now_unix_time_stamp(); let now = now_unix_time_stamp();
let url = db_core::clean_url(&r.url); let url = db_core::clean_url(&r.url);
sqlx::query!( sqlx::query!(
"INSERT OR REPLACE INTO "INSERT INTO
starchart_repositories ( starchart_repositories (
hostname_id, owner_id, name, description, html_url, website, created, hostname_id, owner_id, name, description, html_url, website, created,
last_crawl, imported last_crawl, imported

37
forge/forgeflux/Cargo.toml
View file

@ -1,37 +0,0 @@
[package]
name = "forgeflux"
version = "0.1.0"
authors = ["realaravinth <realaravinth@batsense.net>"]
description = "ForgeFlux StarChart - Federated forge spider"
documentation = "https://forgeflux.org/"
edition = "2021"
license = "AGPLv3 or later version"
[lib]
name = "forgeflux"
path = "src/lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
async-trait = "0.1.51"
url = { version = "2.2.2", features = ["serde"] }
tokio = { version = "1.17", features = ["time"] }
[dependencies.forge-core]
path = "../forge-core"
[dependencies.reqwest]
features = ["rustls-tls-native-roots", "gzip", "deflate", "brotli", "json"]
version = "0.11.10"
[dependencies.serde]
features = ["derive"]
version = "1"
[dependencies.serde_json]
version = "1"
[dev-dependencies]
actix-rt = "2.7"

198
forge/forgeflux/src/lib.rs
View file

@ -1,198 +0,0 @@
/*
* ForgeFlux StarChart - A federated software forge spider
* Copyright © 2022 Aravinth Manivannan <realaravinth@batsense.net>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
use std::sync::Arc;
use std::time::Duration;
use reqwest::Client;
use tokio::task::JoinHandle;
use url::Url;
use db_core::ForgeImplementation;
use forge_core::dev::*;
use forge_core::Repository;
pub mod schema;
const REPO_SEARCH_PATH: &str = "/search/repositories";
const FORGEFLUX_NODEINFO: &str = "/nodeinfo/2.0";
const FORGEFLUX_IDENTIFIER: &str = "forgeflux";
#[derive(Clone)]
pub struct ForgeFlux {
pub instance_url: Url,
pub client: Client,
url: Url,
}
impl ForgeFlux {
pub fn new(instance_url: Url, client: Client) -> Self {
let url = Url::parse(&db_core::clean_url(&instance_url)).unwrap();
Self {
instance_url,
client,
url,
}
}
}
impl PartialEq for ForgeFlux {
fn eq(&self, other: &Self) -> bool {
self.url == other.url && self.instance_url == other.instance_url
}
}
#[async_trait]
impl SCForge for ForgeFlux {
async fn is_forge(&self) -> bool {
let u = self.instance_url.clone();
let mut node_info_url = self.instance_url.clone();
node_info_url.set_path(FORGEFLUX_NODEINFO);
let resp = self.client.get(node_info_url).send().await.unwrap();
if resp.status() == 200 {
let res: schema::Nodeinfo = resp.json().await.unwrap();
return res.software.name == FORGEFLUX_IDENTIFIER;
} else {
false
}
}
fn get_url(&self) -> &Url {
&self.url
}
fn forge_type(&self) -> ForgeImplementation {
ForgeImplementation::ForgeFlux
}
async fn crawl(&self, limit: u64, page: u64, rate_limit: u64) -> CrawlResp {
fn empty_is_none(s: &str) -> Option<String> {
let s = s.trim();
if s.is_empty() {
None
} else {
Some(s.to_owned())
}
}
let tags = Tags::default();
let mut users = UserMap::default();
let mut internal_users = UserMap::default();
let mut repos = Repositories::default();
let instance_url = self.instance_url.clone();
let mut url = instance_url.clone();
url.set_path(REPO_SEARCH_PATH);
url.set_query(Some(&format!("page={page}&limit={limit}")));
let mut res: Vec<schema::Repository> = self
.client
.get(url)
.send()
.await
.unwrap()
.json()
.await
.unwrap();
let mut sleep_fut: Option<JoinHandle<()>> = None;
for repo in res.drain(0..) {
let user = { //if internal_users.contains_key(&repo.attributed_to.to_string()) {
if let Some(sleep_fut) = sleep_fut {
sleep_fut.await.unwrap();
}
let user: schema::User = self
.client
.get(repo.attributed_to.clone())
.send()
.await
.unwrap()
.json()
.await
.unwrap();
sleep_fut = Some(tokio::spawn(tokio::time::sleep(Duration::new(
rate_limit, 0,
))));
let profile_photo = if let Some(profile_photo) = user.icon {
Some(profile_photo.url.to_string())
} else {
None
};
let f_user = Arc::new(User {
url: user.id.clone(),
username: Arc::new(user.preferred_username),
html_link: user.id.to_string(),
profile_photo,
});
users.insert(f_user.username.clone(), f_user.clone());
users.insert(Arc::new(f_user.url.to_string()), f_user.clone());
internal_users.insert(Arc::new(repo.attributed_to.to_string()), f_user.clone());
f_user
};
// } else {
// internal_users
// .get(&repo.attributed_to.to_string())
// .unwrap()
// .clone()
// };
let frepo = Repository {
url: repo.id.clone(),
website: None,
name: repo.name,
owner: user,
html_link: repo.id.to_string(),
tags: None,
description: Some(repo.summary),
};
repos.push(frepo);
}
CrawlResp { repos, tags, users }
}
}
#[cfg(test)]
mod tests {
use super::*;
use url::Url;
pub const NET_REPOSITORIES: u64 = 0;
pub const PER_CRAWL: u64 = 10;
#[actix_rt::test]
async fn forgeflux_works() {
let ctx = ForgeFlux::new(
Url::parse(&std::env::var("FORGEFLUX_HOST").unwrap()).unwrap(),
Client::new(),
);
assert!(ctx.is_forge().await);
let steps = NET_REPOSITORIES / PER_CRAWL;
for i in 0..steps {
let res = ctx.crawl(PER_CRAWL, i, 0).await;
assert_eq!(res.repos.len() as u64, PER_CRAWL);
}
}
}

79
forge/forgeflux/src/schema.rs
View file

@ -1,79 +0,0 @@
/*
* ForgeFlux StarChart - A federated software forge spider
* Copyright © 2usize22 Aravinth Manivannan <realaravinth@batsense.net>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
use serde::{Deserialize, Serialize};
use url::Url;
#[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct ProfilePhoto {
pub url: Url,
pub media_type: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct User {
pub name: String,
pub preferred_username: String,
pub id: Url,
pub url: Option<Url>,
pub icon: Option<ProfilePhoto>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Repository {
pub id: Url,
pub clone_uri: Url,
pub inbox: Url,
pub name: String,
pub attributed_to: Url,
pub summary: String,
}
#[derive(Debug, Clone, PartialEq, Hash, Eq, Serialize, Deserialize)]
pub struct Software {
pub name: String,
}
#[derive(Debug, Clone, PartialEq, Hash, Eq, Serialize, Deserialize)]
pub struct Nodeinfo {
pub software: Software,
}
#[cfg(test)]
mod tests {
use super::*;
use std::fs;
#[test]
/// Tests if Gitea responses panic when deserialized with serde into structs defined in this
/// module/file. Since Go doesn't have abilities to describe nullable values, I(@realaravinth)
/// am forced to do this as I my knowledge about Gitea codebase is very limited.
fn schema_doesnt_panic() {
let files = ["./tests/schema/forgeflux/net.forgeflux.org.json"];
for file in files.iter() {
let contents = fs::read_to_string(file).unwrap();
for line in contents.lines() {
let _: Vec<Repository> =
serde_json::from_str(line).expect("Forgeflux schema paniced");
}
}
}
}

1
forge/forgeflux/tests/schema/forgeflux/net.forgeflux.org.json
View file

File diff suppressed because one or more lines are too long

2
src/pages/auth/add.rs
View file

@ -105,7 +105,7 @@ mod tests {
use crate::errors::ServiceError; use crate::errors::ServiceError;
use crate::pages::auth::add::{AddChallenge, AddChallengePayload, ReadableError}; use crate::pages::auth::add::{AddChallenge, AddChallengePayload, ReadableError};
use crate::pages::errors::*; use crate::pages::errors::*;
use crate::settings::Settings;
#[actix_rt::test] #[actix_rt::test]
async fn add_page_works() { async fn add_page_works() {

6
src/pages/routes.rs
View file

@ -63,12 +63,6 @@ pub struct Auth {
pub verify: &'static str, pub verify: &'static str,
} }
impl Default for Auth {
fn default() -> Self {
Self::new()
}
}
impl Auth { impl Auth {
/// create new instance of Authentication route /// create new instance of Authentication route
pub const fn new() -> Auth { pub const fn new() -> Auth {

108
src/settings.rs
View file

@ -318,6 +318,60 @@ fn set_separator_field(mut s: ConfigBuilder<DefaultState>) -> ConfigBuilder<Defa
s s
} }
#[cfg(test)]
mod tests {
use super::*;
use crate::utils::get_random;
#[test]
fn root_dir_is_created_test() {
let dir;
loop {
let mut tmp = env::temp_dir();
tmp = tmp.join(get_random(10));
if tmp.exists() {
continue;
} else {
dir = tmp;
break;
}
}
let repo = Repository {
root: dir.to_str().unwrap().to_owned(),
};
repo.create_root_dir();
assert!(dir.exists());
assert!(dir.is_dir());
let file = dir.join("foo");
fs::write(&file, "foo").unwrap();
repo.create_root_dir();
assert!(dir.exists());
assert!(dir.is_dir());
assert!(file.exists());
assert!(file.is_file());
let repo = Repository {
root: file.to_str().unwrap().to_owned(),
};
repo.create_root_dir();
assert!(file.exists());
assert!(file.is_dir());
let mut license_path = Path::new(&repo.root).to_path_buf();
license_path.push(LICENSE_FILE);
assert!(license_path.exists());
assert!(license_path.is_file());
assert!(fs::read_to_string(license_path)
.unwrap()
.contains(CC0_LICENSE_TXT));
}
}
const CC0_LICENSE_TXT: &str = r#" const CC0_LICENSE_TXT: &str = r#"
Creative Commons Legal Code Creative Commons Legal Code
@ -443,57 +497,3 @@ express Statement of Purpose.
"#; "#;
const LICENSE_FILE: &str = "LICENSE.txt"; const LICENSE_FILE: &str = "LICENSE.txt";
#[cfg(test)]
mod tests {
use super::*;
use crate::utils::get_random;
#[test]
fn root_dir_is_created_test() {
let dir;
loop {
let mut tmp = env::temp_dir();
tmp = tmp.join(get_random(10));
if tmp.exists() {
continue;
} else {
dir = tmp;
break;
}
}
let repo = Repository {
root: dir.to_str().unwrap().to_owned(),
};
repo.create_root_dir();
assert!(dir.exists());
assert!(dir.is_dir());
let file = dir.join("foo");
fs::write(&file, "foo").unwrap();
repo.create_root_dir();
assert!(dir.exists());
assert!(dir.is_dir());
assert!(file.exists());
assert!(file.is_file());
let repo = Repository {
root: file.to_str().unwrap().to_owned(),
};
repo.create_root_dir();
assert!(file.exists());
assert!(file.is_dir());
let mut license_path = Path::new(&repo.root).to_path_buf();
license_path.push(LICENSE_FILE);
assert!(license_path.exists());
assert!(license_path.is_file());
assert!(fs::read_to_string(license_path)
.unwrap()
.contains(CC0_LICENSE_TXT));
}
}

16
src/spider.rs
View file

@ -25,7 +25,6 @@ use url::Url;
use db_core::prelude::*; use db_core::prelude::*;
use forge_core::prelude::*; use forge_core::prelude::*;
use forgeflux::ForgeFlux;
use gitea::Gitea; use gitea::Gitea;
use crate::ctx::Ctx; use crate::ctx::Ctx;
@ -37,17 +36,10 @@ impl Ctx {
pub async fn crawl(&self, instance_url: &Url, db: &BoxDB, federate: &ArcFederate) { pub async fn crawl(&self, instance_url: &Url, db: &BoxDB, federate: &ArcFederate) {
info!("[crawl][{instance_url}] Init crawling"); info!("[crawl][{instance_url}] Init crawling");
let forge: Box<dyn SCForge> = let forge: Box<dyn SCForge> =
Box::new(ForgeFlux::new(instance_url.clone(), self.client.clone())); Box::new(Gitea::new(instance_url.clone(), self.client.clone()));
let forge = if forge.is_forge().await { if !forge.is_forge().await {
forge unimplemented!("Forge type unimplemented");
} else { }
let gitea = Box::new(Gitea::new(instance_url.clone(), self.client.clone()));
if gitea.is_forge().await {
gitea
} else {
unimplemented!("Forge type unimplemented");
}
};
let mut page = 1; let mut page = 1;
let url = forge.get_url(); let url = forge.get_url();

5
src/tests.rs
View file

@ -17,6 +17,7 @@
*/ */
use std::env; use std::env;
pub use std::sync::Arc;
use crate::ctx::Ctx; use crate::ctx::Ctx;
pub use crate::db::BoxDB; pub use crate::db::BoxDB;
@ -57,10 +58,6 @@ pub mod sqlx_sqlite {
env::set_var("DATABASE_URL", &url); env::set_var("DATABASE_URL", &url);
println!("found db url: {url}"); println!("found db url: {url}");
let tmp_dir = Temp::new_dir().unwrap(); let tmp_dir = Temp::new_dir().unwrap();
let d = tmp_dir.as_path();
let _ = std::fs::remove_dir_all(d);
let _ = std::fs::create_dir(d);
env::set_var("STARCHART__REPOSITORY__ROOT", tmp_dir.to_str().unwrap()); env::set_var("STARCHART__REPOSITORY__ROOT", tmp_dir.to_str().unwrap());
let mut settings = Settings::new().unwrap(); let mut settings = Settings::new().unwrap();