diff --git a/.github/workflows/clippy-fmt.yml b/.github/workflows/clippy-fmt.yml deleted file mode 100644 index 325e539..0000000 --- a/.github/workflows/clippy-fmt.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Lint - -on: - pull_request: - types: [opened, synchronize, reopened] - push: - branches: - - master - -jobs: - fmt: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - components: rustfmt - - name: Check with rustfmt - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --all -- --check - - clippy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - components: clippy - override: true - - - name: Check with Clippy - uses: actions-rs/clippy-check@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - args: --workspace --tests --all-features diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml deleted file mode 100644 index 7190ede..0000000 --- a/.github/workflows/coverage.yml +++ /dev/null @@ -1,93 +0,0 @@ -name: Coverage - -on: - pull_request: - types: [opened, synchronize, reopened] - push: - branches: - - master - -jobs: - build_and_test: - strategy: - fail-fast: false - matrix: - version: - - stable - - # services: - # postgres: - # image: postgres - # env: - # POSTGRES_PASSWORD: password - # POSTGRES_USER: postgres - # POSTGRES_DB: postgres - # options: >- - # --health-cmd pg_isready - # --health-interval 10s - # --health-timeout 5s - # --health-retries 5 - # ports: - # - 5432:5432 - - name: ${{ matrix.version }} - x86_64-unknown-linux-gnu - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - # - name: ⚡ Cache - # uses: actions/cache@v3 - # with: - # path: | - # ~/.cargo/registry - # ~/.cargo/git - # target - # key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - name: Install ${{ matrix.version }} - uses: actions-rs/toolchain@v1 - with: - toolchain: ${{ matrix.version }}-x86_64-unknown-linux-gnu - profile: minimal - override: true - - - name: load env - run: | - mkdir -p db/db-sqlx-sqlite/tmp && - source .env-sample \ - && echo "POSTGRES_DATABASE_URL=$POSTGRES_DATABASE_URL" >> $GITHUB_ENV \ - && echo "SQLITE_DATABASE_URL=$SQLITE_DATABASE_URL" >> $GITHUB_ENV - - # usually run as part of `make test` but because this workflow doesn't run - # that command, `make dev-env` is used - - name: setup dev environment - run: make dev-env - env: - GIT_HASH: 8e77345f1597e40c2e266cb4e6dee74888918a61 # dummy value - POSTGRES_DATABASE_URL: "${{ env.POSTGRES_DATABASE_URL }}" - SQLITE_DATABASE_URL: "${{ env.SQLITE_DATABASE_URL }}" - - - name: run migrations - run: make migrate - env: - GIT_HASH: 8e77345f1597e40c2e266cb4e6dee74888918a61 # dummy value - POSTGRES_DATABASE_URL: "${{ env.POSTGRES_DATABASE_URL }}" - SQLITE_DATABASE_URL: "${{ env.SQLITE_DATABASE_URL }}" - - - name: Generate coverage file - if: matrix.version == 'stable' && (github.ref == 'refs/heads/master' || github.event_name == 'pull_request') - uses: actions-rs/tarpaulin@v0.1 - env: - # GIT_HASH is dummy value. I guess build.rs is skipped in tarpaulin - # execution so this value is required for preventing meta tests from - # panicking - GIT_HASH: 8e77345f1597e40c2e266cb4e6dee74888918a61 - POSTGRES_DATABASE_URL: "${{ env.POSTGRES_DATABASE_URL }}" - SQLITE_DATABASE_URL: "${{ env.SQLITE_DATABASE_URL }}" - with: - args: "--all-features --no-fail-fast --workspace=db/db-sqlx-sqlite,. -t 1200" - # args: "--all-features --no-fail-fast --workspace=database/db-sqlx-postgres,database/db-sqlx-sqlite,. -t 1200" - - - name: Upload to Codecov - if: matrix.version == 'stable' && (github.ref == 'refs/heads/master' || github.event_name == 'pull_request') - uses: codecov/codecov-action@v2 diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml deleted file mode 100644 index b0becbd..0000000 --- a/.github/workflows/linux.yml +++ /dev/null @@ -1,128 +0,0 @@ -name: Build - -on: - pull_request: - types: [opened, synchronize, reopened] - push: - branches: - - master - -jobs: - build_and_test: - strategy: - fail-fast: false - matrix: - version: - - stable - # - nightly - - name: ${{ matrix.version }} - x86_64-unknown-linux-gnu - runs-on: - ubuntu-latest - - # services: - # postgres: - # image: postgres - # env: - # POSTGRES_PASSWORD: password - # POSTGRES_USER: postgres - # POSTGRES_DB: postgres - # options: >- - # --health-cmd pg_isready - # --health-interval 10s - # --health-timeout 5s - # --health-retries 5 - # ports: - # - 5432:5432 - # - steps: - - uses: actions/checkout@v2 - - # - name: ⚡ Cache - # uses: actions/cache@v3 - # with: - # path: | - # /var/lib/docker - # ~/.cargo/registry - # ~/.cargo/git - # target - # key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - - name: Cache - uses: Swatinem/rust-cache@v1 - - - name: Login to DockerHub - if: (github.ref == 'refs/heads/master' || github.event_name == 'push') && github.repository == 'forgeflux-org/starchart' - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Install ${{ matrix.version }} - uses: actions-rs/toolchain@v1 - with: - toolchain: ${{ matrix.version }}-x86_64-unknown-linux-gnu - profile: minimal - override: true - - - name: load env - run: | - mkdir -p db/db-sqlx-sqlite/tmp && - source .env-sample \ - && echo "POSTGRES_DATABASE_URL=$POSTGRES_DATABASE_URL" >> $GITHUB_ENV \ - && echo "SQLITE_DATABASE_URL=$SQLITE_DATABASE_URL" >> $GITHUB_ENV - - - name: run migrations - run: make migrate - env: - GIT_HASH: 8e77345f1597e40c2e266cb4e6dee74888918a61 # dummy value - POSTGRES_DATABASE_URL: "${{ env.POSTGRES_DATABASE_URL }}" - SQLITE_DATABASE_URL: "${{ env.SQLITE_DATABASE_URL }}" - - - name: build - run: - make - env: - POSTGRES_DATABASE_URL: "${{ env.POSTGRES_DATABASE_URL }}" - SQLITE_DATABASE_URL: "${{ env.SQLITE_DATABASE_URL }}" - - - name: build docker images - if: matrix.version == 'stable' - run: make docker - - - name: publish docker images - if: matrix.version == 'stable' && (github.ref == 'refs/heads/master' || github.event_name == 'push') && github.repository == 'forgeflux-org/starchart' - run: make docker-publish - - - name: run tests - timeout-minutes: 40 - run: - make test - env: - GIT_HASH: 8e77345f1597e40c2e266cb4e6dee74888918a61 # dummy value - POSTGRES_DATABASE_URL: "${{ env.POSTGRES_DATABASE_URL }}" - SQLITE_DATABASE_URL: "${{ env.SQLITE_DATABASE_URL }}" - - - name: generate documentation - if: matrix.version == 'stable' && (github.ref == 'refs/heads/master' || github.event_name == 'push') && github.repository == 'forgeflux-org/starchart' - run: - make doc - env: - GIT_HASH: 8e77345f1597e40c2e266cb4e6dee74888918a61 # dummy value - POSTGRES_DATABASE_URL: "${{ env.POSTGRES_DATABASE_URL }}" - SQLITE_DATABASE_URL: "${{ env.SQLITE_DATABASE_URL }}" - - - name: Deploy to GitHub Pages - if: matrix.version == 'stable' && (github.ref == 'refs/heads/master' || github.event_name == 'push') && github.repository == 'forgeflux-org/starchart' - uses: JamesIves/github-pages-deploy-action@3.7.1 - with: - branch: gh-pages - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - FOLDER: - ./target/doc/ - - # - name: deploy - # if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'realaravinth/realaravinth' }} - # run: >- - # curl --location --request POST "https://deploy.batsense.net/api/v1/update" --header 'Content-Type: application/json' --data-raw "{ \"secret\": \"${{ secrets.DEPLOY_TOKEN }}\", \"branch\": \"gh-pages\" }" diff --git a/.woodpecker.yml b/.woodpecker.yml new file mode 100644 index 0000000..4550d20 --- /dev/null +++ b/.woodpecker.yml @@ -0,0 +1,57 @@ +steps: + setup: + image: python + when: + event: [push, pull_request, tag, deployment] + environment: + POSTGRES_DATABASE_URL: postgres://postgres:password@database:5432/postgres + commands: + - pip install requests + - sed -i 's/localhost/forgejo/' scripts/gitea.py + - python ./scripts/gitea.py + + + test: + image: rust + when: + event: [push, pull_request, tag, deployment] + environment: + POSTGRES_DATABASE_URL: postgres://postgres:password@database:5432/postgres + SQLITE_DATABASE_URL: sqlite:///tmp/admin.db + FORGEJO_HOST: http://forgejo:3000 + commands: + # - curl -fsSL https://deb.nodesource.com/setup_16.x | bash - &&\ + # - apt update && apt-get -y --no-install-recommends install nodejs tar gpg curl wget + # - rustup toolchain install nightly + # - rustup override set nightly + - rustup component add rustfmt + - rustup component add clippy + # - export FORGEFLUX_server_COOKIE_SECRET=d05604b56ffd0f57200be49313a9e51436ca8f1678b9ffdf8d2602d5e9839f05 + - make migrate + - make + # - make lint + - make test.workspaces + + + build_docker_img: + image: plugins/docker + when: + event: [pull_request] + settings: + dry_run: true + repo: forgeflux/starchart + tags: latest + +services: + forgejo: + image: codeberg.org/forgejo/forgejo:9 + environment: + FORGEJO__security__INSTALL_LOCK: true + FORGEJO__federation__ENABLED: true + FORGEJO__server__ROOT_URL: http://forgejo + FORGEJO__server__HTTP_PORT: 3000 + + database: + image: postgres + environment: + POSTGRES_PASSWORD: password diff --git a/Makefile b/Makefile index 7290d06..9e89594 100644 --- a/Makefile +++ b/Makefile @@ -131,6 +131,10 @@ test: migrate ## Run tests $(call cache_bust) $(call test_workspaces) +test.workspaces: + $(call cache_bust) + $(call test_workspaces) + # cd database/db-sqlx-postgres &&\ # DATABASE_URL=${POSTGRES_DATABASE_URL}\ # cargo test --no-fail-fast diff --git a/docker-compose-dev-deps.yml b/docker-compose-dev-deps.yml index e140eab..d5bf491 100644 --- a/docker-compose-dev-deps.yml +++ b/docker-compose-dev-deps.yml @@ -5,19 +5,12 @@ networks: external: false services: - server: - image: gitea/gitea:1.16.5 - container_name: gitea + forgejo: + image: codeberg.org/forgejo/forgejo:9 environment: - - USER_UID=1000 - - USER_GID=1000 - restart: always - networks: - - gitea - volumes: - - ./tmp/gitea:/data - - /etc/timezone:/etc/timezone:ro - - /etc/localtime:/etc/localtime:ro + - FORGEJO__security__INSTALL_LOCK=true + - FORGEJO__federation__ENABLED=true + - FORGEJO__server__ROOT_URL=http://localhost + - FORGEJO__server__HTTP_PORT=3000 ports: - - "8080:3000" - - "2221:22" + - "3000:3000" diff --git a/forge/gitea/src/lib.rs b/forge/gitea/src/lib.rs index b179328..13ddc83 100644 --- a/forge/gitea/src/lib.rs +++ b/forge/gitea/src/lib.rs @@ -31,6 +31,7 @@ pub mod schema; const REPO_SEARCH_PATH: &str = "/api/v1/repos/search"; const GITEA_NODEINFO: &str = "/api/v1/nodeinfo"; const GITEA_IDENTIFIER: &str = "gitea"; +const FORGEJO_IDENTIFIER: &str = "forgejo"; #[derive(Clone)] pub struct Gitea { @@ -177,13 +178,13 @@ mod tests { use super::*; use url::Url; - pub const GITEA_HOST: &str = "http://localhost:8080"; pub const NET_REPOSITORIES: u64 = 100; pub const PER_CRAWL: u64 = 10; #[actix_rt::test] async fn gitea_works() { - let ctx = Gitea::new(Url::parse(GITEA_HOST).unwrap(), Client::new()); + let ctx = Gitea::new(Url::parse(&std::env::var("FORGEJO_HOST").unwrap()).unwrap(), Client::new()); + assert!(ctx.is_forge().await); let steps = NET_REPOSITORIES / PER_CRAWL; diff --git a/scripts/gitea.py b/scripts/gitea.py index 5f4e827..e6d07af 100755 --- a/scripts/gitea.py +++ b/scripts/gitea.py @@ -20,7 +20,7 @@ def check_online(): while True: try: res = requests.get( - "http://localhost:8080/api/v1/nodeinfo", allow_redirects=False + "http://localhost:3000/api/v1/nodeinfo", allow_redirects=False ) if any([res.status_code == 302, res.status_code == 200]): break @@ -30,46 +30,6 @@ def check_online(): count += 1 continue - -def install(): - INSTALL_PAYLOAD = { - "db_type": "sqlite3", - "db_host": "localhost:3306", - "db_user": "root", - "db_passwd": "", - "db_name": "gitea", - "ssl_mode": "disable", - "db_schema": "", - "charset": "utf8", - "db_path": "/data/gitea/gitea.db", - "app_name": "Gitea:+Git+with+a+cup+of+tea", - "repo_root_path": "/data/git/repositories", - "lfs_root_path": "/data/git/lfs", - "run_user": "git", - "domain": "localhost", - "ssh_port": "2221", - "http_port": "3000", - "app_url": "http://localhost:8080/", - "log_root_path": "/data/gitea/log", - "smtp_host": "", - "smtp_from": "", - "smtp_user": "", - "smtp_passwd": "", - "enable_federated_avatar": "on", - "enable_open_id_sign_in": "on", - "enable_open_id_sign_up": "on", - "default_allow_create_organization": "on", - "default_enable_timetracking": "on", - "no_reply_address": "noreply.localhost", - "password_algorithm": "pbkdf2", - "admin_name": "", - "admin_passwd": "", - "admin_confirm_passwd": "", - "admin_email": "", - } - requests.post(f"http://localhost:8080", data=INSTALL_PAYLOAD) - - class ParseCSRFGiteaForm(HTMLParser): token: str = None @@ -110,7 +70,7 @@ class HTMLClient: def get_csrf_token(self, url: str) -> str: resp = self.session.get(url, allow_redirects=False) - if resp.status_code != 200 and resp.status_code != 302: + if resp.status_code != 200 and resp.status_code != 302 and resp.status_code != 303: print(resp.status_code, resp.text) raise Exception(f"Can't get csrf token: {resp.status_code}") csrf = self.__get_csrf_token(resp.text) @@ -118,7 +78,7 @@ class HTMLClient: def register(client: HTMLClient): - url = "http://localhost:8080/user/sign_up" + url = "http://localhost:3000/user/sign_up" csrf = client.get_csrf_token(url) payload = { "_csrf": csrf, @@ -131,7 +91,7 @@ def register(client: HTMLClient): def login(client: HTMLClient): - url = "http://localhost:8080/user/login" + url = "http://localhost:3000/user/login" csrf = client.get_csrf_token(url) payload = { "_csrf": csrf, @@ -141,7 +101,7 @@ def login(client: HTMLClient): } resp = client.session.post(url, data=payload, allow_redirects=False) print(f"login {client.session.cookies}") - if resp.status_code == 302: + if resp.status_code == 303: print("User logged in") return @@ -167,7 +127,7 @@ def create_repositories(client: HTMLClient): } return data - url = "http://localhost:8080/repo/create" + url = "http://localhost:3000/repo/create" for repo in REPOS: csrf = client.get_csrf_token(url) resp = client.session.post(url, data=get_repository_payload(csrf, repo)) @@ -182,7 +142,7 @@ def create_repositories(client: HTMLClient): def add_tag(repo: str, client: HTMLClient): print("adding tags") tag = "testing" - url = f"http://{GITEA_USER}:{GITEA_PASSWORD}@localhost:8080/api/v1/repos/{GITEA_USER}/{repo}/topics/{tag}" + url = f"http://{GITEA_USER}:{GITEA_PASSWORD}@localhost:3000/api/v1/repos/{GITEA_USER}/{repo}/topics/{tag}" resp = requests.put(url) if resp.status_code != 204: print(f"Error while adding tags repository: {repo} {resp.status_code}") @@ -196,8 +156,6 @@ if __name__ == "__main__": REPOS.append(f"repository_{i}") check_online() print("Instance online") - install() - print("Instance configured and installed") client = HTMLClient() count = 0 while True: diff --git a/src/spider.rs b/src/spider.rs index 419bdfb..6857e79 100644 --- a/src/spider.rs +++ b/src/spider.rs @@ -229,13 +229,14 @@ mod tests { use url::Url; - pub const GITEA_HOST: &str = "http://localhost:8080"; pub const GITEA_USERNAME: &str = "bot"; #[actix_rt::test] async fn crawl_gitea() { let (db, ctx, federate, _tmp_dir) = sqlx_sqlite::get_ctx().await; - let url = Url::parse(GITEA_HOST).unwrap(); + let url = Url::parse( + &std::env::var("FORGEJO_HOST").unwrap() + ).unwrap(); ctx.crawl(&url, &db, &federate).await; // let hostname = get_hostname(&Url::parse(GITEA_HOST).unwrap()); assert!(db.forge_exists(&url).await.unwrap());