From 6c83545d0ec44e2ba6aaa1bf83921e437dc8b4ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Dachary?= Date: Sat, 5 Nov 2022 00:00:07 +0100 Subject: [PATCH] implementation: Woodpecker based CI Refs: https://codeberg.org/forgejo/forgejo/issues/73 Refs: https://codeberg.org/forgejo/forgejo/issues/101 Refs: https://codeberg.org/woodpecker-plugins/plugin-docker-buildx/issues/51 Refs: https://codeberg.org/forgejo/forgejo/issues/117 Refs: https://codeberg.org/forgejo/forgejo/issues/25 Refs: https://codeberg.org/forgejo/forgejo/issues/154 --- .woodpecker/compliance.yml | 74 +++++++++ .woodpecker/docker-release-version.yml | 80 ++++++++++ .woodpecker/release-version.yml | 111 ++++++++++++++ .woodpecker/releases-helper.yml | 24 +++ .woodpecker/testing-amd64.yml | 141 ++++++++++++++++++ Makefile | 2 +- releases/Dockerfile | 3 + releases/Dockerfile-rootless | 3 + .../container-images-pull-verify-push-test.sh | 74 +++++++++ releases/container-images-pull-verify-push.sh | 115 ++++++++++++++ 10 files changed, 626 insertions(+), 1 deletion(-) create mode 100644 .woodpecker/compliance.yml create mode 100644 .woodpecker/docker-release-version.yml create mode 100644 .woodpecker/release-version.yml create mode 100644 .woodpecker/releases-helper.yml create mode 100644 .woodpecker/testing-amd64.yml create mode 100644 releases/Dockerfile create mode 100644 releases/Dockerfile-rootless create mode 100755 releases/container-images-pull-verify-push-test.sh create mode 100755 releases/container-images-pull-verify-push.sh diff --git a/.woodpecker/compliance.yml b/.woodpecker/compliance.yml new file mode 100644 index 000000000..1872ecc57 --- /dev/null +++ b/.woodpecker/compliance.yml @@ -0,0 +1,74 @@ +platform: linux/amd64 + +branches: + exclude: [ forgejo-development, main, release/*, soft-fork/*/*, soft-fork/*/*/* ] + +variables: + - &golang_image 'golang:1.19' + - &gitea_test_image 'gitea/test_env:linux-amd64' + - &goproxy_override '' + - &goproxy_setup |- + if [ -n "$${GOPROXY_OVERRIDE:-}" ]; then + export GOPROXY="$${GOPROXY_OVERRIDE}"; + echo "Using goproxy from goproxy_override \"$${GOPROXY}\""; + elif [ -n "$${GOPROXY_DEFAULT:-}" ]; then + export GOPROXY="$${GOPROXY_DEFAULT}"; + echo "Using goproxy from goproxy_default (secret) not displaying"; + else + export GOPROXY="https://proxy.golang.org,direct"; + echo "No goproxy overrides or defaults given, using \"$${GOPROXY}\""; + fi + +workspace: + base: /go + path: src/codeberg/gitea + +pipeline: + deps-backend: + image: *golang_image + pull: true + environment: + GOPROXY_OVERRIDE: *goproxy_override + secrets: + - goproxy_default + commands: + - *goproxy_setup + - make deps-backend + + security-check: + image: *golang_image + group: checks + pull: true + environment: + GOPROXY_OVERRIDE: *goproxy_override + secrets: + - goproxy_default + commands: + - *goproxy_setup + - make security-check + + lint-backend: + image: *gitea_test_image + group: checks + pull: true + environment: + GOPROXY_OVERRIDE: *goproxy_override + TAGS: 'bindata sqlite sqlite_unlock_notify' + GOSUMDB: 'sum.golang.org' + secrets: + - goproxy_default + commands: + - *goproxy_setup + - make lint-backend + + checks-backend: + image: *gitea_test_image + pull: true + group: checks + environment: + GOPROXY_OVERRIDE: *goproxy_override + secrets: + - goproxy_default + commands: + - *goproxy_setup + - make --always-make checks-backend diff --git a/.woodpecker/docker-release-version.yml b/.woodpecker/docker-release-version.yml new file mode 100644 index 000000000..73d987415 --- /dev/null +++ b/.woodpecker/docker-release-version.yml @@ -0,0 +1,80 @@ +platform: linux/amd64 + +when: + event: tag + +depends_on: +- testing-amd64 + +variables: + - &git_image 'docker:git' + - &dind_image 'docker:20.10-dind' + - &buildx_image 'woodpeckerci/plugin-docker-buildx:2.0.0' + - &integration_image 'codeberg.org/forgejo-integration/forgejo' + - &dockerfile_root 'Dockerfile' +# for testing purposes +# - &dockerfile_root 'releases/Dockerfile' + - &dockerfile_rootless 'Dockerfile.rootless' +# for testing purposes +# - &dockerfile_rootless 'releases/Dockerfile-rootless' + - &verify 'true' +# for testing purposes +# - &verify 'false' + - &archs 'amd64 arm64' + +pipeline: + fetch-tags: + image: *git_image + pull: true + commands: + - git config --add safe.directory '*' + - git fetch --tags --force + + build-root: + image: *buildx_image + group: integration + pull: true + settings: + platforms: linux/amd64,linux/arm64 + dockerfile: *dockerfile_root + registry: + from_secret: domain + tag: ${CI_COMMIT_TAG##v} + repo: *integration_image + build_args: + - GOPROXY=https://proxy.golang.org + password: + from_secret: releaseteamtoken + username: + from_secret: releaseteamuser + + build-rootless: + image: *buildx_image + group: integration + pull: true + settings: + platforms: linux/amd64,linux/arm64 + dockerfile: *dockerfile_rootless + registry: + from_secret: domain + tag: ${CI_COMMIT_TAG##v}-rootless + repo: *integration_image + build_args: + - GOPROXY=https://proxy.golang.org + password: + from_secret: releaseteamtoken + username: + from_secret: releaseteamuser + + publish: + image: *dind_image + environment: + INTEGRATION_IMAGE: *integration_image + VERIFY: *verify + ARCHS: *archs + commands: + - ./releases/container-images-pull-verify-push.sh + secrets: + - releaseteamtoken + - releaseteamuser + - domain diff --git a/.woodpecker/release-version.yml b/.woodpecker/release-version.yml new file mode 100644 index 000000000..7186ef30e --- /dev/null +++ b/.woodpecker/release-version.yml @@ -0,0 +1,111 @@ +platform: linux/amd64 + +when: + event: tag + +depends_on: +- testing-amd64 + +variables: + - &git_image 'docker:git' + - &node_image 'node:18' + - &golang_image 'golang:1.19' + - &gpg_sign_image 'plugins/gpgsign:1' + - &xgo_image 'techknowlogick/xgo:go-1.19.x' + - &gpg_sign_image 'plugins/gpgsign:1' + - &goproxy_override '' + - &goproxy_setup |- + if [ -n "$${GOPROXY_OVERRIDE:-}" ]; then + export GOPROXY="$${GOPROXY_OVERRIDE}"; + echo "Using goproxy from goproxy_override \"$${GOPROXY}\""; + elif [ -n "$${GOPROXY_DEFAULT:-}" ]; then + export GOPROXY="$${GOPROXY_DEFAULT}"; + echo "Using goproxy from goproxy_default (secret) not displaying"; + else + export GOPROXY="https://proxy.golang.org,direct"; + echo "No goproxy overrides or defaults given, using \"$${GOPROXY}\""; + fi + +workspace: + base: /source + path: / + +pipeline: + fetch-tags: + image: *git_image + pull: true + group: deps + commands: + - git config --add safe.directory '*' + - git fetch --tags --force + + deps-frontend: + image: *node_image + pull: true + group: deps + commands: + - make deps-frontend + + deps-backend: + image: *golang_image + pull: true + group: deps + environment: + GOPROXY_OVERRIDE: *goproxy_override + secrets: + - goproxy_default + commands: + - *goproxy_setup + - make deps-backend + + static: + image: *xgo_image + pull: true + commands: + - *goproxy_setup + - curl -sL https://deb.nodesource.com/setup_16.x | bash - && apt-get -qqy install nodejs + - export PATH=$PATH:$GOPATH/bin + - make CI=true LINUX_ARCHS=linux/amd64,linux/arm64,linux/arm-6 release + environment: + TAGS: 'bindata sqlite sqlite_unlock_notify' + DEBIAN_FRONTEND: 'noninteractive' + GOPROXY_OVERRIDE: *goproxy_override + secrets: + - goproxy_default + + verifyruns: + image: *golang_image + commands: + - ./dist/release/forgejo-*-amd64 --version | grep 'built with' + - apt-get update + - apt-get install -y qemu-user-static + - /usr/bin/qemu-aarch64-static ./dist/release/forgejo-*-arm64 --version | grep 'built with' + - /usr/bin/qemu-arm-static ./dist/release/forgejo-*-arm-6 --version | grep 'built with' + + gpg-sign: + image: *gpg_sign_image + pull: true + settings: + detach_sign: true + excludes: + - "dist/release/*.sha256" + files: + - "dist/release/*" + key: + from_secret: releaseteamgpg + + release: + image: *golang_image + commands: + - curl -sL https://dl.gitea.io/tea/0.9.0/tea-0.9.0-linux-amd64 > /bin/tea && chmod +x /bin/tea + - REMOTE=$(echo $CI_REPO_LINK | sed -e 's|.*://||' -e 's|/.*||') + - GITEA_SERVER_URL=$CI_REPO_LINK GITEA_SERVER_TOKEN=$RELEASETEAMTOKEN tea login add --name $RELEASETEAMUSER --url $REMOTE + - ASSETS=$(ls dist/release/* | sed -e 's/^/-a /') + - echo "$${CI_COMMIT_TAG##v}" | grep -qi '\-rc' && export RELEASETYPE="--prerelease" && echo "Uploading as Pre-Release" + - echo "$${CI_COMMIT_TAG##v}" | grep -qi '\-test' && export RELEASETYPE="--draft" && echo "Uploading as Draft" + - test $${RELEASETYPE+false} || echo "Uploading as Stable" + - anchor=$(echo $CI_COMMIT_TAG | sed -e 's/^v//' -e 's/[^a-zA-Z0-9]/-/g') + - tea release create $ASSETS --note "See https://codeberg.org/forgejo/forgejo/src/branch/forgejo/RELEASE-NOTES.md#$${anchor}" --tag $CI_COMMIT_TAG --title $CI_COMMIT_TAG $${RELEASETYPE} + secrets: + - releaseteamtoken + - releaseteamuser diff --git a/.woodpecker/releases-helper.yml b/.woodpecker/releases-helper.yml new file mode 100644 index 000000000..c22bf23ad --- /dev/null +++ b/.woodpecker/releases-helper.yml @@ -0,0 +1,24 @@ +platform: linux/amd64 + +branches: + includes: [ wip-ci-* ] + +when: + event: push + +variables: + - &dind_image 'docker:20.10-dind' + +pipeline: + container-images-pull-verify-push: + image: *dind_image + commands: +# arm64 would require qemu-user-static which is not available on alpline +# the test coverage does not change much and running the tests test locally +# is possible if there is a doubt + - ARCHS=amd64 ./releases/container-images-pull-verify-push-test.sh test + - ./releases/container-images-pull-verify-push-test.sh test_teardown + secrets: + - releaseteamuser + - releaseteamtoken + - domain diff --git a/.woodpecker/testing-amd64.yml b/.woodpecker/testing-amd64.yml new file mode 100644 index 000000000..82a99fb6c --- /dev/null +++ b/.woodpecker/testing-amd64.yml @@ -0,0 +1,141 @@ +platform: linux/amd64 + +branches: + exclude: [ forgejo-development, main, release/*, soft-fork/*/*, soft-fork/*/*/* ] + +depends_on: +- compliance + +variables: + - &git_image 'docker:git' + - &golang_image 'golang:1.19' + - &gitea_test_image 'gitea/test_env:linux-amd64' + - &mysql_image 'mysql:8' + - &pgsql_image 'postgres:10' + - &goproxy_override '' + - &goproxy_setup |- + if [ -n "$${GOPROXY_OVERRIDE:-}" ]; then + export GOPROXY="$${GOPROXY_OVERRIDE}"; + echo "Using goproxy from goproxy_override \"$${GOPROXY}\""; + elif [ -n "$${GOPROXY_DEFAULT:-}" ]; then + export GOPROXY="$${GOPROXY_DEFAULT}"; + echo "Using goproxy from goproxy_default (secret) not displaying"; + else + export GOPROXY="https://proxy.golang.org,direct"; + echo "No goproxy overrides or defaults given, using \"$${GOPROXY}\""; + fi + +services: + mysql8: + image: *mysql_image + pull: true + environment: + MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_DATABASE: testgitea + + pgsql: + image: *pgsql_image + pull: true + environment: + POSTGRES_DB: test + POSTGRES_PASSWORD: postgres + +workspace: + base: /go + path: src/codeberg/gitea + +pipeline: + fetch-tags: + image: *git_image + pull: true + commands: + - git config --add safe.directory '*' + - git fetch --tags --force + + deps-backend: + image: *golang_image + pull: true + environment: + GOPROXY_OVERRIDE: *goproxy_override + secrets: + - goproxy_default + commands: + - *goproxy_setup + - make deps-backend + + tag-pre-condition: + image: *git_image + pull: true + commands: + - git update-ref refs/heads/tag_test ${CI_COMMIT_SHA} + + prepare-test-env: + image: *gitea_test_image + pull: true + commands: + - ./build/test-env-prepare.sh + + build: + image: *gitea_test_image + environment: + GOSUMDB: sum.golang.org + TAGS: bindata sqlite sqlite_unlock_notify + GOPROXY_OVERRIDE: *goproxy_override + secrets: + - goproxy_default + commands: + - *goproxy_setup + - su gitea -c './build/test-env-check.sh' + - su gitea -c 'make backend' + + unit-test: + image: *gitea_test_image + environment: + TAGS: 'bindata sqlite sqlite_unlock_notify' + RACE_ENABLED: 'true' + GOPROXY_OVERRIDE: *goproxy_override + secrets: + - github_read_token + - goproxy_default + commands: + - *goproxy_setup + - su gitea -c 'make unit-test-coverage test-check' + + test-mysql8: + group: integration + image: *gitea_test_image + commands: + - *goproxy_setup + - su gitea -c 'timeout -s ABRT 50m make test-mysql8-migration test-mysql8' + environment: + TAGS: 'bindata' + RACE_ENABLED: 'true' + USE_REPO_TEST_DIR: '1' + GOPROXY_OVERRIDE: *goproxy_override + secrets: + - goproxy_default + + test-pgsql: + group: integration + image: *gitea_test_image + commands: + - *goproxy_setup + - su gitea -c 'timeout -s ABRT 50m make test-pgsql-migration test-pgsql' + environment: + TAGS: 'bindata' + RACE_ENABLED: 'true' + USE_REPO_TEST_DIR: '1' + GOPROXY_OVERRIDE: *goproxy_override + secrets: + - goproxy_default + + test-sqlite: + group: integration + image: *gitea_test_image + environment: + - USE_REPO_TEST_DIR=1 + - GOPROXY=off + - TAGS=bindata gogit sqlite sqlite_unlock_notify + - TEST_TAGS=bindata gogit sqlite sqlite_unlock_notify + commands: + - su gitea -c 'timeout -s ABRT 120m make test-sqlite-migration test-sqlite' diff --git a/Makefile b/Makefile index 5531bf192..b6ec9a580 100644 --- a/Makefile +++ b/Makefile @@ -751,7 +751,7 @@ $(EXECUTABLE): $(GO_SOURCES) $(TAGS_PREREQ) CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) build $(GOFLAGS) $(EXTRA_GOFLAGS) -tags '$(TAGS)' -ldflags '-s -w $(LDFLAGS)' -o $@ .PHONY: release -release: frontend generate release-windows release-linux release-darwin release-copy release-compress vendor release-sources release-docs release-check +release: frontend generate release-linux release-copy release-compress vendor release-sources release-check $(DIST_DIRS): mkdir -p $(DIST_DIRS) diff --git a/releases/Dockerfile b/releases/Dockerfile new file mode 100644 index 000000000..bef4e4f6d --- /dev/null +++ b/releases/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.17 + +RUN echo root > state diff --git a/releases/Dockerfile-rootless b/releases/Dockerfile-rootless new file mode 100644 index 000000000..561b67e9a --- /dev/null +++ b/releases/Dockerfile-rootless @@ -0,0 +1,3 @@ +FROM alpine:3.17 + +RUN echo rootless > state diff --git a/releases/container-images-pull-verify-push-test.sh b/releases/container-images-pull-verify-push-test.sh new file mode 100755 index 000000000..e492c9768 --- /dev/null +++ b/releases/container-images-pull-verify-push-test.sh @@ -0,0 +1,74 @@ +#!/bin/sh + +# +# Tests are run when on a wip-ci-* branch, see .woodpecker/releases-helper.yml +# It should be changed to run it every time this file is changed when 1.18 is used because 1.17 does not have +# webhooks with the information for that filtering. +# + +set -ex + +image_delete() { + curl -sS -H "Authorization: token $token" -X DELETE https://$DOMAIN/v2/$1/forgejo/manifests/$2 +} + +# +# Create the same set of images that buildx would +# +test_setup() { + dir=$(dirname $0) + + for suffix in '' '-rootless' ; do + ( + cd $dir + manifests="" + for arch in $ARCHS ; do + image=$(arch_image_name $INTEGRATION_USER $arch $suffix) + docker build -f Dockerfile$suffix --platform linux/$arch -t $image . + docker push $image + images="$images $image" + done + manifest=$(image_name $INTEGRATION_USER $suffix) + docker manifest rm $manifest || true + docker manifest create $manifest $images + image_put $INTEGRATION_USER $(image_tag $suffix) $manifest + ) + done +} + +test_teardown() { + authenticate + for suffix in '' '-rootless' ; do + image_delete $INTEGRATION_USER $(image_tag $suffix) + image_delete $CI_REPO_OWNER $(image_tag $suffix) + image_delete $CI_REPO_OWNER $(short_image_tag $suffix) + for arch in $ARCHS ; do + image_delete $INTEGRATION_USER $(arch_image_tag $arch $suffix) + image_delete $CI_REPO_OWNER $(arch_image_tag $arch $suffix) + done + done +} + +# +# Running the test locally instead of withing Woodpecker +# +# 1. Setup: obtain a token at https://codeberg.org/user/settings/applications +# 2. Run: RELEASETEAMUSER= RELEASETEAMTOKEn= container-images-pull-verify-push.sh test +# 3. Verify: (optional) manual verification at https://codeberg.org//-/packages/container/forgejo/versions +# 4. Cleanup: RELEASETEAMUSER= RELEASETEAMTOKEn= container-images-pull-verify-push.sh test_teardown +# +test() { + boot + test_teardown + test_setup + VERIFY_STRING=something + VERIFY_COMMAND="echo $VERIFY_STRING" + echo "================================ TEST BEGIN" + main + echo "================================ TEST END" +} + +: ${CI_REPO_OWNER:=dachary} +: ${CI_COMMIT_TAG:=v17.1.42-2} + +. $(dirname $0)/container-images-pull-verify-push.sh diff --git a/releases/container-images-pull-verify-push.sh b/releases/container-images-pull-verify-push.sh new file mode 100755 index 000000000..69bc2cf68 --- /dev/null +++ b/releases/container-images-pull-verify-push.sh @@ -0,0 +1,115 @@ +#!/bin/sh + +set -ex + +: ${DOCKER_HOST:=unix:///var/run/docker.sock} +: ${ARCHS:=amd64 arm64} +: ${INTEGRATION_USER:=forgejo-integration} +: ${INTEGRATION_IMAGE:=codeberg.org/$INTEGRATION_USER/forgejo} +: ${TAG:=${CI_COMMIT_TAG##v}} +: ${SHORT_TAG=${TAG%.*-*}} +: ${DOMAIN:=codeberg.org} + +: ${VERIFY:=true} +VERIFY_COMMAND='gitea --version' +VERIFY_STRING='built with' + +publish() { + for suffix in '' '-rootless' ; do + images="" + for arch in $ARCHS ; do + # + # Get the image from the integration user + # + image=$(image_name $INTEGRATION_USER $suffix) + docker pull --platform linux/$arch $image + # + # Verify it is usable + # + if $VERIFY ; then + docker run --platform linux/$arch --rm $image $VERIFY_COMMAND | grep "$VERIFY_STRING" + fi + # + # Push the image with a tag reflecting the architecture to the repo owner + # + arch_image=$(arch_image_name $CI_REPO_OWNER $arch $suffix) + docker tag $image $arch_image + docker push $arch_image + images="$images $arch_image" + done + + # + # Push a manifest with all the architectures to the repo owner + # + manifest=$(image_name $CI_REPO_OWNER $suffix) + docker manifest rm $manifest || true + docker manifest create $manifest $images + image_put $CI_REPO_OWNER $(image_tag $suffix) $manifest + image_put $CI_REPO_OWNER $(short_image_tag $suffix) $manifest + # + # Sanity check to ensure the manifest that are published can actualy + # be used. + # + for arch in $ARCHS ; do + docker pull --platform linux/$arch $(image_name $CI_REPO_OWNER $suffix) + docker pull --platform linux/$arch $(short_image_name $CI_REPO_OWNER $suffix) + done + done +} + +boot() { + if docker version ; then + return + fi + apk --update --no-cache add coredns jq curl + ( echo ".:53 {" ; echo " forward . /etc/resolv.conf"; echo "}" ) > /etc/coredns/Corefile + coredns -conf /etc/coredns/Corefile & + /usr/local/bin/dockerd --data-root /var/lib/docker --host=$DOCKER_HOST --dns 172.17.0.3 & + for i in $(seq 60) ; do + docker version && break + sleep 1 + done + docker version || exit 1 +} + +authenticate() { + echo "$RELEASETEAMTOKEN" | docker login --password-stdin --username "$RELEASETEAMUSER" $DOMAIN + token=$(curl -u$RELEASETEAMUSER:$RELEASETEAMTOKEN -sS https://$DOMAIN/v2/token | jq --raw-output .token) +} + +image_put() { + docker manifest inspect $3 > /tmp/manifest.json + curl -sS -H "Authorization: token $token" -X PUT --data-binary @/tmp/manifest.json https://$DOMAIN/v2/$1/forgejo/manifests/$2 +} + +main() { + boot + authenticate + publish +} + +image_name() { + echo $DOMAIN/$1/forgejo:$(image_tag $2) +} + +image_tag() { + echo $TAG$1 +} + +short_image_name() { + echo $DOMAIN/$1/forgejo:$(short_image_tag $2) +} + +short_image_tag() { + echo $SHORT_TAG$1 +} + +arch_image_name() { + echo $DOMAIN/$1/forgejo:$(arch_image_tag $2 $3) +} + +arch_image_tag() { + echo $TAG-$1$2 +} + +${@:-main}