realaravinth/bench-forgejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

implementation: Woodpecker based CI

Browse source 
Refs: https://codeberg.org/forgejo/forgejo/issues/73
Refs: https://codeberg.org/forgejo/forgejo/issues/101
Refs: https://codeberg.org/woodpecker-plugins/plugin-docker-buildx/issues/51
Refs: https://codeberg.org/forgejo/forgejo/issues/117
Refs: https://codeberg.org/forgejo/forgejo/issues/25
Refs: https://codeberg.org/forgejo/forgejo/issues/154
This commit is contained in:
Loïc Dachary 2022-11-05 00:00:07 +01:00
parent 9c305030b1
commit 6c83545d0e
Signed by untrusted user: dachary
GPG key ID: 992D23B392F9E4F2
10 changed files with 626 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

74
.woodpecker/compliance.yml Normal file
View file

@ -0,0 +1,74 @@
platform: linux/amd64
branches:
exclude: [ forgejo-development, main, release/*, soft-fork/*/*, soft-fork/*/*/* ]
variables:
- &golang_image 'golang:1.19'
- &gitea_test_image 'gitea/test_env:linux-amd64'
- &goproxy_override ''
- &goproxy_setup |-
if [ -n "$${GOPROXY_OVERRIDE:-}" ]; then
export GOPROXY="$${GOPROXY_OVERRIDE}";
echo "Using goproxy from goproxy_override \"$${GOPROXY}\"";
elif [ -n "$${GOPROXY_DEFAULT:-}" ]; then
export GOPROXY="$${GOPROXY_DEFAULT}";
echo "Using goproxy from goproxy_default (secret) not displaying";
else
export GOPROXY="https://proxy.golang.org,direct";
echo "No goproxy overrides or defaults given, using \"$${GOPROXY}\"";
fi
workspace:
base: /go
path: src/codeberg/gitea
pipeline:
deps-backend:
image: *golang_image
pull: true
environment:
GOPROXY_OVERRIDE: *goproxy_override
secrets:
- goproxy_default
commands:
- *goproxy_setup
- make deps-backend
security-check:
image: *golang_image
group: checks
pull: true
environment:
GOPROXY_OVERRIDE: *goproxy_override
secrets:
- goproxy_default
commands:
- *goproxy_setup
- make security-check
lint-backend:
image: *gitea_test_image
group: checks
pull: true
environment:
GOPROXY_OVERRIDE: *goproxy_override
TAGS: 'bindata sqlite sqlite_unlock_notify'
GOSUMDB: 'sum.golang.org'
secrets:
- goproxy_default
commands:
- *goproxy_setup
- make lint-backend
checks-backend:
image: *gitea_test_image
pull: true
group: checks
environment:
GOPROXY_OVERRIDE: *goproxy_override
secrets:
- goproxy_default
commands:
- *goproxy_setup
- make --always-make checks-backend

80
.woodpecker/docker-release-version.yml Normal file
View file

@ -0,0 +1,80 @@
platform: linux/amd64
when:
event: tag
depends_on:
- testing-amd64
variables:
- &git_image 'docker:git'
- &dind_image 'docker:20.10-dind'
- &buildx_image 'woodpeckerci/plugin-docker-buildx:2.0.0'
- &integration_image 'codeberg.org/forgejo-integration/forgejo'
- &dockerfile_root 'Dockerfile'
# for testing purposes
# - &dockerfile_root 'releases/Dockerfile'
- &dockerfile_rootless 'Dockerfile.rootless'
# for testing purposes
# - &dockerfile_rootless 'releases/Dockerfile-rootless'
- &verify 'true'
# for testing purposes
# - &verify 'false'
- &archs 'amd64 arm64'
pipeline:
fetch-tags:
image: *git_image
pull: true
commands:
- git config --add safe.directory '*'
- git fetch --tags --force
build-root:
image: *buildx_image
group: integration
pull: true
settings:
platforms: linux/amd64,linux/arm64
dockerfile: *dockerfile_root
registry:
from_secret: domain
tag: ${CI_COMMIT_TAG##v}
repo: *integration_image
build_args:
- GOPROXY=https://proxy.golang.org
password:
from_secret: releaseteamtoken
username:
from_secret: releaseteamuser
build-rootless:
image: *buildx_image
group: integration
pull: true
settings:
platforms: linux/amd64,linux/arm64
dockerfile: *dockerfile_rootless
registry:
from_secret: domain
tag: ${CI_COMMIT_TAG##v}-rootless
repo: *integration_image
build_args:
- GOPROXY=https://proxy.golang.org
password:
from_secret: releaseteamtoken
username:
from_secret: releaseteamuser
publish:
image: *dind_image
environment:
INTEGRATION_IMAGE: *integration_image
VERIFY: *verify
ARCHS: *archs
commands:
- ./releases/container-images-pull-verify-push.sh
secrets:
- releaseteamtoken
- releaseteamuser
- domain

111
.woodpecker/release-version.yml Normal file
View file

@ -0,0 +1,111 @@
platform: linux/amd64
when:
event: tag
depends_on:
- testing-amd64
variables:
- &git_image 'docker:git'
- &node_image 'node:18'
- &golang_image 'golang:1.19'
- &gpg_sign_image 'plugins/gpgsign:1'
- &xgo_image 'techknowlogick/xgo:go-1.19.x'
- &gpg_sign_image 'plugins/gpgsign:1'
- &goproxy_override ''
- &goproxy_setup |-
if [ -n "$${GOPROXY_OVERRIDE:-}" ]; then
export GOPROXY="$${GOPROXY_OVERRIDE}";
echo "Using goproxy from goproxy_override \"$${GOPROXY}\"";
elif [ -n "$${GOPROXY_DEFAULT:-}" ]; then
export GOPROXY="$${GOPROXY_DEFAULT}";
echo "Using goproxy from goproxy_default (secret) not displaying";
else
export GOPROXY="https://proxy.golang.org,direct";
echo "No goproxy overrides or defaults given, using \"$${GOPROXY}\"";
fi
workspace:
base: /source
path: /
pipeline:
fetch-tags:
image: *git_image
pull: true
group: deps
commands:
- git config --add safe.directory '*'
- git fetch --tags --force
deps-frontend:
image: *node_image
pull: true
group: deps
commands:
- make deps-frontend
deps-backend:
image: *golang_image
pull: true
group: deps
environment:
GOPROXY_OVERRIDE: *goproxy_override
secrets:
- goproxy_default
commands:
- *goproxy_setup
- make deps-backend
static:
image: *xgo_image
pull: true
commands:
- *goproxy_setup
- curl -sL https://deb.nodesource.com/setup_16.x | bash - && apt-get -qqy install nodejs
- export PATH=$PATH:$GOPATH/bin
- make CI=true LINUX_ARCHS=linux/amd64,linux/arm64,linux/arm-6 release
environment:
TAGS: 'bindata sqlite sqlite_unlock_notify'
DEBIAN_FRONTEND: 'noninteractive'
GOPROXY_OVERRIDE: *goproxy_override
secrets:
- goproxy_default
verifyruns:
image: *golang_image
commands:
- ./dist/release/forgejo-*-amd64 --version | grep 'built with'
- apt-get update
- apt-get install -y qemu-user-static
- /usr/bin/qemu-aarch64-static ./dist/release/forgejo-*-arm64 --version | grep 'built with'
- /usr/bin/qemu-arm-static ./dist/release/forgejo-*-arm-6 --version | grep 'built with'
gpg-sign:
image: *gpg_sign_image
pull: true
settings:
detach_sign: true
excludes:
- "dist/release/*.sha256"
files:
- "dist/release/*"
key:
from_secret: releaseteamgpg
release:
image: *golang_image
commands:
- curl -sL https://dl.gitea.io/tea/0.9.0/tea-0.9.0-linux-amd64 > /bin/tea && chmod +x /bin/tea
- REMOTE=$(echo $CI_REPO_LINK | sed -e 's|.*://||' -e 's|/.*||')
- GITEA_SERVER_URL=$CI_REPO_LINK GITEA_SERVER_TOKEN=$RELEASETEAMTOKEN tea login add --name $RELEASETEAMUSER --url $REMOTE
- ASSETS=$(ls dist/release/* | sed -e 's/^/-a /')
- echo "$${CI_COMMIT_TAG##v}" | grep -qi '\-rc' && export RELEASETYPE="--prerelease" && echo "Uploading as Pre-Release"
- echo "$${CI_COMMIT_TAG##v}" | grep -qi '\-test' && export RELEASETYPE="--draft" && echo "Uploading as Draft"
- test $${RELEASETYPE+false} || echo "Uploading as Stable"
- anchor=$(echo $CI_COMMIT_TAG | sed -e 's/^v//' -e 's/[^a-zA-Z0-9]/-/g')
- tea release create $ASSETS --note "See https://codeberg.org/forgejo/forgejo/src/branch/forgejo/RELEASE-NOTES.md#$${anchor}" --tag $CI_COMMIT_TAG --title $CI_COMMIT_TAG $${RELEASETYPE}
secrets:
- releaseteamtoken
- releaseteamuser

24
.woodpecker/releases-helper.yml Normal file
View file

@ -0,0 +1,24 @@
platform: linux/amd64
branches:
includes: [ wip-ci-* ]
when:
event: push
variables:
- &dind_image 'docker:20.10-dind'
pipeline:
container-images-pull-verify-push:
image: *dind_image
commands:
# arm64 would require qemu-user-static which is not available on alpline
# the test coverage does not change much and running the tests test locally
# is possible if there is a doubt
- ARCHS=amd64 ./releases/container-images-pull-verify-push-test.sh test
- ./releases/container-images-pull-verify-push-test.sh test_teardown
secrets:
- releaseteamuser
- releaseteamtoken
- domain

141
.woodpecker/testing-amd64.yml Normal file
View file

@ -0,0 +1,141 @@
platform: linux/amd64
branches:
exclude: [ forgejo-development, main, release/*, soft-fork/*/*, soft-fork/*/*/* ]
depends_on:
- compliance
variables:
- &git_image 'docker:git'
- &golang_image 'golang:1.19'
- &gitea_test_image 'gitea/test_env:linux-amd64'
- &mysql_image 'mysql:8'
- &pgsql_image 'postgres:10'
- &goproxy_override ''
- &goproxy_setup |-
if [ -n "$${GOPROXY_OVERRIDE:-}" ]; then
export GOPROXY="$${GOPROXY_OVERRIDE}";
echo "Using goproxy from goproxy_override \"$${GOPROXY}\"";
elif [ -n "$${GOPROXY_DEFAULT:-}" ]; then
export GOPROXY="$${GOPROXY_DEFAULT}";
echo "Using goproxy from goproxy_default (secret) not displaying";
else
export GOPROXY="https://proxy.golang.org,direct";
echo "No goproxy overrides or defaults given, using \"$${GOPROXY}\"";
fi
services:
mysql8:
image: *mysql_image
pull: true
environment:
MYSQL_ALLOW_EMPTY_PASSWORD: yes
MYSQL_DATABASE: testgitea
pgsql:
image: *pgsql_image
pull: true
environment:
POSTGRES_DB: test
POSTGRES_PASSWORD: postgres
workspace:
base: /go
path: src/codeberg/gitea
pipeline:
fetch-tags:
image: *git_image
pull: true
commands:
- git config --add safe.directory '*'
- git fetch --tags --force
deps-backend:
image: *golang_image
pull: true
environment:
GOPROXY_OVERRIDE: *goproxy_override
secrets:
- goproxy_default
commands:
- *goproxy_setup
- make deps-backend
tag-pre-condition:
image: *git_image
pull: true
commands:
- git update-ref refs/heads/tag_test ${CI_COMMIT_SHA}
prepare-test-env:
image: *gitea_test_image
pull: true
commands:
- ./build/test-env-prepare.sh
build:
image: *gitea_test_image
environment:
GOSUMDB: sum.golang.org
TAGS: bindata sqlite sqlite_unlock_notify
GOPROXY_OVERRIDE: *goproxy_override
secrets:
- goproxy_default
commands:
- *goproxy_setup
- su gitea -c './build/test-env-check.sh'
- su gitea -c 'make backend'
unit-test:
image: *gitea_test_image
environment:
TAGS: 'bindata sqlite sqlite_unlock_notify'
RACE_ENABLED: 'true'
GOPROXY_OVERRIDE: *goproxy_override
secrets:
- github_read_token
- goproxy_default
commands:
- *goproxy_setup
- su gitea -c 'make unit-test-coverage test-check'
test-mysql8:
group: integration
image: *gitea_test_image
commands:
- *goproxy_setup
- su gitea -c 'timeout -s ABRT 50m make test-mysql8-migration test-mysql8'
environment:
TAGS: 'bindata'
RACE_ENABLED: 'true'
USE_REPO_TEST_DIR: '1'
GOPROXY_OVERRIDE: *goproxy_override
secrets:
- goproxy_default
test-pgsql:
group: integration
image: *gitea_test_image
commands:
- *goproxy_setup
- su gitea -c 'timeout -s ABRT 50m make test-pgsql-migration test-pgsql'
environment:
TAGS: 'bindata'
RACE_ENABLED: 'true'
USE_REPO_TEST_DIR: '1'
GOPROXY_OVERRIDE: *goproxy_override
secrets:
- goproxy_default
test-sqlite:
group: integration
image: *gitea_test_image
environment:
- USE_REPO_TEST_DIR=1
- GOPROXY=off
- TAGS=bindata gogit sqlite sqlite_unlock_notify
- TEST_TAGS=bindata gogit sqlite sqlite_unlock_notify
commands:
- su gitea -c 'timeout -s ABRT 120m make test-sqlite-migration test-sqlite'

2
Makefile
View file

@ -751,7 +751,7 @@ $(EXECUTABLE): $(GO_SOURCES) $(TAGS_PREREQ)
CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) build $(GOFLAGS) $(EXTRA_GOFLAGS) -tags '$(TAGS)' -ldflags '-s -w $(LDFLAGS)' -o $@
.PHONY: release
release: frontend generate release-windows release-linux release-darwin release-copy release-compress vendor release-sources release-docs release-check
release: frontend generate release-linux release-copy release-compress vendor release-sources release-check
$(DIST_DIRS):
mkdir -p $(DIST_DIRS)

3
releases/Dockerfile Normal file
View file

@ -0,0 +1,3 @@
FROM alpine:3.17
RUN echo root > state

3
releases/Dockerfile-rootless Normal file
View file

@ -0,0 +1,3 @@
FROM alpine:3.17
RUN echo rootless > state

74
releases/container-images-pull-verify-push-test.sh Executable file
View file

@ -0,0 +1,74 @@
#!/bin/sh
#
# Tests are run when on a wip-ci-* branch, see .woodpecker/releases-helper.yml
# It should be changed to run it every time this file is changed when 1.18 is used because 1.17 does not have
# webhooks with the information for that filtering.
#
set -ex
image_delete() {
curl -sS -H "Authorization: token $token" -X DELETE https://$DOMAIN/v2/$1/forgejo/manifests/$2
}
#
# Create the same set of images that buildx would
#
test_setup() {
dir=$(dirname $0)
for suffix in '' '-rootless' ; do
(
cd $dir
manifests=""
for arch in $ARCHS ; do
image=$(arch_image_name $INTEGRATION_USER $arch $suffix)
docker build -f Dockerfile$suffix --platform linux/$arch -t $image .
docker push $image
images="$images $image"
done
manifest=$(image_name $INTEGRATION_USER $suffix)
docker manifest rm $manifest || true
docker manifest create $manifest $images
image_put $INTEGRATION_USER $(image_tag $suffix) $manifest
)
done
}
test_teardown() {
authenticate
for suffix in '' '-rootless' ; do
image_delete $INTEGRATION_USER $(image_tag $suffix)
image_delete $CI_REPO_OWNER $(image_tag $suffix)
image_delete $CI_REPO_OWNER $(short_image_tag $suffix)
for arch in $ARCHS ; do
image_delete $INTEGRATION_USER $(arch_image_tag $arch $suffix)
image_delete $CI_REPO_OWNER $(arch_image_tag $arch $suffix)
done
done
}
#
# Running the test locally instead of withing Woodpecker
#
# 1. Setup: obtain a token at https://codeberg.org/user/settings/applications
# 2. Run: RELEASETEAMUSER=<username> RELEASETEAMTOKEn=<apptoken> container-images-pull-verify-push.sh test
# 3. Verify: (optional) manual verification at https://codeberg.org/<username>/-/packages/container/forgejo/versions
# 4. Cleanup: RELEASETEAMUSER=<username> RELEASETEAMTOKEn=<apptoken> container-images-pull-verify-push.sh test_teardown
#
test() {
boot
test_teardown
test_setup
VERIFY_STRING=something
VERIFY_COMMAND="echo $VERIFY_STRING"
echo "================================ TEST BEGIN"
main
echo "================================ TEST END"
}
: ${CI_REPO_OWNER:=dachary}
: ${CI_COMMIT_TAG:=v17.1.42-2}
. $(dirname $0)/container-images-pull-verify-push.sh

115
releases/container-images-pull-verify-push.sh Executable file
View file

@ -0,0 +1,115 @@
#!/bin/sh
set -ex
: ${DOCKER_HOST:=unix:///var/run/docker.sock}
: ${ARCHS:=amd64 arm64}
: ${INTEGRATION_USER:=forgejo-integration}
: ${INTEGRATION_IMAGE:=codeberg.org/$INTEGRATION_USER/forgejo}
: ${TAG:=${CI_COMMIT_TAG##v}}
: ${SHORT_TAG=${TAG%.*-*}}
: ${DOMAIN:=codeberg.org}
: ${VERIFY:=true}
VERIFY_COMMAND='gitea --version'
VERIFY_STRING='built with'
publish() {
for suffix in '' '-rootless' ; do
images=""
for arch in $ARCHS ; do
#
# Get the image from the integration user
#
image=$(image_name $INTEGRATION_USER $suffix)
docker pull --platform linux/$arch $image
#
# Verify it is usable
#
if $VERIFY ; then
docker run --platform linux/$arch --rm $image $VERIFY_COMMAND | grep "$VERIFY_STRING"
fi
#
# Push the image with a tag reflecting the architecture to the repo owner
#
arch_image=$(arch_image_name $CI_REPO_OWNER $arch $suffix)
docker tag $image $arch_image
docker push $arch_image
images="$images $arch_image"
done
#
# Push a manifest with all the architectures to the repo owner
#
manifest=$(image_name $CI_REPO_OWNER $suffix)
docker manifest rm $manifest || true
docker manifest create $manifest $images
image_put $CI_REPO_OWNER $(image_tag $suffix) $manifest
image_put $CI_REPO_OWNER $(short_image_tag $suffix) $manifest
#
# Sanity check to ensure the manifest that are published can actualy
# be used.
#
for arch in $ARCHS ; do
docker pull --platform linux/$arch $(image_name $CI_REPO_OWNER $suffix)
docker pull --platform linux/$arch $(short_image_name $CI_REPO_OWNER $suffix)
done
done
}
boot() {
if docker version ; then
return
fi
apk --update --no-cache add coredns jq curl
( echo ".:53 {" ; echo " forward . /etc/resolv.conf"; echo "}" ) > /etc/coredns/Corefile
coredns -conf /etc/coredns/Corefile &
/usr/local/bin/dockerd --data-root /var/lib/docker --host=$DOCKER_HOST --dns 172.17.0.3 &
for i in $(seq 60) ; do
docker version && break
sleep 1
done
docker version || exit 1
}
authenticate() {
echo "$RELEASETEAMTOKEN" | docker login --password-stdin --username "$RELEASETEAMUSER" $DOMAIN
token=$(curl -u$RELEASETEAMUSER:$RELEASETEAMTOKEN -sS https://$DOMAIN/v2/token | jq --raw-output .token)
}
image_put() {
docker manifest inspect $3 > /tmp/manifest.json
curl -sS -H "Authorization: token $token" -X PUT --data-binary @/tmp/manifest.json https://$DOMAIN/v2/$1/forgejo/manifests/$2
}
main() {
boot
authenticate
publish
}
image_name() {
echo $DOMAIN/$1/forgejo:$(image_tag $2)
}
image_tag() {
echo $TAG$1
}
short_image_name() {
echo $DOMAIN/$1/forgejo:$(short_image_tag $2)
}
short_image_tag() {
echo $SHORT_TAG$1
}
arch_image_name() {
echo $DOMAIN/$1/forgejo:$(arch_image_tag $2 $3)
}
arch_image_tag() {
echo $TAG-$1$2
}
${@:-main}