implementation: Woodpecker based CI

Refs: https://codeberg.org/forgejo/forgejo/issues/73 Refs: https://codeberg.org/forgejo/forgejo/issues/101 Refs: https://codeberg.org/woodpecker-plugins/plugin-docker-buildx/issues/51 Refs: https://codeberg.org/forgejo/forgejo/issues/117 Refs: https://codeberg.org/forgejo/forgejo/issues/25 Refs: https://codeberg.org/forgejo/forgejo/issues/154
2022-11-05 00:00:07 +01:00 · 2022-11-05 00:00:07 +01:00 · 6c83545d0e
parent 9c305030b1
commit 6c83545d0e
10 changed files with 626 additions and 1 deletions
--- a/.woodpecker/compliance.yml
+++ b/.woodpecker/compliance.yml
@ -0,0 +1,74 @@
+platform: linux/amd64
+
+branches:
+  exclude: [ forgejo-development, main, release/*, soft-fork/*/*, soft-fork/*/*/* ]
+
+variables:
+ - &golang_image 'golang:1.19'
+ - &gitea_test_image 'gitea/test_env:linux-amd64'
+ - &goproxy_override ''
+ - &goproxy_setup |-
+      if [ -n "$${GOPROXY_OVERRIDE:-}" ]; then
+        export GOPROXY="$${GOPROXY_OVERRIDE}";
+        echo "Using goproxy from goproxy_override \"$${GOPROXY}\"";
+      elif [ -n "$${GOPROXY_DEFAULT:-}" ]; then
+        export GOPROXY="$${GOPROXY_DEFAULT}";
+        echo "Using goproxy from goproxy_default (secret) not displaying";
+      else
+        export GOPROXY="https://proxy.golang.org,direct";
+        echo "No goproxy overrides or defaults given, using \"$${GOPROXY}\"";
+      fi
+
+workspace:
+  base: /go
+  path: src/codeberg/gitea
+
+pipeline:
+  deps-backend:
+    image: *golang_image
+    pull: true
+    environment:
+      GOPROXY_OVERRIDE: *goproxy_override
+    secrets:
+      - goproxy_default
+    commands:
+    - *goproxy_setup
+    - make deps-backend
+
+  security-check:
+    image: *golang_image
+    group: checks
+    pull: true
+    environment:
+      GOPROXY_OVERRIDE: *goproxy_override
+    secrets:
+      - goproxy_default
+    commands:
+    - *goproxy_setup
+    - make security-check
+
+  lint-backend:
+    image: *gitea_test_image
+    group: checks
+    pull: true
+    environment:
+      GOPROXY_OVERRIDE: *goproxy_override
+      TAGS: 'bindata sqlite sqlite_unlock_notify'
+      GOSUMDB: 'sum.golang.org'
+    secrets:
+      - goproxy_default
+    commands:
+    - *goproxy_setup
+    - make lint-backend
+
+  checks-backend:
+    image: *gitea_test_image
+    pull: true
+    group: checks
+    environment:
+      GOPROXY_OVERRIDE: *goproxy_override
+    secrets:
+      - goproxy_default
+    commands:
+    - *goproxy_setup
+    - make --always-make checks-backend
--- a/.woodpecker/docker-release-version.yml
+++ b/.woodpecker/docker-release-version.yml
@ -0,0 +1,80 @@
+platform: linux/amd64
+
+when:
+  event: tag
+
+depends_on:
+- testing-amd64
+
+variables:
+ - &git_image 'docker:git'
+ - &dind_image 'docker:20.10-dind'
+ - &buildx_image 'woodpeckerci/plugin-docker-buildx:2.0.0'
+ - &integration_image 'codeberg.org/forgejo-integration/forgejo'
+ - &dockerfile_root 'Dockerfile'
+# for testing purposes
+# - &dockerfile_root 'releases/Dockerfile'
+ - &dockerfile_rootless 'Dockerfile.rootless'
+# for testing purposes
+# - &dockerfile_rootless 'releases/Dockerfile-rootless'
+ - &verify 'true'
+# for testing purposes
+# - &verify 'false'
+ - &archs 'amd64 arm64'
+
+pipeline:
+  fetch-tags:
+    image: *git_image
+    pull: true
+    commands:
+     - git config --add safe.directory '*'
+     - git fetch --tags --force
+
+  build-root:
+    image: *buildx_image
+    group: integration
+    pull: true
+    settings:
+      platforms: linux/amd64,linux/arm64
+      dockerfile: *dockerfile_root
+      registry:
+        from_secret: domain
+      tag: ${CI_COMMIT_TAG##v}
+      repo: *integration_image
+      build_args:
+        - GOPROXY=https://proxy.golang.org
+      password:
+        from_secret: releaseteamtoken
+      username:
+        from_secret: releaseteamuser
+
+  build-rootless:
+    image: *buildx_image
+    group: integration
+    pull: true
+    settings:
+      platforms: linux/amd64,linux/arm64
+      dockerfile: *dockerfile_rootless
+      registry:
+        from_secret: domain
+      tag: ${CI_COMMIT_TAG##v}-rootless
+      repo: *integration_image
+      build_args:
+        - GOPROXY=https://proxy.golang.org
+      password:
+        from_secret: releaseteamtoken
+      username:
+        from_secret: releaseteamuser
+
+  publish:
+    image: *dind_image
+    environment:
+      INTEGRATION_IMAGE: *integration_image
+      VERIFY: *verify
+      ARCHS: *archs
+    commands:
+      - ./releases/container-images-pull-verify-push.sh
+    secrets:
+      - releaseteamtoken
+      - releaseteamuser
+      - domain
--- a/.woodpecker/release-version.yml
+++ b/.woodpecker/release-version.yml
@ -0,0 +1,111 @@
+platform: linux/amd64
+
+when:
+  event: tag
+
+depends_on:
+- testing-amd64
+
+variables:
+ - &git_image 'docker:git'
+ - &node_image 'node:18'
+ - &golang_image 'golang:1.19'
+ - &gpg_sign_image 'plugins/gpgsign:1'
+ - &xgo_image 'techknowlogick/xgo:go-1.19.x'
+ - &gpg_sign_image 'plugins/gpgsign:1'
+ - &goproxy_override ''
+ - &goproxy_setup |-
+      if [ -n "$${GOPROXY_OVERRIDE:-}" ]; then
+        export GOPROXY="$${GOPROXY_OVERRIDE}";
+        echo "Using goproxy from goproxy_override \"$${GOPROXY}\"";
+      elif [ -n "$${GOPROXY_DEFAULT:-}" ]; then
+        export GOPROXY="$${GOPROXY_DEFAULT}";
+        echo "Using goproxy from goproxy_default (secret) not displaying";
+      else
+        export GOPROXY="https://proxy.golang.org,direct";
+        echo "No goproxy overrides or defaults given, using \"$${GOPROXY}\"";
+      fi
+
+workspace:
+  base: /source
+  path: /
+
+pipeline:
+  fetch-tags:
+    image: *git_image
+    pull: true
+    group: deps
+    commands:
+     - git config --add safe.directory '*'
+     - git fetch --tags --force
+
+  deps-frontend:
+    image: *node_image
+    pull: true
+    group: deps
+    commands:
+      - make deps-frontend
+
+  deps-backend:
+    image: *golang_image
+    pull: true
+    group: deps
+    environment:
+      GOPROXY_OVERRIDE: *goproxy_override
+    secrets:
+      - goproxy_default
+    commands:
+    - *goproxy_setup
+    - make deps-backend
+
+  static:
+    image: *xgo_image
+    pull: true
+    commands:
+      - *goproxy_setup
+      - curl -sL https://deb.nodesource.com/setup_16.x | bash - && apt-get -qqy install nodejs
+      - export PATH=$PATH:$GOPATH/bin
+      - make CI=true LINUX_ARCHS=linux/amd64,linux/arm64,linux/arm-6 release
+    environment:
+      TAGS: 'bindata sqlite sqlite_unlock_notify'
+      DEBIAN_FRONTEND: 'noninteractive'
+      GOPROXY_OVERRIDE: *goproxy_override
+    secrets:
+      - goproxy_default
+
+  verifyruns:
+    image: *golang_image
+    commands:
+      - ./dist/release/forgejo-*-amd64 --version | grep 'built with'
+      - apt-get update
+      - apt-get install -y qemu-user-static
+      - /usr/bin/qemu-aarch64-static ./dist/release/forgejo-*-arm64 --version | grep 'built with'
+      - /usr/bin/qemu-arm-static ./dist/release/forgejo-*-arm-6 --version | grep 'built with'
+
+  gpg-sign:
+    image: *gpg_sign_image
+    pull: true
+    settings:
+      detach_sign: true
+      excludes:
+        - "dist/release/*.sha256"
+      files:
+        - "dist/release/*"
+      key:
+        from_secret: releaseteamgpg
+
+  release:
+    image: *golang_image
+    commands:
+    - curl -sL https://dl.gitea.io/tea/0.9.0/tea-0.9.0-linux-amd64 > /bin/tea && chmod +x /bin/tea
+    - REMOTE=$(echo $CI_REPO_LINK | sed -e 's|.*://||' -e 's|/.*||')
+    - GITEA_SERVER_URL=$CI_REPO_LINK GITEA_SERVER_TOKEN=$RELEASETEAMTOKEN tea login add --name $RELEASETEAMUSER --url $REMOTE
+    - ASSETS=$(ls dist/release/* | sed -e 's/^/-a /')
+    - echo "$${CI_COMMIT_TAG##v}" | grep -qi '\-rc' && export RELEASETYPE="--prerelease" && echo "Uploading as Pre-Release"
+    - echo "$${CI_COMMIT_TAG##v}" | grep -qi '\-test' && export RELEASETYPE="--draft" && echo "Uploading as Draft"
+    - test $${RELEASETYPE+false} || echo "Uploading as Stable"
+    - anchor=$(echo $CI_COMMIT_TAG | sed -e 's/^v//' -e 's/[^a-zA-Z0-9]/-/g')
+    - tea release create $ASSETS --note "See https://codeberg.org/forgejo/forgejo/src/branch/forgejo/RELEASE-NOTES.md#$${anchor}" --tag $CI_COMMIT_TAG --title $CI_COMMIT_TAG $${RELEASETYPE}
+    secrets:
+      - releaseteamtoken
+      - releaseteamuser
--- a/.woodpecker/releases-helper.yml
+++ b/.woodpecker/releases-helper.yml
@ -0,0 +1,24 @@
+platform: linux/amd64
+
+branches:
+  includes: [ wip-ci-* ]
+
+when:
+  event: push
+
+variables:
+ - &dind_image 'docker:20.10-dind'
+
+pipeline:
+  container-images-pull-verify-push:
+    image: *dind_image
+    commands:
+# arm64 would require qemu-user-static which is not available on alpline
+# the test coverage does not change much and running the tests test locally
+# is possible if there is a doubt
+      - ARCHS=amd64 ./releases/container-images-pull-verify-push-test.sh test
+      - ./releases/container-images-pull-verify-push-test.sh test_teardown
+    secrets:
+      - releaseteamuser
+      - releaseteamtoken
+      - domain
--- a/.woodpecker/testing-amd64.yml
+++ b/.woodpecker/testing-amd64.yml
@ -0,0 +1,141 @@
+platform: linux/amd64
+
+branches:
+  exclude: [ forgejo-development, main, release/*, soft-fork/*/*, soft-fork/*/*/* ]
+
+depends_on:
+- compliance
+
+variables:
+ - &git_image 'docker:git'
+ - &golang_image 'golang:1.19'
+ - &gitea_test_image 'gitea/test_env:linux-amd64'
+ - &mysql_image 'mysql:8'
+ - &pgsql_image 'postgres:10'
+ - &goproxy_override ''
+ - &goproxy_setup |-
+      if [ -n "$${GOPROXY_OVERRIDE:-}" ]; then
+        export GOPROXY="$${GOPROXY_OVERRIDE}";
+        echo "Using goproxy from goproxy_override \"$${GOPROXY}\"";
+      elif [ -n "$${GOPROXY_DEFAULT:-}" ]; then
+        export GOPROXY="$${GOPROXY_DEFAULT}";
+        echo "Using goproxy from goproxy_default (secret) not displaying";
+      else
+        export GOPROXY="https://proxy.golang.org,direct";
+        echo "No goproxy overrides or defaults given, using \"$${GOPROXY}\"";
+      fi
+
+services:
+  mysql8:
+    image: *mysql_image
+    pull: true
+    environment:
+      MYSQL_ALLOW_EMPTY_PASSWORD: yes
+      MYSQL_DATABASE: testgitea
+
+  pgsql:
+    image: *pgsql_image
+    pull: true
+    environment:
+      POSTGRES_DB: test
+      POSTGRES_PASSWORD: postgres
+
+workspace:
+  base: /go
+  path: src/codeberg/gitea
+
+pipeline:
+  fetch-tags:
+    image: *git_image
+    pull: true
+    commands:
+    - git config --add safe.directory '*'
+    - git fetch --tags --force
+
+  deps-backend:
+    image: *golang_image
+    pull: true
+    environment:
+      GOPROXY_OVERRIDE: *goproxy_override
+    secrets:
+      - goproxy_default
+    commands:
+    - *goproxy_setup
+    - make deps-backend
+
+  tag-pre-condition:
+    image: *git_image
+    pull: true
+    commands:
+    - git update-ref refs/heads/tag_test ${CI_COMMIT_SHA}
+
+  prepare-test-env:
+    image: *gitea_test_image
+    pull: true
+    commands:
+    - ./build/test-env-prepare.sh
+
+  build:
+    image: *gitea_test_image
+    environment:
+      GOSUMDB: sum.golang.org
+      TAGS: bindata sqlite sqlite_unlock_notify
+      GOPROXY_OVERRIDE: *goproxy_override
+    secrets:
+      - goproxy_default
+    commands:
+    - *goproxy_setup
+    - su gitea -c './build/test-env-check.sh'
+    - su gitea -c 'make backend'
+
+  unit-test:
+    image: *gitea_test_image
+    environment:
+      TAGS: 'bindata sqlite sqlite_unlock_notify'
+      RACE_ENABLED: 'true'
+      GOPROXY_OVERRIDE: *goproxy_override
+    secrets:
+    - github_read_token
+    - goproxy_default
+    commands:
+    - *goproxy_setup
+    - su gitea -c 'make unit-test-coverage test-check'
+
+  test-mysql8:
+    group: integration
+    image: *gitea_test_image
+    commands:
+      - *goproxy_setup
+      - su gitea -c 'timeout -s ABRT 50m make test-mysql8-migration test-mysql8'
+    environment:
+      TAGS: 'bindata'
+      RACE_ENABLED: 'true'
+      USE_REPO_TEST_DIR: '1'
+      GOPROXY_OVERRIDE: *goproxy_override
+    secrets:
+      - goproxy_default
+
+  test-pgsql:
+    group: integration
+    image: *gitea_test_image
+    commands:
+      - *goproxy_setup
+      - su gitea -c 'timeout -s ABRT 50m make test-pgsql-migration test-pgsql'
+    environment:
+      TAGS: 'bindata'
+      RACE_ENABLED: 'true'
+      USE_REPO_TEST_DIR: '1'
+      GOPROXY_OVERRIDE: *goproxy_override
+    secrets:
+      - goproxy_default
+
+  test-sqlite:
+    group: integration
+    image: *gitea_test_image
+    environment:
+    - USE_REPO_TEST_DIR=1
+    - GOPROXY=off
+    - TAGS=bindata gogit sqlite sqlite_unlock_notify
+    - TEST_TAGS=bindata gogit sqlite sqlite_unlock_notify
+    commands:
+    - su gitea -c 'timeout -s ABRT 120m make test-sqlite-migration test-sqlite'
--- a/2
+++ b/2
@ -751,7 +751,7 @@ $(EXECUTABLE): $(GO_SOURCES) $(TAGS_PREREQ)
 	CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) build $(GOFLAGS) $(EXTRA_GOFLAGS) -tags '$(TAGS)' -ldflags '-s -w $(LDFLAGS)' -o $@

 .PHONY: release
-release: frontend generate release-windows release-linux release-darwin release-copy release-compress vendor release-sources release-docs release-check
+release: frontend generate release-linux release-copy release-compress vendor release-sources release-check

 $(DIST_DIRS):
 	mkdir -p $(DIST_DIRS)
--- a/releases/Dockerfile
+++ b/releases/Dockerfile
@ -0,0 +1,3 @@
+FROM alpine:3.17
+
+RUN echo root > state
--- a/releases/Dockerfile-rootless
+++ b/releases/Dockerfile-rootless
@ -0,0 +1,3 @@
+FROM alpine:3.17
+
+RUN echo rootless > state
--- a/releases/container-images-pull-verify-push-test.sh
+++ b/releases/container-images-pull-verify-push-test.sh
@ -0,0 +1,74 @@
+#!/bin/sh
+
+#
+# Tests are run when on a wip-ci-* branch, see .woodpecker/releases-helper.yml
+# It should be changed to run it every time this file is changed when 1.18 is used because 1.17 does not have
+# webhooks with the information for that filtering.
+#
+
+set -ex
+
+image_delete() {
+    curl -sS  -H "Authorization: token $token" -X DELETE https://$DOMAIN/v2/$1/forgejo/manifests/$2
+}
+
+#
+# Create the same set of images that buildx would
+#
+test_setup() {
+    dir=$(dirname $0)
+
+    for suffix in '' '-rootless' ; do
+	(
+	    cd $dir
+	    manifests=""
+	    for arch in $ARCHS ; do
+		image=$(arch_image_name $INTEGRATION_USER $arch $suffix)
+		docker build -f Dockerfile$suffix --platform linux/$arch -t $image .
+		docker push $image
+		images="$images $image"
+	    done
+	    manifest=$(image_name $INTEGRATION_USER $suffix)
+	    docker manifest rm $manifest || true
+	    docker manifest create $manifest $images
+	    image_put $INTEGRATION_USER $(image_tag $suffix) $manifest
+	)
+    done
+}
+
+test_teardown() {
+    authenticate
+    for suffix in '' '-rootless' ; do
+	image_delete $INTEGRATION_USER $(image_tag $suffix)
+	image_delete $CI_REPO_OWNER $(image_tag $suffix)
+	image_delete $CI_REPO_OWNER $(short_image_tag $suffix)
+	for arch in $ARCHS ; do
+	    image_delete $INTEGRATION_USER $(arch_image_tag $arch $suffix)
+	    image_delete $CI_REPO_OWNER $(arch_image_tag $arch $suffix)
+	done
+    done
+}
+
+#
+# Running the test locally instead of withing Woodpecker
+#
+# 1. Setup: obtain a token at https://codeberg.org/user/settings/applications
+# 2. Run: RELEASETEAMUSER=<username> RELEASETEAMTOKEn=<apptoken> container-images-pull-verify-push.sh test
+# 3. Verify: (optional) manual verification at https://codeberg.org/<username>/-/packages/container/forgejo/versions
+# 4. Cleanup: RELEASETEAMUSER=<username> RELEASETEAMTOKEn=<apptoken> container-images-pull-verify-push.sh test_teardown
+#
+test() {
+    boot
+    test_teardown
+    test_setup
+    VERIFY_STRING=something
+    VERIFY_COMMAND="echo $VERIFY_STRING"
+    echo "================================ TEST BEGIN"
+    main
+    echo "================================ TEST END"
+}
+
+: ${CI_REPO_OWNER:=dachary}
+: ${CI_COMMIT_TAG:=v17.1.42-2}
+
+. $(dirname $0)/container-images-pull-verify-push.sh
--- a/releases/container-images-pull-verify-push.sh
+++ b/releases/container-images-pull-verify-push.sh
@ -0,0 +1,115 @@
+#!/bin/sh
+
+set -ex
+
+: ${DOCKER_HOST:=unix:///var/run/docker.sock}
+: ${ARCHS:=amd64 arm64}
+: ${INTEGRATION_USER:=forgejo-integration}
+: ${INTEGRATION_IMAGE:=codeberg.org/$INTEGRATION_USER/forgejo}
+: ${TAG:=${CI_COMMIT_TAG##v}}
+: ${SHORT_TAG=${TAG%.*-*}}
+: ${DOMAIN:=codeberg.org}
+
+: ${VERIFY:=true}
+VERIFY_COMMAND='gitea --version'
+VERIFY_STRING='built with'
+
+publish() {
+    for suffix in '' '-rootless' ; do
+	images=""
+	for arch in $ARCHS ; do
+	    #
+	    # Get the image from the integration user
+	    #
+	    image=$(image_name $INTEGRATION_USER $suffix)
+	    docker pull --platform linux/$arch $image
+	    #
+	    # Verify it is usable
+	    #
+	    if $VERIFY ; then
+		docker run --platform linux/$arch --rm $image $VERIFY_COMMAND | grep "$VERIFY_STRING"
+	    fi
+	    #
+	    # Push the image with a tag reflecting the architecture to the repo owner
+	    #
+	    arch_image=$(arch_image_name $CI_REPO_OWNER $arch $suffix)
+	    docker tag $image $arch_image
+	    docker push $arch_image
+	    images="$images $arch_image"
+	done
+
+	#
+	# Push a manifest with all the architectures to the repo owner
+	#
+	manifest=$(image_name $CI_REPO_OWNER $suffix)
+	docker manifest rm $manifest || true
+	docker manifest create $manifest $images
+	image_put $CI_REPO_OWNER $(image_tag $suffix) $manifest
+	image_put $CI_REPO_OWNER $(short_image_tag $suffix) $manifest
+	#
+	# Sanity check to ensure the manifest that are published can actualy
+	# be used.
+	#
+	for arch in $ARCHS ; do
+	    docker pull --platform linux/$arch $(image_name $CI_REPO_OWNER $suffix)
+	    docker pull --platform linux/$arch $(short_image_name $CI_REPO_OWNER $suffix)
+	done
+    done
+}
+
+boot() {
+    if docker version ; then
+	return
+    fi
+    apk --update --no-cache add coredns jq curl
+    ( echo ".:53 {" ; echo "  forward . /etc/resolv.conf"; echo "}" ) > /etc/coredns/Corefile
+    coredns -conf /etc/coredns/Corefile &
+    /usr/local/bin/dockerd --data-root /var/lib/docker --host=$DOCKER_HOST --dns 172.17.0.3 &
+    for i in $(seq 60) ; do
+	docker version && break
+	sleep 1
+    done
+    docker version || exit 1
+}
+
+authenticate() {
+    echo "$RELEASETEAMTOKEN" | docker login --password-stdin --username "$RELEASETEAMUSER" $DOMAIN
+    token=$(curl -u$RELEASETEAMUSER:$RELEASETEAMTOKEN -sS https://$DOMAIN/v2/token | jq --raw-output .token)
+}
+
+image_put() {
+    docker manifest inspect $3 > /tmp/manifest.json
+    curl -sS  -H "Authorization: token $token" -X PUT --data-binary @/tmp/manifest.json https://$DOMAIN/v2/$1/forgejo/manifests/$2
+}
+
+main() {
+    boot
+    authenticate
+    publish
+}
+
+image_name() {
+    echo $DOMAIN/$1/forgejo:$(image_tag $2)
+}
+
+image_tag() {
+    echo $TAG$1
+}
+
+short_image_name() {
+    echo $DOMAIN/$1/forgejo:$(short_image_tag $2)
+}
+
+short_image_tag() {
+    echo $SHORT_TAG$1
+}
+
+arch_image_name() {
+    echo $DOMAIN/$1/forgejo:$(arch_image_tag $2 $3)
+}
+
+arch_image_tag() {
+    echo $TAG-$1$2
+}
+
+${@:-main}