From ab3694d136d2a0ea60acb8c427b3287ba74530b5 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 1 Nov 2021 13:23:41 +0800 Subject: [PATCH] Support build tag --- .gitignore | 3 +- Makefile | 162 +++++++++++++++++++++++++++++++++++++ bindata_dynamic.go | 17 ++++ bindata_embed.go | 33 ++++++++ cmd/deployer.go | 8 +- cmd/web.go | 2 +- docker-compose.yml | 40 +++++++++ go.mod | 6 +- main.go | 2 +- pkgs/services/cmd.go | 8 ++ pkgs/services/container.go | 36 ++++++++- pkgs/services/dns.go | 12 ++- pkgs/services/git.go | 2 +- pkgs/services/services.go | 6 ++ pkgs/settings/settings.go | 12 +++ routers/routers.go | 40 ++++++++- 16 files changed, 372 insertions(+), 17 deletions(-) create mode 100644 Makefile create mode 100644 bindata_dynamic.go create mode 100644 bindata_embed.go create mode 100644 docker-compose.yml diff --git a/.gitignore b/.gitignore index aefcb24..89800b9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ pr-deployer config.yaml -data/ \ No newline at end of file +data/ +dist \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a88abcb --- /dev/null +++ b/Makefile @@ -0,0 +1,162 @@ +DIST := dist +export GO111MODULE=on +export CGO_ENABLED=0 + +GO ?= go +SHASUM ?= shasum -a 256 + +export PATH := $($(GO) env GOPATH)/bin:$(PATH) + +GOFILES := $(shell find . -name "*.go" -type f ! -path "./vendor/*" ! -path "*/bindata.go") +GOFMT ?= gofmt -s + +ifneq ($(DRONE_TAG),) + VERSION ?= $(subst v,,$(DRONE_TAG)) + TEA_VERSION ?= $(VERSION) +else + ifneq ($(DRONE_BRANCH),) + VERSION ?= $(subst release/v,,$(DRONE_BRANCH)) + else + VERSION ?= master + endif + TEA_VERSION ?= $(shell git describe --tags --always | sed 's/-/+/' | sed 's/^v//') +endif +VERSION_TAG ?= $(shell sed 's/+/_/' <<< $(TEA_VERSION)) + +TAGS ?= +LDFLAGS := -X "main.Version=$(TEA_VERSION)" -X "main.Tags=$(TAGS)" -s -w + +ifeq ($(STATIC),true) + # NOTE: clean up this mess, when https://github.com/golang/go/issues/26492 is resolved + # static_build is a defacto standard tag used in go packages + TAGS := osusergo,netgo,static_build,$(TAGS) + LDFLAGS := $(LDFLAGS) -linkmode=external -extldflags "-static-pie" -X "main.Tags=$(TAGS)" + export CGO_ENABLED=1 # needed for linkmode=external +endif + +# override to allow passing additional goflags via make CLI +override GOFLAGS := $(GOFLAGS) -mod=vendor -tags '$(TAGS)' -ldflags '$(LDFLAGS)' + +PACKAGES ?= $(shell $(GO) list ./... | grep -v /vendor/) +SOURCES ?= $(shell find . -name "*.go" -type f) + +ifeq ($(OS), Windows_NT) + EXECUTABLE := pr-deployer.exe +else + EXECUTABLE := pr-deployer +endif + +.PHONY: all +all: build + +.PHONY: clean +clean: + $(GO) clean -mod=vendor -i ./... + rm -rf $(EXECUTABLE) $(DIST) + +.PHONY: fmt +fmt: + $(GOFMT) -w $(GOFILES) + +.PHONY: vet +vet: + # Default vet + $(GO) vet -mod=vendor $(PACKAGES) + +.PHONY: lint +lint: + @hash revive > /dev/null 2>&1; if [ $$? -ne 0 ]; then \ + cd /tmp && $(GO) get -u github.com/mgechev/revive; \ + fi + revive -config .revive.toml -exclude=./vendor/... ./... || exit 1 + +.PHONY: misspell-check +misspell-check: + @hash misspell > /dev/null 2>&1; if [ $$? -ne 0 ]; then \ + cd /tmp && $(GO) get -u github.com/client9/misspell/cmd/misspell; \ + fi + misspell -error -i unknwon,destory $(GOFILES) + +.PHONY: misspell +misspell: + @hash misspell > /dev/null 2>&1; if [ $$? -ne 0 ]; then \ + cd /tmp && $(GO) get -u github.com/client9/misspell/cmd/misspell; \ + fi + misspell -w -i unknwon $(GOFILES) + +.PHONY: fmt-check +fmt-check: + # get all go files and run go fmt on them + @diff=$$($(GOFMT) -d $(GOFILES)); \ + if [ -n "$$diff" ]; then \ + echo "Please run 'make fmt' and commit the result:"; \ + echo "$${diff}"; \ + exit 1; \ + fi; + +.PHONY: test +test: + $(GO) test -mod=vendor $(PACKAGES) + +.PHONY: unit-test-coverage +unit-test-coverage: + $(GO) test -mod=vendor -cover -coverprofile coverage.out $(PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1 + +.PHONY: vendor +vendor: + $(GO) mod tidy && $(GO) mod vendor + +.PHONY: test-vendor +test-vendor: vendor + @diff=$$(git diff vendor/); \ + if [ -n "$$diff" ]; then \ + echo "Please run 'make vendor' and commit the result:"; \ + echo "$${diff}"; \ + exit 1; \ + fi; + +.PHONY: check +check: test + +.PHONY: install +install: $(SOURCES) + @echo "installing to $(GOPATH)/bin/$(EXECUTABLE)" + $(GO) install -tags="bindata" -v -buildmode=pie $(GOFLAGS) + +.PHONY: build +build: $(EXECUTABLE) + +$(EXECUTABLE): $(SOURCES) +ifeq ($(STATIC),true) + @echo "enabling static build, make sure you have glibc-static (or equivalent) installed" +endif + $(GO) build $(GOFLAGS) -o $@ + +.PHONY: build-image +build-image: + docker build --build-arg VERSION=$(TEA_VERSION) -t gitea/pr-deployer:$(VERSION_TAG) . + +.PHONY: release +release: release-dirs release-os release-compress release-check + +.PHONY: release-dirs +release-dirs: + mkdir -p $(DIST)/binaries $(DIST)/release + +.PHONY: release-os +release-os: + @hash gox > /dev/null 2>&1; if [ $$? -ne 0 ]; then \ + cd /tmp && $(GO) get -u github.com/mitchellh/gox; \ + fi + CGO_ENABLED=0 gox -verbose -tags="bindata" -cgo=false $(GOFLAGS) -osarch='!darwin/386 !darwin/arm' -os="windows linux darwin" -arch="amd64 arm arm64" -output="$(DIST)/release/pr-deployer-$(VERSION)-{{.OS}}-{{.Arch}}" + +.PHONY: release-compress +release-compress: + @hash gxz > /dev/null 2>&1; if [ $$? -ne 0 ]; then \ + GO111MODULE=off $(GO) get -u github.com/ulikunitz/xz/cmd/gxz; \ + fi + cd $(DIST)/release/; for file in `find . -type f -name "*"`; do echo "compressing $${file}" && gxz -k -9 $${file}; done; + +.PHONY: release-check +release-check: + cd $(DIST)/release/; for file in `find . -type f -name "*"`; do echo "checksumming $${file}" && $(SHASUM) `echo $${file} | sed 's/^..//'` > $${file}.sha256; done; \ No newline at end of file diff --git a/bindata_dynamic.go b/bindata_dynamic.go new file mode 100644 index 0000000..8578a56 --- /dev/null +++ b/bindata_dynamic.go @@ -0,0 +1,17 @@ +//go:build !bindata +// +build !bindata + +package main + +import ( + "io/fs" + "os" +) + +func getPublicAssets() fs.FS { + return os.DirFS("public/") +} + +func getTemplateAssets() fs.FS { + return os.DirFS("templates/") +} diff --git a/bindata_embed.go b/bindata_embed.go new file mode 100644 index 0000000..1b01879 --- /dev/null +++ b/bindata_embed.go @@ -0,0 +1,33 @@ +//go:build bindata +// +build bindata + +package main + +import ( + "embed" + "io/fs" + + log "github.com/sirupsen/logrus" +) + +//go:embed public/* +var publicAssets embed.FS + +//go:embed templates/* +var templateAssets embed.FS + +func getPublicAssets() fs.FS { + res, err := fs.Sub(publicAssets, "public") + if err != nil { + log.Fatal(err) + } + return res +} + +func getTemplateAssets() fs.FS { + res, err := fs.Sub(templateAssets, "templates") + if err != nil { + log.Fatal(err) + } + return res +} diff --git a/cmd/deployer.go b/cmd/deployer.go index f66c040..6d85591 100644 --- a/cmd/deployer.go +++ b/cmd/deployer.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "io/fs" "github.com/spf13/cobra" ) @@ -26,6 +27,9 @@ var ( runWeb(cmd, args) }, } + + publicFS fs.FS + templateFS fs.FS ) func init() { @@ -33,6 +37,8 @@ func init() { } // Execute represnets execute command -func Execute() error { +func Execute(pFS, tFS fs.FS) error { + publicFS = pFS + templateFS = tFS return deployerCmd.Execute() } diff --git a/cmd/web.go b/cmd/web.go index 8048d81..18645d6 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -7,5 +7,5 @@ import ( ) func runWeb(cmd *cobra.Command, args []string) { - routers.Web() + routers.Web(publicFS, templateFS) } diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..310727d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,40 @@ +version: "3" + +networks: + traefik: + external: + name: traefik_general + internal: + external: false + +volumes: + git: + driver: local + gitea: + driver: local + ssh: + driver: local + +services: + server: + image: ${DEMO_CONTAINER} + restart: always + networks: + - traefik + - internal + labels: + - traefik.docker.network=traefik_general + - traefik.port=3000 + - traefik.frontend.rule=Host:${DEMO_DOMAIN} + healthcheck: + test: ["NONE"] + interval: 30s + timeout: 10s + retries: 5 + volumes: + - /etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt:ro + - ${DEMO_DATA_DIR}/git:/data/git + - ${DEMO_DATA_DIR}/gitea:/data/gitea + - ${DEMO_DATA_DIR}/ssh:/data/ssh + ports: + - ${DEMO_SSH}:22 \ No newline at end of file diff --git a/go.mod b/go.mod index ca12551..09f86a1 100644 --- a/go.mod +++ b/go.mod @@ -5,12 +5,11 @@ go 1.17 require ( gitea.com/go-chi/session v0.0.0-20211013065435-7d334f340c09 github.com/cloudflare/cloudflare-go v0.26.0 - github.com/docker/docker v20.10.10+incompatible github.com/go-chi/chi/v5 v5.0.4 - github.com/go-git/go-git/v5 v5.4.2 github.com/go-sql-driver/mysql v1.6.0 github.com/gobwas/glob v0.2.3 github.com/google/go-github/v39 v39.2.0 + github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.2.1 github.com/spf13/viper v1.8.1 @@ -25,12 +24,14 @@ require ( github.com/acomagu/bufpipe v1.0.3 // indirect github.com/containerd/containerd v1.5.7 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect + github.com/docker/docker v20.10.10+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/emirpasic/gods v1.12.0 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.3.1 // indirect + github.com/go-git/go-git/v5 v5.4.2 // indirect github.com/goccy/go-json v0.7.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect @@ -53,7 +54,6 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect github.com/pelletier/go-toml v1.9.3 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/sergi/go-diff v1.1.0 // indirect github.com/spf13/afero v1.6.0 // indirect github.com/spf13/cast v1.3.1 // indirect diff --git a/main.go b/main.go index 614bd10..08e063b 100644 --- a/main.go +++ b/main.go @@ -21,7 +21,7 @@ func main() { } log.Info("models init") - if err := cmd.Execute(); err != nil { + if err := cmd.Execute(getPublicAssets(), getTemplateAssets()); err != nil { log.Fatal(err) } } diff --git a/pkgs/services/cmd.go b/pkgs/services/cmd.go index 8dbb65e..b2ec75e 100644 --- a/pkgs/services/cmd.go +++ b/pkgs/services/cmd.go @@ -12,6 +12,14 @@ func getPRDir(number int) string { return filepath.Join(settings.CodeCacheDir, strconv.Itoa(number)) } +func getPRGitDir(number int) string { + return filepath.Join(getPRDir(number), "git") +} + +func getPRRuntimeDir(number int) string { + return filepath.Join(getPRDir(number), "container") +} + func WithDir(cmd *exec.Cmd, dir string) *exec.Cmd { cmd.Dir = dir return cmd diff --git a/pkgs/services/container.go b/pkgs/services/container.go index 4abd30c..064a525 100644 --- a/pkgs/services/container.go +++ b/pkgs/services/container.go @@ -4,7 +4,9 @@ import ( "bytes" "context" "fmt" + "os" "os/exec" + "path/filepath" "gitea.com/gitea/pr-deployer/pkgs/settings" @@ -17,7 +19,7 @@ func getImageTag(number int) string { // buildImage build the docker image via Dockerfile func buildImage(ctx context.Context, number int) (string, error) { - p := getPRDir(number) + p := getPRGitDir(number) tagName := fmt.Sprintf("%s/%s:%s", settings.RepoOwner, settings.RepoName, getImageTag(number)) cmd := exec.Command("docker", "build", "-t", tagName, @@ -30,12 +32,38 @@ func buildImage(ctx context.Context, number int) (string, error) { return tagName, nil } +func contractErr(errs ...error) error { + var errContent string + for _, e := range errs { + if e != nil { + errContent += e.Error() + ";" + } + } + return errors.New(errContent) +} + func runImage(ctx context.Context, number int, image string) (string, error) { - p := getPRDir(number) + p := getPRRuntimeDir(number) + if err := os.MkdirAll(p, os.ModePerm); err != nil { + return "", err + } + + composeDir := filepath.Dir(settings.DockerComposeTemplateFile) + var buf bytes.Buffer - cmd := exec.Command("docker", "run", "-d", image) + var errBuf bytes.Buffer + cmd := exec.Command("docker-compose", "up", "-d") + cmd.Env = append(os.Environ(), + "DEMO_CONTAINER="+image, + "DEMO_DOMAIN="+getFullSubDomain(number), + "DEMO_SSH=22", + "DEMO_DATA_DIR="+p, + ) cmd.Stdout = &buf - if err := WithDir(cmd, p).Run(); err != nil { + cmd.Stderr = &errBuf + err := WithDir(cmd, composeDir).Run() + if err != nil || errBuf.String() != "" { + err = contractErr(err, errors.New(errBuf.String())) return "", errors.Wrap(err, "docker run") } diff --git a/pkgs/services/dns.go b/pkgs/services/dns.go index 7ff826b..49f4fa3 100644 --- a/pkgs/services/dns.go +++ b/pkgs/services/dns.go @@ -9,6 +9,14 @@ import ( "github.com/google/go-github/v39/github" ) +func getSubDomainName(number int) string { + return fmt.Sprintf("try-pr-%d", number) +} + +func getFullSubDomain(number int) string { + return getSubDomainName(number) + "." + settings.PRParentDomain +} + func checkAndUpdateSubDomain(number int) error { api, err := cloudflare.NewWithAPIToken(settings.CloudflareToken) if err != nil { @@ -21,10 +29,10 @@ func checkAndUpdateSubDomain(number int) error { } var found bool - var name = fmt.Sprintf("try-pr-%d", number) + var name = getSubDomainName(number) var filter = cloudflare.DNSRecord{ Type: "A", - Name: name + "." + settings.PRParentDomain, + Name: getFullSubDomain(number), } recs, err := api.DNSRecords(context.Background(), zoneID, filter) if err != nil { diff --git a/pkgs/services/git.go b/pkgs/services/git.go index 2258caa..1f661ef 100644 --- a/pkgs/services/git.go +++ b/pkgs/services/git.go @@ -14,7 +14,7 @@ import ( ) func updateGitRepo(number int, sha string) (string, error) { - p := getPRDir(number) + p := getPRGitDir(number) log.Trace("clone code into", p) var local = fmt.Sprintf("pull/%d/head:pr/%d", number, number) diff --git a/pkgs/services/services.go b/pkgs/services/services.go index e7949f0..7f3247c 100644 --- a/pkgs/services/services.go +++ b/pkgs/services/services.go @@ -3,6 +3,7 @@ package services import ( "context" "fmt" + "os" "gitea.com/gitea/pr-deployer/pkgs/settings" log "github.com/sirupsen/logrus" @@ -21,6 +22,11 @@ func UpdateAndStartPullRequest(ctx context.Context, client Client, number int, s return err } + prDir := getPRDir(number) + if err := os.MkdirAll(prDir, os.ModePerm); err != nil { + return err + } + // 0 download the git log.Trace("updateGitRepo") diff --git a/pkgs/settings/settings.go b/pkgs/settings/settings.go index c569d39..008328c 100644 --- a/pkgs/settings/settings.go +++ b/pkgs/settings/settings.go @@ -38,6 +38,8 @@ var ( DBType string DBConnStr string + + DockerComposeTemplateFile string ) var ( @@ -149,5 +151,15 @@ func Init() error { return errors.New("DBConnStr is empty") } + DockerComposeTemplateFile = viper.GetString("DockerComposeTemplateFile") + if DockerComposeTemplateFile == "" { + return errors.New("DockerComposeTemplateFile should not be empty") + } + + DockerComposeTemplateFile, err = filepath.Abs(DockerComposeTemplateFile) + if err != nil { + return err + } + return nil } diff --git a/routers/routers.go b/routers/routers.go index 7c4e5e8..5eb5d6e 100644 --- a/routers/routers.go +++ b/routers/routers.go @@ -3,7 +3,10 @@ package routers import ( "context" "fmt" + "io" + "io/fs" "net/http" + "path/filepath" "strconv" "gitea.com/gitea/pr-deployer/pkgs/services" @@ -20,9 +23,38 @@ import ( var rnd *render.Render -func Web() { +type tmplFS struct { + fs.FS +} + +func (tfs tmplFS) Walk(root string, walkFn filepath.WalkFunc) error { + return fs.WalkDir(tfs, root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + info, err := d.Info() + return walkFn(path, info, err) + }) +} + +func (tfs tmplFS) ReadFile(filename string) ([]byte, error) { + f, err := tfs.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + return io.ReadAll(f) +} + +func convertFS(templateFS fs.FS) render.FileSystem { + return tmplFS{templateFS} +} + +func Web(publicFS, templateFS fs.FS) { rnd = render.New(render.Options{ + Directory: ".", IsDevelopment: true, + FileSystem: convertFS(templateFS), }) c := chi.NewRouter() @@ -35,9 +67,11 @@ func Web() { c.Post("/pr/{index}/run", RunPR) c.Post("/pr/{index}/stop", StopPR) c.Post("/webhook", Webhook) + + fs := http.StripPrefix("/public", http.FileServer(http.FS(publicFS))) + c.Get("/public/*", func(w http.ResponseWriter, r *http.Request) { - p := chi.URLParam(r, "*") - http.ServeFile(w, r, "public/"+p) + fs.ServeHTTP(w, r) }) http.ListenAndServe(":3001", c)