Run lint as part of the docker build.

2022-01-18 18:03:21 +00:00
23 changed files with 70 additions and 616 deletions
--- a/.buildkite/lint.sh
+++ b/.buildkite/lint.sh
@ -8,4 +8,4 @@ go get golang.org/x/lint/golint
 go install golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow
 go get github.com/fzipp/gocyclo/cmd/gocyclo

-./scripts/lint.sh
+/bin/sh ./scripts/lint.sh
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@ -1 +0,0 @@
-*       @richvdh
--- a/.github/workflows/docker.yaml
+++ b/.github/workflows/docker.yaml
@ -1,84 +0,0 @@
-name: Docker
-
-on:
-  push:
-  pull_request:
-    branches: [ master ]
-
-jobs:
-  build:
-    name: Build and push Docker image
-    runs-on: ubuntu-latest
-    env:
-      IMAGE: ghcr.io/${{ github.repository }}
-
-    permissions:
-      packages: write
-      contents: read
-
-    steps:
-      - name: Checkout the code
-        uses: actions/checkout@v2
-
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v3
-        with:
-          images: "${{ env.IMAGE }}"
-          bake-target: docker-metadata-action
-          tags: |
-            type=ref,event=branch
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=semver,pattern={{major}}
-            type=sha
-
-      - name: Docker meta (debug variant)
-        id: meta-debug
-        uses: docker/metadata-action@v3
-        with:
-          images: "${{ env.IMAGE }}"
-          bake-target: docker-metadata-action-debug
-          tags: |
-            type=ref,event=branch,suffix=-debug
-            type=semver,pattern={{version}},suffix=-debug
-            type=semver,pattern={{major}}.{{minor}},suffix=-debug
-            type=semver,pattern={{major}},suffix=-debug
-            type=sha,suffix=-debug
-
-      - name: Merge buildx bake files
-        run: |
-            jq -s '.[0] * .[1]' ${{ steps.meta.outputs.bake-file }} ${{ steps.meta-debug.outputs.bake-file }} > docker-bake.override.json
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          config-inline: |
-            [registry."docker.io"]
-              mirrors = ["mirror.gcr.io"]
-
-      - name: Login to GitHub Container Registry
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      # For pull-requests, only read from the cache, do not try to push to the
-      # cache or the image itself
-      - name: Build
-        uses: docker/bake-action@v1
-        if: github.event_name == 'pull_request'
-        with:
-          set: |
-            base.cache-from=type=registry,ref=${{ env.IMAGE }}:buildcache
-
-      - name: Build and push
-        uses: docker/bake-action@v1
-        if: github.event_name != 'pull_request'
-        with:
-          set: |
-            base.output=type=image,push=true
-            base.cache-from=type=registry,ref=${{ env.IMAGE }}:buildcache
-            base.cache-to=type=registry,ref=${{ env.IMAGE }}:buildcache,mode=max
--- a/.woodpecker.yml
+++ b/.woodpecker.yml
@ -1,15 +0,0 @@
-pipeline:
-  build-test:
-    image: golang
-    commands:
-      -  go build
-      -  go test
-
-  publish:
-    image: plugins/docker
-    settings:
-      username: realaravinth
-      password:
-        from_secret: DOCKER_TOKEN
-      repo: realaravinth/rageshake
-      tags: latest
--- a/CHANGES.md
+++ b/CHANGES.md
@ -1,64 +1,3 @@
-1.7 (2022-04-14)
-================
-
-Features
--------
-
- Pass the prefix as a unique ID for the rageshake to the generic webhook mechanism. ([\#54](https://github.com/matrix-org/rageshake/issues/54))
-
-
-1.6 (2022-02-22)
-================
-
-Features
--------
-
- Provide ?format=tar.gz option on directory listings to download tarball. ([\#53](https://github.com/matrix-org/rageshake/issues/53))
-
-
-1.5 (2022-02-08)
-================
-
-Features
--------
-
- Allow upload of Files with a .json postfix. ([\#52](https://github.com/matrix-org/rageshake/issues/52))
-
-
-1.4 (2022-02-01)
-================
-
-Features
--------
-
- Allow forwarding of a request to a webhook endpoint. ([\#50](https://github.com/matrix-org/rageshake/issues/50))
-
-
-1.3 (2022-01-25)
-================
-
-Features
--------
-
- Add support for creating GitLab issues. Contributed by @tulir. ([\#37](https://github.com/matrix-org/rageshake/issues/37))
- Support element-android submitting logs with .gz suffix. ([\#40](https://github.com/matrix-org/rageshake/issues/40))
-
-
-Bugfixes
--------
-
- Prevent timestamp collisions when reports are submitted within 1 second of each other. ([\#39](https://github.com/matrix-org/rageshake/issues/39))
-
-
-Internal Changes
----------------
-
- Update minimum Go version to 1.16. ([\#37](https://github.com/matrix-org/rageshake/issues/37), [\#42](https://github.com/matrix-org/rageshake/issues/42))
- Add documentation on the types and formats of files submitted to the rageshake server. ([\#44](https://github.com/matrix-org/rageshake/issues/44))
- Build and push a multi-arch Docker image on the GitHub Container Registry. ([\#47](https://github.com/matrix-org/rageshake/issues/47))
- Add a /health endpoint that always replies with a 200 OK. ([\#48](https://github.com/matrix-org/rageshake/issues/48))
-
-
 1.2 (2020-09-16)
 ================

--- a/33
+++ b/33
@ -1,23 +1,22 @@
-## Build stage ##
-FROM golang as builder
+FROM golang:alpine as builder
+RUN apk add --update --no-cache git ca-certificates
+
+RUN mkdir /build 
 WORKDIR /build
-COPY go.mod go.sum ./
+COPY go.mod .
+COPY go.sum .
 RUN go mod download
+
+
 COPY . .
-RUN go build -o rageshake

-## Runtime stage, debug variant ##
-FROM  debian:bullseye as debug
-COPY --from=builder /build/rageshake /rageshake/
+RUN CGO_ENABLED=0 sh .buildkite/lint.sh
+
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o rageshake
+
+FROM scratch
+COPY --from=builder /build/rageshake /rageshake
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 WORKDIR /
 EXPOSE 9110
-ENTRYPOINT ["/rageshake/rageshake"]
-
-## Runtime stage ##
-FROM  debian:bullseye as rageshake
-LABEL org.opencontainers.image.source https://git.batsense.net/mystiq/rageshake
-RUN apt-get update && apt-get install -y ca-certificates
-WORKDIR /
-COPY --from=builder /build/rageshake /rageshake/
-EXPOSE 9110
-ENTRYPOINT ["/rageshake/rageshake"]
+CMD ["/rageshake"]
--- a/README.md
+++ b/README.md
@ -1,9 +1,8 @@
-WOODPECKER: [![status-badge](https://ci.batsense.net/api/badges/mystiq/rageshake/status.svg)](https://ci.batsense.net/mystiq/rageshake)
 # rageshake [![Build status](https://badge.buildkite.com/76a4362a20b12dcd589f9308a905ffcc537278b9c363c0b5f1.svg?branch=master)](https://buildkite.com/matrix-dot-org/rageshake)

 Web service which collects and serves bug reports.

-rageshake requires Go version 1.16 or later.
+rageshake requires Go version 1.15 or later.

 To run it, do:

@ -29,8 +28,6 @@ Serves submitted bug reports. Protected by basic HTTP auth using the
 username/password provided in the environment. A browsable list, collated by
 report submission date and time.

-A whole directory can be downloaded as a tarball by appending the parameter `?format=tar.gz` to the end of the URL path
-
 ### POST `/api/submit`

 Submission endpoint: this is where applications should send their reports.
@ -71,16 +68,10 @@ logs.)
    * `id`: textual identifier for the logs. Used as the filename, as above.
    * `lines`: log data. Newlines should be  encoded as `\n`, as normal in JSON).

-  A summary of the current log file formats that are uploaded for `log` and
-  `compressed-log`  is [available](docs/submitted_reports.md).
-
 * `compressed-log`: a gzipped logfile. Decompressed and then treated the same as
  `log`.

  Compressed logs are not supported for the JSON upload encoding.
-  
-  A summary of the current log file formats that are uploaded for `log` and
-  `compressed-log` is [available](docs/submitted_reports.md).

 * `file`: an arbitrary file to attach to the report. Saved as-is to disk, and
  a link is added to the github issue. The filename must be in the format
@ -107,14 +98,4 @@ You can get notifications when a new rageshake arrives on the server.

 Currently this tool supports pushing notifications as GitHub issues in a repo,
 through a Slack webhook or by email, cf sample config file for how to
-configure them.
-
-### Generic Webhook Notifications
-
-You can receive a webhook notifications when a new rageshake arrives on the server.
-
-These requests contain all the parsed metadata, and links to the uploaded files, and any github/gitlab
-issues created.
-
-Details on the request and expected response are [available](docs/generic\_webhook.md).
-
+configure them.
--- a/changelog.d/37.feature
+++ b/changelog.d/37.feature
@ -0,0 +1 @@
+Add support for creating GitLab issues. Contributed by @tulir.
--- a/changelog.d/37.misc
+++ b/changelog.d/37.misc
@ -0,0 +1 @@
+Update minimum Go version to 1.15.
--- a/changelog.d/39.bugfix
+++ b/changelog.d/39.bugfix
@ -0,0 +1 @@
+Prevent timestamp collisions when reports are submitted within 1 second of each other.
--- a/changelog.d/40.feature
+++ b/changelog.d/40.feature
@ -0,0 +1 @@
+Support element-android submitting logs with .gz suffix.
--- a/docker-bake.hcl
+++ b/docker-bake.hcl
@ -1,25 +0,0 @@
-// This is what is baked by GitHub Actions
-group "default" { targets = ["regular", "debug"] }
-
-// Targets filled by GitHub Actions: one for the regular tag, one for the debug tag
-target "docker-metadata-action" {}
-target "docker-metadata-action-debug" {}
-
-// This sets the platforms and is further extended by GitHub Actions to set the
-// output and the cache locations
-target "base" {
-  platforms = [
-    "linux/amd64",
-    "linux/arm64",
-    "linux/arm",
-  ]
-}
-
-target "regular" {
-  inherits = ["base", "docker-metadata-action"]
-}
-
-target "debug" {
-  inherits = ["base", "docker-metadata-action-debug"]
-  target = "debug"
-}
--- a/docs/generic_webhook.md
+++ b/docs/generic_webhook.md
@ -1,40 +0,0 @@
-## Generic webhook request
-
-If the configuration option `generic_webhook_urls` is set, then an asynchronous request to
-each endpoint listed will be sent in parallel, after the incoming request is parsed and the
-files are uploaded.
-
-The webhook is designed for notification or other tracking services, and does not contain
-the original log files uploaded.
-
-(If you want the original log files, we suggest to implement the rageshake interface itself).
-
-A sample JSON body is as follows:
-
-```
-{
-  'user_text': 'test\r\n\r\nIssue: No issue link given',
-  'app': 'element-web',
-  'data': {
-    'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:96.0) Gecko/20100101 Firefox/96.0',
-    'Version': '0f15ba34cdf5-react-0f15ba34cdf5-js-0f15ba34cdf5',
-    ...
-    'user_id': '@michaelgoesforawalk:matrix.org'},
-  'labels': None,
-  'logs': [
-    'logs-0000.log.gz',
-    'logs-0001.log.gz',
-    'logs-0002.log.gz',
-  ],
-  'logErrors': None,
-  'files': [
-    'screenshot.png'
-  ],
-  'fileErrors': None,
-  'report_url': 'https://github.com/your-org/your-repo/issues/1251',
-  'listing_url': 'http://your-rageshake-server/api/listing/2022-01-25/154742-OOXBVGIX'
-}
-```
-
-The log and other files can be individually downloaded by concatenating the `listing_url` and the `logs` or `files` name.
-You may need to provide a HTTP basic auth user/pass if configured on your rageshake server.
--- a/docs/submitted_reports.md
+++ b/docs/submitted_reports.md
@ -1,87 +0,0 @@
-# Common report styles
-
-Rageshakes can come from a number of applications, and we provide some practical notes on the generated format.
-
-At present these should not be considered absolute nor a structure to follow; but an attempt to document the currently visible formats as of January 2022.
-
-## Feedback 
-
-Log files are not transmitted; the main feedback is entirely within the user message body.
-
-This occurs from all platforms.
-
-## Element Web / Element Desktop
-
-Log files are transmitted in reverse order (0000 is the youngest) 
-
-Log line format:
-```
-2022-01-17T14:57:20.806Z I Using WebAssembly Olm
-< ---- TIMESTAMP ------> L <-- Message ----
-
-L = log level, (W=Warn, I=Info, etc)
-```
-
-New log files are started each restart of the app, but some log files may not contain all data from the start of the session.
-
-## Element iOS
-
-Crash Log is special and is sent only once (and deleted on the device afterwards)
-
-`crash.log`
-
-Following logs are available, going back in time with ascending number.
-console.log with no number is the current log file.
-```
-console.log (newest)
-console-1.log 
-...
-console-49.log (oldest)
-
-console-nse.log (newest)
-console-nse-1.log 
-...
-console-nse-49.log (oldest)
-
-console-share.log (newest)
-console-share-1.log
-console-share-49.log (oldest)
-```
-
-## Element Android
-
-There is a historical issue with the naming of files, documented in [issue #40](https://github.com/matrix-org/rageshake/issues/40).
-
-Log file 0000 is odd, it contains the logcat data if sent.
-
-Log line format:
-```
-01-17 14:59:30.657 14303 14303 W Activity: Slow Operation: 
-<-- TIMESTAMP ---> <-P-> <-T-> L <-- Message --
-
-L = Log Level (W=Warn, I=Info etc)
-P = Process ID
-T = Thread ID
-```
-Remaining log files are transmitted according to their position in the round-robin logging to file - there will be (up to) 7 files written to in a continious loop; one of the seven will be the oldest, the rest will be in order.
-
-Log line format:
-```
-2022-01-17T13:06:36*838GMT+00:00Z 12226 D/ /Tag: Migration: Importing legacy session
-< ---- TIMESTAMP ---------------> <-P-> L        <-- Message ----
-
-L = log level, (W=Warn, I=Info, etc)
-P = Process ID
-```
-
-Once the fix to #40 is in place, we will see the following files:
-
-```
-logcatError.log
-logcat.log
-crash.log
-keyrequests.log
-log-[1-7].log
-```
-
-Log 1-7 are logs from a round-robin buffer and are ordered but the start point is undefined
--- a/go.mod
+++ b/go.mod
@ -1,11 +1,11 @@
 module github.com/matrix-org/rageshake

-go 1.16
+go 1.15

 require (
 	github.com/google/go-github v0.0.0-20170401000335-12363ffc1001
 	github.com/jordan-wright/email v4.0.1-0.20200824153738-3f5bafa1cd84+incompatible
 	github.com/xanzy/go-gitlab v0.50.2
 	golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288
-	gopkg.in/yaml.v2 v2.2.8
+	gopkg.in/yaml.v2 v2.2.2
 )
--- a/go.sum
+++ b/go.sum
@ -45,6 +45,5 @@ google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQ
 google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
--- a/logserver.go
+++ b/logserver.go
@ -17,7 +17,6 @@ limitations under the License.
 package main

 import (
-	"archive/tar"
 	"compress/gzip"
 	"io"
 	"log"
@ -80,9 +79,10 @@ func serveFile(w http.ResponseWriter, r *http.Request, path string) {
 	// for anti-XSS belt-and-braces, set a very restrictive CSP
 	w.Header().Set("Content-Security-Policy", "default-src: none")

-	// if it's a directory, serve a listing or a tarball
+	// if it's a directory, serve a listing
 	if d.IsDir() {
-		serveDirectory(w, r, path)
+		log.Println("Serving", path)
+		http.ServeFile(w, r, path)
 		return
 	}

@ -119,122 +119,9 @@ func extensionToMimeType(path string) string {
 		return "image/jpeg"
 	}

-	if strings.HasSuffix(path, ".json") {
-		return "application/json"
-	}
 	return "application/octet-stream"
 }

-// Chooses to serve either a directory listing or tarball based on the 'format' parameter.
-func serveDirectory(w http.ResponseWriter, r *http.Request, path string) {
-	format, _ := r.URL.Query()["format"]
-	if len(format) == 1 && format[0] == "tar.gz" {
-		log.Println("Serving tarball of", path)
-		err := serveTarball(w, r, path)
-		if err != nil {
-			msg, code := toHTTPError(err)
-			http.Error(w, msg, code)
-			log.Println("Error", err)
-		}
-		return
-	}
-	log.Println("Serving directory listing of", path)
-	http.ServeFile(w, r, path)
-}
-
-// Streams a dynamically created tar.gz file with the contents of the given directory
-// Will serve a partial, corrupted response if there is a error partway through the
-// operation as we stream the response.
-//
-// The resultant tarball will contain a single directory containing all the files
-// so it can unpack cleanly without overwriting other files.
-//
-// Errors are only returned if generated before the tarball has started being
-// written to the ResponseWriter
-func serveTarball(w http.ResponseWriter, r *http.Request, dir string) error {
-	directory, err := os.Open(dir)
-	if err != nil {
-		return err
-	}
-
-	// Creates a "disposition filename"
-	// Take a URL.path like `/2022-01-10/184843-BZZXEGYH/`
-	// and removes leading and trailing `/` and replaces internal `/` with `_`
-	// to form a suitable filename for use in the content-disposition header
-	// dfilename would turn into `2022-01-10_184843-BZZXEGYH`
-	dfilename := strings.Trim(r.URL.Path, "/")
-	dfilename = strings.Replace(dfilename, "/", "_", -1)
-
-	// There is no application/tgz or similar; return a gzip file as best option.
-	// This tends to trigger archive type tools, which will then use the filename to
-	// identify the contents correctly.
-	w.Header().Set("Content-Type", "application/gzip")
-	w.Header().Set("Content-Disposition", "attachment; filename="+dfilename+".tar.gz")
-
-	files, err := directory.Readdir(-1)
-	if err != nil {
-		return err
-	}
-
-	gzip := gzip.NewWriter(w)
-	defer gzip.Close()
-	targz := tar.NewWriter(gzip)
-	defer targz.Close()
-
-	for _, file := range files {
-		if file.IsDir() {
-			// We avoid including nested directories
-			// This will result in requests for directories with only directories in
-			// to return an empty tarball instead of recursively including directories.
-			// This helps the server remain performant as a download of 'everything' would be slow
-			continue
-		}
-		path := dir + "/" + file.Name()
-		// We use the existing disposition filename to create a base directory structure for the files
-		// so when they are unpacked, they are grouped in a unique folder on disk
-		err := addToArchive(targz, dfilename, path)
-		if err != nil {
-			// From this point we assume that data may have been sent to the client already.
-			// We therefore do not http.Error() after this point, instead closing the stream and
-			// allowing the client to deal with a partial file as if there was a network issue.
-			log.Println("Error streaming tarball", err)
-			return nil
-		}
-	}
-	return nil
-}
-
-// Add a single file into the archive.
-func addToArchive(targz *tar.Writer, dfilename string, filename string) error {
-	file, err := os.Open(filename)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	info, err := file.Stat()
-	if err != nil {
-		return err
-	}
-
-	header, err := tar.FileInfoHeader(info, info.Name())
-	if err != nil {
-		return err
-	}
-	header.Name = dfilename + "/" + info.Name()
-
-	err = targz.WriteHeader(header)
-	if err != nil {
-		return err
-	}
-
-	_, err = io.Copy(targz, file)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
 func serveGzippedFile(w http.ResponseWriter, r *http.Request, path string, size int64) {
 	w.Header().Set("Content-Type", "text/plain; charset=utf-8")

--- a/main.go
+++ b/main.go
@ -71,8 +71,6 @@ type config struct {
 	SMTPUsername string `yaml:"smtp_username"`

 	SMTPPassword string `yaml:"smtp_password"`
-
-	GenericWebhookURLs []string `yaml:"generic_webhook_urls"`
 }

 func basicAuth(handler http.Handler, username, password, realm string) http.Handler {
@ -136,8 +134,6 @@ func main() {
 		log.Fatal("Email address(es) specified but no smtp_server configured. Wrong configuration, aborting...")
 	}

-	genericWebhookClient := configureGenericWebhookClient(cfg)
-
 	apiPrefix := cfg.APIPrefix
 	if apiPrefix == "" {
 		_, port, err := net.SplitHostPort(*bindAddr)
@ -152,7 +148,7 @@ func main() {
 	log.Printf("Using %s/listing as public URI", apiPrefix)

 	rand.Seed(time.Now().UnixNano())
-	http.Handle("/api/submit", &submitServer{ghClient, glClient, apiPrefix, slack, genericWebhookClient, cfg})
+	http.Handle("/api/submit", &submitServer{ghClient, glClient, apiPrefix, slack, cfg})

 	// Make sure bugs directory exists
 	_ = os.Mkdir("bugs", os.ModePerm)
@ -171,26 +167,11 @@ func main() {
 	}
 	http.Handle("/api/listing/", fs)

-	http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprint(w, "ok")
-	})
-
 	log.Println("Listening on", *bindAddr)

 	log.Fatal(http.ListenAndServe(*bindAddr, nil))
 }

-func configureGenericWebhookClient(cfg *config) *http.Client {
-	if len(cfg.GenericWebhookURLs) == 0 {
-		fmt.Println("No generic_webhook_urls configured.")
-		return nil
-	}
-	fmt.Println("Will forward metadata of all requests to ", cfg.GenericWebhookURLs)
-	return &http.Client{
-		Timeout: time.Second * 300,
-	}
-}
-
 func loadConfig(configPath string) (*config, error) {
 	contents, err := ioutil.ReadFile(configPath)
 	if err != nil {
--- a/BIN
+++ b/BIN
--- a/rageshake.sample.yaml
+++ b/rageshake.sample.yaml
@ -50,9 +50,3 @@ email_from: Rageshake <rageshake@matrix.org>
 smtp_server: localhost:25
 smtp_username: myemailuser
 smtp_password: myemailpass
-
-
-# a list of webhook URLs, (see docs/generic_webhook.md)
-generic_webhook_urls: 
-  - https://server.example.com/your-server/api
-  - http://another-server.com/api
--- a/scripts/lint.sh
+++ b/scripts/lint.sh
@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 #
 # check the go source for lint. This is run by CI, and the pre-commit hook.

--- a/submit.go
+++ b/submit.go
@ -57,8 +57,7 @@ type submitServer struct {

 	slack *slackClient

-	genericWebhookClient *http.Client
-	cfg                  *config
+	cfg *config
 }

 // the type of payload which can be uploaded as JSON to the submit endpoint
@ -77,38 +76,19 @@ type jsonLogEntry struct {
 	Lines string `json:"lines"`
 }

-// Stores additional information created during processing of a payload
-type genericWebhookPayload struct {
-	payload
-	// If a github/gitlab report is generated, this is set.
-	ReportURL  string `json:"report_url"`
-	// Complete link to the listing URL that contains all uploaded logs
-	ListingURL string `json:"listing_url"`
+// the payload after parsing
+type parsedPayload struct {
+	UserText   string
+	AppName    string
+	Data       map[string]string
+	Labels     []string
+	Logs       []string
+	LogErrors  []string
+	Files      []string
+	FileErrors []string
 }

-// Stores information about a request made to this server
-type payload struct {
-	// A unique ID for this payload, generated within this server 
-	ID         string            `json:"id"`
-	// A multi-line string containing the user description of the fault. 
-	UserText   string            `json:"user_text"`
-	// A short slug to identify the app making the report
-	AppName    string            `json:"app"`
-	// Arbitrary data to annotate the report
-	Data       map[string]string `json:"data"`
-	// Short labels to group reports
-	Labels     []string          `json:"labels"`
-	// A list of names of logs recognised by the server
-	Logs       []string          `json:"logs"`
-	// Set if there are log parsing errors
-	LogErrors  []string          `json:"logErrors"`
-	// A list of other files (not logs) uploaded as part of the rageshake
-	Files      []string          `json:"files"`
-	// Set if there are file parsing errors
-	FileErrors []string          `json:"fileErrors"`
-}
-
-func (p payload) WriteTo(out io.Writer) {
+func (p parsedPayload) WriteTo(out io.Writer) {
 	fmt.Fprintf(
 		out,
 		"%s\n\nNumber of logs: %d\nApplication: %s\n",
@ -191,11 +171,6 @@ func (s *submitServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		return
 	}

-	// We use this prefix (eg, 2022-05-01/125223-abcde) as a unique identifier for this rageshake.
-	// This is going to be used to uniquely identify rageshakes, even if they are not submitted to
-	// an issue tracker for instance with automatic rageshakes that can be plentiful
-	p.ID = prefix
-
 	resp, err := s.saveReport(req.Context(), *p, reportDir, listingURL)
 	if err != nil {
 		log.Println("Error handling report submission:", err)
@ -210,7 +185,7 @@ func (s *submitServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {

 // parseRequest attempts to parse a received request as a bug report. If
 // the request cannot be parsed, it responds with an error and returns nil.
-func parseRequest(w http.ResponseWriter, req *http.Request, reportDir string) *payload {
+func parseRequest(w http.ResponseWriter, req *http.Request, reportDir string) *parsedPayload {
 	length, err := strconv.Atoi(req.Header.Get("Content-Length"))
 	if err != nil {
 		log.Println("Couldn't parse content-length", err)
@ -246,13 +221,13 @@ func parseRequest(w http.ResponseWriter, req *http.Request, reportDir string) *p
 	return p
 }

-func parseJSONRequest(w http.ResponseWriter, req *http.Request, reportDir string) (*payload, error) {
+func parseJSONRequest(w http.ResponseWriter, req *http.Request, reportDir string) (*parsedPayload, error) {
 	var p jsonPayload
 	if err := json.NewDecoder(req.Body).Decode(&p); err != nil {
 		return nil, err
 	}

-	parsed := payload{
+	parsed := parsedPayload{
 		UserText: strings.TrimSpace(p.Text),
 		Data:     make(map[string]string),
 		Labels:   p.Labels,
@ -307,13 +282,13 @@ func parseJSONRequest(w http.ResponseWriter, req *http.Request, reportDir string
 	return &parsed, nil
 }

-func parseMultipartRequest(w http.ResponseWriter, req *http.Request, reportDir string) (*payload, error) {
+func parseMultipartRequest(w http.ResponseWriter, req *http.Request, reportDir string) (*parsedPayload, error) {
 	rdr, err := req.MultipartReader()
 	if err != nil {
 		return nil, err
 	}

-	p := payload{
+	p := parsedPayload{
 		Data: make(map[string]string),
 	}

@ -332,7 +307,7 @@ func parseMultipartRequest(w http.ResponseWriter, req *http.Request, reportDir s
 	return &p, nil
 }

-func parseFormPart(part *multipart.Part, p *payload, reportDir string) error {
+func parseFormPart(part *multipart.Part, p *parsedPayload, reportDir string) error {
 	defer part.Close()
 	field := part.FormName()
 	partName := part.FileName()
@ -393,7 +368,7 @@ func parseFormPart(part *multipart.Part, p *payload, reportDir string) error {

 // formPartToPayload updates the relevant part of *p from a name/value pair
 // read from the form data.
-func formPartToPayload(field, data string, p *payload) {
+func formPartToPayload(field, data string, p *parsedPayload) {
 	if field == "text" {
 		p.UserText = data
 	} else if field == "app" {
@ -419,7 +394,7 @@ func formPartToPayload(field, data string, p *payload) {
 // * no silly characters (/, ctrl chars, etc)
 //
 // * nothing starting with '.'
-var filenameRegexp = regexp.MustCompile(`^[a-zA-Z0-9_-]+\.(jpg|png|txt|json)$`)
+var filenameRegexp = regexp.MustCompile(`^[a-zA-Z0-9_-]+\.(jpg|png|txt)$`)

 // saveFormPart saves a file upload to the report directory.
 //
@ -460,15 +435,13 @@ func saveLogPart(logNum int, filename string, reader io.Reader, reportDir string
 	// some clients use sensible names (foo.N.log), which we preserve. For
 	// others, we just make up a filename.
 	//
-	// We append a ".gz" extension if not already present, as the final file we store on
-	// disk will be gzipped. The original filename may or may not contain a '.gz' depending
-	// on the client that uploaded it, and if it was uploaded already compressed.
-
+	// Either way, we need to append .gz, because we're compressing it.
 	var leafName string
 	if logRegexp.MatchString(filename) {
-		leafName = filename
-		if !strings.HasSuffix(filename, ".gz") {
-			leafName += ".gz"
+		if strings.HasSuffix(filename, ".gz") {
+			leafName = filename
+		} else {
+			leafName = filename + ".gz"
 		}
 	} else {
 		leafName = fmt.Sprintf("logs-%04d.log.gz", logNum)
@ -493,7 +466,7 @@ func saveLogPart(logNum int, filename string, reader io.Reader, reportDir string
 	return leafName, nil
 }

-func (s *submitServer) saveReport(ctx context.Context, p payload, reportDir, listingURL string) (*submitResponse, error) {
+func (s *submitServer) saveReport(ctx context.Context, p parsedPayload, reportDir, listingURL string) (*submitResponse, error) {
 	var summaryBuf bytes.Buffer
 	resp := submitResponse{}
 	p.WriteTo(&summaryBuf)
@ -517,61 +490,10 @@ func (s *submitServer) saveReport(ctx context.Context, p payload, reportDir, lis
 		return nil, err
 	}

-	if err := s.submitGenericWebhook(p, listingURL, resp.ReportURL); err != nil {
-		return nil, err
-	}
-
 	return &resp, nil
 }

-// submitGenericWebhook submits a basic JSON body to an endpoint configured in the config
-//
-// The request does not include the log body, only the metadata in the payload,
-// with the required listingURL to obtain the logs over http if required.
-//
-// If a github or gitlab issue was previously made, the reportURL will also be passed.
-//
-// Uses a goroutine to handle the http request asynchronously as by this point all critical
-// information has been stored.
-
-func (s *submitServer) submitGenericWebhook(p payload, listingURL string, reportURL string) error {
-	if s.genericWebhookClient == nil {
-		return nil
-	}
-	genericHookPayload := genericWebhookPayload{
-		payload: p,
-		ReportURL:     reportURL,
-		ListingURL:    listingURL,
-	}
-	for _, url := range s.cfg.GenericWebhookURLs {
-		// Enrich the payload with a reportURL and listingURL, to convert a single struct
-		// to JSON easily
-
-		payloadBuffer := new(bytes.Buffer)
-		json.NewEncoder(payloadBuffer).Encode(genericHookPayload)
-		req, err := http.NewRequest("POST", url, payloadBuffer)
-		req.Header.Set("Content-Type", "application/json")
-		if err != nil {
-			log.Println("Unable to submit to URL ", url, " ", err)
-			return err
-		}
-		log.Println("Making generic webhook request to URL ", url)
-		go s.sendGenericWebhook(req)
-	}
-	return nil
-}
-
-func (s *submitServer) sendGenericWebhook(req *http.Request) {
-	resp, err := s.genericWebhookClient.Do(req)
-	if err != nil {
-		log.Println("Unable to submit notification", err)
-	} else {
-		defer resp.Body.Close()
-		log.Println("Got response", resp.Status)
-	}
-}
-
-func (s *submitServer) submitGithubIssue(ctx context.Context, p payload, listingURL string, resp *submitResponse) error {
+func (s *submitServer) submitGithubIssue(ctx context.Context, p parsedPayload, listingURL string, resp *submitResponse) error {
 	if s.ghClient == nil {
 		return nil
 	}
@ -602,7 +524,7 @@ func (s *submitServer) submitGithubIssue(ctx context.Context, p payload, listing
 	return nil
 }

-func (s *submitServer) submitGitlabIssue(p payload, listingURL string, resp *submitResponse) error {
+func (s *submitServer) submitGitlabIssue(p parsedPayload, listingURL string, resp *submitResponse) error {
 	if s.glClient == nil {
 		return nil
 	}
@ -625,7 +547,7 @@ func (s *submitServer) submitGitlabIssue(p payload, listingURL string, resp *sub
 	return nil
 }

-func (s *submitServer) submitSlackNotification(p payload, listingURL string) error {
+func (s *submitServer) submitSlackNotification(p parsedPayload, listingURL string) error {
 	if s.slack == nil {
 		return nil
 	}
@ -643,7 +565,7 @@ func (s *submitServer) submitSlackNotification(p payload, listingURL string) err
 	return nil
 }

-func buildReportTitle(p payload) string {
+func buildReportTitle(p parsedPayload) string {
 	// set the title to the first (non-empty) line of the user's report, if any
 	trimmedUserText := strings.TrimSpace(p.UserText)
 	if trimmedUserText == "" {
@ -657,7 +579,7 @@ func buildReportTitle(p payload) string {
 	return trimmedUserText
 }

-func buildReportBody(p payload, newline, quoteChar string) *bytes.Buffer {
+func buildReportBody(p parsedPayload, newline, quoteChar string) *bytes.Buffer {
 	var bodyBuf bytes.Buffer
 	fmt.Fprintf(&bodyBuf, "User message:\n\n%s\n\n", p.UserText)
 	var dataKeys []string
@ -673,7 +595,7 @@ func buildReportBody(p payload, newline, quoteChar string) *bytes.Buffer {
 	return &bodyBuf
 }

-func buildGenericIssueRequest(p payload, listingURL string) (title, body string) {
+func buildGenericIssueRequest(p parsedPayload, listingURL string) (title, body string) {
 	bodyBuf := buildReportBody(p, "  \n", "`")

 	// Add log links to the body
@ -695,7 +617,7 @@ func buildGenericIssueRequest(p payload, listingURL string) (title, body string)
 	return
 }

-func buildGithubIssueRequest(p payload, listingURL string) github.IssueRequest {
+func buildGithubIssueRequest(p parsedPayload, listingURL string) github.IssueRequest {
 	title, body := buildGenericIssueRequest(p, listingURL)

 	labels := p.Labels
@ -710,7 +632,7 @@ func buildGithubIssueRequest(p payload, listingURL string) github.IssueRequest {
 	}
 }

-func buildGitlabIssueRequest(p payload, listingURL string, labels []string, confidential bool) *gitlab.CreateIssueOptions {
+func buildGitlabIssueRequest(p parsedPayload, listingURL string, labels []string, confidential bool) *gitlab.CreateIssueOptions {
 	title, body := buildGenericIssueRequest(p, listingURL)

 	if p.Labels != nil {
@ -725,7 +647,7 @@ func buildGitlabIssueRequest(p payload, listingURL string, labels []string, conf
 	}
 }

-func (s *submitServer) sendEmail(p payload, reportDir string) error {
+func (s *submitServer) sendEmail(p parsedPayload, reportDir string) error {
 	if len(s.cfg.EmailAddresses) == 0 {
 		return nil
 	}
--- a/submit_test.go
+++ b/submit_test.go
@ -35,7 +35,7 @@ import (
 //
 // if tempDir is empty, a new temp dir is created, and deleted when the test
 // completes.
-func testParsePayload(t *testing.T, body, contentType string, tempDir string) (*payload, *http.Response) {
+func testParsePayload(t *testing.T, body, contentType string, tempDir string) (*parsedPayload, *http.Response) {
 	req, err := http.NewRequest("POST", "/api/submit", strings.NewReader(body))
 	if err != nil {
 		t.Fatal(err)
@ -232,7 +232,7 @@ Content-Type: application/octet-stream
 	return
 }

-func checkParsedMultipartUpload(t *testing.T, p *payload) {
+func checkParsedMultipartUpload(t *testing.T, p *parsedPayload) {
 	wanted := "test words."
 	if p.UserText != wanted {
 		t.Errorf("User text: got %s, want %s", p.UserText, wanted)
@ -478,7 +478,7 @@ user_id: id
 	}
 	var buf bytes.Buffer
 	for _, v := range sample {
-		p := payload{Data: v.data}
+		p := parsedPayload{Data: v.data}
 		buf.Reset()
 		p.WriteTo(&buf)
 		got := strings.TrimSpace(buf.String())
@ -488,7 +488,7 @@ user_id: id
 	}

 	for k, v := range sample {
-		p := payload{Data: v.data}
+		p := parsedPayload{Data: v.data}
 		res := buildGithubIssueRequest(p, "")
 		got := *res.Body
 		if k == 0 {
				`@ -0,0 +1 @@`
				`Add support for creating GitLab issues. Contributed by @tulir.`
				`@ -0,0 +1 @@`
				`Prevent timestamp collisions when reports are submitted within 1 second of each other.`
				`@ -0,0 +1 @@`
				`Support element-android submitting logs with .gz suffix.`