fix: install ca certs

rageshake stores logs and screenshot in the same directory as the binary.
Copying binary to /bin/ will, well, mess /bin/ up. Not good.

.buildkite/build-and-test.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+set -e
+
+cd `dirname $0`/..
+
+go build
+go test

.buildkite/check_changelog.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+#
+# this script is run by buildkite to check that a changelog file exists
+#
+set -e
+
+# we need 19.9 to read config from towncrier.toml
+pip3 install --pre 'towncrier>19.2'
+python3 -m towncrier.check

.buildkite/lint.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+set -e
+
+cd `dirname $0`/..
+
+go get golang.org/x/lint/golint
+go install golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow
+go get github.com/fzipp/gocyclo/cmd/gocyclo
+
+./scripts/lint.sh

.github/CODEOWNERS

@@ -0,0 +1 @@
+*       @richvdh

.github/workflows/docker.yaml

@@ -0,0 +1,84 @@
+name: Docker
+
+on:
+  push:
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    name: Build and push Docker image
+    runs-on: ubuntu-latest
+    env:
+      IMAGE: ghcr.io/${{ github.repository }}
+
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@v2
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: "${{ env.IMAGE }}"
+          bake-target: docker-metadata-action
+          tags: |
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=sha
+
+      - name: Docker meta (debug variant)
+        id: meta-debug
+        uses: docker/metadata-action@v3
+        with:
+          images: "${{ env.IMAGE }}"
+          bake-target: docker-metadata-action-debug
+          tags: |
+            type=ref,event=branch,suffix=-debug
+            type=semver,pattern={{version}},suffix=-debug
+            type=semver,pattern={{major}}.{{minor}},suffix=-debug
+            type=semver,pattern={{major}},suffix=-debug
+            type=sha,suffix=-debug
+
+      - name: Merge buildx bake files
+        run: |
+            jq -s '.[0] * .[1]' ${{ steps.meta.outputs.bake-file }} ${{ steps.meta-debug.outputs.bake-file }} > docker-bake.override.json
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          config-inline: |
+            [registry."docker.io"]
+              mirrors = ["mirror.gcr.io"]
+
+      - name: Login to GitHub Container Registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # For pull-requests, only read from the cache, do not try to push to the
+      # cache or the image itself
+      - name: Build
+        uses: docker/bake-action@v1
+        if: github.event_name == 'pull_request'
+        with:
+          set: |
+            base.cache-from=type=registry,ref=${{ env.IMAGE }}:buildcache
+
+      - name: Build and push
+        uses: docker/bake-action@v1
+        if: github.event_name != 'pull_request'
+        with:
+          set: |
+            base.output=type=image,push=true
+            base.cache-from=type=registry,ref=${{ env.IMAGE }}:buildcache
+            base.cache-to=type=registry,ref=${{ env.IMAGE }}:buildcache,mode=max

.gitignore

@@ -1,3 +1,4 @@
 /bin
 /bugs
 /pkg
+/rageshake.yaml

.travis.yml

@@ -1,8 +0,0 @@
-language: go
-go:
- - 1.7
-install:
- - go get github.com/constabulary/gb/...
- - go get github.com/golang/lint/golint
- - go get github.com/fzipp/gocyclo
-script: bash -x ./hooks/pre-commit

.woodpecker.yml

@@ -0,0 +1,15 @@
+pipeline:
+  build-test:
+    image: golang
+    commands:
+      -  go build
+      -  go test
+
+  publish:
+    image: plugins/docker
+    settings:
+      username: realaravinth
+      password:
+        from_secret: DOCKER_TOKEN
+      repo: realaravinth/rageshake
+      tags: latest

CHANGES.md

@@ -0,0 +1,85 @@
+1.7 (2022-04-14)
+================
+
+Features
+--------
+
+- Pass the prefix as a unique ID for the rageshake to the generic webhook mechanism. ([\#54](https://github.com/matrix-org/rageshake/issues/54))
+
+
+1.6 (2022-02-22)
+================
+
+Features
+--------
+
+- Provide ?format=tar.gz option on directory listings to download tarball. ([\#53](https://github.com/matrix-org/rageshake/issues/53))
+
+
+1.5 (2022-02-08)
+================
+
+Features
+--------
+
+- Allow upload of Files with a .json postfix. ([\#52](https://github.com/matrix-org/rageshake/issues/52))
+
+
+1.4 (2022-02-01)
+================
+
+Features
+--------
+
+- Allow forwarding of a request to a webhook endpoint. ([\#50](https://github.com/matrix-org/rageshake/issues/50))
+
+
+1.3 (2022-01-25)
+================
+
+Features
+--------
+
+- Add support for creating GitLab issues. Contributed by @tulir. ([\#37](https://github.com/matrix-org/rageshake/issues/37))
+- Support element-android submitting logs with .gz suffix. ([\#40](https://github.com/matrix-org/rageshake/issues/40))
+
+
+Bugfixes
+--------
+
+- Prevent timestamp collisions when reports are submitted within 1 second of each other. ([\#39](https://github.com/matrix-org/rageshake/issues/39))
+
+
+Internal Changes
+----------------
+
+- Update minimum Go version to 1.16. ([\#37](https://github.com/matrix-org/rageshake/issues/37), [\#42](https://github.com/matrix-org/rageshake/issues/42))
+- Add documentation on the types and formats of files submitted to the rageshake server. ([\#44](https://github.com/matrix-org/rageshake/issues/44))
+- Build and push a multi-arch Docker image on the GitHub Container Registry. ([\#47](https://github.com/matrix-org/rageshake/issues/47))
+- Add a /health endpoint that always replies with a 200 OK. ([\#48](https://github.com/matrix-org/rageshake/issues/48))
+
+
+1.2 (2020-09-16)
+================
+
+Features
+--------
+
+- Add email support. ([\#35](https://github.com/matrix-org/rageshake/issues/35))
+
+
+1.1 (2020-06-04)
+================
+
+Features
+--------
+
+- Add support for Slack notifications. Contributed by @awesome-manuel. ([\#28](https://github.com/matrix-org/rageshake/issues/28))
+
+
+Internal Changes
+----------------
+
+- Update minimum go version to 1.11. ([\#29](https://github.com/matrix-org/rageshake/issues/29), [\#30](https://github.com/matrix-org/rageshake/issues/30))
+- Replace vendored libraries with `go mod`. ([\#31](https://github.com/matrix-org/rageshake/issues/31))
+- Add Dockerfile. Contributed by @awesome-manuel. ([\#32](https://github.com/matrix-org/rageshake/issues/32))

CONTRIBUTING.rst

@@ -21,10 +21,9 @@ merge this back into the matrix.org 'official' master branch. We use github's
 pull request workflow to review the contribution, and either ask you to make
 any refinements needed or merge it and make them ourselves.
 
-We use Travis for continuous integration, and all pull requests get
+We use Buildkite for continuous integration, and all pull requests get
-automatically tested by Travis: if your change breaks the build, then the PR
+automatically tested: if your change breaks the build, then the PR will show
-will show that there are failed checks, so please check back after a few
+that there are failed checks, so please check back after a few minutes.
-minutes.
 
 Code style
 ~~~~~~~~~~

Dockerfile

@@ -0,0 +1,23 @@
+## Build stage ##
+FROM golang as builder
+WORKDIR /build
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+RUN go build -o rageshake
+
+## Runtime stage, debug variant ##
+FROM  debian:bullseye as debug
+COPY --from=builder /build/rageshake /rageshake/
+WORKDIR /
+EXPOSE 9110
+ENTRYPOINT ["/rageshake/rageshake"]
+
+## Runtime stage ##
+FROM  debian:bullseye as rageshake
+LABEL org.opencontainers.image.source https://git.batsense.net/mystiq/rageshake
+RUN apt-get update && apt-get install -y ca-certificates
+WORKDIR /
+COPY --from=builder /build/rageshake /rageshake/
+EXPOSE 9110
+ENTRYPOINT ["/rageshake/rageshake"]

README.md

@@ -1,18 +1,23 @@
-# rageshake [![Build Status](https://travis-ci.org/matrix-org/rageshake.svg?branch=master)](https://travis-ci.org/matrix-org/rageshake)
+WOODPECKER: [![status-badge](https://ci.batsense.net/api/badges/mystiq/rageshake/status.svg)](https://ci.batsense.net/mystiq/rageshake)
+# rageshake [![Build status](https://badge.buildkite.com/76a4362a20b12dcd589f9308a905ffcc537278b9c363c0b5f1.svg?branch=master)](https://buildkite.com/matrix-dot-org/rageshake)
 
 Web service which collects and serves bug reports.
 
+rageshake requires Go version 1.16 or later.
+
 To run it, do:
 
 ```
-BUGS_USER=<user> BUGS_PASS=<password> go run src/github.com/matrix-org/rageshake/main.go PORT
+go build
+./bin/rageshake
 ```
 
-Example:
+Optional parameters:
 
-```
+ * `-config <path>`: The path to a YAML config file; see
-BUGS_USER=alice BUGS_PASS=secret go run src/github.com/matrix-org/rageshake/main.go 8080
+   [rageshake.sample.yaml](rageshake.sample.yaml) for more information.
-```
+ * `-listen <address>`: TCP network address to listen for HTTP requests
+   on. Example: `:9110`.
 
 ## HTTP endpoints
 
@@ -24,22 +29,92 @@ Serves submitted bug reports. Protected by basic HTTP auth using the
 username/password provided in the environment. A browsable list, collated by
 report submission date and time.
 
+A whole directory can be downloaded as a tarball by appending the parameter `?format=tar.gz` to the end of the URL path
+
 ### POST `/api/submit`
 
 Submission endpoint: this is where applications should send their reports.
 
-The body of the request should be a JSON object with the following fields:
+The body of the request should be a multipart form-data submission, with the
+following form field names. (For backwards compatibility, it can also be a JSON
+object, but multipart is preferred as it allows more efficient transfer of the
+logs.)
 
 * `text`: A textual description of the problem. Included in the
   `details.log.gz` file.
 
 * `user_agent`: Application user-agent.  Included in the `details.log.gz` file.
 
+* `app`: Identifier for the application (eg 'riot-web'). Should correspond to a
+  mapping configured in the configuration file for github issue reporting to
+  work.
+
 * `version`: Application version. Included in the `details.log.gz` file.
 
-* `logs`: an of log files. Each entry in the list should be an object with the
+* `label`: Label to attach to the github issue, and include in the details file.
-  following fields:
+
+  If using the JSON upload encoding, this should be encoded as a `labels` field,
+  whose value should be a list of strings.
+
+* `log`: a log file, with lines separated by newline characters. Multiple log
+  files can be included by including several `log` parts.
+
+  If the log is uploaded with a filename `name.ext`, where `name` contains only
+  alphanumerics, `.`, `-` or `_`, and `ext` is one of `log` or `txt`, then the
+  file saved to disk is based on that. Otherwise, a suitable name is
+  constructed.
+
+  If using the JSON upload encoding, the request object should instead include
+  a single `logs` field, which is an array of objects with the following
+  fields:
+
+    * `id`: textual identifier for the logs. Used as the filename, as above.
+    * `lines`: log data. Newlines should be  encoded as `\n`, as normal in JSON).
+
+  A summary of the current log file formats that are uploaded for `log` and
+  `compressed-log`  is [available](docs/submitted_reports.md).
+
+* `compressed-log`: a gzipped logfile. Decompressed and then treated the same as
+  `log`.
+
+  Compressed logs are not supported for the JSON upload encoding.
+  
+  A summary of the current log file formats that are uploaded for `log` and
+  `compressed-log` is [available](docs/submitted_reports.md).
+
+* `file`: an arbitrary file to attach to the report. Saved as-is to disk, and
+  a link is added to the github issue. The filename must be in the format
+  `name.ext`, where `name` contains only alphanumerics, `-` or `_`, and `ext`
+  is one of `jpg`, `png`, or `txt`.
+
+  Not supported for the JSON upload encoding.
+
+* Any other form field names are interpreted as arbitrary name/value strings to
+  include in the `details.log.gz` file.
+
+  If using the JSON upload encoding, this additional metadata should insted be
+  encoded as a `data` field, whose value should be a JSON map. (Note that the
+  values must be strings; numbers, objects and arrays will be rejected.)
+
+The response (if successful) will be a JSON object with the following fields:
+
+* `report_url`: A URL where the user can track their bug report. Omitted if
+  issue submission was disabled.
+
+## Notifications
+
+You can get notifications when a new rageshake arrives on the server.
+
+Currently this tool supports pushing notifications as GitHub issues in a repo,
+through a Slack webhook or by email, cf sample config file for how to
+configure them.
+
+### Generic Webhook Notifications
+
+You can receive a webhook notifications when a new rageshake arrives on the server.
+
+These requests contain all the parsed metadata, and links to the uploaded files, and any github/gitlab
+issues created.
+
+Details on the request and expected response are [available](docs/generic\_webhook.md).
 
-  * `id`: textual identifier for the logs. Currently ignored.
-  * `lines`: log data. Lines should be separated by newline characters (encoded
-    as `\n`, as normal in JSON).

RELEASING.md

@@ -0,0 +1,30 @@
+1. Set a variable to the version number for convenience:
+   ```sh
+   ver=x.y
+   ```
+1. Update the changelog:
+   ```sh
+   # we need 19.9 to read config from towncrier.toml
+   pip3 install --pre 'towncrier>19.2'
+   towncrier --version=$ver
+   ```
+1. Push your changes:
+   ```sh
+   git add -u && git commit -m $ver && git push
+   ```
+1. Sanity-check the
+   [changelog](https://github.com/matrix-org/rageshake/blob/master/CHANGES.md)
+   and update if need be.
+1. Create a signed tag for the release:
+   ```sh
+   git tag -s v$ver
+   ```
+   Base the tag message on the changelog.
+1. Push the tag:
+   ```sh
+   git push origin tag v$ver
+   ```
+1. Create release on GH project page:
+   ```sh
+   xdg-open https://github.com/matrix-org/rageshake/releases/edit/v$ver
+   ```

changelog.d/.gitignore

@@ -0,0 +1 @@
+!.gitignore

docker-bake.hcl

@@ -0,0 +1,25 @@
+// This is what is baked by GitHub Actions
+group "default" { targets = ["regular", "debug"] }
+
+// Targets filled by GitHub Actions: one for the regular tag, one for the debug tag
+target "docker-metadata-action" {}
+target "docker-metadata-action-debug" {}
+
+// This sets the platforms and is further extended by GitHub Actions to set the
+// output and the cache locations
+target "base" {
+  platforms = [
+    "linux/amd64",
+    "linux/arm64",
+    "linux/arm",
+  ]
+}
+
+target "regular" {
+  inherits = ["base", "docker-metadata-action"]
+}
+
+target "debug" {
+  inherits = ["base", "docker-metadata-action-debug"]
+  target = "debug"
+}

docs/generic_webhook.md

@@ -0,0 +1,40 @@
+## Generic webhook request
+
+If the configuration option `generic_webhook_urls` is set, then an asynchronous request to
+each endpoint listed will be sent in parallel, after the incoming request is parsed and the
+files are uploaded.
+
+The webhook is designed for notification or other tracking services, and does not contain
+the original log files uploaded.
+
+(If you want the original log files, we suggest to implement the rageshake interface itself).
+
+A sample JSON body is as follows:
+
+```
+{
+  'user_text': 'test\r\n\r\nIssue: No issue link given',
+  'app': 'element-web',
+  'data': {
+    'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:96.0) Gecko/20100101 Firefox/96.0',
+    'Version': '0f15ba34cdf5-react-0f15ba34cdf5-js-0f15ba34cdf5',
+    ...
+    'user_id': '@michaelgoesforawalk:matrix.org'},
+  'labels': None,
+  'logs': [
+    'logs-0000.log.gz',
+    'logs-0001.log.gz',
+    'logs-0002.log.gz',
+  ],
+  'logErrors': None,
+  'files': [
+    'screenshot.png'
+  ],
+  'fileErrors': None,
+  'report_url': 'https://github.com/your-org/your-repo/issues/1251',
+  'listing_url': 'http://your-rageshake-server/api/listing/2022-01-25/154742-OOXBVGIX'
+}
+```
+
+The log and other files can be individually downloaded by concatenating the `listing_url` and the `logs` or `files` name.
+You may need to provide a HTTP basic auth user/pass if configured on your rageshake server.

docs/submitted_reports.md

@@ -0,0 +1,87 @@
+# Common report styles
+
+Rageshakes can come from a number of applications, and we provide some practical notes on the generated format.
+
+At present these should not be considered absolute nor a structure to follow; but an attempt to document the currently visible formats as of January 2022.
+
+## Feedback 
+
+Log files are not transmitted; the main feedback is entirely within the user message body.
+
+This occurs from all platforms.
+
+## Element Web / Element Desktop
+
+Log files are transmitted in reverse order (0000 is the youngest) 
+
+Log line format:
+```
+2022-01-17T14:57:20.806Z I Using WebAssembly Olm
+< ---- TIMESTAMP ------> L <-- Message ----
+
+L = log level, (W=Warn, I=Info, etc)
+```
+
+New log files are started each restart of the app, but some log files may not contain all data from the start of the session.
+
+## Element iOS
+
+Crash Log is special and is sent only once (and deleted on the device afterwards)
+
+`crash.log`
+
+Following logs are available, going back in time with ascending number.
+console.log with no number is the current log file.
+```
+console.log (newest)
+console-1.log 
+...
+console-49.log (oldest)
+
+console-nse.log (newest)
+console-nse-1.log 
+...
+console-nse-49.log (oldest)
+
+console-share.log (newest)
+console-share-1.log
+console-share-49.log (oldest)
+```
+
+## Element Android
+
+There is a historical issue with the naming of files, documented in [issue #40](https://github.com/matrix-org/rageshake/issues/40).
+
+Log file 0000 is odd, it contains the logcat data if sent.
+
+Log line format:
+```
+01-17 14:59:30.657 14303 14303 W Activity: Slow Operation: 
+<-- TIMESTAMP ---> <-P-> <-T-> L <-- Message --
+
+L = Log Level (W=Warn, I=Info etc)
+P = Process ID
+T = Thread ID
+```
+Remaining log files are transmitted according to their position in the round-robin logging to file - there will be (up to) 7 files written to in a continious loop; one of the seven will be the oldest, the rest will be in order.
+
+Log line format:
+```
+2022-01-17T13:06:36*838GMT+00:00Z 12226 D/ /Tag: Migration: Importing legacy session
+< ---- TIMESTAMP ---------------> <-P-> L        <-- Message ----
+
+L = log level, (W=Warn, I=Info, etc)
+P = Process ID
+```
+
+Once the fix to #40 is in place, we will see the following files:
+
+```
+logcatError.log
+logcat.log
+crash.log
+keyrequests.log
+log-[1-7].log
+```
+
+Log 1-7 are logs from a round-robin buffer and are ordered but the start point is undefined

go.mod

@@ -0,0 +1,11 @@
+module github.com/matrix-org/rageshake
+
+go 1.16
+
+require (
+	github.com/google/go-github v0.0.0-20170401000335-12363ffc1001
+	github.com/jordan-wright/email v4.0.1-0.20200824153738-3f5bafa1cd84+incompatible
+	github.com/xanzy/go-gitlab v0.50.2
+	golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288
+	gopkg.in/yaml.v2 v2.2.8
+)

go.sum

@@ -0,0 +1,50 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/go-github v0.0.0-20170401000335-12363ffc1001 h1:OK4gfzCBCtPg14E4sYsczwFhjVu1jQJZI+OEOpiTigw=
+github.com/google/go-github v0.0.0-20170401000335-12363ffc1001/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
+github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
+github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs=
+github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
+github.com/jordan-wright/email v4.0.1-0.20200824153738-3f5bafa1cd84+incompatible h1:d60x4RsAHk/UX/0OT8Gc6D7scVvhBbEANpTAWrDhA/I=
+github.com/jordan-wright/email v4.0.1-0.20200824153738-3f5bafa1cd84+incompatible/go.mod h1:1c7szIrayyPPB/987hsnvNzLushdWf4o/79s3P08L8A=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/xanzy/go-gitlab v0.50.2 h1:Qm/um2Jryuqusc6VmN7iZYVTQVzNynzSiuMJDnCU1wE=
+github.com/xanzy/go-gitlab v0.50.2/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288 h1:JIqe8uIcRBHXDQVvZtHwp80ai3Lw3IJAeJEs55Dc1W0=
+golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQk=
+google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

hooks/pre-commit

@@ -2,29 +2,36 @@
 
 set -eu
 
-# make the GIT_DIR and GIT_INDEX_FILE absolute, before we change dir
+# make git_dir and GIT_INDEX_FILE absolute, before we change dir
-export GIT_DIR=$(readlink -f `git rev-parse --git-dir`)
+#
+# (don't actually set GIT_DIR, because it messes up `go get`, and several of
+# the go commands run `go get` indirectly)
+#
+git_dir=$(readlink -f `git rev-parse --git-dir`)
 if [ -n "${GIT_INDEX_FILE:+x}" ]; then
     export GIT_INDEX_FILE=$(readlink -f "$GIT_INDEX_FILE")
 fi
 
 wd=`pwd`
 
-# create a temp dir. The `trap` incantaion will ensure that it is removed again
+# create a temp dir. The `trap` incantation will ensure that it is removed
-# when this script completes.
+# again when this script completes.
 tmpdir=`mktemp -d`
 trap 'rm -rf "$tmpdir"' EXIT
 cd "$tmpdir"
 
-# get a copy of the index
+# get a clean copy of the index (ie, what has been `git add`ed), so that we can
-git checkout-index -a
+# run the checks against what we are about to commit, rather than what is in
+# the working copy.
+git --git-dir="${git_dir}" checkout-index -a
 
 # run our checks
-golint src/...
+go fmt
-go fmt ./src/...
+./scripts/lint.sh
-go tool vet --shadow ./src
+go test
-gocyclo -over 12 src/
+
-gb test
+# we're done with go so can set GIT_DIR
+export GIT_DIR="$git_dir"
 
 # if there are no changes from the index, we are done
 git diff --quiet && exit 0

logserver.go

@@ -0,0 +1,320 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+)
+
+// logServer is an http.handler which will serve up bugreports
+type logServer struct {
+	root string
+}
+
+func (f *logServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	upath := r.URL.Path
+
+	if !strings.HasPrefix(upath, "/") {
+		upath = "/" + upath
+		r.URL.Path = upath
+	}
+
+	log.Println("Serving", upath)
+
+	// eliminate ., .., //, etc
+	upath = path.Clean(upath)
+
+	// reject some dodgy paths. This is based on the code for http.Dir.Open (see https://golang.org/src/net/http/fs.go#L37).
+	//
+	// the check for '..' is a sanity-check because my understanding of `path.Clean` is that it should never return
+	// a value including '..' for input starting with '/'. It's taken from the code for http.ServeFile
+	// (https://golang.org/src/net/http/fs.go#L637).
+	if containsDotDot(upath) || strings.Contains(upath, "\x00") || (filepath.Separator != '/' && strings.IndexRune(upath, filepath.Separator) >= 0) {
+		http.Error(w, "invalid URL path", http.StatusBadRequest)
+		return
+	}
+
+	// convert to abs path
+	upath, err := filepath.Abs(filepath.Join(f.root, filepath.FromSlash(upath)))
+
+	if err != nil {
+		msg, code := toHTTPError(err)
+		http.Error(w, msg, code)
+		return
+	}
+
+	serveFile(w, r, upath)
+}
+
+func serveFile(w http.ResponseWriter, r *http.Request, path string) {
+	d, err := os.Stat(path)
+	if err != nil {
+		msg, code := toHTTPError(err)
+		http.Error(w, msg, code)
+		return
+	}
+
+	// for anti-XSS belt-and-braces, set a very restrictive CSP
+	w.Header().Set("Content-Security-Policy", "default-src: none")
+
+	// if it's a directory, serve a listing or a tarball
+	if d.IsDir() {
+		serveDirectory(w, r, path)
+		return
+	}
+
+	// if it's a gzipped log file, serve it as text
+	if strings.HasSuffix(path, ".gz") {
+		serveGzippedFile(w, r, path, d.Size())
+		return
+	}
+
+	// otherwise, limit ourselves to a number of known-safe content-types, to
+	// guard against XSS vulnerabilities.
+	// http.serveFile preserves the content-type header if one is already set.
+	w.Header().Set("Content-Type", extensionToMimeType(path))
+
+	http.ServeFile(w, r, path)
+}
+
+// extensionToMimeType returns a suitable mime type for the given filename
+//
+// Unlike mime.TypeByExtension, the results are limited to a set of types which
+// should be safe to serve to a browser without introducing XSS vulnerabilities.
+func extensionToMimeType(path string) string {
+	if strings.HasSuffix(path, ".txt") {
+		// anyone uploading text in anything other than utf-8 needs to be
+		// re-educated.
+		return "text/plain; charset=utf-8"
+	}
+
+	if strings.HasSuffix(path, ".png") {
+		return "image/png"
+	}
+
+	if strings.HasSuffix(path, ".jpg") {
+		return "image/jpeg"
+	}
+
+	if strings.HasSuffix(path, ".json") {
+		return "application/json"
+	}
+	return "application/octet-stream"
+}
+
+// Chooses to serve either a directory listing or tarball based on the 'format' parameter.
+func serveDirectory(w http.ResponseWriter, r *http.Request, path string) {
+	format, _ := r.URL.Query()["format"]
+	if len(format) == 1 && format[0] == "tar.gz" {
+		log.Println("Serving tarball of", path)
+		err := serveTarball(w, r, path)
+		if err != nil {
+			msg, code := toHTTPError(err)
+			http.Error(w, msg, code)
+			log.Println("Error", err)
+		}
+		return
+	}
+	log.Println("Serving directory listing of", path)
+	http.ServeFile(w, r, path)
+}
+
+// Streams a dynamically created tar.gz file with the contents of the given directory
+// Will serve a partial, corrupted response if there is a error partway through the
+// operation as we stream the response.
+//
+// The resultant tarball will contain a single directory containing all the files
+// so it can unpack cleanly without overwriting other files.
+//
+// Errors are only returned if generated before the tarball has started being
+// written to the ResponseWriter
+func serveTarball(w http.ResponseWriter, r *http.Request, dir string) error {
+	directory, err := os.Open(dir)
+	if err != nil {
+		return err
+	}
+
+	// Creates a "disposition filename"
+	// Take a URL.path like `/2022-01-10/184843-BZZXEGYH/`
+	// and removes leading and trailing `/` and replaces internal `/` with `_`
+	// to form a suitable filename for use in the content-disposition header
+	// dfilename would turn into `2022-01-10_184843-BZZXEGYH`
+	dfilename := strings.Trim(r.URL.Path, "/")
+	dfilename = strings.Replace(dfilename, "/", "_", -1)
+
+	// There is no application/tgz or similar; return a gzip file as best option.
+	// This tends to trigger archive type tools, which will then use the filename to
+	// identify the contents correctly.
+	w.Header().Set("Content-Type", "application/gzip")
+	w.Header().Set("Content-Disposition", "attachment; filename="+dfilename+".tar.gz")
+
+	files, err := directory.Readdir(-1)
+	if err != nil {
+		return err
+	}
+
+	gzip := gzip.NewWriter(w)
+	defer gzip.Close()
+	targz := tar.NewWriter(gzip)
+	defer targz.Close()
+
+	for _, file := range files {
+		if file.IsDir() {
+			// We avoid including nested directories
+			// This will result in requests for directories with only directories in
+			// to return an empty tarball instead of recursively including directories.
+			// This helps the server remain performant as a download of 'everything' would be slow
+			continue
+		}
+		path := dir + "/" + file.Name()
+		// We use the existing disposition filename to create a base directory structure for the files
+		// so when they are unpacked, they are grouped in a unique folder on disk
+		err := addToArchive(targz, dfilename, path)
+		if err != nil {
+			// From this point we assume that data may have been sent to the client already.
+			// We therefore do not http.Error() after this point, instead closing the stream and
+			// allowing the client to deal with a partial file as if there was a network issue.
+			log.Println("Error streaming tarball", err)
+			return nil
+		}
+	}
+	return nil
+}
+
+// Add a single file into the archive.
+func addToArchive(targz *tar.Writer, dfilename string, filename string) error {
+	file, err := os.Open(filename)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	info, err := file.Stat()
+	if err != nil {
+		return err
+	}
+
+	header, err := tar.FileInfoHeader(info, info.Name())
+	if err != nil {
+		return err
+	}
+	header.Name = dfilename + "/" + info.Name()
+
+	err = targz.WriteHeader(header)
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(targz, file)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func serveGzippedFile(w http.ResponseWriter, r *http.Request, path string, size int64) {
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+
+	acceptsGzip := false
+	splitRune := func(s rune) bool { return s == ' ' || s == '\t' || s == '\n' || s == ',' }
+	for _, hdr := range r.Header["Accept-Encoding"] {
+		for _, enc := range strings.FieldsFunc(hdr, splitRune) {
+			if enc == "gzip" {
+				acceptsGzip = true
+				break
+			}
+		}
+	}
+
+	if acceptsGzip {
+		serveGzip(w, r, path, size)
+	} else {
+		serveUngzipped(w, r, path)
+	}
+}
+
+// serveGzip serves a gzipped file with gzip content-encoding
+func serveGzip(w http.ResponseWriter, r *http.Request, path string, size int64) {
+	f, err := os.Open(path)
+	if err != nil {
+		msg, code := toHTTPError(err)
+		http.Error(w, msg, code)
+		return
+	}
+	defer f.Close()
+
+	w.Header().Set("Content-Encoding", "gzip")
+	w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
+
+	w.WriteHeader(http.StatusOK)
+	io.Copy(w, f)
+}
+
+// serveUngzipped ungzips a gzipped file and serves it
+func serveUngzipped(w http.ResponseWriter, r *http.Request, path string) {
+	f, err := os.Open(path)
+	if err != nil {
+		msg, code := toHTTPError(err)
+		http.Error(w, msg, code)
+		return
+	}
+	defer f.Close()
+
+	gz, err := gzip.NewReader(f)
+	if err != nil {
+		msg, code := toHTTPError(err)
+		http.Error(w, msg, code)
+		return
+	}
+	defer gz.Close()
+
+	w.WriteHeader(http.StatusOK)
+	io.Copy(w, gz)
+}
+
+func toHTTPError(err error) (msg string, httpStatus int) {
+	if os.IsNotExist(err) {
+		return "404 page not found", http.StatusNotFound
+	}
+	if os.IsPermission(err) {
+		return "403 Forbidden", http.StatusForbidden
+	}
+	// Default:
+	return "500 Internal Server Error", http.StatusInternalServerError
+}
+
+func containsDotDot(v string) bool {
+	if !strings.Contains(v, "..") {
+		return false
+	}
+	for _, ent := range strings.FieldsFunc(v, isSlashRune) {
+		if ent == ".." {
+			return true
+		}
+	}
+	return false
+}
+func isSlashRune(r rune) bool { return r == '/' || r == '\\' }

main.go

@@ -0,0 +1,204 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"crypto/subtle"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"net"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/google/go-github/github"
+	"github.com/xanzy/go-gitlab"
+	"golang.org/x/oauth2"
+
+	"gopkg.in/yaml.v2"
+)
+
+var configPath = flag.String("config", "rageshake.yaml", "The path to the config file. For more information, see the config file in this repository.")
+var bindAddr = flag.String("listen", ":9110", "The port to listen on.")
+
+type config struct {
+	// Username and password required to access the bug report listings
+	BugsUser string `yaml:"listings_auth_user"`
+	BugsPass string `yaml:"listings_auth_pass"`
+
+	// External URI to /api
+	APIPrefix string `yaml:"api_prefix"`
+
+	// A GitHub personal access token, to create a GitHub issue for each report.
+	GithubToken string `yaml:"github_token"`
+
+	GithubProjectMappings map[string]string `yaml:"github_project_mappings"`
+
+	GitlabURL   string `yaml:"gitlab_url"`
+	GitlabToken string `yaml:"gitlab_token"`
+
+	GitlabProjectMappings   map[string]int      `yaml:"gitlab_project_mappings"`
+	GitlabProjectLabels     map[string][]string `yaml:"gitlab_project_labels"`
+	GitlabIssueConfidential bool                `yaml:"gitlab_issue_confidential"`
+
+	SlackWebhookURL string `yaml:"slack_webhook_url"`
+
+	EmailAddresses []string `yaml:"email_addresses"`
+
+	EmailFrom string `yaml:"email_from"`
+
+	SMTPServer string `yaml:"smtp_server"`
+
+	SMTPUsername string `yaml:"smtp_username"`
+
+	SMTPPassword string `yaml:"smtp_password"`
+
+	GenericWebhookURLs []string `yaml:"generic_webhook_urls"`
+}
+
+func basicAuth(handler http.Handler, username, password, realm string) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		user, pass, ok := r.BasicAuth() // pull creds from the request
+
+		// check user and pass securely
+		if !ok || subtle.ConstantTimeCompare([]byte(user), []byte(username)) != 1 || subtle.ConstantTimeCompare([]byte(pass), []byte(password)) != 1 {
+			w.Header().Set("WWW-Authenticate", `Basic realm="`+realm+`"`)
+			w.WriteHeader(401)
+			w.Write([]byte("Unauthorised.\n"))
+			return
+		}
+
+		handler.ServeHTTP(w, r)
+	})
+}
+
+func main() {
+	flag.Parse()
+
+	cfg, err := loadConfig(*configPath)
+	if err != nil {
+		log.Fatalf("Invalid config file: %s", err)
+	}
+
+	var ghClient *github.Client
+
+	if cfg.GithubToken == "" {
+		fmt.Println("No github_token configured. Reporting bugs to github is disabled.")
+	} else {
+		ctx := context.Background()
+		ts := oauth2.StaticTokenSource(
+			&oauth2.Token{AccessToken: cfg.GithubToken},
+		)
+		tc := oauth2.NewClient(ctx, ts)
+		tc.Timeout = time.Duration(5) * time.Minute
+		ghClient = github.NewClient(tc)
+	}
+
+	var glClient *gitlab.Client
+	if cfg.GitlabToken == "" {
+		fmt.Println("No gitlab_token configured. Reporting bugs to gitlab is disaled.")
+	} else {
+		glClient, err = gitlab.NewClient(cfg.GitlabToken, gitlab.WithBaseURL(cfg.GitlabURL))
+		if err != nil {
+			// This probably only happens if the base URL is invalid
+			log.Fatalln("Failed to create GitLab client:", err)
+		}
+	}
+
+	var slack *slackClient
+
+	if cfg.SlackWebhookURL == "" {
+		fmt.Println("No slack_webhook_url configured. Reporting bugs to slack is disabled.")
+	} else {
+		slack = newSlackClient(cfg.SlackWebhookURL)
+	}
+
+	if len(cfg.EmailAddresses) > 0 && cfg.SMTPServer == "" {
+		log.Fatal("Email address(es) specified but no smtp_server configured. Wrong configuration, aborting...")
+	}
+
+	genericWebhookClient := configureGenericWebhookClient(cfg)
+
+	apiPrefix := cfg.APIPrefix
+	if apiPrefix == "" {
+		_, port, err := net.SplitHostPort(*bindAddr)
+		if err != nil {
+			log.Fatal(err)
+		}
+		apiPrefix = fmt.Sprintf("http://localhost:%s/api", port)
+	} else {
+		// remove trailing /
+		apiPrefix = strings.TrimRight(apiPrefix, "/")
+	}
+	log.Printf("Using %s/listing as public URI", apiPrefix)
+
+	rand.Seed(time.Now().UnixNano())
+	http.Handle("/api/submit", &submitServer{ghClient, glClient, apiPrefix, slack, genericWebhookClient, cfg})
+
+	// Make sure bugs directory exists
+	_ = os.Mkdir("bugs", os.ModePerm)
+
+	// serve files under "bugs"
+	ls := &logServer{"bugs"}
+	fs := http.StripPrefix("/api/listing/", ls)
+
+	// set auth if env vars exist
+	usr := cfg.BugsUser
+	pass := cfg.BugsPass
+	if usr == "" || pass == "" {
+		fmt.Println("No listings_auth_user/pass configured. No authentication is running for /api/listing")
+	} else {
+		fs = basicAuth(fs, usr, pass, "Riot bug reports")
+	}
+	http.Handle("/api/listing/", fs)
+
+	http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, "ok")
+	})
+
+	log.Println("Listening on", *bindAddr)
+
+	log.Fatal(http.ListenAndServe(*bindAddr, nil))
+}
+
+func configureGenericWebhookClient(cfg *config) *http.Client {
+	if len(cfg.GenericWebhookURLs) == 0 {
+		fmt.Println("No generic_webhook_urls configured.")
+		return nil
+	}
+	fmt.Println("Will forward metadata of all requests to ", cfg.GenericWebhookURLs)
+	return &http.Client{
+		Timeout: time.Second * 300,
+	}
+}
+
+func loadConfig(configPath string) (*config, error) {
+	contents, err := ioutil.ReadFile(configPath)
+	if err != nil {
+		return nil, err
+	}
+	var cfg config
+	if err = yaml.Unmarshal(contents, &cfg); err != nil {
+		return nil, err
+	}
+	return &cfg, nil
+}

rageshake.sample.yaml

@@ -0,0 +1,58 @@
+# username/password pair which will be required to access the bug report
+# listings at `/api/listing`, via HTTP basic auth.  If omitted, there will be
+# *no* authentication on this access!
+listings_auth_user: alice
+listings_auth_pass: secret
+
+# the external URL at which /api is accessible; it is used to add a link to the
+# report to the GitHub issue. If unspecified, based on the listen address.
+# api_prefix: https://riot.im/bugreports
+
+# a GitHub personal access token (https://github.com/settings/tokens), which
+# will be used to create a GitHub issue for each report. It requires
+# `public_repo` scope. If omitted, no issues will be created.
+github_token: secrettoken
+
+# mappings from app name (as submitted in the API) to github repo for issue reporting.
+github_project_mappings:
+   my-app: octocat/HelloWorld
+
+# a GitLab personal access token (https://gitlab.com/-/profile/personal_access_tokens), which
+# will be used to create a GitLab issue for each report. It requires
+# `api` scope. If omitted, no issues will be created.
+gitlab_token: secrettoken
+# the base URL of the GitLab instance to use
+gitlab_url: https://gitlab.com
+
+# mappings from app name (as submitted in the API) to the GitLab Project ID (not name!) for issue reporting.
+gitlab_project_mappings:
+  my-app: 12345
+# mappings from app name to a list of GitLab label names for issue reporting.
+gitlab_project_labels:
+  my-app:
+    - client::my-app
+# whether GitLab issues should be created as confidential issues. Defaults to false.
+gitlab_issue_confidential: true
+
+# a Slack personal webhook URL (https://api.slack.com/incoming-webhooks), which
+# will be used to post a notification on Slack for each report.
+slack_webhook_url: https://hooks.slack.com/services/TTTTTTT/XXXXXXXXXX/YYYYYYYYYYY
+
+# notification can also be pushed by email.
+# this param controls the target emails
+email_addresses:
+  - support@matrix.org
+
+# this is the from field that will be used in the email notifications
+email_from: Rageshake <rageshake@matrix.org>
+
+# SMTP server configuration
+smtp_server: localhost:25
+smtp_username: myemailuser
+smtp_password: myemailpass
+
+
+# a list of webhook URLs, (see docs/generic_webhook.md)
+generic_webhook_urls: 
+  - https://server.example.com/your-server/api
+  - http://another-server.com/api

scripts/lint.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# check the go source for lint. This is run by CI, and the pre-commit hook.
+
+# we *don't* check gofmt here, following the advice at
+# https://golang.org/doc/go1.10#gofmt
+
+set -eu
+
+golint -set_exit_status
+go vet -vettool=$(which shadow)
+gocyclo -over 12 .

slack.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+)
+
+type slackClient struct {
+	webHook string
+	name    string
+	face    string
+}
+
+func newSlackClient(webHook string) *slackClient {
+	return &slackClient{
+		webHook: webHook,
+		name:    "Notifier",
+		face:    "robot_face"}
+}
+
+func (slack *slackClient) Name(name string) {
+	slack.name = name
+}
+
+func (slack *slackClient) Face(face string) {
+	slack.face = face
+}
+
+func (slack slackClient) Notify(text string) error {
+	json := buildRequest(text, slack)
+
+	req, err := http.NewRequest("POST", slack.webHook, strings.NewReader(json))
+	if err != nil {
+		return fmt.Errorf("Can't connect to host %s: %s", slack.webHook, err.Error())
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+
+	client := http.Client{}
+	_, err = client.Do(req)
+
+	return err
+}
+
+func buildRequest(text string, slack slackClient) string {
+	return fmt.Sprintf(`{"text":"%s", "username": "%s", "icon_emoji": ":%s:"}`, text, slack.name, slack.face)
+}

src/github.com/matrix-org/rageshake/main.go

@@ -1,158 +0,0 @@
-/*
-Copyright 2017 Vector Creations Ltd
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
-	"bytes"
-	"compress/gzip"
-	"crypto/subtle"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"os"
-	"path/filepath"
-	"strconv"
-	"time"
-)
-
-var maxPayloadSize = 1024 * 1024 * 55 // 55 MB
-
-type LogEntry struct {
-	ID    string `json:"id"`
-	Lines string `json:"lines"`
-}
-
-type Payload struct {
-	Text      string     `json:"text"`
-	Version   string     `json:"version"`
-	UserAgent string     `json:"user_agent"`
-	Logs      []LogEntry `json:"logs"`
-}
-
-func respond(code int, w http.ResponseWriter) {
-	w.WriteHeader(code)
-	w.Write([]byte("{}"))
-}
-
-func gzipAndSave(data []byte, dirname, fpath string) error {
-	_ = os.MkdirAll(filepath.Join("bugs", dirname), os.ModePerm)
-	fpath = filepath.Join("bugs", dirname, fpath)
-
-	if _, err := os.Stat(fpath); err == nil {
-		return fmt.Errorf("file already exists") // the user can just retry
-	}
-	var b bytes.Buffer
-	gz := gzip.NewWriter(&b)
-	if _, err := gz.Write(data); err != nil {
-		return err
-	}
-	if err := gz.Flush(); err != nil {
-		return err
-	}
-	if err := gz.Close(); err != nil {
-		return err
-	}
-	if err := ioutil.WriteFile(fpath, b.Bytes(), 0644); err != nil {
-		return err
-	}
-	return nil
-}
-
-func basicAuth(handler http.Handler, username, password, realm string) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		user, pass, ok := r.BasicAuth() // pull creds from the request
-
-		// check user and pass securely
-		if !ok || subtle.ConstantTimeCompare([]byte(user), []byte(username)) != 1 || subtle.ConstantTimeCompare([]byte(pass), []byte(password)) != 1 {
-			w.Header().Set("WWW-Authenticate", `Basic realm="`+realm+`"`)
-			w.WriteHeader(401)
-			w.Write([]byte("Unauthorised.\n"))
-			return
-		}
-
-		handler.ServeHTTP(w, r)
-	})
-}
-
-func main() {
-	http.HandleFunc("/api/submit", func(w http.ResponseWriter, req *http.Request) {
-		if req.Method != "POST" && req.Method != "OPTIONS" {
-			respond(405, w)
-			return
-		}
-		// Set CORS
-		w.Header().Set("Access-Control-Allow-Origin", "*")
-		w.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS")
-		w.Header().Set("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept")
-		if req.Method == "OPTIONS" {
-			respond(200, w)
-			return
-		}
-		if length, err := strconv.Atoi(req.Header.Get("Content-Length")); err != nil || length > maxPayloadSize {
-			respond(413, w)
-			return
-		}
-		var p Payload
-		if err := json.NewDecoder(req.Body).Decode(&p); err != nil {
-			respond(400, w)
-			return
-		}
-		// Dump bug report to disk as form:
-		//  "bugreport-20170115-112233.log.gz" => user text, version, user agent, # logs
-		//  "bugreport-20170115-112233-0.log.gz" => most recent log
-		//  "bugreport-20170115-112233-1.log.gz" => ...
-		//  "bugreport-20170115-112233-N.log.gz" => oldest log
-		t := time.Now().UTC()
-		prefix := t.Format("2006-01-02/150405")
-		summary := fmt.Sprintf(
-			"%s\n\nNumber of logs: %d\nVersion: %s\nUser-Agent: %s\n", p.Text, len(p.Logs), p.Version, p.UserAgent,
-		)
-		if err := gzipAndSave([]byte(summary), prefix, "details.log.gz"); err != nil {
-			respond(500, w)
-			return
-		}
-		for i, log := range p.Logs {
-			if err := gzipAndSave([]byte(log.Lines), prefix, fmt.Sprintf("logs-%d.log.gz", i)); err != nil {
-				respond(500, w)
-				return // TODO: Rollback?
-			}
-		}
-		respond(200, w)
-	})
-
-	// Make sure bugs directory exists
-	_ = os.Mkdir("bugs", os.ModePerm)
-
-	// serve files under "bugs"
-	fs := http.FileServer(http.Dir("bugs"))
-	fs = http.StripPrefix("/api/listing/", fs)
-
-	// set auth if env vars exist
-	usr := os.Getenv("BUGS_USER")
-	pass := os.Getenv("BUGS_PASS")
-	if usr == "" || pass == "" {
-		fmt.Println("BUGS_USER and BUGS_PASS env vars not found. No authentication is running for /api/listing")
-	} else {
-		fs = basicAuth(fs, usr, pass, "Riot bug reports")
-	}
-	http.Handle("/api/listing/", fs)
-
-	port := os.Args[1]
-	log.Fatal(http.ListenAndServe(":"+port, nil))
-}

submit.go

@@ -0,0 +1,790 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"encoding/base32"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"mime"
+	"mime/multipart"
+	"net/http"
+	"net/smtp"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/google/go-github/github"
+	"github.com/jordan-wright/email"
+	"github.com/xanzy/go-gitlab"
+)
+
+var maxPayloadSize = 1024 * 1024 * 55 // 55 MB
+
+type submitServer struct {
+	// github client for reporting bugs. may be nil, in which case,
+	// reporting is disabled.
+	ghClient *github.Client
+	glClient *gitlab.Client
+
+	// External URI to /api
+	apiPrefix string
+
+	slack *slackClient
+
+	genericWebhookClient *http.Client
+	cfg                  *config
+}
+
+// the type of payload which can be uploaded as JSON to the submit endpoint
+type jsonPayload struct {
+	Text      string            `json:"text"`
+	AppName   string            `json:"app"`
+	Version   string            `json:"version"`
+	UserAgent string            `json:"user_agent"`
+	Logs      []jsonLogEntry    `json:"logs"`
+	Data      map[string]string `json:"data"`
+	Labels    []string          `json:"labels"`
+}
+
+type jsonLogEntry struct {
+	ID    string `json:"id"`
+	Lines string `json:"lines"`
+}
+
+// Stores additional information created during processing of a payload
+type genericWebhookPayload struct {
+	payload
+	// If a github/gitlab report is generated, this is set.
+	ReportURL  string `json:"report_url"`
+	// Complete link to the listing URL that contains all uploaded logs
+	ListingURL string `json:"listing_url"`
+}
+
+// Stores information about a request made to this server
+type payload struct {
+	// A unique ID for this payload, generated within this server 
+	ID         string            `json:"id"`
+	// A multi-line string containing the user description of the fault. 
+	UserText   string            `json:"user_text"`
+	// A short slug to identify the app making the report
+	AppName    string            `json:"app"`
+	// Arbitrary data to annotate the report
+	Data       map[string]string `json:"data"`
+	// Short labels to group reports
+	Labels     []string          `json:"labels"`
+	// A list of names of logs recognised by the server
+	Logs       []string          `json:"logs"`
+	// Set if there are log parsing errors
+	LogErrors  []string          `json:"logErrors"`
+	// A list of other files (not logs) uploaded as part of the rageshake
+	Files      []string          `json:"files"`
+	// Set if there are file parsing errors
+	FileErrors []string          `json:"fileErrors"`
+}
+
+func (p payload) WriteTo(out io.Writer) {
+	fmt.Fprintf(
+		out,
+		"%s\n\nNumber of logs: %d\nApplication: %s\n",
+		p.UserText, len(p.Logs), p.AppName,
+	)
+	fmt.Fprintf(out, "Labels: %s\n", strings.Join(p.Labels, ", "))
+
+	var dataKeys []string
+	for k := range p.Data {
+		dataKeys = append(dataKeys, k)
+	}
+	sort.Strings(dataKeys)
+	for _, k := range dataKeys {
+		v := p.Data[k]
+		fmt.Fprintf(out, "%s: %s\n", k, v)
+	}
+	if len(p.LogErrors) > 0 {
+		fmt.Fprint(out, "Log upload failures:\n")
+		for _, e := range p.LogErrors {
+			fmt.Fprintf(out, "    %s\n", e)
+		}
+	}
+	if len(p.FileErrors) > 0 {
+		fmt.Fprint(out, "Attachment upload failures:\n")
+		for _, e := range p.FileErrors {
+			fmt.Fprintf(out, "    %s\n", e)
+		}
+	}
+}
+
+type submitResponse struct {
+	ReportURL string `json:"report_url,omitempty"`
+}
+
+func (s *submitServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	// if we attempt to return a response without reading the request body,
+	// apache gets upset and returns a 500. Let's try this.
+	defer req.Body.Close()
+	defer io.Copy(ioutil.Discard, req.Body)
+
+	if req.Method != "POST" && req.Method != "OPTIONS" {
+		respond(405, w)
+		return
+	}
+
+	// Set CORS
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	w.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS")
+	w.Header().Set("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept")
+	if req.Method == "OPTIONS" {
+		respond(200, w)
+		return
+	}
+
+	// create the report dir before parsing the request, so that we can dump
+	// files straight in
+	t := time.Now().UTC()
+	prefix := t.Format("2006-01-02/150405")
+	randBytes := make([]byte, 5)
+	rand.Read(randBytes)
+	prefix += "-" + base32.StdEncoding.EncodeToString(randBytes)
+	reportDir := filepath.Join("bugs", prefix)
+	if err := os.MkdirAll(reportDir, os.ModePerm); err != nil {
+		log.Println("Unable to create report directory", err)
+		http.Error(w, "Internal error", 500)
+		return
+	}
+
+	listingURL := s.apiPrefix + "/listing/" + prefix
+	log.Println("Handling report submission; listing URI will be", listingURL)
+
+	p := parseRequest(w, req, reportDir)
+	if p == nil {
+		// parseRequest already wrote an error, but now let's delete the
+		// useless report dir
+		if err := os.RemoveAll(reportDir); err != nil {
+			log.Printf("Unable to remove report dir %s after invalid upload: %v\n",
+				reportDir, err)
+		}
+		return
+	}
+
+	// We use this prefix (eg, 2022-05-01/125223-abcde) as a unique identifier for this rageshake.
+	// This is going to be used to uniquely identify rageshakes, even if they are not submitted to
+	// an issue tracker for instance with automatic rageshakes that can be plentiful
+	p.ID = prefix
+
+	resp, err := s.saveReport(req.Context(), *p, reportDir, listingURL)
+	if err != nil {
+		log.Println("Error handling report submission:", err)
+		http.Error(w, "Internal error", 500)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(200)
+	json.NewEncoder(w).Encode(resp)
+}
+
+// parseRequest attempts to parse a received request as a bug report. If
+// the request cannot be parsed, it responds with an error and returns nil.
+func parseRequest(w http.ResponseWriter, req *http.Request, reportDir string) *payload {
+	length, err := strconv.Atoi(req.Header.Get("Content-Length"))
+	if err != nil {
+		log.Println("Couldn't parse content-length", err)
+		http.Error(w, "Bad content-length", 400)
+		return nil
+	}
+	if length > maxPayloadSize {
+		log.Println("Content-length", length, "too large")
+		http.Error(w, fmt.Sprintf("Content too large (max %d)", maxPayloadSize), 413)
+		return nil
+	}
+
+	contentType := req.Header.Get("Content-Type")
+	if contentType != "" {
+		d, _, _ := mime.ParseMediaType(contentType)
+		if d == "multipart/form-data" {
+			p, err1 := parseMultipartRequest(w, req, reportDir)
+			if err1 != nil {
+				log.Println("Error parsing multipart data:", err1)
+				http.Error(w, "Bad multipart data", 400)
+				return nil
+			}
+			return p
+		}
+	}
+
+	p, err := parseJSONRequest(w, req, reportDir)
+	if err != nil {
+		log.Println("Error parsing JSON body", err)
+		http.Error(w, fmt.Sprintf("Could not decode payload: %s", err.Error()), 400)
+		return nil
+	}
+	return p
+}
+
+func parseJSONRequest(w http.ResponseWriter, req *http.Request, reportDir string) (*payload, error) {
+	var p jsonPayload
+	if err := json.NewDecoder(req.Body).Decode(&p); err != nil {
+		return nil, err
+	}
+
+	parsed := payload{
+		UserText: strings.TrimSpace(p.Text),
+		Data:     make(map[string]string),
+		Labels:   p.Labels,
+	}
+
+	if p.Data != nil {
+		parsed.Data = p.Data
+	}
+
+	for i, logfile := range p.Logs {
+		buf := bytes.NewBufferString(logfile.Lines)
+		leafName, err := saveLogPart(i, logfile.ID, buf, reportDir)
+		if err != nil {
+			log.Printf("Error saving log %s: %v", leafName, err)
+			parsed.LogErrors = append(parsed.LogErrors, fmt.Sprintf("Error saving log %s: %v", leafName, err))
+		} else {
+			parsed.Logs = append(parsed.Logs, leafName)
+		}
+	}
+
+	// backwards-compatibility hack: current versions of riot-android
+	// don't set 'app', so we don't correctly file github issues.
+	if p.AppName == "" && p.UserAgent == "Android" {
+		parsed.AppName = "riot-android"
+
+		// they also shove lots of stuff into 'Version' which we don't really
+		// want in the github report
+		for _, line := range strings.Split(p.Version, "\n") {
+			line = strings.TrimSpace(line)
+			if line == "" {
+				continue
+			}
+			parts := strings.SplitN(line, ":", 2)
+			key := strings.TrimSpace(parts[0])
+			val := ""
+			if len(parts) > 1 {
+				val = strings.TrimSpace(parts[1])
+			}
+			parsed.Data[key] = val
+		}
+	} else {
+		parsed.AppName = p.AppName
+
+		if p.UserAgent != "" {
+			parsed.Data["User-Agent"] = p.UserAgent
+		}
+		if p.Version != "" {
+			parsed.Data["Version"] = p.Version
+		}
+	}
+
+	return &parsed, nil
+}
+
+func parseMultipartRequest(w http.ResponseWriter, req *http.Request, reportDir string) (*payload, error) {
+	rdr, err := req.MultipartReader()
+	if err != nil {
+		return nil, err
+	}
+
+	p := payload{
+		Data: make(map[string]string),
+	}
+
+	for true {
+		part, err := rdr.NextPart()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+
+		if err = parseFormPart(part, &p, reportDir); err != nil {
+			return nil, err
+		}
+	}
+	return &p, nil
+}
+
+func parseFormPart(part *multipart.Part, p *payload, reportDir string) error {
+	defer part.Close()
+	field := part.FormName()
+	partName := part.FileName()
+
+	var partReader io.Reader
+	if field == "compressed-log" {
+		// decompress logs as we read them.
+		//
+		// we could save the log directly rather than unzipping and re-zipping,
+		// but doing so conveys the benefit of checking the validity of the
+		// gzip at upload time.
+		zrdr, err := gzip.NewReader(part)
+		if err != nil {
+			// we don't reject the whole request if there is an
+			// error reading one attachment.
+			log.Printf("Error unzipping %s: %v", partName, err)
+
+			p.LogErrors = append(p.LogErrors, fmt.Sprintf("Error unzipping %s: %v", partName, err))
+			return nil
+		}
+		defer zrdr.Close()
+		partReader = zrdr
+	} else {
+		// read the field data directly from the multipart part
+		partReader = part
+	}
+
+	if field == "file" {
+		leafName, err := saveFormPart(partName, partReader, reportDir)
+		if err != nil {
+			log.Printf("Error saving %s %s: %v", field, partName, err)
+			p.FileErrors = append(p.FileErrors, fmt.Sprintf("Error saving %s: %v", partName, err))
+		} else {
+			p.Files = append(p.Files, leafName)
+		}
+		return nil
+	}
+
+	if field == "log" || field == "compressed-log" {
+		leafName, err := saveLogPart(len(p.Logs), partName, partReader, reportDir)
+		if err != nil {
+			log.Printf("Error saving %s %s: %v", field, partName, err)
+			p.LogErrors = append(p.LogErrors, fmt.Sprintf("Error saving %s: %v", partName, err))
+		} else {
+			p.Logs = append(p.Logs, leafName)
+		}
+		return nil
+	}
+
+	b, err := ioutil.ReadAll(partReader)
+	if err != nil {
+		return err
+	}
+	data := string(b)
+	formPartToPayload(field, data, p)
+	return nil
+}
+
+// formPartToPayload updates the relevant part of *p from a name/value pair
+// read from the form data.
+func formPartToPayload(field, data string, p *payload) {
+	if field == "text" {
+		p.UserText = data
+	} else if field == "app" {
+		p.AppName = data
+	} else if field == "version" {
+		p.Data["Version"] = data
+	} else if field == "user_agent" {
+		p.Data["User-Agent"] = data
+	} else if field == "label" {
+		p.Labels = append(p.Labels, data)
+	} else {
+		p.Data[field] = data
+	}
+}
+
+// we use a quite restrictive regexp for the filenames; in particular:
+//
+// * a limited set of extensions. We are careful to limit the content-types
+//   we will serve the files with, but somebody might accidentally point an
+//   Apache or nginx at the upload directory, which would serve js files as
+//   application/javascript and open XSS vulnerabilities.
+//
+// * no silly characters (/, ctrl chars, etc)
+//
+// * nothing starting with '.'
+var filenameRegexp = regexp.MustCompile(`^[a-zA-Z0-9_-]+\.(jpg|png|txt|json)$`)
+
+// saveFormPart saves a file upload to the report directory.
+//
+// Returns the leafname of the saved file.
+func saveFormPart(leafName string, reader io.Reader, reportDir string) (string, error) {
+	if !filenameRegexp.MatchString(leafName) {
+		return "", fmt.Errorf("Invalid upload filename")
+	}
+
+	fullName := filepath.Join(reportDir, leafName)
+
+	log.Println("Saving uploaded file", leafName, "to", fullName)
+
+	f, err := os.Create(fullName)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	_, err = io.Copy(f, reader)
+	if err != nil {
+		return "", err
+	}
+
+	return leafName, nil
+}
+
+// we require a sensible extension, and don't allow the filename to start with
+// '.'
+var logRegexp = regexp.MustCompile(`^[a-zA-Z0-9_-][a-zA-Z0-9_.-]*\.(log|txt)(\.gz)?$`)
+
+// saveLogPart saves a log upload to the report directory.
+//
+// Returns the leafname of the saved file.
+func saveLogPart(logNum int, filename string, reader io.Reader, reportDir string) (string, error) {
+	// pick a name to save the log file with.
+	//
+	// some clients use sensible names (foo.N.log), which we preserve. For
+	// others, we just make up a filename.
+	//
+	// We append a ".gz" extension if not already present, as the final file we store on
+	// disk will be gzipped. The original filename may or may not contain a '.gz' depending
+	// on the client that uploaded it, and if it was uploaded already compressed.
+
+	var leafName string
+	if logRegexp.MatchString(filename) {
+		leafName = filename
+		if !strings.HasSuffix(filename, ".gz") {
+			leafName += ".gz"
+		}
+	} else {
+		leafName = fmt.Sprintf("logs-%04d.log.gz", logNum)
+	}
+
+	fullname := filepath.Join(reportDir, leafName)
+
+	f, err := os.Create(fullname)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	gz := gzip.NewWriter(f)
+	defer gz.Close()
+
+	_, err = io.Copy(gz, reader)
+	if err != nil {
+		return "", err
+	}
+
+	return leafName, nil
+}
+
+func (s *submitServer) saveReport(ctx context.Context, p payload, reportDir, listingURL string) (*submitResponse, error) {
+	var summaryBuf bytes.Buffer
+	resp := submitResponse{}
+	p.WriteTo(&summaryBuf)
+	if err := gzipAndSave(summaryBuf.Bytes(), reportDir, "details.log.gz"); err != nil {
+		return nil, err
+	}
+
+	if err := s.submitGithubIssue(ctx, p, listingURL, &resp); err != nil {
+		return nil, err
+	}
+
+	if err := s.submitGitlabIssue(p, listingURL, &resp); err != nil {
+		return nil, err
+	}
+
+	if err := s.submitSlackNotification(p, listingURL); err != nil {
+		return nil, err
+	}
+
+	if err := s.sendEmail(p, reportDir); err != nil {
+		return nil, err
+	}
+
+	if err := s.submitGenericWebhook(p, listingURL, resp.ReportURL); err != nil {
+		return nil, err
+	}
+
+	return &resp, nil
+}
+
+// submitGenericWebhook submits a basic JSON body to an endpoint configured in the config
+//
+// The request does not include the log body, only the metadata in the payload,
+// with the required listingURL to obtain the logs over http if required.
+//
+// If a github or gitlab issue was previously made, the reportURL will also be passed.
+//
+// Uses a goroutine to handle the http request asynchronously as by this point all critical
+// information has been stored.
+
+func (s *submitServer) submitGenericWebhook(p payload, listingURL string, reportURL string) error {
+	if s.genericWebhookClient == nil {
+		return nil
+	}
+	genericHookPayload := genericWebhookPayload{
+		payload: p,
+		ReportURL:     reportURL,
+		ListingURL:    listingURL,
+	}
+	for _, url := range s.cfg.GenericWebhookURLs {
+		// Enrich the payload with a reportURL and listingURL, to convert a single struct
+		// to JSON easily
+
+		payloadBuffer := new(bytes.Buffer)
+		json.NewEncoder(payloadBuffer).Encode(genericHookPayload)
+		req, err := http.NewRequest("POST", url, payloadBuffer)
+		req.Header.Set("Content-Type", "application/json")
+		if err != nil {
+			log.Println("Unable to submit to URL ", url, " ", err)
+			return err
+		}
+		log.Println("Making generic webhook request to URL ", url)
+		go s.sendGenericWebhook(req)
+	}
+	return nil
+}
+
+func (s *submitServer) sendGenericWebhook(req *http.Request) {
+	resp, err := s.genericWebhookClient.Do(req)
+	if err != nil {
+		log.Println("Unable to submit notification", err)
+	} else {
+		defer resp.Body.Close()
+		log.Println("Got response", resp.Status)
+	}
+}
+
+func (s *submitServer) submitGithubIssue(ctx context.Context, p payload, listingURL string, resp *submitResponse) error {
+	if s.ghClient == nil {
+		return nil
+	}
+
+	// submit a github issue
+	ghProj := s.cfg.GithubProjectMappings[p.AppName]
+	if ghProj == "" {
+		log.Println("Not creating GH issue for unknown app", p.AppName)
+		return nil
+	}
+	splits := strings.SplitN(ghProj, "/", 2)
+	if len(splits) < 2 {
+		log.Println("Can't create GH issue for invalid repo", ghProj)
+	}
+	owner, repo := splits[0], splits[1]
+
+	issueReq := buildGithubIssueRequest(p, listingURL)
+
+	issue, _, err := s.ghClient.Issues.Create(ctx, owner, repo, &issueReq)
+	if err != nil {
+		return err
+	}
+
+	log.Println("Created issue:", *issue.HTMLURL)
+
+	resp.ReportURL = *issue.HTMLURL
+
+	return nil
+}
+
+func (s *submitServer) submitGitlabIssue(p payload, listingURL string, resp *submitResponse) error {
+	if s.glClient == nil {
+		return nil
+	}
+
+	glProj := s.cfg.GitlabProjectMappings[p.AppName]
+	glLabels := s.cfg.GitlabProjectLabels[p.AppName]
+
+	issueReq := buildGitlabIssueRequest(p, listingURL, glLabels, s.cfg.GitlabIssueConfidential)
+
+	issue, _, err := s.glClient.Issues.CreateIssue(glProj, issueReq)
+
+	if err != nil {
+		return err
+	}
+
+	log.Println("Created issue:", issue.WebURL)
+
+	resp.ReportURL = issue.WebURL
+
+	return nil
+}
+
+func (s *submitServer) submitSlackNotification(p payload, listingURL string) error {
+	if s.slack == nil {
+		return nil
+	}
+
+	slackBuf := fmt.Sprintf(
+		"%s\nApplication: %s\nReport: %s",
+		p.UserText, p.AppName, listingURL,
+	)
+
+	err := s.slack.Notify(slackBuf)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func buildReportTitle(p payload) string {
+	// set the title to the first (non-empty) line of the user's report, if any
+	trimmedUserText := strings.TrimSpace(p.UserText)
+	if trimmedUserText == "" {
+		return "Untitled report"
+	}
+
+	if i := strings.IndexAny(trimmedUserText, "\r\n"); i >= 0 {
+		return trimmedUserText[0:i]
+	}
+
+	return trimmedUserText
+}
+
+func buildReportBody(p payload, newline, quoteChar string) *bytes.Buffer {
+	var bodyBuf bytes.Buffer
+	fmt.Fprintf(&bodyBuf, "User message:\n\n%s\n\n", p.UserText)
+	var dataKeys []string
+	for k := range p.Data {
+		dataKeys = append(dataKeys, k)
+	}
+	sort.Strings(dataKeys)
+	for _, k := range dataKeys {
+		v := p.Data[k]
+		fmt.Fprintf(&bodyBuf, "%s: %s%s%s%s", k, quoteChar, v, quoteChar, newline)
+	}
+
+	return &bodyBuf
+}
+
+func buildGenericIssueRequest(p payload, listingURL string) (title, body string) {
+	bodyBuf := buildReportBody(p, "  \n", "`")
+
+	// Add log links to the body
+	fmt.Fprintf(bodyBuf, "\n[Logs](%s)", listingURL)
+
+	for _, file := range p.Files {
+		fmt.Fprintf(
+			bodyBuf,
+			" / [%s](%s)",
+			file,
+			listingURL+"/"+file,
+		)
+	}
+
+	title = buildReportTitle(p)
+
+	body = bodyBuf.String()
+
+	return
+}
+
+func buildGithubIssueRequest(p payload, listingURL string) github.IssueRequest {
+	title, body := buildGenericIssueRequest(p, listingURL)
+
+	labels := p.Labels
+	// go-github doesn't like nils
+	if labels == nil {
+		labels = []string{}
+	}
+	return github.IssueRequest{
+		Title:  &title,
+		Body:   &body,
+		Labels: &labels,
+	}
+}
+
+func buildGitlabIssueRequest(p payload, listingURL string, labels []string, confidential bool) *gitlab.CreateIssueOptions {
+	title, body := buildGenericIssueRequest(p, listingURL)
+
+	if p.Labels != nil {
+		labels = append(labels, p.Labels...)
+	}
+
+	return &gitlab.CreateIssueOptions{
+		Title:        &title,
+		Description:  &body,
+		Confidential: &confidential,
+		Labels:       labels,
+	}
+}
+
+func (s *submitServer) sendEmail(p payload, reportDir string) error {
+	if len(s.cfg.EmailAddresses) == 0 {
+		return nil
+	}
+
+	e := email.NewEmail()
+
+	e.From = "Rageshake <rageshake@matrix.org>"
+	if s.cfg.EmailFrom != "" {
+		e.From = s.cfg.EmailFrom
+	}
+
+	e.To = s.cfg.EmailAddresses
+
+	e.Subject = fmt.Sprintf("[%s] %s", p.AppName, buildReportTitle(p))
+
+	e.Text = buildReportBody(p, "\n", "\"").Bytes()
+
+	allFiles := append(p.Files, p.Logs...)
+	for _, file := range allFiles {
+		fullPath := filepath.Join(reportDir, file)
+		e.AttachFile(fullPath)
+	}
+
+	var auth smtp.Auth = nil
+	if s.cfg.SMTPPassword != "" || s.cfg.SMTPUsername != "" {
+		auth = smtp.PlainAuth("", s.cfg.SMTPUsername, s.cfg.SMTPPassword, s.cfg.SMTPServer)
+	}
+	err := e.Send(s.cfg.SMTPServer, auth)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func respond(code int, w http.ResponseWriter) {
+	w.WriteHeader(code)
+	w.Write([]byte("{}"))
+}
+
+func gzipAndSave(data []byte, dirname, fpath string) error {
+	fpath = filepath.Join(dirname, fpath)
+
+	if _, err := os.Stat(fpath); err == nil {
+		return fmt.Errorf("file already exists") // the user can just retry
+	}
+	var b bytes.Buffer
+	gz := gzip.NewWriter(&b)
+	if _, err := gz.Write(data); err != nil {
+		return err
+	}
+	if err := gz.Flush(); err != nil {
+		return err
+	}
+	if err := gz.Close(); err != nil {
+		return err
+	}
+	if err := ioutil.WriteFile(fpath, b.Bytes(), 0644); err != nil {
+		return err
+	}
+	return nil
+}

submit_test.go

@@ -0,0 +1,502 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+// testParsePayload builds a /submit request with the given body, and calls
+// parseRequest with it.
+//
+// if tempDir is empty, a new temp dir is created, and deleted when the test
+// completes.
+func testParsePayload(t *testing.T, body, contentType string, tempDir string) (*payload, *http.Response) {
+	req, err := http.NewRequest("POST", "/api/submit", strings.NewReader(body))
+	if err != nil {
+		t.Fatal(err)
+	}
+	req.Header.Set("Content-Length", strconv.Itoa(len(body)))
+	if contentType != "" {
+		req.Header.Set("Content-Type", contentType)
+	}
+
+	// temporary dir for the uploaded files
+	if tempDir == "" {
+		tempDir = mkTempDir(t)
+		defer os.RemoveAll(tempDir)
+	}
+
+	rr := httptest.NewRecorder()
+	p := parseRequest(rr, req, tempDir)
+	return p, rr.Result()
+}
+
+func TestEmptyJson(t *testing.T) {
+	body := "{}"
+
+	// we just test it is parsed without errors for now
+	p, _ := testParsePayload(t, body, "application/json", "")
+	if p == nil {
+		t.Fatal("parseRequest returned nil")
+	}
+	if len(p.Labels) != 0 {
+		t.Errorf("Labels: got %#v, want []", p.Labels)
+	}
+}
+
+func TestJsonUpload(t *testing.T) {
+	reportDir := mkTempDir(t)
+	defer os.RemoveAll(reportDir)
+
+	body := `{
+    "app": "riot-web",
+    "logs": [
+        {
+            "id": "instance-0.99152119701215051494400738905",
+            "lines": "line1\nline2"
+        }
+    ],
+    "text": "test message",
+    "user_agent": "Mozilla",
+    "version": "0.9.9"
+}`
+
+	p, _ := testParsePayload(t, body, "application/json", reportDir)
+
+	if p == nil {
+		t.Fatal("parseRequest returned nil")
+	}
+
+	wanted := "test message"
+	if p.UserText != wanted {
+		t.Errorf("user text: got %s, want %s", p.UserText, wanted)
+	}
+	wanted = "riot-web"
+	if p.AppName != wanted {
+		t.Errorf("appname: got %s, want %s", p.AppName, wanted)
+	}
+	wanted = "0.9.9"
+	if p.Data["Version"] != wanted {
+		t.Errorf("version: got %s, want %s", p.Data["Version"], wanted)
+	}
+
+	checkUploadedFile(t, reportDir, "logs-0000.log.gz", true, "line1\nline2")
+}
+
+// check that we can unpick the json submitted by the android clients
+func TestUnpickAndroidMangling(t *testing.T) {
+	body := `{"text": "test ylc 001",
+"version": "User : @ylc8001:matrix.org\nPhone : Lenovo P2a42\nVector version: 0:6:9\n",
+"user_agent": "Android"
+}`
+	p, _ := testParsePayload(t, body, "", "")
+	if p == nil {
+		t.Fatal("parseRequest returned nil")
+	}
+	if p.UserText != "test ylc 001" {
+		t.Errorf("user text: got %s, want %s", p.UserText, "test ylc 001")
+	}
+	if p.AppName != "riot-android" {
+		t.Errorf("appname: got %s, want %s", p.AppName, "riot-android")
+	}
+	if p.Data["Version"] != "" {
+		t.Errorf("version: got %s, want ''", p.Data["Version"])
+	}
+	if p.Data["User"] != "@ylc8001:matrix.org" {
+		t.Errorf("data.user: got %s, want %s", p.Data["User"], "@ylc8001:matrix.org")
+	}
+	if p.Data["Phone"] != "Lenovo P2a42" {
+		t.Errorf("data.phone: got %s, want %s", p.Data["Phone"], "Lenovo P2a42")
+	}
+	if p.Data["Vector version"] != "0:6:9" {
+		t.Errorf("data.version: got %s, want %s", p.Data["Version"], "0:6:9")
+	}
+}
+
+func TestMultipartUpload(t *testing.T) {
+	reportDir := mkTempDir(t)
+	defer os.RemoveAll(reportDir)
+
+	p, _ := testParsePayload(t, multipartBody(),
+		"multipart/form-data; boundary=----WebKitFormBoundarySsdgl8Nq9voFyhdO",
+		reportDir,
+	)
+
+	if p == nil {
+		t.Fatal("parseRequest returned nil")
+	}
+
+	checkParsedMultipartUpload(t, p)
+
+	// check logs uploaded correctly
+	checkUploadedFile(t, reportDir, "logs-0000.log.gz", true, "log\nlog\nlog")
+	checkUploadedFile(t, reportDir, "console.0.log.gz", true, "log")
+	checkUploadedFile(t, reportDir, "logs-0002.log.gz", true, "test\n")
+
+	// check file uploaded correctly
+	checkUploadedFile(t, reportDir, "passwd.txt", false, "bibblybobbly")
+	checkUploadedFile(t, reportDir, "crash.log.gz", true, "test\n")
+}
+
+func multipartBody() (body string) {
+	body = `------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="text"
+
+test words.
+------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="app"
+
+riot-web
+------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="version"
+
+UNKNOWN
+------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="user_agent"
+
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36
+------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="test-field"
+
+Test data
+------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="log"; filename="instance-0.215954445471346461492087122412"
+Content-Type: text/plain
+
+log
+log
+log
+------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="log"; filename="console.0.log"
+Content-Type: text/plain
+
+log
+`
+
+	body += `------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="compressed-log"; filename="instance-0.0109372050779190651492004373866"
+Content-Type: application/octet-stream
+
+`
+	body += string([]byte{
+		0x1f, 0x8b, 0x08, 0x00, 0xbf, 0xd8, 0xf5, 0x58, 0x00, 0x03,
+		0x2b, 0x49, 0x2d, 0x2e, 0xe1, 0x02, 0x00,
+		0xc6, 0x35, 0xb9, 0x3b, 0x05, 0x00, 0x00, 0x00,
+		0x0a,
+	})
+
+	body += `------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="file"; filename="passwd.txt"
+Content-Type: application/octet-stream
+
+bibblybobbly
+`
+	body += `------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="compressed-log"; filename="crash.log.gz"
+Content-Type: application/octet-stream
+
+`
+	body += string([]byte{
+		0x1f, 0x8b, 0x08, 0x00, 0xbf, 0xd8, 0xf5, 0x58, 0x00, 0x03,
+		0x2b, 0x49, 0x2d, 0x2e, 0xe1, 0x02, 0x00,
+		0xc6, 0x35, 0xb9, 0x3b, 0x05, 0x00, 0x00, 0x00,
+		0x0a,
+	})
+
+	body += "------WebKitFormBoundarySsdgl8Nq9voFyhdO--\n"
+	return
+}
+
+func checkParsedMultipartUpload(t *testing.T, p *payload) {
+	wanted := "test words."
+	if p.UserText != wanted {
+		t.Errorf("User text: got %s, want %s", p.UserText, wanted)
+	}
+	if len(p.Logs) != 4 {
+		t.Errorf("Log length: got %d, want 4", len(p.Logs))
+	}
+	if len(p.Data) != 3 {
+		t.Errorf("Data length: got %d, want 3", len(p.Data))
+	}
+	if len(p.Labels) != 0 {
+		t.Errorf("Labels: got %#v, want []", p.Labels)
+	}
+	wanted = "Test data"
+	if p.Data["test-field"] != wanted {
+		t.Errorf("test-field: got %s, want %s", p.Data["test-field"], wanted)
+	}
+	wanted = "logs-0000.log.gz"
+	if p.Logs[0] != wanted {
+		t.Errorf("Log 0: got %s, want %s", p.Logs[0], wanted)
+	}
+	wanted = "console.0.log.gz"
+	if p.Logs[1] != wanted {
+		t.Errorf("Log 1: got %s, want %s", p.Logs[1], wanted)
+	}
+	wanted = "logs-0002.log.gz"
+	if p.Logs[2] != wanted {
+		t.Errorf("Log 2: got %s, want %s", p.Logs[2], wanted)
+	}
+	wanted = "crash.log.gz"
+	if p.Logs[3] != wanted {
+		t.Errorf("Log 3: got %s, want %s", p.Logs[3], wanted)
+	}
+}
+
+func TestLabels(t *testing.T) {
+	body := `------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="label"
+
+label1
+------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="label"
+
+label2
+------WebKitFormBoundarySsdgl8Nq9voFyhdO--
+`
+	p, _ := testParsePayload(t, body,
+		"multipart/form-data; boundary=----WebKitFormBoundarySsdgl8Nq9voFyhdO",
+		"",
+	)
+
+	if p == nil {
+		t.Fatal("parseRequest returned nil")
+	}
+
+	wantedLabels := []string{"label1", "label2"}
+	if !stringSlicesEqual(p.Labels, wantedLabels) {
+		t.Errorf("Labels: got %v, want %v", p.Labels, wantedLabels)
+	}
+}
+
+func stringSlicesEqual(got, want []string) bool {
+	if len(got) != len(want) {
+		return false
+	}
+
+	for i := range got {
+		if got[i] != want[i] {
+			return false
+		}
+	}
+	return true
+}
+
+/* FIXME these should just give a message in the details file now
+func TestEmptyFilename(t *testing.T) {
+	body := `------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="file"
+
+file
+------WebKitFormBoundarySsdgl8Nq9voFyhdO--
+`
+	p, resp := testParsePayload(t, body, "multipart/form-data; boundary=----WebKitFormBoundarySsdgl8Nq9voFyhdO", "")
+	if p != nil {
+		t.Error("parsePayload accepted upload with no filename")
+	}
+
+	if resp.StatusCode != 400 {
+		t.Errorf("response code: got %v, want %v", resp.StatusCode, 400)
+	}
+}
+
+func TestBadFilename(t *testing.T) {
+	body := `------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="file"; filename="etc/passwd"
+
+file
+------WebKitFormBoundarySsdgl8Nq9voFyhdO--
+`
+	p, resp := testParsePayload(t, body, "multipart/form-data; boundary=----WebKitFormBoundarySsdgl8Nq9voFyhdO", "")
+	if p != nil {
+		t.Error("parsePayload accepted upload with bad filename")
+	}
+
+	if resp.StatusCode != 400 {
+		t.Errorf("response code: got %v, want %v", resp.StatusCode, 400)
+	}
+}
+*/
+
+func checkUploadedFile(t *testing.T, reportDir, leafName string, gzipped bool, wanted string) {
+	fi, err := os.Open(filepath.Join(reportDir, leafName))
+	if err != nil {
+		t.Errorf("unable to open uploaded file %s: %v", leafName, err)
+		return
+	}
+	defer fi.Close()
+	var rdr io.Reader
+	if !gzipped {
+		rdr = fi
+	} else {
+		gz, err2 := gzip.NewReader(fi)
+		if err2 != nil {
+			t.Errorf("unable to ungzip uploaded file %s: %v", leafName, err2)
+			return
+		}
+		defer gz.Close()
+		rdr = gz
+	}
+	dat, err := ioutil.ReadAll(rdr)
+	if err != nil {
+		t.Errorf("unable to read uploaded file %s: %v", leafName, err)
+		return
+	}
+
+	datstr := string(dat)
+	if datstr != wanted {
+		t.Errorf("File %s: got %s, want %s", leafName, datstr, wanted)
+	}
+}
+
+func mkTempDir(t *testing.T) string {
+	td, err := ioutil.TempDir("", "rageshake_test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return td
+}
+
+/*****************************************************************************
+ *
+ * buildGithubIssueRequest tests
+ */
+
+func TestBuildGithubIssueLeadingNewline(t *testing.T) {
+	body := `------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="text"
+
+
+test words.
+------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="app"
+
+riot-web
+------WebKitFormBoundarySsdgl8Nq9voFyhdO--
+`
+	p, _ := testParsePayload(t, body,
+		"multipart/form-data; boundary=----WebKitFormBoundarySsdgl8Nq9voFyhdO",
+		"",
+	)
+
+	if p == nil {
+		t.Fatal("parseRequest returned nil")
+	}
+
+	issueReq := buildGithubIssueRequest(*p, "http://test/listing/foo")
+
+	if *issueReq.Title != "test words." {
+		t.Errorf("Title: got %s, want %s", *issueReq.Title, "test words.")
+	}
+	expectedBody := "User message:\n\n\ntest words.\n"
+	if !strings.HasPrefix(*issueReq.Body, expectedBody) {
+		t.Errorf("Body: got %s, want %s", *issueReq.Body, expectedBody)
+	}
+}
+
+func TestBuildGithubIssueEmptyBody(t *testing.T) {
+	body := `------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="text"
+
+------WebKitFormBoundarySsdgl8Nq9voFyhdO--
+`
+	p, _ := testParsePayload(t, body,
+		"multipart/form-data; boundary=----WebKitFormBoundarySsdgl8Nq9voFyhdO",
+		"",
+	)
+
+	if p == nil {
+		t.Fatal("parseRequest returned nil")
+	}
+
+	issueReq := buildGithubIssueRequest(*p, "http://test/listing/foo")
+
+	if *issueReq.Title != "Untitled report" {
+		t.Errorf("Title: got %s, want %s", *issueReq.Title, "Untitled report")
+	}
+	expectedBody := "User message:\n\n\n"
+	if !strings.HasPrefix(*issueReq.Body, expectedBody) {
+		t.Errorf("Body: got %s, want %s", *issueReq.Body, expectedBody)
+	}
+}
+
+func TestTestSortDataKeys(t *testing.T) {
+	expect := `
+Number of logs: 0
+Application: 
+Labels: 
+User-Agent: xxx
+Version: 1
+device_id: id
+user_id: id
+	`
+	expect = strings.TrimSpace(expect)
+	sample := []struct {
+		data map[string]string
+	}{
+		{
+			map[string]string{
+				"Version":    "1",
+				"User-Agent": "xxx",
+				"user_id":    "id",
+				"device_id":  "id",
+			},
+		},
+		{
+			map[string]string{
+				"user_id":    "id",
+				"device_id":  "id",
+				"Version":    "1",
+				"User-Agent": "xxx",
+			},
+		},
+	}
+	var buf bytes.Buffer
+	for _, v := range sample {
+		p := payload{Data: v.data}
+		buf.Reset()
+		p.WriteTo(&buf)
+		got := strings.TrimSpace(buf.String())
+		if got != expect {
+			t.Errorf("expected %s got %s", expect, got)
+		}
+	}
+
+	for k, v := range sample {
+		p := payload{Data: v.data}
+		res := buildGithubIssueRequest(p, "")
+		got := *res.Body
+		if k == 0 {
+			expect = got
+			continue
+		}
+		if got != expect {
+			t.Errorf("expected %s got %s", expect, got)
+		}
+	}
+}

towncrier.toml

@@ -0,0 +1,19 @@
+[tool.towncrier]
+    filename = "CHANGES.md"
+    directory = "changelog.d"
+    issue_format = "[\\#{issue}](https://github.com/matrix-org/rageshake/issues/{issue})"
+
+    [[tool.towncrier.type]]
+        directory = "feature"
+        name = "Features"
+        showcontent = true
+
+    [[tool.towncrier.type]]
+        directory = "bugfix"
+        name = "Bugfixes"
+        showcontent = true
+
+    [[tool.towncrier.type]]
+        directory = "misc"
+        name = "Internal Changes"
+        showcontent = true