fix: install ca certs

rageshake stores logs and screenshot in the same directory as the binary.
Copying binary to /bin/ will, well, mess /bin/ up. Not good.

.buildkite/check_changelog.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+#
+# this script is run by buildkite to check that a changelog file exists
+#
+set -e
+
+# we need 19.9 to read config from towncrier.toml
+pip3 install --pre 'towncrier>19.2'
+python3 -m towncrier.check

.buildkite/lint.sh

@@ -6,6 +6,6 @@ cd `dirname $0`/..
 
 go get golang.org/x/lint/golint
 go install golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow
-go get github.com/fzipp/gocyclo
+go get github.com/fzipp/gocyclo/cmd/gocyclo
 
 ./scripts/lint.sh

.github/CODEOWNERS

@@ -0,0 +1 @@
+*       @richvdh

.github/workflows/docker.yaml

@@ -0,0 +1,84 @@
+name: Docker
+
+on:
+  push:
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    name: Build and push Docker image
+    runs-on: ubuntu-latest
+    env:
+      IMAGE: ghcr.io/${{ github.repository }}
+
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@v2
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: "${{ env.IMAGE }}"
+          bake-target: docker-metadata-action
+          tags: |
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=sha
+
+      - name: Docker meta (debug variant)
+        id: meta-debug
+        uses: docker/metadata-action@v3
+        with:
+          images: "${{ env.IMAGE }}"
+          bake-target: docker-metadata-action-debug
+          tags: |
+            type=ref,event=branch,suffix=-debug
+            type=semver,pattern={{version}},suffix=-debug
+            type=semver,pattern={{major}}.{{minor}},suffix=-debug
+            type=semver,pattern={{major}},suffix=-debug
+            type=sha,suffix=-debug
+
+      - name: Merge buildx bake files
+        run: |
+            jq -s '.[0] * .[1]' ${{ steps.meta.outputs.bake-file }} ${{ steps.meta-debug.outputs.bake-file }} > docker-bake.override.json
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          config-inline: |
+            [registry."docker.io"]
+              mirrors = ["mirror.gcr.io"]
+
+      - name: Login to GitHub Container Registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # For pull-requests, only read from the cache, do not try to push to the
+      # cache or the image itself
+      - name: Build
+        uses: docker/bake-action@v1
+        if: github.event_name == 'pull_request'
+        with:
+          set: |
+            base.cache-from=type=registry,ref=${{ env.IMAGE }}:buildcache
+
+      - name: Build and push
+        uses: docker/bake-action@v1
+        if: github.event_name != 'pull_request'
+        with:
+          set: |
+            base.output=type=image,push=true
+            base.cache-from=type=registry,ref=${{ env.IMAGE }}:buildcache
+            base.cache-to=type=registry,ref=${{ env.IMAGE }}:buildcache,mode=max

.woodpecker.yml

@@ -0,0 +1,15 @@
+pipeline:
+  build-test:
+    image: golang
+    commands:
+      -  go build
+      -  go test
+
+  publish:
+    image: plugins/docker
+    settings:
+      username: realaravinth
+      password:
+        from_secret: DOCKER_TOKEN
+      repo: realaravinth/rageshake
+      tags: latest

CHANGES.md

@@ -1,3 +1,73 @@
+1.7 (2022-04-14)
+================
+
+Features
+--------
+
+- Pass the prefix as a unique ID for the rageshake to the generic webhook mechanism. ([\#54](https://github.com/matrix-org/rageshake/issues/54))
+
+
+1.6 (2022-02-22)
+================
+
+Features
+--------
+
+- Provide ?format=tar.gz option on directory listings to download tarball. ([\#53](https://github.com/matrix-org/rageshake/issues/53))
+
+
+1.5 (2022-02-08)
+================
+
+Features
+--------
+
+- Allow upload of Files with a .json postfix. ([\#52](https://github.com/matrix-org/rageshake/issues/52))
+
+
+1.4 (2022-02-01)
+================
+
+Features
+--------
+
+- Allow forwarding of a request to a webhook endpoint. ([\#50](https://github.com/matrix-org/rageshake/issues/50))
+
+
+1.3 (2022-01-25)
+================
+
+Features
+--------
+
+- Add support for creating GitLab issues. Contributed by @tulir. ([\#37](https://github.com/matrix-org/rageshake/issues/37))
+- Support element-android submitting logs with .gz suffix. ([\#40](https://github.com/matrix-org/rageshake/issues/40))
+
+
+Bugfixes
+--------
+
+- Prevent timestamp collisions when reports are submitted within 1 second of each other. ([\#39](https://github.com/matrix-org/rageshake/issues/39))
+
+
+Internal Changes
+----------------
+
+- Update minimum Go version to 1.16. ([\#37](https://github.com/matrix-org/rageshake/issues/37), [\#42](https://github.com/matrix-org/rageshake/issues/42))
+- Add documentation on the types and formats of files submitted to the rageshake server. ([\#44](https://github.com/matrix-org/rageshake/issues/44))
+- Build and push a multi-arch Docker image on the GitHub Container Registry. ([\#47](https://github.com/matrix-org/rageshake/issues/47))
+- Add a /health endpoint that always replies with a 200 OK. ([\#48](https://github.com/matrix-org/rageshake/issues/48))
+
+
+1.2 (2020-09-16)
+================
+
+Features
+--------
+
+- Add email support. ([\#35](https://github.com/matrix-org/rageshake/issues/35))
+
+
 1.1 (2020-06-04)
 ================
 

Dockerfile

@@ -1,18 +1,23 @@
-FROM golang:alpine as builder
+## Build stage ##
-RUN apk add --update --no-cache git ca-certificates
+FROM golang as builder
-
-RUN mkdir /build 
 WORKDIR /build
-COPY go.mod .
+COPY go.mod go.sum ./
-COPY go.sum .
 RUN go mod download
-
 COPY . .
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o rageshake
+RUN go build -o rageshake
 
-FROM scratch
+## Runtime stage, debug variant ##
-COPY --from=builder /build/rageshake /rageshake
+FROM  debian:bullseye as debug
-COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=builder /build/rageshake /rageshake/
 WORKDIR /
 EXPOSE 9110
-CMD ["/rageshake"]
+ENTRYPOINT ["/rageshake/rageshake"]
+
+## Runtime stage ##
+FROM  debian:bullseye as rageshake
+LABEL org.opencontainers.image.source https://git.batsense.net/mystiq/rageshake
+RUN apt-get update && apt-get install -y ca-certificates
+WORKDIR /
+COPY --from=builder /build/rageshake /rageshake/
+EXPOSE 9110
+ENTRYPOINT ["/rageshake/rageshake"]

README.md

@@ -1,8 +1,9 @@
+WOODPECKER: [![status-badge](https://ci.batsense.net/api/badges/mystiq/rageshake/status.svg)](https://ci.batsense.net/mystiq/rageshake)
 # rageshake [![Build status](https://badge.buildkite.com/76a4362a20b12dcd589f9308a905ffcc537278b9c363c0b5f1.svg?branch=master)](https://buildkite.com/matrix-dot-org/rageshake)
 
 Web service which collects and serves bug reports.
 
-rageshake requires Go version 1.11 or later.
+rageshake requires Go version 1.16 or later.
 
 To run it, do:
 
@@ -28,6 +29,8 @@ Serves submitted bug reports. Protected by basic HTTP auth using the
 username/password provided in the environment. A browsable list, collated by
 report submission date and time.
 
+A whole directory can be downloaded as a tarball by appending the parameter `?format=tar.gz` to the end of the URL path
+
 ### POST `/api/submit`
 
 Submission endpoint: this is where applications should send their reports.
@@ -68,10 +71,16 @@ logs.)
     * `id`: textual identifier for the logs. Used as the filename, as above.
     * `lines`: log data. Newlines should be  encoded as `\n`, as normal in JSON).
 
+  A summary of the current log file formats that are uploaded for `log` and
+  `compressed-log`  is [available](docs/submitted_reports.md).
+
 * `compressed-log`: a gzipped logfile. Decompressed and then treated the same as
   `log`.
 
   Compressed logs are not supported for the JSON upload encoding.
+  
+  A summary of the current log file formats that are uploaded for `log` and
+  `compressed-log` is [available](docs/submitted_reports.md).
 
 * `file`: an arbitrary file to attach to the report. Saved as-is to disk, and
   a link is added to the github issue. The filename must be in the format
@@ -91,3 +100,21 @@ The response (if successful) will be a JSON object with the following fields:
 
 * `report_url`: A URL where the user can track their bug report. Omitted if
   issue submission was disabled.
+
+## Notifications
+
+You can get notifications when a new rageshake arrives on the server.
+
+Currently this tool supports pushing notifications as GitHub issues in a repo,
+through a Slack webhook or by email, cf sample config file for how to
+configure them.
+
+### Generic Webhook Notifications
+
+You can receive a webhook notifications when a new rageshake arrives on the server.
+
+These requests contain all the parsed metadata, and links to the uploaded files, and any github/gitlab
+issues created.
+
+Details on the request and expected response are [available](docs/generic\_webhook.md).
+

RELEASING.md

@@ -0,0 +1,30 @@
+1. Set a variable to the version number for convenience:
+   ```sh
+   ver=x.y
+   ```
+1. Update the changelog:
+   ```sh
+   # we need 19.9 to read config from towncrier.toml
+   pip3 install --pre 'towncrier>19.2'
+   towncrier --version=$ver
+   ```
+1. Push your changes:
+   ```sh
+   git add -u && git commit -m $ver && git push
+   ```
+1. Sanity-check the
+   [changelog](https://github.com/matrix-org/rageshake/blob/master/CHANGES.md)
+   and update if need be.
+1. Create a signed tag for the release:
+   ```sh
+   git tag -s v$ver
+   ```
+   Base the tag message on the changelog.
+1. Push the tag:
+   ```sh
+   git push origin tag v$ver
+   ```
+1. Create release on GH project page:
+   ```sh
+   xdg-open https://github.com/matrix-org/rageshake/releases/edit/v$ver
+   ```

docker-bake.hcl

@@ -0,0 +1,25 @@
+// This is what is baked by GitHub Actions
+group "default" { targets = ["regular", "debug"] }
+
+// Targets filled by GitHub Actions: one for the regular tag, one for the debug tag
+target "docker-metadata-action" {}
+target "docker-metadata-action-debug" {}
+
+// This sets the platforms and is further extended by GitHub Actions to set the
+// output and the cache locations
+target "base" {
+  platforms = [
+    "linux/amd64",
+    "linux/arm64",
+    "linux/arm",
+  ]
+}
+
+target "regular" {
+  inherits = ["base", "docker-metadata-action"]
+}
+
+target "debug" {
+  inherits = ["base", "docker-metadata-action-debug"]
+  target = "debug"
+}

docs/generic_webhook.md

@@ -0,0 +1,40 @@
+## Generic webhook request
+
+If the configuration option `generic_webhook_urls` is set, then an asynchronous request to
+each endpoint listed will be sent in parallel, after the incoming request is parsed and the
+files are uploaded.
+
+The webhook is designed for notification or other tracking services, and does not contain
+the original log files uploaded.
+
+(If you want the original log files, we suggest to implement the rageshake interface itself).
+
+A sample JSON body is as follows:
+
+```
+{
+  'user_text': 'test\r\n\r\nIssue: No issue link given',
+  'app': 'element-web',
+  'data': {
+    'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:96.0) Gecko/20100101 Firefox/96.0',
+    'Version': '0f15ba34cdf5-react-0f15ba34cdf5-js-0f15ba34cdf5',
+    ...
+    'user_id': '@michaelgoesforawalk:matrix.org'},
+  'labels': None,
+  'logs': [
+    'logs-0000.log.gz',
+    'logs-0001.log.gz',
+    'logs-0002.log.gz',
+  ],
+  'logErrors': None,
+  'files': [
+    'screenshot.png'
+  ],
+  'fileErrors': None,
+  'report_url': 'https://github.com/your-org/your-repo/issues/1251',
+  'listing_url': 'http://your-rageshake-server/api/listing/2022-01-25/154742-OOXBVGIX'
+}
+```
+
+The log and other files can be individually downloaded by concatenating the `listing_url` and the `logs` or `files` name.
+You may need to provide a HTTP basic auth user/pass if configured on your rageshake server.

docs/submitted_reports.md

@@ -0,0 +1,87 @@
+# Common report styles
+
+Rageshakes can come from a number of applications, and we provide some practical notes on the generated format.
+
+At present these should not be considered absolute nor a structure to follow; but an attempt to document the currently visible formats as of January 2022.
+
+## Feedback 
+
+Log files are not transmitted; the main feedback is entirely within the user message body.
+
+This occurs from all platforms.
+
+## Element Web / Element Desktop
+
+Log files are transmitted in reverse order (0000 is the youngest) 
+
+Log line format:
+```
+2022-01-17T14:57:20.806Z I Using WebAssembly Olm
+< ---- TIMESTAMP ------> L <-- Message ----
+
+L = log level, (W=Warn, I=Info, etc)
+```
+
+New log files are started each restart of the app, but some log files may not contain all data from the start of the session.
+
+## Element iOS
+
+Crash Log is special and is sent only once (and deleted on the device afterwards)
+
+`crash.log`
+
+Following logs are available, going back in time with ascending number.
+console.log with no number is the current log file.
+```
+console.log (newest)
+console-1.log 
+...
+console-49.log (oldest)
+
+console-nse.log (newest)
+console-nse-1.log 
+...
+console-nse-49.log (oldest)
+
+console-share.log (newest)
+console-share-1.log
+console-share-49.log (oldest)
+```
+
+## Element Android
+
+There is a historical issue with the naming of files, documented in [issue #40](https://github.com/matrix-org/rageshake/issues/40).
+
+Log file 0000 is odd, it contains the logcat data if sent.
+
+Log line format:
+```
+01-17 14:59:30.657 14303 14303 W Activity: Slow Operation: 
+<-- TIMESTAMP ---> <-P-> <-T-> L <-- Message --
+
+L = Log Level (W=Warn, I=Info etc)
+P = Process ID
+T = Thread ID
+```
+Remaining log files are transmitted according to their position in the round-robin logging to file - there will be (up to) 7 files written to in a continious loop; one of the seven will be the oldest, the rest will be in order.
+
+Log line format:
+```
+2022-01-17T13:06:36*838GMT+00:00Z 12226 D/ /Tag: Migration: Importing legacy session
+< ---- TIMESTAMP ---------------> <-P-> L        <-- Message ----
+
+L = log level, (W=Warn, I=Info, etc)
+P = Process ID
+```
+
+Once the fix to #40 is in place, we will see the following files:
+
+```
+logcatError.log
+logcat.log
+crash.log
+keyrequests.log
+log-[1-7].log
+```
+
+Log 1-7 are logs from a round-robin buffer and are ordered but the start point is undefined

go.mod

@@ -1,16 +1,11 @@
 module github.com/matrix-org/rageshake
 
+go 1.16
+
 require (
-	cloud.google.com/go v0.0.0-20170406015231-675fad27ef35
-	github.com/golang/protobuf v0.0.0-20170331031902-2bba0603135d
-	github.com/google/go-genproto v0.0.0-20170404132009-411e09b969b1
 	github.com/google/go-github v0.0.0-20170401000335-12363ffc1001
-	github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135
+	github.com/jordan-wright/email v4.0.1-0.20200824153738-3f5bafa1cd84+incompatible
-	github.com/googleapis/gax-go v0.0.0-20170321005343-9af46dd5a171
+	github.com/xanzy/go-gitlab v0.50.2
-	github.com/pkg/errors v0.0.0-20171018195549-f15c970de5b7
+	golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288
-	golang.org/x/net v0.0.0-20170329170435-ffcf1bedda3b
+	gopkg.in/yaml.v2 v2.2.8
-	golang.org/x/oauth2 v0.0.0-20170321013421-7fdf09982454
-	golang.org/x/text v0.0.0-20170401064109-f4b4367115ec
-	google.golang.org/grpc v0.0.0-20170405173540-b5071124392b
-	gopkg.in/yaml.v2 v2.0.0-20170407172122-cd8b52f8269e
 )

go.sum

@@ -1,17 +1,50 @@
-cloud.google.com/go v0.0.0-20170406015231-675fad27ef35/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/golang/protobuf v0.0.0-20170331031902-2bba0603135d/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/google/go-genproto v0.0.0-20170404132009-411e09b969b1/go.mod h1:3Rcd9jSoLVkV/osPrt5CogLvLiarfI8U9/x78NwhuDU=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/go-github v0.0.0-20170401000335-12363ffc1001 h1:OK4gfzCBCtPg14E4sYsczwFhjVu1jQJZI+OEOpiTigw=
 github.com/google/go-github v0.0.0-20170401000335-12363ffc1001/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
-github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 h1:zLTLjkaOFEFIOxY5BWLFLwh+cL8vOBW4XJ2aqLE/Tf0=
+github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
-github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
-github.com/googleapis/gax-go v0.0.0-20170321005343-9af46dd5a171/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
+github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
-github.com/pkg/errors v0.0.0-20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-golang.org/x/net v0.0.0-20170329170435-ffcf1bedda3b h1:Co3zyosPfwWowmu8+roHGC+aDgizpCPH3ukhubZ0Ttg=
+github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
-golang.org/x/net v0.0.0-20170329170435-ffcf1bedda3b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
-golang.org/x/oauth2 v0.0.0-20170321013421-7fdf09982454 h1:qH7SPXL1bLgpFB+ycaFjqQ2lI54cG8OGelAQGpmZSnc=
+github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs=
-golang.org/x/oauth2 v0.0.0-20170321013421-7fdf09982454/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
-golang.org/x/text v0.0.0-20170401064109-f4b4367115ec/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+github.com/jordan-wright/email v4.0.1-0.20200824153738-3f5bafa1cd84+incompatible h1:d60x4RsAHk/UX/0OT8Gc6D7scVvhBbEANpTAWrDhA/I=
-google.golang.org/grpc v0.0.0-20170405173540-b5071124392b/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
+github.com/jordan-wright/email v4.0.1-0.20200824153738-3f5bafa1cd84+incompatible/go.mod h1:1c7szIrayyPPB/987hsnvNzLushdWf4o/79s3P08L8A=
-gopkg.in/yaml.v2 v2.0.0-20170407172122-cd8b52f8269e h1:o/mfNjxpTLivuKEfxzzwrJ8PmulH2wEp7t713uMwKAA=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-gopkg.in/yaml.v2 v2.0.0-20170407172122-cd8b52f8269e/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/xanzy/go-gitlab v0.50.2 h1:Qm/um2Jryuqusc6VmN7iZYVTQVzNynzSiuMJDnCU1wE=
+github.com/xanzy/go-gitlab v0.50.2/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288 h1:JIqe8uIcRBHXDQVvZtHwp80ai3Lw3IJAeJEs55Dc1W0=
+golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQk=
+google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

logserver.go

@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+	"archive/tar"
 	"compress/gzip"
 	"io"
 	"log"
@@ -79,10 +80,9 @@ func serveFile(w http.ResponseWriter, r *http.Request, path string) {
 	// for anti-XSS belt-and-braces, set a very restrictive CSP
 	w.Header().Set("Content-Security-Policy", "default-src: none")
 
-	// if it's a directory, serve a listing
+	// if it's a directory, serve a listing or a tarball
 	if d.IsDir() {
-		log.Println("Serving", path)
+		serveDirectory(w, r, path)
-		http.ServeFile(w, r, path)
 		return
 	}
 
@@ -119,9 +119,122 @@ func extensionToMimeType(path string) string {
 		return "image/jpeg"
 	}
 
+	if strings.HasSuffix(path, ".json") {
+		return "application/json"
+	}
 	return "application/octet-stream"
 }
 
+// Chooses to serve either a directory listing or tarball based on the 'format' parameter.
+func serveDirectory(w http.ResponseWriter, r *http.Request, path string) {
+	format, _ := r.URL.Query()["format"]
+	if len(format) == 1 && format[0] == "tar.gz" {
+		log.Println("Serving tarball of", path)
+		err := serveTarball(w, r, path)
+		if err != nil {
+			msg, code := toHTTPError(err)
+			http.Error(w, msg, code)
+			log.Println("Error", err)
+		}
+		return
+	}
+	log.Println("Serving directory listing of", path)
+	http.ServeFile(w, r, path)
+}
+
+// Streams a dynamically created tar.gz file with the contents of the given directory
+// Will serve a partial, corrupted response if there is a error partway through the
+// operation as we stream the response.
+//
+// The resultant tarball will contain a single directory containing all the files
+// so it can unpack cleanly without overwriting other files.
+//
+// Errors are only returned if generated before the tarball has started being
+// written to the ResponseWriter
+func serveTarball(w http.ResponseWriter, r *http.Request, dir string) error {
+	directory, err := os.Open(dir)
+	if err != nil {
+		return err
+	}
+
+	// Creates a "disposition filename"
+	// Take a URL.path like `/2022-01-10/184843-BZZXEGYH/`
+	// and removes leading and trailing `/` and replaces internal `/` with `_`
+	// to form a suitable filename for use in the content-disposition header
+	// dfilename would turn into `2022-01-10_184843-BZZXEGYH`
+	dfilename := strings.Trim(r.URL.Path, "/")
+	dfilename = strings.Replace(dfilename, "/", "_", -1)
+
+	// There is no application/tgz or similar; return a gzip file as best option.
+	// This tends to trigger archive type tools, which will then use the filename to
+	// identify the contents correctly.
+	w.Header().Set("Content-Type", "application/gzip")
+	w.Header().Set("Content-Disposition", "attachment; filename="+dfilename+".tar.gz")
+
+	files, err := directory.Readdir(-1)
+	if err != nil {
+		return err
+	}
+
+	gzip := gzip.NewWriter(w)
+	defer gzip.Close()
+	targz := tar.NewWriter(gzip)
+	defer targz.Close()
+
+	for _, file := range files {
+		if file.IsDir() {
+			// We avoid including nested directories
+			// This will result in requests for directories with only directories in
+			// to return an empty tarball instead of recursively including directories.
+			// This helps the server remain performant as a download of 'everything' would be slow
+			continue
+		}
+		path := dir + "/" + file.Name()
+		// We use the existing disposition filename to create a base directory structure for the files
+		// so when they are unpacked, they are grouped in a unique folder on disk
+		err := addToArchive(targz, dfilename, path)
+		if err != nil {
+			// From this point we assume that data may have been sent to the client already.
+			// We therefore do not http.Error() after this point, instead closing the stream and
+			// allowing the client to deal with a partial file as if there was a network issue.
+			log.Println("Error streaming tarball", err)
+			return nil
+		}
+	}
+	return nil
+}
+
+// Add a single file into the archive.
+func addToArchive(targz *tar.Writer, dfilename string, filename string) error {
+	file, err := os.Open(filename)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	info, err := file.Stat()
+	if err != nil {
+		return err
+	}
+
+	header, err := tar.FileInfoHeader(info, info.Name())
+	if err != nil {
+		return err
+	}
+	header.Name = dfilename + "/" + info.Name()
+
+	err = targz.WriteHeader(header)
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(targz, file)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func serveGzippedFile(w http.ResponseWriter, r *http.Request, path string, size int64) {
 	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
 

main.go

@@ -21,17 +21,20 @@ import (
 	"crypto/subtle"
 	"flag"
 	"fmt"
-	"github.com/google/go-github/github"
-	"golang.org/x/oauth2"
 	"io/ioutil"
 	"log"
+	"math/rand"
 	"net"
 	"net/http"
 	"os"
 	"strings"
 	"time"
 
-	yaml "gopkg.in/yaml.v2"
+	"github.com/google/go-github/github"
+	"github.com/xanzy/go-gitlab"
+	"golang.org/x/oauth2"
+
+	"gopkg.in/yaml.v2"
 )
 
 var configPath = flag.String("config", "rageshake.yaml", "The path to the config file. For more information, see the config file in this repository.")
@@ -50,7 +53,26 @@ type config struct {
 
 	GithubProjectMappings map[string]string `yaml:"github_project_mappings"`
 
+	GitlabURL   string `yaml:"gitlab_url"`
+	GitlabToken string `yaml:"gitlab_token"`
+
+	GitlabProjectMappings   map[string]int      `yaml:"gitlab_project_mappings"`
+	GitlabProjectLabels     map[string][]string `yaml:"gitlab_project_labels"`
+	GitlabIssueConfidential bool                `yaml:"gitlab_issue_confidential"`
+
 	SlackWebhookURL string `yaml:"slack_webhook_url"`
+
+	EmailAddresses []string `yaml:"email_addresses"`
+
+	EmailFrom string `yaml:"email_from"`
+
+	SMTPServer string `yaml:"smtp_server"`
+
+	SMTPUsername string `yaml:"smtp_username"`
+
+	SMTPPassword string `yaml:"smtp_password"`
+
+	GenericWebhookURLs []string `yaml:"generic_webhook_urls"`
 }
 
 func basicAuth(handler http.Handler, username, password, realm string) http.Handler {
@@ -91,6 +113,17 @@ func main() {
 		ghClient = github.NewClient(tc)
 	}
 
+	var glClient *gitlab.Client
+	if cfg.GitlabToken == "" {
+		fmt.Println("No gitlab_token configured. Reporting bugs to gitlab is disaled.")
+	} else {
+		glClient, err = gitlab.NewClient(cfg.GitlabToken, gitlab.WithBaseURL(cfg.GitlabURL))
+		if err != nil {
+			// This probably only happens if the base URL is invalid
+			log.Fatalln("Failed to create GitLab client:", err)
+		}
+	}
+
 	var slack *slackClient
 
 	if cfg.SlackWebhookURL == "" {
@@ -99,6 +132,12 @@ func main() {
 		slack = newSlackClient(cfg.SlackWebhookURL)
 	}
 
+	if len(cfg.EmailAddresses) > 0 && cfg.SMTPServer == "" {
+		log.Fatal("Email address(es) specified but no smtp_server configured. Wrong configuration, aborting...")
+	}
+
+	genericWebhookClient := configureGenericWebhookClient(cfg)
+
 	apiPrefix := cfg.APIPrefix
 	if apiPrefix == "" {
 		_, port, err := net.SplitHostPort(*bindAddr)
@@ -112,7 +151,8 @@ func main() {
 	}
 	log.Printf("Using %s/listing as public URI", apiPrefix)
 
-	http.Handle("/api/submit", &submitServer{ghClient, apiPrefix, cfg.GithubProjectMappings, slack})
+	rand.Seed(time.Now().UnixNano())
+	http.Handle("/api/submit", &submitServer{ghClient, glClient, apiPrefix, slack, genericWebhookClient, cfg})
 
 	// Make sure bugs directory exists
 	_ = os.Mkdir("bugs", os.ModePerm)
@@ -131,11 +171,26 @@ func main() {
 	}
 	http.Handle("/api/listing/", fs)
 
+	http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, "ok")
+	})
+
 	log.Println("Listening on", *bindAddr)
 
 	log.Fatal(http.ListenAndServe(*bindAddr, nil))
 }
 
+func configureGenericWebhookClient(cfg *config) *http.Client {
+	if len(cfg.GenericWebhookURLs) == 0 {
+		fmt.Println("No generic_webhook_urls configured.")
+		return nil
+	}
+	fmt.Println("Will forward metadata of all requests to ", cfg.GenericWebhookURLs)
+	return &http.Client{
+		Timeout: time.Second * 300,
+	}
+}
+
 func loadConfig(configPath string) (*config, error) {
 	contents, err := ioutil.ReadFile(configPath)
 	if err != nil {

rageshake.sample.yaml

@@ -17,6 +17,42 @@ github_token: secrettoken
 github_project_mappings:
    my-app: octocat/HelloWorld
 
+# a GitLab personal access token (https://gitlab.com/-/profile/personal_access_tokens), which
+# will be used to create a GitLab issue for each report. It requires
+# `api` scope. If omitted, no issues will be created.
+gitlab_token: secrettoken
+# the base URL of the GitLab instance to use
+gitlab_url: https://gitlab.com
+
+# mappings from app name (as submitted in the API) to the GitLab Project ID (not name!) for issue reporting.
+gitlab_project_mappings:
+  my-app: 12345
+# mappings from app name to a list of GitLab label names for issue reporting.
+gitlab_project_labels:
+  my-app:
+    - client::my-app
+# whether GitLab issues should be created as confidential issues. Defaults to false.
+gitlab_issue_confidential: true
+
 # a Slack personal webhook URL (https://api.slack.com/incoming-webhooks), which
 # will be used to post a notification on Slack for each report.
 slack_webhook_url: https://hooks.slack.com/services/TTTTTTT/XXXXXXXXXX/YYYYYYYYYYY
+
+# notification can also be pushed by email.
+# this param controls the target emails
+email_addresses:
+  - support@matrix.org
+
+# this is the from field that will be used in the email notifications
+email_from: Rageshake <rageshake@matrix.org>
+
+# SMTP server configuration
+smtp_server: localhost:25
+smtp_username: myemailuser
+smtp_password: myemailpass
+
+
+# a list of webhook URLs, (see docs/generic_webhook.md)
+generic_webhook_urls: 
+  - https://server.example.com/your-server/api
+  - http://another-server.com/api

submit.go

@@ -20,14 +20,17 @@ import (
 	"bytes"
 	"compress/gzip"
 	"context"
+	"encoding/base32"
 	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
+	"math/rand"
 	"mime"
 	"mime/multipart"
 	"net/http"
+	"net/smtp"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -37,6 +40,8 @@ import (
 	"time"
 
 	"github.com/google/go-github/github"
+	"github.com/jordan-wright/email"
+	"github.com/xanzy/go-gitlab"
 )
 
 var maxPayloadSize = 1024 * 1024 * 55 // 55 MB
@@ -45,14 +50,15 @@ type submitServer struct {
 	// github client for reporting bugs. may be nil, in which case,
 	// reporting is disabled.
 	ghClient *github.Client
+	glClient *gitlab.Client
 
 	// External URI to /api
 	apiPrefix string
 
-	// mappings from application to github owner/project
-	githubProjectMappings map[string]string
-
 	slack *slackClient
+
+	genericWebhookClient *http.Client
+	cfg                  *config
 }
 
 // the type of payload which can be uploaded as JSON to the submit endpoint
@@ -71,19 +77,38 @@ type jsonLogEntry struct {
 	Lines string `json:"lines"`
 }
 
-// the payload after parsing
+// Stores additional information created during processing of a payload
-type parsedPayload struct {
+type genericWebhookPayload struct {
-	UserText   string
+	payload
-	AppName    string
+	// If a github/gitlab report is generated, this is set.
-	Data       map[string]string
+	ReportURL  string `json:"report_url"`
-	Labels     []string
+	// Complete link to the listing URL that contains all uploaded logs
-	Logs       []string
+	ListingURL string `json:"listing_url"`
-	LogErrors  []string
-	Files      []string
-	FileErrors []string
 }
 
-func (p parsedPayload) WriteTo(out io.Writer) {
+// Stores information about a request made to this server
+type payload struct {
+	// A unique ID for this payload, generated within this server 
+	ID         string            `json:"id"`
+	// A multi-line string containing the user description of the fault. 
+	UserText   string            `json:"user_text"`
+	// A short slug to identify the app making the report
+	AppName    string            `json:"app"`
+	// Arbitrary data to annotate the report
+	Data       map[string]string `json:"data"`
+	// Short labels to group reports
+	Labels     []string          `json:"labels"`
+	// A list of names of logs recognised by the server
+	Logs       []string          `json:"logs"`
+	// Set if there are log parsing errors
+	LogErrors  []string          `json:"logErrors"`
+	// A list of other files (not logs) uploaded as part of the rageshake
+	Files      []string          `json:"files"`
+	// Set if there are file parsing errors
+	FileErrors []string          `json:"fileErrors"`
+}
+
+func (p payload) WriteTo(out io.Writer) {
 	fmt.Fprintf(
 		out,
 		"%s\n\nNumber of logs: %d\nApplication: %s\n",
@@ -142,6 +167,9 @@ func (s *submitServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	// files straight in
 	t := time.Now().UTC()
 	prefix := t.Format("2006-01-02/150405")
+	randBytes := make([]byte, 5)
+	rand.Read(randBytes)
+	prefix += "-" + base32.StdEncoding.EncodeToString(randBytes)
 	reportDir := filepath.Join("bugs", prefix)
 	if err := os.MkdirAll(reportDir, os.ModePerm); err != nil {
 		log.Println("Unable to create report directory", err)
@@ -163,6 +191,11 @@ func (s *submitServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 
+	// We use this prefix (eg, 2022-05-01/125223-abcde) as a unique identifier for this rageshake.
+	// This is going to be used to uniquely identify rageshakes, even if they are not submitted to
+	// an issue tracker for instance with automatic rageshakes that can be plentiful
+	p.ID = prefix
+
 	resp, err := s.saveReport(req.Context(), *p, reportDir, listingURL)
 	if err != nil {
 		log.Println("Error handling report submission:", err)
@@ -177,7 +210,7 @@ func (s *submitServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 
 // parseRequest attempts to parse a received request as a bug report. If
 // the request cannot be parsed, it responds with an error and returns nil.
-func parseRequest(w http.ResponseWriter, req *http.Request, reportDir string) *parsedPayload {
+func parseRequest(w http.ResponseWriter, req *http.Request, reportDir string) *payload {
 	length, err := strconv.Atoi(req.Header.Get("Content-Length"))
 	if err != nil {
 		log.Println("Couldn't parse content-length", err)
@@ -213,13 +246,13 @@ func parseRequest(w http.ResponseWriter, req *http.Request, reportDir string) *p
 	return p
 }
 
-func parseJSONRequest(w http.ResponseWriter, req *http.Request, reportDir string) (*parsedPayload, error) {
+func parseJSONRequest(w http.ResponseWriter, req *http.Request, reportDir string) (*payload, error) {
 	var p jsonPayload
 	if err := json.NewDecoder(req.Body).Decode(&p); err != nil {
 		return nil, err
 	}
 
-	parsed := parsedPayload{
+	parsed := payload{
 		UserText: strings.TrimSpace(p.Text),
 		Data:     make(map[string]string),
 		Labels:   p.Labels,
@@ -274,13 +307,13 @@ func parseJSONRequest(w http.ResponseWriter, req *http.Request, reportDir string
 	return &parsed, nil
 }
 
-func parseMultipartRequest(w http.ResponseWriter, req *http.Request, reportDir string) (*parsedPayload, error) {
+func parseMultipartRequest(w http.ResponseWriter, req *http.Request, reportDir string) (*payload, error) {
 	rdr, err := req.MultipartReader()
 	if err != nil {
 		return nil, err
 	}
 
-	p := parsedPayload{
+	p := payload{
 		Data: make(map[string]string),
 	}
 
@@ -299,7 +332,7 @@ func parseMultipartRequest(w http.ResponseWriter, req *http.Request, reportDir s
 	return &p, nil
 }
 
-func parseFormPart(part *multipart.Part, p *parsedPayload, reportDir string) error {
+func parseFormPart(part *multipart.Part, p *payload, reportDir string) error {
 	defer part.Close()
 	field := part.FormName()
 	partName := part.FileName()
@@ -360,7 +393,7 @@ func parseFormPart(part *multipart.Part, p *parsedPayload, reportDir string) err
 
 // formPartToPayload updates the relevant part of *p from a name/value pair
 // read from the form data.
-func formPartToPayload(field, data string, p *parsedPayload) {
+func formPartToPayload(field, data string, p *payload) {
 	if field == "text" {
 		p.UserText = data
 	} else if field == "app" {
@@ -386,7 +419,7 @@ func formPartToPayload(field, data string, p *parsedPayload) {
 // * no silly characters (/, ctrl chars, etc)
 //
 // * nothing starting with '.'
-var filenameRegexp = regexp.MustCompile(`^[a-zA-Z0-9_-]+\.(jpg|png|txt)$`)
+var filenameRegexp = regexp.MustCompile(`^[a-zA-Z0-9_-]+\.(jpg|png|txt|json)$`)
 
 // saveFormPart saves a file upload to the report directory.
 //
@@ -416,7 +449,7 @@ func saveFormPart(leafName string, reader io.Reader, reportDir string) (string,
 
 // we require a sensible extension, and don't allow the filename to start with
 // '.'
-var logRegexp = regexp.MustCompile(`^[a-zA-Z0-9_-][a-zA-Z0-9_.-]*\.(log|txt)$`)
+var logRegexp = regexp.MustCompile(`^[a-zA-Z0-9_-][a-zA-Z0-9_.-]*\.(log|txt)(\.gz)?$`)
 
 // saveLogPart saves a log upload to the report directory.
 //
@@ -427,10 +460,16 @@ func saveLogPart(logNum int, filename string, reader io.Reader, reportDir string
 	// some clients use sensible names (foo.N.log), which we preserve. For
 	// others, we just make up a filename.
 	//
-	// Either way, we need to append .gz, because we're compressing it.
+	// We append a ".gz" extension if not already present, as the final file we store on
+	// disk will be gzipped. The original filename may or may not contain a '.gz' depending
+	// on the client that uploaded it, and if it was uploaded already compressed.
+
 	var leafName string
 	if logRegexp.MatchString(filename) {
-		leafName = filename + ".gz"
+		leafName = filename
+		if !strings.HasSuffix(filename, ".gz") {
+			leafName += ".gz"
+		}
 	} else {
 		leafName = fmt.Sprintf("logs-%04d.log.gz", logNum)
 	}
@@ -454,7 +493,7 @@ func saveLogPart(logNum int, filename string, reader io.Reader, reportDir string
 	return leafName, nil
 }
 
-func (s *submitServer) saveReport(ctx context.Context, p parsedPayload, reportDir, listingURL string) (*submitResponse, error) {
+func (s *submitServer) saveReport(ctx context.Context, p payload, reportDir, listingURL string) (*submitResponse, error) {
 	var summaryBuf bytes.Buffer
 	resp := submitResponse{}
 	p.WriteTo(&summaryBuf)
@@ -466,74 +505,159 @@ func (s *submitServer) saveReport(ctx context.Context, p parsedPayload, reportDi
 		return nil, err
 	}
 
+	if err := s.submitGitlabIssue(p, listingURL, &resp); err != nil {
+		return nil, err
+	}
+
 	if err := s.submitSlackNotification(p, listingURL); err != nil {
 		return nil, err
 	}
 
+	if err := s.sendEmail(p, reportDir); err != nil {
+		return nil, err
+	}
+
+	if err := s.submitGenericWebhook(p, listingURL, resp.ReportURL); err != nil {
+		return nil, err
+	}
+
 	return &resp, nil
 }
 
-func (s *submitServer) submitGithubIssue(ctx context.Context, p parsedPayload, listingURL string, resp *submitResponse) error {
+// submitGenericWebhook submits a basic JSON body to an endpoint configured in the config
+//
+// The request does not include the log body, only the metadata in the payload,
+// with the required listingURL to obtain the logs over http if required.
+//
+// If a github or gitlab issue was previously made, the reportURL will also be passed.
+//
+// Uses a goroutine to handle the http request asynchronously as by this point all critical
+// information has been stored.
+
+func (s *submitServer) submitGenericWebhook(p payload, listingURL string, reportURL string) error {
+	if s.genericWebhookClient == nil {
+		return nil
+	}
+	genericHookPayload := genericWebhookPayload{
+		payload: p,
+		ReportURL:     reportURL,
+		ListingURL:    listingURL,
+	}
+	for _, url := range s.cfg.GenericWebhookURLs {
+		// Enrich the payload with a reportURL and listingURL, to convert a single struct
+		// to JSON easily
+
+		payloadBuffer := new(bytes.Buffer)
+		json.NewEncoder(payloadBuffer).Encode(genericHookPayload)
+		req, err := http.NewRequest("POST", url, payloadBuffer)
+		req.Header.Set("Content-Type", "application/json")
+		if err != nil {
+			log.Println("Unable to submit to URL ", url, " ", err)
+			return err
+		}
+		log.Println("Making generic webhook request to URL ", url)
+		go s.sendGenericWebhook(req)
+	}
+	return nil
+}
+
+func (s *submitServer) sendGenericWebhook(req *http.Request) {
+	resp, err := s.genericWebhookClient.Do(req)
+	if err != nil {
+		log.Println("Unable to submit notification", err)
+	} else {
+		defer resp.Body.Close()
+		log.Println("Got response", resp.Status)
+	}
+}
+
+func (s *submitServer) submitGithubIssue(ctx context.Context, p payload, listingURL string, resp *submitResponse) error {
 	if s.ghClient == nil {
-		log.Println("GH issue submission disabled")
+		return nil
-	} else {
-		// submit a github issue
-		ghProj := s.githubProjectMappings[p.AppName]
-		if ghProj == "" {
-			log.Println("Not creating GH issue for unknown app", p.AppName)
-			return nil
-		}
-		splits := strings.SplitN(ghProj, "/", 2)
-		if len(splits) < 2 {
-			log.Println("Can't create GH issue for invalid repo", ghProj)
-		}
-		owner, repo := splits[0], splits[1]
-
-		issueReq := buildGithubIssueRequest(p, listingURL)
-
-		issue, _, err := s.ghClient.Issues.Create(ctx, owner, repo, &issueReq)
-		if err != nil {
-			return err
-		}
-
-		log.Println("Created issue:", *issue.HTMLURL)
-
-		resp.ReportURL = *issue.HTMLURL
 	}
+
+	// submit a github issue
+	ghProj := s.cfg.GithubProjectMappings[p.AppName]
+	if ghProj == "" {
+		log.Println("Not creating GH issue for unknown app", p.AppName)
+		return nil
+	}
+	splits := strings.SplitN(ghProj, "/", 2)
+	if len(splits) < 2 {
+		log.Println("Can't create GH issue for invalid repo", ghProj)
+	}
+	owner, repo := splits[0], splits[1]
+
+	issueReq := buildGithubIssueRequest(p, listingURL)
+
+	issue, _, err := s.ghClient.Issues.Create(ctx, owner, repo, &issueReq)
+	if err != nil {
+		return err
+	}
+
+	log.Println("Created issue:", *issue.HTMLURL)
+
+	resp.ReportURL = *issue.HTMLURL
+
 	return nil
 }
 
-func (s *submitServer) submitSlackNotification(p parsedPayload, listingURL string) error {
+func (s *submitServer) submitGitlabIssue(p payload, listingURL string, resp *submitResponse) error {
+	if s.glClient == nil {
+		return nil
+	}
+
+	glProj := s.cfg.GitlabProjectMappings[p.AppName]
+	glLabels := s.cfg.GitlabProjectLabels[p.AppName]
+
+	issueReq := buildGitlabIssueRequest(p, listingURL, glLabels, s.cfg.GitlabIssueConfidential)
+
+	issue, _, err := s.glClient.Issues.CreateIssue(glProj, issueReq)
+
+	if err != nil {
+		return err
+	}
+
+	log.Println("Created issue:", issue.WebURL)
+
+	resp.ReportURL = issue.WebURL
+
+	return nil
+}
+
+func (s *submitServer) submitSlackNotification(p payload, listingURL string) error {
 	if s.slack == nil {
-		log.Println("Slack notifications disabled")
+		return nil
-	} else {
-		slackBuf := fmt.Sprintf(
-			"%s\nApplication: %s\nReport: %s",
-			p.UserText, p.AppName, listingURL,
-		)
-
-		err := s.slack.Notify(slackBuf)
-		if err != nil {
-			return err
-		}
 	}
+
+	slackBuf := fmt.Sprintf(
+		"%s\nApplication: %s\nReport: %s",
+		p.UserText, p.AppName, listingURL,
+	)
+
+	err := s.slack.Notify(slackBuf)
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 
-func buildGithubIssueRequest(p parsedPayload, listingURL string) github.IssueRequest {
+func buildReportTitle(p payload) string {
 	// set the title to the first (non-empty) line of the user's report, if any
-	var title string
 	trimmedUserText := strings.TrimSpace(p.UserText)
 	if trimmedUserText == "" {
-		title = "Untitled report"
+		return "Untitled report"
-	} else {
-		if i := strings.IndexAny(trimmedUserText, "\r\n"); i < 0 {
-			title = trimmedUserText
-		} else {
-			title = trimmedUserText[0:i]
-		}
 	}
 
+	if i := strings.IndexAny(trimmedUserText, "\r\n"); i >= 0 {
+		return trimmedUserText[0:i]
+	}
+
+	return trimmedUserText
+}
+
+func buildReportBody(p payload, newline, quoteChar string) *bytes.Buffer {
 	var bodyBuf bytes.Buffer
 	fmt.Fprintf(&bodyBuf, "User message:\n\n%s\n\n", p.UserText)
 	var dataKeys []string
@@ -543,20 +667,36 @@ func buildGithubIssueRequest(p parsedPayload, listingURL string) github.IssueReq
 	sort.Strings(dataKeys)
 	for _, k := range dataKeys {
 		v := p.Data[k]
-		fmt.Fprintf(&bodyBuf, "%s: `%s`\n", k, v)
+		fmt.Fprintf(&bodyBuf, "%s: %s%s%s%s", k, quoteChar, v, quoteChar, newline)
 	}
-	fmt.Fprintf(&bodyBuf, "[Logs](%s)", listingURL)
+
+	return &bodyBuf
+}
+
+func buildGenericIssueRequest(p payload, listingURL string) (title, body string) {
+	bodyBuf := buildReportBody(p, "  \n", "`")
+
+	// Add log links to the body
+	fmt.Fprintf(bodyBuf, "\n[Logs](%s)", listingURL)
 
 	for _, file := range p.Files {
 		fmt.Fprintf(
-			&bodyBuf,
+			bodyBuf,
 			" / [%s](%s)",
 			file,
 			listingURL+"/"+file,
 		)
 	}
 
-	body := bodyBuf.String()
+	title = buildReportTitle(p)
+
+	body = bodyBuf.String()
+
+	return
+}
+
+func buildGithubIssueRequest(p payload, listingURL string) github.IssueRequest {
+	title, body := buildGenericIssueRequest(p, listingURL)
 
 	labels := p.Labels
 	// go-github doesn't like nils
@@ -570,6 +710,57 @@ func buildGithubIssueRequest(p parsedPayload, listingURL string) github.IssueReq
 	}
 }
 
+func buildGitlabIssueRequest(p payload, listingURL string, labels []string, confidential bool) *gitlab.CreateIssueOptions {
+	title, body := buildGenericIssueRequest(p, listingURL)
+
+	if p.Labels != nil {
+		labels = append(labels, p.Labels...)
+	}
+
+	return &gitlab.CreateIssueOptions{
+		Title:        &title,
+		Description:  &body,
+		Confidential: &confidential,
+		Labels:       labels,
+	}
+}
+
+func (s *submitServer) sendEmail(p payload, reportDir string) error {
+	if len(s.cfg.EmailAddresses) == 0 {
+		return nil
+	}
+
+	e := email.NewEmail()
+
+	e.From = "Rageshake <rageshake@matrix.org>"
+	if s.cfg.EmailFrom != "" {
+		e.From = s.cfg.EmailFrom
+	}
+
+	e.To = s.cfg.EmailAddresses
+
+	e.Subject = fmt.Sprintf("[%s] %s", p.AppName, buildReportTitle(p))
+
+	e.Text = buildReportBody(p, "\n", "\"").Bytes()
+
+	allFiles := append(p.Files, p.Logs...)
+	for _, file := range allFiles {
+		fullPath := filepath.Join(reportDir, file)
+		e.AttachFile(fullPath)
+	}
+
+	var auth smtp.Auth = nil
+	if s.cfg.SMTPPassword != "" || s.cfg.SMTPUsername != "" {
+		auth = smtp.PlainAuth("", s.cfg.SMTPUsername, s.cfg.SMTPPassword, s.cfg.SMTPServer)
+	}
+	err := e.Send(s.cfg.SMTPServer, auth)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func respond(code int, w http.ResponseWriter) {
 	w.WriteHeader(code)
 	w.Write([]byte("{}"))

submit_test.go

@@ -35,7 +35,7 @@ import (
 //
 // if tempDir is empty, a new temp dir is created, and deleted when the test
 // completes.
-func testParsePayload(t *testing.T, body, contentType string, tempDir string) (*parsedPayload, *http.Response) {
+func testParsePayload(t *testing.T, body, contentType string, tempDir string) (*payload, *http.Response) {
 	req, err := http.NewRequest("POST", "/api/submit", strings.NewReader(body))
 	if err != nil {
 		t.Fatal(err)
@@ -160,6 +160,7 @@ func TestMultipartUpload(t *testing.T) {
 
 	// check file uploaded correctly
 	checkUploadedFile(t, reportDir, "passwd.txt", false, "bibblybobbly")
+	checkUploadedFile(t, reportDir, "crash.log.gz", true, "test\n")
 }
 
 func multipartBody() (body string) {
@@ -215,17 +216,29 @@ Content-Type: application/octet-stream
 
 bibblybobbly
 `
+	body += `------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="compressed-log"; filename="crash.log.gz"
+Content-Type: application/octet-stream
+
+`
+	body += string([]byte{
+		0x1f, 0x8b, 0x08, 0x00, 0xbf, 0xd8, 0xf5, 0x58, 0x00, 0x03,
+		0x2b, 0x49, 0x2d, 0x2e, 0xe1, 0x02, 0x00,
+		0xc6, 0x35, 0xb9, 0x3b, 0x05, 0x00, 0x00, 0x00,
+		0x0a,
+	})
+
 	body += "------WebKitFormBoundarySsdgl8Nq9voFyhdO--\n"
 	return
 }
 
-func checkParsedMultipartUpload(t *testing.T, p *parsedPayload) {
+func checkParsedMultipartUpload(t *testing.T, p *payload) {
 	wanted := "test words."
 	if p.UserText != wanted {
 		t.Errorf("User text: got %s, want %s", p.UserText, wanted)
 	}
-	if len(p.Logs) != 3 {
+	if len(p.Logs) != 4 {
-		t.Errorf("Log length: got %d, want 3", len(p.Logs))
+		t.Errorf("Log length: got %d, want 4", len(p.Logs))
 	}
 	if len(p.Data) != 3 {
 		t.Errorf("Data length: got %d, want 3", len(p.Data))
@@ -249,6 +262,10 @@ func checkParsedMultipartUpload(t *testing.T, p *parsedPayload) {
 	if p.Logs[2] != wanted {
 		t.Errorf("Log 2: got %s, want %s", p.Logs[2], wanted)
 	}
+	wanted = "crash.log.gz"
+	if p.Logs[3] != wanted {
+		t.Errorf("Log 3: got %s, want %s", p.Logs[3], wanted)
+	}
 }
 
 func TestLabels(t *testing.T) {
@@ -461,7 +478,7 @@ user_id: id
 	}
 	var buf bytes.Buffer
 	for _, v := range sample {
-		p := parsedPayload{Data: v.data}
+		p := payload{Data: v.data}
 		buf.Reset()
 		p.WriteTo(&buf)
 		got := strings.TrimSpace(buf.String())
@@ -471,7 +488,7 @@ user_id: id
 	}
 
 	for k, v := range sample {
-		p := parsedPayload{Data: v.data}
+		p := payload{Data: v.data}
 		res := buildGithubIssueRequest(p, "")
 		got := *res.Body
 		if k == 0 {