Merge branch 't3chguy/autocomplete' of https://github.com/matrix-org/rageshake into t3chguy/autocomplete

Co-Authored-By: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>

.buildkite/build-and-test.sh

@@ -1,7 +0,0 @@
-#!/bin/sh
-set -e
-
-cd `dirname $0`/..
-
-go build
-go test

.buildkite/check_changelog.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-#
-# this script is run by buildkite to check that a changelog file exists
-#
-set -e
-
-# we need 19.9 to read config from towncrier.toml
-pip3 install --pre 'towncrier>19.2'
-python3 -m towncrier.check

.buildkite/lint.sh

@@ -1,11 +0,0 @@
-#!/bin/sh
-
-set -e
-
-cd `dirname $0`/..
-
-go get golang.org/x/lint/golint
-go install golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow
-go get github.com/fzipp/gocyclo/cmd/gocyclo
-
-./scripts/lint.sh

.github/CODEOWNERS

@@ -1 +0,0 @@
-*       @richvdh

.github/workflows/docker.yaml

@@ -1,84 +0,0 @@
-name: Docker
-
-on:
-  push:
-  pull_request:
-    branches: [ master ]
-
-jobs:
-  build:
-    name: Build and push Docker image
-    runs-on: ubuntu-latest
-    env:
-      IMAGE: ghcr.io/${{ github.repository }}
-
-    permissions:
-      packages: write
-      contents: read
-
-    steps:
-      - name: Checkout the code
-        uses: actions/checkout@v2
-
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v3
-        with:
-          images: "${{ env.IMAGE }}"
-          bake-target: docker-metadata-action
-          tags: |
-            type=ref,event=branch
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=semver,pattern={{major}}
-            type=sha
-
-      - name: Docker meta (debug variant)
-        id: meta-debug
-        uses: docker/metadata-action@v3
-        with:
-          images: "${{ env.IMAGE }}"
-          bake-target: docker-metadata-action-debug
-          tags: |
-            type=ref,event=branch,suffix=-debug
-            type=semver,pattern={{version}},suffix=-debug
-            type=semver,pattern={{major}}.{{minor}},suffix=-debug
-            type=semver,pattern={{major}},suffix=-debug
-            type=sha,suffix=-debug
-
-      - name: Merge buildx bake files
-        run: |
-            jq -s '.[0] * .[1]' ${{ steps.meta.outputs.bake-file }} ${{ steps.meta-debug.outputs.bake-file }} > docker-bake.override.json
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          config-inline: |
-            [registry."docker.io"]
-              mirrors = ["mirror.gcr.io"]
-
-      - name: Login to GitHub Container Registry
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      # For pull-requests, only read from the cache, do not try to push to the
-      # cache or the image itself
-      - name: Build
-        uses: docker/bake-action@v1
-        if: github.event_name == 'pull_request'
-        with:
-          set: |
-            base.cache-from=type=registry,ref=${{ env.IMAGE }}:buildcache
-
-      - name: Build and push
-        uses: docker/bake-action@v1
-        if: github.event_name != 'pull_request'
-        with:
-          set: |
-            base.output=type=image,push=true
-            base.cache-from=type=registry,ref=${{ env.IMAGE }}:buildcache
-            base.cache-to=type=registry,ref=${{ env.IMAGE }}:buildcache,mode=max

.travis.yml

@@ -0,0 +1,7 @@
+language: go
+go:
+  - "1.11"
+install:
+  - go get golang.org/x/lint/golint
+  - go get github.com/fzipp/gocyclo
+script: ./hooks/pre-commit

.woodpecker.yml

@@ -1,15 +0,0 @@
-pipeline:
-  build-test:
-    image: golang
-    commands:
-      -  go build
-      -  go test
-
-  publish:
-    image: plugins/docker
-    settings:
-      username: realaravinth
-      password:
-        from_secret: DOCKER_TOKEN
-      repo: realaravinth/rageshake
-      tags: latest

CHANGES.md

@@ -1,85 +0,0 @@
-1.7 (2022-04-14)
-================
-
-Features
---------
-
-- Pass the prefix as a unique ID for the rageshake to the generic webhook mechanism. ([\#54](https://github.com/matrix-org/rageshake/issues/54))
-
-
-1.6 (2022-02-22)
-================
-
-Features
---------
-
-- Provide ?format=tar.gz option on directory listings to download tarball. ([\#53](https://github.com/matrix-org/rageshake/issues/53))
-
-
-1.5 (2022-02-08)
-================
-
-Features
---------
-
-- Allow upload of Files with a .json postfix. ([\#52](https://github.com/matrix-org/rageshake/issues/52))
-
-
-1.4 (2022-02-01)
-================
-
-Features
---------
-
-- Allow forwarding of a request to a webhook endpoint. ([\#50](https://github.com/matrix-org/rageshake/issues/50))
-
-
-1.3 (2022-01-25)
-================
-
-Features
---------
-
-- Add support for creating GitLab issues. Contributed by @tulir. ([\#37](https://github.com/matrix-org/rageshake/issues/37))
-- Support element-android submitting logs with .gz suffix. ([\#40](https://github.com/matrix-org/rageshake/issues/40))
-
-
-Bugfixes
---------
-
-- Prevent timestamp collisions when reports are submitted within 1 second of each other. ([\#39](https://github.com/matrix-org/rageshake/issues/39))
-
-
-Internal Changes
-----------------
-
-- Update minimum Go version to 1.16. ([\#37](https://github.com/matrix-org/rageshake/issues/37), [\#42](https://github.com/matrix-org/rageshake/issues/42))
-- Add documentation on the types and formats of files submitted to the rageshake server. ([\#44](https://github.com/matrix-org/rageshake/issues/44))
-- Build and push a multi-arch Docker image on the GitHub Container Registry. ([\#47](https://github.com/matrix-org/rageshake/issues/47))
-- Add a /health endpoint that always replies with a 200 OK. ([\#48](https://github.com/matrix-org/rageshake/issues/48))
-
-
-1.2 (2020-09-16)
-================
-
-Features
---------
-
-- Add email support. ([\#35](https://github.com/matrix-org/rageshake/issues/35))
-
-
-1.1 (2020-06-04)
-================
-
-Features
---------
-
-- Add support for Slack notifications. Contributed by @awesome-manuel. ([\#28](https://github.com/matrix-org/rageshake/issues/28))
-
-
-Internal Changes
-----------------
-
-- Update minimum go version to 1.11. ([\#29](https://github.com/matrix-org/rageshake/issues/29), [\#30](https://github.com/matrix-org/rageshake/issues/30))
-- Replace vendored libraries with `go mod`. ([\#31](https://github.com/matrix-org/rageshake/issues/31))
-- Add Dockerfile. Contributed by @awesome-manuel. ([\#32](https://github.com/matrix-org/rageshake/issues/32))

CONTRIBUTING.rst

@@ -21,9 +21,10 @@ merge this back into the matrix.org 'official' master branch. We use github's
 pull request workflow to review the contribution, and either ask you to make
 any refinements needed or merge it and make them ourselves.
 
-We use Buildkite for continuous integration, and all pull requests get
+We use Travis for continuous integration, and all pull requests get
-automatically tested: if your change breaks the build, then the PR will show
+automatically tested by Travis: if your change breaks the build, then the PR
-that there are failed checks, so please check back after a few minutes.
+will show that there are failed checks, so please check back after a few
+minutes.
 
 Code style
 ~~~~~~~~~~

Dockerfile

@@ -1,23 +1,18 @@
-## Build stage ##
+FROM golang:alpine as builder
-FROM golang as builder
+RUN apk add --update --no-cache git ca-certificates
+
+RUN mkdir /build 
 WORKDIR /build
-COPY go.mod go.sum ./
+COPY go.mod .
+COPY go.sum .
 RUN go mod download
+
 COPY . .
-RUN go build -o rageshake
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o rageshake
 
-## Runtime stage, debug variant ##
+FROM scratch
-FROM  debian:bullseye as debug
+COPY --from=builder /build/rageshake /rageshake
-COPY --from=builder /build/rageshake /rageshake/
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 WORKDIR /
 EXPOSE 9110
-ENTRYPOINT ["/rageshake/rageshake"]
+CMD ["/rageshake"]
-
-## Runtime stage ##
-FROM  debian:bullseye as rageshake
-LABEL org.opencontainers.image.source https://git.batsense.net/mystiq/rageshake
-RUN apt-get update && apt-get install -y ca-certificates
-WORKDIR /
-COPY --from=builder /build/rageshake /rageshake/
-EXPOSE 9110
-ENTRYPOINT ["/rageshake/rageshake"]

README.md

@@ -1,9 +1,8 @@
-WOODPECKER: [![status-badge](https://ci.batsense.net/api/badges/mystiq/rageshake/status.svg)](https://ci.batsense.net/mystiq/rageshake)
+# rageshake [![Build Status](https://travis-ci.org/matrix-org/rageshake.svg?branch=master)](https://travis-ci.org/matrix-org/rageshake)
-# rageshake [![Build status](https://badge.buildkite.com/76a4362a20b12dcd589f9308a905ffcc537278b9c363c0b5f1.svg?branch=master)](https://buildkite.com/matrix-dot-org/rageshake)
 
 Web service which collects and serves bug reports.
 
-rageshake requires Go version 1.16 or later.
+rageshake requires Go version 1.11 or later.
 
 To run it, do:
 
@@ -29,8 +28,6 @@ Serves submitted bug reports. Protected by basic HTTP auth using the
 username/password provided in the environment. A browsable list, collated by
 report submission date and time.
 
-A whole directory can be downloaded as a tarball by appending the parameter `?format=tar.gz` to the end of the URL path
-
 ### POST `/api/submit`
 
 Submission endpoint: this is where applications should send their reports.
@@ -71,16 +68,10 @@ logs.)
     * `id`: textual identifier for the logs. Used as the filename, as above.
     * `lines`: log data. Newlines should be  encoded as `\n`, as normal in JSON).
 
-  A summary of the current log file formats that are uploaded for `log` and
-  `compressed-log`  is [available](docs/submitted_reports.md).
-
 * `compressed-log`: a gzipped logfile. Decompressed and then treated the same as
   `log`.
 
   Compressed logs are not supported for the JSON upload encoding.
-  
-  A summary of the current log file formats that are uploaded for `log` and
-  `compressed-log` is [available](docs/submitted_reports.md).
 
 * `file`: an arbitrary file to attach to the report. Saved as-is to disk, and
   a link is added to the github issue. The filename must be in the format
@@ -100,21 +91,3 @@ The response (if successful) will be a JSON object with the following fields:
 
 * `report_url`: A URL where the user can track their bug report. Omitted if
   issue submission was disabled.
-
-## Notifications
-
-You can get notifications when a new rageshake arrives on the server.
-
-Currently this tool supports pushing notifications as GitHub issues in a repo,
-through a Slack webhook or by email, cf sample config file for how to
-configure them.
-
-### Generic Webhook Notifications
-
-You can receive a webhook notifications when a new rageshake arrives on the server.
-
-These requests contain all the parsed metadata, and links to the uploaded files, and any github/gitlab
-issues created.
-
-Details on the request and expected response are [available](docs/generic\_webhook.md).
-

RELEASING.md

@@ -1,30 +0,0 @@
-1. Set a variable to the version number for convenience:
-   ```sh
-   ver=x.y
-   ```
-1. Update the changelog:
-   ```sh
-   # we need 19.9 to read config from towncrier.toml
-   pip3 install --pre 'towncrier>19.2'
-   towncrier --version=$ver
-   ```
-1. Push your changes:
-   ```sh
-   git add -u && git commit -m $ver && git push
-   ```
-1. Sanity-check the
-   [changelog](https://github.com/matrix-org/rageshake/blob/master/CHANGES.md)
-   and update if need be.
-1. Create a signed tag for the release:
-   ```sh
-   git tag -s v$ver
-   ```
-   Base the tag message on the changelog.
-1. Push the tag:
-   ```sh
-   git push origin tag v$ver
-   ```
-1. Create release on GH project page:
-   ```sh
-   xdg-open https://github.com/matrix-org/rageshake/releases/edit/v$ver
-   ```

changelog.d/.gitignore

@@ -1 +0,0 @@
-!.gitignore

docker-bake.hcl

@@ -1,25 +0,0 @@
-// This is what is baked by GitHub Actions
-group "default" { targets = ["regular", "debug"] }
-
-// Targets filled by GitHub Actions: one for the regular tag, one for the debug tag
-target "docker-metadata-action" {}
-target "docker-metadata-action-debug" {}
-
-// This sets the platforms and is further extended by GitHub Actions to set the
-// output and the cache locations
-target "base" {
-  platforms = [
-    "linux/amd64",
-    "linux/arm64",
-    "linux/arm",
-  ]
-}
-
-target "regular" {
-  inherits = ["base", "docker-metadata-action"]
-}
-
-target "debug" {
-  inherits = ["base", "docker-metadata-action-debug"]
-  target = "debug"
-}

docs/generic_webhook.md

@@ -1,40 +0,0 @@
-## Generic webhook request
-
-If the configuration option `generic_webhook_urls` is set, then an asynchronous request to
-each endpoint listed will be sent in parallel, after the incoming request is parsed and the
-files are uploaded.
-
-The webhook is designed for notification or other tracking services, and does not contain
-the original log files uploaded.
-
-(If you want the original log files, we suggest to implement the rageshake interface itself).
-
-A sample JSON body is as follows:
-
-```
-{
-  'user_text': 'test\r\n\r\nIssue: No issue link given',
-  'app': 'element-web',
-  'data': {
-    'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:96.0) Gecko/20100101 Firefox/96.0',
-    'Version': '0f15ba34cdf5-react-0f15ba34cdf5-js-0f15ba34cdf5',
-    ...
-    'user_id': '@michaelgoesforawalk:matrix.org'},
-  'labels': None,
-  'logs': [
-    'logs-0000.log.gz',
-    'logs-0001.log.gz',
-    'logs-0002.log.gz',
-  ],
-  'logErrors': None,
-  'files': [
-    'screenshot.png'
-  ],
-  'fileErrors': None,
-  'report_url': 'https://github.com/your-org/your-repo/issues/1251',
-  'listing_url': 'http://your-rageshake-server/api/listing/2022-01-25/154742-OOXBVGIX'
-}
-```
-
-The log and other files can be individually downloaded by concatenating the `listing_url` and the `logs` or `files` name.
-You may need to provide a HTTP basic auth user/pass if configured on your rageshake server.

docs/submitted_reports.md

@@ -1,87 +0,0 @@
-# Common report styles
-
-Rageshakes can come from a number of applications, and we provide some practical notes on the generated format.
-
-At present these should not be considered absolute nor a structure to follow; but an attempt to document the currently visible formats as of January 2022.
-
-## Feedback 
-
-Log files are not transmitted; the main feedback is entirely within the user message body.
-
-This occurs from all platforms.
-
-## Element Web / Element Desktop
-
-Log files are transmitted in reverse order (0000 is the youngest) 
-
-Log line format:
-```
-2022-01-17T14:57:20.806Z I Using WebAssembly Olm
-< ---- TIMESTAMP ------> L <-- Message ----
-
-L = log level, (W=Warn, I=Info, etc)
-```
-
-New log files are started each restart of the app, but some log files may not contain all data from the start of the session.
-
-## Element iOS
-
-Crash Log is special and is sent only once (and deleted on the device afterwards)
-
-`crash.log`
-
-Following logs are available, going back in time with ascending number.
-console.log with no number is the current log file.
-```
-console.log (newest)
-console-1.log 
-...
-console-49.log (oldest)
-
-console-nse.log (newest)
-console-nse-1.log 
-...
-console-nse-49.log (oldest)
-
-console-share.log (newest)
-console-share-1.log
-console-share-49.log (oldest)
-```
-
-## Element Android
-
-There is a historical issue with the naming of files, documented in [issue #40](https://github.com/matrix-org/rageshake/issues/40).
-
-Log file 0000 is odd, it contains the logcat data if sent.
-
-Log line format:
-```
-01-17 14:59:30.657 14303 14303 W Activity: Slow Operation: 
-<-- TIMESTAMP ---> <-P-> <-T-> L <-- Message --
-
-L = Log Level (W=Warn, I=Info etc)
-P = Process ID
-T = Thread ID
-```
-Remaining log files are transmitted according to their position in the round-robin logging to file - there will be (up to) 7 files written to in a continious loop; one of the seven will be the oldest, the rest will be in order.
-
-Log line format:
-```
-2022-01-17T13:06:36*838GMT+00:00Z 12226 D/ /Tag: Migration: Importing legacy session
-< ---- TIMESTAMP ---------------> <-P-> L        <-- Message ----
-
-L = log level, (W=Warn, I=Info, etc)
-P = Process ID
-```
-
-Once the fix to #40 is in place, we will see the following files:
-
-```
-logcatError.log
-logcat.log
-crash.log
-keyrequests.log
-log-[1-7].log
-```
-
-Log 1-7 are logs from a round-robin buffer and are ordered but the start point is undefined

go.mod

@@ -1,11 +1,16 @@
 module github.com/matrix-org/rageshake
 
-go 1.16
-
 require (
+	cloud.google.com/go v0.0.0-20170406015231-675fad27ef35
+	github.com/golang/protobuf v0.0.0-20170331031902-2bba0603135d
+	github.com/google/go-genproto v0.0.0-20170404132009-411e09b969b1
 	github.com/google/go-github v0.0.0-20170401000335-12363ffc1001
-	github.com/jordan-wright/email v4.0.1-0.20200824153738-3f5bafa1cd84+incompatible
+	github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135
-	github.com/xanzy/go-gitlab v0.50.2
+	github.com/googleapis/gax-go v0.0.0-20170321005343-9af46dd5a171
-	golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288
+	github.com/pkg/errors v0.0.0-20171018195549-f15c970de5b7
-	gopkg.in/yaml.v2 v2.2.8
+	golang.org/x/net v0.0.0-20170329170435-ffcf1bedda3b
+	golang.org/x/oauth2 v0.0.0-20170321013421-7fdf09982454
+	golang.org/x/text v0.0.0-20170401064109-f4b4367115ec
+	google.golang.org/grpc v0.0.0-20170405173540-b5071124392b
+	gopkg.in/yaml.v2 v2.0.0-20170407172122-cd8b52f8269e
 )

go.sum

@@ -1,50 +1,17 @@
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+cloud.google.com/go v0.0.0-20170406015231-675fad27ef35/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/golang/protobuf v0.0.0-20170331031902-2bba0603135d/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/go-genproto v0.0.0-20170404132009-411e09b969b1/go.mod h1:3Rcd9jSoLVkV/osPrt5CogLvLiarfI8U9/x78NwhuDU=
-github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/go-github v0.0.0-20170401000335-12363ffc1001 h1:OK4gfzCBCtPg14E4sYsczwFhjVu1jQJZI+OEOpiTigw=
 github.com/google/go-github v0.0.0-20170401000335-12363ffc1001/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
-github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
+github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 h1:zLTLjkaOFEFIOxY5BWLFLwh+cL8vOBW4XJ2aqLE/Tf0=
-github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
-github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
+github.com/googleapis/gax-go v0.0.0-20170321005343-9af46dd5a171/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
-github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/pkg/errors v0.0.0-20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
+golang.org/x/net v0.0.0-20170329170435-ffcf1bedda3b h1:Co3zyosPfwWowmu8+roHGC+aDgizpCPH3ukhubZ0Ttg=
-github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
+golang.org/x/net v0.0.0-20170329170435-ffcf1bedda3b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs=
+golang.org/x/oauth2 v0.0.0-20170321013421-7fdf09982454 h1:qH7SPXL1bLgpFB+ycaFjqQ2lI54cG8OGelAQGpmZSnc=
-github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
+golang.org/x/oauth2 v0.0.0-20170321013421-7fdf09982454/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-github.com/jordan-wright/email v4.0.1-0.20200824153738-3f5bafa1cd84+incompatible h1:d60x4RsAHk/UX/0OT8Gc6D7scVvhBbEANpTAWrDhA/I=
+golang.org/x/text v0.0.0-20170401064109-f4b4367115ec/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-github.com/jordan-wright/email v4.0.1-0.20200824153738-3f5bafa1cd84+incompatible/go.mod h1:1c7szIrayyPPB/987hsnvNzLushdWf4o/79s3P08L8A=
+google.golang.org/grpc v0.0.0-20170405173540-b5071124392b/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+gopkg.in/yaml.v2 v2.0.0-20170407172122-cd8b52f8269e h1:o/mfNjxpTLivuKEfxzzwrJ8PmulH2wEp7t713uMwKAA=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+gopkg.in/yaml.v2 v2.0.0-20170407172122-cd8b52f8269e/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/xanzy/go-gitlab v0.50.2 h1:Qm/um2Jryuqusc6VmN7iZYVTQVzNynzSiuMJDnCU1wE=
-github.com/xanzy/go-gitlab v0.50.2/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288 h1:JIqe8uIcRBHXDQVvZtHwp80ai3Lw3IJAeJEs55Dc1W0=
-golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQk=
-google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

hooks/pre-commit

@@ -26,8 +26,10 @@ cd "$tmpdir"
 git --git-dir="${git_dir}" checkout-index -a
 
 # run our checks
+golint
 go fmt
-./scripts/lint.sh
+go vet --shadow
+gocyclo -over 12 .
 go test
 
 # we're done with go so can set GIT_DIR

logserver.go

@@ -17,7 +17,6 @@ limitations under the License.
 package main
 
 import (
-	"archive/tar"
 	"compress/gzip"
 	"io"
 	"log"
@@ -80,9 +79,10 @@ func serveFile(w http.ResponseWriter, r *http.Request, path string) {
 	// for anti-XSS belt-and-braces, set a very restrictive CSP
 	w.Header().Set("Content-Security-Policy", "default-src: none")
 
-	// if it's a directory, serve a listing or a tarball
+	// if it's a directory, serve a listing
 	if d.IsDir() {
-		serveDirectory(w, r, path)
+		log.Println("Serving", path)
+		http.ServeFile(w, r, path)
 		return
 	}
 
@@ -119,122 +119,9 @@ func extensionToMimeType(path string) string {
 		return "image/jpeg"
 	}
 
-	if strings.HasSuffix(path, ".json") {
-		return "application/json"
-	}
 	return "application/octet-stream"
 }
 
-// Chooses to serve either a directory listing or tarball based on the 'format' parameter.
-func serveDirectory(w http.ResponseWriter, r *http.Request, path string) {
-	format, _ := r.URL.Query()["format"]
-	if len(format) == 1 && format[0] == "tar.gz" {
-		log.Println("Serving tarball of", path)
-		err := serveTarball(w, r, path)
-		if err != nil {
-			msg, code := toHTTPError(err)
-			http.Error(w, msg, code)
-			log.Println("Error", err)
-		}
-		return
-	}
-	log.Println("Serving directory listing of", path)
-	http.ServeFile(w, r, path)
-}
-
-// Streams a dynamically created tar.gz file with the contents of the given directory
-// Will serve a partial, corrupted response if there is a error partway through the
-// operation as we stream the response.
-//
-// The resultant tarball will contain a single directory containing all the files
-// so it can unpack cleanly without overwriting other files.
-//
-// Errors are only returned if generated before the tarball has started being
-// written to the ResponseWriter
-func serveTarball(w http.ResponseWriter, r *http.Request, dir string) error {
-	directory, err := os.Open(dir)
-	if err != nil {
-		return err
-	}
-
-	// Creates a "disposition filename"
-	// Take a URL.path like `/2022-01-10/184843-BZZXEGYH/`
-	// and removes leading and trailing `/` and replaces internal `/` with `_`
-	// to form a suitable filename for use in the content-disposition header
-	// dfilename would turn into `2022-01-10_184843-BZZXEGYH`
-	dfilename := strings.Trim(r.URL.Path, "/")
-	dfilename = strings.Replace(dfilename, "/", "_", -1)
-
-	// There is no application/tgz or similar; return a gzip file as best option.
-	// This tends to trigger archive type tools, which will then use the filename to
-	// identify the contents correctly.
-	w.Header().Set("Content-Type", "application/gzip")
-	w.Header().Set("Content-Disposition", "attachment; filename="+dfilename+".tar.gz")
-
-	files, err := directory.Readdir(-1)
-	if err != nil {
-		return err
-	}
-
-	gzip := gzip.NewWriter(w)
-	defer gzip.Close()
-	targz := tar.NewWriter(gzip)
-	defer targz.Close()
-
-	for _, file := range files {
-		if file.IsDir() {
-			// We avoid including nested directories
-			// This will result in requests for directories with only directories in
-			// to return an empty tarball instead of recursively including directories.
-			// This helps the server remain performant as a download of 'everything' would be slow
-			continue
-		}
-		path := dir + "/" + file.Name()
-		// We use the existing disposition filename to create a base directory structure for the files
-		// so when they are unpacked, they are grouped in a unique folder on disk
-		err := addToArchive(targz, dfilename, path)
-		if err != nil {
-			// From this point we assume that data may have been sent to the client already.
-			// We therefore do not http.Error() after this point, instead closing the stream and
-			// allowing the client to deal with a partial file as if there was a network issue.
-			log.Println("Error streaming tarball", err)
-			return nil
-		}
-	}
-	return nil
-}
-
-// Add a single file into the archive.
-func addToArchive(targz *tar.Writer, dfilename string, filename string) error {
-	file, err := os.Open(filename)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	info, err := file.Stat()
-	if err != nil {
-		return err
-	}
-
-	header, err := tar.FileInfoHeader(info, info.Name())
-	if err != nil {
-		return err
-	}
-	header.Name = dfilename + "/" + info.Name()
-
-	err = targz.WriteHeader(header)
-	if err != nil {
-		return err
-	}
-
-	_, err = io.Copy(targz, file)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
 func serveGzippedFile(w http.ResponseWriter, r *http.Request, path string, size int64) {
 	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
 

main.go

@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -21,20 +22,17 @@ import (
 	"crypto/subtle"
 	"flag"
 	"fmt"
+	"github.com/google/go-github/github"
+	"golang.org/x/oauth2"
 	"io/ioutil"
 	"log"
-	"math/rand"
 	"net"
 	"net/http"
 	"os"
 	"strings"
 	"time"
 
-	"github.com/google/go-github/github"
+	yaml "gopkg.in/yaml.v2"
-	"github.com/xanzy/go-gitlab"
-	"golang.org/x/oauth2"
-
-	"gopkg.in/yaml.v2"
 )
 
 var configPath = flag.String("config", "rageshake.yaml", "The path to the config file. For more information, see the config file in this repository.")
@@ -51,28 +49,13 @@ type config struct {
 	// A GitHub personal access token, to create a GitHub issue for each report.
 	GithubToken string `yaml:"github_token"`
 
+	// Mappings from app name (as submitted in the API) to github repo for issue reporting.
 	GithubProjectMappings map[string]string `yaml:"github_project_mappings"`
-
+	// Mappings from app name (as submitted in the API) to github repo to which the issues pertain.
-	GitlabURL   string `yaml:"gitlab_url"`
+	// Not needed if the issues are reported to the main repo as github will complete the ambiguous references correctly.
-	GitlabToken string `yaml:"gitlab_token"`
+	AutocompleteProjectMappings map[string]string `yaml:"autocomplete_project_mappings"`
-
-	GitlabProjectMappings   map[string]int      `yaml:"gitlab_project_mappings"`
-	GitlabProjectLabels     map[string][]string `yaml:"gitlab_project_labels"`
-	GitlabIssueConfidential bool                `yaml:"gitlab_issue_confidential"`
 
 	SlackWebhookURL string `yaml:"slack_webhook_url"`
-
-	EmailAddresses []string `yaml:"email_addresses"`
-
-	EmailFrom string `yaml:"email_from"`
-
-	SMTPServer string `yaml:"smtp_server"`
-
-	SMTPUsername string `yaml:"smtp_username"`
-
-	SMTPPassword string `yaml:"smtp_password"`
-
-	GenericWebhookURLs []string `yaml:"generic_webhook_urls"`
 }
 
 func basicAuth(handler http.Handler, username, password, realm string) http.Handler {
@@ -113,31 +96,14 @@ func main() {
 		ghClient = github.NewClient(tc)
 	}
 
-	var glClient *gitlab.Client
-	if cfg.GitlabToken == "" {
-		fmt.Println("No gitlab_token configured. Reporting bugs to gitlab is disaled.")
-	} else {
-		glClient, err = gitlab.NewClient(cfg.GitlabToken, gitlab.WithBaseURL(cfg.GitlabURL))
-		if err != nil {
-			// This probably only happens if the base URL is invalid
-			log.Fatalln("Failed to create GitLab client:", err)
-		}
-	}
-
 	var slack *slackClient
 
 	if cfg.SlackWebhookURL == "" {
 		fmt.Println("No slack_webhook_url configured. Reporting bugs to slack is disabled.")
 	} else {
-		slack = newSlackClient(cfg.SlackWebhookURL)
+		slack = NewSlackClient(cfg.SlackWebhookURL)
 	}
 
-	if len(cfg.EmailAddresses) > 0 && cfg.SMTPServer == "" {
-		log.Fatal("Email address(es) specified but no smtp_server configured. Wrong configuration, aborting...")
-	}
-
-	genericWebhookClient := configureGenericWebhookClient(cfg)
-
 	apiPrefix := cfg.APIPrefix
 	if apiPrefix == "" {
 		_, port, err := net.SplitHostPort(*bindAddr)
@@ -151,8 +117,7 @@ func main() {
 	}
 	log.Printf("Using %s/listing as public URI", apiPrefix)
 
-	rand.Seed(time.Now().UnixNano())
+	http.Handle("/api/submit", &submitServer{ghClient, apiPrefix, cfg.GithubProjectMappings, cfg.AutocompleteProjectMappings, slack})
-	http.Handle("/api/submit", &submitServer{ghClient, glClient, apiPrefix, slack, genericWebhookClient, cfg})
 
 	// Make sure bugs directory exists
 	_ = os.Mkdir("bugs", os.ModePerm)
@@ -171,26 +136,11 @@ func main() {
 	}
 	http.Handle("/api/listing/", fs)
 
-	http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprint(w, "ok")
-	})
-
 	log.Println("Listening on", *bindAddr)
 
 	log.Fatal(http.ListenAndServe(*bindAddr, nil))
 }
 
-func configureGenericWebhookClient(cfg *config) *http.Client {
-	if len(cfg.GenericWebhookURLs) == 0 {
-		fmt.Println("No generic_webhook_urls configured.")
-		return nil
-	}
-	fmt.Println("Will forward metadata of all requests to ", cfg.GenericWebhookURLs)
-	return &http.Client{
-		Timeout: time.Second * 300,
-	}
-}
-
 func loadConfig(configPath string) (*config, error) {
 	contents, err := ioutil.ReadFile(configPath)
 	if err != nil {

rageshake.sample.yaml

@@ -17,42 +17,11 @@ github_token: secrettoken
 github_project_mappings:
    my-app: octocat/HelloWorld
 
-# a GitLab personal access token (https://gitlab.com/-/profile/personal_access_tokens), which
+# mappings from app name (as submitted in the API) to github repo to which the issues pertain.
-# will be used to create a GitLab issue for each report. It requires
+# not needed if the issues are reported to the main repo as github will complete the ambiguous references correctly.
-# `api` scope. If omitted, no issues will be created.
+autocomplete_project_mappings:
-gitlab_token: secrettoken
+   my-app: octocat/HelloWorld_src
-# the base URL of the GitLab instance to use
-gitlab_url: https://gitlab.com
-
-# mappings from app name (as submitted in the API) to the GitLab Project ID (not name!) for issue reporting.
-gitlab_project_mappings:
-  my-app: 12345
-# mappings from app name to a list of GitLab label names for issue reporting.
-gitlab_project_labels:
-  my-app:
-    - client::my-app
-# whether GitLab issues should be created as confidential issues. Defaults to false.
-gitlab_issue_confidential: true
 
 # a Slack personal webhook URL (https://api.slack.com/incoming-webhooks), which
 # will be used to post a notification on Slack for each report.
 slack_webhook_url: https://hooks.slack.com/services/TTTTTTT/XXXXXXXXXX/YYYYYYYYYYY
-
-# notification can also be pushed by email.
-# this param controls the target emails
-email_addresses:
-  - support@matrix.org
-
-# this is the from field that will be used in the email notifications
-email_from: Rageshake <rageshake@matrix.org>
-
-# SMTP server configuration
-smtp_server: localhost:25
-smtp_username: myemailuser
-smtp_password: myemailpass
-
-
-# a list of webhook URLs, (see docs/generic_webhook.md)
-generic_webhook_urls: 
-  - https://server.example.com/your-server/api
-  - http://another-server.com/api

scripts/lint.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-#
-# check the go source for lint. This is run by CI, and the pre-commit hook.
-
-# we *don't* check gofmt here, following the advice at
-# https://golang.org/doc/go1.10#gofmt
-
-set -eu
-
-golint -set_exit_status
-go vet -vettool=$(which shadow)
-gocyclo -over 12 .

slack.go

@@ -1,9 +1,9 @@
 package main
 
 import (
+	"strings"
 	"fmt"
 	"net/http"
-	"strings"
 )
 
 type slackClient struct {
@@ -12,7 +12,7 @@ type slackClient struct {
 	face    string
 }
 
-func newSlackClient(webHook string) *slackClient {
+func NewSlackClient(webHook string) *slackClient {
 	return &slackClient{
 		webHook: webHook,
 		name:    "Notifier",

submit.go

@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -20,17 +21,14 @@ import (
 	"bytes"
 	"compress/gzip"
 	"context"
-	"encoding/base32"
 	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
-	"math/rand"
 	"mime"
 	"mime/multipart"
 	"net/http"
-	"net/smtp"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -40,8 +38,6 @@ import (
 	"time"
 
 	"github.com/google/go-github/github"
-	"github.com/jordan-wright/email"
-	"github.com/xanzy/go-gitlab"
 )
 
 var maxPayloadSize = 1024 * 1024 * 55 // 55 MB
@@ -50,15 +46,15 @@ type submitServer struct {
 	// github client for reporting bugs. may be nil, in which case,
 	// reporting is disabled.
 	ghClient *github.Client
-	glClient *gitlab.Client
 
 	// External URI to /api
 	apiPrefix string
 
-	slack *slackClient
+	// mappings from application to github owner/project
+	githubProjectMappings       map[string]string
+	autocompleteProjectMappings map[string]string
 
-	genericWebhookClient *http.Client
+	slack *slackClient
-	cfg                  *config
 }
 
 // the type of payload which can be uploaded as JSON to the submit endpoint
@@ -77,38 +73,19 @@ type jsonLogEntry struct {
 	Lines string `json:"lines"`
 }
 
-// Stores additional information created during processing of a payload
+// the payload after parsing
-type genericWebhookPayload struct {
+type parsedPayload struct {
-	payload
+	UserText   string
-	// If a github/gitlab report is generated, this is set.
+	AppName    string
-	ReportURL  string `json:"report_url"`
+	Data       map[string]string
-	// Complete link to the listing URL that contains all uploaded logs
+	Labels     []string
-	ListingURL string `json:"listing_url"`
+	Logs       []string
+	LogErrors  []string
+	Files      []string
+	FileErrors []string
 }
 
-// Stores information about a request made to this server
+func (p parsedPayload) WriteTo(out io.Writer) {
-type payload struct {
-	// A unique ID for this payload, generated within this server 
-	ID         string            `json:"id"`
-	// A multi-line string containing the user description of the fault. 
-	UserText   string            `json:"user_text"`
-	// A short slug to identify the app making the report
-	AppName    string            `json:"app"`
-	// Arbitrary data to annotate the report
-	Data       map[string]string `json:"data"`
-	// Short labels to group reports
-	Labels     []string          `json:"labels"`
-	// A list of names of logs recognised by the server
-	Logs       []string          `json:"logs"`
-	// Set if there are log parsing errors
-	LogErrors  []string          `json:"logErrors"`
-	// A list of other files (not logs) uploaded as part of the rageshake
-	Files      []string          `json:"files"`
-	// Set if there are file parsing errors
-	FileErrors []string          `json:"fileErrors"`
-}
-
-func (p payload) WriteTo(out io.Writer) {
 	fmt.Fprintf(
 		out,
 		"%s\n\nNumber of logs: %d\nApplication: %s\n",
@@ -143,6 +120,16 @@ type submitResponse struct {
 	ReportURL string `json:"report_url,omitempty"`
 }
 
+// regex to match and substitute ambiguous issue references in the rageshake body text
+// matches a hash followed by digits optionally surrounded by whitespace or some punctuation
+// also matches if the input starts with digits where the hash becomes optional
+var ambiguousIssueRegex = regexp.MustCompile(`(^|[([{\s])(?:^|#)(\d+)([^\w]|$)`)
+
+func replaceAmbiguousIssueReferences(ownerRepo, text string) string {
+	t := ambiguousIssueRegex.ReplaceAllString(text, fmt.Sprintf("${1}%s#$2$3", ownerRepo))
+	return t
+}
+
 func (s *submitServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	// if we attempt to return a response without reading the request body,
 	// apache gets upset and returns a 500. Let's try this.
@@ -167,9 +154,6 @@ func (s *submitServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	// files straight in
 	t := time.Now().UTC()
 	prefix := t.Format("2006-01-02/150405")
-	randBytes := make([]byte, 5)
-	rand.Read(randBytes)
-	prefix += "-" + base32.StdEncoding.EncodeToString(randBytes)
 	reportDir := filepath.Join("bugs", prefix)
 	if err := os.MkdirAll(reportDir, os.ModePerm); err != nil {
 		log.Println("Unable to create report directory", err)
@@ -191,10 +175,9 @@ func (s *submitServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	// We use this prefix (eg, 2022-05-01/125223-abcde) as a unique identifier for this rageshake.
+	if s.autocompleteProjectMappings[p.AppName] != "" {
-	// This is going to be used to uniquely identify rageshakes, even if they are not submitted to
+		p.UserText = replaceAmbiguousIssueReferences(s.autocompleteProjectMappings[p.AppName], p.UserText)
-	// an issue tracker for instance with automatic rageshakes that can be plentiful
+	}
-	p.ID = prefix
 
 	resp, err := s.saveReport(req.Context(), *p, reportDir, listingURL)
 	if err != nil {
@@ -210,7 +193,7 @@ func (s *submitServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 
 // parseRequest attempts to parse a received request as a bug report. If
 // the request cannot be parsed, it responds with an error and returns nil.
-func parseRequest(w http.ResponseWriter, req *http.Request, reportDir string) *payload {
+func parseRequest(w http.ResponseWriter, req *http.Request, reportDir string) *parsedPayload {
 	length, err := strconv.Atoi(req.Header.Get("Content-Length"))
 	if err != nil {
 		log.Println("Couldn't parse content-length", err)
@@ -246,13 +229,13 @@ func parseRequest(w http.ResponseWriter, req *http.Request, reportDir string) *p
 	return p
 }
 
-func parseJSONRequest(w http.ResponseWriter, req *http.Request, reportDir string) (*payload, error) {
+func parseJSONRequest(w http.ResponseWriter, req *http.Request, reportDir string) (*parsedPayload, error) {
 	var p jsonPayload
 	if err := json.NewDecoder(req.Body).Decode(&p); err != nil {
 		return nil, err
 	}
 
-	parsed := payload{
+	parsed := parsedPayload{
 		UserText: strings.TrimSpace(p.Text),
 		Data:     make(map[string]string),
 		Labels:   p.Labels,
@@ -307,13 +290,13 @@ func parseJSONRequest(w http.ResponseWriter, req *http.Request, reportDir string
 	return &parsed, nil
 }
 
-func parseMultipartRequest(w http.ResponseWriter, req *http.Request, reportDir string) (*payload, error) {
+func parseMultipartRequest(w http.ResponseWriter, req *http.Request, reportDir string) (*parsedPayload, error) {
 	rdr, err := req.MultipartReader()
 	if err != nil {
 		return nil, err
 	}
 
-	p := payload{
+	p := parsedPayload{
 		Data: make(map[string]string),
 	}
 
@@ -332,7 +315,7 @@ func parseMultipartRequest(w http.ResponseWriter, req *http.Request, reportDir s
 	return &p, nil
 }
 
-func parseFormPart(part *multipart.Part, p *payload, reportDir string) error {
+func parseFormPart(part *multipart.Part, p *parsedPayload, reportDir string) error {
 	defer part.Close()
 	field := part.FormName()
 	partName := part.FileName()
@@ -393,7 +376,7 @@ func parseFormPart(part *multipart.Part, p *payload, reportDir string) error {
 
 // formPartToPayload updates the relevant part of *p from a name/value pair
 // read from the form data.
-func formPartToPayload(field, data string, p *payload) {
+func formPartToPayload(field, data string, p *parsedPayload) {
 	if field == "text" {
 		p.UserText = data
 	} else if field == "app" {
@@ -419,7 +402,7 @@ func formPartToPayload(field, data string, p *payload) {
 // * no silly characters (/, ctrl chars, etc)
 //
 // * nothing starting with '.'
-var filenameRegexp = regexp.MustCompile(`^[a-zA-Z0-9_-]+\.(jpg|png|txt|json)$`)
+var filenameRegexp = regexp.MustCompile(`^[a-zA-Z0-9_-]+\.(jpg|png|txt)$`)
 
 // saveFormPart saves a file upload to the report directory.
 //
@@ -449,7 +432,7 @@ func saveFormPart(leafName string, reader io.Reader, reportDir string) (string,
 
 // we require a sensible extension, and don't allow the filename to start with
 // '.'
-var logRegexp = regexp.MustCompile(`^[a-zA-Z0-9_-][a-zA-Z0-9_.-]*\.(log|txt)(\.gz)?$`)
+var logRegexp = regexp.MustCompile(`^[a-zA-Z0-9_-][a-zA-Z0-9_.-]*\.(log|txt)$`)
 
 // saveLogPart saves a log upload to the report directory.
 //
@@ -460,16 +443,10 @@ func saveLogPart(logNum int, filename string, reader io.Reader, reportDir string
 	// some clients use sensible names (foo.N.log), which we preserve. For
 	// others, we just make up a filename.
 	//
-	// We append a ".gz" extension if not already present, as the final file we store on
+	// Either way, we need to append .gz, because we're compressing it.
-	// disk will be gzipped. The original filename may or may not contain a '.gz' depending
-	// on the client that uploaded it, and if it was uploaded already compressed.
-
 	var leafName string
 	if logRegexp.MatchString(filename) {
-		leafName = filename
+		leafName = filename + ".gz"
-		if !strings.HasSuffix(filename, ".gz") {
-			leafName += ".gz"
-		}
 	} else {
 		leafName = fmt.Sprintf("logs-%04d.log.gz", logNum)
 	}
@@ -493,7 +470,7 @@ func saveLogPart(logNum int, filename string, reader io.Reader, reportDir string
 	return leafName, nil
 }
 
-func (s *submitServer) saveReport(ctx context.Context, p payload, reportDir, listingURL string) (*submitResponse, error) {
+func (s *submitServer) saveReport(ctx context.Context, p parsedPayload, reportDir, listingURL string) (*submitResponse, error) {
 	var summaryBuf bytes.Buffer
 	resp := submitResponse{}
 	p.WriteTo(&summaryBuf)
@@ -505,159 +482,74 @@ func (s *submitServer) saveReport(ctx context.Context, p payload, reportDir, lis
 		return nil, err
 	}
 
-	if err := s.submitGitlabIssue(p, listingURL, &resp); err != nil {
-		return nil, err
-	}
-
 	if err := s.submitSlackNotification(p, listingURL); err != nil {
 		return nil, err
 	}
 
-	if err := s.sendEmail(p, reportDir); err != nil {
-		return nil, err
-	}
-
-	if err := s.submitGenericWebhook(p, listingURL, resp.ReportURL); err != nil {
-		return nil, err
-	}
-
 	return &resp, nil
 }
 
-// submitGenericWebhook submits a basic JSON body to an endpoint configured in the config
+func (s *submitServer) submitGithubIssue(ctx context.Context, p parsedPayload, listingURL string, resp *submitResponse) error {
-//
+	if s.ghClient == nil {
-// The request does not include the log body, only the metadata in the payload,
+		log.Println("GH issue submission disabled")
-// with the required listingURL to obtain the logs over http if required.
+	} else {
-//
+		// submit a github issue
-// If a github or gitlab issue was previously made, the reportURL will also be passed.
+		ghProj := s.githubProjectMappings[p.AppName]
-//
+		if ghProj == "" {
-// Uses a goroutine to handle the http request asynchronously as by this point all critical
+			log.Println("Not creating GH issue for unknown app", p.AppName)
-// information has been stored.
+			return nil
+		}
+		splits := strings.SplitN(ghProj, "/", 2)
+		if len(splits) < 2 {
+			log.Println("Can't create GH issue for invalid repo", ghProj)
+		}
+		owner, repo := splits[0], splits[1]
 
-func (s *submitServer) submitGenericWebhook(p payload, listingURL string, reportURL string) error {
+		issueReq := buildGithubIssueRequest(p, listingURL)
-	if s.genericWebhookClient == nil {
-		return nil
-	}
-	genericHookPayload := genericWebhookPayload{
-		payload: p,
-		ReportURL:     reportURL,
-		ListingURL:    listingURL,
-	}
-	for _, url := range s.cfg.GenericWebhookURLs {
-		// Enrich the payload with a reportURL and listingURL, to convert a single struct
-		// to JSON easily
 
-		payloadBuffer := new(bytes.Buffer)
+		issue, _, err := s.ghClient.Issues.Create(ctx, owner, repo, &issueReq)
-		json.NewEncoder(payloadBuffer).Encode(genericHookPayload)
-		req, err := http.NewRequest("POST", url, payloadBuffer)
-		req.Header.Set("Content-Type", "application/json")
 		if err != nil {
-			log.Println("Unable to submit to URL ", url, " ", err)
 			return err
 		}
-		log.Println("Making generic webhook request to URL ", url)
+
-		go s.sendGenericWebhook(req)
+		log.Println("Created issue:", *issue.HTMLURL)
+
+		resp.ReportURL = *issue.HTMLURL
 	}
 	return nil
 }
 
-func (s *submitServer) sendGenericWebhook(req *http.Request) {
+func (s *submitServer) submitSlackNotification(p parsedPayload, listingURL string) error {
-	resp, err := s.genericWebhookClient.Do(req)
-	if err != nil {
-		log.Println("Unable to submit notification", err)
-	} else {
-		defer resp.Body.Close()
-		log.Println("Got response", resp.Status)
-	}
-}
-
-func (s *submitServer) submitGithubIssue(ctx context.Context, p payload, listingURL string, resp *submitResponse) error {
-	if s.ghClient == nil {
-		return nil
-	}
-
-	// submit a github issue
-	ghProj := s.cfg.GithubProjectMappings[p.AppName]
-	if ghProj == "" {
-		log.Println("Not creating GH issue for unknown app", p.AppName)
-		return nil
-	}
-	splits := strings.SplitN(ghProj, "/", 2)
-	if len(splits) < 2 {
-		log.Println("Can't create GH issue for invalid repo", ghProj)
-	}
-	owner, repo := splits[0], splits[1]
-
-	issueReq := buildGithubIssueRequest(p, listingURL)
-
-	issue, _, err := s.ghClient.Issues.Create(ctx, owner, repo, &issueReq)
-	if err != nil {
-		return err
-	}
-
-	log.Println("Created issue:", *issue.HTMLURL)
-
-	resp.ReportURL = *issue.HTMLURL
-
-	return nil
-}
-
-func (s *submitServer) submitGitlabIssue(p payload, listingURL string, resp *submitResponse) error {
-	if s.glClient == nil {
-		return nil
-	}
-
-	glProj := s.cfg.GitlabProjectMappings[p.AppName]
-	glLabels := s.cfg.GitlabProjectLabels[p.AppName]
-
-	issueReq := buildGitlabIssueRequest(p, listingURL, glLabels, s.cfg.GitlabIssueConfidential)
-
-	issue, _, err := s.glClient.Issues.CreateIssue(glProj, issueReq)
-
-	if err != nil {
-		return err
-	}
-
-	log.Println("Created issue:", issue.WebURL)
-
-	resp.ReportURL = issue.WebURL
-
-	return nil
-}
-
-func (s *submitServer) submitSlackNotification(p payload, listingURL string) error {
 	if s.slack == nil {
-		return nil
+		log.Println("Slack notifications disabled")
+	} else {
+		slackBuf := fmt.Sprintf(
+			"%s\nApplication: %s\nReport: %s",
+			p.UserText, p.AppName, listingURL,
+		)
+
+		err := s.slack.Notify(slackBuf)
+		if err != nil {
+			return err
+		}
 	}
-
-	slackBuf := fmt.Sprintf(
-		"%s\nApplication: %s\nReport: %s",
-		p.UserText, p.AppName, listingURL,
-	)
-
-	err := s.slack.Notify(slackBuf)
-	if err != nil {
-		return err
-	}
-
 	return nil
 }
 
-func buildReportTitle(p payload) string {
+func buildGithubIssueRequest(p parsedPayload, listingURL string) github.IssueRequest {
 	// set the title to the first (non-empty) line of the user's report, if any
+	var title string
 	trimmedUserText := strings.TrimSpace(p.UserText)
 	if trimmedUserText == "" {
-		return "Untitled report"
+		title = "Untitled report"
+	} else {
+		if i := strings.IndexAny(trimmedUserText, "\r\n"); i < 0 {
+			title = trimmedUserText
+		} else {
+			title = trimmedUserText[0:i]
+		}
 	}
 
-	if i := strings.IndexAny(trimmedUserText, "\r\n"); i >= 0 {
-		return trimmedUserText[0:i]
-	}
-
-	return trimmedUserText
-}
-
-func buildReportBody(p payload, newline, quoteChar string) *bytes.Buffer {
 	var bodyBuf bytes.Buffer
 	fmt.Fprintf(&bodyBuf, "User message:\n\n%s\n\n", p.UserText)
 	var dataKeys []string
@@ -667,36 +559,20 @@ func buildReportBody(p payload, newline, quoteChar string) *bytes.Buffer {
 	sort.Strings(dataKeys)
 	for _, k := range dataKeys {
 		v := p.Data[k]
-		fmt.Fprintf(&bodyBuf, "%s: %s%s%s%s", k, quoteChar, v, quoteChar, newline)
+		fmt.Fprintf(&bodyBuf, "%s: `%s`\n", k, v)
 	}
-
+	fmt.Fprintf(&bodyBuf, "[Logs](%s)", listingURL)
-	return &bodyBuf
-}
-
-func buildGenericIssueRequest(p payload, listingURL string) (title, body string) {
-	bodyBuf := buildReportBody(p, "  \n", "`")
-
-	// Add log links to the body
-	fmt.Fprintf(bodyBuf, "\n[Logs](%s)", listingURL)
 
 	for _, file := range p.Files {
 		fmt.Fprintf(
-			bodyBuf,
+			&bodyBuf,
 			" / [%s](%s)",
 			file,
 			listingURL+"/"+file,
 		)
 	}
 
-	title = buildReportTitle(p)
+	body := bodyBuf.String()
-
-	body = bodyBuf.String()
-
-	return
-}
-
-func buildGithubIssueRequest(p payload, listingURL string) github.IssueRequest {
-	title, body := buildGenericIssueRequest(p, listingURL)
 
 	labels := p.Labels
 	// go-github doesn't like nils
@@ -710,57 +586,6 @@ func buildGithubIssueRequest(p payload, listingURL string) github.IssueRequest {
 	}
 }
 
-func buildGitlabIssueRequest(p payload, listingURL string, labels []string, confidential bool) *gitlab.CreateIssueOptions {
-	title, body := buildGenericIssueRequest(p, listingURL)
-
-	if p.Labels != nil {
-		labels = append(labels, p.Labels...)
-	}
-
-	return &gitlab.CreateIssueOptions{
-		Title:        &title,
-		Description:  &body,
-		Confidential: &confidential,
-		Labels:       labels,
-	}
-}
-
-func (s *submitServer) sendEmail(p payload, reportDir string) error {
-	if len(s.cfg.EmailAddresses) == 0 {
-		return nil
-	}
-
-	e := email.NewEmail()
-
-	e.From = "Rageshake <rageshake@matrix.org>"
-	if s.cfg.EmailFrom != "" {
-		e.From = s.cfg.EmailFrom
-	}
-
-	e.To = s.cfg.EmailAddresses
-
-	e.Subject = fmt.Sprintf("[%s] %s", p.AppName, buildReportTitle(p))
-
-	e.Text = buildReportBody(p, "\n", "\"").Bytes()
-
-	allFiles := append(p.Files, p.Logs...)
-	for _, file := range allFiles {
-		fullPath := filepath.Join(reportDir, file)
-		e.AttachFile(fullPath)
-	}
-
-	var auth smtp.Auth = nil
-	if s.cfg.SMTPPassword != "" || s.cfg.SMTPUsername != "" {
-		auth = smtp.PlainAuth("", s.cfg.SMTPUsername, s.cfg.SMTPPassword, s.cfg.SMTPServer)
-	}
-	err := e.Send(s.cfg.SMTPServer, auth)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 func respond(code int, w http.ResponseWriter) {
 	w.WriteHeader(code)
 	w.Write([]byte("{}"))

submit_test.go

@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -35,7 +36,7 @@ import (
 //
 // if tempDir is empty, a new temp dir is created, and deleted when the test
 // completes.
-func testParsePayload(t *testing.T, body, contentType string, tempDir string) (*payload, *http.Response) {
+func testParsePayload(t *testing.T, body, contentType string, tempDir string) (*parsedPayload, *http.Response) {
 	req, err := http.NewRequest("POST", "/api/submit", strings.NewReader(body))
 	if err != nil {
 		t.Fatal(err)
@@ -160,7 +161,6 @@ func TestMultipartUpload(t *testing.T) {
 
 	// check file uploaded correctly
 	checkUploadedFile(t, reportDir, "passwd.txt", false, "bibblybobbly")
-	checkUploadedFile(t, reportDir, "crash.log.gz", true, "test\n")
 }
 
 func multipartBody() (body string) {
@@ -216,29 +216,17 @@ Content-Type: application/octet-stream
 
 bibblybobbly
 `
-	body += `------WebKitFormBoundarySsdgl8Nq9voFyhdO
-Content-Disposition: form-data; name="compressed-log"; filename="crash.log.gz"
-Content-Type: application/octet-stream
-
-`
-	body += string([]byte{
-		0x1f, 0x8b, 0x08, 0x00, 0xbf, 0xd8, 0xf5, 0x58, 0x00, 0x03,
-		0x2b, 0x49, 0x2d, 0x2e, 0xe1, 0x02, 0x00,
-		0xc6, 0x35, 0xb9, 0x3b, 0x05, 0x00, 0x00, 0x00,
-		0x0a,
-	})
-
 	body += "------WebKitFormBoundarySsdgl8Nq9voFyhdO--\n"
 	return
 }
 
-func checkParsedMultipartUpload(t *testing.T, p *payload) {
+func checkParsedMultipartUpload(t *testing.T, p *parsedPayload) {
 	wanted := "test words."
 	if p.UserText != wanted {
 		t.Errorf("User text: got %s, want %s", p.UserText, wanted)
 	}
-	if len(p.Logs) != 4 {
+	if len(p.Logs) != 3 {
-		t.Errorf("Log length: got %d, want 4", len(p.Logs))
+		t.Errorf("Log length: got %d, want 3", len(p.Logs))
 	}
 	if len(p.Data) != 3 {
 		t.Errorf("Data length: got %d, want 3", len(p.Data))
@@ -262,10 +250,6 @@ func checkParsedMultipartUpload(t *testing.T, p *payload) {
 	if p.Logs[2] != wanted {
 		t.Errorf("Log 2: got %s, want %s", p.Logs[2], wanted)
 	}
-	wanted = "crash.log.gz"
-	if p.Logs[3] != wanted {
-		t.Errorf("Log 3: got %s, want %s", p.Logs[3], wanted)
-	}
 }
 
 func TestLabels(t *testing.T) {
@@ -478,7 +462,7 @@ user_id: id
 	}
 	var buf bytes.Buffer
 	for _, v := range sample {
-		p := payload{Data: v.data}
+		p := parsedPayload{Data: v.data}
 		buf.Reset()
 		p.WriteTo(&buf)
 		got := strings.TrimSpace(buf.String())
@@ -488,7 +472,7 @@ user_id: id
 	}
 
 	for k, v := range sample {
-		p := payload{Data: v.data}
+		p := parsedPayload{Data: v.data}
 		res := buildGithubIssueRequest(p, "")
 		got := *res.Body
 		if k == 0 {
@@ -500,3 +484,22 @@ user_id: id
 		}
 	}
 }
+
+func TestAutocompleteIssueReferences(t *testing.T) {
+	tests := map[string]string{
+		"Testing #123 Foobar": "Testing owner/repo#123 Foobar", // standard
+		"#123":                "owner/repo#123",                // first/last word
+		"test (#123) bar":     "test (owner/repo#123) bar",     // brackets
+		"Start #123. Now":     "Start owner/repo#123. Now",     // followed by punctuation
+		"#123foo":             "#123foo",                       // ignore
+		"foo/#123":            "foo/#123",                      // ignore
+		"123":                 "owner/repo#123",                // special case for entire body being a number
+	}
+
+	for text, expect := range tests {
+		got := replaceAmbiguousIssueReferences("owner/repo", text)
+		if expect != got {
+			t.Errorf("expected %s got %s", expect, got)
+		}
+	}
+}

towncrier.toml

@@ -1,19 +0,0 @@
-[tool.towncrier]
-    filename = "CHANGES.md"
-    directory = "changelog.d"
-    issue_format = "[\\#{issue}](https://github.com/matrix-org/rageshake/issues/{issue})"
-
-    [[tool.towncrier.type]]
-        directory = "feature"
-        name = "Features"
-        showcontent = true
-
-    [[tool.towncrier.type]]
-        directory = "bugfix"
-        name = "Bugfixes"
-        showcontent = true
-
-    [[tool.towncrier.type]]
-        directory = "misc"
-        name = "Internal Changes"
-        showcontent = true