mystiq/rageshake
Archived
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #54 from matrix-org/michaelk/persist_unique_id_in_shakes

Browse source 
Persist prefix as a unique id in rageshakes
This commit is contained in:
Michael Kaye 2022-04-14 14:13:08 +01:00 committed by GitHub
commit 02237888c9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 62 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
changelog.d/54.feature Normal file
View file

@ -0,0 +1 @@
Pass the prefix as a unique ID for the rageshake to the generic webhook mechanism.

8
logserver.go
View file

@ -69,7 +69,6 @@ func (f *logServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
serveFile(w, r, upath) serveFile(w, r, upath)
} }
func serveFile(w http.ResponseWriter, r *http.Request, path string) { func serveFile(w http.ResponseWriter, r *http.Request, path string) {
d, err := os.Stat(path) d, err := os.Stat(path)
if err != nil { if err != nil {
@ -163,14 +162,14 @@ func serveTarball(w http.ResponseWriter, r *http.Request, dir string) error {
// and removes leading and trailing `/` and replaces internal `/` with `_` // and removes leading and trailing `/` and replaces internal `/` with `_`
// to form a suitable filename for use in the content-disposition header // to form a suitable filename for use in the content-disposition header
// dfilename would turn into `2022-01-10_184843-BZZXEGYH` // dfilename would turn into `2022-01-10_184843-BZZXEGYH`
dfilename := strings.Trim(r.URL.Path,"/") dfilename := strings.Trim(r.URL.Path, "/")
dfilename = strings.Replace(dfilename, "/","_",-1) dfilename = strings.Replace(dfilename, "/", "_", -1)
// There is no application/tgz or similar; return a gzip file as best option. // There is no application/tgz or similar; return a gzip file as best option.
// This tends to trigger archive type tools, which will then use the filename to // This tends to trigger archive type tools, which will then use the filename to
// identify the contents correctly. // identify the contents correctly.
w.Header().Set("Content-Type", "application/gzip") w.Header().Set("Content-Type", "application/gzip")
w.Header().Set("Content-Disposition", "attachment; filename=" + dfilename + ".tar.gz") w.Header().Set("Content-Disposition", "attachment; filename="+dfilename+".tar.gz")
files, err := directory.Readdir(-1) files, err := directory.Readdir(-1)
if err != nil { if err != nil {
@ -182,7 +181,6 @@ func serveTarball(w http.ResponseWriter, r *http.Request, dir string) error {
targz := tar.NewWriter(gzip) targz := tar.NewWriter(gzip)
defer targz.Close() defer targz.Close()
for _, file := range files { for _, file := range files {
if file.IsDir() { if file.IsDir() {
// We avoid including nested directories // We avoid including nested directories

2
main.go
View file

@ -180,7 +180,7 @@ func main() {
log.Fatal(http.ListenAndServe(*bindAddr, nil)) log.Fatal(http.ListenAndServe(*bindAddr, nil))
} }
func configureGenericWebhookClient(cfg *config) (*http.Client) { func configureGenericWebhookClient(cfg *config) *http.Client {
if len(cfg.GenericWebhookURLs) == 0 { if len(cfg.GenericWebhookURLs) == 0 {
fmt.Println("No generic_webhook_urls configured.") fmt.Println("No generic_webhook_urls configured.")
return nil return nil

70
submit.go
View file

@ -77,26 +77,38 @@ type jsonLogEntry struct {
Lines string `json:"lines"` Lines string `json:"lines"`
} }
// Stores additional information created during processing of a payload
type genericWebhookPayload struct { type genericWebhookPayload struct {
parsedPayload payload
// If a github/gitlab report is generated, this is set.
ReportURL string `json:"report_url"` ReportURL string `json:"report_url"`
// Complete link to the listing URL that contains all uploaded logs
ListingURL string `json:"listing_url"` ListingURL string `json:"listing_url"`
} }
// the payload after parsing // Stores information about a request made to this server
type parsedPayload struct { type payload struct {
// A unique ID for this payload, generated within this server
ID string `json:"id"`
// A multi-line string containing the user description of the fault.
UserText string `json:"user_text"` UserText string `json:"user_text"`
// A short slug to identify the app making the report
AppName string `json:"app"` AppName string `json:"app"`
// Arbitrary data to annotate the report
Data map[string]string `json:"data"` Data map[string]string `json:"data"`
// Short labels to group reports
Labels []string `json:"labels"` Labels []string `json:"labels"`
// A list of names of logs recognised by the server
Logs []string `json:"logs"` Logs []string `json:"logs"`
// Set if there are log parsing errors
LogErrors []string `json:"logErrors"` LogErrors []string `json:"logErrors"`
// A list of other files (not logs) uploaded as part of the rageshake
Files []string `json:"files"` Files []string `json:"files"`
// Set if there are file parsing errors
FileErrors []string `json:"fileErrors"` FileErrors []string `json:"fileErrors"`
} }
func (p parsedPayload) WriteTo(out io.Writer) { func (p payload) WriteTo(out io.Writer) {
fmt.Fprintf( fmt.Fprintf(
out, out,
"%s\n\nNumber of logs: %d\nApplication: %s\n", "%s\n\nNumber of logs: %d\nApplication: %s\n",
@ -179,6 +191,11 @@ func (s *submitServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return return
} }
// We use this prefix (eg, 2022-05-01/125223-abcde) as a unique identifier for this rageshake.
// This is going to be used to uniquely identify rageshakes, even if they are not submitted to
// an issue tracker for instance with automatic rageshakes that can be plentiful
p.ID = prefix
resp, err := s.saveReport(req.Context(), *p, reportDir, listingURL) resp, err := s.saveReport(req.Context(), *p, reportDir, listingURL)
if err != nil { if err != nil {
log.Println("Error handling report submission:", err) log.Println("Error handling report submission:", err)
@ -193,7 +210,7 @@ func (s *submitServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// parseRequest attempts to parse a received request as a bug report. If // parseRequest attempts to parse a received request as a bug report. If
// the request cannot be parsed, it responds with an error and returns nil. // the request cannot be parsed, it responds with an error and returns nil.
func parseRequest(w http.ResponseWriter, req *http.Request, reportDir string) *parsedPayload { func parseRequest(w http.ResponseWriter, req *http.Request, reportDir string) *payload {
length, err := strconv.Atoi(req.Header.Get("Content-Length")) length, err := strconv.Atoi(req.Header.Get("Content-Length"))
if err != nil { if err != nil {
log.Println("Couldn't parse content-length", err) log.Println("Couldn't parse content-length", err)
@ -229,13 +246,13 @@ func parseRequest(w http.ResponseWriter, req *http.Request, reportDir string) *p
return p return p
} }
func parseJSONRequest(w http.ResponseWriter, req *http.Request, reportDir string) (*parsedPayload, error) { func parseJSONRequest(w http.ResponseWriter, req *http.Request, reportDir string) (*payload, error) {
var p jsonPayload var p jsonPayload
if err := json.NewDecoder(req.Body).Decode(&p); err != nil { if err := json.NewDecoder(req.Body).Decode(&p); err != nil {
return nil, err return nil, err
} }
parsed := parsedPayload{ parsed := payload{
UserText: strings.TrimSpace(p.Text), UserText: strings.TrimSpace(p.Text),
Data: make(map[string]string), Data: make(map[string]string),
Labels: p.Labels, Labels: p.Labels,
@ -290,13 +307,13 @@ func parseJSONRequest(w http.ResponseWriter, req *http.Request, reportDir string
return &parsed, nil return &parsed, nil
} }
func parseMultipartRequest(w http.ResponseWriter, req *http.Request, reportDir string) (*parsedPayload, error) { func parseMultipartRequest(w http.ResponseWriter, req *http.Request, reportDir string) (*payload, error) {
rdr, err := req.MultipartReader() rdr, err := req.MultipartReader()
if err != nil { if err != nil {
return nil, err return nil, err
} }
p := parsedPayload{ p := payload{
Data: make(map[string]string), Data: make(map[string]string),
} }
@ -315,7 +332,7 @@ func parseMultipartRequest(w http.ResponseWriter, req *http.Request, reportDir s
return &p, nil return &p, nil
} }
func parseFormPart(part *multipart.Part, p *parsedPayload, reportDir string) error { func parseFormPart(part *multipart.Part, p *payload, reportDir string) error {
defer part.Close() defer part.Close()
field := part.FormName() field := part.FormName()
partName := part.FileName() partName := part.FileName()
@ -376,7 +393,7 @@ func parseFormPart(part *multipart.Part, p *parsedPayload, reportDir string) err
// formPartToPayload updates the relevant part of *p from a name/value pair // formPartToPayload updates the relevant part of *p from a name/value pair
// read from the form data. // read from the form data.
func formPartToPayload(field, data string, p *parsedPayload) { func formPartToPayload(field, data string, p *payload) {
if field == "text" { if field == "text" {
p.UserText = data p.UserText = data
} else if field == "app" { } else if field == "app" {
@ -476,7 +493,7 @@ func saveLogPart(logNum int, filename string, reader io.Reader, reportDir string
return leafName, nil return leafName, nil
} }
func (s *submitServer) saveReport(ctx context.Context, p parsedPayload, reportDir, listingURL string) (*submitResponse, error) { func (s *submitServer) saveReport(ctx context.Context, p payload, reportDir, listingURL string) (*submitResponse, error) {
var summaryBuf bytes.Buffer var summaryBuf bytes.Buffer
resp := submitResponse{} resp := submitResponse{}
p.WriteTo(&summaryBuf) p.WriteTo(&summaryBuf)
@ -509,7 +526,7 @@ func (s *submitServer) saveReport(ctx context.Context, p parsedPayload, reportDi
// submitGenericWebhook submits a basic JSON body to an endpoint configured in the config // submitGenericWebhook submits a basic JSON body to an endpoint configured in the config
// //
// The request does not include the log body, only the metadata in the parsedPayload, // The request does not include the log body, only the metadata in the payload,
// with the required listingURL to obtain the logs over http if required. // with the required listingURL to obtain the logs over http if required.
// //
// If a github or gitlab issue was previously made, the reportURL will also be passed. // If a github or gitlab issue was previously made, the reportURL will also be passed.
@ -517,17 +534,17 @@ func (s *submitServer) saveReport(ctx context.Context, p parsedPayload, reportDi
// Uses a goroutine to handle the http request asynchronously as by this point all critical // Uses a goroutine to handle the http request asynchronously as by this point all critical
// information has been stored. // information has been stored.
func (s *submitServer) submitGenericWebhook(p parsedPayload, listingURL string, reportURL string) error { func (s *submitServer) submitGenericWebhook(p payload, listingURL string, reportURL string) error {
if s.genericWebhookClient == nil { if s.genericWebhookClient == nil {
return nil return nil
} }
genericHookPayload := genericWebhookPayload{ genericHookPayload := genericWebhookPayload{
parsedPayload: p, payload: p,
ReportURL: reportURL, ReportURL: reportURL,
ListingURL: listingURL, ListingURL: listingURL,
} }
for _, url := range s.cfg.GenericWebhookURLs { for _, url := range s.cfg.GenericWebhookURLs {
// Enrich the parsedPayload with a reportURL and listingURL, to convert a single struct // Enrich the payload with a reportURL and listingURL, to convert a single struct
// to JSON easily // to JSON easily
payloadBuffer := new(bytes.Buffer) payloadBuffer := new(bytes.Buffer)
@ -554,8 +571,7 @@ func (s *submitServer) sendGenericWebhook(req *http.Request) {
} }
} }
func (s *submitServer) submitGithubIssue(ctx context.Context, p payload, listingURL string, resp *submitResponse) error {
func (s *submitServer) submitGithubIssue(ctx context.Context, p parsedPayload, listingURL string, resp *submitResponse) error {
if s.ghClient == nil { if s.ghClient == nil {
return nil return nil
} }
@ -586,7 +602,7 @@ func (s *submitServer) submitGithubIssue(ctx context.Context, p parsedPayload, l
return nil return nil
} }
func (s *submitServer) submitGitlabIssue(p parsedPayload, listingURL string, resp *submitResponse) error { func (s *submitServer) submitGitlabIssue(p payload, listingURL string, resp *submitResponse) error {
if s.glClient == nil { if s.glClient == nil {
return nil return nil
} }
@ -609,7 +625,7 @@ func (s *submitServer) submitGitlabIssue(p parsedPayload, listingURL string, res
return nil return nil
} }
func (s *submitServer) submitSlackNotification(p parsedPayload, listingURL string) error { func (s *submitServer) submitSlackNotification(p payload, listingURL string) error {
if s.slack == nil { if s.slack == nil {
return nil return nil
} }
@ -627,7 +643,7 @@ func (s *submitServer) submitSlackNotification(p parsedPayload, listingURL strin
return nil return nil
} }
func buildReportTitle(p parsedPayload) string { func buildReportTitle(p payload) string {
// set the title to the first (non-empty) line of the user's report, if any // set the title to the first (non-empty) line of the user's report, if any
trimmedUserText := strings.TrimSpace(p.UserText) trimmedUserText := strings.TrimSpace(p.UserText)
if trimmedUserText == "" { if trimmedUserText == "" {
@ -641,7 +657,7 @@ func buildReportTitle(p parsedPayload) string {
return trimmedUserText return trimmedUserText
} }
func buildReportBody(p parsedPayload, newline, quoteChar string) *bytes.Buffer { func buildReportBody(p payload, newline, quoteChar string) *bytes.Buffer {
var bodyBuf bytes.Buffer var bodyBuf bytes.Buffer
fmt.Fprintf(&bodyBuf, "User message:\n\n%s\n\n", p.UserText) fmt.Fprintf(&bodyBuf, "User message:\n\n%s\n\n", p.UserText)
var dataKeys []string var dataKeys []string
@ -657,7 +673,7 @@ func buildReportBody(p parsedPayload, newline, quoteChar string) *bytes.Buffer {
return &bodyBuf return &bodyBuf
} }
func buildGenericIssueRequest(p parsedPayload, listingURL string) (title, body string) { func buildGenericIssueRequest(p payload, listingURL string) (title, body string) {
bodyBuf := buildReportBody(p, " \n", "`") bodyBuf := buildReportBody(p, " \n", "`")
// Add log links to the body // Add log links to the body
@ -679,7 +695,7 @@ func buildGenericIssueRequest(p parsedPayload, listingURL string) (title, body s
return return
} }
func buildGithubIssueRequest(p parsedPayload, listingURL string) github.IssueRequest { func buildGithubIssueRequest(p payload, listingURL string) github.IssueRequest {
title, body := buildGenericIssueRequest(p, listingURL) title, body := buildGenericIssueRequest(p, listingURL)
labels := p.Labels labels := p.Labels
@ -694,7 +710,7 @@ func buildGithubIssueRequest(p parsedPayload, listingURL string) github.IssueReq
} }
} }
func buildGitlabIssueRequest(p parsedPayload, listingURL string, labels []string, confidential bool) *gitlab.CreateIssueOptions { func buildGitlabIssueRequest(p payload, listingURL string, labels []string, confidential bool) *gitlab.CreateIssueOptions {
title, body := buildGenericIssueRequest(p, listingURL) title, body := buildGenericIssueRequest(p, listingURL)
if p.Labels != nil { if p.Labels != nil {
@ -709,7 +725,7 @@ func buildGitlabIssueRequest(p parsedPayload, listingURL string, labels []string
} }
} }
func (s *submitServer) sendEmail(p parsedPayload, reportDir string) error { func (s *submitServer) sendEmail(p payload, reportDir string) error {
if len(s.cfg.EmailAddresses) == 0 { if len(s.cfg.EmailAddresses) == 0 {
return nil return nil
} }

8
submit_test.go
View file

@ -35,7 +35,7 @@ import (
// //
// if tempDir is empty, a new temp dir is created, and deleted when the test // if tempDir is empty, a new temp dir is created, and deleted when the test
// completes. // completes.
func testParsePayload(t *testing.T, body, contentType string, tempDir string) (*parsedPayload, *http.Response) { func testParsePayload(t *testing.T, body, contentType string, tempDir string) (*payload, *http.Response) {
req, err := http.NewRequest("POST", "/api/submit", strings.NewReader(body)) req, err := http.NewRequest("POST", "/api/submit", strings.NewReader(body))
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -232,7 +232,7 @@ Content-Type: application/octet-stream
return return
} }
func checkParsedMultipartUpload(t *testing.T, p *parsedPayload) { func checkParsedMultipartUpload(t *testing.T, p *payload) {
wanted := "test words." wanted := "test words."
if p.UserText != wanted { if p.UserText != wanted {
t.Errorf("User text: got %s, want %s", p.UserText, wanted) t.Errorf("User text: got %s, want %s", p.UserText, wanted)
@ -478,7 +478,7 @@ user_id: id
} }
var buf bytes.Buffer var buf bytes.Buffer
for _, v := range sample { for _, v := range sample {
p := parsedPayload{Data: v.data} p := payload{Data: v.data}
buf.Reset() buf.Reset()
p.WriteTo(&buf) p.WriteTo(&buf)
got := strings.TrimSpace(buf.String()) got := strings.TrimSpace(buf.String())
@ -488,7 +488,7 @@ user_id: id
} }
for k, v := range sample { for k, v := range sample {
p := parsedPayload{Data: v.data} p := payload{Data: v.data}
res := buildGithubIssueRequest(p, "") res := buildGithubIssueRequest(p, "")
got := *res.Body got := *res.Body
if k == 0 { if k == 0 {