From 7cb9486333bee9d5e25736dca057f8deea6bd233 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 18 Apr 2017 15:43:04 +0100 Subject: [PATCH] Multipart upload support (#12) Allow submission of the input data as a multipart form, instead of JSON. --- README.md | 25 ++++-- src/github.com/matrix-org/rageshake/submit.go | 89 ++++++++++++++++++- .../matrix-org/rageshake/submit_test.go | 68 ++++++++++++++ 3 files changed, 172 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 67a329d..540df23 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,10 @@ report submission date and time. Submission endpoint: this is where applications should send their reports. -The body of the request should be a JSON object with the following fields: +The body of the request should be a multipart form-data submission, with the +following form field names. (For backwards compatibility, it can also be a JSON +object, but multipart is preferred as it allows more efficient transfer of the +logs.) * `text`: A textual description of the problem. Included in the `details.log.gz` file. @@ -46,12 +49,22 @@ The body of the request should be a JSON object with the following fields: * `version`: Application version. Included in the `details.log.gz` file. -* `logs`: an of log files. Each entry in the list should be an object with the - following fields: +* `log`: a log file, with lines separated by newline characters. Multiple log + files can be included by including several `log` parts. - * `id`: textual identifier for the logs. Currently ignored. - * `lines`: log data. Lines should be separated by newline characters (encoded - as `\n`, as normal in JSON). + If using the JSON upload encoding, the request object should instead include + a single `logs` field, which is an array of objects with the following + fields: + + * `id`: textual identifier for the logs. Currently ignored. + * `lines`: log data. Newlines should be encoded as `\n`, as normal in JSON). + +* Any other form field names are interpreted as arbitrary name/value strings to + include in the `details.log.gz` file. + + If using the JSON upload encoding, this additional metadata should insted be + encoded as a `data` field, whose value should be a JSON map. (Note that the + values must be strings; numbers, objects and arrays will be rejected.) The response (if successful) will be a JSON object with the following fields: diff --git a/src/github.com/matrix-org/rageshake/submit.go b/src/github.com/matrix-org/rageshake/submit.go index 63b4236..46b3f2e 100644 --- a/src/github.com/matrix-org/rageshake/submit.go +++ b/src/github.com/matrix-org/rageshake/submit.go @@ -23,8 +23,11 @@ import ( "encoding/json" "fmt" "github.com/google/go-github/github" + "io" "io/ioutil" "log" + "mime" + "mime/multipart" "net/http" "os" "path/filepath" @@ -112,12 +115,35 @@ func parseRequest(w http.ResponseWriter, req *http.Request) *payload { http.Error(w, fmt.Sprintf("Content too large (max %i)", maxPayloadSize), 413) return nil } - var p payload - if err := json.NewDecoder(req.Body).Decode(&p); err != nil { - log.Println("Couldn't decode request body", err) + + contentType := req.Header.Get("Content-Type") + if contentType != "" { + d, _, _ := mime.ParseMediaType(contentType) + if d == "multipart/form-data" { + p, err1 := parseMultipartRequest(w, req) + if err1 != nil { + log.Println("Error parsing multipart data", err1) + http.Error(w, "Bad multipart data", 400) + return nil + } + return p + } + } + + p, err := parseJSONRequest(w, req) + if err != nil { + log.Println("Error parsing JSON body", err) http.Error(w, fmt.Sprintf("Could not decode payload: %s", err.Error()), 400) return nil } + return p +} + +func parseJSONRequest(w http.ResponseWriter, req *http.Request) (*payload, error) { + var p payload + if err := json.NewDecoder(req.Body).Decode(&p); err != nil { + return nil, err + } p.Text = strings.TrimSpace(p.Text) @@ -148,7 +174,62 @@ func parseRequest(w http.ResponseWriter, req *http.Request) *payload { p.Version = "" } - return &p + return &p, nil +} + +func parseMultipartRequest(w http.ResponseWriter, req *http.Request) (*payload, error) { + rdr, err := req.MultipartReader() + if err != nil { + return nil, err + } + + p := payload{ + Logs: make([]logEntry, 0), + Data: make(map[string]string), + } + + for true { + part, err := rdr.NextPart() + if err == io.EOF { + break + } else if err != nil { + return nil, err + } + + if err = parseFormPart(part, &p); err != nil { + return nil, err + } + } + return &p, nil +} + +func parseFormPart(part *multipart.Part, p *payload) error { + defer part.Close() + field := part.FormName() + + b, err := ioutil.ReadAll(part) + if err != nil { + return err + } + data := string(b) + + if field == "text" { + p.Text = data + } else if field == "app" { + p.AppName = data + } else if field == "version" { + p.Version = data + } else if field == "user_agent" { + p.UserAgent = data + } else if field == "log" { + p.Logs = append(p.Logs, logEntry{ + ID: part.FileName(), + Lines: data, + }) + } else { + p.Data[field] = data + } + return nil } func (s *submitServer) saveReport(ctx context.Context, p payload) (*submitResponse, error) { diff --git a/src/github.com/matrix-org/rageshake/submit_test.go b/src/github.com/matrix-org/rageshake/submit_test.go index 18a8e9a..d5a5ebe 100644 --- a/src/github.com/matrix-org/rageshake/submit_test.go +++ b/src/github.com/matrix-org/rageshake/submit_test.go @@ -77,3 +77,71 @@ func TestUnpickAndroidMangling(t *testing.T) { t.Errorf("data.version: got %s, want %s", p.Data["Version"], "0:6:9") } } + +func TestMultipartUpload(t *testing.T) { + body := `------WebKitFormBoundarySsdgl8Nq9voFyhdO +Content-Disposition: form-data; name="text" + +test words. +------WebKitFormBoundarySsdgl8Nq9voFyhdO +Content-Disposition: form-data; name="app" + +riot-web +------WebKitFormBoundarySsdgl8Nq9voFyhdO +Content-Disposition: form-data; name="version" + +UNKNOWN +------WebKitFormBoundarySsdgl8Nq9voFyhdO +Content-Disposition: form-data; name="user_agent" + +Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36 +------WebKitFormBoundarySsdgl8Nq9voFyhdO +Content-Disposition: form-data; name="test-field" + +Test data +------WebKitFormBoundarySsdgl8Nq9voFyhdO +Content-Disposition: form-data; name="log"; filename="instance-0.215954445471346461492087122412" +Content-Type: text/plain + +log +log +log +------WebKitFormBoundarySsdgl8Nq9voFyhdO +Content-Disposition: form-data; name="log"; filename="instance-0.067644760733513781492004890379" +Content-Type: text/plain + +log +------WebKitFormBoundarySsdgl8Nq9voFyhdO-- +` + p := testParsePayload(t, body, "multipart/form-data; boundary=----WebKitFormBoundarySsdgl8Nq9voFyhdO") + wanted := "test words." + if p.Text != wanted { + t.Errorf("User text: got %s, want %s", p.Text, wanted) + } + if len(p.Logs) != 2 { + t.Errorf("Log length: got %d, want 2", len(p.Logs)) + } + if len(p.Data) != 1 { + t.Errorf("Data length: got %d, want 1", len(p.Data)) + } + wanted = "Test data" + if p.Data["test-field"] != wanted { + t.Errorf("test-field: got %s, want %s", p.Data["test-field"], wanted) + } + wanted = "log\nlog\nlog" + if p.Logs[0].Lines != wanted { + t.Errorf("Log 0: got %s, want %s", p.Logs[0].Lines, wanted) + } + wanted = "instance-0.215954445471346461492087122412" + if p.Logs[0].ID != wanted { + t.Errorf("Log 0 ID: got %s, want %s", p.Logs[0].ID, wanted) + } + wanted = "log" + if p.Logs[1].Lines != wanted { + t.Errorf("Log 1: got %s, want %s", p.Logs[1].Lines, wanted) + } + wanted = "instance-0.067644760733513781492004890379" + if p.Logs[1].ID != wanted { + t.Errorf("Log 1 ID: got %s, want %s", p.Logs[1].ID, wanted) + } +}