Multipart upload support (#12)

Allow submission of the input data as a multipart form, instead of JSON.

README.md

@@ -33,7 +33,10 @@ report submission date and time.
 
 Submission endpoint: this is where applications should send their reports.
 
-The body of the request should be a JSON object with the following fields:
+The body of the request should be a multipart form-data submission, with the
+following form field names. (For backwards compatibility, it can also be a JSON
+object, but multipart is preferred as it allows more efficient transfer of the
+logs.)
 
 * `text`: A textual description of the problem. Included in the
   `details.log.gz` file.
@@ -46,12 +49,22 @@ The body of the request should be a JSON object with the following fields:
 
 * `version`: Application version. Included in the `details.log.gz` file.
 
-* `logs`: an of log files. Each entry in the list should be an object with the
+* `log`: a log file, with lines separated by newline characters. Multiple log
-  following fields:
+  files can be included by including several `log` parts.
 
-  * `id`: textual identifier for the logs. Currently ignored.
+  If using the JSON upload encoding, the request object should instead include
-  * `lines`: log data. Lines should be separated by newline characters (encoded
+  a single `logs` field, which is an array of objects with the following
-    as `\n`, as normal in JSON).
+  fields:
+
+    * `id`: textual identifier for the logs. Currently ignored.
+    * `lines`: log data. Newlines should be  encoded as `\n`, as normal in JSON).
+
+* Any other form field names are interpreted as arbitrary name/value strings to
+  include in the `details.log.gz` file.
+
+  If using the JSON upload encoding, this additional metadata should insted be
+  encoded as a `data` field, whose value should be a JSON map. (Note that the
+  values must be strings; numbers, objects and arrays will be rejected.)
 
 The response (if successful) will be a JSON object with the following fields:
 

src/github.com/matrix-org/rageshake/submit.go

@@ -23,8 +23,11 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/google/go-github/github"
+	"io"
 	"io/ioutil"
 	"log"
+	"mime"
+	"mime/multipart"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -112,12 +115,35 @@ func parseRequest(w http.ResponseWriter, req *http.Request) *payload {
 		http.Error(w, fmt.Sprintf("Content too large (max %i)", maxPayloadSize), 413)
 		return nil
 	}
-	var p payload
+
-	if err := json.NewDecoder(req.Body).Decode(&p); err != nil {
+	contentType := req.Header.Get("Content-Type")
-		log.Println("Couldn't decode request body", err)
+	if contentType != "" {
+		d, _, _ := mime.ParseMediaType(contentType)
+		if d == "multipart/form-data" {
+			p, err1 := parseMultipartRequest(w, req)
+			if err1 != nil {
+				log.Println("Error parsing multipart data", err1)
+				http.Error(w, "Bad multipart data", 400)
+				return nil
+			}
+			return p
+		}
+	}
+
+	p, err := parseJSONRequest(w, req)
+	if err != nil {
+		log.Println("Error parsing JSON body", err)
 		http.Error(w, fmt.Sprintf("Could not decode payload: %s", err.Error()), 400)
 		return nil
 	}
+	return p
+}
+
+func parseJSONRequest(w http.ResponseWriter, req *http.Request) (*payload, error) {
+	var p payload
+	if err := json.NewDecoder(req.Body).Decode(&p); err != nil {
+		return nil, err
+	}
 
 	p.Text = strings.TrimSpace(p.Text)
 
@@ -148,7 +174,62 @@ func parseRequest(w http.ResponseWriter, req *http.Request) *payload {
 		p.Version = ""
 	}
 
-	return &p
+	return &p, nil
+}
+
+func parseMultipartRequest(w http.ResponseWriter, req *http.Request) (*payload, error) {
+	rdr, err := req.MultipartReader()
+	if err != nil {
+		return nil, err
+	}
+
+	p := payload{
+		Logs: make([]logEntry, 0),
+		Data: make(map[string]string),
+	}
+
+	for true {
+		part, err := rdr.NextPart()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+
+		if err = parseFormPart(part, &p); err != nil {
+			return nil, err
+		}
+	}
+	return &p, nil
+}
+
+func parseFormPart(part *multipart.Part, p *payload) error {
+	defer part.Close()
+	field := part.FormName()
+
+	b, err := ioutil.ReadAll(part)
+	if err != nil {
+		return err
+	}
+	data := string(b)
+
+	if field == "text" {
+		p.Text = data
+	} else if field == "app" {
+		p.AppName = data
+	} else if field == "version" {
+		p.Version = data
+	} else if field == "user_agent" {
+		p.UserAgent = data
+	} else if field == "log" {
+		p.Logs = append(p.Logs, logEntry{
+			ID:    part.FileName(),
+			Lines: data,
+		})
+	} else {
+		p.Data[field] = data
+	}
+	return nil
 }
 
 func (s *submitServer) saveReport(ctx context.Context, p payload) (*submitResponse, error) {

src/github.com/matrix-org/rageshake/submit_test.go

@@ -77,3 +77,71 @@ func TestUnpickAndroidMangling(t *testing.T) {
 		t.Errorf("data.version: got %s, want %s", p.Data["Version"], "0:6:9")
 	}
 }
+
+func TestMultipartUpload(t *testing.T) {
+	body := `------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="text"
+
+test words.
+------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="app"
+
+riot-web
+------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="version"
+
+UNKNOWN
+------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="user_agent"
+
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36
+------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="test-field"
+
+Test data
+------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="log"; filename="instance-0.215954445471346461492087122412"
+Content-Type: text/plain
+
+log
+log
+log
+------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="log"; filename="instance-0.067644760733513781492004890379"
+Content-Type: text/plain
+
+log
+------WebKitFormBoundarySsdgl8Nq9voFyhdO--
+`
+	p := testParsePayload(t, body, "multipart/form-data; boundary=----WebKitFormBoundarySsdgl8Nq9voFyhdO")
+	wanted := "test words."
+	if p.Text != wanted {
+		t.Errorf("User text: got %s, want %s", p.Text, wanted)
+	}
+	if len(p.Logs) != 2 {
+		t.Errorf("Log length: got %d, want 2", len(p.Logs))
+	}
+	if len(p.Data) != 1 {
+		t.Errorf("Data length: got %d, want 1", len(p.Data))
+	}
+	wanted = "Test data"
+	if p.Data["test-field"] != wanted {
+		t.Errorf("test-field: got %s, want %s", p.Data["test-field"], wanted)
+	}
+	wanted = "log\nlog\nlog"
+	if p.Logs[0].Lines != wanted {
+		t.Errorf("Log 0: got %s, want %s", p.Logs[0].Lines, wanted)
+	}
+	wanted = "instance-0.215954445471346461492087122412"
+	if p.Logs[0].ID != wanted {
+		t.Errorf("Log 0 ID: got %s, want %s", p.Logs[0].ID, wanted)
+	}
+	wanted = "log"
+	if p.Logs[1].Lines != wanted {
+		t.Errorf("Log 1: got %s, want %s", p.Logs[1].Lines, wanted)
+	}
+	wanted = "instance-0.067644760733513781492004890379"
+	if p.Logs[1].ID != wanted {
+		t.Errorf("Log 1 ID: got %s, want %s", p.Logs[1].ID, wanted)
+	}
+}