From 0cf9e84bddc08f7c49326289c8c9da6faf939031 Mon Sep 17 00:00:00 2001
From: Bruno Windels <brunow@matrix.org>
Date: Fri, 21 Dec 2018 14:35:24 +0100
Subject: [PATCH] WIP

---
 GOAL.md                 |   5 ++
 matrix.mjs              |  95 +++++++++++++++++++++
 room/room.js            |  20 +++++
 room/summary.js         | 185 ++++++++++++++++++++++++++++++++++++++++
 src/error.js            |   6 ++
 src/network.js          |  58 +++++++++++++
 src/session.js          |  34 ++++++++
 src/sync/common.js      |  10 +++
 src/sync/incremental.js |  64 ++++++++++++++
 src/sync/initial.js     |  19 +++++
 10 files changed, 496 insertions(+)
 create mode 100644 GOAL.md
 create mode 100644 matrix.mjs
 create mode 100644 room/room.js
 create mode 100644 room/summary.js
 create mode 100644 src/error.js
 create mode 100644 src/network.js
 create mode 100644 src/session.js
 create mode 100644 src/sync/common.js
 create mode 100644 src/sync/incremental.js
 create mode 100644 src/sync/initial.js

diff --git a/GOAL.md b/GOAL.md
new file mode 100644
index 00000000..00037ae1
--- /dev/null
+++ b/GOAL.md
@@ -0,0 +1,5 @@
+goal:
+
+to write a minimal matrix client that should you all your rooms, allows you to pick one and read and write messages in it.
+
+on the technical side, the goal is to go low-memory, and test the performance of storing every event individually in indexeddb.
\ No newline at end of file
diff --git a/matrix.mjs b/matrix.mjs
new file mode 100644
index 00000000..f484b55e
--- /dev/null
+++ b/matrix.mjs
@@ -0,0 +1,95 @@
+
+/*
+idb stores:
+all in one database per stored session:
+	- session
+		- device id
+		- last sync token
+		- access token
+		- home server
+		- user id
+		- user name
+		- avatar
+		- filter(s)?
+	- room_summaries
+		- room_id
+		- heroes
+		- room_name
+		- room_avatar (just the url)
+		- tags (account_data?)
+		- is_direct
+		- unread_message_count ?
+		- unread_message_with_mention ?
+	- roomstate_{room_id}
+		how about every state event gets a revision number
+		for each state event, we store the min and max revision number where they form part of the room state
+		then we "just" do a where revision_range includes revision, and every state event event/gap in the timeline we store the revision number, and we have an index on it? so we can easily look for the nearest one 
+
+	
+		it's like every state event we know about has a range where it is relevant
+		we want the intersection of a revision with all ranges
+        1           2 3      *   4  5       6
+		|   topic   |     oth*er topic     |
+		|     power levels   *   |
+		| member a'1  | membe*r a'2 |
+							 *-------- get intersection for all or some type & state_keys for revision 3 (forward) or 4 (backwards)
+
+		tricky to do a > && < in indexeddb
+		we'll need to do either > or < for min or max revision and iterate through the cursor and apply the rest of the conditions in code ...
+
+		all current state for last event would have max revision of some special value to indicate it hasn't been replaced yet.
+
+		the idea is that we can easily load just the state for a given event in the timeline,
+		can be the last synced event, or a permalink event
+	- members_{room_id}
+		historical?
+	- timeline_{room_id}
+	- search?
+
+where to store avatars?
+	we could cache the requested ones in a table ...
+	or service worker, but won't work on my phone
+*/
+
+class Credentials {
+	accessToken,
+	deviceId
+}
+
+class LoginFlow {
+
+	constructor(network) {
+
+	}
+//differentiate between next stage and Credentials?
+	async next(stage) {}
+
+	static async attemptPasswordLogin(username, password) {
+
+	}
+}
+
+class LoginStage {
+	get type() {}
+	serialize() {} 	//called by LoginFlow::next
+}
+
+class PasswordStage extends LoginStage {
+	set password() {
+
+	}
+
+	set username() {
+
+	}
+
+	serialize() {
+		return {
+			identifier: {
+				type: "m.id.user",
+				user: this._username
+			},
+			password: this._password
+		};
+	}
+}
\ No newline at end of file
diff --git a/room/room.js b/room/room.js
new file mode 100644
index 00000000..323e564d
--- /dev/null
+++ b/room/room.js
@@ -0,0 +1,20 @@
+class Room {
+
+	constructor(roomId, storage) {
+		this._roomId = roomId;
+		this._storage = storage;
+		this._summary = new RoomSummary(this._roomId, this._storage);
+	}
+
+	async applyInitialSync(roomResponse, membership) {
+
+	}
+
+	async applyIncrementalSync(roomResponse, membership) {
+
+	}
+
+	async loadFromStorage() {
+
+	}
+}
\ No newline at end of file
diff --git a/room/summary.js b/room/summary.js
new file mode 100644
index 00000000..a38b6bc1
--- /dev/null
+++ b/room/summary.js
@@ -0,0 +1,185 @@
+const SUMMARY_NAME_COUNT = 3;
+
+function disambiguateMember(name, userId) {
+	return `${name} (${userId})`;
+}
+
+export class RoomSummary {
+	constructor(roomId, storage) {
+		this._storage = storage;
+		this._members = new SummaryMembers();
+		this._roomId = roomId;
+		this._inviteCount = 0;
+		this._joinCount = 0;
+		this._calculatedName = null;
+		this._nameFromEvent = null;
+		this._lastMessageBody = null;
+	}
+
+	get name() {
+		return this._nameFromEvent || this._calculatedName;
+	}
+
+	get lastMessage() {
+		return this._lastMessageBody;
+	}
+
+	get inviteCount() {
+		return this._inviteCount;
+	}
+
+	get joinCount() {
+		return this._joinCount;
+	}
+
+	async applySync(roomResponse) {
+		const changed = this._processSyncResponse(roomResponse);
+		if (changed) {
+			await this._persist();
+		}
+		return changed;
+	}
+
+	async loadFromStorage() {
+		const summary = await storage.getSummary(this._roomId);
+		this._roomId = summary.roomId;
+		this._inviteCount = summary.inviteCount;
+		this._joinCount = summary.joinCount;
+		this._calculatedName = summary.calculatedName;
+		this._nameFromEvent = summary.nameFromEvent;
+		this._lastMessageBody = summary.lastMessageBody;
+		this._members = new SummaryMembers(summary.members);
+	}
+
+	_persist() {
+		const summary = {
+			roomId: this._roomId,
+			heroes: this._heroes,
+			inviteCount: this._inviteCount,
+			joinCount: this._joinCount,
+			calculatedName: this._calculatedName,
+			nameFromEvent: this._nameFromEvent,
+			lastMessageBody: this._lastMessageBody,
+			members: this._members.asArray()
+		};
+		return this.storage.saveSummary(this.room_id, summary);
+	}
+
+	_processSyncResponse(roomResponse) {
+		// lets not do lazy loading for now
+		// if (roomResponse.summary) {
+		// 	this._updateSummary(roomResponse.summary);
+		// }
+		let changed = false;
+		if (roomResponse.limited) {
+			changed = roomResponse.state_events.events.reduce((changed, e) => {
+				return this._processEvent(e) || changed;
+			}, changed);
+		}
+		changed = roomResponse.timeline.events.reduce((changed, e) => {
+			return this._processEvent(e) || changed;
+		}, changed);
+
+		return changed;
+	}
+
+	_processEvent(event) {
+		if (event.type === "m.room.name") {
+			const newName = event.content && event.content.name;
+			if (newName !== this._nameFromEvent) {
+				this._nameFromEvent = newName;
+				return true;
+			}
+		} else if (event.type === "m.room.member") {
+			return this._processMembership(event);
+		} else if (event.type === "m.room.message") {
+			const content = event.content;
+			const body = content && content.body;
+			const msgtype = content && content.msgtype;
+			if (msgtype === "m.text") {
+				this._lastMessageBody = body;
+				return true;
+			}
+		}
+		return false;
+	}
+
+	_processMembership(event) {
+		let changed = false;
+		const prevMembership = event.prev_content && event.prev_content.membership;
+		const membership = event.content && event.content.membership;
+		// danger of a replayed event getting the count out of sync
+		// but summary api will solve this.
+		// otherwise we'd have to store all the member ids in here
+		if (membership !== prevMembership) {
+			switch (prevMembership) {
+				case "invite": --this._inviteCount;
+				case "join": --this._joinCount;
+			}
+			switch (membership) {
+				case "invite": ++this._inviteCount;
+				case "join": ++this._joinCount;
+			}
+			changed = true;
+		}
+		if (membership === "join" && content.name) {
+			// TODO: avatar_url
+			changed = this._members.applyMember(content.name, content.state_key) || changed;
+		}
+		return changed;
+	}
+
+	_updateSummary(summary) {
+		const heroes = summary["m.heroes"];
+		const inviteCount = summary["m.joined_member_count"];
+		const joinCount = summary["m.invited_member_count"];
+
+		if (heroes) {
+			this._heroes = heroes;
+		}
+		if (Number.isInteger(inviteCount)) {
+			this._inviteCount = inviteCount;
+		}
+		if (Number.isInteger(joinCount)) {
+			this._joinCount = joinCount;
+		}
+		// this._recaculateNameIfNoneSet();
+	}
+}
+
+class SummaryMembers {
+	constructor(initialMembers = []) {
+		this._alphabeticalNames = initialMembers.map(m => m.name);
+	}
+
+	applyMember(name, userId) {
+		let insertionIndex = 0;
+		for (var i = this._alphabeticalNames.length - 1; i >= 0; i--) {
+			const cmp = this._alphabeticalNames[i].localeCompare(name);
+			// name is already in the list, disambiguate
+			if (cmp === 0) {
+				name = disambiguateMember(name, userId);
+			}
+			// name should come after already present name, stop
+			if (cmp >= 0) {
+				insertionIndex = i + 1;
+				break;	
+			}
+		}
+		// don't append names if list is full already
+		if (insertionIndex < SUMMARY_NAME_COUNT) {
+			this._alphabeticalNames.splice(insertionIndex, 0, name);
+		}
+		if (this._alphabeticalNames > SUMMARY_NAME_COUNT) {
+			this._alphabeticalNames = this._alphabeticalNames.slice(0, SUMMARY_NAME_COUNT);
+		}
+	}
+
+	get names() {
+		return this._alphabeticalNames;
+	}
+
+	asArray() {
+		return this._alphabeticalNames.map(n => {name: n});
+	}
+}
diff --git a/src/error.js b/src/error.js
new file mode 100644
index 00000000..f40751b9
--- /dev/null
+++ b/src/error.js
@@ -0,0 +1,6 @@
+export class HomeServerError extends Error {
+	constructor(body) {
+		super(body.error);
+		this.errcode = body.errcode;
+	}
+}
\ No newline at end of file
diff --git a/src/network.js b/src/network.js
new file mode 100644
index 00000000..3d45d876
--- /dev/null
+++ b/src/network.js
@@ -0,0 +1,58 @@
+class Request {
+	constructor(promise, controller) {
+		this._promise = promise;
+		this._controller = controller;
+	}
+
+	abort() {
+		this._controller.abort();
+	}
+
+	response() {
+		return this._promise;
+	}
+}
+
+export class Network {
+	constructor(homeserver, accessToken) {
+		this._homeserver = homeserver;
+		this._accessToken = accessToken;
+	}
+
+	_url(csPath) {
+		return `${this._homeserver}/_matrix/client/r0/${csPath}`;
+	}
+
+	_request(method, csPath, queryParams = {}) {
+		const queryString = Object.entries(queryParams)
+			.filter(([name, value]) => value !== undefined)
+			.map(([name, value]) => `${encodeURIComponent(name)}=${encodeURIComponent(value)}`);
+			.join("&");
+		const url = this._url(`${csPath}?${queryString}`);
+		const request = new Request(url);
+		const headers = request.headers;
+		headers.append("Authorization", `Bearer ${this._accessToken}`);
+		headers.append("Accept", "application/json");
+		if (false/* body */) {
+			headers.append("Content-Type", "application/json");
+		}
+		const controller = new AbortController();
+		// TODO: set authenticated headers with second arguments, cache them
+		let promise = fetch(request, {signal: controller.signal});
+		promise = promise.then(response => {
+			if (response.ok) {
+				return response.json();
+			} else {
+				switch (response.status) {
+					default:
+						throw new HomeServerError(response.json())
+				}
+			}
+		});
+		return new Request(promise, controller);
+	}
+
+	sync(timeout = 0, since = null) {
+		return this._request("GET", "/sync", {since, timeout});
+	}
+}
\ No newline at end of file
diff --git a/src/session.js b/src/session.js
new file mode 100644
index 00000000..9ae9b98c
--- /dev/null
+++ b/src/session.js
@@ -0,0 +1,34 @@
+class Session {
+	// sessionData has device_id and access_token
+	constructor(sessionData) {
+		this._sessionData = sessionData;
+	}
+
+	loadFromStorage() {
+		// what is the PK for a session [user_id, device_id], a uuid?
+	}
+
+	start() {
+		if (!this._syncToken) {
+			do initial sync
+		}
+		do incremental sync
+	}
+
+	stop() {
+		if (this._initialSync) {
+			this._initialSync.abort();
+		}
+		if (this._incrementalSync) {
+			this._incrementalSync.stop();
+		}
+	}
+
+	getRoom(roomId) {
+		return this._rooms[roomId];
+	}
+
+	applySync(newRooms, syncToken, accountData) {
+
+	}
+}
\ No newline at end of file
diff --git a/src/sync/common.js b/src/sync/common.js
new file mode 100644
index 00000000..0e411e71
--- /dev/null
+++ b/src/sync/common.js
@@ -0,0 +1,10 @@
+export function parseRooms(responseSections, roomMapper) {
+	return ["join", "invite", "leave"].map(membership => {
+		const membershipSection = responseSections[membership];
+		const results = Object.entries(membershipSection).map(([roomId, roomResponse]) => {
+			const room = roomMapper(roomId, membership);
+			return room.processInitialSync(roomResponse);
+		});
+		return results;
+	}).reduce((allResults, sectionResults) => allResults.concat(sectionResults), []);
+}
\ No newline at end of file
diff --git a/src/sync/incremental.js b/src/sync/incremental.js
new file mode 100644
index 00000000..a0368c58
--- /dev/null
+++ b/src/sync/incremental.js
@@ -0,0 +1,64 @@
+import {parseRooms} from "./common";
+import {RequestAbortError} from "../network";
+import {HomeServerError} from "../error";
+
+const TIMEOUT = 30;
+
+export class IncrementalSync {
+	constructor(network, session, roomCreator) {
+		this._network = network;
+		this._session = session;
+		this._roomCreator = roomCreator;
+		this._isSyncing = false;
+		this._currentRequest = null;
+	}
+
+	start() {
+		if (this._isSyncing) {
+			return;
+		}
+		this._isSyncing = true;
+		try {
+			this._syncLoop(session.syncToken);
+		} catch(err) {
+			//expected when stop is called
+			if (err instanceof RequestAbortError) {
+
+			} else if (err instanceof HomeServerError) {
+
+			} else {
+				// something threw something
+			}
+		}
+	}
+
+	async _syncLoop(syncToken) {
+		while(this._isSyncing) {
+			this._currentRequest = this._network.sync(TIMEOUT, syncToken);
+			const response = await this._currentRequest.response();
+			syncToken = response.next_batch;
+			const sessionPromise = session.applySync(syncToken, response.account_data);
+			// to_device
+			// presence
+			const roomPromises = parseRooms(response.rooms, async (roomId, roomResponse, membership) => {
+				let room = session.getRoom(roomId);
+				if (!room) {
+					room = await session.createRoom(roomId);
+				}
+				return room.applyIncrementalSync(roomResponse, membership);
+			});
+			await Promise.all(roomPromises.concat(sessionPromise));
+		}
+	}
+
+	stop() {
+		if (!this._isSyncing) {
+			return;
+		}
+		this._isSyncing = false;
+		if (this._currentRequest) {
+			this._currentRequest.abort();
+			this._currentRequest = null;
+		}
+	}
+}
\ No newline at end of file
diff --git a/src/sync/initial.js b/src/sync/initial.js
new file mode 100644
index 00000000..2a2b26a0
--- /dev/null
+++ b/src/sync/initial.js
@@ -0,0 +1,19 @@
+import {parseRooms} from "./common";
+
+// TODO make abortable
+export async function initialSync(network, session) {
+	const response = await network.sync().response();
+	const rooms = await createRooms(response.rooms, session);
+	const sessionData = {syncToken: response.next_batch};
+	const accountData = response.account_data;
+	await session.applySync(rooms, response.next_batch, response.account_data);
+}
+
+function createRooms(responseSections, session) {
+	const roomPromises = parseRooms(responseSections, (roomId, roomResponse, membership) => {
+		const room = await session.createRoom(roomId);
+		await room.initialSync(roomResponse, membership);
+		return room;
+	});
+	return Promise.all(roomPromises);
+}