s/rooms/ranges/

This now allows you to read incoming encrypted events. This is
VERY flakey due to not having device lists, OTK counts and a
fudged session sync lifecycle callbacks but it does appear to
work empirically!

package.json

@@ -58,6 +58,7 @@
     "@rollup/plugin-commonjs": "^15.0.0",
     "@rollup/plugin-json": "^4.1.0",
     "@rollup/plugin-node-resolve": "^9.0.0",
+    "@types/assert": "^1.5.6",
     "aes-js": "^3.1.2",
     "another-json": "^0.2.0",
     "base64-arraybuffer": "^0.2.0",

src/domain/session/SessionViewModel.js

@@ -39,7 +39,9 @@ export class SessionViewModel extends ViewModel {
         })));
         this._leftPanelViewModel = this.track(new LeftPanelViewModel(this.childOptions({
             invites: this._sessionContainer.session.invites,
-            rooms: this._sessionContainer.session.rooms
+            rooms: this._sessionContainer.session.rooms,
+            sync: this._sessionContainer.sync,
+            compareFn: this._sessionContainer.sync.compare.bind(this._sessionContainer.sync),
         })));
         this._settingsViewModel = null;
         this._roomViewModelObservable = null;

src/domain/session/leftpanel/BaseTileViewModel.js

@@ -57,6 +57,10 @@ export class BaseTileViewModel extends ViewModel {
     }
 
     compare(other) {
+        // don't use KIND_ORDER for placeholder|room kinds as they are comparable
+        if (this.kind !== "invite" && other.kind !== "invite") {
+            return 0;
+        }
         if (other.kind !== this.kind) {
             return KIND_ORDER.indexOf(this.kind) - KIND_ORDER.indexOf(other.kind);
         }

src/domain/session/leftpanel/LeftPanelViewModel.js

@@ -19,38 +19,59 @@ import {ViewModel} from "../../ViewModel.js";
 import {RoomTileViewModel} from "./RoomTileViewModel.js";
 import {InviteTileViewModel} from "./InviteTileViewModel.js";
 import {RoomFilter} from "./RoomFilter.js";
+import { ConcatList, MappedList } from "../../../observable/index.js";
+import {FilteredMap} from "../../../observable/map/FilteredMap.js";
+import {Sync3ObservableList} from "../../../matrix/Sync3ObservableList";
 import {ApplyMap} from "../../../observable/map/ApplyMap.js";
 import {addPanelIfNeeded} from "../../navigation/index.js";
+import { PlaceholderRoomTileViewModel } from "./PlaceholderRoomTileViewModel.js";
 
 export class LeftPanelViewModel extends ViewModel {
     constructor(options) {
         super(options);
-        const {rooms, invites} = options;
-        this._tileViewModelsMap = this._mapTileViewModels(rooms, invites);
-        this._tileViewModelsFilterMap = new ApplyMap(this._tileViewModelsMap);
-        this._tileViewModels = this._tileViewModelsFilterMap.sortValues((a, b) => a.compare(b));
+        const {rooms, invites, compareFn, sync} = options;
+        this._sync = sync;
+        const sync3List = new Sync3ObservableList(sync, rooms);
+        const list = new ConcatList(invites.sortValues((a,b) => a.compare(b)), sync3List);
+        this._tileViewModels = this._mapTileViewModels(list);
         this._currentTileVM = null;
         this._setupNavigation();
         this._closeUrl = this.urlCreator.urlForSegment("session");
         this._settingsUrl = this.urlCreator.urlForSegment("settings");
+        this._compareFn = compareFn;
     }
 
-    _mapTileViewModels(rooms, invites) {
-        // join is not commutative, invites will take precedence over rooms
-        return invites.join(rooms).mapValues((roomOrInvite, emitChange) => {
+    _subscribeToRoom(roomId) {
+        this._sync.setRoomSubscriptions([roomId]);
+    }
+
+    _mapTileViewModels(list) {
+        const mapper = (roomOrInvite, emitChange) => {
             let vm;
-            if (roomOrInvite.isInvite) {
+            if (roomOrInvite === null) {
+                vm = new PlaceholderRoomTileViewModel(this.childOptions({room: null, emitChange}));
+            } else if (roomOrInvite.isInvite) {
                 vm = new InviteTileViewModel(this.childOptions({invite: roomOrInvite, emitChange}));
             } else {
-                vm = new RoomTileViewModel(this.childOptions({room: roomOrInvite, emitChange}));
+                vm = new RoomTileViewModel(this.childOptions({
+                    room: roomOrInvite,
+                    compareFn: this._compareFn,
+                    emitChange,
+                }));
             }
+            if (roomOrInvite) {
                 const isOpen = this.navigation.path.get("room")?.value === roomOrInvite.id;
                 if (isOpen) {
                     vm.open();
                     this._updateCurrentVM(vm);
                 }
+            }
             return vm;
-        });
+        };
+        const updater = (tileViewModel, noIdea, roomOrInvite) => {
+            return mapper(roomOrInvite);
+        }
+        return new MappedList(list, mapper, updater);
     }
 
     _updateCurrentVM(vm) {
@@ -88,10 +109,24 @@ export class LeftPanelViewModel extends ViewModel {
         this._currentTileVM?.close();
         this._currentTileVM = null;
         if (roomId) {
-            this._currentTileVM = this._tileViewModelsMap.get(roomId);
+            // find the vm for the room. Previously we used a map to do this but sync3 only gives
+            // us a list. We could've re-mapped things in the observable pipeline but we don't need
+            // these values to be kept up-to-date when a O(n) search on click isn't particularly
+            // expensive.
+            let targetVM;
+            for ( let vm of this._tileViewModels ) {
+                if (vm.id === roomId) {
+                    targetVM = vm;
+                    break;
+                }
+            }
+            if (targetVM) {
+                this._subscribeToRoom(roomId);
+                this._currentTileVM = targetVM;
                 this._currentTileVM?.open();
             }
         }
+    }
 
     toggleGrid() {
         const room = this.navigation.path.get("room");
@@ -137,4 +172,8 @@ export class LeftPanelViewModel extends ViewModel {
             return startFiltering;
         }
     }
+
+    loadRoomRange(range) {
+        this._sync.loadRange(range.start, range.end);
+    }
 }

src/domain/session/leftpanel/PlaceholderRoomTileViewModel.js

@@ -0,0 +1,66 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {getIdentifierColorNumber} from "../../avatar.js";
+import {BaseTileViewModel} from "./BaseTileViewModel.js";
+
+export class PlaceholderRoomTileViewModel extends BaseTileViewModel {
+    constructor(options) {
+        super(options);
+        // Placeholder tiles can be sorted with Room tiles, so we need to ensure we have the same
+        // fields else the comparison needs to take into account the kind().
+        // We need a fake room so we can do compare(other) with RoomTileViewModels 
+        const {room} = options;
+        this._room = room;
+    }
+
+    get busy() {
+        return false;
+    }
+
+    get kind() {
+        return "placeholder";
+    }
+
+    compare(other) {
+        // TODO: factor this out with the compare(other) of the room tile as it does this check as well.
+        // TODO _room is null
+        if (other._room.index !== undefined) {
+            return this._room.index > other._room.index ? 1 : -1;
+        }
+        return super.compare(other);
+    }
+
+    get name() {
+        return "Placeholder";
+    }
+
+    get avatarLetter() {
+        return " ";
+    }
+
+    get avatarColorNumber() {
+        return getIdentifierColorNumber("placeholder"); // TODO: randomise
+    }
+
+    avatarUrl(size) {
+        return null;
+    }
+
+    get avatarTitle() {
+        return "Placeholder";
+    }
+}

src/domain/session/leftpanel/RoomTileViewModel.js

@@ -20,9 +20,10 @@ import {BaseTileViewModel} from "./BaseTileViewModel.js";
 export class RoomTileViewModel extends BaseTileViewModel {
     constructor(options) {
         super(options);
-        const {room} = options;
+        const {room, compareFn} = options;
         this._room = room;
         this._url = this.urlCreator.openRoomActionUrl(this._room.id);
+        this._compareFn = compareFn;
     }
 
     get kind() {
@@ -33,11 +34,23 @@ export class RoomTileViewModel extends BaseTileViewModel {
         return this._url;
     }
 
+    get id() {
+        return this._room.id;
+    }
+
     compare(other) {
+        return this._compareFn(this._room.id, other._room.id);
+        
         const parentComparison = super.compare(other);
         if (parentComparison !== 0) {
             return parentComparison;
         }
+
+        // sync v3 has its own ordering, use it if we have an index
+        if (this._room.index !== undefined && other._room.index !== undefined) {
+            return this._room.index > other._room.index ? 1 : -1;
+        }
+
         /*
         put unread rooms first
         then put rooms with a timestamp first, and sort by name

src/matrix/SessionContainer.js

@@ -24,7 +24,8 @@ import {Reconnector, ConnectionStatus} from "./net/Reconnector.js";
 import {ExponentialRetryDelay} from "./net/ExponentialRetryDelay.js";
 import {MediaRepository} from "./net/MediaRepository.js";
 import {RequestScheduler} from "./net/RequestScheduler.js";
-import {Sync, SyncStatus} from "./Sync.js";
+// import {Sync, SyncStatus} from "./Sync.js";
+import {Sync3, SyncStatus} from "./Sync3";
 import {Session} from "./Session.js";
 import {PasswordLoginMethod} from "./login/PasswordLoginMethod";
 import {TokenLoginMethod} from "./login/TokenLoginMethod";
@@ -254,7 +255,12 @@ export class SessionContainer {
             await log.wrap("createIdentity", log => this._session.createIdentity(log));
         }
         
-        this._sync = new Sync({hsApi: this._requestScheduler.hsApi, storage: this._storage, session: this._session, logger: this._platform.logger});
+        this._sync = new Sync3(
+            this._requestScheduler.hsApi,
+            this._session,
+            this._storage,
+            this._platform.logger,
+        );
         // notify sync and session when back online
         this._reconnectSubscription = this._reconnector.connectionStatus.subscribe(state => {
             if (state === ConnectionStatus.Online) {

src/matrix/Sync3ObservableList.ts

@@ -0,0 +1,226 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import * as assert from "assert";
+import { BaseObservableList } from "../observable/list/BaseObservableList";
+import { ObservableMap } from "../observable/map/ObservableMap.js";
+import { SubscriptionHandle } from "../observable/BaseObservable";
+import { Room } from "./room/Room.js";
+
+// subset of Sync3 functions used in this list; interfaced out for testing
+interface ISync {
+    count(): number;
+    roomAtIndex(i: number): string
+    indexOfRoom(roomId: string): number
+}
+
+/**
+ * An observable list that produces a subset of rooms based on Sync3 responses.
+ */
+export class Sync3ObservableList extends BaseObservableList<Room | null> {
+    sync: ISync;
+    rooms: ObservableMap;
+    subscription: SubscriptionHandle | null;
+
+    /**
+     * Construct an observable list that produces Rooms within the sliding window of Sync3 responses
+     * or `null` to indicate that a placeholder room should be used.
+     * @param sync3 The Sync3 class, which tracks up-to-date information on the sliding window / room counts.
+     * @param rooms The entire set of rooms known to the client (e.g from Session.rooms).
+     */
+    constructor(sync3: ISync, rooms: ObservableMap) {
+        super();
+        this.sync = sync3;
+        this.subscription = null;
+        this.rooms = rooms;
+    }
+
+    onSubscribeFirst() {
+        this.subscription = this.rooms.subscribe(this);
+    }
+
+    onUnsubscribeLast() {
+        if (!this.subscription) {
+            return;
+        }
+        this.subscription();
+        this.subscription = null;
+    }
+
+    onAdd(index, entry) {
+        let i = this.sync.indexOfRoom(entry.id);
+        this.emitUpdate(i, entry); // we always emit updates as we have num_joined_rooms entries (placeholders)
+    }
+
+    onUpdate(index, entry, params) {
+        let i = this.sync.indexOfRoom(entry.id);
+        if (i === -1) {
+            return;
+        }
+        this.emitUpdate(i, entry); // we always emit updates as we have num_joined_rooms entries (placeholders)
+    }
+
+    onRemove(index, entry) {
+        let i = this.sync.indexOfRoom(entry.id);
+        this.emitUpdate(i, entry); // we always emit updates as we have num_joined_rooms entries (placeholders)
+    }
+
+    get length(): number {
+        return this.sync.count();
+    }
+
+    [Symbol.iterator](): Iterator<Room | null, any, undefined> {
+        let i = 0;
+        return {
+            next: (): any => {
+                // base case
+                if (i >= this.length) {
+                    return {
+                        done: true,
+                    };
+                }
+                let roomId = this.sync.roomAtIndex(i);
+                i += 1;
+                if (!roomId) {
+                    return {
+                        value: null
+                    }
+                }
+                return {
+                    value: this.rooms.get(roomId) || null,
+                }
+            }
+        }
+    }
+}
+
+export function tests() {
+    const makeRooms = function (len) {
+        let rooms: any[] = [];
+        for (let i = 0; i < len; i++) {
+            const roomId = "!room-" + i;
+            rooms.push({
+                id: roomId,
+                data: {
+                    some_key: i,
+                },
+            });
+        }
+        return rooms;
+    }
+
+    const assertList = function (assert, rooms, gotList, wantListRoomIds) {
+        assert.equal(wantListRoomIds.length, gotList.length);
+        if (wantListRoomIds.length === 0) {
+            for (const room of gotList) {
+                assert.equal(0, 1); // fail
+            }
+            return;
+        }
+        let i = 0;
+        for (const room of gotList) {
+            const wantRoomId = wantListRoomIds[i];
+            const gotRoomId = room ? room.id : null;
+            assert.strictEqual(wantRoomId, gotRoomId);
+            if (wantRoomId !== null && gotRoomId !== null) {
+                assert.deepEqual(room, rooms.get(wantRoomId));
+            }
+            i += 1;
+        }
+    }
+
+    return {
+        "iterator": (assert) => {
+            const rooms = new ObservableMap();
+            let indexToRoomId = {};
+            let roomCount = 0;
+            const sync3 = {
+                count: (): number => {
+                    return roomCount;
+                },
+                roomAtIndex: (i: number): string => {
+                    return indexToRoomId[i];
+                },
+                indexOfRoom(roomId: string): number {
+                    for (let i of Object.keys(indexToRoomId)) {
+                        let r = indexToRoomId[i];
+                        if (r === roomId) {
+                            return Number(i);
+                        }
+                    }
+                    return -1;
+                }
+            };
+
+            makeRooms(100).forEach((r) => {
+                rooms.add(r.id, r);
+            });
+            const list = new Sync3ObservableList(sync3, rooms);
+
+            // iterate over the list (0 items) as sync3 has no indexes for the rooms
+            assertList(assert, rooms, list, []);
+
+
+            // 'load' 5 rooms from sync v3 and set the total count to 5 so we load the entire room list in one go!
+            // [R,R,R,R,R]
+            let slidingWindow: any[] = [
+                "!room-50", "!room-53", "!room-1", "!room-52", "!room-97"
+            ]
+            roomCount = 5;
+            for (let i = 0; i < slidingWindow.length; i++) {
+                indexToRoomId[i] = slidingWindow[i];
+            }
+            assertList(assert, rooms, list, slidingWindow);
+
+            // now add 5 more rooms which we don't know about, we should iterate through them with `null` entries
+            // [R,R,R,R,R,P,P,P,P,P] (R=room, P=placeholder)
+            roomCount = 10;
+            assertList(assert, rooms, list, slidingWindow.concat([null, null, null, null, null]));
+
+
+            // now track a window in the middle of the list (5-10)
+            indexToRoomId = {};
+            for (let i = 0; i < slidingWindow.length; i++) {
+                indexToRoomId[i + 5] = slidingWindow[i];
+            }
+            roomCount = 15;
+            // [P,P,P,P,P,R,R,R,R,R,P,P,P,P,P]
+            assertList(assert, rooms, list, [null, null, null, null, null].concat(slidingWindow.concat([null, null, null, null, null])));
+
+
+            // now track multiple ranges
+            const anotherSlidingWindow: any[] = [
+                "!room-30", "!room-33", "!room-36", "!room-29", "!room-21"
+            ]
+            for (let i = 0; i < anotherSlidingWindow.length; i++) {
+                indexToRoomId[i + 15] = anotherSlidingWindow[i];
+            }
+            roomCount = 20;
+            // [P,P,P,P,P,R,R,R,R,R,P,P,P,P,P,R,R,R,R,R]
+            assertList(
+                assert, rooms, list,
+                [null, null, null, null, null].concat(
+                    slidingWindow.concat(
+                        [null, null, null, null, null].concat(
+                            anotherSlidingWindow
+                        )
+                    )
+                )
+            );
+
+        },
+    };
+};

src/matrix/net/HomeServerApi.js

@@ -110,6 +110,12 @@ export class HomeServerApi {
         return this._get("/sync", {since, timeout, filter}, null, options);
     }
 
+    sync3(body, pos, timeout, options = null) {
+        // FIXME TODO
+        const syncURL = "http://localhost:8008/_matrix/client/v3/sync";
+        return this._authedRequest("POST", syncURL, {timeout, pos}, body, options);
+    }
+
     // params is from, dir and optionally to, limit, filter.
     messages(roomId, params, options = null) {
         return this._get(`/rooms/${encodeURIComponent(roomId)}/messages`, params, null, options);

src/matrix/room/BaseRoom.js

@@ -76,6 +76,11 @@ export class BaseRoom extends EventEmitter {
         return retryTimelineEntries;
     }
 
+    // Forcibly update this room in collections
+    forceRefresh() {
+        this._emitUpdate();
+    }
+
     /**
      * Used for retrying decryption from other sources than sync, like key backup.
      * @internal

src/matrix/room/Room.js

@@ -54,6 +54,7 @@ export class Room extends BaseRoom {
         return false;
     }
 
+    // {}, string, bool?, [], txn, log
     async prepareSync(roomResponse, membership, invite, newKeys, txn, log) {
         log.set("id", this.id);
         if (newKeys) {
@@ -65,7 +66,7 @@ export class Room extends BaseRoom {
         }
         let roomEncryption = this._roomEncryption;
         // encryption is enabled in this sync
-        if (!roomEncryption && summaryChanges.encryption) {
+        if (!roomEncryption && summaryChanges.encryption && false) { // TODO: re-enable and ensure we call Session._setupEncryption first
             log.set("enableEncryption", true);
             roomEncryption = this._createRoomEncryption(this, summaryChanges.encryption);
         }

src/matrix/room/timeline/persistence/SyncWriter.js

@@ -42,12 +42,16 @@ export class SyncWriter {
         this._relationWriter = relationWriter;
         this._fragmentIdComparer = fragmentIdComparer;
         this._lastLiveKey = null;
+        this._lastEventId = null;
     }
 
     async load(txn, log) {
         const liveFragment = await txn.timelineFragments.liveFragment(this._roomId);
         if (liveFragment) {
             const [lastEvent] = await txn.timelineEvents.lastEvents(this._roomId, liveFragment.id, 1);
+            if (lastEvent && lastEvent.event) {
+                this._lastEventId = lastEvent.event.event_id;
+            }
             // fall back to the default event index in case the fragment was somehow written but no events
             // we should only create fragments when really writing timeline events now
             // (see https://github.com/vector-im/hydrogen-web/issues/112) but can't hurt to be extra robust.
@@ -148,12 +152,39 @@ export class SyncWriter {
     async _writeTimeline(timelineEvents, timeline, memberSync, currentKey, txn, log) {
         const entries = [];
         const updatedEntries = [];
+        timelineEvents = timelineEvents || [];
+        // Sync v3 hack:
+        // Events can come in multiple times currently in the following situation:
+        // - Click on room A, makes a room subscription, fetches most recent 20 events.
+        // - Click on room B, unsubs from room A and subs on room B.
+        // - Click on room A, remakes a room sub, fetches the same most recent 20 events.
+        // This confuses H which then thinks there are timeline fragments and it inserts events into
+        // the wrong places. To hack around this, we track the latest event in the room and drop all
+        // events <= this event ID to ensure we don't see duplicates. We can still get timeline
+        // fragments (e.g in busy rooms when you re-click on the room there may be >20 events) but
+        // this should guard against having dupes.
+        // NB: It is critical that the sliding window request has `timeline_limit: 0`, and ONLY the
+        // room subscription has `timeline_limit: N`. Failure to do this means when you click on a
+        // room you are actually getting _scrollback_ (if originally you want 1 event then want 20)
+        // and this results in dropped events due to not having the correct event key ordering.
+        let lastEventIndex = -1;
+        for (let i = 0; i < timelineEvents.length; i++) {
+            if (timelineEvents[i].event_id === this._lastEventId) {
+                lastEventIndex = i;
+                break;
+            }
+        }
+        if (lastEventIndex >= 0) {
+            timelineEvents.splice(0, lastEventIndex+1);
+        }
+
         if (timelineEvents?.length) {
             // only create a fragment when we will really write an event
             currentKey = await this._ensureLiveFragment(currentKey, entries, timeline, txn, log);
             log.set("timelineEvents", timelineEvents.length);
             let timelineStateEventCount = 0;
             for(const event of timelineEvents) {
+                this._lastEventId = event.event_id;
                 // store event in timeline
                 currentKey = currentKey.nextKey();
                 const storageEntry = createEventEntry(currentKey, this._roomId, event);

src/observable/list/BaseMappedList.ts

@@ -15,21 +15,21 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import {BaseObservableList} from "./BaseObservableList";
-import {findAndUpdateInArray} from "./common";
+import { BaseObservableList } from "./BaseObservableList";
+import { findAndUpdateInArray } from "./common";
 
-export type Mapper<F,T> = (value: F) => T
-export type Updater<F,T> = (mappedValue: T, params: any, value: F) => void;
+export type Mapper<F, T> = (value: F) => T
+export type Updater<F, T> = (mappedValue: T, params: any, value: F) => any;
 
-export class BaseMappedList<F,T,R = T> extends BaseObservableList<T> {
+export class BaseMappedList<F, T, R = T> extends BaseObservableList<T> {
     protected _sourceList: BaseObservableList<F>;
     protected _sourceUnsubscribe: (() => void) | null = null;
-    _mapper: Mapper<F,R>;
-    _updater?: Updater<F,T>;
+    _mapper: Mapper<F, R>;
+    _updater?: Updater<F, T>;
     _removeCallback?: (value: T) => void;
     _mappedValues: T[] | null = null;
 
-    constructor(sourceList: BaseObservableList<F>, mapper: Mapper<F,R>, updater?: Updater<F,T>, removeCallback?: (value: T) => void) {
+    constructor(sourceList: BaseObservableList<F>, mapper: Mapper<F, R>, updater?: Updater<F, T>, removeCallback?: (value: T) => void) {
         super();
         this._sourceList = sourceList;
         this._mapper = mapper;
@@ -50,20 +50,31 @@ export class BaseMappedList<F,T,R = T> extends BaseObservableList<T> {
     }
 }
 
-export function runAdd<F,T,R>(list: BaseMappedList<F,T,R>, index: number, mappedValue: T): void {
+export function runAdd<F, T, R>(list: BaseMappedList<F, T, R>, index: number, mappedValue: T): void {
     list._mappedValues!.splice(index, 0, mappedValue);
     list.emitAdd(index, mappedValue);
 }
 
-export function runUpdate<F,T,R>(list: BaseMappedList<F,T,R>, index: number, value: F, params: any): void {
-    const mappedValue = list._mappedValues![index];
+export function runUpdate<F, T, R>(list: BaseMappedList<F, T, R>, index: number, value: F, params: any): void {
+    let mappedValue = list._mappedValues![index];
     if (list._updater) {
-        list._updater(mappedValue, params, value);
+        // allow updates to completely remap the underlying data type
+        // TODO: do we need to unsubscribe from anything here?
+        let newMappedValue = list._updater(mappedValue, params, value);
+        if (newMappedValue) {
+            if (!params) {
+                params = {};
             }
+            params.oldValue = mappedValue;
+            mappedValue = newMappedValue;
+            list._mappedValues![index] = mappedValue;
+        }
+    }
+    // pass the new updated value down the chain
     list.emitUpdate(index, mappedValue, params);
 }
 
-export function runRemove<F,T,R>(list: BaseMappedList<F,T,R>, index: number): void {
+export function runRemove<F, T, R>(list: BaseMappedList<F, T, R>, index: number): void {
     const mappedValue = list._mappedValues![index];
     list._mappedValues!.splice(index, 1);
     if (list._removeCallback) {
@@ -72,14 +83,14 @@ export function runRemove<F,T,R>(list: BaseMappedList<F,T,R>, index: number): vo
     list.emitRemove(index, mappedValue);
 }
 
-export function runMove<F,T,R>(list: BaseMappedList<F,T,R>, fromIdx: number, toIdx: number): void {
+export function runMove<F, T, R>(list: BaseMappedList<F, T, R>, fromIdx: number, toIdx: number): void {
     const mappedValue = list._mappedValues![fromIdx];
     list._mappedValues!.splice(fromIdx, 1);
     list._mappedValues!.splice(toIdx, 0, mappedValue);
     list.emitMove(fromIdx, toIdx, mappedValue);
 }
 
-export function runReset<F,T,R>(list: BaseMappedList<F,T,R>): void {
+export function runReset<F, T, R>(list: BaseMappedList<F, T, R>): void {
     list._mappedValues = [];
     list.emitReset();
 }

src/observable/list/ConcatList.ts

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import {BaseObservableList, IListObserver} from "./BaseObservableList";
+import { BaseObservableList, IListObserver } from "./BaseObservableList";
 
 export class ConcatList<T> extends BaseObservableList<T> implements IListObserver<T> {
     protected _sourceLists: BaseObservableList<T>[];
@@ -50,7 +50,7 @@ export class ConcatList<T> extends BaseObservableList<T> implements IListObserve
         // reset, and 
         this.emitReset();
         let idx = 0;
-        for(const item of this) {
+        for (const item of this) {
             this.emitAdd(idx, item);
             idx += 1;
         }
@@ -106,8 +106,8 @@ export class ConcatList<T> extends BaseObservableList<T> implements IListObserve
     }
 }
 
-import {ObservableArray} from "./ObservableArray";
-import {defaultObserverWith} from "./BaseObservableList";
+import { ObservableArray } from "./ObservableArray";
+import { defaultObserverWith } from "./BaseObservableList";
 export async function tests() {
     return {
         test_length(assert) {

src/observable/list/SortedMapList.js

@@ -63,7 +63,20 @@ export class SortedMapList extends BaseObservableList {
 
     onRemove(key, value) {
         const pair = {key, value};
-        const idx = sortedIndex(this._sortedPairs, pair, this._comparator);
+        // Don't call sortedIndex as it does a binary search for the removed item.
+        // Whilst that is faster than the O(n) search we're doing here, it's not valid to compare
+        // removed items as the system may have no ability to compare them at this point.
+        let idx = -1;
+        for (let i = 0; i < this._sortedPairs.length; i++) {
+            if (this._sortedPairs[i].key === key) {
+                idx = i;
+                break;
+            }
+        }
+        if (idx === -1) {
+            return;
+        }
+        console.log("removing ", key, idx, value);
         // assert key === this._sortedPairs[idx].key;
         this._sortedPairs.splice(idx, 1);
         this.emitRemove(idx, value);

src/placeholder-rooms.html

@@ -0,0 +1,118 @@
+<!DOCTYPE html>
+<html>
+	<head>
+		<meta charset="utf-8">
+        <link rel="stylesheet" type="text/css" href="platform/web/ui/css/main.css">
+        <link rel="stylesheet" type="text/css" href="platform/web/ui/css/themes/element/theme.css">
+        <style type="text/css">
+        .LeftPanel{
+            height: 100%;
+        }
+        </style>
+	</head>
+	<body class="not-ie11">
+        <input id="tokeninput" type="password" placeholder="access_token"/>
+        <input id="tokensubmit" type="button" value="Start" />
+        <div id="session-status" class="hydrogen" style="height: 500px;"></div>
+        <script id="main" type="module">
+            // core hydrogen imports
+            import {Platform} from "./platform/web/Platform";
+            import {createNavigation, createRouter} from "./domain/navigation/index.js";
+            import {StorageFactory} from "./matrix/storage/idb/StorageFactory";
+            import {Session} from "./matrix/Session.js";
+            import {ObservableMap} from "./observable/index.js";
+            import {MediaRepository} from "./matrix/net/MediaRepository.js";
+
+            // left panel specific
+            import {LeftPanelView} from "./platform/web/ui/session/leftpanel/LeftPanelView.js";
+            import {LeftPanelViewModel} from "./domain/session/leftpanel/LeftPanelViewModel";
+
+            // matrix specific bits
+            import {HomeServerApi} from "./matrix/net/HomeServerApi.js";
+            import {Sync3} from "./matrix/Sync3";
+
+            const sleep = (ms) => {
+                return new Promise((resolve) => setTimeout(resolve, ms));
+            };
+            
+
+            // dependency inject everything...
+            const platform = new Platform(document.body, {
+                // worker: "src/worker.js",
+                downloadSandbox: "assets/download-sandbox.html",
+                defaultHomeServer: "localhost",
+                // NOTE: uncomment this if you want the service worker for local development
+                // serviceWorker: "sw.js",
+                // NOTE: provide push config if you want push notifs for local development
+                //       see assets/config.json for what the config looks like
+                // push: {...},
+                olm: {
+                    wasm: "lib/olm/olm.wasm",
+                    legacyBundle: "lib/olm/olm_legacy.js",
+                    wasmBundle: "lib/olm/olm.js",
+                }
+            }, null, {development: true});
+            const navigation = createNavigation();
+            platform.setNavigation(navigation);
+            const urlRouter = createRouter({navigation, history: platform.history});
+            urlRouter.attach();
+            const hydrogenSessionID = "demo";
+            const factory = new StorageFactory(null); // TODO: this needs to be a fake idb
+            let storage;
+            const loadStorage = async () => {
+                await platform.logger.run("login", async log => {
+                    try {
+                        await factory.delete(hydrogenSessionID);
+                    } catch (err) {
+                        console.log("failed to delete old indexeddb:", err);
+                    }
+                    storage = await factory.create(hydrogenSessionID, log);
+                });
+            };
+            await loadStorage();
+
+            // kick off a sync v3 loop when an access token is provided
+            document.getElementById("tokensubmit").addEventListener("click", () => {
+                const accessToken = document.getElementById("tokeninput").value;
+                const sessionId = new Date().getTime() + "";
+                const hs = new HomeServerApi({
+                    homeserver: "http://localhost:8008",
+                    accessToken: accessToken,
+                    request: platform.request,
+                });
+                const session = new Session({
+                    storage: storage,
+                    hsApi: hs,
+                    mediaRepository: new MediaRepository({
+                        homeserver: "https://matrix.org",
+                        platform: platform,
+                    }),
+                    sessionInfo: {
+                        id: hydrogenSessionID,
+                        deviceId: null,
+                        userId: null,
+                        homeserver: "http://localhost:8008",
+                    },
+                });
+                const syncer = new Sync3(hs, session, storage, platform.logger);
+                syncer.start();
+
+                // make a left panel
+                const leftPanel = new LeftPanelViewModel({
+                    invites: new ObservableMap(),
+                    rooms: session.rooms,
+                    navigation: navigation,
+                    urlCreator: urlRouter,
+                    platform: platform,
+                    compareFn: syncer.compare.bind(syncer),
+                });
+                leftPanel.loadRoomRange = async (range) => {
+                    // pretend to load something
+                };
+                const view = new LeftPanelView(leftPanel);
+                document.getElementById("session-status").appendChild(view.mount());
+            });
+
+        </script>
+	</body>
+</html>

src/platform/web/ui/css/left-panel.css

@@ -45,6 +45,19 @@ limitations under the License.
     align-items: center;
 }
 
+.RoomList > .placeholder {
+    display: flex;
+    align-items: center;
+}
+
+.RoomList > .placeholder > .phdescription {
+    /* the avatar icon doesn't pad right because the placeholder lacks an <a> tag, so pad left to get the equiv layout */
+    margin-left: 8px;
+    /* make grey rectangles where the description should be */
+    background: rgb(236,237,238);
+    color: rgb(236,237,238);
+}
+
 .RoomList .description {
     margin: 0;
     flex: 1 1 0;

src/platform/web/ui/general/LazyListView.ts

@@ -14,15 +14,17 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import {tag} from "./html";
-import {removeChildren, mountView} from "./utils";
-import {ListRange, ResultType, AddRemoveResult} from "./ListRange";
-import {ListView, IOptions as IParentOptions} from "./ListView";
-import {IView} from "./types";
+import { tag } from "./html";
+import { removeChildren, mountView } from "./utils";
+import { ListRange, ResultType, AddRemoveResult } from "./ListRange";
+import { ListView, IOptions as IParentOptions } from "./ListView";
+import { IView } from "./types";
 
 export interface IOptions<T, V> extends IParentOptions<T, V> {
     itemHeight: number;
     overflowItems?: number;
+    shouldRecreateItem?: (value: any, oldValue: any) => boolean;
+    onRangeVisible?: (range: ListRange) => void;
 }
 
 export class LazyListView<T, V extends IView> extends ListView<T, V> {
@@ -31,14 +33,18 @@ export class LazyListView<T, V extends IView> extends ListView<T, V> {
     private itemHeight: number;
     private overflowItems: number;
     private scrollContainer?: HTMLElement;
+    private onRangeVisible?: (range: ListRange) => void;
+    private shouldRecreateItem?: (value: any, oldValue: any) => boolean;
 
     constructor(
-        {itemHeight, overflowItems = 20, ...options}: IOptions<T, V>, 
+        { itemHeight, onRangeVisible, shouldRecreateItem, overflowItems = 20, ...options }: IOptions<T, V>,
         childCreator: (value: T) => V
     ) {
         super(options, childCreator);
         this.itemHeight = itemHeight;
         this.overflowItems = overflowItems;
+        this.onRangeVisible = onRangeVisible;
+        this.shouldRecreateItem = shouldRecreateItem;
     }
 
     handleEvent(e: Event) {
@@ -82,7 +88,7 @@ export class LazyListView<T, V extends IView> extends ListView<T, V> {
     }
 
     private _getVisibleRange() {
-        const {clientHeight, scrollTop} = this.root()!;
+        const { clientHeight, scrollTop } = this.root()!;
         if (clientHeight === 0) {
             throw new Error("LazyListView height is 0");
         }
@@ -101,6 +107,9 @@ export class LazyListView<T, V extends IView> extends ListView<T, V> {
         });
         this._listElement!.appendChild(fragment);
         this.adjustPadding(range);
+        if (this.onRangeVisible) {
+            this.onRangeVisible(range);
+        }
     }
 
     private renderUpdate(prevRange: ListRange, newRange: ListRange) {
@@ -121,6 +130,9 @@ export class LazyListView<T, V extends IView> extends ListView<T, V> {
                 }
             });
             this.adjustPadding(newRange);
+            if (this.onRangeVisible) {
+                this.onRangeVisible(newRange);
+            }
         } else {
             this.reRenderFullRange(newRange);
         }
@@ -136,7 +148,7 @@ export class LazyListView<T, V extends IView> extends ListView<T, V> {
 
     mount() {
         const listElement = super.mount();
-        this.scrollContainer = tag.div({className: "LazyListParent"}, listElement) as HTMLElement;
+        this.scrollContainer = tag.div({ className: "LazyListParent" }, listElement) as HTMLElement;
         this.scrollContainer.addEventListener("scroll", this);
         return this.scrollContainer;
     }
@@ -181,9 +193,13 @@ export class LazyListView<T, V extends IView> extends ListView<T, V> {
 
     onUpdate(i: number, value: T, params: any) {
         if (this.renderRange!.containsIndex(i)) {
+            if (this.shouldRecreateItem && this.shouldRecreateItem(value, params?.oldValue)) {
+                super.recreateItem(this.renderRange!.toLocalIndex(i), value);
+            } else {
                 this.updateChild(this.renderRange!.toLocalIndex(i), value, params);
             }
         }
+    }
 
     private applyRemoveAddResult(result: AddRemoveResult<T>) {
         // order is important here, the new range can have a different start

src/platform/web/ui/session/leftpanel/LeftPanelView.js

@@ -14,10 +14,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import {ListView} from "../../general/ListView";
+import {LazyListView} from "../../general/LazyListView";
 import {TemplateView} from "../../general/TemplateView";
 import {RoomTileView} from "./RoomTileView.js";
 import {InviteTileView} from "./InviteTileView.js";
+import { PlaceholderRoomTileView } from "./PlaceholderRoomTileView";
+import { PlaceholderRoomTileViewModel } from "../../../../../domain/session/leftpanel/PlaceholderRoomTileViewModel";
+import { RoomTileViewModel } from "../../../../../domain/session/leftpanel/RoomTileViewModel";
 
 class FilterField extends TemplateView {
     render(t, options) {
@@ -58,14 +61,35 @@ export class LeftPanelView extends TemplateView {
                 vm.i18n`Show single room` :
                 vm.i18n`Enable grid layout`;
         };
-        const roomList = t.view(new ListView(
+        const roomList = t.view(new LazyListView(
             {
                 className: "RoomList",
+                itemHeight: 44,
                 list: vm.tileViewModels,
+                onRangeVisible: (range) => {
+                    vm.loadRoomRange(range);
+                },
+                shouldRecreateItem: (value, oldValue) => {
+                    const oldIsPlaceholder = oldValue instanceof PlaceholderRoomTileViewModel;
+                    const newIsPlaceholder = value instanceof PlaceholderRoomTileViewModel;
+                    if (oldIsPlaceholder != newIsPlaceholder) {
+                        return true;
+                    }
+                    // We used to just recreate the item if the underlying view model was swapped out e.g ph->room
+                    // but there is also a need to recreate items on room->room transitions (to re-make the
+                    // subviews)
+                    // views can be recycled so if the room ID is different then also recreate
+                    if (oldValue.id !== value.id) {
+                        return true;
+                    }
+                    return false;
+                }
             },
             tileVM => {
                 if (tileVM.kind === "invite") {
                     return new InviteTileView(tileVM);
+                } if (tileVM.kind === "placeholder") {
+                    return new PlaceholderRoomTileView(tileVM);
                 } else {
                     return new RoomTileView(tileVM);
                 }

src/platform/web/ui/session/leftpanel/PlaceholderRoomTileView.js

@@ -0,0 +1,34 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {TemplateView} from "../../general/TemplateView";
+import {renderStaticAvatar} from "../../avatar.js";
+
+export class PlaceholderRoomTileView extends TemplateView {
+    render(t, vm) {
+        const classes = {
+            "active": vm => vm.isOpen,
+            "hidden": vm => vm.hidden,
+            "placeholder": true,
+        };
+        return t.li({"className": classes}, [
+            renderStaticAvatar(vm, 32),
+            t.div({className: "phdescription"}, [
+                t.div({className: "name"}, vm.name),
+            ])
+        ]);
+    }
+}

yarn.lock

@@ -1164,6 +1164,11 @@
   resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82"
   integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==
 
+"@types/assert@^1.5.6":
+  version "1.5.6"
+  resolved "https://registry.yarnpkg.com/@types/assert/-/assert-1.5.6.tgz#a8b5a94ce5fb8f4ba65fdc37fc9507609114189e"
+  integrity sha512-Y7gDJiIqb9qKUHfBQYOWGngUpLORtirAVPuj/CWJrU2C6ZM4/y3XLwuwfGMF8s7QzW746LQZx23m0+1FSgjfug==
+
 "@types/cacheable-request@^6.0.1":
   version "6.0.2"
   resolved "https://registry.yarnpkg.com/@types/cacheable-request/-/cacheable-request-6.0.2.tgz#c324da0197de0a98a2312156536ae262429ff6b9"