diff --git a/src/matrix/room/timeline/FragmentIdComparer.js b/src/matrix/room/timeline/FragmentIdComparer.js
index df5c3ac7..029c1333 100644
--- a/src/matrix/room/timeline/FragmentIdComparer.js
+++ b/src/matrix/room/timeline/FragmentIdComparer.js
@@ -108,6 +108,7 @@ index for fast lookup of how two fragments can be sorted
 */
 export default class FragmentIdComparer {
     constructor(fragments) {
+        this._fragmentsById = fragments.reduce((map, f) => {map.set(f.id, f); return map;}, new Map());
         this.rebuild(fragments);
     }
 
@@ -157,6 +158,10 @@ export default class FragmentIdComparer {
     // linkFragments(txn, firstFragmentId, secondFragmentId) {
 
     // }
+    add(fragment) {
+        this._fragmentsById[fragment.id] = fragment;
+        this.rebuild(this._fragmentsById.values());
+    }
 }
 
 //#ifdef TESTS
diff --git a/src/matrix/room/timeline/persistence/GapWriter.js b/src/matrix/room/timeline/persistence/GapWriter.js
index 06d450a6..415f2292 100644
--- a/src/matrix/room/timeline/persistence/GapWriter.js
+++ b/src/matrix/room/timeline/persistence/GapWriter.js
@@ -64,8 +64,9 @@ export default class GapWriter {
             txn.timelineFragments.set(neighbourFragmentEntry.fragment);
             directionalAppend(entries, neighbourFragmentEntry, direction);
 
-            // update fragmentIdComparer here after linking up fragments?
-            this._fragmentIdComparer.rebuild(await txn.timelineFragments.all());
+            // update fragmentIdComparer here after linking up fragments
+            this._fragmentIdComparer.add(fragmentEntry.fragment);
+            this._fragmentIdComparer.add(neighbourFragmentEntry.fragment);
         }
         fragmentEntry.token = end;
         txn.timelineFragments.set(fragmentEntry.fragment);
diff --git a/src/matrix/room/timeline/persistence/SyncWriter.js b/src/matrix/room/timeline/persistence/SyncWriter.js
index f27e8a2e..ebec1bae 100644
--- a/src/matrix/room/timeline/persistence/SyncWriter.js
+++ b/src/matrix/room/timeline/persistence/SyncWriter.js
@@ -40,6 +40,7 @@ export default class SyncWriter {
                 nextToken: null
             };
             txn.timelineFragments.add(fragment);
+            this._fragmentIdComparer.add(fragment);
             return fragment;
         } else {
             return liveFragment;
@@ -62,6 +63,7 @@ export default class SyncWriter {
             nextToken: null
         };
         txn.timelineFragments.add(newFragment);
+        this._fragmentIdComparer.add(newFragment);
         return {oldFragment, newFragment};
     }
 
@@ -117,11 +119,6 @@ export default class SyncWriter {
             }
         }
 
-        if (timeline.limited) {
-            const fragments = await txn.timelineFragments.all(this._roomId);
-            this._fragmentIdComparer.rebuild(fragments);
-        }
-
         return entries;
     }
 }
diff --git a/src/matrix/room/timeline/persistence/TimelineReader.js b/src/matrix/room/timeline/persistence/TimelineReader.js
index 75402aee..ee7c7865 100644
--- a/src/matrix/room/timeline/persistence/TimelineReader.js
+++ b/src/matrix/room/timeline/persistence/TimelineReader.js
@@ -16,7 +16,6 @@ export default class TimelineReader {
             this._storage.storeNames.timelineFragments,
         ]);
         let entries = [];
-        let loadedFragment = false;
 
         const timelineStore = txn.timelineEvents;
         const fragmentStore = txn.timelineFragments;
@@ -36,28 +35,21 @@ export default class TimelineReader {
                 const fragment = await fragmentStore.get(this._roomId, eventKey.fragmentId);
                 // this._fragmentIdComparer.addFragment(fragment);
                 let fragmentEntry = new FragmentBoundaryEntry(fragment, direction.isBackward, this._fragmentIdComparer);
-                // append or prepend fragmentEntry, reuse func from GapPersister?
+                // append or prepend fragmentEntry, reuse func from GapWriter?
                 directionalAppend(entries, fragmentEntry, direction);
                 // don't count it in amount perhaps? or do?
                 if (fragmentEntry.linkedFragmentId) {
                     const nextFragment = await fragmentStore.get(this._roomId, fragmentEntry.linkedFragmentId);
-                    // this._fragmentIdComparer.addFragment(nextFragment);
+                    this._fragmentIdComparer.add(nextFragment);
                     const nextFragmentEntry = new FragmentBoundaryEntry(nextFragment, direction.isForward, this._fragmentIdComparer);
                     directionalAppend(entries, nextFragmentEntry, direction);
                     eventKey = new EventKey(nextFragmentEntry.fragmentId, nextFragmentEntry.eventIndex);
-                    loadedFragment = true;
                 } else {
                     eventKey = null;
                 }
             }
         }
 
-        // reload fragments
-        if (loadedFragment) {
-            const fragments = await fragmentStore.all(this._roomId);
-            this._fragmentIdComparer.rebuild(fragments);
-        }
-
         return entries;
     }
 }