diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js
index 3659c447..95e6c579 100644
--- a/src/matrix/e2ee/RoomEncryption.js
+++ b/src/matrix/e2ee/RoomEncryption.js
@@ -210,11 +210,12 @@ export class RoomEncryption {
                 }
                 let roomKey = this._megolmDecryption.roomKeyFromBackup(this._room.id, sessionId, session);
                 if (roomKey) {
+                    let keyIsBestOne = false;
                     let retryEventIds;
                     try {
                         const txn = await this._storage.readWriteTxn([this._storage.storeNames.inboundGroupSessions]);
                         try {
-                            const keyIsBestOne = await this._megolmDecryption.writeRoomKey(roomKey, txn);
+                            keyIsBestOne = await this._megolmDecryption.writeRoomKey(roomKey, txn);
                             if (keyIsBestOne) {
                                 retryEventIds = roomKey.eventIds;
                             }
@@ -228,8 +229,8 @@ export class RoomEncryption {
                         // this is just clearing the internal sessionInfo
                         roomKey.dispose();
                     }
-                    if (retryEventIds?.length) {
-                        await this._room.retryDecryption(retryEventIds);
+                    if (keyIsBestOne) {
+                        await this._room.retryDecryption(roomKey, retryEventIds || []);
                     }
                 }
             } else if (session?.algorithm) {
diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index f5dd38de..605af83d 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -66,13 +66,22 @@ export class Room extends EventEmitter {
         return retryEntries;
     }
 
+    _getAdditionalTimelineRetryEntries(otherRetryEntries, roomKeys) {
+        let retryTimelineEntries = this._roomEncryption.filterUndecryptedEventEntriesForKeys(this._timeline.remoteEntries, roomKeys);
+        // filter out any entries already in retryEntries so we don't decrypt them twice
+        const existingIds = otherRetryEntries.reduce((ids, e) => {ids.add(e.id); return ids;}, new Set());
+        retryTimelineEntries = retryTimelineEntries.filter(e => !existingIds.has(e.id));
+        return retryTimelineEntries;
+    }
+
     /**
      * Used for retrying decryption from other sources than sync, like key backup.
      * @internal
-     * @param  {Array<string>} eventIds
+     * @param  {RoomKey} roomKey
+     * @param  {Array<string>} eventIds any event ids that should be retried. There might be more in the timeline though for this key.
      * @return {Promise}
      */
-    async retryDecryption(eventIds) {
+    async retryDecryption(roomKey, eventIds) {
         if (!this._roomEncryption) {
             return;
         }
@@ -80,7 +89,11 @@ export class Room extends EventEmitter {
             this._storage.storeNames.timelineEvents,
             this._storage.storeNames.inboundGroupSessions,
         ]);
-        const retryEntries = await this._eventIdsToEntries(eventIds, txn);
+        let retryEntries = await this._eventIdsToEntries(eventIds, txn);
+        if (this._timeline) {
+            const retryTimelineEntries = this._getAdditionalTimelineRetryEntries(retryEntries, [roomKey]);
+            retryEntries = retryEntries.concat(retryTimelineEntries);
+        }
         if (retryEntries.length) {
             const decryptRequest = this._decryptEntries(DecryptionSource.Retry, retryEntries, txn);
             // this will close txn while awaiting decryption
@@ -166,10 +179,7 @@ export class Room extends EventEmitter {
         // We want to decrypt all events we can though if the user is looking
         // at them when the timeline is open
         if (this._timeline) {
-            let retryTimelineEntries = this._roomEncryption.filterUndecryptedEventEntriesForKeys(this._timeline.remoteEntries, newKeys);
-            // filter out any entries already in retryEntries so we don't decrypt them twice
-            const existingIds = retryEntries.reduce((ids, e) => {ids.add(e.id); return ids;}, new Set());
-            retryTimelineEntries = retryTimelineEntries.filter(e => !existingIds.has(e.id));
+            const retryTimelineEntries = this._getAdditionalTimelineRetryEntries(retryEntries, newKeys);
             // make copies so we don't modify the original entry in writeSync, before the afterSync stage
             const retryTimelineEntriesCopies = retryTimelineEntries.map(e => e.clone());
             // add to other retry entries