diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index af1bf9a8..0c257728 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -26,7 +26,7 @@ import {DeviceMessageHandler} from "./DeviceMessageHandler.js";
 import {Account as E2EEAccount} from "./e2ee/Account.js";
 import {Decryption as OlmDecryption} from "./e2ee/olm/Decryption.js";
 import {Encryption as OlmEncryption} from "./e2ee/olm/Encryption.js";
-import {Decryption as MegOlmDecryption} from "./e2ee/megolm/Decryption.js";
+import {Decryption as MegOlmDecryption} from "./e2ee/megolm/Decryption";
 import {SessionBackup} from "./e2ee/megolm/SessionBackup.js";
 import {Encryption as MegOlmEncryption} from "./e2ee/megolm/Encryption.js";
 import {MEGOLM_ALGORITHM} from "./e2ee/common.js";
@@ -137,11 +137,8 @@ export class Session {
             now: this._platform.clock.now,
             ownDeviceId: this._sessionInfo.deviceId,
         });
-        this._megolmDecryption = new MegOlmDecryption({
-            pickleKey: PICKLE_KEY,
-            olm: this._olm,
-            olmWorker: this._olmWorker,
-        });
+        const keyLoader = new KeyLoader(this._olm, PICKLE_KEY, 20);
+        this._megolmDecryption = new MegOlmDecryption(keyLoader, this._olmWorker);
         this._deviceMessageHandler.enableEncryption({olmDecryption, megolmDecryption: this._megolmDecryption});
     }
 
@@ -319,6 +316,7 @@ export class Session {
     dispose() {
         this._olmWorker?.dispose();
         this._sessionBackup?.dispose();
+        this._megolmDecryption.dispose();
         for (const room of this._rooms.values()) {
             room.dispose();
         }
diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js
index cd2b5d59..66b3366f 100644
--- a/src/matrix/e2ee/RoomEncryption.js
+++ b/src/matrix/e2ee/RoomEncryption.js
@@ -36,8 +36,6 @@ export class RoomEncryption {
         this._megolmDecryption = megolmDecryption;
         // content of the m.room.encryption event
         this._encryptionParams = encryptionParams;
-        this._megolmBackfillCache = this._megolmDecryption.createSessionCache();
-        this._megolmSyncCache = this._megolmDecryption.createSessionCache(1);
         // caches devices to verify events
         this._senderDeviceCache = new Map();
         this._storage = storage;
@@ -76,9 +74,6 @@ export class RoomEncryption {
     }
 
     notifyTimelineClosed() {
-        // empty the backfill cache when closing the timeline
-        this._megolmBackfillCache.dispose();
-        this._megolmBackfillCache = this._megolmDecryption.createSessionCache();
         this._senderDeviceCache = new Map();    // purge the sender device cache
     }
 
@@ -112,27 +107,8 @@ export class RoomEncryption {
             }
             validEvents.push(event);
         }
-        let customCache;
-        let sessionCache;
-        // we have different caches so we can keep them small but still
-        // have backfill and sync not invalidate each other
-        if (source === DecryptionSource.Sync) {
-            sessionCache = this._megolmSyncCache;
-        } else if (source === DecryptionSource.Timeline) {
-            sessionCache = this._megolmBackfillCache;
-        } else if (source === DecryptionSource.Retry) {
-            // when retrying, we could have mixed events from at the bottom of the timeline (sync)
-            // and somewhere else, so create a custom cache we use just for this operation.
-            customCache = this._megolmDecryption.createSessionCache();
-            sessionCache = customCache;
-        } else {
-            throw new Error("Unknown source: " + source);
-        }
         const preparation = await this._megolmDecryption.prepareDecryptAll(
-            this._room.id, validEvents, newKeys, sessionCache, txn);
-        if (customCache) {
-            customCache.dispose();
-        }
+            this._room.id, validEvents, newKeys, txn);
         return new DecryptionPreparation(preparation, errors, source, this, events);
     }
 
@@ -204,37 +180,31 @@ export class RoomEncryption {
             return;
         }
         log.set("id", sessionId);
-        log.set("senderKey", senderKey);           
+        log.set("senderKey", senderKey);
         try {
             const session = await this._sessionBackup.getSession(this._room.id, sessionId, log);
             if (session?.algorithm === MEGOLM_ALGORITHM) {
-                if (session["sender_key"] !== senderKey) {
-                    log.set("wrong_sender_key", session["sender_key"]);
-                    log.logLevel = log.level.Warn;
-                    return;
-                }
                 let roomKey = this._megolmDecryption.roomKeyFromBackup(this._room.id, sessionId, session);
                 if (roomKey) {
+                    if (roomKey.senderKey !== senderKey) {
+                        log.set("wrong_sender_key", roomKey.senderKey);
+                        log.logLevel = log.level.Warn;
+                        return;
+                    }
                     let keyIsBestOne = false;
                     let retryEventIds;
+                    const txn = await this._storage.readWriteTxn([this._storage.storeNames.inboundGroupSessions]);
                     try {
-                        const txn = await this._storage.readWriteTxn([this._storage.storeNames.inboundGroupSessions]);
-                        try {
-                            keyIsBestOne = await this._megolmDecryption.writeRoomKey(roomKey, txn);
-                            log.set("isBetter", keyIsBestOne);
-                            if (keyIsBestOne) {
-                                retryEventIds = roomKey.eventIds;
-                            }
-                        } catch (err) {
-                            txn.abort();
-                            throw err;
+                        keyIsBestOne = await this._megolmDecryption.writeRoomKey(roomKey, txn);
+                        log.set("isBetter", keyIsBestOne);
+                        if (keyIsBestOne) {
+                            retryEventIds = roomKey.eventIds;
                         }
-                        await txn.complete();
-                    } finally {
-                        // can still access properties on it afterwards
-                        // this is just clearing the internal sessionInfo
-                        roomKey.dispose();
+                    } catch (err) {
+                        txn.abort();
+                        throw err;
                     }
+                    await txn.complete();
                     if (keyIsBestOne) {
                         await log.wrap("retryDecryption", log => this._room.notifyRoomKey(roomKey, retryEventIds || [], log));
                     }
@@ -466,8 +436,6 @@ export class RoomEncryption {
 
     dispose() {
         this._disposed = true;
-        this._megolmBackfillCache.dispose();
-        this._megolmSyncCache.dispose();
     }
 }