diff --git a/src/matrix/DeviceMessageHandler.js b/src/matrix/DeviceMessageHandler.js
index 537b948d..51a2378c 100644
--- a/src/matrix/DeviceMessageHandler.js
+++ b/src/matrix/DeviceMessageHandler.js
@@ -15,6 +15,7 @@ limitations under the License.
 */
 
 import {OLM_ALGORITHM, MEGOLM_ALGORITHM} from "./e2ee/common.js";
+import {groupBy} from "../utils/groupBy.js";
 
 // key to store in session store
 const PENDING_ENCRYPTED_EVENTS = "pendingEncryptedDeviceEvents";
@@ -44,21 +45,21 @@ export class DeviceMessageHandler {
         const megOlmRoomKeysPayloads = payloads.filter(p => {
             return p.event?.type === "m.room_key" && p.event.content?.algorithm === MEGOLM_ALGORITHM;
         });
-        let megolmChanges;
+        let roomKeys;
         if (megOlmRoomKeysPayloads.length) {
-            megolmChanges = await this._megolmDecryption.addRoomKeys(megOlmRoomKeysPayloads, txn);
+            roomKeys = await this._megolmDecryption.addRoomKeys(megOlmRoomKeysPayloads, txn);
         }
-        return {megolmChanges};
+        return {roomKeys};
     }
 
-    _applyDecryptChanges({megolmChanges}) {
-        if (megolmChanges) {
-            this._megolmDecryption.applyRoomKeyChanges(megolmChanges);
+    _applyDecryptChanges(rooms, {roomKeys}) {
+        const roomKeysByRoom = groupBy(roomKeys, s => s.roomId);
+        for (const [roomId, roomKeys] of roomKeysByRoom) {
         }
     }
 
     // not safe to call multiple times without awaiting first call
-    async decryptPending() {
+    async decryptPending(rooms) {
         if (!this._olmDecryption) {
             return;
         }
@@ -89,7 +90,7 @@ export class DeviceMessageHandler {
             throw err;
         }
         await txn.complete();
-        this._applyDecryptChanges(changes);
+        this._applyDecryptChanges(rooms, changes);
     }
 
     async _getPendingEvents(txn) {
diff --git a/src/matrix/Session.js b/src/matrix/Session.js
index 1d0ac73e..c3429075 100644
--- a/src/matrix/Session.js
+++ b/src/matrix/Session.js
@@ -150,7 +150,7 @@ export class Session {
             }
             await this._e2eeAccount.generateOTKsIfNeeded(this._storage);
             await this._e2eeAccount.uploadKeys(this._storage);
-            await this._deviceMessageHandler.decryptPending();
+            await this._deviceMessageHandler.decryptPending(this.rooms);
         }
     }
 
@@ -285,7 +285,7 @@ export class Session {
 
     async afterSyncCompleted() {
         const needsToUploadOTKs = await this._e2eeAccount.generateOTKsIfNeeded(this._storage);
-        const promises = [this._deviceMessageHandler.decryptPending()];
+        const promises = [this._deviceMessageHandler.decryptPending(this.rooms)];
         if (needsToUploadOTKs) {
             // TODO: we could do this in parallel with sync if it proves to be too slow
             // but I'm not sure how to not swallow errors in that case
diff --git a/src/matrix/e2ee/RoomEncryption.js b/src/matrix/e2ee/RoomEncryption.js
index 5f0c4cc1..f5e91913 100644
--- a/src/matrix/e2ee/RoomEncryption.js
+++ b/src/matrix/e2ee/RoomEncryption.js
@@ -27,12 +27,39 @@ export class RoomEncryption {
         this._megolmEncryption = megolmEncryption;
         // content of the m.room.encryption event
         this._encryptionParams = encryptionParams;
+
+        this._megolmBackfillCache = this._megolmDecryption.createSessionCache();
+        this._megolmSyncCache = this._megolmDecryption.createSessionCache();
+    }
+
+    notifyTimelineClosed() {
+        // empty the backfill cache when closing the timeline
+        this._megolmBackfillCache.dispose();
+        this._megolmBackfillCache = this._megolmDecryption.createSessionCache();
     }
 
     async writeMemberChanges(memberChanges, txn) {
         return await this._deviceTracker.writeMemberChanges(this._room, memberChanges, txn);
     }
 
+    async decryptNewSyncEvent(id, event, txn) {
+        const payload = await this._megolmDecryption.decryptNewEvent(
+            this._room.id, event, this._megolmSyncCache, txn);
+        return payload;
+    }
+
+    async decryptNewGapEvent(id, event, txn) {
+        const payload = await this._megolmDecryption.decryptNewEvent(
+            this._room.id, event, this._megolmBackfillCache, txn);
+        return payload;
+    }
+
+    async decryptStoredEvent(id, event, txn) {
+        const payload = await this._megolmDecryption.decryptStoredEvent(
+            this._room.id, event, this._megolmBackfillCache, txn);
+        return payload;
+    }
+
     async encrypt(type, content, hsApi) {
         const megolmResult = await this._megolmEncryption.encrypt(this._room.id, type, content, this._encryptionParams);
         // share the new megolm session if needed
diff --git a/src/matrix/room/Room.js b/src/matrix/room/Room.js
index d2272b3d..14579b73 100644
--- a/src/matrix/room/Room.js
+++ b/src/matrix/room/Room.js
@@ -45,6 +45,22 @@ export class Room extends EventEmitter {
         this._roomEncryption = null;
 	}
 
+    async _decryptSyncEntries(entries, txn) {
+        await Promise.all(entries.map(async e => {
+            if (e.eventType === "m.room.encrypted") {
+                try {
+                    const decryptedEvent = await this._roomEncryption.decryptNewSyncEvent(e.internalId, e.event, txn);
+                    if (decryptedEvent) {
+                        e.replaceWithDecrypted(decryptedEvent);
+                    }
+                } catch (err) {
+                    e.setDecryptionError(err);
+                }
+            }
+        }));
+        return entries;
+    }
+
     /** @package */
     async writeSync(roomResponse, membership, isInitialSync, txn) {
         const isTimelineOpen = !!this._timeline;
@@ -53,7 +69,13 @@ export class Room extends EventEmitter {
             membership,
             isInitialSync, isTimelineOpen,
             txn);
-		const {entries, newLiveKey, memberChanges} = await this._syncWriter.writeSync(roomResponse, txn);
+		const {entries: encryptedEntries, newLiveKey, memberChanges} =
+            await this._syncWriter.writeSync(roomResponse, txn);
+        // decrypt if applicable
+        let entries = encryptedEntries;
+        if (this._roomEncryption) {
+            entries = await this._decryptSyncEntries(encryptedEntries, txn);
+        }
         // fetch new members while we have txn open,
         // but don't make any in-memory changes yet
         let heroChanges;
@@ -341,6 +363,9 @@ export class Room extends EventEmitter {
             closeCallback: () => {
                 console.log(`closing the timeline for ${this._roomId}`);
                 this._timeline = null;
+                if (this._roomEncryption) {
+                    this._roomEncryption.notifyTimelineClosed();
+                }
             },
             user: this._user,
         });
diff --git a/src/matrix/room/timeline/entries/EventEntry.js b/src/matrix/room/timeline/entries/EventEntry.js
index 4dce9834..2bf5d941 100644
--- a/src/matrix/room/timeline/entries/EventEntry.js
+++ b/src/matrix/room/timeline/entries/EventEntry.js
@@ -21,6 +21,7 @@ export class EventEntry extends BaseEntry {
     constructor(eventEntry, fragmentIdComparer) {
         super(fragmentIdComparer);
         this._eventEntry = eventEntry;
+        this._decryptionError = null;
     }
 
     get fragmentId() {
@@ -31,6 +32,10 @@ export class EventEntry extends BaseEntry {
         return this._eventEntry.eventIndex;
     }
 
+    get internalId() {
+        return `${this.fragmentId}|${this.entryIndex}`;
+    }
+
     get content() {
         return this._eventEntry.event.content;
     }
@@ -66,4 +71,12 @@ export class EventEntry extends BaseEntry {
     get id() {
         return this._eventEntry.event.event_id;
     }
+
+    replaceWithDecrypted(event) {
+        this._eventEntry.event = event;
+    }
+
+    setDecryptionError(err) {
+        this._decryptionError = err;
+    }
 }