diff --git a/src/matrix/room/sending/SendQueue.js b/src/matrix/room/sending/SendQueue.js
index 4b1088c3..1ec789e7 100644
--- a/src/matrix/room/sending/SendQueue.js
+++ b/src/matrix/room/sending/SendQueue.js
@@ -63,7 +63,17 @@ export class SendQueue {
                                 pendingEvent.setWaiting();
                             } else {
                                 log.catch(err);
-                                pendingEvent.setError(err);
+                                const isPermanentError = err.name === "HomeServerError" && (
+                                    err.statusCode === 400 ||   // bad request, must be a bug on our end
+                                    err.statusCode === 403 ||   // forbidden
+                                    err.statusCode === 404      // not found
+                                );
+                                if (isPermanentError) {
+                                    log.set("remove", true);
+                                    await pendingEvent.abort();
+                                } else {
+                                    pendingEvent.setError(err);
+                                }
                             }
                         }
                     });