feat: switch to matrix.test.mystiq.app

Fix develop server breaking due to import syntax

.gitignore

@@ -10,3 +10,4 @@ lib
 *.tar.gz
 .eslintcache
 .tmp
+tmp/

.woodpecker.yml

@@ -0,0 +1,18 @@
+pipeline:
+    buildfrontend:
+        image: node:16
+        commands:
+            - yarn install --prefer-offline --frozen-lockfile
+            - yarn test
+            - yarn run lint-ci
+            - yarn run tsc
+            - yarn build
+
+    deploy:
+        image: python
+        when:
+            event: push
+            branch: master
+        commands:
+            - make ci-deploy
+        secrets: [ GITEA_WRITE_DEPLOY_KEY, LIBREPAGES_DEPLOY_SECRET ]

Makefile

@@ -0,0 +1,14 @@
+ci-deploy: ## Deploy from CI/CD. Only call from within CI
+	@if [ "${CI}" != "woodpecker" ]; \
+	then echo "Only call from within CI. Will re-write your local Git configuration. To override, set export CI=woodpecker"; \
+		 exit 1; \
+	fi
+	git config --global user.email "${CI_COMMIT_AUTHOR_EMAIL}"
+	git config --global user.name "${CI_COMMIT_AUTHOR}"
+	./scripts/ci.sh --commit-files librepages target "${CI_COMMIT_AUTHOR} <${CI_COMMIT_AUTHOR_EMAIL}>"
+	./scripts/ci.sh --init "$$GITEA_WRITE_DEPLOY_KEY"
+	./scripts/ci.sh --deploy ${LIBREPAGES_DEPLOY_SECRET}  librepages
+	./scripts/ci.sh --clean
+
+help: ## Prints help for targets with comments
+	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'

README.md

@@ -1,3 +1,5 @@
+[![status-badge](https://ci.batsense.net/api/badges/mystiq/hydrogen-web/status.svg)](https://ci.batsense.net/mystiq/hydrogen-web)
+
 # Hydrogen
 
 A minimal [Matrix](https://matrix.org/) chat client, focused on performance, offline functionality, and broad browser support. This is work in progress and not yet ready for primetime. Bug reports are welcome, but please don't file any feature requests or other missing things to be on par with Element Web.

doc/IMPORT-ISSUES.md

@@ -0,0 +1,11 @@
+## How to import common-js dependency using ES6 syntax
+---
+Until [#6632](https://github.com/vitejs/vite/issues/6632) is fixed, such imports should be done as follows:
+
+```ts
+import * as pkg from "off-color";
+// @ts-ignore 
+const offColor = pkg.offColor ?? pkg.default.offColor;
+```
+
+This way build, dev server and unit tests should all work.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "hydrogen-web",
-  "version": "0.2.33",
+  "version": "0.3.1",
   "description": "A javascript matrix client prototype, trying to minize RAM usage by offloading as much as possible to IndexedDB",
   "directories": {
     "doc": "doc"
@@ -50,8 +50,9 @@
     "postcss-flexbugs-fixes": "^5.0.2",
     "postcss-value-parser": "^4.2.0",
     "regenerator-runtime": "^0.13.7",
+    "svgo": "^2.8.0",
     "text-encoding": "^0.7.0",
-    "typescript": "^4.3.5",
+    "typescript": "^4.7.0",
     "vite": "^2.9.8",
     "xxhashjs": "^0.2.2"
   },

scripts/build-plugins/rollup-plugin-build-themes.js

@@ -14,10 +14,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 const path = require('path').posix;
+const {optimize} = require('svgo');
 
 async function readCSSSource(location) {
     const fs = require("fs").promises;
-    const path = require("path");
     const resolvedLocation = path.resolve(__dirname, "../../", `${location}/theme.css`);
     const data = await fs.readFile(resolvedLocation);
     return data;
@@ -43,6 +43,45 @@ function addThemesToConfig(bundle, manifestLocations, defaultThemes) {
     }
 }
 
+/**
+ * Returns an object where keys are the svg file names and the values
+ * are the svg code (optimized)
+ * @param {*} icons Object where keys are css variable names and values are locations of the svg
+ * @param {*} manifestLocation Location of manifest used for resolving path 
+ */
+async function generateIconSourceMap(icons, manifestLocation) {
+    const sources = {};
+    const fileNames = [];
+    const promises = [];
+    const fs = require("fs").promises;
+    for (const icon of Object.values(icons)) {
+        const [location] = icon.split("?");
+        // resolve location against manifestLocation
+        const resolvedLocation = path.resolve(manifestLocation, location);
+        const iconData = fs.readFile(resolvedLocation);
+        promises.push(iconData);
+        const fileName = path.basename(resolvedLocation);
+        fileNames.push(fileName);
+    }
+    const results = await Promise.all(promises);
+    for (let i = 0; i < results.length; ++i)  {
+        const svgString = results[i].toString();
+        const result = optimize(svgString, {
+            plugins: [
+                {
+                    name: "preset-default",
+                    params: {
+                        overrides: { convertColors: false, },
+                    },
+                },
+            ],
+        });
+        const optimizedSvgString = result.data;
+        sources[fileNames[i]] = optimizedSvgString;
+    }
+    return sources;
+}
+
 /**
  * Returns a mapping from location (of manifest file) to an array containing all the chunks (of css files) generated from that location.
  * To understand what chunk means in this context, see https://rollupjs.org/guide/en/#generatebundle.
@@ -278,7 +317,7 @@ module.exports = function buildThemes(options) {
             ];
 },
 
-        generateBundle(_, bundle) {
+        async generateBundle(_, bundle) {
             const assetMap = getMappingFromFileNameToAssetInfo(bundle);
             const chunkMap = getMappingFromLocationToChunkArray(bundle);
             const runtimeThemeChunkMap = getMappingFromLocationToRuntimeChunk(bundle);
@@ -299,13 +338,29 @@ module.exports = function buildThemes(options) {
                     const locationRelativeToManifest = path.relative(manifestLocation, locationRelativeToBuildRoot);
                     builtAssets[`${name}-${variant}`] = locationRelativeToManifest;
                 }
+                // Emit the base svg icons as asset
+                const nameToAssetHashedLocation = [];
+                const nameToSource = await generateIconSourceMap(icon, location);
+                for (const [name, source] of Object.entries(nameToSource)) {
+                    const ref = this.emitFile({ type: "asset", name, source });
+                    const assetHashedName = this.getFileName(ref);
+                    nameToAssetHashedLocation[name] = assetHashedName;
+                }
+                // Update icon section in output manifest with paths to the icon in build output 
+                for (const [variable, location] of Object.entries(icon)) {
+                    const [locationWithoutQueryParameters, queryParameters] = location.split("?");
+                    const name = path.basename(locationWithoutQueryParameters);
+                    const locationRelativeToBuildRoot = nameToAssetHashedLocation[name];
+                    const locationRelativeToManifest = path.relative(manifestLocation, locationRelativeToBuildRoot);
+                    icon[variable] = `${locationRelativeToManifest}?${queryParameters}`;
+                }
                 const runtimeThemeChunk = runtimeThemeChunkMap.get(location);
                 const runtimeAssetLocation = path.relative(manifestLocation, assetMap.get(runtimeThemeChunk.fileName).fileName); 
                 manifest.source = {
                     "built-assets": builtAssets,
                     "runtime-asset": runtimeAssetLocation,
                     "derived-variables": derivedVariables,
-                    "icon": icon
+                    "icon": icon,
                 };
                 const name = `theme-${themeKey}.json`;
                 manifestLocations.push(`${manifestLocation}/${name}`);

scripts/ci.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+# ci.sh: Helper script to automate deployment operations on CI/CD
+# Copyright © 2022 Aravinth Manivannan <realaravinth@batsense.net>
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+# 
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+set -xEeuo  pipefail
+#source $(pwd)/scripts/lib.sh
+
+readonly SSH_ID_FILE=/tmp/ci-ssh-id
+readonly SSH_REMOTE_NAME=origin-ssh
+readonly PROJECT_ROOT=$(pwd)
+
+match_arg() {
+    if [ $1 == $2 ] || [ $1 == $3 ]
+    then
+        return 0
+    else
+        return 1
+    fi
+}
+
+help() {
+	cat << EOF
+USAGE: ci.sh [SUBCOMMAND]
+Helper script to automate deployment operations on CI/CD
+
+Subcommands
+
+  -c  --clean	cleanup secrets, SSH key and other runtime data
+  -i  --init <SSH_PRIVATE_KEY>	initialize environment, write SSH private to file
+  -d  --deploy <PAGES-SECRET> <TARGET BRANCH>	push branch to Gitea and call Pages server
+  -h  --help	print this help menu
+EOF
+}
+
+# $1: SSH private key
+write_ssh(){
+	truncate --size 0 $SSH_ID_FILE
+	echo "$1" > $SSH_ID_FILE
+	chmod 600 $SSH_ID_FILE
+}
+
+set_ssh_remote() {
+	http_remote_url=$(git remote get-url origin)
+	remote_hostname=$(echo $http_remote_url | cut -d '/' -f 3) 
+	repository_owner=$(echo $http_remote_url | cut -d '/' -f 4)
+	repository_name=$(echo $http_remote_url | cut -d '/' -f 5)
+	ssh_remote="git@$remote_hostname:$repository_owner/$repository_name"
+	ssh_remote="git@git.batsense.net:mystiq/hydrogen-web.git"
+	git remote add $SSH_REMOTE_NAME $ssh_remote
+}
+
+clean() {
+	if [ -f $SSH_ID_FILE ]
+	then
+		shred $SSH_ID_FILE
+		rm $SSH_ID_FILE
+	fi
+}
+
+# $1: branch name
+# $2: directory containing build assets
+# $3: Author in <author-name author@example.com> format
+commit_files() {
+	cd $PROJECT_ROOT
+	original_branch=$(git branch --show-current)
+	tmp_dir=$(mktemp -d)
+	cp -r $2/* $tmp_dir
+
+	if [[ -z $(git ls-remote --heads origin ${1}) ]]
+	then
+		echo "[*] Creating deployment branch $1"
+		git checkout --orphan $1
+	else
+		echo "[*] Deployment branch $1 exists, pulling changes from remote"
+		git fetch origin $1
+		git switch $1 
+	fi
+
+	git rm -rf .
+	/bin/rm -rf *
+	cp -r $tmp_dir/* .
+	git add --all
+	if [ $(git status --porcelain | xargs | sed '/^$/d' | wc -l) -gt 0 ];
+	then
+		echo "[*] Repository has changed, committing changes"
+		git commit \
+			--author="$3" \
+			--message="new deploy: $(date --iso-8601=seconds)"
+	fi
+	git checkout $original_branch
+}
+
+# $1: Pages API secret
+# $2: Deployment target branch
+deploy() {
+	if (( "$#" < 2 ))
+	then
+		help
+	else
+		git -c core.sshCommand="/usr/bin/ssh -oStrictHostKeyChecking=no -i $SSH_ID_FILE"\
+			push --force  $SSH_REMOTE_NAME $2
+		curl -vv --location --request \
+				POST "https://deploy.batsense.net/api/v1/update"\
+				--header 'Content-Type: application/json' \
+				--data-raw "{ \"secret\": \"$1\", \"branch\": \"$2\" }"
+	fi
+}
+
+if (( "$#" < 1 ))
+then
+	help
+	exit -1
+fi
+
+
+if match_arg $1 '-i' '--init'
+then
+	if (( "$#" < 2 ))
+	then
+		help
+		exit -1
+	fi
+	set_ssh_remote
+	write_ssh "$2"
+elif match_arg $1 '-c' '--clean'
+then
+	clean
+elif match_arg $1 '-cf' '--commit-files'
+then
+	if (( "$#" < 4 ))
+	then
+		help
+		exit -1
+	fi
+    commit_files $2 $3 $4
+elif match_arg $1 '-d' '--deploy'
+then
+	if (( "$#" < 3 ))
+	then
+		help
+		exit -1
+	fi
+	deploy $2 $3
+elif match_arg $1 '-h' '--help'
+then
+	help
+else
+	help
+fi
+
+
+

scripts/postcss/css-url-to-variables.js

@@ -81,7 +81,8 @@ module.exports = (opts = {}) => {
             const urlVariables = new Map();
             const counter = createCounter();
             root.walkDecls(decl => findAndReplaceUrl(decl, urlVariables, counter));
-            if (urlVariables.size) {
+            const cssFileLocation = root.source.input.from;
+            if (urlVariables.size && !cssFileLocation.includes("type=runtime")) {
                 addResolvedVariablesToRootSelector(root, { Rule, Declaration }, urlVariables);
             }
             if (opts.compiledVariables){

scripts/postcss/svg-builder.mjs

@@ -14,12 +14,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-const fs = require("fs");
+import {readFileSync, mkdirSync, writeFileSync} from "fs";
-const path = require("path");
+import {resolve} from "path";
-const xxhash = require('xxhashjs');
+import {h32} from "xxhashjs";
+import {getColoredSvgString} from "../../src/platform/web/theming/shared/svg-colorizer.mjs";
 
 function createHash(content) {
-    const hasher = new xxhash.h32(0);
+    const hasher = new h32(0);
     hasher.update(content);
     return hasher.digest();
 }
@@ -30,18 +31,14 @@ function createHash(content) {
  * @param {string} primaryColor Primary color for the new svg
  * @param {string} secondaryColor Secondary color for the new svg
  */
-module.exports.buildColorizedSVG = function (svgLocation, primaryColor, secondaryColor) {
+export function buildColorizedSVG(svgLocation, primaryColor, secondaryColor) {
-    const svgCode = fs.readFileSync(svgLocation, { encoding: "utf8"});
+    const svgCode = readFileSync(svgLocation, { encoding: "utf8"});
-    let coloredSVGCode = svgCode.replaceAll("#ff00ff", primaryColor);
+    const coloredSVGCode = getColoredSvgString(svgCode, primaryColor, secondaryColor);
-    coloredSVGCode = coloredSVGCode.replaceAll("#00ffff", secondaryColor);
-    if (svgCode === coloredSVGCode) {
-        throw new Error("svg-colorizer made no color replacements! The input svg should only contain colors #ff00ff (primary, case-sensitive) and #00ffff (secondary, case-sensitive).");
-    }
     const fileName = svgLocation.match(/.+[/\\](.+\.svg)/)[1];
     const outputName = `${fileName.substring(0, fileName.length - 4)}-${createHash(coloredSVGCode)}.svg`;
-    const outputPath = path.resolve(__dirname, "../../.tmp");
+    const outputPath = resolve(__dirname, "./.tmp");
     try {
-       fs.mkdirSync(outputPath);
+       mkdirSync(outputPath);
     }
     catch (e) {
         if (e.code !== "EEXIST") {
@@ -49,6 +46,6 @@ module.exports.buildColorizedSVG = function (svgLocation, primaryColor, secondar
         }
     }
     const outputFile = `${outputPath}/${outputName}`;
-    fs.writeFileSync(outputFile, coloredSVGCode);
+    writeFileSync(outputFile, coloredSVGCode);
     return outputFile;
 }

scripts/sdk/base-manifest.json

@@ -1,7 +1,7 @@
 {
     "name": "hydrogen-view-sdk",
     "description": "Embeddable matrix client library, including view components",
-    "version": "0.0.15",
+    "version": "0.1.0",
     "main": "./lib-build/hydrogen.cjs.js",
     "exports": {
         ".": {

scripts/test-derived-theme/test-theme.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+cp scripts/test-derived-theme/theme.json target/assets/theme-customer.json
+cat target/config.json | jq '.themeManifests += ["assets/theme-customer.json"]' | cat > target/config.temp.json
+rm target/config.json
+mv target/config.temp.json target/config.json

scripts/test-derived-theme/theme.json

@@ -0,0 +1,51 @@
+{
+	"name": "Customer",
+    "extends": "element",
+    "id": "customer",
+	"values": {
+        "variants": {
+            "dark": {
+                "dark": true,
+                "default": true,
+                "name": "Dark",
+                "variables": {
+                    "background-color-primary": "#21262b",
+                    "background-color-secondary": "#2D3239",
+                    "text-color": "#fff",
+                    "accent-color": "#F03F5B",
+                    "error-color": "#FF4B55",
+                    "fixed-white": "#fff",
+                    "room-badge": "#61708b",
+                    "link-color": "#238cf5"
+                }
+            },
+            "light": {
+                "default": true,
+                "name": "Dark",
+                "variables": {
+                    "background-color-primary": "#21262b",
+                    "background-color-secondary": "#2D3239",
+                    "text-color": "#fff",
+                    "accent-color": "#F03F5B",
+                    "error-color": "#FF4B55",
+                    "fixed-white": "#fff",
+                    "room-badge": "#61708b",
+                    "link-color": "#238cf5"
+                }
+            },
+            "red": {
+                "name": "Gruvbox",
+                "variables": {
+                    "background-color-primary": "#282828",
+                    "background-color-secondary": "#3c3836",
+                    "text-color": "#fbf1c7",
+                    "accent-color": "#8ec07c",
+                    "error-color": "#fb4934",
+                    "fixed-white": "#fff",
+                    "room-badge": "#cc241d",
+                    "link-color": "#fe8019"
+                }
+            }
+        }
+    }
+}

src/matrix/e2ee/DeviceTracker.js

@@ -15,11 +15,13 @@ limitations under the License.
 */
 
 import {verifyEd25519Signature, SIGNATURE_ALGORITHM} from "./common.js";
+import {HistoryVisibility, shouldShareKey} from "./common.js";
+import {RoomMember} from "../room/members/RoomMember.js";
 
 const TRACKING_STATUS_OUTDATED = 0;
 const TRACKING_STATUS_UPTODATE = 1;
 
-export function addRoomToIdentity(identity, userId, roomId) {
+function addRoomToIdentity(identity, userId, roomId) {
     if (!identity) {
         identity = {
             userId: userId,
@@ -79,28 +81,57 @@ export class DeviceTracker {
         }));
     }
 
-    writeMemberChanges(room, memberChanges, txn) {
+    /** @return Promise<{added: string[], removed: string[]}> the user ids for who the room was added or removed to the userIdentity,
-        return Promise.all(Array.from(memberChanges.values()).map(async memberChange => {
+     *                                                        and with who a key should be now be shared
-            return this._applyMemberChange(memberChange, txn);
+     **/
+    async writeMemberChanges(room, memberChanges, historyVisibility, txn) {
+        const added = [];
+        const removed = [];
+        await Promise.all(Array.from(memberChanges.values()).map(async memberChange => {
+            // keys should now be shared with this member?
+            // add the room to the userIdentity if so
+            if (shouldShareKey(memberChange.membership, historyVisibility)) {
+                if (await this._addRoomToUserIdentity(memberChange.roomId, memberChange.userId, txn)) {
+                    added.push(memberChange.userId);
+                }
+            } else if (shouldShareKey(memberChange.previousMembership, historyVisibility)) {
+                // try to remove room we were previously sharing the key with the member but not anymore
+                const {roomId} = memberChange;
+                // if we left the room, remove room from all user identities in the room
+                if (memberChange.userId === this._ownUserId) {
+                    const userIds = await txn.roomMembers.getAllUserIds(roomId);
+                    await Promise.all(userIds.map(userId => {
+                        return this._removeRoomFromUserIdentity(roomId, userId, txn);
+                    }));
+                } else {
+                    await this._removeRoomFromUserIdentity(roomId, memberChange.userId, txn);
+                }
+                removed.push(memberChange.userId);
+            }
         }));
+        return {added, removed};
     }
 
-    async trackRoom(room, log) {
+    async trackRoom(room, historyVisibility, log) {
         if (room.isTrackingMembers || !room.isEncrypted) {
             return;
         }
-        const memberList = await room.loadMemberList(log);
+        const memberList = await room.loadMemberList(undefined, log);
+        const txn = await this._storage.readWriteTxn([
+            this._storage.storeNames.roomSummary,
+            this._storage.storeNames.userIdentities,
+        ]);
         try {
-            const txn = await this._storage.readWriteTxn([
-                this._storage.storeNames.roomSummary,
-                this._storage.storeNames.userIdentities,
-            ]);
             let isTrackingChanges;
             try {
                 isTrackingChanges = room.writeIsTrackingMembers(true, txn);
                 const members = Array.from(memberList.members.values());
                 log.set("members", members.length);
-                await this._writeJoinedMembers(members, txn);
+                await Promise.all(members.map(async member => {
+                    if (shouldShareKey(member.membership, historyVisibility)) {
+                        await this._addRoomToUserIdentity(member.roomId, member.userId, txn);
+                    }
+                }));
             } catch (err) {
                 txn.abort();
                 throw err;
@@ -112,21 +143,43 @@ export class DeviceTracker {
         }
     }
 
-    async _writeJoinedMembers(members, txn) {
+    async writeHistoryVisibility(room, historyVisibility, syncTxn, log) {
-        await Promise.all(members.map(async member => {
+        const added = [];
-            if (member.membership === "join") {
+        const removed = [];
-                await this._writeMember(member, txn);
+        if (room.isTrackingMembers && room.isEncrypted) {
-            }
+            await log.wrap("rewriting userIdentities", async log => {
-        }));
+                const memberList = await room.loadMemberList(syncTxn, log);
+                try {
+                    const members = Array.from(memberList.members.values());
+                    log.set("members", members.length);
+                    await Promise.all(members.map(async member => {
+                        if (shouldShareKey(member.membership, historyVisibility)) {
+                            if (await this._addRoomToUserIdentity(member.roomId, member.userId, syncTxn)) {
+                                added.push(member.userId);
+                            }
+                        } else {
+                            if (await this._removeRoomFromUserIdentity(member.roomId, member.userId, syncTxn)) {
+                                removed.push(member.userId);
+                            }
+                        }
+                    }));
+                } finally {
+                    memberList.release();
+                }
+            });
+        }
+        return {added, removed};
     }
 
-    async _writeMember(member, txn) {
+    async _addRoomToUserIdentity(roomId, userId, txn) {
         const {userIdentities} = txn;
-        const identity = await userIdentities.get(member.userId);
+        const identity = await userIdentities.get(userId);
-        const updatedIdentity = addRoomToIdentity(identity, member.userId, member.roomId);
+        const updatedIdentity = addRoomToIdentity(identity, userId, roomId);
         if (updatedIdentity) {
             userIdentities.set(updatedIdentity);
+            return true;
         }
+        return false;
     }
 
     async _removeRoomFromUserIdentity(roomId, userId, txn) {
@@ -141,28 +194,9 @@ export class DeviceTracker {
             } else {
                 userIdentities.set(identity);
             }
+            return true;
         }
-    }
+        return false;
-
-    async _applyMemberChange(memberChange, txn) {
-        // TODO: depends whether we encrypt for invited users??
-        // add room
-        if (memberChange.hasJoined) {
-            await this._writeMember(memberChange.member, txn);
-        }
-        // remove room
-        else if (memberChange.hasLeft) {
-            const {roomId} = memberChange;
-            // if we left the room, remove room from all user identities in the room
-            if (memberChange.userId === this._ownUserId) {
-                const userIds = await txn.roomMembers.getAllUserIds(roomId);
-                await Promise.all(userIds.map(userId => {
-                    return this._removeRoomFromUserIdentity(roomId, userId, txn);
-                }));
-            } else {
-                await this._removeRoomFromUserIdentity(roomId, memberChange.userId, txn);
-            }
-        }
     }
 
     async _queryKeys(userIds, hsApi, log) {
@@ -367,16 +401,18 @@ export class DeviceTracker {
 
 import {createMockStorage} from "../../mocks/Storage";
 import {Instance as NullLoggerInstance} from "../../logging/NullLogger";
+import {MemberChange} from "../room/members/RoomMember";
 
 export function tests() {
 
     function createUntrackedRoomMock(roomId, joinedUserIds, invitedUserIds = []) {
         return {
+            id: roomId,
             isTrackingMembers: false,
             isEncrypted: true,
             loadMemberList: () => {
-                const joinedMembers = joinedUserIds.map(userId => {return {membership: "join", roomId, userId};});
+                const joinedMembers = joinedUserIds.map(userId => {return RoomMember.fromUserId(roomId, userId, "join");});
-                const invitedMembers = invitedUserIds.map(userId => {return {membership: "invite", roomId, userId};});
+                const invitedMembers = invitedUserIds.map(userId => {return RoomMember.fromUserId(roomId, userId, "invite");});
                 const members = joinedMembers.concat(invitedMembers);
                 const memberMap = members.reduce((map, member) => {
                     map.set(member.userId, member);
@@ -440,10 +476,29 @@ export function tests() {
             }
         };
     }
+
+    async function writeMemberListToStorage(room, storage) {
+        const txn = await storage.readWriteTxn([
+            storage.storeNames.roomMembers,
+        ]);
+        const memberList = await room.loadMemberList(txn);
+        try {
+            for (const member of memberList.members.values()) {
+                txn.roomMembers.set(member.serialize());
+            }
+        } catch (err) {
+            txn.abort();
+            throw err;
+        } finally {
+            memberList.release();
+        }
+        await txn.complete();
+    }
+
     const roomId = "!abc:hs.tld";
 
     return {
-        "trackRoom only writes joined members": async assert => {
+        "trackRoom only writes joined members with history visibility of joined": async assert => {
             const storage = await createMockStorage();
             const tracker = new DeviceTracker({
                 storage,
@@ -453,7 +508,7 @@ export function tests() {
                 ownDeviceId: "ABCD",
             });
             const room = createUntrackedRoomMock(roomId, ["@alice:hs.tld", "@bob:hs.tld"], ["@charly:hs.tld"]);
-            await tracker.trackRoom(room, NullLoggerInstance.item);
+            await tracker.trackRoom(room, HistoryVisibility.Joined, NullLoggerInstance.item);
             const txn = await storage.readTxn([storage.storeNames.userIdentities]);
             assert.deepEqual(await txn.userIdentities.get("@alice:hs.tld"), {
                 userId: "@alice:hs.tld",
@@ -477,7 +532,7 @@ export function tests() {
                 ownDeviceId: "ABCD",
             });
             const room = createUntrackedRoomMock(roomId, ["@alice:hs.tld", "@bob:hs.tld"]);
-            await tracker.trackRoom(room, NullLoggerInstance.item);
+            await tracker.trackRoom(room, HistoryVisibility.Joined, NullLoggerInstance.item);
             const hsApi = createQueryKeysHSApiMock();
             const devices = await tracker.devicesForRoomMembers(roomId, ["@alice:hs.tld", "@bob:hs.tld"], hsApi, NullLoggerInstance.item);
             assert.equal(devices.length, 2);
@@ -494,7 +549,7 @@ export function tests() {
                 ownDeviceId: "ABCD",
             });
             const room = createUntrackedRoomMock(roomId, ["@alice:hs.tld", "@bob:hs.tld"]);
-            await tracker.trackRoom(room, NullLoggerInstance.item);
+            await tracker.trackRoom(room, HistoryVisibility.Joined, NullLoggerInstance.item);
             const hsApi = createQueryKeysHSApiMock();
             // query devices first time
             await tracker.devicesForRoomMembers(roomId, ["@alice:hs.tld", "@bob:hs.tld"], hsApi, NullLoggerInstance.item);
@@ -512,6 +567,169 @@ export function tests() {
             const txn2 = await storage.readTxn([storage.storeNames.deviceIdentities]);
             // also check the modified key was not stored
             assert.equal((await txn2.deviceIdentities.get("@alice:hs.tld", "device1")).ed25519Key, "ed25519:@alice:hs.tld:device1:key");
-        }
+        },
+        "change history visibility from joined to invited adds invitees": async assert => {
+            const storage = await createMockStorage();
+            const tracker = new DeviceTracker({
+                storage,
+                getSyncToken: () => "token",
+                olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw
+                ownUserId: "@alice:hs.tld",
+                ownDeviceId: "ABCD",
+            });
+            // alice is joined, bob is invited
+            const room = await createUntrackedRoomMock(roomId, 
+                ["@alice:hs.tld"], ["@bob:hs.tld"]);
+            await tracker.trackRoom(room, HistoryVisibility.Joined, NullLoggerInstance.item);
+            const txn = await storage.readWriteTxn([storage.storeNames.userIdentities, storage.storeNames.deviceIdentities]);
+            assert.equal(await txn.userIdentities.get("@bob:hs.tld"), undefined);
+            const {added, removed} = await tracker.writeHistoryVisibility(room, HistoryVisibility.Invited, txn, NullLoggerInstance.item);
+            assert.equal((await txn.userIdentities.get("@bob:hs.tld")).userId, "@bob:hs.tld");
+            assert.deepEqual(added, ["@bob:hs.tld"]);
+            assert.deepEqual(removed, []);
+        },
+        "change history visibility from invited to joined removes invitees": async assert => {
+            const storage = await createMockStorage();
+            const tracker = new DeviceTracker({
+                storage,
+                getSyncToken: () => "token",
+                olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw
+                ownUserId: "@alice:hs.tld",
+                ownDeviceId: "ABCD",
+            });
+            // alice is joined, bob is invited
+            const room = await createUntrackedRoomMock(roomId, 
+                ["@alice:hs.tld"], ["@bob:hs.tld"]);
+            await tracker.trackRoom(room, HistoryVisibility.Invited, NullLoggerInstance.item);
+            const txn = await storage.readWriteTxn([storage.storeNames.userIdentities, storage.storeNames.deviceIdentities]);
+            assert.equal((await txn.userIdentities.get("@bob:hs.tld")).userId, "@bob:hs.tld");
+            const {added, removed} = await tracker.writeHistoryVisibility(room, HistoryVisibility.Joined, txn, NullLoggerInstance.item);
+            assert.equal(await txn.userIdentities.get("@bob:hs.tld"), undefined);
+            assert.deepEqual(added, []);
+            assert.deepEqual(removed, ["@bob:hs.tld"]);
+        },
+        "adding invitee with history visibility of invited adds room to userIdentities": async assert => {
+            const storage = await createMockStorage();
+            const tracker = new DeviceTracker({
+                storage,
+                getSyncToken: () => "token",
+                olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw
+                ownUserId: "@alice:hs.tld",
+                ownDeviceId: "ABCD",
+            });
+            const room = await createUntrackedRoomMock(roomId, ["@alice:hs.tld"]);
+            await tracker.trackRoom(room, HistoryVisibility.Invited, NullLoggerInstance.item);
+            const txn = await storage.readWriteTxn([storage.storeNames.userIdentities, storage.storeNames.deviceIdentities]);
+            // inviting a new member
+            const inviteChange = new MemberChange(RoomMember.fromUserId(roomId, "@bob:hs.tld", "invite"));
+            const {added, removed} = await tracker.writeMemberChanges(room, [inviteChange], HistoryVisibility.Invited, txn);
+            assert.deepEqual(added, ["@bob:hs.tld"]);
+            assert.deepEqual(removed, []);
+            assert.equal((await txn.userIdentities.get("@bob:hs.tld")).userId, "@bob:hs.tld");
+        },
+        "adding invitee with history visibility of joined doesn't add room": async assert => {
+            const storage = await createMockStorage();
+            const tracker = new DeviceTracker({
+                storage,
+                getSyncToken: () => "token",
+                olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw
+                ownUserId: "@alice:hs.tld",
+                ownDeviceId: "ABCD",
+            });
+            const room = await createUntrackedRoomMock(roomId, ["@alice:hs.tld"]);
+            await tracker.trackRoom(room, HistoryVisibility.Joined, NullLoggerInstance.item);
+            const txn = await storage.readWriteTxn([storage.storeNames.userIdentities, storage.storeNames.deviceIdentities]);
+            // inviting a new member
+            const inviteChange = new MemberChange(RoomMember.fromUserId(roomId, "@bob:hs.tld", "invite"));
+            const memberChanges = new Map([[inviteChange.userId, inviteChange]]);
+            const {added, removed} = await tracker.writeMemberChanges(room, memberChanges, HistoryVisibility.Joined, txn);
+            assert.deepEqual(added, []);
+            assert.deepEqual(removed, []);
+            assert.equal(await txn.userIdentities.get("@bob:hs.tld"), undefined);
+        },
+        "getting all devices after changing history visibility now includes invitees": async assert => {
+            const storage = await createMockStorage();
+            const tracker = new DeviceTracker({
+                storage,
+                getSyncToken: () => "token",
+                olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw
+                ownUserId: "@alice:hs.tld",
+                ownDeviceId: "ABCD",
+            });
+            const room = createUntrackedRoomMock(roomId, ["@alice:hs.tld"], ["@bob:hs.tld"]);
+            await tracker.trackRoom(room, HistoryVisibility.Invited, NullLoggerInstance.item);
+            const hsApi = createQueryKeysHSApiMock();
+            // write memberlist from room mock to mock storage,
+            // as devicesForTrackedRoom reads directly from roomMembers store.
+            await writeMemberListToStorage(room, storage);
+            const devices = await tracker.devicesForTrackedRoom(roomId, hsApi, NullLoggerInstance.item);
+            assert.equal(devices.length, 2);
+            assert.equal(devices.find(d => d.userId === "@alice:hs.tld").ed25519Key, "ed25519:@alice:hs.tld:device1:key");
+            assert.equal(devices.find(d => d.userId === "@bob:hs.tld").ed25519Key, "ed25519:@bob:hs.tld:device1:key");
+        },
+        "rejecting invite with history visibility of invited removes room from user identity": async assert => {
+            const storage = await createMockStorage();
+            const tracker = new DeviceTracker({
+                storage,
+                getSyncToken: () => "token",
+                olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw
+                ownUserId: "@alice:hs.tld",
+                ownDeviceId: "ABCD",
+            });
+            // alice is joined, bob is invited
+            const room = await createUntrackedRoomMock(roomId, ["@alice:hs.tld"], ["@bob:hs.tld"]);
+            await tracker.trackRoom(room, HistoryVisibility.Invited, NullLoggerInstance.item);
+            const txn = await storage.readWriteTxn([storage.storeNames.userIdentities, storage.storeNames.deviceIdentities]);
+            // reject invite
+            const inviteChange = new MemberChange(RoomMember.fromUserId(roomId, "@bob:hs.tld", "leave"), "invite");
+            const memberChanges = new Map([[inviteChange.userId, inviteChange]]);
+            const {added, removed} = await tracker.writeMemberChanges(room, memberChanges, HistoryVisibility.Invited, txn);
+            assert.deepEqual(added, []);
+            assert.deepEqual(removed, ["@bob:hs.tld"]);
+            assert.equal(await txn.userIdentities.get("@bob:hs.tld"), undefined);
+        },
+        "remove room from user identity sharing multiple rooms with us preserves other room": async assert => {
+            const storage = await createMockStorage();
+            const tracker = new DeviceTracker({
+                storage,
+                getSyncToken: () => "token",
+                olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw
+                ownUserId: "@alice:hs.tld",
+                ownDeviceId: "ABCD",
+            });
+            // alice is joined, bob is invited
+            const room1 = await createUntrackedRoomMock("!abc:hs.tld", ["@alice:hs.tld", "@bob:hs.tld"]);
+            const room2 = await createUntrackedRoomMock("!def:hs.tld", ["@alice:hs.tld", "@bob:hs.tld"]);
+            await tracker.trackRoom(room1, HistoryVisibility.Joined, NullLoggerInstance.item);
+            await tracker.trackRoom(room2, HistoryVisibility.Joined, NullLoggerInstance.item);
+            const txn1 = await storage.readTxn([storage.storeNames.userIdentities]);
+            assert.deepEqual((await txn1.userIdentities.get("@bob:hs.tld")).roomIds, ["!abc:hs.tld", "!def:hs.tld"]);
+            const leaveChange = new MemberChange(RoomMember.fromUserId(room2.id, "@bob:hs.tld", "leave"), "join");
+            const memberChanges = new Map([[leaveChange.userId, leaveChange]]);
+            const txn2 = await storage.readWriteTxn([storage.storeNames.userIdentities, storage.storeNames.deviceIdentities]);
+            await tracker.writeMemberChanges(room2, memberChanges, HistoryVisibility.Joined, txn2);
+            await txn2.complete();
+            const txn3 = await storage.readTxn([storage.storeNames.userIdentities]);
+            assert.deepEqual((await txn3.userIdentities.get("@bob:hs.tld")).roomIds, ["!abc:hs.tld"]);
+        },
+        "add room to user identity sharing multiple rooms with us preserves other room": async assert => {
+            const storage = await createMockStorage();
+            const tracker = new DeviceTracker({
+                storage,
+                getSyncToken: () => "token",
+                olmUtil: {ed25519_verify: () => {}}, // valid if it does not throw
+                ownUserId: "@alice:hs.tld",
+                ownDeviceId: "ABCD",
+            });
+            // alice is joined, bob is invited
+            const room1 = await createUntrackedRoomMock("!abc:hs.tld", ["@alice:hs.tld", "@bob:hs.tld"]);
+            const room2 = await createUntrackedRoomMock("!def:hs.tld", ["@alice:hs.tld", "@bob:hs.tld"]);
+            await tracker.trackRoom(room1, HistoryVisibility.Joined, NullLoggerInstance.item);
+            const txn1 = await storage.readTxn([storage.storeNames.userIdentities]);
+            assert.deepEqual((await txn1.userIdentities.get("@bob:hs.tld")).roomIds, ["!abc:hs.tld"]);
+            await tracker.trackRoom(room2, HistoryVisibility.Joined, NullLoggerInstance.item);
+            const txn2 = await storage.readTxn([storage.storeNames.userIdentities]);
+            assert.deepEqual((await txn2.userIdentities.get("@bob:hs.tld")).roomIds, ["!abc:hs.tld", "!def:hs.tld"]);
+        },
     }
 }

src/matrix/e2ee/RoomEncryption.js

@@ -19,8 +19,10 @@ import {groupEventsBySession} from "./megolm/decryption/utils";
 import {mergeMap} from "../../utils/mergeMap";
 import {groupBy} from "../../utils/groupBy";
 import {makeTxnId} from "../common.js";
+import {iterateResponseStateEvents} from "../room/common";
 
 const ENCRYPTED_TYPE = "m.room.encrypted";
+const ROOM_HISTORY_VISIBILITY_TYPE = "m.room.history_visibility";
 // how often ensureMessageKeyIsShared can check if it needs to
 // create a new outbound session
 // note that encrypt could still create a new session
@@ -45,6 +47,7 @@ export class RoomEncryption {
         this._isFlushingRoomKeyShares = false;
         this._lastKeyPreShareTime = null;
         this._keySharePromise = null;
+        this._historyVisibility = undefined;
         this._disposed = false;
     }
 
@@ -77,22 +80,68 @@ export class RoomEncryption {
         this._senderDeviceCache = new Map();    // purge the sender device cache
     }
 
-    async writeMemberChanges(memberChanges, txn, log) {
+    async writeSync(roomResponse, memberChanges, txn, log) {
-        let shouldFlush = false;
+        let historyVisibility = await this._loadHistoryVisibilityIfNeeded(this._historyVisibility, txn);
-        const memberChangesArray = Array.from(memberChanges.values());
+        const addedMembers = [];
-        // this also clears our session if we leave the room ourselves
+        const removedMembers = [];
-        if (memberChangesArray.some(m => m.hasLeft)) {
+        // update the historyVisibility if needed
+        await iterateResponseStateEvents(roomResponse, event => {
+            // TODO: can the same state event appear twice? Hence we would be rewriting the useridentities twice...
+            // we'll see in the logs
+            if(event.state_key === "" && event.type === ROOM_HISTORY_VISIBILITY_TYPE) {
+                const newHistoryVisibility = event?.content?.history_visibility;
+                if (newHistoryVisibility !== historyVisibility) {
+                    return log.wrap({
+                        l: "history_visibility changed",
+                        from: historyVisibility,
+                        to: newHistoryVisibility
+                    }, async log => {
+                        historyVisibility = newHistoryVisibility;
+                        const result = await this._deviceTracker.writeHistoryVisibility(this._room, historyVisibility, txn, log);
+                        addedMembers.push(...result.added);
+                        removedMembers.push(...result.removed);
+                    });
+                }
+            }
+        });
+        // process member changes
+        if (memberChanges.size) {
+            const result = await this._deviceTracker.writeMemberChanges(
+                this._room, memberChanges, historyVisibility, txn);
+            addedMembers.push(...result.added);
+            removedMembers.push(...result.removed);
+        }
+        // discard key if somebody (including ourselves) left
+        if (removedMembers.length) {
             log.log({
                 l: "discardOutboundSession",
-                leftUsers: memberChangesArray.filter(m => m.hasLeft).map(m => m.userId),
+                leftUsers: removedMembers,
             });
             this._megolmEncryption.discardOutboundSession(this._room.id, txn);
         }
-        if (memberChangesArray.some(m => m.hasJoined)) {
+        let shouldFlush = false;
-            shouldFlush = await this._addShareRoomKeyOperationForNewMembers(memberChangesArray, txn, log);
+        // add room to userIdentities if needed, and share the current key with them
+        if (addedMembers.length) {
+            shouldFlush = await this._addShareRoomKeyOperationForMembers(addedMembers, txn, log);
         }
-        await this._deviceTracker.writeMemberChanges(this._room, memberChanges, txn);
+        return {shouldFlush, historyVisibility};
-        return shouldFlush;
+    }
+
+    afterSync({historyVisibility}) {
+        this._historyVisibility = historyVisibility;
+    }
+
+    async _loadHistoryVisibilityIfNeeded(historyVisibility, txn = undefined) {
+        if (!historyVisibility) {
+            if (!txn) {
+                txn = await this._storage.readTxn([this._storage.storeNames.roomState]);
+            }
+            const visibilityEntry = await txn.roomState.get(this._room.id, ROOM_HISTORY_VISIBILITY_TYPE, "");
+            if (visibilityEntry) {
+                return visibilityEntry.event?.content?.history_visibility;
+            }
+        }
+        return historyVisibility;
     }
 
     async prepareDecryptAll(events, newKeys, source, txn) {
@@ -274,10 +323,15 @@ export class RoomEncryption {
     }
 
     async _shareNewRoomKey(roomKeyMessage, hsApi, log) {
+        this._historyVisibility = await this._loadHistoryVisibilityIfNeeded(this._historyVisibility);
+        await this._deviceTracker.trackRoom(this._room, this._historyVisibility, log);
+        const devices = await this._deviceTracker.devicesForTrackedRoom(this._room.id, hsApi, log);
+        const userIds = Array.from(devices.reduce((set, device) => set.add(device.userId), new Set()));
+            
         let writeOpTxn = await this._storage.readWriteTxn([this._storage.storeNames.operations]);
         let operation;
         try {
-            operation = this._writeRoomKeyShareOperation(roomKeyMessage, null, writeOpTxn);
+            operation = this._writeRoomKeyShareOperation(roomKeyMessage, userIds, writeOpTxn);
         } catch (err) {
             writeOpTxn.abort();
             throw err;
@@ -288,8 +342,7 @@ export class RoomEncryption {
         await this._processShareRoomKeyOperation(operation, hsApi, log);
     }
 
-    async _addShareRoomKeyOperationForNewMembers(memberChangesArray, txn, log) {
+    async _addShareRoomKeyOperationForMembers(userIds, txn, log) {
-        const userIds = memberChangesArray.filter(m => m.hasJoined).map(m => m.userId);
         const roomKeyMessage = await this._megolmEncryption.createRoomKeyMessage(
             this._room.id, txn);
         if (roomKeyMessage) {
@@ -342,18 +395,9 @@ export class RoomEncryption {
 
     async _processShareRoomKeyOperation(operation, hsApi, log) {
         log.set("id", operation.id);
-
+        this._historyVisibility = await this._loadHistoryVisibilityIfNeeded(this._historyVisibility);
-        await this._deviceTracker.trackRoom(this._room, log);
+        await this._deviceTracker.trackRoom(this._room, this._historyVisibility, log);
-        let devices;
+        const devices = await this._deviceTracker.devicesForRoomMembers(this._room.id, operation.userIds, hsApi, log);
-        if (operation.userIds === null) {
-            devices = await this._deviceTracker.devicesForTrackedRoom(this._room.id, hsApi, log);
-            const userIds = Array.from(devices.reduce((set, device) => set.add(device.userId), new Set()));
-            operation.userIds = userIds;
-            await this._updateOperationsStore(operations => operations.update(operation));
-        } else {
-            devices = await this._deviceTracker.devicesForRoomMembers(this._room.id, operation.userIds, hsApi, log);
-        }
-        
         const messages = await log.wrap("olm encrypt", log => this._olmEncryption.encrypt(
             "m.room_key", operation.roomKeyMessage, devices, hsApi, log));
         const missingDevices = devices.filter(d => !messages.some(m => m.device === d));
@@ -507,3 +551,143 @@ class BatchDecryptionResult {
         }));
     }
 }
+
+import {createMockStorage} from "../../mocks/Storage";
+import {Clock as MockClock} from "../../mocks/Clock";
+import {poll} from "../../mocks/poll";
+import {Instance as NullLoggerInstance} from "../../logging/NullLogger";
+import {ConsoleLogger} from "../../logging/ConsoleLogger";
+import {HomeServer as MockHomeServer} from "../../mocks/HomeServer.js";
+
+export function tests() {
+    const roomId = "!abc:hs.tld";
+    return {
+        "ensureMessageKeyIsShared tracks room and passes correct history visibility to deviceTracker": async assert => {
+            const storage = await createMockStorage();
+            const megolmMock = {
+                async ensureOutboundSession() { return { }; }
+            };
+            const olmMock = {
+                async encrypt() { return []; }
+            }
+            let isRoomTracked = false;
+            let isDevicesRequested = false;
+            const deviceTracker = {
+                async trackRoom(room, historyVisibility) {
+                    // only assert on first call
+                    if (isRoomTracked) { return; }
+                    assert(!isDevicesRequested);
+                    assert.equal(room.id, roomId);
+                    assert.equal(historyVisibility, "invited");
+                    isRoomTracked = true;
+                },
+                async devicesForTrackedRoom() {
+                    assert(isRoomTracked);
+                    isDevicesRequested = true;
+                    return [];
+                },
+                async devicesForRoomMembers() {
+                    return [];
+                }
+            }
+            const writeTxn = await storage.readWriteTxn([storage.storeNames.roomState]);
+            writeTxn.roomState.set(roomId, {state_key: "", type: ROOM_HISTORY_VISIBILITY_TYPE, content: {
+                history_visibility: "invited"
+            }});
+            await writeTxn.complete();
+            const roomEncryption = new RoomEncryption({
+                room: {id: roomId},
+                megolmEncryption: megolmMock,
+                olmEncryption: olmMock,
+                storage,
+                deviceTracker,
+                clock: new MockClock()
+            });
+            const homeServer = new MockHomeServer();
+            const promise = roomEncryption.ensureMessageKeyIsShared(homeServer.api, NullLoggerInstance.item);
+            // need to poll because sendToDevice isn't first async step
+            const request = await poll(() => homeServer.requests.sendToDevice?.[0]);
+            request.respond({});
+            await promise;
+            assert(isRoomTracked);
+            assert(isDevicesRequested);
+        },
+        "encrypt tracks room and passes correct history visibility to deviceTracker": async assert => {
+            const storage = await createMockStorage();
+            const megolmMock = {
+                async encrypt() { return { roomKeyMessage: {} }; }
+            };
+            const olmMock = {
+                async encrypt() { return []; }
+            }
+            let isRoomTracked = false;
+            let isDevicesRequested = false;
+            const deviceTracker = {
+                async trackRoom(room, historyVisibility) {
+                    // only assert on first call
+                    if (isRoomTracked) { return; }
+                    assert(!isDevicesRequested);
+                    assert.equal(room.id, roomId);
+                    assert.equal(historyVisibility, "invited");
+                    isRoomTracked = true;
+                },
+                async devicesForTrackedRoom() {
+                    assert(isRoomTracked);
+                    isDevicesRequested = true;
+                    return [];
+                },
+                async devicesForRoomMembers() {
+                    return [];
+                }
+            }
+            const writeTxn = await storage.readWriteTxn([storage.storeNames.roomState]);
+            writeTxn.roomState.set(roomId, {state_key: "", type: ROOM_HISTORY_VISIBILITY_TYPE, content: {
+                history_visibility: "invited"
+            }});
+            await writeTxn.complete();
+            const roomEncryption = new RoomEncryption({
+                room: {id: roomId},
+                megolmEncryption: megolmMock,
+                olmEncryption: olmMock,
+                storage,
+                deviceTracker
+            });
+            const homeServer = new MockHomeServer();
+            const promise = roomEncryption.encrypt("m.room.message", {body: "hello"}, homeServer.api, NullLoggerInstance.item);
+            // need to poll because sendToDevice isn't first async step
+            const request = await poll(() => homeServer.requests.sendToDevice?.[0]);
+            request.respond({});
+            await promise;
+            assert(isRoomTracked);
+            assert(isDevicesRequested);
+        },
+        "writeSync passes correct history visibility to deviceTracker": async assert => {
+            const storage = await createMockStorage();
+            let isMemberChangesCalled = false;
+            const deviceTracker = {
+                async writeMemberChanges(room, memberChanges, historyVisibility, txn) {
+                    assert.equal(historyVisibility, "invited");
+                    isMemberChangesCalled = true;
+                    return {removed: [], added: []};
+                },
+                async devicesForRoomMembers() {
+                    return [];
+                }
+            }
+            const writeTxn = await storage.readWriteTxn([storage.storeNames.roomState]);
+            writeTxn.roomState.set(roomId, {state_key: "", type: ROOM_HISTORY_VISIBILITY_TYPE, content: {
+                history_visibility: "invited"
+            }});
+            const memberChanges = new Map([["@alice:hs.tld", {}]]);
+            const roomEncryption = new RoomEncryption({
+                room: {id: roomId},
+                storage,
+                deviceTracker
+            });
+            const roomResponse = {};
+            const txn = await storage.readWriteTxn([storage.storeNames.roomState]);
+            await roomEncryption.writeSync(roomResponse, memberChanges, txn, NullLoggerInstance.item);
+            assert(isMemberChangesCalled);
+        },
+    }
+}

src/matrix/e2ee/common.js

@@ -69,3 +69,28 @@ export function createRoomEncryptionEvent() {
         }
     }
 }
+
+
+// Use enum when converting to TS
+export const HistoryVisibility = Object.freeze({
+    Joined: "joined",
+    Invited: "invited",
+    WorldReadable: "world_readable",
+    Shared: "shared",
+});
+
+export function shouldShareKey(membership, historyVisibility) {
+    switch (historyVisibility) {
+        case HistoryVisibility.WorldReadable:
+            return true;
+        case HistoryVisibility.Shared:
+            // was part of room at some time
+            return membership !== undefined;
+        case HistoryVisibility.Joined:
+            return membership === "join";
+        case HistoryVisibility.Invited:
+            return membership === "invite" || membership === "join";
+        default:
+            return false;
+    }
+}

src/matrix/room/BaseRoom.js

@@ -243,7 +243,7 @@ export class BaseRoom extends EventEmitter {
 
 
     /** @public */
-    async loadMemberList(log = null) {
+    async loadMemberList(txn = undefined, log = null) {
         if (this._memberList) {
             // TODO: also await fetchOrLoadMembers promise here
             this._memberList.retain();
@@ -254,6 +254,9 @@ export class BaseRoom extends EventEmitter {
                 roomId: this._roomId,
                 hsApi: this._hsApi,
                 storage: this._storage,
+                // pass in a transaction if we know we won't need to fetch (which would abort the transaction)
+                // and we want to make this operation part of the larger transaction
+                txn,
                 syncToken: this._getSyncToken(),
                 // to handle race between /members and /sync
                 setChangedMembersMap: map => this._changedMembersDuringSync = map,

src/matrix/room/Room.js

@@ -139,11 +139,11 @@ export class Room extends BaseRoom {
         }
         log.set("newEntries", newEntries.length);
         log.set("updatedEntries", updatedEntries.length);
-        let shouldFlushKeyShares = false;
+        let encryptionChanges;
         // pass member changes to device tracker
-        if (roomEncryption && this.isTrackingMembers && memberChanges?.size) {
+        if (roomEncryption) {
-            shouldFlushKeyShares = await roomEncryption.writeMemberChanges(memberChanges, txn, log);
+            encryptionChanges = await roomEncryption.writeSync(roomResponse, memberChanges, txn, log);
-            log.set("shouldFlushKeyShares", shouldFlushKeyShares);
+            log.set("shouldFlushKeyShares", encryptionChanges.shouldFlush);
         }
         const allEntries = newEntries.concat(updatedEntries);
         // also apply (decrypted) timeline entries to the summary changes
@@ -188,7 +188,7 @@ export class Room extends BaseRoom {
             memberChanges,
             heroChanges,
             powerLevelsEvent,
-            shouldFlushKeyShares,
+            encryptionChanges,
         };
     }
 
@@ -201,11 +201,14 @@ export class Room extends BaseRoom {
         const {
             summaryChanges, newEntries, updatedEntries, newLiveKey,
             removedPendingEvents, memberChanges, powerLevelsEvent,
-            heroChanges, roomEncryption
+            heroChanges, roomEncryption, encryptionChanges
         } = changes;
         log.set("id", this.id);
         this._syncWriter.afterSync(newLiveKey);
         this._setEncryption(roomEncryption);
+        if (this._roomEncryption) {
+            this._roomEncryption.afterSync(encryptionChanges);
+        }
         if (memberChanges.size) {
             if (this._changedMembersDuringSync) {
                 for (const [userId, memberChange] of memberChanges.entries()) {
@@ -288,8 +291,8 @@ export class Room extends BaseRoom {
         }
     }
 
-    needsAfterSyncCompleted({shouldFlushKeyShares}) {
+    needsAfterSyncCompleted({encryptionChanges}) {
-        return shouldFlushKeyShares;
+        return encryptionChanges?.shouldFlush;
     }
 
     /**

src/matrix/room/common.ts

@@ -14,6 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import type {StateEvent} from "../storage/types";
+
 export function getPrevContentFromStateEvent(event) {
     // where to look for prev_content is a bit of a mess,
     // see https://matrix.to/#/!NasysSDfxKxZBzJJoE:matrix.org/$DvrAbZJiILkOmOIuRsNoHmh2v7UO5CWp_rYhlGk34fQ?via=matrix.org&via=pixie.town&via=amorgan.xyz
@@ -40,3 +42,83 @@ export enum RoomType {
     Private,
     Public
 }
+
+type RoomResponse = {
+    state?: {
+        events?: Array<StateEvent>
+    },
+    timeline?: {
+        events?: Array<StateEvent>
+    }
+}
+
+/** iterates over any state events in a sync room response, in the order that they should be applied (from older to younger events) */
+export function iterateResponseStateEvents(roomResponse: RoomResponse, callback: (StateEvent) => Promise<void> | void): Promise<void> | void {
+    let promises: Promise<void>[] | undefined = undefined;
+    const callCallback = stateEvent => {
+        const result = callback(stateEvent);
+        if (result instanceof Promise) {
+            promises = promises ?? [];
+            promises.push(result);
+        }
+    };
+    // first iterate over state events, they precede the timeline
+    const stateEvents = roomResponse.state?.events;
+    if (stateEvents) {
+        for (let i = 0; i < stateEvents.length; i++) {
+            callCallback(stateEvents[i]);
+        }
+    }
+    // now see if there are any state events within the timeline
+    let timelineEvents = roomResponse.timeline?.events;
+    if (timelineEvents) {
+        for (let i = 0; i < timelineEvents.length; i++) {
+            const event = timelineEvents[i];
+            if (typeof event.state_key === "string") {
+                callCallback(event);
+            }
+        }
+    }
+    if (promises) {
+        return Promise.all(promises).then(() => undefined);
+    }
+}
+
+export function tests() {
+    return {
+        "test iterateResponseStateEvents with both state and timeline sections": assert => {
+            const roomResponse = {
+                state: {
+                    events: [
+                        {type: "m.room.member", state_key: "1"},
+                        {type: "m.room.member", state_key: "2", content: {a: 1}},
+                    ]
+                },
+                timeline: {
+                    events: [
+                        {type: "m.room.message"},
+                        {type: "m.room.member", state_key: "3"},
+                        {type: "m.room.message"},
+                        {type: "m.room.member", state_key: "2", content: {a: 2}},
+                    ]
+                }
+            } as unknown as RoomResponse;
+            const expectedStateKeys = ["1", "2", "3", "2"];
+            const expectedAForMember2 = [1, 2];
+            iterateResponseStateEvents(roomResponse, event => {
+                assert.strictEqual(event.type, "m.room.member");
+                assert.strictEqual(expectedStateKeys.shift(), event.state_key);
+                if (event.state_key === "2") {
+                    assert.strictEqual(expectedAForMember2.shift(), event.content.a);
+                }
+            });
+            assert.strictEqual(expectedStateKeys.length, 0);
+            assert.strictEqual(expectedAForMember2.length, 0);
+        },
+        "test iterateResponseStateEvents with empty response": assert => {
+            iterateResponseStateEvents({}, () => {
+                assert.fail("no events expected");
+            });
+        }
+    }
+}

src/matrix/room/members/RoomMember.js

@@ -137,6 +137,10 @@ export class MemberChange {
         return this.member.membership;
     }
 
+    get wasInvited() {
+        return this.previousMembership === "invite" && this.membership !== "invite";
+    }
+
     get hasLeft() {
         return this.previousMembership === "join" && this.membership !== "join";
     }

src/matrix/room/members/load.js

@@ -17,10 +17,12 @@ limitations under the License.
 
 import {RoomMember} from "./RoomMember.js";
 
-async function loadMembers({roomId, storage}) {
+async function loadMembers({roomId, storage, txn}) {
-    const txn = await storage.readTxn([
+    if (!txn) {
-        storage.storeNames.roomMembers,
+        txn = await storage.readTxn([
-    ]);
+            storage.storeNames.roomMembers,
+        ]);
+    }
     const memberDatas = await txn.roomMembers.getAll(roomId);
     return memberDatas.map(d => new RoomMember(d));
 }

src/matrix/storage/idb/schema.ts

@@ -2,7 +2,6 @@ import {IDOMStorage} from "./types";
 import {ITransaction} from "./QueryTarget";
 import {iterateCursor, NOT_DONE, reqAsPromise} from "./utils";
 import {RoomMember, EVENT_TYPE as MEMBER_EVENT_TYPE} from "../../room/members/RoomMember.js";
-import {addRoomToIdentity} from "../../e2ee/DeviceTracker.js";
 import {SESSION_E2EE_KEY_PREFIX} from "../../e2ee/common.js";
 import {SummaryData} from "../../room/RoomSummary";
 import {RoomMemberStore, MemberData} from "./stores/RoomMemberStore";
@@ -183,51 +182,12 @@ function createTimelineRelationsStore(db: IDBDatabase) : void {
     db.createObjectStore("timelineRelations", {keyPath: "key"});
 }
 
-//v11 doesn't change the schema, but ensures all userIdentities have all the roomIds they should (see #470)
+//v11 doesn't change the schema,
-async function fixMissingRoomsInUserIdentities(db: IDBDatabase, txn: IDBTransaction, localStorage: IDOMStorage, log: ILogItem) {
+// but ensured all userIdentities have all the roomIds they should (see #470)
-    const roomSummaryStore = txn.objectStore("roomSummary");
+
-    const trackedRoomIds: string[] = [];
+// 2022-07-20: The fix dated from August 2021, and have removed it now because of a
-    await iterateCursor<SummaryData>(roomSummaryStore.openCursor(), roomSummary => {
+// refactoring needed in the device tracker, which made it inconvenient to expose addRoomToIdentity
-        if (roomSummary.isTrackingMembers) {
+function fixMissingRoomsInUserIdentities() {}
-            trackedRoomIds.push(roomSummary.roomId);
-        }
-        return NOT_DONE;
-    });
-    const outboundGroupSessionsStore = txn.objectStore("outboundGroupSessions");
-    const userIdentitiesStore: IDBObjectStore = txn.objectStore("userIdentities");
-    const roomMemberStore = txn.objectStore("roomMembers");
-    for (const roomId of trackedRoomIds) {
-        let foundMissing = false;
-        const joinedUserIds: string[] = [];
-        const memberRange = IDBKeyRange.bound(roomId, `${roomId}|${MAX_UNICODE}`, true, true);
-        await log.wrap({l: "room", id: roomId}, async log => {
-            await iterateCursor<MemberData>(roomMemberStore.openCursor(memberRange), member => {
-                if (member.membership === "join") {
-                    joinedUserIds.push(member.userId);
-                }
-                return NOT_DONE;
-            });
-            log.set("joinedUserIds", joinedUserIds.length);
-            for (const userId of joinedUserIds) {
-                const identity = await reqAsPromise(userIdentitiesStore.get(userId));
-                const originalRoomCount = identity?.roomIds?.length;
-                const updatedIdentity = addRoomToIdentity(identity, userId, roomId);
-                if (updatedIdentity) {
-                    log.log({l: `fixing up`, id: userId,
-                        roomsBefore: originalRoomCount, roomsAfter: updatedIdentity.roomIds.length});
-                    userIdentitiesStore.put(updatedIdentity);
-                    foundMissing = true;
-                }
-            }
-            log.set("foundMissing", foundMissing);
-            if (foundMissing) {
-                // clear outbound megolm session,
-                // so we'll create a new one on the next message that will be properly shared
-                outboundGroupSessionsStore.delete(roomId);
-            }
-        });
-    }
-}
 
 // v12 move ssssKey to e2ee:ssssKey so it will get backed up in the next step
 async function changeSSSSKeyPrefix(db: IDBDatabase, txn: IDBTransaction) {

src/platform/types/theme.ts

@@ -22,6 +22,13 @@ export type ThemeManifest = Partial<{
     version: number;
     // A user-facing string that is the name for this theme-collection.
     name: string;
+    // An identifier for this theme
+    id: string;
+    /**
+     * Id of the theme that this theme derives from.
+     * Only present for derived/runtime themes.
+    */
+    extends: string;
     /**
      * This is added to the manifest during the build process and includes data
      * that is needed to load themes at runtime.
@@ -42,6 +49,12 @@ export type ThemeManifest = Partial<{
         "runtime-asset": string;
         // Array of derived-variables
         "derived-variables": Array<string>;
+        /**
+         * Mapping from icon variable to location of icon in build output with query parameters
+         * indicating how it should be colored for this particular theme.
+         * eg: "icon-url-1": "element-logo.86bc8565.svg?primary=accent-color"
+         */
+        icon: Record<string, string>;
     };
     values: {
         /**
@@ -60,6 +73,8 @@ type Variant = Partial<{
     default: boolean;
     // A user-facing string that is the name for this variant.
     name: string;
+    // A boolean indicating whether this is a dark theme or not
+    dark: boolean;
     /**
      * Mapping from css variable to its value.
      * eg: {"background-color-primary": "#21262b", ...} 

src/platform/web/Platform.js

@@ -38,7 +38,7 @@ import {downloadInIframe} from "./dom/download.js";
 import {Disposables} from "../../utils/Disposables";
 import {parseHTML} from "./parsehtml.js";
 import {handleAvatarError} from "./ui/avatar";
-import {ThemeLoader} from "./ThemeLoader";
+import {ThemeLoader} from "./theming/ThemeLoader";
 
 function addScript(src) {
     return new Promise(function (resolve, reject) {

src/platform/web/ThemeLoader.ts

@@ -1,217 +0,0 @@
-/*
-Copyright 2020 The Matrix.org Foundation C.I.C.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-import type {ILogItem} from "../../logging/types.js";
-import type {Platform} from "./Platform.js";
-
-type NormalVariant = {
-    id: string;
-    cssLocation: string;
-};
-
-type DefaultVariant = {
-    dark: {
-        id: string;
-        cssLocation: string;
-        variantName: string;
-    };
-    light: {
-        id: string;
-        cssLocation: string;
-        variantName: string;
-    };
-    default: {
-        id: string;
-        cssLocation: string;
-        variantName: string;
-    };
-}
-
-type ThemeInformation = NormalVariant | DefaultVariant; 
-
-export enum ColorSchemePreference {
-    Dark,
-    Light
-};
-
-export class ThemeLoader {
-    private _platform: Platform;
-    private _themeMapping: Record<string, ThemeInformation>;
-
-    constructor(platform: Platform) {
-        this._platform = platform;
-    }
-
-    async init(manifestLocations: string[], log?: ILogItem): Promise<void> {
-        await this._platform.logger.wrapOrRun(log, "ThemeLoader.init", async (log) => {
-            this._themeMapping = {};
-            const results = await Promise.all(
-                manifestLocations.map( location => this._platform.request(location, { method: "GET", format: "json", cache: true, }).response())
-            );
-            results.forEach(({ body }, i) => this._populateThemeMap(body, manifestLocations[i], log));
-        });
-    }
-
-    private _populateThemeMap(manifest, manifestLocation: string, log: ILogItem) {
-        log.wrap("populateThemeMap", (l) => {
-            /*
-            After build has finished, the source section of each theme manifest
-            contains `built-assets` which is a mapping from the theme-id to
-            cssLocation of theme
-            */
-            const builtAssets: Record<string, string> = manifest.source?.["built-assets"];
-            const themeName = manifest.name;
-            let defaultDarkVariant: any = {}, defaultLightVariant: any = {};
-            for (let [themeId, cssLocation] of Object.entries(builtAssets)) {
-                try {
-                    /**
-                     * This cssLocation is relative to the location of the manifest file.
-                     * So we first need to resolve it relative to the root of this hydrogen instance.
-                     */
-                    cssLocation = new URL(cssLocation, new URL(manifestLocation, window.location.origin)).href;
-                }
-                catch {
-                    continue;
-                }
-                const variant = themeId.match(/.+-(.+)/)?.[1];
-                const { name: variantName, default: isDefault, dark } = manifest.values.variants[variant!];
-                const themeDisplayName = `${themeName} ${variantName}`;
-                if (isDefault) {
-                    /**
-                     * This is a default variant!
-                     * We'll add these to the themeMapping (separately) keyed with just the
-                     * theme-name (i.e "Element" instead of "Element Dark").
-                     * We need to be able to distinguish them from other variants!
-                     * 
-                     * This allows us to render radio-buttons with "dark" and
-                     * "light" options.
-                    */
-                    const defaultVariant = dark ? defaultDarkVariant : defaultLightVariant;
-                    defaultVariant.variantName = variantName;
-                    defaultVariant.id = themeId
-                    defaultVariant.cssLocation = cssLocation;
-                    continue;
-                }
-                // Non-default variants are keyed in themeMapping with "theme_name variant_name"
-                // eg: "Element Dark"
-                this._themeMapping[themeDisplayName] = {
-                    cssLocation,
-                    id: themeId
-                };
-            }
-            if (defaultDarkVariant.id && defaultLightVariant.id) {
-                /**
-                 * As mentioned above, if there's both a default dark and a default light variant,
-                 * add them to themeMapping separately.
-                 */
-                const defaultVariant = this.preferredColorScheme === ColorSchemePreference.Dark ? defaultDarkVariant : defaultLightVariant;
-                this._themeMapping[themeName] = { dark: defaultDarkVariant, light: defaultLightVariant, default: defaultVariant };
-            }
-            else {
-                /**
-                 * If only one default variant is found (i.e only dark default or light default but not both),
-                 * treat it like any other variant.
-                 */
-                const variant = defaultDarkVariant.id ? defaultDarkVariant : defaultLightVariant;
-                this._themeMapping[`${themeName} ${variant.variantName}`] = { id: variant.id, cssLocation: variant.cssLocation };
-            }
-            //Add the default-theme as an additional option to the mapping
-            const defaultThemeId = this.getDefaultTheme();
-            if (defaultThemeId) {
-                const themeDetails = this._findThemeDetailsFromId(defaultThemeId);
-                if (themeDetails) {
-                    this._themeMapping["Default"] = { id: "default", cssLocation: themeDetails.cssLocation };
-                }
-            }
-            l.log({ l: "Default Theme", theme: defaultThemeId});
-            l.log({ l: "Preferred colorscheme", scheme: this.preferredColorScheme === ColorSchemePreference.Dark ? "dark" : "light" });
-            l.log({ l: "Result", themeMapping: this._themeMapping });
-         });
-    }
-
-    setTheme(themeName: string, themeVariant?: "light" | "dark" | "default", log?: ILogItem) {
-        this._platform.logger.wrapOrRun(log, { l: "change theme", name: themeName, variant: themeVariant }, () => {
-            let cssLocation: string;
-            let themeDetails = this._themeMapping[themeName];
-            if ("id" in themeDetails) {
-                cssLocation = themeDetails.cssLocation;
-            }
-            else {
-                if (!themeVariant) {
-                    throw new Error("themeVariant is undefined!");
-                }
-                cssLocation = themeDetails[themeVariant].cssLocation;
-            }
-            this._platform.replaceStylesheet(cssLocation);
-            this._platform.settingsStorage.setString("theme-name", themeName);
-            if (themeVariant) {
-                this._platform.settingsStorage.setString("theme-variant", themeVariant);
-            }
-            else {
-                this._platform.settingsStorage.remove("theme-variant");
-            }
-        });
-    }
-
-    /** Maps theme display name to theme information */
-    get themeMapping(): Record<string, ThemeInformation> {
-        return this._themeMapping;
-    }
-
-    async getActiveTheme(): Promise<{themeName: string, themeVariant?: string}> {
-        let themeName = await this._platform.settingsStorage.getString("theme-name");
-        let themeVariant = await this._platform.settingsStorage.getString("theme-variant");
-        if (!themeName || !this._themeMapping[themeName]) {
-            themeName = "Default" in this._themeMapping ? "Default" : Object.keys(this._themeMapping)[0];
-            if (!this._themeMapping[themeName][themeVariant]) {
-                themeVariant = "default" in this._themeMapping[themeName] ? "default" : undefined;
-            }
-        }
-        return { themeName, themeVariant };
-    }
-
-    getDefaultTheme(): string | undefined {
-        switch (this.preferredColorScheme) {
-            case ColorSchemePreference.Dark:
-                return this._platform.config["defaultTheme"]?.dark;
-            case ColorSchemePreference.Light:
-                return this._platform.config["defaultTheme"]?.light;
-        }
-    }
-
-    private _findThemeDetailsFromId(themeId: string): {themeName: string, cssLocation: string, variant?: string} | undefined {
-        for (const [themeName, themeData] of Object.entries(this._themeMapping)) {
-            if ("id" in themeData && themeData.id === themeId) {
-                return { themeName, cssLocation: themeData.cssLocation };
-            }
-            else if ("light" in themeData && themeData.light?.id === themeId) {
-                return { themeName, cssLocation: themeData.light.cssLocation, variant: "light" };
-            }
-            else if ("dark" in themeData && themeData.dark?.id === themeId) {
-                return { themeName, cssLocation: themeData.dark.cssLocation, variant: "dark" };
-            }
-        }
-    }
-
-    get preferredColorScheme(): ColorSchemePreference | undefined {
-        if (window.matchMedia("(prefers-color-scheme: dark)").matches) {
-            return ColorSchemePreference.Dark;
-        }
-        else if (window.matchMedia("(prefers-color-scheme: light)").matches) {
-            return ColorSchemePreference.Light;
-        }
-    }
-}

src/platform/web/assets/config.json

@@ -4,6 +4,6 @@
     "gatewayUrl": "https://matrix.org",
     "applicationServerKey": "BC-gpSdVHEXhvHSHS0AzzWrQoukv2BE7KzpoPO_FfPacqOo3l1pdqz7rSgmB04pZCWaHPz7XRe6fjLaC-WPDopM"
   },
-  "defaultHomeServer": "matrix.org",
+  "defaultHomeServer": "matrix.test.mystiq.app",
-  "bugReportEndpointUrl": "https://element.io/bugreports/submit"
+  "bugReportEndpointUrl": "https://rageshake.test.mystiq.app/api/submit"
 }

src/platform/web/dom/request/fetch.js

@@ -115,6 +115,9 @@ export function createFetchRequest(createTimeout, serviceWorkerHandler) {
                 } else if (format === "buffer") {
                     body = await response.arrayBuffer();
                 }
+                else if (format === "text") {
+                    body = await response.text();
+                }
             } catch (err) {
                 // some error pages return html instead of json, ignore error
                 if (!(err.name === "SyntaxError" && status >= 400)) {

src/platform/web/theming/DerivedVariables.ts

@@ -0,0 +1,131 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+ 
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+import {derive} from "./shared/color.mjs";
+
+export class DerivedVariables {
+    private _baseVariables: Record<string, string>;
+    private _variablesToDerive: string[]
+    private _isDark: boolean
+    private _aliases: Record<string, string> = {};
+    private _derivedAliases: string[] = [];
+
+    constructor(baseVariables: Record<string, string>, variablesToDerive: string[], isDark: boolean) {
+        this._baseVariables = baseVariables;
+        this._variablesToDerive = variablesToDerive;
+        this._isDark = isDark;
+    }
+
+    toVariables(): Record<string, string> {
+        const resolvedVariables: any = {};
+        this._detectAliases();
+        for (const variable of this._variablesToDerive) {
+            const resolvedValue = this._derive(variable);
+            if (resolvedValue) {
+                resolvedVariables[variable] = resolvedValue;
+            }
+        }
+        for (const [alias, variable] of Object.entries(this._aliases) as any) {
+            resolvedVariables[alias] = this._baseVariables[variable] ?? resolvedVariables[variable];
+        }
+        for (const variable of this._derivedAliases) {
+            const resolvedValue = this._deriveAlias(variable, resolvedVariables);
+            if (resolvedValue) {
+                resolvedVariables[variable] = resolvedValue;
+            }
+        }
+        return resolvedVariables;
+    }
+
+    private _detectAliases(): void {
+        const newVariablesToDerive: string[] = [];
+        for (const variable of this._variablesToDerive) {
+            const [alias, value] = variable.split("=");
+            if (value) {
+                this._aliases[alias] = value;
+            }
+            else {
+                newVariablesToDerive.push(variable);
+            }
+        }
+        this._variablesToDerive = newVariablesToDerive;
+    }
+
+    private _derive(variable: string): string | undefined {
+        const RE_VARIABLE_VALUE = /(.+)--(.+)-(.+)/;
+        const matches = variable.match(RE_VARIABLE_VALUE);
+        if (matches) {
+            const [, baseVariable, operation, argument] = matches;
+            const value = this._baseVariables[baseVariable];
+            if (!value ) {
+                if (this._aliases[baseVariable]) {
+                    this._derivedAliases.push(variable);
+                    return;
+                }
+                else {
+                    throw new Error(`Cannot find value for base variable "${baseVariable}"!`);
+                }
+            }
+            const resolvedValue = derive(value, operation, argument, this._isDark);
+            return resolvedValue;
+        }
+    }
+
+    private _deriveAlias(variable: string, resolvedVariables: Record<string, string>): string | undefined {
+        const RE_VARIABLE_VALUE = /(.+)--(.+)-(.+)/;
+        const matches = variable.match(RE_VARIABLE_VALUE);
+        if (matches) {
+            const [, baseVariable, operation, argument] = matches;
+            const value = resolvedVariables[baseVariable];
+            if (!value ) {
+                throw new Error(`Cannot find value for alias "${baseVariable}" when trying to derive ${variable}!`);
+            }
+            const resolvedValue = derive(value, operation, argument, this._isDark);
+            return resolvedValue;
+        }
+    }
+}
+
+import * as pkg from "off-color";
+// @ts-ignore 
+const offColor = pkg.offColor ?? pkg.default.offColor;
+
+export function tests() {
+    return {
+        "Simple variable derivation": assert => {
+            const deriver = new DerivedVariables({ "background-color": "#ff00ff" }, ["background-color--darker-5"], false);
+            const result = deriver.toVariables();
+            const resultColor = offColor("#ff00ff").darken(5/100).hex();
+            assert.deepEqual(result, {"background-color--darker-5": resultColor});
+        },
+
+        "For dark themes, lighten and darken are inverted": assert => {
+            const deriver = new DerivedVariables({ "background-color": "#ff00ff" }, ["background-color--darker-5"], true);
+            const result = deriver.toVariables();
+            const resultColor = offColor("#ff00ff").lighten(5/100).hex();
+            assert.deepEqual(result, {"background-color--darker-5": resultColor});
+        },
+
+        "Aliases can be derived": assert => {
+            const deriver = new DerivedVariables({ "background-color": "#ff00ff" }, ["my-awesome-alias=background-color","my-awesome-alias--darker-5"], false);
+            const result = deriver.toVariables();
+            const resultColor = offColor("#ff00ff").darken(5/100).hex();
+            assert.deepEqual(result, {
+                "my-awesome-alias": "#ff00ff",
+                "my-awesome-alias--darker-5": resultColor,
+            });
+        },
+    }
+}

src/platform/web/theming/IconColorizer.ts

@@ -0,0 +1,79 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+import type {Platform} from "../Platform.js";
+import {getColoredSvgString} from "./shared/svg-colorizer.mjs";
+
+type ParsedStructure = {
+    [variableName: string]: {
+        svg: Promise<{ status: number; body: string }>;
+        primary: string | null;
+        secondary: string | null;
+    };
+};
+
+export class IconColorizer {
+    private _iconVariables: Record<string, string>;
+    private _resolvedVariables: Record<string, string>;
+    private _manifestLocation: string;
+    private _platform: Platform;
+
+    constructor(platform: Platform, iconVariables: Record<string, string>, resolvedVariables: Record<string, string>, manifestLocation: string) {
+        this._platform = platform;
+        this._iconVariables = iconVariables;
+        this._resolvedVariables = resolvedVariables;
+        this._manifestLocation = manifestLocation;
+    }
+
+    async toVariables(): Promise<Record<string, string>> {
+        const { parsedStructure, promises } = await this._fetchAndParseIcons();
+        await Promise.all(promises);
+        return this._produceColoredIconVariables(parsedStructure);
+    }
+
+    private async _fetchAndParseIcons(): Promise<{ parsedStructure: ParsedStructure, promises: any[] }> {
+        const promises: any[] = [];
+        const parsedStructure: ParsedStructure = {};
+        for (const [variable, url] of Object.entries(this._iconVariables)) {
+            const urlObject = new URL(`https://${url}`);
+            const pathWithoutQueryParams = urlObject.hostname;
+            const relativePath = new URL(pathWithoutQueryParams, new URL(this._manifestLocation, window.location.origin));
+            const responsePromise = this._platform.request(relativePath, { method: "GET", format: "text", cache: true, }).response()
+            promises.push(responsePromise);
+            const searchParams = urlObject.searchParams;
+            parsedStructure[variable] = {
+                svg: responsePromise,
+                primary: searchParams.get("primary"),
+                secondary: searchParams.get("secondary")
+            };
+        }
+        return { parsedStructure, promises };
+    }
+
+    private async _produceColoredIconVariables(parsedStructure: ParsedStructure): Promise<Record<string, string>> {
+        let coloredVariables: Record<string, string> = {};
+        for (const [variable, { svg, primary, secondary }] of Object.entries(parsedStructure)) {
+            const { body: svgCode } = await svg;
+            if (!primary) {
+                throw new Error(`Primary color variable ${primary} not in list of variables!`);
+            }
+            const primaryColor = this._resolvedVariables[primary], secondaryColor = this._resolvedVariables[secondary!];
+            const coloredSvgCode = getColoredSvgString(svgCode, primaryColor, secondaryColor);
+            const dataURI = `url('data:image/svg+xml;utf8,${encodeURIComponent(coloredSvgCode)}')`;
+            coloredVariables[variable] = dataURI;
+        }
+        return coloredVariables;
+    }
+}

src/platform/web/theming/ThemeLoader.ts

@@ -0,0 +1,188 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import type {ILogItem} from "../../../logging/types";
+import type {Platform} from "../Platform.js";
+import {RuntimeThemeParser} from "./parsers/RuntimeThemeParser";
+import type {Variant, ThemeInformation} from "./parsers/types";
+import {ColorSchemePreference} from "./parsers/types";
+import {BuiltThemeParser} from "./parsers/BuiltThemeParser";
+
+export class ThemeLoader {
+    private _platform: Platform;
+    private _themeMapping: Record<string, ThemeInformation>;
+    private _injectedVariables?: Record<string, string>;
+
+    constructor(platform: Platform) {
+        this._platform = platform;
+    }
+
+    async init(manifestLocations: string[], log?: ILogItem): Promise<void> {
+        await this._platform.logger.wrapOrRun(log, "ThemeLoader.init", async (log) => {
+            const results = await Promise.all(
+                manifestLocations.map(location => this._platform.request(location, { method: "GET", format: "json", cache: true, }).response())
+            );
+            const runtimeThemeParser = new RuntimeThemeParser(this._platform, this.preferredColorScheme);
+            const builtThemeParser = new BuiltThemeParser(this.preferredColorScheme);
+            const runtimeThemePromises: Promise<void>[] = [];
+            for (let i = 0; i < results.length; ++i) {
+                const { body } = results[i];
+                try {
+                    if (body.extends) {
+                        const indexOfBaseManifest = results.findIndex(manifest => manifest.body.id === body.extends);
+                        if (indexOfBaseManifest === -1) {
+                            throw new Error(`Base manifest for derived theme at ${manifestLocations[i]} not found!`);
+                        }
+                        const {body: baseManifest} = results[indexOfBaseManifest];
+                        const baseManifestLocation = manifestLocations[indexOfBaseManifest];
+                        const promise = runtimeThemeParser.parse(body, baseManifest, baseManifestLocation, log);
+                        runtimeThemePromises.push(promise);
+                    }
+                    else {
+                        builtThemeParser.parse(body, manifestLocations[i], log);
+                    }
+                }
+                catch(e) {
+                    console.error(e);
+                }
+            }
+            await Promise.all(runtimeThemePromises);
+            this._themeMapping = { ...builtThemeParser.themeMapping, ...runtimeThemeParser.themeMapping };
+            Object.assign(this._themeMapping, builtThemeParser.themeMapping, runtimeThemeParser.themeMapping);
+            this._addDefaultThemeToMapping(log);
+            log.log({ l: "Preferred colorscheme", scheme: this.preferredColorScheme === ColorSchemePreference.Dark ? "dark" : "light" });
+            log.log({ l: "Result", themeMapping: this._themeMapping });
+        });
+    }
+
+    setTheme(themeName: string, themeVariant?: "light" | "dark" | "default", log?: ILogItem) {
+        this._platform.logger.wrapOrRun(log, { l: "change theme", name: themeName, variant: themeVariant }, () => {
+            let cssLocation: string, variables: Record<string, string>;
+            let themeDetails = this._themeMapping[themeName];
+            if ("id" in themeDetails) {
+                cssLocation = themeDetails.cssLocation;
+                variables = themeDetails.variables;
+            }
+            else {
+                if (!themeVariant) {
+                    throw new Error("themeVariant is undefined!");
+                }
+                cssLocation = themeDetails[themeVariant].cssLocation;
+                variables = themeDetails[themeVariant].variables;
+            }
+            this._platform.replaceStylesheet(cssLocation);
+            if (variables) {
+                log?.log({l: "Derived Theme", variables});
+                this._injectCSSVariables(variables);
+            }
+            else {
+                this._removePreviousCSSVariables();
+            }
+            this._platform.settingsStorage.setString("theme-name", themeName);
+            if (themeVariant) {
+                this._platform.settingsStorage.setString("theme-variant", themeVariant);
+            }
+            else {
+                this._platform.settingsStorage.remove("theme-variant");
+            }
+        });
+    }
+
+    private _injectCSSVariables(variables: Record<string, string>): void {
+        const root = document.documentElement;
+        for (const [variable, value] of Object.entries(variables)) {
+            root.style.setProperty(`--${variable}`, value);
+        }
+        this._injectedVariables = variables;
+    }
+
+    private _removePreviousCSSVariables(): void {
+        if (!this._injectedVariables) {
+            return;
+        }
+        const root = document.documentElement;
+        for (const variable of Object.keys(this._injectedVariables)) {
+            root.style.removeProperty(`--${variable}`);
+        }
+        this._injectedVariables = undefined;
+    }
+
+    /** Maps theme display name to theme information */
+    get themeMapping(): Record<string, ThemeInformation> {
+        return this._themeMapping;
+    }
+
+    async getActiveTheme(): Promise<{themeName: string, themeVariant?: string}> {
+        let themeName = await this._platform.settingsStorage.getString("theme-name");
+        let themeVariant = await this._platform.settingsStorage.getString("theme-variant");
+        if (!themeName || !this._themeMapping[themeName]) {
+            themeName = "Default" in this._themeMapping ? "Default" : Object.keys(this._themeMapping)[0];
+            if (!this._themeMapping[themeName][themeVariant]) {
+                themeVariant = "default" in this._themeMapping[themeName] ? "default" : undefined;
+            }
+        }
+        return { themeName, themeVariant };
+    }
+
+    getDefaultTheme(): string | undefined {
+        switch (this.preferredColorScheme) {
+            case ColorSchemePreference.Dark:
+                return this._platform.config["defaultTheme"]?.dark;
+            case ColorSchemePreference.Light:
+                return this._platform.config["defaultTheme"]?.light;
+        }
+    }
+
+    private _findThemeDetailsFromId(themeId: string): {themeName: string, themeData: Partial<Variant>} | undefined {
+        for (const [themeName, themeData] of Object.entries(this._themeMapping)) {
+            if ("id" in themeData && themeData.id === themeId) {
+                return { themeName, themeData };
+            }
+            else if ("light" in themeData && themeData.light?.id === themeId) {
+                return { themeName, themeData: themeData.light };
+            }
+            else if ("dark" in themeData && themeData.dark?.id === themeId) {
+                return { themeName, themeData: themeData.dark };
+            }
+        }
+    }
+
+    private _addDefaultThemeToMapping(log: ILogItem) {
+        log.wrap("addDefaultThemeToMapping", l => { 
+            const defaultThemeId = this.getDefaultTheme();
+            if (defaultThemeId) {
+                const themeDetails = this._findThemeDetailsFromId(defaultThemeId);
+                if (themeDetails) {
+                    this._themeMapping["Default"] = { id: "default", cssLocation: themeDetails.themeData.cssLocation! };
+                    const variables = themeDetails.themeData.variables;
+                    if (variables) {
+                        this._themeMapping["Default"].variables = variables;
+                    }
+                }
+            }
+            l.log({ l: "Default Theme", theme: defaultThemeId});
+        });
+    }
+
+    get preferredColorScheme(): ColorSchemePreference | undefined {
+        if (window.matchMedia("(prefers-color-scheme: dark)").matches) {
+            return ColorSchemePreference.Dark;
+        }
+        else if (window.matchMedia("(prefers-color-scheme: light)").matches) {
+            return ColorSchemePreference.Light;
+        }
+    }
+}

src/platform/web/theming/parsers/BuiltThemeParser.ts

@@ -0,0 +1,106 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import type {ThemeInformation} from "./types";
+import type {ThemeManifest} from "../../../types/theme";
+import type {ILogItem} from "../../../../logging/types";
+import {ColorSchemePreference} from "./types";
+
+export class BuiltThemeParser {
+    private _themeMapping: Record<string, ThemeInformation> = {};
+    private _preferredColorScheme?: ColorSchemePreference;
+
+    constructor(preferredColorScheme?: ColorSchemePreference) {
+        this._preferredColorScheme = preferredColorScheme;
+    }
+
+    parse(manifest: ThemeManifest, manifestLocation: string, log: ILogItem) {
+        log.wrap("BuiltThemeParser.parse", () => {
+            /*
+            After build has finished, the source section of each theme manifest
+            contains `built-assets` which is a mapping from the theme-id to
+            cssLocation of theme
+            */
+            const builtAssets: Record<string, string> = manifest.source?.["built-assets"];
+            const themeName = manifest.name;
+            if (!themeName) {
+                throw new Error(`Theme name not found in manifest at ${manifestLocation}`);
+            }
+            let defaultDarkVariant: any = {}, defaultLightVariant: any = {};
+            for (let [themeId, cssLocation] of Object.entries(builtAssets)) {
+                try {
+                    /**
+                     * This cssLocation is relative to the location of the manifest file.
+                     * So we first need to resolve it relative to the root of this hydrogen instance.
+                     */
+                    cssLocation = new URL(cssLocation, new URL(manifestLocation, window.location.origin)).href;
+                }
+                catch {
+                    continue;
+                }
+                const variant = themeId.match(/.+-(.+)/)?.[1];
+                const variantDetails = manifest.values?.variants[variant!];
+                if (!variantDetails) {
+                    throw new Error(`Variant ${variant} is missing in manifest at ${manifestLocation}`);
+                }
+                const { name: variantName, default: isDefault, dark } = variantDetails;
+                const themeDisplayName = `${themeName} ${variantName}`;
+                if (isDefault) {
+                    /**
+                     * This is a default variant!
+                     * We'll add these to the themeMapping (separately) keyed with just the
+                     * theme-name (i.e "Element" instead of "Element Dark").
+                     * We need to be able to distinguish them from other variants!
+                     * 
+                     * This allows us to render radio-buttons with "dark" and
+                     * "light" options.
+                    */
+                    const defaultVariant = dark ? defaultDarkVariant : defaultLightVariant;
+                    defaultVariant.variantName = variantName;
+                    defaultVariant.id = themeId
+                    defaultVariant.cssLocation = cssLocation;
+                    continue;
+                }
+                // Non-default variants are keyed in themeMapping with "theme_name variant_name"
+                // eg: "Element Dark"
+                this._themeMapping[themeDisplayName] = {
+                    cssLocation,
+                    id: themeId
+                };
+            }
+            if (defaultDarkVariant.id && defaultLightVariant.id) {
+                /**
+                 * As mentioned above, if there's both a default dark and a default light variant,
+                 * add them to themeMapping separately.
+                 */
+                const defaultVariant = this._preferredColorScheme === ColorSchemePreference.Dark ? defaultDarkVariant : defaultLightVariant;
+                this._themeMapping[themeName] = { dark: defaultDarkVariant, light: defaultLightVariant, default: defaultVariant };
+            }
+            else {
+                /**
+                 * If only one default variant is found (i.e only dark default or light default but not both),
+                 * treat it like any other variant.
+                 */
+                const variant = defaultDarkVariant.id ? defaultDarkVariant : defaultLightVariant;
+                this._themeMapping[`${themeName} ${variant.variantName}`] = { id: variant.id, cssLocation: variant.cssLocation };
+            }
+         });
+    }
+
+    get themeMapping(): Record<string, ThemeInformation> {
+        return this._themeMapping;
+    }
+}

src/platform/web/theming/parsers/RuntimeThemeParser.ts

@@ -0,0 +1,98 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+import type {ThemeInformation} from "./types";
+import type {Platform} from "../../Platform.js";
+import type {ThemeManifest} from "../../../types/theme";
+import {ColorSchemePreference} from "./types";
+import {IconColorizer} from "../IconColorizer";
+import {DerivedVariables} from "../DerivedVariables";
+import {ILogItem} from "../../../../logging/types";
+
+export class RuntimeThemeParser {
+    private _themeMapping: Record<string, ThemeInformation> = {};
+    private _preferredColorScheme?: ColorSchemePreference;
+    private _platform: Platform;
+
+    constructor(platform: Platform, preferredColorScheme?: ColorSchemePreference) {
+        this._preferredColorScheme = preferredColorScheme;
+        this._platform = platform;
+    }
+
+    async parse(manifest: ThemeManifest, baseManifest: ThemeManifest, baseManifestLocation: string, log: ILogItem): Promise<void> {
+        await log.wrap("RuntimeThemeParser.parse", async () => {
+            const {cssLocation, derivedVariables, icons} = this._getSourceData(baseManifest, baseManifestLocation, log);
+            const themeName = manifest.name;
+            if (!themeName) {
+                throw new Error(`Theme name not found in manifest!`);
+            }
+            let defaultDarkVariant: any = {}, defaultLightVariant: any = {};
+            for (const [variant, variantDetails] of Object.entries(manifest.values?.variants!) as [string, any][]) {
+                try {
+                    const themeId = `${manifest.id}-${variant}`;
+                    const { name: variantName, default: isDefault, dark, variables } = variantDetails;
+                    const resolvedVariables = new DerivedVariables(variables, derivedVariables, dark).toVariables();
+                    Object.assign(variables, resolvedVariables);
+                    const iconVariables = await new IconColorizer(this._platform, icons, variables, baseManifestLocation).toVariables();
+                    Object.assign(variables, resolvedVariables, iconVariables);
+                    const themeDisplayName = `${themeName} ${variantName}`;
+                    if (isDefault) {
+                        const defaultVariant = dark ? defaultDarkVariant : defaultLightVariant;
+                        Object.assign(defaultVariant, { variantName, id: themeId, cssLocation, variables });
+                        continue;
+                    }
+                    this._themeMapping[themeDisplayName] = { cssLocation, id: themeId, variables: variables, };
+                }
+                catch (e) {
+                    console.error(e);
+                    continue;
+                }
+            }
+            if (defaultDarkVariant.id && defaultLightVariant.id) {
+                const defaultVariant = this._preferredColorScheme === ColorSchemePreference.Dark ? defaultDarkVariant : defaultLightVariant;
+                this._themeMapping[themeName] = { dark: defaultDarkVariant, light: defaultLightVariant, default: defaultVariant };
+            }
+            else {
+                const variant = defaultDarkVariant.id ? defaultDarkVariant : defaultLightVariant;
+                this._themeMapping[`${themeName} ${variant.variantName}`] = { id: variant.id, cssLocation: variant.cssLocation };
+            }
+        });
+    }
+
+    private _getSourceData(manifest: ThemeManifest, location: string, log: ILogItem)
+        : { cssLocation: string, derivedVariables: string[], icons: Record<string, string>} {
+        return log.wrap("getSourceData", () => {
+            const runtimeCSSLocation = manifest.source?.["runtime-asset"];
+            if (!runtimeCSSLocation) {
+                throw new Error(`Run-time asset not found in source section for theme at ${location}`);
+            }
+            const cssLocation = new URL(runtimeCSSLocation, new URL(location, window.location.origin)).href;
+            const derivedVariables = manifest.source?.["derived-variables"];
+            if (!derivedVariables) {
+                throw new Error(`Derived variables not found in source section for theme at ${location}`);
+            }
+            const icons = manifest.source?.["icon"];
+            if (!icons) {
+                throw new Error(`Icon mapping not found in source section for theme at ${location}`);
+            }
+            return { cssLocation, derivedVariables, icons };
+         });
+    }
+
+    get themeMapping(): Record<string, ThemeInformation> {
+        return this._themeMapping;
+    }
+
+}

src/platform/web/theming/parsers/types.ts

@@ -0,0 +1,38 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export type NormalVariant = {
+    id: string;
+    cssLocation: string;
+    variables?: any;
+};
+
+export type Variant = NormalVariant & {
+    variantName: string;
+};
+
+export type DefaultVariant = {
+    dark: Variant;
+    light: Variant;
+    default: Variant;
+}
+
+export type ThemeInformation = NormalVariant | DefaultVariant; 
+
+export enum ColorSchemePreference {
+    Dark,
+    Light
+};

src/platform/web/theming/shared/color.mjs

@@ -13,10 +13,10 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
+import * as pkg from 'off-color';
+const offColor = pkg.offColor ?? pkg.default.offColor;
 
-const offColor = require("off-color").offColor;
+export function derive(value, operation, argument, isDark) {
-
-module.exports.derive = function (value, operation, argument, isDark) {
     const argumentAsNumber = parseInt(argument);
     if (isDark) {
         // For dark themes, invert the operation

src/platform/web/theming/shared/svg-colorizer.mjs

@@ -0,0 +1,24 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export function getColoredSvgString(svgString, primaryColor, secondaryColor) {
+    let coloredSVGCode = svgString.replaceAll("#ff00ff", primaryColor);
+    coloredSVGCode = coloredSVGCode.replaceAll("#00ffff", secondaryColor);
+    if (svgString === coloredSVGCode) {
+        throw new Error("svg-colorizer made no color replacements! The input svg should only contain colors #ff00ff (primary, case-sensitive) and #00ffff (secondary, case-sensitive).");
+    }
+    return coloredSVGCode;
+}

vite.common-config.js

@@ -8,8 +8,8 @@ const path = require("path");
 const manifest = require("./package.json");
 const version = manifest.version;
 const compiledVariables = new Map();
-const derive = require("./scripts/postcss/color").derive;
+import {buildColorizedSVG as replacer} from "./scripts/postcss/svg-builder.mjs";
-const replacer = require("./scripts/postcss/svg-colorizer").buildColorizedSVG;
+import {derive} from "./src/platform/web/theming/shared/color.mjs";
 
 const commonOptions = {
     logLevel: "warn",

vite.sdk-assets-config.js

@@ -36,7 +36,7 @@ export default mergeOptions(commonOptions, {
     plugins: [
         themeBuilder({
             themeConfig: {
-                themes: { element: "./src/platform/web/ui/css/themes/element" },
+                themes: ["./src/platform/web/ui/css/themes/element"],
                 default: "element",
             },
             compiledVariables,

yarn.lock

@@ -77,6 +77,11 @@
     "@nodelib/fs.scandir" "2.1.5"
     fastq "^1.6.0"
 
+"@trysound/sax@0.2.0":
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/@trysound/sax/-/sax-0.2.0.tgz#cccaab758af56761eb7bf37af6f03f326dd798ad"
+  integrity sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==
+
 "@types/json-schema@^7.0.7":
   version "7.0.9"
   resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.9.tgz#97edc9037ea0c38585320b28964dde3b39e4660d"
@@ -347,6 +352,11 @@ commander@^6.1.0:
   resolved "https://registry.yarnpkg.com/commander/-/commander-6.2.1.tgz#0792eb682dfbc325999bb2b84fddddba110ac73c"
   integrity sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==
 
+commander@^7.2.0:
+  version "7.2.0"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7"
+  integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==
+
 concat-map@0.0.1:
   version "0.0.1"
   resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
@@ -382,11 +392,26 @@ css-select@^4.1.3:
     domutils "^2.6.0"
     nth-check "^2.0.0"
 
+css-tree@^1.1.2, css-tree@^1.1.3:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/css-tree/-/css-tree-1.1.3.tgz#eb4870fb6fd7707327ec95c2ff2ab09b5e8db91d"
+  integrity sha512-tRpdppF7TRazZrjJ6v3stzv93qxRcSsFmW6cX0Zm2NVKpxE1WV1HblnghVv9TreireHkqI/VDEsfolRF1p6y7Q==
+  dependencies:
+    mdn-data "2.0.14"
+    source-map "^0.6.1"
+
 css-what@^5.0.0:
   version "5.0.1"
   resolved "https://registry.yarnpkg.com/css-what/-/css-what-5.0.1.tgz#3efa820131f4669a8ac2408f9c32e7c7de9f4cad"
   integrity sha512-FYDTSHb/7KXsWICVsxdmiExPjCfRC4qRFBdVwv7Ax9hMnvMmEjP9RfxTEZ3qPZGmADDn2vAKSo9UcN1jKVYscg==
 
+csso@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/csso/-/csso-4.2.0.tgz#ea3a561346e8dc9f546d6febedd50187cf389529"
+  integrity sha512-wvlcdIbf6pwKEk7vHj8/Bkc0B4ylXZruLvOgs9doS5eOsOpuodOV2zJChSpkp+pRpYQLQMeF04nr3Z68Sta9jA==
+  dependencies:
+    css-tree "^1.1.2"
+
 cuint@^0.2.2:
   version "0.2.2"
   resolved "https://registry.yarnpkg.com/cuint/-/cuint-0.2.2.tgz#408086d409550c2631155619e9fa7bcadc3b991b"
@@ -1197,6 +1222,11 @@ lru-cache@^6.0.0:
   dependencies:
     yallist "^4.0.0"
 
+mdn-data@2.0.14:
+  version "2.0.14"
+  resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-2.0.14.tgz#7113fc4281917d63ce29b43446f701e68c25ba50"
+  integrity sha512-dn6wd0uw5GsdswPFfsgMp5NSB0/aDe6fK94YJV/AJDYXL6HVLWBsxeq7js7Ad+mU2K9LAlwpk6kN2D5mwCPVow==
+
 mdn-polyfills@^5.20.0:
   version "5.20.0"
   resolved "https://registry.yarnpkg.com/mdn-polyfills/-/mdn-polyfills-5.20.0.tgz#ca8247edf20a4f60dec6804372229812b348260b"
@@ -1500,7 +1530,7 @@ source-map-js@^1.0.2:
   resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.0.2.tgz#adbc361d9c62df380125e7f161f71c826f1e490c"
   integrity sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==
 
-source-map@~0.6.1:
+source-map@^0.6.1, source-map@~0.6.1:
   version "0.6.1"
   resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263"
   integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==
@@ -1510,6 +1540,11 @@ sprintf-js@~1.0.2:
   resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"
   integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=
 
+stable@^0.1.8:
+  version "0.1.8"
+  resolved "https://registry.yarnpkg.com/stable/-/stable-0.1.8.tgz#836eb3c8382fe2936feaf544631017ce7d47a3cf"
+  integrity sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w==
+
 string-width@^4.2.0:
   version "4.2.2"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.2.tgz#dafd4f9559a7585cfba529c6a0a4f73488ebd4c5"
@@ -1550,6 +1585,19 @@ supports-preserve-symlinks-flag@^1.0.0:
   resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09"
   integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==
 
+svgo@^2.8.0:
+  version "2.8.0"
+  resolved "https://registry.yarnpkg.com/svgo/-/svgo-2.8.0.tgz#4ff80cce6710dc2795f0c7c74101e6764cfccd24"
+  integrity sha512-+N/Q9kV1+F+UeWYoSiULYo4xYSDQlTgb+ayMobAXPwMnLvop7oxKMo9OzIrX5x3eS4L4f2UHhc9axXwY8DpChg==
+  dependencies:
+    "@trysound/sax" "0.2.0"
+    commander "^7.2.0"
+    css-select "^4.1.3"
+    css-tree "^1.1.3"
+    csso "^4.2.0"
+    picocolors "^1.0.0"
+    stable "^0.1.8"
+
 table@^6.0.9:
   version "6.7.1"
   resolved "https://registry.yarnpkg.com/table/-/table-6.7.1.tgz#ee05592b7143831a8c94f3cee6aae4c1ccef33e2"
@@ -1617,10 +1665,10 @@ type-fest@^0.20.2:
   resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4"
   integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==
 
-typescript@^4.3.5:
+typescript@^4.7.0:
-  version "4.3.5"
+  version "4.7.4"
-  resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.3.5.tgz#4d1c37cc16e893973c45a06886b7113234f119f4"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.7.4.tgz#1a88596d1cf47d59507a1bcdfb5b9dfe4d488235"
-  integrity sha512-DqQgihaQ9cUrskJo9kIyW/+g0Vxsk8cDtZ52a3NGh0YNTfpUSArXSohyUGnvbPazEPLu398C0UxmKSOrPumUzA==
+  integrity sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==
 
 typeson-registry@^1.0.0-alpha.20:
   version "1.0.0-alpha.39"