Merge pull request #13 from vector-im/bwindels/cache-invalidation

Add content hashes to all immutable assets

package.json

@@ -29,6 +29,7 @@
     "@rollup/plugin-multi-entry": "^3.0.1",
     "@rollup/plugin-node-resolve": "^8.4.0",
     "cheerio": "^1.0.0-rc.3",
+    "commander": "^6.0.0",
     "core-js": "^3.6.5",
     "finalhandler": "^1.1.1",
     "impunity": "^0.0.11",
@@ -37,8 +38,10 @@
     "postcss-css-variables": "^0.17.0",
     "postcss-flexbugs-fixes": "^4.2.1",
     "postcss-import": "^12.0.1",
+    "postcss-url": "^8.0.0",
     "regenerator-runtime": "^0.13.7",
     "rollup": "^1.15.6",
-    "serve-static": "^1.13.2"
+    "serve-static": "^1.13.2",
+    "xxhash": "^0.3.0"
   }
 }

scripts/build.mjs

@@ -19,11 +19,13 @@ import cheerio from "cheerio";
 import fsRoot from "fs";
 const fs = fsRoot.promises;
 import path from "path";
+import XXHash from 'xxhash';
 import rollup from 'rollup';
 import postcss from "postcss";
 import postcssImport from "postcss-import";
 import { fileURLToPath } from 'url';
 import { dirname } from 'path';
+import commander from "commander";
 // needed for legacy bundle
 import babel from '@rollup/plugin-babel';
 // needed to find the polyfill modules in the main-legacy.js bundle
@@ -32,6 +34,8 @@ import { nodeResolve } from '@rollup/plugin-node-resolve';
 import commonjs from '@rollup/plugin-commonjs';
 // multi-entry plugin so we can add polyfill file to main
 import multi from '@rollup/plugin-multi-entry';
+// replace urls of asset names with content hashed version
+import postcssUrl from "postcss-url";
 
 import cssvariables from "postcss-css-variables";
 import flexbugsFixes from "postcss-flexbugs-fixes";
@@ -43,32 +47,20 @@ const PROJECT_NAME = "Hydrogen Chat";
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 const projectDir = path.join(__dirname, "../");
-const cssDir = path.join(projectDir, "src/ui/web/css/");
+const cssSrcDir = path.join(projectDir, "src/ui/web/css/");
-const targetDir = path.join(projectDir, "target");
+const targetDir = path.join(projectDir, "target/");
 
-const {debug, noOffline, legacy} = process.argv.reduce((params, param) => {
+const program = new commander.Command();
-    if (param.startsWith("--")) {
+program
-        params[param.substr(2)] = true;
+    .option("--legacy", "make a build for IE11")
-    }
+    .option("--no-offline", "make a build without a service worker or appcache manifest")
-    return params;
+program.parse(process.argv);
-}, {
+const {debug, noOffline, legacy} = program;
-    debug: false,
-    noOffline: false,
-    legacy: false
-});
 const offline = !noOffline;
 
 async function build() {
     // get version number
     const version = JSON.parse(await fs.readFile(path.join(projectDir, "package.json"), "utf8")).version;
-    // clear target dir
-    await removeDirIfExists(targetDir);
-    await fs.mkdir(targetDir);
-    let bundleName = `${PROJECT_ID}.js`;
-    if (legacy) {
-        bundleName = `${PROJECT_ID}-legacy.js`;
-    }
-
 
     const devHtml = await fs.readFile(path.join(projectDir, "index.html"), "utf8");
     const doc = cheerio.load(devHtml);
@@ -76,25 +68,41 @@ async function build() {
     findThemes(doc, themeName => {
         themes.push(themeName);
     });
-
+    // clear target dir
+    await removeDirIfExists(targetDir);
+    await createDirs(targetDir, themes);
     // also creates the directories where the theme css bundles are placed in,
     // so do it first
     const themeAssets = await copyThemeAssets(themes, legacy);
+    const jsBundlePath = await (legacy ? buildJsLegacy() : buildJs());
+    const cssBundlePaths = await buildCssBundles(legacy ? buildCssLegacy : buildCss, themes, themeAssets);
+    const assetPaths = createAssetPaths(jsBundlePath, cssBundlePaths, themeAssets);
 
-    await buildHtml(doc, version, bundleName);
+    let manifestPath;
-    if (legacy) {
-        await buildJsLegacy(bundleName);
-    } else {
-        await buildJs(bundleName);
-    }
-    await buildCssBundles(legacy ? buildCssLegacy : buildCss, themes);
     if (offline) {
-        await buildOffline(version, bundleName, themeAssets);
+        manifestPath = await buildOffline(version, assetPaths);
     }
+    await buildHtml(doc, version, assetPaths, manifestPath);
 
     console.log(`built ${PROJECT_ID}${legacy ? " legacy" : ""} ${version} successfully`);
 }
 
+function createAssetPaths(jsBundlePath, cssBundlePaths, themeAssets) {
+    function trim(path) {
+        if (!path.startsWith(targetDir)) {
+            throw new Error("invalid target path: " + targetDir);
+        }
+        return path.substr(targetDir.length);
+    }
+    return {
+        jsBundle: () => trim(jsBundlePath),
+        cssMainBundle: () => trim(cssBundlePaths.main),
+        cssThemeBundle: themeName => trim(cssBundlePaths.themes[themeName]),
+        cssThemeBundles: () => Object.values(cssBundlePaths.themes).map(a => trim(a)),
+        otherAssets: () => Object.values(themeAssets).map(a => trim(a))
+    };
+}
+
 async function findThemes(doc, callback) {
     doc("link[rel~=stylesheet][title]").each((i, el) => {
         const theme = doc(el);
@@ -110,37 +118,39 @@ async function findThemes(doc, callback) {
     });
 }
 
-async function copyThemeAssets(themes, legacy) {
+async function createDirs(targetDir, themes) {
-    const assets = [];
+    await fs.mkdir(targetDir);
-    // create theme directories and copy assets
+    const themeDir = path.join(targetDir, "themes");
-    await fs.mkdir(path.join(targetDir, "themes"));
+    await fs.mkdir(themeDir);
+    for (const theme of themes) {
+        await fs.mkdir(path.join(themeDir, theme));
+    }
+}
+
+async function copyThemeAssets(themes, legacy) {
+    const assets = {};
     for (const theme of themes) {
-        assets.push(`themes/${theme}/bundle.css`);
         const themeDstFolder = path.join(targetDir, `themes/${theme}`);
-        await fs.mkdir(themeDstFolder);
+        const themeSrcFolder = path.join(cssSrcDir, `themes/${theme}`);
-        const themeSrcFolder = path.join(cssDir, `themes/${theme}`);
+        const themeAssets = await copyFolder(themeSrcFolder, themeDstFolder, file => {
-        await copyFolder(themeSrcFolder, themeDstFolder, file => {
             const isUnneededFont = legacy ? file.endsWith(".woff2") : file.endsWith(".woff");
-            if (!file.endsWith(".css") && !isUnneededFont) {
+            return !file.endsWith(".css") && !isUnneededFont;
-                assets.push(file.substr(cssDir.length));
-                return true;
-            }
-            return false;
         });
+        Object.assign(assets, themeAssets);
     }
     return assets;
 }
 
-async function buildHtml(doc, version, bundleName) {
+async function buildHtml(doc, version, assetPaths, manifestPath) {
     // transform html file
     // change path to main.css to css bundle
-    doc("link[rel=stylesheet]:not([title])").attr("href", `${PROJECT_ID}.css`);
+    doc("link[rel=stylesheet]:not([title])").attr("href", assetPaths.cssMainBundle());
     // change paths to all theme stylesheets
     findThemes(doc, (themeName, theme) => {
-        theme.attr("href", `themes/${themeName}/bundle.css`);
+        theme.attr("href", assetPaths.cssThemeBundle(themeName));
     });
     doc("script#main").replaceWith(
-        `<script type="text/javascript" src="${bundleName}"></script>` +
+        `<script type="text/javascript" src="${assetPaths.jsBundle()}"></script>` +
         `<script type="text/javascript">${PROJECT_ID}Bundle.main(document.body);</script>`);
     removeOrEnableScript(doc("script#service-worker"), offline);
 
@@ -152,22 +162,25 @@ async function buildHtml(doc, version, bundleName) {
 
     if (offline) {
         doc("html").attr("manifest", "manifest.appcache");
-        doc("head").append(`<link rel="manifest" href="manifest.json">`);
+        doc("head").append(`<link rel="manifest" href="${manifestPath.substr(targetDir.length)}">`);
     }
     await fs.writeFile(path.join(targetDir, "index.html"), doc.html(), "utf8");
 }
 
-async function buildJs(bundleName) {
+async function buildJs() {
     // create js bundle
     const bundle = await rollup.rollup({input: 'src/main.js'});
-    await bundle.write({
+    const {output} = await bundle.generate({
-        file: path.join(targetDir, bundleName),
         format: 'iife',
         name: `${PROJECT_ID}Bundle`
     });
+    const code = output[0].code;
+    const bundlePath = resource(`${PROJECT_ID}.js`, code);
+    await fs.writeFile(bundlePath, code, "utf8");
+    return bundlePath;
 }
 
-async function buildJsLegacy(bundleName) {
+async function buildJsLegacy() {
     // compile down to whatever IE 11 needs
     const babelPlugin = babel.babel({
         babelHelpers: 'bundled',
@@ -189,24 +202,24 @@ async function buildJsLegacy(bundleName) {
         plugins: [multi(), commonjs(), nodeResolve(), babelPlugin]
     };
     const bundle = await rollup.rollup(rollupConfig);
-    await bundle.write({
+    const {output} = await bundle.generate({
-        file: path.join(targetDir, bundleName),
         format: 'iife',
         name: `${PROJECT_ID}Bundle`
     });
+    const code = output[0].code;
+    const bundlePath = resource(`${PROJECT_ID}-legacy.js`, code);
+    await fs.writeFile(bundlePath, code, "utf8");
+    return bundlePath;
 }
 
-async function buildOffline(version, bundleName, themeAssets) {
+async function buildOffline(version, assetPaths) {
-    const {offlineAssets, cacheAssets} = themeAssets.reduce((result, asset) => {
-        if (asset.endsWith(".css")) {
-            result.offlineAssets.push(asset);
-        } else {
-            result.cacheAssets.push(asset);
-        }
-        return result;
-    }, {offlineAssets: [], cacheAssets: []});
     // write offline availability
-    const offlineFiles = [bundleName, `${PROJECT_ID}.css`, "index.html", "icon-192.png"].concat(offlineAssets);
+    const offlineFiles = [
+        assetPaths.jsBundle(),
+        assetPaths.cssMainBundle(),
+        "index.html",
+        "icon-192.png",
+    ].concat(assetPaths.cssThemeBundles());
 
     // write appcache manifest
     const manifestLines = [
@@ -223,7 +236,7 @@ async function buildOffline(version, bundleName, themeAssets) {
     let swSource = await fs.readFile(path.join(projectDir, "src/service-worker.template.js"), "utf8");
     swSource = swSource.replace(`"%%VERSION%%"`, `"${version}"`);
     swSource = swSource.replace(`"%%OFFLINE_FILES%%"`, JSON.stringify(offlineFiles));
-    swSource = swSource.replace(`"%%CACHE_FILES%%"`, JSON.stringify(cacheAssets));
+    swSource = swSource.replace(`"%%CACHE_FILES%%"`, JSON.stringify(assetPaths.otherAssets()));
     await fs.writeFile(path.join(targetDir, "sw.js"), swSource, "utf8");
     // write web manifest
     const webManifest = {
@@ -233,35 +246,61 @@ async function buildOffline(version, bundleName, themeAssets) {
         start_url: "index.html",
         icons: [{"src": "icon-192.png", "sizes": "192x192", "type": "image/png"}],
     };
-    await fs.writeFile(path.join(targetDir, "manifest.json"), JSON.stringify(webManifest), "utf8");
+    const manifestJson = JSON.stringify(webManifest);
+    const manifestPath = resource("manifest.json", manifestJson);
+    await fs.writeFile(manifestPath, manifestJson, "utf8");
     // copy icon
+    // should this icon have a content hash as well?
     let icon = await fs.readFile(path.join(projectDir, "icon.png"));
     await fs.writeFile(path.join(targetDir, "icon-192.png"), icon);
+    return manifestPath;
 }
 
-async function buildCssBundles(buildFn, themes) {
+async function buildCssBundles(buildFn, themes, themeAssets) {
-    const cssMainFile = path.join(cssDir, "main.css");
+    const bundleCss = await buildFn(path.join(cssSrcDir, "main.css"));
-    await buildFn(cssMainFile, path.join(targetDir, `${PROJECT_ID}.css`));
+    const mainDstPath = resource(`${PROJECT_ID}.css`, bundleCss);
+    await fs.writeFile(mainDstPath, bundleCss, "utf8");
+    const bundlePaths = {main: mainDstPath, themes: {}};
     for (const theme of themes) {
-        await buildFn(
+        const urlBase = path.join(targetDir, `themes/${theme}/`);
-            path.join(cssDir, `themes/${theme}/theme.css`),
+        const assetUrlMapper = ({absolutePath}) => {
-            path.join(targetDir, `themes/${theme}/bundle.css`)
+            const hashedDstPath = themeAssets[absolutePath];
-        );
+            if (hashedDstPath && hashedDstPath.startsWith(urlBase)) {
+                return hashedDstPath.substr(urlBase.length);
+            }
+        };
+        const themeCss = await buildFn(path.join(cssSrcDir, `themes/${theme}/theme.css`), assetUrlMapper);
+        const themeDstPath = resource(`themes/${theme}/bundle.css`, themeCss);
+        await fs.writeFile(themeDstPath, themeCss, "utf8");
+        bundlePaths.themes[theme] = themeDstPath;
     }
+    return bundlePaths;
 }
 
-async function buildCss(entryPath, bundlePath) {
+async function buildCss(entryPath, urlMapper = null) {
     const preCss = await fs.readFile(entryPath, "utf8");
-    const cssBundler = postcss([postcssImport]);
+    const options = [postcssImport];
+    if (urlMapper) {
+        options.push(postcssUrl({url: urlMapper}));
+    }
+    const cssBundler = postcss(options);
     const result = await cssBundler.process(preCss, {from: entryPath});
-    await fs.writeFile(bundlePath, result.css, "utf8");
+    return result.css;
 }
 
-async function buildCssLegacy(entryPath, bundlePath) {
+async function buildCssLegacy(entryPath, urlMapper = null) {
     const preCss = await fs.readFile(entryPath, "utf8");
-    const cssBundler = postcss([postcssImport, cssvariables(), flexbugsFixes()]);
+    const options = [
+        postcssImport,
+        cssvariables(),
+        flexbugsFixes()
+    ];
+    if (urlMapper) {
+        options.push(postcssUrl({url: urlMapper}));
+    }
+    const cssBundler = postcss(options);
     const result = await cssBundler.process(preCss, {from: entryPath});
-    await fs.writeFile(bundlePath, result.css, "utf8");
+    return result.css;
 }
 
 function removeOrEnableScript(scriptNode, enable) {
@@ -283,17 +322,41 @@ async function removeDirIfExists(targetDir) {
 }
 
 async function copyFolder(srcRoot, dstRoot, filter) {
+    const assetPaths = {};
     const dirEnts = await fs.readdir(srcRoot, {withFileTypes: true});
     for (const dirEnt of dirEnts) {
         const dstPath = path.join(dstRoot, dirEnt.name);
         const srcPath = path.join(srcRoot, dirEnt.name);
         if (dirEnt.isDirectory()) {
             await fs.mkdir(dstPath);
-            await copyFolder(srcPath, dstPath, filter);
+            Object.assign(assetPaths, await copyFolder(srcPath, dstPath, filter));
         } else if (dirEnt.isFile() && filter(srcPath)) {
-            await fs.copyFile(srcPath, dstPath);
+            const content = await fs.readFile(srcPath);
+            const hashedDstPath = resource(dstPath, content);
+            await fs.writeFile(hashedDstPath, content);
+            assetPaths[srcPath] = hashedDstPath;
         }
     }
+    return assetPaths;
 }
 
+function resource(relPath, content) {
+    let fullPath = relPath;
+    if (!relPath.startsWith("/")) {
+        fullPath = path.join(targetDir, relPath);
+    }
+    const hash = contentHash(Buffer.from(content));
+    const dir = path.dirname(fullPath);
+    const extname = path.extname(fullPath);
+    const basename = path.basename(fullPath, extname);
+    return path.join(dir, `${basename}-${hash}${extname}`);
+}
+
+function contentHash(str) {
+    var hasher = new XXHash(0);
+    hasher.update(str);
+    return hasher.digest();
+}
+
+
 build().catch(err => console.error(err));