From 661bd65229780875e9c55d7cd251b205b6b9ed71 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Thu, 13 Aug 2020 08:09:30 +0000 Subject: [PATCH 1/4] Update README.md clarify status and external input --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a02e6cd4..aabdf89a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Hydrogen -A minimal [Matrix](https://matrix.org/) chat client, focused on performance, offline functionality, and broad browser support. +A minimal [Matrix](https://matrix.org/) chat client, focused on performance, offline functionality, and broad browser support. This is work in progress and not yet ready for primetime. We're currently not accepting any externally reported issues (features, bug reports, ...) at this time. ## Goals From 044360afaa4ddd58b6dedd101267819e08ec9e1e Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Thu, 13 Aug 2020 18:59:31 +0200 Subject: [PATCH 2/4] add content hashes to build assets --- package.json | 4 +- scripts/build.mjs | 184 ++++++++++++++++++++++++++++------------------ 2 files changed, 114 insertions(+), 74 deletions(-) diff --git a/package.json b/package.json index f44d51a6..c43220c7 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "@rollup/plugin-multi-entry": "^3.0.1", "@rollup/plugin-node-resolve": "^8.4.0", "cheerio": "^1.0.0-rc.3", + "commander": "^6.0.0", "core-js": "^3.6.5", "finalhandler": "^1.1.1", "impunity": "^0.0.11", @@ -39,6 +40,7 @@ "postcss-import": "^12.0.1", "regenerator-runtime": "^0.13.7", "rollup": "^1.15.6", - "serve-static": "^1.13.2" + "serve-static": "^1.13.2", + "xxhash": "^0.3.0" } } diff --git a/scripts/build.mjs b/scripts/build.mjs index 611738cb..30189714 100644 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -19,11 +19,13 @@ import cheerio from "cheerio"; import fsRoot from "fs"; const fs = fsRoot.promises; import path from "path"; +import XXHash from 'xxhash'; import rollup from 'rollup'; import postcss from "postcss"; import postcssImport from "postcss-import"; import { fileURLToPath } from 'url'; import { dirname } from 'path'; +import commander from "commander"; // needed for legacy bundle import babel from '@rollup/plugin-babel'; // needed to find the polyfill modules in the main-legacy.js bundle @@ -43,32 +45,20 @@ const PROJECT_NAME = "Hydrogen Chat"; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); const projectDir = path.join(__dirname, "../"); -const cssDir = path.join(projectDir, "src/ui/web/css/"); -const targetDir = path.join(projectDir, "target"); +const cssSrcDir = path.join(projectDir, "src/ui/web/css/"); +const targetDir = path.join(projectDir, "target/"); -const {debug, noOffline, legacy} = process.argv.reduce((params, param) => { - if (param.startsWith("--")) { - params[param.substr(2)] = true; - } - return params; -}, { - debug: false, - noOffline: false, - legacy: false -}); +const program = new commander.Command(); +program + .option("--legacy", "make a build for IE11") + .option("--no-offline", "make a build without a service worker or appcache manifest") +program.parse(process.argv); +const {debug, noOffline, legacy} = program; const offline = !noOffline; async function build() { // get version number const version = JSON.parse(await fs.readFile(path.join(projectDir, "package.json"), "utf8")).version; - // clear target dir - await removeDirIfExists(targetDir); - await fs.mkdir(targetDir); - let bundleName = `${PROJECT_ID}.js`; - if (legacy) { - bundleName = `${PROJECT_ID}-legacy.js`; - } - const devHtml = await fs.readFile(path.join(projectDir, "index.html"), "utf8"); const doc = cheerio.load(devHtml); @@ -76,25 +66,40 @@ async function build() { findThemes(doc, themeName => { themes.push(themeName); }); - + // clear target dir + await removeDirIfExists(targetDir); + await createDirs(targetDir, themes); // also creates the directories where the theme css bundles are placed in, // so do it first const themeAssets = await copyThemeAssets(themes, legacy); + const jsBundlePath = await (legacy ? buildJsLegacy() : buildJs()); + const cssBundlePaths = await buildCssBundles(legacy ? buildCssLegacy : buildCss, themes); + const assetPaths = createAssetPaths(jsBundlePath, cssBundlePaths, themeAssets); - await buildHtml(doc, version, bundleName); - if (legacy) { - await buildJsLegacy(bundleName); - } else { - await buildJs(bundleName); - } - await buildCssBundles(legacy ? buildCssLegacy : buildCss, themes); if (offline) { - await buildOffline(version, bundleName, themeAssets); + await buildOffline(version, assetPaths); } + await buildHtml(doc, version, assetPaths); console.log(`built ${PROJECT_ID}${legacy ? " legacy" : ""} ${version} successfully`); } +function createAssetPaths(jsBundlePath, cssBundlePaths, themeAssets) { + function trim(path) { + if (!path.startsWith(targetDir)) { + throw new Error("invalid target path: " + targetDir); + } + return path.substr(targetDir.length); + } + return { + jsBundle: () => trim(jsBundlePath), + cssMainBundle: () => trim(cssBundlePaths.main), + cssThemeBundle: themeName => trim(cssBundlePaths.themes[themeName]), + cssThemeBundles: () => Object.values(cssBundlePaths.themes).map(a => trim(a)), + otherAssets: () => themeAssets.map(a => trim(a)) + }; +} + async function findThemes(doc, callback) { doc("link[rel~=stylesheet][title]").each((i, el) => { const theme = doc(el); @@ -110,37 +115,39 @@ async function findThemes(doc, callback) { }); } +async function createDirs(targetDir, themes) { + await fs.mkdir(targetDir); + const themeDir = path.join(targetDir, "themes"); + await fs.mkdir(themeDir); + for (const theme of themes) { + await fs.mkdir(path.join(themeDir, theme)); + } +} + async function copyThemeAssets(themes, legacy) { const assets = []; - // create theme directories and copy assets - await fs.mkdir(path.join(targetDir, "themes")); for (const theme of themes) { - assets.push(`themes/${theme}/bundle.css`); const themeDstFolder = path.join(targetDir, `themes/${theme}`); - await fs.mkdir(themeDstFolder); - const themeSrcFolder = path.join(cssDir, `themes/${theme}`); - await copyFolder(themeSrcFolder, themeDstFolder, file => { + const themeSrcFolder = path.join(cssSrcDir, `themes/${theme}`); + const themeAssets = await copyFolder(themeSrcFolder, themeDstFolder, file => { const isUnneededFont = legacy ? file.endsWith(".woff2") : file.endsWith(".woff"); - if (!file.endsWith(".css") && !isUnneededFont) { - assets.push(file.substr(cssDir.length)); - return true; - } - return false; + return !file.endsWith(".css") && !isUnneededFont; }); + assets.push(...themeAssets); } return assets; } -async function buildHtml(doc, version, bundleName) { +async function buildHtml(doc, version, assetPaths) { // transform html file // change path to main.css to css bundle - doc("link[rel=stylesheet]:not([title])").attr("href", `${PROJECT_ID}.css`); + doc("link[rel=stylesheet]:not([title])").attr("href", assetPaths.cssMainBundle()); // change paths to all theme stylesheets findThemes(doc, (themeName, theme) => { - theme.attr("href", `themes/${themeName}/bundle.css`); + theme.attr("href", assetPaths.cssThemeBundle(themeName)); }); doc("script#main").replaceWith( - `` + + `` + ``); removeOrEnableScript(doc("script#service-worker"), offline); @@ -157,17 +164,20 @@ async function buildHtml(doc, version, bundleName) { await fs.writeFile(path.join(targetDir, "index.html"), doc.html(), "utf8"); } -async function buildJs(bundleName) { +async function buildJs() { // create js bundle const bundle = await rollup.rollup({input: 'src/main.js'}); - await bundle.write({ - file: path.join(targetDir, bundleName), + const {output} = await bundle.generate({ format: 'iife', name: `${PROJECT_ID}Bundle` }); + const code = output[0].code; + const bundlePath = resource(`${PROJECT_ID}.js`, code); + await fs.writeFile(bundlePath, code, "utf8"); + return bundlePath; } -async function buildJsLegacy(bundleName) { +async function buildJsLegacy() { // compile down to whatever IE 11 needs const babelPlugin = babel.babel({ babelHelpers: 'bundled', @@ -189,24 +199,24 @@ async function buildJsLegacy(bundleName) { plugins: [multi(), commonjs(), nodeResolve(), babelPlugin] }; const bundle = await rollup.rollup(rollupConfig); - await bundle.write({ - file: path.join(targetDir, bundleName), + const {output} = await bundle.generate({ format: 'iife', name: `${PROJECT_ID}Bundle` }); + const code = output[0].code; + const bundlePath = resource(`${PROJECT_ID}-legacy.js`, code); + await fs.writeFile(bundlePath, code, "utf8"); + return bundlePath; } -async function buildOffline(version, bundleName, themeAssets) { - const {offlineAssets, cacheAssets} = themeAssets.reduce((result, asset) => { - if (asset.endsWith(".css")) { - result.offlineAssets.push(asset); - } else { - result.cacheAssets.push(asset); - } - return result; - }, {offlineAssets: [], cacheAssets: []}); +async function buildOffline(version, assetPaths) { // write offline availability - const offlineFiles = [bundleName, `${PROJECT_ID}.css`, "index.html", "icon-192.png"].concat(offlineAssets); + const offlineFiles = [ + assetPaths.jsBundle(), + assetPaths.cssMainBundle(), + "index.html", + "icon-192.png", + ].concat(assetPaths.cssThemeBundles()); // write appcache manifest const manifestLines = [ @@ -223,7 +233,7 @@ async function buildOffline(version, bundleName, themeAssets) { let swSource = await fs.readFile(path.join(projectDir, "src/service-worker.template.js"), "utf8"); swSource = swSource.replace(`"%%VERSION%%"`, `"${version}"`); swSource = swSource.replace(`"%%OFFLINE_FILES%%"`, JSON.stringify(offlineFiles)); - swSource = swSource.replace(`"%%CACHE_FILES%%"`, JSON.stringify(cacheAssets)); + swSource = swSource.replace(`"%%CACHE_FILES%%"`, JSON.stringify(assetPaths.otherAssets())); await fs.writeFile(path.join(targetDir, "sw.js"), swSource, "utf8"); // write web manifest const webManifest = { @@ -235,33 +245,37 @@ async function buildOffline(version, bundleName, themeAssets) { }; await fs.writeFile(path.join(targetDir, "manifest.json"), JSON.stringify(webManifest), "utf8"); // copy icon + // should this icon have a content hash as well? let icon = await fs.readFile(path.join(projectDir, "icon.png")); await fs.writeFile(path.join(targetDir, "icon-192.png"), icon); } async function buildCssBundles(buildFn, themes) { - const cssMainFile = path.join(cssDir, "main.css"); - await buildFn(cssMainFile, path.join(targetDir, `${PROJECT_ID}.css`)); + const bundleCss = await buildFn(path.join(cssSrcDir, "main.css")); + const mainDstPath = resource(`${PROJECT_ID}.css`, bundleCss); + await fs.writeFile(mainDstPath, bundleCss, "utf8"); + const bundlePaths = {main: mainDstPath, themes: {}}; for (const theme of themes) { - await buildFn( - path.join(cssDir, `themes/${theme}/theme.css`), - path.join(targetDir, `themes/${theme}/bundle.css`) - ); + const themeCss = await buildFn(path.join(cssSrcDir, `themes/${theme}/theme.css`)); + const themeDstPath = resource(`themes/${theme}/bundle.css`, themeCss); + await fs.writeFile(themeDstPath, themeCss, "utf8"); + bundlePaths.themes[theme] = themeDstPath; } + return bundlePaths; } -async function buildCss(entryPath, bundlePath) { +async function buildCss(entryPath) { const preCss = await fs.readFile(entryPath, "utf8"); const cssBundler = postcss([postcssImport]); const result = await cssBundler.process(preCss, {from: entryPath}); - await fs.writeFile(bundlePath, result.css, "utf8"); + return result.css; } -async function buildCssLegacy(entryPath, bundlePath) { +async function buildCssLegacy(entryPath) { const preCss = await fs.readFile(entryPath, "utf8"); const cssBundler = postcss([postcssImport, cssvariables(), flexbugsFixes()]); const result = await cssBundler.process(preCss, {from: entryPath}); - await fs.writeFile(bundlePath, result.css, "utf8"); + return result.css; } function removeOrEnableScript(scriptNode, enable) { @@ -283,17 +297,41 @@ async function removeDirIfExists(targetDir) { } async function copyFolder(srcRoot, dstRoot, filter) { + const assetPaths = []; const dirEnts = await fs.readdir(srcRoot, {withFileTypes: true}); for (const dirEnt of dirEnts) { const dstPath = path.join(dstRoot, dirEnt.name); const srcPath = path.join(srcRoot, dirEnt.name); if (dirEnt.isDirectory()) { await fs.mkdir(dstPath); - await copyFolder(srcPath, dstPath, filter); + assetPaths.push(... await copyFolder(srcPath, dstPath, filter)); } else if (dirEnt.isFile() && filter(srcPath)) { - await fs.copyFile(srcPath, dstPath); + const content = await fs.readFile(srcPath); + const hashedDstPath = resource(dstPath, content); + await fs.writeFile(hashedDstPath, content); + assetPaths.push(hashedDstPath); } } + return assetPaths; } +function resource(relPath, content) { + let fullPath = relPath; + if (!relPath.startsWith("/")) { + fullPath = path.join(targetDir, relPath); + } + const hash = contentHash(Buffer.from(content)); + const dir = path.dirname(fullPath); + const extname = path.extname(fullPath); + const basename = path.basename(fullPath, extname); + return path.join(dir, `${basename}-${hash}${extname}`); +} + +function contentHash(str) { + var hasher = new XXHash(0); + hasher.update(str); + return hasher.digest(); +} + + build().catch(err => console.error(err)); From 0104e14e0b6b905ff8fe5757154f05a6d43e762b Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 14 Aug 2020 10:45:14 +0200 Subject: [PATCH 3/4] map urls in theme css bundles to their content-hashed counterparts --- package.json | 1 + scripts/build.mjs | 47 ++++++++++++++++++++++++++++++++++------------- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/package.json b/package.json index c43220c7..a2531924 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "postcss-css-variables": "^0.17.0", "postcss-flexbugs-fixes": "^4.2.1", "postcss-import": "^12.0.1", + "postcss-url": "^8.0.0", "regenerator-runtime": "^0.13.7", "rollup": "^1.15.6", "serve-static": "^1.13.2", diff --git a/scripts/build.mjs b/scripts/build.mjs index 30189714..96fee0c8 100644 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -34,6 +34,8 @@ import { nodeResolve } from '@rollup/plugin-node-resolve'; import commonjs from '@rollup/plugin-commonjs'; // multi-entry plugin so we can add polyfill file to main import multi from '@rollup/plugin-multi-entry'; +// replace urls of asset names with content hashed version +import postcssUrl from "postcss-url"; import cssvariables from "postcss-css-variables"; import flexbugsFixes from "postcss-flexbugs-fixes"; @@ -73,7 +75,7 @@ async function build() { // so do it first const themeAssets = await copyThemeAssets(themes, legacy); const jsBundlePath = await (legacy ? buildJsLegacy() : buildJs()); - const cssBundlePaths = await buildCssBundles(legacy ? buildCssLegacy : buildCss, themes); + const cssBundlePaths = await buildCssBundles(legacy ? buildCssLegacy : buildCss, themes, themeAssets); const assetPaths = createAssetPaths(jsBundlePath, cssBundlePaths, themeAssets); if (offline) { @@ -96,7 +98,7 @@ function createAssetPaths(jsBundlePath, cssBundlePaths, themeAssets) { cssMainBundle: () => trim(cssBundlePaths.main), cssThemeBundle: themeName => trim(cssBundlePaths.themes[themeName]), cssThemeBundles: () => Object.values(cssBundlePaths.themes).map(a => trim(a)), - otherAssets: () => themeAssets.map(a => trim(a)) + otherAssets: () => Object.values(themeAssets).map(a => trim(a)) }; } @@ -125,7 +127,7 @@ async function createDirs(targetDir, themes) { } async function copyThemeAssets(themes, legacy) { - const assets = []; + const assets = {}; for (const theme of themes) { const themeDstFolder = path.join(targetDir, `themes/${theme}`); const themeSrcFolder = path.join(cssSrcDir, `themes/${theme}`); @@ -133,7 +135,7 @@ async function copyThemeAssets(themes, legacy) { const isUnneededFont = legacy ? file.endsWith(".woff2") : file.endsWith(".woff"); return !file.endsWith(".css") && !isUnneededFont; }); - assets.push(...themeAssets); + Object.assign(assets, themeAssets); } return assets; } @@ -250,13 +252,20 @@ async function buildOffline(version, assetPaths) { await fs.writeFile(path.join(targetDir, "icon-192.png"), icon); } -async function buildCssBundles(buildFn, themes) { +async function buildCssBundles(buildFn, themes, themeAssets) { const bundleCss = await buildFn(path.join(cssSrcDir, "main.css")); const mainDstPath = resource(`${PROJECT_ID}.css`, bundleCss); await fs.writeFile(mainDstPath, bundleCss, "utf8"); const bundlePaths = {main: mainDstPath, themes: {}}; for (const theme of themes) { - const themeCss = await buildFn(path.join(cssSrcDir, `themes/${theme}/theme.css`)); + const urlBase = path.join(targetDir, `themes/${theme}/`); + const assetUrlMapper = ({absolutePath}) => { + const hashedDstPath = themeAssets[absolutePath]; + if (hashedDstPath && hashedDstPath.startsWith(urlBase)) { + return hashedDstPath.substr(urlBase.length); + } + }; + const themeCss = await buildFn(path.join(cssSrcDir, `themes/${theme}/theme.css`), assetUrlMapper); const themeDstPath = resource(`themes/${theme}/bundle.css`, themeCss); await fs.writeFile(themeDstPath, themeCss, "utf8"); bundlePaths.themes[theme] = themeDstPath; @@ -264,16 +273,28 @@ async function buildCssBundles(buildFn, themes) { return bundlePaths; } -async function buildCss(entryPath) { +async function buildCss(entryPath, urlMapper = null) { const preCss = await fs.readFile(entryPath, "utf8"); - const cssBundler = postcss([postcssImport]); + const options = [postcssImport]; + if (urlMapper) { + options.push(postcssUrl({url: urlMapper})); + } + const cssBundler = postcss(options); const result = await cssBundler.process(preCss, {from: entryPath}); return result.css; } -async function buildCssLegacy(entryPath) { +async function buildCssLegacy(entryPath, urlMapper = null) { const preCss = await fs.readFile(entryPath, "utf8"); - const cssBundler = postcss([postcssImport, cssvariables(), flexbugsFixes()]); + const options = [ + postcssImport, + cssvariables(), + flexbugsFixes() + ]; + if (urlMapper) { + options.push(postcssUrl({url: urlMapper})); + } + const cssBundler = postcss(options); const result = await cssBundler.process(preCss, {from: entryPath}); return result.css; } @@ -297,19 +318,19 @@ async function removeDirIfExists(targetDir) { } async function copyFolder(srcRoot, dstRoot, filter) { - const assetPaths = []; + const assetPaths = {}; const dirEnts = await fs.readdir(srcRoot, {withFileTypes: true}); for (const dirEnt of dirEnts) { const dstPath = path.join(dstRoot, dirEnt.name); const srcPath = path.join(srcRoot, dirEnt.name); if (dirEnt.isDirectory()) { await fs.mkdir(dstPath); - assetPaths.push(... await copyFolder(srcPath, dstPath, filter)); + Object.assign(assetPaths, await copyFolder(srcPath, dstPath, filter)); } else if (dirEnt.isFile() && filter(srcPath)) { const content = await fs.readFile(srcPath); const hashedDstPath = resource(dstPath, content); await fs.writeFile(hashedDstPath, content); - assetPaths.push(hashedDstPath); + assetPaths[srcPath] = hashedDstPath; } } return assetPaths; From 3c4805b2678b0958aa03bfd8297a61eb9f87bf63 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 14 Aug 2020 11:06:39 +0200 Subject: [PATCH 4/4] also content-hash the webapp manifest --- scripts/build.mjs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/build.mjs b/scripts/build.mjs index 96fee0c8..5ddcbafa 100644 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -78,10 +78,11 @@ async function build() { const cssBundlePaths = await buildCssBundles(legacy ? buildCssLegacy : buildCss, themes, themeAssets); const assetPaths = createAssetPaths(jsBundlePath, cssBundlePaths, themeAssets); + let manifestPath; if (offline) { - await buildOffline(version, assetPaths); + manifestPath = await buildOffline(version, assetPaths); } - await buildHtml(doc, version, assetPaths); + await buildHtml(doc, version, assetPaths, manifestPath); console.log(`built ${PROJECT_ID}${legacy ? " legacy" : ""} ${version} successfully`); } @@ -140,7 +141,7 @@ async function copyThemeAssets(themes, legacy) { return assets; } -async function buildHtml(doc, version, assetPaths) { +async function buildHtml(doc, version, assetPaths, manifestPath) { // transform html file // change path to main.css to css bundle doc("link[rel=stylesheet]:not([title])").attr("href", assetPaths.cssMainBundle()); @@ -161,7 +162,7 @@ async function buildHtml(doc, version, assetPaths) { if (offline) { doc("html").attr("manifest", "manifest.appcache"); - doc("head").append(``); + doc("head").append(``); } await fs.writeFile(path.join(targetDir, "index.html"), doc.html(), "utf8"); } @@ -245,11 +246,14 @@ async function buildOffline(version, assetPaths) { start_url: "index.html", icons: [{"src": "icon-192.png", "sizes": "192x192", "type": "image/png"}], }; - await fs.writeFile(path.join(targetDir, "manifest.json"), JSON.stringify(webManifest), "utf8"); + const manifestJson = JSON.stringify(webManifest); + const manifestPath = resource("manifest.json", manifestJson); + await fs.writeFile(manifestPath, manifestJson, "utf8"); // copy icon // should this icon have a content hash as well? let icon = await fs.readFile(path.join(projectDir, "icon.png")); await fs.writeFile(path.join(targetDir, "icon-192.png"), icon); + return manifestPath; } async function buildCssBundles(buildFn, themes, themeAssets) {