From 07b6458f19c086e6886fefb025e3f4413dba62d6 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Thu, 1 Oct 2020 19:21:27 +0200 Subject: [PATCH 1/9] track assets centrally in build script --- scripts/build.mjs | 281 +++++++++++++++++++++++----------------------- 1 file changed, 141 insertions(+), 140 deletions(-) diff --git a/scripts/build.mjs b/scripts/build.mjs index a8f86fe9..f18c04a4 100644 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -41,10 +41,6 @@ import postcssUrl from "postcss-url"; import cssvariables from "postcss-css-variables"; import flexbugsFixes from "postcss-flexbugs-fixes"; -const PROJECT_ID = "hydrogen"; -const PROJECT_SHORT_NAME = "Hydrogen"; -const PROJECT_NAME = "Hydrogen Chat"; - const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); const projectDir = path.join(__dirname, "../"); @@ -55,21 +51,10 @@ const program = new commander.Command(); program .option("--no-offline", "make a build without a service worker or appcache manifest") program.parse(process.argv); -const {debug, noOffline} = program; +const {noOffline} = program; const offline = !noOffline; -const olmFiles = { - wasm: "olm-4289088762.wasm", - legacyBundle: "olm_legacy-3232457086.js", - wasmBundle: "olm-1421970081.js", -}; - -// IDEA: how about instead of assetPaths we maintain a mapping between the source file and the target file -// so throughout the build script we can refer to files by their source name - async function build() { - // only used for CSS for now, using legacy for all targets for now - const legacy = true; // get version number const version = JSON.parse(await fs.readFile(path.join(projectDir, "package.json"), "utf8")).version; @@ -82,46 +67,23 @@ async function build() { // clear target dir await removeDirIfExists(targetDir); await createDirs(targetDir, themes); - // copy assets - await copyFolder(path.join(projectDir, "lib/olm/"), targetDir); - // also creates the directories where the theme css bundles are placed in, - // so do it first - const themeAssets = await copyThemeAssets(themes, legacy); - const jsBundlePath = await buildJs("src/main.js", `${PROJECT_ID}.js`); - const jsLegacyBundlePath = await buildJsLegacy("src/main.js", `${PROJECT_ID}-legacy.js`, 'src/legacy-extras.js'); - const jsWorkerPath = await buildWorkerJsLegacy("src/worker.js", `worker.js`); - const cssBundlePaths = await buildCssBundles(legacy ? buildCssLegacy : buildCss, themes, themeAssets); - - let manifestPath; - - const assetPaths = createAssetPaths(jsBundlePath, jsLegacyBundlePath, jsWorkerPath, - cssBundlePaths, themeAssets); - + const assets = new AssetMap(targetDir); + // copy olm assets + const olmAssets = await copyFolder(path.join(projectDir, "lib/olm/"), assets.directory); + assets.addSubMap(olmAssets); + await buildJs("src/main.js", code => assets.create(`hydrogen.js`, code)); + await buildJsLegacy("src/main.js", code => assets.create(`hydrogen-legacy.js`, code), 'src/legacy-extras.js'); + await buildWorkerJsLegacy("src/worker.js", code => assets.create(`worker.js`, code)); + // creates the directories where the theme css bundles are placed in, + // and writes to assets, so the build bundles can translate them, so do it first + await copyThemeAssets(themes, assets); + await buildCssBundles(buildCssLegacy, themes, assets); if (offline) { - manifestPath = await buildOffline(version, assetPaths); + await buildOffline(version, assets); } - await buildHtml(doc, version, assetPaths, manifestPath); - - console.log(`built ${PROJECT_ID} ${version} successfully`); -} - -function trim(path) { - if (!path.startsWith(targetDir)) { - throw new Error("invalid target path: " + targetDir); - } - return path.substr(targetDir.length); -} - -function createAssetPaths(jsBundlePath, jsLegacyBundlePath, jsWorkerPath, cssBundlePaths, themeAssets) { - return { - jsBundle: () => trim(jsBundlePath), - jsLegacyBundle: () => trim(jsLegacyBundlePath), - jsWorker: () => trim(jsWorkerPath), - cssMainBundle: () => trim(cssBundlePaths.main), - cssThemeBundle: themeName => trim(cssBundlePaths.themes[themeName]), - cssThemeBundles: () => Object.values(cssBundlePaths.themes).map(a => trim(a)), - otherAssets: () => Object.values(themeAssets).map(a => trim(a)), - }; + await buildHtml(doc, version, assets); + // 3 unhashed assets: index.html, manifest.appcache and sw.js + console.log(`built hydrogen ${version} successfully with ${assets.all().length + 3} files`); } async function findThemes(doc, callback) { @@ -148,36 +110,38 @@ async function createDirs(targetDir, themes) { } } -async function copyThemeAssets(themes, legacy) { - const assets = {}; +async function copyThemeAssets(themes, assets) { for (const theme of themes) { const themeDstFolder = path.join(targetDir, `themes/${theme}`); const themeSrcFolder = path.join(cssSrcDir, `themes/${theme}`); const themeAssets = await copyFolder(themeSrcFolder, themeDstFolder, file => { - const isUnneededFont = legacy ? file.endsWith(".woff2") : file.endsWith(".woff"); - return !file.endsWith(".css") && !isUnneededFont; + return !file.endsWith(".css"); }); - Object.assign(assets, themeAssets); + assets.addSubMap(themeAssets); } return assets; } -async function buildHtml(doc, version, assetPaths, manifestPath) { +async function buildHtml(doc, version, assets) { // transform html file // change path to main.css to css bundle - doc("link[rel=stylesheet]:not([title])").attr("href", assetPaths.cssMainBundle()); + doc("link[rel=stylesheet]:not([title])").attr("href", assets.resolve(`hydrogen.css`)); // change paths to all theme stylesheets findThemes(doc, (themeName, theme) => { - theme.attr("href", assetPaths.cssThemeBundle(themeName)); + theme.attr("href", assets.resolve(`themes/${themeName}/bundle.css`)); }); const pathsJSON = JSON.stringify({ - worker: assetPaths.jsWorker(), - olm: olmFiles + worker: assets.resolve(`worker.js`), + olm: { + wasm: assets.resolve("olm.wasm"), + legacyBundle: assets.resolve("olm_legacy.js"), + wasmBundle: assets.resolve("olm.js"), + } }); doc("script#main").replaceWith( - `` + - `` + - ``); + `` + + `` + + ``); removeOrEnableScript(doc("script#service-worker"), offline); const versionScript = doc("script#version"); @@ -188,12 +152,12 @@ async function buildHtml(doc, version, assetPaths, manifestPath) { if (offline) { doc("html").attr("manifest", "manifest.appcache"); - doc("head").append(``); + doc("head").append(``); } await fs.writeFile(path.join(targetDir, "index.html"), doc.html(), "utf8"); } -async function buildJs(inputFile, outputName) { +async function buildJs(inputFile, outputNameFn) { // create js bundle const bundle = await rollup({ input: inputFile, @@ -202,15 +166,13 @@ async function buildJs(inputFile, outputName) { const {output} = await bundle.generate({ format: 'es', // TODO: can remove this? - name: `${PROJECT_ID}Bundle` + name: `hydrogenBundle` }); const code = output[0].code; - const bundlePath = resource(outputName, code); - await fs.writeFile(bundlePath, code, "utf8"); - return bundlePath; + await fs.writeFile(outputNameFn(code), code, "utf8"); } -async function buildJsLegacy(inputFile, outputName, extraFile, polyfillFile) { +async function buildJsLegacy(inputFile, outputNameFn, extraFile, polyfillFile) { // compile down to whatever IE 11 needs const babelPlugin = babel.babel({ babelHelpers: 'bundled', @@ -245,35 +207,26 @@ async function buildJsLegacy(inputFile, outputName, extraFile, polyfillFile) { const bundle = await rollup(rollupConfig); const {output} = await bundle.generate({ format: 'iife', - name: `${PROJECT_ID}Bundle` + name: `hydrogenBundle` }); const code = output[0].code; - const bundlePath = resource(outputName, code); - await fs.writeFile(bundlePath, code, "utf8"); - return bundlePath; + await fs.writeFile(outputNameFn(code), code, "utf8"); } -function buildWorkerJsLegacy(inputFile, outputName) { +function buildWorkerJsLegacy(inputFile, outputNameFn) { const polyfillFile = 'src/worker-polyfill.js'; - return buildJsLegacy(inputFile, outputName, null, polyfillFile); + return buildJsLegacy(inputFile, outputNameFn, null, polyfillFile); } -async function buildOffline(version, assetPaths) { - // write web manifest +async function buildOffline(version, assets) { const webManifest = JSON.parse(await fs.readFile(path.join(projectDir, "assets/manifest.json"), "utf8")); + // copy manifest icons for (const icon of webManifest.icons) { let iconData = await fs.readFile(path.join(projectDir, icon.src)); - let iconPath = resource(path.basename(icon.src), iconData); - await fs.writeFile(iconPath, iconData); - icon.src = trim(iconPath); + const iconTargetPath = path.basename(icon.src); + await fs.writeFile(assets.create(iconTargetPath, iconData), iconData); + icon.src = assets.resolve(iconTargetPath); } - // write offline availability - const offlineFiles = [ - assetPaths.cssMainBundle(), - "index.html", - ].concat(assetPaths.cssThemeBundles()) - .concat(webManifest.icons.map(i => i.src)); - // write appcache manifest const appCacheLines = [ `CACHE MANIFEST`, @@ -282,53 +235,51 @@ async function buildOffline(version, assetPaths) { `"*"`, `CACHE`, ]; - appCacheLines.push(assetPaths.jsLegacyBundle(), ...offlineFiles); - const swOfflineFiles = [assetPaths.jsBundle(), ...offlineFiles]; + appCacheLines.push(...assets.allWithout(["hydrogen.js"])); + const swOfflineFiles = assets.allWithout(["hydrogen-legacy.js", "olm_legacy.js"]); const appCacheManifest = appCacheLines.join("\n") + "\n"; await fs.writeFile(path.join(targetDir, "manifest.appcache"), appCacheManifest, "utf8"); // write service worker let swSource = await fs.readFile(path.join(projectDir, "src/service-worker.template.js"), "utf8"); swSource = swSource.replace(`"%%VERSION%%"`, `"${version}"`); swSource = swSource.replace(`"%%OFFLINE_FILES%%"`, JSON.stringify(swOfflineFiles)); - swSource = swSource.replace(`"%%CACHE_FILES%%"`, JSON.stringify(assetPaths.otherAssets())); + // service worker should not have a hashed name as it is polled by the browser for updates await fs.writeFile(path.join(targetDir, "sw.js"), swSource, "utf8"); const manifestJson = JSON.stringify(webManifest); - const manifestPath = resource("manifest.json", manifestJson); - await fs.writeFile(manifestPath, manifestJson, "utf8"); - return manifestPath; + await fs.writeFile(assets.create("manifest.json", manifestJson), manifestJson, "utf8"); } -async function buildCssBundles(buildFn, themes, themeAssets) { +async function buildCssBundles(buildFn, themes, assets) { const bundleCss = await buildFn(path.join(cssSrcDir, "main.css")); - const mainDstPath = resource(`${PROJECT_ID}.css`, bundleCss); - await fs.writeFile(mainDstPath, bundleCss, "utf8"); - const bundlePaths = {main: mainDstPath, themes: {}}; + await fs.writeFile(assets.create(`hydrogen.css`, bundleCss), bundleCss, "utf8"); for (const theme of themes) { - const urlBase = path.join(targetDir, `themes/${theme}/`); + const themeRelPath = `themes/${theme}/`; + const themeRoot = path.join(cssSrcDir, themeRelPath); const assetUrlMapper = ({absolutePath}) => { - const hashedDstPath = themeAssets[absolutePath]; - if (hashedDstPath && hashedDstPath.startsWith(urlBase)) { - return hashedDstPath.substr(urlBase.length); + if (!absolutePath.startsWith(themeRoot)) { + throw new Error("resource is out of theme directory: " + absolutePath); + } + const relPath = absolutePath.substr(themeRoot.length); + const hashedDstPath = assets.resolve(path.join(themeRelPath, relPath)); + if (hashedDstPath) { + return hashedDstPath.substr(themeRelPath.length); } }; - const themeCss = await buildFn(path.join(cssSrcDir, `themes/${theme}/theme.css`), assetUrlMapper); - const themeDstPath = resource(`themes/${theme}/bundle.css`, themeCss); - await fs.writeFile(themeDstPath, themeCss, "utf8"); - bundlePaths.themes[theme] = themeDstPath; + const themeCss = await buildFn(path.join(themeRoot, `theme.css`), assetUrlMapper); + await fs.writeFile(assets.create(path.join(themeRelPath, `bundle.css`), themeCss), themeCss, "utf8"); } - return bundlePaths; } -async function buildCss(entryPath, urlMapper = null) { - const preCss = await fs.readFile(entryPath, "utf8"); - const options = [postcssImport]; - if (urlMapper) { - options.push(postcssUrl({url: urlMapper})); - } - const cssBundler = postcss(options); - const result = await cssBundler.process(preCss, {from: entryPath}); - return result.css; -} +// async function buildCss(entryPath, urlMapper = null) { +// const preCss = await fs.readFile(entryPath, "utf8"); +// const options = [postcssImport]; +// if (urlMapper) { +// options.push(postcssUrl({url: urlMapper})); +// } +// const cssBundler = postcss(options); +// const result = await cssBundler.process(preCss, {from: entryPath}); +// return result.css; +// } async function buildCssLegacy(entryPath, urlMapper = null) { const preCss = await fs.readFile(entryPath, "utf8"); @@ -363,35 +314,21 @@ async function removeDirIfExists(targetDir) { } } -async function copyFolder(srcRoot, dstRoot, filter) { - const assetPaths = {}; +async function copyFolder(srcRoot, dstRoot, filter, assets = null) { + assets = assets || new AssetMap(dstRoot); const dirEnts = await fs.readdir(srcRoot, {withFileTypes: true}); for (const dirEnt of dirEnts) { const dstPath = path.join(dstRoot, dirEnt.name); const srcPath = path.join(srcRoot, dirEnt.name); if (dirEnt.isDirectory()) { await fs.mkdir(dstPath); - Object.assign(assetPaths, await copyFolder(srcPath, dstPath, filter)); + await copyFolder(srcPath, dstPath, filter, assets); } else if ((dirEnt.isFile() || dirEnt.isSymbolicLink()) && (!filter || filter(srcPath))) { const content = await fs.readFile(srcPath); - const hashedDstPath = resource(dstPath, content); - await fs.writeFile(hashedDstPath, content); - assetPaths[srcPath] = hashedDstPath; + await fs.writeFile(assets.create(dstPath, content), content); } } - return assetPaths; -} - -function resource(relPath, content) { - let fullPath = relPath; - if (!path.isAbsolute(relPath)) { - fullPath = path.join(targetDir, relPath); - } - const hash = contentHash(Buffer.from(content)); - const dir = path.dirname(fullPath); - const extname = path.extname(fullPath); - const basename = path.basename(fullPath, extname); - return path.join(dir, `${basename}-${hash}${extname}`); + return assets; } function contentHash(str) { @@ -400,5 +337,69 @@ function contentHash(str) { return hasher.digest(); } +class AssetMap { + constructor(targetDir) { + // remove last / if any, so substr in create works well + this._targetDir = path.resolve(targetDir); + this._assets = new Map(); + } + + _toRelPath(resourcePath) { + let relPath = resourcePath; + if (path.isAbsolute(resourcePath)) { + if (!resourcePath.startsWith(this._targetDir)) { + throw new Error(`absolute path ${resourcePath} that is not within target dir ${this._targetDir}`); + } + relPath = resourcePath.substr(this._targetDir.length + 1); // + 1 for the / + } + return relPath; + } + + create(resourcePath, content) { + const relPath = this._toRelPath(resourcePath); + const hash = contentHash(Buffer.from(content)); + const dir = path.dirname(relPath); + const extname = path.extname(relPath); + const basename = path.basename(relPath, extname); + const dstRelPath = path.join(dir, `${basename}-${hash}${extname}`); + this._assets.set(relPath, dstRelPath); + return path.join(this._targetDir, dstRelPath); + } + + get directory() { + return this._targetDir; + } + + resolve(resourcePath) { + const relPath = this._toRelPath(resourcePath); + const result = this._assets.get(relPath); + if (!result) { + throw new Error(`unknown path: ${relPath}, only know ${Array.from(this._assets.keys()).join(", ")}`); + } + return result; + } + + addSubMap(assetMap) { + if (!assetMap.directory.startsWith(this.directory)) { + throw new Error(`map directory doesn't start with this directory: ${assetMap.directory} ${this.directory}`); + } + console.log("adding submap from", assetMap.directory, this.directory); + const relSubRoot = assetMap.directory.substr(this.directory.length + 1); + for (const [key, value] of assetMap._assets.entries()) { + this._assets.set(path.join(relSubRoot, key), path.join(relSubRoot, value)); + } + } + + all() { + return Array.from(this._assets.values()); + } + + allWithout(excluded) { + excluded = excluded.map(p => this.resolve(p)); + return this.all().filter(p => excluded.indexOf(p) === -1); + } +} + + build().catch(err => console.error(err)); From 4a19c8d575537e4b2296e04bdc63bdd132bba0da Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 2 Oct 2020 09:33:07 +0200 Subject: [PATCH 2/9] move asset writing into AssetMap as well --- scripts/build.mjs | 54 +++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/scripts/build.mjs b/scripts/build.mjs index f18c04a4..335f620c 100644 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -71,9 +71,9 @@ async function build() { // copy olm assets const olmAssets = await copyFolder(path.join(projectDir, "lib/olm/"), assets.directory); assets.addSubMap(olmAssets); - await buildJs("src/main.js", code => assets.create(`hydrogen.js`, code)); - await buildJsLegacy("src/main.js", code => assets.create(`hydrogen-legacy.js`, code), 'src/legacy-extras.js'); - await buildWorkerJsLegacy("src/worker.js", code => assets.create(`worker.js`, code)); + await assets.write(`hydrogen.js`, await buildJs("src/main.js")); + await assets.write(`hydrogen-legacy.js`, await buildJsLegacy(["src/main.js", 'src/legacy-polyfill.js', 'src/legacy-extras.js'])); + await assets.write(`worker.js`, await buildJsLegacy(["src/worker.js", 'src/worker-polyfill.js'])); // creates the directories where the theme css bundles are placed in, // and writes to assets, so the build bundles can translate them, so do it first await copyThemeAssets(themes, assets); @@ -157,7 +157,7 @@ async function buildHtml(doc, version, assets) { await fs.writeFile(path.join(targetDir, "index.html"), doc.html(), "utf8"); } -async function buildJs(inputFile, outputNameFn) { +async function buildJs(inputFile) { // create js bundle const bundle = await rollup({ input: inputFile, @@ -169,10 +169,10 @@ async function buildJs(inputFile, outputNameFn) { name: `hydrogenBundle` }); const code = output[0].code; - await fs.writeFile(outputNameFn(code), code, "utf8"); + return code; } -async function buildJsLegacy(inputFile, outputNameFn, extraFile, polyfillFile) { +async function buildJsLegacy(inputFiles) { // compile down to whatever IE 11 needs const babelPlugin = babel.babel({ babelHelpers: 'bundled', @@ -192,13 +192,6 @@ async function buildJsLegacy(inputFile, outputNameFn, extraFile, polyfillFile) { ] ] }); - if (!polyfillFile) { - polyfillFile = 'src/legacy-polyfill.js'; - } - const inputFiles = [polyfillFile, inputFile]; - if (extraFile) { - inputFiles.push(extraFile); - } // create js bundle const rollupConfig = { input: inputFiles, @@ -210,12 +203,7 @@ async function buildJsLegacy(inputFile, outputNameFn, extraFile, polyfillFile) { name: `hydrogenBundle` }); const code = output[0].code; - await fs.writeFile(outputNameFn(code), code, "utf8"); -} - -function buildWorkerJsLegacy(inputFile, outputNameFn) { - const polyfillFile = 'src/worker-polyfill.js'; - return buildJsLegacy(inputFile, outputNameFn, null, polyfillFile); + return code; } async function buildOffline(version, assets) { @@ -224,8 +212,7 @@ async function buildOffline(version, assets) { for (const icon of webManifest.icons) { let iconData = await fs.readFile(path.join(projectDir, icon.src)); const iconTargetPath = path.basename(icon.src); - await fs.writeFile(assets.create(iconTargetPath, iconData), iconData); - icon.src = assets.resolve(iconTargetPath); + icon.src = await assets.write(iconTargetPath, iconData); } // write appcache manifest const appCacheLines = [ @@ -245,13 +232,12 @@ async function buildOffline(version, assets) { swSource = swSource.replace(`"%%OFFLINE_FILES%%"`, JSON.stringify(swOfflineFiles)); // service worker should not have a hashed name as it is polled by the browser for updates await fs.writeFile(path.join(targetDir, "sw.js"), swSource, "utf8"); - const manifestJson = JSON.stringify(webManifest); - await fs.writeFile(assets.create("manifest.json", manifestJson), manifestJson, "utf8"); + await assets.write("manifest.json", JSON.stringify(webManifest)); } async function buildCssBundles(buildFn, themes, assets) { const bundleCss = await buildFn(path.join(cssSrcDir, "main.css")); - await fs.writeFile(assets.create(`hydrogen.css`, bundleCss), bundleCss, "utf8"); + await assets.write(`hydrogen.css`, bundleCss); for (const theme of themes) { const themeRelPath = `themes/${theme}/`; const themeRoot = path.join(cssSrcDir, themeRelPath); @@ -266,7 +252,7 @@ async function buildCssBundles(buildFn, themes, assets) { } }; const themeCss = await buildFn(path.join(themeRoot, `theme.css`), assetUrlMapper); - await fs.writeFile(assets.create(path.join(themeRelPath, `bundle.css`), themeCss), themeCss, "utf8"); + await assets.write(path.join(themeRelPath, `bundle.css`), themeCss); } } @@ -325,7 +311,7 @@ async function copyFolder(srcRoot, dstRoot, filter, assets = null) { await copyFolder(srcPath, dstPath, filter, assets); } else if ((dirEnt.isFile() || dirEnt.isSymbolicLink()) && (!filter || filter(srcPath))) { const content = await fs.readFile(srcPath); - await fs.writeFile(assets.create(dstPath, content), content); + await assets.write(dstPath, content); } } return assets; @@ -355,7 +341,7 @@ class AssetMap { return relPath; } - create(resourcePath, content) { + _create(resourcePath, content) { const relPath = this._toRelPath(resourcePath); const hash = contentHash(Buffer.from(content)); const dir = path.dirname(relPath); @@ -363,7 +349,19 @@ class AssetMap { const basename = path.basename(relPath, extname); const dstRelPath = path.join(dir, `${basename}-${hash}${extname}`); this._assets.set(relPath, dstRelPath); - return path.join(this._targetDir, dstRelPath); + return dstRelPath; + } + + async write(resourcePath, content) { + const relPath = this._create(resourcePath, content); + const fullPath = path.join(this.directory, relPath); + if (typeof content === "string") { + await fs.writeFile(fullPath, content, "utf8"); + } else { + await fs.writeFile(fullPath, content); + } + return relPath; + } } get directory() { From 441bfa48904ee1f4df43725b5f1af1f6a6266a21 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 2 Oct 2020 09:33:35 +0200 Subject: [PATCH 3/9] also write unhashed assets from asset map --- scripts/build.mjs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/scripts/build.mjs b/scripts/build.mjs index 335f620c..f8e1a110 100644 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -82,8 +82,7 @@ async function build() { await buildOffline(version, assets); } await buildHtml(doc, version, assets); - // 3 unhashed assets: index.html, manifest.appcache and sw.js - console.log(`built hydrogen ${version} successfully with ${assets.all().length + 3} files`); + console.log(`built hydrogen ${version} successfully with ${assets.all().length} files`); } async function findThemes(doc, callback) { @@ -151,10 +150,10 @@ async function buildHtml(doc, version, assets) { versionScript.text(vSource); if (offline) { - doc("html").attr("manifest", "manifest.appcache"); + doc("html").attr("manifest", assets.resolve("manifest.appcache")); doc("head").append(``); } - await fs.writeFile(path.join(targetDir, "index.html"), doc.html(), "utf8"); + await assets.writeUnhashed("index.html", doc.html()); } async function buildJs(inputFile) { @@ -225,13 +224,13 @@ async function buildOffline(version, assets) { appCacheLines.push(...assets.allWithout(["hydrogen.js"])); const swOfflineFiles = assets.allWithout(["hydrogen-legacy.js", "olm_legacy.js"]); const appCacheManifest = appCacheLines.join("\n") + "\n"; - await fs.writeFile(path.join(targetDir, "manifest.appcache"), appCacheManifest, "utf8"); + await assets.writeUnhashed("manifest.appcache", appCacheManifest); // write service worker let swSource = await fs.readFile(path.join(projectDir, "src/service-worker.template.js"), "utf8"); swSource = swSource.replace(`"%%VERSION%%"`, `"${version}"`); swSource = swSource.replace(`"%%OFFLINE_FILES%%"`, JSON.stringify(swOfflineFiles)); // service worker should not have a hashed name as it is polled by the browser for updates - await fs.writeFile(path.join(targetDir, "sw.js"), swSource, "utf8"); + await assets.writeUnhashed("sw.js", swSource); await assets.write("manifest.json", JSON.stringify(webManifest)); } @@ -362,6 +361,17 @@ class AssetMap { } return relPath; } + + async writeUnhashed(resourcePath, content) { + const relPath = this._toRelPath(resourcePath); + this._assets.set(relPath, relPath); + const fullPath = path.join(this.directory, relPath); + if (typeof content === "string") { + await fs.writeFile(fullPath, content, "utf8"); + } else { + await fs.writeFile(fullPath, content); + } + return relPath; } get directory() { From 38a30bcc49b51de1993c3c6e781db3dbb6c11b05 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 2 Oct 2020 09:33:57 +0200 Subject: [PATCH 4/9] targetDir doesn't need to be global anymore --- scripts/build.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build.mjs b/scripts/build.mjs index f8e1a110..165354c5 100644 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -45,7 +45,6 @@ const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); const projectDir = path.join(__dirname, "../"); const cssSrcDir = path.join(projectDir, "src/ui/web/css/"); -const targetDir = path.join(projectDir, "target/"); const program = new commander.Command(); program @@ -65,6 +64,7 @@ async function build() { themes.push(themeName); }); // clear target dir + const targetDir = path.join(projectDir, "target/"); await removeDirIfExists(targetDir); await createDirs(targetDir, themes); const assets = new AssetMap(targetDir); @@ -111,7 +111,7 @@ async function createDirs(targetDir, themes) { async function copyThemeAssets(themes, assets) { for (const theme of themes) { - const themeDstFolder = path.join(targetDir, `themes/${theme}`); + const themeDstFolder = path.join(assets.directory, `themes/${theme}`); const themeSrcFolder = path.join(cssSrcDir, `themes/${theme}`); const themeAssets = await copyFolder(themeSrcFolder, themeDstFolder, file => { return !file.endsWith(".css"); From dff8ddb53273570f35c65a866620ad496ae0d002 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 2 Oct 2020 09:34:12 +0200 Subject: [PATCH 5/9] remove logging --- scripts/build.mjs | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/build.mjs b/scripts/build.mjs index 165354c5..d1b70d1c 100644 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -391,7 +391,6 @@ class AssetMap { if (!assetMap.directory.startsWith(this.directory)) { throw new Error(`map directory doesn't start with this directory: ${assetMap.directory} ${this.directory}`); } - console.log("adding submap from", assetMap.directory, this.directory); const relSubRoot = assetMap.directory.substr(this.directory.length + 1); for (const [key, value] of assetMap._assets.entries()) { this._assets.set(path.join(relSubRoot, key), path.join(relSubRoot, value)); From 8dc2816d6e57e1ecc846b15046c57de1ffc4750b Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Fri, 2 Oct 2020 19:54:24 +0200 Subject: [PATCH 6/9] keep hashed files in cache as long as they don't change --- index.html | 1 + scripts/build.mjs | 140 ++++++++++++++++++++------------- src/service-worker.template.js | 104 ++++++++++++++++++------ src/ui/web/login/common.js | 2 +- 4 files changed, 165 insertions(+), 82 deletions(-) diff --git a/index.html b/index.html index 74e44c99..b7b55d75 100644 --- a/index.html +++ b/index.html @@ -15,6 +15,7 @@ ` + - `` + - ``); - removeOrEnableScript(doc("script#service-worker"), offline); + const mainScripts = [ + `` + ]; + if (!modernOnly) { + mainScripts.push( + ``, + `` + ); + } + doc("script#main").replaceWith(mainScripts.join("")); + doc("script#service-worker").attr("type", "text/javascript"); const versionScript = doc("script#version"); versionScript.attr("type", "text/javascript"); let vSource = versionScript.contents().text(); vSource = vSource.replace(`"%%VERSION%%"`, `"${version}"`); + vSource = vSource.replace(`"%%GLOBAL_HASH%%"`, `"${globalHash}"`); versionScript.text(vSource); - - if (offline) { - doc("html").attr("manifest", assets.resolve("manifest.appcache")); - doc("head").append(``); - } + doc("head").append(``); await assets.writeUnhashed("index.html", doc.html()); } @@ -205,7 +211,22 @@ async function buildJsLegacy(inputFiles) { return code; } -async function buildOffline(version, assets) { +const SERVICEWORKER_NONCACHED_ASSETS = [ + "hydrogen-legacy.js", + "olm_legacy.js", + "sw.js", +]; + +function isPreCached(asset) { + return asset.endsWith(".svg") || + asset.endsWith(".png") || + asset.endsWith(".css") || + asset.endsWith(".wasm") || + // most environments don't need the worker + asset.endsWith(".js") && asset !== "worker.js"; +} + +async function buildManifest(assets) { const webManifest = JSON.parse(await fs.readFile(path.join(projectDir, "assets/manifest.json"), "utf8")); // copy manifest icons for (const icon of webManifest.icons) { @@ -213,25 +234,33 @@ async function buildOffline(version, assets) { const iconTargetPath = path.basename(icon.src); icon.src = await assets.write(iconTargetPath, iconData); } - // write appcache manifest - const appCacheLines = [ - `CACHE MANIFEST`, - `# v${version}`, - `NETWORK`, - `"*"`, - `CACHE`, - ]; - appCacheLines.push(...assets.allWithout(["hydrogen.js"])); - const swOfflineFiles = assets.allWithout(["hydrogen-legacy.js", "olm_legacy.js"]); - const appCacheManifest = appCacheLines.join("\n") + "\n"; - await assets.writeUnhashed("manifest.appcache", appCacheManifest); + await assets.write("manifest.json", JSON.stringify(webManifest)); +} + +async function buildServiceWorker(globalHash, assets) { + const unhashedPreCachedAssets = ["index.html"]; + const hashedPreCachedAssets = []; + const hashedCachedOnRequestAssets = []; + + for (const [unresolved, resolved] of assets) { + if (SERVICEWORKER_NONCACHED_ASSETS.includes(unresolved)) { + continue; + } else if (unresolved === resolved) { + unhashedPreCachedAssets.push(resolved); + } else if (isPreCached(unresolved)) { + hashedPreCachedAssets.push(resolved); + } else { + hashedCachedOnRequestAssets.push(resolved); + } + } // write service worker let swSource = await fs.readFile(path.join(projectDir, "src/service-worker.template.js"), "utf8"); - swSource = swSource.replace(`"%%VERSION%%"`, `"${version}"`); - swSource = swSource.replace(`"%%OFFLINE_FILES%%"`, JSON.stringify(swOfflineFiles)); + swSource = swSource.replace(`"%%GLOBAL_HASH%%"`, `"${globalHash}"`); + swSource = swSource.replace(`"%%UNHASHED_PRECACHED_ASSETS%%"`, JSON.stringify(unhashedPreCachedAssets)); + swSource = swSource.replace(`"%%HASHED_PRECACHED_ASSETS%%"`, JSON.stringify(hashedPreCachedAssets)); + swSource = swSource.replace(`"%%HASHED_CACHED_ON_REQUEST_ASSETS%%"`, JSON.stringify(hashedCachedOnRequestAssets)); // service worker should not have a hashed name as it is polled by the browser for updates await assets.writeUnhashed("sw.js", swSource); - await assets.write("manifest.json", JSON.stringify(webManifest)); } async function buildCssBundles(buildFn, themes, assets) { @@ -281,14 +310,6 @@ async function buildCssLegacy(entryPath, urlMapper = null) { return result.css; } -function removeOrEnableScript(scriptNode, enable) { - if (enable) { - scriptNode.attr("type", "text/javascript"); - } else { - scriptNode.remove(); - } -} - async function removeDirIfExists(targetDir) { try { await fs.rmdir(targetDir, {recursive: true}); @@ -397,16 +418,25 @@ class AssetMap { } } - all() { - return Array.from(this._assets.values()); + [Symbol.iterator]() { + return this._assets.entries(); } - allWithout(excluded) { - excluded = excluded.map(p => this.resolve(p)); - return this.all().filter(p => excluded.indexOf(p) === -1); + isUnhashed(relPath) { + const resolvedPath = this._assets.get(relPath); + if (!resolvedPath) { + throw new Error("Unknown asset: " + relPath); + } + return relPath === resolvedPath; + } + + get size() { + return this._assets.size; + } + + has(relPath) { + return this._assets.has(relPath); } } - - -build().catch(err => console.error(err)); +build(program).catch(err => console.error(err)); diff --git a/src/service-worker.template.js b/src/service-worker.template.js index 485afba8..cad5843a 100644 --- a/src/service-worker.template.js +++ b/src/service-worker.template.js @@ -14,38 +14,90 @@ See the License for the specific language governing permissions and limitations under the License. */ -const VERSION = "%%VERSION%%"; -const OFFLINE_FILES = "%%OFFLINE_FILES%%"; -// TODO: cache these files when requested -// The difficulty is that these are relative filenames, and we don't have access to document.baseURI -// Clients.match({type: "window"}).url and assume they are all the same? they really should be ... safari doesn't support this though -const CACHE_FILES = "%%CACHE_FILES%%"; -const cacheName = `hydrogen-${VERSION}`; +const GLOBAL_HASH = "%%GLOBAL_HASH%%"; +const UNHASHED_PRECACHED_ASSETS = "%%UNHASHED_PRECACHED_ASSETS%%"; +const HASHED_PRECACHED_ASSETS = "%%HASHED_PRECACHED_ASSETS%%"; +const HASHED_CACHED_ON_REQUEST_ASSETS = "%%HASHED_CACHED_ON_REQUEST_ASSETS%%"; +const unhashedCacheName = `hydrogen-assets-${GLOBAL_HASH}`; +const hashedCacheName = `hydrogen-assets`; self.addEventListener('install', function(e) { - e.waitUntil( - caches.open(cacheName).then(function(cache) { - return cache.addAll(OFFLINE_FILES); - }) - ); + e.waitUntil((async () => { + const unhashedCache = await caches.open(unhashedCacheName); + await unhashedCache.addAll(UNHASHED_PRECACHED_ASSETS); + const hashedCache = await caches.open(hashedCacheName); + await Promise.all(HASHED_PRECACHED_ASSETS.map(async asset => { + if (!await hashedCache.match(asset)) { + await hashedCache.add(asset); + } + })); + })()); }); -self.addEventListener('activate', (event) => { - event.waitUntil( - caches.keys().then((keyList) => { - return Promise.all(keyList.map((key) => { - if (key !== cacheName) { - return caches.delete(key); +async function purgeOldCaches() { + // remove any caches we don't know about + const keyList = await caches.keys(); + for (const key of keyList) { + if (key !== unhashedCacheName && key !== hashedCacheName) { + await caches.delete(key); } - })); - }) - ); + } + // remove the cache for any old hashed resource + const hashedCache = await caches.open(hashedCacheName); + const keys = await hashedCache.keys(); + const hashedAssetURLs = + HASHED_PRECACHED_ASSETS + .concat(HASHED_CACHED_ON_REQUEST_ASSETS) + .map(a => new URL(a, self.registration.scope).href); + + for (const request of keys) { + if (!hashedAssetURLs.some(url => url === request.url)) { + hashedCache.delete(request); + } + } +} + +self.addEventListener('activate', (event) => { + event.waitUntil(purgeOldCaches()); }); self.addEventListener('fetch', (event) => { - event.respondWith( - caches.open(cacheName) - .then(cache => cache.match(event.request)) - .then((response) => response || fetch(event.request)) - ); + event.respondWith(handleRequest(event.request)); }); + +async function handleRequest(request) { + const baseURL = self.registration.scope; + if (request.url === baseURL) { + request = new Request(new URL("index.html", baseURL)); + } + let response = await readCache(request); + if (!response) { + response = await fetch(request); + await updateCache(request, response); + } + return response; +} + +async function updateCache(request, response) { + const baseURL = self.registration.scope; + if(!request.url.startsWith(baseURL)) { + return; + } + let assetName = request.url.substr(baseURL.length); + if (HASHED_CACHED_ON_REQUEST_ASSETS.includes(assetName)) { + const cache = await caches.open(hashedCacheName); + await cache.put(request, response.clone()); + } +} + +async function readCache(request) { + const unhashedCache = await caches.open(unhashedCacheName); + let response = await unhashedCache.match(request); + if (response) { + return response; + } + const hashedCache = await caches.open(hashedCacheName); + response = await hashedCache.match(request); + return response; +} + diff --git a/src/ui/web/login/common.js b/src/ui/web/login/common.js index f1e6496f..b84e5e6a 100644 --- a/src/ui/web/login/common.js +++ b/src/ui/web/login/common.js @@ -18,7 +18,7 @@ export function hydrogenGithubLink(t) { if (window.HYDROGEN_VERSION) { return t.a({target: "_blank", href: `https://github.com/vector-im/hydrogen-web/releases/tag/v${window.HYDROGEN_VERSION}`}, - `Hydrogen v${window.HYDROGEN_VERSION} on Github`); + `Hydrogen v${window.HYDROGEN_VERSION} (${window.HYDROGEN_GLOBAL_HASH}) on Github`); } else { return t.a({target: "_blank", href: "https://github.com/vector-im/hydrogen-web"}, "Hydrogen on Github"); From 6a8c0b52e0e7c2e2bddff368f3e82afc82bd2acc Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Mon, 5 Oct 2020 15:10:06 +0200 Subject: [PATCH 7/9] indenting --- src/service-worker.template.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/service-worker.template.js b/src/service-worker.template.js index cad5843a..df77cad3 100644 --- a/src/service-worker.template.js +++ b/src/service-worker.template.js @@ -1,5 +1,6 @@ /* Copyright 2020 Bruno Windels +Copyright 2020 The Matrix.org Foundation C.I.C. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -58,11 +59,11 @@ async function purgeOldCaches() { } self.addEventListener('activate', (event) => { - event.waitUntil(purgeOldCaches()); + event.waitUntil(purgeOldCaches()); }); self.addEventListener('fetch', (event) => { - event.respondWith(handleRequest(event.request)); + event.respondWith(handleRequest(event.request)); }); async function handleRequest(request) { From c8fd9d423da7c1b2ffed87a85c5a1900e34aba6f Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Mon, 5 Oct 2020 15:10:15 +0200 Subject: [PATCH 8/9] log requests to dev server --- scripts/serve-local.js | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/serve-local.js b/scripts/serve-local.js index b6ff8cf3..ae07d180 100644 --- a/scripts/serve-local.js +++ b/scripts/serve-local.js @@ -35,6 +35,7 @@ const serve = serveStatic( // Create server const server = http.createServer(function onRequest (req, res) { + console.log(req.method, req.url); serve(req, res, finalhandler(req, res)) }); From 8f5d678698bf4ca243b10a6b54e1086d9564ee1e Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Mon, 5 Oct 2020 15:58:29 +0200 Subject: [PATCH 9/9] cache small media repo thumbnails in service worker --- src/service-worker.template.js | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/src/service-worker.template.js b/src/service-worker.template.js index df77cad3..bbb8c892 100644 --- a/src/service-worker.template.js +++ b/src/service-worker.template.js @@ -21,6 +21,7 @@ const HASHED_PRECACHED_ASSETS = "%%HASHED_PRECACHED_ASSETS%%"; const HASHED_CACHED_ON_REQUEST_ASSETS = "%%HASHED_CACHED_ON_REQUEST_ASSETS%%"; const unhashedCacheName = `hydrogen-assets-${GLOBAL_HASH}`; const hashedCacheName = `hydrogen-assets`; +const mediaThumbnailCacheName = `hydrogen-media-thumbnails`; self.addEventListener('install', function(e) { e.waitUntil((async () => { @@ -39,7 +40,7 @@ async function purgeOldCaches() { // remove any caches we don't know about const keyList = await caches.keys(); for (const key of keyList) { - if (key !== unhashedCacheName && key !== hashedCacheName) { + if (key !== unhashedCacheName && key !== hashedCacheName && key !== mediaThumbnailCacheName) { await caches.delete(key); } } @@ -80,14 +81,21 @@ async function handleRequest(request) { } async function updateCache(request, response) { + const url = new URL(request.url); const baseURL = self.registration.scope; - if(!request.url.startsWith(baseURL)) { - return; - } - let assetName = request.url.substr(baseURL.length); - if (HASHED_CACHED_ON_REQUEST_ASSETS.includes(assetName)) { - const cache = await caches.open(hashedCacheName); - await cache.put(request, response.clone()); + if (url.pathname.startsWith("/_matrix/media/r0/thumbnail/")) { + const width = parseInt(url.searchParams.get("width"), 10); + const height = parseInt(url.searchParams.get("height"), 10); + if (width <= 50 && height <= 50) { + const cache = await caches.open(mediaThumbnailCacheName); + cache.put(request, response.clone()); + } + } else if (request.url.startsWith(baseURL)) { + let assetName = request.url.substr(baseURL.length); + if (HASHED_CACHED_ON_REQUEST_ASSETS.includes(assetName)) { + const cache = await caches.open(hashedCacheName); + await cache.put(request, response.clone()); + } } } @@ -99,6 +107,15 @@ async function readCache(request) { } const hashedCache = await caches.open(hashedCacheName); response = await hashedCache.match(request); + if (response) { + return response; + } + + const url = new URL(request.url); + if (url.pathname.startsWith("/_matrix/media/r0/thumbnail/")) { + const mediaThumbnailCache = await caches.open(mediaThumbnailCacheName); + response = await mediaThumbnailCache.match(request); + } return response; }