From 07b6458f19c086e6886fefb025e3f4413dba62d6 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Thu, 1 Oct 2020 19:21:27 +0200 Subject: [PATCH] track assets centrally in build script --- scripts/build.mjs | 281 +++++++++++++++++++++++----------------------- 1 file changed, 141 insertions(+), 140 deletions(-) diff --git a/scripts/build.mjs b/scripts/build.mjs index a8f86fe9..f18c04a4 100644 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -41,10 +41,6 @@ import postcssUrl from "postcss-url"; import cssvariables from "postcss-css-variables"; import flexbugsFixes from "postcss-flexbugs-fixes"; -const PROJECT_ID = "hydrogen"; -const PROJECT_SHORT_NAME = "Hydrogen"; -const PROJECT_NAME = "Hydrogen Chat"; - const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); const projectDir = path.join(__dirname, "../"); @@ -55,21 +51,10 @@ const program = new commander.Command(); program .option("--no-offline", "make a build without a service worker or appcache manifest") program.parse(process.argv); -const {debug, noOffline} = program; +const {noOffline} = program; const offline = !noOffline; -const olmFiles = { - wasm: "olm-4289088762.wasm", - legacyBundle: "olm_legacy-3232457086.js", - wasmBundle: "olm-1421970081.js", -}; - -// IDEA: how about instead of assetPaths we maintain a mapping between the source file and the target file -// so throughout the build script we can refer to files by their source name - async function build() { - // only used for CSS for now, using legacy for all targets for now - const legacy = true; // get version number const version = JSON.parse(await fs.readFile(path.join(projectDir, "package.json"), "utf8")).version; @@ -82,46 +67,23 @@ async function build() { // clear target dir await removeDirIfExists(targetDir); await createDirs(targetDir, themes); - // copy assets - await copyFolder(path.join(projectDir, "lib/olm/"), targetDir); - // also creates the directories where the theme css bundles are placed in, - // so do it first - const themeAssets = await copyThemeAssets(themes, legacy); - const jsBundlePath = await buildJs("src/main.js", `${PROJECT_ID}.js`); - const jsLegacyBundlePath = await buildJsLegacy("src/main.js", `${PROJECT_ID}-legacy.js`, 'src/legacy-extras.js'); - const jsWorkerPath = await buildWorkerJsLegacy("src/worker.js", `worker.js`); - const cssBundlePaths = await buildCssBundles(legacy ? buildCssLegacy : buildCss, themes, themeAssets); - - let manifestPath; - - const assetPaths = createAssetPaths(jsBundlePath, jsLegacyBundlePath, jsWorkerPath, - cssBundlePaths, themeAssets); - + const assets = new AssetMap(targetDir); + // copy olm assets + const olmAssets = await copyFolder(path.join(projectDir, "lib/olm/"), assets.directory); + assets.addSubMap(olmAssets); + await buildJs("src/main.js", code => assets.create(`hydrogen.js`, code)); + await buildJsLegacy("src/main.js", code => assets.create(`hydrogen-legacy.js`, code), 'src/legacy-extras.js'); + await buildWorkerJsLegacy("src/worker.js", code => assets.create(`worker.js`, code)); + // creates the directories where the theme css bundles are placed in, + // and writes to assets, so the build bundles can translate them, so do it first + await copyThemeAssets(themes, assets); + await buildCssBundles(buildCssLegacy, themes, assets); if (offline) { - manifestPath = await buildOffline(version, assetPaths); + await buildOffline(version, assets); } - await buildHtml(doc, version, assetPaths, manifestPath); - - console.log(`built ${PROJECT_ID} ${version} successfully`); -} - -function trim(path) { - if (!path.startsWith(targetDir)) { - throw new Error("invalid target path: " + targetDir); - } - return path.substr(targetDir.length); -} - -function createAssetPaths(jsBundlePath, jsLegacyBundlePath, jsWorkerPath, cssBundlePaths, themeAssets) { - return { - jsBundle: () => trim(jsBundlePath), - jsLegacyBundle: () => trim(jsLegacyBundlePath), - jsWorker: () => trim(jsWorkerPath), - cssMainBundle: () => trim(cssBundlePaths.main), - cssThemeBundle: themeName => trim(cssBundlePaths.themes[themeName]), - cssThemeBundles: () => Object.values(cssBundlePaths.themes).map(a => trim(a)), - otherAssets: () => Object.values(themeAssets).map(a => trim(a)), - }; + await buildHtml(doc, version, assets); + // 3 unhashed assets: index.html, manifest.appcache and sw.js + console.log(`built hydrogen ${version} successfully with ${assets.all().length + 3} files`); } async function findThemes(doc, callback) { @@ -148,36 +110,38 @@ async function createDirs(targetDir, themes) { } } -async function copyThemeAssets(themes, legacy) { - const assets = {}; +async function copyThemeAssets(themes, assets) { for (const theme of themes) { const themeDstFolder = path.join(targetDir, `themes/${theme}`); const themeSrcFolder = path.join(cssSrcDir, `themes/${theme}`); const themeAssets = await copyFolder(themeSrcFolder, themeDstFolder, file => { - const isUnneededFont = legacy ? file.endsWith(".woff2") : file.endsWith(".woff"); - return !file.endsWith(".css") && !isUnneededFont; + return !file.endsWith(".css"); }); - Object.assign(assets, themeAssets); + assets.addSubMap(themeAssets); } return assets; } -async function buildHtml(doc, version, assetPaths, manifestPath) { +async function buildHtml(doc, version, assets) { // transform html file // change path to main.css to css bundle - doc("link[rel=stylesheet]:not([title])").attr("href", assetPaths.cssMainBundle()); + doc("link[rel=stylesheet]:not([title])").attr("href", assets.resolve(`hydrogen.css`)); // change paths to all theme stylesheets findThemes(doc, (themeName, theme) => { - theme.attr("href", assetPaths.cssThemeBundle(themeName)); + theme.attr("href", assets.resolve(`themes/${themeName}/bundle.css`)); }); const pathsJSON = JSON.stringify({ - worker: assetPaths.jsWorker(), - olm: olmFiles + worker: assets.resolve(`worker.js`), + olm: { + wasm: assets.resolve("olm.wasm"), + legacyBundle: assets.resolve("olm_legacy.js"), + wasmBundle: assets.resolve("olm.js"), + } }); doc("script#main").replaceWith( - `` + - `` + - ``); + `` + + `` + + ``); removeOrEnableScript(doc("script#service-worker"), offline); const versionScript = doc("script#version"); @@ -188,12 +152,12 @@ async function buildHtml(doc, version, assetPaths, manifestPath) { if (offline) { doc("html").attr("manifest", "manifest.appcache"); - doc("head").append(``); + doc("head").append(``); } await fs.writeFile(path.join(targetDir, "index.html"), doc.html(), "utf8"); } -async function buildJs(inputFile, outputName) { +async function buildJs(inputFile, outputNameFn) { // create js bundle const bundle = await rollup({ input: inputFile, @@ -202,15 +166,13 @@ async function buildJs(inputFile, outputName) { const {output} = await bundle.generate({ format: 'es', // TODO: can remove this? - name: `${PROJECT_ID}Bundle` + name: `hydrogenBundle` }); const code = output[0].code; - const bundlePath = resource(outputName, code); - await fs.writeFile(bundlePath, code, "utf8"); - return bundlePath; + await fs.writeFile(outputNameFn(code), code, "utf8"); } -async function buildJsLegacy(inputFile, outputName, extraFile, polyfillFile) { +async function buildJsLegacy(inputFile, outputNameFn, extraFile, polyfillFile) { // compile down to whatever IE 11 needs const babelPlugin = babel.babel({ babelHelpers: 'bundled', @@ -245,35 +207,26 @@ async function buildJsLegacy(inputFile, outputName, extraFile, polyfillFile) { const bundle = await rollup(rollupConfig); const {output} = await bundle.generate({ format: 'iife', - name: `${PROJECT_ID}Bundle` + name: `hydrogenBundle` }); const code = output[0].code; - const bundlePath = resource(outputName, code); - await fs.writeFile(bundlePath, code, "utf8"); - return bundlePath; + await fs.writeFile(outputNameFn(code), code, "utf8"); } -function buildWorkerJsLegacy(inputFile, outputName) { +function buildWorkerJsLegacy(inputFile, outputNameFn) { const polyfillFile = 'src/worker-polyfill.js'; - return buildJsLegacy(inputFile, outputName, null, polyfillFile); + return buildJsLegacy(inputFile, outputNameFn, null, polyfillFile); } -async function buildOffline(version, assetPaths) { - // write web manifest +async function buildOffline(version, assets) { const webManifest = JSON.parse(await fs.readFile(path.join(projectDir, "assets/manifest.json"), "utf8")); + // copy manifest icons for (const icon of webManifest.icons) { let iconData = await fs.readFile(path.join(projectDir, icon.src)); - let iconPath = resource(path.basename(icon.src), iconData); - await fs.writeFile(iconPath, iconData); - icon.src = trim(iconPath); + const iconTargetPath = path.basename(icon.src); + await fs.writeFile(assets.create(iconTargetPath, iconData), iconData); + icon.src = assets.resolve(iconTargetPath); } - // write offline availability - const offlineFiles = [ - assetPaths.cssMainBundle(), - "index.html", - ].concat(assetPaths.cssThemeBundles()) - .concat(webManifest.icons.map(i => i.src)); - // write appcache manifest const appCacheLines = [ `CACHE MANIFEST`, @@ -282,53 +235,51 @@ async function buildOffline(version, assetPaths) { `"*"`, `CACHE`, ]; - appCacheLines.push(assetPaths.jsLegacyBundle(), ...offlineFiles); - const swOfflineFiles = [assetPaths.jsBundle(), ...offlineFiles]; + appCacheLines.push(...assets.allWithout(["hydrogen.js"])); + const swOfflineFiles = assets.allWithout(["hydrogen-legacy.js", "olm_legacy.js"]); const appCacheManifest = appCacheLines.join("\n") + "\n"; await fs.writeFile(path.join(targetDir, "manifest.appcache"), appCacheManifest, "utf8"); // write service worker let swSource = await fs.readFile(path.join(projectDir, "src/service-worker.template.js"), "utf8"); swSource = swSource.replace(`"%%VERSION%%"`, `"${version}"`); swSource = swSource.replace(`"%%OFFLINE_FILES%%"`, JSON.stringify(swOfflineFiles)); - swSource = swSource.replace(`"%%CACHE_FILES%%"`, JSON.stringify(assetPaths.otherAssets())); + // service worker should not have a hashed name as it is polled by the browser for updates await fs.writeFile(path.join(targetDir, "sw.js"), swSource, "utf8"); const manifestJson = JSON.stringify(webManifest); - const manifestPath = resource("manifest.json", manifestJson); - await fs.writeFile(manifestPath, manifestJson, "utf8"); - return manifestPath; + await fs.writeFile(assets.create("manifest.json", manifestJson), manifestJson, "utf8"); } -async function buildCssBundles(buildFn, themes, themeAssets) { +async function buildCssBundles(buildFn, themes, assets) { const bundleCss = await buildFn(path.join(cssSrcDir, "main.css")); - const mainDstPath = resource(`${PROJECT_ID}.css`, bundleCss); - await fs.writeFile(mainDstPath, bundleCss, "utf8"); - const bundlePaths = {main: mainDstPath, themes: {}}; + await fs.writeFile(assets.create(`hydrogen.css`, bundleCss), bundleCss, "utf8"); for (const theme of themes) { - const urlBase = path.join(targetDir, `themes/${theme}/`); + const themeRelPath = `themes/${theme}/`; + const themeRoot = path.join(cssSrcDir, themeRelPath); const assetUrlMapper = ({absolutePath}) => { - const hashedDstPath = themeAssets[absolutePath]; - if (hashedDstPath && hashedDstPath.startsWith(urlBase)) { - return hashedDstPath.substr(urlBase.length); + if (!absolutePath.startsWith(themeRoot)) { + throw new Error("resource is out of theme directory: " + absolutePath); + } + const relPath = absolutePath.substr(themeRoot.length); + const hashedDstPath = assets.resolve(path.join(themeRelPath, relPath)); + if (hashedDstPath) { + return hashedDstPath.substr(themeRelPath.length); } }; - const themeCss = await buildFn(path.join(cssSrcDir, `themes/${theme}/theme.css`), assetUrlMapper); - const themeDstPath = resource(`themes/${theme}/bundle.css`, themeCss); - await fs.writeFile(themeDstPath, themeCss, "utf8"); - bundlePaths.themes[theme] = themeDstPath; + const themeCss = await buildFn(path.join(themeRoot, `theme.css`), assetUrlMapper); + await fs.writeFile(assets.create(path.join(themeRelPath, `bundle.css`), themeCss), themeCss, "utf8"); } - return bundlePaths; } -async function buildCss(entryPath, urlMapper = null) { - const preCss = await fs.readFile(entryPath, "utf8"); - const options = [postcssImport]; - if (urlMapper) { - options.push(postcssUrl({url: urlMapper})); - } - const cssBundler = postcss(options); - const result = await cssBundler.process(preCss, {from: entryPath}); - return result.css; -} +// async function buildCss(entryPath, urlMapper = null) { +// const preCss = await fs.readFile(entryPath, "utf8"); +// const options = [postcssImport]; +// if (urlMapper) { +// options.push(postcssUrl({url: urlMapper})); +// } +// const cssBundler = postcss(options); +// const result = await cssBundler.process(preCss, {from: entryPath}); +// return result.css; +// } async function buildCssLegacy(entryPath, urlMapper = null) { const preCss = await fs.readFile(entryPath, "utf8"); @@ -363,35 +314,21 @@ async function removeDirIfExists(targetDir) { } } -async function copyFolder(srcRoot, dstRoot, filter) { - const assetPaths = {}; +async function copyFolder(srcRoot, dstRoot, filter, assets = null) { + assets = assets || new AssetMap(dstRoot); const dirEnts = await fs.readdir(srcRoot, {withFileTypes: true}); for (const dirEnt of dirEnts) { const dstPath = path.join(dstRoot, dirEnt.name); const srcPath = path.join(srcRoot, dirEnt.name); if (dirEnt.isDirectory()) { await fs.mkdir(dstPath); - Object.assign(assetPaths, await copyFolder(srcPath, dstPath, filter)); + await copyFolder(srcPath, dstPath, filter, assets); } else if ((dirEnt.isFile() || dirEnt.isSymbolicLink()) && (!filter || filter(srcPath))) { const content = await fs.readFile(srcPath); - const hashedDstPath = resource(dstPath, content); - await fs.writeFile(hashedDstPath, content); - assetPaths[srcPath] = hashedDstPath; + await fs.writeFile(assets.create(dstPath, content), content); } } - return assetPaths; -} - -function resource(relPath, content) { - let fullPath = relPath; - if (!path.isAbsolute(relPath)) { - fullPath = path.join(targetDir, relPath); - } - const hash = contentHash(Buffer.from(content)); - const dir = path.dirname(fullPath); - const extname = path.extname(fullPath); - const basename = path.basename(fullPath, extname); - return path.join(dir, `${basename}-${hash}${extname}`); + return assets; } function contentHash(str) { @@ -400,5 +337,69 @@ function contentHash(str) { return hasher.digest(); } +class AssetMap { + constructor(targetDir) { + // remove last / if any, so substr in create works well + this._targetDir = path.resolve(targetDir); + this._assets = new Map(); + } + + _toRelPath(resourcePath) { + let relPath = resourcePath; + if (path.isAbsolute(resourcePath)) { + if (!resourcePath.startsWith(this._targetDir)) { + throw new Error(`absolute path ${resourcePath} that is not within target dir ${this._targetDir}`); + } + relPath = resourcePath.substr(this._targetDir.length + 1); // + 1 for the / + } + return relPath; + } + + create(resourcePath, content) { + const relPath = this._toRelPath(resourcePath); + const hash = contentHash(Buffer.from(content)); + const dir = path.dirname(relPath); + const extname = path.extname(relPath); + const basename = path.basename(relPath, extname); + const dstRelPath = path.join(dir, `${basename}-${hash}${extname}`); + this._assets.set(relPath, dstRelPath); + return path.join(this._targetDir, dstRelPath); + } + + get directory() { + return this._targetDir; + } + + resolve(resourcePath) { + const relPath = this._toRelPath(resourcePath); + const result = this._assets.get(relPath); + if (!result) { + throw new Error(`unknown path: ${relPath}, only know ${Array.from(this._assets.keys()).join(", ")}`); + } + return result; + } + + addSubMap(assetMap) { + if (!assetMap.directory.startsWith(this.directory)) { + throw new Error(`map directory doesn't start with this directory: ${assetMap.directory} ${this.directory}`); + } + console.log("adding submap from", assetMap.directory, this.directory); + const relSubRoot = assetMap.directory.substr(this.directory.length + 1); + for (const [key, value] of assetMap._assets.entries()) { + this._assets.set(path.join(relSubRoot, key), path.join(relSubRoot, value)); + } + } + + all() { + return Array.from(this._assets.values()); + } + + allWithout(excluded) { + excluded = excluded.map(p => this.resolve(p)); + return this.all().filter(p => excluded.indexOf(p) === -1); + } +} + + build().catch(err => console.error(err));