import {Express, Router} from "express"; import * as fs from "fs"; import path from "path"; import config from "config"; import ApplicationComponent from "../ApplicationComponent"; import {logger} from "../Logger"; import {ServerError} from "../HttpError"; import * as crypto from "crypto"; import requireFromString from "require-from-string"; import "svelte/register"; import {compile, preprocess} from "svelte/compiler"; import {sveltePreprocess} from "svelte-preprocess/dist/autoProcess"; import chokidar from "chokidar"; import {CssResult} from "svelte/types/compiler/interfaces"; import {PreRenderedChunk, rollup, RollupBuild, RollupOptions, RollupWatcher, RollupWatchOptions, watch} from "rollup"; import svelte from "rollup-plugin-svelte"; import resolve from "@rollup/plugin-node-resolve"; import commonjs from "@rollup/plugin-commonjs"; import {terser} from "rollup-plugin-terser"; import cssOnlyRollupPlugin from "rollup-plugin-css-only"; import livereloadRollupPlugin from "rollup-plugin-livereload"; const BACKEND_CODE_PREFIX = 'swaf.'; const COMPILED_SVELTE_EXTENSION = '.swafview'; export default class FrontendToolsComponent extends ApplicationComponent { public static getSveltePreCompileSeparator(canonicalViewName: string): string { return '\n---' + crypto.createHash('sha1') .update(path.basename(path.resolve(canonicalViewName))) .digest('base64') + '---\n'; } private content: Map = new Map(); private readonly viewPaths: string[]; private readonly svelteDevViewsPath: string; private readonly dependencyCache: Record> = {}; private readonly backendCodeCache: Record = {}; private readonly fileCache: Record = {}; private rollup?: RollupBuild | RollupWatcher; public constructor( private readonly publicAssetsPath: string, private readonly svelteOutputPath: string, ...viewPaths: string[] ) { super(); this.viewPaths = [ ...viewPaths.map(p => path.resolve(p)), path.resolve(__dirname, '../../../views'), path.resolve(__dirname, '../../views'), path.resolve(__dirname, '../views'), ]; this.svelteDevViewsPath = path.resolve('views'); if (!fs.existsSync(svelteOutputPath)) { fs.mkdirSync(svelteOutputPath); } } public async start(app: Express): Promise { await this.cachePublicAssets(); await this.preCompileSvelteViews(); const watcher = chokidar.watch(this.svelteDevViewsPath, {persistent: true}); watcher.on('ready', () => { logger.debug('Watching svelte assets for changes'); watcher.on('add', (path) => { if (path.endsWith('.svelte')) { this.resetBundle() .then(() => this.preCompileSvelte(path, true)) .then(() => this.bundle(...Object.keys(this.backendCodeCache))) .catch(err => logger.error(err)); } }); watcher.on('change', (path) => { if (path.endsWith('.svelte')) { delete this.backendCodeCache[this.toCanonicalViewName(path)]; this.preCompileSvelte(path, true) .then(() => this.bundle(...Object.keys(this.backendCodeCache))) .catch(err => logger.error(err)); } }); }); app.engine('svelte', (path, options, callback) => { this.renderSvelte(path, options as Record, callback) .catch(err => callback(err)); }); app.set('views', this.viewPaths); app.set('view engine', 'svelte'); } public async stop(): Promise { await this.resetBundle(); } private async cachePublicAssets(): Promise { if (config.get('view.enable_asset_cache')) { logger.info('Caching assets from', this.publicAssetsPath, '...'); await this.forEachFileInDirRecursively(this.publicAssetsPath, file => this.cacheFile(file)); } else { logger.info('Asset cache disabled.'); } } private async preCompileSvelteViews(): Promise { logger.info('Pre-compiling svelte views...'); await this.forEachFileInDirRecursively(this.svelteDevViewsPath, async file => { if (file.endsWith('.svelte')) { await this.preCompileSvelte(file); } }); await this.bundle(...Object.keys(this.backendCodeCache)); } public async handle(router: Router): Promise { router.use((req, res, next) => { res.locals.inlineAsset = (urlPath: string) => { if (!config.get('view.enable_asset_cache')) { return fs.readFileSync(path.resolve(this.publicAssetsPath + urlPath)); } const content = this.content.get(urlPath); if (!content) { throw new ServerError('Asset ' + path + ' was not loaded.'); } return content; }; next(); }); } private async forEachFileInDirRecursively(dir: string, consumer: (file: string) => Promise): Promise { await new Promise((resolve, reject) => { fs.readdir(dir, (err, files) => { if (err) return reject(err); resolve(Promise.all(files.map(file => new Promise((resolve, reject) => { file = path.join(dir, file); fs.stat(file, (err, stat) => { if (err) return reject(err); if (stat.isDirectory()) { resolve(this.forEachFileInDirRecursively(file, consumer)); } else { resolve(consumer(file)); } }); })))); }); }); } private async cacheFile(file: string): Promise { await new Promise((resolve, reject) => { fs.readFile(file, (err, data) => { if (err) return reject(data); const urlPath = file.replace(this.publicAssetsPath, ''); this.content.set(urlPath, data.toString()); logger.debug('Loaded', file, 'as', urlPath); resolve(); }); }); } private async preCompileSvelte(file: string, alsoCompileDependents: boolean = false): Promise { file = path.relative('.', file); const canonicalViewName = this.toCanonicalViewName(file); const intermediateFile = path.join(this.svelteOutputPath, canonicalViewName); logger.debug(canonicalViewName + ' > ', 'Pre-compiling', file, '->', intermediateFile); const source = await new Promise((resolve, reject) => { fs.readFile(file, (err, data) => { if (err) return reject(err); resolve(data.toString()); }); }); const allBackendLines: string[] = []; for (const dependency of this.resolveDependencies(source, canonicalViewName)) { allBackendLines.push(...(await this.replaceBackendCode(dependency)).backendLines); } const {backendReplacedCode, backendLines} = await this.replaceBackendCode(canonicalViewName, source); allBackendLines.push(...backendLines); const preprocessedCode = await this.preProcessSvelte(backendReplacedCode, intermediateFile, canonicalViewName); // Server Side Render (initial HTML, no-js) const ssr = this.compileSvelteSsr(preprocessedCode, intermediateFile, canonicalViewName); const separator = FrontendToolsComponent.getSveltePreCompileSeparator(canonicalViewName); const finalCode = [ [...new Set(allBackendLines).values()].join('\n'), ssr.head, ssr.html, ssr.css.code, ].join(separator); const swafViewFile = path.join(this.svelteOutputPath, canonicalViewName + COMPILED_SVELTE_EXTENSION); await new Promise((resolve, reject) => { fs.mkdir(path.dirname(swafViewFile), {recursive: true}, (err) => { if (err) return reject(err); resolve(); }); }); await new Promise((resolve, reject) => fs.writeFile(swafViewFile, finalCode, err => { if (err) return reject(err); resolve(); })); if (alsoCompileDependents && Object.keys(this.dependencyCache).indexOf(canonicalViewName) >= 0) { logger.debug(canonicalViewName + ' > ', 'Compiling dependents...'); for (const dependent of [...this.dependencyCache[canonicalViewName]]) { await this.preCompileSvelte(await this.resolveViewFromCanonicalName(dependent), true); } } } private resolveDependencies(source: string, canonicalViewName: string): string[] { const dependencies: string[] = []; for (const match of source.matchAll(/import .+ from ['"](.+?\.svelte)['"];/gm)) { dependencies.push(path.join(path.dirname(canonicalViewName), match[1])); } // Clear existing links from cache for (const dependency of Object.keys(this.dependencyCache)) { this.dependencyCache[dependency].delete(canonicalViewName); } // Add new links to cache for (const dependency of dependencies) { if (Object.keys(this.dependencyCache).indexOf(dependency) < 0) { this.dependencyCache[dependency] = new Set(); } this.dependencyCache[dependency].add(canonicalViewName); } return dependencies; } private async replaceBackendCode(canonicalViewName: string, code?: string): Promise<{ backendReplacedCode: string, backendLines: string[], }> { // Cache if (Object.keys(this.backendCodeCache).indexOf(canonicalViewName) >= 0) { return this.backendCodeCache[canonicalViewName]; } const outputFile = path.join(this.svelteOutputPath, canonicalViewName); await new Promise((resolve, reject) => { fs.mkdir(path.dirname(outputFile), {recursive: true}, (err) => { if (err) return reject(err); resolve(); }); }); // Read source file if code was not already provided if (!code) { const file = await this.resolveViewFromCanonicalName(canonicalViewName); code = await new Promise((resolve, reject) => { fs.readFile(file, (err, data) => { if (err) return reject(err); resolve(data.toString()); }); }); } // Skip replace if there is no swaf export if (code.indexOf(`export let swaf = {};`) < 0) { const generated = { backendReplacedCode: code, backendLines: [], }; await new Promise((resolve, reject) => { fs.writeFile(outputFile, generated.backendReplacedCode, (err) => { if (err) return reject(err); resolve(); }); }); this.backendCodeCache[canonicalViewName] = generated; return generated; } let output = code; const backendLines = new Set(); let index = 0; while ((index = output.indexOf(BACKEND_CODE_PREFIX, index + 1)) >= 0) { // Escaping if (index > 0 && output[index - 1] === '\\') { const isEscapingEscaped: boolean = index > 1 && output[index - 2] === '\\'; output = output.substring(0, index - 1 - (isEscapingEscaped ? 1 : 0)) + output.substring(index, output.length); continue; } const startIndex = index + BACKEND_CODE_PREFIX.length; let endIndex = startIndex; let struct = 0; while (endIndex < output.length) { if (['(', '[', '{'].indexOf(output[endIndex]) >= 0) struct++; if ([')', ']', '}'].indexOf(output[endIndex]) >= 0) { struct--; if (struct <= 0) { if (struct === 0) endIndex++; break; } } if ([' ', '\n', '<'].indexOf(output[endIndex]) >= 0 && struct === 0) break; endIndex++; } let backendLine = output.substring(startIndex, endIndex); if (backendLine.match(/([^()]+)\((.+?)\)/)) { backendLine = backendLine.replace(/([^()]+)\((.+?)\)/, "'$1', [$2]"); } else { backendLine = backendLine.replace(/([^()]+)/, "'$1'"); } backendLines.add(backendLine); output = output.substring(0, index) + 'swaf(' + backendLine + ')' + output.substring(endIndex, output.length); } const generated = { backendReplacedCode: output, backendLines: [...backendLines], }; await new Promise((resolve, reject) => { fs.writeFile(outputFile, generated.backendReplacedCode, (err) => { if (err) return reject(err); resolve(); }); }); this.backendCodeCache[canonicalViewName] = generated; return generated; } private async preProcessSvelte(code: string, filename: string, canonicalViewName: string): Promise { logger.debug(canonicalViewName + ' > ', 'Preprocessing svelte', filename); const preprocessed = await preprocess( code, sveltePreprocess({ typescript: { tsconfigFile: 'tsconfig.views.json', }, }), { filename: filename, }, ); return preprocessed.code; } private compileSvelteSsr(code: string, filename: string, canonicalViewName: string): { head: string, css: CssResult, html: string, } { logger.debug(canonicalViewName + ' > ', 'Compiling svelte ssr', filename); const svelteSsr = compile(code, { dev: config.get('view.dev'), generate: 'ssr', format: 'cjs', cssOutputFilename: filename + '.css', }); const locals = this.getGlobals(); return requireFromString(svelteSsr.js.code, filename).default.render({ swaf: (key: string, args?: unknown[]) => { if (!args) return locals[key]; const f = locals[key]; if (typeof f !== 'function') throw new Error(key + ' is not a function.'); return f.call(locals, ...args); }, }); } private async bundle(...canonicalViewNames: string[]): Promise { logger.debug('Bundling'); const production = !config.get('view.dev'); const options: RollupOptions | RollupWatchOptions = { input: canonicalViewNames.map(name => path.join(this.svelteOutputPath, name)), output: { sourcemap: true, format: 'es', name: 'bundle', dir: path.join(this.publicAssetsPath, 'js'), entryFileNames: (chunkInfo: PreRenderedChunk): string => { const name = chunkInfo.facadeModuleId ? path.relative(this.svelteOutputPath, chunkInfo.facadeModuleId) : chunkInfo.name; return name + '.js'; }, }, plugins: [ svelte({ preprocess: sveltePreprocess({ typescript: { tsconfigFile: 'tsconfig.views.json', }, }), compilerOptions: { // enable run-time checks when not in production dev: !production, hydratable: true, }, }), // we'll extract any component CSS out into // a separate file - better for performance cssOnlyRollupPlugin({output: 'bundle.css'}), // If you have external dependencies installed from // npm, you'll most likely need these plugins. In // some cases you'll need additional configuration - // consult the documentation for details: // https://github.com/rollup/plugins/tree/master/packages/commonjs resolve({ browser: true, dedupe: ['svelte'], }), commonjs(), ], watch: { buildDelay: 1000, clearScreen: false, }, }; if (production) { // If we're building for production (npm run build // instead of npm run dev), minify options.plugins?.push(terser()); } else { // Watch the `public` directory and refresh the // browser on changes when not in production const plugin = livereloadRollupPlugin('public'); options.plugins?.push(plugin); } for (const name of canonicalViewNames) { await new Promise((resolve, reject) => { fs.mkdir(path.dirname(path.join(this.publicAssetsPath, 'js', name)), {recursive: true}, err => { if (err) return reject(err); resolve(); }); }); } if (production) { if (!this.rollup) { this.rollup = await rollup(options); await this.rollup.write({ format: 'es', dir: path.join(this.publicAssetsPath, 'js'), }); } } else { if (!this.rollup) { this.rollup = watch(options); this.rollup.on('event', (event) => { if (event.code === 'ERROR' || event.code === 'BUNDLE_END') { event.result?.close().catch(err => logger.error(err)); logger.debug('Bundled from watch'); } }); } } } private async resetBundle(): Promise { if (this.rollup) { logger.debug('Stopping rollup...'); await this.rollup.close(); this.rollup = undefined; } } private async renderSvelte( file: string, locals: Record, callback: (err: Error | null, rendered?: string) => void, ): Promise { const canonicalViewName = this.toCanonicalViewName(file); const actualFile = path.join(this.svelteOutputPath, canonicalViewName + COMPILED_SVELTE_EXTENSION); if (!config.get('view.enable_asset_cache')) delete this.fileCache[actualFile]; const view = await this.getFileContentsFromCache(actualFile); const templateFile = await this.resolveViewFromCanonicalName('layouts/svelte_layout.html'); if (!config.get('view.enable_asset_cache')) delete this.fileCache[templateFile]; let output = await this.getFileContentsFromCache(templateFile); const [ backendLines, head, html, css, ] = view.split(FrontendToolsComponent.getSveltePreCompileSeparator(canonicalViewName)); locals = Object.assign(this.getGlobals(), locals); const localMap: Record = {}; backendLines.split('\n').forEach(line => { const key = line.substring(1, line.indexOf(',') >= 0 ? line.indexOf(',') - 1 : line.length - 1); if (line.indexOf('[') >= 0) { const args = line.substring(line.indexOf('[') + 1, line.length - 1) .split(/, *?/) .map(arg => { if (arg.startsWith("'")) return '"' + arg.substring(1, arg.length - 1) + '"'; return arg; }) .map(arg => JSON.parse(arg)); const f = locals[key]; if (typeof f !== 'function') throw new Error(key + ' is not a function.'); localMap[`'${key}', ${JSON.stringify(args)}`] = f.call(locals, ...args); } else { localMap[`'${key}'`] = locals[key]; } }); const props = JSON.stringify(localMap); output = output.replace('%canonicalViewName%', canonicalViewName); output = output.replace('%props%', props); output = output.replace('%head%', head); output = output.replace('%html%', html); output = output.replace('%css%', css); callback(null, output); } private async getFileContentsFromCache(file: string): Promise { if (!this.fileCache[file]) { this.fileCache[file] = await new Promise((resolve, reject) => { fs.readFile(file, (err, data) => { if (err) return reject(err); resolve(data.toString()); }); }); } return this.fileCache[file]; } private toCanonicalViewName(file: string): string { const resolvedFilePath = path.resolve(file); let canonicalViewName: string | null = null; for (const viewPath of this.viewPaths) { if (resolvedFilePath.startsWith(viewPath)) { canonicalViewName = resolvedFilePath.substring(viewPath.length + 1); } } if (!canonicalViewName) throw new Error('view ' + file + ' not found'); return canonicalViewName; } private async resolveViewFromCanonicalName(canonicalName: string): Promise { for (const viewPath of this.viewPaths) { const tryPath = path.join(viewPath, canonicalName); if (await new Promise((resolve, reject) => { fs.stat(tryPath, (err) => { if (err == null) { resolve(true); } else if (err.code === 'ENOENT') { resolve(false); } else { reject(err); } }); })) { return tryPath; } } throw new Error('View not found from canonical name ' + canonicalName); } /** * TODO: add a way to add locals from anywhere */ private getGlobals(): Record { return { route: (name: string) => 'unimplemented route ' + name, direct: 'access', }; } } export type SourceMap = string | Record | undefined;