From e5a9b9908db6c100c9e8d988844cc32c65dfbf5d Mon Sep 17 00:00:00 2001 From: Alice Gaudon Date: Wed, 24 Mar 2021 21:41:13 +0100 Subject: [PATCH] Refactor and make it nicer codewise --- package.json | 2 +- src/TestApp.ts | 3 +- src/Utils.ts | 39 ++ src/components/FrontendToolsComponent.ts | 616 +---------------------- src/frontend/SvelteViewEngine.ts | 427 ++++++++++++++++ src/frontend/ViewEngine.ts | 142 ++++++ src/utils/FileCache.ts | 35 ++ 7 files changed, 673 insertions(+), 591 deletions(-) create mode 100644 src/frontend/SvelteViewEngine.ts create mode 100644 src/frontend/ViewEngine.ts create mode 100644 src/utils/FileCache.ts diff --git a/package.json b/package.json index 797e74d..8318a49 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "prepare-sources": "node scripts/prepare-sources.js", "compile": "yarn clean && tsc", "build": "yarn prepare-sources && yarn compile && node scripts/dist.js", - "dev": "yarn prepare-sources && concurrently -k -n \"Typescript,Node,Webpack,Maildev\" -p \"[{name}]\" -c \"blue,green,red,yellow\" \"tsc --watch\" \"nodemon -i public\" \"maildev\"", + "dev": "yarn prepare-sources && concurrently -k -n \"Typescript,Node,Webpack,Maildev\" -p \"[{name}]\" -c \"blue,green,red,yellow\" \"tsc --watch\" \"nodemon -i build\" \"maildev\"", "lint": "eslint .", "release": "yarn build && yarn lint && yarn test && cd dist && yarn publish" }, diff --git a/src/TestApp.ts b/src/TestApp.ts index 806f089..9c761ed 100644 --- a/src/TestApp.ts +++ b/src/TestApp.ts @@ -33,6 +33,7 @@ import PreviousUrlComponent from "./components/PreviousUrlComponent"; import AddNameChangedAtToUsersMigration from "./auth/migrations/AddNameChangedAtToUsersMigration"; import BackendController from "./helpers/BackendController"; import FrontendToolsComponent from "./components/FrontendToolsComponent"; +import SvelteViewEngine from "./frontend/SvelteViewEngine"; import packageJson = require('./package.json'); export const MIGRATIONS = [ @@ -76,7 +77,7 @@ export default class TestApp extends Application { // Dynamic views and routes this.use(new NunjucksComponent(['test/views', 'views'])); - this.use(new FrontendToolsComponent('public', 'build')); + this.use(new FrontendToolsComponent(new SvelteViewEngine('build/views', 'public', 'views'))); this.use(new PreviousUrlComponent()); // Services diff --git a/src/Utils.ts b/src/Utils.ts index 53d0891..53f56cc 100644 --- a/src/Utils.ts +++ b/src/Utils.ts @@ -1,3 +1,7 @@ +import fs from "fs"; +import path from "path"; +import * as util from "util"; + export async function sleep(ms: number): Promise { return await new Promise(resolve => { setTimeout(() => resolve(), ms); @@ -44,3 +48,38 @@ export function getMethods(obj: T): string[] } while (currentObj); return [...properties.keys()].filter(item => typeof obj[item] === 'function'); } + +export const afs = { + readFile: util.promisify(fs.readFile), + writeFile: util.promisify(fs.writeFile), + mkdir: util.promisify(fs.mkdir), + readdir: util.promisify(fs.readdir), + exists: async (file: string): Promise => + await new Promise((resolve, reject) => + fs.stat(file, (err) => { + if (err == null) { + resolve(true); + } else if (err.code === 'ENOENT') { + resolve(false); + } else { + reject(err); + } + })), +}; + +export async function readdirRecursively(dir: string, consumer: (file: string) => Promise): Promise { + const files = await afs.readdir(dir); + + await Promise.all(files.map(file => new Promise((resolve, reject) => { + file = path.join(dir, file); + fs.stat(file, (err, stat) => { + if (err) return reject(err); + + if (stat.isDirectory()) { + resolve(readdirRecursively(file, consumer)); + } else { + resolve(consumer(file)); + } + }); + }))); +} diff --git a/src/components/FrontendToolsComponent.ts b/src/components/FrontendToolsComponent.ts index 3f35181..e375b9d 100644 --- a/src/components/FrontendToolsComponent.ts +++ b/src/components/FrontendToolsComponent.ts @@ -1,622 +1,60 @@ import {Express, Router} from "express"; -import * as fs from "fs"; import path from "path"; import config from "config"; import ApplicationComponent from "../ApplicationComponent"; import {logger} from "../Logger"; -import {ServerError} from "../HttpError"; -import * as crypto from "crypto"; -import requireFromString from "require-from-string"; import "svelte/register"; -import {compile, preprocess} from "svelte/compiler"; -import {sveltePreprocess} from "svelte-preprocess/dist/autoProcess"; -import chokidar from "chokidar"; -import {CssResult} from "svelte/types/compiler/interfaces"; -import {PreRenderedChunk, rollup, RollupBuild, RollupOptions, RollupWatcher, RollupWatchOptions, watch} from "rollup"; -import svelte from "rollup-plugin-svelte"; -import resolve from "@rollup/plugin-node-resolve"; -import commonjs from "@rollup/plugin-commonjs"; -import {terser} from "rollup-plugin-terser"; -import cssOnlyRollupPlugin from "rollup-plugin-css-only"; -import livereloadRollupPlugin from "rollup-plugin-livereload"; - -const BACKEND_CODE_PREFIX = 'swaf.'; -const COMPILED_SVELTE_EXTENSION = '.swafview'; +import ViewEngine from "../frontend/ViewEngine"; +import {readdirRecursively} from "../Utils"; +import FileCache from "../utils/FileCache"; export default class FrontendToolsComponent extends ApplicationComponent { - public static getSveltePreCompileSeparator(canonicalViewName: string): string { - return '\n---' + - crypto.createHash('sha1') - .update(path.basename(path.resolve(canonicalViewName))) - .digest('base64') + - '---\n'; - } - - private content: Map = new Map(); - private readonly viewPaths: string[]; - private readonly svelteDevViewsPath: string; - - private readonly dependencyCache: Record> = {}; - private readonly backendCodeCache: Record = {}; - private readonly fileCache: Record = {}; - - private rollup?: RollupBuild | RollupWatcher; + private readonly publicAssetsCache: FileCache = new FileCache(); public constructor( - private readonly publicAssetsPath: string, - private readonly svelteOutputPath: string, - ...viewPaths: string[] + private readonly viewEngine: ViewEngine, ) { super(); - this.viewPaths = [ - ...viewPaths.map(p => path.resolve(p)), - path.resolve(__dirname, '../../../views'), - path.resolve(__dirname, '../../views'), - path.resolve(__dirname, '../views'), - ]; - this.svelteDevViewsPath = path.resolve('views'); - - if (!fs.existsSync(svelteOutputPath)) { - fs.mkdirSync(svelteOutputPath); - } } public async start(app: Express): Promise { - await this.cachePublicAssets(); + // Cache public assets + if (config.get('view.enable_asset_cache')) { + logger.info('Caching assets from', this.viewEngine.getPublicDir(), '...'); + await readdirRecursively( + this.viewEngine.getPublicDir(), + async file => await this.publicAssetsCache.load(file), + ); + } else { + logger.info('Asset cache disabled.'); + } - await this.preCompileSvelteViews(); + // Pre-compile and watch in dev mode + if (config.get('view.dev')) { + await this.viewEngine.preCompileAll(); + await this.viewEngine.watch(); + } - const watcher = chokidar.watch(this.svelteDevViewsPath, {persistent: true}); - watcher.on('ready', () => { - logger.debug('Watching svelte assets for changes'); - - watcher.on('add', (path) => { - if (path.endsWith('.svelte')) { - this.resetBundle() - .then(() => this.preCompileSvelte(path, true)) - .then(() => this.bundle(...Object.keys(this.backendCodeCache))) - .catch(err => logger.error(err)); - } - }); - - watcher.on('change', (path) => { - if (path.endsWith('.svelte')) { - delete this.backendCodeCache[this.toCanonicalViewName(path)]; - this.preCompileSvelte(path, true) - .then(() => this.bundle(...Object.keys(this.backendCodeCache))) - .catch(err => logger.error(err)); - } - }); - }); - - app.engine('svelte', (path, options, callback) => { - this.renderSvelte(path, options as Record, callback) + // Setup express view engine + app.engine(this.viewEngine.getExtension(), (path, options, callback) => { + this.viewEngine.render(path, options as Record, callback) .catch(err => callback(err)); }); - app.set('views', this.viewPaths); + app.set('views', this.viewEngine.getViewPaths()); app.set('view engine', 'svelte'); } public async stop(): Promise { - await this.resetBundle(); - } - - private async cachePublicAssets(): Promise { - if (config.get('view.enable_asset_cache')) { - logger.info('Caching assets from', this.publicAssetsPath, '...'); - await this.forEachFileInDirRecursively(this.publicAssetsPath, file => this.cacheFile(file)); - } else { - logger.info('Asset cache disabled.'); - } - } - - private async preCompileSvelteViews(): Promise { - logger.info('Pre-compiling svelte views...'); - await this.forEachFileInDirRecursively(this.svelteDevViewsPath, async file => { - if (file.endsWith('.svelte')) { - await this.preCompileSvelte(file); - } - }); - await this.bundle(...Object.keys(this.backendCodeCache)); + await this.viewEngine.end?.(); } public async handle(router: Router): Promise { router.use((req, res, next) => { res.locals.inlineAsset = (urlPath: string) => { - if (!config.get('view.enable_asset_cache')) { - return fs.readFileSync(path.resolve(this.publicAssetsPath + urlPath)); - } - - const content = this.content.get(urlPath); - if (!content) { - throw new ServerError('Asset ' + path + ' was not loaded.'); - } - return content; + return this.publicAssetsCache.getOrFail(path.join(this.viewEngine.getPublicDir(), urlPath)); }; + next(); }); } - - private async forEachFileInDirRecursively(dir: string, consumer: (file: string) => Promise): Promise { - await new Promise((resolve, reject) => { - fs.readdir(dir, (err, files) => { - if (err) return reject(err); - - resolve(Promise.all(files.map(file => new Promise((resolve, reject) => { - file = path.join(dir, file); - fs.stat(file, (err, stat) => { - if (err) return reject(err); - - if (stat.isDirectory()) { - resolve(this.forEachFileInDirRecursively(file, consumer)); - } else { - resolve(consumer(file)); - } - }); - })))); - }); - }); - } - - private async cacheFile(file: string): Promise { - await new Promise((resolve, reject) => { - fs.readFile(file, (err, data) => { - if (err) return reject(data); - - const urlPath = file.replace(this.publicAssetsPath, ''); - this.content.set(urlPath, data.toString()); - logger.debug('Loaded', file, 'as', urlPath); - resolve(); - }); - }); - } - - private async preCompileSvelte(file: string, alsoCompileDependents: boolean = false): Promise { - file = path.relative('.', file); - const canonicalViewName = this.toCanonicalViewName(file); - const intermediateFile = path.join(this.svelteOutputPath, canonicalViewName); - - logger.debug(canonicalViewName + ' > ', 'Pre-compiling', file, '->', intermediateFile); - const source = await new Promise((resolve, reject) => { - fs.readFile(file, (err, data) => { - if (err) return reject(err); - - resolve(data.toString()); - }); - }); - - const allBackendLines: string[] = []; - for (const dependency of this.resolveDependencies(source, canonicalViewName)) { - allBackendLines.push(...(await this.replaceBackendCode(dependency)).backendLines); - } - - const {backendReplacedCode, backendLines} = await this.replaceBackendCode(canonicalViewName, source); - allBackendLines.push(...backendLines); - - const preprocessedCode = await this.preProcessSvelte(backendReplacedCode, intermediateFile, canonicalViewName); - - // Server Side Render (initial HTML, no-js) - const ssr = this.compileSvelteSsr(preprocessedCode, intermediateFile, canonicalViewName); - - const separator = FrontendToolsComponent.getSveltePreCompileSeparator(canonicalViewName); - const finalCode = [ - [...new Set(allBackendLines).values()].join('\n'), - ssr.head, - ssr.html, - ssr.css.code, - ].join(separator); - - const swafViewFile = path.join(this.svelteOutputPath, canonicalViewName + COMPILED_SVELTE_EXTENSION); - await new Promise((resolve, reject) => { - fs.mkdir(path.dirname(swafViewFile), {recursive: true}, (err) => { - if (err) return reject(err); - resolve(); - }); - }); - await new Promise((resolve, reject) => fs.writeFile(swafViewFile, finalCode, err => { - if (err) return reject(err); - resolve(); - })); - - if (alsoCompileDependents && Object.keys(this.dependencyCache).indexOf(canonicalViewName) >= 0) { - logger.debug(canonicalViewName + ' > ', 'Compiling dependents...'); - for (const dependent of [...this.dependencyCache[canonicalViewName]]) { - await this.preCompileSvelte(await this.resolveViewFromCanonicalName(dependent), true); - } - } - } - - private resolveDependencies(source: string, canonicalViewName: string): string[] { - const dependencies: string[] = []; - - for (const match of source.matchAll(/import .+ from ['"](.+?\.svelte)['"];/gm)) { - dependencies.push(path.join(path.dirname(canonicalViewName), match[1])); - } - - // Clear existing links from cache - for (const dependency of Object.keys(this.dependencyCache)) { - this.dependencyCache[dependency].delete(canonicalViewName); - } - - // Add new links to cache - for (const dependency of dependencies) { - if (Object.keys(this.dependencyCache).indexOf(dependency) < 0) { - this.dependencyCache[dependency] = new Set(); - } - this.dependencyCache[dependency].add(canonicalViewName); - } - - return dependencies; - } - - private async replaceBackendCode(canonicalViewName: string, code?: string): Promise<{ - backendReplacedCode: string, - backendLines: string[], - }> { - // Cache - if (Object.keys(this.backendCodeCache).indexOf(canonicalViewName) >= 0) { - return this.backendCodeCache[canonicalViewName]; - } - - const outputFile = path.join(this.svelteOutputPath, canonicalViewName); - await new Promise((resolve, reject) => { - fs.mkdir(path.dirname(outputFile), {recursive: true}, (err) => { - if (err) return reject(err); - resolve(); - }); - }); - - // Read source file if code was not already provided - if (!code) { - const file = await this.resolveViewFromCanonicalName(canonicalViewName); - code = await new Promise((resolve, reject) => { - fs.readFile(file, (err, data) => { - if (err) return reject(err); - - resolve(data.toString()); - }); - }); - } - - // Skip replace if there is no swaf export - if (code.indexOf(`export let swaf = {};`) < 0) { - const generated = { - backendReplacedCode: code, - backendLines: [], - }; - await new Promise((resolve, reject) => { - fs.writeFile(outputFile, generated.backendReplacedCode, (err) => { - if (err) return reject(err); - - resolve(); - }); - }); - this.backendCodeCache[canonicalViewName] = generated; - return generated; - } - - - let output = code; - const backendLines = new Set(); - - let index = 0; - while ((index = output.indexOf(BACKEND_CODE_PREFIX, index + 1)) >= 0) { - // Escaping - if (index > 0 && output[index - 1] === '\\') { - const isEscapingEscaped: boolean = index > 1 && output[index - 2] === '\\'; - output = output.substring(0, index - 1 - (isEscapingEscaped ? 1 : 0)) + - output.substring(index, output.length); - continue; - } - - const startIndex = index + BACKEND_CODE_PREFIX.length; - let endIndex = startIndex; - let struct = 0; - - while (endIndex < output.length) { - if (['(', '[', '{'].indexOf(output[endIndex]) >= 0) struct++; - if ([')', ']', '}'].indexOf(output[endIndex]) >= 0) { - struct--; - if (struct <= 0) { - if (struct === 0) endIndex++; - break; - } - } - if ([' ', '\n', '<'].indexOf(output[endIndex]) >= 0 && struct === 0) break; - endIndex++; - } - - let backendLine = output.substring(startIndex, endIndex); - if (backendLine.match(/([^()]+)\((.+?)\)/)) { - backendLine = backendLine.replace(/([^()]+)\((.+?)\)/, "'$1', [$2]"); - } else { - backendLine = backendLine.replace(/([^()]+)/, "'$1'"); - } - - backendLines.add(backendLine); - output = output.substring(0, index) + - 'swaf(' + backendLine + ')' + - output.substring(endIndex, output.length); - } - - const generated = { - backendReplacedCode: output, - backendLines: [...backendLines], - }; - await new Promise((resolve, reject) => { - fs.writeFile(outputFile, generated.backendReplacedCode, (err) => { - if (err) return reject(err); - - resolve(); - }); - }); - this.backendCodeCache[canonicalViewName] = generated; - - return generated; - } - - private async preProcessSvelte(code: string, filename: string, canonicalViewName: string): Promise { - logger.debug(canonicalViewName + ' > ', 'Preprocessing svelte', filename); - const preprocessed = await preprocess( - code, - sveltePreprocess({ - typescript: { - tsconfigFile: 'tsconfig.views.json', - }, - }), - { - filename: filename, - }, - ); - - return preprocessed.code; - } - - private compileSvelteSsr(code: string, filename: string, canonicalViewName: string): { - head: string, - css: CssResult, - html: string, - } { - logger.debug(canonicalViewName + ' > ', 'Compiling svelte ssr', filename); - const svelteSsr = compile(code, { - dev: config.get('view.dev'), - generate: 'ssr', - format: 'cjs', - cssOutputFilename: filename + '.css', - }); - - const locals = this.getGlobals(); - - return requireFromString(svelteSsr.js.code, filename).default.render({ - swaf: (key: string, args?: unknown[]) => { - if (!args) return locals[key]; - - const f = locals[key]; - if (typeof f !== 'function') throw new Error(key + ' is not a function.'); - return f.call(locals, ...args); - }, - }); - } - - private async bundle(...canonicalViewNames: string[]): Promise { - logger.debug('Bundling'); - - const production = !config.get('view.dev'); - const options: RollupOptions | RollupWatchOptions = { - input: canonicalViewNames.map(name => path.join(this.svelteOutputPath, name)), - output: { - sourcemap: true, - format: 'es', - name: 'bundle', - dir: path.join(this.publicAssetsPath, 'js'), - entryFileNames: (chunkInfo: PreRenderedChunk): string => { - const name = chunkInfo.facadeModuleId ? - path.relative(this.svelteOutputPath, chunkInfo.facadeModuleId) : - chunkInfo.name; - return name + '.js'; - }, - }, - plugins: [ - svelte({ - preprocess: sveltePreprocess({ - typescript: { - tsconfigFile: 'tsconfig.views.json', - }, - }), - compilerOptions: { - // enable run-time checks when not in production - dev: !production, - hydratable: true, - }, - }), - // we'll extract any component CSS out into - // a separate file - better for performance - cssOnlyRollupPlugin({output: 'bundle.css'}), - - // If you have external dependencies installed from - // npm, you'll most likely need these plugins. In - // some cases you'll need additional configuration - - // consult the documentation for details: - // https://github.com/rollup/plugins/tree/master/packages/commonjs - resolve({ - browser: true, - dedupe: ['svelte'], - }), - commonjs(), - ], - watch: { - buildDelay: 1000, - clearScreen: false, - }, - }; - - if (production) { - // If we're building for production (npm run build - // instead of npm run dev), minify - options.plugins?.push(terser()); - } else { - // Watch the `public` directory and refresh the - // browser on changes when not in production - const plugin = livereloadRollupPlugin('public'); - options.plugins?.push(plugin); - } - - for (const name of canonicalViewNames) { - await new Promise((resolve, reject) => { - fs.mkdir(path.dirname(path.join(this.publicAssetsPath, 'js', name)), {recursive: true}, err => { - if (err) return reject(err); - resolve(); - }); - }); - } - - if (production) { - if (!this.rollup) { - this.rollup = await rollup(options); - - await this.rollup.write({ - format: 'es', - dir: path.join(this.publicAssetsPath, 'js'), - }); - } - } else { - if (!this.rollup) { - this.rollup = watch(options); - this.rollup.on('event', (event) => { - if (event.code === 'ERROR' || event.code === 'BUNDLE_END') { - event.result?.close().catch(err => logger.error(err)); - logger.debug('Bundled from watch'); - } - }); - } - } - } - - private async resetBundle(): Promise { - if (this.rollup) { - logger.debug('Stopping rollup...'); - await this.rollup.close(); - this.rollup = undefined; - } - } - - private async renderSvelte( - file: string, - locals: Record, - callback: (err: Error | null, rendered?: string) => void, - ): Promise { - const canonicalViewName = this.toCanonicalViewName(file); - const actualFile = path.join(this.svelteOutputPath, canonicalViewName + COMPILED_SVELTE_EXTENSION); - - if (!config.get('view.enable_asset_cache')) delete this.fileCache[actualFile]; - const view = await this.getFileContentsFromCache(actualFile); - - - const templateFile = await this.resolveViewFromCanonicalName('layouts/svelte_layout.html'); - - if (!config.get('view.enable_asset_cache')) delete this.fileCache[templateFile]; - let output = await this.getFileContentsFromCache(templateFile); - - - const [ - backendLines, - head, - html, - css, - ] = view.split(FrontendToolsComponent.getSveltePreCompileSeparator(canonicalViewName)); - - locals = Object.assign(this.getGlobals(), locals); - - const localMap: Record = {}; - backendLines.split('\n').forEach(line => { - const key = line.substring(1, line.indexOf(',') >= 0 ? line.indexOf(',') - 1 : line.length - 1); - if (line.indexOf('[') >= 0) { - const args = line.substring(line.indexOf('[') + 1, line.length - 1) - .split(/, *?/) - .map(arg => { - if (arg.startsWith("'")) return '"' + arg.substring(1, arg.length - 1) + '"'; - return arg; - }) - .map(arg => JSON.parse(arg)); - - const f = locals[key]; - if (typeof f !== 'function') throw new Error(key + ' is not a function.'); - localMap[`'${key}', ${JSON.stringify(args)}`] = f.call(locals, ...args); - } else { - localMap[`'${key}'`] = locals[key]; - } - }); - const props = JSON.stringify(localMap); - - output = output.replace('%canonicalViewName%', canonicalViewName); - output = output.replace('%props%', props); - output = output.replace('%head%', head); - output = output.replace('%html%', html); - output = output.replace('%css%', css); - - callback(null, output); - } - - private async getFileContentsFromCache(file: string): Promise { - if (!this.fileCache[file]) { - this.fileCache[file] = await new Promise((resolve, reject) => { - fs.readFile(file, (err, data) => { - if (err) return reject(err); - resolve(data.toString()); - }); - }); - } - - return this.fileCache[file]; - } - - private toCanonicalViewName(file: string): string { - const resolvedFilePath = path.resolve(file); - - let canonicalViewName: string | null = null; - for (const viewPath of this.viewPaths) { - if (resolvedFilePath.startsWith(viewPath)) { - canonicalViewName = resolvedFilePath.substring(viewPath.length + 1); - } - } - - if (!canonicalViewName) throw new Error('view ' + file + ' not found'); - return canonicalViewName; - } - - private async resolveViewFromCanonicalName(canonicalName: string): Promise { - for (const viewPath of this.viewPaths) { - const tryPath = path.join(viewPath, canonicalName); - if (await new Promise((resolve, reject) => { - fs.stat(tryPath, (err) => { - if (err == null) { - resolve(true); - } else if (err.code === 'ENOENT') { - resolve(false); - } else { - reject(err); - } - }); - })) { - return tryPath; - } - } - - throw new Error('View not found from canonical name ' + canonicalName); - } - - /** - * TODO: add a way to add locals from anywhere - */ - private getGlobals(): Record { - return { - route: (name: string) => 'unimplemented route ' + name, - direct: 'access', - }; - } } - -export type SourceMap = string | Record | undefined; diff --git a/src/frontend/SvelteViewEngine.ts b/src/frontend/SvelteViewEngine.ts new file mode 100644 index 0000000..9607828 --- /dev/null +++ b/src/frontend/SvelteViewEngine.ts @@ -0,0 +1,427 @@ +import crypto from "crypto"; +import path from "path"; +import {PreRenderedChunk, rollup, RollupBuild, RollupOptions, RollupWatcher, RollupWatchOptions, watch} from "rollup"; +import config from "config"; +import ViewEngine from "./ViewEngine"; +import {logger} from "../Logger"; +import FileCache from "../utils/FileCache"; +import {afs} from "../Utils"; +import {compile, preprocess} from "svelte/compiler"; +import {sveltePreprocess} from "svelte-preprocess/dist/autoProcess"; +import requireFromString from "require-from-string"; +import {CssResult} from "svelte/types/compiler/interfaces"; +import svelte from "rollup-plugin-svelte"; +import cssOnlyRollupPlugin from "rollup-plugin-css-only"; +import resolve from "@rollup/plugin-node-resolve"; +import commonjs from "@rollup/plugin-commonjs"; +import {terser} from "rollup-plugin-terser"; +import livereloadRollupPlugin from "rollup-plugin-livereload"; + +const BACKEND_CODE_PREFIX = 'swaf.'; +const COMPILED_SVELTE_EXTENSION = '.swafview'; + +export default class SvelteViewEngine extends ViewEngine { + public static getPreCompileSeparator(canonicalViewName: string): string { + return '\n---' + + crypto.createHash('sha1') + .update(path.basename(path.resolve(canonicalViewName))) + .digest('base64') + + '---\n'; + } + + + private readonly fileCache: FileCache = new FileCache(); + private readonly dependencyCache: Record> = {}; + private readonly backendCodeCache: Record = {}; + + private rollup?: RollupBuild | RollupWatcher; + + public constructor( + buildDir: string, + publicDir: string, + devWatchedViewPath: string, + ...additionalViewPaths: string[] + ) { + super(buildDir, publicDir, devWatchedViewPath, ...additionalViewPaths); + } + + public getExtension(): string { + return 'svelte'; + } + + /** + * TODO: add replaces on ssr html + */ + public async render( + file: string, + locals: Record, + callback: (err: Error | null, output?: string) => void, + ): Promise { + const canonicalViewName = this.toCanonicalName(file); + + // View + const actualFile = path.join(this.getBuildDir(), canonicalViewName + COMPILED_SVELTE_EXTENSION); + const view = await this.fileCache.get(actualFile, config.get('view.dev')); + + // Root template + const templateFile = await this.resolveFileFromCanonicalName('layouts/svelte_layout.html'); + let output = await this.fileCache.get(templateFile, config.get('view.dev')); + + // Pre-compiled parts + const [ + backendLines, + head, + html, + css, + ] = view.split(SvelteViewEngine.getPreCompileSeparator(canonicalViewName)); + + // Props (locals) + locals = Object.assign(this.getGlobals(), locals); + + const localMap: Record = {}; + backendLines.split('\n').forEach(line => { + const key = line.substring(1, line.indexOf(',') >= 0 ? line.indexOf(',') - 1 : line.length - 1); + if (line.indexOf('[') >= 0) { + const args = line.substring(line.indexOf('[') + 1, line.length - 1) + .split(/, *?/) + .map(arg => { + if (arg.startsWith("'")) return '"' + arg.substring(1, arg.length - 1) + '"'; + return arg; + }) + .map(arg => JSON.parse(arg)); + + const f = locals[key]; + if (typeof f !== 'function') throw new Error(key + ' is not a function.'); + localMap[`'${key}', ${JSON.stringify(args)}`] = f.call(locals, ...args); + } else { + localMap[`'${key}'`] = locals[key]; + } + }); + const props = JSON.stringify(localMap); + + // Replaces + output = output.replace('%canonicalViewName%', canonicalViewName); + output = output.replace('%props%', props); + output = output.replace('%head%', head); + output = output.replace('%html%', html); + output = output.replace('%css%', css); + + callback(null, output); + } + + public async end(): Promise { + await this.stopRollup(); + } + + /** + * TODO: add a way to add globals from anywhere + */ + private getGlobals(): Record { + return { + route: (name: string) => 'unimplemented route ' + name, + direct: 'access', + }; + } + + public async preCompile(canonicalName: string, alsoCompileDependents: boolean): Promise { + const file = await this.resolveFileFromCanonicalName(canonicalName); + const intermediateFile = path.join(this.getBuildDir(), canonicalName); + + logger.debug(canonicalName + ' > ', 'Pre-compiling', file, '->', intermediateFile); + const source = await this.fileCache.get(file, config.get('view.dev')); + + const allBackendLines: string[] = []; + for (const dependency of this.resolveDependencies(source, canonicalName)) { + allBackendLines.push(...(await this.replaceBackendCode(dependency)).backendLines); + } + + const {backendReplacedCode, backendLines} = await this.replaceBackendCode(canonicalName, source); + allBackendLines.push(...backendLines); + + // Server Side Render (initial HTML and CSS, no-js) + const ssr = await this.compileSsr(canonicalName, intermediateFile, backendReplacedCode); + + const separator = SvelteViewEngine.getPreCompileSeparator(canonicalName); + const finalCode = [ + [...new Set(allBackendLines).values()].join('\n'), + ssr.head, + ssr.html, + ssr.css.code, + ].join(separator); + + const swafViewFile = path.join(this.getBuildDir(), canonicalName + COMPILED_SVELTE_EXTENSION); + await afs.mkdir(path.dirname(swafViewFile), {recursive: true}); + await afs.writeFile(swafViewFile, finalCode); + + if (alsoCompileDependents && Object.keys(this.dependencyCache).indexOf(canonicalName) >= 0) { + logger.debug(canonicalName + ' > ', 'Compiling dependents...'); + for (const dependent of [...this.dependencyCache[canonicalName]]) { + await this.preCompile(dependent, true); + } + } + } + + private resolveDependencies(source: string, canonicalViewName: string): string[] { + const dependencies: string[] = []; + + for (const match of source.matchAll(/import .+ from ['"](.+?\.svelte)['"];/gm)) { + dependencies.push(path.join(path.dirname(canonicalViewName), match[1])); + } + + // Clear existing links from cache + for (const dependency of Object.keys(this.dependencyCache)) { + this.dependencyCache[dependency].delete(canonicalViewName); + } + + // Add new links to cache + for (const dependency of dependencies) { + if (Object.keys(this.dependencyCache).indexOf(dependency) < 0) { + this.dependencyCache[dependency] = new Set(); + } + this.dependencyCache[dependency].add(canonicalViewName); + } + + return dependencies; + } + + private async replaceBackendCode(canonicalViewName: string, code?: string): Promise<{ + backendReplacedCode: string, + backendLines: string[], + }> { + // Cache + if (Object.keys(this.backendCodeCache).indexOf(canonicalViewName) >= 0) { + return this.backendCodeCache[canonicalViewName]; + } + + // mkdir output file dir + const outputFile = path.join(this.getBuildDir(), canonicalViewName); + await afs.mkdir(path.dirname(outputFile), {recursive: true}); + + // Read source file if code was not already provided + if (!code) { + const file = await this.resolveFileFromCanonicalName(canonicalViewName); + code = await this.fileCache.get(file, config.get('view.dev')); + } + + // Skip replace if there is no swaf export + if (!code.match(/export[ \n]+let[ \n]+swaf[ \n]*=[ \n]*{[ \n]*}/)) { + const generated = { + backendReplacedCode: code, + backendLines: [], + }; + await afs.writeFile(outputFile, generated.backendReplacedCode); + this.backendCodeCache[canonicalViewName] = generated; + return generated; + } + + + let output = code; + const backendLines = new Set(); + + let index = 0; + while ((index = output.indexOf(BACKEND_CODE_PREFIX, index + 1)) >= 0) { + // Escaping + if (index > 0 && output[index - 1] === '\\') { + const isEscapingEscaped: boolean = index > 1 && output[index - 2] === '\\'; + output = output.substring(0, index - 1 - (isEscapingEscaped ? 1 : 0)) + + output.substring(index, output.length); + continue; + } + + const startIndex = index + BACKEND_CODE_PREFIX.length; + let endIndex = startIndex; + let struct = 0; + + while (endIndex < output.length) { + if (['(', '[', '{'].indexOf(output[endIndex]) >= 0) struct++; + if ([')', ']', '}'].indexOf(output[endIndex]) >= 0) { + struct--; + if (struct <= 0) { + if (struct === 0) endIndex++; + break; + } + } + if ([' ', '\n', '<'].indexOf(output[endIndex]) >= 0 && struct === 0) break; + endIndex++; + } + + let backendLine = output.substring(startIndex, endIndex); + if (backendLine.match(/([^()]+)\((.+?)\)/)) { + backendLine = backendLine.replace(/([^()]+)\((.+?)\)/, "'$1', [$2]"); + } else { + backendLine = backendLine.replace(/([^()]+)/, "'$1'"); + } + + backendLines.add(backendLine); + output = output.substring(0, index) + + 'swaf(' + backendLine + ')' + + output.substring(endIndex, output.length); + } + + const generated = { + backendReplacedCode: output, + backendLines: [...backendLines], + }; + await afs.writeFile(outputFile, generated.backendReplacedCode); + this.backendCodeCache[canonicalViewName] = generated; + + return generated; + } + + public async afterPreCompile(): Promise { + await this.bundle(...Object.keys(this.backendCodeCache)); + } + + + public async onNewFile(): Promise { + await this.stopRollup(); + } + + public async onFileChange(file: string): Promise { + delete this.backendCodeCache[this.toCanonicalName(file)]; + } + + private async compileSsr(canonicalName: string, file: string, code: string): Promise<{ + head: string, + css: CssResult, + html: string, + }> { + // Svelte preprocess + logger.debug(canonicalName + ' > ', 'Preprocessing svelte', file); + const preprocessed = await preprocess( + code, + sveltePreprocess({ + typescript: { + tsconfigFile: 'tsconfig.views.json', + }, + }), + { + filename: file, + }, + ); + + // Svelte compile + logger.debug(canonicalName + ' > ', 'Compiling svelte ssr', file); + const svelteSsr = compile(preprocessed.code, { + dev: config.get('view.dev'), + generate: 'ssr', + format: 'cjs', + cssOutputFilename: file + '.css', + }); + + const locals = this.getGlobals(); + return requireFromString(svelteSsr.js.code, file).default.render({ + swaf: (key: string, args?: unknown[]) => { + if (!args) return locals[key]; + + const f = locals[key]; + if (typeof f !== 'function') throw new Error(key + ' is not a function.'); + return f.call(locals, ...args); + }, + }); + } + + /** + * TODO: load rollup config from external source + */ + private async bundle(...canonicalViewNames: string[]): Promise { + logger.debug('Bundling'); + + const production = !config.get('view.dev'); + const options: RollupOptions | RollupWatchOptions = { + input: canonicalViewNames.map(name => path.join(this.getBuildDir(), name)), + output: { + sourcemap: true, + format: 'es', + name: 'bundle', + dir: path.join(this.getPublicDir(), 'js'), + entryFileNames: (chunkInfo: PreRenderedChunk): string => { + const name = chunkInfo.facadeModuleId ? + path.relative(this.getBuildDir(), chunkInfo.facadeModuleId) : + chunkInfo.name; + return name + '.js'; + }, + }, + plugins: [ + svelte({ + preprocess: sveltePreprocess({ + typescript: { + tsconfigFile: 'tsconfig.views.json', + }, + }), + compilerOptions: { + // enable run-time checks when not in production + dev: !production, + hydratable: true, + }, + }), + // we'll extract any component CSS out into + // a separate file - better for performance + cssOnlyRollupPlugin({output: 'bundle.css'}), + + // If you have external dependencies installed from + // npm, you'll most likely need these plugins. In + // some cases you'll need additional configuration - + // consult the documentation for details: + // https://github.com/rollup/plugins/tree/master/packages/commonjs + resolve({ + browser: true, + dedupe: ['svelte'], + }), + commonjs(), + ], + watch: { + buildDelay: 1000, + clearScreen: false, + }, + }; + + if (production) { + // If we're building for production (npm run build + // instead of npm run dev), minify + options.plugins?.push(terser()); + } else { + // Watch the `public` directory and refresh the + // browser on changes when not in production + const plugin = livereloadRollupPlugin('public'); + options.plugins?.push(plugin); + } + + for (const name of canonicalViewNames) { + await afs.mkdir(path.dirname(path.join(this.getPublicDir(), 'js', name)), {recursive: true}); + } + + if (production) { + if (!this.rollup) { + this.rollup = await rollup(options); + + await this.rollup.write({ + format: 'es', + dir: path.join(this.getPublicDir(), 'js'), + }); + } + } else { + if (!this.rollup) { + this.rollup = watch(options); + this.rollup.on('event', (event) => { + if (event.code === 'ERROR' || event.code === 'BUNDLE_END') { + event.result?.close().catch(err => logger.error(err)); + logger.debug('Bundled from watch'); + } + }); + } + } + } + + private async stopRollup(): Promise { + if (this.rollup) { + logger.debug('Stopping rollup...'); + await this.rollup.close(); + this.rollup = undefined; + } + } +} + diff --git a/src/frontend/ViewEngine.ts b/src/frontend/ViewEngine.ts new file mode 100644 index 0000000..e16db1d --- /dev/null +++ b/src/frontend/ViewEngine.ts @@ -0,0 +1,142 @@ +import path from "path"; +import fs from "fs"; +import chokidar from "chokidar"; +import {logger} from "../Logger"; +import {afs, readdirRecursively} from "../Utils"; + +export default abstract class ViewEngine { + protected readonly viewPaths: string[]; + + /** + * @param buildDir A temporary directory that will contain any non-final or final non-public asset. + * @param publicDir The output directory that should contain all final and public assets. + * @param devWatchedViewDir The directory that should be watched in dev environment. + * @param additionalViewPaths By order of priority, the directories that contain all the views of the app. + * Swaf provided views and default ./view directory are automatically added (don't add them yourself). + * @protected + */ + protected constructor( + private readonly buildDir: string, + private readonly publicDir: string, + private readonly devWatchedViewDir: string, + ...additionalViewPaths: string[] + ) { + this.viewPaths = [ + ...additionalViewPaths.map(p => path.resolve(p)), + path.resolve(__dirname, '../../../views'), + path.resolve(__dirname, '../../views'), + path.resolve(__dirname, '../views'), + ].filter(dir => fs.existsSync(dir)); + + if (!fs.existsSync(this.buildDir)) { + fs.mkdirSync(this.buildDir, {recursive: true}); + } + } + + public abstract getExtension(): string; + + public abstract render( + file: string, + locals: Record, + callback: (err: Error | null, output?: string) => void, + ): Promise; + + public end?(): Promise; + + public getViewPaths(): string[] { + return this.viewPaths; + } + + public getBuildDir(): string { + return this.buildDir; + } + + public getPublicDir(): string { + return this.publicDir; + } + + public getDevWatchedViewDir(): string { + return this.devWatchedViewDir; + } + + public preCompile?(canonicalName: string, alsoCompileDependents: boolean): Promise; + + public afterPreCompile?(): Promise; + + public async preCompileAll(): Promise { + if (this.preCompile) { + logger.info(`Pre-compiling ${this.getExtension()} views...`); + + // List all views + const views: string[] = []; + for (const viewPath of this.viewPaths) { + await readdirRecursively(viewPath, async file => { + if (file.endsWith('.' + this.getExtension())) { + views.push(this.toCanonicalName(file)); + } + }); + } + + // Deduplicate and pre-compile + for (const canonicalName of [...new Set(views)]) { + await this.preCompile(canonicalName, false); + } + } + + await this.afterPreCompile?.(); + } + + public onNewFile?(file: string): Promise; + + public onFileChange?(file: string): Promise; + + public async watch(): Promise { + const watcher = chokidar.watch(this.devWatchedViewDir, {persistent: true}); + watcher.on('ready', () => { + logger.debug(`Watching ${this.getExtension()} assets for changes`); + + watcher.on('add', (file) => { + if (file.endsWith('.' + this.getExtension())) { + (this.onNewFile ? this.onNewFile(file) : Promise.resolve()) + .then(() => this.preCompile?.(this.toCanonicalName(file), true)) + .then(() => this.afterPreCompile?.()) + .catch(err => logger.error(err)); + } + }); + + watcher.on('change', (file) => { + if (file.endsWith('.' + this.getExtension())) { + (this.onFileChange ? this.onFileChange(file) : Promise.resolve()) + .then(() => this.preCompile?.(this.toCanonicalName(file), true)) + .then(() => this.afterPreCompile?.()) + .catch(err => logger.error(err)); + } + }); + }); + } + + protected toCanonicalName(file: string): string { + const resolvedFilePath = path.resolve(file); + + let canonicalViewName: string | null = null; + for (const viewPath of this.viewPaths) { + if (resolvedFilePath.startsWith(viewPath)) { + canonicalViewName = resolvedFilePath.substring(viewPath.length + 1); + } + } + + if (!canonicalViewName) throw new Error('view ' + file + ' not found'); + return canonicalViewName; + } + + protected async resolveFileFromCanonicalName(canonicalName: string): Promise { + for (const viewPath of this.viewPaths) { + const tryPath = path.join(viewPath, canonicalName); + if (await afs.exists(tryPath)) { + return tryPath; + } + } + + throw new Error('View not found from canonical name ' + canonicalName); + } +} diff --git a/src/utils/FileCache.ts b/src/utils/FileCache.ts new file mode 100644 index 0000000..d6e2d66 --- /dev/null +++ b/src/utils/FileCache.ts @@ -0,0 +1,35 @@ +import fs from "fs"; +import {afs} from "../Utils"; + +export default class FileCache { + private readonly cache: Record = {}; + + public async get(file: string, forceRead: boolean = false): Promise { + if (forceRead || !this.cache[file]) { + await this.load(file); + } + + return this.cache[file]; + } + + public getSync(file: string, forceRead: boolean = false): string { + if (forceRead || !this.cache[file]) { + this.loadSync(file); + } + + return this.cache[file]; + } + + public getOrFail(file: string): string { + if (!this.cache[file]) throw new Error(file + ' not cached.'); + return this.cache[file]; + } + + public async load(file: string): Promise { + this.cache[file] = (await afs.readFile(file)).toString(); + } + + public loadSync(file: string): void { + this.cache[file] = fs.readFileSync(file).toString(); + } +}