swaf/src/components/FrontendToolsComponent.ts

import {Express, Router} from "express";
import * as fs from "fs";
import path from "path";
import config from "config";
import ApplicationComponent from "../ApplicationComponent";
import {logger} from "../Logger";
import {ServerError} from "../HttpError";
import * as crypto from "crypto";
import requireFromString from "require-from-string";
import "svelte/register";
import {compile, preprocess} from "svelte/compiler";
import {sveltePreprocess} from "svelte-preprocess/dist/autoProcess";
import chokidar from "chokidar";
import {CssResult} from "svelte/types/compiler/interfaces";
import {PreRenderedChunk, rollup, RollupBuild, RollupOptions, RollupWatcher, RollupWatchOptions, watch} from "rollup";
import svelte from "rollup-plugin-svelte";
import resolve from "@rollup/plugin-node-resolve";
import commonjs from "@rollup/plugin-commonjs";
import {terser} from "rollup-plugin-terser";
import cssOnlyRollupPlugin from "rollup-plugin-css-only";
import livereloadRollupPlugin from "rollup-plugin-livereload";

const BACKEND_CODE_PREFIX = 'swaf.';
const COMPILED_SVELTE_EXTENSION = '.swafview';

export default class FrontendToolsComponent extends ApplicationComponent {
    public static getSveltePreCompileSeparator(canonicalViewName: string): string {
        return '\n---' +
            crypto.createHash('sha1')
                .update(path.basename(path.resolve(canonicalViewName)))
                .digest('base64') +
            '---\n';
    }

    private content: Map<string, string> = new Map();
    private readonly viewPaths: string[];
    private readonly svelteDevViewsPath: string;

    private readonly dependencyCache: Record<string, Set<string>> = {};
    private readonly backendCodeCache: Record<string, {
        backendReplacedCode: string,
        backendLines: string[],
    }> = {};
    private readonly fileCache: Record<string, string> = {};

    private rollup?: RollupBuild | RollupWatcher;

    public constructor(
        private readonly publicAssetsPath: string,
        private readonly svelteOutputPath: string,
        ...viewPaths: string[]
    ) {
        super();
        this.viewPaths = [
            ...viewPaths.map(p => path.resolve(p)),
            path.resolve(__dirname, '../../../views'),
            path.resolve(__dirname, '../../views'),
            path.resolve(__dirname, '../views'),
        ];
        this.svelteDevViewsPath = path.resolve('views');

        if (!fs.existsSync(svelteOutputPath)) {
            fs.mkdirSync(svelteOutputPath);
        }
    }

    public async start(app: Express): Promise<void> {
        await this.cachePublicAssets();

        await this.preCompileSvelteViews();

        const watcher = chokidar.watch(this.svelteDevViewsPath, {persistent: true});
        watcher.on('ready', () => {
            logger.debug('Watching svelte assets for changes');

            watcher.on('add', (path) => {
                if (path.endsWith('.svelte')) {
                    this.resetBundle()
                        .then(() => this.preCompileSvelte(path, true))
                        .then(() => this.bundle(...Object.keys(this.backendCodeCache)))
                        .catch(err => logger.error(err));
                }
            });

            watcher.on('change', (path) => {
                if (path.endsWith('.svelte')) {
                    delete this.backendCodeCache[this.toCanonicalViewName(path)];
                    this.preCompileSvelte(path, true)
                        .then(() => this.bundle(...Object.keys(this.backendCodeCache)))
                        .catch(err => logger.error(err));
                }
            });
        });

        app.engine('svelte', (path, options, callback) => {
            this.renderSvelte(path, options as Record<string, unknown>, callback)
                .catch(err => callback(err));
        });
        app.set('views', this.viewPaths);
        app.set('view engine', 'svelte');
    }

    public async stop(): Promise<void> {
        await this.resetBundle();
    }

    private async cachePublicAssets(): Promise<void> {
        if (config.get<boolean>('view.enable_asset_cache')) {
            logger.info('Caching assets from', this.publicAssetsPath, '...');
            await this.forEachFileInDirRecursively(this.publicAssetsPath, file => this.cacheFile(file));
        } else {
            logger.info('Asset cache disabled.');
        }
    }

    private async preCompileSvelteViews(): Promise<void> {
        logger.info('Pre-compiling svelte views...');
        await this.forEachFileInDirRecursively(this.svelteDevViewsPath, async file => {
            if (file.endsWith('.svelte')) {
                await this.preCompileSvelte(file);
            }
        });
        await this.bundle(...Object.keys(this.backendCodeCache));
    }

    public async handle(router: Router): Promise<void> {
        router.use((req, res, next) => {
            res.locals.inlineAsset = (urlPath: string) => {
                if (!config.get<boolean>('view.enable_asset_cache')) {
                    return fs.readFileSync(path.resolve(this.publicAssetsPath + urlPath));
                }

                const content = this.content.get(urlPath);
                if (!content) {
                    throw new ServerError('Asset ' + path + ' was not loaded.');
                }
                return content;
            };
            next();
        });
    }

    private async forEachFileInDirRecursively(dir: string, consumer: (file: string) => Promise<void>): Promise<void> {
        await new Promise<void[]>((resolve, reject) => {
            fs.readdir(dir, (err, files) => {
                if (err) return reject(err);

                resolve(Promise.all<void>(files.map(file => new Promise<void>((resolve, reject) => {
                    file = path.join(dir, file);
                    fs.stat(file, (err, stat) => {
                        if (err) return reject(err);

                        if (stat.isDirectory()) {
                            resolve(this.forEachFileInDirRecursively(file, consumer));
                        } else {
                            resolve(consumer(file));
                        }
                    });
                }))));
            });
        });
    }

    private async cacheFile(file: string): Promise<void> {
        await new Promise<void>((resolve, reject) => {
            fs.readFile(file, (err, data) => {
                if (err) return reject(data);

                const urlPath = file.replace(this.publicAssetsPath, '');
                this.content.set(urlPath, data.toString());
                logger.debug('Loaded', file, 'as', urlPath);
                resolve();
            });
        });
    }

    private async preCompileSvelte(file: string, alsoCompileDependents: boolean = false): Promise<void> {
        file = path.relative('.', file);
        const canonicalViewName = this.toCanonicalViewName(file);
        const intermediateFile = path.join(this.svelteOutputPath, canonicalViewName);

        logger.debug(canonicalViewName + ' > ', 'Pre-compiling', file, '->', intermediateFile);
        const source = await new Promise<string>((resolve, reject) => {
            fs.readFile(file, (err, data) => {
                if (err) return reject(err);

                resolve(data.toString());
            });
        });

        const allBackendLines: string[] = [];
        for (const dependency of this.resolveDependencies(source, canonicalViewName)) {
            allBackendLines.push(...(await this.replaceBackendCode(dependency)).backendLines);
        }

        const {backendReplacedCode, backendLines} = await this.replaceBackendCode(canonicalViewName, source);
        allBackendLines.push(...backendLines);

        const preprocessedCode = await this.preProcessSvelte(backendReplacedCode, intermediateFile, canonicalViewName);

        // Server Side Render (initial HTML, no-js)
        const ssr = this.compileSvelteSsr(preprocessedCode, intermediateFile, canonicalViewName);

        const separator = FrontendToolsComponent.getSveltePreCompileSeparator(canonicalViewName);
        const finalCode = [
            [...new Set<string>(allBackendLines).values()].join('\n'),
            ssr.head,
            ssr.html,
            ssr.css.code,
        ].join(separator);

        const swafViewFile = path.join(this.svelteOutputPath, canonicalViewName + COMPILED_SVELTE_EXTENSION);
        await new Promise<void>((resolve, reject) => {
            fs.mkdir(path.dirname(swafViewFile), {recursive: true}, (err) => {
                if (err) return reject(err);
                resolve();
            });
        });
        await new Promise<void>((resolve, reject) => fs.writeFile(swafViewFile, finalCode, err => {
            if (err) return reject(err);
            resolve();
        }));

        if (alsoCompileDependents && Object.keys(this.dependencyCache).indexOf(canonicalViewName) >= 0) {
            logger.debug(canonicalViewName + ' > ', 'Compiling dependents...');
            for (const dependent of [...this.dependencyCache[canonicalViewName]]) {
                await this.preCompileSvelte(await this.resolveViewFromCanonicalName(dependent), true);
            }
        }
    }

    private resolveDependencies(source: string, canonicalViewName: string): string[] {
        const dependencies: string[] = [];

        for (const match of source.matchAll(/import .+ from ['"](.+?\.svelte)['"];/gm)) {
            dependencies.push(path.join(path.dirname(canonicalViewName), match[1]));
        }

        // Clear existing links from cache
        for (const dependency of Object.keys(this.dependencyCache)) {
            this.dependencyCache[dependency].delete(canonicalViewName);
        }

        // Add new links to cache
        for (const dependency of dependencies) {
            if (Object.keys(this.dependencyCache).indexOf(dependency) < 0) {
                this.dependencyCache[dependency] = new Set<string>();
            }
            this.dependencyCache[dependency].add(canonicalViewName);
        }

        return dependencies;
    }

    private async replaceBackendCode(canonicalViewName: string, code?: string): Promise<{
        backendReplacedCode: string,
        backendLines: string[],
    }> {
        // Cache
        if (Object.keys(this.backendCodeCache).indexOf(canonicalViewName) >= 0) {
            return this.backendCodeCache[canonicalViewName];
        }

        const outputFile = path.join(this.svelteOutputPath, canonicalViewName);
        await new Promise<void>((resolve, reject) => {
            fs.mkdir(path.dirname(outputFile), {recursive: true}, (err) => {
                if (err) return reject(err);
                resolve();
            });
        });

        // Read source file if code was not already provided
        if (!code) {
            const file = await this.resolveViewFromCanonicalName(canonicalViewName);
            code = await new Promise<string>((resolve, reject) => {
                fs.readFile(file, (err, data) => {
                    if (err) return reject(err);

                    resolve(data.toString());
                });
            });
        }

        // Skip replace if there is no swaf export
        if (code.indexOf(`export let swaf = {};`) < 0) {
            const generated = {
                backendReplacedCode: code,
                backendLines: [],
            };
            await new Promise<void>((resolve, reject) => {
                fs.writeFile(outputFile, generated.backendReplacedCode, (err) => {
                    if (err) return reject(err);

                    resolve();
                });
            });
            this.backendCodeCache[canonicalViewName] = generated;
            return generated;
        }


        let output = code;
        const backendLines = new Set<string>();

        let index = 0;
        while ((index = output.indexOf(BACKEND_CODE_PREFIX, index + 1)) >= 0) {
            // Escaping
            if (index > 0 && output[index - 1] === '\\') {
                const isEscapingEscaped: boolean = index > 1 && output[index - 2] === '\\';
                output = output.substring(0, index - 1 - (isEscapingEscaped ? 1 : 0)) +
                    output.substring(index, output.length);
                continue;
            }

            const startIndex = index + BACKEND_CODE_PREFIX.length;
            let endIndex = startIndex;
            let struct = 0;

            while (endIndex < output.length) {
                if (['(', '[', '{'].indexOf(output[endIndex]) >= 0) struct++;
                if ([')', ']', '}'].indexOf(output[endIndex]) >= 0) {
                    struct--;
                    if (struct <= 0) {
                        if (struct === 0) endIndex++;
                        break;
                    }
                }
                if ([' ', '\n', '<'].indexOf(output[endIndex]) >= 0 && struct === 0) break;
                endIndex++;
            }

            let backendLine = output.substring(startIndex, endIndex);
            if (backendLine.match(/([^()]+)\((.+?)\)/)) {
                backendLine = backendLine.replace(/([^()]+)\((.+?)\)/, "'$1', [$2]");
            } else {
                backendLine = backendLine.replace(/([^()]+)/, "'$1'");
            }

            backendLines.add(backendLine);
            output = output.substring(0, index) +
                'swaf(' + backendLine + ')' +
                output.substring(endIndex, output.length);
        }

        const generated = {
            backendReplacedCode: output,
            backendLines: [...backendLines],
        };
        await new Promise<void>((resolve, reject) => {
            fs.writeFile(outputFile, generated.backendReplacedCode, (err) => {
                if (err) return reject(err);

                resolve();
            });
        });
        this.backendCodeCache[canonicalViewName] = generated;

        return generated;
    }

    private async preProcessSvelte(code: string, filename: string, canonicalViewName: string): Promise<string> {
        logger.debug(canonicalViewName + ' > ', 'Preprocessing svelte', filename);
        const preprocessed = await preprocess(
            code,
            sveltePreprocess({
                typescript: {
                    tsconfigFile: 'tsconfig.views.json',
                },
            }),
            {
                filename: filename,
            },
        );

        return preprocessed.code;
    }

    private compileSvelteSsr(code: string, filename: string, canonicalViewName: string): {
        head: string,
        css: CssResult,
        html: string,
    } {
        logger.debug(canonicalViewName + ' > ', 'Compiling svelte ssr', filename);
        const svelteSsr = compile(code, {
            dev: config.get<boolean>('view.dev'),
            generate: 'ssr',
            format: 'cjs',
            cssOutputFilename: filename + '.css',
        });

        const locals = this.getGlobals();

        return requireFromString(svelteSsr.js.code, filename).default.render({
            swaf: (key: string, args?: unknown[]) => {
                if (!args) return locals[key];

                const f = locals[key];
                if (typeof f !== 'function') throw new Error(key + ' is not a function.');
                return f.call(locals, ...args);
            },
        });
    }

    private async bundle(...canonicalViewNames: string[]): Promise<void> {
        logger.debug('Bundling');

        const production = !config.get<boolean>('view.dev');
        const options: RollupOptions | RollupWatchOptions = {
            input: canonicalViewNames.map(name => path.join(this.svelteOutputPath, name)),
            output: {
                sourcemap: true,
                format: 'es',
                name: 'bundle',
                dir: path.join(this.publicAssetsPath, 'js'),
                entryFileNames: (chunkInfo: PreRenderedChunk): string => {
                    const name = chunkInfo.facadeModuleId ?
                        path.relative(this.svelteOutputPath, chunkInfo.facadeModuleId) :
                        chunkInfo.name;
                    return name + '.js';
                },
            },
            plugins: [
                svelte({
                    preprocess: sveltePreprocess({
                        typescript: {
                            tsconfigFile: 'tsconfig.views.json',
                        },
                    }),
                    compilerOptions: {
                        // enable run-time checks when not in production
                        dev: !production,
                        hydratable: true,
                    },
                }),
                // we'll extract any component CSS out into
                // a separate file - better for performance
                cssOnlyRollupPlugin({output: 'bundle.css'}),

                // If you have external dependencies installed from
                // npm, you'll most likely need these plugins. In
                // some cases you'll need additional configuration -
                // consult the documentation for details:
                // https://github.com/rollup/plugins/tree/master/packages/commonjs
                resolve({
                    browser: true,
                    dedupe: ['svelte'],
                }),
                commonjs(),
            ],
            watch: {
                buildDelay: 1000,
                clearScreen: false,
            },
        };

        if (production) {
            // If we're building for production (npm run build
            // instead of npm run dev), minify
            options.plugins?.push(terser());
        } else {
            // Watch the `public` directory and refresh the
            // browser on changes when not in production
            const plugin = livereloadRollupPlugin('public');
            options.plugins?.push(plugin);
        }

        for (const name of canonicalViewNames) {
            await new Promise<void>((resolve, reject) => {
                fs.mkdir(path.dirname(path.join(this.publicAssetsPath, 'js', name)), {recursive: true}, err => {
                    if (err) return reject(err);
                    resolve();
                });
            });
        }

        if (production) {
            if (!this.rollup) {
                this.rollup = await rollup(options);

                await this.rollup.write({
                    format: 'es',
                    dir: path.join(this.publicAssetsPath, 'js'),
                });
            }
        } else {
            if (!this.rollup) {
                this.rollup = watch(options);
                this.rollup.on('event', (event) => {
                    if (event.code === 'ERROR' || event.code === 'BUNDLE_END') {
                        event.result?.close().catch(err => logger.error(err));
                        logger.debug('Bundled from watch');
                    }
                });
            }
        }
    }

    private async resetBundle(): Promise<void> {
        if (this.rollup) {
            logger.debug('Stopping rollup...');
            await this.rollup.close();
            this.rollup = undefined;
        }
    }

    private async renderSvelte(
        file: string,
        locals: Record<string, unknown>,
        callback: (err: Error | null, rendered?: string) => void,
    ): Promise<void> {
        const canonicalViewName = this.toCanonicalViewName(file);
        const actualFile = path.join(this.svelteOutputPath, canonicalViewName + COMPILED_SVELTE_EXTENSION);

        if (!config.get<boolean>('view.enable_asset_cache')) delete this.fileCache[actualFile];
        const view = await this.getFileContentsFromCache(actualFile);


        const templateFile = await this.resolveViewFromCanonicalName('layouts/svelte_layout.html');

        if (!config.get<boolean>('view.enable_asset_cache')) delete this.fileCache[templateFile];
        let output = await this.getFileContentsFromCache(templateFile);


        const [
            backendLines,
            head,
            html,
            css,
        ] = view.split(FrontendToolsComponent.getSveltePreCompileSeparator(canonicalViewName));

        locals = Object.assign(this.getGlobals(), locals);

        const localMap: Record<string, unknown> = {};
        backendLines.split('\n').forEach(line => {
            const key = line.substring(1, line.indexOf(',') >= 0 ? line.indexOf(',') - 1 : line.length - 1);
            if (line.indexOf('[') >= 0) {
                const args = line.substring(line.indexOf('[') + 1, line.length - 1)
                    .split(/, *?/)
                    .map(arg => {
                        if (arg.startsWith("'")) return '"' + arg.substring(1, arg.length - 1) + '"';
                        return arg;
                    })
                    .map(arg => JSON.parse(arg));

                const f = locals[key];
                if (typeof f !== 'function') throw new Error(key + ' is not a function.');
                localMap[`'${key}', ${JSON.stringify(args)}`] = f.call(locals, ...args);
            } else {
                localMap[`'${key}'`] = locals[key];
            }
        });
        const props = JSON.stringify(localMap);

        output = output.replace('%canonicalViewName%', canonicalViewName);
        output = output.replace('%props%', props);
        output = output.replace('%head%', head);
        output = output.replace('%html%', html);
        output = output.replace('%css%', css);

        callback(null, output);
    }

    private async getFileContentsFromCache(file: string): Promise<string> {
        if (!this.fileCache[file]) {
            this.fileCache[file] = await new Promise<string>((resolve, reject) => {
                fs.readFile(file, (err, data) => {
                    if (err) return reject(err);
                    resolve(data.toString());
                });
            });
        }

        return this.fileCache[file];
    }

    private toCanonicalViewName(file: string): string {
        const resolvedFilePath = path.resolve(file);

        let canonicalViewName: string | null = null;
        for (const viewPath of this.viewPaths) {
            if (resolvedFilePath.startsWith(viewPath)) {
                canonicalViewName = resolvedFilePath.substring(viewPath.length + 1);
            }
        }

        if (!canonicalViewName) throw new Error('view ' + file + ' not found');
        return canonicalViewName;
    }

    private async resolveViewFromCanonicalName(canonicalName: string): Promise<string> {
        for (const viewPath of this.viewPaths) {
            const tryPath = path.join(viewPath, canonicalName);
            if (await new Promise<boolean>((resolve, reject) => {
                fs.stat(tryPath, (err) => {
                    if (err == null) {
                        resolve(true);
                    } else if (err.code === 'ENOENT') {
                        resolve(false);
                    } else {
                        reject(err);
                    }
                });
            })) {
                return tryPath;
            }
        }

        throw new Error('View not found from canonical name ' + canonicalName);
    }

    /**
     * TODO: add a way to add locals from anywhere
     */
    private getGlobals(): Record<string, unknown> {
        return {
            route: (name: string) => 'unimplemented route ' + name,
            direct: 'access',
        };
    }
}

export type SourceMap = string | Record<string, unknown> | undefined;