Move svelte compilation to external rollup and add pre-compile-views cli

.eslintrc.js

@@ -88,6 +88,7 @@ module.exports = {
     },
     ignorePatterns: [
         '.eslintrc.js',
+        'rollup.config.js',
         'jest.config.js',
         'scripts/**/*',
         'dist/**/*',

package.json

@@ -15,10 +15,11 @@
   "scripts": {
     "test": "jest --verbose --runInBand",
     "clean": "node scripts/clean.js",
-    "prepareSources": "node scripts/prepareSources.js",
+    "prepare-sources": "node scripts/prepare-sources.js",
     "compile": "yarn clean && tsc",
-    "build": "yarn prepareSources && yarn compile && node scripts/dist.js",
+    "build": "yarn prepare-sources && yarn compile && node . pre-compile-views && node scripts/dist.js",
-    "dev": "yarn prepareSources && concurrently -k -n \"Typescript,Node,Webpack,Maildev\" -p \"[{name}]\" -c \"blue,green,red,yellow\" \"tsc --watch\" \"nodemon -i public -i build\" \"maildev\"",
+    "build-production": "NODE_ENV=production yarn build",
+    "dev": "yarn prepare-sources && concurrently -k -n \"Typescript,Node,ViewPreCompile,Maildev\" -p \"[{name}]\" -c \"blue,green,red,yellow\" \"tsc --watch\" \"nodemon -i public -i build\" \"nodemon -i public -i build -- pre-compile-views --watch\" \"maildev\"",
     "lint": "eslint .",
     "release": "yarn build && yarn lint && yarn test && cd dist && yarn publish"
   },

rollup.config.js

@@ -0,0 +1,70 @@
+import path from "path";
+import svelte from "rollup-plugin-svelte";
+import {sveltePreprocess} from "svelte-preprocess/dist/autoProcess";
+import cssOnlyRollupPlugin from "rollup-plugin-css-only";
+import resolve from "@rollup/plugin-node-resolve";
+import commonjs from "@rollup/plugin-commonjs";
+import {terser} from "rollup-plugin-terser";
+import livereloadRollupPlugin from "rollup-plugin-livereload";
+
+const production = process.env.ENV === 'production';
+const buildDir = process.env.BUILD_DIR;
+const publicDir = process.env.PUBLIC_DIR;
+const input = process.env.INPUT.split(':');
+
+export default commandLineArgs => {
+    const type = commandLineArgs.input;
+    delete commandLineArgs.input;
+
+    return {
+        input: input,
+        output: {
+            sourcemap: true,
+            format: 'es',
+            name: 'bundle',
+            dir: path.join(publicDir, 'js'),
+            entryFileNames: (chunkInfo) => {
+                const name = chunkInfo.facadeModuleId ?
+                    path.relative(buildDir, chunkInfo.facadeModuleId) :
+                    chunkInfo.name;
+                return name + '.js';
+            },
+        },
+        plugins: [
+            svelte({
+                preprocess: sveltePreprocess({
+                    typescript: {
+                        tsconfigFile: 'tsconfig.views.json',
+                    },
+                }),
+                compilerOptions: {
+                    dev: !production,
+                    hydratable: true,
+                },
+            }),
+
+            // Extract css into separate files
+            cssOnlyRollupPlugin({output: 'bundle.css'}),
+
+            // If you have external dependencies installed from
+            // npm, you'll most likely need these plugins. In
+            // some cases you'll need additional configuration -
+            // consult the documentation for details:
+            // https://github.com/rollup/plugins/tree/master/packages/commonjs
+            resolve({
+                browser: true,
+                dedupe: ['svelte'],
+            }),
+            commonjs(),
+
+            // Live reload in dev
+            !production && !!commandLineArgs.watch && livereloadRollupPlugin('public'),
+
+            // Minify in production
+            production && terser(),
+        ],
+        watch: {
+            clearScreen: false,
+        },
+    };
+};

src/Application.ts

@@ -17,6 +17,7 @@ import CacheProvider from "./CacheProvider";
 import RedisComponent from "./components/RedisComponent";
 import Extendable from "./Extendable";
 import {logger, loggingContextMiddleware} from "./Logger";
+import FrontendToolsComponent from "./components/FrontendToolsComponent";
 import TemplateError = lib.TemplateError;
 
 export default abstract class Application implements Extendable<ApplicationComponent | WebSocketListener<Application>> {
@@ -225,6 +226,7 @@ export default abstract class Application implements Extendable<ApplicationCompo
         const flags = {
             verbose: false,
             fullHttpRequests: false,
+            watch: false,
         };
         let mainCommand: string | null = null;
         const mainCommandArgs: string[] = [];
@@ -236,7 +238,11 @@ export default abstract class Application implements Extendable<ApplicationCompo
                 case '--full-http-requests':
                     flags.fullHttpRequests = true;
                     break;
+                case '--watch':
+                    flags.watch = true;
+                    break;
                 case 'migration':
+                case 'pre-compile-views':
                     if (mainCommand === null) mainCommand = args[i];
                     else throw new Error(`Only one main command can be used at once (${mainCommand},${args[i]})`);
                     break;
@@ -256,6 +262,9 @@ export default abstract class Application implements Extendable<ApplicationCompo
                     await MysqlConnectionManager.migrationCommand(mainCommandArgs);
                     await this.stop();
                     break;
+                case 'pre-compile-views':
+                    await this.as(FrontendToolsComponent).preCompileViews(flags.watch);
+                    break;
                 default:
                     logger.fatal('Unimplemented main command', mainCommand);
                     break;

src/components/FrontendToolsComponent.ts

@@ -29,12 +29,6 @@ export default class FrontendToolsComponent extends ApplicationComponent {
             logger.info('Asset cache disabled.');
         }
 
-        // Pre-compile and watch in dev mode
-        if (config.get<boolean>('view.dev')) {
-            await this.viewEngine.preCompileAll();
-            await this.viewEngine.watch();
-        }
-
         // Setup express view engine
         app.engine(this.viewEngine.getExtension(), (path, options, callback) => {
             this.viewEngine.render(path, options as Record<string, unknown>, callback)
@@ -45,7 +39,7 @@ export default class FrontendToolsComponent extends ApplicationComponent {
     }
 
     public async stop(): Promise<void> {
-        await this.viewEngine.end?.();
+        await this.viewEngine.stop();
     }
 
     public async handle(router: Router): Promise<void> {
@@ -57,4 +51,8 @@ export default class FrontendToolsComponent extends ApplicationComponent {
             next();
         });
     }
+
+    public async preCompileViews(watch: boolean): Promise<void> {
+        await this.viewEngine.preCompileAll(watch);
+    }
 }

src/frontend/SvelteViewEngine.ts

@@ -1,6 +1,5 @@
 import crypto from "crypto";
 import path from "path";
-import {PreRenderedChunk, rollup, RollupBuild, RollupOptions, RollupWatcher, RollupWatchOptions, watch} from "rollup";
 import config from "config";
 import ViewEngine from "./ViewEngine";
 import {logger} from "../Logger";
@@ -10,12 +9,7 @@ import {compile, preprocess} from "svelte/compiler";
 import {sveltePreprocess} from "svelte-preprocess/dist/autoProcess";
 import requireFromString from "require-from-string";
 import {CssResult} from "svelte/types/compiler/interfaces";
-import svelte from "rollup-plugin-svelte";
+import * as child_process from "child_process";
-import cssOnlyRollupPlugin from "rollup-plugin-css-only";
-import resolve from "@rollup/plugin-node-resolve";
-import commonjs from "@rollup/plugin-commonjs";
-import {terser} from "rollup-plugin-terser";
-import livereloadRollupPlugin from "rollup-plugin-livereload";
 
 const BACKEND_CODE_PREFIX = 'swaf.';
 const COMPILED_SVELTE_EXTENSION = '.swafview';
@@ -37,7 +31,7 @@ export default class SvelteViewEngine extends ViewEngine {
         backendLines: string[],
     }> = {};
 
-    private rollup?: RollupBuild | RollupWatcher;
+    private rollup?: child_process.ChildProcess;
 
     public constructor(
         buildDir: string,
@@ -112,7 +106,8 @@ export default class SvelteViewEngine extends ViewEngine {
         callback(null, output);
     }
 
-    public async end(): Promise<void> {
+    public async stop(): Promise<void> {
+        await super.stop();
         await this.stopRollup();
     }
 
@@ -130,7 +125,7 @@ export default class SvelteViewEngine extends ViewEngine {
         const file = await this.resolveFileFromCanonicalName(canonicalName);
         const intermediateFile = path.join(this.getBuildDir(), canonicalName);
 
-        logger.debug(canonicalName + ' > ', 'Pre-compiling', file, '->', intermediateFile);
+        logger.info(canonicalName + ' > ', 'Pre-compiling', file, '->', intermediateFile);
         const source = await this.fileCache.get(file, config.get<boolean>('view.dev'));
 
         const allBackendLines: string[] = [];
@@ -157,7 +152,7 @@ export default class SvelteViewEngine extends ViewEngine {
         await afs.writeFile(swafViewFile, finalCode);
 
         if (alsoCompileDependents && Object.keys(this.dependencyCache).indexOf(canonicalName) >= 0) {
-            logger.debug(canonicalName + ' > ', 'Compiling dependents...');
+            logger.info(canonicalName + ' > ', 'Compiling dependents...');
             for (const dependent of [...this.dependencyCache[canonicalName]]) {
                 await this.preCompile(dependent, true);
             }
@@ -271,8 +266,8 @@ export default class SvelteViewEngine extends ViewEngine {
         return generated;
     }
 
-    public async afterPreCompile(): Promise<void> {
+    public async afterPreCompile(watch: boolean): Promise<void> {
-        await this.bundle(...Object.keys(this.backendCodeCache));
+        await this.bundle(watch, ...Object.keys(this.backendCodeCache));
     }
 
 
@@ -284,13 +279,22 @@ export default class SvelteViewEngine extends ViewEngine {
         delete this.backendCodeCache[this.toCanonicalName(file)];
     }
 
+    public async onFileRemove(file: string): Promise<void> {
+        const canonicalName = this.toCanonicalName(file);
+        delete this.backendCodeCache[canonicalName];
+        delete this.dependencyCache[canonicalName];
+        Object.values(this.dependencyCache).forEach(set => set.delete(canonicalName));
+        await this.stopRollup();
+        await this.afterPreCompile(true);
+    }
+
     private async compileSsr(canonicalName: string, file: string, code: string): Promise<{
         head: string,
         css: CssResult,
         html: string,
     }> {
         // Svelte preprocess
-        logger.debug(canonicalName + ' > ', 'Preprocessing svelte', file);
+        logger.info(canonicalName + ' > ', 'Preprocessing svelte', file);
         const preprocessed = await preprocess(
             code,
             sveltePreprocess({
@@ -304,7 +308,7 @@ export default class SvelteViewEngine extends ViewEngine {
         );
 
         // Svelte compile
-        logger.debug(canonicalName + ' > ', 'Compiling svelte ssr', file);
+        logger.info(canonicalName + ' > ', 'Compiling svelte ssr', file);
         const svelteSsr = compile(preprocessed.code, {
             dev: config.get<boolean>('view.dev'),
             generate: 'ssr',
@@ -324,103 +328,43 @@ export default class SvelteViewEngine extends ViewEngine {
         });
     }
 
-    /**
+    private async bundle(watch: boolean, ...canonicalViewNames: string[]): Promise<void> {
-     * TODO: load rollup config from external source
+        logger.info('Bundling...');
-     */
-    private async bundle(...canonicalViewNames: string[]): Promise<void> {
-        logger.debug('Bundling');
-
-        const production = !config.get<boolean>('view.dev');
-        const options: RollupOptions | RollupWatchOptions = {
-            input: canonicalViewNames.map(name => path.join(this.getBuildDir(), name)),
-            output: {
-                sourcemap: true,
-                format: 'es',
-                name: 'bundle',
-                dir: path.join(this.getPublicDir(), 'js'),
-                entryFileNames: (chunkInfo: PreRenderedChunk): string => {
-                    const name = chunkInfo.facadeModuleId ?
-                        path.relative(this.getBuildDir(), chunkInfo.facadeModuleId) :
-                        chunkInfo.name;
-                    return name + '.js';
-                },
-            },
-            plugins: [
-                svelte({
-                    preprocess: sveltePreprocess({
-                        typescript: {
-                            tsconfigFile: 'tsconfig.views.json',
-                        },
-                    }),
-                    compilerOptions: {
-                        // enable run-time checks when not in production
-                        dev: !production,
-                        hydratable: true,
-                    },
-                }),
-                // we'll extract any component CSS out into
-                // a separate file - better for performance
-                cssOnlyRollupPlugin({output: 'bundle.css'}),
-
-                // If you have external dependencies installed from
-                // npm, you'll most likely need these plugins. In
-                // some cases you'll need additional configuration -
-                // consult the documentation for details:
-                // https://github.com/rollup/plugins/tree/master/packages/commonjs
-                resolve({
-                    browser: true,
-                    dedupe: ['svelte'],
-                }),
-                commonjs(),
-            ],
-            watch: {
-                buildDelay: 1000,
-                clearScreen: false,
-            },
-        };
-
-        if (production) {
-            // If we're building for production (npm run build
-            // instead of npm run dev), minify
-            options.plugins?.push(terser());
-        } else {
-            // Watch the `public` directory and refresh the
-            // browser on changes when not in production
-            const plugin = livereloadRollupPlugin('public');
-            options.plugins?.push(plugin);
-        }
 
+        // Prepare output dir
         for (const name of canonicalViewNames) {
             await afs.mkdir(path.dirname(path.join(this.getPublicDir(), 'js', name)), {recursive: true});
         }
 
-        if (production) {
+        const production = !config.get<boolean>('view.dev');
-            if (!this.rollup) {
+        const input = canonicalViewNames.map(name => path.join(this.getBuildDir(), name));
-                this.rollup = await rollup(options);
 
-                await this.rollup.write({
+        if (!this.rollup) {
-                    format: 'es',
+            const args = [
-                    dir: path.join(this.getPublicDir(), 'js'),
+                'rollup',
-                });
+                '-c', 'rollup.config.js',
-            }
+                '--environment', `ENV:${production ? 'production' : 'dev'},BUILD_DIR:${this.getBuildDir()},PUBLIC_DIR:${this.getPublicDir()},INPUT:${input.join(':')}`,
-        } else {
+            ];
-            if (!this.rollup) {
+            if (watch) args.push('--watch');
-                this.rollup = watch(options);
+            this.rollup = child_process.spawn('yarn', args, {stdio: [process.stdin, process.stdout, process.stderr]});
-                this.rollup.on('event', (event) => {
+            logger.info('Rollup started');
-                    if (event.code === 'ERROR' || event.code === 'BUNDLE_END') {
+            this.rollup.once('exit', () => {
-                        event.result?.close().catch(err => logger.error(err));
+                logger.info('Rollup stopped');
-                        logger.debug('Bundled from watch');
+                this.rollup = undefined;
-                    }
+            });
-                });
-            }
         }
     }
 
     private async stopRollup(): Promise<void> {
         if (this.rollup) {
-            logger.debug('Stopping rollup...');
+            logger.info(`Stopping rollup (${this.rollup.pid})...`);
-            await this.rollup.close();
+            await new Promise<void>((resolve, reject) => {
-            this.rollup = undefined;
+                if (!this.rollup) return resolve();
+                this.rollup.once('exit', () => {
+                    resolve();
+                });
+                if (!this.rollup.kill("SIGTERM")) reject('Could not stop rollup.');
+            });
         }
     }
 }

src/frontend/ViewEngine.ts

@@ -1,11 +1,12 @@
 import path from "path";
 import fs from "fs";
-import chokidar from "chokidar";
+import chokidar, {FSWatcher} from "chokidar";
 import {logger} from "../Logger";
 import {afs, readdirRecursively} from "../Utils";
 
 export default abstract class ViewEngine {
     protected readonly viewPaths: string[];
+    private watcher?: FSWatcher;
 
     /**
      * @param buildDir A temporary directory that will contain any non-final or final non-public asset.
@@ -41,8 +42,6 @@ export default abstract class ViewEngine {
         callback: (err: Error | null, output?: string) => void,
     ): Promise<void>;
 
-    public end?(): Promise<void>;
-
     public getViewPaths(): string[] {
         return this.viewPaths;
     }
@@ -61,9 +60,9 @@ export default abstract class ViewEngine {
 
     public preCompile?(canonicalName: string, alsoCompileDependents: boolean): Promise<void>;
 
-    public afterPreCompile?(): Promise<void>;
+    public afterPreCompile?(watch: boolean): Promise<void>;
 
-    public async preCompileAll(): Promise<void> {
+    public async preCompileAll(watch: boolean): Promise<void> {
         if (this.preCompile) {
             logger.info(`Pre-compiling ${this.getExtension()} views...`);
 
@@ -83,38 +82,60 @@ export default abstract class ViewEngine {
             }
         }
 
-        await this.afterPreCompile?.();
+        await this.afterPreCompile?.(watch);
+
+        if (watch) {
+            await this.watch();
+        }
     }
 
     public onNewFile?(file: string): Promise<void>;
 
     public onFileChange?(file: string): Promise<void>;
 
-    public async watch(): Promise<void> {
+    public onFileRemove?(file: string): Promise<void>;
-        const watcher = chokidar.watch(this.devWatchedViewDir, {persistent: true});
-        watcher.on('ready', () => {
-            logger.debug(`Watching ${this.getExtension()} assets for changes`);
 
-            watcher.on('add', (file) => {
+    public async watch(): Promise<void> {
+        this.watcher = chokidar.watch(this.devWatchedViewDir, {persistent: true});
+        this.watcher.on('ready', () => {
+            if (!this.watcher) return;
+
+            logger.info(`Watching ${this.getExtension()} assets for changes`);
+
+            this.watcher.on('add', (file) => {
                 if (file.endsWith('.' + this.getExtension())) {
                     (this.onNewFile ? this.onNewFile(file) : Promise.resolve())
                         .then(() => this.preCompile?.(this.toCanonicalName(file), true))
-                        .then(() => this.afterPreCompile?.())
+                        .then(() => this.afterPreCompile?.(true))
                         .catch(err => logger.error(err));
                 }
             });
 
-            watcher.on('change', (file) => {
+            this.watcher.on('change', (file) => {
                 if (file.endsWith('.' + this.getExtension())) {
                     (this.onFileChange ? this.onFileChange(file) : Promise.resolve())
                         .then(() => this.preCompile?.(this.toCanonicalName(file), true))
-                        .then(() => this.afterPreCompile?.())
+                        .then(() => this.afterPreCompile?.(true))
+                        .catch(err => logger.error(err));
+                }
+            });
+
+            this.watcher.on('unlink', (file) => {
+                if (file.endsWith('.' + this.getExtension())) {
+                    (this.onFileRemove ? this.onFileRemove(file) : Promise.resolve())
                         .catch(err => logger.error(err));
                 }
             });
         });
     }
 
+    public async stop(): Promise<void> {
+        if (this.watcher) {
+            await this.watcher.close();
+            this.watcher = undefined;
+        }
+    }
+
     protected toCanonicalName(file: string): string {
         const resolvedFilePath = path.resolve(file);