Add two step pre-compile/compile asset processing

Reorganize views into new "assets" folder structure
Turn locals into a store so locals don't have to be passed through files that don't need them
Some fixes to previous commit (esm) 82ab0b963c
Remove afs in favor of fs.promises (renamed afs.exists to Utils.doesFileExist
Rename Utils.readdirRecursively to Utils.listFilesRecursively

assets/ts/stores.ts

@@ -0,0 +1,3 @@
+import {writable} from "svelte/store";
+
+export const locals = writable({});

assets/views/layouts/svelte_layout.html

@@ -5,9 +5,10 @@
 
     <style>%css%</style>
     <script type="module" defer>
-        import View from '/js/%canonicalViewName%.js';
+        import View from '/js/views/%canonicalViewName%.js';
+        import {locals} from '/js/stores.js';
 
-        const locals = %locals%;
+        locals.set(%locals%);
 
         new View({
             hydrate: true,

rollup.config.js

@@ -25,12 +25,13 @@ export default commandLineArgs => ({
                 chunkInfo.name;
             return name + '.js';
         },
+        chunkFileNames: '[name].js',
     },
     plugins: [
         svelte({
             preprocess: sveltePreprocess({
                 typescript: {
-                    tsconfigFile: 'tsconfig.views.json',
+                    tsconfigFile: 'tsconfig.svelte.json',
                 },
             }),
             compilerOptions: {

scripts/dist.js

@@ -22,7 +22,7 @@ function copyRecursively(file, destination) {
     'yarn.lock',
     'README.md',
     'config/',
-    'views/',
+    'assets/',
 ].forEach(file => {
     copyRecursively(file, 'dist');
 });

src/TestApp.ts

@@ -29,9 +29,12 @@ import SessionComponent from "./components/SessionComponent.js";
 import WebSocketServerComponent from "./components/WebSocketServerComponent.js";
 import Controller from "./Controller.js";
 import Migration, {MigrationType} from "./db/Migration.js";
+import AssetCompiler from "./frontend/AssetCompiler.js";
+import CopyAssetPreCompiler from "./frontend/CopyAssetPreCompiler.js";
 import MailViewEngine from "./frontend/MailViewEngine.js";
 import NunjucksViewEngine from "./frontend/NunjucksViewEngine.js";
 import SvelteViewEngine from "./frontend/SvelteViewEngine.js";
+import TypeScriptPreCompiler from "./frontend/TypeScriptPreCompiler.js";
 import BackendController from "./helpers/BackendController.js";
 import MailController from "./mail/MailController.js";
 import {MAGIC_LINK_MAIL} from "./Mails.js";
@@ -77,16 +80,19 @@ export default class TestApp extends Application {
         this.use(new ServeStaticDirectoryComponent('public'));
 
         // Dynamic views and routes
+        const intermediateDirectory = 'build';
         this.use(new FrontendToolsComponent(
-            'public',
-            new SvelteViewEngine('build/views', 'public', 'views', 'test/views'),
-            new NunjucksViewEngine('views', 'test/views'),
+            new AssetCompiler(intermediateDirectory, 'public'),
+            new CopyAssetPreCompiler(intermediateDirectory, '', 'json', ['test/assets'], false),
+            new TypeScriptPreCompiler(intermediateDirectory, ['test/assets']),
+            new SvelteViewEngine(intermediateDirectory, 'test/assets'),
+            new NunjucksViewEngine(intermediateDirectory, 'test/assets'),
         ));
         this.use(new PreviousUrlComponent());
 
         // Services
         this.use(new MysqlComponent());
-        this.use(new MailComponent(new MailViewEngine('views', 'test/views')));
+        this.use(new MailComponent(new MailViewEngine('build', 'test/assets')));
 
         // Session
         this.use(new RedisComponent());

src/Utils.ts

@@ -1,6 +1,5 @@
-import fs from "fs";
+import {promises as fs} from "fs";
 import path from "path";
-import * as util from "util";
 
 export async function sleep(ms: number): Promise<void> {
     return await new Promise(resolve => {
@@ -49,37 +48,31 @@ export function getMethods<T extends { [p: string]: unknown }>(obj: T): string[]
     return [...properties.keys()].filter(item => typeof obj[item] === 'function');
 }
 
-export const afs = {
-    readFile: util.promisify(fs.readFile),
-    writeFile: util.promisify(fs.writeFile),
-    mkdir: util.promisify(fs.mkdir),
-    readdir: util.promisify(fs.readdir),
-    exists: async (file: string): Promise<boolean> =>
-        await new Promise<boolean>((resolve, reject) =>
-            fs.stat(file, (err) => {
-                if (err == null) {
-                    resolve(true);
-                } else if (err.code === 'ENOENT') {
-                    resolve(false);
-                } else {
-                    reject(err);
-                }
-            })),
-};
+export async function listFilesRecursively(dir: string): Promise<string[]> {
+    const localFiles = await fs.readdir(dir);
+    const files: string[] = [];
+    for (const file of localFiles.map(file => path.join(dir, file))) {
+        const stat = await fs.stat(file);
 
-export async function readdirRecursively(dir: string, consumer: (file: string) => Promise<void>): Promise<void> {
-    const files = await afs.readdir(dir);
-
-    await Promise.all<void>(files.map(file => new Promise<void>((resolve, reject) => {
-        file = path.join(dir, file);
-        fs.stat(file, (err, stat) => {
-            if (err) return reject(err);
-
-            if (stat.isDirectory()) {
-                resolve(readdirRecursively(file, consumer));
-            } else {
-                resolve(consumer(file));
-            }
-        });
-    })));
+        if (stat.isDirectory()) {
+            files.push(...await listFilesRecursively(file));
+        } else {
+            files.push(file);
+        }
+    }
+    return files;
+}
+
+
+export async function doesFileExist(file: string): Promise<boolean> {
+    try {
+        await fs.stat(file);
+    } catch (err) {
+        if (err.code === 'ENOENT') {
+            return false;
+        } else {
+            throw err;
+        }
+    }
+    return true;
 }

src/components/FrontendToolsComponent.ts

@@ -1,5 +1,3 @@
-import "../../node_modules/svelte/register.js";
-
 import config from "config";
 import {Express, Router} from "express";
 import path from "path";
@@ -9,40 +7,53 @@ import util from "util";
 
 import ApplicationComponent from "../ApplicationComponent.js";
 import Controller, {RouteParams} from "../Controller.js";
+import AssetCompiler from "../frontend/AssetCompiler.js";
+import AssetPreCompiler from "../frontend/AssetPreCompiler.js";
 import ViewEngine from "../frontend/ViewEngine.js";
 import {logger} from "../Logger.js";
-import {readdirRecursively} from "../Utils.js";
+import {listFilesRecursively} from "../Utils.js";
 import FileCache from "../utils/FileCache.js";
 
 export default class FrontendToolsComponent extends ApplicationComponent {
+    private readonly publicDir: string;
     private readonly publicAssetsCache: FileCache = new FileCache();
-    private readonly viewEngines: ViewEngine[];
+    private readonly assetPreCompilers: AssetPreCompiler[];
 
     public constructor(
-        private readonly publicAssetsDir: string,
-        ...viewEngines: ViewEngine[]
+        private readonly assetCompiler: AssetCompiler,
+        ...assetPreCompilers: AssetPreCompiler[]
     ) {
         super();
-        this.viewEngines = viewEngines;
+        this.assetPreCompilers = assetPreCompilers;
+        this.publicDir = this.assetCompiler.targetDir;
+
+        for (const assetPreCompiler of this.assetPreCompilers) {
+            if (assetPreCompiler.isPublic()) {
+                this.assetCompiler.addExtension(assetPreCompiler.getExtension());
+            }
+        }
     }
 
     public async start(app: Express): Promise<void> {
         // Cache public assets
         if (config.get<boolean>('asset_cache')) {
-            logger.info('Caching assets from', this.publicAssetsDir, '...');
-            await readdirRecursively(
-                this.publicAssetsDir,
-                async file => await this.publicAssetsCache.load(file),
-            );
+            logger.info('Caching assets from', this.publicDir, '...');
+            for (const file of await listFilesRecursively(this.publicDir)) {
+                await this.publicAssetsCache.load(file);
+            }
         } else {
             logger.info('Asset cache disabled.');
         }
 
+        this.hookPreCompilers();
+
         // Setup express view engine
         let main = true;
-        for (const viewEngine of this.viewEngines) {
-            viewEngine.setup(app, main);
-            main = false;
+        for (const assetPreCompiler of this.assetPreCompilers) {
+            if (assetPreCompiler instanceof ViewEngine) {
+                assetPreCompiler.setup(app, main);
+                main = false;
+            }
         }
 
         // Add util globals
@@ -50,15 +61,15 @@ export default class FrontendToolsComponent extends ApplicationComponent {
     }
 
     public async stop(): Promise<void> {
-        for (const viewEngine of this.viewEngines) {
-            await viewEngine.stop();
+        for (const assetPreCompiler of this.assetPreCompilers) {
+            await assetPreCompiler.stop();
         }
     }
 
     public async handle(router: Router): Promise<void> {
         router.use((req, res, next) => {
             res.locals.inlineAsset = (urlPath: string) => {
-                return this.publicAssetsCache.getOrFail(path.join(this.publicAssetsDir, urlPath));
+                return this.publicAssetsCache.getOrFail(path.join(this.publicDir, urlPath));
             };
 
             next();
@@ -75,12 +86,6 @@ export default class FrontendToolsComponent extends ApplicationComponent {
         });
     }
 
-    public async preCompileViews(watch: boolean): Promise<void> {
-        for (const viewEngine of this.viewEngines) {
-            await viewEngine.preCompileAll(watch);
-        }
-    }
-
     public setupGlobals(): void {
         ViewEngine.setGlobal('route', (
             route: string,
@@ -99,4 +104,27 @@ export default class FrontendToolsComponent extends ApplicationComponent {
             return v.toString(16);
         });
     }
+
+    public hookPreCompilers(): void {
+        for (const assetPreCompiler of this.assetPreCompilers) {
+            assetPreCompiler.onPreCompile(async watch => {
+                await this.assetCompiler.compile(watch);
+            });
+            assetPreCompiler.onInputChange(async restart => {
+                await this.assetCompiler.stopWatching(restart);
+            });
+        }
+    }
+
+    public async preCompileViews(watch: boolean): Promise<void> {
+        for (const viewEngine of this.assetPreCompilers) {
+            await viewEngine.preCompileAll(watch);
+        }
+
+        await this.assetCompiler.compile(watch);
+
+        if (watch) {
+            this.hookPreCompilers();
+        }
+    }
 }

src/frontend/AssetCompiler.ts

@@ -0,0 +1,71 @@
+import child_process from "child_process";
+import config from "config";
+import fs from "fs";
+
+import {logger} from "../Logger.js";
+import {listFilesRecursively} from "../Utils.js";
+
+export default class AssetCompiler {
+    private rollup?: child_process.ChildProcess;
+    private extensions: string[] = [];
+
+
+    /**
+     * @param sourceDir The source assets directory.
+     * @param targetDir The output directory that should contain all final and public assets.
+     */
+    public constructor(
+        public readonly sourceDir: string,
+        public readonly targetDir: string,
+    ) {
+        if (!fs.existsSync(this.targetDir)) {
+            fs.mkdirSync(this.targetDir, {recursive: true});
+        }
+    }
+
+    public addExtension(extension: string): void {
+        this.extensions.push(extension);
+    }
+
+    public async compile(watch: boolean): Promise<void> {
+        // Prepare input list
+        const input = (await listFilesRecursively(this.sourceDir))
+            .filter(f => this.extensions.find(ext => f.endsWith('.' + ext)));
+        logger.info('Input list:', input);
+
+        const production = !config.get<boolean>('view.dev');
+        if (!this.rollup) {
+            const args = [
+                'rollup',
+                '-c', 'rollup.config.js',
+                '--environment', `ENV:${production ? 'production' : 'dev'},BUILD_DIR:${this.sourceDir},PUBLIC_DIR:${this.targetDir},INPUT:${input.join(':')}`,
+            ];
+            if (watch) args.push('--watch');
+            this.rollup = child_process.spawn('yarn', args, {stdio: [process.stdin, process.stdout, process.stderr]});
+            logger.info('Rollup started');
+            this.rollup.once('exit', (code: number) => {
+                logger.info('Rollup stopped (' + code + ')');
+                this.rollup = undefined;
+
+                if (!watch && code !== 0) process.exit(code);
+            });
+        }
+    }
+
+    public async stopWatching(restart: boolean): Promise<void> {
+        if (this.rollup) {
+            logger.info(`Stopping rollup (${this.rollup.pid})...`);
+            await new Promise<void>((resolve, reject) => {
+                if (!this.rollup) return resolve();
+                this.rollup.once('exit', () => {
+                    resolve();
+                });
+                if (!this.rollup.kill("SIGTERM")) reject('Could not stop rollup.');
+            });
+        }
+
+        if (restart) {
+            await this.compile(true);
+        }
+    }
+}

src/frontend/AssetPreCompiler.ts

@@ -0,0 +1,179 @@
+import chokidar, {FSWatcher} from "chokidar";
+import {existsSync, mkdirSync,promises as fs} from "fs";
+import path from "path";
+
+import {logger} from "../Logger.js";
+import {doesFileExist, listFilesRecursively} from "../Utils.js";
+
+export default abstract class AssetPreCompiler {
+    protected readonly assetPaths: string[];
+    private watcher?: FSWatcher;
+    private afterPreCompileHandlers: ((watch: boolean) => Promise<void>)[] = [];
+    private inputChangeHandler?: (restart: boolean) => Promise<void>;
+
+    /**
+     * @param targetDir The directory to put pre-compiled assets into.
+     * @param assetType This must be the assets sub-directory name of the asset type this pre-compiler will handle.
+     * Example: ts
+     * Example: views
+     * @param extension The asset files extension.
+     * @param outputToPublicDir should the assets be compiled into the publicly served directory?
+     * @param additionalFallbackAssetPaths By order of priority, fallback asset directories.
+     * Swaf provided assets and default ./assets directory are automatically added (don't add them yourself).
+     */
+    protected constructor(
+        protected readonly targetDir: string,
+        protected readonly assetType: string,
+        protected readonly extension: string,
+        private readonly outputToPublicDir: boolean,
+        ...additionalFallbackAssetPaths: string[]
+    ) {
+        this.assetPaths = [
+            ...additionalFallbackAssetPaths,
+            'assets',
+            'node_modules/swaf/assets',
+        ].map(p => path.resolve(p, assetType))
+            .filter(dir => existsSync(dir));
+        this.targetDir = path.join(targetDir, assetType);
+
+        if (!existsSync(this.targetDir)) {
+            mkdirSync(this.targetDir, {recursive: true});
+        }
+    }
+
+    public getExtension(): string {
+        return this.extension;
+    }
+
+    public isPublic(): boolean {
+        return this.outputToPublicDir;
+    }
+
+    public async stop(): Promise<void> {
+        if (this.watcher) {
+            await this.watcher.close();
+            this.watcher = undefined;
+        }
+    }
+
+    public getViewPaths(): string[] {
+        return this.assetPaths;
+    }
+
+    public getPrimaryAssetPath(): string {
+        if (this.assetPaths.length === 0) throw new Error('No asset path was found.');
+        return this.assetPaths[0];
+    }
+
+    public abstract preCompile(canonicalName: string, alsoCompileDependents: boolean): Promise<void>;
+
+    public onPreCompile(afterPreCompileHandler: (watch: boolean) => Promise<void>): void {
+        this.afterPreCompileHandlers.push(afterPreCompileHandler);
+    }
+
+    protected async afterPreCompile(watch: boolean): Promise<void> {
+        await Promise.all(this.afterPreCompileHandlers.map(handler => handler(watch)));
+    }
+
+    public async preCompileAll(watch: boolean): Promise<void> {
+        logger.info(`Pre-compiling ${this.extension} views...`);
+
+        // List all views
+        const views: string[] = [];
+        for (const viewPath of this.assetPaths) {
+            for (const file of await listFilesRecursively(viewPath)) {
+                if (file.endsWith('.' + this.extension)) {
+                    views.push(this.toCanonicalName(file));
+                }
+            }
+        }
+
+        // Deduplicate and pre-compile
+        for (const canonicalName of [...new Set<string>(views)]) {
+            await this.preCompile(canonicalName, false);
+        }
+
+        await this.afterPreCompile(watch);
+
+        if (watch) {
+            await this.watch();
+        }
+    }
+
+    public async watch(): Promise<void> {
+        this.watcher = chokidar.watch(this.getPrimaryAssetPath(), {persistent: true});
+        this.watcher.on('ready', () => {
+            if (!this.watcher) return;
+
+            logger.info(`Watching ${this.extension} assets for changes`);
+
+            this.watcher.on('add', (file) => {
+                if (file.endsWith('.' + this.extension)) {
+                    this.onNewFile(file)
+                        .then(() => this.preCompile(this.toCanonicalName(file), true))
+                        .then(() => {
+                            return this.afterPreCompile(true);
+                        })
+                        .catch(err => logger.error(err));
+                }
+            });
+
+            this.watcher.on('change', (file) => {
+                if (file.endsWith('.' + this.extension)) {
+                    (this.onFileChange ? this.onFileChange(file) : Promise.resolve())
+                        .then(() => this.preCompile(this.toCanonicalName(file), true))
+                        .then(() => {
+                            return this.afterPreCompile(true);
+                        })
+                        .catch(err => logger.error(err));
+                }
+            });
+
+            this.watcher.on('unlink', (file) => {
+                if (file.endsWith('.' + this.extension)) {
+                    this.onFileRemove(file)
+                        .catch(err => logger.error(err));
+                }
+            });
+        });
+    }
+
+    public async onNewFile(_file: string): Promise<void> {
+        await this.inputChangeHandler?.(false);
+    }
+
+    public onFileChange?(file: string): Promise<void>;
+
+    public async onFileRemove(_file: string): Promise<void> {
+        await this.inputChangeHandler?.(true);
+    }
+
+    public onInputChange(inputChangeHandler: (restart: boolean) => Promise<void>): void {
+        this.inputChangeHandler = inputChangeHandler;
+    }
+
+    protected toCanonicalName(file: string): string {
+        const resolvedFilePath = path.resolve(file);
+
+        let canonicalViewName: string | null = null;
+        for (const viewPath of this.assetPaths) {
+            if (resolvedFilePath.startsWith(viewPath)) {
+                canonicalViewName = resolvedFilePath.substring(viewPath.length + 1);
+            }
+        }
+
+        if (!canonicalViewName) throw new Error('view ' + file + ' not found');
+        return canonicalViewName;
+    }
+
+    protected async resolveFileFromCanonicalName(canonicalName: string): Promise<string> {
+        for (const viewPath of this.assetPaths) {
+            const tryPath = path.join(viewPath, canonicalName);
+            if (await doesFileExist(tryPath)) {
+                return tryPath;
+            }
+        }
+
+        throw new Error('View not found from canonical name ' + canonicalName);
+    }
+}

src/frontend/CopyAssetPreCompiler.ts

@@ -0,0 +1,31 @@
+import {promises as fs} from "fs";
+import path from "path";
+
+import {logger} from "../Logger.js";
+import AssetPreCompiler from "./AssetPreCompiler.js";
+
+export default class CopyAssetPreCompiler extends AssetPreCompiler {
+    protected readonly actualTargetDir: string;
+
+    public constructor(
+        targetDir: string,
+        assetType: string,
+        extension: string,
+        additionalFallbackAssetPaths: string[] = [],
+        outputToPublic: boolean = true,
+        overrideTargetAssetType?: string,
+    ) {
+        super(targetDir, assetType, extension, outputToPublic, ...additionalFallbackAssetPaths);
+        this.actualTargetDir = overrideTargetAssetType ?
+            path.join(targetDir, overrideTargetAssetType) :
+            this.targetDir;
+    }
+
+    public async preCompile(canonicalName: string): Promise<void> {
+        const inputFile = await this.resolveFileFromCanonicalName(canonicalName);
+        const outputFile = path.join(this.actualTargetDir, canonicalName);
+        logger.info('Copying', inputFile, '->', outputFile);
+        await fs.mkdir(path.dirname(outputFile), {recursive: true});
+        await fs.copyFile(inputFile, outputFile);
+    }
+}

src/frontend/NunjucksViewEngine.ts

@@ -6,8 +6,8 @@ import ViewEngine from "./ViewEngine.js";
 export default class NunjucksViewEngine extends ViewEngine {
     private readonly environment: Environment;
 
-    public constructor(devWatchedViewDir: string, ...additionalViewPaths: string[]) {
-        super(devWatchedViewDir, ...additionalViewPaths);
+    public constructor(targetDir: string, ...additionalViewPaths: string[]) {
+        super(targetDir, 'views', 'njk', false, ...additionalViewPaths);
 
         const opts = {
             autoescape: true,
@@ -15,12 +15,12 @@ export default class NunjucksViewEngine extends ViewEngine {
             throwOnUndefined: true,
         };
         this.environment = new nunjucks.Environment([
-            ...this.viewPaths.map(path => new nunjucks.FileSystemLoader(path, opts)),
+            ...this.assetPaths.map(path => new nunjucks.FileSystemLoader(path, opts)),
         ], opts);
     }
 
-    public getExtension(): string {
-        return 'njk';
+    public async preCompile(): Promise<void> {
+        // Nunjucks doesn't need preCompilation or compilation
     }
 
     public async render(file: string, locals: Record<string, unknown>): Promise<string> {

src/frontend/SvelteViewEngine.ts

@@ -1,7 +1,8 @@
-import child_process from "child_process";
+import "../../node_modules/svelte/register.js";
+
 import config from "config";
 import crypto from "crypto";
-import fs from "fs";
+import {promises as fs} from 'fs';
 import {nanoid} from "nanoid";
 import path from "path";
 import requireFromString from "require-from-string";
@@ -10,7 +11,6 @@ import {CssResult} from "svelte/types/compiler/interfaces";
 import {sveltePreprocess} from "svelte-preprocess/dist/autoProcess.js";
 
 import {logger} from "../Logger.js";
-import {afs} from "../Utils.js";
 import FileCache from "../utils/FileCache.js";
 import ViewEngine from "./ViewEngine.js";
 
@@ -35,29 +35,23 @@ export default class SvelteViewEngine extends ViewEngine {
         backendCalls: string[],
     }> = {};
 
-    private rollup?: child_process.ChildProcess;
-
-    /**
-     * @param buildDir A temporary directory that will contain any non-final or final non-public asset.
-     * @param publicDir The output directory that should contain all final and public assets.
-     * @param devWatchedViewDir see {@link ViewEngine}.
-     * @param additionalViewPaths see {@link ViewEngine}.
-     */
     public constructor(
-        private readonly buildDir: string,
-        private readonly publicDir: string,
-        devWatchedViewDir: string,
+        targetDir: string,
         ...additionalViewPaths: string[]
     ) {
-        super(devWatchedViewDir, ...additionalViewPaths);
-
-        if (!fs.existsSync(this.buildDir)) {
-            fs.mkdirSync(this.buildDir, {recursive: true});
-        }
+        super(targetDir, 'views', 'svelte', true, ...additionalViewPaths);
     }
 
-    public getExtension(): string {
-        return 'svelte';
+    public async onFileChange(file: string): Promise<void> {
+        delete this.backendCallsCache[this.toCanonicalName(file)];
+    }
+
+    public async onFileRemove(file: string): Promise<void> {
+        const canonicalName = this.toCanonicalName(file);
+        delete this.backendCallsCache[canonicalName];
+        delete this.dependencyCache[canonicalName];
+        Object.values(this.dependencyCache).forEach(set => set.delete(canonicalName));
+        await super.onFileRemove(file);
     }
 
     public async render(
@@ -67,7 +61,7 @@ export default class SvelteViewEngine extends ViewEngine {
         const canonicalViewName = this.toCanonicalName(file);
 
         // View
-        const actualFile = path.join(this.buildDir, canonicalViewName + COMPILED_SVELTE_EXTENSION);
+        const actualFile = path.join(this.targetDir, canonicalViewName + COMPILED_SVELTE_EXTENSION);
         const view = await this.fileCache.get(actualFile, !config.get<boolean>('view.cache'));
 
         // Root template
@@ -121,14 +115,9 @@ export default class SvelteViewEngine extends ViewEngine {
         );
     }
 
-    public async stop(): Promise<void> {
-        await super.stop();
-        await this.stopRollup();
-    }
-
     public async preCompile(canonicalName: string, alsoCompileDependents: boolean): Promise<void> {
         const file = await this.resolveFileFromCanonicalName(canonicalName);
-        const intermediateFile = path.join(this.buildDir, canonicalName);
+        const intermediateFile = path.join(this.targetDir, canonicalName);
 
         logger.info(canonicalName + ' > ', 'Pre-compiling', file, '->', intermediateFile);
         const source = await this.fileCache.get(file, !config.get<boolean>('view.cache'));
@@ -152,9 +141,9 @@ export default class SvelteViewEngine extends ViewEngine {
             ssr.css.code,
         ].join(separator);
 
-        const swafViewFile = path.join(this.buildDir, canonicalName + COMPILED_SVELTE_EXTENSION);
-        await afs.mkdir(path.dirname(swafViewFile), {recursive: true});
-        await afs.writeFile(swafViewFile, finalCode);
+        const swafViewFile = path.join(this.targetDir, canonicalName + COMPILED_SVELTE_EXTENSION);
+        await fs.mkdir(path.dirname(swafViewFile), {recursive: true});
+        await fs.writeFile(swafViewFile, finalCode);
 
         if (alsoCompileDependents && Object.keys(this.dependencyCache).indexOf(canonicalName) >= 0) {
             logger.info(canonicalName + ' > ', 'Compiling dependents...');
@@ -197,8 +186,8 @@ export default class SvelteViewEngine extends ViewEngine {
         }
 
         // mkdir output file dir
-        const outputFile = path.join(this.buildDir, canonicalViewName);
-        await afs.mkdir(path.dirname(outputFile), {recursive: true});
+        const outputFile = path.join(this.targetDir, canonicalViewName);
+        await fs.mkdir(path.dirname(outputFile), {recursive: true});
 
         // Read source file if code was not already provided
         if (!code) {
@@ -212,7 +201,7 @@ export default class SvelteViewEngine extends ViewEngine {
                 replacedBackendCall: code,
                 backendCalls: [],
             };
-            await afs.writeFile(outputFile, generated.replacedBackendCall);
+            await fs.writeFile(outputFile, generated.replacedBackendCall);
             this.backendCallsCache[canonicalViewName] = generated;
             return generated;
         }
@@ -252,7 +241,7 @@ export default class SvelteViewEngine extends ViewEngine {
             if (backendCall.match(/([^()]+)\((.+?)\)/)) {
                 backendCall = backendCall.replace(/([^()]+)\((.+?)\)/, "'$1', `[$2]`");
             } else {
-                backendCall = backendCall.replace(/([^()]+)/, "'$1'");
+                backendCall = backendCall.replace(/([^()]+)(\(\))?/, "'$1'");
             }
             backendCalls.add(backendCall);
 
@@ -266,34 +255,12 @@ export default class SvelteViewEngine extends ViewEngine {
             replacedBackendCall: output,
             backendCalls: [...backendCalls],
         };
-        await afs.writeFile(outputFile, generated.replacedBackendCall);
+        await fs.writeFile(outputFile, generated.replacedBackendCall);
         this.backendCallsCache[canonicalViewName] = generated;
 
         return generated;
     }
 
-    public async afterPreCompile(watch: boolean): Promise<void> {
-        await this.bundle(watch, ...Object.keys(this.backendCallsCache));
-    }
-
-
-    public async onNewFile(): Promise<void> {
-        await this.stopRollup();
-    }
-
-    public async onFileChange(file: string): Promise<void> {
-        delete this.backendCallsCache[this.toCanonicalName(file)];
-    }
-
-    public async onFileRemove(file: string): Promise<void> {
-        const canonicalName = this.toCanonicalName(file);
-        delete this.backendCallsCache[canonicalName];
-        delete this.dependencyCache[canonicalName];
-        Object.values(this.dependencyCache).forEach(set => set.delete(canonicalName));
-        await this.stopRollup();
-        await this.afterPreCompile(true);
-    }
-
     private async compileSsr(canonicalName: string, file: string, code: string): Promise<{
         head: string,
         css: CssResult,
@@ -305,7 +272,7 @@ export default class SvelteViewEngine extends ViewEngine {
             code,
             sveltePreprocess({
                 typescript: {
-                    tsconfigFile: 'tsconfig.views.json',
+                    tsconfigFile: 'tsconfig.svelte.json',
                 },
             }),
             {
@@ -322,58 +289,13 @@ export default class SvelteViewEngine extends ViewEngine {
             cssOutputFilename: file + '.css',
         });
 
-        const globals = ViewEngine.getGlobals();
-        const localsFunction = (key: string, rawArgs?: string) => {
-            if (!rawArgs) return globals[key];
-            const args = Function(`"use strict";const locals = Object.assign(arguments[0], arguments[1]);return (${rawArgs});`)(localsFunction, globals) as string[];
+        // Load locals into locals store
+        const localsModulePath = "../../build/ts/stores.js";
+        const localsModule = await import(localsModulePath);
+        localsModule.locals.set(ViewEngine.getGlobals());
 
-            const f = globals[key];
-            if (typeof f !== 'function') throw new Error(key + ' is not a function.');
-            return f.call(globals, ...args);
-        };
-        return requireFromString(svelteSsr.js.code, file + nanoid()).default.render({
-            locals: localsFunction,
-        });
-    }
-
-    private async bundle(watch: boolean, ...canonicalViewNames: string[]): Promise<void> {
-        logger.info('Bundling...');
-
-        // Prepare output dir
-        for (const name of canonicalViewNames) {
-            await afs.mkdir(path.dirname(path.join(this.publicDir, 'js', name)), {recursive: true});
-        }
-
-        const production = !config.get<boolean>('view.dev');
-        const input = canonicalViewNames.map(name => path.join(this.buildDir, name));
-
-        if (!this.rollup) {
-            const args = [
-                'rollup',
-                '-c', 'rollup.config.js',
-                '--environment', `ENV:${production ? 'production' : 'dev'},BUILD_DIR:${this.buildDir},PUBLIC_DIR:${this.publicDir},INPUT:${input.join(':')}`,
-            ];
-            if (watch) args.push('--watch');
-            this.rollup = child_process.spawn('yarn', args, {stdio: [process.stdin, process.stdout, process.stderr]});
-            logger.info('Rollup started');
-            this.rollup.once('exit', () => {
-                logger.info('Rollup stopped');
-                this.rollup = undefined;
-            });
-        }
-    }
-
-    private async stopRollup(): Promise<void> {
-        if (this.rollup) {
-            logger.info(`Stopping rollup (${this.rollup.pid})...`);
-            await new Promise<void>((resolve, reject) => {
-                if (!this.rollup) return resolve();
-                this.rollup.once('exit', () => {
-                    resolve();
-                });
-                if (!this.rollup.kill("SIGTERM")) reject('Could not stop rollup.');
-            });
-        }
+        // Load module and render
+        return requireFromString(svelteSsr.js.code, file + nanoid() + '.js').default.render();
     }
 }
 

src/frontend/TypeScriptPreCompiler.ts

@@ -0,0 +1,18 @@
+import child_process from "child_process";
+
+import {logger} from "../Logger.js";
+import CopyAssetPreCompiler from "./CopyAssetPreCompiler.js";
+
+export default class TypeScriptPreCompiler extends CopyAssetPreCompiler {
+    public constructor(
+        targetDir: string,
+        additionalFallbackAssetPaths: string[] = [],
+    ) {
+        super(targetDir, 'ts', 'ts', additionalFallbackAssetPaths, false, 'ts-source');
+
+        this.onPreCompile(async _watch => {
+            logger.info('Building ts assets...');
+            await child_process.execSync(`yarn tsc -b tsconfig.frontend.json`, {stdio: [process.stdin, process.stdout, process.stderr]});
+        });
+    }
+}

src/frontend/ViewEngine.ts

@@ -1,12 +1,9 @@
-import chokidar, {FSWatcher} from "chokidar";
 import {Express} from "express";
-import fs from "fs";
-import path from "path";
 
 import {logger} from "../Logger.js";
-import {afs, readdirRecursively} from "../Utils.js";
+import AssetPreCompiler from "./AssetPreCompiler.js";
 
-export default abstract class ViewEngine {
+export default abstract class ViewEngine extends AssetPreCompiler {
     private static readonly globals: Record<string, unknown> = {};
 
     public static getGlobals(): Record<string, unknown> {
@@ -18,37 +15,23 @@ export default abstract class ViewEngine {
     }
 
 
-    protected readonly viewPaths: string[];
-    private watcher?: FSWatcher;
-
-    /**
-     * @param devWatchedViewDir The directory that should be watched in dev environment.
-     * @param additionalViewPaths By order of priority, the directories that contain all the views of the app.
-     * Swaf provided views and default ./view directory are automatically added (don't add them yourself).
-     * @protected
-     */
     protected constructor(
-        private readonly devWatchedViewDir: string,
+        targetDir: string,
+        assetType: string,
+        extension: string,
+        outputToPublicDir: boolean,
         ...additionalViewPaths: string[]
     ) {
-        this.viewPaths = [
-            ...additionalViewPaths,
-            'views',
-            'node_modules/swaf/views',
-        ].map(p => path.resolve(p))
-            .filter(dir => fs.existsSync(dir));
+        super(targetDir, assetType, extension, outputToPublicDir, ...additionalViewPaths);
     }
 
-    public abstract getExtension(): string;
-
-
     public abstract render(
         file: string,
         locals: Record<string, unknown>,
     ): Promise<string>;
 
     public setup(app: Express, main: boolean): void {
-        app.engine(this.getExtension(), (path, options, callback) => {
+        app.engine(this.extension, (path, options, callback) => {
             // Props (locals)
             const locals = Object.assign(options, ViewEngine.getGlobals());
 
@@ -58,124 +41,15 @@ export default abstract class ViewEngine {
         });
 
         const existingViewPaths = app.get('views');
-        app.set('views', existingViewPaths ?
+        app.set('views', Array.isArray(existingViewPaths) ?
             [...new Set([
-                ...typeof existingViewPaths === 'string' ?
-                    [existingViewPaths] :
-                    existingViewPaths,
+                ...existingViewPaths,
                 ...this.getViewPaths(),
             ])] :
             this.getViewPaths());
 
         if (main) {
-            app.set('view engine', this.getExtension());
+            app.set('view engine', this.extension);
         }
     }
-
-    public getViewPaths(): string[] {
-        return this.viewPaths;
-    }
-
-    public preCompile?(canonicalName: string, alsoCompileDependents: boolean): Promise<void>;
-
-    public afterPreCompile?(watch: boolean): Promise<void>;
-
-    public async preCompileAll(watch: boolean): Promise<void> {
-        if (this.preCompile) {
-            logger.info(`Pre-compiling ${this.getExtension()} views...`);
-
-            // List all views
-            const views: string[] = [];
-            for (const viewPath of this.viewPaths) {
-                await readdirRecursively(viewPath, async file => {
-                    if (file.endsWith('.' + this.getExtension())) {
-                        views.push(this.toCanonicalName(file));
-                    }
-                });
-            }
-
-            // Deduplicate and pre-compile
-            for (const canonicalName of [...new Set<string>(views)]) {
-                await this.preCompile(canonicalName, false);
-            }
-        }
-
-        await this.afterPreCompile?.(watch);
-
-        if (watch) {
-            await this.watch();
-        }
-    }
-
-    public onNewFile?(file: string): Promise<void>;
-
-    public onFileChange?(file: string): Promise<void>;
-
-    public onFileRemove?(file: string): Promise<void>;
-
-    public async watch(): Promise<void> {
-        this.watcher = chokidar.watch(this.devWatchedViewDir, {persistent: true});
-        this.watcher.on('ready', () => {
-            if (!this.watcher) return;
-
-            logger.info(`Watching ${this.getExtension()} assets for changes`);
-
-            this.watcher.on('add', (file) => {
-                if (file.endsWith('.' + this.getExtension())) {
-                    (this.onNewFile ? this.onNewFile(file) : Promise.resolve())
-                        .then(() => this.preCompile?.(this.toCanonicalName(file), true))
-                        .then(() => this.afterPreCompile?.(true))
-                        .catch(err => logger.error(err));
-                }
-            });
-
-            this.watcher.on('change', (file) => {
-                if (file.endsWith('.' + this.getExtension())) {
-                    (this.onFileChange ? this.onFileChange(file) : Promise.resolve())
-                        .then(() => this.preCompile?.(this.toCanonicalName(file), true))
-                        .then(() => this.afterPreCompile?.(true))
-                        .catch(err => logger.error(err));
-                }
-            });
-
-            this.watcher.on('unlink', (file) => {
-                if (file.endsWith('.' + this.getExtension())) {
-                    (this.onFileRemove ? this.onFileRemove(file) : Promise.resolve())
-                        .catch(err => logger.error(err));
-                }
-            });
-        });
-    }
-
-    public async stop(): Promise<void> {
-        if (this.watcher) {
-            await this.watcher.close();
-            this.watcher = undefined;
-        }
-    }
-
-    protected toCanonicalName(file: string): string {
-        const resolvedFilePath = path.resolve(file);
-
-        let canonicalViewName: string | null = null;
-        for (const viewPath of this.viewPaths) {
-            if (resolvedFilePath.startsWith(viewPath)) {
-                canonicalViewName = resolvedFilePath.substring(viewPath.length + 1);
-            }
-        }
-
-        if (!canonicalViewName) throw new Error('view ' + file + ' not found');
-        return canonicalViewName;
-    }
-
-    protected async resolveFileFromCanonicalName(canonicalName: string): Promise<string> {
-        for (const viewPath of this.viewPaths) {
-            const tryPath = path.join(viewPath, canonicalName);
-            if (await afs.exists(tryPath)) {
-                return tryPath;
-            }
-        }
-
-        throw new Error('View not found from canonical name ' + canonicalName);
-    }
 }

src/main.ts

@@ -7,15 +7,15 @@ process.env['NODE_CONFIG_DIR'] =
     + (process.env['NODE_CONFIG_DIR'] || path.resolve('config'));
 
 import config from "config";
+import {promises as fs} from "fs";
 
 import {logger} from "./Logger.js";
 import TestApp from "./TestApp.js";
-import {afs} from "./Utils.js";
 
 (async () => {
     logger.debug('Config path:', process.env['NODE_CONFIG_DIR']);
 
-    const packageJson = JSON.parse((await afs.readFile('package.json')).toString());
+    const packageJson = JSON.parse((await fs.readFile('package.json')).toString());
 
     const app = new TestApp(packageJson.version, config.get<string>('listen_addr'), config.get<number>('port'));
     await app.start();

src/utils/FileCache.ts

@@ -1,6 +1,4 @@
-import fs from "fs";
-
-import {afs} from "../Utils.js";
+import {promises as fs, readFileSync} from "fs";
 
 export default class FileCache {
     private readonly cache: Record<string, string> = {};
@@ -27,10 +25,10 @@ export default class FileCache {
     }
 
     public async load(file: string): Promise<void> {
-        this.cache[file] = (await afs.readFile(file)).toString();
+        this.cache[file] = (await fs.readFile(file)).toString();
     }
 
     public loadSync(file: string): void {
-        this.cache[file] = fs.readFileSync(file).toString();
+        this.cache[file] = readFileSync(file).toString();
     }
 }

test/assets/views/home_dep.svelte

@@ -0,0 +1,26 @@
+<script lang="ts">
+    import {locals} from "../ts/stores.js";
+
+    export let depTest = 'Success';
+
+    let locallyDefinedVar = 'correct value';
+</script>
+
+<style>
+    p {
+        color: brown;
+    }
+</style>
+
+
+<p>Simple dep test</p>
+<p>\$locals.</p>
+<p>\$locals.</p>
+
+<p>$locals: {$locals}</p>
+<p>\$locals.route: {$locals.route} <br> {$locals.dump(typeof $locals.route)}</p>
+<p>\$locals.dump: {$locals.dump('bonjour')}</p>
+<p>\$locals.dump: {$locals.dump($locals.app)}</p>
+<p>\$locals.dump: {$locals.dump(NaN)}</p>
+<p>\$locals.route: {$locals.route('auth')}</p>
+<p>\$locals.route: {$locals.route('home')}</p>

test/assets/views/home_test.svelte

@@ -1,9 +1,8 @@
 <script>
+    import {locals} from "../ts/stores.js";
     import HomeDep from "./home_dep.svelte";
     import Layout from "./layout.svelte";
 
-    export let locals = {};
-
     let count = 5;
 
     function handleClick() {
@@ -11,6 +10,8 @@
     }
 
     let depTest;
+
+    let mode = ($locals).name === 'locals' ? 'RENDU' : 'PRE RENDU'
 </script>
 
 <style lang="scss">
@@ -25,6 +26,8 @@
 	}
 </style>
 
+<h1 id="rendertest">{mode}</h1>
+
 <Layout title="Home">
     <svelte:fragment slot="body">
         <h1>BONJOUR lol</h1>
@@ -35,24 +38,24 @@
 
 <button on:click={handleClick}>More hellos!!</button>
 
-<p>Direct access: {locals.direct}</p>
+<p>Direct access: {$locals.direct}</p>
 
-{#if locals.route('auth') === '/'}
+{#if $locals.route('auth') === '/'}
     We're home!
 {:else}
-    We're somewhere else... {locals.route('auth')}
+    We're somewhere else... {$locals.route('auth')}
 {/if}
 
-<p>The route to auth is {locals.route('auth')}</p>
+<p>The route to auth is {$locals.route('auth')}</p>
 
-<p>\locals.notcode</p>
+<p>\$locals.notcode</p>
 
-<p>{`{\\locals.escaped}`}</p>
+<p>{`{\\$locals.escaped}`}</p>
 
 <div class="style-test">
     <p>Blue!</p>
 </div>
 
-<HomeDep locals={locals} bind:depTest={depTest}/>
+<HomeDep bind:depTest={depTest}/>
 
 <p>Dependency test: {depTest}</p>

test/views/layouts/base.njk

@@ -1,59 +0,0 @@
-{% extends 'layouts/barebone.njk' %}
-{% import 'macros.njk' as macros %}
-
-{% block _stylesheets %}
-    {{ super() }}
-    <link rel="stylesheet" href="/css/app.css">
-    {% block stylesheets %}{% endblock %}
-{% endblock %}
-{% block _scripts %}
-    {{ super() }}
-    {% block scripts %}{% endblock %}
-{% endblock %}
-
-{% block header %}
-    <a href="/" class="logo"><img src="/img/logo.svg" alt="Logo"> {{ app.name }}</a>
-    <nav>
-        <button id="menu-button"><i data-feather="menu"></i></button>
-        <ul id="main-menu">
-            {% if user %}
-                {% if user.is_admin %}
-                    <li><a href="{{ route('backend') }}"><i data-feather="settings"></i> <span class="tip">Backend</span></a></li>
-                {% endif %}
-
-                <li>
-                    <a href="{{ route('account') }}"><i data-feather="user"></i>
-                        <span class="tip">{{ user.name | default('Account') }}</span></a>
-                </li>
-
-                <li>
-                    <form action="{{ route('logout') }}" method="POST">
-                        <button><i data-feather="log-out"></i> <span class="tip">Logout</span></button>
-                        {{ macros.csrf(getCsrfToken) }}
-                    </form>
-                </li>
-            {% else %}
-                <li><a href="{{ route('auth') }}"><i data-feather="log-in"></i> Log in / Register</a></li>
-            {% endif %}
-        </ul>
-    </nav>
-{% endblock %}
-
-{% block _body %}
-    <div class="container">
-        {{ macros.messages(flash) }}
-    </div>
-
-    <main>
-        {% if h1 %}
-            <h1>{{ h1 }}</h1>
-        {% endif %}
-        {% if subtitle %}
-            <p>{{ subtitle }}</p>
-        {% endif %}
-
-        {% block body %}{% endblock %}
-    </main>
-{% endblock %}
-
-{% block footer %}{{ app.name }} v{{ app_version }} - all rights reserved.{% endblock %}

tsconfig.frontend.json

@@ -0,0 +1,18 @@
+{
+    "extends": "./tsconfig.json",
+    "compilerOptions": {
+        "target": "ESNext",
+        "module": "CommonJS",
+        "baseUrl": "build",
+        "rootDir": "build/ts-source",
+        "sourceRoot": "build/ts-source",
+        "outDir": "build/ts",
+        "declaration": false,
+
+        "typeRoots": [],
+        "resolveJsonModule": false
+    },
+    "include": [
+        "build/ts-source/**/*"
+    ]
+}

tsconfig.svelte.json

@@ -0,0 +1,10 @@
+{
+    "extends": "@tsconfig/svelte/tsconfig.json",
+    "compilerOptions": {
+        "outDir": "public/js",
+        "rootDir": "build",
+    },
+    "include": [
+        "assets/ts/**/*"
+    ],
+}

tsconfig.views.json

@@ -1,19 +0,0 @@
-{
-    "extends": "@tsconfig/svelte/tsconfig.json",
-    "compilerOptions": {
-        "outDir": "public/js",
-        "rootDir": "./build",
-        "target": "ES6",
-        "strict": true,
-        "lib": [
-            "es2020",
-            "DOM"
-        ],
-        "typeRoots": [
-            "./node_modules/@types"
-        ]
-    },
-    "include": [
-        "assets/ts/**/*"
-    ]
-}

views/home_dep.svelte

@@ -1,25 +0,0 @@
-<script lang="ts">
-    export let depTest = 'Success';
-    export let locals = {};
-
-    let locallyDefinedVar = 'correct value';
-</script>
-
-<style>
-    p {
-        color: brown;
-    }
-</style>
-
-
-<p>Simple dep test</p>
-<p>\locals.</p>
-<p>\locals.</p>
-
-<p>locals: {locals}</p>
-<p>\locals.route: {locals.route} <br> {locals.dump(typeof locals.route)}</p>
-<p>\locals.dump: {locals.dump('bonjour')}</p>
-<p>\locals.dump: {locals.dump(locals.app)}</p>
-<p>\locals.dump: {locals.dump(NaN)}</p>
-<p>\locals.route: {locals.route('auth')}</p>
-<p>\locals.route: {locals.route('home')}</p>