Add two step pre-compile/compile asset processing

Reorganize views into new "assets" folder structure
Turn locals into a store so locals don't have to be passed through files that don't need them
Some fixes to previous commit (esm) 82ab0b963c
Remove afs in favor of fs.promises (renamed afs.exists to Utils.doesFileExist
Rename Utils.readdirRecursively to Utils.listFilesRecursively
This commit is contained in:
Alice Gaudon 2021-05-04 17:04:14 +02:00
parent 82ab0b963c
commit 6aa37eb9e4
50 changed files with 516 additions and 437 deletions

3
assets/ts/stores.ts Normal file
View File

@ -0,0 +1,3 @@
import {writable} from "svelte/store";
export const locals = writable({});

View File

@ -5,9 +5,10 @@
<style>%css%</style> <style>%css%</style>
<script type="module" defer> <script type="module" defer>
import View from '/js/%canonicalViewName%.js'; import View from '/js/views/%canonicalViewName%.js';
import {locals} from '/js/stores.js';
const locals = %locals%; locals.set(%locals%);
new View({ new View({
hydrate: true, hydrate: true,

View File

@ -25,12 +25,13 @@ export default commandLineArgs => ({
chunkInfo.name; chunkInfo.name;
return name + '.js'; return name + '.js';
}, },
chunkFileNames: '[name].js',
}, },
plugins: [ plugins: [
svelte({ svelte({
preprocess: sveltePreprocess({ preprocess: sveltePreprocess({
typescript: { typescript: {
tsconfigFile: 'tsconfig.views.json', tsconfigFile: 'tsconfig.svelte.json',
}, },
}), }),
compilerOptions: { compilerOptions: {

View File

@ -22,7 +22,7 @@ function copyRecursively(file, destination) {
'yarn.lock', 'yarn.lock',
'README.md', 'README.md',
'config/', 'config/',
'views/', 'assets/',
].forEach(file => { ].forEach(file => {
copyRecursively(file, 'dist'); copyRecursively(file, 'dist');
}); });

View File

@ -29,9 +29,12 @@ import SessionComponent from "./components/SessionComponent.js";
import WebSocketServerComponent from "./components/WebSocketServerComponent.js"; import WebSocketServerComponent from "./components/WebSocketServerComponent.js";
import Controller from "./Controller.js"; import Controller from "./Controller.js";
import Migration, {MigrationType} from "./db/Migration.js"; import Migration, {MigrationType} from "./db/Migration.js";
import AssetCompiler from "./frontend/AssetCompiler.js";
import CopyAssetPreCompiler from "./frontend/CopyAssetPreCompiler.js";
import MailViewEngine from "./frontend/MailViewEngine.js"; import MailViewEngine from "./frontend/MailViewEngine.js";
import NunjucksViewEngine from "./frontend/NunjucksViewEngine.js"; import NunjucksViewEngine from "./frontend/NunjucksViewEngine.js";
import SvelteViewEngine from "./frontend/SvelteViewEngine.js"; import SvelteViewEngine from "./frontend/SvelteViewEngine.js";
import TypeScriptPreCompiler from "./frontend/TypeScriptPreCompiler.js";
import BackendController from "./helpers/BackendController.js"; import BackendController from "./helpers/BackendController.js";
import MailController from "./mail/MailController.js"; import MailController from "./mail/MailController.js";
import {MAGIC_LINK_MAIL} from "./Mails.js"; import {MAGIC_LINK_MAIL} from "./Mails.js";
@ -77,16 +80,19 @@ export default class TestApp extends Application {
this.use(new ServeStaticDirectoryComponent('public')); this.use(new ServeStaticDirectoryComponent('public'));
// Dynamic views and routes // Dynamic views and routes
const intermediateDirectory = 'build';
this.use(new FrontendToolsComponent( this.use(new FrontendToolsComponent(
'public', new AssetCompiler(intermediateDirectory, 'public'),
new SvelteViewEngine('build/views', 'public', 'views', 'test/views'), new CopyAssetPreCompiler(intermediateDirectory, '', 'json', ['test/assets'], false),
new NunjucksViewEngine('views', 'test/views'), new TypeScriptPreCompiler(intermediateDirectory, ['test/assets']),
new SvelteViewEngine(intermediateDirectory, 'test/assets'),
new NunjucksViewEngine(intermediateDirectory, 'test/assets'),
)); ));
this.use(new PreviousUrlComponent()); this.use(new PreviousUrlComponent());
// Services // Services
this.use(new MysqlComponent()); this.use(new MysqlComponent());
this.use(new MailComponent(new MailViewEngine('views', 'test/views'))); this.use(new MailComponent(new MailViewEngine('build', 'test/assets')));
// Session // Session
this.use(new RedisComponent()); this.use(new RedisComponent());

View File

@ -1,6 +1,5 @@
import fs from "fs"; import {promises as fs} from "fs";
import path from "path"; import path from "path";
import * as util from "util";
export async function sleep(ms: number): Promise<void> { export async function sleep(ms: number): Promise<void> {
return await new Promise(resolve => { return await new Promise(resolve => {
@ -49,37 +48,31 @@ export function getMethods<T extends { [p: string]: unknown }>(obj: T): string[]
return [...properties.keys()].filter(item => typeof obj[item] === 'function'); return [...properties.keys()].filter(item => typeof obj[item] === 'function');
} }
export const afs = { export async function listFilesRecursively(dir: string): Promise<string[]> {
readFile: util.promisify(fs.readFile), const localFiles = await fs.readdir(dir);
writeFile: util.promisify(fs.writeFile), const files: string[] = [];
mkdir: util.promisify(fs.mkdir), for (const file of localFiles.map(file => path.join(dir, file))) {
readdir: util.promisify(fs.readdir), const stat = await fs.stat(file);
exists: async (file: string): Promise<boolean> =>
await new Promise<boolean>((resolve, reject) =>
fs.stat(file, (err) => {
if (err == null) {
resolve(true);
} else if (err.code === 'ENOENT') {
resolve(false);
} else {
reject(err);
}
})),
};
export async function readdirRecursively(dir: string, consumer: (file: string) => Promise<void>): Promise<void> { if (stat.isDirectory()) {
const files = await afs.readdir(dir); files.push(...await listFilesRecursively(file));
} else {
await Promise.all<void>(files.map(file => new Promise<void>((resolve, reject) => { files.push(file);
file = path.join(dir, file); }
fs.stat(file, (err, stat) => { }
if (err) return reject(err); return files;
}
if (stat.isDirectory()) {
resolve(readdirRecursively(file, consumer));
} else { export async function doesFileExist(file: string): Promise<boolean> {
resolve(consumer(file)); try {
} await fs.stat(file);
}); } catch (err) {
}))); if (err.code === 'ENOENT') {
return false;
} else {
throw err;
}
}
return true;
} }

View File

@ -1,5 +1,3 @@
import "../../node_modules/svelte/register.js";
import config from "config"; import config from "config";
import {Express, Router} from "express"; import {Express, Router} from "express";
import path from "path"; import path from "path";
@ -9,40 +7,53 @@ import util from "util";
import ApplicationComponent from "../ApplicationComponent.js"; import ApplicationComponent from "../ApplicationComponent.js";
import Controller, {RouteParams} from "../Controller.js"; import Controller, {RouteParams} from "../Controller.js";
import AssetCompiler from "../frontend/AssetCompiler.js";
import AssetPreCompiler from "../frontend/AssetPreCompiler.js";
import ViewEngine from "../frontend/ViewEngine.js"; import ViewEngine from "../frontend/ViewEngine.js";
import {logger} from "../Logger.js"; import {logger} from "../Logger.js";
import {readdirRecursively} from "../Utils.js"; import {listFilesRecursively} from "../Utils.js";
import FileCache from "../utils/FileCache.js"; import FileCache from "../utils/FileCache.js";
export default class FrontendToolsComponent extends ApplicationComponent { export default class FrontendToolsComponent extends ApplicationComponent {
private readonly publicDir: string;
private readonly publicAssetsCache: FileCache = new FileCache(); private readonly publicAssetsCache: FileCache = new FileCache();
private readonly viewEngines: ViewEngine[]; private readonly assetPreCompilers: AssetPreCompiler[];
public constructor( public constructor(
private readonly publicAssetsDir: string, private readonly assetCompiler: AssetCompiler,
...viewEngines: ViewEngine[] ...assetPreCompilers: AssetPreCompiler[]
) { ) {
super(); super();
this.viewEngines = viewEngines; this.assetPreCompilers = assetPreCompilers;
this.publicDir = this.assetCompiler.targetDir;
for (const assetPreCompiler of this.assetPreCompilers) {
if (assetPreCompiler.isPublic()) {
this.assetCompiler.addExtension(assetPreCompiler.getExtension());
}
}
} }
public async start(app: Express): Promise<void> { public async start(app: Express): Promise<void> {
// Cache public assets // Cache public assets
if (config.get<boolean>('asset_cache')) { if (config.get<boolean>('asset_cache')) {
logger.info('Caching assets from', this.publicAssetsDir, '...'); logger.info('Caching assets from', this.publicDir, '...');
await readdirRecursively( for (const file of await listFilesRecursively(this.publicDir)) {
this.publicAssetsDir, await this.publicAssetsCache.load(file);
async file => await this.publicAssetsCache.load(file), }
);
} else { } else {
logger.info('Asset cache disabled.'); logger.info('Asset cache disabled.');
} }
this.hookPreCompilers();
// Setup express view engine // Setup express view engine
let main = true; let main = true;
for (const viewEngine of this.viewEngines) { for (const assetPreCompiler of this.assetPreCompilers) {
viewEngine.setup(app, main); if (assetPreCompiler instanceof ViewEngine) {
main = false; assetPreCompiler.setup(app, main);
main = false;
}
} }
// Add util globals // Add util globals
@ -50,15 +61,15 @@ export default class FrontendToolsComponent extends ApplicationComponent {
} }
public async stop(): Promise<void> { public async stop(): Promise<void> {
for (const viewEngine of this.viewEngines) { for (const assetPreCompiler of this.assetPreCompilers) {
await viewEngine.stop(); await assetPreCompiler.stop();
} }
} }
public async handle(router: Router): Promise<void> { public async handle(router: Router): Promise<void> {
router.use((req, res, next) => { router.use((req, res, next) => {
res.locals.inlineAsset = (urlPath: string) => { res.locals.inlineAsset = (urlPath: string) => {
return this.publicAssetsCache.getOrFail(path.join(this.publicAssetsDir, urlPath)); return this.publicAssetsCache.getOrFail(path.join(this.publicDir, urlPath));
}; };
next(); next();
@ -75,12 +86,6 @@ export default class FrontendToolsComponent extends ApplicationComponent {
}); });
} }
public async preCompileViews(watch: boolean): Promise<void> {
for (const viewEngine of this.viewEngines) {
await viewEngine.preCompileAll(watch);
}
}
public setupGlobals(): void { public setupGlobals(): void {
ViewEngine.setGlobal('route', ( ViewEngine.setGlobal('route', (
route: string, route: string,
@ -99,4 +104,27 @@ export default class FrontendToolsComponent extends ApplicationComponent {
return v.toString(16); return v.toString(16);
}); });
} }
public hookPreCompilers(): void {
for (const assetPreCompiler of this.assetPreCompilers) {
assetPreCompiler.onPreCompile(async watch => {
await this.assetCompiler.compile(watch);
});
assetPreCompiler.onInputChange(async restart => {
await this.assetCompiler.stopWatching(restart);
});
}
}
public async preCompileViews(watch: boolean): Promise<void> {
for (const viewEngine of this.assetPreCompilers) {
await viewEngine.preCompileAll(watch);
}
await this.assetCompiler.compile(watch);
if (watch) {
this.hookPreCompilers();
}
}
} }

View File

@ -0,0 +1,71 @@
import child_process from "child_process";
import config from "config";
import fs from "fs";
import {logger} from "../Logger.js";
import {listFilesRecursively} from "../Utils.js";
export default class AssetCompiler {
private rollup?: child_process.ChildProcess;
private extensions: string[] = [];
/**
* @param sourceDir The source assets directory.
* @param targetDir The output directory that should contain all final and public assets.
*/
public constructor(
public readonly sourceDir: string,
public readonly targetDir: string,
) {
if (!fs.existsSync(this.targetDir)) {
fs.mkdirSync(this.targetDir, {recursive: true});
}
}
public addExtension(extension: string): void {
this.extensions.push(extension);
}
public async compile(watch: boolean): Promise<void> {
// Prepare input list
const input = (await listFilesRecursively(this.sourceDir))
.filter(f => this.extensions.find(ext => f.endsWith('.' + ext)));
logger.info('Input list:', input);
const production = !config.get<boolean>('view.dev');
if (!this.rollup) {
const args = [
'rollup',
'-c', 'rollup.config.js',
'--environment', `ENV:${production ? 'production' : 'dev'},BUILD_DIR:${this.sourceDir},PUBLIC_DIR:${this.targetDir},INPUT:${input.join(':')}`,
];
if (watch) args.push('--watch');
this.rollup = child_process.spawn('yarn', args, {stdio: [process.stdin, process.stdout, process.stderr]});
logger.info('Rollup started');
this.rollup.once('exit', (code: number) => {
logger.info('Rollup stopped (' + code + ')');
this.rollup = undefined;
if (!watch && code !== 0) process.exit(code);
});
}
}
public async stopWatching(restart: boolean): Promise<void> {
if (this.rollup) {
logger.info(`Stopping rollup (${this.rollup.pid})...`);
await new Promise<void>((resolve, reject) => {
if (!this.rollup) return resolve();
this.rollup.once('exit', () => {
resolve();
});
if (!this.rollup.kill("SIGTERM")) reject('Could not stop rollup.');
});
}
if (restart) {
await this.compile(true);
}
}
}

View File

@ -0,0 +1,179 @@
import chokidar, {FSWatcher} from "chokidar";
import {existsSync, mkdirSync,promises as fs} from "fs";
import path from "path";
import {logger} from "../Logger.js";
import {doesFileExist, listFilesRecursively} from "../Utils.js";
export default abstract class AssetPreCompiler {
protected readonly assetPaths: string[];
private watcher?: FSWatcher;
private afterPreCompileHandlers: ((watch: boolean) => Promise<void>)[] = [];
private inputChangeHandler?: (restart: boolean) => Promise<void>;
/**
* @param targetDir The directory to put pre-compiled assets into.
* @param assetType This must be the assets sub-directory name of the asset type this pre-compiler will handle.
* Example: ts
* Example: views
* @param extension The asset files extension.
* @param outputToPublicDir should the assets be compiled into the publicly served directory?
* @param additionalFallbackAssetPaths By order of priority, fallback asset directories.
* Swaf provided assets and default ./assets directory are automatically added (don't add them yourself).
*/
protected constructor(
protected readonly targetDir: string,
protected readonly assetType: string,
protected readonly extension: string,
private readonly outputToPublicDir: boolean,
...additionalFallbackAssetPaths: string[]
) {
this.assetPaths = [
...additionalFallbackAssetPaths,
'assets',
'node_modules/swaf/assets',
].map(p => path.resolve(p, assetType))
.filter(dir => existsSync(dir));
this.targetDir = path.join(targetDir, assetType);
if (!existsSync(this.targetDir)) {
mkdirSync(this.targetDir, {recursive: true});
}
}
public getExtension(): string {
return this.extension;
}
public isPublic(): boolean {
return this.outputToPublicDir;
}
public async stop(): Promise<void> {
if (this.watcher) {
await this.watcher.close();
this.watcher = undefined;
}
}
public getViewPaths(): string[] {
return this.assetPaths;
}
public getPrimaryAssetPath(): string {
if (this.assetPaths.length === 0) throw new Error('No asset path was found.');
return this.assetPaths[0];
}
public abstract preCompile(canonicalName: string, alsoCompileDependents: boolean): Promise<void>;
public onPreCompile(afterPreCompileHandler: (watch: boolean) => Promise<void>): void {
this.afterPreCompileHandlers.push(afterPreCompileHandler);
}
protected async afterPreCompile(watch: boolean): Promise<void> {
await Promise.all(this.afterPreCompileHandlers.map(handler => handler(watch)));
}
public async preCompileAll(watch: boolean): Promise<void> {
logger.info(`Pre-compiling ${this.extension} views...`);
// List all views
const views: string[] = [];
for (const viewPath of this.assetPaths) {
for (const file of await listFilesRecursively(viewPath)) {
if (file.endsWith('.' + this.extension)) {
views.push(this.toCanonicalName(file));
}
}
}
// Deduplicate and pre-compile
for (const canonicalName of [...new Set<string>(views)]) {
await this.preCompile(canonicalName, false);
}
await this.afterPreCompile(watch);
if (watch) {
await this.watch();
}
}
public async watch(): Promise<void> {
this.watcher = chokidar.watch(this.getPrimaryAssetPath(), {persistent: true});
this.watcher.on('ready', () => {
if (!this.watcher) return;
logger.info(`Watching ${this.extension} assets for changes`);
this.watcher.on('add', (file) => {
if (file.endsWith('.' + this.extension)) {
this.onNewFile(file)
.then(() => this.preCompile(this.toCanonicalName(file), true))
.then(() => {
return this.afterPreCompile(true);
})
.catch(err => logger.error(err));
}
});
this.watcher.on('change', (file) => {
if (file.endsWith('.' + this.extension)) {
(this.onFileChange ? this.onFileChange(file) : Promise.resolve())
.then(() => this.preCompile(this.toCanonicalName(file), true))
.then(() => {
return this.afterPreCompile(true);
})
.catch(err => logger.error(err));
}
});
this.watcher.on('unlink', (file) => {
if (file.endsWith('.' + this.extension)) {
this.onFileRemove(file)
.catch(err => logger.error(err));
}
});
});
}
public async onNewFile(_file: string): Promise<void> {
await this.inputChangeHandler?.(false);
}
public onFileChange?(file: string): Promise<void>;
public async onFileRemove(_file: string): Promise<void> {
await this.inputChangeHandler?.(true);
}
public onInputChange(inputChangeHandler: (restart: boolean) => Promise<void>): void {
this.inputChangeHandler = inputChangeHandler;
}
protected toCanonicalName(file: string): string {
const resolvedFilePath = path.resolve(file);
let canonicalViewName: string | null = null;
for (const viewPath of this.assetPaths) {
if (resolvedFilePath.startsWith(viewPath)) {
canonicalViewName = resolvedFilePath.substring(viewPath.length + 1);
}
}
if (!canonicalViewName) throw new Error('view ' + file + ' not found');
return canonicalViewName;
}
protected async resolveFileFromCanonicalName(canonicalName: string): Promise<string> {
for (const viewPath of this.assetPaths) {
const tryPath = path.join(viewPath, canonicalName);
if (await doesFileExist(tryPath)) {
return tryPath;
}
}
throw new Error('View not found from canonical name ' + canonicalName);
}
}

View File

@ -0,0 +1,31 @@
import {promises as fs} from "fs";
import path from "path";
import {logger} from "../Logger.js";
import AssetPreCompiler from "./AssetPreCompiler.js";
export default class CopyAssetPreCompiler extends AssetPreCompiler {
protected readonly actualTargetDir: string;
public constructor(
targetDir: string,
assetType: string,
extension: string,
additionalFallbackAssetPaths: string[] = [],
outputToPublic: boolean = true,
overrideTargetAssetType?: string,
) {
super(targetDir, assetType, extension, outputToPublic, ...additionalFallbackAssetPaths);
this.actualTargetDir = overrideTargetAssetType ?
path.join(targetDir, overrideTargetAssetType) :
this.targetDir;
}
public async preCompile(canonicalName: string): Promise<void> {
const inputFile = await this.resolveFileFromCanonicalName(canonicalName);
const outputFile = path.join(this.actualTargetDir, canonicalName);
logger.info('Copying', inputFile, '->', outputFile);
await fs.mkdir(path.dirname(outputFile), {recursive: true});
await fs.copyFile(inputFile, outputFile);
}
}

View File

@ -6,8 +6,8 @@ import ViewEngine from "./ViewEngine.js";
export default class NunjucksViewEngine extends ViewEngine { export default class NunjucksViewEngine extends ViewEngine {
private readonly environment: Environment; private readonly environment: Environment;
public constructor(devWatchedViewDir: string, ...additionalViewPaths: string[]) { public constructor(targetDir: string, ...additionalViewPaths: string[]) {
super(devWatchedViewDir, ...additionalViewPaths); super(targetDir, 'views', 'njk', false, ...additionalViewPaths);
const opts = { const opts = {
autoescape: true, autoescape: true,
@ -15,12 +15,12 @@ export default class NunjucksViewEngine extends ViewEngine {
throwOnUndefined: true, throwOnUndefined: true,
}; };
this.environment = new nunjucks.Environment([ this.environment = new nunjucks.Environment([
...this.viewPaths.map(path => new nunjucks.FileSystemLoader(path, opts)), ...this.assetPaths.map(path => new nunjucks.FileSystemLoader(path, opts)),
], opts); ], opts);
} }
public getExtension(): string { public async preCompile(): Promise<void> {
return 'njk'; // Nunjucks doesn't need preCompilation or compilation
} }
public async render(file: string, locals: Record<string, unknown>): Promise<string> { public async render(file: string, locals: Record<string, unknown>): Promise<string> {

View File

@ -1,7 +1,8 @@
import child_process from "child_process"; import "../../node_modules/svelte/register.js";
import config from "config"; import config from "config";
import crypto from "crypto"; import crypto from "crypto";
import fs from "fs"; import {promises as fs} from 'fs';
import {nanoid} from "nanoid"; import {nanoid} from "nanoid";
import path from "path"; import path from "path";
import requireFromString from "require-from-string"; import requireFromString from "require-from-string";
@ -10,7 +11,6 @@ import {CssResult} from "svelte/types/compiler/interfaces";
import {sveltePreprocess} from "svelte-preprocess/dist/autoProcess.js"; import {sveltePreprocess} from "svelte-preprocess/dist/autoProcess.js";
import {logger} from "../Logger.js"; import {logger} from "../Logger.js";
import {afs} from "../Utils.js";
import FileCache from "../utils/FileCache.js"; import FileCache from "../utils/FileCache.js";
import ViewEngine from "./ViewEngine.js"; import ViewEngine from "./ViewEngine.js";
@ -35,29 +35,23 @@ export default class SvelteViewEngine extends ViewEngine {
backendCalls: string[], backendCalls: string[],
}> = {}; }> = {};
private rollup?: child_process.ChildProcess;
/**
* @param buildDir A temporary directory that will contain any non-final or final non-public asset.
* @param publicDir The output directory that should contain all final and public assets.
* @param devWatchedViewDir see {@link ViewEngine}.
* @param additionalViewPaths see {@link ViewEngine}.
*/
public constructor( public constructor(
private readonly buildDir: string, targetDir: string,
private readonly publicDir: string,
devWatchedViewDir: string,
...additionalViewPaths: string[] ...additionalViewPaths: string[]
) { ) {
super(devWatchedViewDir, ...additionalViewPaths); super(targetDir, 'views', 'svelte', true, ...additionalViewPaths);
if (!fs.existsSync(this.buildDir)) {
fs.mkdirSync(this.buildDir, {recursive: true});
}
} }
public getExtension(): string { public async onFileChange(file: string): Promise<void> {
return 'svelte'; delete this.backendCallsCache[this.toCanonicalName(file)];
}
public async onFileRemove(file: string): Promise<void> {
const canonicalName = this.toCanonicalName(file);
delete this.backendCallsCache[canonicalName];
delete this.dependencyCache[canonicalName];
Object.values(this.dependencyCache).forEach(set => set.delete(canonicalName));
await super.onFileRemove(file);
} }
public async render( public async render(
@ -67,7 +61,7 @@ export default class SvelteViewEngine extends ViewEngine {
const canonicalViewName = this.toCanonicalName(file); const canonicalViewName = this.toCanonicalName(file);
// View // View
const actualFile = path.join(this.buildDir, canonicalViewName + COMPILED_SVELTE_EXTENSION); const actualFile = path.join(this.targetDir, canonicalViewName + COMPILED_SVELTE_EXTENSION);
const view = await this.fileCache.get(actualFile, !config.get<boolean>('view.cache')); const view = await this.fileCache.get(actualFile, !config.get<boolean>('view.cache'));
// Root template // Root template
@ -121,14 +115,9 @@ export default class SvelteViewEngine extends ViewEngine {
); );
} }
public async stop(): Promise<void> {
await super.stop();
await this.stopRollup();
}
public async preCompile(canonicalName: string, alsoCompileDependents: boolean): Promise<void> { public async preCompile(canonicalName: string, alsoCompileDependents: boolean): Promise<void> {
const file = await this.resolveFileFromCanonicalName(canonicalName); const file = await this.resolveFileFromCanonicalName(canonicalName);
const intermediateFile = path.join(this.buildDir, canonicalName); const intermediateFile = path.join(this.targetDir, canonicalName);
logger.info(canonicalName + ' > ', 'Pre-compiling', file, '->', intermediateFile); logger.info(canonicalName + ' > ', 'Pre-compiling', file, '->', intermediateFile);
const source = await this.fileCache.get(file, !config.get<boolean>('view.cache')); const source = await this.fileCache.get(file, !config.get<boolean>('view.cache'));
@ -152,9 +141,9 @@ export default class SvelteViewEngine extends ViewEngine {
ssr.css.code, ssr.css.code,
].join(separator); ].join(separator);
const swafViewFile = path.join(this.buildDir, canonicalName + COMPILED_SVELTE_EXTENSION); const swafViewFile = path.join(this.targetDir, canonicalName + COMPILED_SVELTE_EXTENSION);
await afs.mkdir(path.dirname(swafViewFile), {recursive: true}); await fs.mkdir(path.dirname(swafViewFile), {recursive: true});
await afs.writeFile(swafViewFile, finalCode); await fs.writeFile(swafViewFile, finalCode);
if (alsoCompileDependents && Object.keys(this.dependencyCache).indexOf(canonicalName) >= 0) { if (alsoCompileDependents && Object.keys(this.dependencyCache).indexOf(canonicalName) >= 0) {
logger.info(canonicalName + ' > ', 'Compiling dependents...'); logger.info(canonicalName + ' > ', 'Compiling dependents...');
@ -197,8 +186,8 @@ export default class SvelteViewEngine extends ViewEngine {
} }
// mkdir output file dir // mkdir output file dir
const outputFile = path.join(this.buildDir, canonicalViewName); const outputFile = path.join(this.targetDir, canonicalViewName);
await afs.mkdir(path.dirname(outputFile), {recursive: true}); await fs.mkdir(path.dirname(outputFile), {recursive: true});
// Read source file if code was not already provided // Read source file if code was not already provided
if (!code) { if (!code) {
@ -212,7 +201,7 @@ export default class SvelteViewEngine extends ViewEngine {
replacedBackendCall: code, replacedBackendCall: code,
backendCalls: [], backendCalls: [],
}; };
await afs.writeFile(outputFile, generated.replacedBackendCall); await fs.writeFile(outputFile, generated.replacedBackendCall);
this.backendCallsCache[canonicalViewName] = generated; this.backendCallsCache[canonicalViewName] = generated;
return generated; return generated;
} }
@ -252,7 +241,7 @@ export default class SvelteViewEngine extends ViewEngine {
if (backendCall.match(/([^()]+)\((.+?)\)/)) { if (backendCall.match(/([^()]+)\((.+?)\)/)) {
backendCall = backendCall.replace(/([^()]+)\((.+?)\)/, "'$1', `[$2]`"); backendCall = backendCall.replace(/([^()]+)\((.+?)\)/, "'$1', `[$2]`");
} else { } else {
backendCall = backendCall.replace(/([^()]+)/, "'$1'"); backendCall = backendCall.replace(/([^()]+)(\(\))?/, "'$1'");
} }
backendCalls.add(backendCall); backendCalls.add(backendCall);
@ -266,34 +255,12 @@ export default class SvelteViewEngine extends ViewEngine {
replacedBackendCall: output, replacedBackendCall: output,
backendCalls: [...backendCalls], backendCalls: [...backendCalls],
}; };
await afs.writeFile(outputFile, generated.replacedBackendCall); await fs.writeFile(outputFile, generated.replacedBackendCall);
this.backendCallsCache[canonicalViewName] = generated; this.backendCallsCache[canonicalViewName] = generated;
return generated; return generated;
} }
public async afterPreCompile(watch: boolean): Promise<void> {
await this.bundle(watch, ...Object.keys(this.backendCallsCache));
}
public async onNewFile(): Promise<void> {
await this.stopRollup();
}
public async onFileChange(file: string): Promise<void> {
delete this.backendCallsCache[this.toCanonicalName(file)];
}
public async onFileRemove(file: string): Promise<void> {
const canonicalName = this.toCanonicalName(file);
delete this.backendCallsCache[canonicalName];
delete this.dependencyCache[canonicalName];
Object.values(this.dependencyCache).forEach(set => set.delete(canonicalName));
await this.stopRollup();
await this.afterPreCompile(true);
}
private async compileSsr(canonicalName: string, file: string, code: string): Promise<{ private async compileSsr(canonicalName: string, file: string, code: string): Promise<{
head: string, head: string,
css: CssResult, css: CssResult,
@ -305,7 +272,7 @@ export default class SvelteViewEngine extends ViewEngine {
code, code,
sveltePreprocess({ sveltePreprocess({
typescript: { typescript: {
tsconfigFile: 'tsconfig.views.json', tsconfigFile: 'tsconfig.svelte.json',
}, },
}), }),
{ {
@ -322,58 +289,13 @@ export default class SvelteViewEngine extends ViewEngine {
cssOutputFilename: file + '.css', cssOutputFilename: file + '.css',
}); });
const globals = ViewEngine.getGlobals(); // Load locals into locals store
const localsFunction = (key: string, rawArgs?: string) => { const localsModulePath = "../../build/ts/stores.js";
if (!rawArgs) return globals[key]; const localsModule = await import(localsModulePath);
const args = Function(`"use strict";const locals = Object.assign(arguments[0], arguments[1]);return (${rawArgs});`)(localsFunction, globals) as string[]; localsModule.locals.set(ViewEngine.getGlobals());
const f = globals[key]; // Load module and render
if (typeof f !== 'function') throw new Error(key + ' is not a function.'); return requireFromString(svelteSsr.js.code, file + nanoid() + '.js').default.render();
return f.call(globals, ...args);
};
return requireFromString(svelteSsr.js.code, file + nanoid()).default.render({
locals: localsFunction,
});
}
private async bundle(watch: boolean, ...canonicalViewNames: string[]): Promise<void> {
logger.info('Bundling...');
// Prepare output dir
for (const name of canonicalViewNames) {
await afs.mkdir(path.dirname(path.join(this.publicDir, 'js', name)), {recursive: true});
}
const production = !config.get<boolean>('view.dev');
const input = canonicalViewNames.map(name => path.join(this.buildDir, name));
if (!this.rollup) {
const args = [
'rollup',
'-c', 'rollup.config.js',
'--environment', `ENV:${production ? 'production' : 'dev'},BUILD_DIR:${this.buildDir},PUBLIC_DIR:${this.publicDir},INPUT:${input.join(':')}`,
];
if (watch) args.push('--watch');
this.rollup = child_process.spawn('yarn', args, {stdio: [process.stdin, process.stdout, process.stderr]});
logger.info('Rollup started');
this.rollup.once('exit', () => {
logger.info('Rollup stopped');
this.rollup = undefined;
});
}
}
private async stopRollup(): Promise<void> {
if (this.rollup) {
logger.info(`Stopping rollup (${this.rollup.pid})...`);
await new Promise<void>((resolve, reject) => {
if (!this.rollup) return resolve();
this.rollup.once('exit', () => {
resolve();
});
if (!this.rollup.kill("SIGTERM")) reject('Could not stop rollup.');
});
}
} }
} }

View File

@ -0,0 +1,18 @@
import child_process from "child_process";
import {logger} from "../Logger.js";
import CopyAssetPreCompiler from "./CopyAssetPreCompiler.js";
export default class TypeScriptPreCompiler extends CopyAssetPreCompiler {
public constructor(
targetDir: string,
additionalFallbackAssetPaths: string[] = [],
) {
super(targetDir, 'ts', 'ts', additionalFallbackAssetPaths, false, 'ts-source');
this.onPreCompile(async _watch => {
logger.info('Building ts assets...');
await child_process.execSync(`yarn tsc -b tsconfig.frontend.json`, {stdio: [process.stdin, process.stdout, process.stderr]});
});
}
}

View File

@ -1,12 +1,9 @@
import chokidar, {FSWatcher} from "chokidar";
import {Express} from "express"; import {Express} from "express";
import fs from "fs";
import path from "path";
import {logger} from "../Logger.js"; import {logger} from "../Logger.js";
import {afs, readdirRecursively} from "../Utils.js"; import AssetPreCompiler from "./AssetPreCompiler.js";
export default abstract class ViewEngine { export default abstract class ViewEngine extends AssetPreCompiler {
private static readonly globals: Record<string, unknown> = {}; private static readonly globals: Record<string, unknown> = {};
public static getGlobals(): Record<string, unknown> { public static getGlobals(): Record<string, unknown> {
@ -18,37 +15,23 @@ export default abstract class ViewEngine {
} }
protected readonly viewPaths: string[];
private watcher?: FSWatcher;
/**
* @param devWatchedViewDir The directory that should be watched in dev environment.
* @param additionalViewPaths By order of priority, the directories that contain all the views of the app.
* Swaf provided views and default ./view directory are automatically added (don't add them yourself).
* @protected
*/
protected constructor( protected constructor(
private readonly devWatchedViewDir: string, targetDir: string,
assetType: string,
extension: string,
outputToPublicDir: boolean,
...additionalViewPaths: string[] ...additionalViewPaths: string[]
) { ) {
this.viewPaths = [ super(targetDir, assetType, extension, outputToPublicDir, ...additionalViewPaths);
...additionalViewPaths,
'views',
'node_modules/swaf/views',
].map(p => path.resolve(p))
.filter(dir => fs.existsSync(dir));
} }
public abstract getExtension(): string;
public abstract render( public abstract render(
file: string, file: string,
locals: Record<string, unknown>, locals: Record<string, unknown>,
): Promise<string>; ): Promise<string>;
public setup(app: Express, main: boolean): void { public setup(app: Express, main: boolean): void {
app.engine(this.getExtension(), (path, options, callback) => { app.engine(this.extension, (path, options, callback) => {
// Props (locals) // Props (locals)
const locals = Object.assign(options, ViewEngine.getGlobals()); const locals = Object.assign(options, ViewEngine.getGlobals());
@ -58,124 +41,15 @@ export default abstract class ViewEngine {
}); });
const existingViewPaths = app.get('views'); const existingViewPaths = app.get('views');
app.set('views', existingViewPaths ? app.set('views', Array.isArray(existingViewPaths) ?
[...new Set([ [...new Set([
...typeof existingViewPaths === 'string' ? ...existingViewPaths,
[existingViewPaths] :
existingViewPaths,
...this.getViewPaths(), ...this.getViewPaths(),
])] : ])] :
this.getViewPaths()); this.getViewPaths());
if (main) { if (main) {
app.set('view engine', this.getExtension()); app.set('view engine', this.extension);
} }
} }
public getViewPaths(): string[] {
return this.viewPaths;
}
public preCompile?(canonicalName: string, alsoCompileDependents: boolean): Promise<void>;
public afterPreCompile?(watch: boolean): Promise<void>;
public async preCompileAll(watch: boolean): Promise<void> {
if (this.preCompile) {
logger.info(`Pre-compiling ${this.getExtension()} views...`);
// List all views
const views: string[] = [];
for (const viewPath of this.viewPaths) {
await readdirRecursively(viewPath, async file => {
if (file.endsWith('.' + this.getExtension())) {
views.push(this.toCanonicalName(file));
}
});
}
// Deduplicate and pre-compile
for (const canonicalName of [...new Set<string>(views)]) {
await this.preCompile(canonicalName, false);
}
}
await this.afterPreCompile?.(watch);
if (watch) {
await this.watch();
}
}
public onNewFile?(file: string): Promise<void>;
public onFileChange?(file: string): Promise<void>;
public onFileRemove?(file: string): Promise<void>;
public async watch(): Promise<void> {
this.watcher = chokidar.watch(this.devWatchedViewDir, {persistent: true});
this.watcher.on('ready', () => {
if (!this.watcher) return;
logger.info(`Watching ${this.getExtension()} assets for changes`);
this.watcher.on('add', (file) => {
if (file.endsWith('.' + this.getExtension())) {
(this.onNewFile ? this.onNewFile(file) : Promise.resolve())
.then(() => this.preCompile?.(this.toCanonicalName(file), true))
.then(() => this.afterPreCompile?.(true))
.catch(err => logger.error(err));
}
});
this.watcher.on('change', (file) => {
if (file.endsWith('.' + this.getExtension())) {
(this.onFileChange ? this.onFileChange(file) : Promise.resolve())
.then(() => this.preCompile?.(this.toCanonicalName(file), true))
.then(() => this.afterPreCompile?.(true))
.catch(err => logger.error(err));
}
});
this.watcher.on('unlink', (file) => {
if (file.endsWith('.' + this.getExtension())) {
(this.onFileRemove ? this.onFileRemove(file) : Promise.resolve())
.catch(err => logger.error(err));
}
});
});
}
public async stop(): Promise<void> {
if (this.watcher) {
await this.watcher.close();
this.watcher = undefined;
}
}
protected toCanonicalName(file: string): string {
const resolvedFilePath = path.resolve(file);
let canonicalViewName: string | null = null;
for (const viewPath of this.viewPaths) {
if (resolvedFilePath.startsWith(viewPath)) {
canonicalViewName = resolvedFilePath.substring(viewPath.length + 1);
}
}
if (!canonicalViewName) throw new Error('view ' + file + ' not found');
return canonicalViewName;
}
protected async resolveFileFromCanonicalName(canonicalName: string): Promise<string> {
for (const viewPath of this.viewPaths) {
const tryPath = path.join(viewPath, canonicalName);
if (await afs.exists(tryPath)) {
return tryPath;
}
}
throw new Error('View not found from canonical name ' + canonicalName);
}
} }

View File

@ -7,15 +7,15 @@ process.env['NODE_CONFIG_DIR'] =
+ (process.env['NODE_CONFIG_DIR'] || path.resolve('config')); + (process.env['NODE_CONFIG_DIR'] || path.resolve('config'));
import config from "config"; import config from "config";
import {promises as fs} from "fs";
import {logger} from "./Logger.js"; import {logger} from "./Logger.js";
import TestApp from "./TestApp.js"; import TestApp from "./TestApp.js";
import {afs} from "./Utils.js";
(async () => { (async () => {
logger.debug('Config path:', process.env['NODE_CONFIG_DIR']); logger.debug('Config path:', process.env['NODE_CONFIG_DIR']);
const packageJson = JSON.parse((await afs.readFile('package.json')).toString()); const packageJson = JSON.parse((await fs.readFile('package.json')).toString());
const app = new TestApp(packageJson.version, config.get<string>('listen_addr'), config.get<number>('port')); const app = new TestApp(packageJson.version, config.get<string>('listen_addr'), config.get<number>('port'));
await app.start(); await app.start();

View File

@ -1,6 +1,4 @@
import fs from "fs"; import {promises as fs, readFileSync} from "fs";
import {afs} from "../Utils.js";
export default class FileCache { export default class FileCache {
private readonly cache: Record<string, string> = {}; private readonly cache: Record<string, string> = {};
@ -27,10 +25,10 @@ export default class FileCache {
} }
public async load(file: string): Promise<void> { public async load(file: string): Promise<void> {
this.cache[file] = (await afs.readFile(file)).toString(); this.cache[file] = (await fs.readFile(file)).toString();
} }
public loadSync(file: string): void { public loadSync(file: string): void {
this.cache[file] = fs.readFileSync(file).toString(); this.cache[file] = readFileSync(file).toString();
} }
} }

View File

@ -0,0 +1,26 @@
<script lang="ts">
import {locals} from "../ts/stores.js";
export let depTest = 'Success';
let locallyDefinedVar = 'correct value';
</script>
<style>
p {
color: brown;
}
</style>
<p>Simple dep test</p>
<p>\$locals.</p>
<p>\$locals.</p>
<p>$locals: {$locals}</p>
<p>\$locals.route: {$locals.route} <br> {$locals.dump(typeof $locals.route)}</p>
<p>\$locals.dump: {$locals.dump('bonjour')}</p>
<p>\$locals.dump: {$locals.dump($locals.app)}</p>
<p>\$locals.dump: {$locals.dump(NaN)}</p>
<p>\$locals.route: {$locals.route('auth')}</p>
<p>\$locals.route: {$locals.route('home')}</p>

View File

@ -1,9 +1,8 @@
<script> <script>
import {locals} from "../ts/stores.js";
import HomeDep from "./home_dep.svelte"; import HomeDep from "./home_dep.svelte";
import Layout from "./layout.svelte"; import Layout from "./layout.svelte";
export let locals = {};
let count = 5; let count = 5;
function handleClick() { function handleClick() {
@ -11,6 +10,8 @@
} }
let depTest; let depTest;
let mode = ($locals).name === 'locals' ? 'RENDU' : 'PRE RENDU'
</script> </script>
<style lang="scss"> <style lang="scss">
@ -25,6 +26,8 @@
} }
</style> </style>
<h1 id="rendertest">{mode}</h1>
<Layout title="Home"> <Layout title="Home">
<svelte:fragment slot="body"> <svelte:fragment slot="body">
<h1>BONJOUR lol</h1> <h1>BONJOUR lol</h1>
@ -35,24 +38,24 @@
<button on:click={handleClick}>More hellos!!</button> <button on:click={handleClick}>More hellos!!</button>
<p>Direct access: {locals.direct}</p> <p>Direct access: {$locals.direct}</p>
{#if locals.route('auth') === '/'} {#if $locals.route('auth') === '/'}
We're home! We're home!
{:else} {:else}
We're somewhere else... {locals.route('auth')} We're somewhere else... {$locals.route('auth')}
{/if} {/if}
<p>The route to auth is {locals.route('auth')}</p> <p>The route to auth is {$locals.route('auth')}</p>
<p>\locals.notcode</p> <p>\$locals.notcode</p>
<p>{`{\\locals.escaped}`}</p> <p>{`{\\$locals.escaped}`}</p>
<div class="style-test"> <div class="style-test">
<p>Blue!</p> <p>Blue!</p>
</div> </div>
<HomeDep locals={locals} bind:depTest={depTest}/> <HomeDep bind:depTest={depTest}/>
<p>Dependency test: {depTest}</p> <p>Dependency test: {depTest}</p>

View File

@ -1,59 +0,0 @@
{% extends 'layouts/barebone.njk' %}
{% import 'macros.njk' as macros %}
{% block _stylesheets %}
{{ super() }}
<link rel="stylesheet" href="/css/app.css">
{% block stylesheets %}{% endblock %}
{% endblock %}
{% block _scripts %}
{{ super() }}
{% block scripts %}{% endblock %}
{% endblock %}
{% block header %}
<a href="/" class="logo"><img src="/img/logo.svg" alt="Logo"> {{ app.name }}</a>
<nav>
<button id="menu-button"><i data-feather="menu"></i></button>
<ul id="main-menu">
{% if user %}
{% if user.is_admin %}
<li><a href="{{ route('backend') }}"><i data-feather="settings"></i> <span class="tip">Backend</span></a></li>
{% endif %}
<li>
<a href="{{ route('account') }}"><i data-feather="user"></i>
<span class="tip">{{ user.name | default('Account') }}</span></a>
</li>
<li>
<form action="{{ route('logout') }}" method="POST">
<button><i data-feather="log-out"></i> <span class="tip">Logout</span></button>
{{ macros.csrf(getCsrfToken) }}
</form>
</li>
{% else %}
<li><a href="{{ route('auth') }}"><i data-feather="log-in"></i> Log in / Register</a></li>
{% endif %}
</ul>
</nav>
{% endblock %}
{% block _body %}
<div class="container">
{{ macros.messages(flash) }}
</div>
<main>
{% if h1 %}
<h1>{{ h1 }}</h1>
{% endif %}
{% if subtitle %}
<p>{{ subtitle }}</p>
{% endif %}
{% block body %}{% endblock %}
</main>
{% endblock %}
{% block footer %}{{ app.name }} v{{ app_version }} - all rights reserved.{% endblock %}

18
tsconfig.frontend.json Normal file
View File

@ -0,0 +1,18 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"target": "ESNext",
"module": "CommonJS",
"baseUrl": "build",
"rootDir": "build/ts-source",
"sourceRoot": "build/ts-source",
"outDir": "build/ts",
"declaration": false,
"typeRoots": [],
"resolveJsonModule": false
},
"include": [
"build/ts-source/**/*"
]
}

10
tsconfig.svelte.json Normal file
View File

@ -0,0 +1,10 @@
{
"extends": "@tsconfig/svelte/tsconfig.json",
"compilerOptions": {
"outDir": "public/js",
"rootDir": "build",
},
"include": [
"assets/ts/**/*"
],
}

View File

@ -1,19 +0,0 @@
{
"extends": "@tsconfig/svelte/tsconfig.json",
"compilerOptions": {
"outDir": "public/js",
"rootDir": "./build",
"target": "ES6",
"strict": true,
"lib": [
"es2020",
"DOM"
],
"typeRoots": [
"./node_modules/@types"
]
},
"include": [
"assets/ts/**/*"
]
}

View File

@ -1,25 +0,0 @@
<script lang="ts">
export let depTest = 'Success';
export let locals = {};
let locallyDefinedVar = 'correct value';
</script>
<style>
p {
color: brown;
}
</style>
<p>Simple dep test</p>
<p>\locals.</p>
<p>\locals.</p>
<p>locals: {locals}</p>
<p>\locals.route: {locals.route} <br> {locals.dump(typeof locals.route)}</p>
<p>\locals.dump: {locals.dump('bonjour')}</p>
<p>\locals.dump: {locals.dump(locals.app)}</p>
<p>\locals.dump: {locals.dump(NaN)}</p>
<p>\locals.route: {locals.route('auth')}</p>
<p>\locals.route: {locals.route('home')}</p>