swaf/src/components/FrontendToolsComponent.ts

623 lines
23 KiB
TypeScript
Raw Normal View History

2021-03-24 16:07:50 +01:00
import {Express, Router} from "express";
2021-03-18 16:22:25 +01:00
import * as fs from "fs";
import path from "path";
import config from "config";
import ApplicationComponent from "../ApplicationComponent";
import {logger} from "../Logger";
import {ServerError} from "../HttpError";
import * as crypto from "crypto";
import requireFromString from "require-from-string";
2021-03-24 16:07:50 +01:00
import "svelte/register";
import {compile, preprocess} from "svelte/compiler";
2021-03-18 16:22:25 +01:00
import {sveltePreprocess} from "svelte-preprocess/dist/autoProcess";
import chokidar from "chokidar";
import {CssResult} from "svelte/types/compiler/interfaces";
2021-03-24 16:07:50 +01:00
import {PreRenderedChunk, rollup, RollupBuild, RollupOptions, RollupWatcher, RollupWatchOptions, watch} from "rollup";
import svelte from "rollup-plugin-svelte";
import resolve from "@rollup/plugin-node-resolve";
import commonjs from "@rollup/plugin-commonjs";
import {terser} from "rollup-plugin-terser";
import cssOnlyRollupPlugin from "rollup-plugin-css-only";
import livereloadRollupPlugin from "rollup-plugin-livereload";
2021-03-18 16:22:25 +01:00
const BACKEND_CODE_PREFIX = 'swaf.';
const COMPILED_SVELTE_EXTENSION = '.swafview';
export default class FrontendToolsComponent extends ApplicationComponent {
2021-03-24 16:07:50 +01:00
public static getSveltePreCompileSeparator(canonicalViewName: string): string {
return '\n---' +
crypto.createHash('sha1')
.update(path.basename(path.resolve(canonicalViewName)))
.digest('base64') +
'---\n';
2021-03-18 16:22:25 +01:00
}
private content: Map<string, string> = new Map();
2021-03-24 16:07:50 +01:00
private readonly viewPaths: string[];
private readonly svelteDevViewsPath: string;
private readonly dependencyCache: Record<string, Set<string>> = {};
private readonly backendCodeCache: Record<string, {
backendReplacedCode: string,
backendLines: string[],
}> = {};
private readonly fileCache: Record<string, string> = {};
private rollup?: RollupBuild | RollupWatcher;
2021-03-18 16:22:25 +01:00
public constructor(
private readonly publicAssetsPath: string,
private readonly svelteOutputPath: string,
2021-03-24 16:07:50 +01:00
...viewPaths: string[]
2021-03-18 16:22:25 +01:00
) {
super();
2021-03-24 16:07:50 +01:00
this.viewPaths = [
...viewPaths.map(p => path.resolve(p)),
path.resolve(__dirname, '../../../views'),
path.resolve(__dirname, '../../views'),
path.resolve(__dirname, '../views'),
];
this.svelteDevViewsPath = path.resolve('views');
2021-03-18 16:22:25 +01:00
if (!fs.existsSync(svelteOutputPath)) {
fs.mkdirSync(svelteOutputPath);
}
}
2021-03-24 16:07:50 +01:00
public async start(app: Express): Promise<void> {
2021-03-18 16:22:25 +01:00
await this.cachePublicAssets();
await this.preCompileSvelteViews();
2021-03-24 16:07:50 +01:00
const watcher = chokidar.watch(this.svelteDevViewsPath, {persistent: true});
2021-03-18 16:22:25 +01:00
watcher.on('ready', () => {
logger.debug('Watching svelte assets for changes');
watcher.on('add', (path) => {
2021-03-24 16:07:50 +01:00
if (path.endsWith('.svelte')) {
this.resetBundle()
.then(() => this.preCompileSvelte(path, true))
.then(() => this.bundle(...Object.keys(this.backendCodeCache)))
.catch(err => logger.error(err));
}
2021-03-18 16:22:25 +01:00
});
watcher.on('change', (path) => {
2021-03-24 16:07:50 +01:00
if (path.endsWith('.svelte')) {
delete this.backendCodeCache[this.toCanonicalViewName(path)];
this.preCompileSvelte(path, true)
.then(() => this.bundle(...Object.keys(this.backendCodeCache)))
.catch(err => logger.error(err));
}
2021-03-18 16:22:25 +01:00
});
});
2021-03-24 16:07:50 +01:00
app.engine('svelte', (path, options, callback) => {
this.renderSvelte(path, options as Record<string, unknown>, callback)
.catch(err => callback(err));
});
app.set('views', this.viewPaths);
app.set('view engine', 'svelte');
}
public async stop(): Promise<void> {
await this.resetBundle();
2021-03-18 16:22:25 +01:00
}
private async cachePublicAssets(): Promise<void> {
if (config.get<boolean>('view.enable_asset_cache')) {
logger.info('Caching assets from', this.publicAssetsPath, '...');
await this.forEachFileInDirRecursively(this.publicAssetsPath, file => this.cacheFile(file));
} else {
logger.info('Asset cache disabled.');
}
}
private async preCompileSvelteViews(): Promise<void> {
logger.info('Pre-compiling svelte views...');
2021-03-24 16:07:50 +01:00
await this.forEachFileInDirRecursively(this.svelteDevViewsPath, async file => {
2021-03-18 16:22:25 +01:00
if (file.endsWith('.svelte')) {
await this.preCompileSvelte(file);
}
});
2021-03-24 16:07:50 +01:00
await this.bundle(...Object.keys(this.backendCodeCache));
2021-03-18 16:22:25 +01:00
}
public async handle(router: Router): Promise<void> {
router.use((req, res, next) => {
res.locals.inlineAsset = (urlPath: string) => {
if (!config.get<boolean>('view.enable_asset_cache')) {
return fs.readFileSync(path.resolve(this.publicAssetsPath + urlPath));
}
const content = this.content.get(urlPath);
if (!content) {
throw new ServerError('Asset ' + path + ' was not loaded.');
}
return content;
};
next();
});
}
private async forEachFileInDirRecursively(dir: string, consumer: (file: string) => Promise<void>): Promise<void> {
await new Promise<void[]>((resolve, reject) => {
fs.readdir(dir, (err, files) => {
if (err) return reject(err);
resolve(Promise.all<void>(files.map(file => new Promise<void>((resolve, reject) => {
file = path.join(dir, file);
fs.stat(file, (err, stat) => {
if (err) return reject(err);
if (stat.isDirectory()) {
resolve(this.forEachFileInDirRecursively(file, consumer));
} else {
resolve(consumer(file));
}
});
}))));
});
});
}
private async cacheFile(file: string): Promise<void> {
await new Promise<void>((resolve, reject) => {
fs.readFile(file, (err, data) => {
if (err) return reject(data);
const urlPath = file.replace(this.publicAssetsPath, '');
this.content.set(urlPath, data.toString());
logger.debug('Loaded', file, 'as', urlPath);
resolve();
});
});
}
2021-03-24 16:07:50 +01:00
private async preCompileSvelte(file: string, alsoCompileDependents: boolean = false): Promise<void> {
file = path.relative('.', file);
const canonicalViewName = this.toCanonicalViewName(file);
const intermediateFile = path.join(this.svelteOutputPath, canonicalViewName);
logger.debug(canonicalViewName + ' > ', 'Pre-compiling', file, '->', intermediateFile);
2021-03-18 16:22:25 +01:00
const source = await new Promise<string>((resolve, reject) => {
fs.readFile(file, (err, data) => {
if (err) return reject(err);
resolve(data.toString());
});
});
2021-03-24 16:07:50 +01:00
const allBackendLines: string[] = [];
for (const dependency of this.resolveDependencies(source, canonicalViewName)) {
allBackendLines.push(...(await this.replaceBackendCode(dependency)).backendLines);
}
2021-03-18 16:22:25 +01:00
2021-03-24 16:07:50 +01:00
const {backendReplacedCode, backendLines} = await this.replaceBackendCode(canonicalViewName, source);
allBackendLines.push(...backendLines);
2021-03-18 16:22:25 +01:00
2021-03-24 16:07:50 +01:00
const preprocessedCode = await this.preProcessSvelte(backendReplacedCode, intermediateFile, canonicalViewName);
2021-03-18 16:22:25 +01:00
2021-03-24 16:07:50 +01:00
// Server Side Render (initial HTML, no-js)
const ssr = this.compileSvelteSsr(preprocessedCode, intermediateFile, canonicalViewName);
2021-03-18 16:22:25 +01:00
2021-03-24 16:07:50 +01:00
const separator = FrontendToolsComponent.getSveltePreCompileSeparator(canonicalViewName);
2021-03-18 16:22:25 +01:00
const finalCode = [
2021-03-24 16:07:50 +01:00
[...new Set<string>(allBackendLines).values()].join('\n'),
2021-03-18 16:22:25 +01:00
ssr.head,
ssr.html,
ssr.css.code,
].join(separator);
2021-03-24 16:07:50 +01:00
const swafViewFile = path.join(this.svelteOutputPath, canonicalViewName + COMPILED_SVELTE_EXTENSION);
await new Promise<void>((resolve, reject) => {
fs.mkdir(path.dirname(swafViewFile), {recursive: true}, (err) => {
if (err) return reject(err);
resolve();
});
});
await new Promise<void>((resolve, reject) => fs.writeFile(swafViewFile, finalCode, err => {
2021-03-18 16:22:25 +01:00
if (err) return reject(err);
resolve();
}));
2021-03-24 16:07:50 +01:00
if (alsoCompileDependents && Object.keys(this.dependencyCache).indexOf(canonicalViewName) >= 0) {
logger.debug(canonicalViewName + ' > ', 'Compiling dependents...');
for (const dependent of [...this.dependencyCache[canonicalViewName]]) {
await this.preCompileSvelte(await this.resolveViewFromCanonicalName(dependent), true);
}
}
2021-03-18 16:22:25 +01:00
}
2021-03-24 16:07:50 +01:00
private resolveDependencies(source: string, canonicalViewName: string): string[] {
const dependencies: string[] = [];
for (const match of source.matchAll(/import .+ from ['"](.+?\.svelte)['"];/gm)) {
dependencies.push(path.join(path.dirname(canonicalViewName), match[1]));
}
// Clear existing links from cache
for (const dependency of Object.keys(this.dependencyCache)) {
this.dependencyCache[dependency].delete(canonicalViewName);
}
// Add new links to cache
for (const dependency of dependencies) {
if (Object.keys(this.dependencyCache).indexOf(dependency) < 0) {
this.dependencyCache[dependency] = new Set<string>();
}
this.dependencyCache[dependency].add(canonicalViewName);
}
return dependencies;
}
private async replaceBackendCode(canonicalViewName: string, code?: string): Promise<{
2021-03-18 16:22:25 +01:00
backendReplacedCode: string,
backendLines: string[],
2021-03-24 16:07:50 +01:00
}> {
// Cache
if (Object.keys(this.backendCodeCache).indexOf(canonicalViewName) >= 0) {
return this.backendCodeCache[canonicalViewName];
}
const outputFile = path.join(this.svelteOutputPath, canonicalViewName);
await new Promise<void>((resolve, reject) => {
fs.mkdir(path.dirname(outputFile), {recursive: true}, (err) => {
if (err) return reject(err);
resolve();
});
});
// Read source file if code was not already provided
if (!code) {
const file = await this.resolveViewFromCanonicalName(canonicalViewName);
code = await new Promise<string>((resolve, reject) => {
fs.readFile(file, (err, data) => {
if (err) return reject(err);
resolve(data.toString());
});
});
}
// Skip replace if there is no swaf export
2021-03-18 16:22:25 +01:00
if (code.indexOf(`export let swaf = {};`) < 0) {
2021-03-24 16:07:50 +01:00
const generated = {
2021-03-18 16:22:25 +01:00
backendReplacedCode: code,
backendLines: [],
};
2021-03-24 16:07:50 +01:00
await new Promise<void>((resolve, reject) => {
fs.writeFile(outputFile, generated.backendReplacedCode, (err) => {
if (err) return reject(err);
resolve();
});
});
this.backendCodeCache[canonicalViewName] = generated;
return generated;
2021-03-18 16:22:25 +01:00
}
2021-03-24 16:07:50 +01:00
let output = code;
2021-03-18 16:22:25 +01:00
const backendLines = new Set<string>();
let index = 0;
2021-03-24 16:07:50 +01:00
while ((index = output.indexOf(BACKEND_CODE_PREFIX, index + 1)) >= 0) {
2021-03-18 16:22:25 +01:00
// Escaping
2021-03-24 16:07:50 +01:00
if (index > 0 && output[index - 1] === '\\') {
const isEscapingEscaped: boolean = index > 1 && output[index - 2] === '\\';
output = output.substring(0, index - 1 - (isEscapingEscaped ? 1 : 0)) +
output.substring(index, output.length);
2021-03-18 16:22:25 +01:00
continue;
}
const startIndex = index + BACKEND_CODE_PREFIX.length;
let endIndex = startIndex;
let struct = 0;
2021-03-24 16:07:50 +01:00
while (endIndex < output.length) {
if (['(', '[', '{'].indexOf(output[endIndex]) >= 0) struct++;
if ([')', ']', '}'].indexOf(output[endIndex]) >= 0) {
2021-03-18 16:22:25 +01:00
struct--;
if (struct <= 0) {
if (struct === 0) endIndex++;
break;
}
}
2021-03-24 16:07:50 +01:00
if ([' ', '\n', '<'].indexOf(output[endIndex]) >= 0 && struct === 0) break;
2021-03-18 16:22:25 +01:00
endIndex++;
}
2021-03-24 16:07:50 +01:00
let backendLine = output.substring(startIndex, endIndex);
if (backendLine.match(/([^()]+)\((.+?)\)/)) {
backendLine = backendLine.replace(/([^()]+)\((.+?)\)/, "'$1', [$2]");
} else {
backendLine = backendLine.replace(/([^()]+)/, "'$1'");
}
2021-03-18 16:22:25 +01:00
backendLines.add(backendLine);
2021-03-24 16:07:50 +01:00
output = output.substring(0, index) +
'swaf(' + backendLine + ')' +
output.substring(endIndex, output.length);
2021-03-18 16:22:25 +01:00
}
2021-03-24 16:07:50 +01:00
const generated = {
backendReplacedCode: output,
2021-03-18 16:22:25 +01:00
backendLines: [...backendLines],
};
2021-03-24 16:07:50 +01:00
await new Promise<void>((resolve, reject) => {
fs.writeFile(outputFile, generated.backendReplacedCode, (err) => {
if (err) return reject(err);
resolve();
});
});
this.backendCodeCache[canonicalViewName] = generated;
return generated;
2021-03-18 16:22:25 +01:00
}
2021-03-24 16:07:50 +01:00
private async preProcessSvelte(code: string, filename: string, canonicalViewName: string): Promise<string> {
logger.debug(canonicalViewName + ' > ', 'Preprocessing svelte', filename);
2021-03-18 16:22:25 +01:00
const preprocessed = await preprocess(
code,
sveltePreprocess({
typescript: {
tsconfigFile: 'tsconfig.views.json',
},
}),
{
filename: filename,
},
);
2021-03-24 16:07:50 +01:00
return preprocessed.code;
2021-03-18 16:22:25 +01:00
}
2021-03-24 16:07:50 +01:00
private compileSvelteSsr(code: string, filename: string, canonicalViewName: string): {
2021-03-18 16:22:25 +01:00
head: string,
css: CssResult,
html: string,
} {
2021-03-24 16:07:50 +01:00
logger.debug(canonicalViewName + ' > ', 'Compiling svelte ssr', filename);
2021-03-18 16:22:25 +01:00
const svelteSsr = compile(code, {
2021-03-24 16:07:50 +01:00
dev: config.get<boolean>('view.dev'),
2021-03-18 16:22:25 +01:00
generate: 'ssr',
format: 'cjs',
cssOutputFilename: filename + '.css',
});
2021-03-24 16:07:50 +01:00
const locals = this.getGlobals();
2021-03-18 16:22:25 +01:00
return requireFromString(svelteSsr.js.code, filename).default.render({
2021-03-24 16:07:50 +01:00
swaf: (key: string, args?: unknown[]) => {
if (!args) return locals[key];
const f = locals[key];
if (typeof f !== 'function') throw new Error(key + ' is not a function.');
return f.call(locals, ...args);
},
2021-03-18 16:22:25 +01:00
});
}
2021-03-24 16:07:50 +01:00
private async bundle(...canonicalViewNames: string[]): Promise<void> {
logger.debug('Bundling');
const production = !config.get<boolean>('view.dev');
const options: RollupOptions | RollupWatchOptions = {
input: canonicalViewNames.map(name => path.join(this.svelteOutputPath, name)),
output: {
sourcemap: true,
format: 'es',
name: 'bundle',
dir: path.join(this.publicAssetsPath, 'js'),
entryFileNames: (chunkInfo: PreRenderedChunk): string => {
const name = chunkInfo.facadeModuleId ?
path.relative(this.svelteOutputPath, chunkInfo.facadeModuleId) :
chunkInfo.name;
return name + '.js';
},
},
plugins: [
svelte({
preprocess: sveltePreprocess({
typescript: {
tsconfigFile: 'tsconfig.views.json',
},
}),
compilerOptions: {
// enable run-time checks when not in production
dev: !production,
hydratable: true,
},
}),
// we'll extract any component CSS out into
// a separate file - better for performance
cssOnlyRollupPlugin({output: 'bundle.css'}),
// If you have external dependencies installed from
// npm, you'll most likely need these plugins. In
// some cases you'll need additional configuration -
// consult the documentation for details:
// https://github.com/rollup/plugins/tree/master/packages/commonjs
resolve({
browser: true,
dedupe: ['svelte'],
}),
commonjs(),
],
watch: {
buildDelay: 1000,
clearScreen: false,
},
};
if (production) {
// If we're building for production (npm run build
// instead of npm run dev), minify
options.plugins?.push(terser());
} else {
// Watch the `public` directory and refresh the
// browser on changes when not in production
const plugin = livereloadRollupPlugin('public');
options.plugins?.push(plugin);
}
for (const name of canonicalViewNames) {
await new Promise<void>((resolve, reject) => {
fs.mkdir(path.dirname(path.join(this.publicAssetsPath, 'js', name)), {recursive: true}, err => {
if (err) return reject(err);
resolve();
});
});
}
if (production) {
if (!this.rollup) {
this.rollup = await rollup(options);
await this.rollup.write({
format: 'es',
dir: path.join(this.publicAssetsPath, 'js'),
});
}
} else {
if (!this.rollup) {
this.rollup = watch(options);
this.rollup.on('event', (event) => {
if (event.code === 'ERROR' || event.code === 'BUNDLE_END') {
event.result?.close().catch(err => logger.error(err));
logger.debug('Bundled from watch');
}
});
}
}
}
private async resetBundle(): Promise<void> {
if (this.rollup) {
logger.debug('Stopping rollup...');
await this.rollup.close();
this.rollup = undefined;
}
}
private async renderSvelte(
file: string,
locals: Record<string, unknown>,
callback: (err: Error | null, rendered?: string) => void,
): Promise<void> {
const canonicalViewName = this.toCanonicalViewName(file);
const actualFile = path.join(this.svelteOutputPath, canonicalViewName + COMPILED_SVELTE_EXTENSION);
if (!config.get<boolean>('view.enable_asset_cache')) delete this.fileCache[actualFile];
const view = await this.getFileContentsFromCache(actualFile);
const templateFile = await this.resolveViewFromCanonicalName('layouts/svelte_layout.html');
if (!config.get<boolean>('view.enable_asset_cache')) delete this.fileCache[templateFile];
let output = await this.getFileContentsFromCache(templateFile);
const [
backendLines,
head,
html,
css,
] = view.split(FrontendToolsComponent.getSveltePreCompileSeparator(canonicalViewName));
locals = Object.assign(this.getGlobals(), locals);
const localMap: Record<string, unknown> = {};
backendLines.split('\n').forEach(line => {
const key = line.substring(1, line.indexOf(',') >= 0 ? line.indexOf(',') - 1 : line.length - 1);
if (line.indexOf('[') >= 0) {
const args = line.substring(line.indexOf('[') + 1, line.length - 1)
.split(/, *?/)
.map(arg => {
if (arg.startsWith("'")) return '"' + arg.substring(1, arg.length - 1) + '"';
return arg;
})
.map(arg => JSON.parse(arg));
const f = locals[key];
if (typeof f !== 'function') throw new Error(key + ' is not a function.');
localMap[`'${key}', ${JSON.stringify(args)}`] = f.call(locals, ...args);
} else {
localMap[`'${key}'`] = locals[key];
}
2021-03-18 16:22:25 +01:00
});
2021-03-24 16:07:50 +01:00
const props = JSON.stringify(localMap);
output = output.replace('%canonicalViewName%', canonicalViewName);
output = output.replace('%props%', props);
output = output.replace('%head%', head);
output = output.replace('%html%', html);
output = output.replace('%css%', css);
callback(null, output);
}
private async getFileContentsFromCache(file: string): Promise<string> {
if (!this.fileCache[file]) {
this.fileCache[file] = await new Promise<string>((resolve, reject) => {
fs.readFile(file, (err, data) => {
if (err) return reject(err);
resolve(data.toString());
});
});
}
return this.fileCache[file];
}
private toCanonicalViewName(file: string): string {
const resolvedFilePath = path.resolve(file);
let canonicalViewName: string | null = null;
for (const viewPath of this.viewPaths) {
if (resolvedFilePath.startsWith(viewPath)) {
canonicalViewName = resolvedFilePath.substring(viewPath.length + 1);
}
}
if (!canonicalViewName) throw new Error('view ' + file + ' not found');
return canonicalViewName;
}
private async resolveViewFromCanonicalName(canonicalName: string): Promise<string> {
for (const viewPath of this.viewPaths) {
const tryPath = path.join(viewPath, canonicalName);
if (await new Promise<boolean>((resolve, reject) => {
fs.stat(tryPath, (err) => {
if (err == null) {
resolve(true);
} else if (err.code === 'ENOENT') {
resolve(false);
} else {
reject(err);
}
});
})) {
return tryPath;
}
}
throw new Error('View not found from canonical name ' + canonicalName);
}
/**
* TODO: add a way to add locals from anywhere
*/
private getGlobals(): Record<string, unknown> {
2021-03-18 16:22:25 +01:00
return {
2021-03-24 16:07:50 +01:00
route: (name: string) => 'unimplemented route ' + name,
direct: 'access',
2021-03-18 16:22:25 +01:00
};
}
}
export type SourceMap = string | Record<string, unknown> | undefined;