From 9b5a0b2824fced5a4dba088674f168428f981f66 Mon Sep 17 00:00:00 2001 From: Alice Gaudon Date: Mon, 6 Jul 2020 10:49:18 +0200 Subject: [PATCH] Add URL shortening --- assets/config.json | 1 + assets/files/shorten_url.sh | 17 ++++ assets/files/upload_file.sh | 26 +++-- assets/js/url-shortener.js | 11 ++ src/App.ts | 8 ++ src/SlugGenerator.ts | 17 ++++ src/controllers/AuthTokenController.ts | 36 +++++++ src/controllers/FileController.ts | 50 ++------- src/controllers/URLRedirectController.ts | 82 +++++++++++++++ src/migrations/CreateUrlRedirectsTable.ts | 20 ++++ src/models/URLRedirect.ts | 44 ++++++++ views/desktop-utility.njk | 119 ++++++++++++++++++++++ views/file-upload.njk | 117 +-------------------- views/layouts/base.njk | 5 +- views/url-manager.njk | 44 ++++++++ views/url-shortener.njk | 40 ++++++++ yarn.lock | 52 +++++++--- 17 files changed, 498 insertions(+), 191 deletions(-) create mode 100755 assets/files/shorten_url.sh create mode 100644 assets/js/url-shortener.js create mode 100644 src/SlugGenerator.ts create mode 100644 src/controllers/AuthTokenController.ts create mode 100644 src/controllers/URLRedirectController.ts create mode 100644 src/migrations/CreateUrlRedirectsTable.ts create mode 100644 src/models/URLRedirect.ts create mode 100644 views/desktop-utility.njk create mode 100644 views/url-manager.njk create mode 100644 views/url-shortener.njk diff --git a/assets/config.json b/assets/config.json index a181d7c..60bc2c0 100644 --- a/assets/config.json +++ b/assets/config.json @@ -2,6 +2,7 @@ "bundles": { "app": "js/app.js", "fm": "js/fm.js", + "url-shortener": "js/url-shortener.js", "layout": "sass/layout.scss", "error": "sass/error.scss", "logo": "img/logo.svg", diff --git a/assets/files/shorten_url.sh b/assets/files/shorten_url.sh new file mode 100755 index 0000000..667e434 --- /dev/null +++ b/assets/files/shorten_url.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +target_url=$1 +slug=$2 +token=$(cat "$HOME/.ily_token") + +if test -z "$slug"; then + method=POST +else + method=PUT +fi + +curl -X $method -H 'Accept: text/plain' \ + -H "Authorization: $token" \ + -F 'type=url' \ + -F "target_url=$target_url" \ + "https://ily.li/$slug" diff --git a/assets/files/upload_file.sh b/assets/files/upload_file.sh index 5c0d36e..d4a1890 100755 --- a/assets/files/upload_file.sh +++ b/assets/files/upload_file.sh @@ -1,20 +1,18 @@ #!/bin/sh local_file=$1 -token=$(cat $HOME/.ily_token) +slug=$2 +token=$(cat "$HOME/.ily_token") -if test -z "$2"; then - curl -X POST -H 'Accept: text/plain' \ - -H "Authorization: $token" \ - -F 'type=file' \ - -F 'upload=@'$local_file \ - -F 'ttl=2592000' \ - https://ily.li/ +if test -z "$slug"; then + method=POST else - curl -X PUT -H 'Accept: text/plain' \ - -H "Authorization: $token" \ - -F 'type=file' \ - -F 'upload=@'$local_file \ - -F 'ttl=2592000' \ - "https://ily.li/$2" + method=PUT fi + +curl -X $method -H 'Accept: text/plain' \ + -H "Authorization: $token" \ + -F 'type=file' \ + -F "upload=@$local_file" \ + -F 'ttl=2592000' \ + "https://ily.li/$slug" diff --git a/assets/js/url-shortener.js b/assets/js/url-shortener.js new file mode 100644 index 0000000..efc5190 --- /dev/null +++ b/assets/js/url-shortener.js @@ -0,0 +1,11 @@ +document.addEventListener('DOMContentLoaded', () => { + const form = document.getElementById('url-shortener-form'); + if (!form) return; + + const autogenUrlCheckbox = document.getElementById('field-autogen_url'); + const slugField = document.getElementById('field-slug'); + + autogenUrlCheckbox.addEventListener('change', () => { + slugField.disabled = autogenUrlCheckbox.checked; + }); +}); diff --git a/src/App.ts b/src/App.ts index 68c7770..2ca486f 100644 --- a/src/App.ts +++ b/src/App.ts @@ -36,6 +36,9 @@ import CreateFilesTable from "./migrations/CreateFilesTable"; import IncreaseFilesSizeField from "./migrations/IncreaseFilesSizeField"; import AddApprovedFieldToUsersTable from "wms-core/auth/migrations/AddApprovedFieldToUsersTable"; import BackendController from "./controllers/BackendController"; +import CreateUrlRedirectsTable from "./migrations/CreateUrlRedirectsTable"; +import AuthTokenController from "./controllers/AuthTokenController"; +import URLRedirectController from "./controllers/URLRedirectController"; export default class App extends Application { private readonly port: number; @@ -56,6 +59,7 @@ export default class App extends Application { CreateFilesTable, IncreaseFilesSizeField, AddApprovedFieldToUsersTable, + CreateUrlRedirectsTable, ]; } @@ -136,10 +140,14 @@ export default class App extends Application { // Core functionality this.use(new MailController()); + // Other functionality + this.use(new AuthTokenController()); + // Semi-static this.use(new AboutController()); // Global slug this.use(new FileController()); + this.use(new URLRedirectController()); } } \ No newline at end of file diff --git a/src/SlugGenerator.ts b/src/SlugGenerator.ts new file mode 100644 index 0000000..b95997a --- /dev/null +++ b/src/SlugGenerator.ts @@ -0,0 +1,17 @@ +import {cryptoRandomDictionary} from "wms-core/Utils"; +import config from "config"; +import FileModel from "./models/FileModel"; +import {ServerError} from "wms-core/HttpError"; + +const SLUG_DICTIONARY = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; +export default async function generateSlug(tries: number): Promise { + let i = 0; + do { + const slug = cryptoRandomDictionary(config.get('newlyGeneratedSlugSize'), SLUG_DICTIONARY); + if (!await FileModel.getBySlug(slug)) { + return slug; + } + i++; + } while (i < tries); + throw new ServerError('Failed to generate slug; newly generated slug size should be increased by 1.'); +}; \ No newline at end of file diff --git a/src/controllers/AuthTokenController.ts b/src/controllers/AuthTokenController.ts new file mode 100644 index 0000000..625d027 --- /dev/null +++ b/src/controllers/AuthTokenController.ts @@ -0,0 +1,36 @@ +import Controller from "wms-core/Controller"; +import {REQUIRE_AUTH_MIDDLEWARE} from "wms-core/auth/AuthComponent"; +import {Request, Response} from "express"; +import AuthToken from "../models/AuthToken"; +import {BadRequestError, ForbiddenHttpError, NotFoundHttpError} from "wms-core/HttpError"; + +export default class AuthTokenController extends Controller { + routes(): void { + this.post('/gen-auth-token', this.postGenAuthToken, 'generate-token', REQUIRE_AUTH_MIDDLEWARE); + this.post('/revoke-auth-token/:id', this.postRevokeAuthToken, 'revoke-token', REQUIRE_AUTH_MIDDLEWARE); + } + + protected async postGenAuthToken(req: Request, res: Response): Promise { + const authToken = new AuthToken({ + user_id: req.models.user!.id, + ttl: req.body.ttl ? parseInt(req.body.ttl) : 365 * 24 * 3600, + }); + await authToken.save(); + req.flash('success', 'Successfully created auth token.'); + res.redirectBack(Controller.route('file-upload')); + } + + protected async postRevokeAuthToken(req: Request, res: Response): Promise { + const id = req.params.id; + if (!id) throw new BadRequestError('Cannot revoke token without an id.', 'Please provide an id.', req.url); + + const authToken = await AuthToken.getById(parseInt(id)); + if (!authToken) throw new NotFoundHttpError('Auth token', req.url); + if (!authToken.canDelete(req.models.user!.id!)) throw new ForbiddenHttpError('auth token', req.url); + + await authToken.delete(); + + req.flash('success', 'Successfully deleted auth token.'); + res.redirectBack(Controller.route('file-upload')); + } +} \ No newline at end of file diff --git a/src/controllers/FileController.ts b/src/controllers/FileController.ts index 3ba5c25..cc2ce06 100644 --- a/src/controllers/FileController.ts +++ b/src/controllers/FileController.ts @@ -1,27 +1,24 @@ import Controller from "wms-core/Controller"; import {REQUIRE_AUTH_MIDDLEWARE, REQUIRE_REQUEST_AUTH_MIDDLEWARE} from "wms-core/auth/AuthComponent"; import {NextFunction, Request, Response} from "express"; -import {BadRequestError, ForbiddenHttpError, NotFoundHttpError, ServerError} from "wms-core/HttpError"; +import {BadRequestError, ForbiddenHttpError, ServerError} from "wms-core/HttpError"; import FileModel from "../models/FileModel"; -import {cryptoRandomDictionary} from "wms-core/Utils"; import config from "config"; import * as fs from "fs"; import AuthToken from "../models/AuthToken"; import {IncomingForm} from "formidable"; import {FILE_UPLOAD_MIDDLEWARE} from "wms-core/components/ExpressAppComponent"; import Logger from "wms-core/Logger"; +import generateSlug from "../SlugGenerator"; -const SLUG_DICTIONARY = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; export default class FileController extends Controller { routes(): void { this.get('/files/upload', this.getFileUploader, 'file-upload', REQUIRE_AUTH_MIDDLEWARE); - this.get('/files/upload/script', this.downloadLinuxScript, 'linux-script'); + this.get('/files/upload/script', this.downloadLinuxScript, 'file-linux-script'); this.post('/files/post', this.postFileFrontend, 'post-file-frontend', REQUIRE_AUTH_MIDDLEWARE, FILE_UPLOAD_FORM_MIDDLEWARE); this.get('/files/:page([0-9]+)?', this.getFileManager, 'file-manager', REQUIRE_AUTH_MIDDLEWARE); this.post('/files/delete/:slug', this.deleteFile, 'delete-file-frontend', REQUIRE_AUTH_MIDDLEWARE); - this.post('/gen-auth-token', this.postGenAuthToken, 'generate-token', REQUIRE_AUTH_MIDDLEWARE); - this.post('/revoke-auth-token/:id', this.postRevokeAuthToken, 'revoke-token', REQUIRE_AUTH_MIDDLEWARE); this.post('/', this.postFile, 'post-file', REQUIRE_REQUEST_AUTH_MIDDLEWARE, FILE_UPLOAD_FORM_MIDDLEWARE); this.delete('/:slug', this.deleteFile, 'delete-file', REQUIRE_REQUEST_AUTH_MIDDLEWARE); @@ -46,32 +43,9 @@ export default class FileController extends Controller { }); } - protected async postGenAuthToken(req: Request, res: Response): Promise { - const authToken = new AuthToken({ - user_id: req.models.user!.id, - ttl: req.body.ttl ? parseInt(req.body.ttl) : 365 * 24 * 3600, - }); - await authToken.save(); - req.flash('success', 'Successfully created auth token.'); - res.redirectBack(Controller.route('file-upload')); - } - - protected async postRevokeAuthToken(req: Request, res: Response): Promise { - const id = req.params.id; - if (!id) throw new BadRequestError('Cannot revoke token without an id.', 'Please provide an id.', req.url); - - const authToken = await AuthToken.getById(parseInt(id)); - if (!authToken) throw new NotFoundHttpError('Auth token', req.url); - if (!authToken.canDelete(req.models.user!.id!)) throw new ForbiddenHttpError('auth token', req.url); - - await authToken.delete(); - - req.flash('success', 'Successfully deleted auth token.'); - res.redirectBack(Controller.route('file-upload')); - } - protected async postFileFrontend(req: Request, res: Response): Promise { - await this.handleFileUpload(req.body.autogen_url === undefined && req.body.slug ? req.body.slug : await this.generateSlug(10), req, res); + req.body.type = 'file'; + await this.handleFileUpload(req.body.autogen_url === undefined && req.body.slug ? req.body.slug : await generateSlug(10), req, res); } protected async downloadFile(req: Request, res: Response, next: NextFunction): Promise { @@ -96,7 +70,7 @@ export default class FileController extends Controller { protected async postFile(req: Request, res: Response, next: NextFunction): Promise { if (req.body.type !== 'file') return next(); - await this.handleFileUpload(req.body.slug || await this.generateSlug(10), req, res); + await this.handleFileUpload(req.body.slug || await generateSlug(10), req, res); } protected async putFile(req: Request, res: Response, next: NextFunction): Promise { @@ -174,18 +148,6 @@ export default class FileController extends Controller { }, }); } - - private async generateSlug(tries: number): Promise { - let i = 0; - do { - const slug = cryptoRandomDictionary(config.get('newlyGeneratedSlugSize'), SLUG_DICTIONARY); - if (!await FileModel.getBySlug(slug)) { - return slug; - } - i++; - } while (i < tries); - throw new ServerError('Failed to generate slug; newly generated slug size should be increased by 1.'); - } } diff --git a/src/controllers/URLRedirectController.ts b/src/controllers/URLRedirectController.ts new file mode 100644 index 0000000..aa51a38 --- /dev/null +++ b/src/controllers/URLRedirectController.ts @@ -0,0 +1,82 @@ +import Controller from "wms-core/Controller"; +import {NextFunction, Request, Response} from "express"; +import URLRedirect from "../models/URLRedirect"; +import {REQUIRE_AUTH_MIDDLEWARE, REQUIRE_REQUEST_AUTH_MIDDLEWARE} from "wms-core/auth/AuthComponent"; +import generateSlug from "../SlugGenerator"; +import {BadRequestError} from "wms-core/HttpError"; + +export default class URLRedirectController extends Controller { + routes(): void { + this.get('/url/shorten', this.getURLShortener, 'url-shortener', REQUIRE_AUTH_MIDDLEWARE); + this.get('/url/shorten/script', this.downloadLinuxScript, 'url-linux-script'); + this.post('/url/shorten', this.addURLFrontend, 'shorten-url', REQUIRE_AUTH_MIDDLEWARE); + this.get('/urls/:page([0-9]+)?', this.getURLRedirectManager, 'url-manager', REQUIRE_AUTH_MIDDLEWARE); + + this.post('/', this.addURL, 'post-url', REQUIRE_REQUEST_AUTH_MIDDLEWARE); + this.delete('/:slug', this.deleteURL, 'delete-url', REQUIRE_REQUEST_AUTH_MIDDLEWARE); + this.get('/:slug', this.getURLRedirect, 'get-url'); + this.put('/:slug', this.addURL, 'put-url', REQUIRE_REQUEST_AUTH_MIDDLEWARE); + } + + protected async getURLShortener(req: Request, res: Response): Promise { + res.render('url-shortener'); + } + + protected async downloadLinuxScript(req: Request, res: Response): Promise { + res.download('assets/files/shorten_url.sh', 'shorten_url.sh'); + } + + protected async getURLRedirectManager(req: Request, res: Response): Promise { + res.render('url-manager', { + urls: await URLRedirect.paginateForUser(req, 100, req.models.user!.id!), + }); + } + + protected async getURLRedirect(req: Request, res: Response, next: NextFunction): Promise { + const url = await URLRedirect.getBySlug(req.params.slug); + if (!url) return next(); + + res.redirect(url.target_url, 301); + } + + protected async deleteURL(req: Request, res: Response, next: NextFunction): Promise { + const urlRedirect = await URLRedirect.getBySlug(req.params.slug); + if (!urlRedirect) return next(); + + throw new BadRequestError( + 'Deleting url redirects is disabled for security reasons.', + 'If you still want to disable the redirection, please contact us via email.', + req.url + ); + } + + protected async addURLFrontend(req: Request, res: Response, next: NextFunction): Promise { + req.body.type = 'url'; + await this.addURL(req, res, next, req.body.autogen_url === undefined && req.body.slug ? req.body.slug : await generateSlug(10)); + } + + protected async addURL(req: Request, res: Response, next: NextFunction, slug?: string): Promise { + if (req.body.type !== 'url') return next(); + + slug = slug || req.params.slug || req.body.slug || await generateSlug(10); + const urlRedirect = new URLRedirect({ + user_id: req.models.user!.id, + slug: slug, + target_url: req.body.target_url, + }); + + await urlRedirect.save(); + + res.format({ + json: () => res.json({ + url: urlRedirect.getURL(), + }), + text: () => res.send(urlRedirect.getURL()), + html: () => { + req.flash('success', 'URL shortened successfully!'); + req.flash('url', urlRedirect.getURL()); + res.redirectBack('/'); + }, + }); + } +} \ No newline at end of file diff --git a/src/migrations/CreateUrlRedirectsTable.ts b/src/migrations/CreateUrlRedirectsTable.ts new file mode 100644 index 0000000..b82fb6d --- /dev/null +++ b/src/migrations/CreateUrlRedirectsTable.ts @@ -0,0 +1,20 @@ +import Migration from "wms-core/db/Migration"; +import {Connection} from "mysql"; + +export default class CreateUrlRedirectsTable extends Migration { + public async install(connection: Connection): Promise { + await this.query('CREATE TABLE url_redirects(' + + 'id INT NOT NULL AUTO_INCREMENT,' + + 'user_id INT NOT NULL,' + + 'slug VARCHAR(259) UNIQUE NOT NULL,' + + 'target_url VARCHAR(1745) NOT NULL,' + + 'created_at DATETIME NOT NULL DEFAULT NOW(),' + + 'PRIMARY KEY (id)' + + ')', connection); + } + + public async rollback(connection: Connection): Promise { + await this.query('DROP TABLE IF EXISTS url_redirects', connection); + } + +} \ No newline at end of file diff --git a/src/models/URLRedirect.ts b/src/models/URLRedirect.ts new file mode 100644 index 0000000..b3bd2c2 --- /dev/null +++ b/src/models/URLRedirect.ts @@ -0,0 +1,44 @@ +import Model from "wms-core/db/Model"; +import Validator from "wms-core/db/Validator"; +import User from "wms-core/auth/models/User"; +import FileModel from "./FileModel"; +import {Request} from "express"; +import config from "config"; +import Controller from "wms-core/Controller"; + +export default class URLRedirect extends Model { + public static get table(): string { + return 'url_redirects'; + } + + public static async getBySlug(slug: string): Promise { + return await this.select().where('slug', slug).first(); + } + + public static async paginateForUser(req: Request, perPage: number, user_id: number): Promise { + return await this.paginate(req, perPage, this.select().where('user_id', user_id)); + } + + public readonly user_id!: number; + public readonly slug!: string; + public readonly target_url!: string; + public created_at?: Date; + + constructor(data: any) { + super(data); + } + + protected init(): void { + this.addProperty('user_id', new Validator().defined().exists(User, 'id')); + this.addProperty('slug', new Validator().defined().minLength(1).maxLength(259).unique(URLRedirect, 'slug').unique(FileModel, 'slug')); + this.addProperty('target_url', new Validator().defined().maxLength(1745).regexp(/^https?:\/\/.{3,259}?\/?/i)); + this.addProperty('created_at', new Validator()); + } + + public getURL(): string { + return config.get('base_url') + Controller.route('get-url', { + slug: this.slug, + }); + } + +} \ No newline at end of file diff --git a/views/desktop-utility.njk b/views/desktop-utility.njk new file mode 100644 index 0000000..3c2a975 --- /dev/null +++ b/views/desktop-utility.njk @@ -0,0 +1,119 @@ +
+
+

Setup a desktop utility

+

There may be a desktop client at some point. For now, if you're an advanced user, you can setup + scripts/macros.

+ +
+
+

First alternative: sh script (native on linux)

+

If you have the sh shell on your machine (i.e. you are on linux, git bash on windows...), you can + download this script.

+

You must put a valid auth token (generated in the form at the bottom of this page) in a .ily_token + file in your home directory ($HOME/.ily_token).

+

Examples:

+ 
upload_script.sh path/to/file
+ 
upload_script.sh path/to/file my_very_important_file.png
+
+ +
+ +
+

Second alternative: implement your own client

+

+ To upload the file, you must: +

+
    +
  • Set the "Authorization" HTTP header to an auth token (generate one with the form below)
    • +
  • Make a proper file upload request either with the method "POST" on / (auto-generates a short + url) or "PUT" (choose the target url you want, alphanum) +
    • +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
Field nameDescriptionOptional?Example
uploadThe file fieldNo-
ttlHow much time (in seconds) to keep the fileYes0 (never delete), 30 (delete after 30s)
+

Example with curl:

+ 
curl -X POST -H 'Accept: application/json' \
+-H "Authorization: very_secret_and_personal_token" \
+-F 'type=file' \
+-F 'upload=@path/to/local/file' \
+https://ily.li/
+ 
curl -X PUT -H 'Accept: application/json' \
+-H "Authorization: very_secret_and_personal_token" \
+-F 'type=file' \
+-F 'upload=@path/to/local/file' \
+https://ily.li/my_very_important_file.png
+ 
curl -X POST -H 'Accept: application/json' \
+-H "Authorization: very_secret_and_personal_token" \
+-F 'type=file' \
+-F 'upload=@path/to/local/file' \
+-F 'ttl=30' \
+https://ily.li/
+
+
+
+ +
+

Auth tokens

+
+ {{ macros.csrf(getCSRFToken) }} + + +
+ + + + + + + + + + + + + + {% for token in auth_tokens %} + + + + + + + + {% endfor %} + +
#SecretCreated atLast used atActions
{{ token.id }} +
+
{{ token.secret }}
+ +
+ 		{{ token.created_at.toISOString() }}{{ token.used_at.toISOString() }} +
+ +
+
+
\ No newline at end of file diff --git a/views/file-upload.njk b/views/file-upload.njk index 16b4dd6..ef54f77 100644 --- a/views/file-upload.njk +++ b/views/file-upload.njk @@ -41,122 +41,7 @@ - -
-

Setup a desktop utility

-

There may be a desktop client at some point. For now, if you're an advanced user, you can setup - scripts/macros.

- -
-
-

First alternative: sh script (native on linux)

-

If you have the sh shell on your machine (i.e. you are on linux, git bash on windows...), you can - download this script.

-

You must put a valid auth token (generated in the form at the bottom of this page) in a .ily_token file in your home directory ($HOME/.ily_token).

-

Examples:

- 
upload_script.sh path/to/file
- 
upload_script.sh path/to/file my_very_important_file.png
-
- -
- -
-

Second alternative: implement your own client

-

- To upload the file, you must: -

-
    -
  • Set the "Authorization" HTTP header to an auth token (generate one with the form below)
    • -
  • Make a proper file upload request either with the method "POST" on / (auto-generates a short - url) or "PUT" (choose the target url you want, alphanum) -
    • -
- - - - - - - - - - - - - - - - - - - - - - - - - -
Field nameDescriptionOptional?Example
uploadThe file fieldNo-
ttlHow much time (in seconds) to keep the fileYes0 (never delete), 30 (delete after 30s)
-

Example with curl:

- 
curl -X POST -H 'Accept: application/json' \
--H "Authorization: very_secret_and_personal_token" \
--F 'type=file' \
--F 'upload=@path/to/local/file' \
-https://ily.li/
- 
curl -X PUT -H 'Accept: application/json' \
--H "Authorization: very_secret_and_personal_token" \
--F 'type=file' \
--F 'upload=@path/to/local/file' \
-https://ily.li/my_very_important_file.png
- 
curl -X POST -H 'Accept: application/json' \
--H "Authorization: very_secret_and_personal_token" \
--F 'type=file' \
--F 'upload=@path/to/local/file' \
--F 'ttl=30' \
-https://ily.li/
-
-
-
-

Auth tokens

-
- {{ macros.csrf(getCSRFToken) }} - - -
- - - - - - - - - - - - - - {% for token in auth_tokens %} - - - - - - - - {% endfor %} - -
#SecretCreated atLast used atActions
{{ token.id }} -
-
{{ token.secret }}
- -
- 		{{ token.created_at.toISOString() }}{{ token.used_at.toISOString() }} -
- -
-
-
+ {% include 'desktop-utility.njk' %} {% endblock %} \ No newline at end of file diff --git a/views/layouts/base.njk b/views/layouts/base.njk index a622ad1..b979698 100644 --- a/views/layouts/base.njk +++ b/views/layouts/base.njk @@ -15,10 +15,11 @@