From 9db3562584b51453daa6141eb28b0135b83a42b8 Mon Sep 17 00:00:00 2001 From: Alice Gaudon Date: Sun, 14 Jun 2020 17:37:47 +0200 Subject: [PATCH] Make file upload work --- .gitignore | 2 + config/default.ts | 2 +- config/production.ts | 1 + package.json | 4 +- src/App.ts | 24 ++++---- src/controllers/FileController.ts | 53 +++++++++--------- src/models/FileModel.ts | 7 ++- storage/tmp/.gitkeep | 1 + storage/uploads/.gitkeep | 0 views/file-manager.njk | 27 +++++++-- views/layouts/base.njk | 2 +- yarn.lock | 91 +++++++++++++++++++++---------- 12 files changed, 140 insertions(+), 74 deletions(-) create mode 100644 storage/tmp/.gitkeep create mode 100644 storage/uploads/.gitkeep diff --git a/.gitignore b/.gitignore index 7626c60..94f3f7d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ node_modules public dist yarn-error.log +storage/tmp +storage/uploads \ No newline at end of file diff --git a/config/default.ts b/config/default.ts index 9160d20..31e5ddf 100644 --- a/config/default.ts +++ b/config/default.ts @@ -17,5 +17,5 @@ export default Object.assign(require("wms-core/config/default").default, { gitlab_webhook_token: 'secret', newlyGeneratedSlugSize: 3, default_file_ttl: 30, // 30 seconds - max_upload_size: '100mb', + max_upload_size: 1, // MB }); \ No newline at end of file diff --git a/config/production.ts b/config/production.ts index 5aff414..1c22849 100644 --- a/config/production.ts +++ b/config/production.ts @@ -15,4 +15,5 @@ export default Object.assign(require("wms-core/config/production").default, { gitlab_webhook_token: 'CHANGEME', newlyGeneratedSlugSize: 5, default_file_ttl: 30 * 24 * 3600, // 30 days + max_upload_size: 8192, // MB }); \ No newline at end of file diff --git a/package.json b/package.json index e1a56eb..8bdab9c 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "@types/config": "^0.0.36", "@types/express": "^4.17.6", "@types/jest": "^25.2.1", + "@types/multer": "^1.4.3", "@types/node": "^13.13.2", "babel-loader": "^8.1.0", "concurrently": "^5.1.0", @@ -44,9 +45,8 @@ "wms-core": "^0" }, "dependencies": { - "@types/express-fileupload": "^1.1.3", "config": "^3.3.1", "express": "^4.17.1", - "express-fileupload": "^1.1.7-alpha.3" + "multer": "^1.4.2" } } diff --git a/src/App.ts b/src/App.ts index e8f4dba..ae036ad 100644 --- a/src/App.ts +++ b/src/App.ts @@ -64,15 +64,23 @@ export default class App extends Application { const redisComponent = new RedisComponent(); const mysqlComponent = new MysqlComponent(); - const expressAppComponent = new ExpressAppComponent(this.port); - this.use(expressAppComponent); - this.use(new NunjucksComponent()); - this.use(new LogRequestsComponent()); + // Session + this.use(redisComponent); + this.use(new SessionComponent(redisComponent)); // Static files this.use(new ServeStaticDirectoryComponent('public')); this.use(new ServeStaticDirectoryComponent('node_modules/feather-icons/dist', '/icons')); + // Utils + this.use(new RedirectBackComponent()); + this.use(new FormHelperComponent()); + + const expressAppComponent = new ExpressAppComponent(this.port); + this.use(expressAppComponent); + this.use(new NunjucksComponent()); + this.use(new LogRequestsComponent()); + // Maintenance this.use(new MaintenanceComponent(this, () => { return redisComponent.canServe() && mysqlComponent.canServe(); @@ -83,9 +91,7 @@ export default class App extends Application { this.use(mysqlComponent); this.use(new MailComponent()); - // Session - this.use(redisComponent); - this.use(new SessionComponent(redisComponent)); + // Auth this.use(new AuthComponent(new class extends AuthGuard { public async getProofForSession(session: Express.Session): Promise { return await MagicLink.bySessionID(session.id, [MagicLinkActionType.LOGIN, MagicLinkActionType.REGISTER]); @@ -101,10 +107,6 @@ export default class App extends Application { } })); - // Utils - this.use(new RedirectBackComponent()); - this.use(new FormHelperComponent()); - // Middlewares this.use(new CsrfProtectionComponent()); diff --git a/src/controllers/FileController.ts b/src/controllers/FileController.ts index 05125fd..41b4feb 100644 --- a/src/controllers/FileController.ts +++ b/src/controllers/FileController.ts @@ -1,22 +1,27 @@ import Controller from "wms-core/Controller"; import {REQUIRE_AUTH_MIDDLEWARE} from "wms-core/auth/AuthComponent"; -import express, {Request, Response} from "express"; +import {Request, Response, Router} from "express"; import {BadRequestError, NotFoundHttpError, ServerError} from "wms-core/HttpError"; import FileModel from "../models/FileModel"; -import fileUpload from "express-fileupload"; import {cryptoRandomDictionary} from "wms-core/Utils"; import config from "config"; import * as fs from "fs"; +import multer from "multer"; const SLUG_DICTIONARY = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; export default class FileController extends Controller { + setupRequestParsingMiddlewares(router: Router) { + router.use('/', FILE_UPLOAD_MIDDLEWARE); + router.use('/:slug', FILE_UPLOAD_MIDDLEWARE); + } + routes(): void { this.get('/files/:page?', this.getFileManager, 'file-manager', REQUIRE_AUTH_MIDDLEWARE); this.get('/files/delete/:slug', this.deleteFile, 'delete-file-frontend', REQUIRE_AUTH_MIDDLEWARE); this.get('/:slug', this.downloadFile, 'get-file'); - this.post('/', this.postFile, 'post-file', REQUIRE_AUTH_MIDDLEWARE, FILE_UPLOAD_MAX_SIZE_MIDDLEWARE, FILE_UPLOAD_MIDDLEWARE); - this.put('/:slug', this.putFile, 'put-file', REQUIRE_AUTH_MIDDLEWARE, FILE_UPLOAD_MAX_SIZE_MIDDLEWARE, FILE_UPLOAD_MIDDLEWARE); + this.post('/', this.postFile, 'post-file', REQUIRE_AUTH_MIDDLEWARE); + this.put('/:slug', this.putFile, 'put-file', REQUIRE_AUTH_MIDDLEWARE); this.delete('/:slug', this.deleteFile, 'delete-file', REQUIRE_AUTH_MIDDLEWARE); } @@ -52,26 +57,29 @@ export default class FileController extends Controller { } protected async handleFileUpload(slug: string, req: Request, res: Response): Promise { - if (!req.files || !req.files.upload) { + // Check for file upload + if (!req.file) { throw new BadRequestError('No file received.', 'You must upload exactly one (1) file.', req.url); } - let upload = req.files.upload; - if (Array.isArray(upload) && upload.length !== 1) { - throw new BadRequestError((req.files ? req.files.length : 0) + ' files received.', 'You must upload exactly one (1) file.', req.url); - } - if (Array.isArray(upload)) upload = upload[0]; + let upload = req.file; + + // TTL + let ttl = config.get('default_file_ttl'); + if (req.body.never_expire !== undefined) ttl = 0; + if (req.body.ttl !== undefined) ttl = parseInt(req.body.ttl); + else if (req.body.expire_after_days !== undefined) ttl = parseInt(req.body.expire_after_days) * 24 * 3600; const file = new FileModel({ slug: slug, - real_name: upload.name, + real_name: upload.originalname, storage_type: 'local', storage_path: 'storage/uploads/' + slug, - ttl: req.body.ttl !== undefined ? parseInt(req.body.ttl) : config.get('default_file_ttl'), + ttl: ttl, }); await file.save(); - await upload.mv(file.storage_path); + fs.renameSync(upload.path, file.storage_path); res.format({ json: () => res.json({ @@ -79,7 +87,7 @@ export default class FileController extends Controller { }), text: () => res.send(file.getURL()), html: () => { - req.flash('success', 'Upload success! ' + file.getURL()); + req.flash('success', 'Upload success!'); res.redirectBack('/'); }, }); @@ -123,14 +131,9 @@ export default class FileController extends Controller { } } -const FILE_UPLOAD_MAX_SIZE_MIDDLEWARE = express.urlencoded({ - limit: config.get('max_upload_size'), - extended: true, -}); - -const FILE_UPLOAD_MIDDLEWARE = fileUpload({ - tempFileDir: 'storage/tmp', - useTempFiles: true, - abortOnLimit: true, - createParentPath: true, -}); +const FILE_UPLOAD_MIDDLEWARE = multer({ + dest: 'storage/tmp', + limits: { + fileSize: config.get('max_upload_size') * 1024 * 1024, + }, +}).single('upload'); diff --git a/src/models/FileModel.ts b/src/models/FileModel.ts index 20c1dd5..ba7060e 100644 --- a/src/models/FileModel.ts +++ b/src/models/FileModel.ts @@ -35,14 +35,17 @@ export default class FileModel extends Model { }); } - public getExpirationDate(): Date { + public getExpirationDate(): Date | null { if (!this.created_at) return new Date(); + if (this.ttl === 0) return null; return new Date(this.created_at.getTime() + this.ttl * 1000); } public shouldBeDeleted(): boolean { - return new Date().getTime() >= this.getExpirationDate().getTime(); + const expirationDate = this.getExpirationDate(); + if (!expirationDate) return false; + return new Date().getTime() >= expirationDate.getTime(); } } diff --git a/storage/tmp/.gitkeep b/storage/tmp/.gitkeep new file mode 100644 index 0000000..22e8364 --- /dev/null +++ b/storage/tmp/.gitkeep @@ -0,0 +1 @@ +/* \ No newline at end of file diff --git a/storage/uploads/.gitkeep b/storage/uploads/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/views/file-manager.njk b/views/file-manager.njk index f54c164..a6348d8 100644 --- a/views/file-manager.njk +++ b/views/file-manager.njk @@ -14,21 +14,23 @@

Upload a file

-
- {{ macros.field(_locals, 'file', 'upload', '', 'Choose wisely', 'The maximum upload size is ' + max_upload_size, validation_attributes='required') }} + + {{ macros.field(_locals, 'file', 'upload', '', 'Choose wisely', 'The maximum upload size is ' + max_upload_size + 'MiB', validation_attributes='required') }} {{ macros.field(_locals, 'number', 'expire_after_days', '30', 'How many days to delete this file after', null, validation_attributes='max="1825"') }} {{ macros.field(_locals, 'checkbox', 'never_expire', '', 'Never delete this file') }} - + {{ macros.csrf(getCSRFToken) }} + +

File list

- +
@@ -37,6 +39,23 @@ + + + {% for file in files %} + + + + + {% set expires_at = file.getExpirationDate() %} + + + {% endfor %} +
#Expires at
{{ file.id }} +
+ {{ file.getURL() }} + +
+ 		{{ file.real_name }}{% if expires_at %}{{ expires_at.toISOString() }}{% else %}Never{% endif %}
{% endblock %} \ No newline at end of file diff --git a/views/layouts/base.njk b/views/layouts/base.njk index edbb5c7..9b3ed9c 100644 --- a/views/layouts/base.njk +++ b/views/layouts/base.njk @@ -18,7 +18,7 @@
  • About
    • {% if user %}
  • File manager
    • -
  • Logout
    • +
  • Logout
    • {% else %}
  • Login / Register
    • {% endif %} diff --git a/yarn.lock b/yarn.lock index 6987dad..8aeabe5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1110,13 +1110,6 @@ dependencies: "@types/node" "*" -"@types/express-fileupload@^1.1.3": - version "1.1.3" - resolved "https://registry.toot.party/@types%2fexpress-fileupload/-/express-fileupload-1.1.3.tgz#7cccbcb2d0b423b1c53ec651115ea9d910feb8a4" - integrity sha512-J5Ft7O0Zg+fn8h6dUVf8aK/vgO4mU62i2zurH7S+yLYnrPjF6hXdLy2dnfE3m5pSWmletyQ3dHm5IX1XHmwhWQ== - dependencies: - "@types/express" "*" - "@types/express-serve-static-core@*": version "4.17.7" resolved "https://registry.toot.party/@types%2fexpress-serve-static-core/-/express-serve-static-core-4.17.7.tgz#dfe61f870eb549dc6d7e12050901847c7d7e915b" @@ -1202,6 +1195,13 @@ resolved "https://registry.toot.party/@types%2fminimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d" integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA== +"@types/multer@^1.4.3": + version "1.4.3" + resolved "https://registry.toot.party/@types%2fmulter/-/multer-1.4.3.tgz#bdff74b334c38a8ee1de9fbedb5d1d3dbc377422" + integrity sha512-tWsKbF5LYtXrJ7eOfI0aLBgEv9B7fnJe1JRXTj5+Z6EMfX0yHVsRFsNGnKyN8Bs0gtDv+JR37xAqsPnALyVTqg== + dependencies: + "@types/express" "*" + "@types/mysql@^2.15.10": version "2.15.13" resolved "https://registry.toot.party/@types%2fmysql/-/mysql-2.15.13.tgz#153dc2e2f8dffd39f7bba556c2679f14bdbecde1" @@ -1591,6 +1591,11 @@ anymatch@^3.0.3, anymatch@~3.1.1: normalize-path "^3.0.0" picomatch "^2.0.4" +append-field@^1.0.0: + version "1.0.0" + resolved "https://registry.toot.party/append-field/-/append-field-1.0.0.tgz#1e3440e915f0b1203d23748e78edd7b9b5b43e56" + integrity sha1-HjRA6RXwsSA9I3SOeO3XubW0PlY= + aproba@^1.0.3, aproba@^1.1.1: version "1.2.0" resolved "https://registry.toot.party/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a" @@ -2205,12 +2210,13 @@ builtin-status-codes@^3.0.0: resolved "https://registry.toot.party/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz#85982878e21b98e1c66425e03d0174788f569ee8" integrity sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug= -busboy@^0.3.1: - version "0.3.1" - resolved "https://registry.toot.party/busboy/-/busboy-0.3.1.tgz#170899274c5bf38aae27d5c62b71268cd585fd1b" - integrity sha512-y7tTxhGKXcyBxRKAni+awqx8uqaJKrSFSNFSeRG5CsWNdmy2BIK+6VGWEW7TZnIO/533mtMEA4rOevQV815YJw== +busboy@^0.2.11: + version "0.2.14" + resolved "https://registry.toot.party/busboy/-/busboy-0.2.14.tgz#6c2a622efcf47c57bbbe1e2a9c37ad36c7925453" + integrity sha1-bCpiLvz0fFe7vh4qnDetNseSVFM= dependencies: - dicer "0.3.0" + dicer "0.2.5" + readable-stream "1.1.x" bytes@3.1.0: version "3.1.0" @@ -2617,7 +2623,7 @@ concat-map@0.0.1: resolved "https://registry.toot.party/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s= -concat-stream@^1.5.0: +concat-stream@^1.5.0, concat-stream@^1.5.2: version "1.6.2" resolved "https://registry.toot.party/concat-stream/-/concat-stream-1.6.2.tgz#904bdf194cd3122fc675c77fc4ac3d4ff0fd1a34" integrity sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw== @@ -3204,11 +3210,12 @@ detect-newline@^3.0.0: resolved "https://registry.toot.party/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651" integrity sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA== -dicer@0.3.0: - version "0.3.0" - resolved "https://registry.toot.party/dicer/-/dicer-0.3.0.tgz#eacd98b3bfbf92e8ab5c2fdb71aaac44bb06b872" - integrity sha512-MdceRRWqltEG2dZqO769g27N/3PXfcKl04VhYnBlo2YhH7zPi88VebsjTKclaOyiuMaGU72hTfw3VkUitGcVCA== +dicer@0.2.5: + version "0.2.5" + resolved "https://registry.toot.party/dicer/-/dicer-0.2.5.tgz#5996c086bb33218c812c090bddc09cd12facb70f" + integrity sha1-WZbAhrszIYyBLAkL3cCc0S+stw8= dependencies: + readable-stream "1.1.x" streamsearch "0.1.2" diff-sequences@^25.2.6: @@ -3706,13 +3713,6 @@ expect@^25.5.0: jest-message-util "^25.5.0" jest-regex-util "^25.2.6" -express-fileupload@^1.1.7-alpha.3: - version "1.1.7-alpha.3" - resolved "https://registry.toot.party/express-fileupload/-/express-fileupload-1.1.7-alpha.3.tgz#7c09f42aeacb835a50979f241d7b2850d54ed92d" - integrity sha512-2YRJQqjgfFcYiMr8inico+UQ0UsxuOUyO9wkWkx+vjsEcUI7c1ae38Nv5NKdGjHqL5+J01P6StT9mjZTI7Qzjg== - dependencies: - busboy "^0.3.1" - express-session@^1.17.1: version "1.17.1" resolved "https://registry.toot.party/express-session/-/express-session-1.17.1.tgz#36ecbc7034566d38c8509885c044d461c11bf357" @@ -5190,6 +5190,11 @@ is-yarn-global@^0.3.0: resolved "https://registry.toot.party/is-yarn-global/-/is-yarn-global-0.3.0.tgz#d502d3382590ea3004893746754c89139973e232" integrity sha512-VjSeb/lHmkoyd8ryPVIKvOCn4D1koMqY+vqyjjUfc3xyKtP4dYOxM44sZrnqQSzSds3xyOrUTLTC9LVCVgLngw== +isarray@0.0.1: + version "0.0.1" + resolved "https://registry.toot.party/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf" + integrity sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8= + isarray@1.0.0, isarray@^1.0.0, isarray@~1.0.0: version "1.0.0" resolved "https://registry.toot.party/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" @@ -6779,6 +6784,20 @@ ms@^2.1.1: resolved "https://registry.toot.party/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== +multer@^1.4.2: + version "1.4.2" + resolved "https://registry.toot.party/multer/-/multer-1.4.2.tgz#2f1f4d12dbaeeba74cb37e623f234bf4d3d2057a" + integrity sha512-xY8pX7V+ybyUpbYMxtjM9KAiD9ixtg5/JkeKUTD6xilfDv0vzzOFcCp4Ljb1UU3tSOM3VTZtKo63OmzOrGi3Cg== + dependencies: + append-field "^1.0.0" + busboy "^0.2.11" + concat-stream "^1.5.2" + mkdirp "^0.5.1" + object-assign "^4.1.1" + on-finished "^2.3.0" + type-is "^1.6.4" + xtend "^4.0.0" + mysql@^2.18.1: version "2.18.1" resolved "https://registry.toot.party/mysql/-/mysql-2.18.1.tgz#2254143855c5a8c73825e4522baf2ea021766717" @@ -8030,6 +8049,16 @@ read-pkg@^5.2.0: string_decoder "~1.1.1" util-deprecate "~1.0.1" +readable-stream@1.1.x: + version "1.1.14" + resolved "https://registry.toot.party/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9" + integrity sha1-fPTFTvZI44EwhMY23SB54WbAgdk= + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.1" + isarray "0.0.1" + string_decoder "~0.10.x" + readable-stream@^3.1.1, readable-stream@^3.6.0: version "3.6.0" resolved "https://registry.toot.party/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" @@ -8978,6 +9007,11 @@ string_decoder@^1.0.0, string_decoder@^1.1.1: dependencies: safe-buffer "~5.2.0" +string_decoder@~0.10.x: + version "0.10.31" + resolved "https://registry.toot.party/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" + integrity sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ= + string_decoder@~1.1.1: version "1.1.1" resolved "https://registry.toot.party/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" @@ -9446,7 +9480,7 @@ type-fest@^0.8.1: resolved "https://registry.toot.party/type-fest/-/type-fest-0.8.1.tgz#09e249ebde851d3b1e48d27c105444667f17b83d" integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA== -type-is@~1.6.17, type-is@~1.6.18: +type-is@^1.6.4, type-is@~1.6.17, type-is@~1.6.18: version "1.6.18" resolved "https://registry.toot.party/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131" integrity sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g== @@ -9940,12 +9974,13 @@ widest-line@^3.1.0: string-width "^4.0.0" wms-core@^0: - version "0.7.1" - resolved "https://registry.toot.party/wms-core/-/wms-core-0.7.1.tgz#bbc3c6d7e3db93da89818e1c1c5a65c1938df44a" - integrity sha512-EBSP6b4DkTdlwLx6xOvcTN8FYXwiz19JWzFYrmngIXspahOjWEVT12z2IAoa+9uZ7Afk0LWKZRiNvs57c22QHQ== + version "0.7.12" + resolved "https://registry.toot.party/wms-core/-/wms-core-0.7.12.tgz#9e198599df089962f636de7b28fa10d96f4167b7" + integrity sha512-msKHjh+7QnPugpcxrLcSiVFBemtGkSQOG0fLFIRoTk5CORbZHLdghBeL7DGmSsfM1SV97om2XI9xSjz2A/Rj1Q== dependencies: "@types/express" "^4.17.6" "@types/express-session" "^1.17.0" + "@types/multer" "^1.4.3" "@types/mysql" "^2.15.10" "@types/nodemailer" "^6.4.0" "@types/nunjucks" "^3.1.3"