From 9e3286cdbdb31ac588ca2f143420acd68c915a2d Mon Sep 17 00:00:00 2001 From: Jelenkee <59470612+Jelenkee@users.noreply.github.com> Date: Thu, 7 Oct 2021 12:22:31 +0200 Subject: [PATCH] Extended dir-list information (#241) * -refactored dirlist to use promises -added stats and more info to dir list * -added lastModified * -added format url parameter * -added docs * -fixed compatibility with node 10 * -removed unnecessary request -used p-map * -replaced p-map with p-limit * -removed logging * -added error catch * -improved test --- README.md | 81 ++++++++++++++++ index.d.ts | 15 +++ index.js | 13 ++- lib/dirList.js | 135 +++++++++++++++++++-------- package.json | 1 + test/dir-list.test.js | 210 +++++++++++++++++++++++++++++++++++++++++- 6 files changed, 412 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 2ce4bf7..ddc3f3f 100644 --- a/README.md +++ b/README.md @@ -222,6 +222,14 @@ Options: `html`, `json` Directory list can be also in `html` format; in that case, `list.render` function is required. +You can override the option with URL parameter `format`. Options are `html` and `json`. + +```bash +GET .../public/assets?format=json +``` + +will return the response as json independent of `list.format`. + **Example:** ```js @@ -300,6 +308,79 @@ GET .../public/index GET .../public/index.json ``` +#### `list.extendedFolderInfo` + +Default: `undefined` + +If `true` some extended information for folders will be accessible in `list.render` and in the json response. + +```js +render(dirs, files) { + const dir = dirs[0]; + dir.fileCount // number of files in this folder + dir.totalFileCount // number of files in this folder (recursive) + dir.folderCount // number of folders in this folder + dir.totalFolderCount // number of folders in this folder (recursive) + dir.totalSize // size of all files in this folder (recursive) + dir.lastModified // most recent last modified timestamp of all files in this folder (recursive) +} +``` + +Warning: This will slightly decrease the performance, especially for deeply nested file structures. + +#### `list.jsonFormat` + +Default: `names` + +Options: `names`, `extended` + +This option determines the output format when `json` is selected. + +`names`: +```json +{ + "dirs": [ + "dir1", + "dir2" + ], + "files": [ + "file1.txt", + "file2.txt" + ] +} +``` + +`extended`: +```json +{ + "dirs": [ + { + "name": "dir1", + "stats": { + "dev": 2100, + "size": 4096, + ... + }, + "extendedInfo": { + "fileCount": 4, + "totalSize": 51233, + ... + } + } + ], + "files": [ + { + "name": "file1.txt", + "stats": { + "dev": 2200, + "size": 554, + ... + } + } + ] +} +``` + #### `preCompressed` Default: `false` diff --git a/index.d.ts b/index.d.ts index 47a5e55..186550f 100644 --- a/index.d.ts +++ b/index.d.ts @@ -3,6 +3,7 @@ /// import { FastifyPluginCallback, FastifyReply } from 'fastify'; +import { Stats } from 'fs'; declare module "fastify" { interface FastifyReply { @@ -15,14 +16,26 @@ declare module "fastify" { } } +interface ExtendedInformation { + fileCount: number; + totalFileCount: number; + folderCount: number; + totalFolderCount: number; + totalSize: number; + lastModified: number; +} + interface ListDir { href: string; name: string; + stats: Stats; + extendedInfo?: ExtendedInformation; } interface ListFile { href: string; name: string; + stats: Stats; } interface ListRender { @@ -33,6 +46,8 @@ interface ListOptions { format: 'json' | 'html'; names: string[]; render: ListRender; + extendedFolderInfo?: boolean; + jsonFormat?: 'names' | 'extended'; } // Passed on to `send` diff --git a/index.js b/index.js index b6d6cad..b06d771 100644 --- a/index.js +++ b/index.js @@ -142,12 +142,13 @@ async function fastifyStatic (fastify, opts) { stream.on('directory', function (_, path) { if (opts.list) { - return dirList.send({ + dirList.send({ reply, dir: path, options: opts.list, route: pathname - }) + }).catch((err) => reply.send(err)) + return } if (opts.redirect === true) { @@ -161,7 +162,13 @@ async function fastifyStatic (fastify, opts) { if (err.code === 'ENOENT') { // if file exists, send real file, otherwise send dir list if name match if (opts.list && dirList.handle(pathname, opts.list)) { - return dirList.send({ reply, dir: dirList.path(opts.root, pathname), options: opts.list, route: pathname }) + dirList.send({ + reply, + dir: dirList.path(opts.root, pathname), + options: opts.list, + route: pathname + }).catch((err) => reply.send(err)) + return } // root paths left to try? diff --git a/lib/dirList.js b/lib/dirList.js index 76e39d5..1611609 100644 --- a/lib/dirList.js +++ b/lib/dirList.js @@ -1,7 +1,8 @@ 'use strict' const path = require('path') -const fs = require('fs') +const fs = require('fs').promises +const pLimit = require('p-limit') const dirList = { /** @@ -10,36 +11,84 @@ const dirList = { * @param {function(error, entries)} callback * note: can't use glob because don't get error on non existing dir */ - list: function (dir, callback) { + list: async function (dir, options) { const entries = { dirs: [], files: [] } - fs.readdir(dir, (err, files) => { - if (err) { - return callback(err) - } - if (files.length < 1) { - callback(null, entries) + const files = await fs.readdir(dir) + if (files.length < 1) { + return entries + } + + const limit = pLimit(4) + await Promise.all(files.map(filename => limit(async () => { + let stats + try { + stats = await fs.stat(path.join(dir, filename)) + } catch (error) { return } - let j = 0 - for (let i = 0; i < files.length; i++) { - const filename = files[i] - fs.stat(path.join(dir, filename), (err, file) => { - if (!err) { - if (file.isDirectory()) { - entries.dirs.push(filename) - } else { - entries.files.push(filename) - } + const entry = { name: filename, stats } + if (stats.isDirectory()) { + if (options.extendedFolderInfo) { + entry.extendedInfo = await getExtendedInfo(path.join(dir, filename)) + } + entries.dirs.push(entry) + } else { + entries.files.push(entry) + } + }))) + + async function getExtendedInfo (folderPath) { + const depth = folderPath.split(path.sep).length + let totalSize = 0 + let fileCount = 0 + let totalFileCount = 0 + let folderCount = 0 + let totalFolderCount = 0 + let lastModified = 0 + + async function walk (dir) { + const files = await fs.readdir(dir) + const limit = pLimit(4) + await Promise.all(files.map(filename => limit(async () => { + const filePath = path.join(dir, filename) + let stats + try { + stats = await fs.stat(filePath) + } catch (error) { + return } - if (j++ >= files.length - 1) { - entries.dirs.sort() - entries.files.sort() - callback(null, entries) + if (stats.isDirectory()) { + totalFolderCount++ + if (filePath.split(path.sep).length === depth + 1) { + folderCount++ + } + await walk(filePath) + } else { + totalSize += stats.size + totalFileCount++ + if (filePath.split(path.sep).length === depth + 1) { + fileCount++ + } + lastModified = Math.max(lastModified, stats.mtimeMs) } - }) + }))) } - }) + + await walk(folderPath) + return { + totalSize, + fileCount, + totalFileCount, + folderCount, + totalFolderCount, + lastModified + } + } + + entries.dirs.sort((a, b) => a.name.localeCompare(b.name)) + entries.files.sort((a, b) => a.name.localeCompare(b.name)) + return entries }, /** @@ -49,33 +98,41 @@ const dirList = { * @param {ListOptions} options * @param {string} route request route */ - send: function ({ reply, dir, options, route }) { - dirList.list(dir, (err, entries) => { - if (err) { - reply.callNotFound() - return - } + send: async function ({ reply, dir, options, route }) { + let entries + try { + entries = await dirList.list(dir, options) + } catch (error) { + return reply.callNotFound() + } + const format = reply.request.query.format || options.format + if (format !== 'html') { + if (options.jsonFormat !== 'extended') { + const nameEntries = { dirs: [], files: [] } + entries.dirs.forEach(entry => nameEntries.dirs.push(entry.name)) + entries.files.forEach(entry => nameEntries.files.push(entry.name)) - if (options.format !== 'html') { + reply.send(nameEntries) + } else { reply.send(entries) - return } + return + } - const html = options.render( - entries.dirs.map(entry => dirList.htmlInfo(entry, route)), - entries.files.map(entry => dirList.htmlInfo(entry, route))) - reply.type('text/html').send(html) - }) + const html = options.render( + entries.dirs.map(entry => dirList.htmlInfo(entry, route)), + entries.files.map(entry => dirList.htmlInfo(entry, route))) + reply.type('text/html').send(html) }, /** * provide the html information about entry and route, to get name and full route - * @param {string} entry file or dir name + * @param entry file or dir name and stats * @param {string} route request route * @return {ListFile} */ htmlInfo: function (entry, route) { - return { href: path.join(path.dirname(route), entry).replace(/\\/g, '/'), name: entry } + return { href: path.join(path.dirname(route), entry.name).replace(/\\/g, '/'), name: entry.name, stats: entry.stats, extendedInfo: entry.extendedInfo } }, /** diff --git a/package.json b/package.json index 126e78d..d37fead 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "encoding-negotiator": "^2.0.1", "fastify-plugin": "^3.0.0", "glob": "^7.1.4", + "p-limit": "^3.1.0", "readable-stream": "^3.4.0", "send": "^0.17.1" }, diff --git a/test/dir-list.test.js b/test/dir-list.test.js index e5d1f0f..280a4a7 100644 --- a/test/dir-list.test.js +++ b/test/dir-list.test.js @@ -12,8 +12,11 @@ const fastifyStatic = require('..') const helper = { arrange: function (t, options, f) { + return helper.arrangeModule(t, options, fastifyStatic, f) + }, + arrangeModule: function (t, options, mock, f) { const fastify = Fastify() - fastify.register(fastifyStatic, options) + fastify.register(mock, options) t.teardown(fastify.close.bind(fastify)) fastify.listen(0, err => { t.error(err) @@ -237,6 +240,86 @@ t.test('dir list html format', t => { } }) +t.test('dir list html format - stats', t => { + t.plan(7) + + const options1 = { + root: path.join(__dirname, '/static'), + prefix: '/public', + index: false, + list: { + format: 'html', + render (dirs, files) { + t.ok(dirs.length > 0) + t.ok(files.length > 0) + + t.ok(dirs.every(every)) + t.ok(files.every(every)) + + function every (value) { + return value.stats && + value.stats.atime && + !value.extendedInfo + } + } + } + } + + const route = '/public/' + + helper.arrange(t, options1, (url) => { + simple.concat({ + method: 'GET', + url: url + route + }, (err, response, body) => { + t.error(err) + t.equal(response.statusCode, 200) + }) + }) +}) + +t.test('dir list html format - extended info', t => { + t.plan(4) + + const route = '/public/' + + const options = { + root: path.join(__dirname, '/static'), + prefix: '/public', + index: false, + list: { + format: 'html', + extendedFolderInfo: true, + render (dirs, files) { + t.test('dirs', t => { + t.plan(dirs.length * 7) + + for (const value of dirs) { + t.ok(value.extendedInfo) + + t.equal(typeof value.extendedInfo.fileCount, 'number') + t.equal(typeof value.extendedInfo.totalFileCount, 'number') + t.equal(typeof value.extendedInfo.folderCount, 'number') + t.equal(typeof value.extendedInfo.totalFolderCount, 'number') + t.equal(typeof value.extendedInfo.totalSize, 'number') + t.equal(typeof value.extendedInfo.lastModified, 'number') + } + }) + } + } + } + + helper.arrange(t, options, (url) => { + simple.concat({ + method: 'GET', + url: url + route + }, (err, response, body) => { + t.error(err) + t.equal(response.statusCode, 200) + }) + }) +}) + t.test('dir list json format', t => { t.plan(2) @@ -270,6 +353,92 @@ t.test('dir list json format', t => { }) }) +t.test('dir list json format - extended info', t => { + t.plan(2) + + const options = { + root: path.join(__dirname, '/static'), + prefix: '/public', + prefixAvoidTrailingSlash: true, + list: { + format: 'json', + names: ['index', 'index.json', '/'], + extendedFolderInfo: true, + jsonFormat: 'extended' + + } + } + const routes = ['/public/shallow/'] + + helper.arrange(t, options, (url) => { + for (const route of routes) { + t.test(route, t => { + t.plan(5) + simple.concat({ + method: 'GET', + url: url + route + }, (err, response, body) => { + t.error(err) + t.equal(response.statusCode, 200) + const bodyObject = JSON.parse(body.toString()) + t.equal(bodyObject.dirs[0].name, 'empty') + t.equal(typeof bodyObject.dirs[0].stats.atime, 'string') + t.equal(typeof bodyObject.dirs[0].extendedInfo.totalSize, 'number') + }) + }) + } + }) +}) + +t.test('dir list - url parameter format', t => { + t.plan(13) + + const options = { + root: path.join(__dirname, '/static'), + prefix: '/public', + index: false, + list: { + format: 'html', + render (dirs, files) { + return 'html' + } + } + } + const route = '/public/' + + helper.arrange(t, options, (url) => { + simple.concat({ + method: 'GET', + url: url + route + }, (err, response, body) => { + t.error(err) + t.equal(response.statusCode, 200) + t.equal(body.toString(), 'html') + t.ok(response.headers['content-type'].includes('text/html')) + }) + + simple.concat({ + method: 'GET', + url: url + route + '?format=html' + }, (err, response, body) => { + t.error(err) + t.equal(response.statusCode, 200) + t.equal(body.toString(), 'html') + t.ok(response.headers['content-type'].includes('text/html')) + }) + + simple.concat({ + method: 'GET', + url: url + route + '?format=json' + }, (err, response, body) => { + t.error(err) + t.equal(response.statusCode, 200) + t.ok(body.toString()) + t.ok(response.headers['content-type'].includes('application/json')) + }) + }) +}) + t.test('dir list on empty dir', t => { t.plan(2) @@ -387,3 +556,42 @@ t.test('serve a non existent dir and get error', t => { }) }) }) + +t.test('dir list error', t => { + t.plan(7) + + const options = { + root: path.join(__dirname, '/static'), + prefix: '/public', + prefixAvoidTrailingSlash: true, + index: false, + list: { + format: 'html', + names: ['index', 'index.htm'], + render: () => '' + } + } + + const errorMessage = 'mocking send' + const dirList = require('../lib/dirList') + dirList.send = async () => { throw new Error(errorMessage) } + + const mock = t.mock('..', { + '../lib/dirList.js': dirList + }) + + const routes = ['/public/', '/public/index.htm'] + + helper.arrangeModule(t, options, mock, (url) => { + for (const route of routes) { + simple.concat({ + method: 'GET', + url: url + route + }, (err, response, body) => { + t.error(err) + t.equal(JSON.parse(body.toString()).message, errorMessage) + t.equal(response.statusCode, 500) + }) + } + }) +})