Extended dir-list information (#241)

* -refactored dirlist to use promises
-added stats and more info to dir list

* -added lastModified

* -added format url parameter

* -added docs

* -fixed compatibility with node 10

* -removed unnecessary request
-used p-map

* -replaced p-map with p-limit

* -removed logging

* -added error catch

* -improved test

README.md

@@ -222,6 +222,14 @@ Options: `html`, `json`
 
 Directory list can be also in `html` format; in that case, `list.render` function is required.
 
+You can override the option with URL parameter `format`. Options are `html` and `json`.
+
+```bash
+GET .../public/assets?format=json
+```
+
+will return the response as json independent of `list.format`.
+
 **Example:**
 
 ```js
@@ -300,6 +308,79 @@ GET .../public/index
 GET .../public/index.json
 ```
 
+#### `list.extendedFolderInfo`
+
+Default: `undefined`
+
+If `true` some extended information for folders will be accessible in `list.render` and in the json response.
+
+```js
+render(dirs, files) {
+  const dir = dirs[0];
+  dir.fileCount // number of files in this folder
+  dir.totalFileCount // number of files in this folder (recursive)
+  dir.folderCount // number of folders in this folder
+  dir.totalFolderCount // number of folders in this folder (recursive)
+  dir.totalSize // size of all files in this folder (recursive)
+  dir.lastModified // most recent last modified timestamp of all files in this folder (recursive)
+}
+```
+
+Warning: This will slightly decrease the performance, especially for deeply nested file structures.
+
+#### `list.jsonFormat`
+
+Default: `names`
+
+Options: `names`, `extended`
+
+This option determines the output format when `json` is selected.
+
+`names`:
+```json
+{
+  "dirs": [
+    "dir1",
+    "dir2"
+  ],
+  "files": [
+    "file1.txt",
+    "file2.txt"
+  ]
+}
+```
+
+`extended`:
+```json
+{
+  "dirs": [
+    {
+      "name": "dir1",
+      "stats": {
+        "dev": 2100,
+        "size": 4096,
+        ...
+      },
+      "extendedInfo": {
+        "fileCount": 4,
+        "totalSize": 51233,
+        ...
+      }
+    }
+  ],
+  "files": [
+    {
+      "name": "file1.txt",
+      "stats": {
+        "dev": 2200,
+        "size": 554,
+        ...
+      }
+    }
+  ]
+}
+```
+
 #### `preCompressed`
 
 Default: `false`

index.d.ts

@@ -3,6 +3,7 @@
 /// <reference types="node" />
 
 import { FastifyPluginCallback, FastifyReply } from 'fastify';
+import { Stats } from 'fs';
 
 declare module "fastify" {
   interface FastifyReply {
@@ -15,14 +16,26 @@ declare module "fastify" {
   }
 }
 
+interface ExtendedInformation {
+  fileCount: number;
+  totalFileCount: number;
+  folderCount: number;
+  totalFolderCount: number;
+  totalSize: number;
+  lastModified: number;
+}
+
 interface ListDir {
   href: string;
   name: string;
+  stats: Stats;
+  extendedInfo?: ExtendedInformation;
 }
 
 interface ListFile {
   href: string;
   name: string;
+  stats: Stats;
 }
 
 interface ListRender {
@@ -33,6 +46,8 @@ interface ListOptions {
   format: 'json' | 'html';
   names: string[];
   render: ListRender;
+  extendedFolderInfo?: boolean;
+  jsonFormat?: 'names' | 'extended';
 }
 
 // Passed on to `send`

index.js

@@ -142,12 +142,13 @@ async function fastifyStatic (fastify, opts) {
 
     stream.on('directory', function (_, path) {
       if (opts.list) {
-        return dirList.send({
+        dirList.send({
           reply,
           dir: path,
           options: opts.list,
           route: pathname
-        })
+        }).catch((err) => reply.send(err))
+        return
       }
 
       if (opts.redirect === true) {
@@ -161,7 +162,13 @@ async function fastifyStatic (fastify, opts) {
       if (err.code === 'ENOENT') {
         // if file exists, send real file, otherwise send dir list if name match
         if (opts.list && dirList.handle(pathname, opts.list)) {
-          return dirList.send({ reply, dir: dirList.path(opts.root, pathname), options: opts.list, route: pathname })
+          dirList.send({
+            reply,
+            dir: dirList.path(opts.root, pathname),
+            options: opts.list,
+            route: pathname
+          }).catch((err) => reply.send(err))
+          return
         }
 
         // root paths left to try?

lib/dirList.js

@@ -1,7 +1,8 @@
 'use strict'
 
 const path = require('path')
-const fs = require('fs')
+const fs = require('fs').promises
+const pLimit = require('p-limit')
 
 const dirList = {
   /**
@@ -10,36 +11,84 @@ const dirList = {
    * @param {function(error, entries)} callback
    * note: can't use glob because don't get error on non existing dir
    */
-  list: function (dir, callback) {
+  list: async function (dir, options) {
     const entries = { dirs: [], files: [] }
-    fs.readdir(dir, (err, files) => {
-      if (err) {
-        return callback(err)
-      }
-      if (files.length < 1) {
-        callback(null, entries)
+    const files = await fs.readdir(dir)
+    if (files.length < 1) {
+      return entries
+    }
+
+    const limit = pLimit(4)
+    await Promise.all(files.map(filename => limit(async () => {
+      let stats
+      try {
+        stats = await fs.stat(path.join(dir, filename))
+      } catch (error) {
         return
       }
-      let j = 0
-      for (let i = 0; i < files.length; i++) {
-        const filename = files[i]
-        fs.stat(path.join(dir, filename), (err, file) => {
-          if (!err) {
-            if (file.isDirectory()) {
-              entries.dirs.push(filename)
-            } else {
-              entries.files.push(filename)
-            }
+      const entry = { name: filename, stats }
+      if (stats.isDirectory()) {
+        if (options.extendedFolderInfo) {
+          entry.extendedInfo = await getExtendedInfo(path.join(dir, filename))
+        }
+        entries.dirs.push(entry)
+      } else {
+        entries.files.push(entry)
+      }
+    })))
+
+    async function getExtendedInfo (folderPath) {
+      const depth = folderPath.split(path.sep).length
+      let totalSize = 0
+      let fileCount = 0
+      let totalFileCount = 0
+      let folderCount = 0
+      let totalFolderCount = 0
+      let lastModified = 0
+
+      async function walk (dir) {
+        const files = await fs.readdir(dir)
+        const limit = pLimit(4)
+        await Promise.all(files.map(filename => limit(async () => {
+          const filePath = path.join(dir, filename)
+          let stats
+          try {
+            stats = await fs.stat(filePath)
+          } catch (error) {
+            return
           }
 
-          if (j++ >= files.length - 1) {
-            entries.dirs.sort()
-            entries.files.sort()
-            callback(null, entries)
+          if (stats.isDirectory()) {
+            totalFolderCount++
+            if (filePath.split(path.sep).length === depth + 1) {
+              folderCount++
+            }
+            await walk(filePath)
+          } else {
+            totalSize += stats.size
+            totalFileCount++
+            if (filePath.split(path.sep).length === depth + 1) {
+              fileCount++
+            }
+            lastModified = Math.max(lastModified, stats.mtimeMs)
           }
-        })
+        })))
       }
-    })
+
+      await walk(folderPath)
+      return {
+        totalSize,
+        fileCount,
+        totalFileCount,
+        folderCount,
+        totalFolderCount,
+        lastModified
+      }
+    }
+
+    entries.dirs.sort((a, b) => a.name.localeCompare(b.name))
+    entries.files.sort((a, b) => a.name.localeCompare(b.name))
+    return entries
   },
 
   /**
@@ -49,33 +98,41 @@ const dirList = {
    * @param {ListOptions} options
    * @param {string} route request route
    */
-  send: function ({ reply, dir, options, route }) {
-    dirList.list(dir, (err, entries) => {
-      if (err) {
-        reply.callNotFound()
-        return
-      }
+  send: async function ({ reply, dir, options, route }) {
+    let entries
+    try {
+      entries = await dirList.list(dir, options)
+    } catch (error) {
+      return reply.callNotFound()
+    }
+    const format = reply.request.query.format || options.format
+    if (format !== 'html') {
+      if (options.jsonFormat !== 'extended') {
+        const nameEntries = { dirs: [], files: [] }
+        entries.dirs.forEach(entry => nameEntries.dirs.push(entry.name))
+        entries.files.forEach(entry => nameEntries.files.push(entry.name))
 
-      if (options.format !== 'html') {
+        reply.send(nameEntries)
+      } else {
         reply.send(entries)
-        return
       }
+      return
+    }
 
-      const html = options.render(
-        entries.dirs.map(entry => dirList.htmlInfo(entry, route)),
-        entries.files.map(entry => dirList.htmlInfo(entry, route)))
-      reply.type('text/html').send(html)
-    })
+    const html = options.render(
+      entries.dirs.map(entry => dirList.htmlInfo(entry, route)),
+      entries.files.map(entry => dirList.htmlInfo(entry, route)))
+    reply.type('text/html').send(html)
   },
 
   /**
    * provide the html information about entry and route, to get name and full route
-   * @param {string} entry file or dir name
+   * @param entry file or dir name and stats
    * @param {string} route request route
    * @return {ListFile}
    */
   htmlInfo: function (entry, route) {
-    return { href: path.join(path.dirname(route), entry).replace(/\\/g, '/'), name: entry }
+    return { href: path.join(path.dirname(route), entry.name).replace(/\\/g, '/'), name: entry.name, stats: entry.stats, extendedInfo: entry.extendedInfo }
   },
 
   /**

package.json

@@ -33,6 +33,7 @@
     "encoding-negotiator": "^2.0.1",
     "fastify-plugin": "^3.0.0",
     "glob": "^7.1.4",
+    "p-limit": "^3.1.0",
     "readable-stream": "^3.4.0",
     "send": "^0.17.1"
   },