From 39cf9d2743a83b467df05264601407000a17598b Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Tue, 29 Dec 2020 14:05:28 -0800 Subject: [PATCH] feat: support directory listings (#225) In addition to providing the directory listing flag, this swaps the underlying HTTP server from `serve-static` to `serve-handler`. There should be no user facing changes for that swap. --- README.md | 6 ++++ package.json | 6 ++-- src/cli.ts | 6 ++++ src/config.ts | 1 + src/index.ts | 15 ++++++--- src/server.ts | 35 ++++++++++++--------- test/fixtures/config/linkinator.config.json | 3 +- test/fixtures/directoryIndex/README.md | 1 + test/fixtures/directoryIndex/dir1/dir1.md | 1 + test/fixtures/directoryIndex/dir2/dir2.md | 0 test/test.ts | 17 ++++++++++ 11 files changed, 67 insertions(+), 24 deletions(-) create mode 100644 test/fixtures/directoryIndex/README.md create mode 100644 test/fixtures/directoryIndex/dir1/dir1.md create mode 100644 test/fixtures/directoryIndex/dir2/dir2.md diff --git a/README.md b/README.md index b715d44..9cf5646 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,10 @@ $ linkinator LOCATIONS [ --arguments ] --config Path to the config file to use. Looks for `linkinator.config.json` by default. + --directory-listing + Include an automatic directory index file when linking to a directory. + Defaults to 'false'. + --format, -f Return the data in CSV or JSON format. @@ -138,6 +142,7 @@ All options are optional. It should look like this: "concurrency": 100, "timeout": 0, "markdown": true, + "directoryListing": true, "skip": "www.googleapis.com" } ``` @@ -161,6 +166,7 @@ where the server is started. Defaults to the path passed in `path`. - `timeout` (number) - By default, requests made by linkinator do not time out (or follow the settings of the OS). This option (in milliseconds) will fail requests after the configured amount of time. - `markdown` (boolean) - Automatically parse and scan markdown if scanning from a location on disk. - `linksToSkip` (array | function) - An array of regular expression strings that should be skipped, OR an async function that's called for each link with the link URL as its only argument. Return a Promise that resolves to `true` to skip the link or `false` to check it. +- `directoryListing` (boolean) - Automatically serve a static file listing page when serving a directory. Defaults to `false`. #### linkinator.LinkChecker() Constructor method that can be used to create a new `LinkChecker` instance. This is particularly useful if you want to receive events as the crawler crawls. Exposes the following events: diff --git a/package.json b/package.json index 86436d5..2586c70 100644 --- a/package.json +++ b/package.json @@ -23,27 +23,25 @@ "dependencies": { "chalk": "^4.0.0", "cheerio": "^1.0.0-rc.2", - "finalhandler": "^1.1.2", "gaxios": "^4.0.0", "glob": "^7.1.6", "jsonexport": "^3.0.0", "marked": "^1.2.5", "meow": "^8.0.0", "p-queue": "^6.2.1", - "serve-static": "^1.14.1", + "serve-handler": "^6.1.3", "server-destroy": "^1.0.1", "update-notifier": "^5.0.0" }, "devDependencies": { "@types/chai": "^4.2.7", "@types/cheerio": "0.22.22", - "@types/finalhandler": "^1.1.0", "@types/glob": "^7.1.3", "@types/marked": "^1.2.0", "@types/meow": "^5.0.0", "@types/mocha": "^8.0.0", "@types/node": "^12.7.12", - "@types/serve-static": "^1.13.8", + "@types/serve-handler": "^6.1.0", "@types/server-destroy": "^1.0.0", "@types/sinon": "^9.0.0", "@types/update-notifier": "^5.0.0", diff --git a/src/cli.ts b/src/cli.ts index a842c77..e7cefde 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -35,6 +35,10 @@ const cli = meow( --config Path to the config file to use. Looks for \`linkinator.config.json\` by default. + --directory-listing + Include an automatic directory index file when linking to a directory. + Defaults to 'false'. + --format, -f Return the data in CSV or JSON format. @@ -80,6 +84,7 @@ const cli = meow( markdown: {type: 'boolean'}, serverRoot: {type: 'string'}, verbosity: {type: 'string'}, + directoryListing: {type: 'boolean'}, }, booleanDefault: undefined, } @@ -126,6 +131,7 @@ async function main() { markdown: flags.markdown, concurrency: Number(flags.concurrency), serverRoot: flags.serverRoot, + directoryListing: flags.directoryListing, }; if (flags.skip) { if (typeof flags.skip === 'string') { diff --git a/src/config.ts b/src/config.ts index 9de62c9..d37fee6 100644 --- a/src/config.ts +++ b/src/config.ts @@ -13,6 +13,7 @@ export interface Flags { timeout?: number; markdown?: boolean; serverRoot?: string; + directoryListing?: boolean; } export async function getConfig(flags: Flags) { diff --git a/src/index.ts b/src/index.ts index 2e896da..2f222ba 100644 --- a/src/index.ts +++ b/src/index.ts @@ -25,6 +25,7 @@ export interface CheckOptions { markdown?: boolean; linksToSkip?: string[] | ((link: string) => Promise); serverRoot?: string; + directoryListing?: boolean; } export enum LinkState { @@ -82,11 +83,12 @@ export class LinkChecker extends EventEmitter { const hasHttpPaths = options.path.find(x => x.startsWith('http')); if (!hasHttpPaths) { const port = options.port || 5000 + Math.round(Math.random() * 1000); - server = await startWebServer( - options.serverRoot!, + server = await startWebServer({ + root: options.serverRoot!, port, - options.markdown - ); + markdown: options.markdown, + directoryListing: options.directoryListing, + }); for (let i = 0; i < options.path.length; i++) { if (options.path[i].startsWith('/')) { options.path[i] = options.path[i].slice(1); @@ -150,6 +152,11 @@ export class LinkChecker extends EventEmitter { options.path = [options.path]; } + // disable directory listings by default + if (options.directoryListing === undefined) { + options.directoryListing = false; + } + // Ensure we do not mix http:// and file system paths. The paths passed in // must all be filesystem paths, or HTTP paths. let isUrlType: boolean | undefined = undefined; diff --git a/src/server.ts b/src/server.ts index 094737b..742805f 100644 --- a/src/server.ts +++ b/src/server.ts @@ -3,34 +3,36 @@ import * as path from 'path'; import * as util from 'util'; import * as fs from 'fs'; import * as marked from 'marked'; -import finalhandler = require('finalhandler'); -import serveStatic = require('serve-static'); +import serve = require('serve-handler'); import enableDestroy = require('server-destroy'); const readFile = util.promisify(fs.readFile); +export interface WebServerOptions { + // The local path that should be mounted as a static web server + root: string; + // The port on which to start the local web server + port: number; + // If markdown should be automatically compiled and served + markdown?: boolean; + // Should directories automatically serve an inde page + directoryListing?: boolean; +} + /** * Spin up a local HTTP server to serve static requests from disk - * @param root The local path that should be mounted as a static web server - * @param port The port on which to start the local web server - * @param markdown If markdown should be automatically compiled and served * @private * @returns Promise that resolves with the instance of the HTTP server */ -export async function startWebServer( - root: string, - port: number, - markdown?: boolean -) { +export async function startWebServer(options: WebServerOptions) { return new Promise((resolve, reject) => { - const serve = serveStatic(root); const server = http .createServer(async (req, res) => { const pathParts = req.url!.split('/').filter(x => !!x); if (pathParts.length > 0) { const ext = path.extname(pathParts[pathParts.length - 1]); - if (markdown && ext.toLowerCase() === '.md') { - const filePath = path.join(path.resolve(root), req.url!); + if (options.markdown && ext.toLowerCase() === '.md') { + const filePath = path.join(path.resolve(options.root), req.url!); const data = await readFile(filePath, {encoding: 'utf-8'}); const result = marked(data, {gfm: true}); res.writeHead(200, { @@ -40,9 +42,12 @@ export async function startWebServer( return; } } - return serve(req, res, finalhandler(req, res) as () => void); + return serve(req, res, { + public: options.root, + directoryListing: options.directoryListing, + }); }) - .listen(port, () => resolve(server)) + .listen(options.port, () => resolve(server)) .on('error', reject); enableDestroy(server); }); diff --git a/test/fixtures/config/linkinator.config.json b/test/fixtures/config/linkinator.config.json index eba5c3f..a4cc630 100644 --- a/test/fixtures/config/linkinator.config.json +++ b/test/fixtures/config/linkinator.config.json @@ -3,5 +3,6 @@ "recurse": true, "silent": true, "concurrency": 17, - "skip": "🌳" + "skip": "🌳", + "directoryListing": false } diff --git a/test/fixtures/directoryIndex/README.md b/test/fixtures/directoryIndex/README.md new file mode 100644 index 0000000..d705b0c --- /dev/null +++ b/test/fixtures/directoryIndex/README.md @@ -0,0 +1 @@ +This has links to a [directory with files](dir1/) and an [empty directory](dir2/). diff --git a/test/fixtures/directoryIndex/dir1/dir1.md b/test/fixtures/directoryIndex/dir1/dir1.md new file mode 100644 index 0000000..df23bb9 --- /dev/null +++ b/test/fixtures/directoryIndex/dir1/dir1.md @@ -0,0 +1 @@ +👋 diff --git a/test/fixtures/directoryIndex/dir2/dir2.md b/test/fixtures/directoryIndex/dir2/dir2.md new file mode 100644 index 0000000..e69de29 diff --git a/test/test.ts b/test/test.ts index ebde3c7..d9eaf01 100644 --- a/test/test.ts +++ b/test/test.ts @@ -480,4 +480,21 @@ describe('linkinator', () => { scope.done(); assert.strictEqual(results.links.length, 2); }); + + it('should support directory index', async () => { + const results = await check({ + path: 'test/fixtures/directoryIndex/README.md', + directoryListing: true, + }); + assert.ok(results.passed); + assert.strictEqual(results.links.length, 3); + }); + + it('should disabling directory index by default', async () => { + const results = await check({ + path: 'test/fixtures/directoryIndex/README.md', + }); + assert.ok(!results.passed); + assert.strictEqual(results.links.length, 3); + }); });