From 7675846f0d949fcac1e06c45581e11c8d4223ec2 Mon Sep 17 00:00:00 2001 From: jinjor Date: Fri, 4 Jan 2019 05:16:10 +0900 Subject: [PATCH] migrate from playground --- .travis.yml | 15 +++ README.md | 51 +++++++++ main.ts | 299 ++++++++++++++++++++++++++++++++++++++++++++++++ random-files.ts | 127 ++++++++++++++++++++ test.ts | 237 ++++++++++++++++++++++++++++++++++++++ tsconfig.json | 13 +++ 6 files changed, 742 insertions(+) create mode 100644 .travis.yml create mode 100644 README.md create mode 100644 main.ts create mode 100644 random-files.ts create mode 100644 test.ts create mode 100644 tsconfig.json diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..420a77e --- /dev/null +++ b/.travis.yml @@ -0,0 +1,15 @@ +language: minimal + +os: + - osx + - linux + +before_install: + - export PATH=$HOME/.deno/bin:$PATH + +install: + - curl -L https://deno.land/x/install/install.py | python + +script: + - deno -v + - deno test.ts --allow-write --allow-run diff --git a/README.md b/README.md new file mode 100644 index 0000000..023aa1c --- /dev/null +++ b/README.md @@ -0,0 +1,51 @@ +# Watch + +A pure deno file watcher. + +## Example + +```typescript +import watch from "https://raw.githubusercontent.com/jinjor/deno-watch/1.0.0/main.ts"; + +for await (const changes of watch("src")) { + console.log(changes.added); + console.log(changes.modified); + console.log(changes.deleted); +} +``` + +```typescript +const end = watch("src").start(changes => { + console.log(changes); +}); +``` + +## Options + +Written in the [source code](./index.ts). + +## Benchmark + +``` +test Benchmark +generated 10930 files. +[Add] +took 183ms to traverse 11232 files +took 147ms to traverse 11542 files +took 142ms to traverse 11845 files +[Modify] +took 139ms to traverse 11891 files +took 136ms to traverse 11891 files +took 154ms to traverse 11891 files +[Delete] +took 138ms to traverse 11608 files +took 134ms to traverse 11274 files +took 145ms to traverse 10960 files +... ok +``` + +Try yourself: + +``` +deno https://raw.githubusercontent.com/jinjor/deno-watch/master/test.ts --allow-write +``` diff --git a/main.ts b/main.ts new file mode 100644 index 0000000..eedb8ec --- /dev/null +++ b/main.ts @@ -0,0 +1,299 @@ +import { + readDir, + readlink, + lstatSync, + lstat, + readDirSync, + readlinkSync, + FileInfo +} from "deno"; + +/** The result of checking in one loop */ +export class Changes { + /** Paths of added files */ + added: string[] = []; + /** Paths of modified files */ + modified: string[] = []; + /** Paths of deleted files */ + deleted: string[] = []; + /** The time[posix ms] when the checking started. */ + startTime: number; + /** The time[posix ms] when the checking ended. */ + endTime: number; + /** Current file count */ + fileCount = 0; + /** added + modified + deleted */ + get length(): number { + return this.added.length + this.modified.length + this.deleted.length; + } + /** all changed paths */ + get all(): string[] { + return [...this.added, ...this.modified, ...this.deleted]; + } + /** The time[ms] took for checking. */ + get time() { + return this.endTime - this.startTime; + } +} + +/** Options */ +export interface Options { + /** The minimum interval[ms] of checking loop. + * The next checking can be delayed until user program ends. + * + * |<------------------ interval ----------------->|<--------------- + * |<-- checking -->| |<-- checking --> + * |<--- user program --->| + * + * + * |<---------- interval --------->| |<----------------------- + * |<-- checking -->| |<-- checking --> + * |<--- user program --->| + */ + interval?: number; + /** If true, watcher checks the symlinked files/directories too. */ + followSymlink?: boolean; + /** Ignores something like .gitignore, .vscode, etc. */ + ignoreDotFiles?: boolean; + /** Path to search in regex (ex. "\.(ts|css)$") */ + test?: RegExp | string; + /** Path to ignore in regex. */ + ignore?: RegExp | string; +} + +/** The watcher */ +export interface Watcher extends AsyncIterable { + start(callback: (changes: Changes) => Promise | void): () => void; +} + +const defaultOptions = { + interval: 1000, + followSymlink: false, + ignoreDotFiles: true, + test: /.*/, + ignore: /$^/ +}; + +/** + * Watch directories and detect changes. + * @example + * // Basic usage. + * for await (const changes of watch("src")) { + * console.log(changes.added); + * console.log(changes.modified); + * console.log(changes.deleted); + * } + * @example + * // Kill watcher from outside of the loop. + * const end = watch("src").start(changes => { + * console.log(changes); + * }); + * end(); + * @param dirs + * @param options + */ +export default function watch( + dirs: string | string[], + options?: Options +): Watcher { + const dirs_ = Array.isArray(dirs) ? dirs : [dirs]; + options = Object.assign({}, defaultOptions, options); + return { + [Symbol.asyncIterator]() { + return run(dirs_, options); + }, + start: function(callback: (changes: Changes) => Promise | void) { + const state = { + abort: false, + timeout: null + }; + (async () => { + for await (const changes of run(dirs_, options, state)) { + await callback(changes); + } + })(); + return () => { + state.abort = true; + if (state.timeout) { + clearTimeout(state.timeout); + } + }; + } + }; +} +async function* run( + dirs: string[], + options: Options, + state = { + abort: false, + timeout: null + } +) { + const { interval, followSymlink } = options; + const filter = makeFilter(options); + let lastStartTime = Date.now(); + let files = {}; + collect(files, dirs, followSymlink, filter); + + while (!state.abort) { + let waitTime = Math.max(0, interval - (Date.now() - lastStartTime)); + await new Promise(resolve => { + state.timeout = setTimeout(resolve, waitTime); + }); + state.timeout = null; + lastStartTime = Date.now(); + let changes = new Changes(); + changes.startTime = lastStartTime; + + changes.fileCount = Object.keys(files).length; + const newFiles = {}; + await detectChanges(files, newFiles, dirs, followSymlink, filter, changes); + files = newFiles; + + changes.endTime = Date.now(); + if (changes.length) { + yield changes; + } + } +} + +function makeFilter({ test, ignore, ignoreDotFiles }: Options) { + const testRegex = typeof test === "string" ? new RegExp(test) : test; + const ignoreRegex = typeof ignore === "string" ? new RegExp(ignore) : ignore; + return function filter(f: FileInfo, path: string) { + if (ignoreDotFiles) { + const splitted = path.split("/"); + const name = f.name || splitted[splitted.length - 1]; + if (name.charAt(0) === ".") { + return false; + } + } + if (f.isFile()) { + if (!testRegex.test(path)) { + return false; + } + if (ignoreRegex.test(path)) { + return false; + } + } + return true; + }; +} + +async function detectChanges( + prev: any, + curr: any, + dirs: string[], + followSymlink: boolean, + filter: (f: FileInfo, path: string) => boolean, + changes: Changes +): Promise { + await walk(prev, curr, dirs, followSymlink, filter, changes); + Array.prototype.push.apply(changes.deleted, Object.keys(prev)); +} + +async function walk( + prev: any, + curr: any, + files: (string | FileInfo)[], + followSymlink: boolean, + filter: (f: FileInfo, path: string) => boolean, + changes: Changes +): Promise { + const promises = []; + for (let f of files) { + let linkPath; + let path; + let info; + if (typeof f === "string") { + path = f; + info = await (followSymlink ? statTraverse : lstat)(f); + } else if (f.isSymlink() && followSymlink) { + linkPath = f.path; + info = await statTraverse(f.path); + path = info.path; + } else { + path = f.path; + info = f; + } + if (!path) { + throw new Error("path not found"); + } + if (!filter(info, linkPath || path)) { + continue; + } + if (info.isDirectory()) { + const files = await readDir(path); + promises.push(walk(prev, curr, files, followSymlink, filter, changes)); + } else if (info.isFile()) { + if (curr[path]) { + continue; + } + curr[path] = info.modified || info.created; + if (!prev[path]) { + changes.added.push(path); + } else if (prev[path] < curr[path]) { + changes.modified.push(path); + } + delete prev[path]; + } + } + await Promise.all(promises); +} + +function collect( + all: any, + files: (string | FileInfo)[], + followSymlink: boolean, + filter: (f: FileInfo, path?: string) => boolean +): void { + for (let f of files) { + let linkPath; + let path; + let info; + if (typeof f === "string") { + path = f; + info = (followSymlink ? statTraverseSync : lstatSync)(f); + } else if (f.isSymlink() && followSymlink) { + linkPath = f.path; + path = readlinkSync(f.path); + info = statTraverseSync(path); + } else { + path = f.path; + info = f; + } + if (!path) { + throw new Error("path not found"); + } + if (!filter(info, linkPath || path)) { + continue; + } + if (info.isDirectory()) { + collect(all, readDirSync(path), followSymlink, filter); + } else if (info.isFile()) { + all[path] = info.modified || info.created; + } + } +} + +// Workaround for non-linux +async function statTraverse(path: string): Promise { + const info = await lstat(path); + if (info.isSymlink()) { + const targetPath = await readlink(path); + return statTraverse(targetPath); + } else { + info.path = path; + return info; + } +} +function statTraverseSync(path: string): FileInfo { + const info = lstatSync(path); + if (info.isSymlink()) { + const targetPath = readlinkSync(path); + return statTraverseSync(targetPath); + } else { + info.path = path; + return info; + } +} diff --git a/random-files.ts b/random-files.ts new file mode 100644 index 0000000..65a6103 --- /dev/null +++ b/random-files.ts @@ -0,0 +1,127 @@ +import { + removeAll, + makeTempDirSync, + writeFileSync, + removeSync, + symlinkSync, + mkdirSync, + removeAllSync, + run, + DenoError, + ErrorKind +} from "deno"; +import * as path from "https://deno.land/x/path/index.ts"; + +export function genName(pre = "", post = ""): string { + return pre + Math.floor(Math.random() * 100000) + post; +} + +export async function inTmpDir( + f: (dir: string) => Promise | void, + keepOnFailure = false +) { + await inTmpDirs( + 1, + async tmpDirs => { + await f(tmpDirs[0]); + }, + keepOnFailure + ); +} +export async function inTmpDirs( + count: number, + f: (dirs: string[]) => Promise | void, + keepOnFailure = false +) { + const tmpDirs = []; + for (let i = 0; i < count; i++) { + tmpDirs.push(makeTempDirSync()); + } + function cleanup() { + tmpDirs.forEach(d => { + try { + removeAllSync(d); + } catch (e) { + if (e instanceof DenoError && e.kind === ErrorKind.NotFound) { + // not a problem + } else { + console.error("WARN:", e.message); + } + } + }); + } + try { + await f(tmpDirs); + cleanup(); + } catch (e) { + if (!keepOnFailure) { + cleanup(); + } + throw e; + } +} +class F { + constructor(public path: string, public isDir: boolean) {} + modify() { + writeFileSync(this.path, new Uint8Array(0)); + } + remove() { + if (this.isDir) { + removeAll(this.path); + } else { + removeSync(this.path); + } + } +} +interface Options { + prefix?: string; + postfix?: string; + amount?: number; + isDir?: boolean; +} +const defaultOptions = { + prefix: "", + postfix: "", + amount: 1, + isDir: false +}; +export function genFiles(dir: string, options: Options = {}): F[] { + const amount = Math.max(options.amount, 1); + options = { ...defaultOptions, ...options }; + const files = []; + for (let i = 0; i < amount; i++) { + const filePath = path.join(dir, genName(options.prefix, options.postfix)); + if (options.isDir) { + mkdirSync(filePath); + } else { + writeFileSync(filePath, new Uint8Array(0)); + } + files.push(new F(filePath, options.isDir)); + } + return files; +} +export function genFile(dir: string, options: Options = {}): F { + return genFiles(dir, { ...options, amount: 1 })[0]; +} +export function genDirs(dir: string, options: Options = {}): F[] { + return genFiles(dir, { ...options, isDir: true }); +} +export function genDir(dir: string, options: Options = {}): F { + return genDirs(dir, { ...options, isDir: true, amount: 1 })[0]; +} +export function genLink( + dir: string, + pathToFile: string, + options: Options = {} +): F { + const linkPath = path.join(dir, genName(options.prefix, options.postfix)); + symlinkSync(pathToFile, linkPath); + return new F(linkPath, options.isDir); +} +export async function tree(...args: string[]): Promise { + const process = run({ + args: ["tree", ...args], + stdout: "inherit" + }); + await process.status(); +} diff --git a/test.ts b/test.ts new file mode 100644 index 0000000..094c501 --- /dev/null +++ b/test.ts @@ -0,0 +1,237 @@ +import { writeFile, remove } from "deno"; +import watch from "main.ts"; +import { test, assertEqual } from "https://deno.land/x/testing/testing.ts"; +import { + inTmpDir, + genFile, + genDir, + genLink, + tree, + inTmpDirs +} from "random-files.ts"; + +function delay(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} +function assertChanges(changes, a, m, d) { + try { + assertEqual(changes.added.length, a); + assertEqual(changes.modified.length, m); + assertEqual(changes.deleted.length, d); + } catch (e) { + console.log("expected:", `${a} ${m} ${d}`); + console.log("actual:", changes); + throw e; + } +} +test(async function Watch() { + await inTmpDir(async tmpDir => { + let changes = { added: [], modified: [], deleted: [] }; + const end = watch(tmpDir).start(changes_ => { + changes = changes_; + }); + try { + const f = genFile(tmpDir); + await delay(100); + assertChanges(changes, 0, 0, 0); + await delay(1200); + assertChanges(changes, 1, 0, 0); + f.modify(); + await delay(1200); + assertChanges(changes, 0, 1, 0); + f.remove(); + await delay(1200); + assertChanges(changes, 0, 0, 1); + } finally { + end(); + } + }); +}); +test(async function Symlink() { + await inTmpDirs(2, async ([tmpDir, anotherDir]) => { + let changes = { added: [], modified: [], deleted: [] }; + const end = watch(tmpDir, { + followSymlink: true + }).start(changes_ => { + changes = changes_; + }); + try { + { + const f = genFile(anotherDir); + const link = genLink(tmpDir, f.path); + await delay(1200); + assertChanges(changes, 1, 0, 0); + f.modify(); + await delay(1200); + assertChanges(changes, 0, 1, 0); + } + { + const f = genFile(anotherDir); + const link1 = genLink(anotherDir, f.path); + const link2 = genLink(tmpDir, link1.path); + const link3 = genLink(tmpDir, link2.path); + await delay(1200); + assertChanges(changes, 1, 0, 0); + f.modify(); + await delay(1200); + assertChanges(changes, 0, 1, 0); + } + { + const dir = genDir(anotherDir); + const f = genFile(dir.path); + const link = genLink(tmpDir, f.path); + await delay(1200); + assertChanges(changes, 1, 0, 0); + f.modify(); + await delay(1200); + assertChanges(changes, 0, 1, 0); + } + } catch (e) { + await tree(tmpDir); + await tree(anotherDir); + throw e; + } finally { + end(); + } + }); +}); + +test(async function dotFiles() { + await inTmpDir(async tmpDir => { + let changes = { added: [], modified: [], deleted: [] }; + const end = watch(tmpDir).start(changes_ => { + changes = changes_; + }); + try { + const f = genFile(tmpDir, { prefix: "." }); + await delay(1200); + assertChanges(changes, 0, 0, 0); + const link = genLink(tmpDir, f.path); + await delay(1200); + assertChanges(changes, 0, 0, 0); + const dir = genDir(tmpDir, { prefix: "." }); + genFile(dir.path); + await delay(1200); + assertChanges(changes, 0, 0, 0); + f.remove(); + dir.remove(); + assertChanges(changes, 0, 0, 0); + } finally { + end(); + } + }); +}); + +test(async function filter() { + await inTmpDir(async tmpDir => { + let result1 = []; + const end1 = watch(tmpDir).start(changes => { + result1 = result1.concat(changes.added); + }); + let result2 = []; + const end2 = watch(tmpDir, { test: ".ts$" }).start(changes => { + result2 = result2.concat(changes.added); + }); + let result3 = []; + const end3 = watch(tmpDir, { ignore: ".ts$" }).start(changes => { + result3 = result3.concat(changes.added); + }); + let result4 = []; + const end4 = watch(tmpDir, { test: ".(ts|css)$", ignore: ".css$" }).start( + changes => { + result4 = result4.concat(changes.added); + } + ); + try { + genFile(tmpDir, { postfix: ".ts" }); + genFile(tmpDir, { postfix: ".js" }); + genFile(tmpDir, { postfix: ".css" }); + await delay(1200); + assertEqual(result1.length, 3); + assertEqual(result2.length, 1); + assertEqual(result3.length, 2); + assertEqual(result4.length, 1); + } finally { + end1(); + end2(); + end3(); + end4(); + } + }); +}); + +test(async function WatchByGenerator() { + await inTmpDir(async tmpDir => { + setTimeout(async () => { + const f = genFile(tmpDir); + }, 100); + for await (const changes of watch(tmpDir)) { + assertChanges(changes, 1, 0, 0); + break; + } + }); +}); + +test(async function Benchmark() { + await inTmpDir(async tmpDir => { + const files = []; + await generateManyFiles(tmpDir, files); + console.log(`generated ${files.length} files.`); + const end = watch(tmpDir).start(result => { + console.log( + `took ${result.time}ms to traverse ${result.fileCount} files` + ); + }); + try { + console.log("[Add]"); + for (let i = 0; i < 1000; i++) { + await delay(2); + let fileName = files[Math.floor(Math.random() * files.length)]; + fileName = fileName + ".added"; + await writeFile(fileName, new Uint8Array(0)); + } + console.log("[Modify]"); + for (let i = 0; i < 1000; i++) { + await delay(2); + await writeFile( + files[Math.floor(Math.random() * files.length)], + new Uint8Array(0) + ); + } + console.log("[Delete]"); + for (let i = 0; i < 1000; i++) { + await delay(2); + const index = Math.floor(Math.random() * files.length); + const fileName = files[index]; + if (fileName) { + try { + await remove(fileName); + } catch (e) { + console.log("error"); + console.log(e); + } + } + files[index] = null; + } + } finally { + end(); + } + }); +}); + +const DEPTH = 7; +const FILE_PER_DIR = 10; +const DIR_PER_DIR = 3; +async function generateManyFiles(dir, files, depth = DEPTH) { + if (depth <= 0) { + return; + } + for (let i = 0; i < FILE_PER_DIR; i++) { + const f = genFile(dir); + files.push(f.path); + } + for (let i = 0; i < DIR_PER_DIR; i++) { + const d = genDir(dir); + await generateManyFiles(d.path, files, depth - 1); + } +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..1f18a17 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "baseUrl": ".", + "lib": ["es6", "es2018", "dom", "esnext.asynciterable"], + "target": "es6", + "module": "commonjs", + "paths": { + "deno": ["../../.deno/deno.d.ts"], + "http://*": ["../../.deno/deps/http/*"], + "https://*": ["../../.deno/deps/https/*"] + } + } +}