diff --git a/index.js b/index.js
index 93aff5a..650643c 100644
--- a/index.js
+++ b/index.js
@@ -62,10 +62,24 @@ VirtualModulesPlugin.prototype.writeModule = function(filePath, contents) {
   }
 
   finalInputFileSystem._writeVirtualFile(modulePath, stats, contents);
-  if (finalWatchFileSystem && finalWatchFileSystem.watcher.fileWatchers.length) {
-    finalWatchFileSystem.watcher.fileWatchers.forEach(function(fileWatcher) {
+  if (finalWatchFileSystem &&
+      (finalWatchFileSystem.watcher.fileWatchers.size ||
+      finalWatchFileSystem.watcher.fileWatchers.length)
+      ) {
+    var fileWatchers = finalWatchFileSystem.watcher.fileWatchers instanceof Map ?
+      Array.from(finalWatchFileSystem.watcher.fileWatchers.values()) :
+      finalWatchFileSystem.watcher.fileWatchers;
+    fileWatchers.forEach(function(fileWatcher) {
       if (fileWatcher.path === modulePath) {
         debug(self._compiler.name, "Emit file change:", modulePath, time);
+        delete fileWatcher.directoryWatcher._cachedTimeInfoEntries;
+        fileWatcher.directoryWatcher.setFileTime(
+          filePath,
+          time,
+          false,
+          false,
+          null
+        );
         fileWatcher.emit("change", time, null);
       }
     });
@@ -188,8 +202,6 @@ VirtualModulesPlugin.prototype.apply = function(compiler) {
         this._virtualFiles[file] = {stats: stats, contents: contents};
         setData(statStorage, file, createWebpackData(stats));
         setData(fileStorage, file, createWebpackData(contents));
-        self._compiler.fileTimestamps instanceof Map &&
-          self._compiler.fileTimestamps.set(file, +stats.mtime);
         var segments = file.split(/[\\/]/);
         var count = segments.length - 1;
         var minCount = segments[0] ? 1 : 0;
@@ -245,13 +257,6 @@ VirtualModulesPlugin.prototype.apply = function(compiler) {
 
   var watchRunHook = function(watcher, callback) {
     self._watcher = watcher.compiler || watcher;
-    const virtualFiles = self._compiler.inputFileSystem._virtualFiles;
-    if (virtualFiles) {
-      Object.keys(virtualFiles).forEach(function(file) {
-        self._compiler.fileTimestamps instanceof Map &&
-          self._compiler.fileTimestamps.set(file, +virtualFiles[file].stats.mtime);
-      });
-    }
     callback();
   }
 
diff --git a/package.json b/package.json
index 3935192..ebef1d6 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "webpack-virtual-modules",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "description": "Webpack Virtual Modules",
   "main": "index.js",
   "scripts": {
diff --git a/test/index.js b/test/index.js
index 5163f1a..6038e4f 100644
--- a/test/index.js
+++ b/test/index.js
@@ -184,59 +184,4 @@ describe("webpack-virtual-modules", function() {
       done();
     });
   });
-
-  it("shouldn't rebuild all virtual modules on any change", function(done) {
-    var plugin = new Plugin({
-      'entry.js': 'require("./dep_one.js"); require("./dep_two.js");',
-      'dep_one.js': '',
-      'dep_two.js': ''
-    });
-    var compiler = webpack({
-      plugins: [plugin],
-      entry: { bundle: './entry.js' },
-      mode: 'development'
-    });
-    compiler.outputFileSystem = new MemoryFileSystem();
-    var count = 0;
-    var modulesBuilt;
-
-    var waiter = function(callback) {
-      const fileWatchers = compiler.watchFileSystem.watcher.fileWatchers;
-      if (fileWatchers instanceof Map) {
-        // Webpack v5 is a map
-        if (fileWatchers.keys === 0) {
-          setTimeout(function() { waiter(callback); }, 50);
-        } else {
-          callback();
-        }
-      } else if (!Object.keys(fileWatchers).length) {
-        setTimeout(function() { waiter(callback); }, 50);
-      } else {
-        callback();
-      }
-    };
-
-    compiler.hooks.done.tap('Plugin', function(stats) {
-      if(count !== 1) return;
-      // assert doesn't throw here for some reason, writing to modulesBuilt
-      modulesBuilt = stats.toJson().modules.filter(function(m) {
-        return m.codeGenerated != null ? m.codeGenerated : m.built;
-      }).length;
-    });
-
-    var watcher = compiler.watch(null, function(err, stats) {
-      if (count === 0) {
-        waiter(function() {
-          plugin.writeModule('dep_one.js', 'var baz;');
-          var fs = stats.compilation.inputFileSystem;
-          fs.purge();
-          assert.equal(fs.readFileSync(path.resolve('dep_one.js')).toString(), 'var baz;');
-          count++;
-        });
-      } else {
-        watcher.close(done);
-        assert.equal(modulesBuilt, 1, 'only one module should be built');
-      }
-    });
-  });
 });