remove not valid TLD and second level TLD domains

FGRibreau · Oct 14, 2019 · 6a4940c · 6a4940c
1 parent c9a9696
commit 6a4940c
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 63 deletions.
diff --git a/lib/clean.js b/lib/clean.js
@@ -0,0 +1,18 @@
+'use strict';
+
+var fs = require('fs');
+var p = require('path');
+var psl = require('psl');
+
+var path = p.resolve(__dirname, '../list.txt');
+var rawList = fs.readFileSync(path)
+                .toString()
+                .trim()
+                .split("\n");
+
+var cleanedList = rawList.filter(function(domain){
+  var parsed = psl.parse(domain); 
+  return parsed.listed;
+});
+
+fs.writeFileSync(path, cleanedList.join("\n"))