feat: add ip/domain blacklist

.gitignore

@@ -3,4 +3,6 @@
 node_modules/
 coverage/
 dist/
-config/local.cjs
+config/local.cjs
+DOMAIN_BLACKLIST.json
+IP_BLACKLIST.json

package.json

@@ -13,11 +13,12 @@
     "@koa/router": "^10.1.1",
     "@socket.io/redis-adapter": "^7.1.0",
     "@types/koa-response-time": "^2.1.2",
+    "@types/validator": "^13.7.2",
     "any-ascii": "^0.3.0",
     "config": "^3.3.7",
     "countries-list": "^2.6.1",
     "crypto-random-string": "^4.0.0",
-    "got": "^12.0.1",
+    "got": "^12.0.3",
     "gunzip-maybe": "^1.4.2",
     "http-errors": "^2.0.0",
     "joi": "^17.6.0",
@@ -32,6 +33,7 @@
     "request-ip": "^2.1.3",
     "socket.io": "^4.4.1",
     "throng": "^5.0.0",
+    "validator": "^13.7.0",
     "winston": "^3.6.0"
   },
   "devDependencies": {
@@ -68,11 +70,12 @@
     "utf-8-validate": "^5.0.8"
   },
   "scripts": {
-    "build": "tsc",
+    "build": "tsc && npm run blacklist",
     "start": "NODE_ENV=production node dist/index.js",
     "dev": "NODE_ENV=development node dist/index.js",
     "lint": "xo",
     "lint:fix": "xo --fix",
+    "blacklist": "node dist/lib/malware/download.js",
     "test": "NODE_ENV=test mocha",
     "clean": "rimraf coverage",
     "test:coverage": "npm run clean && c8 mocha",

src/lib/malware/client.ts

@@ -0,0 +1,40 @@
+import {
+	updateList as updateIpList,
+	validate as validateIp,
+	validateSync as validateIpSync,
+} from './ip.js';
+
+import {
+	updateList as updateDomainList,
+	validate as validateDomain,
+	validateSync as validateDomainSync,
+} from './domain.js';
+
+export const updateList = async (): Promise<void> => {
+	await updateIpList();
+	await updateDomainList();
+};
+
+export const validate = async (target: string): Promise<boolean> => {
+	const ipCheck = await validateIp(target);
+	const domainCheck = await validateDomain(target);
+
+	return ipCheck && domainCheck;
+};
+
+export const validateSync = (target: string): boolean => {
+	const ipCheck = validateIpSync(target);
+	const domainCheck = validateDomainSync(target);
+
+	return ipCheck && domainCheck;
+};
+
+export const joiValidate = (value: string): string | Error => {
+	const isValid = validateSync(value);
+
+	if (!isValid) {
+		throw new Error('any.blacklisted');
+	}
+
+	return String(value);
+};

src/lib/malware/domain.ts

@@ -0,0 +1,59 @@
+import {writeFile, readFile} from 'node:fs/promises';
+import {readFileSync} from 'node:fs';
+import path from 'node:path';
+import got from 'got';
+import validator from 'validator';
+
+export const sourceList = [
+	'https://zonefiles.io/f/compromised/domains/full/',
+	'https://phishing.army/download/phishing_army_blocklist.txt',
+	'https://osint.digitalside.it/Threat-Intel/lists/latestdomains.txt',
+	'https://urlhaus.abuse.ch/downloads/hostfile/',
+];
+
+export const domainListPath = path.join(path.resolve(), 'DOMAIN_BLACKLIST.json');
+
+const isFulfilled = <T>(input: PromiseSettledResult<T>): input is PromiseFulfilledResult<T> =>
+	input.status === 'fulfilled';
+
+export const query = async (url: string): Promise<string[]> => {
+	const {body} = await got(url, {
+		timeout: {request: 5000},
+	});
+
+	const result = body.split(/r?\n?\s+/).filter(l => validator.isFQDN(l));
+
+	return result;
+};
+
+export const updateList = async (): Promise<void> => {
+	const result = await Promise.allSettled(sourceList.map(async source => query(source)));
+	const list = [...new Set(result.flatMap(r => isFulfilled(r) ? r.value : []))].map(d => d.toLowerCase());
+
+	await writeFile(domainListPath, JSON.stringify(list), {encoding: 'utf8'});
+};
+
+export const validate = async (target: string): Promise<boolean> => {
+	const data = await readFile(domainListPath, 'utf8');
+	const list = JSON.parse(data) as string[];
+
+	return !list.includes(target.toLowerCase());
+};
+
+export const validateSync = (target: string): boolean => {
+	const data = readFileSync(domainListPath, 'utf8');
+	const list = JSON.parse(data) as string[];
+
+	return !list.includes(target.toLowerCase());
+};
+
+export const joiValidate = (value: string): string | Error => {
+	const isValid = validateSync(value);
+
+	if (!isValid) {
+		throw new Error('domain.blacklisted');
+	}
+
+	return String(value);
+};
+

src/lib/malware/download.ts

@@ -0,0 +1,8 @@
+import {scopedLogger} from '../logger.js';
+import {updateList} from './client.js';
+
+const logger = scopedLogger('malware.blacklist');
+
+logger.info('updating malware blacklist');
+await updateList();
+logger.info('update complete');

src/lib/malware/ip.ts

@@ -0,0 +1,57 @@
+import {writeFile, readFile} from 'node:fs/promises';
+import {readFileSync} from 'node:fs';
+import path from 'node:path';
+import got from 'got';
+import validator from 'validator';
+
+export const sourceList = [
+	'https://osint.digitalside.it/Threat-Intel/lists/latestips.txt',
+	'https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level2.netset',
+	'https://raw.githubusercontent.com/stamparm/ipsum/master/levels/2.txt',
+];
+
+export const ipListPath = path.join(path.resolve(), 'IP_BLACKLIST.json');
+
+const isFulfilled = <T>(input: PromiseSettledResult<T>): input is PromiseFulfilledResult<T> =>
+	input.status === 'fulfilled';
+
+export const query = async (url: string): Promise<string[]> => {
+	const {body} = await got(url, {
+		timeout: {request: 5000},
+	});
+
+	const result = body.split(/r?\n/).filter(l => validator.isIP(l));
+
+	return result;
+};
+
+export const updateList = async (): Promise<void> => {
+	const result = await Promise.allSettled(sourceList.map(async source => query(source)));
+	const ipList = [...new Set(result.flatMap(r => isFulfilled(r) ? r.value : []))];
+
+	await writeFile(ipListPath, JSON.stringify(ipList), {encoding: 'utf8'});
+};
+
+export const validate = async (target: string): Promise<boolean> => {
+	const data = await readFile(ipListPath, 'utf8');
+	const ipList = JSON.parse(data) as string[];
+
+	return !ipList.includes(target);
+};
+
+export const validateSync = (target: string): boolean => {
+	const data = readFileSync(ipListPath, 'utf8');
+	const ipList = JSON.parse(data) as string[];
+
+	return !ipList.includes(target);
+};
+
+export const joiValidate = (value: string): string | Error => {
+	const isValid = validateSync(value);
+
+	if (!isValid) {
+		throw new Error('ip.blacklisted');
+	}
+
+	return String(value);
+};

src/measurement/schema/command-schema.ts

@@ -1,15 +1,18 @@
 import Joi from 'joi';
+import {joiValidate} from '../../lib/malware/client.js';
+import {joiValidate as joiValidateIp} from '../../lib/malware/ip.js';
+import {joiValidate as joiValidateDomain} from '../../lib/malware/domain.js';
 
 export const pingSchema = Joi.object({
 	type: Joi.string().valid('ping').insensitive().required(),
-	target: Joi.alternatives().try(Joi.string().ip(), Joi.string().domain()).required(),
+	target: Joi.alternatives().try(Joi.string().ip(), Joi.string().domain()).custom(joiValidate).required(),
 	packets: Joi.number().min(1).max(16).default(3),
 });
 
 export const tracerouteSchema = Joi.object({
 	type: Joi.string().valid('traceroute').insensitive().required(),
-	target: Joi.alternatives().try(Joi.string().ip(), Joi.string().domain()).required(),
-	protocol: Joi.string().valid('TCP', 'UDP', 'ICMP').insensitive().default('UDP'),
+	target: Joi.alternatives().try(Joi.string().ip(), Joi.string().domain()).custom(joiValidate).required(),
+	protocol: Joi.string().valid('TCP', 'UDP', 'ICMP').insensitive().default('ICMP'),
 	port: Joi.number().port().default(80),
 });
 
@@ -18,10 +21,10 @@ const allowedProtocols = ['UDP', 'TCP'];
 
 export const dnsSchema = Joi.object({
 	type: Joi.string().valid('dns').insensitive().required(),
-	target: Joi.string().domain().required(),
+	target: Joi.string().domain().custom(joiValidateDomain).required(),
 	query: Joi.object({
 		type: Joi.string().valid(...allowedTypes).insensitive().default('A'),
-		resolver: Joi.string().ip(),
+		resolver: Joi.string().ip().custom(joiValidateIp),
 		protocol: Joi.string().valid(...allowedProtocols).insensitive().default('UDP'),
 		port: Joi.number().default('53'),
 	}).default({}),