-
Notifications
You must be signed in to change notification settings - Fork 15
/
reduceFP.py
142 lines (124 loc) · 3.89 KB
/
reduceFP.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
from __future__ import division
import optparse
import Levenshtein
import getopt
import socket
import dpkt, dpkt.dns
import sys
import subprocess
import datetime
import re
import math
import socket
import pythonwhois
import time
import smtplib
import numpy
from os import walk
from dnslib import *
from collections import *
from wordsegment import segment
from optparse import OptionParser
from netaddr import IPNetwork, IPAddress
threshCnC=10 # the number of CnC domains mapped to the same IP
threshIP=2 # the number of IPs mapped to the same CnC domain
def outputCnCIP(outputCnCIPFile, singleDomainMultiIPsDict):
global threshIP
fp = open(outputCnCIPFile, 'w')
for CnC in singleDomainMultiIPsDict:
IPset=singleDomainMultiIPsDict[CnC]
if len(IPset)>=threshIP:
fp.write("%s:\t" % (CnC))
for IP in IPset:
fp.write("%s," % IP)
fp.write("\n")
fp.close()
def outputIPCnC(outputIPCnCFile, singleIPMultiDomainsDict):
global threshCnC
fp = open(outputIPCnCFile, 'w')
for IP in singleIPMultiDomainsDict:
CnCset=singleIPMultiDomainsDict[IP]
if len(CnCset)>=threshCnC:
fp.write("%s:\t" % (IP))
for CnC in CnCset:
fp.write("%s," % CnC)
fp.write("\n")
fp.close()
def reduceFP(inputFile, singleIPMultiDomainsDict, singleDomainMultiIPsDict, whitelistSet):
fp = open(inputFile, 'r')
for line in fp:
info = line.strip("\n").split(' ')
CnC=info[2].strip(",")
IP=info[4]
if checkWhitelist(CnC, whitelistSet) == True:
#sys.stdout.write("%s is ok\n" % CnC)
continue
#sys.stdout.write("%s, %s\n" % (CnC, IP))
if CnC not in singleDomainMultiIPsDict:
IPset=set()
IPset.add(IP)
singleDomainMultiIPsDict[CnC] = IPset
else:
IPset=singleDomainMultiIPsDict[CnC]
IPset.add(IP)
singleDomainMultiIPsDict[CnC] = IPset
if IP not in singleIPMultiDomainsDict:
CnCset=set()
CnCset.add(CnC)
singleIPMultiDomainsDict[IP] = CnCset
else:
CnCset=singleIPMultiDomainsDict[IP]
CnCset.add(CnC)
singleIPMultiDomainsDict[IP] = CnCset
fp.close()
def loadWhitelist(whitelistFile, whitelistSet):
fp = open(whitelistFile, 'r')
for line in fp:
info = line.strip("\n")
#sys.stdout.write("%s\n" % info)
whitelistSet.add(info)
def checkWhitelist(domain, whitelistSet):
for whiteDomain in whitelistSet:
if whiteDomain in domain:
return True
return False
def main(argv) :
whitelistFile = ""
whitelistSet = set()
IPtoDomainDict = dict()
DomaintoIPDict = dict()
singleIPMultiDomainsDict = dict()
singleDomainMultiIPsDict = dict()
inputFile = ""
outputIPCnCFile = ""
outputCnCIPFile = ""
parser = optparse.OptionParser()
parser.add_option("-w", "--whitelist", action="store", type="string", dest="whitelistFile", help="specify the file that contains whitelist domains")
parser.add_option("-i", "--input", action="store", type="string", dest="inputFile", help="specify the input file")
parser.add_option("-o", "--outputIPCnC", action="store", type="string", dest="outputIPCnCFile", help="specify the output file")
parser.add_option("-c", "--outputCnCIP", action="store", type="string", dest="outputCnCIPFile", help="specify the output file")
(options, args) = parser.parse_args()
# load required files
if options.inputFile:
inputFile = options.inputFile
else:
parser.error("input file not given, use -i")
if options.outputIPCnCFile:
outputIPCnCFile = options.outputIPCnCFile
else:
parser.error("output file not given, use -o")
if options.outputCnCIPFile:
outputCnCIPFile = options.outputCnCIPFile
else:
parser.error("output file not given, use -c")
if options.whitelistFile:
whitelistFile = options.whitelistFile
loadWhitelist(whitelistFile, whitelistSet)
else:
parser.error("whitelist file not given, use -w")
reduceFP(inputFile, singleIPMultiDomainsDict, singleDomainMultiIPsDict, whitelistSet)
outputIPCnC(outputIPCnCFile, singleIPMultiDomainsDict)
outputCnCIP(outputCnCIPFile, singleDomainMultiIPsDict)
return
if __name__ == "__main__" :
main(sys.argv[1:])