Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 366 lines (306 sloc) 14.226 kb
4b22974 @urbanadventurer first commit
urbanadventurer authored
1 =begin
2 Copyright 2009, 2010 Andrew Horton
3
4 This file is part of WhatWeb.
5
6 WhatWeb is free software: you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation, either version 2 of the License, or
9 at your option) any later version.
10
11 WhatWeb is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with WhatWeb. If not, see <http://www.gnu.org/licenses/>.
18 =end
19
20 class Output
21 def initialize(f=STDOUT)
22 # if no f, output to STDOUT, if f is a filename then open it, if f is a file use it
23 @f = f if f.class == IO or f.class == File
aeb7432 @urbanadventurer All logs are now flushed/synced
urbanadventurer authored
24 @f=File.open(f,"a") if f.class == String
218a006 @urbanadventurer added mongo db logging, changed JSON output structure, added Charset …
urbanadventurer authored
25 @f.sync = true # we want flushed output
4b22974 @urbanadventurer first commit
urbanadventurer authored
26 end
27
28 def close
1487437 @urbanadventurer updated README
urbanadventurer authored
29 @f.close unless @f.class == IO
4b22974 @urbanadventurer first commit
urbanadventurer authored
30 end
31 end
32
33 class OutputFull < Output
34 def out(target, status, results)
35 @f.puts "Identifying: "+ target.to_s
36 @f.puts "HTTP-Status: "+ status.to_s
37 @f.puts results.pretty_inspect unless results.empty?
38 @f.puts
39 end
40 end
41
42 class OutputVerbose < Output
43 def out(target, status, results)
44 results.each do |plugin_name,plugin_results|
45 unless plugin_results.empty?
46 @f.print plugin_name + " " * (30- plugin_name.size )+ " => "
47 matches = plugin_results.map do |pr|
48 if pr[:name]
49 name_of_match = pr[:name]
50 else
51 name_of_match = [pr[:text],pr[:regexp].to_s,pr[:ghdb],pr[:md5],pr[:tagpattern]].compact.join("|")
52 end
53 stuff =[]
54 stuff << "certainty: #{pr[:certainty]}" if pr[:certainty] != 100
55 stuff << "version: #{pr[:version]}" if pr[:version]
56 stuff << "string: #{pr[:string]}" if pr[:string]
57 stuff << "model: #{pr[:model]}" if pr[:model]
58 stuff << "firmware: #{pr[:firmware]}" if pr[:firmware]
59 stuff << "modules: #{pr[:modules]}" if pr[:modules]
60 stuff << "accounts: #{pr[:accounts]}" if pr[:accounts]
b00f0af @urbanadventurer Changed :version_regexp_offset, :modules_regexp_offset, etc to :regex…
urbanadventurer authored
61 stuff << "filepath: #{pr[:filepath]}" if pr[:filepath]
4b22974 @urbanadventurer first commit
urbanadventurer authored
62 stuff << "url: #{pr[:url]}" if pr[:url]
63 name_of_match + ( !stuff.empty? ? " (" + stuff.join(",") +")" : "" )
64 end
65 puts matches.join(", ")
66 end
67 end
68 end
69 end
70
71 class OutputBrief < Output
72 # don't use colours if not to STDOUT
73 def out(target, status, results)
74 brief_results=[]
75
76 # sort results so plugins that are less important at a glance are last
77 last_plugins=%w| CSS MD5 Header-Hash Footer-Hash Div-Span-Structure Tag-Hash|
78 results=results.sort_by {|x,y| last_plugins.include?(x) ? 1 : 0 }
79
80
81 results.each do |plugin_name,plugin_results|
82 unless plugin_results.empty?
83 # important info in brief mode is version, type and ?
84 # what's the highest probability for the match?
28612fa @urbanadventurer Fixed bugs with Output plugins due to regular expressions being treat…
urbanadventurer authored
85 certainty = plugin_results.map {|x| x[:certainty] unless x[:certainty].class==Regexp }.compact.sort.uniq.last
86 version = plugin_results.map {|x| x[:version] unless x[:version].class==Regexp }.compact.sort.uniq.join(",")
87 string = plugin_results.map {|x| x[:string] unless x[:string].class==Regexp }.compact.sort.uniq.join(",")
4b22974 @urbanadventurer first commit
urbanadventurer authored
88 accounts = plugin_results.map {|x| [x[:account],x[:accounts] ] }.flatten.compact.sort.uniq.join(",")
28612fa @urbanadventurer Fixed bugs with Output plugins due to regular expressions being treat…
urbanadventurer authored
89 model = plugin_results.map {|x| x[:model] unless x[:model].class==Regexp }.compact.sort.uniq.join(",")
90 firmware = plugin_results.map {|x| x[:firmware] unless x[:firmware].class==Regexp }.compact.sort.uniq.join(",")
91 modules = plugin_results.map {|x| x[:modules] unless x[:modules].class==Regexp }.compact.sort.uniq.join(",")
92 filepath = plugin_results.map {|x| x[:filepath] unless x[:filepath].class==Regexp }.compact.sort.uniq.join(",")
b00f0af @urbanadventurer Changed :version_regexp_offset, :modules_regexp_offset, etc to :regex…
urbanadventurer authored
93
4b22974 @urbanadventurer first commit
urbanadventurer authored
94 # be more DRY
95 # if plugins have categories or tags this would be better, eg. all hash plugins are grey
96 if (@f == STDOUT and $use_colour=="auto") or ($use_colour=="always")
97 coloured_string = yellow(string)
98 coloured_string = cyan(string) if plugin_name == "HTTPServer"
99 coloured_string = dark_green(string) if plugin_name == "Title"
100 coloured_string = grey(string) if plugin_name == "MD5"
101 coloured_string = grey(string) if plugin_name == "Div-Span-Structure"
102 coloured_string = grey(string) if plugin_name == "Header-Hash"
103 coloured_string = grey(string) if plugin_name == "Footer-Hash"
104 coloured_string = grey(string) if plugin_name == "CSS"
105 coloured_string = grey(string) if plugin_name == "Tag-Hash"
106
107 coloured_plugin = white(plugin_name)
108 coloured_plugin = grey(plugin_name) if plugin_name == "MD5"
109 coloured_plugin = grey(plugin_name) if plugin_name == "Div-Span-Structure"
110 coloured_plugin = grey(plugin_name) if plugin_name == "Header-Hash"
111 coloured_plugin = grey(plugin_name) if plugin_name == "Footer-Hash"
112 coloured_plugin = grey(plugin_name) if plugin_name == "CSS"
113 coloured_plugin = grey(plugin_name) if plugin_name == "Tag-Hash"
114
115 p = ((certainty and certainty < 100) ? grey(certainty_to_words(certainty))+ " " : "") +
116 coloured_plugin + (!version.empty? ? "["+green(version)+"]" : "") +
117 (!string.empty? ? "[" + coloured_string+"]" : "") +
118 (!accounts.empty? ? "["+ accounts+"]" : "" ) +
f2b1d5d @urbanadventurer model and firmware now display in dark green
urbanadventurer authored
119 (!model.empty? ? "["+ dark_green(model)+"]" : "" ) +
120 (!firmware.empty? ? "["+ dark_green(firmware)+"]" : "" ) +
b00f0af @urbanadventurer Changed :version_regexp_offset, :modules_regexp_offset, etc to :regex…
urbanadventurer authored
121 (!filepath.empty? ? "["+ dark_green(firmware)+"]" : "" ) +
4b22974 @urbanadventurer first commit
urbanadventurer authored
122 (!modules.empty? ? "["+ magenta(modules)+"]" : "" )
123
124
125 brief_results << p
126 else
127 brief_results << ((certainty and certainty < 100) ? certainty_to_words(certainty)+ " " : "") +
128 plugin_name + (!version.empty? ? "[" + version +"]" : "") +
129 (!string.empty? ? "[" + string+"]" : "") +
130 (!accounts.empty? ? " ["+ accounts+"]" : "" ) +
131 (!model.empty? ? "["+ model+"]" : "" ) +
132 (!firmware.empty? ? "["+ firmware+"]" : "" ) +
b00f0af @urbanadventurer Changed :version_regexp_offset, :modules_regexp_offset, etc to :regex…
urbanadventurer authored
133 (!filepath.empty? ? "["+ firmware+"]" : "" ) +
4b22974 @urbanadventurer first commit
urbanadventurer authored
134 (!modules.empty? ? "["+ modules+"]" : "" )
135 end
136 end
137 end
138
139 if (@f == STDOUT and $use_colour=="auto") or ($use_colour=="always")
140 @f.puts blue(target) + " [#{status}] " + brief_results.join(", ")
141 else
142 @f.puts target.to_s + " [#{status}] " + brief_results.join(", ")
143 end
144
145 end
146 end
147
148
149 # Hey, do u actually use this XML output? Then I'd love to hear from you for suggestions, changes, etc.
150 class OutputXML < Output
151 def initialize(f=STDOUT)
152 super
61a586e @urbanadventurer Escaping the XML log properly for &, <, >, "
urbanadventurer authored
153 @substitutions={'&'=>'&amp;', '"'=>'&quot;', '<'=>'&lt;', '>'=>'&gt;'}
4b22974 @urbanadventurer first commit
urbanadventurer authored
154 @f.puts '<?xml version="1.0"?><?xml-stylesheet type="text/xml" href="whatweb.xsl"?>'
155 @f.puts "<log>"
156 end
157
158 def close
159 @f.puts "</log>"
160 @f.close
161 end
162
61a586e @urbanadventurer Escaping the XML log properly for &, <, >, "
urbanadventurer authored
163 def escape(t)
164 text=t.to_s.dup
165 # use sort_by so that & is before &quot;, etc.
166 @substitutions.sort_by {|a,b| a=="&" ? 0 : 1 }.map{|from,to| text.gsub!(from,to) }
167 text
168 end
169
4b22974 @urbanadventurer first commit
urbanadventurer authored
170 def out(target, status, results)
171 @f.puts "<target>"
61a586e @urbanadventurer Escaping the XML log properly for &, <, >, "
urbanadventurer authored
172 @f.puts "\t<uri>#{escape(target)}</uri>"
173 @f.puts "\t<http-status>#{escape(status)}</http-status>"
4b22974 @urbanadventurer first commit
urbanadventurer authored
174
175 results.each do |plugin_name,plugin_results|
176 @f.puts "\t<plugin>"
61a586e @urbanadventurer Escaping the XML log properly for &, <, >, "
urbanadventurer authored
177 @f.puts "\t\t<name>#{escape(plugin_name)}</name>"
4b22974 @urbanadventurer first commit
urbanadventurer authored
178
179 unless plugin_results.empty?
180 # important info in brief mode is version, type and ?
181 # what's the highest probability for the match?
28612fa @urbanadventurer Fixed bugs with Output plugins due to regular expressions being treat…
urbanadventurer authored
182 certainty = plugin_results.map {|x| x[:certainty] unless x[:certainty].class==Regexp }.compact.sort.uniq.last
183 version = plugin_results.map {|x| x[:version] unless x[:version].class==Regexp }.flatten.compact.sort.uniq.join(",")
184 string = plugin_results.map {|x| x[:string] unless x[:string].class==Regexp}.flatten.compact.sort.uniq.join(",")
b00f0af @urbanadventurer Changed :version_regexp_offset, :modules_regexp_offset, etc to :regex…
urbanadventurer authored
185 accounts = plugin_results.map {|x| [x[:account],x[:accounts] ] }.flatten.compact.sort.uniq.join(",")
28612fa @urbanadventurer Fixed bugs with Output plugins due to regular expressions being treat…
urbanadventurer authored
186 model = plugin_results.map {|x| x[:model] unless x[:model].class==Regexp}.compact.sort.uniq.join(",")
187 firmware = plugin_results.map {|x| x[:firmware] unless x[:firmware].class==Regexp}.compact.sort.uniq.join(",")
188 modules = plugin_results.map {|x| x[:modules] unless x[:modules].class==Regexp}.flatten.compact.sort.uniq
189 filepath = plugin_results.map {|x| x[:filepath] unless x[:filepath].class==Regexp}.flatten.compact.sort.uniq.join(",")
4b22974 @urbanadventurer first commit
urbanadventurer authored
190
61a586e @urbanadventurer Escaping the XML log properly for &, <, >, "
urbanadventurer authored
191 @f.puts "\t\t<certainty>#{escape(certainty)}</certainty>" if certainty and certainty < 100
192 version.map {|x| @f.puts "\t\t<version>#{escape(x)}</version>" }
193 string.map {|x| @f.puts "\t\t<string>#{escape(x)}</string>" }
194 accounts.map {|x| @f.puts "\t\t<accounts>#{escape(x)}</accounts>" }
195 model.map {|x| @f.puts "\t\t<model>#{escape(x)}</model>" }
196 firmware.map {|x| @f.puts "\t\t<firmware>#{escape(x)}</firmware>" }
b00f0af @urbanadventurer Changed :version_regexp_offset, :modules_regexp_offset, etc to :regex…
urbanadventurer authored
197 filepath.map {|x| @f.puts "\t\t<filepath>#{escape(x)}</filepath>" }
4b22974 @urbanadventurer first commit
urbanadventurer authored
198
199 if modules.size > 0
61a586e @urbanadventurer Escaping the XML log properly for &, <, >, "
urbanadventurer authored
200 @f.puts "\t\t<modules>\n" + modules.map {|x| "\t\t\t<module>#{escape(x)}</module>" }.join("\n") + "\n\t\t</modules>"
4b22974 @urbanadventurer first commit
urbanadventurer authored
201 end
202 end
203 @f.puts "\t</plugin>"
204 end
205 @f.puts "</target>"
206 end
207 end
208
209 class OutputJSON < Output
c55c367 @urbanadventurer Updated MongoDB output so arrays are flattened
urbanadventurer authored
210
4b22974 @urbanadventurer first commit
urbanadventurer authored
211 def out(target, status, results)
212 # nice
d95db65 @urbanadventurer Fixed MongoDB logging bug
urbanadventurer authored
213 foo= {:target=>target, :http_status=>status, :plugins=>{} }
4b22974 @urbanadventurer first commit
urbanadventurer authored
214
215 results.each do |plugin_name,plugin_results|
218a006 @urbanadventurer added mongo db logging, changed JSON output structure, added Charset …
urbanadventurer authored
216 # thisplugin = {:name=>plugin_name}
217 thisplugin = {}
4b22974 @urbanadventurer first commit
urbanadventurer authored
218
219 unless plugin_results.empty?
220 # important info in brief mode is version, type and ?
221 # what's the highest probability for the match?
222
28612fa @urbanadventurer Fixed bugs with Output plugins due to regular expressions being treat…
urbanadventurer authored
223 certainty = plugin_results.map {|x| x[:certainty] unless x[:certainty].class==Regexp }.compact.sort.uniq.last
224 version = plugin_results.map {|x| x[:version] unless x[:version].class==Regexp }.compact.sort.uniq
225 string = plugin_results.map {|x| x[:string] unless x[:string].class==Regexp }.compact.sort.uniq
4b22974 @urbanadventurer first commit
urbanadventurer authored
226 accounts = plugin_results.map {|x| [x[:account],x[:accounts] ] }.flatten.compact.sort.uniq
28612fa @urbanadventurer Fixed bugs with Output plugins due to regular expressions being treat…
urbanadventurer authored
227 model = plugin_results.map {|x| x[:model] unless x[:model].class==Regexp }.compact.sort.uniq
228 firmware = plugin_results.map {|x| x[:firmware] unless x[:firmware].class==Regexp }.compact.sort.uniq
229 modules = plugin_results.map {|x| x[:modules] unless x[:modules].class==Regexp }.compact.sort.uniq
230 filepath = plugin_results.map {|x| x[:filepath] unless x[:filepath].class==Regexp }.compact.sort.uniq
231
4b22974 @urbanadventurer first commit
urbanadventurer authored
232
233 certainty.nil? ? thisplugin[:certainty] = 100 : thisplugin[:certainty] = certainty
234 thisplugin[:version] = version unless version.empty?
235 thisplugin[:string] = string unless string.empty?
236 thisplugin[:accounts] = accounts unless accounts.empty?
237 thisplugin[:model] = model unless model.empty?
238 thisplugin[:firmware] = firmware unless firmware.empty?
239 thisplugin[:modules] = modules unless modules.empty?
b00f0af @urbanadventurer Changed :version_regexp_offset, :modules_regexp_offset, etc to :regex…
urbanadventurer authored
240 thisplugin[:filepath] = filepath unless filepath.empty?
218a006 @urbanadventurer added mongo db logging, changed JSON output structure, added Charset …
urbanadventurer authored
241 # foo[:plugins] << thisplugin
d95db65 @urbanadventurer Fixed MongoDB logging bug
urbanadventurer authored
242 foo[:plugins][plugin_name.to_sym] = thisplugin
4b22974 @urbanadventurer first commit
urbanadventurer authored
243 end
244 end
245
246 @f.puts JSON::fast_generate(foo)
247 end
248 end
249
218a006 @urbanadventurer added mongo db logging, changed JSON output structure, added Charset …
urbanadventurer authored
250
251 # basically the same as OutputJSON
252 class OutputMongo < Output
253
254 def initialize(collection)
255 # $KCODE='u'
d95db65 @urbanadventurer Fixed MongoDB logging bug
urbanadventurer authored
256 # should make databse and collection comma or fullstop delimited, eg. test,scan
218a006 @urbanadventurer added mongo db logging, changed JSON output structure, added Charset …
urbanadventurer authored
257 @db = Mongo::Connection.new("0.0.0.0").db("test") # resolve-replace means we can't connect to localhost by name
d95db65 @urbanadventurer Fixed MongoDB logging bug
urbanadventurer authored
258 @coll=@db.collection(collection)
218a006 @urbanadventurer added mongo db logging, changed JSON output structure, added Charset …
urbanadventurer authored
259 @charset=nil
260 end
261
262 def close
263 # nothing
264 end
265
c55c367 @urbanadventurer Updated MongoDB output so arrays are flattened
urbanadventurer authored
266 def flatten_elements!(obj)
267 if obj.class == Hash
268 obj.each_value {|x|
269 flatten_elements!(x)
270 }
271 end
272
273 if obj.class == Array
274 obj.flatten!
275 end
276
277 # if obj.class == String
278
279 # end
280 end
281
218a006 @urbanadventurer added mongo db logging, changed JSON output structure, added Charset …
urbanadventurer authored
282 def utf8_elements!(obj)
283 if obj.class == Hash
284 obj.each_value {|x|
285 utf8_elements!(x)
286 }
287 end
288
289 if obj.class == Array
290 obj.each {|x|
291 utf8_elements!(x)
292 }
293 end
294
295 if obj.class == String
296 # obj=obj.upcase!
297 # obj=Iconv.iconv("UTF-8",@charset,obj).join
298 obj=obj.gsub!(/^.*$/,Iconv.iconv("UTF-8",@charset,obj).join) # this is a bad way to do this but it works
299 end
300 end
301
302 def out(target, status, results)
303 # nice
d95db65 @urbanadventurer Fixed MongoDB logging bug
urbanadventurer authored
304 foo= {:target=>target, :http_status=>status, :plugins=>{} }
218a006 @urbanadventurer added mongo db logging, changed JSON output structure, added Charset …
urbanadventurer authored
305
306 results.each do |plugin_name,plugin_results|
307 # thisplugin = {:name=>plugin_name}
308 thisplugin = {}
309
310 unless plugin_results.empty?
311 # important info in brief mode is version, type and ?
312 # what's the highest probability for the match?
313
28612fa @urbanadventurer Fixed bugs with Output plugins due to regular expressions being treat…
urbanadventurer authored
314 certainty = plugin_results.map {|x| x[:certainty] unless x[:certainty].class==Regexp }.compact.sort.uniq.last
315 version = plugin_results.map {|x| x[:version] unless x[:version].class==Regexp }.compact.sort.uniq
316 string = plugin_results.map {|x| x[:string] unless x[:string].class==Regexp }.compact.sort.uniq
218a006 @urbanadventurer added mongo db logging, changed JSON output structure, added Charset …
urbanadventurer authored
317 accounts = plugin_results.map {|x| [x[:account],x[:accounts] ] }.flatten.compact.sort.uniq
28612fa @urbanadventurer Fixed bugs with Output plugins due to regular expressions being treat…
urbanadventurer authored
318 model = plugin_results.map {|x| x[:model] unless x[:model].class==Regexp }.compact.sort.uniq
319 firmware = plugin_results.map {|x| x[:firmware] unless x[:firmware].class==Regexp }.compact.sort.uniq
320 modules = plugin_results.map {|x| x[:modules] unless x[:modules].class==Regexp }.compact.sort.uniq
321 filepath = plugin_results.map {|x| x[:filepath] unless x[:filepath].class==Regexp }.compact.sort.uniq
218a006 @urbanadventurer added mongo db logging, changed JSON output structure, added Charset …
urbanadventurer authored
322
323 certainty.nil? ? thisplugin[:certainty] = 100 : thisplugin[:certainty] = certainty
324 thisplugin[:version] = version unless version.empty?
325 thisplugin[:string] = string unless string.empty?
326 thisplugin[:accounts] = accounts unless accounts.empty?
327 thisplugin[:model] = model unless model.empty?
328 thisplugin[:firmware] = firmware unless firmware.empty?
329 thisplugin[:modules] = modules unless modules.empty?
330 thisplugin[:filepath] = filepath unless filepath.empty?
331 # foo[:plugins] << thisplugin
d95db65 @urbanadventurer Fixed MongoDB logging bug
urbanadventurer authored
332 foo[:plugins][plugin_name.to_sym] = thisplugin
218a006 @urbanadventurer added mongo db logging, changed JSON output structure, added Charset …
urbanadventurer authored
333 end
334 end
335 #data=JSON::fast_generate(foo)
336
337 # extract charset from charset plugin
338 @charset=results.map {|n,r| r[0][:string] if n=="Charset" }.compact.first
339 # pp @charset
340 unless @charset.nil? or @charset == "Failed"
341 # puts "here"
342 utf8_elements!(foo) # convert foo to utf-8
c55c367 @urbanadventurer Updated MongoDB output so arrays are flattened
urbanadventurer authored
343 flatten_elements!(foo)
218a006 @urbanadventurer added mongo db logging, changed JSON output structure, added Charset …
urbanadventurer authored
344 @coll.insert(foo)
c55c367 @urbanadventurer Updated MongoDB output so arrays are flattened
urbanadventurer authored
345 pp foo
218a006 @urbanadventurer added mongo db logging, changed JSON output structure, added Charset …
urbanadventurer authored
346 end
347 end
348 end
349
350
351
4b22974 @urbanadventurer first commit
urbanadventurer authored
352 class OutputJSONVerbose < Output
353 def out(target, status, results)
354 # brutal and simple
355 @f.puts JSON::fast_generate([target,status,results])
356 end
357 end
358
359 class OutputErrors < Output
360 def out(error)
361 @f.puts error
362 end
363
364 end
365
Something went wrong with that request. Please try again.