Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

odstranenie zoznamov domen, cachovanie, upravy textov, vela oprav pre…

… nasadenie
  • Loading branch information...
commit 73d08220c36114c241202c3f62f46e64deaec6a0 1 parent c00c497
@martinliptak authored
Showing with 335 additions and 443 deletions.
  1. +2 −0  .gitignore
  2. +117 −210 app/assets/javascripts/application.js
  3. +121 −5 app/controllers/domains_controller.rb
  4. +0 −98 app/controllers/lists_controller.rb
  5. +2 −2 app/helpers/application_helper.rb
  6. +1 −1  app/models/analyzers/analyze_all_domains.rb
  7. +15 −17 app/models/domain.rb
  8. +0 −11 app/models/list_domain.rb
  9. +4 −1 app/models/parsers/ceske_domeny_info_parser.rb
  10. +1 −1  app/models/parsers/dmoz_parser.rb
  11. +5 −7 app/models/parsers/sk_nic_parser.rb
  12. +1 −1  app/views/domains/analyze.html.erb
  13. +5 −0 app/views/domains/countries.html.erb
  14. +7 −5 app/views/{lists/show.html.erb → domains/index.html.erb}
  15. +20 −17 app/views/layouts/application.html.erb
  16. +1 −1  config/database.yml
  17. +1 −1  config/environments/development.rb
  18. +9 −2 config/environments/production.rb
  19. +5 −3 config/routes.rb
  20. +1 −0  db/migrate/20110815142037_create_domains.rb
  21. +0 −9 db/migrate/20110816182407_create_list_domains.rb
  22. 0  {app/assets → public}/images/wslogo.png
  23. +14 −14 spec/models/domain_spec.rb
  24. +0 −21 spec/models/list_domain_spec.rb
  25. +0 −4 spec/models/parsers/dmoz_parser_spec.rb
  26. +3 −12 spec/models/parsers/sk_nic_parser_spec.rb
View
2  .gitignore
@@ -4,3 +4,5 @@ log/*.log
tmp/
.sass-cache/
.idea/
+public/assets
+
View
327 app/assets/javascripts/application.js
@@ -31,237 +31,144 @@ function niceRound(num)
return Math.round(num * c) / c;
}
-$(function() {
- /**
- * Grid theme for Highcharts JS
- * @author Torstein Hønsi
- */
-
- Highcharts.theme = {
- colors: ['#058DC7', '#50B432', '#ED561B', '#DDDF00', '#24CBE5', '#64E572', '#FF9655', '#FFF263', '#6AF9C4'],
- chart: {
- /*
- backgroundColor: {
- linearGradient: [0, 0, 500, 500],
- stops: [
- [0, 'rgb(255, 255, 255)'],
- [1, 'rgb(240, 240, 255)']
- ]
- }
- ,
- borderWidth: 2,
- plotBackgroundColor: 'rgba(255, 255, 255, .9)',
- plotShadow: true,
- plotBorderWidth: 1
- */
- plotShadow: true
- },
- title: {
- style: {
- color: '#000',
- font: 'bold 16px "Trebuchet MS", Verdana, sans-serif'
- }
- },
- subtitle: {
- style: {
- color: '#666666',
- font: 'bold 12px "Trebuchet MS", Verdana, sans-serif'
- }
- },
- xAxis: {
- gridLineWidth: 1,
- lineColor: '#000',
- tickColor: '#000',
- labels: {
- style: {
- color: '#000',
- font: '11px Trebuchet MS, Verdana, sans-serif'
- }
- },
- title: {
- style: {
- color: '#333',
- fontWeight: 'bold',
- fontSize: '12px',
- fontFamily: 'Trebuchet MS, Verdana, sans-serif'
-
- }
- }
- },
- yAxis: {
- minorTickInterval: 'auto',
- lineColor: '#000',
- lineWidth: 1,
- tickWidth: 1,
- tickColor: '#000',
- labels: {
- style: {
- color: '#000',
- font: '11px Trebuchet MS, Verdana, sans-serif'
- }
- },
- title: {
- style: {
- color: '#333',
- fontWeight: 'bold',
- fontSize: '12px',
- fontFamily: 'Trebuchet MS, Verdana, sans-serif'
- }
- }
- },
- legend: {
- itemStyle: {
- font: '9pt Trebuchet MS, Verdana, sans-serif',
- color: 'black'
-
- },
- itemHoverStyle: {
- color: '#039'
- },
- itemHiddenStyle: {
- color: 'gray'
- }
- },
- labels: {
- style: {
- color: '#99b'
- }
- }
- };
-
- // Apply the theme
- // var highchartsOptions = Highcharts.setOptions(Highcharts.theme);
-
- $(".chart").each(function () {
- var chart = this;
-
- options = {
- container: chart,
- chart: {
- renderTo: chart,
- defaultSeriesType: $(chart).attr("data-type"),
- zoomType: 'xy'
- },
- title: {
- text: $(chart).attr("title")
- },
- xAxis: {
- type: $(chart).attr("data-type") != 'line' ? null : 'datetime',
- categories: $.parseJSON($(chart).attr("data-categories")),
- title: {
- text: $(chart).attr("data-x-title")
- }
- },
- yAxis: {
- title: {
- text: $(chart).attr("data-y-title")
- },
- min: $(chart).attr("data-min") || null,
- startOnTick: $(chart).attr("data-type") != 'line',
- showFirstLabel: $(chart).attr("data-type") != 'line'
- },
- legend: {
- enabled: false
- },
- tooltip: {
- formatter: function () {
- return eval($(chart).attr("data-tooltip"));
- }
- },
- plotOptions: {
- column: {
- cursor: $(chart).attr("data-no-pointer") ? "" : "pointer",
- point: {
- events: {
- "click": function () {
- var m = this.category.match(/<a href="(.*)">/);
- if (m)
- window.location = m[1];
- }
+$(".chart").each(function () {
+ var chart = this;
+
+ options = {
+ container: chart,
+ chart: {
+ renderTo: chart,
+ defaultSeriesType: $(chart).attr("data-type"),
+ zoomType: 'xy'
+ },
+ title: {
+ text: $(chart).attr("title")
+ },
+ xAxis: {
+ type: $(chart).attr("data-type") != 'line' ? null : 'datetime',
+ categories: $.parseJSON($(chart).attr("data-categories")),
+ title: {
+ text: $(chart).attr("data-x-title")
+ }
+ },
+ yAxis: {
+ title: {
+ text: $(chart).attr("data-y-title")
+ },
+ min: $(chart).attr("data-min") || null,
+ startOnTick: $(chart).attr("data-type") != 'line',
+ showFirstLabel: $(chart).attr("data-type") != 'line'
+ },
+ legend: {
+ enabled: false
+ },
+ tooltip: {
+ formatter: function () {
+ return eval($(chart).attr("data-tooltip"));
+ }
+ },
+ plotOptions: {
+ column: {
+ cursor: $(chart).attr("data-no-pointer") ? "" : "pointer",
+ point: {
+ events: {
+ "click": function () {
+ var m = this.category.match(/<a href="(.*)">/);
+ if (m)
+ window.location = m[1];
}
}
},
- pie: {
- cursor: $(chart).attr("data-no-pointer") ? "" : "pointer",
- point: {
- events: {
- "click": function () {
- var m = this.name.match(/<a href="(.*)">/);
- if (m)
- window.location = m[1];
- }
+ animation: false
+ },
+ pie: {
+ cursor: $(chart).attr("data-no-pointer") ? "" : "pointer",
+ point: {
+ events: {
+ "click": function () {
+ var m = this.name.match(/<a href="(.*)">/);
+ if (m)
+ window.location = m[1];
}
}
- },
- line: {
- marker: {
- enabled: false,
- states: {
- hover: {
- enabled: true
- }
- }
- }
- }
+ },
+ animation: false
},
- series: [],
- credits: {
- enabled: false
- },
- animation: {
- enabled: false
- }
- }
-
- if ($.parseJSON($(chart).attr("data-series-hash")))
- {
- h = $.parseJSON($(chart).attr("data-series-data"));
- for (s in h)
- {
- options['series'].push({
- type: $(chart).attr("data-type"),
- name: s,
- data: h[s],
- pointInterval: parseInt($(chart).attr("data-series-interval")) || null,
- pointStart: parseInt($(chart).attr("data-series-start")) || null
- });
+ line: {
+ marker: {
+ enabled: false,
+ states: {
+ hover: {
+ enabled: true
+ }
+ }
+ }
}
-
- options.legend.enabled = true;
- }
- else
- {
- options['series'].push({
+ },
+ series: [],
+ credits: {
+ enabled: false
+ },
+ animation: {
+ enabled: false
+ }
+ }
+
+ if ($.parseJSON($(chart).attr("data-series-hash")))
+ {
+ h = $.parseJSON($(chart).attr("data-series-data"));
+ for (s in h)
+ {
+ options['series'].push({
type: $(chart).attr("data-type"),
- data: $.parseJSON($(chart).attr("data-series-data")),
+ name: s,
+ data: h[s],
pointInterval: parseInt($(chart).attr("data-series-interval")) || null,
- pointStart: parseInt($(chart).attr("data-series-start")) || null,
- dataLabels: {
- formatter: function() {
- return this.point.name + ": " + niceRound(this.y) + "%";
- }
- }
+ pointStart: parseInt($(chart).attr("data-series-start")) || null
});
- }
-
- new Highcharts.Chart(options);
- });
-
- if ($('#refresh-in').length)
- {
- setInterval('refresh();', 1000);
- }
+ }
+
+ options.legend.enabled = true;
+ }
+ else
+ {
+ options['series'].push({
+ type: $(chart).attr("data-type"),
+ data: $.parseJSON($(chart).attr("data-series-data")),
+ pointInterval: parseInt($(chart).attr("data-series-interval")) || null,
+ pointStart: parseInt($(chart).attr("data-series-start")) || null,
+ dataLabels: {
+ formatter: function() {
+ return this.point.name + ": " + niceRound(this.y) + "%";
+ }
+ }
+ });
+ }
+ new Highcharts.Chart(options);
});
+var refreshInterval;
+
function refresh()
{
secs = parseInt($('#refresh-in').html());
secs -= 1;
if (secs == 0)
+ {
+ clearInterval(refreshInterval);
location.reload(true);
+ }
else
$('#refresh-in').html(secs);
}
+
+$(function () {
+
+ if ($('#refresh-in').length)
+ {
+ refreshInterval = setInterval('refresh();', 1000);
+ }
+});
View
126 app/controllers/domains_controller.rb
@@ -1,9 +1,88 @@
require "whois"
require "page_rankr"
+COUNTRIES = {
+ 'Global' => ['com', 'org', 'net', 'edu', 'info', 'biz', 'gov'],
+ 'German' => ['de', 'ch', 'at'],
+ 'Italian' => 'it',
+ 'French' => 'fr',
+ 'Czech' => 'cz',
+ 'Russian' => 'ru',
+ 'Japanese' => 'jp'
+}
+
class DomainsController < ApplicationController
include DomainsHelper
+ before_filter :set_params_tld, :except => :search
+ before_filter :set_markers, :only => :index
+
+ caches_action :index, :layout => false, :cache_path => Proc.new { |controller|
+ "domains_index_#{controller.params[:tld]}"
+ }
+ caches_action :countries, :layout => false
+
+ def index
+ @domains = Domain.where(:tld => @tld).analyzed
+
+ @distribution = {}
+
+ @distribution[:framework] = distribution(Domain.connection.execute("
+ select framework, count(*) from pages
+ join domains on page_id = pages.id and tld in #{@tld_set}
+ where framework is not null
+ group by framework order by count(*) desc"), :framework)
+
+ @distribution[:feature] = distribution(Domain.connection.execute("
+ select features.name, count(*) from features
+ join pages on features.page_id = pages.id
+ join domains on domains.page_id = pages.id and tld in #{@tld_set}
+ group by name order by count(*) desc"), :feature)
+
+ for type in [:server, :engine, :doctype]
+ @distribution[type] = distribution(Domain.connection.execute("
+ select #{type}, count(*) from pages
+ join domains on page_id = pages.id and tld in #{@tld_set}
+ group by #{type} order by count(*) desc"), type, @domains.count / 100)
+ end
+
+ @html5 = Domain.connection.execute("
+ select count(*) from pages
+ join domains on page_id = pages.id and tld in #{@tld_set}
+ where doctype = 'html' ").first.first
+
+ @distribution[:ipv6] = Domain.connection.execute("
+ select if(ipv6, 'Yes', 'No'), count(*) from domains
+ where ipv6 is not null and tld in #{@tld_set}
+ group by ipv6 order by count(*) desc")
+
+ @distribution[:country] = distribution(Domain.connection.execute("
+ select country, count(*) from locations
+ join domains on locations.id = domains.location_id and tld in #{@tld_set}
+ group by country order by count(*) desc"), :country, @domains.count / 200)
+
+ @distribution[:city] = distribution(Domain.connection.execute("
+ select city, count(*) from locations
+ join domains on locations.id = domains.location_id and tld in #{@tld_set}
+ where city <> ' '
+ group by city order by count(*) desc limit 12"), :city)
+
+ @markers = Domain.connection.execute("
+ select longitude, latitude, count(*), city from locations
+ join domains on locations.id = domains.location_id and tld in #{@tld_set}
+ where longitude is not null and latitude is not null and city <> ' '
+ group by longitude, latitude order by count(*) desc limit 100").map { |marker|
+ {
+ :latitude => marker[1],
+ :longitude => marker[0],
+ :title => marker[3],
+ :icon => marker_icon(marker[2]),
+ :html => "<b>#{marker[3]}</b><br/>
+ <a href=#{url_for(search_domains_path(:city => marker[3], :tld => params[:tld]))}>#{pluralize(marker[2], 'domain')}</a>"
+ }
+ }
+ end
+
def show
@domain = Domain.find_by_name(decode_domain_name(params[:id]))
@title = "Domain #{@domain.name}"
@@ -67,11 +146,9 @@ def search
where['locations.country'] = params[:country]
end
- unless params[:list].blank?
- tables << :list_domains
- where['list_domains.list'] = params[:list]
-
- @list = params[:list]
+ if params[:tld].present?
+ set_params_tld
+ where['domains.tld'] = @tld
end
@domains = Domain.page(params[:page]).includes(tables.uniq).where(where).where(like)
@@ -140,4 +217,43 @@ def analyze
render :action => "410", :status => '410 Gone'
end
+ def countries
+ @title = 'All countries'
+ @countries = Domain.group(:tld).order('count(*) desc').where('tld is not null').having('count(*) > 1000').map(&:tld)
+ end
+
+ private
+
+ def set_params_tld
+ params[:tld] = 'Global' unless params[:tld] =~ /^\w+$/
+
+ if COUNTRIES[params[:tld]]
+ @tld = COUNTRIES[params[:tld]]
+ else
+ @tld = params[:tld]
+ end
+ @tld_set = "(#{Array.wrap(@tld).map{|t| "'#{t}'" }.join(',')})"
+ end
+
+ def set_markers
+ @markers = []
+ end
+
+ def distribution(data, attribute, threshold = 0)
+ other = 0
+ dist = data.select do |row|
+ row[0] = "Undetected" if row[0] == nil
+ other += row[1] if row[1] <= threshold
+ row[1] > threshold
+ end
+ dist = dist.map do |data|
+ if data[0] == 'Undetected'
+ data
+ else
+ ["<a href=\"#{url_for search_domains_path(attribute => data[0], :tld => params[:tld])}\">#{data[0]}</a>", data[1]]
+ end
+ end
+ dist << ["Other", other] if other > 0
+ dist
+ end
end
View
98 app/controllers/lists_controller.rb
@@ -1,98 +0,0 @@
-class ListsController < ApplicationController
- LISTS = ['sk_nic', 'cz', 'dmoz']
-
- def show
- if params[:id] and LISTS.include?(params[:id])
- @list = params[:id]
- else
- @list = 'dmoz'
- end
- @domains = Domain.includes(:list_domains).where('list_domains.list' => @list).analyzed
-
- @distribution = {}
-
- @distribution[:framework] = distribution(Domain.connection.execute("
- select framework, count(*) from pages
- join domains on page_id = pages.id
- join list_domains on domain_id = domains.id and list = '#{@list}'
- where framework is not null
- group by framework order by count(*) desc"), :framework)
-
- @distribution[:feature] = distribution(Domain.connection.execute("
- select features.name, count(*) from features
- join pages on features.page_id = pages.id
- join domains on domains.page_id = pages.id
- join list_domains on domain_id = domains.id and list = '#{@list}'
- group by name order by count(*) desc"), :feature)
-
- for type in [:server, :engine, :doctype]
- @distribution[type] = distribution(Domain.connection.execute("
- select #{type}, count(*) from pages
- join domains on page_id = pages.id
- join list_domains on domain_id = domains.id and list = '#{@list}'
- group by #{type} order by count(*) desc"), type, @domains.count / 100)
- end
-
- @html5 = Domain.connection.execute("
- select count(*) from pages
- join domains on page_id = pages.id
- join list_domains on domain_id = domains.id and list = '#{@list}'
- where doctype = 'html' ").first.first
-
- @distribution[:ipv6] = Domain.connection.execute("
- select if(ipv6, 'Yes', 'No'), count(*) from domains
- join list_domains on domain_id = domains.id and list = '#{@list}'
- where ipv6 is not null
- group by ipv6 order by count(*) desc")
-
- @distribution[:country] = distribution(Domain.connection.execute("
- select country, count(*) from locations
- join domains on locations.id = domains.location_id
- join list_domains on domain_id = domains.id and list = '#{@list}'
- group by country order by count(*) desc"), :country, @domains.count / 150)
-
- @distribution[:city] = distribution(Domain.connection.execute("
- select city, count(*) from locations
- join domains on locations.id = domains.location_id
- join list_domains on domain_id = domains.id and list = '#{@list}'
- where city <> ' '
- group by city order by count(*) desc limit 12"), :city)
-
- @markers = Domain.connection.execute("
- select longitude, latitude, count(*), city from locations
- join domains on locations.id = domains.location_id
- join list_domains on domain_id = domains.id and list = '#{@list}'
- where longitude is not null and latitude is not null and city <> ' '
- group by longitude, latitude order by count(*) desc limit 1000").map { |marker|
- {
- :latitude => marker[1],
- :longitude => marker[0],
- :title => marker[3],
- :icon => marker_icon(marker[2]),
- :zoom => marker[2] < 80 ? 8 : 0,
- :html => "<b>#{marker[3]}</b><br/>
- <a href=#{url_for(search_domains_path(:city => marker[3],:list => @list))}>#{pluralize(marker[2], 'domain')}</a>"
- }
- }
- end
-
- private
-
- def distribution(data, attribute, threshold = 0)
- other = 0
- dist = data.select do |row|
- row[0] = "Undetected" if row[0] == nil
- other += row[1] if row[1] <= threshold
- row[1] > threshold
- end
- dist = dist.map do |data|
- if data[0] == 'Undetected'
- data
- else
- ["<a href=\"#{url_for search_domains_path(attribute => data[0], :list => @list)}\">#{data[0]}</a>", data[1]]
- end
- end
- dist << ["Other", other] if other > 0
- dist
- end
-end
View
4 app/helpers/application_helper.rb
@@ -4,7 +4,7 @@ def title
@title ? "#{@title} | Statscrawler.com" : "Statscrawler.com"
end
- def list(title, list)
- "<li class=\"right #{list == @list ? "active" : ""}\">#{link_to title, list_path(list)}</li>".html_safe
+ def item(title, tld)
+ "<li class=\"right #{params[:tld] == tld ? "active" : ""}\">#{link_to title, root_path(:tld => tld)}</li>".html_safe
end
end
View
2  app/models/analyzers/analyze_all_domains.rb
@@ -3,7 +3,7 @@ module Analyzers
class AnalyzeAllDomains < AnalyzeDomain
@queue = 'low'
- def self.perform(thread_count = 95)
+ def self.perform(thread_count = 50)
if thread_count > 0
Thread.abort_on_exception = true
View
32 app/models/domain.rb
@@ -1,7 +1,8 @@
class Domain < ActiveRecord::Base
belongs_to :page
belongs_to :location
- has_many :list_domains
+
+ before_save :set_tld_from_name
before_destroy :destroy_orphaned_pages
before_destroy :destroy_orphaned_locations
@@ -9,20 +10,20 @@ class Domain < ActiveRecord::Base
def to_param
name.gsub('.', '_')
end
+
+ def url
+ if name.tr('^.', '').size > 1
+ "http://#{name}"
+ else
+ "http://www.#{name}"
+ end
+ end
def analyze
yield(self)
self.analyzed_at = Time.now
save
end
-
- def self.create_from_list(list, domain_name)
- domain = find_or_create_by_name(domain_name)
- unless domain.list_domains.where(:list => list).any?
- domain.list_domains.create(:list => list)
- end
- domain
- end
def self.to_be_analyzed
where(:analyzed_at => nil)
@@ -32,14 +33,6 @@ def self.analyzed
where('analyzed_at IS NOT NULL')
end
- def url
- if name.tr('^.', '').size > 1
- "http://#{name}"
- else
- "http://www.#{name}"
- end
- end
-
private
def destroy_orphaned_pages
@@ -49,4 +42,9 @@ def destroy_orphaned_pages
def destroy_orphaned_locations
location.destroy if location and location.domains.count == 1
end
+
+ def set_tld_from_name
+ m = /.(\w+)$/.match(name);
+ write_attribute(:tld, m[1]) if m and m[1]
+ end
end
View
11 app/models/list_domain.rb
@@ -1,11 +0,0 @@
-class ListDomain < ActiveRecord::Base
- belongs_to :domain
-
- before_destroy :destroy_orphaned_domains
-
- private
-
- def destroy_orphaned_domains
- domain.destroy if domain.list_domains.count == 1
- end
-end
View
5 app/models/parsers/ceske_domeny_info_parser.rb
@@ -5,7 +5,10 @@ class CeskeDomenyInfoParser < BaseParser
@queue = 'low'
def self.perform
+ Rails.logger.info "#{Time.now} Parsing czech domains"
for letter in (1..9).to_a + ('A'..'Z').to_a
+
+ Rails.logger.info "#{Time.now} Parsing '#{letter}' domains"
for page in (1..10_000)
continue = false
@@ -14,7 +17,7 @@ def self.perform
anchors = doc.css('.left table a')
if anchors.any?
for a in anchors
- Domain.create_from_list('cz', a.text.sub('www.', '').tr('/', ''))
+ Domain.find_or_create_by_name(a.text.sub('www.', '').tr('/', ''))
end
continue = true
end
View
2  app/models/parsers/dmoz_parser.rb
@@ -10,7 +10,7 @@ def self.perform
gz = Zlib::GzipReader.new(file)
gz.each_line { |line|
if line =~ %r{http://([^/]*)}
- Domain.create_from_list('dmoz', $1)
+ Domain.find_or_create_by_name($1)
end
}
gz.close
View
12 app/models/parsers/sk_nic_parser.rb
@@ -33,18 +33,16 @@ def self.perform
}
Rails.logger.info "#{Time.now} Destroying SK-NIC domains"
- ListDomain.find_by_sql("SELECT l.* FROM list_domains l
- JOIN domains d ON d.id = l.domain_id AND l.list = 'sk_nic'
- WHERE NOT EXISTS (SELECT * FROM temp_domains t WHERE t.name = d.name)").each { |list_domain|
- list_domain.destroy
+ Domain.find_by_sql("SELECT * FROM domains d
+ WHERE tld = 'sk' AND NOT EXISTS (SELECT * FROM temp_domains t WHERE t.name = d.name)").each { |domain|
+ domain.destroy
}
Rails.logger.info "#{Time.now} Adding SK-NIC domains"
- ActiveRecord::Base.connection.execute("SELECT name FROM temp_domains t
+ Domain.connection.execute("SELECT name FROM temp_domains t
WHERE NOT EXISTS (SELECT * FROM domains d
- JOIN list_domains l ON d.id = l.domain_id AND l.list = 'sk_nic'
WHERE d.name = t.name)").each { |result|
- Domain.create_from_list('sk_nic', result[0])
+ Domain.create(:name => result[0])
}
ActiveRecord::Base.connection.execute("DROP TEMPORARY TABLE temp_domains")
View
2  app/views/domains/analyze.html.erb
@@ -1,4 +1,4 @@
<h1><%= @title %></h1>
-<p>Domain is being analyzed, please wait <span id="refresh-in">20</span> seconds...</p>
+<p>Domain is being analyzed, please wait <span id="refresh-in">10</span> seconds...</p>
View
5 app/views/domains/countries.html.erb
@@ -0,0 +1,5 @@
+<h1><%= @title %></h1>
+
+<% for tld in @countries %>
+ <li><%= link_to tld, root_path(:tld => tld) %></li>
+<% end %>
View
12 app/views/lists/show.html.erb → app/views/domains/index.html.erb
@@ -1,11 +1,15 @@
-<p><%= number_with_delimiter @domains.analyzed.count %> domains have been analyzed in this list. </p>
+<p><%= number_with_delimiter @domains.analyzed.count %>
+ <%= params[:tld].downcase %>
+ domains
+ <%= "(#{@tld.join(', ')})" if @tld.kind_of? Array %>
+ have been analyzed. </p>
<%= render "shared/column", :title => "Frameworks",
:data => @distribution[:framework],
:tooltip => "'<b>'+ addSpaces(this.y) + '</b> domains are powered by <b>'+ this.x +'</b>. '",
:x_title => "Framework", :y_title => "Domains",
:description => 'There are many ways to detect a framework.
- For example some add X-Powered-By HTTP header, some set unique cookie name or
+ For example some frameworks add X-Powered-By HTTP header, some set unique cookie name or
provide HTTP generator meta tag. Often the only reliable way
is the directory structure of stylesheets, javascripts or images.
Unique names of HTML element classes can also be used.
@@ -22,7 +26,7 @@
<%= render "shared/pie", :title => "Servers",
:data => @distribution[:server],
:total => @domains.count,
- :description => 'Detected from Server header, although it is often hidden for security reasons.' %>
+ :description => 'Detected from Server header, although this header is often hidden for security reasons.' %>
<%= render "shared/pie", :title => "Languages",
:data => @distribution[:engine],
@@ -55,6 +59,4 @@
<article class="block" >
<div id="map" data-markers="<%= @markers.to_json %>"></div>
-
- <p>Zoom in to country level to see the smallest hosting centers. </p>
</article>
View
37 app/views/layouts/application.html.erb
@@ -33,16 +33,18 @@
</head>
<body class="<%= params[:controller] %>">
<div id="page">
- <header>
- <div id="description">What frameworks and javascript libraries
- are used on the Internet? What are the most common languages,
- web servers and doctypes? Where are those domains hosted?
- Statscrawler works hard to find out. </div>
- <a href="/" id="logo">
- <h1>Statscrawler.com</h1>
- </a>
- <div id="slogan">Analyzed <%= number_with_delimiter(Domain.analyzed.count) %> domains and analyzing more...</div>
- </header>
+ <%= cache :expires_in => 1.hour do %>
+ <header>
+ <div id="description">What frameworks and javascript libraries
+ are used on the Internet? What are the most common languages,
+ web servers and doctypes? Where are those domains hosted?
+ Statscrawler works hard to find out. </div>
+ <a href="/" id="logo">
+ <h1>Statscrawler.com</h1>
+ </a>
+ <div id="slogan">Analyzed <%= number_with_delimiter(Domain.analyzed.count) %> domains and analyzing more...</div>
+ </header>
+ <% end %>
<section id="bar">
<div id="analyze">
@@ -56,9 +58,9 @@
<nav>
<ul>
- <%= list 'DMOZ domains', 'dmoz' %>
- <%= list 'Slovak domains', 'sk_nic' %>
- <%= list 'Czech domains', 'cz' %>
+ <% for tld in COUNTRIES.keys %>
+ <%= item tld.titleize, tld %>
+ <% end %>
</ul>
</nav>
</section>
@@ -98,6 +100,7 @@
</div>
<div class="right">
+ <%= link_to 'All countries', countries_domains_path %>
<%= link_to 'All domains', search_domains_path(:name => '') %>
<a href="#page" id="move-to-top">Move to top &uArr;</a>
</div>
@@ -108,10 +111,10 @@
</section>
<footer>
- <%= link_to image_tag('wslogo.png'), 'http://www.websupport.sk' %>
- Statscrawler has been created by <a href="http://twitter.com/martinliptak">@martinliptak</a>.
- Any feedback is welcome. If you are interested in how framework detection works, check out
- <a href="https://github.com/martinliptak/statscrawler-matcher">Github</a>.
+ <%= link_to image_tag('/images/wslogo.png'), 'http://www.websupport.sk' %>
+ Stats were collected in September 2011 by <a href="http://twitter.com/martinliptak">@martinliptak</a>.
+ Any feedback will be highly appreciated. If you are interested in how our framework detection works,
+ check out <a href="https://github.com/martinliptak/statscrawler-matcher">Github</a>.
Virtual server for crawling has been provided by <a href="http://www.websupport.sk">Websupport</a>.
</footer>
</div>
View
2  config/database.yml
@@ -12,7 +12,7 @@ development:
adapter: mysql2
encoding: utf8
reconnect: false
- database: statscrawler.com_development
+ database: statscrawler.com_production
pool: 5
username: root
password:
View
2  config/environments/development.rb
@@ -10,7 +10,7 @@
config.whiny_nils = true
# Show full error reports and disable caching
- config.consider_all_requests_local = true
+ 64
config.action_controller.perform_caching = false
# Don't care if the mailer can't send
View
11 config/environments/production.rb
@@ -5,7 +5,7 @@
config.cache_classes = true
# Full error reports are disabled and caching is turned on
- config.consider_all_requests_local = false
+ config.consider_all_requests_local = true
config.action_controller.perform_caching = true
# Disable Rails's static asset server (Apache or nginx will already do this)
@@ -13,7 +13,14 @@
# Compress JavaScripts and CSS
config.assets.compress = true
-
+
+ # Don't fallback to assets pipeline if a precompiled asset is missed
+ config.assets.compile = false
+ config.assets.precompile << 'map.js'
+
+ # Generate digests for assets URLs
+ config.assets.digest = true
+
# Specifies the header that your server uses for sending files
# (comment out if your front-end server doesn't support this)
config.action_dispatch.x_sendfile_header = "X-Sendfile" # Use 'X-Accel-Redirect' for nginx
View
8 config/routes.rb
@@ -1,7 +1,9 @@
StatscrawlerCom::Application.routes.draw do
- resources :lists
resources :domains do
- get :search, :on => :collection
+ collection do
+ get :search
+ get :countries
+ end
member do
get :whois
@@ -10,5 +12,5 @@
end
end
- root :to => 'lists#show#dmoz'
+ root :to => 'domains#index'
end
View
1  db/migrate/20110815142037_create_domains.rb
@@ -2,6 +2,7 @@ class CreateDomains < ActiveRecord::Migration
def change
create_table :domains do |t|
t.string :name
+ t.string :tld
t.references :page
t.references :location
t.boolean :ipv6
View
9 db/migrate/20110816182407_create_list_domains.rb
@@ -1,9 +0,0 @@
-class CreateListDomains < ActiveRecord::Migration
- def change
- create_table :list_domains do |t|
- t.string :list
- t.references :domain
- end
- add_index :list_domains, :domain_id
- end
-end
View
0  app/assets/images/wslogo.png → public/images/wslogo.png
File renamed without changes
View
28 spec/models/domain_spec.rb
@@ -2,28 +2,28 @@
describe Domain do
- it "should create domain from list" do
- Domain.create_from_list("list", "first")
- Domain.count.should == 1
- ListDomain.count.should == 1
-
+ it "should set tld from domain name before save" do
+ Domain.create(:name => "example.org")
+
first = Domain.first
- first.name.should == 'first'
+ first.name.should == 'example.org'
+ first.tld.should == 'org'
first.page.should be_blank
first.location.should be_blank
first.analyzed_at.should be_blank
-
- Domain.create_from_list("list", "second")
- Domain.count.should == 2
- ListDomain.count.should == 2
-
- Domain.create_from_list("list2", "second")
+
+ Domain.create(:name => "example.co.uk")
Domain.count.should == 2
- ListDomain.count.should == 3
+ last = Domain.last
+ last.name.should == 'example.co.uk'
+ last.tld.should == 'uk'
+ last.page.should be_blank
+ last.location.should be_blank
+ last.analyzed_at.should be_blank
end
it "should provide analyze helper method" do
- first = Domain.create_from_list("list", "first")
+ first = Domain.create(:name => "first")
Timecop.freeze(Time.now) {
first.analyze { |domain|
View
21 spec/models/list_domain_spec.rb
@@ -1,21 +0,0 @@
-require 'spec_helper'
-
-describe ListDomain do
-
- it "should destroy associated domain when no other lists exist" do
- domain = Domain.create :name => 'domain'
- first = domain.list_domains.create :list => 'first'
- second = domain.list_domains.create :list => 'second'
-
- Domain.count.should == 1
- ListDomain.count.should == 2
-
- first.destroy
- Domain.count.should == 1
- ListDomain.count.should == 1
-
- second.destroy
- Domain.count.should == 0
- ListDomain.count.should == 0
- end
-end
View
4 spec/models/parsers/dmoz_parser_spec.rb
@@ -15,9 +15,5 @@
first.page_id.should be_nil
first.location_id.should be_nil
first.analyzed_at.should be_nil
-
- first = ListDomain.first
- first.list.should == 'dmoz'
- first.domain.should == Domain.first
end
end
View
15 spec/models/parsers/sk_nic_parser_spec.rb
@@ -8,17 +8,13 @@
Parsers::SkNicParser.perform
Domain.count.should == 62
- ListDomain.count.should == 62
first = Domain.first
first.name.should == '0-0.sk'
+ first.tld.should == 'sk'
first.page_id.should be_nil
first.location_id.should be_nil
- first.analyzed_at.should be_nil
-
- first = ListDomain.first
- first.list.should == 'sk_nic'
- first.domain.should == Domain.first
+ first.analyzed_at.should be_nil
end
it "should destroy domains removed from list" do
@@ -33,17 +29,12 @@
Parsers::SkNicParser.perform
Domain.count.should == 55
- ListDomain.count.should == 55
first = Domain.first
first.name.should == '001.sk'
first.page_id.should be_nil
first.location_id.should be_nil
- first.analyzed_at.should be_nil
-
- first = ListDomain.first
- first.list.should == 'sk_nic'
- first.domain.should == Domain.first
+ first.analyzed_at.should be_nil
Domain.find_by_name(last.name).should == last
end
Please sign in to comment.
Something went wrong with that request. Please try again.