Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
[FIX] fixing league fetcher
  • Loading branch information
Elad Meidar committed Jul 3, 2014
1 parent 1c66c02 commit c7c4e52
Show file tree
Hide file tree
Showing 8 changed files with 3,738 additions and 16 deletions.
1 change: 1 addition & 0 deletions lib/transfermarkt.rb
Expand Up @@ -3,6 +3,7 @@
module Transfermarkt
require 'httparty'
require 'nokogiri'
require 'useragents'

autoload :EntityBase, 'transfermarkt/entity_base'
autoload :Player, 'transfermarkt/player'
Expand Down
2 changes: 1 addition & 1 deletion lib/transfermarkt/club.rb
Expand Up @@ -9,7 +9,7 @@ class Club < Transfermarkt::EntityBase
def self.fetch_by_club_uri(club_uri, fetch_players = false)
puts "fetching club #{club_uri}"

req = self.get("/#{club_uri}", headers: {"User-Agent" => Transfermarkt::USER_AGENT})
req = self.get("/#{club_uri}", headers: {"User-Agent" => UserAgents.rand()})
if req.code != 200
nil
else
Expand Down
22 changes: 11 additions & 11 deletions lib/transfermarkt/league.rb
Expand Up @@ -8,19 +8,19 @@ class League < Transfermarkt::EntityBase
:club_uris

def self.fetch_clubs_and_uris_by_league_uri(league_uri)
req = self.get("/#{league_uri}", headers: {"User-Agent" => Transfermarkt::USER_AGENT})
req = self.get("/#{league_uri}", headers: {"User-Agent" => ::UserAgents.rand()})
if req.code != 200
nil
else
league_html = Nokogiri::HTML(req.parsed_response)
options = {}

options[:league_uri] = league_uri
options[:name] = league_html.xpath('//*[@id="wb_seite"]/table/tr[1]/td[2]/h1/text()').text.strip.gsub(" -","")
options[:country] = league_html.xpath('//*[@id="wb_seite"]/table/tr[1]/td[2]/h1/a').text
options[:name] = league_html.xpath('//select[@id="wettbewerb_select_breadcrumb"]//option[@selected="selected"]')[0].text
options[:country] = league_html.xpath('//select[@id="land_select_breadcrumb"]//option[@selected="selected"]').text

club_uris = league_html.xpath('//table[@id="vereine"]//tr//td[2]//a[@class="s10"]').collect{|player_html| player_html["href"]}
club_names = league_html.xpath('//table[@id="vereine"]//tr//td[2]//a[@class="s10"]').collect{|player_html| player_html.text }
club_uris = league_html.xpath('//*[@id="yw1"]//table//tr//td[2]//a[1]').collect{|player_html| player_html["href"]}
club_names = league_html.xpath('//*[@id="yw1"]//table//tr//td[2]//a[1]').collect{|player_html| player_html.text }

clubs = Hash[club_names.zip(club_uris)]

Expand All @@ -32,18 +32,18 @@ def self.fetch_clubs_and_uris_by_league_uri(league_uri)
def self.fetch_by_league_uri(league_uri, fetch_clubs = false)
puts "fetching league #{league_uri}"

req = self.get("/#{league_uri}", headers: {"User-Agent" => Transfermarkt::USER_AGENT})
req = self.get("/#{league_uri}", headers: {"User-Agent" => Useragents.rand()})
if req.code != 200
nil
else
league_html = Nokogiri::HTML(req.parsed_response)
options = {}

options[:league_uri] = league_uri
options[:name] = league_html.xpath('//*[@id="wb_seite"]/table/tr[1]/td[2]/h1/text()').text.strip.gsub(" -","")
options[:country] = league_html.xpath('//*[@id="wb_seite"]/table/tr[1]/td[2]/h1/a').text
options[:name] = league_html.xpath('//select[@id="wettbewerb_select_breadcrumb"]//option[@selected="selected"]')[0].text
options[:country] = league_html.xpath('//select[@id="land_select_breadcrumb"]//option[@selected="selected"]').text

options[:club_uris] = league_html.xpath('//table[@id="vereine"]//tr//td[2]//a[@class="s10"]').collect{|player_html| player_html["href"]}
options[:club_uris] = league_html.xpath('//*[@id="yw1"]//table//tr//td[2]//a[1]').collect{|player_html| player_html["href"]}

puts "Found #{options[:club_uris].count} clubs"
options[:clubs] = []
Expand All @@ -62,12 +62,12 @@ def self.fetch_by_league_uri(league_uri, fetch_clubs = false)

def self.fetch_league_uris
root_uri = "/en/ligat-haal/startseite/wettbewerb_ISR1.html"
req = self.get("/#{root_uri}", headers: {"User-Agent" => Transfermarkt::USER_AGENT})
req = self.get("/#{root_uri}", headers: {"User-Agent" => UserAgents.rand()})
if req.code != 200
nil
else
root_html = Nokogiri::HTML(req.parsed_response)
league_uris = root_html.xpath('//*[@id="categorymenu"]/li/ul/li/a').collect{|league| league["href"]}
league_uris = root_html.xpath('//*[@id="yw1"]//table//tr//td[2]//a[1]').collect{|player_html| player_html["href"]}
end
end
end
Expand Down
2 changes: 1 addition & 1 deletion lib/transfermarkt/live_game.rb
Expand Up @@ -5,7 +5,7 @@ class LiveGame
URL = "http://www.transfermarkt.com/en/livescores-und-livetabellen/uebersicht/livescores.html"

def self.fetch
req = self.get(URL, headers: {"User-Agent" => Transfermarkt::USER_AGENT})
req = self.get(URL, headers: {"User-Agent" => UserAgents.rand()})
if req.code != 200
nil
else
Expand Down
6 changes: 3 additions & 3 deletions lib/transfermarkt/player.rb
Expand Up @@ -33,7 +33,7 @@ def initialize(options = {})
def self.fetch_by_profile_uri(profile_uri = "")
puts "fetching player profile #{profile_uri}"

req = self.get("/#{profile_uri}", headers: {"User-Agent" => Transfermarkt::USER_AGENT})
req = self.get("/#{profile_uri}", headers: {"User-Agent" => UserAgents.rand()})
if req.code != 200
nil
else
Expand Down Expand Up @@ -89,7 +89,7 @@ def self.fetch_by_profile_uri(profile_uri = "")
private
def self.fetch_performance_data(performance_uri, is_goalkeeper = false)
puts "Fetching Performance page for #{performance_uri}"
req = self.get("/#{performance_uri}", headers: {"User-Agent" => Transfermarkt::USER_AGENT})
req = self.get("/#{performance_uri}", headers: {"User-Agent" => UserAgents.rand()})
if req.code != 200
nil
else
Expand All @@ -116,7 +116,7 @@ def self.fetch_performance_data(performance_uri, is_goalkeeper = false)
end

def self.fetch_injuries_data(injury_uri)
req = self.get("/#{injury_uri}", headers: {"User-Agent" => Transfermarkt::USER_AGENT})
req = self.get("/#{injury_uri}", headers: {"User-Agent" => UserAgents.rand()})
if req.code != 200
[]
else
Expand Down

0 comments on commit c7c4e52

Please sign in to comment.