Skip to content
Permalink
Browse files

Fiddled around with rubocop, and let it reformat what we have so far.…

…..not sure I'm happy about the single-quote strings but oh well
  • Loading branch information...
Martin Tepper
Martin Tepper committed Nov 5, 2019
1 parent b36f992 commit 1784bd1b18db4230b1b099f1c943ea5d7b883413
Showing with 24 additions and 22 deletions.
  1. +4 −4 ruby/Gemfile
  2. +20 −18 ruby/scrape_crowdfunder
@@ -1,8 +1,8 @@
# frozen_string_literal: true

source "https://rubygems.org"
source 'https://rubygems.org'

git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }

gem "httpx"
gem "nokogiri"
gem 'httpx'
gem 'nokogiri'
@@ -1,30 +1,31 @@
#!/usr/bin/env ruby
# frozen_string_literal: true

require 'rubygems'
require 'bundler/setup'
require 'httpx'
require 'nokogiri'
require 'pp'

index_url = ENV["CROWDFUNDER_PROJECTS_URL"]
index_url = ENV['CROWDFUNDER_PROJECTS_URL']
index_html = HTTPX.get(index_url).to_s
index = Nokogiri.parse(index_html)

def get_n_parse(url)
res = HTTPX.get(url)
unless res.status == 200
puts "AAAAAAAAAA HTTP error for #{url} - #{res.status}"
return nil
end
res = HTTPX.get(url)
unless res.status == 200
puts "AAAAAAAAAA HTTP error for #{url} - #{res.status}"
return nil
end
Nokogiri.parse(res.body.to_s)
end

def parse_detail_page_urls(page)
page.css(".campaign-details a.campaign-link").map { |a| a[:href] }
page.css('.campaign-details a.campaign-link').map { |a| a[:href] }
end

pages = [index]
page_urls = index.css("ul.pagination a.page-link").map { |a| a[:href] }
page_urls = index.css('ul.pagination a.page-link').map { |a| a[:href] }
pp(page_urls)
page_urls.each do |page_url|
pages << get_n_parse(page_url)
@@ -41,22 +42,23 @@ pp(detail_urls)

campaigns = detail_urls.map do |detail_url|
puts detail_url
page = get_n_parse(detail_url)
page = get_n_parse(detail_url)
next unless page
#puts page.css("h5.campaign-goal").text
campaign_goal = Integer(page.css("h5.campaign-goal").text.gsub(/€|,/, ""))
remaining_amount = Integer(page.css("p.remaining-amount").inner_html.gsub(",", "").scan(/€(\d+)?\s/m).flatten.first)

# puts page.css("h5.campaign-goal").text
campaign_goal = Integer(page.css('h5.campaign-goal').text.gsub(/€|,/, ''))
remaining_amount = Integer(page.css('p.remaining-amount').inner_html.gsub(',', '').scan(/€(\d+)?\s/m).flatten.first)
{
url: detail_url,
campaign_goal: campaign_goal,
remaining_amount: remaining_amount,
url: detail_url,
campaign_goal: campaign_goal,
remaining_amount: remaining_amount
}
end.compact

pp(campaigns)

count = campaigns.size
total = campaigns.inject(0) { |t, n| t + n[:campaign_goal]}
remaining = campaigns.inject(0) { |t, n| t + n[:remaining_amount]}
total = campaigns.inject(0) { |t, n| t + n[:campaign_goal] }
remaining = campaigns.inject(0) { |t, n| t + n[:remaining_amount] }

puts "#{count} campaigns, #{total}€ total, #{remaining}€ remaining, #{total - remaining}€ earned"
puts "#{count} campaigns, #{total}€ total, #{remaining}€ remaining, #{total - remaining}€ earned"

0 comments on commit 1784bd1

Please sign in to comment.
You can’t perform that action at this time.