-
Notifications
You must be signed in to change notification settings - Fork 0
/
ingest.rb
88 lines (63 loc) · 1.8 KB
/
ingest.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
require 'json'
require 'net/http'
require 'uri'
require 'yaml'
require './gb-concept'
require './gb-genre'
CONFIG = YAML.load(File.open('ingest.yaml'))
GBAPI_GENRES_BASE = "http://www.giantbomb.com/api/genres/"
GBAPI_GAMES_BASE = "http://www.giantbomb.com/api/games/"
GBAPI_CONCEPTS_BASE = "http://www.giantbomb.com/api/concepts/"
def build_gbapi_uri(base, extra_params)
params = {
api_key: CONFIG["api_key"],
format: "json"
}.merge(extra_params)
uri = URI::parse(base)
uri.query = URI.encode_www_form(params)
uri
end
def get_genres
uri = build_gbapi_uri(GBAPI_GENRES_BASE, field_list: "id,name,date_added,site_detail_url,api_detail_url")
resp = nil
begin
resp = Net::HTTP.get_response(uri)
end until resp.code.to_i == 200
json = JSON.load(resp.body)
GBGenre.new_genres_from_genres_json(json)
end
def get_concepts_page(offset)
uri = build_gbapi_uri(GBAPI_CONCEPTS_BASE, field_list: "id,name,date_added,site_detail_url,api_detail_url", offset: offset)
resp = nil
begin
resp = Net::HTTP.get_response(uri)
end until resp.code.to_i == 200
json = JSON.load(resp.body)
return GBConcept.new_concepts_from_concepts_json(json), json["number_of_total_results"]
end
def get_concepts
n_concepts = Float::INFINITY
offset = 0
concepts = []
while offset < n_concepts do
page, last_n = get_concepts_page(offset)
concepts += page
offset += page.length
if n_concepts != last_n then
puts "note: number of concepts changed from #{n_concepts} to #{last_n}"
n_concepts = last_n
end
puts "fetched #{offset} concept names of #{n_concepts}..."
end
concepts
end
# puts get_genres.inspect
puts get_concepts.inspect
genres = get_genres
concepts = get_concepts
File.open('genres.yml', 'w') do |io|
io.write(genres.to_yaml)
end
File.open('concepts.yml', 'w') do |io|
io.write(concepts.to_yaml)
end