Find file
Fetching contributors…
Cannot retrieve contributors at this time
70 lines (64 sloc) 1.88 KB
#!/usr/bin/ruby
require 'rubygems'
require 'indextank'
require 'json'
require 'yaml'
config = YAML::load(File.open('config.yaml'))
api = IndexTank::Client.new config['api_url']
categories = config['categories'].split('|').map{|x| x.split(",")}
idxname = config['index_name']
index = api.indexes idxname
if not index.exists?
index.add({:public_search => true})
while not index.running?
sleep 0.5
printf "waiting for index %s to be ready...\n", idxname
end
end
printf "Ready.\n"
batches = 0
categories.each do |c|
things = JSON.parse(File.read('data/' +c[1] + '.js'))
count = 0
docs = []
things.each_with_index do |x, i|
description = c[0]
f = 'data/' + c[1] + '/' + x['permalink'] + '.js'
begin
props = JSON.parse(File.read(f))
rescue => e
next
end
if description == "person"
name = props['first_name'] + ' ' + props['last_name']
else
name = props['name']
end
rel = 0
products = 0
overview = Math.log(1 + (props['overview'] or '').length)
if description == "company" #make companies more likely to be a top result
rel = props['relationships'].length
products = props['products'].length
overview *= 2
end
url = props['crunchbase_url']
t = props['image']
if t
thumbnail = 'http://www.crunchbase.com/' + props['image']['available_sizes'][0][-1]
else
thumbnail = 'http://www.gravatar.com/avatar/00000000000000000000000000000000?default=mm'
end
docs << { :docid => url,
:fields => {:name => name, :thumbnail => thumbnail, :url => url, :description => description} ,
:variables => {0 => rel, 1 => products, 2 => overview}}
count +=1
if count == 3000 or i == (things.length - 1)
response = index.batch_insert(docs)
batches += 1
puts "batches: ", batches, docs.length
docs = []
count = 0
end
end
end