Skip to content

Commit

Permalink
Merge branch 'master' of github.com:kkoch986/ExoAPI
Browse files Browse the repository at this point in the history
  • Loading branch information
root committed Oct 2, 2012
2 parents 6e2b6a1 + b8a1cef commit f67452b
Show file tree
Hide file tree
Showing 8 changed files with 689 additions and 240 deletions.
128 changes: 5 additions & 123 deletions app.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,128 +9,6 @@
@mixpanel = Mixpanel::Tracker.new("444779380923a63b1846d1703993333d", request.env, true)
end

#db = Connection.new("localhost", "12002").db('test')
if(production?)
db = Connection.new("localhost", "12002").db('test')
else
db = Connection.new("exoapi.com", "12002").db('test')
end
#db = Connection.new("172.16.3.30","27017").db('test')

###############################################################################################################
##################################### API ... #################################################################
###############################################################################################################

get '/api/planets/search' do

response['Access-Control-Allow-Origin'] = '*'

collection = db.collection('planets')

opts = get_find_opts(params)

# process the parameters into something we can use for search (we would like to enable ranges here)
sort = get_sort_opts(params)

search_p = {}
params.keys.each do |p|
if(p != "fields" && p != "sort" && p != "limit" && p != "start" && p != "jsonp")
# look at the key and see if it contains a range symbol
if(p.split(":").count == 1)
search_p[p] = Float(params[p]) rescue search_p[p] = params[p]
else
# split the operator and field name
parts = p.split(":")
field = parts[0]
operator = parts[1]

begin
search_p[field] = {"$" + operator => Float(params[p])}
rescue
search_p[field] = {"$" + operator => params[p]}
end
end
end
end

planets = collection.find(search_p, opts).sort(sort)

if(!params[:limit].nil?)
planets.limit(params[:limit].to_i)
end

if(!params[:start].nil?)
planets.skip(Integer(params[:start].to_i))
end

@mixpanel.track_event("Planets",
{
:mode => "search",
:opts => opts,
:sort => sort,
:jsonp => params[:jsonp].nil?,
:fields => params[:fields].nil? ? "all" : params[:fields].to_a.join(","),
:ref => request.referrer.nil? ? "none" : request.referrer,
:limit => params[:limit].nil? ? false : params[:limit],
:start => params[:start].nil? ? false : params[:start]
}.merge(search_p))

return_response(planets.to_a, params[:jsonp])
end


get '/api/planets/:id' do

response['Access-Control-Allow-Origin'] = '*'

@mixpanel.append_api("identify", request.ip)

collection = db.collection('planets')

opts = get_find_opts(params)
sort = get_sort_opts(params)
ref = request.referrer.nil? ? "/" : request.referrer

if(params[:id] == "all")
@mixpanel.track_event("Planets",
{
:mode => "all",
:opts => opts,
:sort => sort,
:jsonp => params[:jsonp].nil?,
:ref => request.referrer.nil? ? "none" : request.referrer,
:fields => params[:fields].nil? ? "all" : params[:fields].to_a.join(","),
:limit => params[:limit].nil? ? false : params[:limit],
:start => params[:start].nil? ? false : params[:start]
})
planets = collection.find({}, opts).sort(sort)
else
@mixpanel.track_event("Planets",
{
:mode => "by id",
:opts => opts,
:sort => sort,
:jsonp => params[:jsonp].nil?,
:ref => request.referrer.nil? ? "none" : request.referrer,
:limit => params[:limit].nil? ? false : params[:limit],
:fields => params[:fields].nil? ? "all" : params[:fields].to_a.join(","),
:start => params[:start].nil? ? false : params[:start]
})
planets = collection.find({:_id => params[:id].to_s}, opts).sort(sort)
end


if(!params[:limit].nil?)
planets.limit(params[:limit].to_i)
end

if(!params[:start].nil?)
planets.skip(Integer(params[:start].to_i))
end

return_response(planets.to_a, params[:jsonp])
end

get '/api/debug' do
ref = request.referrer.nil? ? "/" : request.referrer
end
Expand Down Expand Up @@ -185,7 +63,11 @@ def destringify_array(str)
return str.to_s.gsub("[","").gsub("]", "").split(",");
end


###############################################################################################################
##################################### API SITES ###############################################################
###############################################################################################################
require 'v1.rb'
require 'skyhook.rb'

###############################################################################################################
##################################### DEMO SITE VIEWS #########################################################
Expand Down
6 changes: 4 additions & 2 deletions node/process.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ terminal.stdout.on('cd ../exo_data && git pull', function (data) {

/* Connect to Mongo */
var mongodb = require("mongodb"),
mongoserver = new mongodb.Server("localhost", 12002),
mongoserver = new mongodb.Server("localhost", 27017),
//mongoserver = new mongodb.Server("172.16.3.30", 27017),
db_connector = new mongodb.Db("test", mongoserver);

Expand All @@ -24,7 +24,8 @@ setInterval(function(){

function openCollection(err, db)
{
console.log(err);
if(err != undefined)
console.log(err);
db.collection("planets", startParsing);
}

Expand All @@ -50,6 +51,7 @@ function startParsing(err, collection)
delete object.id;
object._id = id;

//collection.update({_id:id}, object, {upsert: true, multi: false, safe: true}, function(err){ console.log(err); });
collection.save(object);
}
console.log("[" + new Date() + "] Complete Update ("+x+") Records Processed");
Expand Down
164 changes: 164 additions & 0 deletions node/process_v2.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
var fs = require('fs');
var libxmljs = require("libxmljs");
var sys = require('util')
var exec = require('child_process').exec;
var child;

function pullUpdate()
{
console.log("Performing Pull Update...");
child = exec("cd ../exo_data && git pull; curl http://nanobio.hpcf.upr.edu/~amendez/phl/phl_hec_all_confirmed.csv -o ../exo_data/data.csv -#", function (error, stdout, stderr) {
console.log('stdout: ' + stdout);
console.log('stderr: ' + stderr + "\n\n");
if (error !== null) {
console.log('exec error: ' + error);
}

console.log("Update Complete.");
// now connect the database
connectDB();

});
}

var mongodb = require("mongodb"),
mongoserver = new mongodb.Server("localhost", 27017),
//mongoserver = new mongodb.Server("172.16.3.30", 27017),
db_connector = new mongodb.Db("skyhook", mongoserver);
function connectDB()
{
/* Connect to Mongo */
var UPDATE_FREQ = 360000;
db_connector.open(openCollection);
setTimeout(pullUpdate, UPDATE_FREQ);
}

function openCollection(err, db)
{
if(err != undefined)
console.log(err);
db.collection("planets", startParsing);
}


function startParsing(err, collection)
{
if(err != undefined)
console.log(err);

var text = fs.readFileSync("../exo_data/data.csv", "UTF-8");
var lines = text.split("\n");

// the first line will be the fields for each successive line
var fields = lines[0].split(",");
for(var x = 0 ; x < fields.length ; x++)
{
var prefix = fields[x].substr(0, 2);
var name = fields[x].substr(3, fields[x].length).replace(/\([a-zA-Z0-9].*\)/, "").trim().split(".").join("").split(" ").join("_");
fields[x] = (prefix + "_" + name).toLowerCase();

// also handle some special cases here i.e. [fe/h]
if(fields[x] == "s._[fe/h]") fields[x] = "s._metallicity";
}

var count = 0;
for(var x = 1; x < lines.length; x++)
{
var values = lines[x].split(",");
// construct an object out of each index in the fields array
if(values.length != fields.length) continue ;

var object = {"star":{}};
for(var y = 0; y < values.length; y++)
{
if(fields[y].substr(0, 3) == "s._")
{
var fname = fields[y].substr(3, fields[y].length);
if(isNumber(values[y])) values[y] = Number(values[y]);
object.star[fname] = values[y];
}
else if(fields[y].substr(0, 3) == "p._")
{
var fname = fields[y].substr(3, fields[y].length);
if(isNumber(values[y])) values[y] = Number(values[y]);
object[fname] = values[y];
}
else
object[fields[y]] = values[y];
}

object._id = object.name;

// now bring in fields from the XML data
// star.angulardistance
// star.magV
// star.magI
// star.magH
// star.magJ
// star.magK
// image
// image description
// multiplicity
// lastupdated?
// new ?
try
{
var text = fs.readFileSync("../exo_data/data/data/" + object._id + ".xml", "UTF-8");
var xml = libxmljs.parseXmlString(text);

object.image = xml.get("//image").text();
object.imagedescription = xml.get("//imagedescription").text();
object.multiplicity = xml.get("//multiplicity").text();
object.star.angulardistance = xml.get("//star/angulardistance").text();

object.star.magV = xml.get("//star/magV").text();
object.star.magI = xml.get("//star/magI").text();
object.star.magH = xml.get("//star/magJ").text();
object.star.magJ = xml.get("//star/magH").text();
object.star.magK = xml.get("//star/magK").text();

// make sure numbers are recorded as numbers not strings
if(isNumber(object.multiplicity)) object.multiplicity = Number(object.multiplicity);
if(isNumber(object.star.angulardistance)) object.star.angulardistance = Number(object.star.angulardistance);
if(isNumber(object.star.magV)) object.star.magV = Number(object.star.magV);
if(isNumber(object.star.magI)) object.star.magI = Number(object.star.magI);
if(isNumber(object.star.magH)) object.star.magH = Number(object.star.magH);
if(isNumber(object.star.magJ)) object.star.magJ = Number(object.star.magJ);
if(isNumber(object.star.magK)) object.star.magK = Number(object.star.magK);

} catch(e) {
// plug in emtpy values for these entrys
object.image = "";
object.imagedescription = "";
object.multiplicity = "";
object.star.angulardistance = "";
object.star.magV = "";
object.star.magI = "";
object.star.magH = "";
object.star.magJ = "";
object.star.magK = "";
}

collection.save(object);

count ++;
}
console.log("Done, " + count + " records processed.");
db_connector.close();
}


function dump(obj) {
var out = '';
for (var i in obj) {
out += i + ": " + obj[i] + "\n";
}

console.log(out);
}

function isNumber(n) {
return !isNaN(parseFloat(n)) && isFinite(n);
}

pullUpdate();
Loading

0 comments on commit f67452b

Please sign in to comment.