diff --git a/bin/import_bd b/bin/import_bd index a92dd074b..ca15948bc 100755 --- a/bin/import_bd +++ b/bin/import_bd @@ -6,6 +6,6 @@ if [ ! -f cache/datastore_bd.xml ]; then wget --ignore-length -O cache/datastore_bd.xml http://datastore.iatistandard.org/api/1/access/activity.xml?recipient-country=bd\&stream=True fi -node serv.js --cmd init -node serv.js --cmd import --xmlfile "cache/datastore_bd.xml" -node serv.js --cmd analyze +node js/cmd init +node js/cmd import "cache/datastore_bd.xml" +node js/cmd analyze diff --git a/bin/import_bd_ug_hn b/bin/import_bd_ug_hn index 21e69ff77..5b5983b17 100755 --- a/bin/import_bd_ug_hn +++ b/bin/import_bd_ug_hn @@ -14,8 +14,8 @@ if [ ! -f cache/datastore_hn.xml ]; then wget --ignore-length -O cache/datastore_hn.xml http://datastore.iatistandard.org/api/1/access/activity.xml?recipient-country=hn\&stream=True fi -node serv.js --cmd init -node serv.js --cmd import --xmlfile "cache/datastore_bd.xml" -node serv.js --cmd import --xmlfile "cache/datastore_ug.xml" -node serv.js --cmd import --xmlfile "cache/datastore_hn.xml" -node serv.js --cmd analyze +node js/cmd init +node js/cmd import "cache/datastore_bd.xml" +node js/cmd import "cache/datastore_ug.xml" +node js/cmd import "cache/datastore_hn.xml" +node js/cmd analyze diff --git a/bin/import_full b/bin/import_full index 85a4d691e..f5c798249 100755 --- a/bin/import_full +++ b/bin/import_full @@ -12,11 +12,11 @@ do fi done -node serv.js --cmd init +node js/cmd init for code in "${codes[@]}" do - node serv.js --cmd import --xmlfile "cache/datastore_${code}.xml" || exit 1 + node js/cmd import "cache/datastore_${code}.xml" || exit 1 done -node serv.js --cmd analyze +node js/cmd analyze diff --git a/dstore/README.md b/dstore/README.md index f902f1433..c0d264d2d 100644 --- a/dstore/README.md +++ b/dstore/README.md @@ -1,7 +1,7 @@ DSTORE ====== -DStore subsidizes the iati-datastore with an optimized nodejs + +DStore subsidises the iati-datastore with an optimised nodejs + SQLite database for use in real time Country Tracker queries. Assuming you are on a Debian derivative. @@ -16,28 +16,32 @@ version of nodejs being installed via apt-get. If you have problems or are not on debian, try building the latest stable version of node rather than using apt-get. -v0.10.24 is current and tested as of the now. +On windows I recommend installing git and node and then using git +bash, the command-line for git to run npm and node as shown bellow. + +v0.10.24 of node is current and tested with this code. Success? Then the following commands can now be run. -NB: There seems to be some confusiuon over the use of node or nodejs -due to package name clashes. Try nodejs if node is not found. +NB: There seems to be some confusion over the use of node or nodejs +due to package name clashes on some distributions. Try nodejs if +node is not found. - node serv.js + node js/serv.js Runs the main server. - node serv.js --port=1337 --database=db/dstore.sqlite + node js/serv.js --port=1337 --database=db/dstore.sqlite Runs the server with some options that could also have been set in the config.json file. - node serv.js --cmd init + node js/cmd.js init Clears the database and creates the default tables ready to be @@ -45,7 +49,7 @@ filled. Alternatively, you could just delete the dstore.sqlite file for a full reset. - node serv.js --cmd import --xmlfile "tmp/bd.xml" + node js/cmd.js import "tmp/bd.xml" Populate the database from just the named xml file which is good for simple tests. @@ -56,6 +60,23 @@ for simple tests. This is a small script that clears the database and then downloads and imports data for Bangladesh, Uganda and Honduras. It's probably best to have a look at this script and see what it does rather than -just run it blindly. +just run it blindly. This script caches downloads so rm the cache +directory to update the data. + + + ../bin/import_full + +This is another script that imports all IATI data from the datastore +and will chug away downloading and processing for a couple of hours. +Otherwise it does the same as the one above. + + +It is recommended that you only import data you wish to use and be +aware that these are just test scripts and will wipe whatever data +is currently in the database before importing. + +They will be replaced with a new import system shortly... + + diff --git a/dstore/changes.txt b/dstore/changes.txt index f694a3bf1..e38fc7482 100644 --- a/dstore/changes.txt +++ b/dstore/changes.txt @@ -1,4 +1,7 @@ +Re jiggled the names in the database slightly and attempted to +optimize for a full data import. + Added orderby result column numbers to Q. Switched from using "xml2js" module to a cooked version of "htmlparse". diff --git a/dstore/dstore b/dstore/dstore new file mode 100755 index 000000000..517b82132 --- /dev/null +++ b/dstore/dstore @@ -0,0 +1,3 @@ +cd `dirname $0` +node js/cmd $* + diff --git a/dstore/js/cmd.js b/dstore/js/cmd.js new file mode 100644 index 000000000..0f0aafe2e --- /dev/null +++ b/dstore/js/cmd.js @@ -0,0 +1,112 @@ +// Copyright (c) 2014 International Aid Transparency Initiative (IATI) +// Licensed under the MIT license whose full text can be found at http://opensource.org/licenses/MIT + +// we expect dstore to be the current directory when this cmd is run +// as we will be creating db/cache directories there + +var wait=require('wait.for'); +var fs = require('fs'); +var express = require('express'); +var util=require('util'); +var path=require('path'); +var app = express(); + +var ls=function(a) { console.log(util.inspect(a,{depth:null})); } + +// global.argv +var argv=require('yargs').argv; global.argv=argv; + +argv.port=argv.port||1337; +argv.database=argv.database||"../dstore/db/dstore.sqlite"; + + +// make sure we have a db dir +fs.mkdir("db",function(e){}); +//ls(argv) +if( argv._[0]=="init" ) +{ + require("./dstore_db").create_tables(); // + return; +} +else +if( argv._[0]=="analyze" ) +{ + require("./dstore_db").analyze(); + return; +} +else +if( argv._[0]=="vacuum" ) +{ + require("./dstore_db").vacuum(); + return; +} +else +if( argv._[0]=="index" ) +{ + require("./dstore_db").create_indexes(); // add indexes to previously inserted data + return; +} +else +if( argv._[0]=="unindex" ) +{ + require("./dstore_db").delete_indexes(); // add indexes to previously inserted data + return; +} +else +if( argv._[0]=="check" ) +{ + require("./dstore_db").check_tables(); + return; +} +else +if( argv._[0]=="exs" ) +{ + wait.launchFiber( require("./dstore_db").hack_exs ); + return; +} +else +if( argv._[0]=="fetch" ) +{ + wait.launchFiber( require("./iati_codes").fetch ); + return; +} +else +if( argv._[0]=="import" ) +{ +// console.log("Attempting Import"); + + var xmlfile=argv._[1]; + var xmlfilename=path.basename(xmlfile,".xml"); + + var fs = require('fs'); + + var data=fs.readFileSync(xmlfile,"UCS-2"); // try 16bit first? + var aa=data.split(//gi)[0]; // trim the end + acts.push(""); // rebuild and add import filename + } + + + console.log("\t\tImporting xmlfile : ("+acts.length+") "+xmlfilename); + +// console.log("activities: "+acts.length); +// console.log(acts[0]); + + + wait.launchFiber(function(){ + require("./dstore_db").fill_acts(acts,xmlfilename); + }); + + return; +} diff --git a/dstore/js/serv.js b/dstore/js/serv.js new file mode 100644 index 000000000..e6b78d619 --- /dev/null +++ b/dstore/js/serv.js @@ -0,0 +1,40 @@ +// Copyright (c) 2014 International Aid Transparency Initiative (IATI) +// Licensed under the MIT license whose full text can be found at http://opensource.org/licenses/MIT + +var wait=require('wait.for'); +var fs = require('fs'); +var express = require('express'); +var util=require('util'); +var path=require('path'); +var app = express(); + +var ls=function(a) { console.log(util.inspect(a,{depth:null})); } + +// global.argv +var argv=require('yargs').argv; global.argv=argv; + +argv.port=argv.port||1337; +argv.database=argv.database||"../dstore/db/dstore.sqlite"; + + +// make sure we have a db dir +fs.mkdir("db",function(e){}); + + +app.use(express.logger()); +app.use(express.json()); + +//app.use("/"); + +app.use("/q",function (req, res) { + require("./query").serv(req,res); +}); + +app.use(express.compress()); +app.use(express.static(__dirname+"/../ctrack")); + +console.log("Starting dstore server at http://localhost:"+argv.port+"/"); + +app.listen(argv.port); + + diff --git a/dstore/serv.js b/dstore/serv.js deleted file mode 100644 index 3aace1c15..000000000 --- a/dstore/serv.js +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright (c) 2014 International Aid Transparency Initiative (IATI) -// Licensed under the MIT license whose full text can be found at http://opensource.org/licenses/MIT - -var wait=require('wait.for'); -var fs = require('fs'); -var express = require('express'); -var util=require('util'); -var path=require('path'); -var app = express(); - -var ls=function(a) { console.log(util.inspect(a,{depth:null})); } - -// global.argv -var argv=require('yargs').argv; global.argv=argv; - -argv.port=argv.port||1337; -argv.database=argv.database||"../dstore/db/dstore.sqlite"; - - -// make sure we have a db dir -fs.mkdir("db",function(e){}); - -if(argv.cmd) -{ -// console.log("cmd found : "+cmd); - if( argv.cmd=="init" ) - { - require("./js/dstore_db").create_tables(); // - return; - } - else - if( argv.cmd=="analyze" ) - { - require("./js/dstore_db").analyze(); - return; - } - else - if( argv.cmd=="vacuum" ) - { - require("./js/dstore_db").vacuum(); - return; - } - else - if( argv.cmd=="index" ) - { - require("./js/dstore_db").create_indexes(); // add indexes to previously inserted data - return; - } - else - if( argv.cmd=="unindex" ) - { - require("./js/dstore_db").delete_indexes(); // add indexes to previously inserted data - return; - } - else - if( argv.cmd=="check" ) - { - require("./js/dstore_db").check_tables(); - return; - } - else - if( argv.cmd=="exs" ) - { - wait.launchFiber( require("./js/dstore_db").hack_exs ); - return; - } - else - if( argv.cmd=="fetch" ) - { - wait.launchFiber( require("./js/iati_codes").fetch ); - return; - } - else - if( argv.cmd=="import" ) - { -// console.log("Attempting Import"); - - var xmlfile=argv.xmlfile; - var xmlfilename=path.basename(xmlfile,".xml"); - - var fs = require('fs'); - - var data=fs.readFileSync(xmlfile,"UCS-2"); // try 16bit first? - var aa=data.split(//gi)[0]; // trim the end - acts.push(""); // rebuild and add import filename - } - - - console.log("\t\tImporting xmlfile : ("+acts.length+") "+xmlfilename); - -// console.log("activities: "+acts.length); -// console.log(acts[0]); - - - wait.launchFiber(function(){ - require("./js/dstore_db").fill_acts(acts,xmlfilename); - }); - - return; - } -} - - - - - -app.use(express.logger()); -app.use(express.json()); - -//app.use("/"); - -app.use("/q",function (req, res) { - require("./js/query").serv(req,res); -}); - -app.use(express.compress()); -app.use(express.static(__dirname+"/../ctrack")); - -console.log("Starting dstore server at http://localhost:"+argv.port+"/"); - -app.listen(argv.port);