tidy up of dstore cmd interface

bin/import_bd

@@ -6,6 +6,6 @@ if [ ! -f cache/datastore_bd.xml ]; then
    wget --ignore-length -O cache/datastore_bd.xml http://datastore.iatistandard.org/api/1/access/activity.xml?recipient-country=bd\&stream=True
 fi
 
-node serv.js --cmd init
-node serv.js --cmd import --xmlfile "cache/datastore_bd.xml"
-node serv.js --cmd analyze
+node js/cmd init
+node js/cmd import "cache/datastore_bd.xml"
+node js/cmd analyze

bin/import_bd_ug_hn

@@ -14,8 +14,8 @@ if [ ! -f cache/datastore_hn.xml ]; then
    wget --ignore-length -O cache/datastore_hn.xml http://datastore.iatistandard.org/api/1/access/activity.xml?recipient-country=hn\&stream=True
 fi
 
-node serv.js --cmd init
-node serv.js --cmd import --xmlfile "cache/datastore_bd.xml"
-node serv.js --cmd import --xmlfile "cache/datastore_ug.xml"
-node serv.js --cmd import --xmlfile "cache/datastore_hn.xml"
-node serv.js --cmd analyze
+node js/cmd init
+node js/cmd import "cache/datastore_bd.xml"
+node js/cmd import "cache/datastore_ug.xml"
+node js/cmd import "cache/datastore_hn.xml"
+node js/cmd analyze

bin/import_full

@@ -12,11 +12,11 @@ do
   fi
 done
 
-node serv.js --cmd init
+node js/cmd init
 
 for code in "${codes[@]}"
 do
-  node serv.js --cmd import --xmlfile "cache/datastore_${code}.xml" || exit 1
+  node js/cmd import "cache/datastore_${code}.xml" || exit 1
 done
 
-node serv.js --cmd analyze
+node js/cmd analyze

dstore/README.md

@@ -1,7 +1,7 @@
 DSTORE
 ======
 
-DStore subsidizes the iati-datastore with an optimized nodejs + 
+DStore subsidises the iati-datastore with an optimised nodejs + 
 SQLite database for use in real time Country Tracker queries.
 
 Assuming you are on a Debian derivative.
@@ -16,36 +16,40 @@ version of nodejs being installed via apt-get. If you have problems
 or are not on debian, try building the latest stable version of 
 node rather than using apt-get.
 
-v0.10.24 is current and tested as of the now.
+On windows I recommend installing git and node and then using git 
+bash, the command-line for git to run npm and node as shown bellow.
+
+v0.10.24 of node is current and tested with this code.
 
 Success?
 
 Then the following commands can now be run.
 
-NB: There seems to be some confusiuon over the use of node or nodejs 
-due to package name clashes. Try nodejs if node is not found.
+NB: There seems to be some confusion over the use of node or nodejs 
+due to package name clashes on some distributions. Try nodejs if 
+node is not found.
 
 
-	node serv.js
+	node js/serv.js
 
 Runs the main server.
 
 
-	node serv.js --port=1337 --database=db/dstore.sqlite
+	node js/serv.js --port=1337 --database=db/dstore.sqlite
 
 Runs the server with some options that could also have been set in 
 the config.json file.
 
 
-	node serv.js --cmd init
+	node js/cmd.js init
 
 
 Clears the database and creates the default tables ready to be 
 filled. Alternatively, you could just delete the dstore.sqlite file 
 for a full reset.
 
 
-	node serv.js --cmd import --xmlfile "tmp/bd.xml"
+	node js/cmd.js import "tmp/bd.xml"
 
 Populate the database from just the named xml file which is good 
 for simple tests.
@@ -56,6 +60,23 @@ for simple tests.
 This is a small script that clears the database and then downloads 
 and imports data for Bangladesh, Uganda and Honduras. It's probably 
 best to have a look at this script and see what it does rather than 
-just run it blindly.
+just run it blindly. This script caches downloads so rm the cache 
+directory to update the data.
+
+
+	../bin/import_full
+
+This is another script that imports all IATI data from the datastore 
+and will chug away downloading and processing for a couple of hours. 
+Otherwise it does the same as the one above.
+
+
+It is recommended that you only import data you wish to use and be 
+aware that these are just test scripts and will wipe whatever data 
+is currently in the database before importing.
+
+They will be replaced with a new import system shortly...
+
+
 
 

dstore/changes.txt

@@ -1,4 +1,7 @@
 
+Re jiggled the names in the database slightly and attempted to 
+optimize for a full data import.
+
 Added orderby result column numbers to Q.
 
 Switched from using "xml2js" module to a cooked version of "htmlparse".

dstore/dstore

@@ -0,0 +1,3 @@
+cd `dirname $0`
+node js/cmd $*
+

dstore/js/cmd.js

@@ -0,0 +1,112 @@
+// Copyright (c) 2014 International Aid Transparency Initiative (IATI)
+// Licensed under the MIT license whose full text can be found at http://opensource.org/licenses/MIT
+
+// we expect dstore to be the current directory when this cmd is run
+// as we will be creating db/cache directories there
+
+var wait=require('wait.for');
+var fs = require('fs');
+var express = require('express');
+var util=require('util');
+var path=require('path');
+var app = express();
+
+var ls=function(a) { console.log(util.inspect(a,{depth:null})); }
+
+// global.argv
+var argv=require('yargs').argv; global.argv=argv;
+
+argv.port=argv.port||1337;
+argv.database=argv.database||"../dstore/db/dstore.sqlite";
+
+
+// make sure we have a db dir
+fs.mkdir("db",function(e){});
+//ls(argv)
+if( argv._[0]=="init" )
+{
+	require("./dstore_db").create_tables(); // 
+	return;
+}
+else
+if( argv._[0]=="analyze" )
+{
+	require("./dstore_db").analyze();
+	return;
+}
+else
+if( argv._[0]=="vacuum" )
+{
+	require("./dstore_db").vacuum();
+	return;
+}
+else
+if( argv._[0]=="index" )
+{
+	require("./dstore_db").create_indexes(); // add indexes to previously inserted data
+	return;
+}
+else
+if( argv._[0]=="unindex" )
+{
+	require("./dstore_db").delete_indexes(); // add indexes to previously inserted data
+	return;
+}
+else
+if( argv._[0]=="check" )
+{
+	require("./dstore_db").check_tables();
+	return;
+}
+else
+if( argv._[0]=="exs" )
+{
+	wait.launchFiber( require("./dstore_db").hack_exs );
+	return;
+}
+else
+if( argv._[0]=="fetch" )
+{
+	wait.launchFiber( require("./iati_codes").fetch );
+	return;
+}
+else
+if( argv._[0]=="import" )
+{
+//		console.log("Attempting Import");
+
+	var xmlfile=argv._[1];
+	var xmlfilename=path.basename(xmlfile,".xml");
+
+	var fs = require('fs');
+
+	var data=fs.readFileSync(xmlfile,"UCS-2"); // try 16bit first?
+	var aa=data.split(/<iati-activity/gi);
+	if(aa.length==1) // nothing found so try utf8
+	{
+		data=fs.readFileSync(xmlfile,"utf8");
+		aa=data.split(/<iati-activity/gi);
+	}
+
+//ls(aa);			
+	var acts=[];
+	for(var i=1;i<aa.length;i++)
+	{
+		var v=aa[i];
+		var v=v.split(/<\/iati-activity>/gi)[0]; // trim the end
+		acts.push("<iati-activity dstore:slug=\""+xmlfilename+"\" dstore:idx=\""+i+"\" "+v+"</iati-activity>"); // rebuild and add import filename
+	}
+
+
+	console.log("\t\tImporting xmlfile : ("+acts.length+") "+xmlfilename);
+
+//		console.log("activities: "+acts.length);
+//			console.log(acts[0]);
+
+
+	wait.launchFiber(function(){
+		require("./dstore_db").fill_acts(acts,xmlfilename);
+	});
+
+	return;		
+}

dstore/js/serv.js

@@ -0,0 +1,40 @@
+// Copyright (c) 2014 International Aid Transparency Initiative (IATI)
+// Licensed under the MIT license whose full text can be found at http://opensource.org/licenses/MIT
+
+var wait=require('wait.for');
+var fs = require('fs');
+var express = require('express');
+var util=require('util');
+var path=require('path');
+var app = express();
+
+var ls=function(a) { console.log(util.inspect(a,{depth:null})); }
+
+// global.argv
+var argv=require('yargs').argv; global.argv=argv;
+
+argv.port=argv.port||1337;
+argv.database=argv.database||"../dstore/db/dstore.sqlite";
+
+
+// make sure we have a db dir
+fs.mkdir("db",function(e){});
+
+
+app.use(express.logger());
+app.use(express.json());
+
+//app.use("/");
+
+app.use("/q",function (req, res) {
+	require("./query").serv(req,res);
+});
+
+app.use(express.compress());
+app.use(express.static(__dirname+"/../ctrack"));
+
+console.log("Starting dstore server at http://localhost:"+argv.port+"/");
+
+app.listen(argv.port);
+
+