From d2f7f1b9261db3921a02aa7b7eefcb465c9d9f36 Mon Sep 17 00:00:00 2001 From: Rob Ennals Date: Tue, 12 May 2009 17:47:37 -0700 Subject: [PATCH] Wrote special Turk interface to allow Turk users to create claims and find evidence and snippets --- client_js/client.js | 89 +++++ firefox_plugin/chrome/content/marker.js | 177 ++++++++++ firefox_plugin/chrome/content/robjson.js | 158 +++++++++ firefox_plugin/marker.js | 44 +++ .../src/com/intel/thinkscala/Datastore.scala | 27 +- .../com/intel/thinkscala/MainServlet.scala | 39 ++- .../src/com/intel/thinkscala/SnipSearch.scala | 12 +- .../com/intel/thinkscala/SqlStatement.scala | 4 +- .../src/com/intel/thinkscala/UrlHandler.scala | 28 +- .../src/com/intel/thinkscala/view/Page.scala | 27 +- .../com/intel/thinkscala/view/Render.scala | 16 +- .../src/com/intel/thinkscala/view/Turk.scala | 42 +++ .../com/intel/thinkscala/view/Widgets.scala | 100 ++++++ turk/props.txt | 8 + webapps/node/WEB-INF/web.xml | 6 + webapps/node/javascript/standard.js | 16 + webapps/node/javascript/turk.js | 328 ++++++++++++++++++ webapps/node/stylesheets/normal.css | 60 +++- webapps/node/stylesheets/turk.css | 150 ++++++++ website/Turker_Instructions.txt | 19 + 20 files changed, 1306 insertions(+), 44 deletions(-) create mode 100644 client_js/client.js create mode 100644 firefox_plugin/chrome/content/marker.js create mode 100644 firefox_plugin/chrome/content/robjson.js create mode 100644 firefox_plugin/marker.js create mode 100644 scala/src/com/intel/thinkscala/view/Turk.scala create mode 100644 scala/src/com/intel/thinkscala/view/Widgets.scala create mode 100644 webapps/node/javascript/turk.js create mode 100644 webapps/node/stylesheets/turk.css create mode 100644 website/Turker_Instructions.txt diff --git a/client_js/client.js b/client_js/client.js new file mode 100644 index 0000000..7491480 --- /dev/null +++ b/client_js/client.js @@ -0,0 +1,89 @@ + +function thinklink_viewClaim(id) { + var apipath = get_api_path(); + var url = apipath+"/claim/"+id; + var that = this; + + if(content.document.getElementById("tl_point_browser")){ + return; + } + + var win = document.createElement("div"); + win.setAttribute("id","tl_point_browser"); + win.className = "tl_dialog"; + win.style.zIndex = "214783647"; + content.document.body.appendChild(win); + + var fader = this.addFader(win); + + win.style.overflow = "hidden"; + win.style.position = "fixed"; + win.style.top = "50px"; + win.style.left = "200px"; + win.style.width = "435px"; + //Math.min((window.innerWidth - 250),550) + "px"; + + var titleBar = content.document.createElement("div"); + win.appendChild(titleBar); + titleBar.setAttribute("id","tl_pb_title"); + titleBar.style.marginBottom = "0px"; + titleBar.style.cursor = "move"; + titleBar.addEventListener("mousedown",function(ev){ + tl_dragStart(ev,that.divID,"tl_point_frame"); + },true); + titleBar.className = "tl_dialog_title"; + + var buttonBox = document.createElement("span"); + buttonBox.style.position = "absolute"; + buttonBox.style.right = "4px"; + titleBar.appendChild(buttonBox); + + var ignoreButton = document.createElement("input"); + ignoreButton.className = "tl_openbutton"; + ignoreButton.setAttribute("type","button"); + ignoreButton.setAttribute("value","Don't highlight this again"); + buttonBox.appendChild(ignoreButton); + ignoreButton.addEventListener("click",function(){ + that.hideMe(); + tl_doAJAX("tl_ignore","scripthack/ignoreclaim.js"+ + "?claim="+snippet.claimid,function(){}); + },true); + + var openButton = document.createElement("input"); + openButton.className = "tl_openbutton"; + openButton.setAttribute("type","button"); + openButton.setAttribute("value","Open Full Interface"); + buttonBox.appendChild(openButton); + openButton.addEventListener("click",function(){ + window.open(thinklink_mainhome); + },true); + + var close = document.createElement("img"); + close.style.width = "64px"; + close.style.paddingTop = "2px"; + close.setAttribute("src",thinklink_imagebase+"bigcancel.png"); + buttonBox.appendChild(close); + close.addEventListener("click",function(){ + that.hideMe(); + content.document.body.removeChild(fader); + },true); + + // add actual content + var frameholder = document.createElement("div"); + frameholder.style.height = "430px"; + + var pointframe = document.createElement("iframe"); + pointframe.src = url; + pointframe.style.width="100%"; + pointframe.style.height="100%"; + pointframe.style.overflow = "hidden"; + pointframe.style.border = "none"; + pointframe.setAttribute("id","tl_point_frame"); + pointframe.setAttribute("allowtransparency","true"); + frameholder.appendChild(pointframe); + frameholder.style.width="100%"; + win.appendChild(frameholder); + + win.style.visibility = "visible"; + win.style.display = "block"; +} diff --git a/firefox_plugin/chrome/content/marker.js b/firefox_plugin/chrome/content/marker.js new file mode 100644 index 0000000..7f70674 --- /dev/null +++ b/firefox_plugin/chrome/content/marker.js @@ -0,0 +1,177 @@ + +var thinklink_imagebase = "http://factextract.cs.berkeley.edu:8180/thinklink/images/" + +function tl_log(msg){ + var consoleService = Components.classes["@mozilla.org/consoleservice;1"].getService(Components.interfaces.nsIConsoleService); + consoleService.logStringMessage("Think Link: " + msg) +} + +function get_api_path(){ + var prefs = Components.classes["@mozilla.org/preferences-service;1"].getService(Components.interfaces.nsIPrefBranch); + var apipath = "http://factextract.cs.berkeley.edu/thinklink"; + if(prefs.prefHasUserValue("extensions.thinklink.api")){ + apipath = prefs.getCharPref("extensions.thinklink.api"); + } + return apipath; +} + +function mark_snippets(){ + tl_log("mark snippets"); + var targeturl = content.document.location.href; + var apipath = get_api_path(); + var url = apipath+"/apianon/search.json?url="+encodeURIComponent(targeturl); + tl_log(url); + ajaxRequest(url,function(snippets){ + tl_log("received "+snippets.length+" snippets"); + for(var i = 0; i < snippets.length; i++){ + var snip = snippets[i]; + tl_log(snip.text); + var frags = snip.text.split(/[\.\n\?\!]/) + for(var j = 0; j < frags.length; j++){ + if(frags[j].length > 10){ + mark_snippet(normalise(frags[j]),snip.claimid,snip.claimtext,content.document.body); + } + } + } + }) +} + +function normalise(str){ + return str.replace(/[^\w]/g,"") +} + +function mark_snippet(text,claimid,claimtext,node){ + if(node.nodeName == "#comment" || normalise(node.textContent).indexOf(text) == -1){ + return; + }else if (node.nodeName == "#text") { + node.parentNode.style.backgroundColor = "#FFD3D3"; + node.parentNode.style.cursor = "pointer"; + node.parentNode.setAttribute("title",claimtext); + node.parentNode.addEventListener("click",function(){ + viewClaim(claimid); + },true); + }else{ + for(var i = 0; i < node.childNodes.length; i++){ + var child = node.childNodes[i]; + mark_snippet(text,claimid,claimtext,child); + } + } +} + +function ajaxRequest(url,callback){ + var req = new XMLHttpRequest(); + req.open("GET",url,true); + req.onreadystatechange = function(){ + if(req.readyState == 4 && req.status == 200){ + tl_log("json = "+req.responseText); + callback(parseJSON(req.responseText)) + } + } + req.send(null); +} + + +function addFader(viewframe){ + var fader = content.document.createElement("div"); + with(fader.style){ + backgroundColor = "black"; + opacity = 0.3; + display = "block"; + height = "100%"; + width = "100%"; + position = "fixed"; + left = "0px"; + top = "0px"; + zIndex = "1000"; + } + fader.addEventListener("click",function(ev){ + content.document.body.removeChild(fader); + content.document.body.removeChild(viewframe); + },true); + content.document.body.appendChild(fader); + return fader; +} + +function viewClaim(id) { + var doc = content.document; + tl_log("viewclaim"); + var apipath = get_api_path(); + var url = apipath+"/node/"+id; + var that = this; + + if(doc.getElementById("tl_point_browser")){ + return; + } + + var win = doc.createElement("div"); + win.setAttribute("id","tl_point_browser"); + win.className = "tl_dialog"; + win.style.zIndex = "214783647"; + doc.body.appendChild(win); + + var fader = this.addFader(win); + + win.style.overflow = "hidden"; + win.style.position = "fixed"; + win.style.top = "50px"; + win.style.left = "200px"; + win.style.width = "435px"; + + var titleBar = doc.createElement("div"); + win.appendChild(titleBar); + titleBar.setAttribute("id","tl_pb_title"); + titleBar.style.marginBottom = "0px"; + titleBar.style.cursor = "move"; + titleBar.addEventListener("mousedown",function(ev){ + tl_dragStart(ev,that.divID,"tl_point_frame"); + },true); + titleBar.className = "tl_dialog_title"; + + var buttonBox = doc.createElement("span"); + buttonBox.style.position = "absolute"; + buttonBox.style.right = "4px"; + titleBar.appendChild(buttonBox); + + var ignoreButton = doc.createElement("input"); + ignoreButton.className = "tl_openbutton"; + ignoreButton.setAttribute("type","button"); + ignoreButton.setAttribute("value","Don't highlight again"); + buttonBox.appendChild(ignoreButton); + ignoreButton.addEventListener("click",function(){ + that.hideMe(); + tl_doAJAX("tl_ignore","scripthack/ignoreclaim.js"+ + "?claim="+snippet.claimid,function(){}); + },true); + + var close = doc.createElement("img"); + close.style.width = "64px"; + close.style.paddingTop = "2px"; + close.style.verticalAlign = "top"; + close.style.cursor = "pointer"; + close.setAttribute("src",thinklink_imagebase+"bigcancel.png"); + buttonBox.appendChild(close); + close.addEventListener("click",function(){ + content.document.body.removeChild(win); + content.document.body.removeChild(fader); + },true); + + // add actual content + var frameholder = doc.createElement("div"); + frameholder.style.height = "430px"; + frameholder.style.marginTop = "26px"; + + var pointframe = doc.createElement("iframe"); + pointframe.src = url; + pointframe.style.width="100%"; + pointframe.style.height="100%"; + pointframe.style.overflow = "hidden"; + pointframe.style.border = "none"; + pointframe.setAttribute("id","tl_point_frame"); + pointframe.setAttribute("allowtransparency","true"); + frameholder.appendChild(pointframe); + frameholder.style.width="100%"; + win.appendChild(frameholder); + + win.style.visibility = "visible"; + win.style.display = "block"; +} diff --git a/firefox_plugin/chrome/content/robjson.js b/firefox_plugin/chrome/content/robjson.js new file mode 100644 index 0000000..c3f54b1 --- /dev/null +++ b/firefox_plugin/chrome/content/robjson.js @@ -0,0 +1,158 @@ + +/* The JSON prototype approach breaks firefox extensions. Here is a less intrusive version */ + +/* + json.js + 2007-08-05 + + Public Domain + + This is a rewritten version of the JSON printer/parser, by Rob Ennals. + It started out as being the original JSON parser but is now quite different. + The rewrite was done because the normal JSON parser breaks for...in loops, + including those used by internal FireFox code. + The main other change is that we strip out "null" and "false" values. +*/ + + +var json_subs = { + '\b': '\\b', + '\t': '\\t', + '\n': '\\n', + '\f': '\\f', + '\r': '\\r', + '"' : '\\"', + '\\': '\\\\' + }; + +function makeJSONString(obj){ + if(typeof obj.sort == 'function'){ + var a = [], // The array holding the partial texts. + i, // Loop counter. + l = obj.length, + v; // The value to be stringified. + + + for (i = 0; i < l; i += 1) { + v = obj[i]; + switch (typeof v) { + case 'object': + if (v) { + a.push(makeJSONString(v)); + } else { + a.push('null'); + } + break; + + case 'string': + case 'number': + case 'boolean': + a.push(makeJSONString(v)); + } + } + +// Join all of the member texts together and wrap them in brackets. + + return '[' + a.join(',') + ']'; + } + switch(typeof obj){ + case 'boolean': + return String(obj); + case 'number': + return isFinite(obj) ? String(obj) : 'null'; + case 'function': + return 'null'; + case 'object': + var a = [], // The array holding the partial texts. + k, // The current key. + v; // The current value. + +// Iterate through all of the keys in the object, ignoring the proto chain +// and keys that are not strings. + + for (k in obj) { + if (typeof k === 'string' && + Object.prototype.hasOwnProperty.apply(obj, [k])) { + v = obj[k]; + if(v === null || v === false || v === "") continue; + switch (typeof v) { + case 'object': + if (v) { + a.push(makeJSONString(k) + ':' + makeJSONString(v)); + } else { + a.push(makeJSONString(k) + ':null'); + } + break; + + case 'string': + case 'number': + case 'boolean': + a.push(makeJSONString(k) + ':' + makeJSONString(v)); + +// Values without a JSON representation are ignored. + + } + } + } + +// Join all of the member texts together and wrap them in braces. + + return '{' + a.join(',') + '}'; + case 'string': + +// If the string contains no control characters, no quote characters, and no +// backslash characters, then we can simply slap some quotes around it. +// Otherwise we must also replace the offending characters with safe +// sequences. + + if (/["\\\x00-\x1f]/.test(obj)) { + return '"' + obj.replace(/[\x00-\x1f\\"]/g, function (a) { + var c = json_subs[a]; + if (c) { + return c; + } + c = a.charCodeAt(); + return '\\u00' + + Math.floor(c / 16).toString(16) + + (c % 16).toString(16); + }) + '"'; + } + return '"' + obj + '"'; + }; +} + +function parseJSON(str){ + var j; + +// Parsing happens in three stages. In the first stage, we run the text against +// a regular expression which looks for non-JSON characters. We are especially +// concerned with '()' and 'new' because they can cause invocation, and '=' +// because it can cause mutation. But just to be safe, we will reject all +// unexpected characters. + +// We split the first stage into 3 regexp operations in order to work around +// crippling deficiencies in Safari's regexp engine. First we replace all +// backslash pairs with '@' (a non-JSON character). Second we delete all of +// the string literals. Third, we look to see if only JSON characters +// remain. If so, then the text is safe for eval. + + if (/^[,:{}\[\]0-9.\-+Eaeflnr-u \n\r\t]*$/.test(str. + replace(/\\./g, '@'). + replace(/"[^"\\\n\r]*"/g, ''))) { +// In the second stage we use the eval function to compile the text into a +// JavaScript structure. The '{' operator is subject to a syntactic ambiguity +// in JavaScript: it can begin a block or an object literal. We wrap the text +// in parens to eliminate the ambiguity. + + try{ + j = eval('(' + str + ')'); + return j; + }catch(e){ + return null; + } + } + +// If the text is not JSON parseable, then a SyntaxError is thrown. + + throw new SyntaxError('parseJSON:' + str); +}; diff --git a/firefox_plugin/marker.js b/firefox_plugin/marker.js new file mode 100644 index 0000000..eea0e3c --- /dev/null +++ b/firefox_plugin/marker.js @@ -0,0 +1,44 @@ + +function get_api_path(){ + var prefs = Components.classes["@mozilla.org/preferences-service;1"].getService(Components.interfaces.nsIPrefBranch); + var apipath = "http://factextract.cs.berkeley.edu/thinklink"; + if(prefs.prefHasUserValue("extensions.thinklink.api")){ + apipath = prefs.getCharPref("extensions.thinklink.api"); + } +} + +function mark_snippets(){ + var targeturl = content.document.location.href; + var apipath = get_api_path(); + var url = apipath+"/anonapi/search.json?url="+encodeURIComponent(targeturl); + ajaxRequest(url,function(snippets){ + for(var i = 0; i < snippets.length; i++){ + var snip = snippets[i]; + mark_snippet(snip.text,snip.claimid,snip.claimtext,content.document.body); + } + }) +} + +function mark_snippet(text,claimid,claimtext,node){ + if(node.nodeName == "#comment" || node.textContent.indexOf(text) == -1){ + return; + }else if (node.nodeName == "#text") { + node.parentNode.style.backgroundColor = "#FFD3D3"; + }else{ + for(var i = 0; i < node.childNodes.length; i++){ + var child = node.childNodes[i]; + mark_snippet(text,claimid,claimtext,node); + } + } +} + +function ajaxRequest(url,callback){ + var req = new XMLHttpRequest(); + req.open("GET",url,true); + req.onreadystatechange = function(){ + if(req.readyState == 4 && req.status == 200){ + callback(parseJSON(req.responseText)) + } + } + req.send(null); +} diff --git a/scala/src/com/intel/thinkscala/Datastore.scala b/scala/src/com/intel/thinkscala/Datastore.scala index 0e2c450..121c79d 100644 --- a/scala/src/com/intel/thinkscala/Datastore.scala +++ b/scala/src/com/intel/thinkscala/Datastore.scala @@ -89,18 +89,20 @@ class Datastore { "AND v2_user.id = v2_node.user_id "+ "GROUP BY v2_node.id ORDER BY v2_history.date DESC LIMIT 10") def getHotClaims = get_hot.queryRows() - + val get_frequent = stmt("SELECT v2_node.*,v2_user.name AS username FROM v2_node,v2_user "+ "WHERE v2_user.id = v2_node.user_id "+ + "AND type=? "+ "ORDER BY instance_count DESC LIMIT 10") - def getFrequentClaims = get_frequent.queryRows() + def getFrequentClaims = get_frequent.queryRows("claim") + def getBigTopics = get_frequent.queryRows("topic") val search_claims = stmt("SELECT v2_node.*,v2_user.name AS username FROM v2_node,v2_user "+ "WHERE type = 'claim' "+ "AND v2_user.id = v2_node.user_id "+ "AND MATCH(text) AGAINST(?) "+ - "LIMIT 20") - def searchClaims(query : String) = search_claims.queryRows(query) + "LIMIT 20 OFFSET ?") + def searchClaims(query : String, offset : Int) = search_claims.queryRows(query,offset * 20) // === Follow links === @@ -116,6 +118,16 @@ class Datastore { def linkedNodes(typ : String, rel : String, target : Int, offset : Int, limit : Int) = linked_nodes.queryRows(typ,rel,target,limit,offset) + val topic_claims = stmt("SELECT v2_node.*,v2_user.name AS username FROM v2_node,v2_user,v2_link "+ + "WHERE v2_link.src = v2_node.id "+ + "AND v2_node.type = 'claim' "+ + "AND v2_link.type = 'about' "+ + "AND v2_link.dst = ? "+ + "AND v2_node.user_id = v2_user.id "+ + "ORDER BY instance_count DESC "+ + "LIMIT 20 OFFSET ?") + def topicClaims(topicid : Int, offset : Int) = topic_claims.queryRows(topicid,offset) + val linkedto_nodes = stmt("SELECT v2_node.*,v2_user.name AS username FROM v2_node,v2_user,v2_link "+ "WHERE v2_link.dst = v2_node.id "+ "AND v2_node.type = ? "+ @@ -205,7 +217,14 @@ class Datastore { update_search_counts.update(searchid) update_instance_count.update(claimid) } + + val update_topic_counts = stmt("UPDATE v2_node SET instance_count = "+ + "(SELECT COUNT(src) FROM v2_link "+ + "WHERE dst = v2_node.id) "+ + "WHERE type='topic')") + def updateTopicCounts = update_topic_counts.update() + val existing_snippet = stmt("SELECT * FROM v2_searchresult,v2_searchurl,v2_snipsearch "+ "WHERE v2_searchurl.id = v2_searchresult.url_id "+ "AND v2_snipsearch.id = v2_searchresult.search_id "+ diff --git a/scala/src/com/intel/thinkscala/MainServlet.scala b/scala/src/com/intel/thinkscala/MainServlet.scala index 148c320..38fb425 100644 --- a/scala/src/com/intel/thinkscala/MainServlet.scala +++ b/scala/src/com/intel/thinkscala/MainServlet.scala @@ -19,8 +19,24 @@ object Urls { def findsnippets(id : Any) = claim(id) + "/findsnippets" def findsnippets(id : Any, query : String) = claim(id) + "/findsnippets?query="+encode(query) def createClaim(query : String) = base+"/claim/new?query="+encode(query) + def searchclaims(query : String) = "/claim/search?query="+encode(query) } +object TurkGetUrls { + val base = Urls.base + "/turk/" + def turker(turkid : Int) = base + turkid + def searchClaims(turkid : Int) = turker(turkid) + "/search" + def setClaim(turkid : Int) = turker(turkid) + "/setclaim" +} + +object TurkPostUrls { + val base = Urls.base + "/turk" + def turksession(turkid : Int) = base + def setClaim(turkid : Int) = base +} + + + object FragUrls { val base = Urls.base + "/fragment/" def snipsearch(id : Any) = base+"snipsearch?claim="+id @@ -62,8 +78,16 @@ class MainServlet extends HttpServlet { c.outputFragment(
{Render.searchQueryList(c,claimid)}
) }) ) - + val gethandlers = List( + UrlHandler("/turk/searchboss", c=> { + },c => { + SnipSearch.searchBoss(c.arg("query"),0,20) + }), + UrlHandler("/turk/(\\d*)",c => { + val turkid = c.urlInt(1) + c.outputRawHtml(Turk.turkClaim(turkid)) + }), UrlHandler("/apianon/search", c => { c.output(c.store.urlSnippets(c.arg("url"))) }), @@ -73,14 +97,16 @@ class MainServlet extends HttpServlet { UrlHandler("/search",c => { // TODO: provide API access val title = c.arg("query")+" - Think Link Claim Search" c.outputHtml(title,Page.search(c)) - }), + },c => { + c.output(c.store.searchClaims(c.arg("query"),c.argInt("page"))) + } + ), UrlHandler("""/claim/(\d*)/findsnippets""", c => { - implicit val ctx = c val claim = c.store.getInfo(c.urlInt(1),c.maybe_userid) val title = claim("text") + "Find Instances with Think Link" var query = c.arg("query") if(query == null) query = claim.str("text") - c.outputHtml(title,Page.findsnippets(claim,query)) + c.outputHtml(title,Page.findsnippets(claim,query)(c)) }), UrlHandler("/claim/new",c => { c.outputHtml("Create New Claim - Think Link",Page.newClaim(c,c.arg("query"))) @@ -109,9 +135,8 @@ class MainServlet extends HttpServlet { c.redirect("/thinklink/") }), - new UrlHandler("/fragment/snipsearch", c => { - implicit val ctx = c - c.outputFragment(Render.snipSearchResults(c.arg("query"))) + UrlHandler("/fragment/snipsearch", c => { + c.outputFragment(Render.snipSearchResults(c.arg("query"))(c)) }) ) diff --git a/scala/src/com/intel/thinkscala/SnipSearch.scala b/scala/src/com/intel/thinkscala/SnipSearch.scala index 9e6c5c6..51df96f 100644 --- a/scala/src/com/intel/thinkscala/SnipSearch.scala +++ b/scala/src/com/intel/thinkscala/SnipSearch.scala @@ -25,7 +25,10 @@ class SnipUrlRes(val snips: Array[SnipInfo], val url: String, val title : String def data = Map("url" -> url, "snips" -> snips, "title" -> title, "error" -> errormsg) } -class BossUrl(val url : String, val title : String, val snips : Array[String]) +class BossUrl(val url : String, val title : String, val snips : Array[String]) extends HasData{ + def data = Map("url" -> url, "title" -> title, "snips" -> snips) +} + class SnipSearch extends HttpServlet { override def doGet(req : HttpServletRequest, res : HttpServletResponse) { @@ -39,14 +42,13 @@ object SnipSearch { val bossKey = "NpeiOwLV34E5KHWPTxBix1HTRHe4zIj2LfTtyyDKvBdeQHOzlC_RIv4SmAPuBh3E"; val bossSvr = "http://boss.yahooapis.com/ysearch/web/v1"; - def searchBoss(claim : String, page : Int) : (Int,Seq[BossUrl]) = { - val url = bossSvr + "/"+encode(claim)+"?appid="+bossKey+"&format=xml&abstract=long&start="+(page*10) + def searchBoss(claim : String, page : Int, count : Int) : Seq[BossUrl] = { + val url = bossSvr + "/"+encode(claim)+"?appid="+bossKey+"&format=xml&abstract=long&start="+(page*count)+"&count="+count val xmltext = download(url) val parser = ConstructingParser.fromSource(Source.fromString(xmltext),false) val doc = parser.document val results = doc \\ "result" - val totalhits = doc \\ "resultset_web" \ "@totalhits" - return (Integer.parseInt(totalhits.toString)/10,results map absForResult) + return results map absForResult } def searchYahoo(claim : String) : Seq[SnipUrlRes] = { diff --git a/scala/src/com/intel/thinkscala/SqlStatement.scala b/scala/src/com/intel/thinkscala/SqlStatement.scala index c426d85..4ffe5b2 100644 --- a/scala/src/com/intel/thinkscala/SqlStatement.scala +++ b/scala/src/com/intel/thinkscala/SqlStatement.scala @@ -24,9 +24,9 @@ class SqlRow extends HashMap[String,Any]{ class SqlStatement(con : Connection, s : String){ val stmt = con.prepareStatement(s,Statement.RETURN_GENERATED_KEYS) - def queryRows(args : Any*) : Iterable[SqlRow] = { + def queryRows(args : Any*) : Seq[SqlRow] = { setArgs(args) - readResults(stmt.executeQuery()) + readResults(stmt.executeQuery()).toSeq } def queryMaybe(args : Any*) : Option[SqlRow] = { diff --git a/scala/src/com/intel/thinkscala/UrlHandler.scala b/scala/src/com/intel/thinkscala/UrlHandler.scala index daf747f..6257c54 100644 --- a/scala/src/com/intel/thinkscala/UrlHandler.scala +++ b/scala/src/com/intel/thinkscala/UrlHandler.scala @@ -24,13 +24,19 @@ class Enum[T](it:java.util.Enumeration[T]) extends Iterator[T]{ class ReqContext(val store : Datastore, m : Match, req : HttpServletRequest, res : HttpServletResponse, path : String){ def urlInt(i : Int) = Integer.parseInt(m.group(i)) - def argInt(name : String) = Integer.parseInt(req.getParameter(name)) + def argInt(name : String) = + if(req.getParameter(name) != null){ + Integer.parseInt(req.getParameter(name)) + }else 0 + def argIntDflt(name : String, dflt : Int) = if(req.getParameter(name) != null) argInt(name) else dflt def arg(name : String) = req.getParameter(name) lazy val user = store.getUser(getCookie("email"), getCookie("password")); def userid = if(user.realuser) user.userid else throw new NoLogin def maybe_userid = user.userid lazy val params = getParams + lazy val format = getFormat(req) + lazy val ishtml = format == null || format == "html" || format == "htm" def getParams : Map[String,String] = { val keys : Iterator[String] = new Enum(req.getParameterNames.asInstanceOf[java.util.Enumeration[String]]) @@ -43,7 +49,6 @@ class ReqContext(val store : Datastore, m : Match, req : HttpServletRequest, res def output(obj : Any) { res.setContentType("text/html; charset=UTF-8") // TODO: set this correctly val writer = res.getWriter - val format = getFormat(req) format match { case "xml" => writer.append(printXML(obj)) case "js" => { @@ -65,6 +70,10 @@ class ReqContext(val store : Datastore, m : Match, req : HttpServletRequest, res UrlHandler.outputHtml(res,Template.normal(this,title,html),true) } + def outputRawHtml(html : NodeSeq){ + UrlHandler.outputHtml(res,html,true) + } + def outputFragment(html : NodeSeq){ UrlHandler.outputHtml(res,html,false) } @@ -113,14 +122,18 @@ class ReqContext(val store : Datastore, m : Match, req : HttpServletRequest, res } } -class UrlHandler(pat : String, func : ReqContext => unit){ +class UrlHandler(pat : String, func : ReqContext => unit, datafunc : ReqContext => Any){ val r = pat.r def tryForUrl(store : Datastore, path : String,req : HttpServletRequest, res : HttpServletResponse) : Boolean = { r.findFirstMatchIn(path) match { case Some(m) => val c = new ReqContext(store,m,req,res,path) try{ - func(new ReqContext(store,m,req,res,path)) + if(c.ishtml || datafunc == null){ + func(c) + }else if(datafunc != null){ + c.output(datafunc(c)) + } }catch{ case e : NotFound => c.notFound case e : NoLogin => c.needLogin @@ -133,7 +146,8 @@ class UrlHandler(pat : String, func : ReqContext => unit){ } object UrlHandler{ - def apply(pat : String, func : ReqContext => unit) = new UrlHandler(pat,func) + def apply(pat : String, func : ReqContext => unit) = new UrlHandler(pat,func,null) + def apply(pat : String, func : ReqContext => unit, datafunc : ReqContext => Any) = new UrlHandler(pat,func,datafunc) def innerRunHandler(store: Datastore, handlers : List[UrlHandler], req : HttpServletRequest, res : HttpServletResponse){ @@ -144,10 +158,12 @@ object UrlHandler{ path+=pathinfo } handlers.foreach(h => { - if(h.tryForUrl(store,path,req,res)){ + if(h.tryForUrl(store,path,req,res)){ return; } }) + val c = new ReqContext(store,null,req,res,path); + c.notFound } def runMatchingHandler(handlers : List[UrlHandler], diff --git a/scala/src/com/intel/thinkscala/view/Page.scala b/scala/src/com/intel/thinkscala/view/Page.scala index cd1525f..cfbadaa 100644 --- a/scala/src/com/intel/thinkscala/view/Page.scala +++ b/scala/src/com/intel/thinkscala/view/Page.scala @@ -6,7 +6,7 @@ import scala.xml._ object Page { import Widgets._ import Render._ - def home(c : ReqContext) = + def home(implicit c : ReqContext) =

Think Link

Are you being duped?
@@ -18,12 +18,17 @@ object Page {
-
Hot Claims
- {c.store.getFrequentClaims flatMap Render.claim} + {Widgets.tabs( + "Hot Claims" -> (() => c.store.getFrequentClaims flatMap Render.claim), + "Hot Topics" -> (() => c.store.getBigTopics flatMap Render.topic) + )}
- - def search(c : ReqContext) = + + def searchResults(query : String, page : Int, c : ReqContext) : NodeSeq = + c.store.searchClaims(query,page).toSeq flatMap Render.claim + + def search(implicit c : ReqContext) =

Search Results

@@ -34,7 +39,7 @@ object Page { Create a new claim
- {c.store.searchClaims(c.arg("query")) flatMap Render.claim} + {Widgets.pagedList(c.store.searchClaims(c.arg("query"),_).toSeq flatMap Render.claim)}
@@ -97,18 +102,18 @@ object Page {

Previous Search Queries

{searchQueryList(c,row.int("id"))} - {simpleSearch("snipsearch", Urls.findsnippets(row("id")), query, - Render.snipSearchResults(query)) - } + {simpleSearch(Urls.findsnippets(row("id")), query, "Enter a search string")} + {Render.snipSearchResults(query)} def topic(c : ReqContext, row : SqlRow) =

{row("text")}

{row("description")}
-
+ {simpleSearch(Urls.topic(row("id")), c.arg("query"), "Search within this topic")} +

Claims about this topic

- {c.store.linkedNodes("claim","about",row.int("id"),0,20) flatMap Render.claim} + {c.store.topicClaims(row.int("id"),0) flatMap Render.claim}

Related Topics

diff --git a/scala/src/com/intel/thinkscala/view/Render.scala b/scala/src/com/intel/thinkscala/view/Render.scala index d327588..a9dda46 100644 --- a/scala/src/com/intel/thinkscala/view/Render.scala +++ b/scala/src/com/intel/thinkscala/view/Render.scala @@ -8,12 +8,20 @@ object Render {
{row("text")}
{row("description")}
- {userref(row.int("user_id"),row.str("username"),"found by")} + {userref(row.int("user_id"),row.str("username"),"created by")} - seen {row("instance_count")} times on the web
+ + def topic(row : SqlRow) = +
+ {row("text")} +
{row("description")}
+ {userref(row.int("user_id"),row.str("username"),"created by")} + - {row("instance_count")} claims in this topic +
def userLinkCount(row : SqlRow) =
@@ -91,9 +99,9 @@ object Render {
} - def bossResults(query : String,page : Int)(implicit c : ReqContext) : (Int,NodeSeq) = { - val (max,bossUrls) = SnipSearch.searchBoss(query,page) - return (max,Util.flatMapWithIndex(bossUrls,Render.bossUrl(_ : BossUrl,_,query))) + def bossResults(query : String,page : Int)(implicit c : ReqContext) : NodeSeq = { + val bossUrls = SnipSearch.searchBoss(query,page,10) + return
{Util.flatMapWithIndex(bossUrls,Render.bossUrl(_ : BossUrl,_,query))}
} def snipSearchResults(query : String)(implicit c : ReqContext) = diff --git a/scala/src/com/intel/thinkscala/view/Turk.scala b/scala/src/com/intel/thinkscala/view/Turk.scala new file mode 100644 index 0000000..de325c5 --- /dev/null +++ b/scala/src/com/intel/thinkscala/view/Turk.scala @@ -0,0 +1,42 @@ +package com.intel.thinkscala.view + +object Turk { + def turkClaim(turkid : Int) = + + + + + Turk Task {turkid} + + + + + + + +// def turkClaim(turkid : Int)(implicit c : ReqContext) = +//
+// {Widgets.ajaxTabs( +// "1. Create Claim" -> createClaim(turkid), +// "2. Mark Snippets" -> markSnippets(turkid), +// "3. Add Opposing Evidence" -> addEvidence(turkid) +// )} +//
+// +// def createClaim(turkid : Int) = +//
+//
+// Enter a disputed claim that is not currently in our database. As you enter words, +// the list box below will show similar claims that we already know. +//
+// {Widgets.suggestBox(TurkGetUrls.searchClaims(turkid),"setClaim")} +//
+} diff --git a/scala/src/com/intel/thinkscala/view/Widgets.scala b/scala/src/com/intel/thinkscala/view/Widgets.scala new file mode 100644 index 0000000..557ae96 --- /dev/null +++ b/scala/src/com/intel/thinkscala/view/Widgets.scala @@ -0,0 +1,100 @@ +package com.intel.thinkscala.view +import scala.xml._ +import scala.collection.mutable.HashMap + +object Widgets { + def greyInput(cls : String, id : String, previewtext : String) = + + + def action(row : SqlRow, action : String, name : String) = + {name} + + + def tabs(options : (String,() => NodeSeq)*)(implicit c : ReqContext) : NodeSeq = { + var selected = c.arg("tab") + if(selected == null){ + selected = options.first._1 + } + return { +
+
+ {options flatMap { + case (name,_) if name == selected => {name} + case (name,_) => {name} + }} +
+
+ {options flatMap { + case (name,func) if name == selected => func() + case _ => NodeSeq.Empty + }} +
+
+ } + } + + def ajaxTabs(options : (String,NodeSeq)*) = +
+
+ {options.first match { + case (name,_) => {name} + }} + {options.drop(1) flatMap { + case (name,_) => {name} + }} +
+
+ {options flatMap { + case (name,content) =>
{content}
+ }} +
+
+ +// def tabs(param : String, options : Array[String], selected : String) = +//
+// options map (s => if(s equals selected){ +// s +// }else{ +// s +// }) +//
+ + def ajaxSearch(id : String, fragurl : String, initquery : String, content : NodeSeq) = +
+ + + + +
+ {content} +
+
+ + def simpleSearch(url : String, initquery : String, emptytext : String) = +
+ +
+ + def pageSelector(current : Int)(implicit c : ReqContext) = +
+ page {current+1} + {if(current > 0){prev}else{}} + next +
+ + def pagedList(content : Int => NodeSeq)(implicit c : ReqContext) : NodeSeq = { + val current = c.argIntDflt("page",0) + (
+ {pageSelector(c.argIntDflt("page",0))} + {content(current)} + {pageSelector(c.argIntDflt("page",0))} +
) + } +} \ No newline at end of file diff --git a/turk/props.txt b/turk/props.txt index 8b13789..4ca9c03 100644 --- a/turk/props.txt +++ b/turk/props.txt @@ -1 +1,9 @@ +Snippet Match For/Against/Relates/Ignore + +Which web snippets make a given claim? + +Given a claim (e.g. "The moon is made of cheese") and a selection of text snippets, tell us which ones are making that claim. + +english, meaning, text, identification, snippet, natural language + diff --git a/webapps/node/WEB-INF/web.xml b/webapps/node/WEB-INF/web.xml index 4919ab4..d5df299 100644 --- a/webapps/node/WEB-INF/web.xml +++ b/webapps/node/WEB-INF/web.xml @@ -69,6 +69,12 @@ /topic/* + + main + /turk/* + + + main /apianon/* diff --git a/webapps/node/javascript/standard.js b/webapps/node/javascript/standard.js index 44b35d0..bd5ad1c 100644 --- a/webapps/node/javascript/standard.js +++ b/webapps/node/javascript/standard.js @@ -8,6 +8,22 @@ function ungrey(obj){ } } +function onInput(textbox,callback){ + textbox.keyup(function(ev){ + var text = textbox.val(); + if(ev.which == 32){ + callback(text); + }else{ + setTimeout(function(){ + if(text == textbox.val()){ + callback(text); + } + },200); + } + }); +} + + function doAdd(obj){ $(obj).text("added") var snip = $(obj).parent(); diff --git a/webapps/node/javascript/turk.js b/webapps/node/javascript/turk.js new file mode 100644 index 0000000..5912e78 --- /dev/null +++ b/webapps/node/javascript/turk.js @@ -0,0 +1,328 @@ + +var global_claimtab; +var global_marktab; +var global_evtab; +var global_subtab; +var global_body; + +function makeTurkUi(){ + var tabs = $("
"); + var header = $("
").appendTo(tabs); + global_claimtab = $("1. Create Disputed Claim").appendTo(header); + global_marktab = $("2. Find Snippets").appendTo(header); + global_evtab = $("3. Add Opposing Evidence").appendTo(header); + global_subtab = $("4. Submit").appendTo(header); + global_body = $("
").appendTo(tabs); + $(document.body).append(tabs); + showClaimPanel(); + + global_claimtab.click(showClaimPanel); + global_marktab.click(showMarkPanel); + global_evtab.click(showEvPanel); + global_subtab.click(submitData); +} + +// DEBUG HACK +var global_claim = "global warming does not exist"; +var global_snippets = []; +var global_urls = {}; +var global_evidence = null; + +var url_base = "/thinklink/"; + +var url_search_claims = url_base+"/claim/search.json" +var url_search_snippets = url_base+"/turk/searchboss.json" +var url_submit = url_base+"/turk/submit" + +function resetTabStatus(){ + global_claimtab.attr("class",""); + global_marktab.attr("class",""); + global_evtab.attr("class",""); + global_subtab.attr("class",""); +} + +function setActiveTab(tab,panel){ + resetTabStatus(); + global_body.empty(); + tab.attr("class","active"); + global_body.append(panel); +} + +//function resetTabStatus(){ + //if(global_claim){ + //global_marktab.attr("class",""); + //} + //if(global_snippets.length >= 10){ + //global_evtab.attr("class",""); + //} + //if(global_evidence){ + //globa + //} + +//} + +function showClaimPanel(){ + var panel = $("
"); + $("
Enter a disputed claim that is not currently in our database
").appendTo(panel); + var bigsearch = $("
").appendTo(panel); + var textbox = $("").appendTo(bigsearch); + var search = $("").appendTo(bigsearch); + var submit = $("").appendTo(bigsearch); + + if(global_claim){ + textbox.val(global_claim); + }else{ + textbox.css("color","grey"); + textbox.val("Enter a disputed claim"); + textbox.focus(function(){ungrey(textbox.get(0))}); + } + + submit.click(function(){ + submit.remove(); + global_claim = textbox.val(); + showMarkPanel(); + }); + + var suggestions = $("
").appendTo(panel); + suggestions.text("existing claims will appear here when you enter a claim above"); + + //$("
As you enter words, the list box above will show similar claims that we already know. Try a couple of alternative wordings to make sure we haven't got the claim already
").appendTo(panel); + + onInput(textbox,updateClaimSuggestions); + search.click(function(){updateClaimSuggestions(textbox.val())}); + setActiveTab(global_claimtab,panel); +} + + + +var global_last_search = null; + +var global_snipsearchbox = null; + +function showMarkPanel(){ + var panel = $("
"); + + if(!global_claim){ + $("
You need to create a disputed claim first
").appendTo(panel); + setActiveTab(global_marktab,panel); + return; + }; + + $("

").text(global_claim).appendTo(panel); + $("
Use the Yahoo search interface to find ten snippets on the web that suggest or imply that this claim is true.
").appendTo(panel); + + if(!global_last_search){ + global_last_search = global_claim; + } + + var bigbox = $("
").appendTo(panel); + var searcher = $("
").appendTo(bigbox); + var searchbox = $("