Integrate callgraph. Add callers: searches, namespace support. Add co…

…ntinuous reindexing.

dxr.config

@@ -1,20 +1,20 @@
 [DXR]
 xrefscripts=/var/www/html/dxr/xref-scripts
 templates=/var/www/html/dxr/templates
-glimpse=/var/www/html/dxr/tools/gcc-dehydra/dehydra/glimpse-4.18.6/bin/glimpse
-glimpseindex=/var/www/html/dxr/tools/gcc-dehydra/dehydra/glimpse-4.18.6/bin/glimpseindex
+glimpse=/var/www/html/dxr/tools/glimpse-4.18.6/bin/glimpse
+glimpseindex=/var/www/html/dxr/tools/glimpse-4.18.6/bin/glimpseindex
 
 [Web]
 wwwdir=/var/www/html/dxr
 virtroot=/dxr
 hosturl=http://scotland.proximity.on.ca
 
+#[comm-central]
+#sourcedir=/var/www/html/dxr/mozilla-trees/comm-central
+#objdir=/var/www/html/dxr/mozilla-trees/objdir-cc-opt
+#mozconfig=/var/www/html/dxr/xref-scripts/dxr-cc-mozconfig
+
 [mozilla-central]
 sourcedir=/var/www/html/dxr/mozilla-trees/mozilla-central
 objdir=/var/www/html/dxr/mozilla-trees/objdir-mc-opt
 mozconfig=/var/www/html/dxr/xref-scripts/dxr-mozconfig
-
-[comm-central]
-sourcedir=/var/www/html/dxr/mozilla-trees/comm-central
-objdir=/var/www/html/dxr/mozilla-trees/objdir-cc-opt
-mozconfig=/var/www/html/dxr/xref-scripts/dxr-cc-mozconfig

search.cgi

@@ -9,6 +9,27 @@ import ConfigParser
 import re
 import subprocess
 
+def split_type(val):
+    parts = val.split('::')
+    # check for 'type' vs. 'type::member' vs. 'namespace::type::member' or 'namespace::namespace2::type::member'
+    n = None
+    t = None
+    m = None
+
+    if len(parts) == 1:
+        # just a single string, stuff it in type
+        t = val
+    elif len(parts) == 2:
+        t = parts[0]
+        m = parts[1]
+    else:
+        m = parts[-1]
+        t = parts[-2]
+        # use the rest as namespace
+        n = '::'.join(parts[0:-2])
+
+    return n, t, m
+
 def GetLine(loc):
     parts = loc.split(':')
     file = ReadFile(os.path.join(wwwdir, tree, parts[0]))
@@ -110,7 +131,7 @@ def processString(string):
     # Check for strings like 'foo::bar'
     halves = string.split('::')
     if len(halves) == 2:
-        count = processMember(halves[1], halves[0])
+        count = processMember(halves[1], halves[0], True)
         if count > 0:
             # we printed results, so don't bother with a text search
             return
@@ -188,7 +209,7 @@ def processMacro(macro):
         mvalue = cgi.escape(m[1])
         print '<h3>%s</h3><pre>%s</pre>' % (mname, mvalue)
 
-def processMember(member, type):
+def processMember(member, type, printDecl):
     members = None
     count = 0 # make sure we find something
     if type:
@@ -204,19 +225,59 @@ def processMember(member, type):
         else:
             print '<h3>%s::%s</h3>' % (cgi.escape(m[1]), cgi.escape(m[0]))
 
-        if m[3]:
+        if printDecl and m[3]:
             print GetLine(m[3])
             count += 1
         if m[4]:
             print GetLine(m[4])
             count += 1
     return count
 
+def processCallers(callers):
+    n, t, m = split_type(callers)
+
+    if not t and not m:
+        return
+
+    if n:
+        # type names will include namespace
+        t = n + '::' + t
+
+    hits = conn.execute("select namespace, type, shortName from node where id in (select caller from edge where callee in (select id from node where type=? COLLATE NOCASE and shortName=? COLLATE NOCASE));", (t, m)).fetchall();
+    count = 0
+    for h in hits:
+        count += 1
+        if h[0]:
+            processMember(h[2], h[0] + '::' + h[1], False)
+        else:
+            processMember(h[2], h[1], False)
+
+#    # No hits on direct type, try bases
+#    if count == 0:
+#        hits = conn.execute("select type, shortName from node where id in (select caller from edge where callee in (select id from node where shortName=? and type in (select tbname from impl where tcname=?)));", (parts[1], parts[0])).fetchall();
+#        for h in hits:
+#            count += 1
+#            processMember(h[1], h[0], False)        
+
+    if count == 0:
+        print "No matches found.  Perhaps you want a base type?"
+
+        # Show base types with this member
+        for type in conn.execute('select tbname, tcloc, direct from impl where tcname = ? order by direct desc COLLATE NOCASE;', (t,)).fetchall():
+            tname = cgi.escape(type[0])
+            tdirect = 'Direct' if type[2] == 1 else 'Indirect'
+            if not path or re.search(path, tloc):
+                print '<h3>%s (%s)</h3>' % (tname, tdirect)
+                print GetLine(type[1])
+
+
 # XXX: enable auto-flush on write - http://mail.python.org/pipermail/python-list/2008-June/668523.html
 # reopen stdout file descriptor with write mode
 # and 0 as the buffer size (unbuffered)
 sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
 
+log = open('./search.log', 'a')
+
 form = cgi.FieldStorage()
 
 string = None
@@ -227,32 +288,47 @@ derived = ''
 member = ''
 tree = '' #mozilla-central' # just a default for now
 macro = ''
+callers = ''
 
 if form.has_key('string'):
     string = form['string'].value
+    log.write('string: ' + string + '\n')
 
 if form.has_key('path'):
     path = form['path'].value
+    log.write('path: ' + path + '\n')
 
 if form.has_key('ext'):
     ext = form['ext'].value
     # remove . if present
     ext = ext.replace('.', '')
+    log.write('ext: ' + ext + '\n')
 
 if form.has_key('type'):
     type = form['type'].value
+    log.write('type: ' + type + '\n')
 
 if form.has_key('derived'):
     derived = form['derived'].value
+    log.write('derived: ' + derived + '\n')
 
 if form.has_key('member'):
     member = form['member'].value
+    log.write('member: ' + member + '\n')
 
 if form.has_key('tree'):
     tree = form['tree'].value
 
 if form.has_key('macro'):
     macro = form['macro'].value
+    log.write('macro: ' + macro + '\n')
+
+if form.has_key('callers'):
+    callers = form['callers'].value
+    log.write('callers: ' + callers + '\n')
+
+log.write('\n')
+log.close()
 
 htmldir = os.path.join('./', tree)
 
@@ -300,14 +376,16 @@ else:
     print '<div id="content">'    
     if type:
         if member:
-            processMember(member, type)
+            processMember(member, type, True)
         else:
             processType(type)
     elif derived:
         processDerived(derived)
     elif member:
-        processMember(member, type)
+        processMember(member, type, True)
     elif macro:
         processMacro(macro)
+    elif callers:
+        processCallers(callers)
 
 print """</div></div></body></html>"""

templates/dxr-index-template.html

@@ -50,7 +50,8 @@
 <p>
 </p>
 
-<font color='#ee3026'><b>Mozilla DXR Source Code Index</b></font>
+<font color='#ee3026'><b>*** WARNING: Development Branch of DXR Source Code Index ***</b></font><br />
+<font color='#ee3026'><b>Stable Demo at <a href="http://dxr.proximity.on.ca/dxr">http://dxr.proximity.on.ca/dxr</a>.</b></font>
 
 <br><br>
 
@@ -63,7 +64,7 @@
       </tr>
       <tr>
         <td nowrap valign=top><font size=-1>simple string</font></td>
-        <td>Search for a word or list of words separated by spaces.<br>  Any bare words (e.g., not path:*) in the search are taken together.<br><a href="search.cgi?tree=mozilla-central&string=Open">Open</a> <a href="search.cgi?tree=mozilla-central&string=nsAccessible::SetParent">nsAccessible::SetParent</a></td>
+        <td>Search for a word or list of words separated by spaces.<br>  Any bare words (e.g., not path:*) in the search are taken together.<br><a href="search.cgi?tree=mozilla-central&string=Open">Open</a> <a href="search.cgi?tree=mozilla-central&string=nsAccessible::SetParent">nsAccessible::SetParent</a> <a href="search.cgi?tree=mozilla-central&string=nsAccessible::">nsAccessible::</a></td>
       </tr>
 <!--
       <tr>
@@ -86,13 +87,17 @@
         <td>Find types that begin with or match <i>name</i>. Can be paired with member:.<br>Bare words are not allowed when doing type: searches.<br><a href="search.cgi?tree=mozilla-central&type=nsIFile">type:nsIFile</a></td>
       </tr>
       <tr>
-        <td nowrap valign=top>derived:<i>name</i></td>
-        <td>Find types that derive from type named <i>name</i>.<br>Bare words are not allowed when doing derived: searches.<br><a href="search.cgi?tree=mozilla-central&derived=nsIFile">derived:nsIFile</a></td>
+        <td nowrap valign=top>derived:<i>name</i><br />derived:<i>name::type</i></td>
+        <td>Find types that derive from type named <i>name</i>.<br>Bare words are not allowed when doing derived: searches.<br><a href="search.cgi?tree=mozilla-central&derived=nsIFile">derived:nsIFile</a> <a href="search.cgi?tree=mozilla-central&amp;derived=nsIFile%3A%3AAppend">derived:nsIFile::Append</a></td>
       </tr>
       <tr>
         <td nowrap valign=top>member:<i>name</i></td>
         <td>Find members that begin with or match <i>name</i>. Can be paired with type:.<br>Bare words are not allowed when doing member: searches.<br><a href="search.cgi?tree=mozilla-central&member=CopyTo">member:CopyTo</a> <a href="search.cgi?tree=mozilla-central&member=CopyTo&type=nsLocalFile">member:CopyTo type:nsLocalFile</a></td>
       </tr>
+      <tr>
+        <td nowrap valign=top>callers:<i>type::member</i></td>
+        <td>Find callers (e.g., containing functions) of <i>type::member</i>.<br>Bare words are not allowed when doing derived: searches.<br><a href="search.cgi?tree=mozilla-central&amp;callers=nsAccessible%3A%3ACacheChildren">callers:nsAccessible::CacheChildren</a></td>
+      </tr>
       <tr>
         <td nowrap valign=top>macro:<i>name</i></td>
         <td>Find macros that begin with or match <i>name</i>.<br><a href="search.cgi?tree=mozilla-central&macro=ns_ensure">macro:ns_ensure</a></td>

xref-scripts/build-xref.sh

@@ -79,7 +79,7 @@ cp ${DXRSCRIPTS}/myrules.mk ${OBJDIR}/config
 echo "Updating ${TREENAME}..."
 hg pull -u
 echo "Top-Level Build of ${TREENAME}..."
-time make -f client.mk build REPORT_BUILD='$(if $(filter %.cpp,$<),$(CXX) -dM -E $(COMPILE_CXXFLAGS) $< > $(subst .o,.macros,$@) 2>&1,@echo $(notdir $<))' > /dev/null 2>&1
+time make -f client.mk build DEHYDRA_PATH='/var/www/html/dxr/tools/gcc-dehydra/dehydra/gcc_treehydra.so' DEHYDRA_MODULES="${DXRSCRIPTS}/dxr.js" TREEHYDRA_MODULES="${DXRSCRIPTS}/callgraph/callgraph_static.js" REPORT_BUILD='$(if $(filter %.cpp,$<),$(CXX) -dM -E $(COMPILE_CXXFLAGS) $< > $(subst .o,.macros,$@) 2>&1,@echo $(notdir $<))' > /dev/null 2>&1
 
 # die if build fails
 if [ "$?" -ne 0 ]; then echo "ERROR - Build failed, aborting."; exit 1; fi
@@ -112,9 +112,15 @@ grep "^insert" ${DBROOT}/all-uniq-fixed-paths.sql > ${DBROOT}/cpp-insert.sql
 grep -v "^insert" ${DBROOT}/all-uniq-fixed-paths.sql > ${DBROOT}/cpp-update.sql
 rm ${DBROOT}/all-uniq-fixed-paths.sql
 
-cat ${DXRSCRIPTS}/dxr-schema.sql > ${DBROOT}/all-cpp.sql
+echo 'PRAGMA journal_mode=off; PRAGMA locking_mode=EXCLUSIVE; BEGIN TRANSACTION;' > ${DBROOT}/all-cpp.sql
+cat ${DXRSCRIPTS}/dxr-schema.sql >> ${DBROOT}/all-cpp.sql
+echo 'COMMIT; PRAGMA locking_mode=NORMAL;' >> ${DBROOT}/all-cpp.sql
 echo 'PRAGMA journal_mode=off; PRAGMA locking_mode=EXCLUSIVE; BEGIN TRANSACTION;' >> ${DBROOT}/all-cpp.sql
 cat ${DBROOT}/cpp-insert.sql >> ${DBROOT}/all-cpp.sql
+echo 'COMMIT; PRAGMA locking_mode=NORMAL;' >> ${DBROOT}/all-cpp.sql
+echo 'PRAGMA journal_mode=off; PRAGMA locking_mode=EXCLUSIVE; BEGIN TRANSACTION;' >> ${DBROOT}/all-cpp.sql
+cat ${DXRSCRIPTS}/dxr-indices.sql >> ${DBROOT}/all-cpp.sql
+echo 'COMMIT; PRAGMA locking_mode=NORMAL;' >> ${DBROOT}/all-cpp.sql
 cat ${DBROOT}/cpp-update.sql >> ${DBROOT}/all-cpp.sql
 echo 'COMMIT; PRAGMA locking_mode=NORMAL;' >> ${DBROOT}/all-cpp.sql
 
@@ -153,7 +159,16 @@ rm ${DBROOT}/macros-uniq-processed
 # XXX: leaving this file for debugging
 #rm ${DBROOT}/all-macros.sql
 
-# add index and then defrag
-sqlite3 ${DBNAME} "CREATE INDEX idx_members ON members (mshortname); VACUUM;"
+# create callgraph
+echo "Process callgraph .sql..."
+(cat ${DXRSCRIPTS}/callgraph/schema.sql
+echo 'BEGIN TRANSACTION;'
+find ${OBJDIR} -name '*.cg.sql' | xargs cat
+cat ${DXRSCRIPTS}/callgraph/indices.sql
+echo 'COMMIT;') > ${DBROOT}/callgraph.sql
+sqlite3 ${DBROOT}/${DBNAME} < ${DBROOT}/callgraph.sql > ${DBROOT}/callgraph.log
+
+# Defrag db
+sqlite3 ${DBNAME} "VACUUM;"
 
 echo "Done - DB created at ${DBROOT}/${DBNAME}"

xref-scripts/dxr-mozconfig

@@ -10,5 +10,6 @@ ac_add_options --enable-application=browser
 export CXX=/var/www/html/dxr/tools/gcc-dehydra/installed/bin/g++
 
 # TODO: need to get arg passing to dxr script working
-export CXXFLAGS=-fplugin="/var/www/html/dxr/tools/gcc-dehydra/dehydra/gcc_dehydra.so -fplugin-arg=/var/www/html/dxr/xref-scripts/dxr.js"
+#export CXXFLAGS=-fplugin="/var/www/html/dxr/tools/gcc-dehydra/dehydra/gcc_dehydra.so -fplugin-arg=/var/www/html/dxr/xref-scripts/dxr.js"
+#export CXXFLAGS=-fplugin="/var/www/html/dxr/tools/gcc-dehydra/dehydra/gcc_dehydra.so"