initial commit

.gitignore

@@ -0,0 +1,9 @@
+*.egg-info*
+.svn
+*~
+#*
+*#
+*.pyc
+.DS_Store
+.#*
+*.orig

LICENSE.txt

@@ -0,0 +1,32 @@
+Copyright (C) 2011-2012 Progressive Change Campaign Committee
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+ 1. Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in
+    the documentation and/or other materials provided with the
+    distribution.
+ 3. The name of the author may not be used to endorse or promote
+    products derived from this software without specific prior
+    written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Portions of this code are derived from http://trac-hacks.org/wiki/RepoSearchPlugin
+which is copyright of Alec Thomas and Ryan J Ollos, and distributed
+under a similar license.

README.txt

@@ -0,0 +1,34 @@
+To install:
+
+{{{
+pip install -r requirements.txt
+python setup.py develop
+}}}
+
+Add to your trac.conf:
+{{{
+[components]
+multireposearch.* = enabled
+}}}
+
+Upgrade your trac environment:
+{{{
+trac-admin path/to/env upgrade
+}}}
+
+Prepare all available repositories with an initial indexing:
+{{{
+trac-admin path/to/env multireposearch reindex_all
+}}}
+
+You will now be able to perform text searches of repository contents through the trac search UI.
+
+As long as you have your trac post-commit or post-receive hooks properly configured, 
+source will remain up-to-date.
+
+Otherwise, to manually reindex a single repository, you ca run:
+{{{
+trac-admin path/to/env multireposearch reindex repo-name
+}}}
+
+Where repo-name is the name assigned to your repository in Trac.

multireposearch/__init__.py

@@ -0,0 +1,2 @@
+from multireposearch.search import *
+from multireposearch.sqlindexer import *

multireposearch/interfaces.py

@@ -0,0 +1,19 @@
+from trac.core import Interface
+
+class IMultiRepoSearchBackend(Interface):
+    """
+    A mechanism that can be queries for full-text search on a single Trac source repository.
+
+    A backend can also optionally provide a indexing facilities.
+    """
+
+    def reindex_repository(reponame):
+        """
+        Reindex a single repository if the backend deems it necessary
+        """
+
+    def find_words(query):
+        """
+        Yield a series of filenames which match the given query
+        """
+

multireposearch/search.py

@@ -0,0 +1,116 @@
+from trac.admin.api import IAdminCommandProvider
+from trac.core import *
+from trac.config import *
+from trac.search import ISearchSource, shorten_result
+from trac.perm import IPermissionRequestor
+from trac.mimeview.api import Mimeview
+from trac.versioncontrol import RepositoryManager
+from trac.versioncontrol.api import Node, IRepositoryChangeListener
+
+from multireposearch.interfaces import IMultiRepoSearchBackend
+
+class MultiRepoSearchPlugin(Component):
+    """ Search the source repository. """
+    implements(ISearchSource, IPermissionRequestor,
+               IAdminCommandProvider,
+               IRepositoryChangeListener)
+
+
+    search_backend = ExtensionOption(
+        'multireposearch', 'search_backend',
+        IMultiRepoSearchBackend,
+        'SqlIndexer',
+        "Name of the component implementing `IMultiRepoSearchBackend`, "
+        "which implements repository indexing and search strategies.")
+
+    def reindex_all(self, verbose=False):
+        repos = RepositoryManager(self.env).get_all_repositories()
+        for reponame in repos:
+            self.search_backend.reindex_repository(reponame, verbose=verbose)
+
+    ## methods for IRepositoryChangeListener
+    def changeset_added(self, repos, changeset):
+        self.search_backend.reindex_repository(repos.reponame)
+
+    def changeset_modified(self, repos, changeset, old_changeset):
+        # TODO: not realy sure what to do here but i think we can ignore it,
+        # because changeset modifications can only pertain to commit-metadata 
+        # which we don't care about
+        pass
+
+    ### methods for IAdminCommandProvider
+
+    """Extension point interface for adding commands to the console
+    administration interface `trac-admin`.
+    """
+
+    def get_admin_commands(self):
+        """Return a list of available admin commands.
+        
+        The items returned by this function must be tuples of the form
+        `(command, args, help, complete, execute)`, where `command` contains
+        the space-separated command and sub-command names, `args` is a string
+        describing the command arguments and `help` is the help text. The
+        first paragraph of the help text is taken as a short help, shown in the
+        list of commands.
+        
+        `complete` is called to auto-complete the command arguments, with the
+        current list of arguments as its only argument. It should return a list
+        of relevant values for the last argument in the list.
+        
+        `execute` is called to execute the command, with the command arguments
+        passed as positional arguments.
+        """
+        return [
+            ('multireposearch reindex_all', '', 'reindex all known repositories', 
+             None,
+             lambda: self.reindex_all(verbose=True)),
+            ('multireposearch reindex', 'reponame', 'reindex a single repository', 
+             None, 
+             lambda reponame: self.search_backend.reindex_repository(reponame, verbose=True)),
+            ]
+
+
+    # IPermissionRequestor methods
+    def get_permission_actions(self):
+        yield 'REPO_SEARCH'
+
+    # ISearchSource methods
+    def get_search_filters(self, req):
+        if req.perm.has_permission('REPO_SEARCH'):
+            yield ('repo', 'Source Repository', 1)
+
+    def get_search_results(self, req, query, filters):
+        if 'repo' not in filters:
+            return
+
+        for filename, reponame in self.search_backend.find_words(query):
+            repo = self.env.get_repository(reponame=reponame, authname=req.authname)
+            node = repo.get_node(filename)
+
+            if node.kind == Node.DIRECTORY:
+                yield (self.env.href.browser(reponame, filename),
+                       "%s (in %s)" % (filename, reponame), change.date, change.author,
+                       'Directory')
+            else:
+                found = 0
+                mimeview = Mimeview(self.env)
+                content = mimeview.to_unicode(node.get_content().read(), node.get_content_type())
+                for n, line in enumerate(content.splitlines()):
+                    line = line.lower()
+                    for q in query:
+                        idx = line.find(q)
+                        if idx != -1:
+                            found = n + 1
+                            break
+                    if found:
+                        break
+
+                change = repo.get_changeset(node.rev)
+
+                yield (self.env.href.browser(reponame, filename
+                                             ) + (found and '#L%i' % found or ''
+                                                  ),
+                       "%s (in %s)" % (filename, reponame), change.date, change.author,
+                       shorten_result(content, query))
+

multireposearch/sqlindexer.py

@@ -0,0 +1,157 @@
+import posixpath
+from trac.core import *
+from trac.db import Table, Column, Index, DatabaseManager
+from trac.env import IEnvironmentSetupParticipant
+from trac.mimeview.api import Mimeview
+from trac.search.api import search_to_sql
+from trac.versioncontrol.api import Node
+
+from tracsqlhelper import get_scalar, execute_non_query, create_table
+
+from multireposearch.interfaces import IMultiRepoSearchBackend
+class SqlIndexer(Component):
+
+    implements(IMultiRepoSearchBackend, 
+               IEnvironmentSetupParticipant)
+
+    ## internal methods
+    def _last_known_rev(self, reponame):
+        with self.env.db_query as db:        
+            indexed_rev = get_scalar(self.env,
+                                     "SELECT version FROM repository_version WHERE repo=%s",
+                                     0, reponame)
+        return indexed_rev
+
+    def _walk_repo(self, repo, path):
+        node = repo.get_node(path)
+        basename = posixpath.basename(path)
+
+        if node.kind == Node.DIRECTORY:
+            for subnode in node.get_entries():
+                for result in self._walk_repo(repo, subnode.path):
+                    yield result
+        else:
+            yield node
+
+    query = """
+SELECT id, filename, repo
+FROM repository_node
+WHERE %s
+"""
+
+    ## methods for IMultiRepoSearchBackend
+
+    def reindex_repository(self, reponame, verbose=False):
+        repo = self.env.get_repository(reponame=reponame)
+
+        last_known_rev = self._last_known_rev(reponame)
+        if last_known_rev is not None and last_known_rev == repo.youngest_rev:
+            if verbose: print "Repo %s doesn't need reindexing" % reponame
+            return
+
+        if verbose: print "Repo %s DOES need reindexing" % reponame
+        mimeview = Mimeview(self.env)
+        with self.env.db_transaction as db:
+            cursor = db.cursor()
+
+            for node in self._walk_repo(repo, "/"):
+                if verbose: print "Fetching content at %s" % node.path
+                content = node.get_content()
+                if content is None:
+                    continue
+                content = mimeview.to_unicode(content.read(), node.get_content_type())
+
+                cursor.execute("""
+DELETE FROM repository_node
+WHERE repo=%s AND filename=%s""", [reponame, node.path])
+                cursor.execute("""
+INSERT INTO repository_node (repo, filename, contents)
+VALUES (%s, %s, %s)""", [reponame, node.path, content])
+
+            if last_known_rev is None:
+                cursor.execute("""
+INSERT INTO repository_version (repo, version)
+VALUES (%s, %s)""", [reponame, repo.youngest_rev])
+            else:
+                cursor.execute("""
+UPDATE repository_version 
+SET version=%s
+WHERE repo=%s""", [repo.youngest_rev, reponame])
+
+
+    def find_words(self, query):
+        with self.env.db_query as db:
+            sql, args = search_to_sql(db, ['contents'], query)
+            for id, filename, repo in db(self.query % sql, args):
+                yield filename, repo
+
+
+    ### methods for IEnvironmentSetupParticipant    
+    """Extension point interface for components that need to participate in the
+    creation and upgrading of Trac environments, for example to create
+    additional database tables."""
+
+    def environment_created(self):
+        """Called when a new Trac environment is created."""
+        if self.environment_needs_upgrade(None):
+            self.upgrade_environment(None)
+
+    def environment_needs_upgrade(self, db):
+        """Called when Trac checks whether the environment needs to be upgraded.
+        
+        Should return `True` if this participant needs an upgrade to be
+        performed, `False` otherwise.
+        """
+        return not self.version()
+
+    def upgrade_environment(self, db):
+        """Actually perform an environment upgrade.
+        
+        Implementations of this method should not commit any database
+        transactions. This is done implicitly after all participants have
+        performed the upgrades they need without an error being raised.
+        """
+        if not self.environment_needs_upgrade(db):
+            return
+
+        version = self.version()
+        for version in range(self.version(), len(self.steps)):
+            for step in self.steps[version]:
+                step(self)
+        execute_non_query(self.env,
+                          "update system set value='1' where name='multireposearch.sqlindexer.db_version';")
+
+
+    def version(self):
+        """returns version of the database (an int)"""
+        version = get_scalar(self.env, 
+                             "select value from system where name = 'multireposearch.sqlindexer.db_version';")
+        if version:
+            return int(version)
+        return 0
+
+    def create_db(self):
+        repo_cache_table = Table('repository_node', key=('id'))[
+            Column('id', auto_increment=True),
+            Column('repo'),
+            Column('filename'),
+            Column('contents'),
+            Index(['contents']),
+            ]
+        create_table(self.env, repo_cache_table)
+
+        repo_version_table = Table('repository_version', key=('id'))[
+            Column('id', auto_increment=True),
+            Column('repo'),
+            Column('version'),
+            ]
+        create_table(self.env, repo_version_table)
+
+        execute_non_query(self.env, "insert into system (name, value) values ('multireposearch.sqlindexer.db_version', '1');")
+
+    # ordered steps for upgrading
+    steps = [ 
+        [ create_db ],
+        ]
+
+

requirements.txt

@@ -0,0 +1 @@
+svn+http://trac-hacks.org/svn/tracsqlhelperscript/0.12/#egg=tracsqlhelper-dev

setup.py

@@ -0,0 +1,18 @@
+from setuptools import setup
+
+try:
+    long_description = open("README.txt").read()
+except:
+    long_description = ''
+
+setup(name='trac-MultiRepoSearchPlugin',
+      version='0.1',
+      description="Search the text of source code in your Trac repositories (0.12 and up)",
+      long_description=long_description,
+      packages=['multireposearch'],
+      author='Ethan Jucovy',
+      author_email='ejucovy@gmail.com',
+      url="http://trac-hacks.org/wiki/MultiRepoSearchPlugin",
+      install_requires=["tracsqlhelper"],
+      license='BSD',
+      entry_points = {'trac.plugins': ['multireposearch = multireposearch']})