Skip to content
Browse files

initial commit

  • Loading branch information...
0 parents commit d744391df6665658ad8963fa769cb94a3d8af55b @ejucovy ejucovy committed Feb 1, 2012
Showing with 388 additions and 0 deletions.
  1. +9 −0 .gitignore
  2. +32 −0 LICENSE.txt
  3. +34 −0 README.txt
  4. +2 −0 multireposearch/__init__.py
  5. +19 −0 multireposearch/interfaces.py
  6. +116 −0 multireposearch/search.py
  7. +157 −0 multireposearch/sqlindexer.py
  8. +1 −0 requirements.txt
  9. +18 −0 setup.py
9 .gitignore
@@ -0,0 +1,9 @@
+*.egg-info*
+.svn
+*~
+#*
+*#
+*.pyc
+.DS_Store
+.#*
+*.orig
32 LICENSE.txt
@@ -0,0 +1,32 @@
+Copyright (C) 2011-2012 Progressive Change Campaign Committee
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the
+ distribution.
+ 3. The name of the author may not be used to endorse or promote
+ products derived from this software without specific prior
+ written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Portions of this code are derived from http://trac-hacks.org/wiki/RepoSearchPlugin
+which is copyright of Alec Thomas and Ryan J Ollos, and distributed
+under a similar license.
34 README.txt
@@ -0,0 +1,34 @@
+To install:
+
+{{{
+pip install -r requirements.txt
+python setup.py develop
+}}}
+
+Add to your trac.conf:
+{{{
+[components]
+multireposearch.* = enabled
+}}}
+
+Upgrade your trac environment:
+{{{
+trac-admin path/to/env upgrade
+}}}
+
+Prepare all available repositories with an initial indexing:
+{{{
+trac-admin path/to/env multireposearch reindex_all
+}}}
+
+You will now be able to perform text searches of repository contents through the trac search UI.
+
+As long as you have your trac post-commit or post-receive hooks properly configured,
+source will remain up-to-date.
+
+Otherwise, to manually reindex a single repository, you ca run:
+{{{
+trac-admin path/to/env multireposearch reindex repo-name
+}}}
+
+Where repo-name is the name assigned to your repository in Trac.
2 multireposearch/__init__.py
@@ -0,0 +1,2 @@
+from multireposearch.search import *
+from multireposearch.sqlindexer import *
19 multireposearch/interfaces.py
@@ -0,0 +1,19 @@
+from trac.core import Interface
+
+class IMultiRepoSearchBackend(Interface):
+ """
+ A mechanism that can be queries for full-text search on a single Trac source repository.
+
+ A backend can also optionally provide a indexing facilities.
+ """
+
+ def reindex_repository(reponame):
+ """
+ Reindex a single repository if the backend deems it necessary
+ """
+
+ def find_words(query):
+ """
+ Yield a series of filenames which match the given query
+ """
+
116 multireposearch/search.py
@@ -0,0 +1,116 @@
+from trac.admin.api import IAdminCommandProvider
+from trac.core import *
+from trac.config import *
+from trac.search import ISearchSource, shorten_result
+from trac.perm import IPermissionRequestor
+from trac.mimeview.api import Mimeview
+from trac.versioncontrol import RepositoryManager
+from trac.versioncontrol.api import Node, IRepositoryChangeListener
+
+from multireposearch.interfaces import IMultiRepoSearchBackend
+
+class MultiRepoSearchPlugin(Component):
+ """ Search the source repository. """
+ implements(ISearchSource, IPermissionRequestor,
+ IAdminCommandProvider,
+ IRepositoryChangeListener)
+
+
+ search_backend = ExtensionOption(
+ 'multireposearch', 'search_backend',
+ IMultiRepoSearchBackend,
+ 'SqlIndexer',
+ "Name of the component implementing `IMultiRepoSearchBackend`, "
+ "which implements repository indexing and search strategies.")
+
+ def reindex_all(self, verbose=False):
+ repos = RepositoryManager(self.env).get_all_repositories()
+ for reponame in repos:
+ self.search_backend.reindex_repository(reponame, verbose=verbose)
+
+ ## methods for IRepositoryChangeListener
+ def changeset_added(self, repos, changeset):
+ self.search_backend.reindex_repository(repos.reponame)
+
+ def changeset_modified(self, repos, changeset, old_changeset):
+ # TODO: not realy sure what to do here but i think we can ignore it,
+ # because changeset modifications can only pertain to commit-metadata
+ # which we don't care about
+ pass
+
+ ### methods for IAdminCommandProvider
+
+ """Extension point interface for adding commands to the console
+ administration interface `trac-admin`.
+ """
+
+ def get_admin_commands(self):
+ """Return a list of available admin commands.
+
+ The items returned by this function must be tuples of the form
+ `(command, args, help, complete, execute)`, where `command` contains
+ the space-separated command and sub-command names, `args` is a string
+ describing the command arguments and `help` is the help text. The
+ first paragraph of the help text is taken as a short help, shown in the
+ list of commands.
+
+ `complete` is called to auto-complete the command arguments, with the
+ current list of arguments as its only argument. It should return a list
+ of relevant values for the last argument in the list.
+
+ `execute` is called to execute the command, with the command arguments
+ passed as positional arguments.
+ """
+ return [
+ ('multireposearch reindex_all', '', 'reindex all known repositories',
+ None,
+ lambda: self.reindex_all(verbose=True)),
+ ('multireposearch reindex', 'reponame', 'reindex a single repository',
+ None,
+ lambda reponame: self.search_backend.reindex_repository(reponame, verbose=True)),
+ ]
+
+
+ # IPermissionRequestor methods
+ def get_permission_actions(self):
+ yield 'REPO_SEARCH'
+
+ # ISearchSource methods
+ def get_search_filters(self, req):
+ if req.perm.has_permission('REPO_SEARCH'):
+ yield ('repo', 'Source Repository', 1)
+
+ def get_search_results(self, req, query, filters):
+ if 'repo' not in filters:
+ return
+
+ for filename, reponame in self.search_backend.find_words(query):
+ repo = self.env.get_repository(reponame=reponame, authname=req.authname)
+ node = repo.get_node(filename)
+
+ if node.kind == Node.DIRECTORY:
+ yield (self.env.href.browser(reponame, filename),
+ "%s (in %s)" % (filename, reponame), change.date, change.author,
+ 'Directory')
+ else:
+ found = 0
+ mimeview = Mimeview(self.env)
+ content = mimeview.to_unicode(node.get_content().read(), node.get_content_type())
+ for n, line in enumerate(content.splitlines()):
+ line = line.lower()
+ for q in query:
+ idx = line.find(q)
+ if idx != -1:
+ found = n + 1
+ break
+ if found:
+ break
+
+ change = repo.get_changeset(node.rev)
+
+ yield (self.env.href.browser(reponame, filename
+ ) + (found and '#L%i' % found or ''
+ ),
+ "%s (in %s)" % (filename, reponame), change.date, change.author,
+ shorten_result(content, query))
+
157 multireposearch/sqlindexer.py
@@ -0,0 +1,157 @@
+import posixpath
+from trac.core import *
+from trac.db import Table, Column, Index, DatabaseManager
+from trac.env import IEnvironmentSetupParticipant
+from trac.mimeview.api import Mimeview
+from trac.search.api import search_to_sql
+from trac.versioncontrol.api import Node
+
+from tracsqlhelper import get_scalar, execute_non_query, create_table
+
+from multireposearch.interfaces import IMultiRepoSearchBackend
+class SqlIndexer(Component):
+
+ implements(IMultiRepoSearchBackend,
+ IEnvironmentSetupParticipant)
+
+ ## internal methods
+ def _last_known_rev(self, reponame):
+ with self.env.db_query as db:
+ indexed_rev = get_scalar(self.env,
+ "SELECT version FROM repository_version WHERE repo=%s",
+ 0, reponame)
+ return indexed_rev
+
+ def _walk_repo(self, repo, path):
+ node = repo.get_node(path)
+ basename = posixpath.basename(path)
+
+ if node.kind == Node.DIRECTORY:
+ for subnode in node.get_entries():
+ for result in self._walk_repo(repo, subnode.path):
+ yield result
+ else:
+ yield node
+
+ query = """
+SELECT id, filename, repo
+FROM repository_node
+WHERE %s
+"""
+
+ ## methods for IMultiRepoSearchBackend
+
+ def reindex_repository(self, reponame, verbose=False):
+ repo = self.env.get_repository(reponame=reponame)
+
+ last_known_rev = self._last_known_rev(reponame)
+ if last_known_rev is not None and last_known_rev == repo.youngest_rev:
+ if verbose: print "Repo %s doesn't need reindexing" % reponame
+ return
+
+ if verbose: print "Repo %s DOES need reindexing" % reponame
+ mimeview = Mimeview(self.env)
+ with self.env.db_transaction as db:
+ cursor = db.cursor()
+
+ for node in self._walk_repo(repo, "/"):
+ if verbose: print "Fetching content at %s" % node.path
+ content = node.get_content()
+ if content is None:
+ continue
+ content = mimeview.to_unicode(content.read(), node.get_content_type())
+
+ cursor.execute("""
+DELETE FROM repository_node
+WHERE repo=%s AND filename=%s""", [reponame, node.path])
+ cursor.execute("""
+INSERT INTO repository_node (repo, filename, contents)
+VALUES (%s, %s, %s)""", [reponame, node.path, content])
+
+ if last_known_rev is None:
+ cursor.execute("""
+INSERT INTO repository_version (repo, version)
+VALUES (%s, %s)""", [reponame, repo.youngest_rev])
+ else:
+ cursor.execute("""
+UPDATE repository_version
+SET version=%s
+WHERE repo=%s""", [repo.youngest_rev, reponame])
+
+
+ def find_words(self, query):
+ with self.env.db_query as db:
+ sql, args = search_to_sql(db, ['contents'], query)
+ for id, filename, repo in db(self.query % sql, args):
+ yield filename, repo
+
+
+ ### methods for IEnvironmentSetupParticipant
+ """Extension point interface for components that need to participate in the
+ creation and upgrading of Trac environments, for example to create
+ additional database tables."""
+
+ def environment_created(self):
+ """Called when a new Trac environment is created."""
+ if self.environment_needs_upgrade(None):
+ self.upgrade_environment(None)
+
+ def environment_needs_upgrade(self, db):
+ """Called when Trac checks whether the environment needs to be upgraded.
+
+ Should return `True` if this participant needs an upgrade to be
+ performed, `False` otherwise.
+ """
+ return not self.version()
+
+ def upgrade_environment(self, db):
+ """Actually perform an environment upgrade.
+
+ Implementations of this method should not commit any database
+ transactions. This is done implicitly after all participants have
+ performed the upgrades they need without an error being raised.
+ """
+ if not self.environment_needs_upgrade(db):
+ return
+
+ version = self.version()
+ for version in range(self.version(), len(self.steps)):
+ for step in self.steps[version]:
+ step(self)
+ execute_non_query(self.env,
+ "update system set value='1' where name='multireposearch.sqlindexer.db_version';")
+
+
+ def version(self):
+ """returns version of the database (an int)"""
+ version = get_scalar(self.env,
+ "select value from system where name = 'multireposearch.sqlindexer.db_version';")
+ if version:
+ return int(version)
+ return 0
+
+ def create_db(self):
+ repo_cache_table = Table('repository_node', key=('id'))[
+ Column('id', auto_increment=True),
+ Column('repo'),
+ Column('filename'),
+ Column('contents'),
+ Index(['contents']),
+ ]
+ create_table(self.env, repo_cache_table)
+
+ repo_version_table = Table('repository_version', key=('id'))[
+ Column('id', auto_increment=True),
+ Column('repo'),
+ Column('version'),
+ ]
+ create_table(self.env, repo_version_table)
+
+ execute_non_query(self.env, "insert into system (name, value) values ('multireposearch.sqlindexer.db_version', '1');")
+
+ # ordered steps for upgrading
+ steps = [
+ [ create_db ],
+ ]
+
+
1 requirements.txt
@@ -0,0 +1 @@
+svn+http://trac-hacks.org/svn/tracsqlhelperscript/0.12/#egg=tracsqlhelper-dev
18 setup.py
@@ -0,0 +1,18 @@
+from setuptools import setup
+
+try:
+ long_description = open("README.txt").read()
+except:
+ long_description = ''
+
+setup(name='trac-MultiRepoSearchPlugin',
+ version='0.1',
+ description="Search the text of source code in your Trac repositories (0.12 and up)",
+ long_description=long_description,
+ packages=['multireposearch'],
+ author='Ethan Jucovy',
+ author_email='ejucovy@gmail.com',
+ url="http://trac-hacks.org/wiki/MultiRepoSearchPlugin",
+ install_requires=["tracsqlhelper"],
+ license='BSD',
+ entry_points = {'trac.plugins': ['multireposearch = multireposearch']})

0 comments on commit d744391

Please sign in to comment.
Something went wrong with that request. Please try again.