diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a203cb1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +*.egg-info* +.svn +*~ +#* +*# +*.pyc +.DS_Store +.#* +*.orig \ No newline at end of file diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..8108037 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,32 @@ +Copyright (C) 2011-2012 Progressive Change Campaign Committee +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. The name of the author may not be used to endorse or promote + products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Portions of this code are derived from http://trac-hacks.org/wiki/RepoSearchPlugin +which is copyright of Alec Thomas and Ryan J Ollos, and distributed +under a similar license. diff --git a/README.txt b/README.txt new file mode 100644 index 0000000..10bc63a --- /dev/null +++ b/README.txt @@ -0,0 +1,34 @@ +To install: + +{{{ +pip install -r requirements.txt +python setup.py develop +}}} + +Add to your trac.conf: +{{{ +[components] +multireposearch.* = enabled +}}} + +Upgrade your trac environment: +{{{ +trac-admin path/to/env upgrade +}}} + +Prepare all available repositories with an initial indexing: +{{{ +trac-admin path/to/env multireposearch reindex_all +}}} + +You will now be able to perform text searches of repository contents through the trac search UI. + +As long as you have your trac post-commit or post-receive hooks properly configured, +source will remain up-to-date. + +Otherwise, to manually reindex a single repository, you ca run: +{{{ +trac-admin path/to/env multireposearch reindex repo-name +}}} + +Where repo-name is the name assigned to your repository in Trac. diff --git a/multireposearch/__init__.py b/multireposearch/__init__.py new file mode 100644 index 0000000..31e6392 --- /dev/null +++ b/multireposearch/__init__.py @@ -0,0 +1,2 @@ +from multireposearch.search import * +from multireposearch.sqlindexer import * diff --git a/multireposearch/interfaces.py b/multireposearch/interfaces.py new file mode 100644 index 0000000..6064526 --- /dev/null +++ b/multireposearch/interfaces.py @@ -0,0 +1,19 @@ +from trac.core import Interface + +class IMultiRepoSearchBackend(Interface): + """ + A mechanism that can be queries for full-text search on a single Trac source repository. + + A backend can also optionally provide a indexing facilities. + """ + + def reindex_repository(reponame): + """ + Reindex a single repository if the backend deems it necessary + """ + + def find_words(query): + """ + Yield a series of filenames which match the given query + """ + diff --git a/multireposearch/search.py b/multireposearch/search.py new file mode 100644 index 0000000..9609ad5 --- /dev/null +++ b/multireposearch/search.py @@ -0,0 +1,116 @@ +from trac.admin.api import IAdminCommandProvider +from trac.core import * +from trac.config import * +from trac.search import ISearchSource, shorten_result +from trac.perm import IPermissionRequestor +from trac.mimeview.api import Mimeview +from trac.versioncontrol import RepositoryManager +from trac.versioncontrol.api import Node, IRepositoryChangeListener + +from multireposearch.interfaces import IMultiRepoSearchBackend + +class MultiRepoSearchPlugin(Component): + """ Search the source repository. """ + implements(ISearchSource, IPermissionRequestor, + IAdminCommandProvider, + IRepositoryChangeListener) + + + search_backend = ExtensionOption( + 'multireposearch', 'search_backend', + IMultiRepoSearchBackend, + 'SqlIndexer', + "Name of the component implementing `IMultiRepoSearchBackend`, " + "which implements repository indexing and search strategies.") + + def reindex_all(self, verbose=False): + repos = RepositoryManager(self.env).get_all_repositories() + for reponame in repos: + self.search_backend.reindex_repository(reponame, verbose=verbose) + + ## methods for IRepositoryChangeListener + def changeset_added(self, repos, changeset): + self.search_backend.reindex_repository(repos.reponame) + + def changeset_modified(self, repos, changeset, old_changeset): + # TODO: not realy sure what to do here but i think we can ignore it, + # because changeset modifications can only pertain to commit-metadata + # which we don't care about + pass + + ### methods for IAdminCommandProvider + + """Extension point interface for adding commands to the console + administration interface `trac-admin`. + """ + + def get_admin_commands(self): + """Return a list of available admin commands. + + The items returned by this function must be tuples of the form + `(command, args, help, complete, execute)`, where `command` contains + the space-separated command and sub-command names, `args` is a string + describing the command arguments and `help` is the help text. The + first paragraph of the help text is taken as a short help, shown in the + list of commands. + + `complete` is called to auto-complete the command arguments, with the + current list of arguments as its only argument. It should return a list + of relevant values for the last argument in the list. + + `execute` is called to execute the command, with the command arguments + passed as positional arguments. + """ + return [ + ('multireposearch reindex_all', '', 'reindex all known repositories', + None, + lambda: self.reindex_all(verbose=True)), + ('multireposearch reindex', 'reponame', 'reindex a single repository', + None, + lambda reponame: self.search_backend.reindex_repository(reponame, verbose=True)), + ] + + + # IPermissionRequestor methods + def get_permission_actions(self): + yield 'REPO_SEARCH' + + # ISearchSource methods + def get_search_filters(self, req): + if req.perm.has_permission('REPO_SEARCH'): + yield ('repo', 'Source Repository', 1) + + def get_search_results(self, req, query, filters): + if 'repo' not in filters: + return + + for filename, reponame in self.search_backend.find_words(query): + repo = self.env.get_repository(reponame=reponame, authname=req.authname) + node = repo.get_node(filename) + + if node.kind == Node.DIRECTORY: + yield (self.env.href.browser(reponame, filename), + "%s (in %s)" % (filename, reponame), change.date, change.author, + 'Directory') + else: + found = 0 + mimeview = Mimeview(self.env) + content = mimeview.to_unicode(node.get_content().read(), node.get_content_type()) + for n, line in enumerate(content.splitlines()): + line = line.lower() + for q in query: + idx = line.find(q) + if idx != -1: + found = n + 1 + break + if found: + break + + change = repo.get_changeset(node.rev) + + yield (self.env.href.browser(reponame, filename + ) + (found and '#L%i' % found or '' + ), + "%s (in %s)" % (filename, reponame), change.date, change.author, + shorten_result(content, query)) + diff --git a/multireposearch/sqlindexer.py b/multireposearch/sqlindexer.py new file mode 100644 index 0000000..6626d1e --- /dev/null +++ b/multireposearch/sqlindexer.py @@ -0,0 +1,157 @@ +import posixpath +from trac.core import * +from trac.db import Table, Column, Index, DatabaseManager +from trac.env import IEnvironmentSetupParticipant +from trac.mimeview.api import Mimeview +from trac.search.api import search_to_sql +from trac.versioncontrol.api import Node + +from tracsqlhelper import get_scalar, execute_non_query, create_table + +from multireposearch.interfaces import IMultiRepoSearchBackend +class SqlIndexer(Component): + + implements(IMultiRepoSearchBackend, + IEnvironmentSetupParticipant) + + ## internal methods + def _last_known_rev(self, reponame): + with self.env.db_query as db: + indexed_rev = get_scalar(self.env, + "SELECT version FROM repository_version WHERE repo=%s", + 0, reponame) + return indexed_rev + + def _walk_repo(self, repo, path): + node = repo.get_node(path) + basename = posixpath.basename(path) + + if node.kind == Node.DIRECTORY: + for subnode in node.get_entries(): + for result in self._walk_repo(repo, subnode.path): + yield result + else: + yield node + + query = """ +SELECT id, filename, repo +FROM repository_node +WHERE %s +""" + + ## methods for IMultiRepoSearchBackend + + def reindex_repository(self, reponame, verbose=False): + repo = self.env.get_repository(reponame=reponame) + + last_known_rev = self._last_known_rev(reponame) + if last_known_rev is not None and last_known_rev == repo.youngest_rev: + if verbose: print "Repo %s doesn't need reindexing" % reponame + return + + if verbose: print "Repo %s DOES need reindexing" % reponame + mimeview = Mimeview(self.env) + with self.env.db_transaction as db: + cursor = db.cursor() + + for node in self._walk_repo(repo, "/"): + if verbose: print "Fetching content at %s" % node.path + content = node.get_content() + if content is None: + continue + content = mimeview.to_unicode(content.read(), node.get_content_type()) + + cursor.execute(""" +DELETE FROM repository_node +WHERE repo=%s AND filename=%s""", [reponame, node.path]) + cursor.execute(""" +INSERT INTO repository_node (repo, filename, contents) +VALUES (%s, %s, %s)""", [reponame, node.path, content]) + + if last_known_rev is None: + cursor.execute(""" +INSERT INTO repository_version (repo, version) +VALUES (%s, %s)""", [reponame, repo.youngest_rev]) + else: + cursor.execute(""" +UPDATE repository_version +SET version=%s +WHERE repo=%s""", [repo.youngest_rev, reponame]) + + + def find_words(self, query): + with self.env.db_query as db: + sql, args = search_to_sql(db, ['contents'], query) + for id, filename, repo in db(self.query % sql, args): + yield filename, repo + + + ### methods for IEnvironmentSetupParticipant + """Extension point interface for components that need to participate in the + creation and upgrading of Trac environments, for example to create + additional database tables.""" + + def environment_created(self): + """Called when a new Trac environment is created.""" + if self.environment_needs_upgrade(None): + self.upgrade_environment(None) + + def environment_needs_upgrade(self, db): + """Called when Trac checks whether the environment needs to be upgraded. + + Should return `True` if this participant needs an upgrade to be + performed, `False` otherwise. + """ + return not self.version() + + def upgrade_environment(self, db): + """Actually perform an environment upgrade. + + Implementations of this method should not commit any database + transactions. This is done implicitly after all participants have + performed the upgrades they need without an error being raised. + """ + if not self.environment_needs_upgrade(db): + return + + version = self.version() + for version in range(self.version(), len(self.steps)): + for step in self.steps[version]: + step(self) + execute_non_query(self.env, + "update system set value='1' where name='multireposearch.sqlindexer.db_version';") + + + def version(self): + """returns version of the database (an int)""" + version = get_scalar(self.env, + "select value from system where name = 'multireposearch.sqlindexer.db_version';") + if version: + return int(version) + return 0 + + def create_db(self): + repo_cache_table = Table('repository_node', key=('id'))[ + Column('id', auto_increment=True), + Column('repo'), + Column('filename'), + Column('contents'), + Index(['contents']), + ] + create_table(self.env, repo_cache_table) + + repo_version_table = Table('repository_version', key=('id'))[ + Column('id', auto_increment=True), + Column('repo'), + Column('version'), + ] + create_table(self.env, repo_version_table) + + execute_non_query(self.env, "insert into system (name, value) values ('multireposearch.sqlindexer.db_version', '1');") + + # ordered steps for upgrading + steps = [ + [ create_db ], + ] + + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..93c19f8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +svn+http://trac-hacks.org/svn/tracsqlhelperscript/0.12/#egg=tracsqlhelper-dev diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..efc5e06 --- /dev/null +++ b/setup.py @@ -0,0 +1,18 @@ +from setuptools import setup + +try: + long_description = open("README.txt").read() +except: + long_description = '' + +setup(name='trac-MultiRepoSearchPlugin', + version='0.1', + description="Search the text of source code in your Trac repositories (0.12 and up)", + long_description=long_description, + packages=['multireposearch'], + author='Ethan Jucovy', + author_email='ejucovy@gmail.com', + url="http://trac-hacks.org/wiki/MultiRepoSearchPlugin", + install_requires=["tracsqlhelper"], + license='BSD', + entry_points = {'trac.plugins': ['multireposearch = multireposearch']})