Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Cataloging for repoze.

  • Loading branch information...
commit d86f180980446ff8583601531610bc86b18b6020 0 parents
@mcdonc mcdonc authored
4 CHANGES.txt
@@ -0,0 +1,4 @@
+0.1
+
+ Initial release.
+
3  COPYRIGHT.txt
@@ -0,0 +1,3 @@
+Copyright (c) 2008 Agendaless Consulting and Contributors.
+(http://www.agendaless.com), All Rights Reserved
+
41 LICENSE.txt
@@ -0,0 +1,41 @@
+License
+
+ A copyright notice accompanies this license document that identifies
+ the copyright holders.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ 1. Redistributions in source code must retain the accompanying
+ copyright notice, this list of conditions, and the following
+ disclaimer.
+
+ 2. Redistributions in binary form must reproduce the accompanying
+ copyright notice, this list of conditions, and the following
+ disclaimer in the documentation and/or other materials provided
+ with the distribution.
+
+ 3. Names of the copyright holders must not be used to endorse or
+ promote products derived from this software without prior
+ written permission from the copyright holders.
+
+ 4. If any files are modified, you must cause the modified files to
+ carry prominent notices stating that you changed the files and
+ the date of any change.
+
+ Disclaimer
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND
+ ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ HOLDERS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+ TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ SUCH DAMAGE.
+
9 README.txt
@@ -0,0 +1,9 @@
+================
+ repoze.catalog
+================
+
+An indexing and searching system based on ``zope.index
+<http://pypi.python.org/pypi/zope.index>`_.
+
+See the ``docs`` subdirectory for documentation.
+
1  TODO.txt
@@ -0,0 +1 @@
+List todo items here.
70 docs/Makefile
@@ -0,0 +1,70 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS =
+SPHINXBUILD = sphinx-build
+PAPER =
+
+# Internal variables.
+PAPEROPT_a4 = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS = -d .build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help clean html web pickle htmlhelp latex changes linkcheck
+
+help:
+ @echo "Please use \`make <target>' where <target> is one of"
+ @echo " html to make standalone HTML files"
+ @echo " pickle to make pickle files (usable by e.g. sphinx-web)"
+ @echo " htmlhelp to make HTML files and a HTML help project"
+ @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+ @echo " changes to make an overview over all changed/added/deprecated items"
+ @echo " linkcheck to check all external links for integrity"
+
+clean:
+ -rm -rf .build/*
+
+html:
+ mkdir -p .build/html .build/doctrees
+ $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) .build/html
+ @echo
+ @echo "Build finished. The HTML pages are in .build/html."
+
+pickle:
+ mkdir -p .build/pickle .build/doctrees
+ $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) .build/pickle
+ @echo
+ @echo "Build finished; now you can process the pickle files or run"
+ @echo " sphinx-web .build/pickle"
+ @echo "to start the sphinx-web server."
+
+web: pickle
+
+htmlhelp:
+ mkdir -p .build/htmlhelp .build/doctrees
+ $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) .build/htmlhelp
+ @echo
+ @echo "Build finished; now you can run HTML Help Workshop with the" \
+ ".hhp project file in .build/htmlhelp."
+
+latex:
+ mkdir -p .build/latex .build/doctrees
+ $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) .build/latex
+ @echo
+ @echo "Build finished; the LaTeX files are in .build/latex."
+ @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \
+ "run these through (pdf)latex."
+
+changes:
+ mkdir -p .build/changes .build/doctrees
+ $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) .build/changes
+ @echo
+ @echo "The overview file is in .build/changes."
+
+linkcheck:
+ mkdir -p .build/linkcheck .build/doctrees
+ $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) .build/linkcheck
+ @echo
+ @echo "Link check complete; look for any errors in the above output " \
+ "or in .build/linkcheck/output.txt."
40 docs/code/index_attributes.py
@@ -0,0 +1,40 @@
+from repoze.catalog.catalog import FileStorageCatalogFactory
+from repoze.catalog.catalog import ConnectionManager
+
+from repoze.catalog.catalog import CatalogFieldIndex
+from repoze.catalog.catalog import CatalogTextIndex
+
+factory = FileStorageCatalogFactory('catalog.db', 'mycatalog')
+
+_initialized = False
+
+def initialize_catalog():
+ global _initialized
+ if not _initialized:
+ # create a catalog
+ manager = ConnectionManager()
+ catalog = factory(manager)
+ # set up indexes
+ catalog['flavors'] = CatalogFieldIndex('flavor')
+ catalog['texts'] = CatalogTextIndex('text')
+ # commit the indexes
+ manager.commit()
+ manager.close()
+ _initialized = True
+
+class Content(object):
+ def __init__(self, flavor, text):
+ self.flavor = flavor
+ self.text = text
+
+if __name__ == '__main__':
+ initialize_catalog()
+ manager = ConnectionManager()
+ catalog = factory(manager)
+ content = {
+ 1:Content('peach', 'i am so very very peachy'),
+ 2:Content('pistachio', 'i am nutty'),
+ }
+ for docid, doc in content.items():
+ catalog.index_doc(docid, doc)
+ manager.commit()
46 docs/code/index_callbacks.py
@@ -0,0 +1,46 @@
+from repoze.catalog.catalog import FileStorageCatalogFactory
+from repoze.catalog.catalog import ConnectionManager
+
+from repoze.catalog.catalog import CatalogFieldIndex
+from repoze.catalog.catalog import CatalogTextIndex
+
+factory = FileStorageCatalogFactory('catalog.db', 'mycatalog')
+
+class Content(object):
+ def __init__(self, flavor, text):
+ self.theflavor = flavor
+ self.thetext = text
+
+def get_flavor(object, default):
+ return getattr(object, 'theflavor', default)
+
+def get_text(object, default):
+ return getattr(object, 'thetext', default)
+
+_initialized = False
+
+def initialize_catalog():
+ global _initialized
+ if not _initialized:
+ # create a catalog
+ manager = ConnectionManager()
+ catalog = factory(manager)
+ # set up indexes
+ catalog['flavors'] = CatalogFieldIndex(get_flavor)
+ catalog['text'] = CatalogTextIndex(get_text)
+ # commit the indexes
+ manager.commit()
+ manager.close()
+ _initialized = True
+
+if __name__ == '__main__':
+ initialize_catalog()
+ manager = ConnectionManager()
+ catalog = factory(manager)
+ content = {
+ 1:Content('peach', 'i am so very very peachy'),
+ 2:Content('pistachio', 'i am nutty'),
+ }
+ for docid, doc in content.items():
+ catalog.index_doc(docid, doc)
+ manager.commit()
186 docs/conf.py
@@ -0,0 +1,186 @@
+# -*- coding: utf-8 -*-
+#
+# repoze.catalog documentation build configuration file
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# The contents of this file are pickled, so don't put values in the namespace
+# that aren't pickleable (module imports are okay, they're removed automatically).
+#
+# All configuration values have a default value; values that are commented out
+# serve to show the default value.
+
+import sys, os
+
+# If your extensions are in another directory, add it here. If the directory
+# is relative to the documentation root, use os.path.abspath to make it
+# absolute, like shown here.
+parent = os.path.dirname(os.path.dirname(__file__))
+sys.path.append(os.path.abspath(parent))
+wd = os.getcwd()
+os.chdir(parent)
+os.system('%s setup.py test -q' % sys.executable)
+os.chdir(wd)
+
+for item in os.listdir(parent):
+ if item.endswith('.egg'):
+ sys.path.append(os.path.join(parent, item))
+
+# General configuration
+# ---------------------
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['.templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General substitutions.
+project = 'repoze.catalog'
+copyright = '2008, Agendaless Consulting'
+
+# The default replacements for |version| and |release|, also used in various
+# other places throughout the built documents.
+#
+# The short X.Y version.
+version = '0.2'
+# The full version, including alpha/beta/rc tags.
+release = '0.2'
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+today_fmt = '%B %d, %Y'
+
+# List of documents that shouldn't be included in the build.
+#unused_docs = []
+
+# List of directories, relative to source directories, that shouldn't be searched
+# for source files.
+#exclude_dirs = []
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+
+# Options for HTML output
+# -----------------------
+
+# The style sheet to use for HTML and HTML Help pages. A file of that name
+# must exist either in Sphinx' static/ path, or in one of the custom paths
+# given in html_static_path.
+html_style = 'default.css'
+
+# The name for this set of Sphinx documents. If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar. Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (within the static path) to place at the top of
+# the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['.static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_use_modindex = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, the reST sources are included in the HTML build as _sources/<name>.
+#html_copy_source = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it. The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = ''
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'repozecatalogdoc'
+
+
+# Options for LaTeX output
+# ------------------------
+
+# The paper size ('letter' or 'a4').
+#latex_paper_size = 'letter'
+
+# The font size ('10pt', '11pt' or '12pt').
+#latex_font_size = '10pt'
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, document class [howto/manual]).
+latex_documents = [
+ ('index', 'repozecatalog.tex', 'repoze.catalog Documentation', 'Agendaless Consulting', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# Additional stuff for the LaTeX preamble.
+#latex_preamble = ''
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_use_modindex = True
42 docs/glossary.rst
@@ -0,0 +1,42 @@
+.. _glossary:
+
+============================
+Glossary
+============================
+
+.. glossary::
+
+ Setuptools
+ `Setuptools <http://peak.telecommunity.com/DevCenter/setuptools>`_
+ builds on Python's ``distutils`` to provide easier building,
+ distribution, and installation of packages.
+ Interface
+ An attribute of a model object that determines its type. It is an
+ instance of a ``zope.interface`` Interface class.
+ Zope
+ `The Z Object Publishing Framework <http://zope.org>`_. The granddaddy
+ of Python web frameworks.
+ ZODB
+ `The Zope Object Database <http://wiki.zope.org/ZODB/FrontPage>`_
+ which is a persistent object store for Python.
+ Field index
+ A type of index that is optimized to index single simple tokenized
+ values. When a field index is searched, it can be searched for
+ one or more values, and it will return a result set that includes
+ these values exacty.
+ Text index
+ A type of index which indexes a value in such a way that parts of
+ it can be searched in a non-exact manner. When a text index is
+ searched, it returns results for values that match based on
+ various properties of the text indexed, such as omitting
+ "stopwords" the text might have.
+ zope.index
+ The `<underlying indexing machinery
+ http://pypi.python.org/pypi/zope.index>` that
+ :mod:``repoze.catalog`` uses.
+ zope.app.catalog
+ The `<cataloging implementation
+ http://pypi.python.org/pypi/zope.app.catalog>` on which
+ :mod:``repoze.catalog`` is based (although it doesn't use any of
+ its code).
+
43 docs/index.rst
@@ -0,0 +1,43 @@
+.. _index:
+
+==============
+repoze.catalog
+==============
+
+:mod:`repoze.catalog` is a Python indexing and searching framework.
+It relies on :term:`zope.index` and most of its internals are taken
+from :term:`zope.app.catalog`. Unlike ``zope.app.catalog``, however,
+it is meant to be useful outside of the larger Zope framework within
+arbitrary Python applications.
+
+Narrative documentation
+-----------------------
+
+Narrative documentation in chapter form explaining how to use
+:mod:`repoze.catalog`.
+
+.. toctree::
+ :maxdepth: 2
+
+ install
+ using
+ glossary
+
+API documentation
+-----------------
+
+Per-module :mod:`repoze.catalog` API documentation.
+
+.. toctree::
+ :maxdepth: 2
+
+ api
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+* :ref:`glossary`
+
36 docs/install.rst
@@ -0,0 +1,36 @@
+Installing :mod:`repoze.catalog`
+================================
+
+How To Install
+--------------
+
+You will need `Python <http://python.org>`_ version 2.4 or better to
+run :mod:`repoze.catalog`. Development of :mod:`repoze.catalog` is
+done primarily under Python 2.4, so that version is recommended.
+:mod:`repoze.bfg` does *not* run under any version of Python before
+2.4, and does *not* run under Python 3.X.
+
+.. warning:: To succesfully install :mod:`repoze.catalog`, you will an
+ environment capable of compiling Python C code. See the
+ documentation about installing, e.g. ``gcc`` and ``python-devel``
+ for your system. You will also need :term:`setuptools` installed
+ on within your Python system in order to run the ``easy_install``
+ command.
+
+It is advisable to install :mod:`repoze.catalog` into a
+:term:`virtualenv` in order to obtain isolation from any "system"
+packages you've got installed in your Python version (and likewise, to
+prevent :mod:`repoze.catalog` from globally installing versions of
+packages that are not compatible with your system Python).
+
+After you've got the requisite dependencies installed, you may install
+:mod:`repoze.catalog` into your Python environment using the following
+command::
+
+ $ easy_install -i http://dist.repoze.org/lemonade/dev/simple repoze.catalog
+
+What Gets Installed
+-------------------
+
+When you ``easy_install`` :mod:`repoze.catalog`, various Zope
+libraries and ZODB are installed.
117 docs/usage.rst
@@ -0,0 +1,117 @@
+.. _usage:
+
+Using :mod:`repoze.catalog`
+===========================
+
+:mod:`repoze.catalog` is an indexing and search system for Python. It
+is inspired by (and uses much code from) Zope's
+:term:`zope.app.catalog`, and uses other :term:`Zope` libraries to do
+much of its work. It manages its own persistence: it stores catalog
+information into a :term:`ZODB` database.
+
+In order to make use of :mod:`repoze.catalog`, your application will
+be required to create objects that are willing to be indexed, and it
+will be responsible for providing each of these objects a unique
+integer identifier, and maintaining the association between the object
+and the unique identifier for the lifetime of your application.
+Objects which are willing to be indexed must either have a particular
+attribute which is guaranteed to have a value *or* you must provide a
+callback that is willing to inspect the content for a value.
+
+The result of searching a catalog is a sequence of integers that
+represent all the document ids that match the query. Your application
+is responsible for being able to (re-) resolve these integers into
+content objects.
+
+Indexing
+--------
+
+Here's a simple example of indexing data within your application.
+This example sets up two indexes.
+
+The first index for ``flavor`` is a :term:`field index`. The second
+index, ``text``, is a :term:`text index`.
+
+.. literalinclude:: code/index_attributes.py
+ :linenos:
+ :language: python
+
+Here's a more complicated example of indexing data into a catalog
+within your application, which uses callbacks to adapt cataloged
+objects to values rather than directly inspecting attributes of the
+content object. We use the same types of indexes, but we set up
+callbacks that allow us to adapt content to a result instead of
+exmaining the object for an attribute directly. This is useful in the
+case that your content objects don't have attributes that match
+exactly what you want to index:
+
+.. literalinclude:: code/index_interfaces.py
+ :linenos:
+ :language: python
+
+Searching
+---------
+
+Searching for values from a previously indexed corpus of content is
+significantly easier than indexing. We pass a query into our
+catalog's ``searchResults`` method, which is composed of the name of
+our index and a value we'd like to find a document for.
+
+.. code-block:: python
+ :linenos:
+
+ factory = FileStorageCatalogFactory('catalog.db', 'mycatalog')
+
+ if __name__ == '__main__':
+ manager = ConnectionManager()
+ catalog = factory(manager)
+ results = catalog.searchResults(flavor='peach')
+ print results
+
+The results of the above search will search the corpus for documents
+which have a result in the ``flavor`` index that matches the value
+``peach``. This particular query will return a sequence, with one
+result::
+
+ [1]
+
+This is the document id for the content we indexed in the above step
+as 'peach'. Your application is responsible for resolving this unique
+identifier back to its constituent content.
+
+You can also pass compound search parameters for multiple indexes.
+The results are intersected to provide a result:
+
+.. code-block:: python
+ :linenos:
+
+ factory = FileStorageCatalogFactory('catalog.db', 'mycatalog')
+
+ if __name__ == '__main__':
+ manager = ConnectionManager()
+ catalog = factory(manager)
+ results = catalog.searchResults(flavor='peach', text='pistachio')
+ print results
+
+The results of the above search will return the empty sequence,
+because no results in our index match a document which has both a
+flavor of 'peach' and text which contains the word 'pistachio'.
+
+See the :term:`zope.index` documentation and implementation for more
+information about special query parameters to indexes.
+
+Restrictions
+------------
+
+Values indexed by a :mod:`repoze.bfg` catalog cannot subclass from the
+ZODB ``Persistent`` class. This is a safeguard to ensure that
+irresolveable cross-database references aren't put into the catalog's
+(separate) database.
+
+Gotchas
+-------
+
+When the ``ConnectionManager`` 's ``commit`` method is called, it will
+commit a transaction for all databases participating in Zope
+transaction management. Don't use this method if you already have
+transaction management enabled in another way.
272 ez_setup.py
@@ -0,0 +1,272 @@
+#!python
+"""Bootstrap setuptools installation
+
+If you want to use setuptools in your package's setup.py, just include this
+file in the same directory with it, and add this to the top of your setup.py::
+
+ from ez_setup import use_setuptools
+ use_setuptools()
+
+If you want to require a specific version of setuptools, set a download
+mirror, or use an alternate download directory, you can do so by supplying
+the appropriate options to ``use_setuptools()``.
+
+This file can also be run as a script to install or upgrade setuptools.
+"""
+import sys
+DEFAULT_VERSION = "0.6c8"
+DEFAULT_URL = "http://pypi.python.org/packages/%s/s/setuptools/" % sys.version[:3]
+
+md5_data = {
+ 'setuptools-0.6b1-py2.3.egg': '8822caf901250d848b996b7f25c6e6ca',
+ 'setuptools-0.6b1-py2.4.egg': 'b79a8a403e4502fbb85ee3f1941735cb',
+ 'setuptools-0.6b2-py2.3.egg': '5657759d8a6d8fc44070a9d07272d99b',
+ 'setuptools-0.6b2-py2.4.egg': '4996a8d169d2be661fa32a6e52e4f82a',
+ 'setuptools-0.6b3-py2.3.egg': 'bb31c0fc7399a63579975cad9f5a0618',
+ 'setuptools-0.6b3-py2.4.egg': '38a8c6b3d6ecd22247f179f7da669fac',
+ 'setuptools-0.6b4-py2.3.egg': '62045a24ed4e1ebc77fe039aa4e6f7e5',
+ 'setuptools-0.6b4-py2.4.egg': '4cb2a185d228dacffb2d17f103b3b1c4',
+ 'setuptools-0.6c1-py2.3.egg': 'b3f2b5539d65cb7f74ad79127f1a908c',
+ 'setuptools-0.6c1-py2.4.egg': 'b45adeda0667d2d2ffe14009364f2a4b',
+ 'setuptools-0.6c2-py2.3.egg': 'f0064bf6aa2b7d0f3ba0b43f20817c27',
+ 'setuptools-0.6c2-py2.4.egg': '616192eec35f47e8ea16cd6a122b7277',
+ 'setuptools-0.6c3-py2.3.egg': 'f181fa125dfe85a259c9cd6f1d7b78fa',
+ 'setuptools-0.6c3-py2.4.egg': 'e0ed74682c998bfb73bf803a50e7b71e',
+ 'setuptools-0.6c3-py2.5.egg': 'abef16fdd61955514841c7c6bd98965e',
+ 'setuptools-0.6c4-py2.3.egg': 'b0b9131acab32022bfac7f44c5d7971f',
+ 'setuptools-0.6c4-py2.4.egg': '2a1f9656d4fbf3c97bf946c0a124e6e2',
+ 'setuptools-0.6c4-py2.5.egg': '8f5a052e32cdb9c72bcf4b5526f28afc',
+ 'setuptools-0.6c5-py2.3.egg': 'ee9fd80965da04f2f3e6b3576e9d8167',
+ 'setuptools-0.6c5-py2.4.egg': 'afe2adf1c01701ee841761f5bcd8aa64',
+ 'setuptools-0.6c5-py2.5.egg': 'a8d3f61494ccaa8714dfed37bccd3d5d',
+ 'setuptools-0.6c6-py2.3.egg': '35686b78116a668847237b69d549ec20',
+ 'setuptools-0.6c6-py2.4.egg': '3c56af57be3225019260a644430065ab',
+ 'setuptools-0.6c6-py2.5.egg': 'b2f8a7520709a5b34f80946de5f02f53',
+ 'setuptools-0.6c7-py2.3.egg': '209fdf9adc3a615e5115b725658e13e2',
+ 'setuptools-0.6c7-py2.4.egg': '5a8f954807d46a0fb67cf1f26c55a82e',
+ 'setuptools-0.6c7-py2.5.egg': '45d2ad28f9750e7434111fde831e8372',
+ 'setuptools-0.6c8-py2.3.egg': '50759d29b349db8cfd807ba8303f1902',
+ 'setuptools-0.6c8-py2.4.egg': 'cba38d74f7d483c06e9daa6070cce6de',
+ 'setuptools-0.6c8-py2.5.egg': '1721747ee329dc150590a58b3e1ac95b',
+}
+
+import sys, os
+
+def _validate_md5(egg_name, data):
+ if egg_name in md5_data:
+ from md5 import md5
+ digest = md5(data).hexdigest()
+ if digest != md5_data[egg_name]:
+ print >>sys.stderr, (
+ "md5 validation of %s failed! (Possible download problem?)"
+ % egg_name
+ )
+ sys.exit(2)
+ return data
+
+
+def use_setuptools(
+ version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir,
+ download_delay=15
+):
+ """Automatically find/download setuptools and make it available on sys.path
+
+ `version` should be a valid setuptools version number that is available
+ as an egg for download under the `download_base` URL (which should end with
+ a '/'). `to_dir` is the directory where setuptools will be downloaded, if
+ it is not already available. If `download_delay` is specified, it should
+ be the number of seconds that will be paused before initiating a download,
+ should one be required. If an older version of setuptools is installed,
+ this routine will print a message to ``sys.stderr`` and raise SystemExit in
+ an attempt to abort the calling script.
+ """
+ was_imported = 'pkg_resources' in sys.modules or 'setuptools' in sys.modules
+ def do_download():
+ egg = download_setuptools(version, download_base, to_dir, download_delay)
+ sys.path.insert(0, egg)
+ import setuptools; setuptools.bootstrap_install_from = egg
+ try:
+ import pkg_resources
+ except ImportError:
+ return do_download()
+ try:
+ pkg_resources.require("setuptools>="+version); return
+ except pkg_resources.VersionConflict, e:
+ if was_imported:
+ print >>sys.stderr, (
+ "The required version of setuptools (>=%s) is not available, and\n"
+ "can't be installed while this script is running. Please install\n"
+ " a more recent version first, using 'easy_install -U setuptools'."
+ "\n\n(Currently using %r)"
+ ) % (version, e.args[0])
+ sys.exit(2)
+ else:
+ del pkg_resources, sys.modules['pkg_resources'] # reload ok
+ return do_download()
+ except pkg_resources.DistributionNotFound:
+ return do_download()
+
+def download_setuptools(
+ version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir,
+ delay = 15
+):
+ """Download setuptools from a specified location and return its filename
+
+ `version` should be a valid setuptools version number that is available
+ as an egg for download under the `download_base` URL (which should end
+ with a '/'). `to_dir` is the directory where the egg will be downloaded.
+ `delay` is the number of seconds to pause before an actual download attempt.
+ """
+ import urllib2, shutil
+ egg_name = "setuptools-%s-py%s.egg" % (version,sys.version[:3])
+ url = download_base + egg_name
+ saveto = os.path.join(to_dir, egg_name)
+ src = dst = None
+ if not os.path.exists(saveto): # Avoid repeated downloads
+ try:
+ from distutils import log
+ if delay:
+ log.warn("""
+---------------------------------------------------------------------------
+This script requires setuptools version %s to run (even to display
+help). I will attempt to download it for you (from
+%s), but
+you may need to enable firewall access for this script first.
+I will start the download in %d seconds.
+
+(Note: if this machine does not have network access, please obtain the file
+
+ %s
+
+and place it in this directory before rerunning this script.)
+---------------------------------------------------------------------------""",
+ version, download_base, delay, url
+ ); from time import sleep; sleep(delay)
+ log.warn("Downloading %s", url)
+ src = urllib2.urlopen(url)
+ # Read/write all in one block, so we don't create a corrupt file
+ # if the download is interrupted.
+ data = _validate_md5(egg_name, src.read())
+ dst = open(saveto,"wb"); dst.write(data)
+ finally:
+ if src: src.close()
+ if dst: dst.close()
+ return os.path.realpath(saveto)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+def main(argv, version=DEFAULT_VERSION):
+ """Install or upgrade setuptools and EasyInstall"""
+ try:
+ import setuptools
+ except ImportError:
+ egg = None
+ try:
+ egg = download_setuptools(version, delay=0)
+ sys.path.insert(0,egg)
+ from setuptools.command.easy_install import main
+ return main(list(argv)+[egg]) # we're done here
+ finally:
+ if egg and os.path.exists(egg):
+ os.unlink(egg)
+ else:
+ if setuptools.__version__ == '0.0.1':
+ print >>sys.stderr, (
+ "You have an obsolete version of setuptools installed. Please\n"
+ "remove it from your system entirely before rerunning this script."
+ )
+ sys.exit(2)
+
+ req = "setuptools>="+version
+ import pkg_resources
+ try:
+ pkg_resources.require(req)
+ except pkg_resources.VersionConflict:
+ try:
+ from setuptools.command.easy_install import main
+ except ImportError:
+ from easy_install import main
+ main(list(argv)+[download_setuptools(delay=0)])
+ sys.exit(0) # try to force an exit
+ else:
+ if argv:
+ from setuptools.command.easy_install import main
+ main(argv)
+ else:
+ print "Setuptools version",version,"or greater has been installed."
+ print '(Run "ez_setup.py -U setuptools" to reinstall or upgrade.)'
+
+def update_md5(filenames):
+ """Update our built-in md5 registry"""
+
+ import re
+ from md5 import md5
+
+ for name in filenames:
+ base = os.path.basename(name)
+ f = open(name,'rb')
+ md5_data[base] = md5(f.read()).hexdigest()
+ f.close()
+
+ data = [" %r: %r,\n" % it for it in md5_data.items()]
+ data.sort()
+ repl = "".join(data)
+
+ import inspect
+ srcfile = inspect.getsourcefile(sys.modules[__name__])
+ f = open(srcfile, 'rb'); src = f.read(); f.close()
+
+ match = re.search("\nmd5_data = {\n([^}]+)}", src)
+ if not match:
+ print >>sys.stderr, "Internal error!"
+ sys.exit(2)
+
+ src = src[:match.start(1)] + repl + src[match.end(1):]
+ f = open(srcfile,'w')
+ f.write(src)
+ f.close()
+
+
+if __name__=='__main__':
+ if len(sys.argv)>2 and sys.argv[1]=='--md5update':
+ update_md5(sys.argv[2:])
+ else:
+ main(sys.argv[1:])
+
+
+
+
+
1  repoze/__init__.py
@@ -0,0 +1 @@
+__import__('pkg_resources').declare_namespace(__name__)
3  repoze/catalog/__init__.py
@@ -0,0 +1,3 @@
+# i have a package for you
+
+
187 repoze/catalog/catalog.py
@@ -0,0 +1,187 @@
+import BTrees
+from persistent.mapping import PersistentMapping
+from persistent import Persistent
+import transaction
+
+from zope.interface import implements
+from zope.interface import Interface
+from zope.interface import Attribute
+
+from zope.index.interfaces import IInjection
+from zope.index.interfaces import IIndexSearch
+
+from zope.index.text import TextIndex
+from zope.index.field import FieldIndex
+from zope.index.keyword import KeywordIndex
+
+class ICatalog(IIndexSearch, IInjection):
+ def searchResults(**query):
+ """Search on the given indexes."""
+
+ def updateIndexes(items):
+ """Reindex all objects in the items sequence [(docid, obj),..] in all
+ indexes."""
+
+ def updateIndex(name, items):
+ """Reindex all objects in the items sequence [(docid, obj),..] in
+ the named index."""
+
+class ICatalogIndex(IIndexSearch, IInjection):
+ """ An index that adapts objects to an attribute or callable value
+ on an object """
+ interface = Attribute('The interface that object to be indexed will '
+ 'be adapted against to provide its catalog value')
+
+class ICatalogAdapter(Interface):
+ def __call__(default):
+ """ Return the value or the default if the object no longer
+ has any value for the adaptation"""
+
+_marker = ()
+
+class CatalogIndex(object):
+ """ Abstract class for interface-based lookup """
+
+ def __init__(self, discriminator, *args, **kwargs):
+ super(CatalogIndex, self).__init__(*args, **kwargs)
+ if not callable(discriminator):
+ if not isinstance(interface, basestring):
+ raise ValueError('discriminator value must be callable or a '
+ 'string')
+ self.discriminator = discriminator
+
+ def index_doc(self, docid, object):
+ if callable(self.discriminator):
+ value = self.discriminator(object, _marker)
+ else:
+ value = getattr(object, self.interface, _marker)
+
+ if value is _marker:
+ # unindex the previous value
+ super(CatalogIndex, self).unindex_doc(docid)
+ return None
+
+ if isinstance(value, Persistent):
+ raise ValueError('Catalog cannot index persistent object %s' %
+ value)
+
+ return super(CatalogIndex, self).index_doc(docid, value)
+
+class CatalogTextIndex(CatalogIndex, TextIndex):
+ implements(ICatalogIndex)
+
+class CatalogFieldIndex(CatalogIndex, FieldIndex):
+ implements(ICatalogIndex)
+
+class CatalogKeywordIndex(CatalogIndex, KeywordIndex):
+ implements(ICatalogIndex)
+
+class Catalog(PersistentMapping):
+
+ implements(ICatalog)
+
+ family = BTrees.family32
+
+ def __init__(self, family=None):
+ PersistentMapping.__init__(self)
+ if family is not None:
+ self.family = family
+
+ def clear(self):
+ for index in self.values():
+ index.clear()
+
+ def index_doc(self, docid, obj):
+ """Register the data in indexes of this catalog."""
+ for index in self.values():
+ assertint(docid)
+ index.index_doc(docid, obj)
+
+ def unindex_doc(self, docid):
+ """Unregister the data from indexes of this catalog."""
+ for index in self.values():
+ assertint(docid)
+ index.unindex_doc(docid)
+
+ def __setitem__(self, name, index):
+ if not ICatalogIndex.providedBy(index):
+ raise ValueError('%s does not provide ICatalogIndex')
+ index.__parent__ = self
+ index.__name__ = name
+ return PersistentMapping.__setitem__(self, name, index)
+
+ def updateIndex(self, name, items):
+ index = self[name]
+ for docid, obj in items:
+ assertint(docid)
+ index.index_doc(docid, obj)
+
+ def updateIndexes(self, items):
+ for docid, obj in items:
+ assertint(docid)
+ for index in self.values():
+ index.index_doc(docid, obj)
+
+ def searchResults(self, **query):
+ results = []
+ for index_name, index_query in query.items():
+ index = self[index_name]
+ r = index.apply(index_query)
+ if r is None:
+ continue
+ if not r:
+ # empty results
+ return r
+ results.append((len(r), r))
+
+ if not results:
+ # no applicable indexes, so catalog was not applicable
+ return None
+
+ results.sort() # order from smallest to largest
+
+ _, result = results.pop(0)
+ for _, r in results:
+ _, result = self.family.IF.weightedIntersection(result, r)
+
+ return result
+
+ def apply(self, query):
+ return self.searchResults(**query)
+
+def assertint(docid):
+ if not isinstance(docid, int):
+ raise ValueError('%r is not an integer value; document ids must be '
+ 'integers' % docid)
+
+class CatalogFactory(object):
+ def __call__(self, connection_handler=None):
+ conn = self.db.open()
+ if connection_handler:
+ connection_handler(conn)
+ root = conn.root()
+ if root.get(self.appname) is None:
+ root[self.appname] = Catalog()
+ return root[self.appname]
+
+class FileStorageCatalogFactory(CatalogFactory):
+ def __init__(self, filename, appname):
+ from ZODB.FileStorage.FileStorage import FileStorage
+ from ZODB.DB import DB
+ f = FileStorage(filename)
+ self.db = DB(f)
+ self.appname = appname
+
+class ConnectionManager(object):
+ def __call__(self, conn):
+ self.conn = conn
+
+ def close(self):
+ self.conn.close()
+
+ def __del__(self):
+ self.close()
+
+ def commit(self, transaction=transaction):
+ transaction.commit()
+
1  repoze/catalog/tests/__init__.py
@@ -0,0 +1 @@
+# i am a package
235 repoze/catalog/tests/test_catalog.py
@@ -0,0 +1,235 @@
+import unittest
+
+class TestCatalogIndex(unittest.TestCase):
+ def _getTargetClass(self):
+ from repoze.catalog.catalog import CatalogIndex
+ return CatalogIndex
+
+ def test_ctor(self):
+ klass = self._getTargetClass()
+ class Test(klass, DummyIndex):
+ pass
+ def callback(object, default):
+ return default
+ index = Test(callback, 'a', a=1)
+ self.assertEqual(index.discriminator, callback)
+ self.assertEqual(index.arg, ('a',))
+ self.assertEqual(index.kw, {'a':1})
+
+ def test_index_doc_callback_returns_nondefault(self):
+ klass = self._getTargetClass()
+ class Test(klass, DummyIndex):
+ pass
+ def callback(ob, default):
+ return ob
+ index = Test(callback)
+ self.assertEqual(index.index_doc(1, 'abc'), 'abc')
+ self.assertEqual(index.value, 'abc')
+ self.assertEqual(index.docid, 1)
+
+class TestCatalog(unittest.TestCase):
+ def _getTargetClass(self):
+ from repoze.catalog.catalog import Catalog
+ return Catalog
+
+ def _makeOne(self):
+ klass = self._getTargetClass()
+ return klass()
+
+ def test_klass_provides_ICatalog(self):
+ klass = self._getTargetClass()
+ from zope.interface.verify import verifyClass
+ from repoze.catalog.catalog import ICatalog
+ verifyClass(ICatalog, klass)
+
+ def test_inst_provides_ICatalog(self):
+ klass = self._getTargetClass()
+ from zope.interface.verify import verifyObject
+ from repoze.catalog.catalog import ICatalog
+ inst = self._makeOne()
+ verifyObject(ICatalog, inst)
+
+ def test_clear(self):
+ catalog = self._makeOne()
+ idx = DummyIndex()
+ catalog['name'] = idx
+ catalog.clear()
+ self.assertEqual(idx.cleared, True)
+
+ def test_index_doc(self):
+ catalog = self._makeOne()
+ idx = DummyIndex()
+ catalog['name'] = idx
+ catalog.index_doc(1, 'value')
+ self.assertEqual(idx.docid, 1)
+ self.assertEqual(idx.value, 'value')
+
+ def test_index_doc_nonint_docid(self):
+ catalog = self._makeOne()
+ idx = DummyIndex()
+ catalog['name'] = idx
+ self.assertRaises(ValueError, catalog.index_doc, 'abc', 'value')
+
+ def test_unindex_doc(self):
+ catalog = self._makeOne()
+ idx = DummyIndex()
+ catalog['name'] = idx
+ catalog.unindex_doc(1)
+ self.assertEqual(idx.unindexed, 1)
+
+ def test_setitem_guard(self):
+ catalog = self._makeOne()
+ self.assertRaises(ValueError, catalog.__setitem__, 'a', None)
+
+ def test_updateIndex(self):
+ catalog = self._makeOne()
+ idx = DummyIndex()
+ catalog['name'] = idx
+ catalog.updateIndex('name', [(1, 1)])
+ self.assertEqual(idx.docid, 1)
+ self.assertEqual(idx.value, 1)
+
+ def test_updateIndexes(self):
+ catalog = self._makeOne()
+ idx1 = DummyIndex()
+ catalog['name1'] = idx1
+ idx2 = DummyIndex()
+ catalog['name2'] = idx2
+ catalog.updateIndexes([(1, 1)])
+ self.assertEqual(idx1.docid, 1)
+ self.assertEqual(idx1.value, 1)
+ self.assertEqual(idx2.docid, 1)
+ self.assertEqual(idx2.value, 1)
+
+ def test_searchResults(self):
+ catalog = self._makeOne()
+ from BTrees.IFBTree import IFSet
+ c1 = IFSet([1, 2, 3])
+ idx1 = DummyIndex(c1)
+ catalog['name1'] = idx1
+ c2 = IFSet([3, 4, 5])
+ idx2 = DummyIndex(c2)
+ catalog['name2'] = idx2
+ result = catalog.searchResults(name1={}, name2={})
+ self.assertEqual(list(result), [3])
+
+ def test_apply(self):
+ catalog = self._makeOne()
+ from BTrees.IFBTree import IFSet
+ c1 = IFSet([1, 2, 3])
+ idx1 = DummyIndex(c1)
+ catalog['name1'] = idx1
+ c2 = IFSet([3, 4, 5])
+ idx2 = DummyIndex(c2)
+ catalog['name2'] = idx2
+ result = catalog.apply({'name1':{}, 'name2':{}})
+ self.assertEqual(list(result), [3])
+
+class TestFileStorageCatalogFactory(unittest.TestCase):
+ def _getTargetClass(self):
+ from repoze.catalog.catalog import FileStorageCatalogFactory
+ return FileStorageCatalogFactory
+
+ def _makeOne(self, filename, appname):
+ klass = self._getTargetClass()
+ return klass(filename, appname)
+
+ def setUp(self):
+ import tempfile
+ self.tempfile = tempfile.mktemp()
+
+ def tearDown(self):
+ import os
+ os.remove(self.tempfile)
+
+ def test_no_conn_handler(self):
+ factory = self._makeOne(self.tempfile, 'catalog')
+ from repoze.catalog.catalog import Catalog
+ catalog = factory()
+ self.failUnless(isinstance(catalog, Catalog))
+ factory.db.close()
+
+ def test_with_conn_handler(self):
+ factory = self._makeOne(self.tempfile, 'catalog')
+ from repoze.catalog.catalog import Catalog
+ e = {}
+ def handle(conn):
+ e['conn'] = conn
+ catalog = factory(handle)
+ self.failUnless(isinstance(catalog, Catalog))
+ self.assertEqual(e['conn']._db, factory.db)
+ factory.db.close()
+
+class TestConnectioManager(unittest.TestCase):
+ def _getTargetClass(self):
+ from repoze.catalog.catalog import ConnectionManager
+ return ConnectionManager
+
+ def _makeOne(self):
+ klass = self._getTargetClass()
+ return klass()
+
+ def test_call(self):
+ conn = DummyConnection()
+ manager = self._makeOne()
+ manager(conn)
+ self.assertEqual(manager.conn, conn)
+
+ def test_close(self):
+ conn = DummyConnection()
+ manager = self._makeOne()
+ manager(conn)
+ manager.close()
+ self.assertEqual(conn.closed, True)
+
+ def test_del(self):
+ conn = DummyConnection()
+ manager = self._makeOne()
+ manager(conn)
+ del manager
+ self.assertEqual(conn.closed, True)
+
+ def test_commit(self):
+ conn = DummyConnection()
+ txn = DummyTransaction()
+ manager = self._makeOne()
+ manager(conn)
+ manager.commit(txn)
+ self.assertEqual(txn.committed, True)
+
+class DummyConnection:
+ def close(self):
+ self.closed = True
+
+class DummyTransaction:
+ def commit(self):
+ self.committed = True
+
+from repoze.catalog.catalog import ICatalogIndex
+from zope.interface import implements
+
+class DummyIndex(object):
+ implements(ICatalogIndex)
+
+ value = None
+ docid = None
+
+ def __init__(self, *arg, **kw):
+ self.arg = arg
+ self.kw = kw
+
+ def index_doc(self, docid, value):
+ self.docid = docid
+ self.value = value
+ return value
+
+ def unindex_doc(self, docid):
+ self.unindexed = docid
+
+ def clear(self):
+ self.cleared = True
+
+ def apply(self, query):
+ return self.arg[0]
+
+
2  setup.cfg
@@ -0,0 +1,2 @@
+[easy_install]
+index_url = http://dist.repoze.org/lemonade/dev/simple
59 setup.py
@@ -0,0 +1,59 @@
+##############################################################################
+#
+# Copyright (c) 2008 Agendaless Consulting and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the BSD-like license at
+# http://www.repoze.org/LICENSE.txt. A copy of the license should accompany
+# this distribution. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL
+# EXPRESS OR IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND
+# FITNESS FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+__version__ = '0.1'
+
+import os
+
+from ez_setup import use_setuptools
+use_setuptools()
+
+from setuptools import setup, find_packages
+
+here = os.path.abspath(os.path.dirname(__file__))
+README = open(os.path.join(here, 'README.txt')).read()
+CHANGES = open(os.path.join(here, 'CHANGES.txt')).read()
+
+setup(name='repoze.catalog',
+ version=__version__,
+ description='Searching and indexing based on zope.index',
+ long_description=README + '\n\n' + CHANGES,
+ classifiers=[
+ "Development Status :: 3 - Alpha",
+ "Intended Audience :: Developers",
+ "Programming Language :: Python",
+ "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
+ ],
+ keywords='indexing catalog search',
+ author="Agendaless Consulting",
+ author_email="repoze-dev@lists.repoze.org",
+ url="http://www.repoze.org",
+ license="BSD-derived (http://www.repoze.org/LICENSE.txt)",
+ packages=find_packages(),
+ include_package_data=True,
+ namespace_packages=['repoze'],
+ zip_safe=False,
+ tests_require = [
+ 'repoze.bfg',
+ 'zope.index',
+ ],
+ install_requires = [
+ 'repoze.bfg',
+ 'zope.index',
+ ],
+ test_suite="repoze.catalog.tests",
+ entry_points = """\
+ """
+ )
+

0 comments on commit d86f180

Please sign in to comment.
Something went wrong with that request. Please try again.