-
Notifications
You must be signed in to change notification settings - Fork 86
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Toby White
committed
May 17, 2011
1 parent
2080e7e
commit 11c1019
Showing
12 changed files
with
1,855 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,130 @@ | ||
# Makefile for Sphinx documentation | ||
# | ||
|
||
# You can set these variables from the command line. | ||
SPHINXOPTS = | ||
SPHINXBUILD = sphinx-build | ||
PAPER = | ||
BUILDDIR = _build | ||
|
||
# Internal variables. | ||
PAPEROPT_a4 = -D latex_paper_size=a4 | ||
PAPEROPT_letter = -D latex_paper_size=letter | ||
ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . | ||
|
||
.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest | ||
|
||
help: | ||
@echo "Please use \`make <target>' where <target> is one of" | ||
@echo " html to make standalone HTML files" | ||
@echo " dirhtml to make HTML files named index.html in directories" | ||
@echo " singlehtml to make a single large HTML file" | ||
@echo " pickle to make pickle files" | ||
@echo " json to make JSON files" | ||
@echo " htmlhelp to make HTML files and a HTML help project" | ||
@echo " qthelp to make HTML files and a qthelp project" | ||
@echo " devhelp to make HTML files and a Devhelp project" | ||
@echo " epub to make an epub" | ||
@echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" | ||
@echo " latexpdf to make LaTeX files and run them through pdflatex" | ||
@echo " text to make text files" | ||
@echo " man to make manual pages" | ||
@echo " changes to make an overview of all changed/added/deprecated items" | ||
@echo " linkcheck to check all external links for integrity" | ||
@echo " doctest to run all doctests embedded in the documentation (if enabled)" | ||
|
||
clean: | ||
-rm -rf $(BUILDDIR)/* | ||
|
||
html: | ||
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html | ||
@echo | ||
@echo "Build finished. The HTML pages are in $(BUILDDIR)/html." | ||
|
||
dirhtml: | ||
$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml | ||
@echo | ||
@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." | ||
|
||
singlehtml: | ||
$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml | ||
@echo | ||
@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." | ||
|
||
pickle: | ||
$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle | ||
@echo | ||
@echo "Build finished; now you can process the pickle files." | ||
|
||
json: | ||
$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json | ||
@echo | ||
@echo "Build finished; now you can process the JSON files." | ||
|
||
htmlhelp: | ||
$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp | ||
@echo | ||
@echo "Build finished; now you can run HTML Help Workshop with the" \ | ||
".hhp project file in $(BUILDDIR)/htmlhelp." | ||
|
||
qthelp: | ||
$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp | ||
@echo | ||
@echo "Build finished; now you can run "qcollectiongenerator" with the" \ | ||
".qhcp project file in $(BUILDDIR)/qthelp, like this:" | ||
@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/sunburnt.qhcp" | ||
@echo "To view the help file:" | ||
@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/sunburnt.qhc" | ||
|
||
devhelp: | ||
$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp | ||
@echo | ||
@echo "Build finished." | ||
@echo "To view the help file:" | ||
@echo "# mkdir -p $$HOME/.local/share/devhelp/sunburnt" | ||
@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/sunburnt" | ||
@echo "# devhelp" | ||
|
||
epub: | ||
$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub | ||
@echo | ||
@echo "Build finished. The epub file is in $(BUILDDIR)/epub." | ||
|
||
latex: | ||
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex | ||
@echo | ||
@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." | ||
@echo "Run \`make' in that directory to run these through (pdf)latex" \ | ||
"(use \`make latexpdf' here to do that automatically)." | ||
|
||
latexpdf: | ||
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex | ||
@echo "Running LaTeX files through pdflatex..." | ||
make -C $(BUILDDIR)/latex all-pdf | ||
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." | ||
|
||
text: | ||
$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text | ||
@echo | ||
@echo "Build finished. The text files are in $(BUILDDIR)/text." | ||
|
||
man: | ||
$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man | ||
@echo | ||
@echo "Build finished. The manual pages are in $(BUILDDIR)/man." | ||
|
||
changes: | ||
$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes | ||
@echo | ||
@echo "The overview file is in $(BUILDDIR)/changes." | ||
|
||
linkcheck: | ||
$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck | ||
@echo | ||
@echo "Link check complete; look for any errors in the above output " \ | ||
"or in $(BUILDDIR)/linkcheck/output.txt." | ||
|
||
doctest: | ||
$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest | ||
@echo "Testing of doctests in the sources finished, look at the " \ | ||
"results in $(BUILDDIR)/doctest/output.txt." |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
.. _about: | ||
|
||
About Sunburnt | ||
============== | ||
|
||
Sunburnt is a library to interface with a Solr instance from Python. It was written by Toby White <toby@timetric.com>, originally for use with the Timetric platform (http://timetric.com). | ||
|
||
Sunburnt is designed to provide a high level API for | ||
|
||
* querying Solr in a Pythonic way, without having to understand Solr's query syntax in depth, and | ||
* inserting Python objects into a Solr index with the minimum of fuss, | ||
|
||
and particularly importantly, to provide Python-level error-checking. If you make a mistake, sunburnt will do its best to tell you why, rather than just throwing back an obscure Solr error. | ||
|
||
For an overview of the design choices, see http://eaddrinu.se/blog/2010/sunburnt.html. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,93 @@ | ||
.. _addingdocuments: | ||
|
||
Adding documents | ||
================ | ||
|
||
The easiest way to add data to the sunburnt instance is to do so using a Python dictionary, of exactly the same form as a query result. The dictionary keys are the names of the fields, and the dictionary values are the values of the corresponding fields. | ||
|
||
:: | ||
|
||
document = {"id":"0553573403", | ||
"cat":"book", | ||
"name":"A Game of Thrones", | ||
"price":7.99, | ||
"inStock": True, | ||
"author_t": | ||
"George R.R. Martin", | ||
"series_t":"A Song of Ice and Fire", | ||
"sequence_i":1, | ||
"genre_s":"fantasy"} | ||
|
||
si.add(document) | ||
|
||
You can add lists of dictionaries in the same way. Given the example "books.csv" file, you could feed it to sunburnt like so: | ||
|
||
:: | ||
|
||
lines = csv.reader(”books.csv”) | ||
field_names = lines.next() | ||
documents = [dict(zip(field_names, line) for line in lines] | ||
si.add(documents) | ||
si.commit() | ||
|
||
.. note:: Committing changes | ||
|
||
Solr separates out the act of adding documents to the index (with ``add()`` above) | ||
and committing them (with ``commit()``). Only after they are committed will they | ||
be searchable. However, you can set your Solr instance up to *autocommit* after | ||
adding documents, so that you don’t need to do a separate commit step. See | ||
http://wiki.apache.org/solr/SolrConfigXml#Update_Handler_Section. For simple Solr | ||
instances, this is probably the easiest approach. For heavily used instances, you | ||
should think carefully about your committing strategy. | ||
|
||
If your data is coming from somewhere else, though, you may not already have it in the | ||
form of a dictionary. So sunburnt will accept arbitrary Python objects as input to ``add()``. | ||
To extract the fields, it will inspect the objects for attributes or methods corresponding | ||
to field names, and use the values of the attributes (or, the result of calling the methods) as values. | ||
|
||
So in the case above, we might have an object that looked like this: | ||
|
||
:: | ||
|
||
class Book(object): | ||
name = “A Game of Thrones” | ||
author_t = “George R.R. Martin” | ||
id = “0553573403” | ||
series_t = “A Song of Ice and Fire” | ||
sequence_i = 1 | ||
|
||
def price(self): | ||
return check_current_price(self) | ||
|
||
def inStock(self): | ||
return check_stock_levels(self) > 0 | ||
|
||
|
||
Adding this to the Solr index is as simple as: | ||
|
||
:: | ||
|
||
si.add(Book()) | ||
|
||
(and you can add a list of books in the same way) | ||
|
||
This is particularly powerful if you’re using something like Django, | ||
which provides you with ORM objects - you can drop these ORM objects | ||
straight into Solr. Given a Django ``Book`` model, you could add the | ||
whole contents of your database with the single call: | ||
|
||
:: | ||
|
||
si.add(Book.objects.all()) | ||
|
||
When adding very large quantities of data, you might have a source | ||
which is lazily constructed. With Django, you'd really rather construct | ||
an ORM iterator, and have sunburnt work its way through the iterator | ||
lazily, in multiple updates, rather than try and construct a single | ||
huge update POST. You can do this by doing: | ||
|
||
:: | ||
|
||
si.add(Book.objects.iterator(), chunk=1000) | ||
|
||
where ``chunk`` controls how many documents are put into each update chunk. |
Oops, something went wrong.