FileFormat.txt Makefile bench.c include/analysis.h include/array.h include/bitvector.h include/defines.h include/document.h include/except.h include/global.h include/hash.h include/hashset.h include/helper.h include/index.h include/mem_pool.h include/posh.h include/priorityqueue.h include/search.h include/similarity.h include/store.h include/threading.h include/vector.h include/win32.h lib/libstemmer_c/MANIFEST lib/libstemmer_c/Makefile lib/libstemmer_c/README lib/libstemmer_c/examples/stemwords.c lib/libstemmer_c/include/libstemmer.h lib/libstemmer_c/libstemmer/libstemmer.c lib/libstemmer_c/libstemmer/modules.h lib/libstemmer_c/mkinc.mak lib/libstemmer_c/runtime/api.c lib/libstemmer_c/runtime/api.h lib/libstemmer_c/runtime/header.h lib/libstemmer_c/runtime/utilities.c lib/libstemmer_c/src_c/stem_ISO_8859_1_danish.c lib/libstemmer_c/src_c/stem_ISO_8859_1_danish.h lib/libstemmer_c/src_c/stem_ISO_8859_1_dutch.c lib/libstemmer_c/src_c/stem_ISO_8859_1_dutch.h lib/libstemmer_c/src_c/stem_ISO_8859_1_english.c lib/libstemmer_c/src_c/stem_ISO_8859_1_english.h lib/libstemmer_c/src_c/stem_ISO_8859_1_finnish.c lib/libstemmer_c/src_c/stem_ISO_8859_1_finnish.h lib/libstemmer_c/src_c/stem_ISO_8859_1_french.c lib/libstemmer_c/src_c/stem_ISO_8859_1_french.h lib/libstemmer_c/src_c/stem_ISO_8859_1_german.c lib/libstemmer_c/src_c/stem_ISO_8859_1_german.h lib/libstemmer_c/src_c/stem_ISO_8859_1_italian.c lib/libstemmer_c/src_c/stem_ISO_8859_1_italian.h lib/libstemmer_c/src_c/stem_ISO_8859_1_norwegian.c lib/libstemmer_c/src_c/stem_ISO_8859_1_norwegian.h lib/libstemmer_c/src_c/stem_ISO_8859_1_porter.c lib/libstemmer_c/src_c/stem_ISO_8859_1_porter.h lib/libstemmer_c/src_c/stem_ISO_8859_1_portuguese.c lib/libstemmer_c/src_c/stem_ISO_8859_1_portuguese.h lib/libstemmer_c/src_c/stem_ISO_8859_1_spanish.c lib/libstemmer_c/src_c/stem_ISO_8859_1_spanish.h lib/libstemmer_c/src_c/stem_ISO_8859_1_swedish.c lib/libstemmer_c/src_c/stem_ISO_8859_1_swedish.h lib/libstemmer_c/src_c/stem_KOI8_R_russian.c lib/libstemmer_c/src_c/stem_KOI8_R_russian.h lib/libstemmer_c/src_c/stem_UTF_8_danish.c lib/libstemmer_c/src_c/stem_UTF_8_danish.h lib/libstemmer_c/src_c/stem_UTF_8_dutch.c lib/libstemmer_c/src_c/stem_UTF_8_dutch.h lib/libstemmer_c/src_c/stem_UTF_8_english.c lib/libstemmer_c/src_c/stem_UTF_8_english.h lib/libstemmer_c/src_c/stem_UTF_8_finnish.c lib/libstemmer_c/src_c/stem_UTF_8_finnish.h lib/libstemmer_c/src_c/stem_UTF_8_french.c lib/libstemmer_c/src_c/stem_UTF_8_french.h lib/libstemmer_c/src_c/stem_UTF_8_german.c lib/libstemmer_c/src_c/stem_UTF_8_german.h lib/libstemmer_c/src_c/stem_UTF_8_italian.c lib/libstemmer_c/src_c/stem_UTF_8_italian.h lib/libstemmer_c/src_c/stem_UTF_8_norwegian.c lib/libstemmer_c/src_c/stem_UTF_8_norwegian.h lib/libstemmer_c/src_c/stem_UTF_8_porter.c lib/libstemmer_c/src_c/stem_UTF_8_porter.h lib/libstemmer_c/src_c/stem_UTF_8_portuguese.c lib/libstemmer_c/src_c/stem_UTF_8_portuguese.h lib/libstemmer_c/src_c/stem_UTF_8_russian.c lib/libstemmer_c/src_c/stem_UTF_8_russian.h lib/libstemmer_c/src_c/stem_UTF_8_spanish.c lib/libstemmer_c/src_c/stem_UTF_8_spanish.h lib/libstemmer_c/src_c/stem_UTF_8_swedish.c lib/libstemmer_c/src_c/stem_UTF_8_swedish.h search_bench.c src/analysis.c src/array.c src/bitvector.c src/compound_io.c src/document.c src/except.c src/filter.c src/fs_store.c src/global.c src/hash.c src/hashset.c src/helper.c src/index.c src/mem_pool.c src/posh.c src/priorityqueue.c src/q_boolean.c src/q_const_score.c src/q_filtered_query.c src/q_fuzzy.c src/q_match_all.c src/q_multi_term.c src/q_parser.c src/q_parser.y src/q_phrase.c src/q_prefix.c src/q_range.c src/q_span.c src/q_term.c src/q_wildcard.c src/ram_store.c src/search.c src/similarity.c src/sort.c src/stopwords.c src/store.c styleguide.html test/all_tests.h test/test.c test/test.h test/test_analysis.c test/test_array.c test/test_bitvector.c test/test_compound_io.c test/test_document.c test/test_except.c test/test_fields.c test/test_filter.c test/test_fs_store.c test/test_global.c test/test_hash.c test/test_hashset.c test/test_helper.c test/test_index.c test/test_mem_pool.c test/test_priorityqueue.c test/test_q_const_score.c test/test_q_filtered.c test/test_q_fuzzy.c test/test_q_parser.c test/test_q_span.c test/test_ram_store.c test/test_search.c test/test_segments.c test/test_similarity.c test/test_sort.c test/test_store.c test/test_store.h test/test_term.c test/test_term_vectors.c test/test_test.c test/testhelper.c test/testhelper.h tools/make_makefile.rb tools/mkmf.pl @@ -1,114 +1,33 @@ -= Ferret +## Lucy -Ferret is a Ruby port of the Java Lucene search engine. -(http://jakarta.apache.org/lucene/) In the same way as Lucene, it is not a -standalone application, but a library you can use to index documents and -search for things in them later. +C Search engine library -== Requirements -* Ruby 1.8 -* (C compiler to build the extension but not required to use Ferret) +## Installation -== Installation +Currently there is nothing to install. You can run the tests by running Make. -If you have gems installed you can simply do; - gem install ferret +## Coding Style -Otherwise, you will need Rake installed. De-compress the archive and enter its top directory. +Lucy is using the Apache coding style. See styleguide.html for a description. +This style can be generated with the following arguments to GNU indent: - tar zxpvf ferret-<version>.tar.gz - cd ferret-<version> + -i4 -npsl -di0 -br -nce -d0 -cli0 -npcs -nfc1 -nut -ncs -Run the following; - $ rake ext - $ ruby setup.rb config - $ ruby setup.rb setup - # sudo ruby setup.rb install - -These simple steps install ferret in the default location of Ruby libraries. -You can also install files into your favorite directory by supplying setup.rb -some options. Try; +## COPYRIGHT AND LICENCE - $ ruby setup.rb --help +Copyright 2006 Marvin Humphrey & David Balmain +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at -== Usage + http://www.apache.org/licenses/LICENSE-2.0 -You can read the TUTORIAL which you'll find in the same directory as this -README. You can also check the following modules for more specific -documentation. - -* Ferret::Analysis: for more information on how the data is processed when it - is tokenized. There are a number of things you can do with your data such as - adding stop lists or perhaps a porter stemmer. There are also a number of - analyzers already available and it is almost trivial to create a new one - with a simple regular expression. - -* Ferret::Search: for more information on querying the index. There are a - number of already available queries and it's unlikely you'll need to create - your own. You may however want to take advantage of the sorting or filtering - abilities of Ferret to present your data the best way you see fit. - -* Ferret::Document: to find out how to create documents. This part of Ferret - is relatively straightforward. The main thing that we haven't gone into here - is the use of term vectors. These allow you to store and retrieve the - positions and offsets of the data which can be very useful in document - comparison amoung other things. == More information - -* Ferret::QueryParser: if you want to find out more about what you can do with - Ferret's Query Parser, this is the place to look. The query parser is one - area that could use a bit of work so please send your suggestions. - -* Ferret::Index: for more advanced access to the index you'll probably want to - use the Ferret::Index::IndexWriter and Ferret::Index::IndexReader. This is - the place to look for more information on them. - -* Ferret::Store: This is the module used to access the actual index storage - and won't be of much interest to most people. - -=== Performance - -Currently Ferret is an order of magnitude slower than Java Lucene which can be -quite a pain at times. I have written some basic C extensions which may or may -not have installed when you installed Ferret. These double the speed but still -leave it a lot slower than the Java version. I have, however, ported the -indexing part of Java Lucene to C and it is an order of magnitude faster then -the Java version. Once I'm pretty certain that the API of Ferret has settled -and won't be changing much, I'll intergrate my C version. So expect to see -Ferret running faster than Java Lucene some time in the future. If you'd like -to try cferret and test my claims, let me know (if you haven't already found -it in my subversion repository). It's not currently portable and will probably -only run on linux. - -== Contact - -For bug reports and patches I have set up Trac here; - - http://ferret.davebalmain.com/trac - -Queries, discussion etc should be addressed to the forum or mailing lists hosted -at; - - http://rubyforge.org/projects/ferret/ - -Alternatively you could create a new page for discussion on the wiki at my Trac -page above. Or, if you're shy, please feel free to email me directly at dbalmain@gmail.com - -Of course, since Ferret is almost a straight port of Java Lucene, -everything said about Lucene at http://jakarta.apache.org/lucene/ should -be true about Ferret. Apart from the bits about it being in Java. - -== Authors - -[<b>David Balmain</b>] Port to Ruby - -[The Apache Software Foundation (Doug Cutting and friends)] Original Apache Lucene - -== License - -Ferret is available under an MIT-style license. - -:include: MIT-LICENSE +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. README MIT-LICENSE Rakefile TODO TUTORIAL ext/dummy.exe ext/extconf.rb ext/ferret.c ext/ferret.h ext/inc/except.h ext/inc/lang.h ext/lang.c ext/lang.h ext/r_analysis.c ext/r_doc.c ext/r_index_io.c ext/r_qparser.c ext/r_search.c ext/r_store.c ext/r_term.c ferret_valgrind.supp lib/ferret.rb lib/ferret/analysis.rb lib/ferret/analysis/analyzers.rb lib/ferret/analysis/standard_tokenizer.rb lib/ferret/analysis/token.rb lib/ferret/analysis/token_filters.rb lib/ferret/analysis/token_stream.rb lib/ferret/analysis/tokenizers.rb lib/ferret/analysis/word_list_loader.rb lib/ferret/document.rb lib/ferret/document/document.rb lib/ferret/document/field.rb lib/ferret/index.rb lib/ferret/index/compound_file_io.rb lib/ferret/index/document_writer.rb lib/ferret/index/field_infos.rb lib/ferret/index/fields_io.rb lib/ferret/index/index.rb lib/ferret/index/index_file_names.rb lib/ferret/index/index_reader.rb lib/ferret/index/index_writer.rb lib/ferret/index/multi_reader.rb lib/ferret/index/multiple_term_doc_pos_enum.rb lib/ferret/index/segment_infos.rb lib/ferret/index/segment_merge_info.rb lib/ferret/index/segment_merge_queue.rb lib/ferret/index/segment_merger.rb lib/ferret/index/segment_reader.rb lib/ferret/index/segment_term_enum.rb lib/ferret/index/segment_term_vector.rb lib/ferret/index/term.rb lib/ferret/index/term_buffer.rb lib/ferret/index/term_doc_enum.rb lib/ferret/index/term_enum.rb lib/ferret/index/term_info.rb lib/ferret/index/term_infos_io.rb lib/ferret/index/term_vector_offset_info.rb lib/ferret/index/term_vectors_io.rb lib/ferret/query_parser.rb lib/ferret/query_parser/query_parser.tab.rb lib/ferret/query_parser/query_parser.y lib/ferret/search.rb lib/ferret/search/Notes.txt lib/ferret/search/boolean_clause.rb lib/ferret/search/boolean_query.rb lib/ferret/search/boolean_scorer.rb lib/ferret/search/caching_wrapper_filter.rb lib/ferret/search/conjunction_scorer.rb lib/ferret/search/disjunction_sum_scorer.rb lib/ferret/search/exact_phrase_scorer.rb lib/ferret/search/explanation.rb lib/ferret/search/field_cache.rb lib/ferret/search/field_doc.rb lib/ferret/search/field_sorted_hit_queue.rb lib/ferret/search/filter.rb lib/ferret/search/filtered_query.rb lib/ferret/search/filtered_term_enum.rb lib/ferret/search/fuzzy_query.rb lib/ferret/search/fuzzy_term_enum.rb lib/ferret/search/hit_collector.rb lib/ferret/search/hit_queue.rb lib/ferret/search/index_searcher.rb lib/ferret/search/match_all_query.rb lib/ferret/search/multi_phrase_query.rb lib/ferret/search/multi_searcher.rb lib/ferret/search/multi_term_query.rb lib/ferret/search/non_matching_scorer.rb lib/ferret/search/package.html lib/ferret/search/phrase_positions.rb lib/ferret/search/phrase_query.rb lib/ferret/search/phrase_scorer.rb lib/ferret/search/prefix_query.rb lib/ferret/search/query.rb lib/ferret/search/query_filter.rb lib/ferret/search/range_filter.rb lib/ferret/search/range_query.rb lib/ferret/search/req_excl_scorer.rb lib/ferret/search/req_opt_sum_scorer.rb lib/ferret/search/score_doc.rb lib/ferret/search/score_doc_comparator.rb lib/ferret/search/scorer.rb lib/ferret/search/similarity.rb lib/ferret/search/sloppy_phrase_scorer.rb lib/ferret/search/sort.rb lib/ferret/search/sort_comparator.rb lib/ferret/search/sort_field.rb lib/ferret/search/spans.rb lib/ferret/search/spans/near_spans_enum.rb lib/ferret/search/spans/package.html lib/ferret/search/spans/span_first_query.rb lib/ferret/search/spans/span_near_query.rb lib/ferret/search/spans/span_not_query.rb lib/ferret/search/spans/span_or_query.rb lib/ferret/search/spans/span_query.rb lib/ferret/search/spans/span_scorer.rb lib/ferret/search/spans/span_term_query.rb lib/ferret/search/spans/span_weight.rb lib/ferret/search/spans/spans_enum.rb lib/ferret/search/term_query.rb lib/ferret/search/term_scorer.rb lib/ferret/search/top_docs.rb lib/ferret/search/top_field_docs.rb lib/ferret/search/weight.rb lib/ferret/search/wildcard_query.rb lib/ferret/search/wildcard_term_enum.rb lib/ferret/stemmers.rb lib/ferret/stemmers/porter_stemmer.rb lib/ferret/store.rb lib/ferret/store/buffered_index_io.rb lib/ferret/store/directory.rb lib/ferret/store/fs_store.rb lib/ferret/store/index_io.rb lib/ferret/store/ram_store.rb lib/ferret/utils.rb lib/ferret/utils/bit_vector.rb lib/ferret/utils/date_tools.rb lib/ferret/utils/number_tools.rb lib/ferret/utils/parameter.rb lib/ferret/utils/priority_queue.rb lib/ferret/utils/string_helper.rb lib/ferret/utils/thread_local.rb lib/ferret/utils/weak_key_hash.rb lib/rferret.rb rake_utils/code_statistics.rb samples/hash_cmp.rb samples/state_analyzer.rb samples/tz_tester.rb setup.rb test/benchmark/fsdir/rw_vint.test test/benchmark/tb_ram_store.rb test/benchmark/tb_rw_vint.rb test/functional/thread_safety_index_test.rb test/functional/thread_safety_test.rb test/longrunning/tc_numbertools.rb test/longrunning/tm_store.rb test/test_all.rb test/test_helper.rb test/unit/analysis/ctc_analyzer.rb test/unit/analysis/ctc_tokenstream.rb test/unit/analysis/data/wordfile test/unit/analysis/rtc_letter_tokenizer.rb test/unit/analysis/rtc_lower_case_filter.rb test/unit/analysis/rtc_lower_case_tokenizer.rb test/unit/analysis/rtc_per_field_analyzer_wrapper.rb test/unit/analysis/rtc_porter_stem_filter.rb test/unit/analysis/rtc_standard_analyzer.rb test/unit/analysis/rtc_standard_tokenizer.rb test/unit/analysis/rtc_stop_analyzer.rb test/unit/analysis/rtc_stop_filter.rb test/unit/analysis/rtc_white_space_analyzer.rb test/unit/analysis/rtc_white_space_tokenizer.rb test/unit/analysis/rtc_word_list_loader.rb test/unit/analysis/tc_analyzer.rb test/unit/analysis/tc_token.rb test/unit/document/rtc_field.rb test/unit/document/tc_document.rb test/unit/document/tc_field.rb test/unit/index/rtc_compound_file_io.rb test/unit/index/rtc_field_infos.rb test/unit/index/rtc_fields_io.rb test/unit/index/rtc_multiple_term_doc_pos_enum.rb test/unit/index/rtc_segment_infos.rb test/unit/index/rtc_segment_term_docs.rb test/unit/index/rtc_segment_term_enum.rb test/unit/index/rtc_segment_term_vector.rb test/unit/index/rtc_term_buffer.rb test/unit/index/rtc_term_info.rb test/unit/index/rtc_term_infos_io.rb test/unit/index/rtc_term_vectors_io.rb test/unit/index/tc_index.rb test/unit/index/tc_index_reader.rb test/unit/index/tc_index_writer.rb test/unit/index/tc_term.rb test/unit/index/tc_term_voi.rb test/unit/index/th_doc.rb test/unit/query_parser/tc_query_parser.rb test/unit/search/rtc_similarity.rb test/unit/search/rtc_sort_field.rb test/unit/search/tc_filter.rb test/unit/search/tc_fuzzy_query.rb test/unit/search/tc_index_searcher.rb test/unit/search/tc_multi_searcher.rb test/unit/search/tc_multi_searcher2.rb test/unit/search/tc_search_and_sort.rb test/unit/search/tc_sort.rb test/unit/search/tc_sort_field.rb test/unit/search/tc_spans.rb test/unit/store/rtc_fs_store.rb test/unit/store/rtc_ram_store.rb test/unit/store/rtm_store.rb test/unit/store/rtm_store_lock.rb test/unit/store/tc_fs_store.rb test/unit/store/tc_ram_store.rb test/unit/store/tm_store.rb test/unit/store/tm_store_lock.rb test/unit/ts_analysis.rb test/unit/ts_document.rb test/unit/ts_index.rb test/unit/ts_query_parser.rb test/unit/ts_search.rb test/unit/ts_store.rb test/unit/ts_utils.rb test/unit/utils/rtc_bit_vector.rb test/unit/utils/rtc_date_tools.rb test/unit/utils/rtc_number_tools.rb test/unit/utils/rtc_parameter.rb test/unit/utils/rtc_priority_queue.rb test/unit/utils/rtc_string_helper.rb test/unit/utils/rtc_thread.rb test/unit/utils/rtc_weak_key_hash.rb test/utils/number_to_spoken.rb 83df7f57e75f82cd938bf3da37ac74a36e71c7e6 44adeb150217b53f93edbb749fa59eb8ce3b4165 dbalmain dbalmain@06fd6eb0-0002-0410-a719-e5602cce40bc http://github.com/dbalmain/ferret/commit/7134d5f61c5ae39a894078a2b5b4afaba1048b43 7134d5f61c5ae39a894078a2b5b4afaba1048b43 2007-02-27T18:14:17-08:00 2007-02-27T18:14:17-08:00 Finally moved the experimental version of ferret into the proper ferret repository. git-svn-id: svn+ssh://davebalmain.com/home/dave/repos/ferret/trunk@737 06fd6eb0-0002-0410-a719-e5602cce40bc 3717d90eba2040894646c1f1b20d78ac2cdde089 dbalmain dbalmain@06fd6eb0-0002-0410-a719-e5602cce40bc