From 932c211412d89078a5ebc2400fd36540f534f0ba Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Thu, 3 Dec 2015 16:06:44 -0800 Subject: [PATCH 01/15] Test Go bindings for Lexicon. Use SegLexicon to test out Lexicon. --- go/lucy/index_test.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/go/lucy/index_test.go b/go/lucy/index_test.go index e7c93a90d..2df680f3d 100644 --- a/go/lucy/index_test.go +++ b/go/lucy/index_test.go @@ -642,3 +642,36 @@ func TestInverterMisc(t *testing.T) { t.Errorf("GetInversion after iterator exhausted: %v", got) } } + +// Use SegLexicon to air out the Lexicon interface. +func TestLexiconBasics(t *testing.T) { + folder := createTestIndex("a", "b", "c") + searcher, _ := OpenIndexSearcher(folder) + segReaders := searcher.GetReader().SegReaders() + lexReader := segReaders[0].(SegReader).Obtain("Lucy::Index::LexiconReader").(LexiconReader) + segLex := lexReader.Lexicon("content", nil).(Lexicon) + if field := segLex.getField(); field != "content" { + t.Errorf("getField: %s", field) + } + segLex.Next() + if got := segLex.GetTerm(); got != "a" { + t.Errorf("GetTerm: %v", got) + } + if docFreq := segLex.docFreq(); docFreq != 1 { + t.Errorf("docFreq: %d", docFreq) + } + if !segLex.Next() || !segLex.Next() { + t.Errorf("Iterate") + } + if segLex.Next() { + t.Errorf("Iteration should be finished") + } + segLex.Seek("b") + if got := segLex.GetTerm(); got != "b" { + t.Errorf("Seek: %v", got) + } + segLex.Reset() + if !segLex.Next() { + t.Errorf("Next after Reset") + } +} From a693aa1decaff2b4e33eaacc180cf95b680d8317 Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Thu, 3 Dec 2015 17:37:07 -0800 Subject: [PATCH 02/15] Test Go bindings for PostingList. --- go/lucy/index_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/go/lucy/index_test.go b/go/lucy/index_test.go index 2df680f3d..5affb0daa 100644 --- a/go/lucy/index_test.go +++ b/go/lucy/index_test.go @@ -675,3 +675,27 @@ func TestLexiconBasics(t *testing.T) { t.Errorf("Next after Reset") } } + +func TestPostingListBasics(t *testing.T) { + folder := createTestIndex("c", "b b b", "a", "b",) + searcher, _ := OpenIndexSearcher(folder) + segReaders := searcher.GetReader().SegReaders() + pListReader := segReaders[0].(SegReader).Obtain("Lucy::Index::PostingListReader").(PostingListReader) + pList := pListReader.PostingList("content", nil) + pList.Seek("b") + if docFreq := pList.GetDocFreq(); docFreq != 2 { + t.Errorf("GetDocFreq: %d", docFreq) + } + if got := pList.Next(); got != 2 { + t.Errorf("Next: %d", got) + } + if docID := pList.GetDocID(); docID != 2 { + t.Errorf("GetDocID: %d", docID) + } + if got := pList.Next(); got != 4 { + t.Errorf("Next (second iter): %d", got) + } + if got := pList.Next(); got != 0 { + t.Error("Next (done): %d", got) + } +} From d262be92763c9d145fd7fdcd3cef07cc8ec18a13 Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Thu, 3 Dec 2015 19:13:19 -0800 Subject: [PATCH 03/15] Test Go bindings for Posting. --- go/lucy/index_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/go/lucy/index_test.go b/go/lucy/index_test.go index 5affb0daa..88e8924e3 100644 --- a/go/lucy/index_test.go +++ b/go/lucy/index_test.go @@ -699,3 +699,16 @@ func TestPostingListBasics(t *testing.T) { t.Error("Next (done): %d", got) } } + +func TestPostingBasics(t *testing.T) { + sim := NewSimilarity() + posting := NewMatchPosting(sim) + posting.SetDocID(42) + if got := posting.GetDocID(); got != 42 { + t.Errorf("Set/GetDocID: %d", got) + } + posting.Reset() + if got := posting.getFreq(); got != 0 { + t.Errorf("getFreq: %d", got) + } +} From 38bea5a604fd4d190bb49719f50d05e1a1c58de5 Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Tue, 8 Dec 2015 18:32:52 -0800 Subject: [PATCH 04/15] Tune and test Go bindings for DataReader. --- go/build.go | 6 ++++++ go/lucy/index.go | 44 +++++++++++++++++++++++++++++++++++++++++++ go/lucy/index_test.go | 38 +++++++++++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+) diff --git a/go/build.go b/go/build.go index 692702dc9..2bdab371a 100644 --- a/go/build.go +++ b/go/build.go @@ -185,6 +185,12 @@ func specClasses(parcel *cfc.Parcel) { indexerBinding.SetSuppressStruct(true) indexerBinding.Register() + dataReaderBinding := cfc.NewGoClass(parcel, "Lucy::Index::DataReader") + dataReaderBinding.SpecMethod("Aggregator", "Aggregator([]DataReader, []int32) (DataReader, error)") + dataReaderBinding.SpecMethod("Get_Segments", "GetSegments() []Segment") + dataReaderBinding.SpecMethod("Close", "Close() error") + dataReaderBinding.Register() + bgMergerBinding := cfc.NewGoClass(parcel, "Lucy::Index::BackgroundMerger") bgMergerBinding.SpecMethod("Prepare_Commit", "PrepareCommit() error") bgMergerBinding.SpecMethod("Commit", "Commit() error") diff --git a/go/lucy/index.go b/go/lucy/index.go index 230e050f0..defa9f216 100644 --- a/go/lucy/index.go +++ b/go/lucy/index.go @@ -18,6 +18,7 @@ package lucy /* #include "Lucy/Index/Indexer.h" +#include "Lucy/Index/DataReader.h" #include "Lucy/Index/IndexManager.h" #include "Lucy/Index/BackgroundMerger.h" #include "Lucy/Index/TermVector.h" @@ -415,3 +416,46 @@ func (s *SortCacheIMP) Find(term interface{}) (retval int32, err error) { }) return retval, err } + +func (d *DataReaderIMP) Aggregator(readers []DataReader, offsets []int32) (retval DataReader, err error) { + err = clownfish.TrapErr(func() { + self := (*C.lucy_DataReader)(clownfish.Unwrap(d, "d")) + size := len(readers) + readersC := C.cfish_Vec_new(C.size_t(size)) + defer C.cfish_decref(unsafe.Pointer(readersC)) + for i := 0; i < size; i++ { + elemC := clownfish.Unwrap(readers[i], "readers[i]") + C.CFISH_Vec_Push(readersC, C.cfish_incref(elemC)) + } + offs := NewI32Array(offsets) + offsetsC := (*C.lucy_I32Array)(clownfish.Unwrap(offs, "offs")) + retvalCF := C.LUCY_DataReader_Aggregator(self, readersC, offsetsC) + defer C.cfish_decref(unsafe.Pointer(retvalCF)) + if retvalCF != nil { + retval = clownfish.ToGo(unsafe.Pointer(retvalCF)).(DataReader) + } + }) + return retval, err +} + +func (d *DataReaderIMP) GetSegments() []Segment { + self := (*C.lucy_DataReader)(clownfish.Unwrap(d, "d")) + retvalCF := C.LUCY_DataReader_Get_Segments(self); + if retvalCF == nil { + return nil + } + size := C.CFISH_Vec_Get_Size(retvalCF) + retval := make([]Segment, int(size)) + for i := 0; i < int(size); i++ { + elem := unsafe.Pointer(C.CFISH_Vec_Fetch(retvalCF, C.size_t(i))) + retval[i] = clownfish.ToGo(unsafe.Pointer(C.cfish_incref(elem))).(Segment) + } + return retval +} + +func (d *DataReaderIMP) Close() error { + return clownfish.TrapErr(func() { + self := (*C.lucy_DataReader)(clownfish.Unwrap(d, "d")) + C.LUCY_DataReader_Close(self) + }) +} diff --git a/go/lucy/index_test.go b/go/lucy/index_test.go index 88e8924e3..40609e671 100644 --- a/go/lucy/index_test.go +++ b/go/lucy/index_test.go @@ -712,3 +712,41 @@ func TestPostingBasics(t *testing.T) { t.Errorf("getFreq: %d", got) } } + +// This function runs Close(), so the reader becomes unusable afterwards. +func runDataReaderCommon(t *testing.T, reader DataReader, runAggregator bool) { + if runAggregator { + got, err := reader.Aggregator([]DataReader{}, []int32{}) + if got == nil || err != nil { + t.Errorf("Aggregator: %#v, %v", got, err) + } + } + if got := reader.GetSchema(); false { + t.Errorf("GetSchema: %v", got) + } + if got := reader.GetFolder(); false { + t.Errorf("GetFolder: %v", got) + } + if got := reader.GetSnapshot(); false { + t.Errorf("GetSnapshot: %v", got) + } + if got := reader.GetSegments(); false { + t.Errorf("GetSegments: %#v", got) + } + if got := reader.GetSegment(); false { + t.Errorf("GetSegment: %#v", got) + } + if got := reader.GetSegTick(); false { + t.Errorf("GetSegTick: %d", got) + } + if err := reader.Close(); err != nil { + t.Errorf("Close: %v", err) + } +} + +func TestIndexReaderMisc(t *testing.T) { + folder := createTestIndex("a", "b", "c") + searcher, _ := OpenIndexSearcher(folder) + reader := searcher.GetReader() + runDataReaderCommon(t, reader, false) +} From c1cdb4d7912d0105ad5e00491769111eb8991d0b Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Tue, 8 Dec 2015 19:17:24 -0800 Subject: [PATCH 05/15] Tune and test Go bindings for DocReader. --- go/build.go | 6 ++++++ go/lucy/index.go | 23 +++++++++++++++++++++++ go/lucy/index_test.go | 23 +++++++++++++++++++++++ go/lucy/lucy.go | 21 ++------------------- go/lucy/search.go | 2 +- 5 files changed, 55 insertions(+), 20 deletions(-) diff --git a/go/build.go b/go/build.go index 2bdab371a..801654ae1 100644 --- a/go/build.go +++ b/go/build.go @@ -191,6 +191,12 @@ func specClasses(parcel *cfc.Parcel) { dataReaderBinding.SpecMethod("Close", "Close() error") dataReaderBinding.Register() + + docReaderBinding := cfc.NewGoClass(parcel, "Lucy::Index::DocReader") + docReaderBinding.SpecMethod("", "ReadDoc(int32, interface{}) error") + docReaderBinding.SpecMethod("Fetch_Doc", "FetchDoc(int32) (HitDoc, error)") + docReaderBinding.Register() + bgMergerBinding := cfc.NewGoClass(parcel, "Lucy::Index::BackgroundMerger") bgMergerBinding.SpecMethod("Prepare_Commit", "PrepareCommit() error") bgMergerBinding.SpecMethod("Commit", "Commit() error") diff --git a/go/lucy/index.go b/go/lucy/index.go index defa9f216..18d5ba156 100644 --- a/go/lucy/index.go +++ b/go/lucy/index.go @@ -19,6 +19,7 @@ package lucy /* #include "Lucy/Index/Indexer.h" #include "Lucy/Index/DataReader.h" +#include "Lucy/Index/DocReader.h" #include "Lucy/Index/IndexManager.h" #include "Lucy/Index/BackgroundMerger.h" #include "Lucy/Index/TermVector.h" @@ -459,3 +460,25 @@ func (d *DataReaderIMP) Close() error { C.LUCY_DataReader_Close(self) }) } + +func (d *DocReaderIMP) ReadDoc(docID int32, doc interface{}) error { + self := (*C.lucy_DocReader)(clownfish.Unwrap(d, "d")) + class := clownfish.GetClass(d) + classC := ((*C.cfish_Class)(clownfish.Unwrap(class, "class"))) + if classC == C.LUCY_DEFAULTDOCREADER { + return doReadDocData((*C.lucy_DefaultDocReader)(self), docID, doc) + } else if classC == C.LUCY_POLYDOCREADER { + return readDocPolyDR((*C.lucy_PolyDocReader)(self), docID, doc) + } else { + panic(clownfish.NewErr(fmt.Sprintf("Unexpected type: %s", class.GetName))) + } +} + +func (d *DocReaderIMP) FetchDoc(docID int32) (doc HitDoc, err error) { + err = clownfish.TrapErr(func() { + self := (*C.lucy_DocReader)(clownfish.Unwrap(d, "d")) + docC := C.LUCY_DocReader_Fetch_Doc(self, C.int32_t(docID)) + doc = WRAPHitDoc(unsafe.Pointer(docC)) + }) + return doc, err +} diff --git a/go/lucy/index_test.go b/go/lucy/index_test.go index 40609e671..893416701 100644 --- a/go/lucy/index_test.go +++ b/go/lucy/index_test.go @@ -750,3 +750,26 @@ func TestIndexReaderMisc(t *testing.T) { reader := searcher.GetReader() runDataReaderCommon(t, reader, false) } + +func TestDefaultDocReaderMisc(t *testing.T) { + folder := createTestIndex("a", "b", "c") + searcher, _ := OpenIndexSearcher(folder) + segReaders := searcher.GetReader().SegReaders() + reader := segReaders[0].(SegReader).Obtain("Lucy::Index::DocReader").(DefaultDocReader) + doc := make(map[string]interface{}) + if err := reader.ReadDoc(2, doc); err != nil { + t.Errorf("ReadDoc: %v", err) + } + runDataReaderCommon(t, reader, true) +} + +func TestPolyDocReaderMisc(t *testing.T) { + folder := createTestIndex("a", "b", "c") + searcher, _ := OpenIndexSearcher(folder) + reader := searcher.GetReader().Obtain("Lucy::Index::DocReader").(PolyDocReader) + doc := make(map[string]interface{}) + if err := reader.ReadDoc(2, doc); err != nil { + t.Errorf("ReadDoc: %v", err) + } + runDataReaderCommon(t, reader, true) +} diff --git a/go/lucy/lucy.go b/go/lucy/lucy.go index 03307bb6e..175ceeebf 100644 --- a/go/lucy/lucy.go +++ b/go/lucy/lucy.go @@ -383,20 +383,8 @@ func fetchEntry(ivars *C.lucy_InverterIVARS, fieldGo string) *C.lucy_InverterEnt return (*C.lucy_InverterEntry)(unsafe.Pointer(entry)) } -func fetchDocFromDocReader(dr DocReader, docID int32, doc interface{}) error { - switch v := dr.(type) { - case *DefaultDocReaderIMP: - return v.readDoc(docID, doc) - case *PolyDocReaderIMP: - return v.readDoc(docID, doc) - default: - panic(clownfish.NewErr(fmt.Sprintf("Unexpected type: %T", v))) - } -} - -func (pdr *PolyDocReaderIMP) readDoc(docID int32, doc interface{}) error { - self := (*C.lucy_PolyDocReader)(clownfish.Unwrap(pdr, "pdr")) - ivars := C.lucy_PolyDocReader_IVARS(self) +func readDocPolyDR(pdr *C.lucy_PolyDocReader, docID int32, doc interface{}) error { + ivars := C.lucy_PolyDocReader_IVARS(pdr) segTick := C.lucy_PolyReader_sub_tick(ivars.offsets, C.int32_t(docID)) offset := C.LUCY_I32Arr_Get(ivars.offsets, segTick) defDocReader := (*C.lucy_DefaultDocReader)(C.CFISH_Vec_Fetch(ivars.readers, C.size_t(segTick))) @@ -414,11 +402,6 @@ func (pdr *PolyDocReaderIMP) readDoc(docID int32, doc interface{}) error { return err } -func (ddr *DefaultDocReaderIMP) readDoc(docID int32, doc interface{}) error { - self := (*C.lucy_DefaultDocReader)(clownfish.Unwrap(ddr, "ddr")) - return doReadDocData(self, docID, doc) -} - func setMapField(store interface{}, field string, val interface{}) error { m := store.(map[string]interface{}) m[field] = val diff --git a/go/lucy/search.go b/go/lucy/search.go index e202b9df5..3f7845ce8 100644 --- a/go/lucy/search.go +++ b/go/lucy/search.go @@ -88,7 +88,7 @@ func (s *SearcherIMP) ReadDoc(docID int32, doc interface{}) error { return clownfish.NewErr("No DocReader available") } docReaderGo := clownfish.WRAPAny(unsafe.Pointer(C.cfish_incref(unsafe.Pointer(docReader)))).(DocReader) - return fetchDocFromDocReader(docReaderGo, docID, doc) + return docReaderGo.ReadDoc(docID, doc) } else { return clownfish.NewErr("Support for ReadDoc not implemented") } From fe25122fadcb9115ddd0810e9986898048e5303e Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Tue, 8 Dec 2015 20:33:58 -0800 Subject: [PATCH 06/15] Tune and test Go bindings for IndexReader. --- go/build.go | 5 +++ go/lucy/index.go | 47 +++++++++++++++++++++++++ go/lucy/index_test.go | 78 +++++++++++++++++++++++++++++++++--------- go/lucy/search_test.go | 9 +++-- 4 files changed, 118 insertions(+), 21 deletions(-) diff --git a/go/build.go b/go/build.go index 801654ae1..fc104dc2f 100644 --- a/go/build.go +++ b/go/build.go @@ -191,6 +191,11 @@ func specClasses(parcel *cfc.Parcel) { dataReaderBinding.SpecMethod("Close", "Close() error") dataReaderBinding.Register() + ixReaderBinding := cfc.NewGoClass(parcel, "Lucy::Index::IndexReader") + ixReaderBinding.SpecMethod("Seg_Readers", "SegReaders() []SegReader") + ixReaderBinding.SpecMethod("Offsets", "Offsets() []int32") + ixReaderBinding.SpecMethod("Obtain", "Obtain(string) (DataReader, error)") + ixReaderBinding.Register() docReaderBinding := cfc.NewGoClass(parcel, "Lucy::Index::DocReader") docReaderBinding.SpecMethod("", "ReadDoc(int32, interface{}) error") diff --git a/go/lucy/index.go b/go/lucy/index.go index 18d5ba156..67d4b4789 100644 --- a/go/lucy/index.go +++ b/go/lucy/index.go @@ -18,6 +18,7 @@ package lucy /* #include "Lucy/Index/Indexer.h" +#include "Lucy/Index/IndexReader.h" #include "Lucy/Index/DataReader.h" #include "Lucy/Index/DocReader.h" #include "Lucy/Index/IndexManager.h" @@ -482,3 +483,49 @@ func (d *DocReaderIMP) FetchDoc(docID int32) (doc HitDoc, err error) { }) return doc, err } + +func OpenIndexReader(index interface{}, snapshot Snapshot, manager IndexManager) (retval IndexReader, err error) { + err = clownfish.TrapErr(func() { + indexC := (*C.cfish_Obj)(clownfish.GoToClownfish(index, unsafe.Pointer(C.CFISH_OBJ), false)) + defer C.cfish_decref(unsafe.Pointer(indexC)) + snapshotC := (*C.lucy_Snapshot)(clownfish.UnwrapNullable(snapshot)) + managerC := (*C.lucy_IndexManager)(clownfish.UnwrapNullable(manager)) + cfObj := C.lucy_IxReader_open(indexC, snapshotC, managerC) + retval = clownfish.WRAPAny(unsafe.Pointer(cfObj)).(IndexReader) + }) + return retval, err +} + +func (r *IndexReaderIMP) SegReaders() []SegReader { + self := (*C.lucy_IndexReader)(clownfish.Unwrap(r, "r")) + retvalCF := C.LUCY_IxReader_Seg_Readers(self); + defer C.cfish_decref(unsafe.Pointer(retvalCF)) + if retvalCF == nil { + return nil + } + size := C.CFISH_Vec_Get_Size(retvalCF) + retval := make([]SegReader, int(size)) + for i := 0; i < int(size); i++ { + elem := unsafe.Pointer(C.CFISH_Vec_Fetch(retvalCF, C.size_t(i))) + retval[i] = clownfish.ToGo(unsafe.Pointer(C.cfish_incref(elem))).(SegReader) + } + return retval +} + +func (r *IndexReaderIMP) Offsets() []int32 { + self := (*C.lucy_IndexReader)(clownfish.Unwrap(r, "r")) + retvalCF := C.LUCY_IxReader_Offsets(self) + defer C.cfish_decref(unsafe.Pointer(retvalCF)) + return i32ArrayToSlice(retvalCF) +} + +func (r *IndexReaderIMP) Obtain(api string) (retval DataReader, err error) { + err = clownfish.TrapErr(func() { + self := (*C.lucy_IndexReader)(clownfish.Unwrap(r, "r")) + apiC := (*C.cfish_String)(clownfish.GoToClownfish(api, unsafe.Pointer(C.CFISH_STRING), false)) + defer C.cfish_decref(unsafe.Pointer(apiC)) + retvalCF := C.LUCY_IxReader_Obtain(self, apiC) + retval = clownfish.WRAPAny(unsafe.Pointer(C.cfish_incref(unsafe.Pointer(retvalCF)))).(DataReader) + }) + return retval, err +} diff --git a/go/lucy/index_test.go b/go/lucy/index_test.go index 893416701..e44bb0977 100644 --- a/go/lucy/index_test.go +++ b/go/lucy/index_test.go @@ -400,9 +400,9 @@ func TestSortCacheMisc(t *testing.T) { indexer.AddDoc(make(map[string]interface{})) indexer.Commit() - searcher, _ := OpenIndexSearcher(folder) - segReaders := searcher.GetReader().SegReaders() - sortReader := segReaders[0].(SegReader).Obtain("Lucy::Index::SortReader").(SortReader) + ixReader, _ := OpenIndexReader(folder, nil, nil) + segReaders := ixReader.SegReaders() + sortReader := segReaders[0].Fetch("Lucy::Index::SortReader").(SortReader) sortCache := sortReader.fetchSortCache("content") if card := sortCache.GetCardinality(); card != 4 { @@ -646,9 +646,9 @@ func TestInverterMisc(t *testing.T) { // Use SegLexicon to air out the Lexicon interface. func TestLexiconBasics(t *testing.T) { folder := createTestIndex("a", "b", "c") - searcher, _ := OpenIndexSearcher(folder) - segReaders := searcher.GetReader().SegReaders() - lexReader := segReaders[0].(SegReader).Obtain("Lucy::Index::LexiconReader").(LexiconReader) + ixReader, _ := OpenIndexReader(folder, nil, nil) + segReaders := ixReader.SegReaders() + lexReader := segReaders[0].Fetch("Lucy::Index::LexiconReader").(LexiconReader) segLex := lexReader.Lexicon("content", nil).(Lexicon) if field := segLex.getField(); field != "content" { t.Errorf("getField: %s", field) @@ -678,9 +678,9 @@ func TestLexiconBasics(t *testing.T) { func TestPostingListBasics(t *testing.T) { folder := createTestIndex("c", "b b b", "a", "b",) - searcher, _ := OpenIndexSearcher(folder) - segReaders := searcher.GetReader().SegReaders() - pListReader := segReaders[0].(SegReader).Obtain("Lucy::Index::PostingListReader").(PostingListReader) + ixReader, _ := OpenIndexReader(folder, nil, nil) + segReaders := ixReader.SegReaders() + pListReader := segReaders[0].Fetch("Lucy::Index::PostingListReader").(PostingListReader) pList := pListReader.PostingList("content", nil) pList.Seek("b") if docFreq := pList.GetDocFreq(); docFreq != 2 { @@ -746,16 +746,62 @@ func runDataReaderCommon(t *testing.T, reader DataReader, runAggregator bool) { func TestIndexReaderMisc(t *testing.T) { folder := createTestIndex("a", "b", "c") - searcher, _ := OpenIndexSearcher(folder) - reader := searcher.GetReader() + reader, _ := OpenIndexReader(folder, nil, nil) + if segReaders := reader.SegReaders(); len(segReaders) != 1 { + t.Errorf("SegReaders: %#v", segReaders) + } + if offsets := reader.Offsets(); offsets[0] != 0 { + t.Errorf("Offsets: %#v", offsets) + } + if got, err := reader.Obtain("Lucy::Index::DocReader"); got == nil || err != nil { + t.Errorf("Obtain should succeed for DocReader: %#v, %v", got, err) + } + if got, err := reader.Obtain("Nope"); got != nil || err == nil { + t.Errorf("Obtain should fail for non-existent API name: %v", err) + } + if got := reader.Fetch("Lucy::Index::DocReader"); got == nil { + t.Errorf("Fetch should succeed for DocReader") + } + if got := reader.Fetch("Nope"); got != nil { + t.Errorf("Fetch should return nil for non-existent API name: %v", got) + } + if got := reader.DocMax(); got != 3 { + t.Errorf("DocMax: %d", got); + } + if got := reader.DocCount(); got != 3 { + t.Errorf("DocCount: %d", got); + } + if got := reader.DelCount(); got != 0 { + t.Errorf("DelCount: %d", got); + } runDataReaderCommon(t, reader, false) } +func TestIndexReaderOpen(t *testing.T) { + folder := createTestIndex("a", "b", "c") + if got, err := OpenIndexReader(folder, nil, nil); got == nil || err != nil { + t.Errorf("nil Snapshot and IndexManager: %v", err) + } + snapshot := NewSnapshot() + snapshot.ReadFile(folder, "") + if got, err := OpenIndexReader(folder, snapshot, nil); got == nil || err != nil { + t.Errorf("With Snapshot: %v", err) + } + manager := NewIndexManager("", nil) + manager.SetFolder(folder) + if got, err := OpenIndexReader(folder, nil, manager); got == nil || err != nil { + t.Errorf("With IndexManager: %v", err) + } + if got, err := OpenIndexReader("no-index-here", nil, nil); got != nil || err == nil { + t.Errorf("Non-existent index path") + } +} + func TestDefaultDocReaderMisc(t *testing.T) { folder := createTestIndex("a", "b", "c") - searcher, _ := OpenIndexSearcher(folder) - segReaders := searcher.GetReader().SegReaders() - reader := segReaders[0].(SegReader).Obtain("Lucy::Index::DocReader").(DefaultDocReader) + ixReader, _ := OpenIndexReader(folder, nil, nil) + segReaders := ixReader.SegReaders() + reader := segReaders[0].Fetch("Lucy::Index::DocReader").(DefaultDocReader) doc := make(map[string]interface{}) if err := reader.ReadDoc(2, doc); err != nil { t.Errorf("ReadDoc: %v", err) @@ -765,8 +811,8 @@ func TestDefaultDocReaderMisc(t *testing.T) { func TestPolyDocReaderMisc(t *testing.T) { folder := createTestIndex("a", "b", "c") - searcher, _ := OpenIndexSearcher(folder) - reader := searcher.GetReader().Obtain("Lucy::Index::DocReader").(PolyDocReader) + ixReader, _ := OpenIndexReader(folder, nil, nil) + reader := ixReader.Fetch("Lucy::Index::DocReader").(PolyDocReader) doc := make(map[string]interface{}) if err := reader.ReadDoc(2, doc); err != nil { t.Errorf("ReadDoc: %v", err) diff --git a/go/lucy/search_test.go b/go/lucy/search_test.go index 3d4fc5037..5061a242c 100644 --- a/go/lucy/search_test.go +++ b/go/lucy/search_test.go @@ -369,12 +369,11 @@ func TestNoMatchMatcherBasics(t *testing.T) { func TestRangeMatcherBasics(t *testing.T) { index := createTestIndex("d", "c", "b", "a", "a", "a", "a") - searcher, _ := OpenIndexSearcher(index) - segReaders := searcher.GetReader().SegReaders() - segReader := segReaders[0].(SegReader) - sortReader := segReader.Obtain("Lucy::Index::SortReader").(SortReader) + ixReader, _ := OpenIndexReader(index, nil, nil) + segReaders := ixReader.SegReaders() + sortReader := segReaders[0].Fetch("Lucy::Index::SortReader").(SortReader) sortCache := sortReader.fetchSortCache("content") - matcher := NewRangeMatcher(0, 0, sortCache, segReader.DocMax()) + matcher := NewRangeMatcher(0, 0, sortCache, segReaders[0].DocMax()) if docID := matcher.Next(); docID != 4 { t.Errorf("Next: %d", docID) } From 6deeba866eab0ffa1a6ad875a2d8912f16c9ae8b Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Thu, 10 Dec 2015 16:08:33 -0800 Subject: [PATCH 07/15] Suppress SegReader and PolyReader Go ctors. For now, providing lucy.OpenIndexReader is enough. --- go/build.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/go/build.go b/go/build.go index fc104dc2f..30a4703af 100644 --- a/go/build.go +++ b/go/build.go @@ -197,6 +197,14 @@ func specClasses(parcel *cfc.Parcel) { ixReaderBinding.SpecMethod("Obtain", "Obtain(string) (DataReader, error)") ixReaderBinding.Register() + polyReaderBinding := cfc.NewGoClass(parcel, "Lucy::Index::PolyReader") + polyReaderBinding.SetSuppressCtor(true) + polyReaderBinding.Register() + + segReaderBinding := cfc.NewGoClass(parcel, "Lucy::Index::SegReader") + segReaderBinding.SetSuppressCtor(true) + segReaderBinding.Register() + docReaderBinding := cfc.NewGoClass(parcel, "Lucy::Index::DocReader") docReaderBinding.SpecMethod("", "ReadDoc(int32, interface{}) error") docReaderBinding.SpecMethod("Fetch_Doc", "FetchDoc(int32) (HitDoc, error)") From a405c39703da14ede15b1b20a54bf63165b32df5 Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Thu, 10 Dec 2015 17:05:05 -0800 Subject: [PATCH 08/15] Tune and test HighlightReader Go bindings. --- go/build.go | 4 ++++ go/lucy/index.go | 10 ++++++++++ go/lucy/index_test.go | 14 ++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/go/build.go b/go/build.go index 30a4703af..a730160da 100644 --- a/go/build.go +++ b/go/build.go @@ -210,6 +210,10 @@ func specClasses(parcel *cfc.Parcel) { docReaderBinding.SpecMethod("Fetch_Doc", "FetchDoc(int32) (HitDoc, error)") docReaderBinding.Register() + hlReaderBinding := cfc.NewGoClass(parcel, "Lucy::Index::HighlightReader") + hlReaderBinding.SpecMethod("Fetch_Doc_Vec", "FetchDocVec(int32) (DocVector, error)") + hlReaderBinding.Register() + bgMergerBinding := cfc.NewGoClass(parcel, "Lucy::Index::BackgroundMerger") bgMergerBinding.SpecMethod("Prepare_Commit", "PrepareCommit() error") bgMergerBinding.SpecMethod("Commit", "Commit() error") diff --git a/go/lucy/index.go b/go/lucy/index.go index 67d4b4789..09ad6ec7c 100644 --- a/go/lucy/index.go +++ b/go/lucy/index.go @@ -21,6 +21,7 @@ package lucy #include "Lucy/Index/IndexReader.h" #include "Lucy/Index/DataReader.h" #include "Lucy/Index/DocReader.h" +#include "Lucy/Index/HighlightReader.h" #include "Lucy/Index/IndexManager.h" #include "Lucy/Index/BackgroundMerger.h" #include "Lucy/Index/TermVector.h" @@ -484,6 +485,15 @@ func (d *DocReaderIMP) FetchDoc(docID int32) (doc HitDoc, err error) { return doc, err } +func (h *HighlightReaderIMP) FetchDocVec(docID int32) (retval DocVector, err error) { + err = clownfish.TrapErr(func() { + self := (*C.lucy_HighlightReader)(clownfish.Unwrap(h, "h")) + retvalCF := C.LUCY_HLReader_Fetch_Doc_Vec(self, C.int32_t(docID)) + retval = WRAPDocVector(unsafe.Pointer(retvalCF)) + }) + return retval, err +} + func OpenIndexReader(index interface{}, snapshot Snapshot, manager IndexManager) (retval IndexReader, err error) { err = clownfish.TrapErr(func() { indexC := (*C.cfish_Obj)(clownfish.GoToClownfish(index, unsafe.Pointer(C.CFISH_OBJ), false)) diff --git a/go/lucy/index_test.go b/go/lucy/index_test.go index e44bb0977..e75ede452 100644 --- a/go/lucy/index_test.go +++ b/go/lucy/index_test.go @@ -819,3 +819,17 @@ func TestPolyDocReaderMisc(t *testing.T) { } runDataReaderCommon(t, reader, true) } + +func TestHighlightReaderMisc(t *testing.T) { + folder := createTestIndex("a", "b", "c") + ixReader, _ := OpenIndexReader(folder, nil, nil) + segReaders := ixReader.SegReaders() + reader := segReaders[0].Fetch("Lucy::Index::HighlightReader").(HighlightReader) + if got, err := reader.FetchDocVec(2); got == nil || err != nil { + t.Errorf("FetchDocVec: %v", err) + } + if got, err := reader.FetchDocVec(4); got != nil || err == nil { + t.Errorf("FetchDocVec catch error: %#v", got) + } + runDataReaderCommon(t, reader, true) +} From bfe8a34733ff43ecf34dc2e7523684381dbec5a9 Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Thu, 10 Dec 2015 17:10:50 -0800 Subject: [PATCH 09/15] Test Go bindings for DeletionsReader. --- go/lucy/index_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/go/lucy/index_test.go b/go/lucy/index_test.go index e75ede452..ebf5d256b 100644 --- a/go/lucy/index_test.go +++ b/go/lucy/index_test.go @@ -833,3 +833,17 @@ func TestHighlightReaderMisc(t *testing.T) { } runDataReaderCommon(t, reader, true) } + +func TestDeletionsReaderMisc(t *testing.T) { + folder := createTestIndex("a", "b", "c") + ixReader, _ := OpenIndexReader(folder, nil, nil) + segReaders := ixReader.SegReaders() + delReader := segReaders[0].Fetch("Lucy::Index::DeletionsReader").(DeletionsReader) + if count := delReader.delCount(); count != 0 { + t.Errorf("delCount: %d", count); + } + if matcher := delReader.iterator(); matcher == nil { + t.Errorf("iterator: %#v", matcher) + } + runDataReaderCommon(t, delReader, true) +} From db08b691bf6a8b5ce450f296f496c76604eef3e6 Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Thu, 10 Dec 2015 18:22:14 -0800 Subject: [PATCH 10/15] Tune and test Go bindings for SortReader. --- go/build.go | 4 ++++ go/lucy/index.go | 14 ++++++++++++++ go/lucy/index_test.go | 16 +++++++++++++++- go/lucy/search_test.go | 2 +- 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/go/build.go b/go/build.go index a730160da..09a31e5be 100644 --- a/go/build.go +++ b/go/build.go @@ -214,6 +214,10 @@ func specClasses(parcel *cfc.Parcel) { hlReaderBinding.SpecMethod("Fetch_Doc_Vec", "FetchDocVec(int32) (DocVector, error)") hlReaderBinding.Register() + sortReaderBinding := cfc.NewGoClass(parcel, "Lucy::Index::SortReader") + sortReaderBinding.SpecMethod("Fetch_Sort_Cache", "fetchSortCache(string) (SortCache, error)") + sortReaderBinding.Register() + bgMergerBinding := cfc.NewGoClass(parcel, "Lucy::Index::BackgroundMerger") bgMergerBinding.SpecMethod("Prepare_Commit", "PrepareCommit() error") bgMergerBinding.SpecMethod("Commit", "Commit() error") diff --git a/go/lucy/index.go b/go/lucy/index.go index 09ad6ec7c..397e86039 100644 --- a/go/lucy/index.go +++ b/go/lucy/index.go @@ -22,6 +22,7 @@ package lucy #include "Lucy/Index/DataReader.h" #include "Lucy/Index/DocReader.h" #include "Lucy/Index/HighlightReader.h" +#include "Lucy/Index/SortReader.h" #include "Lucy/Index/IndexManager.h" #include "Lucy/Index/BackgroundMerger.h" #include "Lucy/Index/TermVector.h" @@ -494,6 +495,19 @@ func (h *HighlightReaderIMP) FetchDocVec(docID int32) (retval DocVector, err err return retval, err } +func (s *SortReaderIMP) fetchSortCache(field string) (retval SortCache, err error) { + err = clownfish.TrapErr(func() { + self := (*C.lucy_SortReader)(clownfish.Unwrap(s, "s")) + fieldC := (*C.cfish_String)(clownfish.GoToClownfish(field, unsafe.Pointer(C.CFISH_STRING), false)) + defer C.cfish_decref(unsafe.Pointer(fieldC)) + retvalCF := C.LUCY_SortReader_Fetch_Sort_Cache(self, fieldC) + if retvalCF != nil { + retval = clownfish.ToGo(unsafe.Pointer(C.cfish_incref(unsafe.Pointer(retvalCF)))).(SortCache) + } + }) + return retval, err +} + func OpenIndexReader(index interface{}, snapshot Snapshot, manager IndexManager) (retval IndexReader, err error) { err = clownfish.TrapErr(func() { indexC := (*C.cfish_Obj)(clownfish.GoToClownfish(index, unsafe.Pointer(C.CFISH_OBJ), false)) diff --git a/go/lucy/index_test.go b/go/lucy/index_test.go index ebf5d256b..efbe0f0cf 100644 --- a/go/lucy/index_test.go +++ b/go/lucy/index_test.go @@ -403,7 +403,7 @@ func TestSortCacheMisc(t *testing.T) { ixReader, _ := OpenIndexReader(folder, nil, nil) segReaders := ixReader.SegReaders() sortReader := segReaders[0].Fetch("Lucy::Index::SortReader").(SortReader) - sortCache := sortReader.fetchSortCache("content") + sortCache, _ := sortReader.fetchSortCache("content") if card := sortCache.GetCardinality(); card != 4 { t.Errorf("GetCardinality: %d", card) @@ -847,3 +847,17 @@ func TestDeletionsReaderMisc(t *testing.T) { } runDataReaderCommon(t, delReader, true) } + +func TestSortReaderMisc(t *testing.T) { + folder := createTestIndex("a", "b", "c") + ixReader, _ := OpenIndexReader(folder, nil, nil) + segReaders := ixReader.SegReaders() + sortReader := segReaders[0].Fetch("Lucy::Index::SortReader").(SortReader) + if got, err := sortReader.fetchSortCache("content"); got == nil || err != nil { + t.Errorf("fetchSortCache should succeed: %v", err) + } + if got, err := sortReader.fetchSortCache("nope"); got != nil || err != nil { + t.Errorf("fetchSortCache for non-field should return nil: %v", err) + } + runDataReaderCommon(t, sortReader, false) +} diff --git a/go/lucy/search_test.go b/go/lucy/search_test.go index 5061a242c..47b809b3f 100644 --- a/go/lucy/search_test.go +++ b/go/lucy/search_test.go @@ -372,7 +372,7 @@ func TestRangeMatcherBasics(t *testing.T) { ixReader, _ := OpenIndexReader(index, nil, nil) segReaders := ixReader.SegReaders() sortReader := segReaders[0].Fetch("Lucy::Index::SortReader").(SortReader) - sortCache := sortReader.fetchSortCache("content") + sortCache, _ := sortReader.fetchSortCache("content") matcher := NewRangeMatcher(0, 0, sortCache, segReaders[0].DocMax()) if docID := matcher.Next(); docID != 4 { t.Errorf("Next: %d", docID) From 3358bc298eaaecd0e705af869d6c59f2e869dc8e Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Thu, 10 Dec 2015 18:53:08 -0800 Subject: [PATCH 11/15] Tune and test Go bindings for LexiconReader. --- go/build.go | 6 ++++++ go/lucy/index.go | 43 +++++++++++++++++++++++++++++++++++++++++++ go/lucy/index_test.go | 31 ++++++++++++++++++++++++++++++- 3 files changed, 79 insertions(+), 1 deletion(-) diff --git a/go/build.go b/go/build.go index 09a31e5be..36b487d0a 100644 --- a/go/build.go +++ b/go/build.go @@ -218,6 +218,12 @@ func specClasses(parcel *cfc.Parcel) { sortReaderBinding.SpecMethod("Fetch_Sort_Cache", "fetchSortCache(string) (SortCache, error)") sortReaderBinding.Register() + lexReaderBinding := cfc.NewGoClass(parcel, "Lucy::Index::LexiconReader") + lexReaderBinding.SpecMethod("Lexicon", "Lexicon(string, interface{}) (Lexicon, error)") + lexReaderBinding.SpecMethod("Doc_Freq", "DocFreq(string, interface{}) (uint32, error)") + lexReaderBinding.SpecMethod("Fetch_Term_Info", "fetchTermInfo(string, interface{}) (TermInfo, error)") + lexReaderBinding.Register() + bgMergerBinding := cfc.NewGoClass(parcel, "Lucy::Index::BackgroundMerger") bgMergerBinding.SpecMethod("Prepare_Commit", "PrepareCommit() error") bgMergerBinding.SpecMethod("Commit", "Commit() error") diff --git a/go/lucy/index.go b/go/lucy/index.go index 397e86039..a0e3396b2 100644 --- a/go/lucy/index.go +++ b/go/lucy/index.go @@ -21,6 +21,7 @@ package lucy #include "Lucy/Index/IndexReader.h" #include "Lucy/Index/DataReader.h" #include "Lucy/Index/DocReader.h" +#include "Lucy/Index/LexiconReader.h" #include "Lucy/Index/HighlightReader.h" #include "Lucy/Index/SortReader.h" #include "Lucy/Index/IndexManager.h" @@ -486,6 +487,48 @@ func (d *DocReaderIMP) FetchDoc(docID int32) (doc HitDoc, err error) { return doc, err } +func (lr *LexiconReaderIMP) Lexicon(field string, term interface{}) (retval Lexicon, err error) { + err = clownfish.TrapErr(func() { + self := (*C.lucy_LexiconReader)(clownfish.Unwrap(lr, "lr")) + fieldC := (*C.cfish_String)(clownfish.GoToClownfish(field, unsafe.Pointer(C.CFISH_STRING), false)) + defer C.cfish_decref(unsafe.Pointer(fieldC)) + termC := (*C.cfish_Obj)(clownfish.GoToClownfish(term, unsafe.Pointer(C.CFISH_OBJ), true)) + defer C.cfish_decref(unsafe.Pointer(termC)) + retvalCF := C.LUCY_LexReader_Lexicon(self, fieldC, termC) + if retvalCF != nil { + retval = clownfish.ToGo(unsafe.Pointer(retvalCF)).(Lexicon) + } + }) + return retval, err +} + +func (lr *LexiconReaderIMP) DocFreq(field string, term interface{}) (retval uint32, err error) { + err = clownfish.TrapErr(func() { + self := (*C.lucy_LexiconReader)(clownfish.Unwrap(lr, "lr")) + fieldC := (*C.cfish_String)(clownfish.GoToClownfish(field, unsafe.Pointer(C.CFISH_STRING), false)) + defer C.cfish_decref(unsafe.Pointer(fieldC)) + termC := (*C.cfish_Obj)(clownfish.GoToClownfish(term, unsafe.Pointer(C.CFISH_OBJ), true)) + defer C.cfish_decref(unsafe.Pointer(termC)) + retval = uint32(C.LUCY_LexReader_Doc_Freq(self, fieldC, termC)) + }) + return retval, err +} + +func (lr *LexiconReaderIMP) fetchTermInfo(field string, term interface{}) (retval TermInfo, err error) { + err = clownfish.TrapErr(func() { + self := (*C.lucy_LexiconReader)(clownfish.Unwrap(lr, "lr")) + fieldC := (*C.cfish_String)(clownfish.GoToClownfish(field, unsafe.Pointer(C.CFISH_STRING), false)) + defer C.cfish_decref(unsafe.Pointer(fieldC)) + termC := (*C.cfish_Obj)(clownfish.GoToClownfish(term, unsafe.Pointer(C.CFISH_OBJ), true)) + defer C.cfish_decref(unsafe.Pointer(termC)) + retvalCF := C.LUCY_LexReader_Fetch_Term_Info(self, fieldC, termC) + if retvalCF != nil { + retval = clownfish.ToGo(unsafe.Pointer(retvalCF)).(TermInfo) + } + }) + return retval, err +} + func (h *HighlightReaderIMP) FetchDocVec(docID int32) (retval DocVector, err error) { err = clownfish.TrapErr(func() { self := (*C.lucy_HighlightReader)(clownfish.Unwrap(h, "h")) diff --git a/go/lucy/index_test.go b/go/lucy/index_test.go index efbe0f0cf..18d1e22b2 100644 --- a/go/lucy/index_test.go +++ b/go/lucy/index_test.go @@ -649,7 +649,7 @@ func TestLexiconBasics(t *testing.T) { ixReader, _ := OpenIndexReader(folder, nil, nil) segReaders := ixReader.SegReaders() lexReader := segReaders[0].Fetch("Lucy::Index::LexiconReader").(LexiconReader) - segLex := lexReader.Lexicon("content", nil).(Lexicon) + segLex, _ := lexReader.Lexicon("content", nil) if field := segLex.getField(); field != "content" { t.Errorf("getField: %s", field) } @@ -820,6 +820,35 @@ func TestPolyDocReaderMisc(t *testing.T) { runDataReaderCommon(t, reader, true) } +func TestLexReaderMisc(t *testing.T) { + folder := createTestIndex("a", "b", "c") + ixReader, _ := OpenIndexReader(folder, nil, nil) + segReaders := ixReader.SegReaders() + lexReader := segReaders[0].Fetch("Lucy::Index::LexiconReader").(LexiconReader) + if got, err := lexReader.Lexicon("content", nil); got == nil || err != nil { + t.Errorf("Lexicon should succeed: %v", err) + } + if got, err := lexReader.Lexicon("content", "foo"); got == nil || err != nil { + t.Errorf("Lexicon with term should succeed: %v", err) + } + if got, err := lexReader.Lexicon("nope", nil); got != nil || err != nil { + t.Errorf("Lexicon for non-field should return nil: %v", err) + } + if got, err := lexReader.DocFreq("content", "b"); got != 1 || err != nil { + t.Errorf("DocFreq: %d, %v", got, err) + } + if got, err := lexReader.DocFreq("content", "nope"); got != 0 || err != nil { + t.Errorf("DocFreq should be 0: %d, %v", got, err) + } + if got, err := lexReader.fetchTermInfo("content", "a"); got == nil || err != nil { + t.Errorf("fetchTermInfo should succeed: %v", err) + } + if got, err := lexReader.fetchTermInfo("content", "nope"); got != nil || err != nil { + t.Errorf("fetchTermInfo with non-existent term should return nil: %v", err) + } + runDataReaderCommon(t, lexReader, false) +} + func TestHighlightReaderMisc(t *testing.T) { folder := createTestIndex("a", "b", "c") ixReader, _ := OpenIndexReader(folder, nil, nil) From e5e4232b6c0c8d22f46c6db5430fb92ee949c855 Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Thu, 10 Dec 2015 19:03:15 -0800 Subject: [PATCH 12/15] Tune and test Go bindings for PostingListReader. --- go/build.go | 4 ++++ go/lucy/index.go | 16 ++++++++++++++++ go/lucy/index_test.go | 19 ++++++++++++++++++- 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/go/build.go b/go/build.go index 36b487d0a..604a9a902 100644 --- a/go/build.go +++ b/go/build.go @@ -224,6 +224,10 @@ func specClasses(parcel *cfc.Parcel) { lexReaderBinding.SpecMethod("Fetch_Term_Info", "fetchTermInfo(string, interface{}) (TermInfo, error)") lexReaderBinding.Register() + pListReaderBinding := cfc.NewGoClass(parcel, "Lucy::Index::PostingListReader") + pListReaderBinding.SpecMethod("Posting_List", "PostingList(string, interface{}) (PostingList, error)") + pListReaderBinding.Register() + bgMergerBinding := cfc.NewGoClass(parcel, "Lucy::Index::BackgroundMerger") bgMergerBinding.SpecMethod("Prepare_Commit", "PrepareCommit() error") bgMergerBinding.SpecMethod("Commit", "Commit() error") diff --git a/go/lucy/index.go b/go/lucy/index.go index a0e3396b2..1f5a75268 100644 --- a/go/lucy/index.go +++ b/go/lucy/index.go @@ -22,6 +22,7 @@ package lucy #include "Lucy/Index/DataReader.h" #include "Lucy/Index/DocReader.h" #include "Lucy/Index/LexiconReader.h" +#include "Lucy/Index/PostingListReader.h" #include "Lucy/Index/HighlightReader.h" #include "Lucy/Index/SortReader.h" #include "Lucy/Index/IndexManager.h" @@ -529,6 +530,21 @@ func (lr *LexiconReaderIMP) fetchTermInfo(field string, term interface{}) (retva return retval, err } +func (p *PostingListReaderIMP) PostingList(field string, term interface{}) (retval PostingList, err error) { + err = clownfish.TrapErr(func() { + self := (*C.lucy_PostingListReader)(clownfish.Unwrap(p, "p")) + fieldC := (*C.cfish_String)(clownfish.GoToClownfish(field, unsafe.Pointer(C.CFISH_STRING), false)) + defer C.cfish_decref(unsafe.Pointer(fieldC)) + termC := (*C.cfish_Obj)(clownfish.GoToClownfish(term, unsafe.Pointer(C.CFISH_OBJ), true)) + defer C.cfish_decref(unsafe.Pointer(termC)) + retvalCF := C.LUCY_PListReader_Posting_List(self, fieldC, termC) + if retvalCF != nil { + retval = clownfish.ToGo(unsafe.Pointer(retvalCF)).(PostingList) + } + }) + return retval, err +} + func (h *HighlightReaderIMP) FetchDocVec(docID int32) (retval DocVector, err error) { err = clownfish.TrapErr(func() { self := (*C.lucy_HighlightReader)(clownfish.Unwrap(h, "h")) diff --git a/go/lucy/index_test.go b/go/lucy/index_test.go index 18d1e22b2..0143fcfad 100644 --- a/go/lucy/index_test.go +++ b/go/lucy/index_test.go @@ -681,7 +681,7 @@ func TestPostingListBasics(t *testing.T) { ixReader, _ := OpenIndexReader(folder, nil, nil) segReaders := ixReader.SegReaders() pListReader := segReaders[0].Fetch("Lucy::Index::PostingListReader").(PostingListReader) - pList := pListReader.PostingList("content", nil) + pList, _ := pListReader.PostingList("content", nil) pList.Seek("b") if docFreq := pList.GetDocFreq(); docFreq != 2 { t.Errorf("GetDocFreq: %d", docFreq) @@ -849,6 +849,23 @@ func TestLexReaderMisc(t *testing.T) { runDataReaderCommon(t, lexReader, false) } +func TestPostingListReaderMisc(t *testing.T) { + folder := createTestIndex("a", "b", "c") + ixReader, _ := OpenIndexReader(folder, nil, nil) + segReaders := ixReader.SegReaders() + pListReader := segReaders[0].Fetch("Lucy::Index::PostingListReader").(PostingListReader) + if got, err := pListReader.PostingList("content", nil); got == nil || err != nil { + t.Errorf("PostingList should succeed: %v", err) + } + if got, err := pListReader.PostingList("content", "foo"); got == nil || err != nil { + t.Errorf("PostingList with term should succeed: %v", err) + } + if got, err := pListReader.PostingList("nope", nil); got != nil || err != nil { + t.Errorf("PostingList for non-field should return nil: %v", err) + } + runDataReaderCommon(t, pListReader, false) +} + func TestHighlightReaderMisc(t *testing.T) { folder := createTestIndex("a", "b", "c") ixReader, _ := OpenIndexReader(folder, nil, nil) From 362aeddca1599907090188035e77aeeba7b57c8b Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Mon, 14 Dec 2015 20:28:27 -0800 Subject: [PATCH 13/15] Tune and test Go bindings for DataWriter. --- go/build.go | 8 ++++++ go/lucy/index.go | 44 +++++++++++++++++++++++++++++++++ go/lucy/index_test.go | 57 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+) diff --git a/go/build.go b/go/build.go index 604a9a902..f78aa5fac 100644 --- a/go/build.go +++ b/go/build.go @@ -228,6 +228,14 @@ func specClasses(parcel *cfc.Parcel) { pListReaderBinding.SpecMethod("Posting_List", "PostingList(string, interface{}) (PostingList, error)") pListReaderBinding.Register() + dwBinding := cfc.NewGoClass(parcel, "Lucy::Index::DataWriter") + dwBinding.SpecMethod("Add_Inverted_Doc", "addInvertedDoc(Inverter, int32) error") + dwBinding.SpecMethod("Add_Segment", "AddSegment(SegReader, []int32) error") + dwBinding.SpecMethod("Delete_Segment", "DeleteSegment(SegReader) error") + dwBinding.SpecMethod("Merge_Segment", "MergeSegment(SegReader, []int32) error") + dwBinding.SpecMethod("Finish", "Finish() error") + dwBinding.Register() + bgMergerBinding := cfc.NewGoClass(parcel, "Lucy::Index::BackgroundMerger") bgMergerBinding.SpecMethod("Prepare_Commit", "PrepareCommit() error") bgMergerBinding.SpecMethod("Commit", "Commit() error") diff --git a/go/lucy/index.go b/go/lucy/index.go index 1f5a75268..0444a30f7 100644 --- a/go/lucy/index.go +++ b/go/lucy/index.go @@ -20,6 +20,7 @@ package lucy #include "Lucy/Index/Indexer.h" #include "Lucy/Index/IndexReader.h" #include "Lucy/Index/DataReader.h" +#include "Lucy/Index/DataWriter.h" #include "Lucy/Index/DocReader.h" #include "Lucy/Index/LexiconReader.h" #include "Lucy/Index/PostingListReader.h" @@ -235,6 +236,49 @@ func (obj *IndexerIMP) Commit() error { }) } +func (d *DataWriterIMP) addInvertedDoc(inverter Inverter, docId int32) error { + return clownfish.TrapErr(func() { + self := (*C.lucy_DataWriter)(clownfish.Unwrap(d, "d")) + inverterCF := (*C.lucy_Inverter)(clownfish.Unwrap(inverter, "inverter")) + C.LUCY_DataWriter_Add_Inverted_Doc(self, inverterCF, C.int32_t(docId)) + }) +} + +func (d *DataWriterIMP) AddSegment(reader SegReader, docMap []int32) error { + return clownfish.TrapErr(func() { + self := (*C.lucy_DataWriter)(clownfish.Unwrap(d, "d")) + readerCF := (*C.lucy_SegReader)(clownfish.Unwrap(reader, "reader")) + docMapConv := NewI32Array(docMap) + docMapCF := (*C.lucy_I32Array)(clownfish.UnwrapNullable(docMapConv)) + C.LUCY_DataWriter_Add_Segment(self, readerCF, docMapCF) + }) +} + +func (d *DataWriterIMP) DeleteSegment(reader SegReader) error { + return clownfish.TrapErr(func() { + self := (*C.lucy_DataWriter)(clownfish.Unwrap(d, "d")) + readerCF := (*C.lucy_SegReader)(clownfish.Unwrap(reader, "reader")) + C.LUCY_DataWriter_Delete_Segment(self, readerCF) + }) +} + +func (d *DataWriterIMP) MergeSegment(reader SegReader, docMap []int32) error { + return clownfish.TrapErr(func() { + self := (*C.lucy_DataWriter)(clownfish.Unwrap(d, "d")) + readerCF := (*C.lucy_SegReader)(clownfish.Unwrap(reader, "reader")) + docMapConv := NewI32Array(docMap) + docMapCF := (*C.lucy_I32Array)(clownfish.UnwrapNullable(docMapConv)) + C.LUCY_DataWriter_Merge_Segment(self, readerCF, docMapCF) + }) +} + +func (d *DataWriterIMP) Finish() error { + return clownfish.TrapErr(func() { + self := (*C.lucy_DataWriter)(clownfish.Unwrap(d, "d")) + C.LUCY_DataWriter_Finish(self) + }) +} + func OpenBackgroundMerger(index interface{}, manager IndexManager) (bgm BackgroundMerger, err error) { err = clownfish.TrapErr(func() { indexC := (*C.cfish_Obj)(clownfish.GoToClownfish(index, unsafe.Pointer(C.CFISH_OBJ), false)) diff --git a/go/lucy/index_test.go b/go/lucy/index_test.go index 0143fcfad..f71cb2f66 100644 --- a/go/lucy/index_test.go +++ b/go/lucy/index_test.go @@ -907,3 +907,60 @@ func TestSortReaderMisc(t *testing.T) { } runDataReaderCommon(t, sortReader, false) } + +func runDataWriterCommon(t *testing.T, api string) { + abcIndex := createTestIndex("a", "b", "c") + indexer, _ := OpenIndexer(&OpenIndexerArgs{Index: abcIndex}) + dataWriter := indexer.getSegWriter().Fetch(api).(DataWriter) + + if got := dataWriter.GetSnapshot(); false { + t.Errorf("GetSnapshot: %v", got) + } + if got := dataWriter.GetSegment(); false { + t.Errorf("GetSegment: %#v", got) + } + if got := dataWriter.GetPolyReader(); false { + t.Errorf("GetPolyReader: %#v", got) + } + if got := dataWriter.GetSchema(); false { + t.Errorf("GetSchema: %v", got) + } + if got := dataWriter.GetFolder(); false { + t.Errorf("GetFolder: %v", got) + } + + doc := NewDoc(1) + doc.Store("content", "three blind mice") + inverter := NewInverter(dataWriter.GetSchema(), dataWriter.GetSegment()) + inverter.SetDoc(doc) + inverter.InvertDoc(doc) + if err := dataWriter.addInvertedDoc(inverter, 1); err != nil { + t.Errorf("addInvertedDoc: %v", err) + } + + segReaders := indexer.getSegWriter().GetPolyReader().SegReaders() + abcSegReader := segReaders[0] + if err := dataWriter.MergeSegment(abcSegReader, []int32{0, 2, 0, 3}); err != nil { + t.Errorf("MergeSegment: %v", err) + } + + // TODO + //if err := dataWriter.DeleteSegment(fooSegReader); err != nil { + // t.Errorf("DeleteSegment: %v", err) + //} + + xyzIndex := createTestIndex("x", "y", "z") + xyzReader, _ := OpenIndexReader(xyzIndex, nil, nil) + xyzSegReaders := xyzReader.SegReaders() + if err := dataWriter.AddSegment(xyzSegReaders[0], []int32{0, 4, 5, 6}); err != nil { + t.Errorf("AddSegment: %v", err) + } + + if err := dataWriter.Finish(); err != nil { + t.Errorf("Finish: %v", err) + } +} + +func TestSortWriterMisc(t *testing.T) { + runDataWriterCommon(t, "Lucy::Index::SortWriter") +} From 2932054958f0fc785d13d1bbfe59dee9ce42ef00 Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Mon, 21 Dec 2015 09:04:49 -0800 Subject: [PATCH 14/15] Tune and test Go bindings for DeletionsWriter. --- go/build.go | 8 +++++++ go/lucy/index.go | 50 +++++++++++++++++++++++++++++++++++++++++++ go/lucy/index_test.go | 30 ++++++++++++++++++++++++++ 3 files changed, 88 insertions(+) diff --git a/go/build.go b/go/build.go index f78aa5fac..82bcd9cf0 100644 --- a/go/build.go +++ b/go/build.go @@ -236,6 +236,14 @@ func specClasses(parcel *cfc.Parcel) { dwBinding.SpecMethod("Finish", "Finish() error") dwBinding.Register() + delWriterBinding := cfc.NewGoClass(parcel, "Lucy::Index::DeletionsWriter") + delWriterBinding.SpecMethod("Delete_By_Term", "DeleteByTerm(string, interface{}) error") + delWriterBinding.SpecMethod("Delete_By_Query", "DeleteByQuery(Query) error") + delWriterBinding.SpecMethod("Delete_By_Doc_ID", "deleteByDocID(int32) error") + delWriterBinding.SpecMethod("Generate_Doc_Map", "generateDocMap(Matcher, int32, int32) ([]int32, error)") + delWriterBinding.SpecMethod("Seg_Deletions", "segDeletions(SegReader) (Matcher, error)") + delWriterBinding.Register() + bgMergerBinding := cfc.NewGoClass(parcel, "Lucy::Index::BackgroundMerger") bgMergerBinding.SpecMethod("Prepare_Commit", "PrepareCommit() error") bgMergerBinding.SpecMethod("Commit", "Commit() error") diff --git a/go/lucy/index.go b/go/lucy/index.go index 0444a30f7..67ec832cc 100644 --- a/go/lucy/index.go +++ b/go/lucy/index.go @@ -21,6 +21,7 @@ package lucy #include "Lucy/Index/IndexReader.h" #include "Lucy/Index/DataReader.h" #include "Lucy/Index/DataWriter.h" +#include "Lucy/Index/DeletionsWriter.h" #include "Lucy/Index/DocReader.h" #include "Lucy/Index/LexiconReader.h" #include "Lucy/Index/PostingListReader.h" @@ -279,6 +280,55 @@ func (d *DataWriterIMP) Finish() error { }) } +func (d *DeletionsWriterIMP) DeleteByTerm(field string, term interface{}) error { + return clownfish.TrapErr(func() { + self := (*C.lucy_DeletionsWriter)(clownfish.Unwrap(d, "d")) + fieldCF := (*C.cfish_String)(clownfish.GoToClownfish(field, unsafe.Pointer(C.CFISH_STRING), false)) + defer C.cfish_decref(unsafe.Pointer(fieldCF)) + termCF := (*C.cfish_Obj)(clownfish.GoToClownfish(term, unsafe.Pointer(C.CFISH_OBJ), false)) + defer C.cfish_decref(unsafe.Pointer(termCF)) + C.LUCY_DelWriter_Delete_By_Term(self, fieldCF, termCF) + }) +} + +func (d *DeletionsWriterIMP) DeleteByQuery(query Query) error { + return clownfish.TrapErr(func() { + self := (*C.lucy_DeletionsWriter)(clownfish.Unwrap(d, "d")) + queryCF := (*C.lucy_Query)(clownfish.Unwrap(query, "query")) + C.LUCY_DelWriter_Delete_By_Query(self, queryCF) + }) +} + +func (d *DeletionsWriterIMP) deleteByDocID(docId int32) error { + return clownfish.TrapErr(func() { + self := (*C.lucy_DeletionsWriter)(clownfish.Unwrap(d, "d")) + C.LUCY_DelWriter_Delete_By_Doc_ID(self, C.int32_t(docId)) + }) +} + +func (d *DeletionsWriterIMP) generateDocMap(deletions Matcher, docMax int32, offset int32) (retval []int32, err error) { + err = clownfish.TrapErr(func() { + self := (*C.lucy_DeletionsWriter)(clownfish.Unwrap(d, "d")) + deletionsCF := (*C.lucy_Matcher)(clownfish.Unwrap(deletions, "deletions")) + retvalCF := C.LUCY_DelWriter_Generate_Doc_Map(self, deletionsCF, C.int32_t(docMax), C.int32_t(offset)) + defer C.cfish_decref(unsafe.Pointer(retvalCF)) + retval = i32ArrayToSlice(retvalCF) + }) + return retval, err +} + +func (d *DeletionsWriterIMP) segDeletions(segReader SegReader) (retval Matcher, err error) { + err = clownfish.TrapErr(func() { + self := (*C.lucy_DeletionsWriter)(clownfish.Unwrap(d, "d")) + segReaderCF := (*C.lucy_SegReader)(clownfish.Unwrap(segReader, "segReader")) + retvalCF := C.LUCY_DelWriter_Seg_Deletions(self, segReaderCF) + if retvalCF != nil { + retval = clownfish.WRAPAny(unsafe.Pointer(retvalCF)).(Matcher) + } + }) + return retval, err +} + func OpenBackgroundMerger(index interface{}, manager IndexManager) (bgm BackgroundMerger, err error) { err = clownfish.TrapErr(func() { indexC := (*C.cfish_Obj)(clownfish.GoToClownfish(index, unsafe.Pointer(C.CFISH_OBJ), false)) diff --git a/go/lucy/index_test.go b/go/lucy/index_test.go index f71cb2f66..9f2aca916 100644 --- a/go/lucy/index_test.go +++ b/go/lucy/index_test.go @@ -964,3 +964,33 @@ func runDataWriterCommon(t *testing.T, api string) { func TestSortWriterMisc(t *testing.T) { runDataWriterCommon(t, "Lucy::Index::SortWriter") } + +func TestDeletionsWriterMisc(t *testing.T) { + index := createTestIndex("a", "b", "c") + indexer, _ := OpenIndexer(&OpenIndexerArgs{Index: index}) + delWriter := indexer.getSegWriter().Fetch("Lucy::Index::DeletionsWriter").(DeletionsWriter) + if delWriter.Updated() { + t.Errorf("Not yet updated") + } + + if err := delWriter.DeleteByTerm("content", "a"); err != nil { + t.Errorf("DeleteByTerm: %v", err) + } + if err := delWriter.DeleteByQuery(NewTermQuery("content", "b")); err != nil { + t.Errorf("DeleteByQuery: %v", err) + } + if err := delWriter.deleteByDocID(3); err != nil { + t.Errorf("deleteByDocID: %v", err) + } + if !delWriter.Updated() { + t.Errorf("Now we're updated") + } + + if got := delWriter.SegDelCount("seg_1"); got != 3 { + t.Errorf("SegDelCount: %d", got) + } + segReaders := delWriter.GetPolyReader().SegReaders() + if dels, err := delWriter.segDeletions(segReaders[0]); dels == nil || err != nil { + t.Errorf("segDeletions: %v", err) + } +} From 126c6ba8907c2f134c9921cafe8d7dc74f1fbc5d Mon Sep 17 00:00:00 2001 From: Marvin Humphrey Date: Mon, 21 Dec 2015 12:38:25 -0800 Subject: [PATCH 15/15] Tune and test Go bindings for SegWriter. --- go/build.go | 5 +++++ go/lucy/index.go | 16 ++++++++++++++++ go/lucy/index_test.go | 17 +++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/go/build.go b/go/build.go index 82bcd9cf0..e3fc79d74 100644 --- a/go/build.go +++ b/go/build.go @@ -236,6 +236,11 @@ func specClasses(parcel *cfc.Parcel) { dwBinding.SpecMethod("Finish", "Finish() error") dwBinding.Register() + segWriterBinding := cfc.NewGoClass(parcel, "Lucy::Index::SegWriter") + segWriterBinding.SpecMethod("Prep_Seg_Dir", "PrepSegDir() error") + segWriterBinding.SpecMethod("Add_Doc", "AddDoc(Doc, float32) error") + segWriterBinding.Register() + delWriterBinding := cfc.NewGoClass(parcel, "Lucy::Index::DeletionsWriter") delWriterBinding.SpecMethod("Delete_By_Term", "DeleteByTerm(string, interface{}) error") delWriterBinding.SpecMethod("Delete_By_Query", "DeleteByQuery(Query) error") diff --git a/go/lucy/index.go b/go/lucy/index.go index 67ec832cc..6f627f033 100644 --- a/go/lucy/index.go +++ b/go/lucy/index.go @@ -21,6 +21,7 @@ package lucy #include "Lucy/Index/IndexReader.h" #include "Lucy/Index/DataReader.h" #include "Lucy/Index/DataWriter.h" +#include "Lucy/Index/SegWriter.h" #include "Lucy/Index/DeletionsWriter.h" #include "Lucy/Index/DocReader.h" #include "Lucy/Index/LexiconReader.h" @@ -280,6 +281,21 @@ func (d *DataWriterIMP) Finish() error { }) } +func (s *SegWriterIMP) PrepSegDir() error { + return clownfish.TrapErr(func() { + self := (*C.lucy_SegWriter)(clownfish.Unwrap(s, "s")) + C.LUCY_SegWriter_Prep_Seg_Dir(self) + }) +} + +func (s *SegWriterIMP) AddDoc(doc Doc, boost float32) error { + return clownfish.TrapErr(func() { + self := (*C.lucy_SegWriter)(clownfish.Unwrap(s, "s")) + docCF := (*C.lucy_Doc)(clownfish.Unwrap(doc, "doc")) + C.LUCY_SegWriter_Add_Doc(self, docCF, C.float(boost)) + }) +} + func (d *DeletionsWriterIMP) DeleteByTerm(field string, term interface{}) error { return clownfish.TrapErr(func() { self := (*C.lucy_DeletionsWriter)(clownfish.Unwrap(d, "d")) diff --git a/go/lucy/index_test.go b/go/lucy/index_test.go index 9f2aca916..bc6bac403 100644 --- a/go/lucy/index_test.go +++ b/go/lucy/index_test.go @@ -994,3 +994,20 @@ func TestDeletionsWriterMisc(t *testing.T) { t.Errorf("segDeletions: %v", err) } } + +func TestSegWriterMisc(t *testing.T) { + index := createTestIndex("a", "b", "c") + ixReader, _ := OpenIndexReader(index, nil, nil) + polyReader := ixReader.(PolyReader) + schema := polyReader.GetSchema() + segment := NewSegment(2) + snapshot := polyReader.GetSnapshot() + segWriter := NewSegWriter(schema, snapshot, segment, polyReader) + if err := segWriter.PrepSegDir(); err != nil { + t.Errorf("PrepSegDir: %v", err) + } + doc := NewDoc(1) + if err := segWriter.AddDoc(doc, 1.0); err != nil { + t.Errorf("AddDoc: %v", err) + } +}