From 36665996f02e090e609a2cdba85087c0954fae68 Mon Sep 17 00:00:00 2001 From: PumpkinSeed Date: Mon, 23 Sep 2019 16:19:04 +0200 Subject: [PATCH 1/6] Add fts --- fts.go | 181 ++++++++++++++++++++++++++++++++++++++++++++++++++++ fts_test.go | 56 ++++++++++++++++ go.mod | 21 ++++++ utils.go | 87 +++++++++++++++++++++++++ 4 files changed, 345 insertions(+) create mode 100644 fts.go create mode 100644 fts_test.go create mode 100644 go.mod create mode 100644 utils.go diff --git a/fts.go b/fts.go new file mode 100644 index 0000000..fd2198f --- /dev/null +++ b/fts.go @@ -0,0 +1,181 @@ +package odatas + +import ( + "errors" + "fmt" + "github.com/couchbase/gocb" + "github.com/volatiletech/null" +) + +var ( + ErrEmptyField = errors.New("field must be filled") + ErrEmptyIndex = errors.New("index must be filled") + + placeholderBucket *gocb.Bucket +) + +type SearchQuery struct { + Match null.String `json:"match,omitempty"` + MatchPhrase null.String `json:"match_phrase,omitempty"` + Term null.String `json:"term,omitempty"` + Prefix null.String `json:"prefix,omitempty"` + Regexp null.String `json:"regexp,omitempty"` + Wildcard null.String `json:"wildcard,omitempty"` + Bool null.Bool `json:"bool,omitempty"` + + Field string `json:"field,omitempty"` + Analyzer null.String `json:"analyzer,omitempty"` + Fuzziness null.Int64 `json:"fuzziness,omitempty"` + PrefixLength null.Int64 `json:"prefix_length,omitempty"` +} + +type CompoundQueries struct { + Conjunction []SearchQuery `json:"conjuction,omitempty"` + Disjunction []SearchQuery `json:"disjunction,omitempty"` +} + +type RangeQuery struct { + StartAsTime null.Time `json:"-"` + EndAsTime null.Time `json:"-"` + Start string `json:"start,omitempty"` + End string `json:"end,omitempty"` + + Min null.Int64 `json:"min,omitempty"` + Max null.Int64 `json:"max,omitempty"` + + InclusiveStart null.Bool `json:"inclusive_start,omitempty"` + InclusiveEnd null.Bool `json:"inclusive_end,omitempty"` + InclusiveMin null.Bool `json:"inclusive_min,omitempty"` + InclusiveMax null.Bool `json:"inclusive_max,omitempty"` + + Field string `json:"field,omitempty"` +} + +// time RFC-3339 +//{ +//"start": "2001-10-09T10:20:30-08:00", +//"end": "2016-10-31", +//"inclusive_start": false, +//"inclusive_end": false, +//"field": "review_date" +//} +//{ +//"min": 100, "max": 1000, +//"inclusive_min": false, +//"inclusive_max": false, +//"field": "id" +//} + +func placeholderInit() { + if placeholderBucket == nil { + var err error + cb, err := gocb.Connect("couchbase://localhost") + if err != nil { + panic(err) + } + + err = cb.Authenticate(gocb.PasswordAuthenticator{ + Username: "Administrator", + Password: "password", + }) + if err != nil { + panic(err) + } + + placeholderBucket, err = cb.OpenBucket("company", "") + if err != nil { + panic(err) + } + } +} + +func (h *Handler) SimpleSearch(index string, q *SearchQuery) error { + placeholderInit() + + if err := q.Setup(); err != nil { + return err + } + + if index == "" { + return ErrEmptyIndex + } + + query := gocb.NewSearchQuery(index, q) + res, err := placeholderBucket.ExecuteSearchQuery(query) + if err != nil { + return err + } + fmt.Println(res.Status()) + for _, hit := range res.Hits() { + fmt.Printf("%s\n", hit.Id) + } + + return nil +} + +func (h *Handler) CompoundSearch(doc string, q *CompoundQueries) { + +} + +func (h *Handler) RangeSearch(doc string, q *RangeQuery) { + +} + +func (s *SearchQuery) Setup() error { + if s.Field == "" { + return ErrEmptyField + } + + switch { + case s.Match.Valid && s.Match.String != "": + s.MatchPhrase = EmptyString() + s.Term = EmptyString() + s.Prefix = EmptyString() + s.Regexp = EmptyString() + s.Wildcard = EmptyString() + s.Bool = EmptyBool() + case s.MatchPhrase.Valid && s.MatchPhrase.String != "": + s.Match = EmptyString() + s.Term = EmptyString() + s.Prefix = EmptyString() + s.Regexp = EmptyString() + s.Wildcard = EmptyString() + s.Bool = EmptyBool() + case s.Term.Valid && s.Term.String != "": + s.Match = EmptyString() + s.MatchPhrase = EmptyString() + s.Prefix = EmptyString() + s.Regexp = EmptyString() + s.Wildcard = EmptyString() + s.Bool = EmptyBool() + case s.Prefix.Valid && s.Prefix.String != "": + s.Match = EmptyString() + s.MatchPhrase = EmptyString() + s.Term = EmptyString() + s.Regexp = EmptyString() + s.Wildcard = EmptyString() + s.Bool = EmptyBool() + case s.Regexp.Valid && s.Regexp.String != "": + s.Match = EmptyString() + s.MatchPhrase = EmptyString() + s.Term = EmptyString() + s.Prefix = EmptyString() + s.Wildcard = EmptyString() + s.Bool = EmptyBool() + case s.Wildcard.Valid && s.Wildcard.String != "": + s.Match = EmptyString() + s.MatchPhrase = EmptyString() + s.Term = EmptyString() + s.Prefix = EmptyString() + s.Regexp = EmptyString() + s.Bool = EmptyBool() + case s.Bool.Valid: + s.Match = EmptyString() + s.MatchPhrase = EmptyString() + s.Term = EmptyString() + s.Prefix = EmptyString() + s.Regexp = EmptyString() + s.Wildcard = EmptyString() + } + return nil +} diff --git a/fts_test.go b/fts_test.go new file mode 100644 index 0000000..8194269 --- /dev/null +++ b/fts_test.go @@ -0,0 +1,56 @@ +package odatas + +import ( + "fmt" + "github.com/volatiletech/null" + "testing" + "time" +) + +type TestData struct { + +} + +func TestSimpleSearchMatch(t *testing.T) { + placeholderInit() + + for i := 0; i< 10; i++ { + order := NewTestStruct1() + _, err := placeholderBucket.Insert("order::"+order.Token, order, 0) + if err != nil { + t.Fatal(err) + } + } + + fields := []string{ + "Status", + "PaymentMethod", + "Email", + "CardHolderName", + "BillingAddressName", + "BillingAddressAddress1", + "BillingAddressCity", + "BillingAddressCountry", + "BillingAddressPostalCode", + "BillingAddressPhone", + "Notes", + "ShippingMethod", + } + manager := placeholderBucket.Manager("", "") + err := manager.CreateIndex("order_idx", fields, true, false) + if err != nil { + t.Fatal(err) + } + + handler := New(&Configuration{}) + searchMatch := "Clay Monahan" + mes := time.Now() + err = handler.SimpleSearch("order_idx", &SearchQuery{ + Match: null.StringFrom(searchMatch), + Field: "CardHolderName", + }) + fmt.Println(time.Since(mes)) + if err != nil { + t.Fatal(err) + } +} \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..fd6fd6b --- /dev/null +++ b/go.mod @@ -0,0 +1,21 @@ +module github.com/PumpkinSeed/odatas + +go 1.12 + +require ( + github.com/brianvoe/gofakeit v3.18.0+incompatible + github.com/couchbase/gocb v1.6.3 + github.com/gofrs/uuid v3.2.0+incompatible // indirect + github.com/golang/snappy v0.0.1 // indirect + github.com/google/uuid v1.1.1 // indirect + github.com/opentracing/opentracing-go v1.1.0 // indirect + github.com/pkg/errors v0.8.1 // indirect + github.com/rs/xid v1.2.1 + github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d // indirect + github.com/volatiletech/null v8.0.0+incompatible + github.com/volatiletech/sqlboiler v3.5.0+incompatible // indirect + golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72 // indirect + gopkg.in/couchbase/gocbcore.v7 v7.1.14 // indirect + gopkg.in/couchbaselabs/gocbconnstr.v1 v1.0.4 // indirect + gopkg.in/couchbaselabs/jsonx.v1 v1.0.0 // indirect +) diff --git a/utils.go b/utils.go new file mode 100644 index 0000000..19caa63 --- /dev/null +++ b/utils.go @@ -0,0 +1,87 @@ +package odatas + +import ( + "fmt" + "github.com/brianvoe/gofakeit" + "github.com/rs/xid" + "github.com/volatiletech/null" + "time" +) + +// TestStruct1 is a struct used for testing and represents an order of a webshop +type testStruct1 struct { + Token string + CreationDate string + ModificationDate string + Status string + PaymentMethod string + InvoiceNumber string + Email string + CardHolderName string + CreditCardLast4Digits string + BillingAddressName string + BillingAddressCompanyName string + BillingAddressAddress1 string + BillingAddressAddress2 string + BillingAddressCity string + BillingAddressCountry string + BillingAddressProvince string + BillingAddressPostalCode string + BillingAddressPhone string + Notes string + ShippingAddressName string + ShippingAddressCompanyName string + ShippingAddressAddress1 string + ShippingAddressAddress2 string + ShippingAddressCity string + ShippingAddressCountry string + ShippingAddressProvince string + ShippingAddressPostalCode string + ShippingAddressPhone string + ShippingAddressSameAsBilling bool + FinalGrandTotal int + ShippingFees int + ShippingMethod string + WillBePaidLater bool + PaymentTransactionId string +} + +func NewTestStruct1() testStruct1 { + addr := gofakeit.Address() + name := gofakeit.Name() + return testStruct1{ + Token: xid.New().String(), + CreationDate: time.Now().UTC().String(), + Status: "processed", + PaymentMethod: "card", + InvoiceNumber: fmt.Sprintf("inv-00%d", gofakeit.Number(10000,99999)), + Email: gofakeit.Email(), + CardHolderName: name, + CreditCardLast4Digits: fmt.Sprintf("%d", gofakeit.Number(1000,9999)), + BillingAddressName: name, + BillingAddressAddress1: addr.Address, + BillingAddressCity: addr.City, + BillingAddressCountry: addr.Country, + BillingAddressPostalCode: addr.Zip, + BillingAddressPhone: gofakeit.Phone(), + Notes: gofakeit.HipsterSentence(10), + ShippingAddressName: name, + ShippingAddressAddress1: addr.Address, + ShippingAddressCity: addr.City, + ShippingAddressCountry: addr.Country, + ShippingAddressPostalCode: addr.Zip, + ShippingAddressPhone: gofakeit.Phone(), + FinalGrandTotal: 443, + ShippingFees: 0, + ShippingMethod: "Free shipping", + PaymentTransactionId: xid.New().String(), + } +} + +func EmptyString() null.String { + return null.String{Valid: false} +} + +func EmptyBool() null.Bool { + return null.Bool{Valid: false} +} From 08703f73520ed9321a7b338710c63e3d76ea867b Mon Sep 17 00:00:00 2001 From: PumpkinSeed Date: Mon, 23 Sep 2019 22:29:01 +0200 Subject: [PATCH 2/6] Add create index --- fts.go | 275 ++++++++++++++++++++++++++++++++++++++++++++-------- fts_test.go | 61 +++++++----- utils.go | 9 +- 3 files changed, 276 insertions(+), 69 deletions(-) diff --git a/fts.go b/fts.go index fd2198f..39c28a4 100644 --- a/fts.go +++ b/fts.go @@ -1,32 +1,40 @@ package odatas import ( + "bytes" + "encoding/base64" + "encoding/json" "errors" "fmt" "github.com/couchbase/gocb" - "github.com/volatiletech/null" + "io/ioutil" + "log" + "net/http" + "time" ) var ( - ErrEmptyField = errors.New("field must be filled") + ErrEmptyField = errors.New("field must be filled") ErrEmptyIndex = errors.New("index must be filled") - placeholderBucket *gocb.Bucket + placeholderBucket *gocb.Bucket + placeholderCluster *gocb.Cluster ) type SearchQuery struct { - Match null.String `json:"match,omitempty"` - MatchPhrase null.String `json:"match_phrase,omitempty"` - Term null.String `json:"term,omitempty"` - Prefix null.String `json:"prefix,omitempty"` - Regexp null.String `json:"regexp,omitempty"` - Wildcard null.String `json:"wildcard,omitempty"` - Bool null.Bool `json:"bool,omitempty"` + Query string `json:"query,omitempty"` + Match string `json:"match,omitempty"` + MatchPhrase string `json:"match_phrase,omitempty"` + Term string `json:"term,omitempty"` + Prefix string `json:"prefix,omitempty"` + Regexp string `json:"regexp,omitempty"` + Wildcard string `json:"wildcard,omitempty"` + Bool bool `json:"bool,omitempty"` - Field string `json:"field,omitempty"` - Analyzer null.String `json:"analyzer,omitempty"` - Fuzziness null.Int64 `json:"fuzziness,omitempty"` - PrefixLength null.Int64 `json:"prefix_length,omitempty"` + Field string `json:"field,omitempty"` + Analyzer string `json:"analyzer,omitempty"` + Fuzziness int64 `json:"fuzziness,omitempty"` + PrefixLength int64 `json:"prefix_length,omitempty"` } type CompoundQueries struct { @@ -35,18 +43,18 @@ type CompoundQueries struct { } type RangeQuery struct { - StartAsTime null.Time `json:"-"` - EndAsTime null.Time `json:"-"` + StartAsTime time.Time `json:"-"` + EndAsTime time.Time `json:"-"` Start string `json:"start,omitempty"` End string `json:"end,omitempty"` - Min null.Int64 `json:"min,omitempty"` - Max null.Int64 `json:"max,omitempty"` + Min int64 `json:"min,omitempty"` + Max int64 `json:"max,omitempty"` - InclusiveStart null.Bool `json:"inclusive_start,omitempty"` - InclusiveEnd null.Bool `json:"inclusive_end,omitempty"` - InclusiveMin null.Bool `json:"inclusive_min,omitempty"` - InclusiveMax null.Bool `json:"inclusive_max,omitempty"` + InclusiveStart bool `json:"inclusive_start,omitempty"` + InclusiveEnd bool `json:"inclusive_end,omitempty"` + InclusiveMin bool `json:"inclusive_min,omitempty"` + InclusiveMax bool `json:"inclusive_max,omitempty"` Field string `json:"field,omitempty"` } @@ -69,12 +77,12 @@ type RangeQuery struct { func placeholderInit() { if placeholderBucket == nil { var err error - cb, err := gocb.Connect("couchbase://localhost") + placeholderCluster, err = gocb.Connect("couchbase://localhost") if err != nil { panic(err) } - err = cb.Authenticate(gocb.PasswordAuthenticator{ + err = placeholderCluster.Authenticate(gocb.PasswordAuthenticator{ Username: "Administrator", Password: "password", }) @@ -82,7 +90,7 @@ func placeholderInit() { panic(err) } - placeholderBucket, err = cb.OpenBucket("company", "") + placeholderBucket, err = placeholderCluster.OpenBucket("company", "") if err != nil { panic(err) } @@ -107,7 +115,10 @@ func (h *Handler) SimpleSearch(index string, q *SearchQuery) error { } fmt.Println(res.Status()) for _, hit := range res.Hits() { - fmt.Printf("%s\n", hit.Id) + fmt.Printf("%+v\n", hit) + } + for _, facet := range res.Facets() { + fmt.Printf("%+v\n", facet) } return nil @@ -127,55 +138,241 @@ func (s *SearchQuery) Setup() error { } switch { - case s.Match.Valid && s.Match.String != "": + case s.Match != "": s.MatchPhrase = EmptyString() s.Term = EmptyString() s.Prefix = EmptyString() s.Regexp = EmptyString() s.Wildcard = EmptyString() s.Bool = EmptyBool() - case s.MatchPhrase.Valid && s.MatchPhrase.String != "": + case s.MatchPhrase != "": s.Match = EmptyString() s.Term = EmptyString() s.Prefix = EmptyString() s.Regexp = EmptyString() s.Wildcard = EmptyString() s.Bool = EmptyBool() - case s.Term.Valid && s.Term.String != "": + case s.Term != "": s.Match = EmptyString() s.MatchPhrase = EmptyString() s.Prefix = EmptyString() s.Regexp = EmptyString() s.Wildcard = EmptyString() s.Bool = EmptyBool() - case s.Prefix.Valid && s.Prefix.String != "": + case s.Prefix != "": s.Match = EmptyString() s.MatchPhrase = EmptyString() s.Term = EmptyString() s.Regexp = EmptyString() s.Wildcard = EmptyString() s.Bool = EmptyBool() - case s.Regexp.Valid && s.Regexp.String != "": + case s.Regexp != "": s.Match = EmptyString() s.MatchPhrase = EmptyString() s.Term = EmptyString() s.Prefix = EmptyString() s.Wildcard = EmptyString() s.Bool = EmptyBool() - case s.Wildcard.Valid && s.Wildcard.String != "": + case s.Wildcard != "": s.Match = EmptyString() s.MatchPhrase = EmptyString() s.Term = EmptyString() s.Prefix = EmptyString() s.Regexp = EmptyString() s.Bool = EmptyBool() - case s.Bool.Valid: - s.Match = EmptyString() - s.MatchPhrase = EmptyString() - s.Term = EmptyString() - s.Prefix = EmptyString() - s.Regexp = EmptyString() - s.Wildcard = EmptyString() + //case s.Bool.Valid: + // s.Match = EmptyString() + // s.MatchPhrase = EmptyString() + // s.Term = EmptyString() + // s.Prefix = EmptyString() + // s.Regexp = EmptyString() + // s.Wildcard = EmptyString() } return nil } + +type FullTextSearchIndexDefinition struct { + Type string `json:"type"` + Name string `json:"name"` + SourceType string `json:"sourceType"` + SourceName string `json:"sourceName"` + PlanParams FullTextSearchIndexPlanParams `json:"planParams"` + Params FullTextSearchIndexParams `json:"params"` +} + +type FullTextSearchIndexPlanParams struct { + MaxPartitionsPerPIndex int64 `json:"maxPartitionsPerPIndex"` +} + +type FullTextSearchIndexParams struct { + DocConfig FullTextSearchIndexDocConfig `json:"doc_config"` + Mapping FullTextSearchIndexMapping `json:"mapping"` + Store FullTextSearchIndexStore `json:"store"` +} + +type FullTextSearchIndexDocConfig struct { + DocIDPrefixDelimiter string `json:"docid_prefix_delim"` + DocIDRegexp string `json:"docid_regexp"` + Mode string `json:"mode"` + TypeField string `json:"type_field"` +} + +type FullTextSearchIndexMapping struct { + DefaultAnalyzer string `json:"default_analyzer"` + DefaultDatetimeParser string `json:"default_datetime_parser"` + DefaultField string `json:"default_field"` + DefaultMapping FullTextSearchIndexDefaultMapping `json:"default_mapping"` + DefaultType string `json:"default_type"` + DocvaluesDynamic bool `json:"docvalues_dynamic"` + IndexDynamic bool `json:"index_dynamic"` + StoreDynamic bool `json:"store_dynamic"` + TypeField string `json:"type_field"` +} + +type FullTextSearchIndexDefaultMapping struct { + Dynamic bool `json:"dynamic"` + Enabled bool `json:"enabled"` +} + +type FullTextSearchIndexStore struct { + IndexType string `json:"indexType"` + KVStoreName string `json:"kvStoreName"` +} + +type FullTextSearchIndexMeta struct { + Name string + SourceType string + SourceName string + DocIDPrefixDelimiter string + DocIDRegexp string + TypeField string +} + +func DefaultFullTextSearchIndexDefinition(meta FullTextSearchIndexMeta) (*FullTextSearchIndexDefinition, error) { + if meta.Name == "" { + return nil, errors.New("index name must set") + } + if meta.SourceType == "" { + return nil, errors.New("source type must set") + } + if meta.SourceName == "" { + return nil, errors.New("source name must set") + } + + var ftsDef = &FullTextSearchIndexDefinition{ + Type: "fulltext-index", + Name: meta.Name, + SourceType: meta.SourceType, + SourceName: meta.SourceName, + PlanParams: FullTextSearchIndexPlanParams{ + MaxPartitionsPerPIndex: 171, + }, + Params: FullTextSearchIndexParams{ + Mapping: FullTextSearchIndexMapping{ + DefaultAnalyzer: "standard", + DefaultDatetimeParser: "dateTimeOptional", + DefaultField: "_all", + DefaultMapping: FullTextSearchIndexDefaultMapping{ + Dynamic: true, + Enabled: true, + }, + DefaultType: "_default", + DocvaluesDynamic: true, + IndexDynamic: true, + StoreDynamic: true, + TypeField: "_type", + }, + Store: FullTextSearchIndexStore{ + IndexType: "scorch", + KVStoreName: "", + }, + }, + } + + switch { + case meta.DocIDPrefixDelimiter != "": + ftsDef.Params.DocConfig = FullTextSearchIndexDocConfig{ + DocIDPrefixDelimiter: meta.DocIDPrefixDelimiter, + Mode: "docid_prefix", + DocIDRegexp: "", + TypeField: "", + } + case meta.DocIDRegexp != "": + ftsDef.Params.DocConfig = FullTextSearchIndexDocConfig{ + DocIDPrefixDelimiter: "", + Mode: "docid_regexp", + DocIDRegexp: meta.DocIDRegexp, + TypeField: "", + } + case meta.TypeField != "": + ftsDef.Params.DocConfig = FullTextSearchIndexDocConfig{ + DocIDPrefixDelimiter: "", + Mode: "type_field", + DocIDRegexp: "", + TypeField: meta.TypeField, + } + } + + return ftsDef, nil +} + +func CreateFullTextSearchIndex(def *FullTextSearchIndexDefinition) error { + domain := "http://localhost:8091" + endpoint := "/_p/fts/api/index/" + url := fmt.Sprintf("%s%s%s", domain, endpoint, def.Name) + + body, err := json.Marshal(def) + if err != nil { + return err + } + req, _ := http.NewRequest("PUT", url, bytes.NewBuffer(body)) + setupBasicAuth(req) + req.Header.Add("Content-Type", "application/json") + client := http.DefaultClient + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + respbody, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + log.Println(string(respbody)) + + return nil +} + +func DeleteFullTextSearchIndex(indexName string) error { + domain := "http://localhost:8091" + endpoint := "/_p/fts/api/index/" + url := fmt.Sprintf("%s%s%s", domain, endpoint, indexName) + + req, _ := http.NewRequest("DELETE", url, nil) + setupBasicAuth(req) + req.Header.Add("Content-Type", "application/json") + client := http.DefaultClient + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + respbody, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + log.Println(string(respbody)) + + return nil +} + +func basicAuth(username, password string) string { + auth := username + ":" + password + return base64.StdEncoding.EncodeToString([]byte(auth)) +} + +func setupBasicAuth(req *http.Request) { + req.Header.Add("Authorization","Basic " + basicAuth("Administrator","password")) +} \ No newline at end of file diff --git a/fts_test.go b/fts_test.go index 8194269..94d82fc 100644 --- a/fts_test.go +++ b/fts_test.go @@ -1,14 +1,45 @@ package odatas import ( + "encoding/json" "fmt" - "github.com/volatiletech/null" "testing" "time" ) -type TestData struct { +func TestSearchQuery(t *testing.T) { + sq := SearchQuery{ + Query: "card", + } + + sqjso, err := json.Marshal(sq) + if err != nil { + t.Fatal(err) + } + fmt.Print(string(sqjso)) + + placeholderInit() +} + +func TestCreateFullTextSearchIndex(t *testing.T) { + err := DeleteFullTextSearchIndex("order_fts_idx") + if err != nil { + t.Fatal(err) + } + def, err := DefaultFullTextSearchIndexDefinition(FullTextSearchIndexMeta{ + Name: "order_fts_idx", + SourceType: "couchbase", + SourceName: "company", + DocIDPrefixDelimiter: "::", + }) + if err != nil { + t.Fatal(err) + } + err = CreateFullTextSearchIndex(def) + if err != nil { + t.Fatal(err) + } } func TestSimpleSearchMatch(t *testing.T) { @@ -22,31 +53,11 @@ func TestSimpleSearchMatch(t *testing.T) { } } - fields := []string{ - "Status", - "PaymentMethod", - "Email", - "CardHolderName", - "BillingAddressName", - "BillingAddressAddress1", - "BillingAddressCity", - "BillingAddressCountry", - "BillingAddressPostalCode", - "BillingAddressPhone", - "Notes", - "ShippingMethod", - } - manager := placeholderBucket.Manager("", "") - err := manager.CreateIndex("order_idx", fields, true, false) - if err != nil { - t.Fatal(err) - } - handler := New(&Configuration{}) - searchMatch := "Clay Monahan" + searchMatch := "Talia Hudson" mes := time.Now() - err = handler.SimpleSearch("order_idx", &SearchQuery{ - Match: null.StringFrom(searchMatch), + err := handler.SimpleSearch("order_fts_idx", &SearchQuery{ + Match: searchMatch, Field: "CardHolderName", }) fmt.Println(time.Since(mes)) diff --git a/utils.go b/utils.go index 19caa63..dd1fd8f 100644 --- a/utils.go +++ b/utils.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/brianvoe/gofakeit" "github.com/rs/xid" - "github.com/volatiletech/null" "time" ) @@ -78,10 +77,10 @@ func NewTestStruct1() testStruct1 { } } -func EmptyString() null.String { - return null.String{Valid: false} +func EmptyString() string { + return "" } -func EmptyBool() null.Bool { - return null.Bool{Valid: false} +func EmptyBool() bool { + return false } From 0352527181b9ff4d553bc5042462fd5304be71a3 Mon Sep 17 00:00:00 2001 From: PumpkinSeed Date: Tue, 24 Sep 2019 09:40:37 +0200 Subject: [PATCH 3/6] Restructure and add inspect --- fts.go | 143 +++++++++++++++++++++++++++++----------------------- fts_test.go | 9 ++-- handler.go | 30 ++++++++++- utils.go | 17 +++++-- 4 files changed, 127 insertions(+), 72 deletions(-) diff --git a/fts.go b/fts.go index 39c28a4..ba9473d 100644 --- a/fts.go +++ b/fts.go @@ -2,7 +2,6 @@ package odatas import ( "bytes" - "encoding/base64" "encoding/json" "errors" "fmt" @@ -139,58 +138,66 @@ func (s *SearchQuery) Setup() error { switch { case s.Match != "": - s.MatchPhrase = EmptyString() - s.Term = EmptyString() - s.Prefix = EmptyString() - s.Regexp = EmptyString() - s.Wildcard = EmptyString() - s.Bool = EmptyBool() + s.MatchPhrase = emptyString() + s.Term = emptyString() + s.Prefix = emptyString() + s.Regexp = emptyString() + s.Wildcard = emptyString() + s.Bool = emptyBool() case s.MatchPhrase != "": - s.Match = EmptyString() - s.Term = EmptyString() - s.Prefix = EmptyString() - s.Regexp = EmptyString() - s.Wildcard = EmptyString() - s.Bool = EmptyBool() + s.Match = emptyString() + s.Term = emptyString() + s.Prefix = emptyString() + s.Regexp = emptyString() + s.Wildcard = emptyString() + s.Bool = emptyBool() case s.Term != "": - s.Match = EmptyString() - s.MatchPhrase = EmptyString() - s.Prefix = EmptyString() - s.Regexp = EmptyString() - s.Wildcard = EmptyString() - s.Bool = EmptyBool() + s.Match = emptyString() + s.MatchPhrase = emptyString() + s.Prefix = emptyString() + s.Regexp = emptyString() + s.Wildcard = emptyString() + s.Bool = emptyBool() case s.Prefix != "": - s.Match = EmptyString() - s.MatchPhrase = EmptyString() - s.Term = EmptyString() - s.Regexp = EmptyString() - s.Wildcard = EmptyString() - s.Bool = EmptyBool() + s.Match = emptyString() + s.MatchPhrase = emptyString() + s.Term = emptyString() + s.Regexp = emptyString() + s.Wildcard = emptyString() + s.Bool = emptyBool() case s.Regexp != "": - s.Match = EmptyString() - s.MatchPhrase = EmptyString() - s.Term = EmptyString() - s.Prefix = EmptyString() - s.Wildcard = EmptyString() - s.Bool = EmptyBool() + s.Match = emptyString() + s.MatchPhrase = emptyString() + s.Term = emptyString() + s.Prefix = emptyString() + s.Wildcard = emptyString() + s.Bool = emptyBool() case s.Wildcard != "": - s.Match = EmptyString() - s.MatchPhrase = EmptyString() - s.Term = EmptyString() - s.Prefix = EmptyString() - s.Regexp = EmptyString() - s.Bool = EmptyBool() + s.Match = emptyString() + s.MatchPhrase = emptyString() + s.Term = emptyString() + s.Prefix = emptyString() + s.Regexp = emptyString() + s.Bool = emptyBool() //case s.Bool.Valid: - // s.Match = EmptyString() - // s.MatchPhrase = EmptyString() - // s.Term = EmptyString() - // s.Prefix = EmptyString() - // s.Regexp = EmptyString() - // s.Wildcard = EmptyString() + // s.Match = emptyString() + // s.MatchPhrase = emptyString() + // s.Term = emptyString() + // s.Prefix = emptyString() + // s.Regexp = emptyString() + // s.Wildcard = emptyString() } return nil } +/* + Index of FTS +*/ + +const ( + ftsEndpoint = "/_p/fts/api/index" +) + type FullTextSearchIndexDefinition struct { Type string `json:"type"` Name string `json:"name"` @@ -316,20 +323,15 @@ func DefaultFullTextSearchIndexDefinition(meta FullTextSearchIndexMeta) (*FullTe return ftsDef, nil } -func CreateFullTextSearchIndex(def *FullTextSearchIndexDefinition) error { - domain := "http://localhost:8091" - endpoint := "/_p/fts/api/index/" - url := fmt.Sprintf("%s%s%s", domain, endpoint, def.Name) - +func (h *Handler) CreateFullTextSearchIndex(def *FullTextSearchIndexDefinition) error { body, err := json.Marshal(def) if err != nil { return err } - req, _ := http.NewRequest("PUT", url, bytes.NewBuffer(body)) + req, _ := http.NewRequest("PUT", h.fullTestSearchURL(def.Name), bytes.NewBuffer(body)) setupBasicAuth(req) req.Header.Add("Content-Type", "application/json") - client := http.DefaultClient - resp, err := client.Do(req) + resp, err := h.http.Do(req) if err != nil { return err } @@ -344,16 +346,12 @@ func CreateFullTextSearchIndex(def *FullTextSearchIndexDefinition) error { return nil } -func DeleteFullTextSearchIndex(indexName string) error { - domain := "http://localhost:8091" - endpoint := "/_p/fts/api/index/" - url := fmt.Sprintf("%s%s%s", domain, endpoint, indexName) - - req, _ := http.NewRequest("DELETE", url, nil) +func (h *Handler) DeleteFullTextSearchIndex(indexName string) error { + req, _ := http.NewRequest("DELETE", h.fullTestSearchURL(indexName), nil) setupBasicAuth(req) req.Header.Add("Content-Type", "application/json") - client := http.DefaultClient - resp, err := client.Do(req) + + resp, err := h.http.Do(req) if err != nil { return err } @@ -368,11 +366,28 @@ func DeleteFullTextSearchIndex(indexName string) error { return nil } -func basicAuth(username, password string) string { - auth := username + ":" + password - return base64.StdEncoding.EncodeToString([]byte(auth)) +func (h *Handler) InspectFullTextSearchIndex(indexName string) (bool, error) { + req, _ := http.NewRequest("GET", h.fullTestSearchURL(""), nil) + setupBasicAuth(req) + req.Header.Add("Content-Type", "application/json") + + resp, err := h.http.Do(req) + if err != nil { + return false, err + } + defer resp.Body.Close() + + respbody, err := ioutil.ReadAll(resp.Body) + if err != nil { + return false, err + } + log.Println(string(respbody)) + return false, nil } -func setupBasicAuth(req *http.Request) { - req.Header.Add("Authorization","Basic " + basicAuth("Administrator","password")) +func (h *Handler) fullTestSearchURL(indexName string) string { + if indexName == "" { + return fmt.Sprintf("%s%s", h.httpAddress, ftsEndpoint) + } + return fmt.Sprintf("%s%s/%s", h.httpAddress, ftsEndpoint, indexName) } \ No newline at end of file diff --git a/fts_test.go b/fts_test.go index 94d82fc..eb8b65c 100644 --- a/fts_test.go +++ b/fts_test.go @@ -22,7 +22,10 @@ func TestSearchQuery(t *testing.T) { } func TestCreateFullTextSearchIndex(t *testing.T) { - err := DeleteFullTextSearchIndex("order_fts_idx") + h := New(&Configuration{}) + h.InspectFullTextSearchIndex("order_fts_idx") + + err := h.DeleteFullTextSearchIndex("order_fts_idx") if err != nil { t.Fatal(err) } @@ -36,7 +39,7 @@ func TestCreateFullTextSearchIndex(t *testing.T) { if err != nil { t.Fatal(err) } - err = CreateFullTextSearchIndex(def) + err = h.CreateFullTextSearchIndex(def) if err != nil { t.Fatal(err) } @@ -46,7 +49,7 @@ func TestSimpleSearchMatch(t *testing.T) { placeholderInit() for i := 0; i< 10; i++ { - order := NewTestStruct1() + order := newTestStruct1() _, err := placeholderBucket.Insert("order::"+order.Token, order, 0) if err != nil { t.Fatal(err) diff --git a/handler.go b/handler.go index f5946de..4d7b5f8 100644 --- a/handler.go +++ b/handler.go @@ -1,7 +1,18 @@ package odatas +import ( + "net" + "net/http" + "time" +) + type Handler struct { state string + + address string + httpAddress string + + http *http.Client } type Configuration struct { @@ -12,5 +23,20 @@ type Configuration struct { } func New(c *Configuration) Handler { - return Handler{} -} \ No newline at end of file + client := &http.Client{ + Transport: &http.Transport{ + Dial: (&net.Dialer{ + Timeout: 5 * time.Second, + }).Dial, + TLSHandshakeTimeout: 5 * time.Second, + }, + Timeout: time.Second * 10, + } + + return Handler{ + http: client, + httpAddress: "http://localhost:8091", + } +} + + diff --git a/utils.go b/utils.go index dd1fd8f..79de1f0 100644 --- a/utils.go +++ b/utils.go @@ -1,9 +1,11 @@ package odatas import ( + "encoding/base64" "fmt" "github.com/brianvoe/gofakeit" "github.com/rs/xid" + "net/http" "time" ) @@ -45,7 +47,7 @@ type testStruct1 struct { PaymentTransactionId string } -func NewTestStruct1() testStruct1 { +func newTestStruct1() testStruct1 { addr := gofakeit.Address() name := gofakeit.Name() return testStruct1{ @@ -77,10 +79,19 @@ func NewTestStruct1() testStruct1 { } } -func EmptyString() string { +func emptyString() string { return "" } -func EmptyBool() bool { +func emptyBool() bool { return false } + +func basicAuth(username, password string) string { + auth := username + ":" + password + return base64.StdEncoding.EncodeToString([]byte(auth)) +} + +func setupBasicAuth(req *http.Request) { + req.Header.Add("Authorization","Basic " + basicAuth("Administrator","password")) +} From 2492a69193300d7a733f82b906bbde8a8ef3773d Mon Sep 17 00:00:00 2001 From: PumpkinSeed Date: Tue, 24 Sep 2019 10:53:36 +0200 Subject: [PATCH 4/6] Finish index creation --- fts.go | 125 +++++++++++++++++++++++++++++++++------------------- fts_test.go | 20 +++++---- 2 files changed, 90 insertions(+), 55 deletions(-) diff --git a/fts.go b/fts.go index ba9473d..02c0d47 100644 --- a/fts.go +++ b/fts.go @@ -7,7 +7,6 @@ import ( "fmt" "github.com/couchbase/gocb" "io/ioutil" - "log" "net/http" "time" ) @@ -198,55 +197,66 @@ const ( ftsEndpoint = "/_p/fts/api/index" ) -type FullTextSearchIndexDefinition struct { - Type string `json:"type"` - Name string `json:"name"` - SourceType string `json:"sourceType"` - SourceName string `json:"sourceName"` - PlanParams FullTextSearchIndexPlanParams `json:"planParams"` - Params FullTextSearchIndexParams `json:"params"` +type apiResponse struct { + Status string `json:"status"` + IndexDefs IndexDefs `json:"indexDefs"` + Error string `json:"error"` } -type FullTextSearchIndexPlanParams struct { +type IndexDefs struct { + UUID string `json:"uuid"` + IndexDefs map[string]IndexDefinition `json:"indexDefs"` +} + +type IndexDefinition struct { + Type string `json:"type"` + Name string `json:"name"` + SourceType string `json:"sourceType"` + SourceName string `json:"sourceName"` + PlanParams IndexPlanParams `json:"planParams"` + Params IndexParams `json:"params"` +} + +type IndexPlanParams struct { MaxPartitionsPerPIndex int64 `json:"maxPartitionsPerPIndex"` } -type FullTextSearchIndexParams struct { - DocConfig FullTextSearchIndexDocConfig `json:"doc_config"` - Mapping FullTextSearchIndexMapping `json:"mapping"` - Store FullTextSearchIndexStore `json:"store"` +type IndexParams struct { + DocConfig IndexDocConfig `json:"doc_config"` + Mapping IndexMapping `json:"mapping"` + Store IndexStore `json:"store"` } -type FullTextSearchIndexDocConfig struct { +type IndexDocConfig struct { DocIDPrefixDelimiter string `json:"docid_prefix_delim"` DocIDRegexp string `json:"docid_regexp"` Mode string `json:"mode"` TypeField string `json:"type_field"` } -type FullTextSearchIndexMapping struct { - DefaultAnalyzer string `json:"default_analyzer"` - DefaultDatetimeParser string `json:"default_datetime_parser"` - DefaultField string `json:"default_field"` - DefaultMapping FullTextSearchIndexDefaultMapping `json:"default_mapping"` - DefaultType string `json:"default_type"` - DocvaluesDynamic bool `json:"docvalues_dynamic"` - IndexDynamic bool `json:"index_dynamic"` - StoreDynamic bool `json:"store_dynamic"` - TypeField string `json:"type_field"` +type IndexMapping struct { + DefaultAnalyzer string `json:"default_analyzer"` + DefaultDatetimeParser string `json:"default_datetime_parser"` + DefaultField string `json:"default_field"` + DefaultMapping IndexDefaultMapping `json:"default_mapping"` + DefaultType string `json:"default_type"` + DocvaluesDynamic bool `json:"docvalues_dynamic"` + IndexDynamic bool `json:"index_dynamic"` + StoreDynamic bool `json:"store_dynamic"` + TypeField string `json:"type_field"` } -type FullTextSearchIndexDefaultMapping struct { +type IndexDefaultMapping struct { Dynamic bool `json:"dynamic"` Enabled bool `json:"enabled"` } -type FullTextSearchIndexStore struct { +type IndexStore struct { IndexType string `json:"indexType"` KVStoreName string `json:"kvStoreName"` } -type FullTextSearchIndexMeta struct { +type IndexMeta struct { Name string SourceType string SourceName string @@ -255,7 +265,7 @@ type FullTextSearchIndexMeta struct { TypeField string } -func DefaultFullTextSearchIndexDefinition(meta FullTextSearchIndexMeta) (*FullTextSearchIndexDefinition, error) { +func DefaultFullTextSearchIndexDefinition(meta IndexMeta) (*IndexDefinition, error) { if meta.Name == "" { return nil, errors.New("index name must set") } @@ -266,20 +276,20 @@ func DefaultFullTextSearchIndexDefinition(meta FullTextSearchIndexMeta) (*FullTe return nil, errors.New("source name must set") } - var ftsDef = &FullTextSearchIndexDefinition{ + var ftsDef = &IndexDefinition{ Type: "fulltext-index", Name: meta.Name, SourceType: meta.SourceType, SourceName: meta.SourceName, - PlanParams: FullTextSearchIndexPlanParams{ + PlanParams: IndexPlanParams{ MaxPartitionsPerPIndex: 171, }, - Params: FullTextSearchIndexParams{ - Mapping: FullTextSearchIndexMapping{ + Params: IndexParams{ + Mapping: IndexMapping{ DefaultAnalyzer: "standard", DefaultDatetimeParser: "dateTimeOptional", DefaultField: "_all", - DefaultMapping: FullTextSearchIndexDefaultMapping{ + DefaultMapping: IndexDefaultMapping{ Dynamic: true, Enabled: true, }, @@ -289,7 +299,7 @@ func DefaultFullTextSearchIndexDefinition(meta FullTextSearchIndexMeta) (*FullTe StoreDynamic: true, TypeField: "_type", }, - Store: FullTextSearchIndexStore{ + Store: IndexStore{ IndexType: "scorch", KVStoreName: "", }, @@ -298,21 +308,21 @@ func DefaultFullTextSearchIndexDefinition(meta FullTextSearchIndexMeta) (*FullTe switch { case meta.DocIDPrefixDelimiter != "": - ftsDef.Params.DocConfig = FullTextSearchIndexDocConfig{ + ftsDef.Params.DocConfig = IndexDocConfig{ DocIDPrefixDelimiter: meta.DocIDPrefixDelimiter, Mode: "docid_prefix", DocIDRegexp: "", TypeField: "", } case meta.DocIDRegexp != "": - ftsDef.Params.DocConfig = FullTextSearchIndexDocConfig{ + ftsDef.Params.DocConfig = IndexDocConfig{ DocIDPrefixDelimiter: "", Mode: "docid_regexp", DocIDRegexp: meta.DocIDRegexp, TypeField: "", } case meta.TypeField != "": - ftsDef.Params.DocConfig = FullTextSearchIndexDocConfig{ + ftsDef.Params.DocConfig = IndexDocConfig{ DocIDPrefixDelimiter: "", Mode: "type_field", DocIDRegexp: "", @@ -323,7 +333,7 @@ func DefaultFullTextSearchIndexDefinition(meta FullTextSearchIndexMeta) (*FullTe return ftsDef, nil } -func (h *Handler) CreateFullTextSearchIndex(def *FullTextSearchIndexDefinition) error { +func (h *Handler) CreateFullTextSearchIndex(def *IndexDefinition) error { body, err := json.Marshal(def) if err != nil { return err @@ -341,7 +351,15 @@ func (h *Handler) CreateFullTextSearchIndex(def *FullTextSearchIndexDefinition) if err != nil { return err } - log.Println(string(respbody)) + + var ar apiResponse + err = json.Unmarshal(respbody, &ar) + if err != nil { + return err + } + if ar.Status == "fail" { + return errors.New(ar.Error) + } return nil } @@ -361,28 +379,43 @@ func (h *Handler) DeleteFullTextSearchIndex(indexName string) error { if err != nil { return err } - log.Println(string(respbody)) + + var ar apiResponse + err = json.Unmarshal(respbody, &ar) + if err != nil { + return err + } + if ar.Status == "fail" { + return errors.New(ar.Error) + } return nil } -func (h *Handler) InspectFullTextSearchIndex(indexName string) (bool, error) { +func (h *Handler) InspectFullTextSearchIndex(indexName string) (bool, *IndexDefinition, error) { req, _ := http.NewRequest("GET", h.fullTestSearchURL(""), nil) setupBasicAuth(req) req.Header.Add("Content-Type", "application/json") resp, err := h.http.Do(req) if err != nil { - return false, err + return false, nil, err } defer resp.Body.Close() respbody, err := ioutil.ReadAll(resp.Body) if err != nil { - return false, err + return false, nil, err } - log.Println(string(respbody)) - return false, nil + var ar apiResponse + err = json.Unmarshal(respbody, &ar) + if err != nil { + return false, nil, err + } + if v, ok := ar.IndexDefs.IndexDefs[indexName]; ok { + return true, &v, nil + } + return false, nil, nil } func (h *Handler) fullTestSearchURL(indexName string) string { @@ -390,4 +423,4 @@ func (h *Handler) fullTestSearchURL(indexName string) string { return fmt.Sprintf("%s%s", h.httpAddress, ftsEndpoint) } return fmt.Sprintf("%s%s/%s", h.httpAddress, ftsEndpoint, indexName) -} \ No newline at end of file +} diff --git a/fts_test.go b/fts_test.go index eb8b65c..eff4282 100644 --- a/fts_test.go +++ b/fts_test.go @@ -22,16 +22,18 @@ func TestSearchQuery(t *testing.T) { } func TestCreateFullTextSearchIndex(t *testing.T) { - h := New(&Configuration{}) - h.InspectFullTextSearchIndex("order_fts_idx") + indexName := "order_fts_idx" - err := h.DeleteFullTextSearchIndex("order_fts_idx") - if err != nil { - t.Fatal(err) + h := New(&Configuration{}) + if ok, _, _ := h.InspectFullTextSearchIndex(indexName); ok { + err := h.DeleteFullTextSearchIndex(indexName) + if err != nil { + t.Fatal(err) + } } - def, err := DefaultFullTextSearchIndexDefinition(FullTextSearchIndexMeta{ - Name: "order_fts_idx", + def, err := DefaultFullTextSearchIndexDefinition(IndexMeta{ + Name: indexName, SourceType: "couchbase", SourceName: "company", DocIDPrefixDelimiter: "::", @@ -48,7 +50,7 @@ func TestCreateFullTextSearchIndex(t *testing.T) { func TestSimpleSearchMatch(t *testing.T) { placeholderInit() - for i := 0; i< 10; i++ { + for i := 0; i < 10; i++ { order := newTestStruct1() _, err := placeholderBucket.Insert("order::"+order.Token, order, 0) if err != nil { @@ -67,4 +69,4 @@ func TestSimpleSearchMatch(t *testing.T) { if err != nil { t.Fatal(err) } -} \ No newline at end of file +} From ccdcbfec1237e4691e07bd4c9554481524013d65 Mon Sep 17 00:00:00 2001 From: PumpkinSeed Date: Tue, 24 Sep 2019 11:26:30 +0200 Subject: [PATCH 5/6] Move indexing, add simple search --- fts.go | 313 +++++++++------------------------------------- fts_index.go | 242 +++++++++++++++++++++++++++++++++++ fts_index_test.go | 29 +++++ fts_test.go | 32 +---- 4 files changed, 334 insertions(+), 282 deletions(-) create mode 100644 fts_index.go create mode 100644 fts_index_test.go diff --git a/fts.go b/fts.go index 02c0d47..b2ff29d 100644 --- a/fts.go +++ b/fts.go @@ -1,16 +1,19 @@ package odatas import ( - "bytes" - "encoding/json" "errors" "fmt" "github.com/couchbase/gocb" - "io/ioutil" - "net/http" + "github.com/couchbase/gocb/cbft" "time" ) +const ( + FacetDate = iota + FacetNumeric + FacetTerm +) + var ( ErrEmptyField = errors.New("field must be filled") ErrEmptyIndex = errors.New("index must be filled") @@ -33,6 +36,16 @@ type SearchQuery struct { Analyzer string `json:"analyzer,omitempty"` Fuzziness int64 `json:"fuzziness,omitempty"` PrefixLength int64 `json:"prefix_length,omitempty"` + + Limit int `json:"-"` + Offset int `json:"-"` +} + +type FacetDef struct { + Name string + Type int + Field string + Size int } type CompoundQueries struct { @@ -57,21 +70,6 @@ type RangeQuery struct { Field string `json:"field,omitempty"` } -// time RFC-3339 -//{ -//"start": "2001-10-09T10:20:30-08:00", -//"end": "2016-10-31", -//"inclusive_start": false, -//"inclusive_end": false, -//"field": "review_date" -//} -//{ -//"min": 100, "max": 1000, -//"inclusive_min": false, -//"inclusive_max": false, -//"field": "id" -//} - func placeholderInit() { if placeholderBucket == nil { var err error @@ -106,7 +104,37 @@ func (h *Handler) SimpleSearch(index string, q *SearchQuery) error { return ErrEmptyIndex } - query := gocb.NewSearchQuery(index, q) + query := gocb.NewSearchQuery(index, q).Limit(q.Limit).Skip(q.Offset) + return h.doSimpleSearch(query) +} + +func (h *Handler) SimpleSearchWithFacets(index string, q *SearchQuery, facets []FacetDef) error { + placeholderInit() + + if err := q.Setup(); err != nil { + return err + } + + if index == "" { + return ErrEmptyIndex + } + + query := gocb.NewSearchQuery(index, q).Limit(q.Limit).Skip(q.Offset) + for _, facet := range facets { + switch facet.Type { + case FacetDate: + query.AddFacet(facet.Name, cbft.NewDateFacet(facet.Field, facet.Size)) + case FacetNumeric: + query.AddFacet(facet.Name, cbft.NewNumericFacet(facet.Field, facet.Size)) + case FacetTerm: + query.AddFacet(facet.Name, cbft.NewTermFacet(facet.Field, facet.Size)) + } + } + + return h.doSimpleSearch(query) +} + +func (h *Handler) doSimpleSearch(query *gocb.SearchQuery) error { res, err := placeholderBucket.ExecuteSearchQuery(query) if err != nil { return err @@ -131,6 +159,17 @@ func (h *Handler) RangeSearch(doc string, q *RangeQuery) { } func (s *SearchQuery) Setup() error { + if s.Query != "" { + s.Match = emptyString() + s.MatchPhrase = emptyString() + s.Term = emptyString() + s.Prefix = emptyString() + s.Regexp = emptyString() + s.Wildcard = emptyString() + s.Bool = emptyBool() + return nil + } + if s.Field == "" { return ErrEmptyField } @@ -191,236 +230,4 @@ func (s *SearchQuery) Setup() error { /* Index of FTS -*/ - -const ( - ftsEndpoint = "/_p/fts/api/index" -) - -type apiResponse struct { - Status string `json:"status"` - IndexDefs IndexDefs `json:"indexDefs"` - Error string `json:"error"` -} - -type IndexDefs struct { - UUID string `json:"uuid"` - IndexDefs map[string]IndexDefinition `json:"indexDefs"` -} - -type IndexDefinition struct { - Type string `json:"type"` - Name string `json:"name"` - SourceType string `json:"sourceType"` - SourceName string `json:"sourceName"` - PlanParams IndexPlanParams `json:"planParams"` - Params IndexParams `json:"params"` -} - -type IndexPlanParams struct { - MaxPartitionsPerPIndex int64 `json:"maxPartitionsPerPIndex"` -} - -type IndexParams struct { - DocConfig IndexDocConfig `json:"doc_config"` - Mapping IndexMapping `json:"mapping"` - Store IndexStore `json:"store"` -} - -type IndexDocConfig struct { - DocIDPrefixDelimiter string `json:"docid_prefix_delim"` - DocIDRegexp string `json:"docid_regexp"` - Mode string `json:"mode"` - TypeField string `json:"type_field"` -} - -type IndexMapping struct { - DefaultAnalyzer string `json:"default_analyzer"` - DefaultDatetimeParser string `json:"default_datetime_parser"` - DefaultField string `json:"default_field"` - DefaultMapping IndexDefaultMapping `json:"default_mapping"` - DefaultType string `json:"default_type"` - DocvaluesDynamic bool `json:"docvalues_dynamic"` - IndexDynamic bool `json:"index_dynamic"` - StoreDynamic bool `json:"store_dynamic"` - TypeField string `json:"type_field"` -} - -type IndexDefaultMapping struct { - Dynamic bool `json:"dynamic"` - Enabled bool `json:"enabled"` -} - -type IndexStore struct { - IndexType string `json:"indexType"` - KVStoreName string `json:"kvStoreName"` -} - -type IndexMeta struct { - Name string - SourceType string - SourceName string - DocIDPrefixDelimiter string - DocIDRegexp string - TypeField string -} - -func DefaultFullTextSearchIndexDefinition(meta IndexMeta) (*IndexDefinition, error) { - if meta.Name == "" { - return nil, errors.New("index name must set") - } - if meta.SourceType == "" { - return nil, errors.New("source type must set") - } - if meta.SourceName == "" { - return nil, errors.New("source name must set") - } - - var ftsDef = &IndexDefinition{ - Type: "fulltext-index", - Name: meta.Name, - SourceType: meta.SourceType, - SourceName: meta.SourceName, - PlanParams: IndexPlanParams{ - MaxPartitionsPerPIndex: 171, - }, - Params: IndexParams{ - Mapping: IndexMapping{ - DefaultAnalyzer: "standard", - DefaultDatetimeParser: "dateTimeOptional", - DefaultField: "_all", - DefaultMapping: IndexDefaultMapping{ - Dynamic: true, - Enabled: true, - }, - DefaultType: "_default", - DocvaluesDynamic: true, - IndexDynamic: true, - StoreDynamic: true, - TypeField: "_type", - }, - Store: IndexStore{ - IndexType: "scorch", - KVStoreName: "", - }, - }, - } - - switch { - case meta.DocIDPrefixDelimiter != "": - ftsDef.Params.DocConfig = IndexDocConfig{ - DocIDPrefixDelimiter: meta.DocIDPrefixDelimiter, - Mode: "docid_prefix", - DocIDRegexp: "", - TypeField: "", - } - case meta.DocIDRegexp != "": - ftsDef.Params.DocConfig = IndexDocConfig{ - DocIDPrefixDelimiter: "", - Mode: "docid_regexp", - DocIDRegexp: meta.DocIDRegexp, - TypeField: "", - } - case meta.TypeField != "": - ftsDef.Params.DocConfig = IndexDocConfig{ - DocIDPrefixDelimiter: "", - Mode: "type_field", - DocIDRegexp: "", - TypeField: meta.TypeField, - } - } - - return ftsDef, nil -} - -func (h *Handler) CreateFullTextSearchIndex(def *IndexDefinition) error { - body, err := json.Marshal(def) - if err != nil { - return err - } - req, _ := http.NewRequest("PUT", h.fullTestSearchURL(def.Name), bytes.NewBuffer(body)) - setupBasicAuth(req) - req.Header.Add("Content-Type", "application/json") - resp, err := h.http.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - respbody, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - - var ar apiResponse - err = json.Unmarshal(respbody, &ar) - if err != nil { - return err - } - if ar.Status == "fail" { - return errors.New(ar.Error) - } - - return nil -} - -func (h *Handler) DeleteFullTextSearchIndex(indexName string) error { - req, _ := http.NewRequest("DELETE", h.fullTestSearchURL(indexName), nil) - setupBasicAuth(req) - req.Header.Add("Content-Type", "application/json") - - resp, err := h.http.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - respbody, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - - var ar apiResponse - err = json.Unmarshal(respbody, &ar) - if err != nil { - return err - } - if ar.Status == "fail" { - return errors.New(ar.Error) - } - - return nil -} - -func (h *Handler) InspectFullTextSearchIndex(indexName string) (bool, *IndexDefinition, error) { - req, _ := http.NewRequest("GET", h.fullTestSearchURL(""), nil) - setupBasicAuth(req) - req.Header.Add("Content-Type", "application/json") - - resp, err := h.http.Do(req) - if err != nil { - return false, nil, err - } - defer resp.Body.Close() - - respbody, err := ioutil.ReadAll(resp.Body) - if err != nil { - return false, nil, err - } - var ar apiResponse - err = json.Unmarshal(respbody, &ar) - if err != nil { - return false, nil, err - } - if v, ok := ar.IndexDefs.IndexDefs[indexName]; ok { - return true, &v, nil - } - return false, nil, nil -} - -func (h *Handler) fullTestSearchURL(indexName string) string { - if indexName == "" { - return fmt.Sprintf("%s%s", h.httpAddress, ftsEndpoint) - } - return fmt.Sprintf("%s%s/%s", h.httpAddress, ftsEndpoint, indexName) -} +*/ \ No newline at end of file diff --git a/fts_index.go b/fts_index.go new file mode 100644 index 0000000..fbc7012 --- /dev/null +++ b/fts_index.go @@ -0,0 +1,242 @@ +package odatas + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" +) + +const ( + ftsEndpoint = "/_p/fts/api/index" +) + +type apiResponse struct { + Status string `json:"status"` + IndexDefs IndexDefs `json:"indexDefs"` + Error string `json:"error"` +} + +type IndexDefs struct { + UUID string `json:"uuid"` + IndexDefs map[string]IndexDefinition `json:"indexDefs"` +} + +type IndexDefinition struct { + Type string `json:"type"` + Name string `json:"name"` + SourceType string `json:"sourceType"` + SourceName string `json:"sourceName"` + PlanParams IndexPlanParams `json:"planParams"` + Params IndexParams `json:"params"` +} + +type IndexPlanParams struct { + MaxPartitionsPerPIndex int64 `json:"maxPartitionsPerPIndex"` +} + +type IndexParams struct { + DocConfig IndexDocConfig `json:"doc_config"` + Mapping IndexMapping `json:"mapping"` + Store IndexStore `json:"store"` +} + +type IndexDocConfig struct { + DocIDPrefixDelimiter string `json:"docid_prefix_delim"` + DocIDRegexp string `json:"docid_regexp"` + Mode string `json:"mode"` + TypeField string `json:"type_field"` +} + +type IndexMapping struct { + DefaultAnalyzer string `json:"default_analyzer"` + DefaultDatetimeParser string `json:"default_datetime_parser"` + DefaultField string `json:"default_field"` + DefaultMapping IndexDefaultMapping `json:"default_mapping"` + DefaultType string `json:"default_type"` + DocvaluesDynamic bool `json:"docvalues_dynamic"` + IndexDynamic bool `json:"index_dynamic"` + StoreDynamic bool `json:"store_dynamic"` + TypeField string `json:"type_field"` +} + +type IndexDefaultMapping struct { + Dynamic bool `json:"dynamic"` + Enabled bool `json:"enabled"` +} + +type IndexStore struct { + IndexType string `json:"indexType"` + KVStoreName string `json:"kvStoreName"` +} + +type IndexMeta struct { + Name string + SourceType string + SourceName string + DocIDPrefixDelimiter string + DocIDRegexp string + TypeField string +} + +func DefaultFullTextSearchIndexDefinition(meta IndexMeta) (*IndexDefinition, error) { + if meta.Name == "" { + return nil, errors.New("index name must set") + } + if meta.SourceType == "" { + return nil, errors.New("source type must set") + } + if meta.SourceName == "" { + return nil, errors.New("source name must set") + } + + var ftsDef = &IndexDefinition{ + Type: "fulltext-index", + Name: meta.Name, + SourceType: meta.SourceType, + SourceName: meta.SourceName, + PlanParams: IndexPlanParams{ + MaxPartitionsPerPIndex: 171, + }, + Params: IndexParams{ + Mapping: IndexMapping{ + DefaultAnalyzer: "standard", + DefaultDatetimeParser: "dateTimeOptional", + DefaultField: "_all", + DefaultMapping: IndexDefaultMapping{ + Dynamic: true, + Enabled: true, + }, + DefaultType: "_default", + DocvaluesDynamic: true, + IndexDynamic: true, + StoreDynamic: true, + TypeField: "_type", + }, + Store: IndexStore{ + IndexType: "scorch", + KVStoreName: "", + }, + }, + } + + switch { + case meta.DocIDPrefixDelimiter != "": + ftsDef.Params.DocConfig = IndexDocConfig{ + DocIDPrefixDelimiter: meta.DocIDPrefixDelimiter, + Mode: "docid_prefix", + DocIDRegexp: "", + TypeField: "", + } + case meta.DocIDRegexp != "": + ftsDef.Params.DocConfig = IndexDocConfig{ + DocIDPrefixDelimiter: "", + Mode: "docid_regexp", + DocIDRegexp: meta.DocIDRegexp, + TypeField: "", + } + case meta.TypeField != "": + ftsDef.Params.DocConfig = IndexDocConfig{ + DocIDPrefixDelimiter: "", + Mode: "type_field", + DocIDRegexp: "", + TypeField: meta.TypeField, + } + } + + return ftsDef, nil +} + +func (h *Handler) CreateFullTextSearchIndex(def *IndexDefinition) error { + body, err := json.Marshal(def) + if err != nil { + return err + } + req, _ := http.NewRequest("PUT", h.fullTestSearchURL(def.Name), bytes.NewBuffer(body)) + setupBasicAuth(req) + req.Header.Add("Content-Type", "application/json") + resp, err := h.http.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + respbody, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + var ar apiResponse + err = json.Unmarshal(respbody, &ar) + if err != nil { + return err + } + if ar.Status == "fail" { + return errors.New(ar.Error) + } + + return nil +} + +func (h *Handler) DeleteFullTextSearchIndex(indexName string) error { + req, _ := http.NewRequest("DELETE", h.fullTestSearchURL(indexName), nil) + setupBasicAuth(req) + req.Header.Add("Content-Type", "application/json") + + resp, err := h.http.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + respbody, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + var ar apiResponse + err = json.Unmarshal(respbody, &ar) + if err != nil { + return err + } + if ar.Status == "fail" { + return errors.New(ar.Error) + } + + return nil +} + +func (h *Handler) InspectFullTextSearchIndex(indexName string) (bool, *IndexDefinition, error) { + req, _ := http.NewRequest("GET", h.fullTestSearchURL(""), nil) + setupBasicAuth(req) + req.Header.Add("Content-Type", "application/json") + + resp, err := h.http.Do(req) + if err != nil { + return false, nil, err + } + defer resp.Body.Close() + + respbody, err := ioutil.ReadAll(resp.Body) + if err != nil { + return false, nil, err + } + var ar apiResponse + err = json.Unmarshal(respbody, &ar) + if err != nil { + return false, nil, err + } + if v, ok := ar.IndexDefs.IndexDefs[indexName]; ok { + return true, &v, nil + } + return false, nil, nil +} + +func (h *Handler) fullTestSearchURL(indexName string) string { + if indexName == "" { + return fmt.Sprintf("%s%s", h.httpAddress, ftsEndpoint) + } + return fmt.Sprintf("%s%s/%s", h.httpAddress, ftsEndpoint, indexName) +} diff --git a/fts_index_test.go b/fts_index_test.go new file mode 100644 index 0000000..2206f8b --- /dev/null +++ b/fts_index_test.go @@ -0,0 +1,29 @@ +package odatas + +import "testing" + +func TestCreateFullTextSearchIndex(t *testing.T) { + indexName := "order_fts_idx" + + h := New(&Configuration{}) + if ok, _, _ := h.InspectFullTextSearchIndex(indexName); ok { + err := h.DeleteFullTextSearchIndex(indexName) + if err != nil { + t.Fatal(err) + } + } + + def, err := DefaultFullTextSearchIndexDefinition(IndexMeta{ + Name: indexName, + SourceType: "couchbase", + SourceName: "company", + DocIDPrefixDelimiter: "::", + }) + if err != nil { + t.Fatal(err) + } + err = h.CreateFullTextSearchIndex(def) + if err != nil { + t.Fatal(err) + } +} \ No newline at end of file diff --git a/fts_test.go b/fts_test.go index eff4282..b41fa65 100644 --- a/fts_test.go +++ b/fts_test.go @@ -21,32 +21,6 @@ func TestSearchQuery(t *testing.T) { placeholderInit() } -func TestCreateFullTextSearchIndex(t *testing.T) { - indexName := "order_fts_idx" - - h := New(&Configuration{}) - if ok, _, _ := h.InspectFullTextSearchIndex(indexName); ok { - err := h.DeleteFullTextSearchIndex(indexName) - if err != nil { - t.Fatal(err) - } - } - - def, err := DefaultFullTextSearchIndexDefinition(IndexMeta{ - Name: indexName, - SourceType: "couchbase", - SourceName: "company", - DocIDPrefixDelimiter: "::", - }) - if err != nil { - t.Fatal(err) - } - err = h.CreateFullTextSearchIndex(def) - if err != nil { - t.Fatal(err) - } -} - func TestSimpleSearchMatch(t *testing.T) { placeholderInit() @@ -59,11 +33,11 @@ func TestSimpleSearchMatch(t *testing.T) { } handler := New(&Configuration{}) - searchMatch := "Talia Hudson" + searchMatch := "Talia" mes := time.Now() err := handler.SimpleSearch("order_fts_idx", &SearchQuery{ - Match: searchMatch, - Field: "CardHolderName", + Query: searchMatch, + //Field: "CardHolderName", }) fmt.Println(time.Since(mes)) if err != nil { From 92a6c5b4580872be77439b9d11f9351c43a73503 Mon Sep 17 00:00:00 2001 From: PumpkinSeed Date: Tue, 24 Sep 2019 15:14:22 +0200 Subject: [PATCH 6/6] Finish fts --- fts.go | 271 +++++++++++++++++++++++++++++++++++----------------- fts_test.go | 97 ++++++++++++++++++- 2 files changed, 276 insertions(+), 92 deletions(-) diff --git a/fts.go b/fts.go index b2ff29d..5dca7b2 100644 --- a/fts.go +++ b/fts.go @@ -30,27 +30,29 @@ type SearchQuery struct { Prefix string `json:"prefix,omitempty"` Regexp string `json:"regexp,omitempty"` Wildcard string `json:"wildcard,omitempty"` - Bool bool `json:"bool,omitempty"` Field string `json:"field,omitempty"` Analyzer string `json:"analyzer,omitempty"` Fuzziness int64 `json:"fuzziness,omitempty"` PrefixLength int64 `json:"prefix_length,omitempty"` - Limit int `json:"-"` + Limit int `json:"-"` Offset int `json:"-"` } type FacetDef struct { - Name string - Type int + Name string + Type int Field string - Size int + Size int } type CompoundQueries struct { - Conjunction []SearchQuery `json:"conjuction,omitempty"` - Disjunction []SearchQuery `json:"disjunction,omitempty"` + Conjunction []SearchQuery `json:"conjuncts,omitempty"` + Disjunction []SearchQuery `json:"disjuncts,omitempty"` + + Limit int `json:"-"` + Offset int `json:"-"` } type RangeQuery struct { @@ -68,6 +70,9 @@ type RangeQuery struct { InclusiveMax bool `json:"inclusive_max,omitempty"` Field string `json:"field,omitempty"` + + Limit int `json:"-"` + Offset int `json:"-"` } func placeholderInit() { @@ -93,80 +98,131 @@ func placeholderInit() { } } -func (h *Handler) SimpleSearch(index string, q *SearchQuery) error { +func (h *Handler) SimpleSearch(index string, q *SearchQuery) ([]gocb.SearchResultHit, error) { placeholderInit() if err := q.Setup(); err != nil { - return err + return nil, err } if index == "" { - return ErrEmptyIndex + return nil, ErrEmptyIndex } query := gocb.NewSearchQuery(index, q).Limit(q.Limit).Skip(q.Offset) - return h.doSimpleSearch(query) + hits, _, err := h.doSearch(query) + return hits, err } -func (h *Handler) SimpleSearchWithFacets(index string, q *SearchQuery, facets []FacetDef) error { +func (h *Handler) SimpleSearchWithFacets(index string, q *SearchQuery, facets []FacetDef) ([]gocb.SearchResultHit, map[string]gocb.SearchResultFacet, error) { placeholderInit() if err := q.Setup(); err != nil { - return err + return nil, nil, err } if index == "" { - return ErrEmptyIndex + return nil, nil, ErrEmptyIndex } query := gocb.NewSearchQuery(index, q).Limit(q.Limit).Skip(q.Offset) - for _, facet := range facets { - switch facet.Type { - case FacetDate: - query.AddFacet(facet.Name, cbft.NewDateFacet(facet.Field, facet.Size)) - case FacetNumeric: - query.AddFacet(facet.Name, cbft.NewNumericFacet(facet.Field, facet.Size)) - case FacetTerm: - query.AddFacet(facet.Name, cbft.NewTermFacet(facet.Field, facet.Size)) - } + h.addFacets(query, facets) + + return h.doSearch(query) +} + +func (h *Handler) CompoundSearch(index string, q *CompoundQueries) ([]gocb.SearchResultHit, error) { + if err := q.Setup(); err != nil { + return nil, err + } + + if index == "" { + return nil, ErrEmptyIndex } - return h.doSimpleSearch(query) + query := gocb.NewSearchQuery(index, q).Limit(q.Limit).Skip(q.Offset) + result, _, err := h.doSearch(query) + return result, err } -func (h *Handler) doSimpleSearch(query *gocb.SearchQuery) error { - res, err := placeholderBucket.ExecuteSearchQuery(query) - if err != nil { - return err +func (h *Handler) CompoundSearchWithFacets(index string, q *CompoundQueries, facets []FacetDef) ([]gocb.SearchResultHit, map[string]gocb.SearchResultFacet, error) { + if err := q.Setup(); err != nil { + return nil, nil, err } - fmt.Println(res.Status()) - for _, hit := range res.Hits() { - fmt.Printf("%+v\n", hit) + + if index == "" { + return nil, nil, ErrEmptyIndex } - for _, facet := range res.Facets() { - fmt.Printf("%+v\n", facet) + + query := gocb.NewSearchQuery(index, q).Limit(q.Limit).Skip(q.Offset) + h.addFacets(query, facets) + result, facetResult, err := h.doSearch(query) + return result, facetResult, err +} + +func (h *Handler) RangeSearch(index string, q *RangeQuery) ([]gocb.SearchResultHit, error) { + if err := q.Setup(); err != nil { + return nil, err } - return nil + if index == "" { + return nil, ErrEmptyIndex + } + + query := gocb.NewSearchQuery(index, q).Limit(q.Limit).Skip(q.Offset) + result, _, err := h.doSearch(query) + return result, err } -func (h *Handler) CompoundSearch(doc string, q *CompoundQueries) { +func (h *Handler) RangeSearchWithFacets(index string, q *RangeQuery, facets []FacetDef) ([]gocb.SearchResultHit, map[string]gocb.SearchResultFacet, error) { + if err := q.Setup(); err != nil { + return nil, nil, err + } + if index == "" { + return nil, nil, ErrEmptyIndex + } + + query := gocb.NewSearchQuery(index, q).Limit(q.Limit).Skip(q.Offset) + h.addFacets(query, facets) + result, facetResult, err := h.doSearch(query) + return result, facetResult, err } -func (h *Handler) RangeSearch(doc string, q *RangeQuery) { +func (h *Handler) doSearch(query *gocb.SearchQuery) ([]gocb.SearchResultHit, map[string]gocb.SearchResultFacet, error) { + res, err := placeholderBucket.ExecuteSearchQuery(query) + if err != nil { + return nil, nil, err + } + fmt.Printf("%+v\n", res.Status()) + for i, v := range res.Hits() { + fmt.Printf("%d ---- %+v\n", i, v) + } + + return res.Hits(), res.Facets(), nil +} +func (h *Handler) addFacets(query *gocb.SearchQuery, facets []FacetDef) { + for _, facet := range facets { + switch facet.Type { + case FacetDate: + query.AddFacet(facet.Name, cbft.NewDateFacet(facet.Field, facet.Size)) + case FacetNumeric: + query.AddFacet(facet.Name, cbft.NewNumericFacet(facet.Field, facet.Size)) + case FacetTerm: + query.AddFacet(facet.Name, cbft.NewTermFacet(facet.Field, facet.Size)) + } + } } func (s *SearchQuery) Setup() error { if s.Query != "" { - s.Match = emptyString() - s.MatchPhrase = emptyString() - s.Term = emptyString() - s.Prefix = emptyString() - s.Regexp = emptyString() - s.Wildcard = emptyString() - s.Bool = emptyBool() + s.Match = "" + s.MatchPhrase = "" + s.Term = "" + s.Prefix = "" + s.Regexp = "" + s.Wildcard = "" return nil } @@ -176,58 +232,95 @@ func (s *SearchQuery) Setup() error { switch { case s.Match != "": - s.MatchPhrase = emptyString() - s.Term = emptyString() - s.Prefix = emptyString() - s.Regexp = emptyString() - s.Wildcard = emptyString() - s.Bool = emptyBool() + s.Query = "" + s.MatchPhrase = "" + s.Term = "" + s.Prefix = "" + s.Regexp = "" + s.Wildcard = "" case s.MatchPhrase != "": - s.Match = emptyString() - s.Term = emptyString() - s.Prefix = emptyString() - s.Regexp = emptyString() - s.Wildcard = emptyString() - s.Bool = emptyBool() + s.Query = "" + s.Match = "" + s.Term = "" + s.Prefix = "" + s.Regexp = "" + s.Wildcard = "" case s.Term != "": - s.Match = emptyString() - s.MatchPhrase = emptyString() - s.Prefix = emptyString() - s.Regexp = emptyString() - s.Wildcard = emptyString() - s.Bool = emptyBool() + s.Query = "" + s.Match = "" + s.MatchPhrase = "" + s.Prefix = "" + s.Regexp = "" + s.Wildcard = "" case s.Prefix != "": - s.Match = emptyString() - s.MatchPhrase = emptyString() - s.Term = emptyString() - s.Regexp = emptyString() - s.Wildcard = emptyString() - s.Bool = emptyBool() + s.Query = "" + s.Match = "" + s.MatchPhrase = "" + s.Term = "" + s.Regexp = "" + s.Wildcard = "" case s.Regexp != "": - s.Match = emptyString() - s.MatchPhrase = emptyString() - s.Term = emptyString() - s.Prefix = emptyString() - s.Wildcard = emptyString() - s.Bool = emptyBool() + s.Query = "" + s.Match = "" + s.MatchPhrase = "" + s.Term = "" + s.Prefix = "" + s.Wildcard = "" case s.Wildcard != "": - s.Match = emptyString() - s.MatchPhrase = emptyString() - s.Term = emptyString() - s.Prefix = emptyString() - s.Regexp = emptyString() - s.Bool = emptyBool() - //case s.Bool.Valid: - // s.Match = emptyString() - // s.MatchPhrase = emptyString() - // s.Term = emptyString() - // s.Prefix = emptyString() - // s.Regexp = emptyString() - // s.Wildcard = emptyString() + s.Query = "" + s.Match = "" + s.MatchPhrase = "" + s.Term = "" + s.Prefix = "" + s.Regexp = "" } return nil } -/* - Index of FTS -*/ \ No newline at end of file +func (c *CompoundQueries) Setup() error { + if c.Conjunction == nil && c.Disjunction == nil { + return errors.New("") + } + + if c.Conjunction != nil { + c.Disjunction = nil + for _, sq := range c.Conjunction { + err := sq.Setup() + if err != nil { + return err + } + } + } else { + for _, sq := range c.Disjunction { + err := sq.Setup() + if err != nil { + return err + } + } + } + + return nil +} + +func (d *RangeQuery) Setup() error { + if d.Field == "" { + return ErrEmptyField + } + + if !d.StartAsTime.IsZero() { + if d.EndAsTime.IsZero() { + return errors.New("") + } + d.Start = d.StartAsTime.Format(time.RFC3339) + d.End = d.EndAsTime.Format(time.RFC3339) + + d.Min = 0 + d.Max = 0 + return nil + } + + d.Start = "" + d.End = "" + + return nil +} \ No newline at end of file diff --git a/fts_test.go b/fts_test.go index b41fa65..3ca65cf 100644 --- a/fts_test.go +++ b/fts_test.go @@ -3,10 +3,15 @@ package odatas import ( "encoding/json" "fmt" + "github.com/brianvoe/gofakeit" "testing" "time" ) +func init() { + gofakeit.Seed(time.Now().UnixNano()) +} + func TestSearchQuery(t *testing.T) { sq := SearchQuery{ Query: "card", @@ -16,9 +21,61 @@ func TestSearchQuery(t *testing.T) { if err != nil { t.Fatal(err) } - fmt.Print(string(sqjso)) + expected := `{"query":"card"}` + if string(sqjso) != expected { + t.Errorf("Query should be %s, instead of %s", expected, string(sqjso)) + } +} - placeholderInit() +func TestConjuncts(t *testing.T) { + cr := CompoundQueries{ + Conjunction: []SearchQuery{ + { + Query: "card", + }, + { + Query: "processed", + }, + }, + } + + err := cr.Setup() + if err != nil { + t.Fatal(err) + } + + json.Marshal(cr) + crjso, err := json.Marshal(cr) + if err != nil { + t.Fatal(err) + } + expected := `{"conjuncts":[{"query":"card"},{"query":"processed"}]}` + if string(crjso) != expected { + t.Errorf("Query should be %s, instead of %s", expected, string(crjso)) + } +} + +func TestRangeQuery(t *testing.T) { + rq := RangeQuery{ + StartAsTime: time.Now().Add(-2000*time.Hour), + EndAsTime: time.Now().Add(-500*time.Hour), + Field: "something", + } + + err := rq.Setup() + if err != nil { + t.Fatal(err) + } + + json.Marshal(rq) + rqjso, err := json.Marshal(rq) + if err != nil { + t.Fatal(err) + } + expected := fmt.Sprintf(`{"start":"%s","end":"%s","field":"something"}`, rq.Start, rq.End) + if string(rqjso) != expected { + t.Errorf("Query should be %s, instead of %s", expected, string(rqjso)) + } } func TestSimpleSearchMatch(t *testing.T) { @@ -35,7 +92,7 @@ func TestSimpleSearchMatch(t *testing.T) { handler := New(&Configuration{}) searchMatch := "Talia" mes := time.Now() - err := handler.SimpleSearch("order_fts_idx", &SearchQuery{ + _, err := handler.SimpleSearch("order_fts_idx", &SearchQuery{ Query: searchMatch, //Field: "CardHolderName", }) @@ -44,3 +101,37 @@ func TestSimpleSearchMatch(t *testing.T) { t.Fatal(err) } } + +func TestSimpleSearchMatchWithFacet(t *testing.T) { + placeholderInit() + + for i := 0; i < 10; i++ { + order := newTestStruct1() + _, err := placeholderBucket.Insert("order::"+order.Token, order, 0) + if err != nil { + t.Fatal(err) + } + } + + handler := New(&Configuration{}) + searchMatch := "Talia" + mes := time.Now() + _, _, err := handler.SimpleSearchWithFacets( + "order_fts_idx", + &SearchQuery{ + Query: searchMatch, + }, + []FacetDef{ + { + Name: "BillingAddressAddress1", + Type: FacetTerm, + Field: "BillingAddressAddress1", + Size: 10, + }, + }, + ) + fmt.Println(time.Since(mes)) + if err != nil { + t.Fatal(err) + } +}