From d68401eb409636883eb50469b18d5d98ea46d7d0 Mon Sep 17 00:00:00 2001 From: Mike Yoon Date: Sat, 21 Mar 2015 23:02:46 -0700 Subject: [PATCH 01/23] Add term search filter and modify search query json Modified QueryDsl to add a query parent node to filtered queries when serializing --- lib/searchfilter.go | 18 +++++++++++++++--- lib/searchquery.go | 2 +- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/lib/searchfilter.go b/lib/searchfilter.go index 82830e75..c9b2eeea 100644 --- a/lib/searchfilter.go +++ b/lib/searchfilter.go @@ -135,6 +135,7 @@ func CompoundFilter(fl ...interface{}) *FilterWrap { type FilterOp struct { curField string TermsMap map[string][]interface{} `json:"terms,omitempty"` + TermMap map[string]interface{} `json:"term,omitempty"` Range map[string]map[string]interface{} `json:"range,omitempty"` Exist map[string]string `json:"exists,omitempty"` MisssingVal map[string]string `json:"missing,omitempty"` @@ -156,6 +157,17 @@ func (f *FilterOp) Field(fld string) *FilterOp { return f } +func (f *FilterOp) Term(field string, value interface{}) *FilterOp { + //Multiple terms in a filter may not be compatible with older versions of + //ElasticSearch + if len(f.TermMap) == 0 { + f.TermMap = make(map[string]interface{}) + } + + f.TermMap[field] = value + return f +} + // Filter Terms // // Filter().Terms("user","kimchy") @@ -164,9 +176,9 @@ func (f *FilterOp) Field(fld string) *FilterOp { // Filter().Terms("user", "kimchy", "elasticsearch") // func (f *FilterOp) Terms(field string, values ...interface{}) *FilterOp { - if len(f.TermsMap) == 0 { - f.TermsMap = make(map[string][]interface{}) - } + //You can only have one terms in a filter + f.TermsMap = make(map[string][]interface{}) + for _, val := range values { f.TermsMap[field] = append(f.TermsMap[field], val) } diff --git a/lib/searchquery.go b/lib/searchquery.go index 3c2ebb8d..a3ab6408 100644 --- a/lib/searchquery.go +++ b/lib/searchquery.go @@ -85,7 +85,7 @@ func (qd *QueryDsl) MarshalJSON() ([]byte, error) { if err != nil { return filterB, err } - return []byte(fmt.Sprintf(`{"filtered":{"query":%s,"filter":%s}}`, queryB, filterB)), nil + return []byte(fmt.Sprintf(`{"query":{"filtered":{"query":%s,"filter":%s}}}`, queryB, filterB)), nil } return json.Marshal(q) } From cf98c144ec0ad34171f23257ef38e1cddc2876f7 Mon Sep 17 00:00:00 2001 From: Mike Yoon Date: Wed, 25 Mar 2015 00:12:20 -0700 Subject: [PATCH 02/23] Add and filter --- lib/searchfilter.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/searchfilter.go b/lib/searchfilter.go index c9b2eeea..387916f9 100644 --- a/lib/searchfilter.go +++ b/lib/searchfilter.go @@ -139,6 +139,7 @@ type FilterOp struct { Range map[string]map[string]interface{} `json:"range,omitempty"` Exist map[string]string `json:"exists,omitempty"` MisssingVal map[string]string `json:"missing,omitempty"` + AndFilters []FilterOp `json:"and,omitempty"` } // A range is a special type of Filter operation @@ -168,6 +169,13 @@ func (f *FilterOp) Term(field string, value interface{}) *FilterOp { return f } +func (f *FilterOp) And(filter *FilterOp) *FilterOp { + if len(f.AndFilters) == 0 { + f.AndFilters = []FilterOp{*filter} + + return f +} + // Filter Terms // // Filter().Terms("user","kimchy") From dc4e366cfa1112e3dd791261aa0cbf7bca880946 Mon Sep 17 00:00:00 2001 From: Mike Yoon Date: Wed, 25 Mar 2015 00:12:34 -0700 Subject: [PATCH 03/23] Fix and filter --- lib/searchfilter.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/searchfilter.go b/lib/searchfilter.go index 387916f9..2bd8573a 100644 --- a/lib/searchfilter.go +++ b/lib/searchfilter.go @@ -172,6 +172,9 @@ func (f *FilterOp) Term(field string, value interface{}) *FilterOp { func (f *FilterOp) And(filter *FilterOp) *FilterOp { if len(f.AndFilters) == 0 { f.AndFilters = []FilterOp{*filter} + } else { + f.AndFilters = append(f.AndFilters, *filter) + } return f } From d6236b0c252981a35c08a1c702d7c201b7b4fb88 Mon Sep 17 00:00:00 2001 From: Mike Yoon Date: Wed, 25 Mar 2015 00:19:40 -0700 Subject: [PATCH 04/23] Add or filter --- lib/searchfilter.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/searchfilter.go b/lib/searchfilter.go index 2bd8573a..364e6ecf 100644 --- a/lib/searchfilter.go +++ b/lib/searchfilter.go @@ -140,6 +140,7 @@ type FilterOp struct { Exist map[string]string `json:"exists,omitempty"` MisssingVal map[string]string `json:"missing,omitempty"` AndFilters []FilterOp `json:"and,omitempty"` + OrFilters []FilterOp `json:"or,omitempty"` } // A range is a special type of Filter operation @@ -173,8 +174,18 @@ func (f *FilterOp) And(filter *FilterOp) *FilterOp { if len(f.AndFilters) == 0 { f.AndFilters = []FilterOp{*filter} } else { - f.AndFilters = append(f.AndFilters, *filter) - } + f.AndFilters = append(f.AndFilters, *filter) + } + + return f +} + +func (f *FilterOp) Or(filter *FilterOp) *FilterOp { + if len(f.OrFilters) == 0 { + f.OrFilters = []FilterOp{*filter} + } else { + f.OrFilters = append(f.OrFilters, *filter) + } return f } From 736af2bf71bffe82b9c02c2813e8b6ac623ec639 Mon Sep 17 00:00:00 2001 From: Mike Yoon Date: Thu, 26 Mar 2015 00:27:57 -0700 Subject: [PATCH 05/23] Cleanup filter dsl a bit * Added an AddRange function for creating a range filter * Change QueryDsl to return a query root node when there's no filter --- lib/searchfilter.go | 98 ++++++++++++++++++--------------------------- lib/searchquery.go | 6 ++- 2 files changed, 43 insertions(+), 61 deletions(-) diff --git a/lib/searchfilter.go b/lib/searchfilter.go index 364e6ecf..3bcbb0ec 100644 --- a/lib/searchfilter.go +++ b/lib/searchfilter.go @@ -134,34 +134,38 @@ func CompoundFilter(fl ...interface{}) *FilterWrap { type FilterOp struct { curField string - TermsMap map[string][]interface{} `json:"terms,omitempty"` - TermMap map[string]interface{} `json:"term,omitempty"` - Range map[string]map[string]interface{} `json:"range,omitempty"` - Exist map[string]string `json:"exists,omitempty"` - MisssingVal map[string]string `json:"missing,omitempty"` - AndFilters []FilterOp `json:"and,omitempty"` - OrFilters []FilterOp `json:"or,omitempty"` + TermsMap map[string][]interface{} `json:"terms,omitempty"` + TermMap map[string]interface{} `json:"term,omitempty"` + RangeMap map[string]RangeFilter `json:"range,omitempty"` + Exist map[string]string `json:"exists,omitempty"` + MisssingVal map[string]string `json:"missing,omitempty"` + AndFilters []FilterOp `json:"and,omitempty"` + OrFilters []FilterOp `json:"or,omitempty"` + Limit *LimitFilter `json:"limit,omitempty"` +} + +type LimitFilter struct { + Value int `json:"value,omitempty"` +} + +type RangeFilter struct { + Gte interface{} `json:"gte,omitempty"` + Lte interface{} `json:"lte,omitempty"` + Gt interface{} `json:"gt,omitempty"` + Lt interface{} `json:"lt,omitempty"` + TimeZone string `json:"time_zone,omitempty"` //Ideally this would be an int } // A range is a special type of Filter operation // // Range().Exists("repository.name") func Range() *FilterOp { - return &FilterOp{Range: make(map[string]map[string]interface{})} -} - -func (f *FilterOp) Field(fld string) *FilterOp { - f.curField = fld - if _, ok := f.Range[fld]; !ok { - m := make(map[string]interface{}) - f.Range[fld] = m - } - return f + return &FilterOp{RangeMap: make(map[string]RangeFilter)} } +// Term will add a term to the filter. +// Multiple Term filters can be added, and ES will OR them. func (f *FilterOp) Term(field string, value interface{}) *FilterOp { - //Multiple terms in a filter may not be compatible with older versions of - //ElasticSearch if len(f.TermMap) == 0 { f.TermMap = make(map[string]interface{}) } @@ -192,11 +196,8 @@ func (f *FilterOp) Or(filter *FilterOp) *FilterOp { // Filter Terms // -// Filter().Terms("user","kimchy") -// -// // we use variadics to allow n arguments, first is the "field" rest are values -// Filter().Terms("user", "kimchy", "elasticsearch") -// +// Filter().Terms("user","kimchy","stuff") +// Note: you can only have one terms clause in a filter. Use a bool filter to combine func (f *FilterOp) Terms(field string, values ...interface{}) *FilterOp { //You can only have one terms in a filter f.TermsMap = make(map[string][]interface{}) @@ -207,42 +208,21 @@ func (f *FilterOp) Terms(field string, values ...interface{}) *FilterOp { return f } -func (f *FilterOp) From(from string) *FilterOp { - f.Range[f.curField]["from"] = from - return f -} -func (f *FilterOp) To(to string) *FilterOp { - f.Range[f.curField]["to"] = to - return f -} -func (f *FilterOp) Gt(gt interface{}) *FilterOp { - f.Range[f.curField]["gt"] = gt - return f -} -func (f *FilterOp) Lt(lt interface{}) *FilterOp { - f.Range[f.curField]["lt"] = lt - return f -} -func (f *FilterOp) Exists(name string) *FilterOp { - f.Exist = map[string]string{"field": name} - return f -} -func (f *FilterOp) Missing(name string) *FilterOp { - f.MisssingVal = map[string]string{"field": name} - return f -} -// Add another Filterop, "combines" two filter ops into one -func (f *FilterOp) Add(fop *FilterOp) *FilterOp { - // TODO, this is invalid, refactor - if len(fop.Exist) > 0 { - f.Exist = fop.Exist - } - if len(fop.MisssingVal) > 0 { - f.MisssingVal = fop.MisssingVal - } - if len(fop.Range) > 0 { - f.Range = fop.Range +// AddRange adds a range filter for the given field. +func (f *FilterOp) AddRange(field string, gte interface{}, + gt interface{}, lte interface{}, lt interface{}, timeZone string) *FilterOp { + + if f.RangeMap == nil { + f.RangeMap = make(map[string]RangeFilter) } + + f.RangeMap[field] = RangeFilter{ + Gte: gte, + Gt: gt, + Lte: lte, + Lt: lt, + TimeZone: timeZone} + return f } diff --git a/lib/searchquery.go b/lib/searchquery.go index a3ab6408..489d6cd4 100644 --- a/lib/searchquery.go +++ b/lib/searchquery.go @@ -87,7 +87,9 @@ func (qd *QueryDsl) MarshalJSON() ([]byte, error) { } return []byte(fmt.Sprintf(`{"query":{"filtered":{"query":%s,"filter":%s}}}`, queryB, filterB)), nil } - return json.Marshal(q) + + retval, err := json.Marshal(q) + return []byte(fmt.Sprintf(`{"query": %s}`, retval)), err } // get all @@ -103,7 +105,7 @@ func (q *QueryDsl) Range(fop *FilterOp) *QueryDsl { return q } // TODO: this is not valid, refactor - q.FilterVal.Add(fop) + //q.FilterVal.Add(fop) return q } From 64df646c43f42f750bd81cc7304d750c5f044344 Mon Sep 17 00:00:00 2001 From: Mike Yoon Date: Thu, 26 Mar 2015 00:49:25 -0700 Subject: [PATCH 06/23] Remove the extra root query node --- lib/searchquery.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/searchquery.go b/lib/searchquery.go index 489d6cd4..1c7028a0 100644 --- a/lib/searchquery.go +++ b/lib/searchquery.go @@ -85,11 +85,9 @@ func (qd *QueryDsl) MarshalJSON() ([]byte, error) { if err != nil { return filterB, err } - return []byte(fmt.Sprintf(`{"query":{"filtered":{"query":%s,"filter":%s}}}`, queryB, filterB)), nil + return []byte(fmt.Sprintf(`{"filtered":{"query":%s,"filter":%s}}`, queryB, filterB)), nil } - - retval, err := json.Marshal(q) - return []byte(fmt.Sprintf(`{"query": %s}`, retval)), err + return json.Marshal(q) } // get all From ce50bb17b90c25e9f0308575902569d550f81416 Mon Sep 17 00:00:00 2001 From: Mike Yoon Date: Thu, 26 Mar 2015 23:23:14 -0700 Subject: [PATCH 07/23] Add limit to filterop and lenient to querydsl --- lib/searchfilter.go | 7 ++++++- lib/searchquery.go | 8 +++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/searchfilter.go b/lib/searchfilter.go index 3bcbb0ec..0d85bbb2 100644 --- a/lib/searchfilter.go +++ b/lib/searchfilter.go @@ -145,7 +145,7 @@ type FilterOp struct { } type LimitFilter struct { - Value int `json:"value,omitempty"` + Value int `json:"value"` } type RangeFilter struct { @@ -226,3 +226,8 @@ func (f *FilterOp) AddRange(field string, gte interface{}, return f } + +func (f *FilterOp) SetLimit(maxResults int) *FilterOp { + f.Limit = &LimitFilter{Value: maxResults} + return f +} diff --git a/lib/searchquery.go b/lib/searchquery.go index 1c7028a0..555c464f 100644 --- a/lib/searchquery.go +++ b/lib/searchquery.go @@ -132,6 +132,11 @@ func (q *QueryDsl) Qs(qs *QueryString) *QueryDsl { return q } +func (q *QueryDsl) SetLenient(lenient bool) *QueryDsl { + q.QueryEmbed.Qs.Lenient = lenient + return q +} + // Fields in query_string search // Fields("fieldname","search_for","","") // @@ -170,7 +175,7 @@ type QueryWrap struct { // QueryString based search func NewQueryString(field, query string) QueryString { - return QueryString{"", field, query, "", "", nil} + return QueryString{"", field, query, "", "", nil, false} } type QueryString struct { @@ -180,6 +185,7 @@ type QueryString struct { Exists string `json:"_exists_,omitempty"` Missing string `json:"_missing_,omitempty"` Fields []string `json:"fields,omitempty"` + Lenient bool `json:"lenient,omitempty"` //_exists_:field1, //_missing_:field1, } From 963428c868bd3b92a6eaa4462ad9ee6c66568a32 Mon Sep 17 00:00:00 2001 From: Mike Yoon Date: Tue, 31 Mar 2015 19:02:49 -0700 Subject: [PATCH 08/23] Add more filters to the filter struct --- lib/searchfilter.go | 49 +++++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/lib/searchfilter.go b/lib/searchfilter.go index 0d85bbb2..b48eabd5 100644 --- a/lib/searchfilter.go +++ b/lib/searchfilter.go @@ -14,7 +14,6 @@ package elastigo import ( "encoding/json" "fmt" - . "github.com/araddon/gou" ) @@ -133,21 +132,40 @@ func CompoundFilter(fl ...interface{}) *FilterWrap { } type FilterOp struct { - curField string - TermsMap map[string][]interface{} `json:"terms,omitempty"` - TermMap map[string]interface{} `json:"term,omitempty"` - RangeMap map[string]RangeFilter `json:"range,omitempty"` - Exist map[string]string `json:"exists,omitempty"` - MisssingVal map[string]string `json:"missing,omitempty"` - AndFilters []FilterOp `json:"and,omitempty"` - OrFilters []FilterOp `json:"or,omitempty"` - Limit *LimitFilter `json:"limit,omitempty"` + TermsMap map[string][]interface{} `json:"terms,omitempty"` + TermMap map[string]interface{} `json:"term,omitempty"` + RangeMap map[string]RangeFilter `json:"range,omitempty"` + ExistMap map[string]string `json:"exists,omitempty"` + MissingMap map[string]string `json:"missing,omitempty"` + AndFilters []FilterOp `json:"and,omitempty"` + OrFilters []FilterOp `json:"or,omitempty"` + NotFilters []FilterOp `json:"not,omitempty"` + Limit *LimitFilter `json:"limit,omitempty"` + Type *TypeFilter `json:"type,omitempty"` + Ids *IdFilter `json:"ids,omitempty"` + Script *ScriptFilter `json:"script,omitempty"` + GeoDistance *GeoDistanceFilter `json"geo_distance,omitempty"` } type LimitFilter struct { Value int `json:"value"` } +type TypeFilter struct { + Value string `json:"value"` +} + +type IdFilter struct { + Type string `json:"type,omitempty"` + Values []string `json:"values,omitempty"` +} + +type ScriptFilter struct { + Script string `json:"script"` + Params map[string]interface{} `json:"params,omitempty"` + IsCached bool `json:"_cache,omitempty"` +} + type RangeFilter struct { Gte interface{} `json:"gte,omitempty"` Lte interface{} `json:"lte,omitempty"` @@ -156,11 +174,8 @@ type RangeFilter struct { TimeZone string `json:"time_zone,omitempty"` //Ideally this would be an int } -// A range is a special type of Filter operation -// -// Range().Exists("repository.name") -func Range() *FilterOp { - return &FilterOp{RangeMap: make(map[string]RangeFilter)} +type GeoDistanceFilter struct { + Distance string `json:"distance"` } // Term will add a term to the filter. @@ -209,8 +224,8 @@ func (f *FilterOp) Terms(field string, values ...interface{}) *FilterOp { return f } -// AddRange adds a range filter for the given field. -func (f *FilterOp) AddRange(field string, gte interface{}, +// Range adds a range filter for the given field. +func (f *FilterOp) Range(field string, gte interface{}, gt interface{}, lte interface{}, lt interface{}, timeZone string) *FilterOp { if f.RangeMap == nil { From 967d7c756418344521104f96903d4fc7a776a8b9 Mon Sep 17 00:00:00 2001 From: Mike Yoon Date: Wed, 1 Apr 2015 19:53:27 -0700 Subject: [PATCH 09/23] Add geodistance filters --- lib/searchfilter.go | 66 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 15 deletions(-) diff --git a/lib/searchfilter.go b/lib/searchfilter.go index b48eabd5..61fb65ae 100644 --- a/lib/searchfilter.go +++ b/lib/searchfilter.go @@ -132,19 +132,20 @@ func CompoundFilter(fl ...interface{}) *FilterWrap { } type FilterOp struct { - TermsMap map[string][]interface{} `json:"terms,omitempty"` - TermMap map[string]interface{} `json:"term,omitempty"` - RangeMap map[string]RangeFilter `json:"range,omitempty"` - ExistMap map[string]string `json:"exists,omitempty"` - MissingMap map[string]string `json:"missing,omitempty"` - AndFilters []FilterOp `json:"and,omitempty"` - OrFilters []FilterOp `json:"or,omitempty"` - NotFilters []FilterOp `json:"not,omitempty"` - Limit *LimitFilter `json:"limit,omitempty"` - Type *TypeFilter `json:"type,omitempty"` - Ids *IdFilter `json:"ids,omitempty"` - Script *ScriptFilter `json:"script,omitempty"` - GeoDistance *GeoDistanceFilter `json"geo_distance,omitempty"` + TermsMap map[string][]interface{} `json:"terms,omitempty"` + TermMap map[string]interface{} `json:"term,omitempty"` + RangeMap map[string]RangeFilter `json:"range,omitempty"` + ExistMap map[string]string `json:"exists,omitempty"` + MissingMap map[string]string `json:"missing,omitempty"` + AndFilters []FilterOp `json:"and,omitempty"` + OrFilters []FilterOp `json:"or,omitempty"` + NotFilters []FilterOp `json:"not,omitempty"` + Limit *LimitFilter `json:"limit,omitempty"` + Type *TypeFilter `json:"type,omitempty"` + Ids *IdFilter `json:"ids,omitempty"` + Script *ScriptFilter `json:"script,omitempty"` + GeoDist map[string]interface{} `json:"geo_distance,omitempty"` + GeoDistRange map[string]interface{} `json:"geo_distance_range,omitempty"` } type LimitFilter struct { @@ -174,8 +175,14 @@ type RangeFilter struct { TimeZone string `json:"time_zone,omitempty"` //Ideally this would be an int } -type GeoDistanceFilter struct { - Distance string `json:"distance"` +type GeoLocation struct { + Latitude float32 `json:"lat"` + Longitude float32 `json:"lon"` +} + +type GeoField struct { + GeoLocation + Field string } // Term will add a term to the filter. @@ -209,6 +216,35 @@ func (f *FilterOp) Or(filter *FilterOp) *FilterOp { return f } +func (f *FilterOp) GeoDistance(distance string, fields ...GeoField) *FilterOp { + f.GeoDist = make(map[string]interface{}) + f.GeoDist["distance"] = distance + for _, val := range fields { + f.GeoDist[val.Field] = val.GeoLocation + } + + return f +} + +func (f *FilterOp) GeoDistanceRange(from string, to string, fields ...GeoField) *FilterOp { + f.GeoDist = make(map[string]interface{}) + f.GeoDist["from"] = from + f.GeoDist["to"] = to + + for _, val := range fields { + f.GeoDist[val.Field] = val.GeoLocation + } + + return f +} + +// Helper to create values for the GeoDistance filters +func NewGeoField(field string, latitude float32, longitude float32) GeoField { + return GeoField{ + GeoLocation: GeoLocation{Latitude: latitude, Longitude: longitude}, + Field: field} +} + // Filter Terms // // Filter().Terms("user","kimchy","stuff") From 61a34658f8e2a2847f8ec6f10d8de527d842f505 Mon Sep 17 00:00:00 2001 From: Mike Yoon Date: Thu, 2 Apr 2015 01:09:27 -0700 Subject: [PATCH 10/23] Add exists and missing filter create functions Also add execution mode to terms filter create Update searchfilter_test to match new Dsl --- lib/searchfilter.go | 35 ++++++++++++++++++++++++++++++++--- lib/searchfilter_test.go | 20 ++++++++++---------- lib/searchsearch.go | 6 +++--- 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/lib/searchfilter.go b/lib/searchfilter.go index 61fb65ae..3b4adbd7 100644 --- a/lib/searchfilter.go +++ b/lib/searchfilter.go @@ -24,6 +24,17 @@ var ( // A bool (and/or) clause type BoolClause string +type TermExecutionMode string + +const ( + TEM_DEFAULT TermExecutionMode = "" + TEM_PLAIN = "plain" + TEM_FIELD = "field_data" + TEM_BOOL = "bool" + TEM_AND = "and" + TEM_OR = "or" +) + // Filter clause is either a boolClause or FilterOp type FilterClause interface { String() string @@ -135,8 +146,8 @@ type FilterOp struct { TermsMap map[string][]interface{} `json:"terms,omitempty"` TermMap map[string]interface{} `json:"term,omitempty"` RangeMap map[string]RangeFilter `json:"range,omitempty"` - ExistMap map[string]string `json:"exists,omitempty"` - MissingMap map[string]string `json:"missing,omitempty"` + ExistsProp *PropertyPathMarker `json:"exists,omitempty"` + MissingProp *PropertyPathMarker `json:"missing,omitempty"` AndFilters []FilterOp `json:"and,omitempty"` OrFilters []FilterOp `json:"or,omitempty"` NotFilters []FilterOp `json:"not,omitempty"` @@ -148,6 +159,10 @@ type FilterOp struct { GeoDistRange map[string]interface{} `json:"geo_distance_range,omitempty"` } +type PropertyPathMarker struct { + Field string `json:"field"` +} + type LimitFilter struct { Value int `json:"value"` } @@ -249,10 +264,14 @@ func NewGeoField(field string, latitude float32, longitude float32) GeoField { // // Filter().Terms("user","kimchy","stuff") // Note: you can only have one terms clause in a filter. Use a bool filter to combine -func (f *FilterOp) Terms(field string, values ...interface{}) *FilterOp { +func (f *FilterOp) Terms(field string, executionMode TermExecutionMode, values ...interface{}) *FilterOp { //You can only have one terms in a filter f.TermsMap = make(map[string][]interface{}) + if executionMode != "" { + f.TermsMap["execution"] = executionMode + } + for _, val := range values { f.TermsMap[field] = append(f.TermsMap[field], val) } @@ -278,6 +297,16 @@ func (f *FilterOp) Range(field string, gte interface{}, return f } +func (f *FilterOp) Exists(field string) *FilterOp { + f.ExistsProp = &PropertyPathMarker{Field: field} + return f +} + +func (f *FilterOp) Missing(field string) *FilterOp { + f.MissingProp = &PropertyPathMarker{Field: field} + return f +} + func (f *FilterOp) SetLimit(maxResults int) *FilterOp { f.Limit = &LimitFilter{Value: maxResults} return f diff --git a/lib/searchfilter_test.go b/lib/searchfilter_test.go index 7f533891..384d04a2 100644 --- a/lib/searchfilter_test.go +++ b/lib/searchfilter_test.go @@ -41,7 +41,7 @@ func TestFilters(t *testing.T) { //actor_attributes: {type: "User", qry = Search("github").Filter( - Filter().Terms("actor_attributes.location", "portland"), + Filter().Terms("actor_attributes.location", TEM_DEFAULT, "portland"), ) out, _ = qry.Result(c) expectedDocs = 10 @@ -53,8 +53,8 @@ func TestFilters(t *testing.T) { Should this be an AND by default? */ qry = Search("github").Filter( - Filter().Terms("actor_attributes.location", "portland"), - Filter().Terms("repository.has_wiki", true), + Filter().Terms("actor_attributes.location", TEM_DEFAULT, "portland"), + Filter().Terms("repository.has_wiki", TEM_DEFAULT, true), ) out, err = qry.Result(c) expectedDocs = 10 @@ -65,10 +65,10 @@ func TestFilters(t *testing.T) { // NOW, lets try with two query calls instead of one qry = Search("github").Filter( - Filter().Terms("actor_attributes.location", "portland"), + Filter().Terms("actor_attributes.location", TEM_DEFAULT, "portland"), ) qry.Filter( - Filter().Terms("repository.has_wiki", true), + Filter().Terms("repository.has_wiki", TEM_DEFAULT, true), ) out, err = qry.Result(c) //gou.Debug(out) @@ -78,8 +78,8 @@ func TestFilters(t *testing.T) { qry = Search("github").Filter( "or", - Filter().Terms("actor_attributes.location", "portland"), - Filter().Terms("repository.has_wiki", true), + Filter().Terms("actor_attributes.location", TEM_DEFAULT, "portland"), + Filter().Terms("repository.has_wiki", TEM_DEFAULT, true), ) out, err = qry.Result(c) expectedHits = 6676 @@ -92,9 +92,9 @@ func TestFilterRange(t *testing.T) { c := NewTestConn() // now lets filter range for repositories with more than 100 forks - out, _ := Search("github").Size("25").Filter( - Range().Field("repository.forks").From("100"), - ).Result(c) + out, _ := Search("github").Size("25").Filter(Filter(). + Range("repository.forks", 100, nil, nil, nil, "")).Result(c) + if out == nil || &out.Hits == nil { t.Fail() return diff --git a/lib/searchsearch.go b/lib/searchsearch.go index 820f23d9..bbab5d14 100644 --- a/lib/searchsearch.go +++ b/lib/searchsearch.go @@ -47,7 +47,7 @@ type SearchDsl struct { FacetVal *FacetDsl `json:"facets,omitempty"` QueryVal *QueryDsl `json:"query,omitempty"` SortBody []*SortDsl `json:"sort,omitempty"` - FilterVal *FilterWrap `json:"filter,omitempty"` + FilterVal *FilterOp `json:"filter,omitempty"` AggregatesVal map[string]*AggregateDsl `json:"aggregations,omitempty"` } @@ -173,12 +173,12 @@ func (s *SearchDsl) Query(q *QueryDsl) *SearchDsl { // Filter().Exists("repository.name"), // Filter().Terms("repository.has_wiki", true) // ) -func (s *SearchDsl) Filter(fl ...interface{}) *SearchDsl { +func (s *SearchDsl) Filter(fl FilterOp) *SearchDsl { if s.FilterVal == nil { s.FilterVal = NewFilterWrap() } - s.FilterVal.addFilters(fl) + s.FilterVal = fl return s } From aa538426af99b59ccc68a79d4fdff4f66cdaadec Mon Sep 17 00:00:00 2001 From: Mike Yoon Date: Thu, 2 Apr 2015 01:18:18 -0700 Subject: [PATCH 11/23] Fix Terms filter and compile error --- lib/searchsearch.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/searchsearch.go b/lib/searchsearch.go index bbab5d14..731d1e6a 100644 --- a/lib/searchsearch.go +++ b/lib/searchsearch.go @@ -173,11 +173,7 @@ func (s *SearchDsl) Query(q *QueryDsl) *SearchDsl { // Filter().Exists("repository.name"), // Filter().Terms("repository.has_wiki", true) // ) -func (s *SearchDsl) Filter(fl FilterOp) *SearchDsl { - if s.FilterVal == nil { - s.FilterVal = NewFilterWrap() - } - +func (s *SearchDsl) Filter(fl *FilterOp) *SearchDsl { s.FilterVal = fl return s } From c394b750b43b433550e760ae194601096cf0c30f Mon Sep 17 00:00:00 2001 From: Mike Yoon Date: Thu, 2 Apr 2015 01:19:16 -0700 Subject: [PATCH 12/23] Forgot this file change for the last commit --- lib/searchfilter.go | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/lib/searchfilter.go b/lib/searchfilter.go index 3b4adbd7..d8d07082 100644 --- a/lib/searchfilter.go +++ b/lib/searchfilter.go @@ -143,20 +143,20 @@ func CompoundFilter(fl ...interface{}) *FilterWrap { } type FilterOp struct { - TermsMap map[string][]interface{} `json:"terms,omitempty"` - TermMap map[string]interface{} `json:"term,omitempty"` - RangeMap map[string]RangeFilter `json:"range,omitempty"` - ExistsProp *PropertyPathMarker `json:"exists,omitempty"` - MissingProp *PropertyPathMarker `json:"missing,omitempty"` - AndFilters []FilterOp `json:"and,omitempty"` - OrFilters []FilterOp `json:"or,omitempty"` - NotFilters []FilterOp `json:"not,omitempty"` - Limit *LimitFilter `json:"limit,omitempty"` - Type *TypeFilter `json:"type,omitempty"` - Ids *IdFilter `json:"ids,omitempty"` - Script *ScriptFilter `json:"script,omitempty"` - GeoDist map[string]interface{} `json:"geo_distance,omitempty"` - GeoDistRange map[string]interface{} `json:"geo_distance_range,omitempty"` + TermsMap map[string]interface{} `json:"terms,omitempty"` + TermMap map[string]interface{} `json:"term,omitempty"` + RangeMap map[string]RangeFilter `json:"range,omitempty"` + ExistsProp *PropertyPathMarker `json:"exists,omitempty"` + MissingProp *PropertyPathMarker `json:"missing,omitempty"` + AndFilters []FilterOp `json:"and,omitempty"` + OrFilters []FilterOp `json:"or,omitempty"` + NotFilters []FilterOp `json:"not,omitempty"` + Limit *LimitFilter `json:"limit,omitempty"` + Type *TypeFilter `json:"type,omitempty"` + Ids *IdFilter `json:"ids,omitempty"` + Script *ScriptFilter `json:"script,omitempty"` + GeoDist map[string]interface{} `json:"geo_distance,omitempty"` + GeoDistRange map[string]interface{} `json:"geo_distance_range,omitempty"` } type PropertyPathMarker struct { @@ -266,15 +266,13 @@ func NewGeoField(field string, latitude float32, longitude float32) GeoField { // Note: you can only have one terms clause in a filter. Use a bool filter to combine func (f *FilterOp) Terms(field string, executionMode TermExecutionMode, values ...interface{}) *FilterOp { //You can only have one terms in a filter - f.TermsMap = make(map[string][]interface{}) + f.TermsMap = make(map[string]interface{}) if executionMode != "" { f.TermsMap["execution"] = executionMode } - for _, val := range values { - f.TermsMap[field] = append(f.TermsMap[field], val) - } + f.TermsMap[field] = values return f } From 7d9074ce2849ad5df1af6184cbf46c99cdb085da Mon Sep 17 00:00:00 2001 From: Mike Yoon Date: Fri, 3 Apr 2015 19:56:30 -0700 Subject: [PATCH 13/23] Update tests to match new filter API --- lib/searchaggregate_test.go | 2 +- lib/searchfilter_test.go | 10 ++++------ lib/searchsearch_test.go | 6 +++--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/lib/searchaggregate_test.go b/lib/searchaggregate_test.go index 984b5d57..5fbf1f32 100644 --- a/lib/searchaggregate_test.go +++ b/lib/searchaggregate_test.go @@ -116,7 +116,7 @@ func TestAggregateFilter(t *testing.T) { avg := Aggregate("avg_price").Avg("price") dateAgg := Aggregate("in_stock_products").Filter( - Range().Field("stock").Gt(0), + Filter().Range("stock", nil, 0, nil, nil, ""), ) dateAgg.Aggregates( diff --git a/lib/searchfilter_test.go b/lib/searchfilter_test.go index 384d04a2..5b265844 100644 --- a/lib/searchfilter_test.go +++ b/lib/searchfilter_test.go @@ -53,9 +53,8 @@ func TestFilters(t *testing.T) { Should this be an AND by default? */ qry = Search("github").Filter( - Filter().Terms("actor_attributes.location", TEM_DEFAULT, "portland"), - Filter().Terms("repository.has_wiki", TEM_DEFAULT, true), - ) + Filter().And(Filter().Terms("actor_attributes.location", TEM_DEFAULT, "portland")). + And(Filter().Terms("repository.has_wiki", TEM_DEFAULT, true))) out, err = qry.Result(c) expectedDocs = 10 expectedHits = 44 @@ -77,9 +76,8 @@ func TestFilters(t *testing.T) { assert.T(t, out.Hits.Total == expectedHits, fmt.Sprintf("Should have %v total got %v", expectedHits, out.Hits.Total)) qry = Search("github").Filter( - "or", - Filter().Terms("actor_attributes.location", TEM_DEFAULT, "portland"), - Filter().Terms("repository.has_wiki", TEM_DEFAULT, true), + Filter().Or(Filter().Terms("actor_attributes.location", TEM_DEFAULT, "portland")). + Or(Filter().Terms("repository.has_wiki", TEM_DEFAULT, true)), ) out, err = qry.Result(c) expectedHits = 6676 diff --git a/lib/searchsearch_test.go b/lib/searchsearch_test.go index 4c44fd9d..9613119d 100644 --- a/lib/searchsearch_test.go +++ b/lib/searchsearch_test.go @@ -194,7 +194,7 @@ func TestSearchFacetRange(t *testing.T) { Facet().Fields("actor").Size("500"), ).Query( Query().Range( - Range().Field("created_at").From("2012-12-10T15:00:00-08:00").To("2012-12-10T15:10:00-08:00"), + Filter().Range("created_at", "2012-12-10T15:00:00-08:00", nil, "2012-12-10T15:10:00-08:00", nil, ""), ).Search("add"), ) out, err = qry.Result(c) @@ -276,7 +276,7 @@ func TestSearchFilterQuery(t *testing.T) { out, _ := Search("github").Size("25").Query( Query().Fields("repository.name", "jas*", "", ""), ).Filter( - Filter().Terms("repository.has_wiki", true), + Filter().Terms("repository.has_wiki", TEM_DEFAULT, true), ).Result(c) if out == nil || &out.Hits == nil { t.Fail() @@ -295,7 +295,7 @@ func TestSearchRange(t *testing.T) { // now lets filter by a subset of the total time out, _ := Search("github").Size("25").Query( Query().Range( - Range().Field("created_at").From("2012-12-10T15:00:00-08:00").To("2012-12-10T15:10:00-08:00"), + Filter().Range("created_at", "2012-12-10T15:00:00-08:00", nil, "2012-12-10T15:10:00-08:00", nil, ""), ).Search("add"), ).Result(c) assert.T(t, out != nil && &out.Hits != nil, "Must not have nil results, or hits") From 99c2cb87f867e5112986a913dede76c8a0d9fc9a Mon Sep 17 00:00:00 2001 From: Mike Yoon Date: Fri, 3 Apr 2015 19:56:54 -0700 Subject: [PATCH 14/23] Update vagrant image and use apt-get to install es --- Vagrantfile | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 860e02e0..ad8fc4cb 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -2,10 +2,19 @@ # vi: set ft=ruby : Vagrant.configure("2") do |config| - config.vm.box = "lucid64" - config.vm.box_url = "http://files.vagrantup.com/lucid64.box" + config.vm.box = "ubuntu/trusty64" + config.vm.network :forwarded_port, guest: 9200, host: 9200, auto_correct: true config.vm.network :forwarded_port, guest: 9300, host: 9300, auto_correct: true - config.vm.provision :shell, :inline => "gem install chef --version 10.26.0 --no-rdoc --no-ri --conservative" + #config.vm.provision :shell, :inline => "curl -L https://www.chef.io/chef/install.sh | sudo bash" + + config.vm.provision "shell", inline: <<-SHELL + wget -qO - https://packages.elasticsearch.org/GPG-KEY-elasticsearch | sudo apt-key add - + sudo apt-add-repository 'deb http://packages.elasticsearch.org/elasticsearch/1.4/debian stable main' + sudo apt-get update + sudo apt-get install -y openjdk-7-jre + sudo apt-get install -y elasticsearch + sudo /etc/init.d/elasticsearch start + SHELL config.vm.provider :virtualbox do |vb| vb.gui = false @@ -16,14 +25,14 @@ Vagrant.configure("2") do |config| # be enabled by default depending on what version of VirtualBox is used. vb.customize ["setextradata", :id, "VBoxInternal2/SharedFoldersEnableSymlinksCreate/v-root", "1"] end - config.vm.provision :chef_solo do |chef| - chef.cookbooks_path = "cookbooks" - chef.add_recipe("apt") - chef.add_recipe("java") - chef.add_recipe("elasticsearch") - chef.add_recipe("git") - chef.add_recipe("mercurial") - chef.add_recipe("build-essential") - chef.add_recipe("golang") - end + #config.vm.provision :chef_solo do |chef| + # chef.cookbooks_path = "cookbooks" + # chef.add_recipe("apt") + # chef.add_recipe("java") + # chef.add_recipe("elasticsearch") + # chef.add_recipe("git") + # chef.add_recipe("mercurial") + # chef.add_recipe("build-essential") + # chef.add_recipe("golang") + #end end \ No newline at end of file From 628e61a666441bb4eee1ac01e2bee2f48b49c080 Mon Sep 17 00:00:00 2001 From: Mike Yoon Date: Fri, 3 Apr 2015 20:06:45 -0700 Subject: [PATCH 15/23] Fix a terms query test --- lib/searchfilter_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/searchfilter_test.go b/lib/searchfilter_test.go index 5b265844..b4e99371 100644 --- a/lib/searchfilter_test.go +++ b/lib/searchfilter_test.go @@ -64,11 +64,11 @@ func TestFilters(t *testing.T) { // NOW, lets try with two query calls instead of one qry = Search("github").Filter( - Filter().Terms("actor_attributes.location", TEM_DEFAULT, "portland"), - ) - qry.Filter( - Filter().Terms("repository.has_wiki", TEM_DEFAULT, true), + Filter(). + And(Filter().Terms("actor_attributes.location", TEM_DEFAULT, "portland")). + And(Filter().Terms("repository.has_wiki", TEM_DEFAULT, true)), ) + out, err = qry.Result(c) //gou.Debug(out) assert.T(t, err == nil, t, "should not have error") From 5e43987e65c08e50b3460da63b0930c468a01d74 Mon Sep 17 00:00:00 2001 From: Mike Yoon Date: Sat, 4 Apr 2015 02:32:45 -0700 Subject: [PATCH 16/23] Start adding filter dsl tests --- lib/searchfilter_test.go | 50 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/lib/searchfilter_test.go b/lib/searchfilter_test.go index b4e99371..20a7db67 100644 --- a/lib/searchfilter_test.go +++ b/lib/searchfilter_test.go @@ -16,8 +16,58 @@ import ( //"github.com/araddon/gou" "github.com/bmizerany/assert" "testing" + "encoding/json" ) +func GetJson(input interface{}) map[string]interface{} { + var result map[string]interface{} + bytes, _ := json.Marshal(input) + + json.Unmarshal(bytes, &result) + return result +} + +func TestTermsDsl(t *testing.T) { + filter := Filter().Terms("Sample", TEM_AND, "asdf", 123, true) + actual := GetJson(filter) + + actualTerms := actual["terms"].(map[string]interface{}) + actualValues := actualTerms["Sample"].([]interface{}) + + assert.Equal(t, 1, len(actual), "JSON should only have one key") + assert.Equal(t, 3, len(actualValues), "Should have 3 term values") + assert.Equal(t, actualValues[0], "asdf") + assert.Equal(t, actualValues[1], float64(123)) + assert.Equal(t, actualValues[2], true) + assert.Equal(t, "and", actualTerms["execution"]) +} + +func TestTermDsl(t *testing.T) { + filter := Filter().Term("Sample", "asdf").Term("field2", 341.4) + actual := GetJson(filter) + + actualTerm := actual["term"].(map[string]interface{}) + + assert.Equal(t, 1, len(actual), "JSON should only have one key") + assert.Equal(t, "asdf", actualTerm["Sample"]) + assert.Equal(t, float64(341.4), actualTerm["field2"]) +} + +func TestRangeDsl(t *testing.T) { + filter := Filter().Range("rangefield", 1, 2, 3, 4, "+08:00") + actual := GetJson(filter) + //A bit lazy, probably should assert keys exist + actualRange := actual["range"].(map[string]interface{})["rangefield"].(map[string]interface{}) + + + assert.Equal(t, 1, len(actual), "JSON should only have one key") + assert.Equal(t, float64(1), actualRange["gte"]) + assert.Equal(t, float64(2), actualRange["gt"]) + assert.Equal(t, float64(3), actualRange["lte"]) + assert.Equal(t, float64(4), actualRange["lt"]) + assert.Equal(t, "+08:00", actualRange["time_zone"]) +} + func TestFilters(t *testing.T) { c := NewTestConn() From eb0ff28844562cf0a63769a25f49d248aa39fff9 Mon Sep 17 00:00:00 2001 From: Mike Yoon Date: Sat, 4 Apr 2015 02:44:10 -0700 Subject: [PATCH 17/23] Merge properly --- lib/searchfilter.go | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/lib/searchfilter.go b/lib/searchfilter.go index cb58c109..e8a28c0e 100644 --- a/lib/searchfilter.go +++ b/lib/searchfilter.go @@ -299,7 +299,6 @@ func (f *FilterOp) Exists(field string) *FilterOp { f.ExistsProp = &PropertyPathMarker{Field: field} return f } -<<<<<<< HEAD func (f *FilterOp) Missing(field string) *FilterOp { f.MissingProp = &PropertyPathMarker{Field: field} @@ -308,24 +307,4 @@ func (f *FilterOp) Missing(field string) *FilterOp { func (f *FilterOp) SetLimit(maxResults int) *FilterOp { f.Limit = &LimitFilter{Value: maxResults} -======= -func (f *FilterOp) Missing(name string) *FilterOp { - f.MissingVal = map[string]string{"field": name} - return f -} - -// Add another Filterop, "combines" two filter ops into one -func (f *FilterOp) Add(fop *FilterOp) *FilterOp { - // TODO, this is invalid, refactor - if len(fop.Exist) > 0 { - f.Exist = fop.Exist - } - if len(fop.MissingVal) > 0 { - f.MissingVal = fop.MissingVal - } - if len(fop.Range) > 0 { - f.Range = fop.Range - } ->>>>>>> mattbaird/master - return f -} +} \ No newline at end of file From de5a0aa32f88386a413c8a2aa699cfebb3f4dfcf Mon Sep 17 00:00:00 2001 From: Mike Yoon Date: Wed, 8 Apr 2015 13:50:43 -0700 Subject: [PATCH 18/23] Finish new filter dsl tests Make new filter op naming more consistent Add Type, Ids, and Not filters functions --- lib/searchfilter.go | 54 +++++++-------- lib/searchfilter_test.go | 145 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 168 insertions(+), 31 deletions(-) diff --git a/lib/searchfilter.go b/lib/searchfilter.go index e8a28c0e..1c0411dd 100644 --- a/lib/searchfilter.go +++ b/lib/searchfilter.go @@ -143,20 +143,6 @@ func CompoundFilter(fl ...interface{}) *FilterWrap { } type FilterOp struct { - TermsMap map[string]interface{} `json:"terms,omitempty"` - TermMap map[string]interface{} `json:"term,omitempty"` - RangeMap map[string]RangeFilter `json:"range,omitempty"` - ExistsProp *PropertyPathMarker `json:"exists,omitempty"` - MissingProp *PropertyPathMarker `json:"missing,omitempty"` - AndFilters []FilterOp `json:"and,omitempty"` - OrFilters []FilterOp `json:"or,omitempty"` - NotFilters []FilterOp `json:"not,omitempty"` - Limit *LimitFilter `json:"limit,omitempty"` - Type *TypeFilter `json:"type,omitempty"` - Ids *IdFilter `json:"ids,omitempty"` - Script *ScriptFilter `json:"script,omitempty"` - GeoDist map[string]interface{} `json:"geo_distance,omitempty"` - GeoDistRange map[string]interface{} `json:"geo_distance_range,omitempty"` } type PropertyPathMarker struct { @@ -171,9 +157,8 @@ type TypeFilter struct { Value string `json:"value"` } -type IdFilter struct { - Type string `json:"type,omitempty"` - Values []string `json:"values,omitempty"` +type IdsFilter struct { + Values []interface{} `json:"values,omitempty"` } type ScriptFilter struct { @@ -231,23 +216,26 @@ func (f *FilterOp) Or(filter *FilterOp) *FilterOp { return f } +func (f *FilterOp) Not(filter *FilterOp) *FilterOp { + if len(f.NotFilters) == 0 { + f.NotFilters = []FilterOp{*filter} + } else { + f.NotFilters = append(f.NotFilters, *filter) + } + + return f +} + func (f *FilterOp) GeoDistance(distance string, fields ...GeoField) *FilterOp { - f.GeoDist = make(map[string]interface{}) - f.GeoDist["distance"] = distance for _, val := range fields { - f.GeoDist[val.Field] = val.GeoLocation } return f } func (f *FilterOp) GeoDistanceRange(from string, to string, fields ...GeoField) *FilterOp { - f.GeoDist = make(map[string]interface{}) - f.GeoDist["from"] = from - f.GeoDist["to"] = to for _, val := range fields { - f.GeoDist[val.Field] = val.GeoLocation } return f @@ -295,6 +283,18 @@ func (f *FilterOp) Range(field string, gte interface{}, return f } +func (f *FilterOp) Type(fieldType string) *FilterOp { + return f +} + +func (f *FilterOp) Ids(ids ...interface{}) *FilterOp { + return f +} + +func (f *FilterOp) IdsByTypes(types []string, ids ...interface{}) *FilterOp { + return f +} + func (f *FilterOp) Exists(field string) *FilterOp { f.ExistsProp = &PropertyPathMarker{Field: field} return f @@ -305,6 +305,6 @@ func (f *FilterOp) Missing(field string) *FilterOp { return f } -func (f *FilterOp) SetLimit(maxResults int) *FilterOp { - f.Limit = &LimitFilter{Value: maxResults} -} \ No newline at end of file +func (f *FilterOp) Limit(maxResults int) *FilterOp { + return f +} diff --git a/lib/searchfilter_test.go b/lib/searchfilter_test.go index 20a7db67..34781616 100644 --- a/lib/searchfilter_test.go +++ b/lib/searchfilter_test.go @@ -14,9 +14,9 @@ package elastigo import ( "fmt" //"github.com/araddon/gou" + "encoding/json" "github.com/bmizerany/assert" "testing" - "encoding/json" ) func GetJson(input interface{}) map[string]interface{} { @@ -27,6 +27,53 @@ func GetJson(input interface{}) map[string]interface{} { return result } +func HasKey(input map[string]interface{}, key string) bool { + if _, ok := input[key]; ok { + return true + } + + return false +} + +func TestAndDsl(t *testing.T) { + filter := Filter().And(Filter().Term("test", "asdf")). + And(Filter().Range("rangefield", 1, 2, 3, 4, "+08:00")) + actual := GetJson(filter) + + actualFilters := actual["and"].([]interface{}) + + assert.Equal(t, 1, len(actual), "JSON should only have one key") + assert.Equal(t, 2, len(actualFilters), "Should have 2 filters") + assert.Equal(t, true, HasKey(actualFilters[0].(map[string]interface{}), "term"), "first filter is term") + assert.Equal(t, true, HasKey(actualFilters[1].(map[string]interface{}), "range"), "second filter is range") +} + +func TestOrDsl(t *testing.T) { + filter := Filter().Or(Filter().Term("test", "asdf")). + Or(Filter().Range("rangefield", 1, 2, 3, 4, "+08:00")) + actual := GetJson(filter) + + actualFilters := actual["or"].([]interface{}) + + assert.Equal(t, 1, len(actual), "JSON should only have one key") + assert.Equal(t, 2, len(actualFilters), "Should have 2 filters") + assert.Equal(t, true, HasKey(actualFilters[0].(map[string]interface{}), "term"), "first filter is term") + assert.Equal(t, true, HasKey(actualFilters[1].(map[string]interface{}), "range"), "second filter is range") +} + +func TestNotDsl(t *testing.T) { + filter := Filter().Not(Filter().Term("test", "asdf")). + Not(Filter().Range("rangefield", 1, 2, 3, 4, "+08:00")) + actual := GetJson(filter) + + actualFilters := actual["not"].([]interface{}) + + assert.Equal(t, 1, len(actual), "JSON should only have one key") + assert.Equal(t, 2, len(actualFilters), "Should have 2 filters") + assert.Equal(t, true, HasKey(actualFilters[0].(map[string]interface{}), "term"), "first filter is term") + assert.Equal(t, true, HasKey(actualFilters[1].(map[string]interface{}), "range"), "second filter is range") +} + func TestTermsDsl(t *testing.T) { filter := Filter().Terms("Sample", TEM_AND, "asdf", 123, true) actual := GetJson(filter) @@ -59,7 +106,6 @@ func TestRangeDsl(t *testing.T) { //A bit lazy, probably should assert keys exist actualRange := actual["range"].(map[string]interface{})["rangefield"].(map[string]interface{}) - assert.Equal(t, 1, len(actual), "JSON should only have one key") assert.Equal(t, float64(1), actualRange["gte"]) assert.Equal(t, float64(2), actualRange["gt"]) @@ -68,6 +114,97 @@ func TestRangeDsl(t *testing.T) { assert.Equal(t, "+08:00", actualRange["time_zone"]) } +func TestExistsDsl(t *testing.T) { + filter := Filter().Exists("field1") + actual := GetJson(filter) + + actualValue := actual["exists"].(map[string]interface{}) + + assert.Equal(t, 1, len(actual), "JSON should only have one key") + assert.Equal(t, "field1", actualValue["field"], "exist field should match") +} + +func TestMissingDsl(t *testing.T) { + filter := Filter().Missing("field1") + actual := GetJson(filter) + + actualValue := actual["missing"].(map[string]interface{}) + + assert.Equal(t, 1, len(actual), "JSON should only have one key") + assert.Equal(t, "field1", actualValue["field"], "missing field should match") +} + +func TestLimitDsl(t *testing.T) { + filter := Filter().Limit(100) + actual := GetJson(filter) + + actualValue := actual["limit"].(map[string]interface{}) + assert.Equal(t, 1, len(actual), "JSON should only have one key") + assert.Equal(t, float64(100), actualValue["value"], "limit value should match") +} + +func TestTypeDsl(t *testing.T) { + filter := Filter().Type("my_type") + actual := GetJson(filter) + + actualValue := actual["type"].(map[string]interface{}) + assert.Equal(t, 1, len(actual), "JSON should only have one key") + assert.Equal(t, "my_type", actualValue["value"], "type value should match") +} + +func TestIdsDsl(t *testing.T) { + filter := Filter().Ids("test", "asdf", "fdsa") + actual := GetJson(filter) + + actualValue := actual["ids"].(map[string]interface{}) + actualValues := actualValue["values"].([]interface{}) + assert.Equal(t, 1, len(actual), "JSON should only have one key") + assert.Equal(t, nil, actualValue["type"], "Should have no type specified") + assert.Equal(t, 3, len(actualValues), "Should have 3 values specified") + assert.Equal(t, "test", actualValues[0], "Should have same value") + assert.Equal(t, "asdf", actualValues[1], "Should have same value") + assert.Equal(t, "fdsa", actualValues[2], "Should have same value") +} + +func TestIdsTypeDsl(t *testing.T) { + filter := Filter().IdsByTypes([]string{"my_type"}, "test", "asdf", "fdsa") + actual := GetJson(filter) + + actualValue := actual["ids"].(map[string]interface{}) + actualTypes := actualValue["type"].([]interface{}) + actualValues := actualValue["values"].([]interface{}) + assert.Equal(t, 1, len(actual), "JSON should only have one key") + assert.Equal(t, 1, len(actualTypes), "Should have one type specified") + assert.Equal(t, "my_type", actualTypes[0], "Should have correct type specified") + assert.Equal(t, 3, len(actualValues), "Should have 3 values specified") + assert.Equal(t, "test", actualValues[0], "Should have same value") + assert.Equal(t, "asdf", actualValues[1], "Should have same value") + assert.Equal(t, "fdsa", actualValues[2], "Should have same value") +} + +func TestGeoDistDsl(t *testing.T) { + filter := Filter().GeoDistance("100km", NewGeoField("pin.location", 32.3, 23.4)) + actual := GetJson(filter) + + actualValue := actual["geo_distance"].(map[string]interface{}) + actualLocation := actualValue["pin.location"].(map[string]interface{}) + assert.Equal(t, "100km", actualValue["distance"], "Distance should be equal") + assert.Equal(t, float64(32.3), actualLocation["lat"], "Latitude should be equal") + assert.Equal(t, float64(23.4), actualLocation["lon"], "Longitude should be equal") +} + +func TestGeoDistRangeDsl(t *testing.T) { + filter := Filter().GeoDistanceRange("100km", "200km", NewGeoField("pin.location", 32.3, 23.4)) + actual := GetJson(filter) + + actualValue := actual["geo_distance_range"].(map[string]interface{}) + actualLocation := actualValue["pin.location"].(map[string]interface{}) + assert.Equal(t, "100km", actualValue["from"], "From should be equal") + assert.Equal(t, "200km", actualValue["to"], "To should be equal") + assert.Equal(t, float64(32.3), actualLocation["lat"], "Latitude should be equal") + assert.Equal(t, float64(23.4), actualLocation["lon"], "Longitude should be equal") +} + func TestFilters(t *testing.T) { c := NewTestConn() @@ -104,7 +241,7 @@ func TestFilters(t *testing.T) { */ qry = Search("github").Filter( Filter().And(Filter().Terms("actor_attributes.location", TEM_DEFAULT, "portland")). - And(Filter().Terms("repository.has_wiki", TEM_DEFAULT, true))) + And(Filter().Terms("repository.has_wiki", TEM_DEFAULT, true))) out, err = qry.Result(c) expectedDocs = 10 expectedHits = 44 @@ -127,7 +264,7 @@ func TestFilters(t *testing.T) { qry = Search("github").Filter( Filter().Or(Filter().Terms("actor_attributes.location", TEM_DEFAULT, "portland")). - Or(Filter().Terms("repository.has_wiki", TEM_DEFAULT, true)), + Or(Filter().Terms("repository.has_wiki", TEM_DEFAULT, true)), ) out, err = qry.Result(c) expectedHits = 6676 From 6649d461dfe3c1be24808276158aa3ced32495c5 Mon Sep 17 00:00:00 2001 From: Mike Yoon Date: Wed, 8 Apr 2015 13:51:26 -0700 Subject: [PATCH 19/23] Make filter name more consistent --- lib/searchfilter.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/lib/searchfilter.go b/lib/searchfilter.go index 1c0411dd..79d630b8 100644 --- a/lib/searchfilter.go +++ b/lib/searchfilter.go @@ -143,6 +143,20 @@ func CompoundFilter(fl ...interface{}) *FilterWrap { } type FilterOp struct { + TermsMap map[string]interface{} `json:"terms,omitempty"` + TermMap map[string]interface{} `json:"term,omitempty"` + RangeMap map[string]RangeFilter `json:"range,omitempty"` + ExistsProp *PropertyPathMarker `json:"exists,omitempty"` + MissingProp *PropertyPathMarker `json:"missing,omitempty"` + AndFilters []FilterOp `json:"and,omitempty"` + OrFilters []FilterOp `json:"or,omitempty"` + NotFilters []FilterOp `json:"not,omitempty"` + LimitProp *LimitFilter `json:"limit,omitempty"` + TypeProp *TypeFilter `json:"type,omitempty"` + IdsProp *IdsFilter `json:"ids,omitempty"` + ScriptProp *ScriptFilter `json:"script,omitempty"` + GeoDistMap map[string]interface{} `json:"geo_distance,omitempty"` + GeoDistRangeMap map[string]interface{} `json:"geo_distance_range,omitempty"` } type PropertyPathMarker struct { @@ -158,6 +172,7 @@ type TypeFilter struct { } type IdsFilter struct { + Type []string `json:"type,omitempty"` Values []interface{} `json:"values,omitempty"` } @@ -227,15 +242,22 @@ func (f *FilterOp) Not(filter *FilterOp) *FilterOp { } func (f *FilterOp) GeoDistance(distance string, fields ...GeoField) *FilterOp { + f.GeoDistMap = make(map[string]interface{}) + f.GeoDistMap["distance"] = distance for _, val := range fields { + f.GeoDistMap[val.Field] = val.GeoLocation } return f } func (f *FilterOp) GeoDistanceRange(from string, to string, fields ...GeoField) *FilterOp { + f.GeoDistRangeMap = make(map[string]interface{}) + f.GeoDistRangeMap["from"] = from + f.GeoDistRangeMap["to"] = to for _, val := range fields { + f.GeoDistRangeMap[val.Field] = val.GeoLocation } return f @@ -284,14 +306,17 @@ func (f *FilterOp) Range(field string, gte interface{}, } func (f *FilterOp) Type(fieldType string) *FilterOp { + f.TypeProp = &TypeFilter{Value: fieldType} return f } func (f *FilterOp) Ids(ids ...interface{}) *FilterOp { + f.IdsProp = &IdsFilter{Values: ids} return f } func (f *FilterOp) IdsByTypes(types []string, ids ...interface{}) *FilterOp { + f.IdsProp = &IdsFilter{Type: types, Values: ids} return f } @@ -306,5 +331,6 @@ func (f *FilterOp) Missing(field string) *FilterOp { } func (f *FilterOp) Limit(maxResults int) *FilterOp { + f.LimitProp = &LimitFilter{Value: maxResults} return f } From 1c8bddf1c93ad451ab7320983ea8104e9d1490f9 Mon Sep 17 00:00:00 2001 From: Mike Yoon Date: Mon, 8 Jun 2015 21:52:23 -0700 Subject: [PATCH 20/23] Make boolean filters variadic --- lib/searchfilter.go | 25 +++++++++++++------------ lib/searchfilter_test.go | 3 +-- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/searchfilter.go b/lib/searchfilter.go index 3070b745..b4323553 100644 --- a/lib/searchfilter.go +++ b/lib/searchfilter.go @@ -154,9 +154,9 @@ type FilterOp struct { RangeMap map[string]RangeFilter `json:"range,omitempty"` ExistsProp *PropertyPathMarker `json:"exists,omitempty"` MissingProp *PropertyPathMarker `json:"missing,omitempty"` - AndFilters []FilterOp `json:"and,omitempty"` - OrFilters []FilterOp `json:"or,omitempty"` - NotFilters []FilterOp `json:"not,omitempty"` + AndFilters []*FilterOp `json:"and,omitempty"` + OrFilters []*FilterOp `json:"or,omitempty"` + NotFilters []*FilterOp `json:"not,omitempty"` LimitProp *LimitFilter `json:"limit,omitempty"` TypeProp *TypeFilter `json:"type,omitempty"` IdsProp *IdsFilter `json:"ids,omitempty"` @@ -217,31 +217,32 @@ func (f *FilterOp) Term(field string, value interface{}) *FilterOp { return f } -func (f *FilterOp) And(filter *FilterOp) *FilterOp { +func (f *FilterOp) And(filters ...*FilterOp) *FilterOp { if len(f.AndFilters) == 0 { - f.AndFilters = []FilterOp{*filter} + f.AndFilters = filters[:] } else { - f.AndFilters = append(f.AndFilters, *filter) + f.AndFilters = append(f.AndFilters, filters...) } return f } -func (f *FilterOp) Or(filter *FilterOp) *FilterOp { +func (f *FilterOp) Or(filters ...*FilterOp) *FilterOp { if len(f.OrFilters) == 0 { - f.OrFilters = []FilterOp{*filter} + f.OrFilters = filters[:] } else { - f.OrFilters = append(f.OrFilters, *filter) + f.OrFilters = append(f.OrFilters, filters...) } return f } -func (f *FilterOp) Not(filter *FilterOp) *FilterOp { +func (f *FilterOp) Not(filters ...*FilterOp) *FilterOp { if len(f.NotFilters) == 0 { - f.NotFilters = []FilterOp{*filter} + f.NotFilters = filters[:] + } else { - f.NotFilters = append(f.NotFilters, *filter) + f.NotFilters = append(f.NotFilters, filters...) } return f diff --git a/lib/searchfilter_test.go b/lib/searchfilter_test.go index 34781616..dfb0c049 100644 --- a/lib/searchfilter_test.go +++ b/lib/searchfilter_test.go @@ -49,8 +49,7 @@ func TestAndDsl(t *testing.T) { } func TestOrDsl(t *testing.T) { - filter := Filter().Or(Filter().Term("test", "asdf")). - Or(Filter().Range("rangefield", 1, 2, 3, 4, "+08:00")) + filter := Filter().Or(Filter().Term("test", "asdf"), Filter().Range("rangefield", 1, 2, 3, 4, "+08:00")) actual := GetJson(filter) actualFilters := actual["or"].([]interface{}) From 9bb50d92e77a65f14bf9d5783bcb15782f7d3da3 Mon Sep 17 00:00:00 2001 From: Mike Yoon Date: Wed, 9 Sep 2015 22:15:01 -0700 Subject: [PATCH 21/23] Fix tests --- lib/searchfilter_test.go | 6 +++--- lib/searchsearch_test.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/searchfilter_test.go b/lib/searchfilter_test.go index 4f0d0102..57671bf7 100644 --- a/lib/searchfilter_test.go +++ b/lib/searchfilter_test.go @@ -232,7 +232,7 @@ func TestFilters(t *testing.T) { Convey("Terms filter", t, func() { qry := Search("oilers").Filter( - Filter().Terms("pos", "RW", "LW"), + Filter().Terms("pos", TEM_DEFAULT, "RW", "LW"), ) out, err := qry.Result(c) So(err, ShouldBeNil) @@ -244,7 +244,7 @@ func TestFilters(t *testing.T) { Convey("Filter involving an AND", t, func() { qry := Search("oilers").Filter( Filter().And( - Filter().Terms("pos", "LW"), + Filter().Terms("pos", TEM_DEFAULT, "LW"), Filter().Exists("PIM"), ), ) @@ -259,7 +259,7 @@ func TestFilters(t *testing.T) { Convey("Filterng filter results", t, func() { qry := Search("oilers").Filter( - Filter().Terms("pos", "LW"), + Filter().Terms("pos", TEM_DEFAULT, "LW"), ) qry.Filter( Filter().Exists("PIM"), diff --git a/lib/searchsearch_test.go b/lib/searchsearch_test.go index 4843cbe8..b09420f0 100644 --- a/lib/searchsearch_test.go +++ b/lib/searchsearch_test.go @@ -155,7 +155,7 @@ func TestSearch(t *testing.T) { Facet().Fields("teams").Size("20"), ).Query( Query().Range( - Filter().Range("dob", 19600101, nil, 19621231, nil, ""), + Filter().Range("dob", "19600101", nil, "19621231", nil, ""), ).Search("*w*"), ) out, err := qry.Result(c) @@ -217,7 +217,7 @@ func TestSearch(t *testing.T) { out, err := Search("oilers").Size("25").Query( Query().Fields("name", "*d*", "", ""), ).Filter( - Filter().Terms("teams", "STL"), + Filter().Terms("teams", TEM_DEFAULT, "STL"), ).Result(c) So(err, ShouldBeNil) So(out, ShouldNotBeNil) @@ -229,7 +229,7 @@ func TestSearch(t *testing.T) { out, err := Search("oilers").Size("25").Query( Query().Range( - Filter().Range("dob", 19600101, nil, 19621231, nil, ""), + Filter().Range("dob", "19600101", nil, "19621231", nil, ""), ).Search("*w*"), ).Result(c) So(err, ShouldBeNil) From 41d3e4073476d775e06bf118a03c4473b31ed135 Mon Sep 17 00:00:00 2001 From: Mike Yoon Date: Tue, 24 Nov 2015 23:16:25 -0800 Subject: [PATCH 22/23] Revert vagrantfile change --- Vagrantfile | 37 ++++++++++++++----------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index ad8fc4cb..cbd1f563 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -2,37 +2,28 @@ # vi: set ft=ruby : Vagrant.configure("2") do |config| - config.vm.box = "ubuntu/trusty64" - config.vm.network :forwarded_port, guest: 9200, host: 9200, auto_correct: true + config.vm.box = "lucid64" + config.vm.box_url = "http://files.vagrantup.com/lucid64.box" config.vm.network :forwarded_port, guest: 9300, host: 9300, auto_correct: true - #config.vm.provision :shell, :inline => "curl -L https://www.chef.io/chef/install.sh | sudo bash" - - config.vm.provision "shell", inline: <<-SHELL - wget -qO - https://packages.elasticsearch.org/GPG-KEY-elasticsearch | sudo apt-key add - - sudo apt-add-repository 'deb http://packages.elasticsearch.org/elasticsearch/1.4/debian stable main' - sudo apt-get update - sudo apt-get install -y openjdk-7-jre - sudo apt-get install -y elasticsearch - sudo /etc/init.d/elasticsearch start - SHELL + config.vm.provision :shell, :inline => "gem install chef --version 10.26.0 --no-rdoc --no-ri --conservative" config.vm.provider :virtualbox do |vb| vb.gui = false vb.customize ["modifyvm", :id, "--memory", "1024"] vb.customize ["modifyvm", :id, "--cpus", "1"] - # This allows symlinks to be created within the /vagrant root directory, + # This allows symlinks to be created within the /vagrant root directory, # which is something librarian-puppet needs to be able to do. This might # be enabled by default depending on what version of VirtualBox is used. vb.customize ["setextradata", :id, "VBoxInternal2/SharedFoldersEnableSymlinksCreate/v-root", "1"] end - #config.vm.provision :chef_solo do |chef| - # chef.cookbooks_path = "cookbooks" - # chef.add_recipe("apt") - # chef.add_recipe("java") - # chef.add_recipe("elasticsearch") - # chef.add_recipe("git") - # chef.add_recipe("mercurial") - # chef.add_recipe("build-essential") - # chef.add_recipe("golang") - #end + config.vm.provision :chef_solo do |chef| + chef.cookbooks_path = "cookbooks" + chef.add_recipe("apt") + chef.add_recipe("java") + chef.add_recipe("elasticsearch") + chef.add_recipe("git") + chef.add_recipe("mercurial") + chef.add_recipe("build-essential") + chef.add_recipe("golang") + end end \ No newline at end of file From 54b3449f77b9861aa7cf5ddcbc21d2bc5abca42d Mon Sep 17 00:00:00 2001 From: Mike Yoon Date: Wed, 25 Nov 2015 00:22:37 -0800 Subject: [PATCH 23/23] Lint and add godoc to searchfilter --- lib/searchfilter.go | 107 ++++++++++++++++++++++++++++++--------- lib/searchfilter_test.go | 10 ++-- lib/searchquery.go | 10 ++-- lib/searchsearch.go | 1 + lib/searchsearch_test.go | 2 +- 5 files changed, 97 insertions(+), 33 deletions(-) diff --git a/lib/searchfilter.go b/lib/searchfilter.go index b4323553..3d10ab57 100644 --- a/lib/searchfilter.go +++ b/lib/searchfilter.go @@ -14,38 +14,50 @@ package elastigo import ( "encoding/json" "fmt" - . "github.com/araddon/gou" + "github.com/araddon/gou" ) var ( - _ = DEBUG + _ = gou.DEBUG ) -// A bool (and/or) clause +// BoolClause represents aa bool (and/or) clause for use with FilterWrap +// Legacy, use new FilterOp functions instead type BoolClause string +// TermExecutionMode refers to how a terms (not term) filter should behave +// The acceptable options are all prefixed with TEM +// See https://www.elastic.co/guide/en/elasticsearch/reference/1.5/query-dsl-terms-filter.html type TermExecutionMode string const ( - TEM_DEFAULT TermExecutionMode = "" - TEM_PLAIN = "plain" - TEM_FIELD = "field_data" - TEM_BOOL = "bool" - TEM_AND = "and" - TEM_OR = "or" + // TEMDefault default ES term filter behavior (plain) + TEMDefault TermExecutionMode = "" + // TEMPlain default ES term filter behavior + TEMPlain TermExecutionMode = "plain" + // TEMField field_data execution mode + TEMField TermExecutionMode = "field_data" + // TEMBool bool execution mode + TEMBool TermExecutionMode = "bool" + // TEMAnd and execution mode + TEMAnd TermExecutionMode = "and" + // TEMOr or execution mode + TEMOr TermExecutionMode = "or" ) -// Filter clause is either a boolClause or FilterOp +// FilterClause is either a boolClause or FilterOp for use with FilterWrap type FilterClause interface { String() string } -// A wrapper to allow for custom serialization +// FilterWrap is the legacy struct for chaining multiple filters with a bool +// Legacy, use new FilterOp functions instead type FilterWrap struct { boolClause string filters []interface{} } +// NewFilterWrap creates a new FilterWrap struct func NewFilterWrap() *FilterWrap { return &FilterWrap{filters: make([]interface{}, 0), boolClause: "and"} } @@ -56,6 +68,7 @@ func (f *FilterWrap) String() string { // Bool sets the type of boolean filter to use. // Accepted values are "and" and "or". +// Legacy, use new FilterOp functions instead func (f *FilterWrap) Bool(s string) { f.boolClause = s } @@ -73,7 +86,7 @@ func (f *FilterWrap) addFilters(fl []interface{}) { f.filters = append(f.filters, fl...) } -// Custom marshalling to support the query dsl +// MarshalJSON override for FilterWrap to match the expected ES syntax with the bool at the root func (f *FilterWrap) MarshalJSON() ([]byte, error) { var root interface{} if len(f.filters) > 1 { @@ -129,7 +142,10 @@ func (f *FilterWrap) MarshalJSON() ([]byte, error) { */ -// Filter Operation + +// Filter creates a blank FilterOp that can be customized with further function calls +// This is the starting point for constructing any filter query +// Examples: // // Filter().Term("user","kimchy") // @@ -137,23 +153,26 @@ func (f *FilterWrap) MarshalJSON() ([]byte, error) { // Filter().Terms("user", "kimchy", "elasticsearch") // // Filter().Exists("repository.name") -// func Filter() *FilterOp { return &FilterOp{} } +// CompoundFilter creates a complete FilterWrap given multiple filters +// Legacy, use new FilterOp functions instead func CompoundFilter(fl ...interface{}) *FilterWrap { FilterVal := NewFilterWrap() FilterVal.addFilters(fl) return FilterVal } +// FilterOp holds all the information for a filter query +// Properties should not be set directly, but instead via the fluent-style API. type FilterOp struct { TermsMap map[string]interface{} `json:"terms,omitempty"` TermMap map[string]interface{} `json:"term,omitempty"` RangeMap map[string]RangeFilter `json:"range,omitempty"` - ExistsProp *PropertyPathMarker `json:"exists,omitempty"` - MissingProp *PropertyPathMarker `json:"missing,omitempty"` + ExistsProp *propertyPathMarker `json:"exists,omitempty"` + MissingProp *propertyPathMarker `json:"missing,omitempty"` AndFilters []*FilterOp `json:"and,omitempty"` OrFilters []*FilterOp `json:"or,omitempty"` NotFilters []*FilterOp `json:"not,omitempty"` @@ -165,29 +184,47 @@ type FilterOp struct { GeoDistRangeMap map[string]interface{} `json:"geo_distance_range,omitempty"` } -type PropertyPathMarker struct { +type propertyPathMarker struct { Field string `json:"field"` } +// LimitFilter holds the Limit filter information +// Value: number of documents to limit type LimitFilter struct { Value int `json:"value"` } +// TypeFilter filters on the document type +// Value: the document type to filter type TypeFilter struct { Value string `json:"value"` } +// IdsFilter holds the type and ids (on the _id field) to filter +// Type: a string or an array of string types. Optional. +// Values: Array of ids to match type IdsFilter struct { Type []string `json:"type,omitempty"` Values []interface{} `json:"values,omitempty"` } +// ScriptFilter will filter using a custom javascript function +// Script: the javascript to run +// Params: map of custom parameters to pass into the function (JSON), if any +// IsCached: whether to cache the results of the filter type ScriptFilter struct { Script string `json:"script"` Params map[string]interface{} `json:"params,omitempty"` IsCached bool `json:"_cache,omitempty"` } +// RangeFilter filters given a range. Parameters need to be comparable for ES to accept. +// Only a minimum of one comparison parameter is required. You probably shouldn't mix GT and GTE parameters. +// Gte: the greater-than-or-equal to value. Should be a number or date. +// Lte: the less-than-or-equal to value. Should be a number or date. +// Gt: the greater-than value. Should be a number or date. +// Lt: the less-than value. Should be a number or date. +// TimeZone: the timezone to use (+|-h:mm format), if the other parameters are dates type RangeFilter struct { Gte interface{} `json:"gte,omitempty"` Lte interface{} `json:"lte,omitempty"` @@ -196,11 +233,14 @@ type RangeFilter struct { TimeZone string `json:"time_zone,omitempty"` //Ideally this would be an int } +// GeoLocation holds the coordinates for a geo query. Currently hashes are not supported. type GeoLocation struct { Latitude float32 `json:"lat"` Longitude float32 `json:"lon"` } +// GeoField holds a GeoLocation and a field to match to. +// This exists so the struct will match the ES schema. type GeoField struct { GeoLocation Field string @@ -208,6 +248,7 @@ type GeoField struct { // Term will add a term to the filter. // Multiple Term filters can be added, and ES will OR them. +// If the term already exists in the FilterOp, the value will be overridden. func (f *FilterOp) Term(field string, value interface{}) *FilterOp { if len(f.TermMap) == 0 { f.TermMap = make(map[string]interface{}) @@ -217,6 +258,7 @@ func (f *FilterOp) Term(field string, value interface{}) *FilterOp { return f } +// And will add an AND op to the filter. One or more FilterOps can be passed in. func (f *FilterOp) And(filters ...*FilterOp) *FilterOp { if len(f.AndFilters) == 0 { f.AndFilters = filters[:] @@ -227,6 +269,7 @@ func (f *FilterOp) And(filters ...*FilterOp) *FilterOp { return f } +// Or will add an OR op to the filter. One or more FilterOps can be passed in. func (f *FilterOp) Or(filters ...*FilterOp) *FilterOp { if len(f.OrFilters) == 0 { f.OrFilters = filters[:] @@ -237,6 +280,7 @@ func (f *FilterOp) Or(filters ...*FilterOp) *FilterOp { return f } +// Not will add a NOT op to the filter. One or more FilterOps can be passed in. func (f *FilterOp) Not(filters ...*FilterOp) *FilterOp { if len(f.NotFilters) == 0 { f.NotFilters = filters[:] @@ -248,6 +292,9 @@ func (f *FilterOp) Not(filters ...*FilterOp) *FilterOp { return f } +// GeoDistance will add a GEO DISTANCE op to the filter. +// distance: distance in ES distance format, i.e. "100km" or "100mi". +// fields: an array of GeoField origin coordinates. Only one coordinate needs to match. func (f *FilterOp) GeoDistance(distance string, fields ...GeoField) *FilterOp { f.GeoDistMap = make(map[string]interface{}) f.GeoDistMap["distance"] = distance @@ -258,6 +305,10 @@ func (f *FilterOp) GeoDistance(distance string, fields ...GeoField) *FilterOp { return f } +// GeoDistanceRange will add a GEO DISTANCE RANGE op to the filter. +// from: minimum distance in ES distance format, i.e. "100km" or "100mi". +// to: maximum distance in ES distance format, i.e. "100km" or "100mi". +// fields: an array of GeoField origin coordinates. Only one coor func (f *FilterOp) GeoDistanceRange(from string, to string, fields ...GeoField) *FilterOp { f.GeoDistRangeMap = make(map[string]interface{}) f.GeoDistRangeMap["from"] = from @@ -270,17 +321,18 @@ func (f *FilterOp) GeoDistanceRange(from string, to string, fields ...GeoField) return f } -// Helper to create values for the GeoDistance filters +// NewGeoField is a helper function to create values for the GeoDistance filters func NewGeoField(field string, latitude float32, longitude float32) GeoField { return GeoField{ GeoLocation: GeoLocation{Latitude: latitude, Longitude: longitude}, Field: field} } -// Filter Terms -// -// Filter().Terms("user","kimchy","stuff") -// Note: you can only have one terms clause in a filter. Use a bool filter to combine +// Terms adds a TERMS op to the filter. +// field: the document field +// executionMode Term execution mode, starts with TEM +// values: array of values to match +// Note: you can only have one terms clause in a filter. Use a bool filter to combine multiple. func (f *FilterOp) Terms(field string, executionMode TermExecutionMode, values ...interface{}) *FilterOp { //You can only have one terms in a filter f.TermsMap = make(map[string]interface{}) @@ -295,6 +347,7 @@ func (f *FilterOp) Terms(field string, executionMode TermExecutionMode, values . } // Range adds a range filter for the given field. +// See the RangeFilter struct documentation for information about the parameters. func (f *FilterOp) Range(field string, gte interface{}, gt interface{}, lte interface{}, lt interface{}, timeZone string) *FilterOp { @@ -312,31 +365,37 @@ func (f *FilterOp) Range(field string, gte interface{}, return f } +// Type adds a TYPE op to the filter. func (f *FilterOp) Type(fieldType string) *FilterOp { f.TypeProp = &TypeFilter{Value: fieldType} return f } +// Ids adds a IDS op to the filter. func (f *FilterOp) Ids(ids ...interface{}) *FilterOp { f.IdsProp = &IdsFilter{Values: ids} return f } +// IdsByTypes adds a IDS op to the filter, but also allows passing in an array of types for the query. func (f *FilterOp) IdsByTypes(types []string, ids ...interface{}) *FilterOp { f.IdsProp = &IdsFilter{Type: types, Values: ids} return f } +// Exists adds an EXISTS op to the filter. func (f *FilterOp) Exists(field string) *FilterOp { - f.ExistsProp = &PropertyPathMarker{Field: field} + f.ExistsProp = &propertyPathMarker{Field: field} return f } +// Missing adds an MISSING op to the filter. func (f *FilterOp) Missing(field string) *FilterOp { - f.MissingProp = &PropertyPathMarker{Field: field} + f.MissingProp = &propertyPathMarker{Field: field} return f } +// Limit adds an LIMIT op to the filter. func (f *FilterOp) Limit(maxResults int) *FilterOp { f.LimitProp = &LimitFilter{Value: maxResults} return f diff --git a/lib/searchfilter_test.go b/lib/searchfilter_test.go index 57671bf7..a4931ddc 100644 --- a/lib/searchfilter_test.go +++ b/lib/searchfilter_test.go @@ -59,7 +59,7 @@ func TestFilterDsl(t *testing.T) { }) Convey("Terms filter", t, func() { - filter := Filter().Terms("Sample", TEM_AND, "asdf", 123, true) + filter := Filter().Terms("Sample", TEMAnd, "asdf", 123, true) actual, err := GetJson(filter) actualTerms := actual["terms"].(map[string]interface{}) @@ -232,7 +232,7 @@ func TestFilters(t *testing.T) { Convey("Terms filter", t, func() { qry := Search("oilers").Filter( - Filter().Terms("pos", TEM_DEFAULT, "RW", "LW"), + Filter().Terms("pos", TEMDefault, "RW", "LW"), ) out, err := qry.Result(c) So(err, ShouldBeNil) @@ -244,7 +244,7 @@ func TestFilters(t *testing.T) { Convey("Filter involving an AND", t, func() { qry := Search("oilers").Filter( Filter().And( - Filter().Terms("pos", TEM_DEFAULT, "LW"), + Filter().Terms("pos", TEMDefault, "LW"), Filter().Exists("PIM"), ), ) @@ -259,7 +259,7 @@ func TestFilters(t *testing.T) { Convey("Filterng filter results", t, func() { qry := Search("oilers").Filter( - Filter().Terms("pos", TEM_DEFAULT, "LW"), + Filter().Terms("pos", TEMDefault, "LW"), ) qry.Filter( Filter().Exists("PIM"), @@ -274,7 +274,7 @@ func TestFilters(t *testing.T) { Convey("Filter involving OR", t, func() { qry := Search("oilers").Filter( Filter().Or( - Filter().Terms("pos", TEM_DEFAULT, "G"), + Filter().Terms("pos", TEMDefault, "G"), Filter().Range("goals", nil, 80, nil, nil, ""), ), ) diff --git a/lib/searchquery.go b/lib/searchquery.go index 8a8630d2..dd01ed71 100644 --- a/lib/searchquery.go +++ b/lib/searchquery.go @@ -98,14 +98,14 @@ func (q *QueryDsl) All() *QueryDsl { return q } -// Limit the query to this range +// Range adds a RANGE FilterOp to the search query +// Legacy. Use the Filter() function instead func (q *QueryDsl) Range(fop *FilterOp) *QueryDsl { if q.FilterVal == nil { q.FilterVal = fop return q } - // TODO: this is not valid, refactor - //q.FilterVal.Add(fop) + return q } @@ -144,6 +144,8 @@ func (q *QueryDsl) Qs(qs *QueryString) *QueryDsl { return q } +// SetLenient sets whether the query should ignore format based failures, +// such as passing in text to a number field. func (q *QueryDsl) SetLenient(lenient bool) *QueryDsl { q.QueryEmbed.Qs.Lenient = lenient return q @@ -213,6 +215,8 @@ type QueryString struct { //_missing_:field1, } +//I don't know how any of the Term stuff below is supposed to work. -mikeyoon + // Generic Term based (used in query, facet, filter) type Term struct { Terms Terms `json:"terms,omitempty"` diff --git a/lib/searchsearch.go b/lib/searchsearch.go index 4b76e4f1..c921ae5a 100644 --- a/lib/searchsearch.go +++ b/lib/searchsearch.go @@ -174,6 +174,7 @@ func (s *SearchDsl) Query(q *QueryDsl) *SearchDsl { // Filter().Exists("repository.name"), // Filter().Terms("repository.has_wiki", true) // ) + func (s *SearchDsl) Filter(fl *FilterOp) *SearchDsl { s.FilterVal = fl return s diff --git a/lib/searchsearch_test.go b/lib/searchsearch_test.go index b09420f0..81f11b3a 100644 --- a/lib/searchsearch_test.go +++ b/lib/searchsearch_test.go @@ -217,7 +217,7 @@ func TestSearch(t *testing.T) { out, err := Search("oilers").Size("25").Query( Query().Fields("name", "*d*", "", ""), ).Filter( - Filter().Terms("teams", TEM_DEFAULT, "STL"), + Filter().Terms("teams", TEMDefault, "STL"), ).Result(c) So(err, ShouldBeNil) So(out, ShouldNotBeNil)