Move ID interface to m3metrics and add m3 id implementation

filters/filter_benchmark_test.go

@@ -24,6 +24,8 @@ import (
 	"bytes"
 	"fmt"
 	"testing"
+
+	"github.com/m3db/m3metrics/metric/id"
 )
 
 var (
@@ -64,7 +66,7 @@ func BenchmarkEquityFilterByValue(b *testing.B) {
 }
 
 func BenchmarkTagsFilterOne(b *testing.B) {
-	filter, _ := NewTagsFilter(testTagsFilterMapOne, NewMockSortedTagIterator, Conjunction)
+	filter, _ := NewTagsFilter(testTagsFilterMapOne, Conjunction, testTagsFilterOptions())
 	benchTagsFilter(b, testFlatID, filter)
 }
 
@@ -73,7 +75,7 @@ func BenchmarkMapTagsFilterOne(b *testing.B) {
 }
 
 func BenchmarkTagsFilterThree(b *testing.B) {
-	filter, _ := NewTagsFilter(testTagsFilterMapThree, NewMockSortedTagIterator, Conjunction)
+	filter, _ := NewTagsFilter(testTagsFilterMapThree, Conjunction, testTagsFilterOptions())
 	benchTagsFilter(b, testFlatID, filter)
 }
 
@@ -225,10 +227,10 @@ func (f testEqualityFilter) Matches(id []byte) bool {
 
 type testMapTagsFilter struct {
 	filters map[string]Filter
-	iterFn  NewSortedTagIteratorFn
+	iterFn  id.SortedTagIteratorFn
 }
 
-func newTestMapTagsFilter(tagFilters map[string]string, iterFn NewSortedTagIteratorFn) Filter {
+func newTestMapTagsFilter(tagFilters map[string]string, iterFn id.SortedTagIteratorFn) Filter {
 	filters := make(map[string]Filter, len(tagFilters))
 	for name, value := range tagFilters {
 		filter, _ := NewFilter([]byte(value))

filters/mock_filter.go

@@ -22,6 +22,8 @@ package filters
 
 import (
 	"strings"
+
+	"github.com/m3db/m3metrics/metric/id"
 )
 
 const (
@@ -45,8 +47,8 @@ type mockSortedTagIterator struct {
 	pairs []mockTagPair
 }
 
-func idToMockTagPairs(id []byte) []mockTagPair {
-	tagPairs := strings.Split(string(id), mockTagPairSeparator)
+func tagsToPairs(tags []byte) []mockTagPair {
+	tagPairs := strings.Split(string(tags), mockTagPairSeparator)
 	var pairs []mockTagPair
 	for _, pair := range tagPairs {
 		p := strings.Split(pair, mockTagValueSeparator)
@@ -56,11 +58,17 @@ func idToMockTagPairs(id []byte) []mockTagPair {
 }
 
 // NewMockSortedTagIterator creates a mock SortedTagIterator based on given ID.
-func NewMockSortedTagIterator(id []byte) SortedTagIterator {
-	pairs := idToMockTagPairs(id)
+func NewMockSortedTagIterator(tags []byte) id.SortedTagIterator {
+	pairs := tagsToPairs(tags)
 	return &mockSortedTagIterator{idx: -1, pairs: pairs}
 }
 
+func (it *mockSortedTagIterator) Reset(tags []byte) {
+	it.idx = -1
+	it.err = nil
+	it.pairs = tagsToPairs(tags)
+}
+
 func (it *mockSortedTagIterator) Next() bool {
 	if it.err != nil || it.idx >= len(it.pairs) {
 		return false

filters/tags_filter.go

@@ -24,26 +24,9 @@ import (
 	"bytes"
 	"fmt"
 	"sort"
-)
-
-// SortedTagIterator iterates over a set of tag names and values
-// sorted by tag names in ascending order.
-type SortedTagIterator interface {
-	// Next returns true if there are more tag names and values.
-	Next() bool
-
-	// Current returns the current tag name and value.
-	Current() ([]byte, []byte)
-
-	// Err returns any errors encountered.
-	Err() error
-
-	// Close closes the iterator.
-	Close()
-}
 
-// NewSortedTagIteratorFn creates a tag iterator given an id.
-type NewSortedTagIteratorFn func(id []byte) SortedTagIterator
+	"github.com/m3db/m3metrics/metric/id"
+)
 
 // tagFilter is a filter associated with a given tag.
 type tagFilter struct {
@@ -61,60 +44,100 @@ func (tn tagFiltersByNameAsc) Len() int           { return len(tn) }
 func (tn tagFiltersByNameAsc) Swap(i, j int)      { tn[i], tn[j] = tn[j], tn[i] }
 func (tn tagFiltersByNameAsc) Less(i, j int) bool { return bytes.Compare(tn[i].name, tn[j].name) < 0 }
 
+// TagsFilterOptions provide a set of tag filter options.
+type TagsFilterOptions struct {
+	// Name of the name tag.
+	NameTagName []byte
+
+	// Function to extract name and tags from an id.
+	NameAndTagsFn id.NameAndTagsFn
+
+	// Function to create a new sorted tag iterator from id tags.
+	SortedTagIteratorFn id.SortedTagIteratorFn
+}
+
 // tagsFilter contains a list of tag filters.
 type tagsFilter struct {
-	filters []tagFilter
-	iterFn  NewSortedTagIteratorFn
-	op      LogicalOp
+	nameFilter Filter
+	tagFilters []tagFilter
+	op         LogicalOp
+	opts       TagsFilterOptions
 }
 
-// NewTagsFilter create a new tags filter.
-func NewTagsFilter(tagFilters map[string]string, iterFn NewSortedTagIteratorFn, op LogicalOp) (Filter, error) {
-	filters := make([]tagFilter, 0, len(tagFilters))
-	for name, value := range tagFilters {
+// NewTagsFilter creates a new tags filter.
+func NewTagsFilter(
+	filters map[string]string,
+	op LogicalOp,
+	opts TagsFilterOptions,
+) (Filter, error) {
+	var (
+		nameFilter Filter
+		tagFilters = make([]tagFilter, 0, len(filters))
+	)
+	for name, value := range filters {
 		valFilter, err := NewFilter([]byte(value))
 		if err != nil {
 			return nil, err
 		}
-
-		filters = append(filters, tagFilter{
-			name:        []byte(name),
-			valueFilter: valFilter,
-		})
+		bName := []byte(name)
+		if bytes.Equal(opts.NameTagName, bName) {
+			nameFilter = valFilter
+		} else {
+			tagFilters = append(tagFilters, tagFilter{
+				name:        bName,
+				valueFilter: valFilter,
+			})
+		}
 	}
-	sort.Sort(tagFiltersByNameAsc(filters))
+	sort.Sort(tagFiltersByNameAsc(tagFilters))
 	return &tagsFilter{
-		filters: filters,
-		iterFn:  iterFn,
-		op:      op,
+		nameFilter: nameFilter,
+		tagFilters: tagFilters,
+		op:         op,
+		opts:       opts,
 	}, nil
 }
 
 func (f *tagsFilter) String() string {
 	separator := " " + string(f.op) + " "
 	var buf bytes.Buffer
-	numFilters := len(f.filters)
-	for i := 0; i < numFilters; i++ {
-		buf.WriteString(f.filters[i].String())
-		if i < numFilters-1 {
+	numTagFilters := len(f.tagFilters)
+	if f.nameFilter != nil {
+		buf.WriteString(fmt.Sprintf("%s:%s", f.opts.NameTagName, f.nameFilter.String()))
+		if numTagFilters > 0 {
+			buf.WriteString(separator)
+		}
+	}
+	for i := 0; i < numTagFilters; i++ {
+		buf.WriteString(f.tagFilters[i].String())
+		if i < numTagFilters-1 {
 			buf.WriteString(separator)
 		}
 	}
 	return buf.String()
 }
 
 func (f *tagsFilter) Matches(id []byte) bool {
-	if len(f.filters) == 0 {
+	if f.nameFilter == nil && len(f.tagFilters) == 0 {
 		return true
 	}
-	iter := f.iterFn(id)
+
+	name, tags, err := f.opts.NameAndTagsFn(id)
+	if err != nil {
+		return false
+	}
+	if f.nameFilter != nil && !f.nameFilter.Matches(name) {
+		return false
+	}
+
+	iter := f.opts.SortedTagIteratorFn(tags)
 	defer iter.Close()
 
 	currIdx := 0
-	for iter.Next() && currIdx < len(f.filters) {
+	for iter.Next() && currIdx < len(f.tagFilters) {
 		name, value := iter.Current()
 
-		comparison := bytes.Compare(name, f.filters[currIdx].name)
+		comparison := bytes.Compare(name, f.tagFilters[currIdx].name)
 		if comparison < 0 {
 			continue
 		}
@@ -125,23 +148,23 @@ func (f *tagsFilter) Matches(id []byte) bool {
 				return false
 			}
 
-			// Iterate filters for the OR case.
+			// Iterate tagFilters for the OR case.
 			currIdx++
-			for currIdx < len(f.filters) && bytes.Compare(name, f.filters[currIdx].name) > 0 {
+			for currIdx < len(f.tagFilters) && bytes.Compare(name, f.tagFilters[currIdx].name) > 0 {
 				currIdx++
 			}
 
-			if currIdx == len(f.filters) {
-				// Past all filters.
+			if currIdx == len(f.tagFilters) {
+				// Past all tagFilters.
 				return false
 			}
 
-			if bytes.Compare(name, f.filters[currIdx].name) < 0 {
+			if bytes.Compare(name, f.tagFilters[currIdx].name) < 0 {
 				continue
 			}
 		}
 
-		match := f.filters[currIdx].valueFilter.Matches(value)
+		match := f.tagFilters[currIdx].valueFilter.Matches(value)
 		if match && f.op == Disjunction {
 			return true
 		}
@@ -157,5 +180,5 @@ func (f *tagsFilter) Matches(id []byte) bool {
 		return false
 	}
 
-	return currIdx == len(f.filters)
+	return currIdx == len(f.tagFilters)
 }

filters/tags_filter_test.go

@@ -27,7 +27,7 @@ import (
 )
 
 func TestEmptyTagsFilterMatches(t *testing.T) {
-	f, err := NewTagsFilter(nil, NewMockSortedTagIterator, Conjunction)
+	f, err := NewTagsFilter(nil, Conjunction, testTagsFilterOptions())
 	require.NoError(t, err)
 	require.True(t, f.Matches([]byte("foo")))
 }
@@ -37,7 +37,7 @@ func TestTagsFilterMatches(t *testing.T) {
 		"tagName1": "tagValue1",
 		"tagName2": "tagValue2",
 	}
-	f, err := NewTagsFilter(filters, NewMockSortedTagIterator, Conjunction)
+	f, err := NewTagsFilter(filters, Conjunction, testTagsFilterOptions())
 	inputs := []mockFilterData{
 		{val: "tagName1=tagValue1,tagName2=tagValue2", match: true},
 		{val: "tagName0=tagValue0,tagName1=tagValue1,tagName2=tagValue2,tagName3=tagValue3", match: true},
@@ -50,7 +50,7 @@ func TestTagsFilterMatches(t *testing.T) {
 		require.Equal(t, input.match, f.Matches([]byte(input.val)))
 	}
 
-	f, err = NewTagsFilter(filters, NewMockSortedTagIterator, Disjunction)
+	f, err = NewTagsFilter(filters, Disjunction, testTagsFilterOptions())
 	inputs = []mockFilterData{
 		{val: "tagName1=tagValue1,tagName2=tagValue2", match: true},
 		{val: "tagName0=tagValue0,tagName1=tagValue1,tagName2=tagValue2,tagName3=tagValue3", match: true},
@@ -73,11 +73,19 @@ func TestTagsFilterString(t *testing.T) {
 		"tagName1": "tagValue1",
 		"tagName2": "tagValue2",
 	}
-	f, err := NewTagsFilter(filters, NewMockSortedTagIterator, Conjunction)
+	f, err := NewTagsFilter(filters, Conjunction, testTagsFilterOptions())
 	require.NoError(t, err)
 	require.Equal(t, `tagName1:Equals("tagValue1") && tagName2:Equals("tagValue2")`, f.String())
 
-	f, err = NewTagsFilter(filters, NewMockSortedTagIterator, Disjunction)
+	f, err = NewTagsFilter(filters, Disjunction, testTagsFilterOptions())
 	require.NoError(t, err)
 	require.Equal(t, `tagName1:Equals("tagValue1") || tagName2:Equals("tagValue2")`, f.String())
 }
+
+func testTagsFilterOptions() TagsFilterOptions {
+	return TagsFilterOptions{
+		NameTagName:         []byte("name"),
+		NameAndTagsFn:       func(b []byte) ([]byte, []byte, error) { return nil, b, nil },
+		SortedTagIteratorFn: NewMockSortedTagIterator,
+	}
+}

metric/aggregated/types.go

@@ -24,13 +24,13 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/m3db/m3metrics/metric"
+	"github.com/m3db/m3metrics/metric/id"
 	"github.com/m3db/m3metrics/policy"
 )
 
 // Metric is a metric, which is essentially a named value at certain time.
 type Metric struct {
-	metric.ID
+	ID        id.RawID
 	TimeNanos int64
 	Value     float64
 }
@@ -47,7 +47,7 @@ func (m Metric) String() string {
 
 // ChunkedMetric is a metric with a chunked ID.
 type ChunkedMetric struct {
-	metric.ChunkedID
+	id.ChunkedID
 	TimeNanos int64
 	Value     float64
 }
@@ -56,7 +56,7 @@ type ChunkedMetric struct {
 // a metric object).
 type RawMetric interface {
 	// ID is the metric identifier.
-	ID() (metric.ID, error)
+	ID() (id.RawID, error)
 
 	// TimeNanos is the metric timestamp in nanoseconds.
 	TimeNanos() (int64, error)

metric/common.go → metric/id/id.go

@@ -18,17 +18,30 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 // THE SOFTWARE.
 
-package metric
+package id
 
 import "fmt"
 
-// ID is the metric id.
-// TODO(xichen): make ID a union of numeric ID and bytes-backed IDs
-// so we can compress IDs on a per-connection basis.
-type ID []byte
+// ID is a metric id.
+type ID interface {
+	// Bytes returns the raw bytes for this id.
+	Bytes() []byte
 
-// String is the string representation of an id.
-func (id ID) String() string { return string(id) }
+	// TagValue looks up the tag value for a tag name.
+	TagValue(tagName []byte) ([]byte, bool)
+}
+
+// NameAndTagsFn returns the name and the tag pairs given an id.
+type NameAndTagsFn func(id []byte) ([]byte, []byte, error)
+
+// NewIDFn creates a new metric ID based on the metric name and metric tag pairs.
+type NewIDFn func(name []byte, tags []TagPair) []byte
+
+// RawID is the raw metric id.
+type RawID []byte
+
+// String is the string representation of a raw id.
+func (rid RawID) String() string { return string(rid) }
 
 // ChunkedID is a three-part id.
 type ChunkedID struct {