resetting again

cmd/sf/sf.go

@@ -33,10 +33,7 @@ import (
 	"github.com/richardlehane/siegfried/pkg/core"
 	"github.com/richardlehane/siegfried/pkg/reader"
 	"github.com/richardlehane/siegfried/pkg/writer"
-	/*// Uncomment to build with profiler
-	"net/http"
-	_ "net/http/pprof"
-	*/)
+)
 
 // defaults
 const maxMulti = 1024
@@ -309,10 +306,6 @@ func replayFile(path string, ctxts chan *context, w writer.Writer) error {
 
 func main() {
 	flag.Parse()
-	/*//UNCOMMENT TO RUN PROFILER
-	go func() {
-		log.Println(http.ListenAndServe("localhost:6060", nil))
-	}()*/
 	// configure home
 	if *home != config.Home() {
 		config.SetHome(*home)

internal/bytematcher/bytematcher.go

@@ -168,9 +168,9 @@ func Add(c core.Matcher, ss core.SignatureSet, priorities priority.List) (core.M
 //       fmt.Print("Success! It is signature 0!")
 //     }
 //   }
-func (b *Matcher) Identify(name string, sb *siegreader.Buffer, exclude ...int) (chan core.Result, error) {
+func (b *Matcher) Identify(name string, sb *siegreader.Buffer, hints ...core.Hint) (chan core.Result, error) {
 	quit, ret := make(chan struct{}), make(chan core.Result)
-	go b.identify(sb, quit, ret, exclude...)
+	go b.identify(sb, quit, ret, hints...)
 	return ret, nil
 }
 
@@ -236,12 +236,12 @@ func (b *Matcher) TestTreeLen() int {
 }
 
 func (b *Matcher) DescribeKeyFrames(i int) []string {
-	if i< 0 || i >= len(b.keyFrames)  {
+	if i < 0 || i >= len(b.keyFrames) {
 		return nil
 	}
 	ret := make([]string, len(b.keyFrames[i]))
 	for j := range ret {
-		ret[j] = b.keyFrames[i][j].String() 
+		ret[j] = b.keyFrames[i][j].String()
 	}
 	return ret
 }

internal/bytematcher/identify.go

@@ -24,14 +24,21 @@ import (
 )
 
 // identify function - brings a new matcher into existence
-func (b *Matcher) identify(buf *siegreader.Buffer, quit chan struct{}, r chan core.Result, exclude ...int) {
+func (b *Matcher) identify(buf *siegreader.Buffer, quit chan struct{}, r chan core.Result, hints ...core.Hint) {
 	buf.Quit = quit
-	waitSet := b.priorities.WaitSet(exclude...)
-	var maxBOF, maxEOF int
-	if len(exclude) > 0 {
-		maxBOF, maxEOF = waitSet.MaxOffsets()
-	} else {
-		maxBOF, maxEOF = b.maxBOF, b.maxEOF
+	waitSet := b.priorities.WaitSet(hints...)
+	maxBOF, maxEOF := b.maxBOF, b.maxEOF
+	if len(hints) > 0 {
+		var hasExclude bool
+		for _, h := range hints {
+			if h.Pivot == nil {
+				hasExclude = true
+				break
+			}
+		}
+		if hasExclude {
+			maxBOF, maxEOF = waitSet.MaxOffsets()
+		}
 	}
 	incoming := b.scorer(buf, waitSet, quit, r)
 	rdr := siegreader.LimitReaderFrom(buf, maxBOF)

internal/containermatcher/container.go

@@ -116,20 +116,19 @@ func (m Matcher) addSigs(i int, nameParts [][]string, sigParts [][]frames.Signat
 	}
 	// give as a starting index the current total of persists in the matcher, except those in the ContainerMatcher in question
 	m[i].startIndexes = append(m[i].startIndexes, m.total(i))
-	prev := len(m[i].parts)
 	for j, n := range nameParts {
 		err = m[i].addSignature(n, sigParts[j])
 		if err != nil {
 			return err
 		}
 	}
-	m[i].priorities.Add(l, len(nameParts), 0, 0)
 	for _, v := range m[i].nameCTest {
-		err := v.commit(l, prev)
+		err = v.commit()
 		if err != nil {
 			return err
 		}
 	}
+	m[i].priorities.Add(l, len(nameParts), 0, 0)
 	return nil
 }
 
@@ -292,32 +291,13 @@ func (ct *cTest) add(s frames.Signature, t int) {
 	ct.buffer = append(ct.buffer, s)
 }
 
-// call for each key after all persists added
-func (ct *cTest) commit(p priority.List, prev int) error {
+// call for each key after all signatures added
+func (ct *cTest) commit() error {
 	if ct.buffer == nil {
 		return nil
 	}
-	// don't set priorities if any of the persists are identical
-	var dupes bool
 	var err error
-	for i, v := range ct.buffer {
-		if i == len(ct.buffer)-1 {
-			break
-		}
-		for _, v2 := range ct.buffer[i+1:] {
-			if v.Equals(v2) {
-				dupes = true
-				break
-			}
-		}
-	}
-	if dupes {
-		ct.bm, _, err = bytematcher.Add(ct.bm, bytematcher.SignatureSet(ct.buffer), nil)
-		ct.bm.(*bytematcher.Matcher).SetLowMem()
-		ct.buffer = nil
-		return err
-	}
-	ct.bm, _, err = bytematcher.Add(ct.bm, bytematcher.SignatureSet(ct.buffer), nil)
+	ct.bm, _, err = bytematcher.Add(ct.bm, bytematcher.SignatureSet(ct.buffer), nil) // don't need to add priorities
 	ct.bm.(*bytematcher.Matcher).SetLowMem()
 	ct.buffer = nil
 	return err

internal/containermatcher/identify.go

@@ -24,7 +24,7 @@ import (
 	"github.com/richardlehane/siegfried/pkg/core"
 )
 
-func (m Matcher) Identify(n string, b *siegreader.Buffer, exclude ...int) (chan core.Result, error) {
+func (m Matcher) Identify(n string, b *siegreader.Buffer, hints ...core.Hint) (chan core.Result, error) {
 	res := make(chan core.Result)
 	// check trigger
 	buf, err := b.Slice(0, 8)

internal/identifier/base.go

@@ -17,6 +17,7 @@ package identifier
 import (
 	"fmt"
 	"strings"
+	"sync"
 
 	"github.com/richardlehane/siegfried/internal/bytematcher"
 	"github.com/richardlehane/siegfried/internal/bytematcher/frames"
@@ -39,34 +40,50 @@ type Base struct {
 	details                                  string
 	multi                                    config.Multi
 	zipDefault                               bool
-	gids, mids, cids, xids, bids, rids, tids indexes
+	gids, mids, cids, xids, bids, rids, tids *indexes
 }
 
 type indexes struct {
-	start int
-	ids   []string
+	start  int
+	ids    []string
+	once   sync.Once
+	lookup map[string][]int
 }
 
-func (ii indexes) hit(i int) (bool, string) {
+func (ii *indexes) find(ks []string) []int {
+	ii.once.Do(func() {
+		ii.lookup = make(map[string][]int)
+		for i, v := range ii.ids {
+			ii.lookup[v] = append(ii.lookup[v], ii.start+i)
+		}
+	})
+	ret := make([]int, 0, len(ks)*2)
+	for _, k := range ks {
+		ret = append(ret, ii.lookup[k]...)
+	}
+	return ret
+}
+
+func (ii *indexes) hit(i int) (bool, string) {
 	if i >= ii.start && i < ii.start+len(ii.ids) {
 		return true, ii.ids[i-ii.start]
 	}
 	return false, ""
 }
 
-func (ii indexes) first(i int) (bool, string) {
+func (ii *indexes) first(i int) (bool, string) {
 	if i == ii.start && len(ii.ids) > 0 {
 		return true, ii.ids[0]
 	}
 	return false, ""
 }
 
-func (ii indexes) save(ls *persist.LoadSaver) {
+func (ii *indexes) save(ls *persist.LoadSaver) {
 	ls.SaveInt(ii.start)
 	ls.SaveStrings(ii.ids)
 }
 
-func (ii indexes) place(i int) (int, int) {
+func (ii *indexes) place(i int) (int, int) {
 	if i >= ii.start && i < ii.start+len(ii.ids) {
 		idx, id := i-ii.start, ii.ids[i-ii.start]
 		var prev, post int
@@ -81,10 +98,10 @@ func (ii indexes) place(i int) (int, int) {
 	return -1, -1
 }
 
-func loadIndexes(ls *persist.LoadSaver) indexes {
-	return indexes{
-		ls.LoadInt(),
-		ls.LoadStrings(),
+func loadIndexes(ls *persist.LoadSaver) *indexes {
+	return &indexes{
+		start: ls.LoadInt(),
+		ids:   ls.LoadStrings(),
 	}
 }
 
@@ -95,6 +112,7 @@ func New(p Parseable, zip string, extra ...string) *Base {
 		details:    config.Details(extra...),
 		multi:      config.GetMulti(),
 		zipDefault: contains(p.IDs(), zip),
+		gids:       &indexes{}, mids: &indexes{}, cids: &indexes{}, xids: &indexes{}, bids: &indexes{}, rids: &indexes{}, tids: &indexes{},
 	}
 }
 
@@ -257,6 +275,27 @@ func (b *Base) Place(m core.MatcherType, idx int) (int, int) {
 	}
 }
 
+func (b *Base) Lookup(m core.MatcherType, keys []string) []int {
+	switch m {
+	default:
+		return nil
+	case core.NameMatcher:
+		return b.gids.find(keys)
+	case core.MIMEMatcher:
+		return b.mids.find(keys)
+	case core.ContainerMatcher:
+		return b.cids.find(keys)
+	case core.XMLMatcher:
+		return b.xids.find(keys)
+	case core.ByteMatcher:
+		return b.bids.find(keys)
+	case core.RIFFMatcher:
+		return b.rids.find(keys)
+	case core.TextMatcher:
+		return b.tids.find(keys)
+	}
+}
+
 func (b *Base) Recognise(m core.MatcherType, idx int) (bool, string) {
 	h, id := b.Hit(m, idx)
 	if h {

internal/mimematcher/mimematcher.go

@@ -109,7 +109,7 @@ func (m Matcher) add(s string, fmt int) {
 }
 
 // Identify tests the supplied MIME-type against the MIMEMatcher. The Buffer is not used.
-func (m Matcher) Identify(s string, na *siegreader.Buffer, exclude ...int) (chan core.Result, error) {
+func (m Matcher) Identify(s string, na *siegreader.Buffer, hints ...core.Hint) (chan core.Result, error) {
 	var (
 		fmts, tfmts []int
 		idx         int

internal/namematcher/namematcher.go

@@ -153,7 +153,7 @@ func normalise(s string) (string, string) {
 	return base, strings.ToLower(strings.TrimPrefix(filepath.Ext(base), "."))
 }
 
-func (m *Matcher) Identify(s string, na *siegreader.Buffer, exclude ...int) (chan core.Result, error) {
+func (m *Matcher) Identify(s string, na *siegreader.Buffer, hints ...core.Hint) (chan core.Result, error) {
 	var efmts, gfmts []int
 	base, ext := normalise(s)
 	var glob string