playing around

asset/memory/main.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"runtime"
+	"sync"
+	"time"
+)
+
+func main() {
+	pageSize := 1024
+	totalSize := pageSize*2_000_000
+	// Assume 1kb per log entry
+	// 1024 bytes to KB
+	// 1000000 means about 1GB
+	fmt.Println(totalSize, "bytes")
+	mySlice := make([]byte, totalSize)
+
+	// warm to ensure timing is fair
+	for j := 0; j < 5; j++ {
+		for i := 0; i < len(mySlice); i += pageSize {
+			bytes.Index(mySlice[i:i+pageSize], []byte("some sort of thing"))
+		}
+	}
+
+	start := time.Now().UnixNano() / int64(time.Millisecond)
+	for i := 0; i < len(mySlice); i += pageSize {
+		bytes.Index(mySlice[i:i+pageSize], []byte("some sort of thing"))
+	}
+	fmt.Println("Single Threaded", time.Now().UnixNano() / int64(time.Millisecond)-start, "ms")
+
+	// https://boyter.org/posts/my-personal-complaints-about-golang/
+	// var input = make(chan []byte, len(toProcess))
+
+	start = time.Now().UnixNano() / int64(time.Millisecond)
+	// Divide the totalsize by number of CPU
+	cpuCount := runtime.NumCPU()
+	chunk := totalSize / cpuCount
+	var wg sync.WaitGroup
+	for i := 0; i < cpuCount; i++ {
+		wg.Add(1)
+
+		// Spawn a routine to search this chunk
+		go func(i int, j int) {
+			for k := i; k < j; k += pageSize {
+				bytes.Index(mySlice[k:k+pageSize], []byte("some sort of thing"))
+			}
+			wg.Done()
+		}(i * chunk, i * chunk + chunk)
+	}
+	wg.Wait()
+	fmt.Println("Multi Threaded", time.Now().UnixNano() / int64(time.Millisecond)-start, "ms")
+}

processor/arguments.go

@@ -27,13 +27,13 @@ var IncludeMinified = false
 var MinifiedLineByteLength = 255
 
 // Disables .gitignore checks
-var GitIgnore = false
+var IgnoreGitIgnore = false
 
 // Disables ignore file checks
-var Ignore = false
+var IgnoreIgnoreFile = false
 
-// DisableCheckBinary toggles checking for binary files using NUL bytes
-var DisableCheckBinary = false
+// IncludeBinaryFiles toggles checking for binary files using NUL bytes
+var IncludeBinaryFiles = false
 
 // Format sets the output format of the formatter
 var Format = ""
@@ -67,6 +67,9 @@ var ResultLimit int64 = 0
 // How many characters out of the file to display in snippets
 var SnippetLength int64 = 0
 
+// Include hidden files and directories in search
+var IncludeHidden = false
+
 var Address string = ":8080"
 var HttpServer bool = false
 var SearchPDF = false

processor/http.go

@@ -47,7 +47,6 @@ type facet struct {
 }
 
 func StartHttpServer() {
-
 	http.HandleFunc("/file/", func(w http.ResponseWriter, r *http.Request) {
 		startTime := makeTimestampMilli()
 		startPos := tryParseInt(r.URL.Query().Get("sp"), 0)
@@ -147,7 +146,9 @@ func StartHttpServer() {
 
 		fileWalker := file.NewFileWalker(directory, fileQueue)
 		fileWalker.PathExclude = PathDenylist
-		fileWalker.EnableIgnoreFile = true
+		fileWalker.IgnoreIgnoreFile = IgnoreIgnoreFile
+		fileWalker.IgnoreGitIgnore = IgnoreGitIgnore
+		fileWalker.IncludeHidden = IncludeHidden
 
 		fileReader := NewFileReaderWorker(fileQueue, toProcessQueue)
 		fileReader.SearchPDF = SearchPDF
@@ -156,7 +157,8 @@ func StartHttpServer() {
 		fileSearcher.SearchString = strings.Split(strings.TrimSpace(query), " ")
 		fileSearcher.IncludeMinified = IncludeMinified
 		fileSearcher.CaseSensitive = CaseSensitive
-		fileSearcher.IncludeBinary = DisableCheckBinary
+		fileSearcher.IncludeBinary = IncludeBinaryFiles
+
 
 		go fileWalker.Start()
 		go fileReader.Start()
@@ -320,5 +322,5 @@ func StartHttpServer() {
 
 	})
 
-	log.Fatal(http.ListenAndServe(":8081", nil))
+	log.Fatal(http.ListenAndServe(Address, nil))
 }

processor/processor.go

@@ -58,7 +58,9 @@ func (process *Process) StartProcess() {
 
 	fileWalker := file.NewFileWalker(process.Directory, fileQueue)
 	fileWalker.PathExclude = PathDenylist
-	fileWalker.EnableIgnoreFile = true
+	fileWalker.IgnoreIgnoreFile = IgnoreIgnoreFile
+	fileWalker.IgnoreGitIgnore = IgnoreGitIgnore
+	fileWalker.IncludeHidden = IncludeHidden
 
 	fileReader := NewFileReaderWorker(fileQueue, toProcessQueue)
 	fileReader.SearchPDF = SearchPDF
@@ -67,7 +69,7 @@ func (process *Process) StartProcess() {
 	fileSearcher.SearchString = SearchString
 	fileSearcher.IncludeMinified = IncludeMinified
 	fileSearcher.CaseSensitive = CaseSensitive
-	fileSearcher.IncludeBinary = DisableCheckBinary
+	fileSearcher.IncludeBinary = IncludeBinaryFiles
 
 	resultSummarizer := NewResultSummarizer(summaryQueue)
 	resultSummarizer.FileReaderWorker = &fileReader

processor/tui.go

@@ -71,10 +71,13 @@ func tuiSearch(app *tview.Application, textView *tview.TextView, searchTerm stri
 	summaryQueue := make(chan *fileJob, runtime.NumCPU())   // Files that match and need to be displayed
 
 	tuiFileWalker = file.NewFileWalker(startDirectory, fileQueue)
-	tuiFileWalker.EnableIgnoreFile = true
+	tuiFileWalker.IgnoreIgnoreFile = IgnoreIgnoreFile
+	tuiFileWalker.IgnoreGitIgnore = IgnoreGitIgnore
+	tuiFileWalker.IncludeHidden = IncludeHidden
 	tuiFileWalker.PathExclude = PathDenylist
 	tuiFileWalker.InstanceId = instanceCount
 
+
 	tuiFileReaderWorker = NewFileReaderWorker(fileQueue, toProcessQueue)
 	tuiFileReaderWorker.InstanceId = instanceCount
 	tuiFileReaderWorker.SearchPDF = SearchPDF
@@ -83,6 +86,7 @@ func tuiSearch(app *tview.Application, textView *tview.TextView, searchTerm stri
 	tuiSearcherWorker.SearchString = SearchString
 	tuiSearcherWorker.MatchLimit = 100
 	tuiSearcherWorker.InstanceId = instanceCount
+	tuiSearcherWorker.IncludeBinary = IncludeBinaryFiles
 
 	instanceCount++
 

processor/woker_searcher.go

@@ -38,6 +38,7 @@ func (f *SearcherWorker) Start() {
 	f.searchParams = parseArguments(f.SearchString)
 
 	for res := range f.input {
+
 		// Check for the presence of a null byte indicating that this
 		// is likely a binary file and if so ignore it
 		if !f.IncludeBinary {