From 9acfa417a23b2d94ce9d3a0e09bf4305242db6b9 Mon Sep 17 00:00:00 2001 From: Stefan Hengl Date: Thu, 4 Mar 2021 13:11:26 +0100 Subject: [PATCH 1/7] search: add support for streaming search This adds the flag `-stream` to the `search` sub-command. Searches submitted with `-stream` are routed to the endpoint `search/stream` instead of the GraphQL api. In addition, users can set `-display ` to set the display limit. We will add support for JSON in a follow-up PR. --- cmd/src/format.go | 10 + cmd/src/search.go | 16 ++ cmd/src/search_alert.go | 11 +- cmd/src/search_alert_test.go | 5 +- cmd/src/search_stream.go | 413 +++++++++++++++++++++++++++++++++++ internal/api/flags.go | 14 ++ 6 files changed, 461 insertions(+), 8 deletions(-) create mode 100644 cmd/src/search_stream.go diff --git a/cmd/src/format.go b/cmd/src/format.go index fa37432932..8ad63edc1f 100644 --- a/cmd/src/format.go +++ b/cmd/src/format.go @@ -60,6 +60,9 @@ func parseTemplate(text string) (*template.Template, error) { "addFloat": func(x, y float64) float64 { return x + y }, + "addInt32": func(x, y int32) int32 { + return x + y + }, "debug": func(v interface{}) string { data, _ := marshalIndent(v) fmt.Println(string(data)) @@ -91,6 +94,13 @@ func parseTemplate(text string) (*template.Template, error) { "buildVersionHasNewSearchInterface": searchTemplateFuncs["buildVersionHasNewSearchInterface"], "renderResult": searchTemplateFuncs["renderResult"], + // Register stream-search specific template functions. + "streamSearchSequentialLineNumber": streamSearchTemplateFuncs["streamSearchSequentialLineNumber"], + "streamSearchHighlightMatch": streamSearchTemplateFuncs["streamSearchHighlightMatch"], + "streamSearchHighlightCommit": streamSearchTemplateFuncs["streamSearchHighlightCommit"], + "streamSearchRenderCommitLabel": streamSearchTemplateFuncs["streamSearchRenderCommitLabel"], + "matchOrMatches": streamSearchTemplateFuncs["matchOrMatches"], + // Alert rendering "searchAlertRender": func(alert searchResultsAlert) string { if content, err := alert.Render(); err != nil { diff --git a/cmd/src/search.go b/cmd/src/search.go index a7e0bb3c5e..4450a52fa4 100644 --- a/cmd/src/search.go +++ b/cmd/src/search.go @@ -54,6 +54,10 @@ Other tips: explainJSONFlag = flagSet.Bool("explain-json", false, "Explain the JSON output schema and exit.") apiFlags = api.NewFlags(flagSet) lessFlag = flagSet.Bool("less", true, "Pipe output to 'less -R' (only if stdout is terminal, and not json flag)") + streamFlag = flagSet.Bool("stream", false, "Consume results as stream.") + + // Streaming. + _ = flagSet.Int("display", -1, "Limit the number of results shown. Only supported for streaming.") ) handler := func(args []string) error { @@ -61,6 +65,18 @@ Other tips: return err } + if *streamFlag { + // Remove -stream from args. + argsWOStream := make([]string, 0, len(args)-1) + for _, a := range args { + if a == "-stream" { + continue + } + argsWOStream = append(argsWOStream, a) + } + return streamHandler(argsWOStream) + } + if *explainJSONFlag { fmt.Println(searchJSONExplanation) return nil diff --git a/cmd/src/search_alert.go b/cmd/src/search_alert.go index 443d0c4c5e..998b3817aa 100644 --- a/cmd/src/search_alert.go +++ b/cmd/src/search_alert.go @@ -19,15 +19,18 @@ func init() { } } +// ProposedQuery is a suggested query to run when we emit an alert. +type ProposedQuery struct { + Description string + Query string +} + // searchResultsAlert is a type that can be used to unmarshal values returned by // the searchResultsAlertFragment GraphQL fragment below. type searchResultsAlert struct { Title string Description string - ProposedQueries []struct { - Description string - Query string - } + ProposedQueries []ProposedQuery } // Render renders an alert to a string ready to be output to a console, diff --git a/cmd/src/search_alert_test.go b/cmd/src/search_alert_test.go index 3c8e76accd..e23016c1d0 100644 --- a/cmd/src/search_alert_test.go +++ b/cmd/src/search_alert_test.go @@ -13,10 +13,7 @@ func TestRender(t *testing.T) { full := &searchResultsAlert{ Title: "foo", Description: "bar", - ProposedQueries: []struct { - Description string - Query string - }{ + ProposedQueries: []ProposedQuery{ { Description: "quux", Query: "xyz:abc", diff --git a/cmd/src/search_stream.go b/cmd/src/search_stream.go new file mode 100644 index 0000000000..0576ae9a02 --- /dev/null +++ b/cmd/src/search_stream.go @@ -0,0 +1,413 @@ +package main + +import ( + "bytes" + "context" + "flag" + "fmt" + "net/url" + "os" + "os/exec" + "regexp" + "strconv" + "strings" + + "github.com/sourcegraph/src-cli/internal/api" + "github.com/sourcegraph/src-cli/internal/streaming" +) + +var labelRegexp *regexp.Regexp + +func init() { + labelRegexp, _ = regexp.Compile("(?:\\[)(.*?)(?:])") +} + +// streamHandler handles search requests which contain the flag "stream". +// Requests are sent to search/stream instead of the GraphQL api. +func streamHandler(args []string) error { + flagSet := flag.NewFlagSet("streaming search", flag.ExitOnError) + var ( + display = flagSet.Int("display", -1, "Limit the number of results that are displayed. Note that the statistics continue to report all results.") + apiFlags = api.StreamingFlags(flagSet) + ) + if err := flagSet.Parse(args); err != nil { + return err + } + + query := flagSet.Arg(0) + + t, err := parseTemplate(streamingTemplate) + if err != nil { + panic(err) + } + + // Create request. + client := cfg.apiClient(apiFlags, flagSet.Output()) + req, err := client.NewHTTPRequest(context.Background(), "GET", "search/stream?q="+url.QueryEscape(query), nil) + if err != nil { + return err + } + req.Header.Set("Accept", "text/event-stream") + if *display >= 0 { + q := req.URL.Query() + q.Add("display", strconv.Itoa(*display)) + req.URL.RawQuery = q.Encode() + } + + // Send request. + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("error sending request: %w", err) + } + defer resp.Body.Close() + + // Process response. + err = streaming.Decoder{ + OnProgress: func(progress *streaming.Progress) { + // We only show the final progress. + if !progress.Done { + return + } + err = t.ExecuteTemplate(os.Stdout, "progress", progress) + if err != nil { + fmt.Printf("ExecuteTemplate: %s\n", err) + } + return + }, + OnError: func(eventError *streaming.EventError) { + fmt.Printf("ERR: %s", eventError.Message) + }, + OnAlert: func(alert *streaming.EventAlert) { + proposedQueries := make([]ProposedQuery, len(alert.ProposedQueries)) + for _, pq := range alert.ProposedQueries { + proposedQueries = append(proposedQueries, ProposedQuery{ + Description: pq.Description, + Query: pq.Query, + }) + } + + err = t.ExecuteTemplate(os.Stdout, "alert", searchResultsAlert{ + Title: alert.Title, + Description: alert.Description, + ProposedQueries: proposedQueries, + }) + if err != nil { + fmt.Printf("ExecuteTemplate: %s\n", err) + return + } + }, + OnMatches: func(matches []streaming.EventMatch) { + for _, match := range matches { + if file, ok := match.(*streaming.EventFileMatch); ok { + err = t.ExecuteTemplate(os.Stdout, "file", struct { + Query string + *streaming.EventFileMatch + }{ + Query: query, + EventFileMatch: file, + }, + ) + if err != nil { + fmt.Printf("ExecuteTemplate: %s\n", err) + return + } + continue + } + + if repo, ok := match.(*streaming.EventRepoMatch); ok { + err = t.ExecuteTemplate(os.Stdout, "repo", struct { + SourcegraphEndpoint string + *streaming.EventRepoMatch + }{ + SourcegraphEndpoint: cfg.Endpoint, + EventRepoMatch: repo, + }) + if err != nil { + fmt.Printf("ExecuteTemplate: %s\n", err) + return + } + continue + } + + if commit, ok := match.(*streaming.EventCommitMatch); ok { + err = t.ExecuteTemplate(os.Stdout, "commit", struct { + SourcegraphEndpoint string + *streaming.EventCommitMatch + }{ + SourcegraphEndpoint: cfg.Endpoint, + EventCommitMatch: commit, + }) + if err != nil { + fmt.Printf("ExecuteTemplate: %s\n", err) + return + } + continue + } + + if symbol, ok := match.(*streaming.EventSymbolMatch); ok { + err = t.ExecuteTemplate(os.Stdout, "symbol", struct { + SourcegraphEndpoint string + *streaming.EventSymbolMatch + }{ + SourcegraphEndpoint: cfg.Endpoint, + EventSymbolMatch: symbol, + }, + ) + if err != nil { + fmt.Printf("ExecuteTemplate: %s\n", err) + return + } + continue + } + } + }, + }.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("error during decoding: %w", err) + } + + // Write trace to output. + if apiFlags.Trace() { + _, err := flagSet.Output().Write([]byte(fmt.Sprintf("x-trace: %s\n", resp.Header.Get("x-trace")))) + if err != nil { + return err + } + } + + return nil +} + +const streamingTemplate = ` +{{define "file"}} + {{- /* Repository and file name */ -}} + {{- color "search-repository"}}{{.Repository}}{{color "nc" -}} + {{- " › " -}} + {{- color "search-filename"}}{{.Path}}{{color "nc" -}} + {{- color "success"}}{{matchOrMatches (len .LineMatches)}}{{color "nc" -}} + {{- "\n" -}} + {{- color "search-border"}}{{"--------------------------------------------------------------------------------\n"}}{{color "nc"}} + + {{- /* Line matches */ -}} + {{- $lineMatches := .LineMatches -}} + {{- range $index, $match := $lineMatches -}} + {{- if not (streamSearchSequentialLineNumber $lineMatches $index) -}} + {{- color "search-border"}}{{" ------------------------------------------------------------------------------\n"}}{{color "nc"}} + {{- end -}} + {{- " "}}{{color "search-line-numbers"}}{{pad (addInt32 $match.LineNumber 1) 6 " "}}{{color "nc" -}} + {{- color "search-border"}}{{" | "}}{{color "nc"}}{{streamSearchHighlightMatch $.Query $match }} + {{- end -}} + {{- "\n" -}} +{{- end -}} + +{{define "symbol"}} + {{- /* Repository and file name */ -}} + {{- color "search-repository"}}{{.Repository}}{{color "nc" -}} + {{- " › " -}} + {{- color "search-filename"}}{{.Path}}{{color "nc" -}} + {{- color "success"}}{{matchOrMatches (len .Symbols)}}{{color "nc" -}} + {{- "\n" -}} + {{- color "search-border"}}{{"--------------------------------------------------------------------------------\n"}}{{color "nc"}} + + {{- /* Symbols */ -}} + {{- $symbols := .Symbols -}} + {{- range $index, $match := $symbols -}} + {{- color "success"}}{{.Name}}{{color "nc" -}} ({{.Kind}}{{if .ContainerName}}{{printf ", %s" .ContainerName}}{{end}}) + {{- color "search-border"}}{{" ("}}{{color "nc" -}} + {{- color "search-repository"}}{{$.SourcegraphEndpoint}}/{{$match.URL}}{{color "nc" -}} + {{- color "search-border"}}{{")\n"}}{{color "nc" -}} + {{- end -}} + {{- "\n" -}} +{{- end -}} + +{{define "repo"}} + {{- /* Link to the result */ -}} + {{- color "success"}}{{.Repository}}{{color "nc" -}} + {{- color "search-border"}}{{" ("}}{{color "nc" -}} + {{- color "search-repository"}}{{$.SourcegraphEndpoint}}/{{.Repository}}{{color "nc" -}} + {{- color "search-border"}}{{")"}}{{color "nc" -}} + {{- color "success"}}{{" ("}}{{"1 match)"}}{{color "nc" -}} + {{- "\n" -}} +{{- end -}} + +{{define "commit"}} + {{- /* Link to the result */ -}} + {{- color "search-border"}}{{"("}}{{color "nc" -}} + {{- color "search-link"}}{{$.SourcegraphEndpoint}}{{.URL}}{{color "nc" -}} + {{- color "search-border"}}{{")\n"}}{{color "nc" -}} + {{- color "nc" -}} + + {{- /* Repository > author name "commit subject" (time ago) */ -}} + {{- color "search-commit-subject"}}{{(streamSearchRenderCommitLabel .Label)}}{{color "nc" -}} + {{- color "success"}}{{matchOrMatches (len .Ranges)}}{{color "nc" -}} + {{- "\n" -}} + {{- color "search-border"}}{{"--------------------------------------------------------------------------------\n"}}{{color "nc"}} + {{- color "search-border"}}{{color "nc"}}{{indent (streamSearchHighlightCommit .Content .Ranges) " "}} +{{end}} + +{{define "alert"}} + {{- searchAlertRender . -}} +{{end}} + +{{define "progress"}} + {{- color "logo" -}}✱{{- color "nc" -}} + {{- " " -}} + {{- if eq .MatchCount 0 -}} + {{- color "warning" -}} + {{- else -}} + {{- color "success" -}} + {{- end -}} + {{- .MatchCount -}}{{if len .Skipped}}+{{end}} results{{- color "nc" -}} + {{- " in " -}}{{color "success"}}{{msDuration .DurationMs}}{{if .RepositoriesCount}}{{- color "nc" -}} + {{- " from " -}}{{color "success"}}{{.RepositoriesCount}}{{- " Repositories" -}}{{- color "nc" -}}{{end}} + {{- "\n" -}} + {{if len .Skipped}} + {{- "\n" -}} + {{- "Some results excluded:" -}} + {{- "\n" -}} + {{- range $index, $skipped := $.Skipped -}} + {{indent $skipped.Title " "}}{{- "\n" -}} + {{- end -}} + {{- "\n" -}} + {{- end -}} +{{- end -}} +` + +var streamSearchTemplateFuncs = map[string]interface{}{ + "streamSearchHighlightMatch": func(query string, match streaming.EventLineMatch) string { + var highlights []highlight + if strings.Contains(query, "patterntype:structural") { + highlights = streamConvertMatchToHighlights(match, false) + return applyHighlightsForFile(match.Line, highlights) + } + + highlights = streamConvertMatchToHighlights(match, true) + return applyHighlights(match.Line, highlights, ansiColors["search-match"], ansiColors["nc"]) + }, + + "streamSearchSequentialLineNumber": func(lineMatches []streaming.EventLineMatch, index int) bool { + prevIndex := index - 1 + if prevIndex < 0 { + return true + } + prevLineNumber := lineMatches[prevIndex].LineNumber + lineNumber := lineMatches[index].LineNumber + return prevLineNumber == lineNumber-1 + }, + + "streamSearchHighlightCommit": func(content string, ranges [][3]int32) string { + highlights := make([]highlight, len(ranges)) + for _, r := range ranges { + highlights = append(highlights, highlight{ + line: int(r[0]), + character: int(r[1]), + length: int(r[2]), + }) + } + if strings.HasPrefix(content, "```diff") { + return streamSearchHighlightDiffPreview(content, highlights) + } + return applyHighlights(stripMarkdownMarkers(content), highlights, ansiColors["search-match"], ansiColors["nc"]) + }, + + "streamSearchRenderCommitLabel": func(label string) string { + m := labelRegexp.FindAllStringSubmatch(label, -1) + if len(m) != 3 || len(m[0]) < 2 || len(m[1]) < 2 || len(m[2]) < 2 { + return label + } + return m[0][1] + " > " + m[1][1] + " : " + m[2][1] + }, + + "matchOrMatches": func(i int) string { + if i == 1 { + return " (1 match)" + } + return fmt.Sprintf(" (%d matches)", i) + }, +} + +func streamSearchHighlightDiffPreview(diffPreview string, highlights []highlight) string { + useColordiff, err := strconv.ParseBool(os.Getenv("COLORDIFF")) + if err != nil { + useColordiff = true + } + if colorDisabled || !useColordiff { + // Only highlight the matches. + return applyHighlights(stripMarkdownMarkers(diffPreview), highlights, ansiColors["search-match"], ansiColors["nc"]) + } + path, err := exec.LookPath("colordiff") + if err != nil { + // colordiff not installed; only highlight the matches. + return applyHighlights(stripMarkdownMarkers(diffPreview), highlights, ansiColors["search-match"], ansiColors["nc"]) + } + + // First highlight the matches, but use a special "end of match" token + // instead of no color (so that we don'streamingTemplate terminate colors that colordiff + // adds). + uniqueStartOfMatchToken := "pXRdMhZbgnPL355429nsO4qFgX86LfXTSmqH4Nr3#*(@)!*#()@!APPJB8ZRutvZ5fdL01273i6OdzLDm0UMC9372891skfJTl2c52yR1v" + uniqueEndOfMatchToken := "v1Ry25c2lTJfks1982739CMU0mDLzdO6i37210Ldf5ZvtuRZ8BJPPA!@)(#*!)@(*#3rN4HqmSTXfL68XgFq4Osn924553LPngbZhMdRXp" + diff := applyHighlights(stripMarkdownMarkers(diffPreview), highlights, uniqueStartOfMatchToken, uniqueEndOfMatchToken) + + // Now highlight our diff with colordiff. + var buf bytes.Buffer + cmd := exec.Command(path) + cmd.Stdin = strings.NewReader(diff) + cmd.Stdout = &buf + if err := cmd.Run(); err != nil { + fmt.Println("warning: colordiff failed to colorize diff:", err) + return diff + } + colorized := buf.String() + var final []string + for _, line := range strings.Split(colorized, "\n") { + // fmt.Println("LINE", line) + // Find where the start-of-match token is in the line. + somToken := strings.Index(line, uniqueStartOfMatchToken) + + // Find which ANSI codes are to the left of our start-of-match token. + indices := ansiRegexp.FindAllStringIndex(line, -1) + matches := ansiRegexp.FindAllString(line, -1) + var left []string + for k, index := range indices { + if index[0] < somToken && index[1] < somToken { + left = append(left, matches[k]) + } + } + + // Replace our start-of-match token with the color we wish. + line = strings.Replace(line, uniqueStartOfMatchToken, ansiColors["search-match"], -1) + + // Replace our end-of-match token with the color terminator, + // and start all colors that were previously started to the left. + line = strings.Replace(line, uniqueEndOfMatchToken, ansiColors["nc"]+strings.Join(left, ""), -1) + + final = append(final, line) + } + return strings.Join(final, "\n") +} + +func stripMarkdownMarkers(content string) string { + content = strings.TrimLeft(content, "```COMMIT_EDITMSG\n") + content = strings.TrimLeft(content, "```diff\n") + return strings.TrimRight(content, "\n```") +} + +// convertMatchToHighlights converts a FileMatch m to a highlight data type. +// When isPreview is true, it is assumed that the result to highlight is only on +// one line, and the offets are relative to this line. When isPreview is false, +// the lineNumber from the FileMatch data is used, which is relative to the file +// content. +func streamConvertMatchToHighlights(m streaming.EventLineMatch, isPreview bool) (highlights []highlight) { + var line int + for _, offsetAndLength := range m.OffsetAndLengths { + ol := offsetAndLength + offset := int(ol[0]) + length := int(ol[1]) + if isPreview { + line = 1 + } else { + line = int(m.LineNumber) + } + highlights = append(highlights, highlight{line: line, character: offset, length: length}) + } + return highlights +} diff --git a/internal/api/flags.go b/internal/api/flags.go index 4a98b6c4f8..38bc371831 100644 --- a/internal/api/flags.go +++ b/internal/api/flags.go @@ -11,6 +11,13 @@ type Flags struct { insecureSkipVerify *bool } +func (f *Flags) Trace() bool { + if f.trace == nil { + return false + } + return *(f.trace) +} + // NewFlags instantiates a new Flags structure and attaches flags to the given // flag set. func NewFlags(flagSet *flag.FlagSet) *Flags { @@ -31,3 +38,10 @@ func defaultFlags() *Flags { insecureSkipVerify: &d, } } + +func StreamingFlags(flagSet *flag.FlagSet) *Flags { + return &Flags{ + trace: flagSet.Bool("trace", false, "Log the trace ID for requests. See https://docs.sourcegraph.com/admin/observability/tracing"), + insecureSkipVerify: flagSet.Bool("insecure-skip-verify", false, "Skip validation of TLS certificates against trusted chains"), + } +} From 07b7051198725ceb4be73307e057e73d3edebf20 Mon Sep 17 00:00:00 2001 From: Stefan Hengl Date: Thu, 4 Mar 2021 16:13:48 +0100 Subject: [PATCH 2/7] use flagSet.Outpout().Write instead of fmt.Printf --- cmd/src/search_stream.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/cmd/src/search_stream.go b/cmd/src/search_stream.go index 0576ae9a02..2d6a6df14c 100644 --- a/cmd/src/search_stream.go +++ b/cmd/src/search_stream.go @@ -70,7 +70,7 @@ func streamHandler(args []string) error { } err = t.ExecuteTemplate(os.Stdout, "progress", progress) if err != nil { - fmt.Printf("ExecuteTemplate: %s\n", err) + _, _ = flagSet.Output().Write([]byte(fmt.Sprintf("error when executing template: %s\n", err))) } return }, @@ -92,7 +92,7 @@ func streamHandler(args []string) error { ProposedQueries: proposedQueries, }) if err != nil { - fmt.Printf("ExecuteTemplate: %s\n", err) + _, _ = flagSet.Output().Write([]byte(fmt.Sprintf("error when executing template: %s\n", err))) return } }, @@ -100,15 +100,13 @@ func streamHandler(args []string) error { for _, match := range matches { if file, ok := match.(*streaming.EventFileMatch); ok { err = t.ExecuteTemplate(os.Stdout, "file", struct { - Query string *streaming.EventFileMatch }{ - Query: query, EventFileMatch: file, }, ) if err != nil { - fmt.Printf("ExecuteTemplate: %s\n", err) + _, _ = flagSet.Output().Write([]byte(fmt.Sprintf("error when executing template: %s\n", err))) return } continue @@ -123,7 +121,7 @@ func streamHandler(args []string) error { EventRepoMatch: repo, }) if err != nil { - fmt.Printf("ExecuteTemplate: %s\n", err) + _, _ = flagSet.Output().Write([]byte(fmt.Sprintf("error when executing template: %s\n", err))) return } continue @@ -138,7 +136,7 @@ func streamHandler(args []string) error { EventCommitMatch: commit, }) if err != nil { - fmt.Printf("ExecuteTemplate: %s\n", err) + _, _ = flagSet.Output().Write([]byte(fmt.Sprintf("error when executing template: %s\n", err))) return } continue @@ -154,7 +152,7 @@ func streamHandler(args []string) error { }, ) if err != nil { - fmt.Printf("ExecuteTemplate: %s\n", err) + _, _ = flagSet.Output().Write([]byte(fmt.Sprintf("error when executing template: %s\n", err))) return } continue From 2020d662c8c71d29e69c2956b4f14577d6ef65b2 Mon Sep 17 00:00:00 2001 From: Stefan Hengl Date: Thu, 4 Mar 2021 16:52:03 +0100 Subject: [PATCH 3/7] use switch instead of if-continue --- cmd/src/search_stream.go | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/cmd/src/search_stream.go b/cmd/src/search_stream.go index 2d6a6df14c..72a2ea08c1 100644 --- a/cmd/src/search_stream.go +++ b/cmd/src/search_stream.go @@ -98,64 +98,57 @@ func streamHandler(args []string) error { }, OnMatches: func(matches []streaming.EventMatch) { for _, match := range matches { - if file, ok := match.(*streaming.EventFileMatch); ok { + switch match := match.(type) { + case *streaming.EventFileMatch: err = t.ExecuteTemplate(os.Stdout, "file", struct { + Query string *streaming.EventFileMatch }{ - EventFileMatch: file, + Query: query, + EventFileMatch: match, }, ) if err != nil { _, _ = flagSet.Output().Write([]byte(fmt.Sprintf("error when executing template: %s\n", err))) return } - continue - } - - if repo, ok := match.(*streaming.EventRepoMatch); ok { + case *streaming.EventRepoMatch: err = t.ExecuteTemplate(os.Stdout, "repo", struct { SourcegraphEndpoint string *streaming.EventRepoMatch }{ SourcegraphEndpoint: cfg.Endpoint, - EventRepoMatch: repo, + EventRepoMatch: match, }) if err != nil { _, _ = flagSet.Output().Write([]byte(fmt.Sprintf("error when executing template: %s\n", err))) return } - continue - } - - if commit, ok := match.(*streaming.EventCommitMatch); ok { + case *streaming.EventCommitMatch: err = t.ExecuteTemplate(os.Stdout, "commit", struct { SourcegraphEndpoint string *streaming.EventCommitMatch }{ SourcegraphEndpoint: cfg.Endpoint, - EventCommitMatch: commit, + EventCommitMatch: match, }) if err != nil { _, _ = flagSet.Output().Write([]byte(fmt.Sprintf("error when executing template: %s\n", err))) return } - continue - } - - if symbol, ok := match.(*streaming.EventSymbolMatch); ok { + case *streaming.EventSymbolMatch: err = t.ExecuteTemplate(os.Stdout, "symbol", struct { SourcegraphEndpoint string *streaming.EventSymbolMatch }{ SourcegraphEndpoint: cfg.Endpoint, - EventSymbolMatch: symbol, + EventSymbolMatch: match, }, ) if err != nil { _, _ = flagSet.Output().Write([]byte(fmt.Sprintf("error when executing template: %s\n", err))) return } - continue } } }, @@ -391,7 +384,7 @@ func stripMarkdownMarkers(content string) string { // convertMatchToHighlights converts a FileMatch m to a highlight data type. // When isPreview is true, it is assumed that the result to highlight is only on -// one line, and the offets are relative to this line. When isPreview is false, +// one line, and the offsets are relative to this line. When isPreview is false, // the lineNumber from the FileMatch data is used, which is relative to the file // content. func streamConvertMatchToHighlights(m streaming.EventLineMatch, isPreview bool) (highlights []highlight) { From cc73500eef348e9129af0c15844c087362036505 Mon Sep 17 00:00:00 2001 From: Stefan Hengl Date: Fri, 5 Mar 2021 15:30:55 +0100 Subject: [PATCH 4/7] add tests --- cmd/src/search_stream.go | 65 ++++++---- cmd/src/search_stream_test.go | 144 +++++++++++++++++++++ cmd/src/testdata/streaming_search_want.txt | 25 ++++ 3 files changed, 212 insertions(+), 22 deletions(-) create mode 100644 cmd/src/search_stream_test.go create mode 100644 cmd/src/testdata/streaming_search_want.txt diff --git a/cmd/src/search_stream.go b/cmd/src/search_stream.go index 72a2ea08c1..a379b9c704 100644 --- a/cmd/src/search_stream.go +++ b/cmd/src/search_stream.go @@ -5,6 +5,7 @@ import ( "context" "flag" "fmt" + "io" "net/url" "os" "os/exec" @@ -26,31 +27,31 @@ func init() { // Requests are sent to search/stream instead of the GraphQL api. func streamHandler(args []string) error { flagSet := flag.NewFlagSet("streaming search", flag.ExitOnError) - var ( - display = flagSet.Int("display", -1, "Limit the number of results that are displayed. Note that the statistics continue to report all results.") - apiFlags = api.StreamingFlags(flagSet) - ) + flags := newStreamingFlags(flagSet) if err := flagSet.Parse(args); err != nil { return err } + client := cfg.apiClient(flags.apiFlags, flagSet.Output()) query := flagSet.Arg(0) + return doStreamSearch(query, flags, client, os.Stdout) +} +func doStreamSearch(query string, flags *streamingFlags, client api.Client, w io.Writer) error { t, err := parseTemplate(streamingTemplate) if err != nil { panic(err) } // Create request. - client := cfg.apiClient(apiFlags, flagSet.Output()) req, err := client.NewHTTPRequest(context.Background(), "GET", "search/stream?q="+url.QueryEscape(query), nil) if err != nil { return err } req.Header.Set("Accept", "text/event-stream") - if *display >= 0 { + if flags.display >= 0 { q := req.URL.Query() - q.Add("display", strconv.Itoa(*display)) + q.Add("display", strconv.Itoa(flags.display)) req.URL.RawQuery = q.Encode() } @@ -61,6 +62,10 @@ func streamHandler(args []string) error { } defer resp.Body.Close() + logError := func(msg string) { + _, _ = fmt.Fprintf(os.Stderr, msg) + } + // Process response. err = streaming.Decoder{ OnProgress: func(progress *streaming.Progress) { @@ -68,9 +73,9 @@ func streamHandler(args []string) error { if !progress.Done { return } - err = t.ExecuteTemplate(os.Stdout, "progress", progress) + err = t.ExecuteTemplate(w, "progress", progress) if err != nil { - _, _ = flagSet.Output().Write([]byte(fmt.Sprintf("error when executing template: %s\n", err))) + logError(fmt.Sprintf("error when executing template: %s\n", err)) } return }, @@ -86,13 +91,13 @@ func streamHandler(args []string) error { }) } - err = t.ExecuteTemplate(os.Stdout, "alert", searchResultsAlert{ + err = t.ExecuteTemplate(w, "alert", searchResultsAlert{ Title: alert.Title, Description: alert.Description, ProposedQueries: proposedQueries, }) if err != nil { - _, _ = flagSet.Output().Write([]byte(fmt.Sprintf("error when executing template: %s\n", err))) + logError(fmt.Sprintf("error when executing template: %s\n", err)) return } }, @@ -100,7 +105,7 @@ func streamHandler(args []string) error { for _, match := range matches { switch match := match.(type) { case *streaming.EventFileMatch: - err = t.ExecuteTemplate(os.Stdout, "file", struct { + err = t.ExecuteTemplate(w, "file", struct { Query string *streaming.EventFileMatch }{ @@ -109,11 +114,11 @@ func streamHandler(args []string) error { }, ) if err != nil { - _, _ = flagSet.Output().Write([]byte(fmt.Sprintf("error when executing template: %s\n", err))) + logError(fmt.Sprintf("error when executing template: %s\n", err)) return } case *streaming.EventRepoMatch: - err = t.ExecuteTemplate(os.Stdout, "repo", struct { + err = t.ExecuteTemplate(w, "repo", struct { SourcegraphEndpoint string *streaming.EventRepoMatch }{ @@ -121,11 +126,11 @@ func streamHandler(args []string) error { EventRepoMatch: match, }) if err != nil { - _, _ = flagSet.Output().Write([]byte(fmt.Sprintf("error when executing template: %s\n", err))) + logError(fmt.Sprintf("error when executing template: %s\n", err)) return } case *streaming.EventCommitMatch: - err = t.ExecuteTemplate(os.Stdout, "commit", struct { + err = t.ExecuteTemplate(w, "commit", struct { SourcegraphEndpoint string *streaming.EventCommitMatch }{ @@ -133,11 +138,11 @@ func streamHandler(args []string) error { EventCommitMatch: match, }) if err != nil { - _, _ = flagSet.Output().Write([]byte(fmt.Sprintf("error when executing template: %s\n", err))) + logError(fmt.Sprintf("error when executing template: %s\n", err)) return } case *streaming.EventSymbolMatch: - err = t.ExecuteTemplate(os.Stdout, "symbol", struct { + err = t.ExecuteTemplate(w, "symbol", struct { SourcegraphEndpoint string *streaming.EventSymbolMatch }{ @@ -146,7 +151,7 @@ func streamHandler(args []string) error { }, ) if err != nil { - _, _ = flagSet.Output().Write([]byte(fmt.Sprintf("error when executing template: %s\n", err))) + logError(fmt.Sprintf("error when executing template: %s\n", err)) return } } @@ -158,13 +163,12 @@ func streamHandler(args []string) error { } // Write trace to output. - if apiFlags.Trace() { - _, err := flagSet.Output().Write([]byte(fmt.Sprintf("x-trace: %s\n", resp.Header.Get("x-trace")))) + if flags.Trace() { + _, err = fmt.Fprintf(os.Stderr, fmt.Sprintf("x-trace: %s\n", resp.Header.Get("x-trace"))) if err != nil { return err } } - return nil } @@ -402,3 +406,20 @@ func streamConvertMatchToHighlights(m streaming.EventLineMatch, isPreview bool) } return highlights } + +type streamingFlags struct { + apiFlags *api.Flags + display int +} + +func newStreamingFlags(flagSet *flag.FlagSet) *streamingFlags { + flags := &streamingFlags{ + apiFlags: api.StreamingFlags(flagSet), + } + flagSet.IntVar(&flags.display, "display", -1, "Limit the number of results that are displayed. Note that the statistics continue to report all results.") + return flags +} + +func (f *streamingFlags) Trace() bool { + return f.apiFlags.Trace() +} diff --git a/cmd/src/search_stream_test.go b/cmd/src/search_stream_test.go new file mode 100644 index 0000000000..b8c76bab88 --- /dev/null +++ b/cmd/src/search_stream_test.go @@ -0,0 +1,144 @@ +package main + +import ( + "flag" + "io/ioutil" + "net" + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/sourcegraph/src-cli/internal/streaming" +) + +var event []streaming.EventMatch + +func mockStreamHandler(w http.ResponseWriter, _ *http.Request) { + writer, _ := streaming.NewWriter(w) + writer.Event("matches", event) + writer.Event("done", nil) +} + +func testServer(t *testing.T, handler http.Handler) *httptest.Server { + t.Helper() + // We need a stable port, because src-cli output contains references to the host. + // Here we exchange the standard listener with our own. + l, err := net.Listen("tcp", "127.0.0.1:55128") + if err != nil { + t.Fatal(err) + } + s := httptest.NewUnstartedServer(handler) + s.Listener.Close() + s.Listener = l + s.Start() + return s +} + +func TestSearchStream(t *testing.T) { + s := testServer(t, http.HandlerFunc(mockStreamHandler)) + defer s.Close() + + cfg = &config{ + Endpoint: s.URL, + } + defer func() { cfg = nil }() + + event = []streaming.EventMatch{ + &streaming.EventFileMatch{ + Type: streaming.FileMatchType, + Path: "path/to/file", + Repository: "org/repo", + Branches: nil, + Version: "", + LineMatches: []streaming.EventLineMatch{ + { + Line: "foo bar", + LineNumber: 4, + OffsetAndLengths: [][2]int32{{4, 3}}, + }, + }, + }, + &streaming.EventRepoMatch{ + Type: streaming.RepoMatchType, + Repository: "sourcegraph/sourcegraph", + Branches: []string{}, + }, + &streaming.EventSymbolMatch{ + Type: streaming.SymbolMatchType, + Path: "path/to/file", + Repository: "org/repo", + Branches: []string{}, + Version: "", + Symbols: []streaming.Symbol{ + { + URL: "github.com/sourcegraph/sourcegraph/-/blob/cmd/frontend/graphqlbackend/search_results.go#L1591:26-1591:35", + Name: "doResults", + ContainerName: "", + Kind: "FUNCTION", + }, + { + URL: "github.com/sourcegraph/sourcegraph/-/blob/cmd/frontend/graphqlbackend/search_results.go#L1591:26-1591:35", + Name: "Results", + ContainerName: "SearchResultsResolver", + Kind: "FIELD", + }, + }, + }, + &streaming.EventCommitMatch{ + Type: streaming.CommitMatchType, + Icon: "", + Label: "[sourcegraph/sourcegraph-atom](/github.com/sourcegraph/sourcegraph-atom) › [Stephen Gutekanst](/github.com/sourcegraph/sourcegraph-atom/-/commit/5b098d7fed963d88e23057ed99d73d3c7a33ad89): [all: release v1.0.5](/github.com/sourcegraph/sourcegraph-atom/-/commit/5b098d7fed963d88e23057ed99d73d3c7a33ad89)^", + URL: "", + Detail: "", + Content: "```COMMIT_EDITMSG\nfoo bar\n```", + Ranges: [][3]int32{ + {1, 3, 3}, + }, + }, + &streaming.EventCommitMatch{ + Type: streaming.CommitMatchType, + Icon: "", + Label: "[sourcegraph/sourcegraph-atom](/github.com/sourcegraph/sourcegraph-atom) › [Stephen Gutekanst](/github.com/sourcegraph/sourcegraph-atom/-/commit/5b098d7fed963d88e23057ed99d73d3c7a33ad89): [all: release v1.0.5](/github.com/sourcegraph/sourcegraph-atom/-/commit/5b098d7fed963d88e23057ed99d73d3c7a33ad89)^", + URL: "", + Detail: "", + Content: "```diff\nsrc/data.ts src/data.ts\n@@ -0,0 +11,4 @@\n+ return of({\n+ title: 'Acme Corp open-source code search',\n+ summary: 'Instant code search across all Acme Corp open-source code.',\n+ githubOrgs: ['sourcegraph'],\n```", + Ranges: [][3]int32{ + {4, 44, 6}, + }, + }, + } + + // Capture output. + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + + flagSet := flag.NewFlagSet("streaming search test", flag.ExitOnError) + flags := newStreamingFlags(flagSet) + client := cfg.apiClient(flags.apiFlags, flagSet.Output()) + + err = doStreamSearch("", flags, client, w) + if err != nil { + t.Fatal(err) + } + err = w.Close() + if err != nil { + t.Fatal(err) + } + got, err := ioutil.ReadAll(r) + if err != nil { + t.Fatal(err) + } + + want, err := ioutil.ReadFile("./testdata/streaming_search_want.txt") + if err != nil { + t.Fatal(err) + } + if d := cmp.Diff(want, got); d != "" { + t.Fatalf("(-want +got): %s", d) + } +} diff --git a/cmd/src/testdata/streaming_search_want.txt b/cmd/src/testdata/streaming_search_want.txt new file mode 100644 index 0000000000..422c14b4ab --- /dev/null +++ b/cmd/src/testdata/streaming_search_want.txt @@ -0,0 +1,25 @@ +org/repo › path/to/file (1 match) +-------------------------------------------------------------------------------- +  5 | foo bar + +sourcegraph/sourcegraph (http://127.0.0.1:55128/sourcegraph/sourcegraph) (1 match) +org/repo › path/to/file (2 matches) +-------------------------------------------------------------------------------- +doResults(FUNCTION) (http://127.0.0.1:55128/github.com/sourcegraph/sourcegraph/-/blob/cmd/frontend/graphqlbackend/search_results.go#L1591:26-1591:35) +Results(FIELD, SearchResultsResolver) (http://127.0.0.1:55128/github.com/sourcegraph/sourcegraph/-/blob/cmd/frontend/graphqlbackend/search_results.go#L1591:26-1591:35) + +(http://127.0.0.1:55128) +sourcegraph/sourcegraph-atom > Stephen Gutekanst : all: release v1.0.5 (1 match) +-------------------------------------------------------------------------------- + oo bar + +(http://127.0.0.1:55128) +sourcegraph/sourcegraph-atom > Stephen Gutekanst : all: release v1.0.5 (1 match) +-------------------------------------------------------------------------------- + src/data.ts src/data.ts + @@ -0,0 +11,4 @@ + + return of({ + + title: 'Acme Corp open-source code search', + + summary: 'Instant code search across all Acme Corp open-source code.', + + githubOrgs: ['sourcegraph'], + From fcf959b1cf03ddda3137b7781fa72739e0bb0ee7 Mon Sep 17 00:00:00 2001 From: Stefan Hengl Date: Fri, 5 Mar 2021 15:49:48 +0100 Subject: [PATCH 5/7] count diff/commit results just like on Cloud --- cmd/src/search_stream.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmd/src/search_stream.go b/cmd/src/search_stream.go index a379b9c704..c0a4a69148 100644 --- a/cmd/src/search_stream.go +++ b/cmd/src/search_stream.go @@ -233,7 +233,13 @@ const streamingTemplate = ` {{- /* Repository > author name "commit subject" (time ago) */ -}} {{- color "search-commit-subject"}}{{(streamSearchRenderCommitLabel .Label)}}{{color "nc" -}} - {{- color "success"}}{{matchOrMatches (len .Ranges)}}{{color "nc" -}} + {{- color "success" -}} + {{- if (len .Ranges) -}} + {{matchOrMatches (len .Ranges)}} + {{- else -}} + {{matchOrMatches 1}} + {{- end -}} + {{- color "nc" -}} {{- "\n" -}} {{- color "search-border"}}{{"--------------------------------------------------------------------------------\n"}}{{color "nc"}} {{- color "search-border"}}{{color "nc"}}{{indent (streamSearchHighlightCommit .Content .Ranges) " "}} From 5c951db14c65dc7cc3efad54711e92eb0c845aab Mon Sep 17 00:00:00 2001 From: Stefan Hengl Date: Tue, 9 Mar 2021 09:22:44 +0100 Subject: [PATCH 6/7] address PR comments This removes the dedicated flagSet for streaming search and move parts of the logic to internal streaming. --- cmd/src/search.go | 24 +++++------- cmd/src/search_stream.go | 72 ++--------------------------------- cmd/src/search_stream_test.go | 10 ++--- internal/api/flags.go | 7 ---- internal/streaming/search.go | 55 ++++++++++++++++++++++++++ 5 files changed, 73 insertions(+), 95 deletions(-) create mode 100644 internal/streaming/search.go diff --git a/cmd/src/search.go b/cmd/src/search.go index 4450a52fa4..6c70ac75b5 100644 --- a/cmd/src/search.go +++ b/cmd/src/search.go @@ -17,6 +17,7 @@ import ( isatty "github.com/mattn/go-isatty" "github.com/sourcegraph/src-cli/internal/api" + "github.com/sourcegraph/src-cli/internal/streaming" "jaytaylor.com/html2text" ) @@ -50,14 +51,12 @@ Other tips: flagSet := flag.NewFlagSet("search", flag.ExitOnError) var ( - jsonFlag = flagSet.Bool("json", false, "Whether or not to output results as JSON") + jsonFlag = flagSet.Bool("json", false, "Whether or not to output results as JSON.") explainJSONFlag = flagSet.Bool("explain-json", false, "Explain the JSON output schema and exit.") apiFlags = api.NewFlags(flagSet) - lessFlag = flagSet.Bool("less", true, "Pipe output to 'less -R' (only if stdout is terminal, and not json flag)") - streamFlag = flagSet.Bool("stream", false, "Consume results as stream.") - - // Streaming. - _ = flagSet.Int("display", -1, "Limit the number of results shown. Only supported for streaming.") + lessFlag = flagSet.Bool("less", true, "Pipe output to 'less -R' (only if stdout is terminal, and not json flag).") + streamFlag = flagSet.Bool("stream", false, "Consume results as stream. Streaming search only supports a subset of flags and parameters: trace, insecure-skip-verify, display.") + display = flagSet.Int("display", -1, "Limit the number of results that are displayed. Only supported together with stream flag. Statistics continue to report all results.") ) handler := func(args []string) error { @@ -66,15 +65,12 @@ Other tips: } if *streamFlag { - // Remove -stream from args. - argsWOStream := make([]string, 0, len(args)-1) - for _, a := range args { - if a == "-stream" { - continue - } - argsWOStream = append(argsWOStream, a) + opts := streaming.Opts{ + Display: *display, + Trace: apiFlags.Trace(), } - return streamHandler(argsWOStream) + client := cfg.apiClient(apiFlags, flagSet.Output()) + return streamSearch(flagSet.Arg(0), opts, client, os.Stdout) } if *explainJSONFlag { diff --git a/cmd/src/search_stream.go b/cmd/src/search_stream.go index c0a4a69148..16bc1851f0 100644 --- a/cmd/src/search_stream.go +++ b/cmd/src/search_stream.go @@ -2,11 +2,8 @@ package main import ( "bytes" - "context" - "flag" "fmt" "io" - "net/url" "os" "os/exec" "regexp" @@ -23,51 +20,15 @@ func init() { labelRegexp, _ = regexp.Compile("(?:\\[)(.*?)(?:])") } -// streamHandler handles search requests which contain the flag "stream". -// Requests are sent to search/stream instead of the GraphQL api. -func streamHandler(args []string) error { - flagSet := flag.NewFlagSet("streaming search", flag.ExitOnError) - flags := newStreamingFlags(flagSet) - if err := flagSet.Parse(args); err != nil { - return err - } - - client := cfg.apiClient(flags.apiFlags, flagSet.Output()) - query := flagSet.Arg(0) - return doStreamSearch(query, flags, client, os.Stdout) -} - -func doStreamSearch(query string, flags *streamingFlags, client api.Client, w io.Writer) error { +func streamSearch(query string, opts streaming.Opts, client api.Client, w io.Writer) error { t, err := parseTemplate(streamingTemplate) if err != nil { panic(err) } - - // Create request. - req, err := client.NewHTTPRequest(context.Background(), "GET", "search/stream?q="+url.QueryEscape(query), nil) - if err != nil { - return err - } - req.Header.Set("Accept", "text/event-stream") - if flags.display >= 0 { - q := req.URL.Query() - q.Add("display", strconv.Itoa(flags.display)) - req.URL.RawQuery = q.Encode() - } - - // Send request. - resp, err := client.Do(req) - if err != nil { - return fmt.Errorf("error sending request: %w", err) - } - defer resp.Body.Close() - logError := func(msg string) { _, _ = fmt.Fprintf(os.Stderr, msg) } - - // Process response. - err = streaming.Decoder{ + decoder := streaming.Decoder{ OnProgress: func(progress *streaming.Progress) { // We only show the final progress. if !progress.Done { @@ -157,19 +118,9 @@ func doStreamSearch(query string, flags *streamingFlags, client api.Client, w io } } }, - }.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("error during decoding: %w", err) } - // Write trace to output. - if flags.Trace() { - _, err = fmt.Fprintf(os.Stderr, fmt.Sprintf("x-trace: %s\n", resp.Header.Get("x-trace"))) - if err != nil { - return err - } - } - return nil + return streaming.Search(query, opts, client, decoder) } const streamingTemplate = ` @@ -412,20 +363,3 @@ func streamConvertMatchToHighlights(m streaming.EventLineMatch, isPreview bool) } return highlights } - -type streamingFlags struct { - apiFlags *api.Flags - display int -} - -func newStreamingFlags(flagSet *flag.FlagSet) *streamingFlags { - flags := &streamingFlags{ - apiFlags: api.StreamingFlags(flagSet), - } - flagSet.IntVar(&flags.display, "display", -1, "Limit the number of results that are displayed. Note that the statistics continue to report all results.") - return flags -} - -func (f *streamingFlags) Trace() bool { - return f.apiFlags.Trace() -} diff --git a/cmd/src/search_stream_test.go b/cmd/src/search_stream_test.go index b8c76bab88..6475e77497 100644 --- a/cmd/src/search_stream_test.go +++ b/cmd/src/search_stream_test.go @@ -11,6 +11,7 @@ import ( "github.com/google/go-cmp/cmp" + "github.com/sourcegraph/src-cli/internal/api" "github.com/sourcegraph/src-cli/internal/streaming" ) @@ -117,11 +118,10 @@ func TestSearchStream(t *testing.T) { t.Fatal(err) } - flagSet := flag.NewFlagSet("streaming search test", flag.ExitOnError) - flags := newStreamingFlags(flagSet) - client := cfg.apiClient(flags.apiFlags, flagSet.Output()) - - err = doStreamSearch("", flags, client, w) + flagSet := flag.NewFlagSet("test", flag.ExitOnError) + flags := api.NewFlags(flagSet) + client := cfg.apiClient(flags, flagSet.Output()) + err = streamSearch("", streaming.Opts{}, client, w) if err != nil { t.Fatal(err) } diff --git a/internal/api/flags.go b/internal/api/flags.go index 38bc371831..4a02dab1cd 100644 --- a/internal/api/flags.go +++ b/internal/api/flags.go @@ -38,10 +38,3 @@ func defaultFlags() *Flags { insecureSkipVerify: &d, } } - -func StreamingFlags(flagSet *flag.FlagSet) *Flags { - return &Flags{ - trace: flagSet.Bool("trace", false, "Log the trace ID for requests. See https://docs.sourcegraph.com/admin/observability/tracing"), - insecureSkipVerify: flagSet.Bool("insecure-skip-verify", false, "Skip validation of TLS certificates against trusted chains"), - } -} diff --git a/internal/streaming/search.go b/internal/streaming/search.go new file mode 100644 index 0000000000..9900c3690c --- /dev/null +++ b/internal/streaming/search.go @@ -0,0 +1,55 @@ +package streaming + +import ( + "context" + "fmt" + "net/url" + "os" + "strconv" + + "github.com/sourcegraph/src-cli/internal/api" +) + +// Opts contains the search options supported by Search. +type Opts struct { + Display int + Trace bool +} + +// Search calls the streaming search endpoint and uses decoder to decode the +// response body. +func Search(query string, opts Opts, client api.Client, decoder Decoder) error { + // Create request. + req, err := client.NewHTTPRequest(context.Background(), "GET", "search/stream?q="+url.QueryEscape(query), nil) + if err != nil { + return err + } + req.Header.Set("Accept", "text/event-stream") + if opts.Display >= 0 { + q := req.URL.Query() + q.Add("display", strconv.Itoa(opts.Display)) + req.URL.RawQuery = q.Encode() + } + + // Send request. + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("error sending request: %w", err) + } + defer resp.Body.Close() + + // Process response. + err = decoder.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("error during decoding: %w", err) + } + + // Output trace. + if opts.Trace { + _, err = fmt.Fprintf(os.Stderr, fmt.Sprintf("x-trace: %s\n", resp.Header.Get("x-trace"))) + if err != nil { + return err + } + } + return nil +} From b6017ee0a21e4dacb2f04834c92c9ee7a9bf91ee Mon Sep 17 00:00:00 2001 From: Stefan Hengl Date: Tue, 9 Mar 2021 10:31:29 +0100 Subject: [PATCH 7/7] minor: add newline before trace output --- cmd/src/search_stream.go | 1 - internal/streaming/search.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/src/search_stream.go b/cmd/src/search_stream.go index 16bc1851f0..f1361825e2 100644 --- a/cmd/src/search_stream.go +++ b/cmd/src/search_stream.go @@ -219,7 +219,6 @@ const streamingTemplate = ` {{- range $index, $skipped := $.Skipped -}} {{indent $skipped.Title " "}}{{- "\n" -}} {{- end -}} - {{- "\n" -}} {{- end -}} {{- end -}} ` diff --git a/internal/streaming/search.go b/internal/streaming/search.go index 9900c3690c..f3ba23f7a5 100644 --- a/internal/streaming/search.go +++ b/internal/streaming/search.go @@ -46,7 +46,7 @@ func Search(query string, opts Opts, client api.Client, decoder Decoder) error { // Output trace. if opts.Trace { - _, err = fmt.Fprintf(os.Stderr, fmt.Sprintf("x-trace: %s\n", resp.Header.Get("x-trace"))) + _, err = fmt.Fprintf(os.Stderr, fmt.Sprintf("\nx-trace: %s\n", resp.Header.Get("x-trace"))) if err != nil { return err }