diff --git a/cmd/src/search.go b/cmd/src/search.go index 6c70ac75b5..29b38d7fe3 100644 --- a/cmd/src/search.go +++ b/cmd/src/search.go @@ -55,7 +55,7 @@ Other tips: explainJSONFlag = flagSet.Bool("explain-json", false, "Explain the JSON output schema and exit.") apiFlags = api.NewFlags(flagSet) lessFlag = flagSet.Bool("less", true, "Pipe output to 'less -R' (only if stdout is terminal, and not json flag).") - streamFlag = flagSet.Bool("stream", false, "Consume results as stream. Streaming search only supports a subset of flags and parameters: trace, insecure-skip-verify, display.") + streamFlag = flagSet.Bool("stream", false, "Consume results as stream. Streaming search only supports a subset of flags and parameters: trace, insecure-skip-verify, display, json.") display = flagSet.Int("display", -1, "Limit the number of results that are displayed. Only supported together with stream flag. Statistics continue to report all results.") ) @@ -68,6 +68,7 @@ Other tips: opts := streaming.Opts{ Display: *display, Trace: apiFlags.Trace(), + Json: *jsonFlag, } client := cfg.apiClient(apiFlags, flagSet.Output()) return streamSearch(flagSet.Arg(0), opts, client, os.Stdout) diff --git a/cmd/src/search_stream.go b/cmd/src/search_stream.go index f1361825e2..939148d849 100644 --- a/cmd/src/search_stream.go +++ b/cmd/src/search_stream.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "encoding/json" "fmt" "io" "os" @@ -9,6 +10,7 @@ import ( "regexp" "strconv" "strings" + "text/template" "github.com/sourcegraph/src-cli/internal/api" "github.com/sourcegraph/src-cli/internal/streaming" @@ -21,20 +23,77 @@ func init() { } func streamSearch(query string, opts streaming.Opts, client api.Client, w io.Writer) error { - t, err := parseTemplate(streamingTemplate) - if err != nil { - panic(err) + var d streaming.Decoder + if opts.Json { + d = jsonDecoder(w) + } else { + t, err := parseTemplate(streamingTemplate) + if err != nil { + return err + } + d = textDecoder(query, t, w) + } + return streaming.Search(query, opts, client, d) +} + +// jsonDecoder streams results as JSON to w. +func jsonDecoder(w io.Writer) streaming.Decoder { + // write json.Marshals data and writes it as one line to w plus a newline. + write := func(data interface{}) error { + b, err := json.Marshal(data) + if err != nil { + return err + } + _, err = w.Write(b) + if err != nil { + return err + } + _, err = w.Write([]byte("\n")) + if err != nil { + return err + } + return nil } - logError := func(msg string) { - _, _ = fmt.Fprintf(os.Stderr, msg) + + return streaming.Decoder{ + OnProgress: func(progress *streaming.Progress) { + if !progress.Done { + return + } + err := write(progress) + if err != nil { + logError(err.Error()) + } + }, + OnMatches: func(matches []streaming.EventMatch) { + for _, match := range matches { + err := write(match) + if err != nil { + logError(err.Error()) + } + } + }, + OnAlert: func(alert *streaming.EventAlert) { + err := write(alert) + if err != nil { + logError(err.Error()) + } + }, + OnError: func(eventError *streaming.EventError) { + // Errors are just written to stderr. + logError(eventError.Message) + }, } - decoder := streaming.Decoder{ +} + +func textDecoder(query string, t *template.Template, w io.Writer) streaming.Decoder { + return streaming.Decoder{ OnProgress: func(progress *streaming.Progress) { // We only show the final progress. if !progress.Done { return } - err = t.ExecuteTemplate(w, "progress", progress) + err := t.ExecuteTemplate(w, "progress", progress) if err != nil { logError(fmt.Sprintf("error when executing template: %s\n", err)) } @@ -52,7 +111,7 @@ func streamSearch(query string, opts streaming.Opts, client api.Client, w io.Wri }) } - err = t.ExecuteTemplate(w, "alert", searchResultsAlert{ + err := t.ExecuteTemplate(w, "alert", searchResultsAlert{ Title: alert.Title, Description: alert.Description, ProposedQueries: proposedQueries, @@ -66,7 +125,7 @@ func streamSearch(query string, opts streaming.Opts, client api.Client, w io.Wri for _, match := range matches { switch match := match.(type) { case *streaming.EventFileMatch: - err = t.ExecuteTemplate(w, "file", struct { + err := t.ExecuteTemplate(w, "file", struct { Query string *streaming.EventFileMatch }{ @@ -79,7 +138,7 @@ func streamSearch(query string, opts streaming.Opts, client api.Client, w io.Wri return } case *streaming.EventRepoMatch: - err = t.ExecuteTemplate(w, "repo", struct { + err := t.ExecuteTemplate(w, "repo", struct { SourcegraphEndpoint string *streaming.EventRepoMatch }{ @@ -91,7 +150,7 @@ func streamSearch(query string, opts streaming.Opts, client api.Client, w io.Wri return } case *streaming.EventCommitMatch: - err = t.ExecuteTemplate(w, "commit", struct { + err := t.ExecuteTemplate(w, "commit", struct { SourcegraphEndpoint string *streaming.EventCommitMatch }{ @@ -103,7 +162,7 @@ func streamSearch(query string, opts streaming.Opts, client api.Client, w io.Wri return } case *streaming.EventSymbolMatch: - err = t.ExecuteTemplate(w, "symbol", struct { + err := t.ExecuteTemplate(w, "symbol", struct { SourcegraphEndpoint string *streaming.EventSymbolMatch }{ @@ -119,8 +178,6 @@ func streamSearch(query string, opts streaming.Opts, client api.Client, w io.Wri } }, } - - return streaming.Search(query, opts, client, decoder) } const streamingTemplate = ` @@ -362,3 +419,7 @@ func streamConvertMatchToHighlights(m streaming.EventLineMatch, isPreview bool) } return highlights } + +func logError(msg string) { + _, _ = fmt.Fprintf(os.Stderr, msg) +} diff --git a/cmd/src/search_stream_test.go b/cmd/src/search_stream_test.go index 6475e77497..2f7fa23936 100644 --- a/cmd/src/search_stream_test.go +++ b/cmd/src/search_stream_test.go @@ -15,8 +15,6 @@ import ( "github.com/sourcegraph/src-cli/internal/streaming" ) -var event []streaming.EventMatch - func mockStreamHandler(w http.ResponseWriter, _ *http.Request) { writer, _ := streaming.NewWriter(w) writer.Event("matches", event) @@ -38,6 +36,71 @@ func testServer(t *testing.T, handler http.Handler) *httptest.Server { return s } +var event = []streaming.EventMatch{ + &streaming.EventFileMatch{ + Type: streaming.FileMatchType, + Path: "path/to/file", + Repository: "org/repo", + Branches: nil, + Version: "", + LineMatches: []streaming.EventLineMatch{ + { + Line: "foo bar", + LineNumber: 4, + OffsetAndLengths: [][2]int32{{4, 3}}, + }, + }, + }, + &streaming.EventRepoMatch{ + Type: streaming.RepoMatchType, + Repository: "sourcegraph/sourcegraph", + Branches: []string{}, + }, + &streaming.EventSymbolMatch{ + Type: streaming.SymbolMatchType, + Path: "path/to/file", + Repository: "org/repo", + Branches: []string{}, + Version: "", + Symbols: []streaming.Symbol{ + { + URL: "github.com/sourcegraph/sourcegraph/-/blob/cmd/frontend/graphqlbackend/search_results.go#L1591:26-1591:35", + Name: "doResults", + ContainerName: "", + Kind: "FUNCTION", + }, + { + URL: "github.com/sourcegraph/sourcegraph/-/blob/cmd/frontend/graphqlbackend/search_results.go#L1591:26-1591:35", + Name: "Results", + ContainerName: "SearchResultsResolver", + Kind: "FIELD", + }, + }, + }, + &streaming.EventCommitMatch{ + Type: streaming.CommitMatchType, + Icon: "", + Label: "[sourcegraph/sourcegraph-atom](/github.com/sourcegraph/sourcegraph-atom) › [Stephen Gutekanst](/github.com/sourcegraph/sourcegraph-atom/-/commit/5b098d7fed963d88e23057ed99d73d3c7a33ad89): [all: release v1.0.5](/github.com/sourcegraph/sourcegraph-atom/-/commit/5b098d7fed963d88e23057ed99d73d3c7a33ad89)^", + URL: "", + Detail: "", + Content: "```COMMIT_EDITMSG\nfoo bar\n```", + Ranges: [][3]int32{ + {1, 3, 3}, + }, + }, + &streaming.EventCommitMatch{ + Type: streaming.CommitMatchType, + Icon: "", + Label: "[sourcegraph/sourcegraph-atom](/github.com/sourcegraph/sourcegraph-atom) › [Stephen Gutekanst](/github.com/sourcegraph/sourcegraph-atom/-/commit/5b098d7fed963d88e23057ed99d73d3c7a33ad89): [all: release v1.0.5](/github.com/sourcegraph/sourcegraph-atom/-/commit/5b098d7fed963d88e23057ed99d73d3c7a33ad89)^", + URL: "", + Detail: "", + Content: "```diff\nsrc/data.ts src/data.ts\n@@ -0,0 +11,4 @@\n+ return of({\n+ title: 'Acme Corp open-source code search',\n+ summary: 'Instant code search across all Acme Corp open-source code.',\n+ githubOrgs: ['sourcegraph'],\n```", + Ranges: [][3]int32{ + {4, 44, 6}, + }, + }, +} + func TestSearchStream(t *testing.T) { s := testServer(t, http.HandlerFunc(mockStreamHandler)) defer s.Close() @@ -47,98 +110,55 @@ func TestSearchStream(t *testing.T) { } defer func() { cfg = nil }() - event = []streaming.EventMatch{ - &streaming.EventFileMatch{ - Type: streaming.FileMatchType, - Path: "path/to/file", - Repository: "org/repo", - Branches: nil, - Version: "", - LineMatches: []streaming.EventLineMatch{ - { - Line: "foo bar", - LineNumber: 4, - OffsetAndLengths: [][2]int32{{4, 3}}, - }, - }, - }, - &streaming.EventRepoMatch{ - Type: streaming.RepoMatchType, - Repository: "sourcegraph/sourcegraph", - Branches: []string{}, - }, - &streaming.EventSymbolMatch{ - Type: streaming.SymbolMatchType, - Path: "path/to/file", - Repository: "org/repo", - Branches: []string{}, - Version: "", - Symbols: []streaming.Symbol{ - { - URL: "github.com/sourcegraph/sourcegraph/-/blob/cmd/frontend/graphqlbackend/search_results.go#L1591:26-1591:35", - Name: "doResults", - ContainerName: "", - Kind: "FUNCTION", - }, - { - URL: "github.com/sourcegraph/sourcegraph/-/blob/cmd/frontend/graphqlbackend/search_results.go#L1591:26-1591:35", - Name: "Results", - ContainerName: "SearchResultsResolver", - Kind: "FIELD", - }, - }, - }, - &streaming.EventCommitMatch{ - Type: streaming.CommitMatchType, - Icon: "", - Label: "[sourcegraph/sourcegraph-atom](/github.com/sourcegraph/sourcegraph-atom) › [Stephen Gutekanst](/github.com/sourcegraph/sourcegraph-atom/-/commit/5b098d7fed963d88e23057ed99d73d3c7a33ad89): [all: release v1.0.5](/github.com/sourcegraph/sourcegraph-atom/-/commit/5b098d7fed963d88e23057ed99d73d3c7a33ad89)^", - URL: "", - Detail: "", - Content: "```COMMIT_EDITMSG\nfoo bar\n```", - Ranges: [][3]int32{ - {1, 3, 3}, - }, + cases := []struct { + name string + opts streaming.Opts + want string + }{ + { + "Text", + streaming.Opts{}, + "./testdata/streaming_search_want.txt", }, - &streaming.EventCommitMatch{ - Type: streaming.CommitMatchType, - Icon: "", - Label: "[sourcegraph/sourcegraph-atom](/github.com/sourcegraph/sourcegraph-atom) › [Stephen Gutekanst](/github.com/sourcegraph/sourcegraph-atom/-/commit/5b098d7fed963d88e23057ed99d73d3c7a33ad89): [all: release v1.0.5](/github.com/sourcegraph/sourcegraph-atom/-/commit/5b098d7fed963d88e23057ed99d73d3c7a33ad89)^", - URL: "", - Detail: "", - Content: "```diff\nsrc/data.ts src/data.ts\n@@ -0,0 +11,4 @@\n+ return of({\n+ title: 'Acme Corp open-source code search',\n+ summary: 'Instant code search across all Acme Corp open-source code.',\n+ githubOrgs: ['sourcegraph'],\n```", - Ranges: [][3]int32{ - {4, 44, 6}, + { + "JSON", + streaming.Opts{ + Json: true, }, + "./testdata/streaming_search_want.json", }, } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + // Capture output. + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } - // Capture output. - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) - } - - flagSet := flag.NewFlagSet("test", flag.ExitOnError) - flags := api.NewFlags(flagSet) - client := cfg.apiClient(flags, flagSet.Output()) - err = streamSearch("", streaming.Opts{}, client, w) - if err != nil { - t.Fatal(err) - } - err = w.Close() - if err != nil { - t.Fatal(err) - } - got, err := ioutil.ReadAll(r) - if err != nil { - t.Fatal(err) + flagSet := flag.NewFlagSet("test", flag.ExitOnError) + flags := api.NewFlags(flagSet) + client := cfg.apiClient(flags, flagSet.Output()) + err = streamSearch("", c.opts, client, w) + if err != nil { + t.Fatal(err) + } + err = w.Close() + if err != nil { + t.Fatal(err) + } + got, err := ioutil.ReadAll(r) + if err != nil { + t.Fatal(err) + } + want, err := ioutil.ReadFile(c.want) + if err != nil { + t.Fatal(err) + } + if d := cmp.Diff(want, got); d != "" { + t.Fatalf("(-want +got): %s", d) + } + }) } - want, err := ioutil.ReadFile("./testdata/streaming_search_want.txt") - if err != nil { - t.Fatal(err) - } - if d := cmp.Diff(want, got); d != "" { - t.Fatalf("(-want +got): %s", d) - } } diff --git a/cmd/src/testdata/streaming_search_want.json b/cmd/src/testdata/streaming_search_want.json new file mode 100755 index 0000000000..702b4d7693 --- /dev/null +++ b/cmd/src/testdata/streaming_search_want.json @@ -0,0 +1,5 @@ +{"type":"file","name":"path/to/file","repository":"org/repo","lineMatches":[{"line":"foo bar","lineNumber":4,"offsetAndLengths":[[4,3]]}]} +{"type":"repo","repository":"sourcegraph/sourcegraph"} +{"type":"symbol","name":"path/to/file","repository":"org/repo","symbols":[{"url":"github.com/sourcegraph/sourcegraph/-/blob/cmd/frontend/graphqlbackend/search_results.go#L1591:26-1591:35","name":"doResults","containerName":"","kind":"FUNCTION"},{"url":"github.com/sourcegraph/sourcegraph/-/blob/cmd/frontend/graphqlbackend/search_results.go#L1591:26-1591:35","name":"Results","containerName":"SearchResultsResolver","kind":"FIELD"}]} +{"type":"commit","icon":"","label":"[sourcegraph/sourcegraph-atom](/github.com/sourcegraph/sourcegraph-atom) › [Stephen Gutekanst](/github.com/sourcegraph/sourcegraph-atom/-/commit/5b098d7fed963d88e23057ed99d73d3c7a33ad89): [all: release v1.0.5](/github.com/sourcegraph/sourcegraph-atom/-/commit/5b098d7fed963d88e23057ed99d73d3c7a33ad89)^","url":"","detail":"","content":"```COMMIT_EDITMSG\nfoo bar\n```","ranges":[[1,3,3]]} +{"type":"commit","icon":"","label":"[sourcegraph/sourcegraph-atom](/github.com/sourcegraph/sourcegraph-atom) › [Stephen Gutekanst](/github.com/sourcegraph/sourcegraph-atom/-/commit/5b098d7fed963d88e23057ed99d73d3c7a33ad89): [all: release v1.0.5](/github.com/sourcegraph/sourcegraph-atom/-/commit/5b098d7fed963d88e23057ed99d73d3c7a33ad89)^","url":"","detail":"","content":"```diff\nsrc/data.ts src/data.ts\n@@ -0,0 +11,4 @@\n+ return of\u003cData\u003e({\n+ title: 'Acme Corp open-source code search',\n+ summary: 'Instant code search across all Acme Corp open-source code.',\n+ githubOrgs: ['sourcegraph'],\n```","ranges":[[4,44,6]]} diff --git a/internal/streaming/search.go b/internal/streaming/search.go index f3ba23f7a5..e85a1d960f 100644 --- a/internal/streaming/search.go +++ b/internal/streaming/search.go @@ -14,6 +14,7 @@ import ( type Opts struct { Display int Trace bool + Json bool } // Search calls the streaming search endpoint and uses decoder to decode the