/
search.go
470 lines (434 loc) · 14 KB
/
search.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package frontend
import (
"context"
"errors"
"fmt"
"net/http"
"path"
"sort"
"strings"
"time"
"unicode"
"unicode/utf8"
"github.com/google/safehtml/template"
"golang.org/x/mod/semver"
"golang.org/x/pkgsite/internal"
"golang.org/x/pkgsite/internal/derrors"
"golang.org/x/pkgsite/internal/experiment"
"golang.org/x/pkgsite/internal/log"
"golang.org/x/pkgsite/internal/middleware"
"golang.org/x/pkgsite/internal/postgres"
"golang.org/x/pkgsite/internal/stdlib"
"golang.org/x/pkgsite/internal/version"
"golang.org/x/text/message"
)
// serveSearch applies database data to the search template. Handles endpoint
// /search?q=<query>. If <query> is an exact match for a package path, the user
// will be redirected to the details page.
func (s *Server) serveSearch(w http.ResponseWriter, r *http.Request, ds internal.DataSource) error {
if r.Method != http.MethodGet && r.Method != http.MethodHead {
return &serverError{status: http.StatusMethodNotAllowed}
}
db, ok := ds.(*postgres.DB)
if !ok {
// The proxydatasource does not support the imported by page.
return proxydatasourceNotSupportedErr()
}
ctx := r.Context()
query, filters := searchQueryAndFilters(r)
if !utf8.ValidString(query) {
return &serverError{status: http.StatusBadRequest}
}
if len(filters) > 1 {
return &serverError{
status: http.StatusBadRequest,
epage: &errorPage{
messageTemplate: template.MakeTrustedTemplate(
`<h3 class="Error-message">Search query contains more than one symbol.</h3>`),
},
}
}
if len(query) > maxSearchQueryLength {
return &serverError{
status: http.StatusBadRequest,
epage: &errorPage{
messageTemplate: template.MakeTrustedTemplate(
`<h3 class="Error-message">Search query too long.</h3>`),
},
}
}
if query == "" {
http.Redirect(w, r, "/", http.StatusFound)
return nil
}
pageParams := newPaginationParams(r, defaultSearchLimit)
if pageParams.offset() > maxSearchOffset {
return &serverError{
status: http.StatusBadRequest,
epage: &errorPage{
messageTemplate: template.MakeTrustedTemplate(
`<h3 class="Error-message">Search page number too large.</h3>`),
},
}
}
if pageParams.limit > maxSearchPageSize {
return &serverError{
status: http.StatusBadRequest,
epage: &errorPage{
messageTemplate: template.MakeTrustedTemplate(
`<h3 class="Error-message">Search page size too large.</h3>`),
},
}
}
if path := searchRequestRedirectPath(ctx, ds, query); path != "" {
http.Redirect(w, r, path, http.StatusFound)
return nil
}
var symbol string
if len(filters) > 0 {
symbol = filters[0]
}
mode := searchMode(r)
page, err := fetchSearchPage(ctx, db, query, symbol, pageParams, mode == searchModeSymbol)
if err != nil {
return fmt.Errorf("fetchSearchPage(ctx, db, %q): %v", query, err)
}
page.basePage = s.newBasePage(r, fmt.Sprintf("%s - Search Results", query))
page.SearchMode = mode
if s.shouldServeJSON(r) {
return s.serveJSONPage(w, r, page)
}
tmpl := "legacy_search"
if experiment.IsActive(ctx, internal.ExperimentSearchGrouping) {
tmpl = "search"
}
s.servePage(ctx, w, tmpl, page)
return nil
}
const (
// defaultSearchLimit is the default number of items that appears on the
// search results page if limit is not specified.
defaultSearchLimit = 25
// maxSearchQueryLength represents the max number of characters that a search
// query can be. For PostgreSQL 11, there is a max length of 2K bytes:
// https://www.postgresql.org/docs/11/textsearch-limitations.html. No valid
// searches on pkg.go.dev will need more than the maxSearchQueryLength.
maxSearchQueryLength = 500
// maxSearchOffset is the maximum allowed offset into the search results.
// This prevents some very CPU-intensive queries from running.
maxSearchOffset = 90
// maxSearchPageSize is the maximum allowed limit for search results.
maxSearchPageSize = 100
// searchModePackage is the keyword prefix and query param for searching
// by packages.
searchModePackage = "package"
// searchModeSymbol is the keyword prefix and query param for searching
// by symbols.
searchModeSymbol = "symbol"
// symbolSearchFilter is a filter that can be used to indicate that the query
// contains a symbol. For example, searching for "#unmarshal json" indicates
// that unmarshal is a symbol.
symbolSearchFilter = "#"
)
// SearchPage contains all of the data that the search template needs to
// populate.
type SearchPage struct {
basePage
Pagination pagination
Results []*SearchResult
}
// SearchResult contains data needed to display a single search result.
type SearchResult struct {
Name string
PackagePath string
ModulePath string
ChipText string
Synopsis string
DisplayVersion string
Licenses []string
CommitTime string
NumImportedBy string
Symbols *subResult
SameModule *subResult // package paths in the same module
OtherMajor *subResult // package paths in lower major versions
SymbolName string
SymbolKind string
SymbolSynopsis string
SymbolGOOS string
SymbolGOARCH string
SymbolLink string
}
type subResult struct {
Heading string
Links []link
}
// fetchSearchPage fetches data matching the search query from the database and
// returns a SearchPage.
func fetchSearchPage(ctx context.Context, db *postgres.DB, query, symbol string,
pageParams paginationParams, searchSymbols bool) (*SearchPage, error) {
maxResultCount := maxSearchOffset + pageParams.limit
offset := pageParams.offset()
if experiment.IsActive(ctx, internal.ExperimentSearchGrouping) {
// When using search grouping, do pageless search: always start from the beginning.
offset = 0
}
dbresults, err := db.Search(ctx, query, postgres.SearchOptions{
MaxResults: pageParams.limit,
Offset: offset,
MaxResultCount: maxResultCount,
SearchSymbols: searchSymbols,
SymbolFilter: symbol,
})
if err != nil {
return nil, err
}
var results []*SearchResult
for _, r := range dbresults {
sr := newSearchResult(r, searchSymbols, message.NewPrinter(middleware.LanguageTag(ctx)))
results = append(results, sr)
}
var numResults int
if len(dbresults) > 0 {
numResults = int(dbresults[0].NumResults)
}
numPageResults := 0
for _, r := range dbresults {
// Grouping will put some results inside others. Each result counts one
// for itself plus one for each sub-result in the SameModule list,
// because each of those is removed from the top-level slice. Results in
// the LowerMajor list are not removed from the top-level slice,
// so we don't add them up.
numPageResults += 1 + len(r.SameModule)
}
pgs := newPagination(pageParams, numPageResults, numResults)
sp := &SearchPage{
Results: results,
Pagination: pgs,
}
return sp, nil
}
func newSearchResult(r *postgres.SearchResult, searchSymbols bool, pr *message.Printer) *SearchResult {
// For commands, change the name from "main" to the last component of the import path.
chipText := ""
name := r.Name
if name == "main" {
chipText = "command"
name = effectiveName(r.PackagePath, r.Name)
}
moduleDesc := "Other packages in module " + r.ModulePath
if r.ModulePath == stdlib.ModulePath {
moduleDesc = "Related packages in the standard library"
chipText = "standard library"
}
sr := &SearchResult{
Name: name,
PackagePath: r.PackagePath,
ModulePath: r.ModulePath,
ChipText: chipText,
Synopsis: r.Synopsis,
DisplayVersion: displayVersion(r.ModulePath, r.Version, r.Version),
Licenses: r.Licenses,
CommitTime: elapsedTime(r.CommitTime),
NumImportedBy: pr.Sprint(r.NumImportedBy),
SameModule: packagePaths(moduleDesc+":", r.SameModule),
// Say "other" instead of "lower" because at some point we may
// prefer to show a tagged, lower major version over an untagged
// higher major version.
OtherMajor: modulePaths("Other major versions:", r.OtherMajor),
}
if searchSymbols {
sr.SymbolName = r.SymbolName
sr.SymbolKind = strings.ToLower(string(r.SymbolKind))
sr.SymbolSynopsis = symbolSynopsis(r)
sr.SymbolGOOS = r.SymbolGOOS
sr.SymbolGOARCH = r.SymbolGOARCH
// If the GOOS is "all" or "linux", it doesn't need to be
// specified as a query param. "linux" is the default GOOS when a
// package has multiple build contexts, since it is first item
// listed in internal.BuildContexts.
if r.SymbolGOOS == internal.All || r.SymbolGOOS == "linux" {
sr.SymbolLink = fmt.Sprintf("/%s#%s", r.PackagePath, r.SymbolName)
} else {
sr.SymbolLink = fmt.Sprintf("/%s?GOOS=%s#%s", r.PackagePath, r.SymbolGOOS, r.SymbolName)
}
}
return sr
}
// searchRequestRedirectPath returns the path that a search request should be
// redirected to, or the empty string if there is no such path. If the user
// types an existing package path into the search bar, we will redirect the
// user to the details page. Standard library packages that only contain one
// element (such as fmt, errors, etc.) will not redirect, to allow users to
// search by those terms.
func searchRequestRedirectPath(ctx context.Context, ds internal.DataSource, query string) string {
urlSchemeIdx := strings.Index(query, "://")
if urlSchemeIdx > -1 {
query = query[urlSchemeIdx+3:]
}
requestedPath := path.Clean(query)
if !strings.Contains(requestedPath, "/") {
return ""
}
_, err := ds.GetUnitMeta(ctx, requestedPath, internal.UnknownModulePath, version.Latest)
if err != nil {
if !errors.Is(err, derrors.NotFound) {
log.Errorf(ctx, "searchRequestRedirectPath(%q): %v", requestedPath, err)
}
return ""
}
return fmt.Sprintf("/%s", requestedPath)
}
// searchMode reports whether the search performed should be in package or
// symbol search mode.
func searchMode(r *http.Request) string {
if !experiment.IsActive(r.Context(), internal.ExperimentSymbolSearch) {
return searchModePackage
}
q := rawSearchQuery(r)
if strings.HasPrefix(q, symbolSearchFilter) {
return searchModeSymbol
}
mode := rawSearchMode(r)
if mode == searchModePackage {
return searchModePackage
}
if mode == searchModeSymbol {
return searchModeSymbol
}
if shouldDefaultToSymbolSearch(q) {
return searchModeSymbol
}
return searchModePackage
}
// searchQueryAndFilters returns the search query, trimmed of any filters, and
// the array of words that had a filter prefix.
func searchQueryAndFilters(r *http.Request) (string, []string) {
words := strings.Fields(rawSearchQuery(r))
var filters []string
for i := range words {
if strings.HasPrefix(words[i], symbolSearchFilter) {
words[i] = strings.TrimLeft(words[i], symbolSearchFilter)
filters = append(filters, words[i])
}
}
return strings.Join(words, " "), filters
}
// rawSearchQuery returns the exact search query by the user.
func rawSearchQuery(r *http.Request) string {
return strings.TrimSpace(r.FormValue("q"))
}
// rawSearchQuery returns the exact search mode from the URL request.
func rawSearchMode(r *http.Request) string {
return strings.TrimSpace(r.FormValue("m"))
}
// shouldDefaultToSymbolSearch reports whether the symbol search mode should
// default to symbol search mode based on the input.
func shouldDefaultToSymbolSearch(q string) bool {
if len(strings.Fields(q)) != 1 {
return false
}
if internal.IsGoPkgInPathElement(q) {
return false
}
parts := strings.Split(q, ".")
if len(parts) > 1 {
if len(parts) == 2 && semver.IsValid(parts[1]) {
// The q has the format <text>.<semver> which is likely a
// gopkg.in host, such as yaml.v2. Default to package search.
return false
}
return !internal.TopLevelDomains[parts[len(parts)-1]]
}
// If a user searches for "Unmarshal", assume that they are searching for
// the symbol name "Unmarshal", not the package unmarshal.
return isCapitalized(q)
}
// symbolSynopsis returns the string to be displayed in the code snippet
// section for a symbol search result.
func symbolSynopsis(r *postgres.SearchResult) string {
switch r.SymbolKind {
case internal.SymbolKindField:
return fmt.Sprintf(`
type %s struct {
%s
}
`, strings.Split(r.SymbolName, ".")[0], r.SymbolSynopsis)
case internal.SymbolKindMethod:
if !strings.HasPrefix(r.SymbolSynopsis, "func (") {
return fmt.Sprintf(`
type %s interface {
%s
}
`, strings.Split(r.SymbolName, ".")[0], r.SymbolSynopsis)
}
}
return r.SymbolSynopsis
}
func packagePaths(heading string, rs []*postgres.SearchResult) *subResult {
if len(rs) == 0 {
return nil
}
var links []link
for _, r := range rs {
links = append(links, link{Href: r.PackagePath, Body: internal.Suffix(r.PackagePath, r.ModulePath)})
}
return &subResult{
Heading: heading,
Links: links,
}
}
func modulePaths(heading string, mpaths map[string]bool) *subResult {
if len(mpaths) == 0 {
return nil
}
var mps []string
for m := range mpaths {
mps = append(mps, m)
}
sort.Slice(mps, func(i, j int) bool {
_, v1 := internal.SeriesPathAndMajorVersion(mps[i])
_, v2 := internal.SeriesPathAndMajorVersion(mps[j])
return v1 > v2
})
links := make([]link, len(mps))
for i, m := range mps {
links[i] = link{Href: m, Body: m}
}
return &subResult{
Heading: heading,
Links: links,
}
}
// isCapitalized reports whether the first letter of s is capitalized.
func isCapitalized(s string) bool {
if len(s) == 0 {
return false
}
return unicode.IsUpper(rune(s[0]))
}
// elapsedTime takes a date and returns returns human-readable,
// relative timestamps based on the following rules:
// (1) 'X hours ago' when X < 6
// (2) 'today' between 6 hours and 1 day ago
// (3) 'Y days ago' when Y < 6
// (4) A date formatted like "Jan 2, 2006" for anything further back
func elapsedTime(date time.Time) string {
elapsedHours := int(time.Since(date).Hours())
if elapsedHours == 1 {
return "1 hour ago"
} else if elapsedHours < 6 {
return fmt.Sprintf("%d hours ago", elapsedHours)
}
elapsedDays := elapsedHours / 24
if elapsedDays < 1 {
return "today"
} else if elapsedDays == 1 {
return "1 day ago"
} else if elapsedDays < 6 {
return fmt.Sprintf("%d days ago", elapsedDays)
}
return absoluteTime(date)
}