forked from golang/gddo
-
Notifications
You must be signed in to change notification settings - Fork 0
/
indexae.go
207 lines (191 loc) · 5.02 KB
/
indexae.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
// Copyright 2016 The Go Authors. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd.
package database
import (
"bytes"
"errors"
"fmt"
"log"
"math"
"strings"
"unicode"
"golang.org/x/net/context"
"google.golang.org/appengine/search"
"github.com/golang/gddo/doc"
)
func (p *Package) Load(fields []search.Field, meta *search.DocumentMetadata) error {
for _, f := range fields {
switch f.Name {
case "Name":
if v, ok := f.Value.(search.Atom); ok {
p.Name = string(v)
}
case "Path":
if v, ok := f.Value.(string); ok {
p.Path = v
}
case "Synopsis":
if v, ok := f.Value.(string); ok {
p.Synopsis = v
}
case "ImportCount":
if v, ok := f.Value.(float64); ok {
p.ImportCount = int(v)
}
case "Stars":
if v, ok := f.Value.(float64); ok {
p.Stars = int(v)
}
case "Score":
if v, ok := f.Value.(float64); ok {
p.Score = v
}
}
}
if p.Path == "" {
return errors.New("Invalid document: missing Path field")
}
for _, f := range meta.Facets {
if f.Name == "Fork" {
p.Fork = f.Value.(search.Atom) == "true"
}
}
return nil
}
func (p *Package) Save() ([]search.Field, *search.DocumentMetadata, error) {
fields := []search.Field{
{Name: "Name", Value: search.Atom(p.Name)},
{Name: "Path", Value: p.Path},
{Name: "Synopsis", Value: p.Synopsis},
{Name: "Score", Value: p.Score},
{Name: "ImportCount", Value: float64(p.ImportCount)},
{Name: "Stars", Value: float64(p.Stars)},
}
fork := fmt.Sprint(p.Fork) // "true" or "false"
meta := &search.DocumentMetadata{
// Customize the rank property by the product of the package score and
// natural logarithm of the import count. Rank must be a positive integer.
// Use 1 as minimum rank and keep 3 digits of precision to distinguish
// close ranks.
Rank: int(math.Max(1, 1000*p.Score*math.Log(math.E+float64(p.ImportCount)))),
Facets: []search.Facet{
{Name: "Fork", Value: search.Atom(fork)},
},
}
return fields, meta, nil
}
// PutIndex creates or updates a package entry in the search index. id identifies the document in the index.
// If pdoc is non-nil, PutIndex will update the package's name, path and synopsis supplied by pdoc.
// pdoc must be non-nil for a package's first call to PutIndex.
// PutIndex updates the Score to score, if non-negative.
func PutIndex(c context.Context, pdoc *doc.Package, id string, score float64, importCount int) error {
if id == "" {
return errors.New("indexae: no id assigned")
}
idx, err := search.Open("packages")
if err != nil {
return err
}
var pkg Package
if err := idx.Get(c, id, &pkg); err != nil {
if err != search.ErrNoSuchDocument {
return err
} else if pdoc == nil {
// Cannot update a non-existing document.
return errors.New("indexae: cannot create new document with nil pdoc")
}
// No such document in the index, fall through.
}
// Update document information accordingly.
if pdoc != nil {
pkg.Name = pdoc.Name
pkg.Path = pdoc.ImportPath
pkg.Synopsis = pdoc.Synopsis
pkg.Stars = pdoc.Stars
pkg.Fork = pdoc.Fork
}
if score >= 0 {
pkg.Score = score
}
pkg.ImportCount = importCount
if _, err := idx.Put(c, id, &pkg); err != nil {
return err
}
return nil
}
// Search searches the packages index for a given query. A path-like query string
// will be passed in unchanged, whereas single words will be stemmed.
func Search(c context.Context, q string) ([]Package, error) {
index, err := search.Open("packages")
if err != nil {
return nil, err
}
var pkgs []Package
opt := &search.SearchOptions{
Limit: 100,
}
for it := index.Search(c, parseQuery2(q), opt); ; {
var p Package
_, err := it.Next(&p)
if err == search.Done {
break
}
if err != nil {
return nil, err
}
pkgs = append(pkgs, p)
}
return pkgs, nil
}
func parseQuery2(q string) string {
var buf bytes.Buffer
for _, s := range strings.FieldsFunc(q, isTermSep2) {
if strings.ContainsAny(s, "./") {
// Quote terms with / or . for path like query.
fmt.Fprintf(&buf, "%q ", s)
} else {
// Stem for single word terms.
fmt.Fprintf(&buf, "~%v ", s)
}
}
return buf.String()
}
func isTermSep2(r rune) bool {
return unicode.IsSpace(r) ||
r != '.' && r != '/' && unicode.IsPunct(r) ||
unicode.IsSymbol(r)
}
func deleteIndex(c context.Context, id string) error {
idx, err := search.Open("packages")
if err != nil {
return err
}
return idx.Delete(c, id)
}
// PurgeIndex deletes all the packages from the search index.
func PurgeIndex(c context.Context) error {
idx, err := search.Open("packages")
if err != nil {
return err
}
n := 0
for it := idx.List(c, &search.ListOptions{IDsOnly: true}); ; n++ {
var pkg Package
id, err := it.Next(&pkg)
if err == search.Done {
break
}
if err != nil {
return err
}
if err := idx.Delete(c, id); err != nil {
log.Printf("Failed to delete package %s: %v", id, err)
continue
}
}
log.Printf("Purged %d packages from the search index.", n)
return nil
}