-
Notifications
You must be signed in to change notification settings - Fork 23
/
search_raw.go
93 lines (75 loc) · 1.87 KB
/
search_raw.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
package yandex
import (
"net/http"
"strings"
"github.com/PuerkitoBio/goquery"
"github.com/corpix/uarand"
"github.com/karust/openserp/core"
"github.com/sirupsen/logrus"
)
func yandexRequest(searchURL string) (*http.Response, error) {
baseClient := &http.Client{}
req, err := http.NewRequest("GET", searchURL, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", uarand.GetRandom())
res, err := baseClient.Do(req)
if err != nil {
return nil, err
}
return res, nil
}
func yandexResultParser(response *http.Response) ([]core.SearchResult, error) {
doc, err := goquery.NewDocumentFromResponse(response)
if err != nil {
return nil, err
}
results := []core.SearchResult{}
rank := 1
// Get individual results
sel := doc.Find("li.serp-item")
for i := range sel.Nodes {
item := sel.Eq(i)
// Find URL
linkTag := item.Find("a")
link, _ := linkTag.Attr("href")
link = strings.Trim(link, " ")
// Find title
titleTag := item.Find("h2")
title := titleTag.Text()
// Find description
descTag := item.Find(`span.OrganicTextContentSpan`)
desc := descTag.Text()
if link != "" && link != "#" {
result := core.SearchResult{
Rank: rank,
URL: link,
Title: title,
Description: desc,
}
results = append(results, result)
rank++
}
}
logrus.Tracef("Yandex search document size: %d", len(doc.Text()))
return results, err
}
func Search(query core.Query) ([]core.SearchResult, error) {
googleURL, err := BuildURL(query, 1)
if err != nil {
return nil, err
}
logrus.Debugf("Yandex URL built: %s", googleURL)
res, err := yandexRequest(googleURL)
if err != nil {
return nil, err
}
logrus.Debugf("Yandex Raw response: code=%d", res.StatusCode)
results, err := yandexResultParser(res)
if err != nil {
return nil, err
}
logrus.Debugf("Yandex Raw results : %v", results)
return results, nil
}