forked from n0madic/twitter-scraper
/
search.go
86 lines (75 loc) · 2.47 KB
/
search.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
package twitterscraper
import (
"context"
"errors"
"fmt"
"strconv"
)
// SearchTweets returns channel with tweets for a given search query
func (s *Scraper) SearchTweets(ctx context.Context, query string, maxTweetsNbr int) <-chan *TweetResult {
return getTweetTimeline(ctx, query, maxTweetsNbr, s.FetchSearchTweets)
}
// SearchProfiles returns channel with profiles for a given search query
func (s *Scraper) SearchProfiles(ctx context.Context, query string, maxProfilesNbr int) <-chan *ProfileResult {
return getUserTimeline(ctx, query, maxProfilesNbr, s.FetchSearchProfiles)
}
// getSearchTimeline gets results for a given search query, via the Twitter frontend API
func (s *Scraper) getSearchTimeline(query string, maxNbr int, cursor string) (*timeline, error) {
if !s.isLogged {
return nil, errors.New("scraper is not logged in for search")
}
if maxNbr > 50 {
maxNbr = 50
}
req, err := s.newRequest("GET", "https://twitter.com/i/api/2/search/adaptive.json")
if err != nil {
return nil, err
}
q := req.URL.Query()
q.Add("q", query)
q.Add("count", strconv.Itoa(maxNbr))
q.Add("query_source", "typed_query")
q.Add("pc", "1")
q.Add("requestContext", "launch")
q.Add("spelling_corrections", "1")
q.Add("include_ext_edit_control", "true")
if cursor != "" {
q.Add("cursor", cursor)
}
switch s.searchMode {
case SearchLatest:
q.Add("tweet_search_mode", "live")
case SearchPhotos:
q.Add("result_filter", "image")
case SearchVideos:
q.Add("result_filter", "video")
case SearchUsers:
q.Add("result_filter", "user")
}
req.URL.RawQuery = q.Encode()
var timeline timeline
err = s.RequestAPI(req, &timeline)
if err != nil {
fmt.Printf("err: %v\n", err)
return nil, err
}
return &timeline, nil
}
// FetchSearchTweets gets tweets for a given search query, via the Twitter frontend API
func (s *Scraper) FetchSearchTweets(query string, maxTweetsNbr int, cursor string) ([]*Tweet, string, error) {
timeline, err := s.getSearchTimeline(query, maxTweetsNbr, cursor)
if err != nil {
return nil, "", err
}
tweets, nextCursor := timeline.parseTweets()
return tweets, nextCursor, nil
}
// FetchSearchProfiles gets users for a given search query, via the Twitter frontend API
func (s *Scraper) FetchSearchProfiles(query string, maxProfilesNbr int, cursor string) ([]*Profile, string, error) {
timeline, err := s.getSearchTimeline(query, maxProfilesNbr, cursor)
if err != nil {
return nil, "", err
}
users, nextCursor := timeline.parseUsers()
return users, nextCursor, nil
}