forked from Funmi4194/twitter-scraper
/
tweets.go
157 lines (136 loc) · 6.54 KB
/
tweets.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
package twitterscraper
import (
"context"
"fmt"
"net/url"
)
// GetTweets returns channel with tweets for a given user. If maxTweetsNbr is 0, all tweets will be returned (use with caution due to Twitter rate limiting).
func (s *Scraper) GetTweets(ctx context.Context, user string, maxTweetsNbr int) <-chan *TweetResult {
return getTweetTimeline(ctx, user, maxTweetsNbr, s.FetchTweets)
}
// FetchTweets gets tweets for a given user, via the Twitter frontend API.
func (s *Scraper) FetchTweets(user string, maxTweetsNbr int, cursor string) ([]*Tweet, string, error) {
userID, err := s.GetUserIDByScreenName(user)
if err != nil {
return nil, "", err
}
return s.FetchTweetsByUserID(userID, maxTweetsNbr, cursor)
}
// FetchTweetsByUserID gets tweets for a given userID, via the Twitter frontend GraphQL API.
func (s *Scraper) FetchTweetsByUserID(userID string, maxTweetsNbr int, cursor string) ([]*Tweet, string, error) {
if maxTweetsNbr > 200 {
maxTweetsNbr = 200
}
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/UGi7tjRPr-d_U3bCPIko5Q/UserTweets")
if err != nil {
return nil, "", err
}
variables := map[string]interface{}{
"userId": userID,
"count": maxTweetsNbr,
"includePromotedContent": false,
"withQuickPromoteEligibilityTweetFields": false,
"withVoice": true,
"withV2Timeline": true,
}
features := map[string]interface{}{
"rweb_lists_timeline_redesign_enabled": true,
"responsive_web_graphql_exclude_directive_enabled": true,
"verified_phone_label_enabled": false,
"creator_subscriptions_tweet_preview_api_enabled": true,
"responsive_web_graphql_timeline_navigation_enabled": true,
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
"tweetypie_unmention_optimization_enabled": true,
"vibe_api_enabled": true,
"responsive_web_edit_tweet_api_enabled": true,
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
"view_counts_everywhere_api_enabled": true,
"longform_notetweets_consumption_enabled": true,
"tweet_awards_web_tipping_enabled": false,
"freedom_of_speech_not_reach_fetch_enabled": true,
"standardized_nudges_misinfo": true,
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
"interactive_text_enabled": true,
"responsive_web_text_conversations_enabled": false,
"longform_notetweets_rich_text_read_enabled": true,
"longform_notetweets_inline_media_enabled": false,
"responsive_web_enhance_cards_enabled": false,
}
if cursor != "" {
variables["cursor"] = cursor
}
query := url.Values{}
query.Set("variables", mapToJSONString(variables))
query.Set("features", mapToJSONString(features))
req.URL.RawQuery = query.Encode()
var timeline timelineV2
err = s.RequestAPI(req, &timeline)
if err != nil {
return nil, "", err
}
tweets, nextCursor := timeline.parseTweets()
return tweets, nextCursor, nil
}
// GetTweet get a single tweet by ID.
func (s *Scraper) GetTweet(id string) (*Tweet, error) {
req, err := s.newRequest("GET", "https://twitter.com/i/api/graphql/VWFGPVAGkZMGRKGe3GFFnA/TweetDetail")
if err != nil {
return nil, err
}
variables := map[string]interface{}{
"focalTweetId": id,
"with_rux_injections": false,
"includePromotedContent": true,
"withCommunity": true,
"withQuickPromoteEligibilityTweetFields": true,
"withBirdwatchNotes": true,
"withVoice": true,
"withV2Timeline": true,
}
features := map[string]interface{}{
"rweb_lists_timeline_redesign_enabled": true,
"responsive_web_graphql_exclude_directive_enabled": true,
"verified_phone_label_enabled": false,
"creator_subscriptions_tweet_preview_api_enabled": true,
"responsive_web_graphql_timeline_navigation_enabled": true,
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
"tweetypie_unmention_optimization_enabled": true,
"responsive_web_edit_tweet_api_enabled": true,
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
"view_counts_everywhere_api_enabled": true,
"longform_notetweets_consumption_enabled": true,
"tweet_awards_web_tipping_enabled": false,
"freedom_of_speech_not_reach_fetch_enabled": true,
"standardized_nudges_misinfo": true,
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
"longform_notetweets_rich_text_read_enabled": true,
"longform_notetweets_inline_media_enabled": true,
"responsive_web_enhance_cards_enabled": false,
}
query := url.Values{}
query.Set("variables", mapToJSONString(variables))
query.Set("features", mapToJSONString(features))
req.URL.RawQuery = query.Encode()
var conversation threadedConversation
// Surprisingly, if bearerToken2 is not set, then animated GIFs are not
// present in the response for tweets with a GIF + a photo like this one:
// https://twitter.com/Twitter/status/1580661436132757506
curBearerToken := s.bearerToken
if curBearerToken != bearerToken2 {
s.setBearerToken(bearerToken2)
}
err = s.RequestAPI(req, &conversation)
if curBearerToken != bearerToken2 {
s.setBearerToken(curBearerToken)
}
if err != nil {
return nil, err
}
tweets := conversation.parse()
for _, tweet := range tweets {
if tweet.ID == id {
return tweet, nil
}
}
return nil, fmt.Errorf("tweet with ID %s not found", id)
}