Merge pull request #70 from stefansundin/twitter-api-v2

Upgrade Twitter API to v2

README.md

@@ -26,7 +26,7 @@ These services do not require API keys: Instagram, Periscope, Mixcloud, Speedrun
 
 #### Twitter
 
-Go to [Twitter Application Management](https://apps.twitter.com/) and create a new app.
+Go to the [Twitter Developer Portal](https://developer.twitter.com/) and create a new app.
 
 Once you have the consumer key and consumer secret, run the following to get the bearer token.
 
@@ -36,6 +36,8 @@ curl -X POST -d grant_type=client_credentials -u CONSUMER_KEY:CONSUMER_SECRET ht
 
 Copy the `access_token` and put it in the config.
 
+You need to create a project and add the app to it for it to work.
+
 #### YouTube
 
 Go to the [Google Developer Console](https://console.developers.google.com/), create a project and a server key. Copy the server key. Enable "YouTube Data API v3" in the project.

app.rb

@@ -136,32 +136,35 @@
   end
 
   if user
-    query = { screen_name: user }
-    cache_key_prefix = "twitter.screen_name"
+    endpoint = "/users/by/username/#{user}"
+    ratelimit_endpoint = "/users/by/username"
+    cache_key_prefix = "twitter.username"
     cache_key = user.downcase
   elsif user_id
-    query = { user_id: user_id }
+    endpoint = "/users/#{user_id}"
+    ratelimit_endpoint = "/users/by/id"
     cache_key_prefix = "twitter.user_id"
     cache_key = user_id
   end
 
   path, _ = App::Cache.cache(cache_key_prefix, cache_key, 60*60, 60) do |cached_data, stat|
-    endpoint = "/users/show"
-    ratelimit_remaining, ratelimit_reset = App::Twitter.ratelimit(endpoint)
+    ratelimit_remaining, ratelimit_reset = App::Twitter.ratelimit(ratelimit_endpoint)
     if cached_data && ratelimit_remaining < 100
       break cached_data, stat.mtime
     end
     if ratelimit_remaining < 10
       return [429, "Too many requests. Please try again in #{((ratelimit_reset-Time.now.to_i)/60)+1} minutes."]
     end
 
-    response = App::Twitter.get(endpoint, query: query)
-    next "Error: #{response.json["errors"][0]["message"]}" if response.json.has_key?("errors")
+    response = App::Twitter.get(endpoint, ratelimit_endpoint)
+    next "Error: #{response.json["detail"]}" if response.code == 403 && response.json["detail"] # App not added to a project, which is required for API v2 access
+    next "Error: #{response.json["errors"][0]["detail"]}" if response.json.has_key?("errors")
     raise(App::TwitterError, response) if !response.success?
 
-    user_id = response.json["id_str"]
-    screen_name = response.json["screen_name"].or(response.json["name"])
-    "#{user_id}/#{screen_name}"
+    data = response.json["data"]
+    user_id = data["id"]
+    username = data["username"].or(data["name"])
+    "#{user_id}/#{username}"
   end
   return [422, "Something went wrong. Try again later."] if path.nil?
   return [422, path] if path.start_with?("Error:")
@@ -177,52 +180,72 @@
   include_rts = %w[0 1].pick(params[:include_rts]) || "1"
   exclude_replies = %w[0 1].pick(params[:exclude_replies]) || "0"
 
-  data, @updated_at = App::Cache.cache("twitter.user_timeline", "#{id}.#{include_rts}.#{exclude_replies}", 60*60, 60) do |cached_data, stat|
-    endpoint = "/statuses/user_timeline"
-    ratelimit_remaining, ratelimit_reset = App::Twitter.ratelimit(endpoint)
+  options = {
+    "max_results" => 100,
+    "expansions" => "author_id,attachments.media_keys,referenced_tweets.id,referenced_tweets.id.author_id",
+    "tweet.fields" => "created_at,entities,referenced_tweets",
+    "user.fields" => "username",
+    "media.fields" => "type,url,width,height,variants",
+  }
+  exclude = []
+  if include_rts == "0"
+    exclude.push("retweets")
+  end
+  if exclude_replies == "1"
+    exclude.push("replies")
+  end
+  if exclude.length > 0
+    options["exclude"] = exclude.join(",")
+  end
+
+  data, @updated_at = App::Cache.cache("twitter.tweets", "#{id}.#{include_rts}.#{exclude_replies}", 60*60, 60) do |cached_data, stat|
+    endpoint = "/users/#{id}/tweets"
+    ratelimit_endpoint = "/users/tweets"
+    ratelimit_remaining, ratelimit_reset = App::Twitter.ratelimit(ratelimit_endpoint)
     if cached_data && ratelimit_remaining < 100
       break cached_data, stat.mtime
     end
     if ratelimit_remaining < 10
       return [429, "Too many requests. Please try again in #{((ratelimit_reset-Time.now.to_i)/60)+1} minutes."]
     end
 
-    response = App::Twitter.get(endpoint, query: {
-      user_id: id,
-      count: 100,
-      tweet_mode: "extended",
-      include_rts: include_rts,
-      exclude_replies: exclude_replies,
-    })
+    response = App::Twitter.get(endpoint, ratelimit_endpoint, query: options)
     next "Error: User has been suspended." if response.code == 401
     next "Error: This user id no longer exists. The user was likely deleted or recreated. Try resubscribing." if response.code == 404
     raise(App::TwitterError, response) if !response.success?
 
-    timeline = response.json
-    screen_name = timeline[0]["user"]["screen_name"] if timeline.length > 0
-    tweets = timeline.map do |tweet|
-      if tweet.has_key?("retweeted_status")
-        t = tweet["retweeted_status"]
-        text = "RT #{t["user"]["screen_name"]}: #{CGI.unescapeHTML(t["full_text"])}"
-      else
-        t = tweet
-        text = CGI.unescapeHTML(t["full_text"])
+    data = response.json
+    username = data["includes"]["users"].find { |user| user["id"] == id }&.[]("username")
+
+    tweets = data["data"].map do |tweet|
+      t = tweet
+      text = CGI.unescapeHTML(t["text"])
+
+      original_tweet_reference = t["referenced_tweets"]&.find { |t| t["type"] == "retweeted" }
+      if original_tweet_reference
+        original_tweet = data["includes"]["tweets"].find { |t| t["id"] == original_tweet_reference["id"] }
+        if original_tweet
+          t = original_tweet
+          author = data["includes"]["users"].find { |u| u["id"] == t["author_id"] }
+          text = "RT #{author["username"]}: #{CGI.unescapeHTML(t["text"])}"
+        end
       end
 
-      t["entities"]["urls"].each do |entity|
-        text = text.gsub(entity["url"], entity["expanded_url"])
-      end
-      t["entities"]["media"]&.each do |entity|
-        text = text.gsub(entity["url"], entity["expanded_url"])
+      if t.has_key?("entities") && t["entities"].has_key?("urls")
+        t["entities"]["urls"].each do |entity|
+          text = text.gsub(entity["url"], entity["expanded_url"])
+        end
       end
 
-      media = []
-      if t.has_key?("extended_entities")
-        t["extended_entities"]["media"].each do |entity|
-          if entity["video_info"]
-            video = entity["video_info"]["variants"].sort do |a,b|
+      tweet_media = []
+      if t.has_key?("attachments")
+        t["attachments"]["media_keys"].each do |media_key|
+          media = response.json["includes"]["media"].find { |media| media["media_key"] == media_key }
+          next if !media
+          if media["type"] == "video"
+            video = media["variants"].sort do |a,b|
               if a["content_type"].start_with?("video/") && b["content_type"].start_with?("video/")
-                b["bitrate"] - a["bitrate"]
+                b["bit_rate"] - a["bit_rate"]
               else
                 b["content_type"].start_with?("video/") <=> a["content_type"].start_with?("video/")
               end
@@ -232,34 +255,34 @@
               text += " #{video["url"]}"
             else
               # no dimension information in the URL, so add some (i.e. /tweet_video/)
-              text += " #{video["url"]}#w=#{entity["sizes"]["large"]["w"]}&h=#{entity["sizes"]["large"]["h"]}"
+              text += " #{video["url"]}#w=#{media["width"]}&h=#{media["height"]}"
             end
-            media.push("video")
-          else
-            text += " #{entity["media_url_https"]}:large"
-            media.push("picture")
+            tweet_media.push("video")
+          elsif media["type"] == "photo"
+            text += " #{media["url"]}:large"
+            tweet_media.push("picture")
           end
         end
       end
 
       {
-        "id" => tweet["id_str"],
+        "id" => tweet["id"],
         "created_at" => tweet["created_at"],
         "text" => text,
-        "media" => media.uniq,
+        "media" => tweet_media.uniq,
       }
     end
 
     {
-      "screen_name" => screen_name,
+      "username" => username,
       "tweets" => tweets,
     }.to_json
   end
   return [422, "Something went wrong. Try again later."] if data.nil?
   return [422, data] if data.start_with?("Error:")
 
   @data = JSON.parse(data)
-  @username = @data["screen_name"] if @data["screen_name"]
+  @username = @data["username"] if @data["username"]
   if params[:with_media] == "video"
     @data["tweets"].select! { |t| t["media"].include?("video") }
   elsif params[:with_media] == "picture"

app/services/twitter.rb

@@ -1,23 +1,26 @@
 # frozen_string_literal: true
-# https://dev.twitter.com/rest/reference/get/statuses/user_timeline
+# https://developer.twitter.com/en/docs/twitter-api/tweets/timelines/api-reference/get-users-id-tweets
 
 module App
   class TwitterError < HTTPError; end
 
   class Twitter < HTTP
-    BASE_URL = "https://api.twitter.com/1.1"
+    BASE_URL = "https://api.twitter.com/2"
     HEADERS = {
       "Accept" => "application/json",
       "Authorization" => "Bearer #{ENV["TWITTER_ACCESS_TOKEN"]}",
     }
     ERROR_CLASS = TwitterError
 
-    # https://developer.twitter.com/en/docs/twitter-api/v1/rate-limits
+    # https://developer.twitter.com/en/docs/twitter-api/rate-limits#v2-limits
     @@ratelimit = {
-      "/users/show" => {
+      "/users/by/id" => {
         limit: 300,
       },
-      "/statuses/user_timeline" => {
+      "/users/by/username" => {
+        limit: 300,
+      },
+      "/users/tweets" => {
         limit: 1500,
       },
     }
@@ -32,16 +35,15 @@ def self.ratelimit(endpoint)
       return @@ratelimit[endpoint][:remaining], @@ratelimit[endpoint][:reset]
     end
 
-    def self.get(*args, &block)
-      response = super(*args, &block)
+    def self.get(url, ratelimit_endpoint, options={})
+      response = super(url, options)
       if response.headers.has_key?("x-rate-limit-limit") \
         && response.headers.has_key?("x-rate-limit-remaining") \
         && response.headers.has_key?("x-rate-limit-reset")
-        endpoint = args[0]
-        @@ratelimit[endpoint][:limit] = response.headers["x-rate-limit-limit"][0].to_i
-        @@ratelimit[endpoint][:remaining] = response.headers["x-rate-limit-remaining"][0].to_i
-        @@ratelimit[endpoint][:reset] = response.headers["x-rate-limit-reset"][0].to_i
-        $metrics[:ratelimit].set(response.headers["x-rate-limit-remaining"][0].to_i, labels: { service: "twitter", endpoint: endpoint })
+        @@ratelimit[ratelimit_endpoint][:limit] = response.headers["x-rate-limit-limit"][0].to_i
+        @@ratelimit[ratelimit_endpoint][:remaining] = response.headers["x-rate-limit-remaining"][0].to_i
+        @@ratelimit[ratelimit_endpoint][:reset] = response.headers["x-rate-limit-reset"][0].to_i
+        $metrics[:ratelimit].set(response.headers["x-rate-limit-remaining"][0].to_i, labels: { service: "twitter", endpoint: ratelimit_endpoint })
       end
       return response
     end