From ef9f0e4964b123720eef2eab6c9b994edcca95da Mon Sep 17 00:00:00 2001 From: Stefan Sundin Date: Sun, 2 Apr 2023 12:42:38 -0700 Subject: [PATCH] Upgrade Twitter API to v2, in case v1.1 is taken offline when the API becomes paid. --- README.md | 4 +- app.rb | 121 ++++++++++++++++++++++++---------------- app/services/twitter.rb | 26 +++++---- 3 files changed, 89 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index 86c63cd..326ce7a 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ These services do not require API keys: Instagram, Periscope, Mixcloud, Speedrun #### Twitter -Go to [Twitter Application Management](https://apps.twitter.com/) and create a new app. +Go to the [Twitter Developer Portal](https://developer.twitter.com/) and create a new app. Once you have the consumer key and consumer secret, run the following to get the bearer token. @@ -36,6 +36,8 @@ curl -X POST -d grant_type=client_credentials -u CONSUMER_KEY:CONSUMER_SECRET ht Copy the `access_token` and put it in the config. +You need to create a project and add the app to it for it to work. + #### YouTube Go to the [Google Developer Console](https://console.developers.google.com/), create a project and a server key. Copy the server key. Enable "YouTube Data API v3" in the project. diff --git a/app.rb b/app.rb index aa82638..5b8430b 100644 --- a/app.rb +++ b/app.rb @@ -132,18 +132,19 @@ end if user - query = { screen_name: user } - cache_key_prefix = "twitter.screen_name" + endpoint = "/users/by/username/#{user}" + ratelimit_endpoint = "/users/by/username" + cache_key_prefix = "twitter.username" cache_key = user.downcase elsif user_id - query = { user_id: user_id } + endpoint = "/users/#{user_id}" + ratelimit_endpoint = "/users/by/id" cache_key_prefix = "twitter.user_id" cache_key = user_id end path, _ = App::Cache.cache(cache_key_prefix, cache_key, 60*60, 60) do |cached_data, stat| - endpoint = "/users/show" - ratelimit_remaining, ratelimit_reset = App::Twitter.ratelimit(endpoint) + ratelimit_remaining, ratelimit_reset = App::Twitter.ratelimit(ratelimit_endpoint) if cached_data && ratelimit_remaining < 100 break cached_data, stat.mtime end @@ -151,13 +152,15 @@ return [429, "Too many requests. Please try again in #{((ratelimit_reset-Time.now.to_i)/60)+1} minutes."] end - response = App::Twitter.get(endpoint, query: query) - next "Error: #{response.json["errors"][0]["message"]}" if response.json.has_key?("errors") + response = App::Twitter.get(endpoint, ratelimit_endpoint) + next "Error: #{response.json["detail"]}" if response.code == 403 && response.json["detail"] # App not added to a project, which is required for API v2 access + next "Error: #{response.json["errors"][0]["detail"]}" if response.json.has_key?("errors") raise(App::TwitterError, response) if !response.success? - user_id = response.json["id_str"] - screen_name = response.json["screen_name"].or(response.json["name"]) - "#{user_id}/#{screen_name}" + data = response.json["data"] + user_id = data["id"] + username = data["username"].or(data["name"]) + "#{user_id}/#{username}" end return [422, "Something went wrong. Try again later."] if path.nil? return [422, path] if path.start_with?("Error:") @@ -173,9 +176,28 @@ include_rts = %w[0 1].pick(params[:include_rts]) || "1" exclude_replies = %w[0 1].pick(params[:exclude_replies]) || "0" - data, @updated_at = App::Cache.cache("twitter.user_timeline", "#{id}.#{include_rts}.#{exclude_replies}", 60*60, 60) do |cached_data, stat| - endpoint = "/statuses/user_timeline" - ratelimit_remaining, ratelimit_reset = App::Twitter.ratelimit(endpoint) + options = { + "max_results" => 100, + "expansions" => "author_id,attachments.media_keys,referenced_tweets.id,referenced_tweets.id.author_id", + "tweet.fields" => "created_at,entities,referenced_tweets", + "user.fields" => "username", + "media.fields" => "type,url,width,height,variants", + } + exclude = [] + if include_rts == "0" + exclude.push("retweets") + end + if exclude_replies == "1" + exclude.push("replies") + end + if exclude.length > 0 + options["exclude"] = exclude.join(",") + end + + data, @updated_at = App::Cache.cache("twitter.tweets", "#{id}.#{include_rts}.#{exclude_replies}", 60*60, 60) do |cached_data, stat| + endpoint = "/users/#{id}/tweets" + ratelimit_endpoint = "/users/tweets" + ratelimit_remaining, ratelimit_reset = App::Twitter.ratelimit(ratelimit_endpoint) if cached_data && ratelimit_remaining < 100 break cached_data, stat.mtime end @@ -183,42 +205,43 @@ return [429, "Too many requests. Please try again in #{((ratelimit_reset-Time.now.to_i)/60)+1} minutes."] end - response = App::Twitter.get(endpoint, query: { - user_id: id, - count: 100, - tweet_mode: "extended", - include_rts: include_rts, - exclude_replies: exclude_replies, - }) + response = App::Twitter.get(endpoint, ratelimit_endpoint, query: options) next "Error: User has been suspended." if response.code == 401 next "Error: This user id no longer exists. The user was likely deleted or recreated. Try resubscribing." if response.code == 404 raise(App::TwitterError, response) if !response.success? - timeline = response.json - screen_name = timeline[0]["user"]["screen_name"] if timeline.length > 0 - tweets = timeline.map do |tweet| - if tweet.has_key?("retweeted_status") - t = tweet["retweeted_status"] - text = "RT #{t["user"]["screen_name"]}: #{CGI.unescapeHTML(t["full_text"])}" - else - t = tweet - text = CGI.unescapeHTML(t["full_text"]) + data = response.json + username = data["includes"]["users"].find { |user| user["id"] == id }&.[]("username") + + tweets = data["data"].map do |tweet| + t = tweet + text = CGI.unescapeHTML(t["text"]) + + original_tweet_reference = t["referenced_tweets"]&.find { |t| t["type"] == "retweeted" } + if original_tweet_reference + original_tweet = data["includes"]["tweets"].find { |t| t["id"] == original_tweet_reference["id"] } + if original_tweet + t = original_tweet + author = data["includes"]["users"].find { |u| u["id"] == t["author_id"] } + text = "RT #{author["username"]}: #{CGI.unescapeHTML(t["text"])}" + end end - t["entities"]["urls"].each do |entity| - text = text.gsub(entity["url"], entity["expanded_url"]) - end - t["entities"]["media"]&.each do |entity| - text = text.gsub(entity["url"], entity["expanded_url"]) + if t.has_key?("entities") && t["entities"].has_key?("urls") + t["entities"]["urls"].each do |entity| + text = text.gsub(entity["url"], entity["expanded_url"]) + end end - media = [] - if t.has_key?("extended_entities") - t["extended_entities"]["media"].each do |entity| - if entity["video_info"] - video = entity["video_info"]["variants"].sort do |a,b| + tweet_media = [] + if t.has_key?("attachments") + t["attachments"]["media_keys"].each do |media_key| + media = response.json["includes"]["media"].find { |media| media["media_key"] == media_key } + next if !media + if media["type"] == "video" + video = media["variants"].sort do |a,b| if a["content_type"].start_with?("video/") && b["content_type"].start_with?("video/") - b["bitrate"] - a["bitrate"] + b["bit_rate"] - a["bit_rate"] else b["content_type"].start_with?("video/") <=> a["content_type"].start_with?("video/") end @@ -228,26 +251,26 @@ text += " #{video["url"]}" else # no dimension information in the URL, so add some (i.e. /tweet_video/) - text += " #{video["url"]}#w=#{entity["sizes"]["large"]["w"]}&h=#{entity["sizes"]["large"]["h"]}" + text += " #{video["url"]}#w=#{media["width"]}&h=#{media["height"]}" end - media.push("video") - else - text += " #{entity["media_url_https"]}:large" - media.push("picture") + tweet_media.push("video") + elsif media["type"] == "photo" + text += " #{media["url"]}:large" + tweet_media.push("picture") end end end { - "id" => tweet["id_str"], + "id" => tweet["id"], "created_at" => tweet["created_at"], "text" => text, - "media" => media.uniq, + "media" => tweet_media.uniq, } end { - "screen_name" => screen_name, + "username" => username, "tweets" => tweets, }.to_json end @@ -255,7 +278,7 @@ return [422, data] if data.start_with?("Error:") @data = JSON.parse(data) - @username = @data["screen_name"] if @data["screen_name"] + @username = @data["username"] if @data["username"] if params[:with_media] == "video" @data["tweets"].select! { |t| t["media"].include?("video") } elsif params[:with_media] == "picture" diff --git a/app/services/twitter.rb b/app/services/twitter.rb index ca23383..d594448 100644 --- a/app/services/twitter.rb +++ b/app/services/twitter.rb @@ -1,23 +1,26 @@ # frozen_string_literal: true -# https://dev.twitter.com/rest/reference/get/statuses/user_timeline +# https://developer.twitter.com/en/docs/twitter-api/tweets/timelines/api-reference/get-users-id-tweets module App class TwitterError < HTTPError; end class Twitter < HTTP - BASE_URL = "https://api.twitter.com/1.1" + BASE_URL = "https://api.twitter.com/2" HEADERS = { "Accept" => "application/json", "Authorization" => "Bearer #{ENV["TWITTER_ACCESS_TOKEN"]}", } ERROR_CLASS = TwitterError - # https://developer.twitter.com/en/docs/twitter-api/v1/rate-limits + # https://developer.twitter.com/en/docs/twitter-api/rate-limits#v2-limits @@ratelimit = { - "/users/show" => { + "/users/by/id" => { limit: 300, }, - "/statuses/user_timeline" => { + "/users/by/username" => { + limit: 300, + }, + "/users/tweets" => { limit: 1500, }, } @@ -32,16 +35,15 @@ def self.ratelimit(endpoint) return @@ratelimit[endpoint][:remaining], @@ratelimit[endpoint][:reset] end - def self.get(*args, &block) - response = super(*args, &block) + def self.get(url, ratelimit_endpoint, options={}) + response = super(url, options) if response.headers.has_key?("x-rate-limit-limit") \ && response.headers.has_key?("x-rate-limit-remaining") \ && response.headers.has_key?("x-rate-limit-reset") - endpoint = args[0] - @@ratelimit[endpoint][:limit] = response.headers["x-rate-limit-limit"][0].to_i - @@ratelimit[endpoint][:remaining] = response.headers["x-rate-limit-remaining"][0].to_i - @@ratelimit[endpoint][:reset] = response.headers["x-rate-limit-reset"][0].to_i - $metrics[:ratelimit].set(response.headers["x-rate-limit-remaining"][0].to_i, labels: { service: "twitter", endpoint: endpoint }) + @@ratelimit[ratelimit_endpoint][:limit] = response.headers["x-rate-limit-limit"][0].to_i + @@ratelimit[ratelimit_endpoint][:remaining] = response.headers["x-rate-limit-remaining"][0].to_i + @@ratelimit[ratelimit_endpoint][:reset] = response.headers["x-rate-limit-reset"][0].to_i + $metrics[:ratelimit].set(response.headers["x-rate-limit-remaining"][0].to_i, labels: { service: "twitter", endpoint: ratelimit_endpoint }) end return response end