Add support for displaying Community Notes

src/api.nim

@@ -31,7 +31,7 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profi
                    of TimelineKind.replies: (graphUserTweetsAndReplies, Api.userTweetsAndReplies)
                    of TimelineKind.media: (graphUserMedia, Api.userMedia)
     js = await fetch(url ? params, apiId)
-  result = parseGraphTimeline(js, "user", after)
+  result = await parseGraphTimeline(js, "user", after)
 
 proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
   if id.len == 0: return
@@ -40,7 +40,7 @@ proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
     variables = listTweetsVariables % [id, cursor]
     params = {"variables": variables, "features": gqlFeatures}
     js = await fetch(graphListTweets ? params, Api.listTweets)
-  result = parseGraphTimeline(js, "list", after).tweets
+  result = (await parseGraphTimeline(js, "list", after)).tweets
 
 proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
   let
@@ -59,7 +59,7 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.}
   var
     variables = %*{
       "listId": list.id,
-      "withBirdwatchPivots": false,
+      "withBirdwatchPivots": true,
       "withDownvotePerspective": false,
       "withReactionsMetadata": false,
       "withReactionsPerspective": false
@@ -75,16 +75,17 @@ proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
     variables = """{"rest_id": "$1"}""" % id
     params = {"variables": variables, "features": gqlFeatures}
     js = await fetch(graphTweetResult ? params, Api.tweetResult)
-  result = parseGraphTweetResult(js)
+  result = await parseGraphTweetResult(js)
+
 
 proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
   if id.len == 0: return
   let
-    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+    cursor = if after.len > 0: "\"cursorp\":\"$1\"," % after else: ""
     variables = tweetVariables % [id, cursor]
     params = {"variables": variables, "features": gqlFeatures}
     js = await fetch(graphTweet ? params, Api.tweetDetail)
-  result = parseGraphConversation(js, id)
+  result = await parseGraphConversation(js, id)
 
 proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
   result = (await getGraphTweet(id, after)).replies
@@ -112,7 +113,7 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
   if after.len > 0:
     variables["cursor"] = % after
   let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
-  result = parseGraphSearch(await fetch(url, Api.search), after)
+  result = await parseGraphSearch(await fetch(url, Api.search), after)
   result.query = query
 
 proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} =
@@ -138,10 +139,10 @@ proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
     ps = genParams({"screen_name": name, "trim_user": "true"},
                     count="18", ext=false)
     url = photoRail ? ps
-  result = parsePhotoRail(await fetch(url, Api.photoRail))
+  result = await parsePhotoRail(await fetch(url, Api.photoRail))
 
 proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
-  let client = newAsyncHttpClient(maxRedirects=0)
+  let client = newAsyncHttpClient(maxRedirects=5)
   try:
     let resp = await client.request(url, HttpHead)
     result = resp.headers["location"].replaceUrls(prefs)

src/apiutils.nim

@@ -46,12 +46,16 @@ proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
 
   return getOauth1RequestHeader(params)["authorization"]
 
-proc genHeaders*(url, oauthToken, oauthTokenSecret: string): HttpHeaders =
-  let header = getOauthHeader(url, oauthToken, oauthTokenSecret)
+proc genHeaders*(url: string, account: GuestAccount, browserapi: bool): HttpHeaders =
+  let authorization = if browserapi:
+    "Bearer " & bearerToken
+  else:
+    getOauthHeader(url, account.oauthToken, account.oauthSecret)
+
 
   result = newHttpHeaders({
     "connection": "keep-alive",
-    "authorization": header,
+    "authorization": authorization,
     "content-type": "application/json",
     "x-twitter-active-user": "yes",
     "authority": "api.twitter.com",
@@ -61,6 +65,9 @@ proc genHeaders*(url, oauthToken, oauthTokenSecret: string): HttpHeaders =
     "DNT": "1"
   })
 
+  if browserapi:
+    result["x-guest-token"] = account.guestToken
+
 template fetchImpl(result, fetchBody) {.dirty.} =
   once:
     pool = HttpPool()
@@ -72,7 +79,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
 
   try:
     var resp: AsyncResponse
-    pool.use(genHeaders($url, account.oauthToken, account.oauthSecret)):
+    pool.use(genHeaders($url, account, browserApi)):
       template getContent =
         resp = await c.get($url)
         result = await resp.body
@@ -133,7 +140,16 @@ template retry(bod) =
     echo "[accounts] Rate limited, retrying ", api, " request..."
     bod
 
-proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
+# `fetch` and `fetchRaw` operate in two modes:
+# 1. `browserApi` is false (the normal mode). We call 'https://api.twitter.com' endpoints,
+#    using the oauth token for a particular GuestAccount. This is used for everything
+#    except Community Notes.
+# 2. `browserApi` is true. We call 'https://twitter.com/i/api' endpoints,
+#    using a hardcoded Bearer token, and an 'x-guest-token' header for a particular GuestAccount.
+#    This is currently only used for retrieving Community Notes, which do not seem to be available
+#    through any of the 'https://api.twitter.com' endpoints.
+#    
+proc fetch*(url: Uri; api: Api, browserApi = false): Future[JsonNode] {.async.} =
   retry:
     var body: string
     fetchImpl body:
@@ -149,9 +165,31 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
         invalidate(account)
         raise rateLimitError()
 
-proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
+proc fetchRaw*(url: Uri; api: Api, browserApi = false): Future[string] {.async.} =
   retry:
     fetchImpl result:
       if not (result.startsWith('{') or result.startsWith('[')):
         echo resp.status, ": ", result, " --- url: ", url
         result.setLen(0)
+
+
+proc parseCommunityNote(js: JsonNode): Option[CommunityNote] = 
+  if js.isNull: return
+  var pivot = js{"data", "tweetResult", "result", "birdwatch_pivot"}
+  if pivot.isNull: return
+
+  result = some CommunityNote(
+    title: pivot{"title"}.getStr,
+    subtitle: expandCommunityNoteEntities(pivot{"subtitle"}),
+    footer: expandCommunityNoteEntities(pivot{"footer"}),
+    url: pivot{"destinationUrl"}.getStr
+  )
+
+
+proc getCommunityNote*(id: string): Future[Option[CommunityNote]] {.async.} =
+  if id.len == 0: return
+  let
+    variables = browserApiTweetVariables % [id]
+    params = {"variables": variables, "features": gqlFeatures}
+    js = await fetch(browserGraphTweetResultByRestId ? params, Api.tweetResultByRestId, true)
+  result = parseCommunityNote(js)

src/consts.nim

@@ -4,8 +4,11 @@ import uri, sequtils, strutils
 const
   consumerKey* = "3nVuSoBZnx6U4vzUxf5w"
   consumerSecret* = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
+  bearerToken* = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
 
   api = parseUri("https://api.twitter.com")
+  # This is the API accessed by the browser, which is different from the developer API
+  browserApi = parseUri("https://twitter.com/i/api")
   activate* = $(api / "1.1/guest/activate.json")
 
   photoRail* = api / "1.1/statuses/media_timeline.json"
@@ -25,6 +28,8 @@ const
   graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers"
   graphListTweets* = graphql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline"
 
+  browserGraphTweetResultByRestId* = browserApi / "/graphql/DJS3BdhUhcaEpZ7B7irJDg/TweetResultByRestId"
+
   timelineParams* = {
     "include_can_media_tag": "1",
     "include_cards": "1",
@@ -91,11 +96,18 @@ const
   $2
   "includeHasBirdwatchNotes": false,
   "includePromotedContent": false,
-  "withBirdwatchNotes": false,
+  "withBirdwatchNotes": true,
   "withVoice": false,
   "withV2Timeline": true
 }""".replace(" ", "").replace("\n", "")
 
+  browserApiTweetVariables* = """{
+  "tweetId": "$1",
+  "includePromotedContent": false,
+  "withCommunity": false,
+  "withVoice": false
+}""".replace(" ", "").replace("\n", "")
+
 #   oldUserTweetsVariables* = """{
 #   "userId": "$1", $2
 #   "count": 20,