Add experimental support for unified_card

Closes #345

nitter.nimble

@@ -22,6 +22,7 @@ requires "redpool#f880f49"
 requires "https://github.com/zedeus/redis#d0a0e6f"
 requires "zippy#0.7.3"
 requires "flatty#0.2.3"
+requires "jsony#1.1.3"
 
 
 # Tasks

src/experimental/parser/unifiedcard.nim

@@ -0,0 +1,91 @@
+import std/[options, tables, strutils, strformat, sugar]
+import jsony
+import ../types/unifiedcard
+from ../../types import Card, CardKind, Video
+from ../../utils import twimg, https
+
+proc getImageUrl(entity: MediaEntity): string =
+  entity.mediaUrlHttps.dup(removePrefix(twimg), removePrefix(https))
+
+proc parseDestination(id: string; card: UnifiedCard; result: var Card) =
+  let destination = card.destinationObjects[id].data
+  result.dest = destination.urlData.vanity
+  result.url = destination.urlData.url
+
+proc parseDetails(data: ComponentData; card: UnifiedCard; result: var Card) =
+  data.destination.parseDestination(card, result)
+
+  result.text = data.title
+  if result.text.len == 0:
+    result.text = data.name
+
+proc parseMediaDetails(data: ComponentData; card: UnifiedCard; result: var Card) =
+  data.destination.parseDestination(card, result)
+
+  result.kind = summary
+  result.image = card.mediaEntities[data.mediaId].getImageUrl
+  result.text = data.topicDetail.title
+  result.dest = "Topic"
+
+proc parseAppDetails(data: ComponentData; card: UnifiedCard; result: var Card) =
+  let app = card.appStoreData[data.appId][0]
+
+  case app.kind
+  of androidApp:
+    result.url = "http://play.google.com/store/apps/details?id=" & app.id
+  of iPhoneApp, iPadApp:
+    result.url = "https://itunes.apple.com/app/id" & app.id
+
+  result.text = app.title
+  result.dest = app.category
+
+proc parseListDetails(data: ComponentData; result: var Card) =
+  result.dest = &"List · {data.memberCount} Members"
+
+proc parseCommunityDetails(data: ComponentData; result: var Card) =
+  result.dest = &"Community · {data.memberCount} Members"
+
+proc parseMedia(component: Component; card: UnifiedCard; result: var Card) =
+  let mediaId =
+    if component.kind == swipeableMedia:
+      component.data.mediaList[0].id
+    else:
+      component.data.id
+
+  let rMedia = card.mediaEntities[mediaId]
+  case rMedia.kind:
+  of photo:
+    result.kind = summaryLarge
+    result.image = rMedia.getImageUrl
+  of video:
+    let videoInfo = rMedia.videoInfo.get
+    result.kind = promoVideo
+    result.video = some Video(
+      available: true,
+      thumb: rMedia.getImageUrl,
+      durationMs: videoInfo.durationMillis,
+      variants: videoInfo.variants
+    )
+
+proc parseUnifiedCard*(json: string): Card =
+  let card = json.fromJson(UnifiedCard)
+
+  for component in card.componentObjects.values:
+    case component.kind
+    of details, communityDetails, twitterListDetails:
+      component.data.parseDetails(card, result)
+    of appStoreDetails:
+      component.data.parseAppDetails(card, result)
+    of mediaWithDetailsHorizontal:
+      component.data.parseMediaDetails(card, result)
+    of media, swipeableMedia:
+      component.parseMedia(card, result)
+    of buttonGroup:
+      discard
+
+    case component.kind
+    of twitterListDetails:
+      component.data.parseListDetails(result)
+    of communityDetails:
+      component.data.parseCommunityDetails(result)
+    else: discard

src/experimental/types/unifiedcard.nim

@@ -0,0 +1,79 @@
+import options, tables
+from ../../types import VideoType, VideoVariant
+
+type
+  UnifiedCard* = object
+    componentObjects*: Table[string, Component]
+    destinationObjects*: Table[string, Destination]
+    mediaEntities*: Table[string, MediaEntity]
+    appStoreData*: Table[string, seq[AppStoreData]]
+
+  ComponentType* = enum
+    details
+    media
+    swipeableMedia
+    buttonGroup
+    appStoreDetails
+    twitterListDetails
+    communityDetails
+    mediaWithDetailsHorizontal
+
+  Component* = object
+    kind*: ComponentType
+    data*: ComponentData
+
+  ComponentData* = object
+    id*: string
+    appId*: string
+    mediaId*: string
+    destination*: string
+    title*: Text
+    subtitle*: Text
+    name*: Text
+    memberCount*: int
+    mediaList*: seq[MediaItem]
+    topicDetail*: tuple[title: Text]
+
+  MediaItem* = object
+    id*: string
+    destination*: string
+
+  Destination* = object
+    kind*: string
+    data*: tuple[urlData: UrlData]
+
+  UrlData* = object
+    url*: string
+    vanity*: string
+
+  MediaType* = enum
+    photo, video
+
+  MediaEntity* = object
+    kind*: MediaType
+    mediaUrlHttps*: string
+    videoInfo*: Option[VideoInfo]
+
+  VideoInfo* = object
+    durationMillis*: int
+    variants*: seq[VideoVariant]
+
+  AppType* = enum
+    androidApp, iPhoneApp, iPadApp
+
+  AppStoreData* = object
+    kind*: AppType
+    id*: string
+    title*: Text
+    category*: Text
+
+  Text = object
+    content: string
+
+  HasTypeField = Component | Destination | MediaEntity | AppStoreData
+
+converter fromText*(text: Text): string = text.content
+
+proc renameHook*(v: var HasTypeField; fieldName: var string) =
+  if fieldName == "type":
+    fieldName = "kind"

src/formatters.nim

@@ -61,7 +61,7 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
     result = result.replace(tco, https & prefs.replaceTwitter & "/t.co")
     result = result.replace(cards, prefs.replaceTwitter & "/cards")
     result = result.replace(twRegex, prefs.replaceTwitter)
-    result = result.replace(twLinkRegex, a(
+    result = result.replacef(twLinkRegex, a(
       prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1"))
 
   if prefs.replaceReddit.len > 0 and ("reddit.com" in result or "redd.it" in result):

src/parser.nim

@@ -1,8 +1,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import strutils, options, tables, times, math
-import packedjson
-import packedjson / deserialiser
+import packedjson, packedjson/deserialiser
 import types, parserutils, utils
+import experimental/parser/unifiedcard
 
 proc parseProfile(js: JsonNode; id=""): Profile =
   if js.isNull: return
@@ -102,7 +102,6 @@ proc parseGif(js: JsonNode): Gif =
 
 proc parseVideo(js: JsonNode): Video =
   result = Video(
-    videoId: js{"id_str"}.getStr,
     thumb: js{"media_url_https"}.getImageStr,
     views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr,
     available: js{"ext_media_availability", "status"}.getStr == "available",
@@ -119,7 +118,7 @@ proc parseVideo(js: JsonNode): Video =
 
   for v in js{"video_info", "variants"}:
     result.variants.add VideoVariant(
-      videoType: parseEnum[VideoType](v{"content_type"}.getStr("summary")),
+      contentType: parseEnum[VideoType](v{"content_type"}.getStr("summary")),
       bitrate: v{"bitrate"}.getInt,
       url: v{"url"}.getStr
     )
@@ -129,19 +128,17 @@ proc parsePromoVideo(js: JsonNode): Video =
     thumb: js{"player_image_large"}.getImageVal,
     available: true,
     durationMs: js{"content_duration_seconds"}.getStrVal("0").parseInt * 1000,
-    playbackType: vmap,
-    videoId: js{"player_content_id"}.getStrVal(js{"card_id"}.getStrVal(
-        js{"amplify_content_id"}.getStrVal())),
+    playbackType: vmap
   )
 
   var variant = VideoVariant(
-    videoType: vmap,
+    contentType: vmap,
     url: js{"player_hls_url"}.getStrVal(js{"player_stream_url"}.getStrVal(
         js{"amplify_url_vmap"}.getStrVal()))
   )
 
   if "m3u8" in variant.url:
-    variant.videoType = m3u8
+    variant.contentType = m3u8
     result.playbackType = m3u8
 
   result.variants.add variant
@@ -154,7 +151,7 @@ proc parseBroadcast(js: JsonNode): Card =
     title: js{"broadcaster_display_name"}.getStrVal,
     text: js{"broadcast_title"}.getStrVal,
     image: image,
-    video: some Video(videoId: js{"broadcast_media_id"}.getStrVal, thumb: image)
+    video: some Video(thumb: image)
   )
 
 proc parseCard(js: JsonNode; urls: JsonNode): Card =
@@ -166,6 +163,9 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
     name = js{"name"}.getStr
     kind = parseEnum[CardKind](name[(name.find(":") + 1) ..< name.len], unknown)
 
+  if kind == unified:
+    return parseUnifiedCard(vals{"unified_card", "string_value"}.getStr)
+
   result = Card(
     kind: kind,
     url: vals.getCardUrl(kind),
@@ -190,7 +190,7 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
     result.url = vals{"player_url"}.getStrVal
     if "youtube.com" in result.url:
       result.url = result.url.replace("/embed/", "/watch?v=")
-  of audiospace, unified, unknown:
+  of audiospace, unknown:
     result.title = "This card type is not supported."
   else: discard
 

src/types.nim

@@ -70,12 +70,11 @@ type
     vmap = "video/vmap"
 
   VideoVariant* = object
-    videoType*: VideoType
+    contentType*: VideoType
     url*: string
     bitrate*: int
 
   Video* = object
-    videoId*: string
     durationMs*: int
     url*: string
     thumb*: string
@@ -147,8 +146,6 @@ type
 
   Card* = object
     kind*: CardKind
-    id*: string
-    query*: string
     url*: string
     title*: string
     dest*: string

src/views/tweet.nim

@@ -97,7 +97,7 @@ proc renderVideo*(video: Video; prefs: Prefs; path: string): VNode =
           img(src=thumb)
           renderVideoDisabled(video, path)
         else:
-          let vid = video.variants.filterIt(it.videoType == video.playbackType)
+          let vid = video.variants.filterIt(it.contentType == video.playbackType)
           let source = getVidUrl(vid[0].url)
           case video.playbackType
           of mp4: