-
Notifications
You must be signed in to change notification settings - Fork 12.2k
/
PartialEntityCleaner.scala
80 lines (69 loc) · 2.8 KB
/
PartialEntityCleaner.scala
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
package com.twitter.tweetypie
package hydrator
import com.twitter.mediaservices.commons.tweetmedia.thriftscala._
import com.twitter.tweetypie.media._
import com.twitter.tweetypie.thriftscala._
import scala.collection.Set
/**
* Removes partial Url, Media, and Mention entities that were not
* fully hydrated. Rather than returning no value or a value with
* incomplete entities on an entity hydration failure, we gracefully
* degrade to just omitting those entities. This step needs to be
* applied in the post-cache filter, so that we don't cache the value
* with missing entities.
*
* A MediaEntity will first be converted back to a UrlEntity if it is only
* partially hydrated. If the resulting UrlEntity is itself then only partially
* hydrated, it will get dropped also.
*/
object PartialEntityCleaner {
def apply(stats: StatsReceiver): Mutation[Tweet] = {
val scopedStats = stats.scope("partial_entity_cleaner")
Mutation
.all(
Seq(
TweetLenses.urls.mutation(urls.countMutations(scopedStats.counter("urls"))),
TweetLenses.media.mutation(media.countMutations(scopedStats.counter("media"))),
TweetLenses.mentions.mutation(mentions.countMutations(scopedStats.counter("mentions")))
)
)
.onlyIf(!isRetweet(_))
}
private[this] def clean[E](isPartial: E => Boolean) =
Mutation[Seq[E]] { items =>
items.partition(isPartial) match {
case (Nil, nonPartial) => None
case (partial, nonPartial) => Some(nonPartial)
}
}
private[this] val mentions =
clean[MentionEntity](e => e.userId.isEmpty || e.name.isEmpty)
private[this] val urls =
clean[UrlEntity](e =>
isNullOrEmpty(e.url) || isNullOrEmpty(e.expanded) || isNullOrEmpty(e.display))
private[this] val media =
Mutation[Seq[MediaEntity]] { mediaEntities =>
mediaEntities.partition(isPartialMedia) match {
case (Nil, nonPartial) => None
case (partial, nonPartial) => Some(nonPartial)
}
}
def isPartialMedia(e: MediaEntity): Boolean =
e.fromIndex < 0 ||
e.toIndex <= 0 ||
isNullOrEmpty(e.url) ||
isNullOrEmpty(e.displayUrl) ||
isNullOrEmpty(e.mediaUrl) ||
isNullOrEmpty(e.mediaUrlHttps) ||
isNullOrEmpty(e.expandedUrl) ||
e.mediaInfo.isEmpty ||
e.mediaKey.isEmpty ||
(MediaKeyClassifier.isImage(MediaKeyUtil.get(e)) && containsInvalidSizeVariant(e.sizes))
private[this] val userMentions =
clean[UserMention](e => e.screenName.isEmpty || e.name.isEmpty)
def isNullOrEmpty(optString: Option[String]): Boolean =
optString.isEmpty || optString.exists(isNullOrEmpty(_))
def isNullOrEmpty(str: String): Boolean = str == null || str.isEmpty
def containsInvalidSizeVariant(sizes: Set[MediaSize]): Boolean =
sizes.exists(size => size.height == 0 || size.width == 0)
}