Cashtag functionality (#421)

* Cashtag functionality

* Update write_meta.py

elasticsearch/index-tweets.json

@@ -12,6 +12,7 @@ PUT twinttweets
           "location": {"type": "keyword"},
           "tweet": {"type": "text"},
           "hashtags": {"type": "keyword"},
+          "cashtags": {"type": "keyword"},
           "user_id": {"type": "long"},
           "user_id_str": {"type": "keyword"},
           "username": {"type": "keyword"},

twint/__version__.py

@@ -1,3 +1,3 @@
-VERSION = (1, 2, 1)
+VERSION = (1, 2, 2)
 
 __version__ = '.'.join(map(str, VERSION))

twint/cli.py

@@ -78,6 +78,7 @@ def initialize(args):
     c.Store_csv = args.csv
     c.Store_json = args.json
     c.Show_hashtags = args.hashtags
+    c.Show_cashtags = args.cashtags
     c.Limit = args.limit
     c.Count = args.count
     c.Stats = args.stats
@@ -136,6 +137,7 @@ def options():
     ap.add_argument("--csv", help="Write as .csv file.", action="store_true")
     ap.add_argument("--json", help="Write as .json file", action="store_true")
     ap.add_argument("--hashtags", help="Output hashtags in seperate column.", action="store_true")
+    ap.add_argument("--cashtags", help="Output cashtags in seperate column.", action="store_true")
     ap.add_argument("--userid", help="Twitter user id.")
     ap.add_argument("--limit", help="Number of Tweets to pull (Increments of 20).")
     ap.add_argument("--count", help="Display number of Tweets scraped at the end of session.",

twint/config.py

@@ -19,6 +19,7 @@ class Config:
     Store_json = False
     Custom = {"tweet": None, "user": None, "username": None}
     Show_hashtags = False
+    Show_cashtags = False
     Limit = None
     Count = None
     Stats = False

twint/format.py

@@ -12,6 +12,7 @@ def Tweet(config, t):
         output = output.replace("{tweet}", t.tweet)
         output = output.replace("{location}", t.location)
         output = output.replace("{hashtags}", str(t.hashtags))
+        output = output.replace("{cashtags}", str(t.cashtags))
         output = output.replace("{replies}", t.replies_count)
         output = output.replace("{retweets}", t.retweets_count)
         output = output.replace("{likes}", t.likes_count)
@@ -30,6 +31,9 @@ def Tweet(config, t):
         if config.Show_hashtags:
             hashtags = ",".join(t.hashtags)
             output += f" {hashtags}"
+        if config.Show_cashtags:
+            cashtags = ",".join(t.cashtags)
+            output += f" {cashtags}"
         if config.Stats:
             output += f" | {t.replies_count} replies {t.retweets_count} retweets {t.likes_count} likes"
         if config.Location:

twint/output.py

@@ -58,6 +58,8 @@ def _output(obj, output, config, **extra):
                 obj.mentions[i] = obj.mentions[i].lower()
             for i in range(len(obj.hashtags)):
                 obj.hashtags[i] = obj.hashtags[i].lower()
+            for i in range(len(obj.cashtags)):
+                obj.cashtags[i] = obj.cashtags[i].lower()
         else:
             logme.info('_output:Lowercase:hiddenTweetFound')
             print("[x] Hidden tweet found, account suspended due to violation of TOS")

twint/storage/db.py

@@ -72,6 +72,7 @@ def init(db):
                     link text,
                     mentions text,
                     hashtags text,
+                    cashtags text,
                     urls text,
                     photos text,
                     quote_url text,
@@ -240,6 +241,7 @@ def tweets(conn, Tweet, config):
                     Tweet.link,
                     ",".join(Tweet.mentions),
                     ",".join(Tweet.hashtags),
+                    ",".join(Tweet.cashtags),
                     ",".join(Tweet.urls),
                     ",".join(Tweet.photos),
                     Tweet.quote_url,

twint/storage/elasticsearch.py

@@ -65,6 +65,7 @@ def createIndex(config, instance, **scope):
                         "location": {"type": "keyword"},
                         "tweet": {"type": "text"},
                         "hashtags": {"type": "keyword"},
+                        "cashtags": {"type": "keyword"},
                         "user_id": {"type": "long"},
                         "user_id_str": {"type": "keyword"},
                         "username": {"type": "keyword"},
@@ -199,6 +200,7 @@ def Tweet(Tweet, config):
                 "location": Tweet.location,
                 "tweet": Tweet.tweet,
                 "hashtags": Tweet.hashtags,
+                "cashtags": Tweet.cashtags,
                 "user_id": Tweet.user_id,
                 "user_id_str": Tweet.user_id_str,
                 "username": Tweet.username,

twint/storage/panda.py

@@ -70,6 +70,7 @@ def update(object, config):
             "location": Tweet.location,
             "tweet": Tweet.tweet,
             "hashtags": Tweet.hashtags,
+            "cashtags": Tweet.cashtags,
             "user_id": Tweet.user_id,
             "user_id_str": Tweet.user_id_str,
             "username": Tweet.username,

twint/storage/write_meta.py

@@ -19,6 +19,7 @@ def tweetData(t):
             "likes_count": int(t.likes_count),
             "location": t.location,
             "hashtags": t.hashtags,
+            "cashtags": t.cashtags,
             "link": t.link,
             "retweet": t.retweet,
             "quote_url": t.quote_url,
@@ -48,6 +49,7 @@ def tweetFieldnames():
             "likes_count",
             "location",
             "hashtags",
+            "cashtags",
             "link",
             "retweet",
             "quote_url",

twint/tweet.py

@@ -88,6 +88,7 @@ def Tweet(tw, location, config):
     t.tweet = getText(tw)
     t.location = location
     t.hashtags = [hashtag.text for hashtag in tw.find_all("a","twitter-hashtag")]
+    t.cashtags = [cashtag.text for cashtag in tw.find_all("a", "twitter-cashtag")]
     t.replies_count = getStat(tw, "reply")
     t.retweets_count = getStat(tw, "retweet")
     t.likes_count = getStat(tw, "favorite")