testing new trending stuff

integration_tests/bchain/postdata_test.py

@@ -78,12 +78,35 @@ def test_test_top20_vote_and_comment(steem):
 
     df = tppp.preprocess(df, ncores=1)
 
-    account = config.ACCOUNT
-
     tbpd.vote_and_comment_on_topK(df, poster, 'laida',
                                   overview_permalink='lll')
 
 
 @pytest.mark.skipif(config.PASSWORD is None, reason="needs posting key")
 def test_create_wallet(steem):
-    tbpd.create_wallet(steem, config.PASSWORD, config.POSTING_KEY)
+    tbpd.create_wallet(steem, config.PASSWORD, config.POSTING_KEY)
+
+
+@pytest.mark.skipif(config.PASSWORD is None, reason="needs posting key")
+def test_test_top_trending_post(steem):
+
+    steem.wallet.unlock(config.PASSWORD)
+
+    poster = Poster(steem=steem,
+                    account=config.ACCOUNT,
+                    waiting_time=0.1)
+
+    posts = random_data.create_n_random_posts(10)
+    df = pd.DataFrame(posts)
+    df['reward'] = df.reward
+    df['predicted_votes'] = df.votes
+
+    df = tppp.preprocess(df, ncores=1)
+
+    date = pd.datetime.utcnow().date()
+
+    tbpd.post_top_trending_list(df, poster, date,
+                                overview_permalink='iii',
+                                trufflepicks_permalink='kkk',
+                                steem_amount=10,
+                                sbd_amount=10)

integration_tests/trending0bot_test.py

@@ -0,0 +1,40 @@
+import pytest
+
+import pandas as pd
+import trufflepig.bchain.getdata as tpgd
+import trufflepig.preprocessing as tppp
+import trufflepig.bchain.getaccountdata as tpad
+from trufflepig.bchain.poster import Poster
+import trufflepig.trending0bidbots as tt0b
+from trufflepig.testutils.pytest_fixtures import steem
+from trufflepig import config
+
+
+@pytest.mark.skipif(config.PASSWORD is None, reason="needs posting key")
+def test_create_trending_post(steem):
+
+    current_datetime = pd.datetime.utcnow()
+
+    data_frame = tpgd.scrape_hour_data(steem=steem,
+                                             current_datetime=current_datetime,
+                                             ncores=32,
+                                             offset_hours=8,
+                                       hours=1, stop_after=20)
+
+
+    min_datetime = data_frame.created.min()
+    max_datetime = data_frame.created.max() + pd.Timedelta(days=8)
+    upvote_payments = tpad.get_upvote_payments_to_bots(steem=steem,
+                                                  min_datetime=min_datetime,
+                                                  max_datetime=max_datetime,
+                                                 bots=['booster'])
+
+    data_frame = df = tppp.preprocess(data_frame, ncores=1)
+
+    data_frame = tppp.compute_bidbot_correction(post_frame=data_frame,
+                                                upvote_payments=upvote_payments)
+    account = config.ACCOUNT
+    poster = Poster(account=account, steem=steem)
+
+    tt0b.create_trending_post(data_frame, upvote_payments, poster, 'test',
+                         'test', current_datetime)

tests/bchain/posts_test.py

@@ -157,4 +157,26 @@ def test_weekly_update():
                   topics=topics)
 
     assert body
+    assert title
+
+
+def test_top_trending_post():
+    posts = random_data.create_n_random_posts(10)
+    df = pd.DataFrame(posts)
+    df = tppp.preprocess(df, ncores=1)
+
+    date = pd.datetime.utcnow().date()
+    df.image_urls = df.body.apply(lambda x: tptf.get_image_urls(x))
+
+    title, post = tbpo.top_trending_post(topN_authors=df.author,
+                                 topN_permalinks=df.permalink,
+                                 topN_titles=df.title,
+                                 topN_filtered_bodies=df.filtered_body,
+                                 topN_image_urls=df.image_urls,
+                                 topN_rewards=df.reward, sbd_amount=10,
+                                 steem_amount=10,
+                                 title_date=date, trufflepicks_link='de.de.de',
+                                 truffle_link='www.de')
+
+    assert post
     assert title

trufflepig/bchain/postdata.py

@@ -17,6 +17,8 @@
 
 PERMALINK_TEMPLATE = 'daily-truffle-picks-{date}'
 
+TRENDING_PERMALINK_TEMPLATE = 'non-bot-trending-{date}'
+
 
 def post_topN_list(sorted_post_frame, poster,
                    current_datetime, overview_permalink, N=10):
@@ -38,7 +40,7 @@ def post_topN_list(sorted_post_frame, poster,
     df = sorted_post_frame.iloc[:N, :]
 
     logger.info('Creating top {} post'.format(N))
-    df.first_image_url = df.body.apply(lambda x: tftf.get_image_urls(x))
+    first_image_urls = df.body.apply(lambda x: tftf.get_image_urls(x))
 
     steem_per_mvests = Converter(poster.steem).steem_per_mvests()
     truffle_link = 'https://steemit.com/steemit/@{}/{}'.format(poster.account,
@@ -48,7 +50,7 @@ def post_topN_list(sorted_post_frame, poster,
                                  topN_permalinks=df.permalink,
                                  topN_titles=df.title,
                                  topN_filtered_bodies=df.filtered_body,
-                                 topN_image_urls=df.first_image_url,
+                                 topN_image_urls=first_image_urls,
                                  topN_rewards=df.predicted_reward,
                                  topN_votes=df.predicted_votes,
                                  title_date=current_datetime,
@@ -57,8 +59,6 @@ def post_topN_list(sorted_post_frame, poster,
 
     permalink = PERMALINK_TEMPLATE.format(date=current_datetime.strftime('%Y-%m-%d'))
     logger.info('Posting top post with permalink: {}'.format(permalink))
-    logger.info(title)
-    logger.info(body)
     poster.post(body=body,
                 permalink=permalink,
                 title=title,
@@ -149,6 +149,46 @@ def vote_and_comment_on_topK(sorted_post_frame, poster,
             poster.steem.reconnect()
 
 
+def post_top_trending_list(sorted_post_frame, poster,
+                            current_datetime, trufflepicks_permalink,
+                           overview_permalink, sbd_amount,
+                           steem_amount, N=10):
+    """ Post the no bot trending toplist to the blockchain"""
+    df = sorted_post_frame.iloc[:N, :]
+
+    logger.info('Creating top {} post'.format(N))
+    first_image_urls = df.body.apply(lambda x: tftf.get_image_urls(x))
+
+    steem_per_mvests = Converter(poster.steem).steem_per_mvests()
+    truffle_link = 'https://steemit.com/steemit/@{}/{}'.format(poster.account,
+                                                               overview_permalink)
+    trufflepicks_link = 'https://steemit.com/steemit/@{}/{}'.format(poster.account,
+                                                               trufflepicks_permalink)
+
+    title, body = tfbp.top_trending_post(topN_authors=df.author,
+                                 topN_permalinks=df.permalink,
+                                 topN_titles=df.title,
+                                 topN_filtered_bodies=df.filtered_body,
+                                 topN_image_urls=first_image_urls,
+                                 topN_rewards=df.reward,
+                                 title_date=current_datetime,
+                                 truffle_link=truffle_link,
+                                 steem_per_mvests=steem_per_mvests,
+                                 trufflepicks_link=trufflepicks_link,
+                                 sbd_amount=sbd_amount,
+                                 steem_amount=steem_amount)
+
+    permalink = TRENDING_PERMALINK_TEMPLATE.format(date=current_datetime.strftime('%Y-%m-%d'))
+    logger.info('Posting top trending post with permalink: {}'.format(permalink))
+    poster.post(body=body,
+                permalink=permalink,
+                title=title,
+                tags=tfbp.TRENDING_TAGS,
+                self_vote=False)
+
+    return permalink
+
+
 def create_wallet(steem, password, posting_key,
                   active_key=None):
     """ Creates a new wallet
@@ -184,3 +224,6 @@ def create_wallet(steem, password, posting_key,
             logger.info('Key already present')
 
     logger.info('Wallet is ready')
+
+
+

trufflepig/bchain/posts.py

@@ -8,6 +8,7 @@
 DELEGATION_LINK = 'https://v2.steemconnect.com/sign/delegateVestingShares?delegator=&delegatee=trufflepig&vesting_shares={shares}%20VESTS'
 QUOTE_MAX_LENGTH = 496
 TAGS = ['steemit', 'curation', 'minnowsupport', 'technology', 'community']
+TRENDING_TAGS = ['steemit', 'curation', 'bots', 'technology', 'community']
 
 BODY_PREFIX = ''  # to announce tests etc.
 
@@ -452,3 +453,110 @@ def weekly_update(current_datetime,
 
     return title, post
 
+
+############### Trending Posts ################################################
+
+def top_trending_list(topN_authors, topN_permalinks, topN_titles,
+              topN_filtered_bodies, topN_image_urls,
+              topN_rewards, quote_max_length, nstart=1):
+    """Creates a toplist string"""
+    topN_entry="""{rank}. [{title}](https://steemit.com/@{author}/{permalink})  --  **by @{author} with a current reward of {reward:d} SBD**
+    
+    {image}{quote}
+
+"""
+
+    result_string = ""
+
+    iterable = zip(topN_authors, topN_permalinks, topN_titles,
+                   topN_filtered_bodies, topN_image_urls,
+                   topN_rewards)
+
+    for idx, (author, permalink, title, filtered_body, img_urls, reward) in enumerate(iterable):
+        rank = idx + nstart
+        quote = '>' + filtered_body[:quote_max_length].replace('\n', ' ').replace('  ', ' ').replace('  ', ' ').replace('  ', ' ') + '...'
+        title = tftf.replace_newlines(title)
+        title = tftf.filter_special_characters(title)
+        if len(img_urls) >= 1:
+            imgstr = """ <div class="pull-right"><img src="{img}" /></div>\n\n    """.format(img=img_urls[0])
+        else:
+            imgstr=''
+        entry = topN_entry.format(rank=rank, author=author, permalink=permalink,
+                                   title=title, quote=quote,
+                                   reward=int(reward), image=imgstr)
+        result_string += entry
+    return result_string
+
+
+def top_trending_post(topN_authors, topN_permalinks, topN_titles, topN_filtered_bodies,
+              topN_image_urls, topN_rewards, title_date,
+              trufflepicks_link, truffle_link, sbd_amount, steem_amount,
+              steem_per_mvests=490, truffle_image=TRUFFLE_IMAGE,
+              quote_max_length=QUOTE_MAX_LENGTH):
+    """Craetes the truffle pig daily toplist post"""
+    title = """Here is how the Steemit Trending Page would look like without Bid Bots! ({date})"""
+
+    post="""## Trending Posts Without Bid Bots
+    
+In the last 24 hours alone people spend at least **{amount}** on post promotions **using bid bots or vote selling services**. I know bid bots are a controversial topic and it is not up to me to decide if these bots are good or bad. Heck, I'm a bot myself, so who am I to judge? However, I can help you with your own judgment by providing data. Besides my [DAILY TRUFFLE PICKS]({trufflepicks_link}), where I try to direct attention to posts that deserve more rewards, I decided to use the data at my disposal to publish another kind of top list.
+
+Nowadays it is incredibly difficult to make it to the trending section without spending about 100 SBD or more on bid bot services. So I asked myself, how would the trending page look like if there were no bid bots? Or to be more precise, how would the trending page look like if we exclude every post bumped by a bid bot? 
+
+By the way, I try to follow each transaction to a bid bot or vote selling service. Yet, if I missed a bot, please do leave a comment so I can include it in the future. Thanks!
+
+
+# The Top 10 Posts NOT Promoted by Bots
+
+So without further ado, here are the top earning and text based posts (excluding dmania etc.) of the last 24 hours of content creators that, to the best of my knowledge, did not pay for voting bots or vote selling services. You can see for yourself how these comapre to the current trending posts on the Steemit front page.
+
+{topN_posts}
+
+So? What is your opinion about these non-bot trending posts? Before I forget, do not miss out on checking my other top list of [DAILY TRUFFLE PICKS]({trufflepicks_link}) to help minnows and promote good content! Moreover, if you want to find out more about me, [here I give a detailed explanation about my inner workings]({truffle_link}).
+
+## You can Help and Contribute
+
+By upvoting and resteeming this top list, you help covering the server costs and finance further development and improvement of my humble self. 
+
+**NEW**: You may further show your support for me and all my daily truffle picks by [**following my curation trail**](https://steemauto.com/dash.php?trail=trufflepig&i=1) on SteemAuto!
+
+## Delegate and Invest in the Bot
+
+If you feel generous, you can delegate Steem Power to me and boost my daily upvotes on the truffle posts in my other top list. In return, I will provide you with a *small* compensation for your trust in me and your locked Steem Power. **Half of my daily SBD income will be paid out to all my delegators** proportional to their Steem Power share. Payouts will start 3 days after your delegation.
+
+Click on one of the following links to delegate **[2]({sp2}), [5]({sp5}), [10]({sp10}), [20]({sp20}), [50]({sp50}), [100]({sp100}), [200]({sp200}), [500]({sp500}), [1000]({sp1000}), [2000]({sp2000}),** or even **[5000 Steem Power]({sp5000})**. Thank You!
+
+Cheers,
+
+{truffle_image}
+
+*`TrufflePig`*
+    """
+
+    if sbd_amount > 0 and steem_amount > 0:
+        amount = '{} SBD and {} STEEM'.format(int(sbd_amount), int(steem_amount))
+    elif sbd_amount > 0:
+        amount = '{} SBD'.format(int(sbd_amount))
+    elif steem_amount > 0:
+        amount = '{} STEEM'.format(int(steem_amount))
+    else:
+        raise RuntimeError('Should not happen!')
+
+    link_dict = get_delegation_link(steem_per_mvests=steem_per_mvests)
+
+    topN_posts = top_trending_list(topN_authors=topN_authors,
+                              topN_permalinks=topN_permalinks,
+                              topN_titles=topN_titles,
+                              topN_filtered_bodies=topN_filtered_bodies,
+                              topN_image_urls=topN_image_urls,
+                              topN_rewards=topN_rewards,
+                              quote_max_length=quote_max_length)
+
+    title = title.format(date=title_date.strftime('%d.%m.%Y'))
+    post = post.format(topN_posts=topN_posts,
+                          truffle_image=truffle_image,
+                          trufflepicks_link=trufflepicks_link,
+                          truffle_link=truffle_link,
+                          amount=amount,
+                          **link_dict)
+    post = BODY_PREFIX + post
+    return title, post

trufflepig/main.py

@@ -20,6 +20,7 @@
 import trufflepig.bchain.postweeklyupdate as tppw
 from trufflepig.bchain.mpsteem import MPSteem
 from trufflepig.bchain.poster import Poster
+import trufflepig.trending0bidbots as tt0b
 
 
 logger = logging.getLogger(__name__)
@@ -229,6 +230,23 @@ def main():
                                   topN_permalink=permalink,
                                   overview_permalink=overview_permalink)
 
+    logger.info('Computing the top trending without bidbots')
+    logger.info('Searching for bid bots and bought votes')
+    min_datetime = sorted_frame.created.min()
+    max_datetime = sorted_frame.created.max() + pd.Timedelta(days=8)
+    upvote_payments = tpad.get_upvote_payments_to_bots(steem=steem,
+                                                  min_datetime=min_datetime,
+                                                  max_datetime=max_datetime)
+    logger.info('Adjusting votes and reward')
+    sorted_frame = tppp.compute_bidbot_correction(post_frame=sorted_frame,
+                                                upvote_payments=upvote_payments)
+    tt0b.create_trending_post(sorted_frame,
+                              upvote_payments=upvote_payments,
+                              poster=poster,
+                              topN_permalink=permalink,
+                              overview_permalink=overview_permalink)
+
+
     logger.info('Done with normal duty, answering manual calls!')
     tfod.call_a_pig(poster=poster,
                     pipeline=pipeline,

trufflepig/trending0bidbots.py

@@ -0,0 +1,43 @@
+from steem.amount import Amount
+import trufflepig.bchain.postdata as tbpd
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+
+def compute_total_sbd(upvote_payments):
+    sbd = 0
+    steem = 0
+    for (author, permalink), payments in upvote_payments.items():
+        for payment in payments.values():
+            amount = Amount(payment['amount'])
+            value = amount.amount
+            asset = amount.asset
+            if asset == 'SBD':
+                sbd += value
+            elif asset == 'STEEM':
+                steem += value
+    return sbd, steem
+
+
+def create_trending_post(post_frame, upvote_payments, poster, topN_permalink,
+                         overview_permalink, current_datetime):
+    total_paid_sbd, total_paid_steem = compute_total_sbd(upvote_payments)
+
+    logger.info('People spend {} SBD and {} Steem on Bid Bots the last 24 '
+                'hours.'.format(total_paid_sbd, total_paid_steem))
+
+    no_bid_bots_frame = post_frame.loc[post_frame.bought_votes == 0, :].copy()
+    no_bid_bots_frame.sort_values('reward', inplace=True, ascending=False)
+
+    logger.info('Voted Posts {} out of {}'.format(len(post_frame) - len(no_bid_bots_frame),
+                                                  len(post_frame)))
+
+    tbpd.post_top_trending_list(no_bid_bots_frame, poster, current_datetime,
+                                overview_permalink=overview_permalink,
+                                trufflepicks_permalink=topN_permalink,
+                                steem_amount=total_paid_steem,
+                                sbd_amount=total_paid_sbd)
+