Merge a087334 into dae41e0

SmokinCaterpillar · Apr 2, 2018 · dc07b65 · dc07b65
2 parents dae41e0 + a087334
commit dc07b65
Show file tree

Hide file tree

Showing 11 changed files with 258 additions and 24 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,10 +1,11 @@
-* 0.6.2a* - 2018-03-27
+* 0.6.2a* - 2018-04-01
 
 * SQLite is now used to store blockchain data
 * New Poster class to allow for waiting
 * New posting of trending without bots
 * BUG FIX: Images and links are now properly fitlered
 * BUG FIX: Better displaying on busy!
+* New vote rep score top list
 
 *0.6.1a* - 2018-03-22
 

diff --git a/integration_tests/trending0bot_test.py b/integration_tests/trending0bot_test.py
@@ -28,7 +28,7 @@ def test_create_trending_post(steem):
                                                   max_datetime=max_datetime,
                                                  bots=['booster'])
 
-    data_frame = df = tppp.preprocess(data_frame, ncores=1)
+    data_frame = tppp.preprocess(data_frame, ncores=1)
 
     data_frame = tppp.compute_bidbot_correction(post_frame=data_frame,
                                                 upvote_payments=upvote_payments)
@@ -37,4 +37,26 @@ def test_create_trending_post(steem):
                     no_posting_key_mode=config.PASSWORD is None)
 
     tt0b.create_trending_post(data_frame, upvote_payments, poster, 'test',
+                         'test', current_datetime)
+
+
+def test_create_community_trending_post(steem):
+
+    current_datetime = pd.datetime.utcnow()
+
+    data_frame = tpgd.scrape_hour_data(steem=steem,
+                                             current_datetime=current_datetime,
+                                             ncores=32,
+                                             offset_hours=8,
+                                       hours=1, stop_after=20)
+
+    data_frame = tppp.preprocess(data_frame, ncores=1)
+
+    data_frame = tppp.compute_reputation_vote_score(data_frame)
+
+    account = config.ACCOUNT
+    poster = Poster(account=account, steem=steem,
+                    no_posting_key_mode=config.PASSWORD is None)
+
+    tt0b.create_contributor_top_post(data_frame, poster, 'test',
                          'test', current_datetime)
diff --git a/tests/bchain/posts_test.py b/tests/bchain/posts_test.py
@@ -179,4 +179,26 @@ def test_top_trending_post():
                                  truffle_link='www.de')
 
     assert post
-    assert title
+    assert title
+
+
+def test_top_rep_score_post():
+    posts = random_data.create_n_random_posts(10)
+    df = pd.DataFrame(posts)
+    df = tppp.preprocess(df, ncores=1)
+    df = tppp.compute_reputation_vote_score(df)
+
+    date = pd.datetime.utcnow().date()
+    df.image_urls = df.body.apply(lambda x: tptf.get_image_urls(x))
+
+    title, post = tbpo.top_contributor_vote_post(topN_authors=df.author,
+                                 topN_permalinks=df.permalink,
+                                 topN_titles=df.title,
+                                 topN_filtered_bodies=df.filtered_body,
+                                 topN_image_urls=df.image_urls,
+                                 topN_rep_votes_score=df.reputation_votes_score,
+                                 title_date=date, trufflepicks_link='de.de.de',
+                                 truffle_link='www.de')
+
+    assert post
+    assert title
diff --git a/tests/preprocessing_test.py b/tests/preprocessing_test.py
@@ -52,3 +52,11 @@ def test_bid_bot_correction():
     assert all(post_frame.adjusted_reward >= 0)
     assert post_frame.adjusted_votes.mean() < post_frame.votes.mean()
     assert all(post_frame.adjusted_votes >= 0)
+
+
+def test_repvote_score():
+    posts = create_n_random_posts(30)
+    post_frame = pd.DataFrame(posts)
+
+    post_frame = tppp.compute_reputation_vote_score(post_frame)
+    assert 'reputation_votes_score' in post_frame.columns
diff --git a/trufflepig/bchain/postdata.py b/trufflepig/bchain/postdata.py
@@ -19,6 +19,8 @@
 
 TRENDING_PERMALINK_TEMPLATE = 'non-bot-trending-{date}'
 
+COMMUNITY_PERMALINK_TEMPLATE = 'community-trending-{date}'
+
 
 def post_topN_list(sorted_post_frame, poster,
                    current_datetime, overview_permalink, N=10):
@@ -189,6 +191,43 @@ def post_top_trending_list(sorted_post_frame, poster,
     return permalink
 
 
+def post_top_rep_votes_score_list(sorted_post_frame, poster,
+                            current_datetime, trufflepicks_permalink,
+                           overview_permalink, N=10):
+    """ Post the no bot trending toplist to the blockchain"""
+    df = sorted_post_frame.iloc[:N, :]
+
+    logger.info('Creating top {} post'.format(N))
+    first_image_urls = df.body.apply(lambda x: tftf.get_image_urls(x))
+
+    steem_per_mvests = Converter(poster.steem).steem_per_mvests()
+    truffle_link = 'https://steemit.com/steemit/@{}/{}'.format(poster.account,
+                                                               overview_permalink)
+    trufflepicks_link = 'https://steemit.com/steemit/@{}/{}'.format(poster.account,
+                                                               trufflepicks_permalink)
+
+    title, body = tfbp.top_contributor_vote_post(topN_authors=df.author,
+                                 topN_permalinks=df.permalink,
+                                 topN_titles=df.title,
+                                 topN_filtered_bodies=df.filtered_body,
+                                 topN_image_urls=first_image_urls,
+                                 topN_rep_votes_score=df.reputation_votes_score,
+                                 title_date=current_datetime,
+                                 truffle_link=truffle_link,
+                                 steem_per_mvests=steem_per_mvests,
+                                 trufflepicks_link=trufflepicks_link)
+
+    permalink = COMMUNITY_PERMALINK_TEMPLATE.format(date=current_datetime.strftime('%Y-%m-%d'))
+    logger.info('Posting top trending post with permalink: {}'.format(permalink))
+    poster.post(body=body,
+                permalink=permalink,
+                title=title,
+                tags=tfbp.TRENDING_TAGS,
+                self_vote=False)
+
+    return permalink
+
+
 def create_wallet(steem, password, posting_key,
                   active_key=None):
     """ Creates a new wallet

diff --git a/trufflepig/bchain/posts.py b/trufflepig/bchain/posts.py
@@ -6,7 +6,7 @@
 TRUFFLE_IMAGE_SMALL = '![trufflepig](https://raw.githubusercontent.com/SmokinCaterpillar/TrufflePig/master/img/trufflepig17_small.png)'
 TRUFFLE_IMAGE = '![trufflepig](https://raw.githubusercontent.com/SmokinCaterpillar/TrufflePig/master/img/trufflepig17.png)'
 DELEGATION_LINK = 'https://v2.steemconnect.com/sign/delegateVestingShares?delegator=&delegatee=trufflepig&vesting_shares={shares}%20VESTS'
-QUOTE_MAX_LENGTH = 496
+QUOTE_MAX_LENGTH = 596
 TAGS = ['steemit', 'curation', 'minnowsupport', 'technology', 'community']
 TRENDING_TAGS = ['steemit', 'curation', 'bots', 'technology', 'community']
 
@@ -559,4 +559,98 @@ def top_trending_post(topN_authors, topN_permalinks, topN_titles, topN_filtered_
                           amount=amount,
                           **link_dict)
     post = BODY_PREFIX + post
-    return title, post
+    return title, post
+
+
+def top_rep_vote_list(topN_authors, topN_permalinks, topN_titles,
+              topN_filtered_bodies, topN_image_urls,
+              topN_rep_votes_score, quote_max_length, nstart=1):
+    """Creates a toplist string"""
+    topN_entry="""**#{rank}** [{title}](https://steemit.com/@{author}/{permalink})  --  **by @{author} with a reputation votes score of {score:d}**
+    
+{image}{quote}
+
+"""
+
+    result_string = ""
+
+    iterable = zip(topN_authors, topN_permalinks, topN_titles,
+                   topN_filtered_bodies, topN_image_urls,
+                   topN_rep_votes_score)
+
+    for idx, (author, permalink, title, filtered_body, img_urls, score) in enumerate(iterable):
+        rank = idx + nstart
+        quote = '>' + filtered_body[:quote_max_length].replace('\n', ' ').replace('  ', ' ').replace('  ', ' ').replace('  ', ' ') + '...'
+        title = tftf.replace_newlines(title)
+        title = tftf.filter_special_characters(title)
+        if len(img_urls) >= 1:
+            imgstr = """ <div class="pull-right"><img src="{img}" /></div>\n\n""".format(img=img_urls[0])
+        else:
+            imgstr=''
+        entry = topN_entry.format(rank=rank, author=author, permalink=permalink,
+                                   title=title, quote=quote,
+                                   score=int(score), image=imgstr)
+        result_string += entry
+    return result_string
+
+
+def top_contributor_vote_post(topN_authors, topN_permalinks, topN_titles, topN_filtered_bodies,
+              topN_image_urls, topN_rep_votes_score, title_date,
+              trufflepicks_link, truffle_link,
+              steem_per_mvests=490, truffle_image=TRUFFLE_IMAGE,
+              quote_max_length=QUOTE_MAX_LENGTH):
+    """Craetes the truffle pig daily toplist post"""
+    title = """Here is how the Steemit Trending Page would look like if Reputation and not Money counts! ({date})"""
+
+    post="""## Trending Posts According to Votes by Active Community Members
+    
+Well, I do publish a lot of top lists, I admit that. However, my aim is to help promote good content and top lists are a perfect tool to achieve this. Besides my [DAILY TRUFFLE PICKS]({trufflepicks_link}) and how the trending page would look like without bots, I want to show you what posts the Steemit community really favors. So on alternating days, I publish either this or the non-bot top list!
+
+Here I ask the question how would Steemit look like if not money talked, but reputation!? In the following, I will order the posts according to a metric I call **reputation votes score**. It is simply the weight of a vote times the voter's reputation. I slightly adjust the reputation by subtracting 24, so that new members have a voting power of only 1 and the major voting power is in the hands of the active and established Steemit members. For instance, if your post got a 100% vote by someone with a reputation of 49 and a 50% vote by someone with 74 reputation, the score would be `1.0 * (49 - 24) + 0.5 * (74 - 24) = 50`.
+
+
+# The Top 10 Posts Selected by the Community
+
+So without further ado, here are the top text based posts (excluding dmania etc.) of the last 24 hours of content creators that are voted by active users of this community and ranked according to my **reputation votes score**. A list of what the active Steemit community really prefers, so to say. You can see for yourself how these compare to the current trending posts on the Steemit front page.
+
+{topN_posts}
+
+So? What is your opinion about these community trending posts? Before I forget, do not miss out on checking my other top list of [DAILY TRUFFLE PICKS]({trufflepicks_link}) to help minnows and promote good content! Moreover, if you want to find out more about me, [here I give a detailed explanation about my inner workings]({truffle_link}).
+
+## You can Help and Contribute
+
+By upvoting and resteeming this top list, you help covering the server costs and finance further development and improvements. 
+
+**NEW**: You may further show your support for me and all my daily truffle picks by [**following my curation trail**](https://steemauto.com/dash.php?trail=trufflepig&i=1) on SteemAuto!
+
+## Delegate and Invest in the Bot
+
+If you feel generous, you can delegate Steem Power to me and boost my daily upvotes on the truffle posts in my other top list. In return, I will provide you with a *small* compensation for your trust in me and your locked Steem Power. **Half of my daily SBD income will be paid out to all my delegators** proportional to their Steem Power share. Payouts will start 3 days after your delegation.
+
+Click on one of the following links to delegate **[2]({sp2}), [5]({sp5}), [10]({sp10}), [20]({sp20}), [50]({sp50}), [100]({sp100}), [200]({sp200}), [500]({sp500}), [1000]({sp1000}), [2000]({sp2000}),** or even **[5000 Steem Power]({sp5000})**. Thank You!
+
+Cheers,
+
+{truffle_image}
+
+*`TrufflePig`*
+    """
+
+    link_dict = get_delegation_link(steem_per_mvests=steem_per_mvests)
+
+    topN_posts = top_rep_vote_list(topN_authors=topN_authors,
+                              topN_permalinks=topN_permalinks,
+                              topN_titles=topN_titles,
+                              topN_filtered_bodies=topN_filtered_bodies,
+                              topN_image_urls=topN_image_urls,
+                              topN_rep_votes_score=topN_rep_votes_score,
+                              quote_max_length=quote_max_length)
+
+    title = title.format(date=title_date.strftime('%d.%m.%Y'))
+    post = post.format(topN_posts=topN_posts,
+                          truffle_image=truffle_image,
+                          trufflepicks_link=trufflepicks_link,
+                          truffle_link=truffle_link,
+                          **link_dict)
+    post = BODY_PREFIX + post
+    return title, post
diff --git a/trufflepig/main.py b/trufflepig/main.py
@@ -230,23 +230,31 @@ def main():
                                   topN_permalink=permalink,
                                   overview_permalink=overview_permalink)
 
-    logger.info('Computing the top trending without bidbots')
-    logger.info('Searching for bid bots and bought votes')
-    min_datetime = sorted_frame.created.min()
-    max_datetime = sorted_frame.created.max() + pd.Timedelta(days=1)
-    upvote_payments = tpad.get_upvote_payments_to_bots(steem=steem,
-                                                  min_datetime=min_datetime,
-                                                  max_datetime=max_datetime)
-    logger.info('Adjusting votes and reward')
-    sorted_frame = tppp.compute_bidbot_correction(post_frame=sorted_frame,
-                                                upvote_payments=upvote_payments)
-    tt0b.create_trending_post(sorted_frame,
-                              upvote_payments=upvote_payments,
-                              poster=poster,
-                              topN_permalink=permalink,
-                              overview_permalink=overview_permalink,
-                              current_datetime=current_datetime)
-
+    if current_datetime.day % 2 == 1:
+        logger.info('Computing the top trending without bidbots')
+        logger.info('Searching for bid bots and bought votes')
+        min_datetime = sorted_frame.created.min()
+        max_datetime = sorted_frame.created.max() + pd.Timedelta(days=1)
+        upvote_payments = tpad.get_upvote_payments_to_bots(steem=steem,
+                                                      min_datetime=min_datetime,
+                                                      max_datetime=max_datetime)
+        logger.info('Adjusting votes and reward')
+        sorted_frame = tppp.compute_bidbot_correction(post_frame=sorted_frame,
+                                                    upvote_payments=upvote_payments)
+        tt0b.create_trending_post(sorted_frame,
+                                  upvote_payments=upvote_payments,
+                                  poster=poster,
+                                  topN_permalink=permalink,
+                                  overview_permalink=overview_permalink,
+                                  current_datetime=current_datetime)
+    else:
+        logger.info('Computing community vote rep top list')
+        sorted_frame = tppp.compute_reputation_vote_score(sorted_frame)
+        tt0b.create_contributor_top_post(sorted_frame,
+                                         poster=poster,
+                                         topN_permalink=permalink,
+                                         overview_permalink=overview_permalink,
+                                         current_datetime=current_datetime)
 
     logger.info('Done with normal duty, answering manual calls!')
     tfod.call_a_pig(poster=poster,

diff --git a/trufflepig/preprocessing.py b/trufflepig/preprocessing.py
@@ -516,3 +516,21 @@ def compute_bidbot_correction(post_frame, upvote_payments, sbd_punishment_factor
                                                                        total_sbd,
                                                                        total_votes))
     return post_frame
+
+
+def compute_adjusted_simple_reputation(x):
+    """Maps raw author reputation to simple reputation - 24"""
+    result = (np.log10(x) - 9) * 9 + 1
+    return result
+
+
+def compute_reputation_vote_score(post_frame):
+    """Adds the reputation vote score"""
+
+    logger.info('Computing Reputation vote score')
+    repvote_score = post_frame.active_votes.apply(lambda x: [y['percent']/10000. *
+                                                            max(0, compute_adjusted_simple_reputation(int(y['reputation'])))
+                                                        for y in x])
+    repvote_score = repvote_score.apply(lambda x: sum(x))
+    post_frame['reputation_votes_score'] = repvote_score
+    return post_frame
diff --git a/trufflepig/testutils/random_data.py b/trufflepig/testutils/random_data.py
@@ -125,7 +125,8 @@ def create_post():
             'title': title,
             'reward': reward,
             'votes': votes,
-            'active_votes': [dict(voter='marc{}'.format(x), percent=42)
+            'active_votes': [dict(voter='marc{}'.format(x), percent=42,
+                                  reputation='39360778478138')
                             for x in range(votes)],
             'created': created,
             'tags': tags,

diff --git a/trufflepig/testutils/raw_data.py b/trufflepig/testutils/raw_data.py
@@ -5,7 +5,7 @@
             'title': 'Ladida',
             'reward': 42,
             'votes': 1,
-            'active_votes': [{'voter': 'marc', 'percent': 42}],
+            'active_votes': [{'voter': 'marc', 'percent': 42, 'reputation': '34524323434'}],
             'created': pd.to_datetime('2018-01-01'),
             'tags': ['ff', 'kk'],
             'body': ('Mery had little lamb! hello, cheat, cheat, woop, brother'

diff --git a/trufflepig/trending0bidbots.py b/trufflepig/trending0bidbots.py
@@ -62,3 +62,24 @@ def create_trending_post(post_frame, upvote_payments, poster, topN_permalink,
                                 steem_amount=total_paid_steem,
                                 sbd_amount=total_paid_sbd)
 
+
+def create_contributor_top_post(post_frame, poster, topN_permalink,
+                         overview_permalink, current_datetime):
+    sorted_frame = post_frame.sort_values('reputation_votes_score',
+                                          ascending=False)
+
+    for x in range(10):
+        what = sorted_frame.iloc[x]
+        logger.info('{rank}. [{title}](https://steemit.com/@{author}/{permalink})  --  '
+                    '**by @{author} with a reputation vote score of {rvs} '
+                    'SBD'.format(rank=x+1,
+                                 title=what.title,
+                                 author=what.author,
+                                 permalink=what.permalink,
+                                 rvs=what.reputation_votes_score))
+
+    tbpd.post_top_rep_votes_score_list(sorted_post_frame=sorted_frame,
+                                       poster=poster,
+                                       current_datetime=current_datetime,
+                                       trufflepicks_permalink=topN_permalink,
+                                       overview_permalink=overview_permalink)