Merge 71406ca into abb924c

SmokinCaterpillar · Apr 1, 2018 · f177604 · f177604
2 parents abb924c + 71406ca
commit f177604
Show file tree

Hide file tree

Showing 17 changed files with 606 additions and 67 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,9 @@
 
 * SQLite is now used to store blockchain data
 * New Poster class to allow for waiting
+* New posting of trending without bots
+* BUG FIX: Images and links are now properly fitlered
+* BUG FIX: Better displaying on busy!
 
 *0.6.1a* - 2018-03-22
 

diff --git a/integration_tests/bchain/postdata_test.py b/integration_tests/bchain/postdata_test.py
@@ -10,14 +10,15 @@
 from trufflepig.bchain.poster import Poster
 
 
-@pytest.mark.skipif(config.PASSWORD is None, reason="needs posting key")
 def test_test_top10post(steem):
 
-    steem.wallet.unlock(config.PASSWORD)
+    if config.PASSWORD:
+        steem.wallet.unlock(config.PASSWORD)
 
     poster = Poster(steem=steem,
                     account=config.ACCOUNT,
-                    waiting_time=0.1)
+                    waiting_time=0.1,
+                    no_posting_key_mode=config.PASSWORD is None)
 
     posts = random_data.create_n_random_posts(10)
     df = pd.DataFrame(posts)
@@ -35,14 +36,15 @@ def test_test_top10post(steem):
     tbpd.comment_on_own_top_list(df, poster, permalink)
 
 
-@pytest.mark.skipif(config.PASSWORD is None, reason="needs posting key")
 def test_test_all_top_with_real_data(steem):
 
-    steem.wallet.unlock(config.PASSWORD)
+    if config.PASSWORD:
+        steem.wallet.unlock(config.PASSWORD)
 
     poster = Poster(steem=steem,
                     account=config.ACCOUNT,
-                    waiting_time=0.1)
+                    waiting_time=0.1,
+                    no_posting_key_mode=config.PASSWORD is None)
 
     df = tpbg.scrape_hour_data(steem, stop_after=10)
 
@@ -62,14 +64,15 @@ def test_test_all_top_with_real_data(steem):
                                   overview_permalink='jjj')
 
 
-@pytest.mark.skipif(config.PASSWORD is None, reason="needs posting key")
 def test_test_top20_vote_and_comment(steem):
 
-    steem.wallet.unlock(config.PASSWORD)
+    if config.PASSWORD:
+        steem.wallet.unlock(config.PASSWORD)
 
     poster = Poster(steem=steem,
                     account=config.ACCOUNT,
-                    waiting_time=0.1)
+                    waiting_time=0.1,
+                    no_posting_key_mode=config.PASSWORD is None)
 
     posts = random_data.create_n_random_posts(10)
     df = pd.DataFrame(posts)
@@ -78,12 +81,36 @@ def test_test_top20_vote_and_comment(steem):
 
     df = tppp.preprocess(df, ncores=1)
 
-    account = config.ACCOUNT
-
     tbpd.vote_and_comment_on_topK(df, poster, 'laida',
                                   overview_permalink='lll')
 
 
 @pytest.mark.skipif(config.PASSWORD is None, reason="needs posting key")
 def test_create_wallet(steem):
-    tbpd.create_wallet(steem, config.PASSWORD, config.POSTING_KEY)
+    tbpd.create_wallet(steem, config.PASSWORD, config.POSTING_KEY)
+
+
+def test_test_top_trending_post(steem):
+
+    if config.PASSWORD:
+        steem.wallet.unlock(config.PASSWORD)
+
+    poster = Poster(steem=steem,
+                    account=config.ACCOUNT,
+                    waiting_time=0.1,
+                    no_posting_key_mode=config.PASSWORD is None)
+
+    posts = random_data.create_n_random_posts(10)
+    df = pd.DataFrame(posts)
+    df['reward'] = df.reward
+    df['predicted_votes'] = df.votes
+
+    df = tppp.preprocess(df, ncores=1)
+
+    date = pd.datetime.utcnow().date()
+
+    tbpd.post_top_trending_list(df, poster, date,
+                                overview_permalink='iii',
+                                trufflepicks_permalink='kkk',
+                                steem_amount=10,
+                                sbd_amount=10)
diff --git a/integration_tests/bchain/postweeklyupdate_test.py b/integration_tests/bchain/postweeklyupdate_test.py
@@ -51,7 +51,6 @@ def test_existence(steem):
     assert isinstance(result, str)
 
 
-@pytest.mark.skipif(config.PASSWORD is None, reason="needs posting key")
 def test_weekly_post(steem):
     posts = create_n_random_posts(300)
 
@@ -72,7 +71,8 @@ def test_weekly_post(steem):
     post_frame['sbd_bought_reward'] = 0
     post_frame['bought_votes'] = 0
 
-    poster = Poster(account=config.ACCOUNT, steem=steem)
+    poster = Poster(account=config.ACCOUNT, steem=steem,
+                    no_posting_key_mode=config.PASSWORD is None)
     permalink = tppw.post_weakly_update(pipeline, post_frame,
                                         poster=poster,
                                         current_datetime=current_date)

diff --git a/integration_tests/pigonduty_test.py b/integration_tests/pigonduty_test.py
@@ -8,28 +8,28 @@
 from trufflepig.bchain.poster import Poster
 
 
-@pytest.mark.skipif(config.PASSWORD is None, reason="needs posting key")
 def test_call_a_pig(steem):
     current_datetime = '2018-03-03-18:21:30'
 
     pipeline = MockPipeline()
     poster = Poster(steem=steem,
                     account=config.ACCOUNT,
-                    waiting_time=0.1)
+                    waiting_time=0.1,
+                    no_posting_key_mode=config.PASSWORD is None)
 
     tppd.call_a_pig(poster=poster,
                     pipeline=pipeline, topN_permalink='www.test.com',
                     current_datetime=current_datetime, hours=0.1,
                     overview_permalink='dsfd')
 
 
-@pytest.mark.skipif(config.PASSWORD is None, reason="needs posting key")
 def test_call_a_pig_empty_frame(steem):
     aacs = (('smcaterpillar','question-is-there-an-api-to-upload-images-to-steemit'),)
 
     poster = Poster(steem=steem,
                     account=config.ACCOUNT,
-                    waiting_time=0.51)
+                    waiting_time=0.51,
+                    no_posting_key_mode=config.PASSWORD is None)
 
     pipeline = MockPipeline()
     tppd.execute_call(comment_authors_and_permalinks=aacs,

diff --git a/integration_tests/preprocessing_test.py b/integration_tests/preprocessing_test.py
@@ -66,3 +66,29 @@ def test_bid_bot_correction_real_data(steem):
 
     assert upvotes
     assert (df.sbd_bought_reward.mean() > 0) or (df.steem_bought_reward.mean() > 0)
+
+
+def test_filtered_body_no_images_regression(steem):
+    """ Test for error in filtering as in these quotes: https://steemit.com/steemit/@trufflepig/daily-truffle-picks-2018-03-31"""
+    posts = tpgd.get_post_data([('colovhis', 'dofus-mastodon-cemetery-basic-tutorial'),
+                               ('joshuaetim','rewarding-hardwork-and-excellence-amongst-school-children-'
+                                             'through-steem-powered-notebooks-and-writing-materials-3rd-phase-of')],
+                               steem=steem)
+
+    post_frame = pd.DataFrame(posts)
+    post_frame = tppp.preprocess(post_frame, ncores=1)
+
+    assert not any(post_frame.filtered_body.apply(lambda x: '.JPG' in x))
+
+
+def test_filtered_body_classtextjustify_regression(steem):
+    """ Test for error in fitlering as in these quotes: https://steemit.com/steemit/@trufflepig/daily-truffle-picks-2018-03-31"""
+    posts = tpgd.get_post_data([('colovhis', 'dofus-mastodon-cemetery-basic-tutorial'),
+                               ('joshuaetim','rewarding-hardwork-and-excellence-amongst-school-children-'
+                                             'through-steem-powered-notebooks-and-writing-materials-3rd-phase-of')],
+                               steem=steem)
+
+    post_frame = pd.DataFrame(posts)
+    post_frame = tppp.preprocess(post_frame, ncores=1)
+
+    assert not any(post_frame.filtered_body.apply(lambda x: 'classtextjustify' in x))
diff --git a/integration_tests/trending0bot_test.py b/integration_tests/trending0bot_test.py
@@ -0,0 +1,40 @@
+import pytest
+
+import pandas as pd
+import trufflepig.bchain.getdata as tpgd
+import trufflepig.preprocessing as tppp
+import trufflepig.bchain.getaccountdata as tpad
+from trufflepig.bchain.poster import Poster
+import trufflepig.trending0bidbots as tt0b
+from trufflepig.testutils.pytest_fixtures import steem
+from trufflepig import config
+
+
+def test_create_trending_post(steem):
+
+    current_datetime = pd.datetime.utcnow()
+
+    data_frame = tpgd.scrape_hour_data(steem=steem,
+                                             current_datetime=current_datetime,
+                                             ncores=32,
+                                             offset_hours=8,
+                                       hours=1, stop_after=20)
+
+
+    min_datetime = data_frame.created.min()
+    max_datetime = data_frame.created.max() + pd.Timedelta(days=8)
+    upvote_payments = tpad.get_upvote_payments_to_bots(steem=steem,
+                                                  min_datetime=min_datetime,
+                                                  max_datetime=max_datetime,
+                                                 bots=['booster'])
+
+    data_frame = df = tppp.preprocess(data_frame, ncores=1)
+
+    data_frame = tppp.compute_bidbot_correction(post_frame=data_frame,
+                                                upvote_payments=upvote_payments)
+    account = config.ACCOUNT
+    poster = Poster(account=account, steem=steem,
+                    no_posting_key_mode=config.PASSWORD is None)
+
+    tt0b.create_trending_post(data_frame, upvote_payments, poster, 'test',
+                         'test', current_datetime)
diff --git a/tests/bchain/posts_test.py b/tests/bchain/posts_test.py
@@ -157,4 +157,26 @@ def test_weekly_update():
                   topics=topics)
 
     assert body
+    assert title
+
+
+def test_top_trending_post():
+    posts = random_data.create_n_random_posts(10)
+    df = pd.DataFrame(posts)
+    df = tppp.preprocess(df, ncores=1)
+
+    date = pd.datetime.utcnow().date()
+    df.image_urls = df.body.apply(lambda x: tptf.get_image_urls(x))
+
+    title, post = tbpo.top_trending_post(topN_authors=df.author,
+                                 topN_permalinks=df.permalink,
+                                 topN_titles=df.title,
+                                 topN_filtered_bodies=df.filtered_body,
+                                 topN_image_urls=df.image_urls,
+                                 topN_rewards=df.reward, sbd_amount=10,
+                                 steem_amount=10,
+                                 title_date=date, trufflepicks_link='de.de.de',
+                                 truffle_link='www.de')
+
+    assert post
     assert title
diff --git a/tests/filters/textfilters_test.py b/tests/filters/textfilters_test.py
@@ -32,7 +32,7 @@ def test_filter_special_characters():
 
 def test_filter_formatting():
     result = tptf.filter_formatting('Hi&nbsphey aligncenter nbsp Styletextalign kk')
-    assert result == 'Hi hey     kk'
+    assert result == 'Hihey    kk'
 
 
 def test_replace_newlines():

diff --git a/trufflepig/bchain/getaccountdata.py b/trufflepig/bchain/getaccountdata.py
@@ -90,6 +90,7 @@ def find_nearest_index(target_datetime,
 
     best_smallest_index = 1
     increase = index_tolerance + 1
+    current_datetime = None
     for _ in range(max_tries):
         try:
             action = next(acc.get_account_history(current_index, limit=1))
@@ -114,10 +115,13 @@ def find_nearest_index(target_datetime,
                                                                 latest_index))
         except Exception:
             logger.exception('Problems for index {}. Reconeccting...'.format(current_index))
-            current_index += 1
-            best_largest_index += 1
+            current_index -= 1
+            best_largest_index -= 1
             steem.reconnect()
             acc = Account(account, steem)
+            if current_index <= 1:
+                return 1, current_datetime
+
 
 
 def get_delegates_and_shares(account, steem):

diff --git a/trufflepig/bchain/postdata.py b/trufflepig/bchain/postdata.py
@@ -17,6 +17,8 @@
 
 PERMALINK_TEMPLATE = 'daily-truffle-picks-{date}'
 
+TRENDING_PERMALINK_TEMPLATE = 'non-bot-trending-{date}'
+
 
 def post_topN_list(sorted_post_frame, poster,
                    current_datetime, overview_permalink, N=10):
@@ -38,7 +40,7 @@ def post_topN_list(sorted_post_frame, poster,
     df = sorted_post_frame.iloc[:N, :]
 
     logger.info('Creating top {} post'.format(N))
-    df.first_image_url = df.body.apply(lambda x: tftf.get_image_urls(x))
+    first_image_urls = df.body.apply(lambda x: tftf.get_image_urls(x))
 
     steem_per_mvests = Converter(poster.steem).steem_per_mvests()
     truffle_link = 'https://steemit.com/steemit/@{}/{}'.format(poster.account,
@@ -48,7 +50,7 @@ def post_topN_list(sorted_post_frame, poster,
                                  topN_permalinks=df.permalink,
                                  topN_titles=df.title,
                                  topN_filtered_bodies=df.filtered_body,
-                                 topN_image_urls=df.first_image_url,
+                                 topN_image_urls=first_image_urls,
                                  topN_rewards=df.predicted_reward,
                                  topN_votes=df.predicted_votes,
                                  title_date=current_datetime,
@@ -57,8 +59,6 @@ def post_topN_list(sorted_post_frame, poster,
 
     permalink = PERMALINK_TEMPLATE.format(date=current_datetime.strftime('%Y-%m-%d'))
     logger.info('Posting top post with permalink: {}'.format(permalink))
-    logger.info(title)
-    logger.info(body)
     poster.post(body=body,
                 permalink=permalink,
                 title=title,
@@ -149,6 +149,46 @@ def vote_and_comment_on_topK(sorted_post_frame, poster,
             poster.steem.reconnect()
 
 
+def post_top_trending_list(sorted_post_frame, poster,
+                            current_datetime, trufflepicks_permalink,
+                           overview_permalink, sbd_amount,
+                           steem_amount, N=10):
+    """ Post the no bot trending toplist to the blockchain"""
+    df = sorted_post_frame.iloc[:N, :]
+
+    logger.info('Creating top {} post'.format(N))
+    first_image_urls = df.body.apply(lambda x: tftf.get_image_urls(x))
+
+    steem_per_mvests = Converter(poster.steem).steem_per_mvests()
+    truffle_link = 'https://steemit.com/steemit/@{}/{}'.format(poster.account,
+                                                               overview_permalink)
+    trufflepicks_link = 'https://steemit.com/steemit/@{}/{}'.format(poster.account,
+                                                               trufflepicks_permalink)
+
+    title, body = tfbp.top_trending_post(topN_authors=df.author,
+                                 topN_permalinks=df.permalink,
+                                 topN_titles=df.title,
+                                 topN_filtered_bodies=df.filtered_body,
+                                 topN_image_urls=first_image_urls,
+                                 topN_rewards=df.reward,
+                                 title_date=current_datetime,
+                                 truffle_link=truffle_link,
+                                 steem_per_mvests=steem_per_mvests,
+                                 trufflepicks_link=trufflepicks_link,
+                                 sbd_amount=sbd_amount,
+                                 steem_amount=steem_amount)
+
+    permalink = TRENDING_PERMALINK_TEMPLATE.format(date=current_datetime.strftime('%Y-%m-%d'))
+    logger.info('Posting top trending post with permalink: {}'.format(permalink))
+    poster.post(body=body,
+                permalink=permalink,
+                title=title,
+                tags=tfbp.TRENDING_TAGS,
+                self_vote=False)
+
+    return permalink
+
+
 def create_wallet(steem, password, posting_key,
                   active_key=None):
     """ Creates a new wallet
@@ -184,3 +224,6 @@ def create_wallet(steem, password, posting_key,
             logger.info('Key already present')
 
     logger.info('Wallet is ready')
+
+
+