Merge pull request #59 from brave-intl/load_publishers_list

Refreshing publishers list periodically

include/bat/ledger/ledger.h

@@ -84,6 +84,9 @@ class LEDGER_EXPORT Ledger {
       const std::string& url,
       const std::map<std::string, std::string>& parts,
       const uint64_t& current_time) = 0;
+
+  virtual void OnTimer(uint32_t timer_id) = 0;
+
   virtual std::string URIEncode(const std::string& value) = 0;
 
   virtual void SetPublisherInfo(std::unique_ptr<PublisherInfo> publisher_info,

include/bat/ledger/ledger_callback_handler.h

@@ -38,6 +38,8 @@ class LEDGER_EXPORT LedgerCallbackHandler {
   virtual void OnURLRequestResponse(uint64_t request_id,
                                     int response_code,
                                     const std::string& response) {};
+
+  virtual void OnPublishersListSaved(Result result) {};
 };
 
 }  // namespace ledger

include/bat/ledger/ledger_client.h

@@ -52,6 +52,11 @@ class LEDGER_EXPORT LedgerClient {
   virtual void SavePublisherState(const std::string& publisher_state,
                                   LedgerCallbackHandler* handler) = 0;
 
+  virtual void SavePublishersList(const std::string& publisher_state,
+    LedgerCallbackHandler* handler) = 0;
+
+
+
   virtual void SavePublisherInfo(std::unique_ptr<PublisherInfo> publisher_info,
                                 PublisherInfoCallback callback) = 0;
   virtual void LoadPublisherInfo(const PublisherInfo::id_type& publisher_id,
@@ -68,6 +73,10 @@ class LEDGER_EXPORT LedgerClient {
   virtual void OnRecoverWallet(Result result, double balance, const std::vector<ledger::Grant>& grants) = 0;
   virtual void OnGrantFinish(ledger::Result result, const ledger::Grant& grant) = 0;
 
+  //uint64_t time_offset (input): timer offset in seconds.
+  //uint32_t timer_id (output) : 0 in case of failure
+  virtual void SetTimer(uint64_t time_offset, uint32_t & timer_id) = 0;
+
   virtual std::string URIEncode(const std::string& value) = 0;
 
   virtual std::unique_ptr<ledger::LedgerURLLoader> LoadURL(

include/bat/ledger/publisher_info.h

@@ -81,7 +81,7 @@ LEDGER_EXPORT struct PublisherInfo {
   std::string year;
 };
 
-using PublisherInfoList = std::vector<const PublisherInfo>;
+using PublisherInfoList = std::vector<PublisherInfo>;
 
 }  // namespace ledger
 

src/bat_helper.cc

@@ -782,6 +782,7 @@ namespace braveledger_bat_helper {
       error = !(d.HasMember("min_pubslisher_duration") && d["min_pubslisher_duration"].IsUint() &&
         d.HasMember("min_visits") && d["min_visits"].IsUint() &&
         d.HasMember("allow_non_verified") && d["allow_non_verified"].IsBool() &&
+        d.HasMember("pubs_load_timestamp") && d["pubs_load_timestamp"].IsUint64() &&
         d.HasMember("allow_videos") && d["allow_videos"].IsBool() &&
         d.HasMember("monthly_balances") && d["monthly_balances"].IsArray() &&
         d.HasMember("recurring_donation") && d["recurring_donation"].IsArray());
@@ -791,7 +792,9 @@ namespace braveledger_bat_helper {
       min_pubslisher_duration_ = d["min_pubslisher_duration"].GetUint();
       min_visits_ = d["min_visits"].GetUint();
       allow_non_verified_ = d["allow_non_verified"].GetBool();
+      pubs_load_timestamp_ = d["pubs_load_timestamp"].GetUint64();
       allow_videos_ = d["allow_videos"].GetBool();
+
       for (const auto & i : d["monthly_balances"].GetArray()) {
         rapidjson::StringBuffer sb;
         rapidjson::Writer<rapidjson::StringBuffer> writer(sb);
@@ -839,6 +842,9 @@ namespace braveledger_bat_helper {
 
     writer.String("allow_non_verified");
     writer.Bool(data.allow_non_verified_);
+
+    writer.String("pubs_load_timestamp");
+    writer.Uint64(data.pubs_load_timestamp_);
 
     writer.String("allow_videos");
     writer.Bool(data.allow_videos_);

src/bat_helper.h

@@ -235,6 +235,7 @@ namespace braveledger_bat_helper {
     uint64_t min_pubslisher_duration_ = braveledger_ledger::_default_min_pubslisher_duration;  // In seconds
     unsigned int min_visits_ = 1u;
     bool allow_non_verified_ = true;
+    uint64_t pubs_load_timestamp_ = 0ull; //last publishers list load timestamp (seconds)
     bool allow_videos_ = true;
     std::map<std::string, REPORT_BALANCE_ST> monthly_balances_;
     std::map<std::string, double> recurring_donation_;

src/bat_publishers.cc

@@ -4,12 +4,12 @@
 
 #include "bat_publishers.h"
 
+#include <ctime>
 #include <cmath>
 #include <algorithm>
 
 #include "bat_helper.h"
 #include "ledger_impl.h"
-#include "leveldb/db.h"
 #include "rapidjson_bat_helper.h"
 #include "static_values.h"
 
@@ -214,6 +214,11 @@ void BatPublishers::setPublisherMinVisits(const unsigned int& visits) {
   saveState();
 }
 
+void BatPublishers::setPublishersLastRefreshTimestamp(uint64_t ts) {
+  state_->pubs_load_timestamp_ = ts;
+  saveState();
+}
+
 void BatPublishers::setPublisherAllowNonVerified(const bool& allow) {
   state_->allow_non_verified_ = allow;
   saveState();
@@ -235,6 +240,9 @@ unsigned int BatPublishers::getPublisherMinVisits() const {
 bool BatPublishers::getPublisherAllowNonVerified() const {
   return state_->allow_non_verified_;
 }
+uint64_t BatPublishers::getLastPublishersListLoadTimestamp() const {
+  return state_->pubs_load_timestamp_;
+}
 
 bool BatPublishers::getPublisherAllowVideos() const {
   return state_->allow_videos_;
@@ -426,4 +434,13 @@ std::vector<ledger::ContributionInfo> BatPublishers::GetRecurringDonationList()
   return res;
 }
 
+void BatPublishers::RefreshPublishersList(const std::string & pubs_list) {
+  ledger_->SavePublishersList(pubs_list);
+}
+
+void BatPublishers::OnPublishersListSaved(ledger::Result result) {
+  uint64_t ts = (ledger::Result::OK == result) ? std::time(nullptr) : 0ull;
+  setPublishersLastRefreshTimestamp(ts);
+}
+
 }  // namespace braveledger_bat_publisher

src/bat_publishers.h

@@ -44,6 +44,8 @@ class BatPublishers : public ledger::LedgerCallbackHandler {
 
   void setPublisherMinVisits(const unsigned int& visits);
 
+  void setPublishersLastRefreshTimestamp(uint64_t ts);
+
   void setPublisherAllowNonVerified(const bool& allow);
   void setPublisherAllowVideos(const bool& allow);
   void setBalanceReport(const std::string& year, ledger::PUBLISHER_MONTH month,
@@ -54,6 +56,7 @@ class BatPublishers : public ledger::LedgerCallbackHandler {
   uint64_t getPublisherMinVisitTime() const; // In milliseconds
   unsigned int getPublisherMinVisits() const;
   bool getPublisherAllowNonVerified() const;
+  uint64_t getLastPublishersListLoadTimestamp() const;
   bool getPublisherAllowVideos() const;
 
   std::vector<braveledger_bat_helper::WINNERS_ST> winners(const unsigned int& ballots);
@@ -67,6 +70,10 @@ class BatPublishers : public ledger::LedgerCallbackHandler {
     ledger::PUBLISHER_MONTH month);
   std::vector<ledger::ContributionInfo> GetRecurringDonationList();
 
+  void RefreshPublishersList(const std::string & pubs_list);
+
+  void OnPublishersListSaved(ledger::Result result) override;
+
  private:
   // LedgerCallbackHandler impl
   void OnPublisherStateSaved(ledger::Result result) override;

src/ledger_impl.cc

@@ -2,6 +2,9 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+#include <ctime>
+#include <random>
+
 #include "ledger_impl.h"
 
 #include "bat_client.h"
@@ -26,7 +29,8 @@ LedgerImpl::LedgerImpl(ledger::LedgerClient* client) :
     bat_get_media_(new BatGetMedia(this)),
     initialized_(false),
     last_tab_active_time_(0),
-    last_shown_tab_id_(-1) {
+    last_shown_tab_id_(-1),
+    last_pub_load_timer_id_ (0u){
 }
 
 LedgerImpl::~LedgerImpl() {
@@ -159,6 +163,7 @@ void LedgerImpl::OnPublisherStateLoaded(ledger::Result result,
   }
 
   OnWalletInitialized(result);
+  RefreshPublishersList(false);
 }
 
 void LedgerImpl::SaveLedgerState(const std::string& data) {
@@ -170,6 +175,11 @@ void LedgerImpl::SavePublisherState(const std::string& data,
   ledger_client_->SavePublisherState(data, handler);
 }
 
+
+void LedgerImpl::SavePublishersList(const std::string& data) {
+  ledger_client_->SavePublishersList(data, this);
+}
+
 std::string LedgerImpl::GenerateGUID() const {
   return ledger_client_->GenerateGUID();
 }
@@ -518,4 +528,68 @@ void LedgerImpl::SetBalanceReport(const std::string& year,
   bat_publishers_->setBalanceReport(year, month, report_info);
 }
 
+void LedgerImpl::OnTimer(uint32_t timer_id) {
+  if (timer_id == last_pub_load_timer_id_) {
+    last_pub_load_timer_id_ = 0;
+
+    //download the list
+    std::string url(PUBLISHERSTAGING_SERVER);
+    url += GET_PUBLISHERS_LIST_V1;
+    auto url_loader = LoadURL(url, std::vector<std::string>(), "", "", ledger::URL_METHOD::GET, &handler_);
+    handler_.AddRequestHandler(std::move(url_loader),
+      std::bind(&LedgerImpl::LoadPublishersListCallback,this,_1,_2));
+  }
+}
+
+
+
+void LedgerImpl::LoadPublishersListCallback(bool result, const std::string& response) {
+  bool retryAfterError = true;
+  if (result && !response.empty()) {
+    retryAfterError = false;
+    bat_publishers_->RefreshPublishersList(response);
+  }
+
+  //set timer
+  RefreshPublishersList(retryAfterError);
+}
+
+void LedgerImpl::RefreshPublishersList(bool retryAfterError) {
+  uint64_t start_timer_in{ 0ull };
+
+  if (last_pub_load_timer_id_ != 0) {
+    //timer in progress
+    return;
+  }
+
+  if (retryAfterError) {
+    std::random_device seeder;
+    const auto seed = seeder.entropy() ? seeder() : time(nullptr);
+    std::mt19937 eng(static_cast<std::mt19937::result_type> (seed));
+    std::uniform_int_distribution <> dist(300, 3600); //retry loading publishers list in 300-3600 seconds if failed
+    start_timer_in = dist(eng);
+  }
+  else {
+    uint64_t now = std::time(nullptr);
+    uint64_t lastLoadTimestamp = bat_publishers_->getLastPublishersListLoadTimestamp();
+
+    //check if lastLoadTimestamp doesn't exist or have erroneous value
+    // (start_timer_in == 0) is expected to call callback function immediately
+    start_timer_in = (lastLoadTimestamp == 0ull || lastLoadTimestamp > now) ? 0ull : now - lastLoadTimestamp;
+  }
+
+  //start timer
+  ledger_client_->SetTimer(start_timer_in, last_pub_load_timer_id_);
+}
+
+void LedgerImpl::OnPublishersListSaved(ledger::Result result) {
+
+  if (ledger::Result::OK == result) {
+    bat_publishers_->OnPublishersListSaved(result);
+  }
+  else {
+    RefreshPublishersList(true);
+  }
+}
+
 }  // namespace bat_ledger

src/ledger_impl.h

@@ -84,6 +84,8 @@ class LedgerImpl : public ledger::Ledger,
   void SaveLedgerState(const std::string& data);
   void SavePublisherState(const std::string& data,
                           ledger::LedgerCallbackHandler* handler);
+  void SavePublishersList(const std::string& data);
+
   void LoadLedgerState(ledger::LedgerCallbackHandler* handler);
   void LoadPublisherState(ledger::LedgerCallbackHandler* handler);
 
@@ -106,6 +108,10 @@ class LedgerImpl : public ledger::Ledger,
   void RecoverWallet(const std::string& passPhrase) const override;
   void OnRecoverWallet(ledger::Result result, double balance, const std::vector<braveledger_bat_helper::GRANT>& grants);
 
+  void LoadPublishersListCallback(bool result, const std::string& response);
+
+  void OnPublishersListSaved(ledger::Result result) override;
+
   std::unique_ptr<ledger::LedgerURLLoader> LoadURL(const std::string& url,
       const std::vector<std::string>& headers,
       const std::string& content,
@@ -135,6 +141,8 @@ class LedgerImpl : public ledger::Ledger,
       const std::map<std::string, std::string>& parts,
       const uint64_t& current_time) override;
 
+  void OnTimer(uint32_t timer_id) override;
+
   void OnSetPublisherInfo(ledger::PublisherInfoCallback callback,
                           ledger::Result result,
                           std::unique_ptr<ledger::PublisherInfo> info);
@@ -157,6 +165,8 @@ class LedgerImpl : public ledger::Ledger,
   void OnLedgerStateLoaded(ledger::Result result,
                            const std::string& data) override;
 
+  void RefreshPublishersList(bool retryAfterError);
+
 
   ledger::LedgerClient* ledger_client_;
   std::unique_ptr<braveledger_bat_client::BatClient> bat_client_;
@@ -170,6 +180,7 @@ class LedgerImpl : public ledger::Ledger,
   std::map<uint32_t, ledger::VisitData> current_pages_;
   uint64_t last_tab_active_time_;
   uint32_t last_shown_tab_id_;
+  uint32_t last_pub_load_timer_id_;
  };
 }  // namespace bat_ledger
 

src/static_values.h

@@ -11,6 +11,9 @@
 #define LEDGER_PRODUCTION_SERVER        "https://ledger.mercury.basicattentiontoken.org"
 #define BALANCE_STAGING_SERVER          "https://balance-staging.mercury.basicattentiontoken.org"
 #define BALANCE_PRODUCTION_SERVER       "https://balance.mercury.basicattentiontoken.org"
+#define PUBLISHERSTAGING_SERVER         "https://publishers-staging.basicattentiontoken.org"
+
+
 
 #define PREFIX_V2                       "/v2"
 #define PREFIX_V3                       "/v3"
@@ -27,6 +30,7 @@
 #define RECOVER_WALLET                  "/v2/wallet/"
 #define GET_SET_PROMOTION                "/v1/grants"
 #define GET_PROMOTION_CAPTCHA            "/v1/captchas/"
+#define GET_PUBLISHERS_LIST_V1           "/api/v1/public/channels"
 
 #define REGISTRARVK_FIELDNAME           "registrarVK"
 #define VERIFICATION_FIELDNAME          "verification"
@@ -79,6 +83,8 @@ static const unsigned int _twitch_events_array_size = 8;
 static const std::string _twitch_events[] = {"buffer-empty", "buffer-refill", "video_end",
   "minute-watched", "video_pause", "player_click_vod_seek", "video-play", "video_error"};
 
+static const uint64_t _publishers_list_load_interval = 48 * 60 * 60; //48 hours in seconds
+
 }  // namespace braveledger_ledger
 
 #endif  // BRAVELEDGER_STATIC_VALUES_H_