updated block header

1. Updated block header, proofs now contain more data Notice, that old proofs may become invalid in the future 2. Fixed message routing 3. Fixed block creator id in block header 4. Support for full proofs in tonlib 5. Support for partial state download 6. Some other bugfixes
ton-blockchain · Sep 18, 2019 · 13140dd · 13140dd
1 parent bce33f5
commit 13140dd
Show file tree

Hide file tree

Showing 73 changed files with 2,083 additions and 303 deletions.
diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt
@@ -9,6 +9,7 @@ set(TON_CRYPTO_SOURCE
   common/bigint.cpp
   common/refcnt.cpp
   common/refint.cpp
+  common/bigexp.cpp
   common/bitstring.cpp
   common/util.cpp
   ellcurve/Ed25519.cpp
@@ -42,6 +43,7 @@ set(TON_CRYPTO_SOURCE
   common/bitstring.h
   common/refcnt.hpp
   common/refint.h
+  common/bigexp.h
   common/util.h
 
   ellcurve/Ed25519.h
@@ -227,7 +229,7 @@ target_link_libraries(ton_crypto PUBLIC ${OPENSSL_CRYPTO_LIBRARY} tdutils)
 if (NOT WIN32)
   target_link_libraries(ton_crypto PUBLIC dl z)
 endif()
-target_include_directories(ton_crypto SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR})
+target_include_directories(ton_crypto SYSTEM PUBLIC $<BUILD_INTERFACE:${OPENSSL_INCLUDE_DIR}>)
 
 add_library(ton_db STATIC ${TON_DB_SOURCE})
 target_include_directories(ton_db PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
@@ -324,10 +326,10 @@ if (WINGETOPT_FOUND)
   target_link_libraries_system(create-state wingetopt)
 endif()
 
-add_executable(test-block block/test-block.cpp)
-target_include_directories(test-block PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+add_executable(dump-block block/dump-block.cpp)
+target_include_directories(dump-block PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
   $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>)
-target_link_libraries(test-block PUBLIC ton_crypto fift-lib ton_block)
+target_link_libraries(dump-block PUBLIC ton_crypto fift-lib ton_block)
 if (WINGETOPT_FOUND)
-  target_link_libraries_system(test-block wingetopt)
+  target_link_libraries_system(dump-block wingetopt)
 endif()
diff --git a/crypto/block/block.cpp b/crypto/block/block.cpp
@@ -21,6 +21,7 @@
 #include "block/block-auto.h"
 #include "block/block-parse.h"
 #include "ton/ton-shard.h"
+#include "common/bigexp.h"
 #include "common/util.h"
 #include "td/utils/crypto.h"
 #include "td/utils/tl_storers.h"
@@ -367,14 +368,14 @@ std::unique_ptr<MsgProcessedUptoCollection> MsgProcessedUptoCollection::unpack(t
   return v && v->valid ? std::move(v) : std::unique_ptr<MsgProcessedUptoCollection>{};
 }
 
-bool MsgProcessedUpto::contains(const MsgProcessedUpto& other) const& {
+bool MsgProcessedUpto::contains(const MsgProcessedUpto& other) const & {
   return ton::shard_is_ancestor(shard, other.shard) && mc_seqno >= other.mc_seqno &&
          (last_inmsg_lt > other.last_inmsg_lt ||
           (last_inmsg_lt == other.last_inmsg_lt && !(last_inmsg_hash < other.last_inmsg_hash)));
 }
 
 bool MsgProcessedUpto::contains(ton::ShardId other_shard, ton::LogicalTime other_lt, td::ConstBitPtr other_hash,
-                                ton::BlockSeqno other_mc_seqno) const& {
+                                ton::BlockSeqno other_mc_seqno) const & {
   return ton::shard_is_ancestor(shard, other_shard) && mc_seqno >= other_mc_seqno &&
          (last_inmsg_lt > other_lt || (last_inmsg_lt == other_lt && !(last_inmsg_hash < other_hash)));
 }
@@ -788,6 +789,14 @@ td::Status ShardState::unpack_state(ton::BlockIdExt blkid, Ref<vm::Cell> prev_st
     if (!global_balance_.validate_unpack(extra.global_balance)) {
       return td::Status::Error(-666, "ShardState of "s + id_.to_str() + " does not contain a valid global_balance");
     }
+    if (extra.r1.flags & 1) {
+      if (extra.r1.block_create_stats->prefetch_ulong(8) != 0x17) {
+        return td::Status::Error(-666, "ShardState of "s + id_.to_str() + " does not contain a valid BlockCreateStats");
+      }
+      block_create_stats_ = std::make_unique<vm::Dictionary>(extra.r1.block_create_stats->prefetch_ref(), 256);
+    } else {
+      block_create_stats_ = std::make_unique<vm::Dictionary>(256);
+    }
   }
   return unpack_out_msg_queue_info(std::move(state.out_msg_queue_info));
 }
@@ -1387,6 +1396,114 @@ std::ostream& operator<<(std::ostream& os, const ValueFlow& vflow) {
   return os;
 }
 
+bool DiscountedCounter::increase_by(unsigned count, ton::UnixTime now) {
+  if (!validate()) {
+    return false;
+  }
+  td::uint64 scaled = (td::uint64(count) << 32);
+  if (!total) {
+    last_updated = now;
+    total = count;
+    cnt2048 = scaled;
+    cnt65536 = scaled;
+    return true;
+  }
+  if (count > ~total || cnt2048 > ~scaled || cnt65536 > ~scaled) {
+    return false /* invalidate() */;  // overflow
+  }
+  unsigned dt = (now >= last_updated ? now - last_updated : 0);
+  if (dt > 0) {
+    // more precise version of cnt2048 = llround(cnt2048 * exp(-dt / 2048.));
+    // (rounding error has absolute value < 1)
+    cnt2048 = (dt >= 48 * 2048 ? 0 : td::umulnexps32(cnt2048, dt << 5));
+    // more precise version of cnt65536 = llround(cnt65536 * exp(-dt / 65536.));
+    // (rounding error has absolute value < 1)
+    cnt65536 = td::umulnexps32(cnt65536, dt);
+  }
+  total += count;
+  cnt2048 += scaled;
+  cnt65536 += scaled;
+  last_updated = now;
+  return true;
+}
+
+bool DiscountedCounter::validate() {
+  if (!is_valid()) {
+    return false;
+  }
+  if (!total) {
+    if (cnt2048 | cnt65536) {
+      return invalidate();
+    }
+  } else if (!last_updated) {
+    return invalidate();
+  }
+  return true;
+}
+
+bool DiscountedCounter::fetch(vm::CellSlice& cs) {
+  valid = (cs.fetch_uint_to(32, last_updated) && cs.fetch_uint_to(64, total) && cs.fetch_uint_to(64, cnt2048) &&
+           cs.fetch_uint_to(64, cnt65536));
+  return validate() || invalidate();
+}
+
+bool DiscountedCounter::unpack(Ref<vm::CellSlice> csr) {
+  return (csr.not_null() && fetch(csr.write()) && csr->empty_ext()) || invalidate();
+}
+
+bool DiscountedCounter::store(vm::CellBuilder& cb) const {
+  return is_valid() && cb.store_long_bool(last_updated, 32) && cb.store_long_bool(total, 64) &&
+         cb.store_long_bool(cnt2048, 64) && cb.store_long_bool(cnt65536, 64);
+}
+
+Ref<vm::CellSlice> DiscountedCounter::pack() const {
+  vm::CellBuilder cb;
+  if (store(cb)) {
+    return vm::load_cell_slice_ref(cb.finalize());
+  } else {
+    return {};
+  }
+}
+
+bool DiscountedCounter::show(std::ostream& os) const {
+  if (!is_valid()) {
+    os << "<invalid-counter>";
+    return false;
+  }
+  os << "(counter last_updated:" << last_updated << " total:" << total << " cnt2048: " << (double)cnt2048 / (1LL << 32)
+     << " cnt65536: " << (double)cnt65536 / (1LL << 32) << ")";
+  return true;
+}
+
+std::string DiscountedCounter::to_str() const {
+  std::ostringstream stream;
+  if (show(stream)) {
+    return stream.str();
+  } else {
+    return "<invalid-counter>";
+  }
+}
+
+bool fetch_CreatorStats(vm::CellSlice& cs, DiscountedCounter& mc_cnt, DiscountedCounter& shard_cnt) {
+  return cs.fetch_ulong(4) == 4   // creator_info#4
+         && mc_cnt.fetch(cs)      // mc_blocks:Counters
+         && shard_cnt.fetch(cs);  // shard_blocks:Counters
+}
+
+bool store_CreatorStats(vm::CellBuilder& cb, const DiscountedCounter& mc_cnt, const DiscountedCounter& shard_cnt) {
+  return cb.store_long_bool(4, 4)  // creator_info#4
+         && mc_cnt.store(cb)       // mc_blocks:Counters
+         && shard_cnt.store(cb);   // shard_blocks:Counters
+}
+
+bool unpack_CreatorStats(Ref<vm::CellSlice> cs, DiscountedCounter& mc_cnt, DiscountedCounter& shard_cnt) {
+  if (cs.is_null()) {
+    return mc_cnt.set_zero() && shard_cnt.set_zero();
+  } else {
+    return fetch_CreatorStats(cs.write(), mc_cnt, shard_cnt) && cs->empty_ext();
+  }
+}
+
 /*
  * 
  *    Other block-related functions
@@ -1565,8 +1682,8 @@ std::pair<int, int> perform_hypercube_routing(ton::AccountIdPrefixFull src, ton:
   if (!ton::shard_contains(cur, transit)) {
     return {-1, -1};
   }
-  if (transit.account_id_prefix == dest.account_id_prefix || ton::shard_contains(cur, dest)) {
-    // if destination is already reached, or is in this shard, set cur:=next_hop:=dest
+  if (ton::shard_contains(cur, dest)) {
+    // if destination is in this shard, set cur:=next_hop:=dest
     return {96, 96};
   }
   if (transit.workchain == ton::masterchainId || dest.workchain == ton::masterchainId) {

diff --git a/crypto/block/block.h b/crypto/block/block.h
@@ -159,12 +159,12 @@ struct MsgProcessedUpto {
   MsgProcessedUpto(ton::ShardId _shard, ton::BlockSeqno _mcseqno, ton::LogicalTime _lt, td::ConstBitPtr _hash)
       : shard(_shard), mc_seqno(_mcseqno), last_inmsg_lt(_lt), last_inmsg_hash(_hash) {
   }
-  bool operator<(const MsgProcessedUpto& other) const& {
+  bool operator<(const MsgProcessedUpto& other) const & {
     return shard < other.shard || (shard == other.shard && mc_seqno < other.mc_seqno);
   }
-  bool contains(const MsgProcessedUpto& other) const&;
+  bool contains(const MsgProcessedUpto& other) const &;
   bool contains(ton::ShardId other_shard, ton::LogicalTime other_lt, td::ConstBitPtr other_hash,
-                ton::BlockSeqno other_mc_seqno) const&;
+                ton::BlockSeqno other_mc_seqno) const &;
   // NB: this is for checking whether we have already imported an internal message
   bool already_processed(const EnqueuedMsgDescr& msg) const;
 };
@@ -392,6 +392,7 @@ struct ShardState {
   CurrencyCollection total_balance_, total_validator_fees_, global_balance_;
   std::unique_ptr<vm::AugmentedDictionary> out_msg_queue_;
   std::unique_ptr<vm::Dictionary> ihr_pending_;
+  std::unique_ptr<vm::Dictionary> block_create_stats_;
   std::shared_ptr<block::MsgProcessedUptoCollection> processed_upto_;
 
   bool is_valid() const {
@@ -467,6 +468,67 @@ struct ValueFlow {
 
 std::ostream& operator<<(std::ostream& os, const ValueFlow& vflow);
 
+struct DiscountedCounter {
+  struct SetZero {};
+  bool valid;
+  ton::UnixTime last_updated;
+  td::uint64 total;
+  td::uint64 cnt2048;
+  td::uint64 cnt65536;
+  DiscountedCounter() : valid(false) {
+  }
+  DiscountedCounter(SetZero) : valid(true), last_updated(0), total(0), cnt2048(0), cnt65536(0) {
+  }
+  DiscountedCounter(ton::UnixTime _lastupd, td::uint64 _total, td::uint64 _cnt2048, td::uint64 _cnt65536)
+      : valid(true), last_updated(_lastupd), total(_total), cnt2048(_cnt2048), cnt65536(_cnt65536) {
+  }
+  static DiscountedCounter Zero() {
+    return SetZero();
+  }
+  bool is_valid() const {
+    return valid;
+  }
+  bool invalidate() {
+    return (valid = false);
+  }
+  bool set_zero() {
+    last_updated = 0;
+    total = cnt2048 = cnt65536 = 0;
+    return (valid = true);
+  }
+  bool is_zero() const {
+    return !total;
+  }
+  bool almost_zero() const {
+    return (cnt2048 | cnt65536) <= 1;
+  }
+  bool operator==(const DiscountedCounter& other) const {
+    return last_updated == other.last_updated && total == other.total && cnt2048 == other.cnt2048 &&
+           cnt65536 == other.cnt65536;
+  }
+  bool almost_equals(const DiscountedCounter& other) const {
+    return last_updated == other.last_updated && total == other.total && cnt2048 <= other.cnt2048 + 1 &&
+           other.cnt2048 <= cnt2048 + 1 && cnt65536 <= other.cnt65536 + 1 && other.cnt65536 <= cnt65536 + 1;
+  }
+  bool validate();
+  bool increase_by(unsigned count, ton::UnixTime now);
+  bool fetch(vm::CellSlice& cs);
+  bool unpack(Ref<vm::CellSlice> csr);
+  bool store(vm::CellBuilder& cb) const;
+  Ref<vm::CellSlice> pack() const;
+  bool show(std::ostream& os) const;
+  std::string to_str() const;
+};
+
+static inline std::ostream& operator<<(std::ostream& os, const DiscountedCounter& dcount) {
+  dcount.show(os);
+  return os;
+}
+
+bool fetch_CreatorStats(vm::CellSlice& cs, DiscountedCounter& mc_cnt, DiscountedCounter& shard_cnt);
+bool store_CreatorStats(vm::CellBuilder& cb, const DiscountedCounter& mc_cnt, const DiscountedCounter& shard_cnt);
+bool unpack_CreatorStats(Ref<vm::CellSlice> cs, DiscountedCounter& mc_cnt, DiscountedCounter& shard_cnt);
+
 struct BlockProofLink {
   ton::BlockIdExt from, to;
   bool is_key{false}, is_fwd{false};

diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb
@@ -499,14 +499,20 @@ _ key:Bool blk_ref:ExtBlkRef = KeyExtBlkRef;
 
 _ (HashmapAugE 32 KeyExtBlkRef KeyMaxLt) = OldMcBlocksInfo;
 
+
+counters#_ last_updated:uint32 total:uint64 cnt2048:uint64 cnt65536:uint64 = Counters; 
+creator_info#4 mc_blocks:Counters shard_blocks:Counters = CreatorStats; 
+block_create_stats#17 counters:(HashmapE 256 CreatorStats) = BlockCreateStats;
+
 masterchain_state_extra#cc26
   shard_hashes:ShardHashes
   config:ConfigParams
-  ^[ flags:(## 16) { flags = 0 }
+  ^[ flags:(## 16) { flags <= 1 }
      validator_info:ValidatorInfo
      prev_blocks:OldMcBlocksInfo
      after_key_block:Bool
-     last_key_block:(Maybe ExtBlkRef) ]
+     last_key_block:(Maybe ExtBlkRef)
+     block_create_stats:(flags . 0)?BlockCreateStats ]
   global_balance:CurrencyCollection
 = McStateExtra;
 

diff --git a/crypto/block/check-proof.h b/crypto/block/check-proof.h
@@ -80,4 +80,15 @@ struct TransactionList {
   td::Result<Info> validate() const;
 };
 
+struct BlockChain {
+  ton::BlockIdExt from;
+  ton::BlockIdExt to;
+  td::int32 mode;
+  td::BufferSlice proof;
+
+  using Info = std::unique_ptr<block::BlockProofChain>;
+
+  td::Result<Info> validate() const;
+};
+
 }  // namespace block