feat(tiamnu): hard code in defs.h (#1481)

storage/tianmu/common/defs.h

@@ -18,6 +18,7 @@
 #define TIANMU_COMMON_DEFS_H_
 #pragma once
 
+#include <stddef.h>
 #include <cstdint>
 
 namespace Tianmu {
@@ -42,19 +43,17 @@ constexpr const char *COL_FILTER_HIST_DIR = "hist";
 constexpr const char *COL_KN_FILE = "KN";
 constexpr const char *COL_META_FILE = "META";
 constexpr const char *COL_DN_FILE = "DN";
-/*
-   The size of the file where the DPN metadata resides, in bytes
-   At present, the size of a single DPN is 88 bytes, and the storage limit is 8.589 billion lines
-*/
-constexpr size_t COL_DN_FILE_SIZE = 11 * 1024 * 1024;
+
+constexpr uint8_t MAX_PSS = 16;  // max pack size shift
+constexpr uint8_t DFT_PSS = 16;
+constexpr size_t MAX_ROW_NUMS_EACH_PACK = 1 << common::MAX_PSS;  // max 64K rows per pack
+
+constexpr size_t MAX_CMPR_SIZE = 0x007D000000;
 constexpr const char *COL_DATA_FILE = "DATA";
 constexpr const char *COL_VERSION_DIR = "v";
 constexpr uint32_t COL_FILE_VERSION = 3;
 constexpr uint32_t MAX_COLUMNS_PER_TABLE = 4000;
-
-constexpr uint8_t MAX_PSS = 16;
-constexpr uint8_t DFT_PSS = 16;
-constexpr size_t MAX_CMPR_SIZE = 0x007D000000;
+constexpr uint32_t MAX_ROW_NUM_SHIFT = 33;  // max 8G (1 << 33) rows per storage limitation
 
 using PACK_INDEX = uint32_t;
 constexpr PACK_INDEX INVALID_PACK_INDEX = -1;

storage/tianmu/core/table_share.h

@@ -35,7 +35,7 @@ struct TABLE_META {
   uint64_t magic = common::FILE_MAGIC;
   uint32_t ver = common::TABLE_DATA_VERSION;
   uint32_t id;
-  uint32_t pss = 16;
+  uint32_t pss = common::MAX_PSS;
 };
 
 class TableShare final {

storage/tianmu/vc/column_share.cpp

@@ -54,7 +54,7 @@ static constexpr size_t DPN_INC_CNT = ALLOC_UNIT / sizeof(DPN);
 
 ColumnShare::~ColumnShare() {
   if (start != nullptr) {
-    if (::munmap(start, common::COL_DN_FILE_SIZE) != 0) {
+    if (::munmap(start, COL_DN_FILE_SIZE) != 0) {
       // DO NOT throw in dtor!
       TIANMU_LOG(LogCtl_Level::WARN, "Failed to unmap DPN file. Error %d(%s)", errno, std::strerror(errno));
     }
@@ -85,12 +85,12 @@ void ColumnShare::map_dpn() {
   ASSERT(sb.st_size % sizeof(DPN) == 0);
   capacity = sb.st_size / sizeof(DPN);
 
-  auto addr = ::mmap(0, common::COL_DN_FILE_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, dn_fd, 0);
+  auto addr = ::mmap(0, COL_DN_FILE_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, dn_fd, 0);
   if (addr == MAP_FAILED) {
     throw std::system_error(errno, std::system_category(), "mmap() " + dpn_file.string());
   }
 
-  // TODO: should we mlock(addr, common::COL_DN_FILE_SIZE)?
+  // TODO: should we mlock(addr, COL_DN_FILE_SIZE)?
 
   start = static_cast<DPN *>(addr);
 }
@@ -238,8 +238,7 @@ int ColumnShare::alloc_dpn(common::TX_ID xid, const DPN *from) {
     return i;
   }
 
-  ASSERT((capacity + DPN_INC_CNT) <= (common::COL_DN_FILE_SIZE / sizeof(DPN)),
-         "Failed to allocate new DN: " + m_path.string());
+  ASSERT((capacity + DPN_INC_CNT) <= (COL_DN_FILE_SIZE / sizeof(DPN)), "Failed to allocate new DN: " + m_path.string());
   capacity += DPN_INC_CNT;
 
   //  NOTICE:
@@ -270,7 +269,7 @@ void ColumnShare::alloc_seg(DPN *dpn) {
 }
 
 void ColumnShare::sync_dpns() {
-  int ret = ::msync(start, common::COL_DN_FILE_SIZE, MS_SYNC);
+  int ret = ::msync(start, COL_DN_FILE_SIZE, MS_SYNC);
   if (ret != 0)
     throw std::system_error(errno, std::system_category(), "msync() " + m_path.string());
 }

storage/tianmu/vc/column_share.h

@@ -22,6 +22,7 @@
 
 #include "common/assert.h"
 #include "common/common_definitions.h"
+#include "common/defs.h"
 #include "common/mysql_gate.h"
 #include "data/dpn.h"
 #include "util/fs.h"
@@ -30,6 +31,16 @@
 namespace Tianmu {
 namespace core {
 class TableShare;
+/*
+   Number of DPN to reach the max row limiration.
+   It is calcualted by (max number of row limiration) / (max number of row per pack )
+*/
+constexpr size_t MAX_DPN_NUMS_EACH_COLUMN =
+    1 << (common::MAX_ROW_NUM_SHIFT - common::MAX_PSS);  // max 128K packs per each column
+/*
+   The size of the file where the DPN metadata resides, in bytes
+*/
+constexpr size_t COL_DN_FILE_SIZE = MAX_DPN_NUMS_EACH_COLUMN * sizeof(core::DPN);
 
 struct COL_META {
   uint32_t magic;
@@ -77,11 +88,11 @@ class ColumnShare final {
   }
 
   DPN *get_dpn_ptr(common::PACK_INDEX i) {
-    ASSERT(i < common::COL_DN_FILE_SIZE / sizeof(DPN), "bad dpn index: " + std::to_string(i));
+    ASSERT(i < COL_DN_FILE_SIZE / sizeof(DPN), "bad dpn index: " + std::to_string(i));
     return &start[i];
   }
   const DPN *get_dpn_ptr(common::PACK_INDEX i) const {
-    ASSERT(i < common::COL_DN_FILE_SIZE / sizeof(DPN), "bad dpn index: " + std::to_string(i));
+    ASSERT(i < COL_DN_FILE_SIZE / sizeof(DPN), "bad dpn index: " + std::to_string(i));
     return &start[i];
   }
 

storage/tianmu/vc/tianmu_attr.cpp

@@ -151,7 +151,7 @@ void TianmuAttr::Create(const fs::path &dir, const AttributeTypeInfo &ati, uint8
     }
     fdn.WriteExact(&dpn, sizeof(dpn));
     fdn.Flush();
-    fs::resize_file(dir / common::COL_DN_FILE, common::COL_DN_FILE_SIZE);
+    fs::resize_file(dir / common::COL_DN_FILE, core::COL_DN_FILE_SIZE);
   }
 
   // create filter directories