Skip to content

Commit

Permalink
Merge branch 'unstable' into jsonnumop
Browse files Browse the repository at this point in the history
  • Loading branch information
@git-hulk
git-hulk committed Nov 26, 2023
2 parents db6bdb8 + 5a4ad7f commit 356bcb5
Show file tree
Hide file tree
Showing 5 changed files with 55 additions and 27 deletions.
13 changes: 13 additions & 0 deletions src/storage/storage.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -531,6 +531,19 @@ rocksdb::Status Storage::Get(const rocksdb::ReadOptions &options, rocksdb::Colum
return db_->Get(options, column_family, key, value);
}

rocksdb::Status Storage::Get(const rocksdb::ReadOptions &options, const rocksdb::Slice &key,
rocksdb::PinnableSlice *value) {
return Get(options, db_->DefaultColumnFamily(), key, value);
}

rocksdb::Status Storage::Get(const rocksdb::ReadOptions &options, rocksdb::ColumnFamilyHandle *column_family,
const rocksdb::Slice &key, rocksdb::PinnableSlice *value) {
if (is_txn_mode_ && txn_write_batch_->GetWriteBatch()->Count() > 0) {
return txn_write_batch_->GetFromBatchAndDB(db_.get(), options, column_family, key, value);
}
return db_->Get(options, column_family, key, value);
}

rocksdb::Iterator *Storage::NewIterator(const rocksdb::ReadOptions &options) {
return NewIterator(options, db_->DefaultColumnFamily());
}
Expand Down
4 changes: 4 additions & 0 deletions src/storage/storage.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,10 @@ class Storage {
[[nodiscard]] rocksdb::Status Get(const rocksdb::ReadOptions &options, const rocksdb::Slice &key, std::string *value);
[[nodiscard]] rocksdb::Status Get(const rocksdb::ReadOptions &options, rocksdb::ColumnFamilyHandle *column_family,
const rocksdb::Slice &key, std::string *value);
[[nodiscard]] rocksdb::Status Get(const rocksdb::ReadOptions &options, const rocksdb::Slice &key,
rocksdb::PinnableSlice *value);
[[nodiscard]] rocksdb::Status Get(const rocksdb::ReadOptions &options, rocksdb::ColumnFamilyHandle *column_family,
const rocksdb::Slice &key, rocksdb::PinnableSlice *value);
void MultiGet(const rocksdb::ReadOptions &options, rocksdb::ColumnFamilyHandle *column_family, size_t num_keys,
const rocksdb::Slice *keys, rocksdb::PinnableSlice *values, rocksdb::Status *statuses);
rocksdb::Iterator *NewIterator(const rocksdb::ReadOptions &options, rocksdb::ColumnFamilyHandle *column_family);
Expand Down
23 changes: 12 additions & 11 deletions src/types/redis_bitmap.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -246,19 +246,19 @@ rocksdb::Status Bitmap::BitCount(const Slice &user_key, int64_t start, int64_t s
uint32_t start_index = u_start / kBitmapSegmentBytes;
uint32_t stop_index = u_stop / kBitmapSegmentBytes;
// Don't use multi get to prevent large range query, and take too much memory
std::string value;
for (uint32_t i = start_index; i <= stop_index; i++) {
rocksdb::PinnableSlice pin_value;
std::string sub_key =
InternalKey(ns_key, std::to_string(i * kBitmapSegmentBytes), metadata.version, storage_->IsSlotIdEncoded())
.Encode();
s = storage_->Get(read_options, sub_key, &value);
s = storage_->Get(read_options, sub_key, &pin_value);
if (!s.ok() && !s.IsNotFound()) return s;
if (s.IsNotFound()) continue;
size_t j = 0;
if (i == start_index) j = u_start % kBitmapSegmentBytes;
auto k = static_cast<int64_t>(value.size());
auto k = static_cast<int64_t>(pin_value.size());
if (i == stop_index) k = u_stop % kBitmapSegmentBytes + 1;
*cnt += BitmapString::RawPopcount(reinterpret_cast<const uint8_t *>(value.data()) + j, k);
*cnt += BitmapString::RawPopcount(reinterpret_cast<const uint8_t *>(pin_value.data()) + j, k);
}
return rocksdb::Status::OK();
}
Expand Down Expand Up @@ -304,12 +304,12 @@ rocksdb::Status Bitmap::BitPos(const Slice &user_key, bool bit, int64_t start, i
uint32_t start_index = u_start / kBitmapSegmentBytes;
uint32_t stop_index = u_stop / kBitmapSegmentBytes;
// Don't use multi get to prevent large range query, and take too much memory
std::string value;
rocksdb::PinnableSlice pin_value;
for (uint32_t i = start_index; i <= stop_index; i++) {
std::string sub_key =
InternalKey(ns_key, std::to_string(i * kBitmapSegmentBytes), metadata.version, storage_->IsSlotIdEncoded())
.Encode();
s = storage_->Get(read_options, sub_key, &value);
s = storage_->Get(read_options, sub_key, &pin_value);
if (!s.ok() && !s.IsNotFound()) return s;
if (s.IsNotFound()) {
if (!bit) {
Expand All @@ -320,17 +320,18 @@ rocksdb::Status Bitmap::BitPos(const Slice &user_key, bool bit, int64_t start, i
}
size_t j = 0;
if (i == start_index) j = u_start % kBitmapSegmentBytes;
for (; j < value.size(); j++) {
for (; j < pin_value.size(); j++) {
if (i == stop_index && j > (u_stop % kBitmapSegmentBytes)) break;
if (bit_pos_in_byte(value[j], bit) != -1) {
*pos = static_cast<int64_t>(i * kBitmapSegmentBits + j * 8 + bit_pos_in_byte(value[j], bit));
if (bit_pos_in_byte(pin_value[j], bit) != -1) {
*pos = static_cast<int64_t>(i * kBitmapSegmentBits + j * 8 + bit_pos_in_byte(pin_value[j], bit));
return rocksdb::Status::OK();
}
}
if (!bit && value.size() < kBitmapSegmentBytes) {
*pos = static_cast<int64_t>(i * kBitmapSegmentBits + value.size() * 8);
if (!bit && pin_value.size() < kBitmapSegmentBytes) {
*pos = static_cast<int64_t>(i * kBitmapSegmentBits + pin_value.size() * 8);
return rocksdb::Status::OK();
}
pin_value.Reset();
}
// bit was not found
*pos = bit ? -1 : static_cast<int64_t>(metadata.size * 8);
Expand Down
37 changes: 23 additions & 14 deletions src/types/redis_bloom_chain.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,17 +45,17 @@ void BloomChain::getBFKeyList(const Slice &ns_key, const BloomChainMetadata &met
}

rocksdb::Status BloomChain::getBFDataList(const std::vector<std::string> &bf_key_list,
std::vector<std::string> *bf_data_list) {
std::vector<rocksdb::PinnableSlice> *bf_data_list) {
LatestSnapShot ss(storage_);
rocksdb::ReadOptions read_options;
read_options.snapshot = ss.GetSnapShot();

bf_data_list->reserve(bf_key_list.size());
for (const auto &bf_key : bf_key_list) {
std::string bf_data;
rocksdb::Status s = storage_->Get(read_options, bf_key, &bf_data);
rocksdb::PinnableSlice pin_value;
rocksdb::Status s = storage_->Get(read_options, bf_key, &pin_value);
if (!s.ok()) return s;
bf_data_list->push_back(std::move(bf_data));
bf_data_list->push_back(std::move(pin_value));
}
return rocksdb::Status::OK();
}
Expand Down Expand Up @@ -113,8 +113,9 @@ void BloomChain::bloomAdd(uint64_t item_hash, std::string &bf_data) {
block_split_bloom_filter.InsertHash(item_hash);
}

bool BloomChain::bloomCheck(uint64_t item_hash, std::string &bf_data) {
const BlockSplitBloomFilter block_split_bloom_filter(bf_data);
bool BloomChain::bloomCheck(uint64_t item_hash, std::string_view &bf_data) {
const BlockSplitBloomFilter block_split_bloom_filter(
nonstd::span<char>(const_cast<char *>(bf_data.data()), bf_data.size()));
return block_split_bloom_filter.FindHash(item_hash);
}

Expand Down Expand Up @@ -163,7 +164,7 @@ rocksdb::Status BloomChain::InsertCommon(const Slice &user_key, const std::vecto
std::vector<std::string> bf_key_list;
getBFKeyList(ns_key, metadata, &bf_key_list);

std::vector<std::string> bf_data_list;
std::vector<rocksdb::PinnableSlice> bf_data_list;
s = getBFDataList(bf_key_list, &bf_data_list);
if (!s.ok()) return s;

Expand All @@ -180,7 +181,8 @@ rocksdb::Status BloomChain::InsertCommon(const Slice &user_key, const std::vecto
bool exist = false;
// TODO: to test which direction for searching is better
for (int ii = static_cast<int>(bf_data_list.size()) - 1; ii >= 0; --ii) {
exist = bloomCheck(item_hash_list[i], bf_data_list[ii]);
std::string_view data = bf_data_list[ii].ToStringView();
exist = bloomCheck(item_hash_list[i], data);
if (exist) break;
}

Expand All @@ -190,17 +192,23 @@ rocksdb::Status BloomChain::InsertCommon(const Slice &user_key, const std::vecto
} else {
if (metadata.size + 1 > metadata.GetCapacity()) {
if (metadata.IsScaling()) {
batch->Put(bf_key_list.back(), bf_data_list.back());
batch->Put(bf_key_list.back(), bf_data_list.back().ToStringView());
std::string bf_data;
createBloomFilterInBatch(ns_key, &metadata, batch, &bf_data);
bf_data_list.push_back(std::move(bf_data));
rocksdb::PinnableSlice pin_slice;
*pin_slice.GetSelf() = std::move(bf_data);
pin_slice.PinSelf();
bf_data_list.push_back(std::move(pin_slice));
bf_key_list.push_back(getBFKey(ns_key, metadata, metadata.n_filters - 1));
} else {
(*rets)[i] = BloomFilterAddResult::kFull;
continue;
}
}
bloomAdd(item_hash_list[i], bf_data_list.back());
std::string data = bf_data_list.back().ToString();
bloomAdd(item_hash_list[i], data);
*bf_data_list.back().GetSelf() = std::move(data);
bf_data_list.back().PinSelf();
(*rets)[i] = BloomFilterAddResult::kOk;
metadata.size += 1;
}
Expand All @@ -210,7 +218,7 @@ rocksdb::Status BloomChain::InsertCommon(const Slice &user_key, const std::vecto
std::string bloom_chain_metadata_bytes;
metadata.Encode(&bloom_chain_metadata_bytes);
batch->Put(metadata_cf_handle_, ns_key, bloom_chain_metadata_bytes);
batch->Put(bf_key_list.back(), bf_data_list.back());
batch->Put(bf_key_list.back(), bf_data_list.back().ToStringView());
}
return storage_->Write(storage_->DefaultWriteOptions(), batch->GetWriteBatch());
}
Expand All @@ -237,7 +245,7 @@ rocksdb::Status BloomChain::MExists(const Slice &user_key, const std::vector<std
std::vector<std::string> bf_key_list;
getBFKeyList(ns_key, metadata, &bf_key_list);

std::vector<std::string> bf_data_list;
std::vector<rocksdb::PinnableSlice> bf_data_list;
s = getBFDataList(bf_key_list, &bf_data_list);
if (!s.ok()) return s;

Expand All @@ -248,7 +256,8 @@ rocksdb::Status BloomChain::MExists(const Slice &user_key, const std::vector<std
// check
// TODO: to test which direction for searching is better
for (int ii = static_cast<int>(bf_data_list.size()) - 1; ii >= 0; --ii) {
(*exists)[i] = bloomCheck(item_hash_list[i], bf_data_list[ii]);
std::string_view data = bf_data_list[ii].ToStringView();
(*exists)[i] = bloomCheck(item_hash_list[i], data);
if ((*exists)[i]) break;
}
}
Expand Down
5 changes: 3 additions & 2 deletions src/types/redis_bloom_chain.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,8 @@ class BloomChain : public Database {
rocksdb::Status getBloomChainMetadata(const Slice &ns_key, BloomChainMetadata *metadata);
std::string getBFKey(const Slice &ns_key, const BloomChainMetadata &metadata, uint16_t filters_index);
void getBFKeyList(const Slice &ns_key, const BloomChainMetadata &metadata, std::vector<std::string> *bf_key_list);
rocksdb::Status getBFDataList(const std::vector<std::string> &bf_key_list, std::vector<std::string> *bf_data_list);
rocksdb::Status getBFDataList(const std::vector<std::string> &bf_key_list,
std::vector<rocksdb::PinnableSlice> *bf_data_list);
static void getItemHashList(const std::vector<std::string> &items, std::vector<uint64_t> *item_hash_list);

rocksdb::Status createBloomChain(const Slice &ns_key, double error_rate, uint32_t capacity, uint16_t expansion,
Expand All @@ -88,6 +89,6 @@ class BloomChain : public Database {
/// bf_data: [in/out] The content string of bloomfilter.
static void bloomAdd(uint64_t item_hash, std::string &bf_data);

static bool bloomCheck(uint64_t item_hash, std::string &bf_data);
static bool bloomCheck(uint64_t item_hash, std::string_view &bf_data);
};
} // namespace redis

0 comments on commit 356bcb5

Please sign in to comment.