Skip to content

Commit

Permalink
MB-47195: Add and map new plasma bloom config
Browse files Browse the repository at this point in the history
Add new config indexer.settings.enable_page_bloom_filter.
Map new config to exisiting plasma bloom filter config
for the backIndex. If either one is modified, both will
change. If both are modified at the same time, then prefer
the new setting.

Change-Id: Id3720843ffaf18cf2a88ee3c0b2a25f62facc454
  • Loading branch information
akhilmd committed Oct 12, 2021
1 parent eaadae8 commit dffd3ed
Show file tree
Hide file tree
Showing 3 changed files with 80 additions and 2 deletions.
67 changes: 67 additions & 0 deletions secondary/common/config.go
Expand Up @@ -1626,6 +1626,13 @@ var SystemConfig = Config{
false, // mutable
false, // case-insensitive
},
"indexer.settings.enable_page_bloom_filter": ConfigValue{
false, // keep in sync with index_settings_manager.erl and indexer.plasma.backIndex.enablePageBloomFilter
"Enable maintenance and use of bloom filter for lookup of swapped out items",
false,
false, // mutable
false, // case-insensitive
},
"indexer.plasma.backIndex.enablePageBloomFilter": ConfigValue{
false,
"Enable maintenance and use of bloom filter for lookup of swapped out items",
Expand Down Expand Up @@ -3028,6 +3035,66 @@ func (config Config) Update(data interface{}) error {
return nil
}

// Ensure these are valid configs
var configMap = map[string]string{
"indexer.settings.enable_page_bloom_filter": "indexer.plasma.backIndex.enablePageBloomFilter",
}

func MapSettings(value []byte) ([]byte, error) {
newConfig, err := NewConfig(value)
if err != nil {
return value, err
}

logging.Infof("MapSettings: config before mapping: %v", string(newConfig.Json()))

var dstKey string
var srcVal interface{}

for key1, key2 := range configMap {
val1, key1IsPresent := newConfig[key1]
val2, key2IsPresent := newConfig[key2]

// If both key1 and key2 are present, then prefer key1
if key1IsPresent {
// Either both key1 and key2 are present or only key1 is present
// map key1 into key2
dstKey = key2
srcVal = val1.Value

} else if key2IsPresent {
// Only key2 is present
// map key2 into key1
dstKey = key1
srcVal = val2.Value

}

if cv, ok := SystemConfig[dstKey]; ok {

if dstCV, ok := newConfig[dstKey]; !ok {
// dstKey is not present, populate default value
newConfig[dstKey] = cv
} else if dstCV.Value == srcVal {
// dstKey already has target value, skip mapping
continue
}

if err := newConfig.SetValue(dstKey, srcVal); err != nil {
return value, fmt.Errorf("MapSettings: error during set : dstKey[%v] srcVal[%v] [%v]", dstKey, srcVal, err)
}

} else {
return value, fmt.Errorf("MapSettings: invalid config param %q in configMap", dstKey)
}
}

newValue := newConfig.Json()
logging.Infof("MapSettings: config after mapping: %v", string(newValue))

return newValue, nil
}

// Clone a new config object.
func (config Config) Clone() Config {
clone := make(Config)
Expand Down
9 changes: 7 additions & 2 deletions secondary/indexer/plasma_slice.go
Expand Up @@ -397,7 +397,10 @@ func (slice *plasmaSlice) initStores() error {
bCfg.EvictRunInterval = time.Duration(slice.sysconf["plasma.backIndex.evictRunInterval"].Int()) * time.Millisecond
bCfg.EvictUseMemEstimate = slice.sysconf["plasma.backIndex.evictUseMemEstimate"].Bool()
bCfg.LogPrefix = fmt.Sprintf("%s/%s/Backstore#%d:%d ", slice.idxDefn.Bucket, slice.idxDefn.Name, slice.idxInstId, slice.idxPartnId)
bCfg.EnablePageBloomFilter = slice.sysconf["plasma.backIndex.enablePageBloomFilter"].Bool()

// Will also change based on indexer.plasma.backIndex.enablePageBloomFilter
bCfg.EnablePageBloomFilter = slice.sysconf["settings.enable_page_bloom_filter"].Bool()

bCfg.BloomFilterFalsePositiveRate = slice.sysconf["plasma.backIndex.bloomFilterFalsePositiveRate"].Float64()
bCfg.BloomFilterExpectedMaxItems = slice.sysconf["plasma.backIndex.bloomFilterExpectedMaxItems"].Uint64()
bCfg.EnableInMemoryCompression = slice.sysconf["plasma.backIndex.enableInMemoryCompression"].Bool()
Expand Down Expand Up @@ -2422,7 +2425,9 @@ func (mdb *plasmaSlice) UpdateConfig(cfg common.Config) {
mdb.backstore.ReaderPurgeThreshold = mdb.sysconf["plasma.reader.purge.threshold"].Float64()
mdb.backstore.ReaderPurgePageRatio = mdb.sysconf["plasma.reader.purge.pageRatio"].Float64()

mdb.backstore.EnablePageBloomFilter = mdb.sysconf["plasma.backIndex.enablePageBloomFilter"].Bool()
// Will also change based on indexer.plasma.backIndex.enablePageBloomFilter
mdb.backstore.EnablePageBloomFilter = mdb.sysconf["settings.enable_page_bloom_filter"].Bool()

mdb.backstore.BloomFilterFalsePositiveRate = mdb.sysconf["plasma.backIndex.bloomFilterFalsePositiveRate"].Float64()
mdb.backstore.BloomFilterExpectedMaxItems = mdb.sysconf["plasma.backIndex.bloomFilterExpectedMaxItems"].Uint64()

Expand Down
6 changes: 6 additions & 0 deletions secondary/indexer/settings.go
Expand Up @@ -168,6 +168,12 @@ func (s *settingsManager) handleSettings(w http.ResponseWriter, r *http.Request,
return
}

if bytes, err = common.MapSettings(bytes); err != nil {
logging.Errorf("Fail to map settings. Error: %v", err)
s.writeError(w, err)
return
}

err = config.Update(bytes)
}

Expand Down

0 comments on commit dffd3ed

Please sign in to comment.