Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: can't not get search_cache_budget_gb in create index #30353

Merged
merged 1 commit into from
Jan 31, 2024
Merged

fix: can't not get search_cache_budget_gb in create index #30353

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion internal/datacoord/index_builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,7 @@
}
if isDiskANNIndex(getIndexType(indexParams)) {
var err error
indexParams, err = indexparams.AppendDiskIndexBuildParams(Params, indexParams)
indexParams, err = indexparams.UpdateDiskIndexBuildParams(Params, indexParams)

Check warning on line 305 in internal/datacoord/index_builder.go

View check run for this annotation

Codecov / codecov/patch

internal/datacoord/index_builder.go#L305 

Added line #L305 was not covered by tests
if err != nil {
log.Ctx(ib.ctx).Warn("failed to append index build params", zap.Int64("buildID", buildID),
zap.Int64("nodeID", nodeID), zap.Error(err))
Expand Down
47 changes: 35 additions & 12 deletions pkg/util/indexparams/index_params.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,11 +154,11 @@ func NewBigDataExtraParamsFromMap(value map[string]string) (*BigDataIndexExtraPa
// FillDiskIndexParams fill ratio params to index param on proxy node
// Which will be used to calculate build and load params
func FillDiskIndexParams(params *paramtable.ComponentParam, indexParams map[string]string) error {
maxDegree := params.CommonCfg.MaxDegree.GetValue()
searchListSize := params.CommonCfg.SearchListSize.GetValue()
pqCodeBudgetGBRatio := params.CommonCfg.PQCodeBudgetGBRatio.GetValue()
buildNumThreadsRatio := params.CommonCfg.BuildNumThreadsRatio.GetValue()
searchCacheBudgetGBRatio := params.CommonCfg.SearchCacheBudgetGBRatio.GetValue()
var maxDegree string
var searchListSize string
var pqCodeBudgetGBRatio string
var buildNumThreadsRatio string
var searchCacheBudgetGBRatio string

if params.AutoIndexConfig.Enable.GetAsBool() {
indexParams := params.AutoIndexConfig.IndexParams.GetAsJSONMap()
Expand All @@ -177,6 +177,13 @@ func FillDiskIndexParams(params *paramtable.ComponentParam, indexParams map[stri
}
pqCodeBudgetGBRatio = fmt.Sprintf("%f", extraParams.PQCodeBudgetGBRatio)
buildNumThreadsRatio = fmt.Sprintf("%f", extraParams.BuildNumThreadsRatio)
searchCacheBudgetGBRatio = fmt.Sprintf("%f", extraParams.SearchCacheBudgetGBRatio)
czs007 marked this conversation as resolved.
Show resolved Hide resolved
} else {
maxDegree = params.CommonCfg.MaxDegree.GetValue()
searchListSize = params.CommonCfg.SearchListSize.GetValue()
pqCodeBudgetGBRatio = params.CommonCfg.PQCodeBudgetGBRatio.GetValue()
buildNumThreadsRatio = params.CommonCfg.BuildNumThreadsRatio.GetValue()
searchCacheBudgetGBRatio = params.CommonCfg.SearchCacheBudgetGBRatio.GetValue()
}

indexParams[MaxDegreeKey] = maxDegree
Expand All @@ -197,12 +204,9 @@ func GetIndexParams(indexParams []*commonpb.KeyValuePair, key string) string {
return ""
}

// AppendDiskIndexBuildParams append index params for `buildIndex` (params not exist in `CreateIndex`)
func AppendDiskIndexBuildParams(params *paramtable.ComponentParam, indexParams []*commonpb.KeyValuePair) ([]*commonpb.KeyValuePair, error) {
// UpdateDiskIndexBuildParams update index params for `buildIndex` (override search cache size in `CreateIndex`)
func UpdateDiskIndexBuildParams(params *paramtable.ComponentParam, indexParams []*commonpb.KeyValuePair) ([]*commonpb.KeyValuePair, error) {
existedVal := GetIndexParams(indexParams, SearchCacheBudgetRatioKey)
if len(existedVal) > 0 {
return indexParams, nil
}

var searchCacheBudgetGBRatio string
if params.AutoIndexConfig.Enable.GetAsBool() {
Expand All @@ -219,14 +223,33 @@ func AppendDiskIndexBuildParams(params *paramtable.ComponentParam, indexParams [
searchCacheBudgetGBRatio = fmt.Sprintf("%f", paramVal)
}

if len(searchCacheBudgetGBRatio) > 0 {
// append when not exist
if len(existedVal) == 0 {
indexParams = append(indexParams,
&commonpb.KeyValuePair{
Key: SearchCacheBudgetRatioKey,
Value: searchCacheBudgetGBRatio,
})
return indexParams, nil
}
// override when exist
updatedParams := make([]*commonpb.KeyValuePair, 0, len(indexParams))
for _, param := range indexParams {
if param.Key == SearchCacheBudgetRatioKey {
updatedParams = append(updatedParams,
&commonpb.KeyValuePair{
Key: SearchCacheBudgetRatioKey,
Value: searchCacheBudgetGBRatio,
})
} else {
updatedParams = append(updatedParams,
&commonpb.KeyValuePair{
Key: param.Key,
Value: param.Value,
})
}
}
return indexParams, nil
return updatedParams, nil
}

// SetDiskIndexBuildParams set index build params with ratio params on indexNode
Expand Down
8 changes: 4 additions & 4 deletions pkg/util/indexparams/index_params_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ func TestDiskIndexParams(t *testing.T) {
Value: "4.0",
})

indexParams, err := AppendDiskIndexBuildParams(&params, indexParams)
indexParams, err := UpdateDiskIndexBuildParams(&params, indexParams)
assert.NoError(t, err)
assert.True(t, len(indexParams) == 4)

Expand Down Expand Up @@ -191,7 +191,7 @@ func TestDiskIndexParams(t *testing.T) {
Value: "4.0",
})

autoParams, err = AppendDiskIndexBuildParams(&params, autoParams)
autoParams, err = UpdateDiskIndexBuildParams(&params, autoParams)
assert.NoError(t, err)
assert.True(t, len(autoParams) == 4)

Expand All @@ -208,15 +208,15 @@ func TestDiskIndexParams(t *testing.T) {
}
`
params.Save(params.AutoIndexConfig.ExtraParams.Key, newJSONStr)
autoParams, err = AppendDiskIndexBuildParams(&params, autoParams)
autoParams, err = UpdateDiskIndexBuildParams(&params, autoParams)

assert.NoError(t, err)
assert.True(t, len(autoParams) == 4)

val = GetIndexParams(autoParams, SearchCacheBudgetRatioKey)
iVal, iErr = strconv.ParseFloat(val, 64)
assert.NoError(t, iErr)
assert.Equal(t, 0.225, iVal)
assert.Equal(t, 0.325, iVal)
})

t.Run("set disk index build params", func(t *testing.T) {
Expand Down