Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,18 @@ message GetAllConfigsRequest {

// required - namespace with which the config resource is associated
string resource_namespace = 2;

// optional - filtering criteria to narrow down the configs.
// Supports relational and logical operators on config fields.
Filter filter = 3;

// optional - list of sorting conditions to order the results.
// Multiple SortBy entries are applied in the specified order of priority.
repeated SortBy sort_by = 4;

// optional - pagination parameters to limit and offset the result set.
// Useful for retrieving configs in pages when total count is large.
Pagination pagination = 5;
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Removed total_count for now, will add it later if required

}

message GetAllConfigsResponse {
Expand Down Expand Up @@ -201,3 +213,25 @@ enum LogicalOperator {
LOGICAL_OPERATOR_AND = 1;
LOGICAL_OPERATOR_OR = 2;
}

message SortBy {
Selection selection = 1;
SortOrder sort_order = 2;
}

message Pagination {
int32 limit = 1;
int32 offset = 2;
}

message Selection {
oneof type {
string config_json_path = 1;
}
}

enum SortOrder {
SORT_ORDER_UNSPECIFIED = 0;
SORT_ORDER_ASC = 1;
SORT_ORDER_DESC = 2;
}
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,10 @@ public void getAllConfigs(
List<ContextSpecificConfig> contextSpecificConfigList =
configStore.getAllConfigs(
new ConfigResource(
request.getResourceName(), request.getResourceNamespace(), getTenantId()));
request.getResourceName(), request.getResourceNamespace(), getTenantId()),
request.getFilter(),
request.getPagination(),
request.getSortByList());
responseObserver.onNext(
GetAllConfigsResponse.newBuilder()
.addAllContextSpecificConfigs(contextSpecificConfigList)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@
import org.hypertrace.config.service.ConfigResource;
import org.hypertrace.config.service.ConfigResourceContext;
import org.hypertrace.config.service.v1.ContextSpecificConfig;
import org.hypertrace.config.service.v1.Filter;
import org.hypertrace.config.service.v1.Pagination;
import org.hypertrace.config.service.v1.SortBy;
import org.hypertrace.config.service.v1.UpsertAllConfigsResponse.UpsertedConfig;
import org.hypertrace.config.service.v1.UpsertConfigRequest;

Expand Down Expand Up @@ -57,10 +60,15 @@ Map<ConfigResourceContext, ContextSpecificConfig> getContextConfigs(
* specified parameters, sorted in the descending order of their creation time.
*
* @param configResource
* @param filter
* @param pagination
* @param sortByList
* @return
* @throws IOException
*/
List<ContextSpecificConfig> getAllConfigs(ConfigResource configResource) throws IOException;
List<ContextSpecificConfig> getAllConfigs(
ConfigResource configResource, Filter filter, Pagination pagination, List<SortBy> sortByList)
throws IOException;

/**
* Write each of the provided config value associated with the specified config resource to the
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package org.hypertrace.config.service.store;

import com.google.protobuf.Value;
import io.grpc.Status;
import java.util.List;
import java.util.stream.Collectors;
import org.hypertrace.core.documentstore.expression.impl.ConstantExpression;

public class ConstantExpressionConverter {

public static ConstantExpression fromProtoValue(Value value) {
switch (value.getKindCase()) {
case STRING_VALUE:
return ConstantExpression.of(value.getStringValue());
case NUMBER_VALUE:
return ConstantExpression.of(value.getNumberValue());
case BOOL_VALUE:
return ConstantExpression.of(value.getBoolValue());
case LIST_VALUE:
List<Value> values = value.getListValue().getValuesList();
if (values.isEmpty()) {
// Default to empty string list — or change logic based on expected behavior
return ConstantExpression.ofStrings(List.of());
}

Value.KindCase elementType = values.get(0).getKindCase();
boolean isHomogeneous = values.stream().allMatch(v -> v.getKindCase() == elementType);

if (!isHomogeneous) {
throw Status.INVALID_ARGUMENT
.withDescription("List contains mixed types. All elements must be of the same type.")
.asRuntimeException();
}
switch (elementType) {
case STRING_VALUE:
return ConstantExpression.ofStrings(
values.stream().map(Value::getStringValue).collect(Collectors.toList()));
case NUMBER_VALUE:
return ConstantExpression.ofNumbers(
values.stream().map(Value::getNumberValue).collect(Collectors.toList()));
case BOOL_VALUE:
return ConstantExpression.ofBooleans(
values.stream().map(Value::getBoolValue).collect(Collectors.toList()));
default:
throw Status.UNIMPLEMENTED
.withDescription("Unsupported list element type: " + elementType)
.asRuntimeException();
}
case STRUCT_VALUE:
throw Status.UNIMPLEMENTED
.withDescription("Struct not supported directly in ConstantExpression")
.asRuntimeException();
case NULL_VALUE:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can't compare to null?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not able to find support Null values in ConstantExpression. Thats why marked as unsupported for now.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK, we should add that at some point. For now, the main use case is probably eq/neq NULL which would represent in doc store as a EXISTS or NOT_EXISTS. So we could consider doing that translation if we need it rather than introducing those extra operators.

case KIND_NOT_SET:
default:
throw Status.INVALID_ARGUMENT
.withDescription("Unsupported or null value in ConstantExpression")
.asRuntimeException();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,14 @@
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.NonNull;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.hypertrace.config.service.ConfigResource;
import org.hypertrace.config.service.ConfigResourceContext;
import org.hypertrace.config.service.ConfigServiceUtils;
import org.hypertrace.config.service.v1.ContextSpecificConfig;
import org.hypertrace.config.service.v1.SortBy;
import org.hypertrace.config.service.v1.UpsertAllConfigsResponse.UpsertedConfig;
import org.hypertrace.config.service.v1.UpsertConfigRequest;
import org.hypertrace.core.documentstore.CloseableIterator;
Expand Down Expand Up @@ -58,12 +61,14 @@ public class DocumentConfigStore implements ConfigStore {
private final Datastore datastore;
private final Collection collection;
private final FilterBuilder filterBuilder;
private final FilterExpressionBuilder filterExpressionBuilder;

public DocumentConfigStore(Clock clock, Datastore datastore) {
this.clock = clock;
this.datastore = datastore;
this.collection = this.datastore.getCollection(CONFIGURATIONS_COLLECTION);
this.filterBuilder = new FilterBuilder();
this.filterExpressionBuilder = new FilterExpressionBuilder();
}

@Override
Expand Down Expand Up @@ -208,30 +213,88 @@ public Map<ConfigResourceContext, ContextSpecificConfig> getContextConfigs(
}

@Override
public List<ContextSpecificConfig> getAllConfigs(ConfigResource configResource)
public List<ContextSpecificConfig> getAllConfigs(
ConfigResource configResource,
org.hypertrace.config.service.v1.Filter filter,
org.hypertrace.config.service.v1.Pagination pagination,
List<SortBy> sortByList)
throws IOException {
Query query =
Query.builder()
.addSort(IdentifierExpression.of(VERSION_FIELD_NAME), SortOrder.DESC)
.setFilter(getConfigResourceFilterTypeExpression(configResource))
.build();
List<ContextSpecificConfig> contextSpecificConfigList = new ArrayList<>();

Query query = buildQuery(configResource, filter, pagination, sortByList);
List<ContextSpecificConfig> configList = new ArrayList<>();
Set<String> seenContexts = new HashSet<>();

try (CloseableIterator<Document> documentIterator =
collection.query(query, QueryOptions.DEFAULT_QUERY_OPTIONS)) {
while (documentIterator.hasNext()) {
String documentString = documentIterator.next().toJson();
ConfigDocument configDocument = ConfigDocument.fromJson(documentString);
String context = configDocument.getContext();
if (seenContexts.add(context)) {
convertToContextSpecificConfig(configDocument).ifPresent(contextSpecificConfigList::add);
}
processDocument(documentIterator.next(), seenContexts, configList);
}
}
Collections.sort(
contextSpecificConfigList,

configList.sort(
Comparator.comparingLong(ContextSpecificConfig::getCreationTimestamp).reversed());
return contextSpecificConfigList;
return configList;
}

private Query buildQuery(
ConfigResource configResource,
@NonNull org.hypertrace.config.service.v1.Filter filter,
@NonNull org.hypertrace.config.service.v1.Pagination pagination,
List<SortBy> sortByList) {

FilterTypeExpression combinedFilter = getCombinedFilter(configResource, filter);
Query.QueryBuilder queryBuilder = Query.builder().setFilter(combinedFilter);
if (!pagination.equals(org.hypertrace.config.service.v1.Pagination.getDefaultInstance())) {
queryBuilder.setPagination(
Pagination.builder().offset(pagination.getOffset()).limit(pagination.getLimit()).build());
}

if (!sortByList.isEmpty()) {
sortByList.forEach(
sortBy ->
queryBuilder.addSort(
IdentifierExpression.of(sortBy.getSelection().getConfigJsonPath()),
convertSortOrder(sortBy)));
} else {
queryBuilder.addSort(IdentifierExpression.of(VERSION_FIELD_NAME), SortOrder.DESC);
}
return queryBuilder.build();
}

private FilterTypeExpression getCombinedFilter(
ConfigResource configResource,
@NonNull org.hypertrace.config.service.v1.Filter additionalFilter) {

FilterTypeExpression resourceFilter = getConfigResourceFilterTypeExpression(configResource);
if (additionalFilter.equals(org.hypertrace.config.service.v1.Filter.getDefaultInstance())) {
return resourceFilter;
}

FilterTypeExpression docStoreFilter =
filterExpressionBuilder.buildFilterTypeExpression(additionalFilter);
return LogicalExpression.and(resourceFilter, docStoreFilter);
}

@SneakyThrows
private void processDocument(
Document document, Set<String> seenContexts, List<ContextSpecificConfig> configList) {

ConfigDocument configDocument = ConfigDocument.fromJson(document.toJson());
String context = configDocument.getContext();

if (seenContexts.add(context)) {
convertToContextSpecificConfig(configDocument).ifPresent(configList::add);
}
}

private static SortOrder convertSortOrder(SortBy sortBy) {
switch (sortBy.getSortOrder()) {
case SORT_ORDER_DESC:
return SortOrder.DESC;
case SORT_ORDER_ASC:
default:
return SortOrder.ASC;
}
}

@Override
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package org.hypertrace.config.service.store;

import static org.hypertrace.config.service.store.ConfigDocument.CONFIG_FIELD_NAME;

import io.grpc.Status;
import java.util.List;
import java.util.stream.Collectors;
import org.hypertrace.config.service.v1.Filter;
import org.hypertrace.config.service.v1.LogicalFilter;
import org.hypertrace.config.service.v1.RelationalFilter;
import org.hypertrace.core.documentstore.expression.impl.IdentifierExpression;
import org.hypertrace.core.documentstore.expression.impl.LogicalExpression;
import org.hypertrace.core.documentstore.expression.impl.RelationalExpression;
import org.hypertrace.core.documentstore.expression.operators.LogicalOperator;
import org.hypertrace.core.documentstore.expression.operators.RelationalOperator;
import org.hypertrace.core.documentstore.expression.type.FilterTypeExpression;

public class FilterExpressionBuilder {

public FilterTypeExpression buildFilterTypeExpression(Filter filter) {
switch (filter.getTypeCase()) {
case LOGICAL_FILTER:
return buildLogicalExpression(filter.getLogicalFilter());
case RELATIONAL_FILTER:
return buildRelationalExpression(filter.getRelationalFilter());
case TYPE_NOT_SET:
default:
throw Status.INVALID_ARGUMENT.withDescription("Filter type unset").asRuntimeException();
}
}

private FilterTypeExpression buildLogicalExpression(LogicalFilter logicalFilter) {
List<FilterTypeExpression> childExpressions =
logicalFilter.getOperandsList().stream()
.map(this::buildFilterTypeExpression)
.collect(Collectors.toUnmodifiableList());

LogicalOperator operator;
switch (logicalFilter.getOperator()) {
case LOGICAL_OPERATOR_AND:
operator = LogicalOperator.AND;
break;
case LOGICAL_OPERATOR_OR:
operator = LogicalOperator.OR;
break;
case LOGICAL_OPERATOR_UNSPECIFIED:
default:
throw Status.INVALID_ARGUMENT
.withDescription("Unknown logical operator while building expression")
.asRuntimeException();
}
return LogicalExpression.builder().operator(operator).operands(childExpressions).build();
}

private FilterTypeExpression buildRelationalExpression(RelationalFilter relationalFilter) {
RelationalOperator operator;
switch (relationalFilter.getOperator()) {
case RELATIONAL_OPERATOR_EQ:
operator = RelationalOperator.EQ;
break;
case RELATIONAL_OPERATOR_NEQ:
operator = RelationalOperator.NEQ;
break;
case RELATIONAL_OPERATOR_IN:
operator = RelationalOperator.IN;
break;
case RELATIONAL_OPERATOR_NOT_IN:
operator = RelationalOperator.NOT_IN;
break;
case RELATIONAL_OPERATOR_LT:
operator = RelationalOperator.LT;
break;
case RELATIONAL_OPERATOR_GT:
operator = RelationalOperator.GT;
break;
case RELATIONAL_OPERATOR_LTE:
operator = RelationalOperator.LTE;
break;
case RELATIONAL_OPERATOR_GTE:
operator = RelationalOperator.GTE;
break;
case UNRECOGNIZED:
default:
throw Status.INVALID_ARGUMENT
.withDescription("Unknown relational operator while building expression")
.asRuntimeException();
}

return RelationalExpression.of(
IdentifierExpression.of(buildConfigFieldPath(relationalFilter.getConfigJsonPath())),
operator,
ConstantExpressionConverter.fromProtoValue(relationalFilter.getValue()));
}

private String buildConfigFieldPath(String configJsonPath) {
return String.format("%s.%s", CONFIG_FIELD_NAME, configJsonPath);
}
}
Loading