Skip to content

Commit

Permalink
RANGER-4630: added multiple tagId search in /tags/tags/paginated API
Browse files Browse the repository at this point in the history 
Signed-off-by: Madhan Neethiraj <madhan@apache.org>
  • Loading branch information
@anandN872 @mneethiraj
anandN872 authored and mneethiraj committed Feb 19, 2024
1 parent ae6481f commit 48520f9
Show file tree
Hide file tree
Showing 6 changed files with 202 additions and 9 deletions.
38 changes: 31 additions & 7 deletions agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ public class SearchFilter {
public static final String TAG_SOURCE = "tagSource"; // search
public static final String TAG_SOURCE_PARTIAL = "tagSourcePartial"; // search
public static final String TAG_ID = "tagId"; // search
public static final String TAG_IDS = "tagIds"; // search
public static final String TAG_GUID = "tagGuid"; // search
public static final String TAG_RESOURCE_ID = "resourceId"; // search
public static final String TAG_RESOURCE_GUID = "resourceGuid"; // search
Expand Down Expand Up @@ -136,13 +137,14 @@ public class SearchFilter {
public static final String RETRIEVE_ALL_PAGES = "retrieveAllPages"; // search
public static final String SHARED_WITH_ME = "sharedWithMe"; // search

private Map<String, String> params;
private int startIndex;
private int maxRows = Integer.MAX_VALUE;
private boolean getCount = true;
private String sortBy;
private String sortType;
private boolean isDistinct = true;
private Map<String, String> params;
private Map<String, Object[]> multiValueParams;
private int startIndex;
private int maxRows = Integer.MAX_VALUE;
private boolean getCount = true;
private String sortBy;
private String sortType;
private boolean isDistinct = true;

public SearchFilter() {
this((Map<String, String>) null);
Expand All @@ -151,13 +153,15 @@ public SearchFilter() {
public SearchFilter(SearchFilter other) {
if (other != null) {
setParams(other.params != null ? new HashMap<>(other.params) : null);
setMultiValueParams(other.multiValueParams != null ? new HashMap<>(other.multiValueParams) : null);
setStartIndex(other.startIndex);
setMaxRows(other.maxRows);
setGetCount(other.getCount);
setSortBy(other.sortBy);
setSortType(other.sortType);
} else {
setParams(null);
setMultiValueParams(null);
}
}

Expand Down Expand Up @@ -228,6 +232,26 @@ public Map<String, String> getParamsWithPrefix(String prefix, boolean stripPrefi
public boolean isEmpty() {
return MapUtils.isEmpty(params);
}

public Map<String, Object[]> getMultiValueParams() {
return multiValueParams;
}

public void setMultiValueParams(Map<String, Object[]> multiValueParams) {
this.multiValueParams = multiValueParams;
}

public void setMultiValueParam(String name, Object[] value) {
if (multiValueParams == null) {
multiValueParams = new HashMap<>();
}

multiValueParams.put(name, value);
}

public Object[] getMultiValueParam(String name) {
return multiValueParams != null ? multiValueParams.get(name) : null;
}

public int getStartIndex() {
return startIndex;
Expand Down
128 changes: 126 additions & 2 deletions security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,12 @@
public class RangerSearchUtil extends SearchUtil {
final static Logger logger = LoggerFactory.getLogger(RangerSearchUtil.class);

int minInListLength = 20;

public RangerSearchUtil() {
minInListLength = PropertiesUtil.getIntProperty("ranger.db.min_inlist", minInListLength);
}

public SearchFilter getSearchFilter(@Nonnull HttpServletRequest request, List<SortField> sortFields) {
Validate.notNull(request, "request");
SearchFilter ret = new SearchFilter();
Expand Down Expand Up @@ -308,7 +314,62 @@ private StringBuilder buildWhereClause(SearchFilter searchCriteria,
continue;
}

if (searchField.getDataType() == SearchField.DATA_TYPE.INTEGER) {
Object[] multiValue = searchCriteria.getMultiValueParam(searchField.getClientFieldName());
boolean isMultiValue = multiValue != null && multiValue.length > 0;

if (searchField.getDataType() == SearchField.DATA_TYPE.INT_LIST || (isMultiValue && searchField.getDataType() == SearchField.DATA_TYPE.INTEGER)) {
List<Number> intValueList = new ArrayList<>();

if (isMultiValue) {
for (Object value : multiValue) {
if (value instanceof Integer || value instanceof Long) {
intValueList.add((Number) value);
} else if (value != null) {
intValueList.add(restErrorUtil.parseInt(value.toString(), "Invalid value for " + searchField.getClientFieldName(), MessageEnums.INVALID_INPUT_DATA, null, searchField.getClientFieldName()));
}
}
} else {
String paramVal = searchCriteria.getParam(searchField.getClientFieldName());

if (paramVal != null) {
intValueList.add(restErrorUtil.parseInt(paramVal, "Invalid value for " + searchField.getClientFieldName(), MessageEnums.INVALID_INPUT_DATA, null, searchField.getClientFieldName()));
}
}

if (!intValueList.isEmpty()) {
if (searchField.getCustomCondition() == null) {
if (intValueList.size() <= minInListLength) {
whereClause.append(" and ");

if (intValueList.size() > 1) {
whereClause.append(" ( ");
}

for (int count = 0; count < intValueList.size(); count++) {
if (count > 0) {
whereClause.append(" or ");
}

whereClause.append(searchField.getFieldName()).append("= :")
.append(searchField.getClientFieldName()).append("_").append(count);
}

if (intValueList.size() > 1) {
whereClause.append(" ) ");
}

logger.debug("Where clause ... :: " + whereClause);
} else {
whereClause.append(" and ")
.append(searchField.getFieldName())
.append(" in ")
.append(" (:").append(searchField.getClientFieldName()).append(")");
}
} else {
whereClause.append(" and ").append(searchField.getCustomCondition());
}
}
} else if (searchField.getDataType() == SearchField.DATA_TYPE.INTEGER) {
Integer paramVal = restErrorUtil.parseInt(searchCriteria.getParam(searchField.getClientFieldName()),
"Invalid value for " + searchField.getClientFieldName(),
MessageEnums.INVALID_INPUT_DATA, null, searchField.getClientFieldName());
Expand Down Expand Up @@ -382,10 +443,42 @@ private StringBuilder buildWhereClause(SearchFilter searchCriteria,
}

protected void resolveQueryParams(Query query, SearchFilter searchCriteria, List<SearchField> searchFields) {
Map<String, String> params = searchCriteria.getParams();
Map<String, Object[]> multiValueParams = searchCriteria.getMultiValueParams();

for (SearchField searchField : searchFields) {
Object[] multiValue = multiValueParams != null ? multiValueParams.get(searchField.getClientFieldName()) : null;
boolean isMultiValue = multiValue != null && multiValue.length > 0;

if (searchField.getDataType() == SearchField.DATA_TYPE.INT_LIST || (isMultiValue && searchField.getDataType() == SearchField.DATA_TYPE.INTEGER)) {
List<Number> intValueList = new ArrayList<>();

if (isMultiValue) {
for (Object value : multiValue) {
if (value instanceof Integer || value instanceof Long) {
intValueList.add((Number) value);
} else if (value != null) {
intValueList.add(restErrorUtil.parseInt(value.toString(), "Invalid value for " + searchField.getClientFieldName(), MessageEnums.INVALID_INPUT_DATA, null, searchField.getClientFieldName()));
}
}
} else {
String paramVal = params != null ? params.get(searchField.getClientFieldName()) : null;

if (searchField.getDataType() == SearchField.DATA_TYPE.INTEGER) {
if (paramVal != null) {
intValueList.add(restErrorUtil.parseInt(paramVal, "Invalid value for " + searchField.getClientFieldName(), MessageEnums.INVALID_INPUT_DATA, null, searchField.getClientFieldName()));
}
}

if (!intValueList.isEmpty()) {
if (intValueList.size() <= minInListLength) {
for (int idx = 0; idx < intValueList.size(); idx++) {
query.setParameter(searchField.getClientFieldName() + "_" + idx, intValueList.get(idx));
}
} else {
query.setParameter(searchField.getClientFieldName(), intValueList);
}
}
} else if (searchField.getDataType() == SearchField.DATA_TYPE.INTEGER) {
Integer paramVal = restErrorUtil.parseInt(searchCriteria.getParam(searchField.getClientFieldName()),
"Invalid value for " + searchField.getClientFieldName(),
MessageEnums.INVALID_INPUT_DATA, null, searchField.getClientFieldName());
Expand Down Expand Up @@ -490,4 +583,35 @@ private String getSortType(SearchFilter searchCriteria) {

return ret;
}

public void extractIntList(HttpServletRequest request, SearchFilter searchFilter, String paramName, String userFriendlyParamName) {
String[] values = getParamMultiValues(request, paramName);

if (values != null) {
List<Integer> intValues = new ArrayList<>(values.length);

for (String value : values) {
Integer intValue = restErrorUtil.parseInt(value, "Invalid value for " + userFriendlyParamName, MessageEnums.INVALID_INPUT_DATA, null, paramName);

intValues.add(intValue);
}

searchFilter.setMultiValueParam(paramName, intValues.toArray());
}
}

/**
* @param request
* @param paramName
* @return
*/
String[] getParamMultiValues(HttpServletRequest request, String paramName) {
String[] values = request.getParameterValues(paramName);

if (values == null || values.length == 0) {
values = request.getParameterValues(paramName + "[]");
}

return values;
}
}
38 changes: 38 additions & 0 deletions security-admin/src/main/java/org/apache/ranger/rest/TagREST.java
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import org.apache.ranger.plugin.util.ServiceTags;
import org.apache.ranger.service.RangerServiceResourceService;
import org.apache.ranger.service.RangerTagDefService;
import org.apache.ranger.service.RangerTagResourceMapService;
import org.apache.ranger.service.RangerTagService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -117,6 +118,9 @@ public class TagREST {
@Autowired
RangerServiceResourceService rangerServiceResourceService;

@Autowired
RangerTagResourceMapService rangerTagResourceMapService;

public TagREST() {
}

Expand Down Expand Up @@ -669,6 +673,8 @@ public PList<RangerTag> getTags(@Context HttpServletRequest request) {
try {
SearchFilter filter = searchUtil.getSearchFilter(request, tagService.sortFields);

searchUtil.extractIntList(request, filter, SearchFilter.TAG_IDS, "Tag Id List");

ret = tagStore.getPaginatedTags(filter);
} catch (Exception excp) {
LOG.error("getTags() failed", excp);
Expand Down Expand Up @@ -1266,6 +1272,38 @@ public List<RangerTagResourceMap> getAllTagResourceMaps() {
return ret;
}

@GET
@Path(TagRESTConstants.TAGRESOURCEMAPS_RESOURCE_PAGINATED)
@Produces({ "application/json" })
@PreAuthorize("hasRole('ROLE_SYS_ADMIN')")
public PList<RangerTagResourceMap> getTagResourceMaps(@Context HttpServletRequest request) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> TagREST.getTagResourceMaps()");
}

final PList<RangerTagResourceMap> ret;

try {
SearchFilter filter = searchUtil.getSearchFilter(request, rangerTagResourceMapService.sortFields);

ret = tagStore.getPaginatedTagResourceMaps(filter);
} catch (Exception excp) {
LOG.error("getTagResourceMaps() failed", excp);

throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true);
}

if (ret == null) {
throw restErrorUtil.createRESTException(HttpServletResponse.SC_NOT_FOUND, "Not found", true);
}

if (LOG.isDebugEnabled()) {
LOG.debug("<== TagREST.getTagResourceMaps(): " + ret);
}

return ret;
}

// This API is used by tag-sync to upload tag-objects

@PUT
Expand Down
1 change: 1 addition & 0 deletions security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ public class TagRESTConstants {
static final String RESOURCES_RESOURCE_PAGINATED = "/resources/paginated";
static final String RESOURCE_RESOURCE = "/resource/";
static final String TAGRESOURCEMAPS_RESOURCE = "/tagresourcemaps/";
static final String TAGRESOURCEMAPS_RESOURCE_PAGINATED = "/tagresourcemaps/paginated";
static final String IMPORT_SERVICETAGS_RESOURCE = "/importservicetags/";
static final String TAGRESOURCEMAP_RESOURCE = "/tagresourcemap/";
static final String TAGTYPES_RESOURCE = "/types/";
Expand Down
5 changes: 5 additions & 0 deletions security-admin/src/main/java/org/apache/ranger/service/RangerTagResourceMapService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.apache.ranger.common.SearchField;
import org.apache.ranger.common.SearchField.DATA_TYPE;
import org.apache.ranger.common.SearchField.SEARCH_TYPE;
import org.apache.ranger.common.SortField;
import org.apache.ranger.entity.XXTagResourceMap;
import org.apache.ranger.plugin.model.RangerTagResourceMap;
import org.apache.ranger.plugin.util.SearchFilter;
Expand All @@ -38,6 +39,10 @@ public RangerTagResourceMapService() {
searchFields.add(new SearchField(SearchFilter.TAG_DEF_ID, "obj.id", DATA_TYPE.INTEGER, SEARCH_TYPE.FULL));
searchFields.add(new SearchField(SearchFilter.TAG_RESOURCE_ID, "obj.resourceId", DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL));
searchFields.add(new SearchField(SearchFilter.TAG_ID, "obj.tagId", DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL));

sortFields.add(new SortField(SearchFilter.TAG_DEF_ID, "obj.id", true, SortField.SORT_ORDER.ASC));
sortFields.add(new SortField(SearchFilter.TAG_RESOURCE_ID, "obj.resourceId"));
sortFields.add(new SortField(SearchFilter.TAG_ID, "obj.tagId"));
}

@Override
Expand Down
1 change: 1 addition & 0 deletions security-admin/src/main/java/org/apache/ranger/service/RangerTagService.java
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ public RangerTagService() {
searchFields.add(new SearchField(SearchFilter.TAG_DEF_ID, "obj.type", SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL));
searchFields.add(new SearchField(SearchFilter.TAG_TYPE, "tagDef.name", DATA_TYPE.STRING, SEARCH_TYPE.FULL, "XXTagDef tagDef", "obj.type = tagDef.id"));
searchFields.add(new SearchField(SearchFilter.TAG_TYPE_PARTIAL, "tagDef.name", DATA_TYPE.STRING, SEARCH_TYPE.PARTIAL, "XXTagDef tagDef", "obj.type = tagDef.id"));
searchFields.add(new SearchField(SearchFilter.TAG_IDS, "obj.id", SearchField.DATA_TYPE.INT_LIST, SearchField.SEARCH_TYPE.FULL));

sortFields.add(new SortField(SearchFilter.TAG_ID, "obj.id", true, SortField.SORT_ORDER.ASC));
sortFields.add(new SortField(SearchFilter.TAG_DEF_ID, "obj.type"));
Expand Down

0 comments on commit 48520f9

Please sign in to comment.