Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

checkpoint POC for smaller GetQueryDetails.api response #5141

Draft
wants to merge 1 commit into
base: develop
Choose a base branch
from
Draft

checkpoint POC for smaller GetQueryDetails.api response #5141

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
80 changes: 75 additions & 5 deletions api/src/org/labkey/api/data/JsonWriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,13 @@
import org.labkey.data.xml.queryCustomView.FilterType;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;

/**
Expand All @@ -52,7 +54,21 @@ public class JsonWriter
{
private static final Logger LOG = LogManager.getLogger(JsonWriter.class);

public static Map<FieldKey, Map<String,Object>> getNativeColProps(TableInfo tableInfo, Collection<FieldKey> fields, FieldKey fieldKeyPrefix, boolean includeDomainFormat, boolean includeAdditionalQueryColumns)
public static Map<String,Object> getTemplateColProps(TableInfo tableInfo)
{
ColumnInfo col = new BaseColumnInfo("__template", tableInfo, JdbcType.VARCHAR);
DisplayColumn dc = col.getDisplayColumnFactory().createRenderer(col);
Map<String,Object> template = getNativeColProps(List.of(dc), null, false).get(col.getFieldKey());
template.remove("name");
template.remove("fieldKey");
template.remove("fieldKeyArray");
template.remove("fieldKeyPath");
template.remove("shortCaption");
template.remove("caption");
return template;
}

public static Map<FieldKey, Map<String,Object>> getNativeColProps(TableInfo tableInfo, Collection<FieldKey> fields, FieldKey fieldKeyPrefix, @Nullable Map<String,Object> template, boolean includeDomainFormat, boolean includeAdditionalQueryColumns)
{
List<DisplayColumn> displayColumns = QueryService.get().getColumns(tableInfo, fields, tableInfo.getColumns())
.values()
Expand All @@ -61,10 +77,15 @@ public static Map<FieldKey, Map<String,Object>> getNativeColProps(TableInfo tabl
.map(cinfo -> cinfo.getDisplayColumnFactory().createRenderer(cinfo))
.collect(Collectors.toList());

return getNativeColProps(displayColumns, fieldKeyPrefix, includeDomainFormat);
return getNativeColProps(displayColumns, fieldKeyPrefix, template, includeDomainFormat);
}

public static Map<FieldKey, Map<String,Object>> getNativeColProps(Collection<DisplayColumn> columns, FieldKey fieldKeyPrefix, boolean includeDomainFormat)
{
return getNativeColProps(columns, fieldKeyPrefix, null, includeDomainFormat);
}

public static Map<FieldKey, Map<String,Object>> getNativeColProps(Collection<DisplayColumn> columns, FieldKey fieldKeyPrefix, @Nullable Map<String,Object> template, boolean includeDomainFormat)
{
Map<FieldKey, Map<String,Object>> colProps = new LinkedHashMap<>();
for (DisplayColumn displayColumn : columns)
Expand All @@ -76,15 +97,53 @@ public static Map<FieldKey, Map<String,Object>> getNativeColProps(Collection<Dis
else
fieldKey = new FieldKey(fieldKeyPrefix, displayColumn.getName());

colProps.put(fieldKey, JsonWriter.getMetaData(displayColumn, fieldKeyPrefix, true, true, includeDomainFormat));
colProps.put(fieldKey, JsonWriter.getMetaData(displayColumn, fieldKeyPrefix, template, true, true, includeDomainFormat));
}
return colProps;
}

public static Map<String, Object> getMetaData(DisplayColumn dc, FieldKey fieldKeyPrefix, boolean useFriendlyAsType, boolean includeLookup, boolean includeDomainFormat)
{
return getMetaData(dc, fieldKeyPrefix, null, useFriendlyAsType, includeLookup, includeDomainFormat );
}


// use helper map class to avoid making the code in getMetaData() less readable */
static class TemplateMap
{
final Map<String,Object> _template;
final LinkedHashMap<String,Object> _map;
TemplateMap(Map<String, Object> template)
{
_template = null == template || template.isEmpty() ? Map.of() : template;
_map = new LinkedHashMap<String,Object>();
}

void put(String key, Object value)
{
if (!_template.containsKey(key) || !Objects.equals(_template.get(key), value))
_map.put(key, value);
}

Map<String,Object> getMap()
{
return _map;
}
}

/** put property into props map if value differs from template */
void templatePut(Map<String, Object> props, String key, Object value, Map<String, Object> template)
{
if (template == null || !template.containsKey(key) || Objects.equals(template.get(key), value))
props.put(key, value);
}

public static Map<String, Object> getMetaData(DisplayColumn dc, FieldKey fieldKeyPrefix, @Nullable Map<String,Object> template, boolean useFriendlyAsType, boolean includeLookup, boolean includeDomainFormat)
{
/* there are already a lot of parameters so use null!=template to indicate that we are "compression" the response */
boolean compressed = null != template;
@Nullable ColumnInfo cinfo = dc.getColumnInfo();
Map<String, Object> props = new LinkedHashMap<>();
var props = new TemplateMap(template);
JSONObject ext = new JSONObject();
props.put("ext", ext);

Expand Down Expand Up @@ -315,7 +374,18 @@ else if (cinfo.getFacetingBehaviorType() != null)
props.put("conceptLabelColumn", cinfo.getConceptLabelColumn());
}

return props;
var ret = props.getMap();
if (compressed)
{
if (ret.get("ext") instanceof JSONObject extProp && extProp.isEmpty())
ret.remove("ext");
for (String key : List.of("fieldKey", "fieldKeyPath", "shortCaption", "caption"))
{
if (name.equals(ret.get(key)))
ret.remove(key);
}
}
return ret;
}

@Nullable
Expand Down
37 changes: 27 additions & 10 deletions query/src/org/labkey/query/CustomViewUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ public static Map<String, Object> toMap(ViewContext context, UserSchema schema,
}
}

Map<String, Object> ret = toMap(view, context.getUser(), includeFieldMeta, columnMetadata);
Map<String, Object> ret = toMap(view, context.getUser(), includeFieldMeta, false, columnMetadata);
if (newView)
ret.put("doesNotExist", true);

Expand All @@ -191,10 +191,10 @@ public static Map<String, Object> toMap(ViewContext context, UserSchema schema,
public static Map<String, Object> toMap(CustomView view, @NotNull User user, boolean includeFieldMeta)
{
assert user != null;
return toMap(view, user, includeFieldMeta, new HashMap<>());
return toMap(view, user, includeFieldMeta, false, new HashMap<>());
}

public static Map<String, Object> toMap(CustomView view, @NotNull User user, boolean includeFieldMeta, Map<FieldKey, Map<String, Object>> columnMetadata)
public static Map<String, Object> toMap(CustomView view, @NotNull User user, boolean includeFieldMeta, boolean compressed, Map<FieldKey, Map<String, Object>> columnMetadata)
{
assert user != null;
Map<String, Object> ret = QueryService.get().getCustomViewProperties(view, user);
Expand Down Expand Up @@ -239,14 +239,31 @@ public static Map<String, Object> toMap(CustomView view, @NotNull User user, boo
allKeys.add(entry.getKey());

Map<String, Object> colInfo = new LinkedHashMap<>();
colInfo.put("name", entry.getKey().getName());
colInfo.put("key", entry.getKey().toString());
colInfo.put("fieldKey", entry.getKey().toString());
if (!entry.getValue().isEmpty())
var name = entry.getKey().getName();
colInfo.put("name", name);
if (compressed)
{
String columnTitle = entry.getValue().get(CustomViewInfo.ColumnProperty.columnTitle);
if (columnTitle != null)
colInfo.put("title", columnTitle);
var fieldkey = entry.getKey().toString();
if (!Objects.equals(name, fieldkey))
colInfo.put("fieldkey", fieldkey);
if (!entry.getValue().isEmpty())
{
String columnTitle = entry.getValue().get(CustomViewInfo.ColumnProperty.columnTitle);
if (columnTitle != null && !Objects.equals(columnTitle, name))
colInfo.put("title", columnTitle);
}
}
else
{
colInfo.put("name", entry.getKey().getName());
colInfo.put("key", entry.getKey().toString());
colInfo.put("fieldKey", entry.getKey().toString());
if (!entry.getValue().isEmpty())
{
String columnTitle = entry.getValue().get(CustomViewInfo.ColumnProperty.columnTitle);
if (columnTitle != null)
colInfo.put("title", columnTitle);
}
}
colInfos.add(colInfo);
}
Expand Down
19 changes: 19 additions & 0 deletions query/src/org/labkey/query/controllers/GetQueryDetails2Action.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package org.labkey.query.controllers;

import org.labkey.api.action.Action;
import org.labkey.api.action.ActionType;
import org.labkey.api.action.ApiResponse;
import org.labkey.api.security.RequiresPermission;
import org.labkey.api.security.permissions.ReadPermission;
import org.springframework.validation.BindException;

@RequiresPermission(ReadPermission.class)
@Action(ActionType.SelectMetaData.class)
public class GetQueryDetails2Action extends GetQueryDetailsAction
{
@Override
public ApiResponse execute(Form form, BindException errors)
{
return _execute(form, true);
}
}
33 changes: 22 additions & 11 deletions query/src/org/labkey/query/controllers/GetQueryDetailsAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,12 @@ protected long getLastModified(Form form)
return QueryService.get().metadataLastModified();
}

@Override
public ApiResponse execute(Form form, BindException errors)
@Override public ApiResponse execute(Form form, BindException errors)
{
return _execute(form, false);
}

protected ApiResponse _execute(Form form, boolean compressed)
{
ApiSimpleResponse resp = new ApiSimpleResponse();

Expand Down Expand Up @@ -223,6 +227,13 @@ public ApiResponse execute(Form form, BindException errors)

Map<FieldKey, Map<String, Object>> columnMetadata;

Map<String,Object> templateColumn = null;
if (compressed)
{
templateColumn = JsonWriter.getTemplateColProps(tinfo);
resp.put("templateColumn", templateColumn);
}

//if the caller asked us to chase a foreign key, do that. Note that any call to get a lookup table can throw a
// QueryParseException, so we wrap all FK accesses in a try/catch.
try
Expand Down Expand Up @@ -270,7 +281,7 @@ public ApiResponse execute(Form form, BindException errors)

//now the native columns plus any additional fields requested. Use a copy of the values
// in the reference columnMetadata map, as it may get additional entries later when the view columns are serialized
columnMetadata = JsonWriter.getNativeColProps(tinfo, fields, fk, false, form.isIncludeSuggestedQueryColumns());
columnMetadata = JsonWriter.getNativeColProps(tinfo, fields, fk, templateColumn, false, form.isIncludeSuggestedQueryColumns());
resp.put("columns", new ArrayList<>(columnMetadata.values()));

// table indices
Expand Down Expand Up @@ -316,7 +327,7 @@ public ApiResponse execute(Form form, BindException errors)
{
//now the columns in the user's default view for this query
QueryView qview = new QueryView(schema, settings, null);
resp.put("defaultView", getDefaultViewProps(qview));
resp.put("defaultView", getDefaultViewProps(qview, templateColumn));

List<Map<String, Object>> viewInfos = new ArrayList<>();
Map<String, CustomView> allViews = queryDef.getCustomViews(getUser(), getViewContext().getRequest(), true, false);
Expand All @@ -331,11 +342,11 @@ public ApiResponse execute(Form form, BindException errors)
{
if (cv.getName() == null)
hasDefault = true;
viewInfos.add(CustomViewUtil.toMap(cv, getUser(), true, columnMetadata));
viewInfos.add(CustomViewUtil.toMap(cv, getUser(), true, compressed, columnMetadata));
}

if (!hasDefault)
viewInfos.add(CustomViewUtil.toMap(queryDef.createCustomView(getUser(), null), getUser(), true, columnMetadata));
viewInfos.add(CustomViewUtil.toMap(queryDef.createCustomView(getUser(), null), getUser(), true, compressed, columnMetadata));
}
else
{
Expand All @@ -348,7 +359,7 @@ public ApiResponse execute(Form form, BindException errors)
if (null == cv && (viewName == null || form.isInitializeMissingView()))
cv = queryDef.createCustomView(getUser(), viewName);
if (null != cv)
viewInfos.add(CustomViewUtil.toMap(cv, getUser(), true, columnMetadata));
viewInfos.add(CustomViewUtil.toMap(cv, getUser(), true, compressed, columnMetadata));
}
}

Expand Down Expand Up @@ -424,20 +435,20 @@ public ApiResponse execute(Form form, BindException errors)
return resp;
}

protected Map<String,Object> getDefaultViewProps(QueryView view)
protected Map<String,Object> getDefaultViewProps(QueryView view, Map<String,Object> columnTemplate)
{
Map<String,Object> defViewProps = new HashMap<>();
defViewProps.put("columns", getDefViewColProps(view));
defViewProps.put("columns", getDefViewColProps(view, columnTemplate));
return defViewProps;
}

protected List<Map<String,Object>> getDefViewColProps(QueryView view)
protected List<Map<String,Object>> getDefViewColProps(QueryView view, Map<String,Object> columnTemplate)
{
List<Map<String,Object>> colProps = new ArrayList<>();
for (DisplayColumn dc : view.getDisplayColumns())
{
if (dc.isQueryColumn() && null != dc.getColumnInfo())
colProps.add(JsonWriter.getMetaData(dc, null, true, true, false));
colProps.add(JsonWriter.getMetaData(dc, null, columnTemplate, true, true, false));
}
return colProps;
}
Expand Down
3 changes: 2 additions & 1 deletion query/src/org/labkey/query/controllers/QueryController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,7 @@ public class QueryController extends SpringActionController
ValidateQueriesAction.class,
GetSchemaQueryTreeAction.class,
GetQueryDetailsAction.class,
GetQueryDetails2Action.class,
ViewQuerySourceAction.class
);

Expand Down Expand Up @@ -6867,7 +6868,7 @@ protected Map<String, Object> getQueryProps(QueryDefinition qdef, ActionURL view
if (useQueryDetailColumns)
{
columns = JsonWriter
.getNativeColProps(table, Collections.emptyList(), null, false, false)
.getNativeColProps(table, Collections.emptyList(), null, null, false, false)
.values();
}
else
Expand Down