Skip to content

Commit

Permalink
Make message list fields sortable (Graylog2/graylog-plugin-enterprise…
Browse files Browse the repository at this point in the history 
…#926)

* Make field selection for MessageList sortable

Prior this change, sorting the fields of the MessagList
required to empty the field select and re add the fields
in the desired order.

With this change the selection box is sortable and the
fields can be ordered via drag and drop.

* Add timestamp to selection of message fields

* Message as a field

* Fix height calculation when not rendered in modal

* Fix fallout from rebase

Prior to this change, the constant variable of
a default message still contained "message" and was called wrongly.
Also fix PropType of TypeSpecificValue.

* Add migration for message list

The migration will add timestamp to the beginning of the fields
of a message widget and will remove message and set "show_message_row"
instead.

It also adds a new MessageList as a replacement for StaticMessageList.

* Fix test

* Fix linter warnings for MessageList

* Fix annotations from @dennis

* Remove StaticMessageList

* Add 'All Messages' Table for new Query Tab

* Switch from Bson fixtures to InMemoryDb

* Switch from constructor to builder in ShowDocumentsHandler

* Use parameter based casting instead of explicit

* finalize assignment of variables

* Catch possible errors while iterating over views

* Refactor createAllMessageWidget

* Fix actual eslint warning not just hiding it

* Enhance test structure

* Use word break for table fields
  • Loading branch information
@kmerz @dennisoelkers
kmerz authored and dennisoelkers committed Jun 11, 2019
1 parent a8bbab2 commit 49aa226
Show file tree
Hide file tree
Showing 23 changed files with 439 additions and 630 deletions.
209 changes: 209 additions & 0 deletions ...rg/graylog/plugins/enterprise/migrations/V20190304102700_MigrateMessageListStructure.java
View file
@@ -0,0 +1,209 @@
package org.graylog.plugins.enterprise.migrations;

import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.auto.value.AutoValue;
import com.mongodb.BasicDBObject;
import com.mongodb.client.FindIterable;
import org.bson.Document;
import org.bson.types.ObjectId;
import org.graylog.autovalue.WithBeanGetter;
import org.graylog2.database.MongoConnection;
import org.graylog2.migrations.Migration;

import com.mongodb.client.MongoCollection;
import org.graylog2.plugin.cluster.ClusterConfigService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.inject.Inject;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;

public class V20190304102700_MigrateMessageListStructure extends Migration {
private static final Logger LOG = LoggerFactory.getLogger(V20190304102700_MigrateMessageListStructure.class);

private final ClusterConfigService clusterConfigService;
private final MongoCollection<Document> viewsCollections;
private final MongoCollection<Document> searchCollections;

@Inject
public V20190304102700_MigrateMessageListStructure(final MongoConnection mongoConnection,
final ClusterConfigService clusterConfigService) {
this.viewsCollections = mongoConnection.getMongoDatabase().getCollection("views");
this.searchCollections = mongoConnection.getMongoDatabase().getCollection("searches");
this.clusterConfigService = clusterConfigService;
}

@Override
public ZonedDateTime createdAt() {
return ZonedDateTime.parse("2019-04-03T10:27:00Z");
}

@Override
public void upgrade() {
if (clusterConfigService.get(MigrationCompleted.class) != null) {
LOG.debug("Migration already completed.");
return;
}

final List<String> viewIds = new ArrayList<>();
final FindIterable<Document> documents = viewsCollections.find();
for (final Document view : documents) {
try {
final Document states = view.get("state", Document.class);
states.forEach((String id, Object obj) -> {
final Document state = (Document) obj;
if (state.get("widgets") instanceof List) {
@SuppressWarnings("unchecked") final List<Document> widgets = (List) state.get("widgets");
for (final Document widget : widgets) {
final String type = widget.getString("type");
if (type.equals("messages")) {
final Document config = widget.get("config", Document.class);
@SuppressWarnings("unchecked") final List<String> fields = (List) config.get("fields");
fields.add(0, "timestamp");
if (fields.contains("message")) {
config.put("show_message_row", true);
config.remove("message");
}
config.put("fields", fields);
}
}
}
createAllMessagesWidget(view, id, state);
});
viewsCollections.updateOne(new BasicDBObject("_id", view.getObjectId("_id")), new Document("$set", view));
final String viewId = view.getObjectId("_id").toString();
viewIds.add(viewId);
} catch (Exception e) {
final String viewId = view.getObjectId("_id").toString();
LOG.error("Could not mirgarte view with ID {}", viewId);
}
}

clusterConfigService.write(MigrationCompleted.create(viewIds));
}

private void createAllMessagesWidget(Document view, String stateId, Document state) {
final String widgetId = UUID.randomUUID().toString();

/* Preparations */
@SuppressWarnings("unchecked")
final List<String> selectedFields = (List) state.get("selected_fields");
selectedFields.add(0, "timestamp");
final boolean showMessageRow = selectedFields.contains("message");
if (showMessageRow) {
selectedFields.remove("message");
}

/* Set title */
final Document titles = state.get("titles", Document.class);
final Document widgetTitles = titles.get("widget", Document.class);
widgetTitles.put(widgetId, "All Messages");


/* Add widget */
@SuppressWarnings("unchecked")
final List<Document> widgets = (List) state.get("widgets");
final Document newMessageList = createMessageList(widgetId, selectedFields, showMessageRow);
widgets.add(newMessageList);

/* Add widget Position */
final Document positions = state.get("positions", Document.class);
final int newRow = findNewRow(positions);
Document widgetPosition = createWidgetPosition(newRow);
positions.put(widgetId, widgetPosition);

/* Add widget mapping */
final Document widgetMappings = state.get("widget_mapping", Document.class);
final List<String> widgetMappingSearchTypeIds = getWidgetMappingSearchTypeIds(widgetMappings);

String search_id = view.getString("search_id");
List<String> searchTypeId = findSearchTypIds(stateId, search_id, widgetMappingSearchTypeIds);

widgetMappings.put(widgetId, searchTypeId);
state.put("static_message_list_id", widgetId);
}

private Document createMessageList(String widgetId, List<String> fields, boolean showMessageRow) {
final Document newMessageList = new Document();
newMessageList.put("id", widgetId);
newMessageList.put("type", "messages");
final Document widgetConfig = new Document();
widgetConfig.put("fields", fields);
widgetConfig.put("show_message_row", showMessageRow);
newMessageList.put("config", widgetConfig);
return newMessageList;
}

private int findNewRow(Document positions) {
final Optional<Integer> maxRow = positions.values().stream()
.map(pos -> ((Document) pos).getInteger("height") + ((Document) pos).getInteger("row"))
.max(Comparator.comparingInt(Integer::intValue));
return maxRow.orElse(1);
}


private Document createWidgetPosition(int newRow) {
final Document widgetPosition = new Document();
widgetPosition.put("col", 1);
widgetPosition.put("row", newRow);
widgetPosition.put("width", Double.POSITIVE_INFINITY);
widgetPosition.put("height", 6);
return widgetPosition;
}

private List<String> getWidgetMappingSearchTypeIds(Document widgetMappings) {
final List<String> widgetMappingSearchTypeIds = new ArrayList<>();
for (final Map.Entry mapping : widgetMappings.entrySet()) {
@SuppressWarnings("unchecked")
final List<String> searchIds = (ArrayList) mapping.getValue();
widgetMappingSearchTypeIds.addAll(searchIds);
}
return widgetMappingSearchTypeIds;
}

private List<String> findSearchTypIds(String stateId, String searchId, List<String> widgetMappingSearchTypeIds) {
final BasicDBObject dbQuery = new BasicDBObject();
dbQuery.put("_id", new ObjectId(searchId));
final FindIterable<Document> searches = this.searchCollections.find(dbQuery);

/* There can be only one search with matching id */
assert this.searchCollections.count(dbQuery) == 1;
final Document search = searches.first();

final List<String> searchTypeId = new ArrayList<>();

@SuppressWarnings("unchecked")
final List<Document> queries = (ArrayList) search.get("queries");
for (final Document query : queries) {
if (query.getString("id").equals(stateId)) {
@SuppressWarnings("unchecked")
final List<Document> searchTypes = (ArrayList) query.get("search_types");
searchTypeId.addAll(searchTypes.stream().map(searchType -> searchType.getString("id"))
.filter(search_id -> !widgetMappingSearchTypeIds.contains(search_id)).collect(Collectors.toList()));
}
}
return searchTypeId;
}

@JsonAutoDetect
@AutoValue
@WithBeanGetter
public static abstract class MigrationCompleted {
@JsonProperty("viewIds")
public abstract List<String> viewIds();

@JsonCreator
public static MigrationCompleted create(@JsonProperty("viewIds") final List<String> viewIds) {
return new AutoValue_V20190304102700_MigrateMessageListStructure_MigrationCompleted(viewIds);
}
}
}
7 changes: 7 additions & 0 deletions enterprise/src/main/java/org/graylog/plugins/enterprise/search/views/ViewStateDTO.java
View file
Expand Up @@ -15,6 +15,7 @@
@WithBeanGetter
public abstract class ViewStateDTO {
static final String FIELD_SELECTED_FIELDS = "selected_fields";
static final String FIELD_STATIC_MESSAGE_LIST_ID = "static_message_list_id";
static final String FIELD_TITLES = "titles";
static final String FIELD_WIDGETS = "widgets";
static final String FIELD_WIDGET_MAPPING = "widget_mapping";
Expand All @@ -23,6 +24,9 @@ public abstract class ViewStateDTO {
@JsonProperty(FIELD_SELECTED_FIELDS)
public abstract Set<String> fields();

@JsonProperty(FIELD_STATIC_MESSAGE_LIST_ID)
public abstract String staticMessageListId();

@JsonProperty(FIELD_TITLES)
public abstract Map<String, Map<String, String>> titles();

Expand All @@ -40,6 +44,9 @@ public static abstract class Builder {
@JsonProperty(FIELD_SELECTED_FIELDS)
public abstract Builder fields(Set<String> fields);

@JsonProperty(FIELD_STATIC_MESSAGE_LIST_ID)
public abstract Builder staticMessageListId(String staticMessageListId);

@JsonProperty(FIELD_TITLES)
public abstract Builder titles(Map<String, Map<String, String>> titles);

Expand Down
7 changes: 3 additions & 4 deletions ...org/graylog/plugins/enterprise/search/views/widgets/messagelist/MessageListConfigDTO.java
View file
Expand Up @@ -5,10 +5,9 @@
import com.fasterxml.jackson.annotation.JsonTypeName;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.google.auto.value.AutoValue;
import com.google.common.collect.ImmutableSet;
import org.graylog.plugins.enterprise.search.views.WidgetConfigDTO;

import java.util.Set;

@AutoValue
@JsonTypeName(MessageListConfigDTO.NAME)
@JsonDeserialize(builder = MessageListConfigDTO.Builder.class)
Expand All @@ -18,15 +17,15 @@ public abstract class MessageListConfigDTO implements WidgetConfigDTO {
private static final String FIELD_SHOW_MESSAGE_ROW = "show_message_row";

@JsonProperty(FIELD_FIELDS)
public abstract Set<String> fields();
public abstract ImmutableSet<String> fields();

@JsonProperty(FIELD_SHOW_MESSAGE_ROW)
public abstract boolean showMessageRow();

@AutoValue.Builder
public abstract static class Builder {
@JsonProperty(FIELD_FIELDS)
public abstract Builder fields(Set<String> fields);
public abstract Builder fields(ImmutableSet<String> fields);

@JsonProperty(FIELD_SHOW_MESSAGE_ROW)
public abstract Builder showMessageRow(boolean showMessageRow);
Expand Down
119 changes: 119 additions & 0 deletions ...graylog/plugins/enterprise/migration/V20190304102700_MigrateMessageListStructureTest.java
View file
@@ -0,0 +1,119 @@
package org.graylog.plugins.enterprise.migration;

import com.lordofthejars.nosqlunit.annotation.UsingDataSet;
import com.lordofthejars.nosqlunit.core.LoadStrategyEnum;
import com.lordofthejars.nosqlunit.mongodb.InMemoryMongoDb;
import com.mongodb.BasicDBObject;
import com.mongodb.client.FindIterable;
import com.mongodb.client.MongoCollection;
import org.bson.Document;
import org.bson.types.ObjectId;
import org.graylog.plugins.database.MongoConnectionRule;
import org.graylog.plugins.enterprise.migrations.V20190304102700_MigrateMessageListStructure;
import org.graylog2.migrations.Migration;
import org.graylog2.plugin.cluster.ClusterConfigService;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnit;
import org.mockito.junit.MockitoRule;

import java.time.ZonedDateTime;
import java.util.List;

import static com.lordofthejars.nosqlunit.mongodb.InMemoryMongoDb.InMemoryMongoRuleBuilder.newInMemoryMongoDbRule;
import static org.assertj.core.api.Assertions.assertThat;

public class V20190304102700_MigrateMessageListStructureTest {

@ClassRule
public static final InMemoryMongoDb IN_MEMORY_MONGO_DB = newInMemoryMongoDbRule().build();
@Rule
public final MongoConnectionRule mongoRule = MongoConnectionRule.build("test");

private Migration migration;

@Rule
public final MockitoRule mockitoRule = MockitoJUnit.rule();
@Mock
private ClusterConfigService clusterConfigService;

@Before
public void setUp() throws Exception {
migration = new V20190304102700_MigrateMessageListStructure(mongoRule.getMongoConnection(), clusterConfigService);
}

@Test
public void createdAt() {
// Test the date to detect accidental changes to it.
assertThat(migration.createdAt()).isEqualTo(ZonedDateTime.parse("2019-04-03T10:27:00Z"));
}

@Test
@UsingDataSet(loadStrategy = LoadStrategyEnum.CLEAN_INSERT)
public void testMigratingViewStructure() {
final BasicDBObject dbQuery1 = new BasicDBObject();
dbQuery1.put("_id", new ObjectId("58458e442f857c314491344e"));
final MongoCollection<Document> collection = mongoRule.getMongoConnection()
.getMongoDatabase()
.getCollection("views");

migration.upgrade();

final FindIterable<Document> views = collection.find(dbQuery1);
final Document view1 = views.first();

@SuppressWarnings("unchecked")
final List<Document> widgets1 = (List) view1.get("state", Document.class).get("a2a804b7-27cf-4cac-8015-58d9a9640d33", Document.class).get("widgets");
assertThat(widgets1.size()).isEqualTo(2);
assertThat(widgets1.stream().filter(widget -> widget.getString("type").equals("messages")).count()).isEqualTo(1);
assertThat(widgets1.stream().filter(widget -> widget.getString("type").equals("messages")).allMatch((widget) -> {
final Document config = widget.get("config", Document.class);
@SuppressWarnings("unchecked")
final List<String> fields = (List) config.get("fields");
final boolean startWithTimestamp = fields.get(0).contains("timestamp");
final boolean showMessageRow = config.getBoolean("show_message_row");
return startWithTimestamp && showMessageRow;
})).isTrue();

final BasicDBObject dbQuery2 = new BasicDBObject();
dbQuery2.put("_id", new ObjectId("58458e442f857c314491344f"));

final FindIterable<Document> views2 = collection.find(dbQuery2);
final Document view2 = views2.first();

final Document states = view2.get("state", Document.class);
assertThat(states.values().size()).isEqualTo(13);
assertThat(states.keySet()).containsExactly(
"7c042319-530a-41b9-9dbb-9676fb1da1a4",
"9e5144be-a445-4289-a4cc-0f55142524bc",
"c13b2482-60e7-4b1e-98c9-0df8d6da8230",
"5adc9297-dfc8-4fd9-b422-cbb097715a62",
"ade8c853-503c-407f-b125-efbe2d368973",
"cc2bf983-b398-4295-bf01-1c10ed1a97e1",
"64feccae-9447-40ef-a401-79a7972078a2",
"7c7e04c6-f9f0-495c-91cc-865f60687f8c",
"eeaa8838-616f-40c0-88c0-1059ac64f37e",
"91c6f8c9-024c-48ec-a869-90548fad218a",
"955a71f2-673a-4e1c-a99f-ef97b1b4ae71",
"343ff7b6-4554-49d4-bc0b-1339fdc5dac0",
"7a84d053-e40a-48c1-a433-97521f7ce7ef");

states.values().forEach(state -> {
@SuppressWarnings("unchecked")
final List<Document> widgets2 = (List) ((Document) state).get("widgets");
assertThat(widgets2.stream().filter(widget -> widget.getString("type").equals("messages")).count()).isGreaterThan(0);
widgets2.stream().filter(widget -> widget.getString("type").equals("messages")).forEach((widget) -> {
final Document config = widget.get("config", Document.class);
@SuppressWarnings("unchecked")
final List<String> fields = (List) config.get("fields");
final boolean startWithTimestamp = fields.get(0).contains("timestamp");
final boolean showMessageRow = config.getBoolean("show_message_row");
assertThat(startWithTimestamp).isTrue();
assertThat(showMessageRow).isTrue();
});
});
}
}
2 changes: 1 addition & 1 deletion enterprise/src/web/enterprise/Constants.js
View file
@@ -1,7 +1,7 @@
// @flow strict

export const TIMESTAMP_FIELD = 'timestamp';
export const DEFAULT_MESSAGE_FIELDS = ['message', 'source'];
export const DEFAULT_MESSAGE_FIELDS = [TIMESTAMP_FIELD, 'source'];
export const Messages = {
DEFAULT_LIMIT: 150,
};

0 comments on commit 49aa226

Please sign in to comment.