/
ElasticSearchOutput.java
151 lines (130 loc) · 5.53 KB
/
ElasticSearchOutput.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
/**
* This file is part of Graylog.
*
* Graylog is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Graylog is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Graylog. If not, see <http://www.gnu.org/licenses/>.
*/
package org.graylog2.outputs;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import com.google.common.base.Joiner;
import com.google.common.collect.Ordering;
import com.google.inject.assistedinject.Assisted;
import com.google.inject.assistedinject.AssistedInject;
import org.graylog2.indexer.IndexSet;
import org.graylog2.indexer.messages.Messages;
import org.graylog2.plugin.Message;
import org.graylog2.plugin.configuration.Configuration;
import org.graylog2.plugin.configuration.ConfigurationRequest;
import org.graylog2.plugin.outputs.MessageOutput;
import org.graylog2.plugin.streams.Stream;
import org.graylog2.shared.journal.Journal;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
import static com.codahale.metrics.MetricRegistry.name;
public class ElasticSearchOutput implements MessageOutput {
private static final String WRITES_METRICNAME = name(ElasticSearchOutput.class, "writes");
private static final String PROCESS_TIME_METRICNAME = name(ElasticSearchOutput.class, "processTime");
private static final String NAME = "ElasticSearch Output";
private static final Logger LOG = LoggerFactory.getLogger(ElasticSearchOutput.class);
private final Meter writes;
private final Timer processTime;
private final Messages messages;
private final Journal journal;
private final AtomicBoolean isRunning = new AtomicBoolean(false);
@AssistedInject
public ElasticSearchOutput(MetricRegistry metricRegistry,
Messages messages,
Journal journal,
@Assisted Stream stream,
@Assisted Configuration configuration) {
this(metricRegistry, messages, journal);
}
@Inject
public ElasticSearchOutput(MetricRegistry metricRegistry,
Messages messages,
Journal journal) {
this.messages = messages;
this.journal = journal;
// Only constructing metrics here. write() get's another Core reference. (because this technically is a plugin)
this.writes = metricRegistry.meter(WRITES_METRICNAME);
this.processTime = metricRegistry.timer(PROCESS_TIME_METRICNAME);
// Should be set in initialize once this becomes a real plugin.
isRunning.set(true);
}
@Override
public void write(Message message) throws Exception {
if (LOG.isTraceEnabled()) {
LOG.trace("Writing message id to [{}]: <{}>", NAME, message.getId());
}
write(Collections.singletonList(message));
}
@Override
public void write(List<Message> messageList) throws Exception {
throw new UnsupportedOperationException("Method not supported!");
}
public void writeMessageEntries(List<Map.Entry<IndexSet, Message>> messageList) throws Exception {
if (LOG.isTraceEnabled()) {
final List<String> sortedIds = Ordering.natural().sortedCopy(messageList.stream()
.map(entry -> entry.getValue().getId())
.collect(Collectors.toList()));
LOG.trace("Writing message ids to [{}]: <{}>", NAME, Joiner.on(", ").join(sortedIds));
}
writes.mark(messageList.size());
try (final Timer.Context ignored = processTime.time()) {
messages.bulkIndex(messageList);
}
for (final Map.Entry<IndexSet, Message> entry : messageList) {
journal.markJournalOffsetCommitted(entry.getValue().getJournalOffset());
}
}
@Override
public void stop() {
// TODO: Move ES stop code here.
//isRunning.set(false);
}
@Override
public boolean isRunning() {
return isRunning.get();
}
public interface Factory extends MessageOutput.Factory<ElasticSearchOutput> {
@Override
ElasticSearchOutput create(Stream stream, Configuration configuration);
@Override
Config getConfig();
@Override
Descriptor getDescriptor();
}
public static class Config extends MessageOutput.Config {
@Override
public ConfigurationRequest getRequestedConfiguration() {
// Built in output. This is just for plugin compat. No special configuration required.
return new ConfigurationRequest();
}
}
public static class Descriptor extends MessageOutput.Descriptor {
public Descriptor() {
super("Elasticsearch Output", false, "", "Elasticsearch Output");
}
public Descriptor(String name, boolean exclusive, String linkToDocs, String humanName) {
super(name, exclusive, linkToDocs, humanName);
}
}
}