Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

add sidecar filebeat collector for windows again #19325

Merged
merged 2 commits into from
May 13, 2024
Merged

add sidecar filebeat collector for windows again #19325

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7f12341
add sidecar filebeat collector for windows
AntonEbel May 13, 2024
882a92c
add changelog
AntonEbel May 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions changelog/unreleased/issue-19195.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
type = "f"
message = "Add sidecar filebeat collector for windows again."

issues = ["19195"]
pulls = ["19325"]
180 changes: 110 additions & 70 deletions ...ain/java/org/graylog/plugins/sidecar/migrations/V20180212165000_AddDefaultCollectors.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import com.mongodb.client.FindIterable;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.result.UpdateResult;
import jakarta.inject.Inject;
import org.bson.Document;
import org.bson.types.ObjectId;
import org.graylog.autovalue.WithBeanGetter;
Expand All @@ -41,9 +42,6 @@
import org.slf4j.LoggerFactory;

import javax.annotation.Nullable;

import jakarta.inject.Inject;

import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.time.ZonedDateTime;
Expand Down Expand Up @@ -141,65 +139,110 @@ private void ensureFilebeatCollectorsAndConfig() {
fields:
event_source_product: apache_httpd""";

ensureFilebeatCollector(OS_LINUX, "/usr/lib/graylog-sidecar/filebeat", filebeatConfigBuilder
.append("""

- type: filestream
id: snort-filestream
enabled: true
paths:
- /var/log/snort/alert_json.txt
- /var/log/snort/appid-output.json
parsers:
- ndjson:
target: "snort3"
add_error_key: true
overwrite_keys: true
fields:
event_source_product: snort3
""")
.append("""

- type: filestream
id: zeek-filestream
enabled: true
paths:
- /opt/zeek/logs/current
parsers:
- ndjson:
target: "zeek"
add_error_key: true
overwrite_keys: true
fields:
event_source_product: zeek
""")
.append(f(apacheConfigType, """
paths:
- /var/log/apache2/access.log
- /var/log/apache2/error.log
- /var/log/httpd/access_log
- /var/log/httpd/error_log
"""))
.toString()
ensureCollector(
"filebeat",
"exec",
OS_LINUX,
"/usr/lib/graylog-sidecar/filebeat",
"-c %s",
"test config -c %s",
filebeatConfigBuilder
.append("""

- type: filestream
id: snort-filestream
enabled: true
paths:
- /var/log/snort/alert_json.txt
- /var/log/snort/appid-output.json
parsers:
- ndjson:
target: "snort3"
add_error_key: true
overwrite_keys: true
fields:
event_source_product: snort3
""")
.append("""

- type: filestream
id: zeek-filestream
enabled: true
paths:
- /opt/zeek/logs/current
parsers:
- ndjson:
target: "zeek"
add_error_key: true
overwrite_keys: true
fields:
event_source_product: zeek
""")
.append(f(apacheConfigType, """
paths:
- /var/log/apache2/access.log
- /var/log/apache2/error.log
- /var/log/httpd/access_log
- /var/log/httpd/error_log
"""))
.toString()
).ifPresent(collector -> ensureDefaultConfiguration("filebeat-linux-default", collector));

ensureFilebeatCollector(OS_FREEBSD, "/usr/share/filebeat/bin/filebeat", filebeatConfigBuilder
.append(f(apacheConfigType, """
paths:
- /var/log/httpd-access.log
- /var/log/httpd-error.log
"""))
.toString()
ensureCollector(
"filebeat",
"exec",
OS_FREEBSD,
"/usr/share/filebeat/bin/filebeat",
"-c %s",
"test config -c %s",
filebeatConfigBuilder
.append(f(apacheConfigType, """
paths:
- /var/log/httpd-access.log
- /var/log/httpd-error.log
"""))
.toString()
).ifPresent(collector -> ensureDefaultConfiguration("filebeat-freebsd-default", collector));

ensureFilebeatCollector(OS_DARWIN, "/usr/share/filebeat/bin/filebeat", filebeatConfigBuilder
.append(f(apacheConfigType, """
paths:
- /etc/httpd/log/access_log
- /etc/httpd/log/error_log
"""))
.toString()
ensureCollector(
"filebeat",
"exec",
OS_DARWIN,
"/usr/share/filebeat/bin/filebeat",
"-c %s",
"test config -c %s",
filebeatConfigBuilder
.append(f(apacheConfigType, """
paths:
- /etc/httpd/log/access_log
- /etc/httpd/log/error_log
"""))
.toString()
).ifPresent(collector -> ensureDefaultConfiguration("filebeat-darwin-default", collector));

ensureCollector(
"filebeat",
"svc",
OS_WINDOWS,
"C:\\Program Files\\Graylog\\sidecar\\filebeat.exe",
"-c \"%s\"",
"test config -c \"%s\"",
f("""
%s
output.logstash:
hosts: ["${user.graylog_host}:5044"]
path:
data: ${sidecar.spoolDir!\"C:\\\\Program Files\\\\Graylog\\\\sidecar\\\\cache\\\\filebeat\"}\\data
logs: ${sidecar.spoolDir!\"C:\\\\Program Files\\\\Graylog\\\\sidecar\"}\\logs
tags:
- windows
filebeat.inputs:
- type: log
enabled: true
paths:
- C:\\logs\\log.log
""",
BEATS_PREAMBEL));
}

private void ensureAuditbeatCollectorsAndConfig() {
Expand Down Expand Up @@ -490,19 +533,6 @@ private void ensureNxLogCollectors() {
);
}


private Optional<Collector> ensureFilebeatCollector(String operatingSystem, String executablePath, String config) {
return ensureCollector(
"filebeat",
"exec",
operatingSystem,
executablePath,
"-c %s",
"test config -c %s",
config
);
}

private void removeConfigPath() {
final FindIterable<Document> documentsWithConfigPath = collection.find(exists("configuration_path"));
for (Document document : documentsWithConfigPath) {
Expand Down Expand Up @@ -661,6 +691,14 @@ public static abstract class MigrationState {
// !!! Extending this list does not work. It was persisted after the first migration run.
);

private static final Set<CollectorChecksum> OLD_FILEBEAT_WINDOWS = java.util.Set.of(
new CollectorChecksum("filebeat", OS_WINDOWS, 1490581247L),
new CollectorChecksum("filebeat", OS_WINDOWS, 3884777971L),
new CollectorChecksum("filebeat", OS_WINDOWS, 341507103L),
new CollectorChecksum("filebeat", OS_WINDOWS, 1931248542L),
new CollectorChecksum("filebeat", OS_WINDOWS, 2559816928L)
);

@JsonProperty("knownChecksums")
public abstract Set<CollectorChecksum> knownChecksums();

Expand All @@ -678,6 +716,8 @@ public MigrationState withNewDefaultTemplate(String collectorName, String platfo
public static MigrationState create(@JsonProperty("knownChecksums") Set<CollectorChecksum> knownChecksums) {
var merged = new HashSet<>(knownChecksums);
merged.addAll(OLD_CHECKSUMS);
merged.addAll(OLD_FILEBEAT_WINDOWS);


return new AutoValue_V20180212165000_AddDefaultCollectors_MigrationState(merged);
}
Expand Down
Loading