diff --git a/data-node/pom.xml b/data-node/pom.xml index 0ab443497e08..9e20ab3ba221 100644 --- a/data-node/pom.xml +++ b/data-node/pom.xml @@ -671,6 +671,34 @@ exec-maven-plugin org.codehaus.mojo + + generate-csv-docs + prepare-package + + java + + + org.graylog.datanode.docs.ConfigurationDocsGenerator + + csv + ${project.build.directory}/datanode-conf-docs.csv + + + + + generate-conf-example + prepare-package + + java + + + org.graylog.datanode.docs.ConfigurationDocsGenerator + + conf + ${project.build.directory}/datanode.conf.example + + + install-required-opensearch-plugins-x64 prepare-package diff --git a/data-node/src/main/assembly/datanode.xml b/data-node/src/main/assembly/datanode.xml index 8c2dc9fca0e7..d6a1757f4398 100644 --- a/data-node/src/main/assembly/datanode.xml +++ b/data-node/src/main/assembly/datanode.xml @@ -75,7 +75,7 @@ . - ${project.basedir}/../misc/datanode.conf + ${project.build.directory}/datanode.conf.example datanode.conf.example . diff --git a/data-node/src/main/java/org/graylog/datanode/Configuration.java b/data-node/src/main/java/org/graylog/datanode/Configuration.java index 91065ce5dcfb..d9ce175f05e6 100644 --- a/data-node/src/main/java/org/graylog/datanode/Configuration.java +++ b/data-node/src/main/java/org/graylog/datanode/Configuration.java @@ -32,8 +32,10 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.net.InetAddresses; import org.graylog.datanode.configuration.DatanodeDirectories; +import org.graylog.datanode.docs.DocumentationConstants; import org.graylog2.Configuration.SafeClassesValidator; import org.graylog2.configuration.Documentation; +import org.graylog2.configuration.DocumentationSection; import org.graylog2.plugin.Tools; import org.graylog2.shared.SuppressForbidden; import org.slf4j.Logger; @@ -57,7 +59,9 @@ * Helper class to hold configuration of DataNode */ @SuppressWarnings("FieldMayBeFinal") +@DocumentationSection(heading = "GRAYLOG DATANODE CONFIGURATION FILE", description = DocumentationConstants.DATANODE_DOCUMENTATION_DESCRIPTION) public class Configuration { + private static final Logger LOG = LoggerFactory.getLogger(Configuration.class); public static final String TRANSPORT_CERTIFICATE_PASSWORD_PROPERTY = "transport_certificate_password"; public static final String HTTP_CERTIFICATE_PASSWORD_PROPERTY = "http_certificate_password"; @@ -86,8 +90,10 @@ public class Configuration { @Parameter(value = "opensearch_location") private String opensearchDistributionRoot = "dist"; - @Documentation("Data directory of the embedded opensearch. Contains indices of the opensearch. May be pointed to an existing" + - "opensearch directory during in-place migration to Datanode") + @Documentation(""" + Data directory of the embedded opensearch. Contains indices of the opensearch. + May be pointed to an existing opensearch directory during in-place migration to Datanode + """) @Parameter(value = "opensearch_data_location", required = true) private Path opensearchDataLocation = Path.of("datanode/data"); @@ -95,8 +101,11 @@ public class Configuration { @Parameter(value = "opensearch_logs_location", required = true, validators = DirectoryWritableValidator.class) private Path opensearchLogsLocation = Path.of("datanode/logs"); - @Documentation("Configuration directory of the embedded opensearch. This is the directory where the opensearch" + - "process will store its configuration files. Caution, each start of the Datanode will regenerate the complete content of the directory!") + @Documentation(""" + Configuration directory of the embedded opensearch. This is the directory where the opensearch + process will store its configuration files. Caution, each start of the Datanode will regenerate + the complete content of the directory! + """) @Parameter(value = "opensearch_config_location", required = true, validators = DirectoryWritableValidator.class) private Path opensearchConfigLocation = Path.of("datanode/config"); @@ -113,7 +122,10 @@ public class Configuration { private Integer opensearchProcessLogsBufferSize = 500; - @Documentation("Unique name of this Datanode instance. use this, if your node name should be different from the hostname that's found by programmatically looking it up") + @Documentation(""" + Unique name of this Datanode instance. use this, if your node name should be different from the hostname + that's found by programmatically looking it up. + """) @Parameter(value = "node_name") private String datanodeNodeName; @@ -122,7 +134,10 @@ public class Configuration { @Parameter(value = "initial_cluster_manager_nodes") private String initialClusterManagerNodes; - @Documentation("Opensearch heap memory. Initial and maxmium heap must be identical for OpenSearch, otherwise the boot fails. So it's only one config option") + @Documentation(""" + Opensearch heap memory. Initial and maxmium heap must be identical for OpenSearch, otherwise the boot fails. + So it's only one config option. + """) @Parameter(value = "opensearch_heap") private String opensearchHeap = "1g"; @@ -138,7 +153,10 @@ public class Configuration { @Parameter(value = "opensearch_discovery_seed_hosts", converter = StringListConverter.class) private List opensearchDiscoverySeedHosts = Collections.emptyList(); - @Documentation("Binds an OpenSearch node to an address. Use 0.0.0.0 to include all available network interfaces, or specify an IP address assigned to a specific interface. ") + @Documentation(""" + Binds an OpenSearch node to an address. Use 0.0.0.0 to include all available network interfaces, + or specify an IP address assigned to a specific interface. + """) @Parameter(value = "opensearch_network_host") private String opensearchNetworkHost = null; @@ -158,23 +176,36 @@ public class Configuration { @Parameter(value = HTTP_CERTIFICATE_PASSWORD_PROPERTY) private String datanodeHttpCertificatePassword; - @Documentation("You MUST set a secret to secure/pepper the stored user passwords here. Use at least 16 characters." + - "Generate one by using for example: pwgen -N 1 -s 96 \n" + - "ATTENTION: This value must be the same on all Graylog and Datanode nodes in the cluster. " + - "Changing this value after installation will render all user sessions and encrypted values in the database invalid. (e.g. encrypted access tokens)") + @Documentation(""" + You MUST set a secret to secure/pepper the stored user passwords here. Use at least 16 characters. + Generate one by using for example: pwgen -N 1 -s 96 + ATTENTION: This value must be the same on all Graylog and Datanode nodes in the cluster. + Changing this value after installation will render all user sessions and encrypted values + in the database invalid. (e.g. encrypted access tokens) + """) @Parameter(value = "password_secret", required = true, validators = StringNotBlankValidator.class) private String passwordSecret; - @Documentation("communication between Graylog and OpenSearch is secured by JWT. This configuration defines interval between token regenerations.") + @DocumentationSection(heading = "OpenSearch JWT token usage",description = """ + communication between Graylog and OpenSearch is secured by JWT. These are the defaults used for the token usage + adjust them, if you have special needs. + """) + @Documentation(value = "This configuration defines interval between token regenerations.") @Parameter(value = "indexer_jwt_auth_token_caching_duration") Duration indexerJwtAuthTokenCachingDuration = Duration.seconds(60); - @Documentation("communication between Graylog and OpenSearch is secured by JWT. This configuration defines validity interval of JWT tokens.") + @DocumentationSection(heading = "OpenSearch JWT token usage",description = """ + communication between Graylog and OpenSearch is secured by JWT. These are the defaults used for the token usage + adjust them, if you have special needs. + """) + @Documentation("This configuration defines validity interval of JWT tokens") @Parameter(value = "indexer_jwt_auth_token_expiration_duration") Duration indexerJwtAuthTokenExpirationDuration = Duration.seconds(180); - @Documentation("The auto-generated node ID will be stored in this file and read after restarts. It is a good idea " + - "to use an absolute file path here if you are starting Graylog DataNode from init scripts or similar.") + @Documentation(""" + The auto-generated node ID will be stored in this file and read after restarts. It is a good idea + to use an absolute file path here if you are starting Graylog DataNode from init scripts or similar. + """) @Parameter(value = "node_id_file", validators = NodeIdFileValidator.class) private String nodeIdFile = "data/node-id"; @@ -195,15 +226,19 @@ public class Configuration { @Parameter(value = "clustername") private String clustername = "datanode-cluster"; - @Documentation("This configuration should be used if you want to connect to this Graylog DataNode's REST API and it is available on " + - "another network interface than $http_bind_address, " + - "for example if the machine has multiple network interfaces or is behind a NAT gateway.") - @Parameter(value = "http_publish_uri", validators = URIAbsoluteValidator.class) + @Documentation(""" + This configuration should be used if you want to connect to this Graylog DataNode's REST API + and it is available on another network interface than $http_bind_address, + for example if the machine has multiple network interfaces or is behind a NAT gateway. + """) + @Parameter(value = "http_publish_uri", validators = URIAbsoluteValidator.class) private URI httpPublishUri; - @Documentation("Enable GZIP support for HTTP interface. This compresses API responses and therefore helps to reduce " + - " overall round trip times.") + @Documentation(""" + Enable GZIP support for HTTP interface. This compresses API responses and therefore helps to reduce + overall round trip times. + """) @Parameter(value = "http_enable_gzip") private boolean httpEnableGzip = true; @@ -286,7 +321,10 @@ public Integer getIndicesQueryBoolMaxClauseCount() { return indicesQueryBoolMaxClauseCount; } - @Documentation("Configures verbosity of embedded opensearch logs. Possible values OFF, FATAL, ERROR, WARN, INFO, DEBUG, and TRACE, default is INFO") + @Documentation(""" + Configures verbosity of embedded opensearch logs. + Possible values OFF, FATAL, ERROR, WARN, INFO, DEBUG, and TRACE, default is INFO + """) @Parameter(value = "opensearch_logger_org_opensearch") private String opensearchDebug; diff --git a/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationDocsGenerator.java b/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationDocsGenerator.java new file mode 100644 index 000000000000..f73d9199fe96 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationDocsGenerator.java @@ -0,0 +1,180 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.docs; + +import com.github.joschi.jadconfig.Parameter; +import com.github.joschi.jadconfig.ReflectionUtils; +import jakarta.annotation.Nonnull; +import org.apache.commons.lang3.ClassUtils; +import org.graylog.datanode.commands.Server; +import org.graylog.datanode.docs.printers.ConfigFileDocsPrinter; +import org.graylog.datanode.docs.printers.ConfigurationSection; +import org.graylog.datanode.docs.printers.CsvDocsPrinter; +import org.graylog.datanode.docs.printers.DocsPrinter; +import org.graylog2.configuration.Documentation; +import org.graylog2.configuration.DocumentationSection; + +import java.io.FileWriter; +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +public class ConfigurationDocsGenerator { + + /** + * This class is linked from the datanode pom.xml and generates conf.example and csv documentation. + */ + public static void main(String[] args) throws IOException { + final ConfigurationDocsGenerator generator = new ConfigurationDocsGenerator(); + generator.generateDocumentation(parseDocumentationFormat(args), ConfigurationDocsGenerator::getDatanodeConfigurationBeans); + } + + @Nonnull + private static DocumentationFormat parseDocumentationFormat(String[] args) { + if (args.length != 2) { + throw new IllegalArgumentException("This command needs two arguments - a format and file path. For example" + "csv ${project.build.directory}/configuration-documentation.csv"); + } + final String format = args[0].toLowerCase(Locale.ROOT).trim(); + final String file = args[1]; + return new DocumentationFormat(format, file); + } + + protected void generateDocumentation(DocumentationFormat format, Supplier> configurationBeans) throws IOException { + final List sections = detectConfigurationSections(configurationBeans); + try (final DocsPrinter writer = createWriter(format)) { + writer.write(sections); + } + } + + private DocsPrinter createWriter(DocumentationFormat format) throws IOException { + final FileWriter fileWriter = new FileWriter(format.outputFile(), StandardCharsets.UTF_8); + return switch (format.format()) { + case "csv" -> new CsvDocsPrinter(fileWriter); + case "conf" -> new ConfigFileDocsPrinter(fileWriter); + default -> throw new IllegalArgumentException("Unsupported format " + format.format()); + }; + } + + /** + * Collects all configuration options from all available configuration beans. + */ + private List detectConfigurationSections(Supplier> configurationBeans) { + return configurationBeans.get() + .stream() + .map(ConfigurationDocsGenerator::beanToConfigSections) + .sorted(Comparator.comparing(ConfigurationSection::hasPriority).reversed()) + .toList(); + } + + @Nonnull + private static ConfigurationSection beanToConfigSections(Object configurationBean) { + + String sectionHeading = null; + String sectionDescription = null; + if (configurationBean.getClass().isAnnotationPresent(DocumentationSection.class)) { + final DocumentationSection documentationSection = configurationBean.getClass().getAnnotation(DocumentationSection.class); + sectionHeading = documentationSection.heading(); + sectionDescription = documentationSection.description(); + } + + final List entries = Arrays.stream(configurationBean.getClass().getDeclaredFields()) + .filter(f -> f.isAnnotationPresent(Parameter.class)) + .filter(ConfigurationDocsGenerator::isPublicFacing) + .map(f -> toConfigurationEntry(f, configurationBean)) + .toList(); + + final List entriesWithoutSection = getEntriesWithoutSection(entries); + final List sortedSections = sectionsFromEntries(entries); + + return new ConfigurationSection(sectionHeading, sectionDescription, sortedSections, entriesWithoutSection); + } + + @Nonnull + private static List sectionsFromEntries(List entries) { + final Collection sections = entries.stream() + .filter(ConfigurationEntryWithSection::hasSection) + .collect(Collectors.groupingBy(ConfigurationEntryWithSection::sectionHeading, Collectors.collectingAndThen(Collectors.toList(), list -> new ConfigurationSection(list.iterator().next().sectionHeading(), list.iterator().next().sectionDescription(), Collections.emptyList(), list.stream().map(ConfigurationEntryWithSection::entry).collect(Collectors.toList()))))).values(); + return sections.stream() + .sorted(Comparator.comparing(ConfigurationSection::hasPriority, Comparator.reverseOrder())).toList(); + } + + @Nonnull + private static List getEntriesWithoutSection(List entries) { + return entries.stream() + .filter(e -> !e.hasSection()) + .map(ConfigurationEntryWithSection::entry) + .sorted(Comparator.comparing(ConfigurationEntry::hasPriority, Comparator.reverseOrder())) + .collect(Collectors.toList()); + } + + private static List getDatanodeConfigurationBeans() { + return new Server().getCommandConfigurationBeans(); + } + + /** + * There are some configuration options not intended for general usage, mainly just for system packages configuration. + * + * @see Documentation#visible() + */ + private static boolean isPublicFacing(Field f) { + return !f.isAnnotationPresent(Documentation.class) || f.getAnnotation(Documentation.class).visible(); + } + + private static ConfigurationEntryWithSection toConfigurationEntry(Field f, Object instance) { + final String documentation = Optional.ofNullable(f.getAnnotation(Documentation.class)).map(Documentation::value).orElse(null); + final Parameter parameter = f.getAnnotation(Parameter.class); + final String propertyName = parameter.value(); + final Object defaultValue = getDefaultValue(f, instance); + final String type = getType(f); + final boolean required = parameter.required(); + + final DocumentationSection documentationSection = f.getAnnotation(DocumentationSection.class); + String sectionHeading = null; + String sectionDescription = null; + if (documentationSection != null) { + sectionHeading = documentationSection.heading(); + sectionDescription = documentationSection.description(); + } + final ConfigurationEntry entry = new ConfigurationEntry(instance.getClass(), f.getName(), type, propertyName, defaultValue, required, documentation); + return new ConfigurationEntryWithSection(entry, sectionHeading, sectionDescription); + } + + private static Object getDefaultValue(Field f, Object instance) { + try { + return ReflectionUtils.getFieldValue(instance, f); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + private static String getType(Field f) { + if (f.getType().isPrimitive()) { // unify primitive types and wrappers, e.g. int -> Integer + return ClassUtils.primitiveToWrapper(f.getType()).getSimpleName(); + } else { + return f.getType().getSimpleName(); + } + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationEntry.java b/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationEntry.java new file mode 100644 index 000000000000..3590da8d2c7d --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationEntry.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.docs; + +import jakarta.annotation.Nullable; + +/** + * @param configurationBean Class that defines this configuration entry + * @param fieldName java field name + * @param type Java type name, e.g. String or Integer. + * @param configName configuration property name, as written in the config file + * @param defaultValue default value declared in the java field, null if not defined + * @param required if the configuration property is mandatory (needs default or entry in the config file) + * @param documentation textual documentation of this configuration propery + */ +public record ConfigurationEntry( + Class configurationBean, + String fieldName, + String type, + String configName, + @Nullable Object defaultValue, + boolean required, + String documentation +) { + + public boolean hasPriority() { + return required && defaultValue == null; + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationEntryWithSection.java b/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationEntryWithSection.java new file mode 100644 index 000000000000..5224246b1a1f --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationEntryWithSection.java @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.docs; + +public record ConfigurationEntryWithSection(ConfigurationEntry entry, String sectionHeading, String sectionDescription) { + public boolean hasSection() { + return sectionHeading != null && !sectionHeading.isBlank(); + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/docs/DocumentationConstants.java b/data-node/src/main/java/org/graylog/datanode/docs/DocumentationConstants.java new file mode 100644 index 000000000000..e3ea120b960c --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/docs/DocumentationConstants.java @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.docs; + +public class DocumentationConstants { + public static final String DATANODE_DOCUMENTATION_DESCRIPTION = """ + This is the Graylog DataNode configuration file. The file has to use ISO 8859-1/Latin-1 character encoding. + Characters that cannot be directly represented in this encoding can be written using Unicode escapes + as defined in https://docs.oracle.com/javase/specs/jls/se8/html/jls-3.html#jls-3.3, using the \\u prefix. + For example, \\u002c. + + * Entries are generally expected to be a single line of the form, one of the following: + + propertyName=propertyValue + propertyName:propertyValue + + * White space that appears between the property name and property value is ignored, + so the following are equivalent: + + name=Stephen + name = Stephen + + * White space at the beginning of the line is also ignored. + + * Lines that start with the comment characters ! or # are ignored. Blank lines are also ignored. + + * The property value is generally terminated by the end of the line. White space following the + property value is not ignored, and is treated as part of the property value. + + * A property value can span several lines if each line is terminated by a backslash (‘\\’) character. + For example: + + targetCities=\\ + Detroit,\\ + Chicago,\\ + Los Angeles + + This is equivalent to targetCities=Detroit,Chicago,Los Angeles (white space at the beginning of lines is ignored). + + * The characters newline, carriage return, and tab can be inserted with characters \\n, \\r, and \\t, respectively. + + * The backslash character must be escaped as a double backslash. For example: + + path=c:\\\\docs\\\\doc1 + + + """; +} diff --git a/data-node/src/main/java/org/graylog/datanode/docs/DocumentationFormat.java b/data-node/src/main/java/org/graylog/datanode/docs/DocumentationFormat.java new file mode 100644 index 000000000000..f1dd593bbded --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/docs/DocumentationFormat.java @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.docs; + +public record DocumentationFormat(String format, String outputFile) { +} diff --git a/data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigFileDocsPrinter.java b/data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigFileDocsPrinter.java new file mode 100644 index 000000000000..302a943077d1 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigFileDocsPrinter.java @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.docs.printers; + +import org.apache.commons.lang.WordUtils; +import org.graylog.datanode.docs.ConfigurationEntry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.stream.Collectors; + +public class ConfigFileDocsPrinter implements DocsPrinter { + + private static final Logger LOG = LoggerFactory.getLogger(ConfigFileDocsPrinter.class); + + private final OutputStreamWriter writer; + + public ConfigFileDocsPrinter(OutputStreamWriter writer) { + this.writer = writer; + } + + @Override + public void close() throws IOException { + writer.close(); + } + + @Override + public void flush() throws IOException { + writer.flush(); + } + + @Override + public void write(List configurationSections) { + configurationSections.forEach(section -> doWriteSection(section, 1)); + } + + private void doWriteSection(ConfigurationSection configurationSection, int level) { + heading(configurationSection.heading(), level).ifPresent(this::append); + description(configurationSection.description()).ifPresent(this::append); + configurationSection.entries().stream().map(this::fieldToString).forEach(this::append); + configurationSection.sections().forEach(section -> doWriteSection(section, level + 1)); + } + + private Optional description(String description) { + return Optional.ofNullable(description) + .map(text -> text.lines().map(l -> "# " + l.trim()).collect(Collectors.joining("\n"))) + .map(text -> text + "\n\n"); + } + + private Writer append(String formatted) { + try { + return writer.append(formatted); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private Optional heading(String heading, int level) { + return Optional.ofNullable(heading).map(h -> headingTemplate(level).formatted(heading)); + } + + @Nonnull + private static String headingTemplate(int level) { + if(level == 1) { + return """ + ##################################### + # %s + ##################################### + """; + } else { + return "#### %s\n"; + } + } + + private String fieldToString(ConfigurationEntry field) { + // enabled with empty value, force to fill in + final boolean forceFillIn = field.required() && field.defaultValue() == null; + + + String template = """ + %s + %s%s = %s + + """; + + return String.format(Locale.ROOT, template, formatDocumentation(field), forceFillIn ? "" : "#", field.configName(), wrapValue(field.defaultValue())); + } + + private String wrapValue(Object value) { + return Optional.ofNullable(value).map(String::valueOf).orElse(""); + } + + private static String formatDocumentation(ConfigurationEntry field) { + final String[] lines = field.documentation().split("\n"); + return Arrays.stream(lines).map(String::trim).peek(line -> { + if (line.length() > 120) { + LOG.warn("Documentation line of " + field.configurationBean().getName() + "." + field.fieldName() + " too long, consider splitting into more lines: " + WordUtils.abbreviate(line, 120, 130, "...")); + } + }).map(line -> "# " + line).collect(Collectors.joining("\n")); + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigurationSection.java b/data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigurationSection.java new file mode 100644 index 000000000000..dcd02f202e70 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigurationSection.java @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.docs.printers; + +import org.graylog.datanode.docs.ConfigurationEntry; + +import java.util.List; + +public record ConfigurationSection(String heading, String description, List sections, List entries) { + public boolean hasPriority() { + return entries.stream().anyMatch(ConfigurationEntry::hasPriority); + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/docs/printers/CsvDocsPrinter.java b/data-node/src/main/java/org/graylog/datanode/docs/printers/CsvDocsPrinter.java new file mode 100644 index 000000000000..937f3ed93b98 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/docs/printers/CsvDocsPrinter.java @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.docs.printers; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; +import org.graylog.datanode.docs.ConfigurationEntry; + +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.util.List; + +public class CsvDocsPrinter implements DocsPrinter { + + public static final String HEADER_PARAMETER = "Parameter"; + public static final String HEADER_TYPE = "Type"; + public static final String HEADER_REQUIRED = "Required"; + public static final String HEADER_DEFAULT_VALUE = "Default value"; + public static final String HEADER_DESCRIPTION = "Description"; + public static final String[] HEADERS = {HEADER_PARAMETER, HEADER_TYPE, HEADER_REQUIRED, HEADER_DEFAULT_VALUE, HEADER_DESCRIPTION}; + private final CSVPrinter csvPRinter; + + public CsvDocsPrinter(OutputStreamWriter streamWriter) throws IOException { + this.csvPRinter = new CSVPrinter(streamWriter, CSVFormat.EXCEL); + } + + @Override + public void write(List sections) { + printHeaders(); + sections.forEach(this::doWriteSection); + } + + private void printHeaders() { + try { + csvPRinter.printRecord(HEADERS); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private void doWriteSection(ConfigurationSection section) { + section.entries().forEach(this::writeField); + section.sections().forEach(this::doWriteSection); + } + + + private void writeField(ConfigurationEntry f) { + try { + this.csvPRinter.printRecord(f.configName(), f.type(), boolToHumanReadable(f.required()), f.defaultValue(), f.documentation()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private String boolToHumanReadable(boolean required) { + if (required) { + return "yes"; + } else { + return "no"; + } + } + + @Override + public void close() throws IOException { + csvPRinter.close(); + } + + @Override + public void flush() throws IOException { + csvPRinter.flush(); + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/docs/printers/DocsPrinter.java b/data-node/src/main/java/org/graylog/datanode/docs/printers/DocsPrinter.java new file mode 100644 index 000000000000..7335e7f8c96d --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/docs/printers/DocsPrinter.java @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.docs.printers; + +import java.io.Closeable; +import java.io.Flushable; +import java.util.List; + +public interface DocsPrinter extends Flushable, Closeable { + + void write(List sections); +} diff --git a/data-node/src/test/java/org/graylog/datanode/ConfigurationDocumentationTest.java b/data-node/src/test/java/org/graylog/datanode/ConfigurationDocumentationTest.java index 7581ee8ef69b..f81089bb6633 100644 --- a/data-node/src/test/java/org/graylog/datanode/ConfigurationDocumentationTest.java +++ b/data-node/src/test/java/org/graylog/datanode/ConfigurationDocumentationTest.java @@ -17,23 +17,14 @@ package org.graylog.datanode; import com.github.joschi.jadconfig.Parameter; -import com.github.joschi.jadconfig.ReflectionUtils; -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVPrinter; -import org.apache.commons.lang3.ClassUtils; import org.bson.assertions.Assertions; import org.graylog.datanode.commands.Server; import org.graylog2.configuration.Documentation; import org.junit.jupiter.api.Test; -import java.io.IOException; -import java.io.StringWriter; import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.util.Arrays; import java.util.List; -import java.util.Optional; import java.util.stream.Collectors; class ConfigurationDocumentationTest { @@ -55,75 +46,4 @@ void testAllFieldsAreDocumented() { Assertions.fail("Following datanode configuration fields require @Documentation annotation: \n" + fields); } } - - /** - * When started, this will output to STDOUT the CSV table of datanode's configuration documentation. - */ - public static void main(String[] args) throws IOException { - final StringWriter stringWriter = new StringWriter(); - try (CSVPrinter printer = new CSVPrinter(stringWriter, CSVFormat.EXCEL)) { - - printer.printRecord("Parameter", "Type", "Required", "Default value", "Description"); - - final List datanodeConfiguration = new Server().getCommandConfigurationBeans(); - - datanodeConfiguration.forEach(configurationBean -> { - Arrays.stream(configurationBean.getClass().getDeclaredFields()) - .filter(f -> f.isAnnotationPresent(Parameter.class)) - .filter(ConfigurationDocumentationTest::isPublicFacing) - .map(f -> toConfigurationField(f, configurationBean)) - .forEach(f -> { - try { - printer.printRecord(f.configName(), f.type(), f.required(), f.defaultValue(), f.documentation()); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - }); - System.out.println(stringWriter); - } - } - - private static boolean isPublicFacing(Field f) { - return !f.isAnnotationPresent(Documentation.class) || f.getAnnotation(Documentation.class).visible(); - } - - private static ConfigurationField toConfigurationField(Field f, Object instance) { - - final String documentation = Optional.ofNullable(f.getAnnotation(Documentation.class)) - .map(Documentation::value).orElse(null); - - final Parameter parameter = f.getAnnotation(Parameter.class); - final String propertyName = parameter.value(); - final Object defaultValue = getDefaultValue(f, instance); - - final String type = getType(f); - - final boolean required = parameter.required(); - - return new ConfigurationField(f.getName(), type, propertyName, defaultValue, required, documentation); - } - - private static Object getDefaultValue(Field f, Object instance) { - final Object defaultValue; - try { - defaultValue = ReflectionUtils.getFieldValue(instance, f); - } catch (IllegalAccessException e) { - throw new RuntimeException(e); - } - return defaultValue; - } - - private static String getType(Field f) { - if (f.getType().isPrimitive()) { - return ClassUtils.primitiveToWrapper(f.getType()).getSimpleName(); - } else { - return f.getType().getSimpleName(); - } - } - - - private record ConfigurationField(String fieldName, String type, String configName, Object defaultValue, - boolean required, String documentation) { - } } diff --git a/data-node/src/test/java/org/graylog/datanode/docs/GenerateConfigDocumentationTest.java b/data-node/src/test/java/org/graylog/datanode/docs/GenerateConfigDocumentationTest.java new file mode 100644 index 000000000000..c6ef9584dfac --- /dev/null +++ b/data-node/src/test/java/org/graylog/datanode/docs/GenerateConfigDocumentationTest.java @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.docs; + +import com.github.joschi.jadconfig.Parameter; +import com.github.joschi.jadconfig.util.Duration; +import com.github.joschi.jadconfig.validators.PositiveIntegerValidator; +import com.github.joschi.jadconfig.validators.StringNotBlankValidator; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.assertj.core.api.Assertions; +import org.graylog.datanode.Configuration; +import org.graylog.datanode.docs.printers.CsvDocsPrinter; +import org.graylog2.configuration.Documentation; +import org.graylog2.configuration.DocumentationSection; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.FileReader; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +class GenerateConfigDocumentationTest { + + private ConfigurationDocsGenerator generator; + + @BeforeEach + void setUp() { + this.generator = new ConfigurationDocsGenerator(); + } + + @Test + void testCsv(@TempDir Path tmpPath) throws IOException { + final Path file = tmpPath.resolve("my-documentation.csv"); + final DocumentationFormat format = new DocumentationFormat("csv", file.toFile().getAbsolutePath()); + generator.generateDocumentation(format, () -> List.of(new DummyConfiguration())); + + final CSVFormat csvFormat = CSVFormat.Builder.create(CSVFormat.EXCEL) + .setHeader(CsvDocsPrinter.HEADERS) + .setSkipHeaderRecord(true) + .build(); + + final CSVParser parser = new CSVParser(new FileReader(file.toFile(), StandardCharsets.UTF_8), csvFormat); + final List lines = parser.getRecords(); + Assertions.assertThat(lines) + .hasSize(3) // one entry only, the invisible should not be present in the output + .anySatisfy(line -> { + Assertions.assertThat(line.get(CsvDocsPrinter.HEADER_DEFAULT_VALUE)).isEqualTo("data/node-id"); + Assertions.assertThat(line.get(CsvDocsPrinter.HEADER_PARAMETER)).isEqualTo("node_id_file"); + Assertions.assertThat(line.get(CsvDocsPrinter.HEADER_TYPE)).isEqualTo("String"); + Assertions.assertThat(line.get(CsvDocsPrinter.HEADER_REQUIRED)).isEqualTo("no"); + Assertions.assertThat(line.get(CsvDocsPrinter.HEADER_DESCRIPTION)).contains("The auto-generated node ID will be stored in this file and read after restarts"); + }); + + // Assert order of the lines. Required password_secret without default value has to go first. Then the rest follows + // order of the properties in the class. + Assertions.assertThat(lines.get(0).get(CsvDocsPrinter.HEADER_PARAMETER)).isEqualTo("password_secret"); + Assertions.assertThat(lines.get(1).get(CsvDocsPrinter.HEADER_PARAMETER)).isEqualTo("node_id_file"); + Assertions.assertThat(lines.get(2).get(CsvDocsPrinter.HEADER_PARAMETER)).isEqualTo("indexer_jwt_auth_token_caching_duration"); + } + + @Test + void testConfFile(@TempDir Path tmpPath) throws IOException { + final Path file = tmpPath.resolve("config-example.conf"); + final DocumentationFormat format = new DocumentationFormat("conf", file.toFile().getAbsolutePath()); + generator.generateDocumentation(format, () -> List.of(new DummyConfiguration())); + final String content = Files.readString(file); + Assertions.assertThat(content).contains("The auto-generated node ID will be stored in this file and read after restarts"); + Assertions.assertThat(content).contains("#node_id_file = data/node-id"); + } + + @DocumentationSection(heading = "my-test-config", description = "this is how you configure your app") + private static class DummyConfiguration { + @Documentation(""" + The auto-generated node ID will be stored in this file and read after restarts. It is a good idea + to use an absolute file path here if you are starting Graylog DataNode from init scripts or similar. + """) + @Parameter(value = "node_id_file", validators = Configuration.NodeIdFileValidator.class) + private String nodeIdFile = "data/node-id"; + + @Documentation(visible = false) + @Parameter(value = "timeout_sec", validators = PositiveIntegerValidator.class) + private Integer timeoutSec; + + @Documentation(""" + You MUST set a secret to secure/pepper the stored user passwords here. Use at least 64 characters. + Generate one by using for example: pwgen -N 1 -s 96 + ATTENTION: This value must be the same on all Graylog and Datanode nodes in the cluster. + Changing this value after installation will render all user sessions and encrypted values + in the database invalid. (e.g. encrypted access tokens) + """) + @Parameter(value = "password_secret", required = true, validators = StringNotBlankValidator.class) + private String passwordSecret; + + @DocumentationSection(heading = "OpenSearch JWT token usage", description = "Communication between Graylog and OpenSearch is secured by JWT.") + @Documentation(""" + This configuration defines interval between token regenerations. + """) + @Parameter(value = "indexer_jwt_auth_token_caching_duration") + Duration indexerJwtAuthTokenCachingDuration = Duration.seconds(60); + } +} diff --git a/graylog2-server/src/main/java/org/graylog2/configuration/DocumentationSection.java b/graylog2-server/src/main/java/org/graylog2/configuration/DocumentationSection.java new file mode 100644 index 000000000000..c048e0e4afc4 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/configuration/DocumentationSection.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog2.configuration; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE, ElementType.FIELD}) +public @interface DocumentationSection { + String heading(); + String description(); +} diff --git a/graylog2-server/src/main/java/org/graylog2/configuration/MongoDbConfiguration.java b/graylog2-server/src/main/java/org/graylog2/configuration/MongoDbConfiguration.java index 4ece0cdf62ee..41d98c1191e6 100644 --- a/graylog2-server/src/main/java/org/graylog2/configuration/MongoDbConfiguration.java +++ b/graylog2-server/src/main/java/org/graylog2/configuration/MongoDbConfiguration.java @@ -25,8 +25,10 @@ import com.mongodb.MongoClientURI; public class MongoDbConfiguration { - @Documentation("Increase this value according to the maximum connections your MongoDB server can handle from a single client " + - "if you encounter MongoDB connection problems.") + @Documentation(""" + Increase this value according to the maximum connections your MongoDB server can handle from a single client + if you encounter MongoDB connection problems. + """) @Parameter(value = "mongodb_max_connections", validator = PositiveIntegerValidator.class) private int maxConnections = 1000; @@ -34,8 +36,10 @@ public class MongoDbConfiguration { @Parameter(value = "mongodb_uri", required = true, validator = StringNotBlankValidator.class) private String uri = "mongodb://localhost/graylog"; - @Documentation("Maximum number of attempts to connect to MongoDB on boot for the version probe." + - "Default 0 means retry indefinitely until a connection can be established") + @Documentation(""" + Maximum number of attempts to connect to MongoDB on boot for the version probe. + Default 0 means retry indefinitely until a connection can be established + """) @Parameter(value = "mongodb_version_probe_attempts", validators = {PositiveIntegerValidator.class}) int mongodbVersionProbeAttempts = 0; @@ -60,7 +64,7 @@ public MongoClientURI getMongoClientURI() { @ValidatorMethod public void validate() throws ValidationException { - if(getMongoClientURI() == null) { + if (getMongoClientURI() == null) { throw new ValidationException("mongodb_uri is not a valid MongoDB connection string"); } } diff --git a/graylog2-server/src/main/java/org/graylog2/configuration/TLSProtocolsConfiguration.java b/graylog2-server/src/main/java/org/graylog2/configuration/TLSProtocolsConfiguration.java index cbf8daf953af..1e1450907323 100644 --- a/graylog2-server/src/main/java/org/graylog2/configuration/TLSProtocolsConfiguration.java +++ b/graylog2-server/src/main/java/org/graylog2/configuration/TLSProtocolsConfiguration.java @@ -33,8 +33,10 @@ */ public class TLSProtocolsConfiguration { - @Documentation("allowed TLS protocols for system wide TLS enabled servers. (e.g. message inputs, http interface)." + - "Setting this to an empty value, leaves it up to system libraries and the used JDK to chose a default.") + @Documentation(""" + allowed TLS protocols for system wide TLS enabled servers. (e.g. message inputs, http interface) + Setting this to an empty value, leaves it up to system libraries and the used JDK to chose a default. + """) @Parameter(value = "enabled_tls_protocols", converter = StringSetConverter.class) private Set enabledTlsProtocols = null;