From db97490a783ac2bbe9c4b5b8932b23ff43199d4e Mon Sep 17 00:00:00 2001 From: Tomas Dvorak Date: Tue, 23 Apr 2024 18:00:31 +0200 Subject: [PATCH 1/5] Automatically generate datanode.conf.example + csv documentation of configuration options --- data-node/pom.xml | 28 ++++ data-node/src/main/assembly/datanode.xml | 2 +- .../org/graylog/datanode/Configuration.java | 90 +++++++---- .../datanode/docs/ConfigFilePrinter.java | 140 ++++++++++++++++++ .../ConfigurationDocumentationPrinter.java | 35 +++++ .../CsvConfigurationDocumentationPrinter.java | 56 +++++++ .../docs/GenerateConfigDocumentation.java | 109 ++++++++++++++ .../ConfigurationDocumentationTest.java | 80 ---------- .../configuration/MongoDbConfiguration.java | 14 +- .../TLSProtocolsConfiguration.java | 6 +- 10 files changed, 444 insertions(+), 116 deletions(-) create mode 100644 data-node/src/main/java/org/graylog/datanode/docs/ConfigFilePrinter.java create mode 100644 data-node/src/main/java/org/graylog/datanode/docs/ConfigurationDocumentationPrinter.java create mode 100644 data-node/src/main/java/org/graylog/datanode/docs/CsvConfigurationDocumentationPrinter.java create mode 100644 data-node/src/main/java/org/graylog/datanode/docs/GenerateConfigDocumentation.java diff --git a/data-node/pom.xml b/data-node/pom.xml index 63c79a232d3d..1e457a608a46 100644 --- a/data-node/pom.xml +++ b/data-node/pom.xml @@ -653,6 +653,34 @@ exec-maven-plugin org.codehaus.mojo + + generate-csv-docs + prepare-package + + java + + + org.graylog.datanode.docs.GenerateConfigDocumentation + + csv + ${project.build.directory}/datanode-conf-docs.csv + + + + + generate-conf-example + prepare-package + + java + + + org.graylog.datanode.docs.GenerateConfigDocumentation + + conf + ${project.build.directory}/datanode.conf.example + + + install-required-opensearch-plugins-x64 prepare-package diff --git a/data-node/src/main/assembly/datanode.xml b/data-node/src/main/assembly/datanode.xml index 8c2dc9fca0e7..d6a1757f4398 100644 --- a/data-node/src/main/assembly/datanode.xml +++ b/data-node/src/main/assembly/datanode.xml @@ -75,7 +75,7 @@ . - ${project.basedir}/../misc/datanode.conf + ${project.build.directory}/datanode.conf.example datanode.conf.example . diff --git a/data-node/src/main/java/org/graylog/datanode/Configuration.java b/data-node/src/main/java/org/graylog/datanode/Configuration.java index 1d1843426d7d..23ad921cc97f 100644 --- a/data-node/src/main/java/org/graylog/datanode/Configuration.java +++ b/data-node/src/main/java/org/graylog/datanode/Configuration.java @@ -86,8 +86,10 @@ public class Configuration { @Parameter(value = "opensearch_location") private String opensearchDistributionRoot = "dist"; - @Documentation("Data directory of the embedded opensearch. Contains indices of the opensearch. May be pointed to an existing" + - "opensearch directory during in-place migration to Datanode") + @Documentation(""" + Data directory of the embedded opensearch. Contains indices of the opensearch. + May be pointed to an existing opensearch directory during in-place migration to Datanode + """) @Parameter(value = "opensearch_data_location", required = true) private Path opensearchDataLocation = Path.of("datanode/data"); @@ -95,8 +97,11 @@ public class Configuration { @Parameter(value = "opensearch_logs_location", required = true, validators = DirectoryWritableValidator.class) private Path opensearchLogsLocation = Path.of("datanode/logs"); - @Documentation("Configuration directory of the embedded opensearch. This is the directory where the opensearch" + - "process will store its configuration files. Caution, each start of the Datanode will regenerate the complete content of the directory!") + @Documentation(""" + Configuration directory of the embedded opensearch. This is the directory where the opensearch + process will store its configuration files. Caution, each start of the Datanode will regenerate + the complete content of the directory! + """) @Parameter(value = "opensearch_config_location", required = true, validators = DirectoryWritableValidator.class) private Path opensearchConfigLocation = Path.of("datanode/config"); @@ -113,7 +118,10 @@ public class Configuration { private Integer opensearchProcessLogsBufferSize = 500; - @Documentation("Unique name of this Datanode instance. use this, if your node name should be different from the hostname that's found by programmatically looking it up") + @Documentation(""" + Unique name of this Datanode instance. use this, if your node name should be different from the hostname + that's found by programmatically looking it up. + """) @Parameter(value = "node_name") private String datanodeNodeName; @@ -122,7 +130,10 @@ public class Configuration { @Parameter(value = "initial_cluster_manager_nodes") private String initialClusterManagerNodes; - @Documentation("Opensearch heap memory. Initial and maxmium heap must be identical for OpenSearch, otherwise the boot fails. So it's only one config option") + @Documentation(""" + Opensearch heap memory. Initial and maxmium heap must be identical for OpenSearch, otherwise the boot fails. + So it's only one config option. + """) @Parameter(value = "opensearch_heap") private String opensearchHeap = "1g"; @@ -138,7 +149,10 @@ public class Configuration { @Parameter(value = "opensearch_discovery_seed_hosts", converter = StringListConverter.class) private List opensearchDiscoverySeedHosts = Collections.emptyList(); - @Documentation("Binds an OpenSearch node to an address. Use 0.0.0.0 to include all available network interfaces, or specify an IP address assigned to a specific interface. ") + @Documentation(""" + Binds an OpenSearch node to an address. Use 0.0.0.0 to include all available network interfaces, + or specify an IP address assigned to a specific interface. + """) @Parameter(value = "opensearch_network_host") private String opensearchNetworkHost = null; @@ -158,31 +172,44 @@ public class Configuration { @Parameter(value = HTTP_CERTIFICATE_PASSWORD_PROPERTY) private String datanodeHttpCertificatePassword; - @Documentation("You MUST specify a hash password for the root user (which you only need to initially set up the " + - "system and in case you lose connectivity to your authentication backend)." + - "This password cannot be changed using the API or via the web interface. If you need to change it, " + - "modify it in this file. " + - "Create one by using for example: echo -n yourpassword | shasum -a 256") - @Parameter(value = "root_password_sha2") + @Documentation(""" + You MUST specify a hash password for the root user (which you only need to initially set up the + system and in case you lose connectivity to your authentication backend). + This password cannot be changed using the API or via the web interface. If you need to change it, + modify it in this file. + Create one by using for example: echo -n yourpassword | shasum -a 256 + """) + @Parameter(value = "root_password_sha2", required = true) private String rootPasswordSha2; - @Documentation("You MUST set a secret to secure/pepper the stored user passwords here. Use at least 64 characters." + - "Generate one by using for example: pwgen -N 1 -s 96 \n" + - "ATTENTION: This value must be the same on all Graylog and Datanode nodes in the cluster. " + - "Changing this value after installation will render all user sessions and encrypted values in the database invalid. (e.g. encrypted access tokens)") + @Documentation(""" + You MUST set a secret to secure/pepper the stored user passwords here. Use at least 64 characters. + Generate one by using for example: pwgen -N 1 -s 96 + ATTENTION: This value must be the same on all Graylog and Datanode nodes in the cluster. + Changing this value after installation will render all user sessions and encrypted values + in the database invalid. (e.g. encrypted access tokens) + """) @Parameter(value = "password_secret", required = true, validators = StringNotBlankValidator.class) private String passwordSecret; - @Documentation("communication between Graylog and OpenSearch is secured by JWT. This configuration defines interval between token regenerations.") + @Documentation(""" + communication between Graylog and OpenSearch is secured by JWT. + This configuration defines interval between token regenerations. + """) @Parameter(value = "indexer_jwt_auth_token_caching_duration") Duration indexerJwtAuthTokenCachingDuration = Duration.seconds(60); - @Documentation("communication between Graylog and OpenSearch is secured by JWT. This configuration defines validity interval of JWT tokens.") + @Documentation(""" + communication between Graylog and OpenSearch is secured by JWT. + This configuration defines validity interval of JWT tokens. + """) @Parameter(value = "indexer_jwt_auth_token_expiration_duration") Duration indexerJwtAuthTokenExpirationDuration = Duration.seconds(180); - @Documentation("The auto-generated node ID will be stored in this file and read after restarts. It is a good idea " + - "to use an absolute file path here if you are starting Graylog DataNode from init scripts or similar.") + @Documentation(""" + The auto-generated node ID will be stored in this file and read after restarts. It is a good idea + to use an absolute file path here if you are starting Graylog DataNode from init scripts or similar. + """) @Parameter(value = "node_id_file", validators = NodeIdFileValidator.class) private String nodeIdFile = "data/node-id"; @@ -207,15 +234,19 @@ public class Configuration { @Parameter(value = "clustername") private String clustername = "datanode-cluster"; - @Documentation("This configuration should be used if you want to connect to this Graylog DataNode's REST API and it is available on " + - "another network interface than $http_bind_address, " + - "for example if the machine has multiple network interfaces or is behind a NAT gateway.") - @Parameter(value = "http_publish_uri", validators = URIAbsoluteValidator.class) + @Documentation(""" + This configuration should be used if you want to connect to this Graylog DataNode's REST API + and it is available on another network interface than $http_bind_address, + for example if the machine has multiple network interfaces or is behind a NAT gateway. + """) + @Parameter(value = "http_publish_uri", validators = URIAbsoluteValidator.class) private URI httpPublishUri; - @Documentation("Enable GZIP support for HTTP interface. This compresses API responses and therefore helps to reduce " + - " overall round trip times.") + @Documentation(""" + Enable GZIP support for HTTP interface. This compresses API responses and therefore helps to reduce + overall round trip times. + """) @Parameter(value = "http_enable_gzip") private boolean httpEnableGzip = true; @@ -298,7 +329,10 @@ public Integer getIndicesQueryBoolMaxClauseCount() { return indicesQueryBoolMaxClauseCount; } - @Documentation("Configures verbosity of embedded opensearch logs. Possible values OFF, FATAL, ERROR, WARN, INFO, DEBUG, and TRACE, default is INFO") + @Documentation(""" + Configures verbosity of embedded opensearch logs. + Possible values OFF, FATAL, ERROR, WARN, INFO, DEBUG, and TRACE, default is INFO + """) @Parameter(value = "opensearch_logger_org_opensearch") private String opensearchDebug; diff --git a/data-node/src/main/java/org/graylog/datanode/docs/ConfigFilePrinter.java b/data-node/src/main/java/org/graylog/datanode/docs/ConfigFilePrinter.java new file mode 100644 index 000000000000..78a9e4b03518 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/docs/ConfigFilePrinter.java @@ -0,0 +1,140 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.docs; + +import org.apache.commons.lang.WordUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.util.Arrays; +import java.util.Locale; +import java.util.Optional; +import java.util.stream.Collectors; + +public class ConfigFilePrinter implements ConfigurationDocumentationPrinter { + + public static final String DATANODE_CONFIG_HEADER = """ + ##################################### + # GRAYLOG DATANODE CONFIGURATION FILE + ##################################### + # + # This is the Graylog DataNode configuration file. The file has to use ISO 8859-1/Latin-1 character encoding. + # Characters that cannot be directly represented in this encoding can be written using Unicode escapes + # as defined in https://docs.oracle.com/javase/specs/jls/se8/html/jls-3.html#jls-3.3, using the \\u prefix. + # For example, \\u002c. + # + # * Entries are generally expected to be a single line of the form, one of the following: + # + # propertyName=propertyValue + # propertyName:propertyValue + # + # * White space that appears between the property name and property value is ignored, + # so the following are equivalent: + # + # name=Stephen + # name = Stephen + # + # * White space at the beginning of the line is also ignored. + # + # * Lines that start with the comment characters ! or # are ignored. Blank lines are also ignored. + # + # * The property value is generally terminated by the end of the line. White space following the + # property value is not ignored, and is treated as part of the property value. + # + # * A property value can span several lines if each line is terminated by a backslash (‘\\’) character. + # For example: + # + # targetCities=\\ + # Detroit,\\ + # Chicago,\\ + # Los Angeles + # + # This is equivalent to targetCities=Detroit,Chicago,Los Angeles (white space at the beginning of lines is ignored). + # + # * The characters newline, carriage return, and tab can be inserted with characters \\n, \\r, and \\t, respectively. + # + # * The backslash character must be escaped as a double backslash. For example: + # + # path=c:\\\\docs\\\\doc1 + # + + + """; + + private static final Logger LOG = LoggerFactory.getLogger(ConfigFilePrinter.class); + + private final OutputStreamWriter writer; + + public ConfigFilePrinter(OutputStreamWriter writer) { + this.writer = writer; + } + + @Override + public void close() throws IOException { + writer.close(); + } + + @Override + public void flush() throws IOException { + writer.flush(); + } + + + + public void writeHeader() throws IOException { + writer.append(DATANODE_CONFIG_HEADER); + } + + @Override + public void writeField(ConfigurationDocumentationPrinter.ConfigurationField field) throws IOException { + writer.append(toString(field)); + } + + private String toString(ConfigurationDocumentationPrinter.ConfigurationField field) { + // enabled with empty value, force to fill in + final boolean forceFillIn = field.required() && field.defaultValue() == null; + + + String template = """ + %s + %s%s=%s + + """; + + return String.format(Locale.ROOT, template, formatDocumentation(field), forceFillIn ? "" : "# ", field.configName(), wrapValue(field.defaultValue())); + } + + private String wrapValue(Object value) { + return Optional.ofNullable(value).map(String::valueOf).orElse(""); + } + + private static String formatDocumentation(ConfigurationDocumentationPrinter.ConfigurationField field) { + final String[] lines = field.documentation().split("\n"); + return Arrays.stream(lines) + .map(String::trim) + .peek(line -> { + if (line.length() > 120) { + LOG.warn("Documentation line of " + field.bean().getName() + "." + field.fieldName() + " too long, consider splitting into more lines: " + WordUtils.abbreviate(line, 120, 130, "...")); + } + }) + //.flatMap(line -> Arrays.stream(WordUtils.wrap(line, 120).split(System.lineSeparator()))) + .map(line -> "# " + line) + .collect(Collectors.joining("\n")); + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationDocumentationPrinter.java b/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationDocumentationPrinter.java new file mode 100644 index 000000000000..9b60184d9140 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationDocumentationPrinter.java @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.docs; + +import java.io.Closeable; +import java.io.Flushable; +import java.io.IOException; + +public interface ConfigurationDocumentationPrinter extends Flushable, Closeable { + void writeHeader() throws IOException; + + void writeField(ConfigurationField configurationField) throws IOException; + + + record ConfigurationField(Class bean, String fieldName, String type, String configName, Object defaultValue, + boolean required, String documentation) { + public boolean isPriority() { + return required && defaultValue == null; + } + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/docs/CsvConfigurationDocumentationPrinter.java b/data-node/src/main/java/org/graylog/datanode/docs/CsvConfigurationDocumentationPrinter.java new file mode 100644 index 000000000000..3b54ab60d3fb --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/docs/CsvConfigurationDocumentationPrinter.java @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.docs; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; + +import java.io.IOException; +import java.io.OutputStreamWriter; + +public class CsvConfigurationDocumentationPrinter implements ConfigurationDocumentationPrinter { + + private final CSVPrinter csvPRinter; + + public CsvConfigurationDocumentationPrinter(OutputStreamWriter streamWriter) { + try { + this.csvPRinter = new CSVPrinter(streamWriter, CSVFormat.EXCEL); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void writeHeader() throws IOException { + csvPRinter.printRecord("Parameter", "Type", "Required", "Default value", "Description"); + } + + @Override + public void writeField(ConfigurationField f) throws IOException { + this.csvPRinter.printRecord(f.configName(), f.type(), f.required(), f.defaultValue(), f.documentation()); + } + + @Override + public void close() throws IOException { + csvPRinter.close(); + } + + @Override + public void flush() throws IOException { + csvPRinter.flush(); + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/docs/GenerateConfigDocumentation.java b/data-node/src/main/java/org/graylog/datanode/docs/GenerateConfigDocumentation.java new file mode 100644 index 000000000000..23c2061e7ca1 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/docs/GenerateConfigDocumentation.java @@ -0,0 +1,109 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.docs; + +import com.github.joschi.jadconfig.Parameter; +import com.github.joschi.jadconfig.ReflectionUtils; +import org.apache.commons.lang3.ClassUtils; +import org.graylog.datanode.commands.Server; +import org.graylog2.configuration.Documentation; + +import java.io.FileWriter; +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.Optional; + +public class GenerateConfigDocumentation { + + /** + * When started, this will output to STDOUT the CSV table of datanode's configuration documentation. + */ + public static void main(String[] args) throws IOException { + final List datanodeConfiguration = new Server().getCommandConfigurationBeans(); + final List configuration = detectConfigurationFields(datanodeConfiguration); + try (final ConfigurationDocumentationPrinter writer = createWriter(args)) { + writer.writeHeader(); + configuration.forEach(f -> { + try { + writer.writeField(f); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + } + + private static ConfigurationDocumentationPrinter createWriter(String[] args) throws IOException { + assert args.length >= 2; + final String format = args[0].toLowerCase(Locale.ROOT).trim(); + final String file = args[1]; + + final FileWriter fileWriter = new FileWriter(file, StandardCharsets.UTF_8); + + return switch (format) { + case "csv" -> new CsvConfigurationDocumentationPrinter(fileWriter); + case "conf" -> new ConfigFilePrinter(fileWriter); + default -> throw new IllegalArgumentException("Unsupported format " + format); + }; + } + + private static List detectConfigurationFields(List datanodeConfiguration) { + return datanodeConfiguration.stream() + .flatMap(configurationBean -> Arrays.stream(configurationBean.getClass().getDeclaredFields()).filter(f -> f.isAnnotationPresent(Parameter.class)).filter(GenerateConfigDocumentation::isPublicFacing).map(f -> toConfigurationField(f, configurationBean))) + .sorted(Comparator.comparing(ConfigurationDocumentationPrinter.ConfigurationField::isPriority).reversed()) + .toList(); + } + + private static boolean isPublicFacing(Field f) { + return !f.isAnnotationPresent(Documentation.class) || f.getAnnotation(Documentation.class).visible(); + } + + private static ConfigurationDocumentationPrinter.ConfigurationField toConfigurationField(Field f, Object instance) { + + final String documentation = Optional.ofNullable(f.getAnnotation(Documentation.class)).map(Documentation::value).orElse(null); + + final Parameter parameter = f.getAnnotation(Parameter.class); + final String propertyName = parameter.value(); + final Object defaultValue = getDefaultValue(f, instance); + final String type = getType(f); + final boolean required = parameter.required(); + return new ConfigurationDocumentationPrinter.ConfigurationField(instance.getClass(), f.getName(), type, propertyName, defaultValue, required, documentation); + } + + private static Object getDefaultValue(Field f, Object instance) { + final Object defaultValue; + try { + defaultValue = ReflectionUtils.getFieldValue(instance, f); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + return defaultValue; + } + + private static String getType(Field f) { + if (f.getType().isPrimitive()) { + return ClassUtils.primitiveToWrapper(f.getType()).getSimpleName(); + } else { + return f.getType().getSimpleName(); + } + } +} diff --git a/data-node/src/test/java/org/graylog/datanode/ConfigurationDocumentationTest.java b/data-node/src/test/java/org/graylog/datanode/ConfigurationDocumentationTest.java index 7581ee8ef69b..f81089bb6633 100644 --- a/data-node/src/test/java/org/graylog/datanode/ConfigurationDocumentationTest.java +++ b/data-node/src/test/java/org/graylog/datanode/ConfigurationDocumentationTest.java @@ -17,23 +17,14 @@ package org.graylog.datanode; import com.github.joschi.jadconfig.Parameter; -import com.github.joschi.jadconfig.ReflectionUtils; -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVPrinter; -import org.apache.commons.lang3.ClassUtils; import org.bson.assertions.Assertions; import org.graylog.datanode.commands.Server; import org.graylog2.configuration.Documentation; import org.junit.jupiter.api.Test; -import java.io.IOException; -import java.io.StringWriter; import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.util.Arrays; import java.util.List; -import java.util.Optional; import java.util.stream.Collectors; class ConfigurationDocumentationTest { @@ -55,75 +46,4 @@ void testAllFieldsAreDocumented() { Assertions.fail("Following datanode configuration fields require @Documentation annotation: \n" + fields); } } - - /** - * When started, this will output to STDOUT the CSV table of datanode's configuration documentation. - */ - public static void main(String[] args) throws IOException { - final StringWriter stringWriter = new StringWriter(); - try (CSVPrinter printer = new CSVPrinter(stringWriter, CSVFormat.EXCEL)) { - - printer.printRecord("Parameter", "Type", "Required", "Default value", "Description"); - - final List datanodeConfiguration = new Server().getCommandConfigurationBeans(); - - datanodeConfiguration.forEach(configurationBean -> { - Arrays.stream(configurationBean.getClass().getDeclaredFields()) - .filter(f -> f.isAnnotationPresent(Parameter.class)) - .filter(ConfigurationDocumentationTest::isPublicFacing) - .map(f -> toConfigurationField(f, configurationBean)) - .forEach(f -> { - try { - printer.printRecord(f.configName(), f.type(), f.required(), f.defaultValue(), f.documentation()); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - }); - System.out.println(stringWriter); - } - } - - private static boolean isPublicFacing(Field f) { - return !f.isAnnotationPresent(Documentation.class) || f.getAnnotation(Documentation.class).visible(); - } - - private static ConfigurationField toConfigurationField(Field f, Object instance) { - - final String documentation = Optional.ofNullable(f.getAnnotation(Documentation.class)) - .map(Documentation::value).orElse(null); - - final Parameter parameter = f.getAnnotation(Parameter.class); - final String propertyName = parameter.value(); - final Object defaultValue = getDefaultValue(f, instance); - - final String type = getType(f); - - final boolean required = parameter.required(); - - return new ConfigurationField(f.getName(), type, propertyName, defaultValue, required, documentation); - } - - private static Object getDefaultValue(Field f, Object instance) { - final Object defaultValue; - try { - defaultValue = ReflectionUtils.getFieldValue(instance, f); - } catch (IllegalAccessException e) { - throw new RuntimeException(e); - } - return defaultValue; - } - - private static String getType(Field f) { - if (f.getType().isPrimitive()) { - return ClassUtils.primitiveToWrapper(f.getType()).getSimpleName(); - } else { - return f.getType().getSimpleName(); - } - } - - - private record ConfigurationField(String fieldName, String type, String configName, Object defaultValue, - boolean required, String documentation) { - } } diff --git a/graylog2-server/src/main/java/org/graylog2/configuration/MongoDbConfiguration.java b/graylog2-server/src/main/java/org/graylog2/configuration/MongoDbConfiguration.java index 4ece0cdf62ee..41d98c1191e6 100644 --- a/graylog2-server/src/main/java/org/graylog2/configuration/MongoDbConfiguration.java +++ b/graylog2-server/src/main/java/org/graylog2/configuration/MongoDbConfiguration.java @@ -25,8 +25,10 @@ import com.mongodb.MongoClientURI; public class MongoDbConfiguration { - @Documentation("Increase this value according to the maximum connections your MongoDB server can handle from a single client " + - "if you encounter MongoDB connection problems.") + @Documentation(""" + Increase this value according to the maximum connections your MongoDB server can handle from a single client + if you encounter MongoDB connection problems. + """) @Parameter(value = "mongodb_max_connections", validator = PositiveIntegerValidator.class) private int maxConnections = 1000; @@ -34,8 +36,10 @@ public class MongoDbConfiguration { @Parameter(value = "mongodb_uri", required = true, validator = StringNotBlankValidator.class) private String uri = "mongodb://localhost/graylog"; - @Documentation("Maximum number of attempts to connect to MongoDB on boot for the version probe." + - "Default 0 means retry indefinitely until a connection can be established") + @Documentation(""" + Maximum number of attempts to connect to MongoDB on boot for the version probe. + Default 0 means retry indefinitely until a connection can be established + """) @Parameter(value = "mongodb_version_probe_attempts", validators = {PositiveIntegerValidator.class}) int mongodbVersionProbeAttempts = 0; @@ -60,7 +64,7 @@ public MongoClientURI getMongoClientURI() { @ValidatorMethod public void validate() throws ValidationException { - if(getMongoClientURI() == null) { + if (getMongoClientURI() == null) { throw new ValidationException("mongodb_uri is not a valid MongoDB connection string"); } } diff --git a/graylog2-server/src/main/java/org/graylog2/configuration/TLSProtocolsConfiguration.java b/graylog2-server/src/main/java/org/graylog2/configuration/TLSProtocolsConfiguration.java index cbf8daf953af..1e1450907323 100644 --- a/graylog2-server/src/main/java/org/graylog2/configuration/TLSProtocolsConfiguration.java +++ b/graylog2-server/src/main/java/org/graylog2/configuration/TLSProtocolsConfiguration.java @@ -33,8 +33,10 @@ */ public class TLSProtocolsConfiguration { - @Documentation("allowed TLS protocols for system wide TLS enabled servers. (e.g. message inputs, http interface)." + - "Setting this to an empty value, leaves it up to system libraries and the used JDK to chose a default.") + @Documentation(""" + allowed TLS protocols for system wide TLS enabled servers. (e.g. message inputs, http interface) + Setting this to an empty value, leaves it up to system libraries and the used JDK to chose a default. + """) @Parameter(value = "enabled_tls_protocols", converter = StringSetConverter.class) private Set enabledTlsProtocols = null; From 228862a8cf2bd17776bf7ae03a18e9d9312ec5f6 Mon Sep 17 00:00:00 2001 From: Tomas Dvorak Date: Thu, 25 Apr 2024 14:12:30 +0200 Subject: [PATCH 2/5] code cleanup, tests --- data-node/pom.xml | 4 +- .../datanode/docs/ConfigFilePrinter.java | 140 ------------------ .../docs/ConfigurationDocsGenerator.java | 138 +++++++++++++++++ .../datanode/docs/ConfigurationEntry.java | 43 ++++++ .../datanode/docs/DocumentationFormat.java | 20 +++ .../docs/GenerateConfigDocumentation.java | 109 -------------- .../docs/printers/ConfigFileDocsPrinter.java | 139 +++++++++++++++++ .../CsvDocsPrinter.java} | 23 +-- .../DocsPrinter.java} | 16 +- .../docs/GenerateConfigDocumentationTest.java | 94 ++++++++++++ 10 files changed, 454 insertions(+), 272 deletions(-) delete mode 100644 data-node/src/main/java/org/graylog/datanode/docs/ConfigFilePrinter.java create mode 100644 data-node/src/main/java/org/graylog/datanode/docs/ConfigurationDocsGenerator.java create mode 100644 data-node/src/main/java/org/graylog/datanode/docs/ConfigurationEntry.java create mode 100644 data-node/src/main/java/org/graylog/datanode/docs/DocumentationFormat.java delete mode 100644 data-node/src/main/java/org/graylog/datanode/docs/GenerateConfigDocumentation.java create mode 100644 data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigFileDocsPrinter.java rename data-node/src/main/java/org/graylog/datanode/docs/{CsvConfigurationDocumentationPrinter.java => printers/CsvDocsPrinter.java} (59%) rename data-node/src/main/java/org/graylog/datanode/docs/{ConfigurationDocumentationPrinter.java => printers/DocsPrinter.java} (62%) create mode 100644 data-node/src/test/java/org/graylog/datanode/docs/GenerateConfigDocumentationTest.java diff --git a/data-node/pom.xml b/data-node/pom.xml index 1e457a608a46..5f4ddef33dc3 100644 --- a/data-node/pom.xml +++ b/data-node/pom.xml @@ -660,7 +660,7 @@ java - org.graylog.datanode.docs.GenerateConfigDocumentation + org.graylog.datanode.docs.ConfigurationDocsGenerator csv ${project.build.directory}/datanode-conf-docs.csv @@ -674,7 +674,7 @@ java - org.graylog.datanode.docs.GenerateConfigDocumentation + org.graylog.datanode.docs.ConfigurationDocsGenerator conf ${project.build.directory}/datanode.conf.example diff --git a/data-node/src/main/java/org/graylog/datanode/docs/ConfigFilePrinter.java b/data-node/src/main/java/org/graylog/datanode/docs/ConfigFilePrinter.java deleted file mode 100644 index 78a9e4b03518..000000000000 --- a/data-node/src/main/java/org/graylog/datanode/docs/ConfigFilePrinter.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (C) 2020 Graylog, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * . - */ -package org.graylog.datanode.docs; - -import org.apache.commons.lang.WordUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.util.Arrays; -import java.util.Locale; -import java.util.Optional; -import java.util.stream.Collectors; - -public class ConfigFilePrinter implements ConfigurationDocumentationPrinter { - - public static final String DATANODE_CONFIG_HEADER = """ - ##################################### - # GRAYLOG DATANODE CONFIGURATION FILE - ##################################### - # - # This is the Graylog DataNode configuration file. The file has to use ISO 8859-1/Latin-1 character encoding. - # Characters that cannot be directly represented in this encoding can be written using Unicode escapes - # as defined in https://docs.oracle.com/javase/specs/jls/se8/html/jls-3.html#jls-3.3, using the \\u prefix. - # For example, \\u002c. - # - # * Entries are generally expected to be a single line of the form, one of the following: - # - # propertyName=propertyValue - # propertyName:propertyValue - # - # * White space that appears between the property name and property value is ignored, - # so the following are equivalent: - # - # name=Stephen - # name = Stephen - # - # * White space at the beginning of the line is also ignored. - # - # * Lines that start with the comment characters ! or # are ignored. Blank lines are also ignored. - # - # * The property value is generally terminated by the end of the line. White space following the - # property value is not ignored, and is treated as part of the property value. - # - # * A property value can span several lines if each line is terminated by a backslash (‘\\’) character. - # For example: - # - # targetCities=\\ - # Detroit,\\ - # Chicago,\\ - # Los Angeles - # - # This is equivalent to targetCities=Detroit,Chicago,Los Angeles (white space at the beginning of lines is ignored). - # - # * The characters newline, carriage return, and tab can be inserted with characters \\n, \\r, and \\t, respectively. - # - # * The backslash character must be escaped as a double backslash. For example: - # - # path=c:\\\\docs\\\\doc1 - # - - - """; - - private static final Logger LOG = LoggerFactory.getLogger(ConfigFilePrinter.class); - - private final OutputStreamWriter writer; - - public ConfigFilePrinter(OutputStreamWriter writer) { - this.writer = writer; - } - - @Override - public void close() throws IOException { - writer.close(); - } - - @Override - public void flush() throws IOException { - writer.flush(); - } - - - - public void writeHeader() throws IOException { - writer.append(DATANODE_CONFIG_HEADER); - } - - @Override - public void writeField(ConfigurationDocumentationPrinter.ConfigurationField field) throws IOException { - writer.append(toString(field)); - } - - private String toString(ConfigurationDocumentationPrinter.ConfigurationField field) { - // enabled with empty value, force to fill in - final boolean forceFillIn = field.required() && field.defaultValue() == null; - - - String template = """ - %s - %s%s=%s - - """; - - return String.format(Locale.ROOT, template, formatDocumentation(field), forceFillIn ? "" : "# ", field.configName(), wrapValue(field.defaultValue())); - } - - private String wrapValue(Object value) { - return Optional.ofNullable(value).map(String::valueOf).orElse(""); - } - - private static String formatDocumentation(ConfigurationDocumentationPrinter.ConfigurationField field) { - final String[] lines = field.documentation().split("\n"); - return Arrays.stream(lines) - .map(String::trim) - .peek(line -> { - if (line.length() > 120) { - LOG.warn("Documentation line of " + field.bean().getName() + "." + field.fieldName() + " too long, consider splitting into more lines: " + WordUtils.abbreviate(line, 120, 130, "...")); - } - }) - //.flatMap(line -> Arrays.stream(WordUtils.wrap(line, 120).split(System.lineSeparator()))) - .map(line -> "# " + line) - .collect(Collectors.joining("\n")); - } -} diff --git a/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationDocsGenerator.java b/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationDocsGenerator.java new file mode 100644 index 000000000000..7d6cc7ca5ca7 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationDocsGenerator.java @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.docs; + +import com.github.joschi.jadconfig.Parameter; +import com.github.joschi.jadconfig.ReflectionUtils; +import jakarta.annotation.Nonnull; +import org.apache.commons.lang3.ClassUtils; +import org.graylog.datanode.commands.Server; +import org.graylog.datanode.docs.printers.ConfigFileDocsPrinter; +import org.graylog.datanode.docs.printers.CsvDocsPrinter; +import org.graylog.datanode.docs.printers.DocsPrinter; +import org.graylog2.configuration.Documentation; + +import java.io.FileWriter; +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.function.Supplier; +import java.util.stream.Stream; + +public class ConfigurationDocsGenerator { + + /** + * This class is linked from the datanode pom.xml and generates conf.example and csv documentation. + */ + public static void main(String[] args) throws IOException { + final ConfigurationDocsGenerator generator = new ConfigurationDocsGenerator(); + generator.generateDocumentation(parseDocumentationFormat(args), ConfigurationDocsGenerator::getDatanodeConfigurationBeans); + } + + @Nonnull + private static DocumentationFormat parseDocumentationFormat(String[] args) { + if (args.length != 2) { + throw new IllegalArgumentException("This command needs two arguments - a format and file path. For example" + "csv ${project.build.directory}/configuration-documentation.csv"); + } + final String format = args[0].toLowerCase(Locale.ROOT).trim(); + final String file = args[1]; + return new DocumentationFormat(format, file); + } + + protected void generateDocumentation(DocumentationFormat format, Supplier> configurationBeans) throws IOException { + final List configuration = detectConfigurationFields(configurationBeans); + try (final DocsPrinter writer = createWriter(format)) { + writer.writeHeader(); + + for (ConfigurationEntry f : configuration) { + writer.writeField(f); + } + } + } + + private DocsPrinter createWriter(DocumentationFormat format) throws IOException { + final FileWriter fileWriter = new FileWriter(format.outputFile(), StandardCharsets.UTF_8); + return switch (format.format()) { + case "csv" -> new CsvDocsPrinter(fileWriter); + case "conf" -> new ConfigFileDocsPrinter(fileWriter); + default -> throw new IllegalArgumentException("Unsupported format " + format.format()); + }; + } + + /** + * Collects all configuration options from all available configuration beans. + */ + private List detectConfigurationFields(Supplier> configurationBeans) { + return configurationBeans.get() + .stream() + .flatMap(ConfigurationDocsGenerator::beanToConfigEntries) + .sorted(Comparator.comparing(ConfigurationEntry::isPriority).reversed()) + .toList(); + } + + @Nonnull + private static Stream beanToConfigEntries(Object configurationBean) { + return Arrays.stream(configurationBean.getClass().getDeclaredFields()) + .filter(f -> f.isAnnotationPresent(Parameter.class)) + .filter(ConfigurationDocsGenerator::isPublicFacing) + .map(f -> toConfigurationEntry(f, configurationBean)); + } + + private static List getDatanodeConfigurationBeans() { + return new Server().getCommandConfigurationBeans(); + } + + /** + * There are some configuration options not intended for general usage, mainly just for system packages configuration. + * + * @see Documentation#visible() + */ + private static boolean isPublicFacing(Field f) { + return !f.isAnnotationPresent(Documentation.class) || f.getAnnotation(Documentation.class).visible(); + } + + private static ConfigurationEntry toConfigurationEntry(Field f, Object instance) { + final String documentation = Optional.ofNullable(f.getAnnotation(Documentation.class)).map(Documentation::value).orElse(null); + final Parameter parameter = f.getAnnotation(Parameter.class); + final String propertyName = parameter.value(); + final Object defaultValue = getDefaultValue(f, instance); + final String type = getType(f); + final boolean required = parameter.required(); + return new ConfigurationEntry(instance.getClass(), f.getName(), type, propertyName, defaultValue, required, documentation); + } + + private static Object getDefaultValue(Field f, Object instance) { + try { + return ReflectionUtils.getFieldValue(instance, f); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + private static String getType(Field f) { + if (f.getType().isPrimitive()) { // unify primitive types and wrappers, e.g. int -> Integer + return ClassUtils.primitiveToWrapper(f.getType()).getSimpleName(); + } else { + return f.getType().getSimpleName(); + } + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationEntry.java b/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationEntry.java new file mode 100644 index 000000000000..64c6883274e9 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationEntry.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.docs; + +import jakarta.annotation.Nullable; + +/** + * @param configurationBean Class that defines this configuration entry + * @param fieldName java field name + * @param type Java type name, e.g. String or Integer. + * @param configName configuration property name, as written in the config file + * @param defaultValue default value declared in the java field, null if not defined + * @param required if the configuration property is mandatory (needs default or entry in the config file) + * @param documentation textual documentation of this configuration propery + */ +public record ConfigurationEntry( + Class configurationBean, + String fieldName, + String type, + String configName, + @Nullable Object defaultValue, + boolean required, + String documentation +) { + + public boolean isPriority() { + return required && defaultValue == null; + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/docs/DocumentationFormat.java b/data-node/src/main/java/org/graylog/datanode/docs/DocumentationFormat.java new file mode 100644 index 000000000000..f1dd593bbded --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/docs/DocumentationFormat.java @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.docs; + +public record DocumentationFormat(String format, String outputFile) { +} diff --git a/data-node/src/main/java/org/graylog/datanode/docs/GenerateConfigDocumentation.java b/data-node/src/main/java/org/graylog/datanode/docs/GenerateConfigDocumentation.java deleted file mode 100644 index 23c2061e7ca1..000000000000 --- a/data-node/src/main/java/org/graylog/datanode/docs/GenerateConfigDocumentation.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (C) 2020 Graylog, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * . - */ -package org.graylog.datanode.docs; - -import com.github.joschi.jadconfig.Parameter; -import com.github.joschi.jadconfig.ReflectionUtils; -import org.apache.commons.lang3.ClassUtils; -import org.graylog.datanode.commands.Server; -import org.graylog2.configuration.Documentation; - -import java.io.FileWriter; -import java.io.IOException; -import java.lang.reflect.Field; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.Comparator; -import java.util.List; -import java.util.Locale; -import java.util.Optional; - -public class GenerateConfigDocumentation { - - /** - * When started, this will output to STDOUT the CSV table of datanode's configuration documentation. - */ - public static void main(String[] args) throws IOException { - final List datanodeConfiguration = new Server().getCommandConfigurationBeans(); - final List configuration = detectConfigurationFields(datanodeConfiguration); - try (final ConfigurationDocumentationPrinter writer = createWriter(args)) { - writer.writeHeader(); - configuration.forEach(f -> { - try { - writer.writeField(f); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - } - } - - private static ConfigurationDocumentationPrinter createWriter(String[] args) throws IOException { - assert args.length >= 2; - final String format = args[0].toLowerCase(Locale.ROOT).trim(); - final String file = args[1]; - - final FileWriter fileWriter = new FileWriter(file, StandardCharsets.UTF_8); - - return switch (format) { - case "csv" -> new CsvConfigurationDocumentationPrinter(fileWriter); - case "conf" -> new ConfigFilePrinter(fileWriter); - default -> throw new IllegalArgumentException("Unsupported format " + format); - }; - } - - private static List detectConfigurationFields(List datanodeConfiguration) { - return datanodeConfiguration.stream() - .flatMap(configurationBean -> Arrays.stream(configurationBean.getClass().getDeclaredFields()).filter(f -> f.isAnnotationPresent(Parameter.class)).filter(GenerateConfigDocumentation::isPublicFacing).map(f -> toConfigurationField(f, configurationBean))) - .sorted(Comparator.comparing(ConfigurationDocumentationPrinter.ConfigurationField::isPriority).reversed()) - .toList(); - } - - private static boolean isPublicFacing(Field f) { - return !f.isAnnotationPresent(Documentation.class) || f.getAnnotation(Documentation.class).visible(); - } - - private static ConfigurationDocumentationPrinter.ConfigurationField toConfigurationField(Field f, Object instance) { - - final String documentation = Optional.ofNullable(f.getAnnotation(Documentation.class)).map(Documentation::value).orElse(null); - - final Parameter parameter = f.getAnnotation(Parameter.class); - final String propertyName = parameter.value(); - final Object defaultValue = getDefaultValue(f, instance); - final String type = getType(f); - final boolean required = parameter.required(); - return new ConfigurationDocumentationPrinter.ConfigurationField(instance.getClass(), f.getName(), type, propertyName, defaultValue, required, documentation); - } - - private static Object getDefaultValue(Field f, Object instance) { - final Object defaultValue; - try { - defaultValue = ReflectionUtils.getFieldValue(instance, f); - } catch (IllegalAccessException e) { - throw new RuntimeException(e); - } - return defaultValue; - } - - private static String getType(Field f) { - if (f.getType().isPrimitive()) { - return ClassUtils.primitiveToWrapper(f.getType()).getSimpleName(); - } else { - return f.getType().getSimpleName(); - } - } -} diff --git a/data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigFileDocsPrinter.java b/data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigFileDocsPrinter.java new file mode 100644 index 000000000000..7601fc23e83c --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigFileDocsPrinter.java @@ -0,0 +1,139 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.docs.printers; + +import org.apache.commons.lang.WordUtils; +import org.graylog.datanode.docs.ConfigurationEntry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.util.Arrays; +import java.util.Locale; +import java.util.Optional; +import java.util.stream.Collectors; + +public class ConfigFileDocsPrinter implements DocsPrinter { + + public static final String DATANODE_CONFIG_HEADER = """ + ##################################### + # GRAYLOG DATANODE CONFIGURATION FILE + ##################################### + # + # This is the Graylog DataNode configuration file. The file has to use ISO 8859-1/Latin-1 character encoding. + # Characters that cannot be directly represented in this encoding can be written using Unicode escapes + # as defined in https://docs.oracle.com/javase/specs/jls/se8/html/jls-3.html#jls-3.3, using the \\u prefix. + # For example, \\u002c. + # + # * Entries are generally expected to be a single line of the form, one of the following: + # + # propertyName=propertyValue + # propertyName:propertyValue + # + # * White space that appears between the property name and property value is ignored, + # so the following are equivalent: + # + # name=Stephen + # name = Stephen + # + # * White space at the beginning of the line is also ignored. + # + # * Lines that start with the comment characters ! or # are ignored. Blank lines are also ignored. + # + # * The property value is generally terminated by the end of the line. White space following the + # property value is not ignored, and is treated as part of the property value. + # + # * A property value can span several lines if each line is terminated by a backslash (‘\\’) character. + # For example: + # + # targetCities=\\ + # Detroit,\\ + # Chicago,\\ + # Los Angeles + # + # This is equivalent to targetCities=Detroit,Chicago,Los Angeles (white space at the beginning of lines is ignored). + # + # * The characters newline, carriage return, and tab can be inserted with characters \\n, \\r, and \\t, respectively. + # + # * The backslash character must be escaped as a double backslash. For example: + # + # path=c:\\\\docs\\\\doc1 + # + + + """; + + private static final Logger LOG = LoggerFactory.getLogger(ConfigFileDocsPrinter.class); + + private final OutputStreamWriter writer; + + public ConfigFileDocsPrinter(OutputStreamWriter writer) { + this.writer = writer; + } + + @Override + public void close() throws IOException { + writer.close(); + } + + @Override + public void flush() throws IOException { + writer.flush(); + } + + + public void writeHeader() throws IOException { + writer.append(DATANODE_CONFIG_HEADER); + } + + @Override + public void writeField(ConfigurationEntry field) throws IOException { + writer.append(toString(field)); + } + + private String toString(ConfigurationEntry field) { + // enabled with empty value, force to fill in + final boolean forceFillIn = field.required() && field.defaultValue() == null; + + + String template = """ + %s + %s%s=%s + + """; + + return String.format(Locale.ROOT, template, formatDocumentation(field), forceFillIn ? "" : "# ", field.configName(), wrapValue(field.defaultValue())); + } + + private String wrapValue(Object value) { + return Optional.ofNullable(value).map(String::valueOf).orElse(""); + } + + private static String formatDocumentation(ConfigurationEntry field) { + final String[] lines = field.documentation().split("\n"); + return Arrays.stream(lines) + .map(String::trim) + .peek(line -> { + if (line.length() > 120) { + LOG.warn("Documentation line of " + field.configurationBean().getName() + "." + field.fieldName() + " too long, consider splitting into more lines: " + WordUtils.abbreviate(line, 120, 130, "...")); + } + }) + .map(line -> "# " + line) + .collect(Collectors.joining("\n")); + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/docs/CsvConfigurationDocumentationPrinter.java b/data-node/src/main/java/org/graylog/datanode/docs/printers/CsvDocsPrinter.java similarity index 59% rename from data-node/src/main/java/org/graylog/datanode/docs/CsvConfigurationDocumentationPrinter.java rename to data-node/src/main/java/org/graylog/datanode/docs/printers/CsvDocsPrinter.java index 3b54ab60d3fb..68e0e3076b32 100644 --- a/data-node/src/main/java/org/graylog/datanode/docs/CsvConfigurationDocumentationPrinter.java +++ b/data-node/src/main/java/org/graylog/datanode/docs/printers/CsvDocsPrinter.java @@ -14,33 +14,36 @@ * along with this program. If not, see * . */ -package org.graylog.datanode.docs; +package org.graylog.datanode.docs.printers; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVPrinter; +import org.graylog.datanode.docs.ConfigurationEntry; import java.io.IOException; import java.io.OutputStreamWriter; -public class CsvConfigurationDocumentationPrinter implements ConfigurationDocumentationPrinter { +public class CsvDocsPrinter implements DocsPrinter { + public static final String HEADER_PARAMETER = "Parameter"; + public static final String HEADER_TYPE = "Type"; + public static final String HEADER_REQUIRED = "Required"; + public static final String HEADER_DEFAULT_VALUE = "Default value"; + public static final String HEADER_DESCRIPTION = "Description"; + public static final String[] HEADERS = {HEADER_PARAMETER, HEADER_TYPE, HEADER_REQUIRED, HEADER_DEFAULT_VALUE, HEADER_DESCRIPTION}; private final CSVPrinter csvPRinter; - public CsvConfigurationDocumentationPrinter(OutputStreamWriter streamWriter) { - try { - this.csvPRinter = new CSVPrinter(streamWriter, CSVFormat.EXCEL); - } catch (IOException e) { - throw new RuntimeException(e); - } + public CsvDocsPrinter(OutputStreamWriter streamWriter) throws IOException { + this.csvPRinter = new CSVPrinter(streamWriter, CSVFormat.EXCEL); } @Override public void writeHeader() throws IOException { - csvPRinter.printRecord("Parameter", "Type", "Required", "Default value", "Description"); + csvPRinter.printRecord(HEADERS); } @Override - public void writeField(ConfigurationField f) throws IOException { + public void writeField(ConfigurationEntry f) throws IOException { this.csvPRinter.printRecord(f.configName(), f.type(), f.required(), f.defaultValue(), f.documentation()); } diff --git a/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationDocumentationPrinter.java b/data-node/src/main/java/org/graylog/datanode/docs/printers/DocsPrinter.java similarity index 62% rename from data-node/src/main/java/org/graylog/datanode/docs/ConfigurationDocumentationPrinter.java rename to data-node/src/main/java/org/graylog/datanode/docs/printers/DocsPrinter.java index 9b60184d9140..7934e35b37f3 100644 --- a/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationDocumentationPrinter.java +++ b/data-node/src/main/java/org/graylog/datanode/docs/printers/DocsPrinter.java @@ -14,22 +14,16 @@ * along with this program. If not, see * . */ -package org.graylog.datanode.docs; +package org.graylog.datanode.docs.printers; + +import org.graylog.datanode.docs.ConfigurationEntry; import java.io.Closeable; import java.io.Flushable; import java.io.IOException; -public interface ConfigurationDocumentationPrinter extends Flushable, Closeable { +public interface DocsPrinter extends Flushable, Closeable { void writeHeader() throws IOException; - void writeField(ConfigurationField configurationField) throws IOException; - - - record ConfigurationField(Class bean, String fieldName, String type, String configName, Object defaultValue, - boolean required, String documentation) { - public boolean isPriority() { - return required && defaultValue == null; - } - } + void writeField(ConfigurationEntry configurationEntry) throws IOException; } diff --git a/data-node/src/test/java/org/graylog/datanode/docs/GenerateConfigDocumentationTest.java b/data-node/src/test/java/org/graylog/datanode/docs/GenerateConfigDocumentationTest.java new file mode 100644 index 000000000000..ab05967e6183 --- /dev/null +++ b/data-node/src/test/java/org/graylog/datanode/docs/GenerateConfigDocumentationTest.java @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.docs; + +import com.github.joschi.jadconfig.Parameter; +import com.github.joschi.jadconfig.validators.PositiveIntegerValidator; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.assertj.core.api.Assertions; +import org.graylog.datanode.Configuration; +import org.graylog.datanode.docs.printers.CsvDocsPrinter; +import org.graylog2.configuration.Documentation; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.FileReader; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +class GenerateConfigDocumentationTest { + + private ConfigurationDocsGenerator generator; + + @BeforeEach + void setUp() { + this.generator = new ConfigurationDocsGenerator(); + } + + @Test + void testCsv(@TempDir Path tmpPath) throws IOException { + final Path file = tmpPath.resolve("my-documentation.csv"); + final DocumentationFormat format = new DocumentationFormat("csv", file.toFile().getAbsolutePath()); + generator.generateDocumentation(format, () -> List.of(new DummyConfiguration())); + + final CSVFormat csvFormat = CSVFormat.Builder.create(CSVFormat.EXCEL) + .setHeader(CsvDocsPrinter.HEADERS) + .setSkipHeaderRecord(true) + .build(); + + final CSVParser parser = new CSVParser(new FileReader(file.toFile(), StandardCharsets.UTF_8), csvFormat); + final List lines = parser.getRecords(); + Assertions.assertThat(lines) + .hasSize(1) // one entry only, the invisible should not be present in the output + .anySatisfy(line -> { + Assertions.assertThat(line.get(CsvDocsPrinter.HEADER_DEFAULT_VALUE)).isEqualTo("data/node-id"); + Assertions.assertThat(line.get(CsvDocsPrinter.HEADER_PARAMETER)).isEqualTo("node_id_file"); + Assertions.assertThat(line.get(CsvDocsPrinter.HEADER_TYPE)).isEqualTo("String"); + Assertions.assertThat(line.get(CsvDocsPrinter.HEADER_REQUIRED)).isEqualTo("false"); + Assertions.assertThat(line.get(CsvDocsPrinter.HEADER_DESCRIPTION)).contains("The auto-generated node ID will be stored in this file and read after restarts"); + }); + } + + @Test + void testConfFile(@TempDir Path tmpPath) throws IOException { + final Path file = tmpPath.resolve("config-example.conf"); + final DocumentationFormat format = new DocumentationFormat("conf", file.toFile().getAbsolutePath()); + generator.generateDocumentation(format, () -> List.of(new DummyConfiguration())); + final String content = Files.readString(file); + Assertions.assertThat(content).contains("The auto-generated node ID will be stored in this file and read after restarts"); + Assertions.assertThat(content).contains("# node_id_file=data/node-id"); + } + + private static class DummyConfiguration { + @Documentation(""" + The auto-generated node ID will be stored in this file and read after restarts. It is a good idea + to use an absolute file path here if you are starting Graylog DataNode from init scripts or similar. + """) + @Parameter(value = "node_id_file", validators = Configuration.NodeIdFileValidator.class) + private String nodeIdFile = "data/node-id"; + + @Documentation(visible = false) + @Parameter(value = "timeout_sec", validators = PositiveIntegerValidator.class) + private Integer timeoutSec; + } +} From 4bf1ff6262717d2432dcd8cc2a0850973bd4c6d3 Mon Sep 17 00:00:00 2001 From: Tomas Dvorak Date: Tue, 14 May 2024 13:26:51 +0200 Subject: [PATCH 3/5] add spaces around = in generated config example --- .../graylog/datanode/docs/printers/ConfigFileDocsPrinter.java | 2 +- .../graylog/datanode/docs/GenerateConfigDocumentationTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigFileDocsPrinter.java b/data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigFileDocsPrinter.java index 7601fc23e83c..86b4d590a5e7 100644 --- a/data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigFileDocsPrinter.java +++ b/data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigFileDocsPrinter.java @@ -113,7 +113,7 @@ private String toString(ConfigurationEntry field) { String template = """ %s - %s%s=%s + %s%s = %s """; diff --git a/data-node/src/test/java/org/graylog/datanode/docs/GenerateConfigDocumentationTest.java b/data-node/src/test/java/org/graylog/datanode/docs/GenerateConfigDocumentationTest.java index ab05967e6183..655fa06e7a29 100644 --- a/data-node/src/test/java/org/graylog/datanode/docs/GenerateConfigDocumentationTest.java +++ b/data-node/src/test/java/org/graylog/datanode/docs/GenerateConfigDocumentationTest.java @@ -76,7 +76,7 @@ void testConfFile(@TempDir Path tmpPath) throws IOException { generator.generateDocumentation(format, () -> List.of(new DummyConfiguration())); final String content = Files.readString(file); Assertions.assertThat(content).contains("The auto-generated node ID will be stored in this file and read after restarts"); - Assertions.assertThat(content).contains("# node_id_file=data/node-id"); + Assertions.assertThat(content).contains("# node_id_file = data/node-id"); } private static class DummyConfiguration { From ffed46e7c403aa2832552a5cc16ac961dbd57387 Mon Sep 17 00:00:00 2001 From: Tomas Dvorak Date: Tue, 14 May 2024 14:05:45 +0200 Subject: [PATCH 4/5] adjust spaces in generated config files, add tests for ordering --- .../docs/printers/ConfigFileDocsPrinter.java | 2 +- .../docs/GenerateConfigDocumentationTest.java | 29 +++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigFileDocsPrinter.java b/data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigFileDocsPrinter.java index 86b4d590a5e7..e7c46a7f8823 100644 --- a/data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigFileDocsPrinter.java +++ b/data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigFileDocsPrinter.java @@ -117,7 +117,7 @@ private String toString(ConfigurationEntry field) { """; - return String.format(Locale.ROOT, template, formatDocumentation(field), forceFillIn ? "" : "# ", field.configName(), wrapValue(field.defaultValue())); + return String.format(Locale.ROOT, template, formatDocumentation(field), forceFillIn ? "" : "#", field.configName(), wrapValue(field.defaultValue())); } private String wrapValue(Object value) { diff --git a/data-node/src/test/java/org/graylog/datanode/docs/GenerateConfigDocumentationTest.java b/data-node/src/test/java/org/graylog/datanode/docs/GenerateConfigDocumentationTest.java index 655fa06e7a29..678b2dfd0c4d 100644 --- a/data-node/src/test/java/org/graylog/datanode/docs/GenerateConfigDocumentationTest.java +++ b/data-node/src/test/java/org/graylog/datanode/docs/GenerateConfigDocumentationTest.java @@ -17,7 +17,9 @@ package org.graylog.datanode.docs; import com.github.joschi.jadconfig.Parameter; +import com.github.joschi.jadconfig.util.Duration; import com.github.joschi.jadconfig.validators.PositiveIntegerValidator; +import com.github.joschi.jadconfig.validators.StringNotBlankValidator; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; @@ -59,7 +61,7 @@ void testCsv(@TempDir Path tmpPath) throws IOException { final CSVParser parser = new CSVParser(new FileReader(file.toFile(), StandardCharsets.UTF_8), csvFormat); final List lines = parser.getRecords(); Assertions.assertThat(lines) - .hasSize(1) // one entry only, the invisible should not be present in the output + .hasSize(3) // one entry only, the invisible should not be present in the output .anySatisfy(line -> { Assertions.assertThat(line.get(CsvDocsPrinter.HEADER_DEFAULT_VALUE)).isEqualTo("data/node-id"); Assertions.assertThat(line.get(CsvDocsPrinter.HEADER_PARAMETER)).isEqualTo("node_id_file"); @@ -67,6 +69,12 @@ void testCsv(@TempDir Path tmpPath) throws IOException { Assertions.assertThat(line.get(CsvDocsPrinter.HEADER_REQUIRED)).isEqualTo("false"); Assertions.assertThat(line.get(CsvDocsPrinter.HEADER_DESCRIPTION)).contains("The auto-generated node ID will be stored in this file and read after restarts"); }); + + // Assert order of the lines. Required password_secret without default value has to go first. Then the rest follows + // order of the properties in the class. + Assertions.assertThat(lines.get(0).get(CsvDocsPrinter.HEADER_PARAMETER)).isEqualTo("password_secret"); + Assertions.assertThat(lines.get(1).get(CsvDocsPrinter.HEADER_PARAMETER)).isEqualTo("node_id_file"); + Assertions.assertThat(lines.get(2).get(CsvDocsPrinter.HEADER_PARAMETER)).isEqualTo("indexer_jwt_auth_token_caching_duration"); } @Test @@ -76,7 +84,7 @@ void testConfFile(@TempDir Path tmpPath) throws IOException { generator.generateDocumentation(format, () -> List.of(new DummyConfiguration())); final String content = Files.readString(file); Assertions.assertThat(content).contains("The auto-generated node ID will be stored in this file and read after restarts"); - Assertions.assertThat(content).contains("# node_id_file = data/node-id"); + Assertions.assertThat(content).contains("#node_id_file = data/node-id"); } private static class DummyConfiguration { @@ -90,5 +98,22 @@ private static class DummyConfiguration { @Documentation(visible = false) @Parameter(value = "timeout_sec", validators = PositiveIntegerValidator.class) private Integer timeoutSec; + + @Documentation(""" + You MUST set a secret to secure/pepper the stored user passwords here. Use at least 64 characters. + Generate one by using for example: pwgen -N 1 -s 96 + ATTENTION: This value must be the same on all Graylog and Datanode nodes in the cluster. + Changing this value after installation will render all user sessions and encrypted values + in the database invalid. (e.g. encrypted access tokens) + """) + @Parameter(value = "password_secret", required = true, validators = StringNotBlankValidator.class) + private String passwordSecret; + + @Documentation(""" + communication between Graylog and OpenSearch is secured by JWT. + This configuration defines interval between token regenerations. + """) + @Parameter(value = "indexer_jwt_auth_token_caching_duration") + Duration indexerJwtAuthTokenCachingDuration = Duration.seconds(60); } } From 5a1c5d2ff33e9e0457983b44ebdc3d51e6913b70 Mon Sep 17 00:00:00 2001 From: Tomas Dvorak Date: Wed, 19 Jun 2024 10:54:08 +0200 Subject: [PATCH 5/5] Added support for sections - both on class level and field level --- .../org/graylog/datanode/Configuration.java | 18 ++- .../docs/ConfigurationDocsGenerator.java | 72 +++++++++--- .../datanode/docs/ConfigurationEntry.java | 2 +- .../docs/ConfigurationEntryWithSection.java | 23 ++++ .../datanode/docs/DocumentationConstants.java | 62 ++++++++++ .../docs/printers/ConfigFileDocsPrinter.java | 110 ++++++++---------- .../docs/printers/ConfigurationSection.java | 27 +++++ .../docs/printers/CsvDocsPrinter.java | 29 ++++- .../datanode/docs/printers/DocsPrinter.java | 7 +- .../docs/GenerateConfigDocumentationTest.java | 4 +- .../configuration/DocumentationSection.java | 31 +++++ 11 files changed, 289 insertions(+), 96 deletions(-) create mode 100644 data-node/src/main/java/org/graylog/datanode/docs/ConfigurationEntryWithSection.java create mode 100644 data-node/src/main/java/org/graylog/datanode/docs/DocumentationConstants.java create mode 100644 data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigurationSection.java create mode 100644 graylog2-server/src/main/java/org/graylog2/configuration/DocumentationSection.java diff --git a/data-node/src/main/java/org/graylog/datanode/Configuration.java b/data-node/src/main/java/org/graylog/datanode/Configuration.java index 8712f1f68568..3dc96b6f9c85 100644 --- a/data-node/src/main/java/org/graylog/datanode/Configuration.java +++ b/data-node/src/main/java/org/graylog/datanode/Configuration.java @@ -32,8 +32,10 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.net.InetAddresses; import org.graylog.datanode.configuration.DatanodeDirectories; +import org.graylog.datanode.docs.DocumentationConstants; import org.graylog2.Configuration.SafeClassesValidator; import org.graylog2.configuration.Documentation; +import org.graylog2.configuration.DocumentationSection; import org.graylog2.plugin.Tools; import org.graylog2.shared.SuppressForbidden; import org.slf4j.Logger; @@ -57,7 +59,9 @@ * Helper class to hold configuration of DataNode */ @SuppressWarnings("FieldMayBeFinal") +@DocumentationSection(heading = "GRAYLOG DATANODE CONFIGURATION FILE", description = DocumentationConstants.DATANODE_DOCUMENTATION_DESCRIPTION) public class Configuration { + private static final Logger LOG = LoggerFactory.getLogger(Configuration.class); public static final String TRANSPORT_CERTIFICATE_PASSWORD_PROPERTY = "transport_certificate_password"; public static final String HTTP_CERTIFICATE_PASSWORD_PROPERTY = "http_certificate_password"; @@ -182,17 +186,19 @@ public class Configuration { @Parameter(value = "password_secret", required = true, validators = StringNotBlankValidator.class) private String passwordSecret; - @Documentation(""" - communication between Graylog and OpenSearch is secured by JWT. - This configuration defines interval between token regenerations. + @DocumentationSection(heading = "OpenSearch JWT token usage",description = """ + communication between Graylog and OpenSearch is secured by JWT. These are the defaults used for the token usage + adjust them, if you have special needs. """) + @Documentation(value = "This configuration defines interval between token regenerations.") @Parameter(value = "indexer_jwt_auth_token_caching_duration") Duration indexerJwtAuthTokenCachingDuration = Duration.seconds(60); - @Documentation(""" - communication between Graylog and OpenSearch is secured by JWT. - This configuration defines validity interval of JWT tokens. + @DocumentationSection(heading = "OpenSearch JWT token usage",description = """ + communication between Graylog and OpenSearch is secured by JWT. These are the defaults used for the token usage + adjust them, if you have special needs. """) + @Documentation("This configuration defines validity interval of JWT tokens") @Parameter(value = "indexer_jwt_auth_token_expiration_duration") Duration indexerJwtAuthTokenExpirationDuration = Duration.seconds(180); diff --git a/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationDocsGenerator.java b/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationDocsGenerator.java index 7d6cc7ca5ca7..f73d9199fe96 100644 --- a/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationDocsGenerator.java +++ b/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationDocsGenerator.java @@ -22,21 +22,25 @@ import org.apache.commons.lang3.ClassUtils; import org.graylog.datanode.commands.Server; import org.graylog.datanode.docs.printers.ConfigFileDocsPrinter; +import org.graylog.datanode.docs.printers.ConfigurationSection; import org.graylog.datanode.docs.printers.CsvDocsPrinter; import org.graylog.datanode.docs.printers.DocsPrinter; import org.graylog2.configuration.Documentation; +import org.graylog2.configuration.DocumentationSection; import java.io.FileWriter; import java.io.IOException; import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Locale; import java.util.Optional; import java.util.function.Supplier; -import java.util.stream.Stream; +import java.util.stream.Collectors; public class ConfigurationDocsGenerator { @@ -59,13 +63,9 @@ private static DocumentationFormat parseDocumentationFormat(String[] args) { } protected void generateDocumentation(DocumentationFormat format, Supplier> configurationBeans) throws IOException { - final List configuration = detectConfigurationFields(configurationBeans); + final List sections = detectConfigurationSections(configurationBeans); try (final DocsPrinter writer = createWriter(format)) { - writer.writeHeader(); - - for (ConfigurationEntry f : configuration) { - writer.writeField(f); - } + writer.write(sections); } } @@ -81,20 +81,53 @@ private DocsPrinter createWriter(DocumentationFormat format) throws IOException /** * Collects all configuration options from all available configuration beans. */ - private List detectConfigurationFields(Supplier> configurationBeans) { + private List detectConfigurationSections(Supplier> configurationBeans) { return configurationBeans.get() .stream() - .flatMap(ConfigurationDocsGenerator::beanToConfigEntries) - .sorted(Comparator.comparing(ConfigurationEntry::isPriority).reversed()) + .map(ConfigurationDocsGenerator::beanToConfigSections) + .sorted(Comparator.comparing(ConfigurationSection::hasPriority).reversed()) .toList(); } @Nonnull - private static Stream beanToConfigEntries(Object configurationBean) { - return Arrays.stream(configurationBean.getClass().getDeclaredFields()) + private static ConfigurationSection beanToConfigSections(Object configurationBean) { + + String sectionHeading = null; + String sectionDescription = null; + if (configurationBean.getClass().isAnnotationPresent(DocumentationSection.class)) { + final DocumentationSection documentationSection = configurationBean.getClass().getAnnotation(DocumentationSection.class); + sectionHeading = documentationSection.heading(); + sectionDescription = documentationSection.description(); + } + + final List entries = Arrays.stream(configurationBean.getClass().getDeclaredFields()) .filter(f -> f.isAnnotationPresent(Parameter.class)) .filter(ConfigurationDocsGenerator::isPublicFacing) - .map(f -> toConfigurationEntry(f, configurationBean)); + .map(f -> toConfigurationEntry(f, configurationBean)) + .toList(); + + final List entriesWithoutSection = getEntriesWithoutSection(entries); + final List sortedSections = sectionsFromEntries(entries); + + return new ConfigurationSection(sectionHeading, sectionDescription, sortedSections, entriesWithoutSection); + } + + @Nonnull + private static List sectionsFromEntries(List entries) { + final Collection sections = entries.stream() + .filter(ConfigurationEntryWithSection::hasSection) + .collect(Collectors.groupingBy(ConfigurationEntryWithSection::sectionHeading, Collectors.collectingAndThen(Collectors.toList(), list -> new ConfigurationSection(list.iterator().next().sectionHeading(), list.iterator().next().sectionDescription(), Collections.emptyList(), list.stream().map(ConfigurationEntryWithSection::entry).collect(Collectors.toList()))))).values(); + return sections.stream() + .sorted(Comparator.comparing(ConfigurationSection::hasPriority, Comparator.reverseOrder())).toList(); + } + + @Nonnull + private static List getEntriesWithoutSection(List entries) { + return entries.stream() + .filter(e -> !e.hasSection()) + .map(ConfigurationEntryWithSection::entry) + .sorted(Comparator.comparing(ConfigurationEntry::hasPriority, Comparator.reverseOrder())) + .collect(Collectors.toList()); } private static List getDatanodeConfigurationBeans() { @@ -110,14 +143,23 @@ private static boolean isPublicFacing(Field f) { return !f.isAnnotationPresent(Documentation.class) || f.getAnnotation(Documentation.class).visible(); } - private static ConfigurationEntry toConfigurationEntry(Field f, Object instance) { + private static ConfigurationEntryWithSection toConfigurationEntry(Field f, Object instance) { final String documentation = Optional.ofNullable(f.getAnnotation(Documentation.class)).map(Documentation::value).orElse(null); final Parameter parameter = f.getAnnotation(Parameter.class); final String propertyName = parameter.value(); final Object defaultValue = getDefaultValue(f, instance); final String type = getType(f); final boolean required = parameter.required(); - return new ConfigurationEntry(instance.getClass(), f.getName(), type, propertyName, defaultValue, required, documentation); + + final DocumentationSection documentationSection = f.getAnnotation(DocumentationSection.class); + String sectionHeading = null; + String sectionDescription = null; + if (documentationSection != null) { + sectionHeading = documentationSection.heading(); + sectionDescription = documentationSection.description(); + } + final ConfigurationEntry entry = new ConfigurationEntry(instance.getClass(), f.getName(), type, propertyName, defaultValue, required, documentation); + return new ConfigurationEntryWithSection(entry, sectionHeading, sectionDescription); } private static Object getDefaultValue(Field f, Object instance) { diff --git a/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationEntry.java b/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationEntry.java index 64c6883274e9..3590da8d2c7d 100644 --- a/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationEntry.java +++ b/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationEntry.java @@ -37,7 +37,7 @@ public record ConfigurationEntry( String documentation ) { - public boolean isPriority() { + public boolean hasPriority() { return required && defaultValue == null; } } diff --git a/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationEntryWithSection.java b/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationEntryWithSection.java new file mode 100644 index 000000000000..5224246b1a1f --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/docs/ConfigurationEntryWithSection.java @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.docs; + +public record ConfigurationEntryWithSection(ConfigurationEntry entry, String sectionHeading, String sectionDescription) { + public boolean hasSection() { + return sectionHeading != null && !sectionHeading.isBlank(); + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/docs/DocumentationConstants.java b/data-node/src/main/java/org/graylog/datanode/docs/DocumentationConstants.java new file mode 100644 index 000000000000..e3ea120b960c --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/docs/DocumentationConstants.java @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.docs; + +public class DocumentationConstants { + public static final String DATANODE_DOCUMENTATION_DESCRIPTION = """ + This is the Graylog DataNode configuration file. The file has to use ISO 8859-1/Latin-1 character encoding. + Characters that cannot be directly represented in this encoding can be written using Unicode escapes + as defined in https://docs.oracle.com/javase/specs/jls/se8/html/jls-3.html#jls-3.3, using the \\u prefix. + For example, \\u002c. + + * Entries are generally expected to be a single line of the form, one of the following: + + propertyName=propertyValue + propertyName:propertyValue + + * White space that appears between the property name and property value is ignored, + so the following are equivalent: + + name=Stephen + name = Stephen + + * White space at the beginning of the line is also ignored. + + * Lines that start with the comment characters ! or # are ignored. Blank lines are also ignored. + + * The property value is generally terminated by the end of the line. White space following the + property value is not ignored, and is treated as part of the property value. + + * A property value can span several lines if each line is terminated by a backslash (‘\\’) character. + For example: + + targetCities=\\ + Detroit,\\ + Chicago,\\ + Los Angeles + + This is equivalent to targetCities=Detroit,Chicago,Los Angeles (white space at the beginning of lines is ignored). + + * The characters newline, carriage return, and tab can be inserted with characters \\n, \\r, and \\t, respectively. + + * The backslash character must be escaped as a double backslash. For example: + + path=c:\\\\docs\\\\doc1 + + + """; +} diff --git a/data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigFileDocsPrinter.java b/data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigFileDocsPrinter.java index e7c46a7f8823..302a943077d1 100644 --- a/data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigFileDocsPrinter.java +++ b/data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigFileDocsPrinter.java @@ -21,63 +21,18 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; import java.io.IOException; import java.io.OutputStreamWriter; +import java.io.Writer; import java.util.Arrays; +import java.util.List; import java.util.Locale; import java.util.Optional; import java.util.stream.Collectors; public class ConfigFileDocsPrinter implements DocsPrinter { - public static final String DATANODE_CONFIG_HEADER = """ - ##################################### - # GRAYLOG DATANODE CONFIGURATION FILE - ##################################### - # - # This is the Graylog DataNode configuration file. The file has to use ISO 8859-1/Latin-1 character encoding. - # Characters that cannot be directly represented in this encoding can be written using Unicode escapes - # as defined in https://docs.oracle.com/javase/specs/jls/se8/html/jls-3.html#jls-3.3, using the \\u prefix. - # For example, \\u002c. - # - # * Entries are generally expected to be a single line of the form, one of the following: - # - # propertyName=propertyValue - # propertyName:propertyValue - # - # * White space that appears between the property name and property value is ignored, - # so the following are equivalent: - # - # name=Stephen - # name = Stephen - # - # * White space at the beginning of the line is also ignored. - # - # * Lines that start with the comment characters ! or # are ignored. Blank lines are also ignored. - # - # * The property value is generally terminated by the end of the line. White space following the - # property value is not ignored, and is treated as part of the property value. - # - # * A property value can span several lines if each line is terminated by a backslash (‘\\’) character. - # For example: - # - # targetCities=\\ - # Detroit,\\ - # Chicago,\\ - # Los Angeles - # - # This is equivalent to targetCities=Detroit,Chicago,Los Angeles (white space at the beginning of lines is ignored). - # - # * The characters newline, carriage return, and tab can be inserted with characters \\n, \\r, and \\t, respectively. - # - # * The backslash character must be escaped as a double backslash. For example: - # - # path=c:\\\\docs\\\\doc1 - # - - - """; - private static final Logger LOG = LoggerFactory.getLogger(ConfigFileDocsPrinter.class); private final OutputStreamWriter writer; @@ -96,17 +51,50 @@ public void flush() throws IOException { writer.flush(); } + @Override + public void write(List configurationSections) { + configurationSections.forEach(section -> doWriteSection(section, 1)); + } - public void writeHeader() throws IOException { - writer.append(DATANODE_CONFIG_HEADER); + private void doWriteSection(ConfigurationSection configurationSection, int level) { + heading(configurationSection.heading(), level).ifPresent(this::append); + description(configurationSection.description()).ifPresent(this::append); + configurationSection.entries().stream().map(this::fieldToString).forEach(this::append); + configurationSection.sections().forEach(section -> doWriteSection(section, level + 1)); } - @Override - public void writeField(ConfigurationEntry field) throws IOException { - writer.append(toString(field)); + private Optional description(String description) { + return Optional.ofNullable(description) + .map(text -> text.lines().map(l -> "# " + l.trim()).collect(Collectors.joining("\n"))) + .map(text -> text + "\n\n"); + } + + private Writer append(String formatted) { + try { + return writer.append(formatted); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private Optional heading(String heading, int level) { + return Optional.ofNullable(heading).map(h -> headingTemplate(level).formatted(heading)); + } + + @Nonnull + private static String headingTemplate(int level) { + if(level == 1) { + return """ + ##################################### + # %s + ##################################### + """; + } else { + return "#### %s\n"; + } } - private String toString(ConfigurationEntry field) { + private String fieldToString(ConfigurationEntry field) { // enabled with empty value, force to fill in final boolean forceFillIn = field.required() && field.defaultValue() == null; @@ -126,14 +114,10 @@ private String wrapValue(Object value) { private static String formatDocumentation(ConfigurationEntry field) { final String[] lines = field.documentation().split("\n"); - return Arrays.stream(lines) - .map(String::trim) - .peek(line -> { - if (line.length() > 120) { - LOG.warn("Documentation line of " + field.configurationBean().getName() + "." + field.fieldName() + " too long, consider splitting into more lines: " + WordUtils.abbreviate(line, 120, 130, "...")); - } - }) - .map(line -> "# " + line) - .collect(Collectors.joining("\n")); + return Arrays.stream(lines).map(String::trim).peek(line -> { + if (line.length() > 120) { + LOG.warn("Documentation line of " + field.configurationBean().getName() + "." + field.fieldName() + " too long, consider splitting into more lines: " + WordUtils.abbreviate(line, 120, 130, "...")); + } + }).map(line -> "# " + line).collect(Collectors.joining("\n")); } } diff --git a/data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigurationSection.java b/data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigurationSection.java new file mode 100644 index 000000000000..dcd02f202e70 --- /dev/null +++ b/data-node/src/main/java/org/graylog/datanode/docs/printers/ConfigurationSection.java @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.datanode.docs.printers; + +import org.graylog.datanode.docs.ConfigurationEntry; + +import java.util.List; + +public record ConfigurationSection(String heading, String description, List sections, List entries) { + public boolean hasPriority() { + return entries.stream().anyMatch(ConfigurationEntry::hasPriority); + } +} diff --git a/data-node/src/main/java/org/graylog/datanode/docs/printers/CsvDocsPrinter.java b/data-node/src/main/java/org/graylog/datanode/docs/printers/CsvDocsPrinter.java index 68e0e3076b32..722b4590e979 100644 --- a/data-node/src/main/java/org/graylog/datanode/docs/printers/CsvDocsPrinter.java +++ b/data-node/src/main/java/org/graylog/datanode/docs/printers/CsvDocsPrinter.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.OutputStreamWriter; +import java.util.List; public class CsvDocsPrinter implements DocsPrinter { @@ -38,13 +39,31 @@ public CsvDocsPrinter(OutputStreamWriter streamWriter) throws IOException { } @Override - public void writeHeader() throws IOException { - csvPRinter.printRecord(HEADERS); + public void write(List sections) { + printHeaders(); + sections.forEach(this::doWriteSection); } - @Override - public void writeField(ConfigurationEntry f) throws IOException { - this.csvPRinter.printRecord(f.configName(), f.type(), f.required(), f.defaultValue(), f.documentation()); + private void printHeaders() { + try { + csvPRinter.printRecord(HEADERS); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private void doWriteSection(ConfigurationSection section) { + section.entries().forEach(this::writeField); + section.sections().forEach(this::doWriteSection); + } + + + private void writeField(ConfigurationEntry f) { + try { + this.csvPRinter.printRecord(f.configName(), f.type(), f.required(), f.defaultValue(), f.documentation()); + } catch (IOException e) { + throw new RuntimeException(e); + } } @Override diff --git a/data-node/src/main/java/org/graylog/datanode/docs/printers/DocsPrinter.java b/data-node/src/main/java/org/graylog/datanode/docs/printers/DocsPrinter.java index 7934e35b37f3..7335e7f8c96d 100644 --- a/data-node/src/main/java/org/graylog/datanode/docs/printers/DocsPrinter.java +++ b/data-node/src/main/java/org/graylog/datanode/docs/printers/DocsPrinter.java @@ -16,14 +16,11 @@ */ package org.graylog.datanode.docs.printers; -import org.graylog.datanode.docs.ConfigurationEntry; - import java.io.Closeable; import java.io.Flushable; -import java.io.IOException; +import java.util.List; public interface DocsPrinter extends Flushable, Closeable { - void writeHeader() throws IOException; - void writeField(ConfigurationEntry configurationEntry) throws IOException; + void write(List sections); } diff --git a/data-node/src/test/java/org/graylog/datanode/docs/GenerateConfigDocumentationTest.java b/data-node/src/test/java/org/graylog/datanode/docs/GenerateConfigDocumentationTest.java index 678b2dfd0c4d..7bf19d398459 100644 --- a/data-node/src/test/java/org/graylog/datanode/docs/GenerateConfigDocumentationTest.java +++ b/data-node/src/test/java/org/graylog/datanode/docs/GenerateConfigDocumentationTest.java @@ -27,6 +27,7 @@ import org.graylog.datanode.Configuration; import org.graylog.datanode.docs.printers.CsvDocsPrinter; import org.graylog2.configuration.Documentation; +import org.graylog2.configuration.DocumentationSection; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -87,6 +88,7 @@ void testConfFile(@TempDir Path tmpPath) throws IOException { Assertions.assertThat(content).contains("#node_id_file = data/node-id"); } + @DocumentationSection(heading = "my-test-config", description = "this is how you configure your app") private static class DummyConfiguration { @Documentation(""" The auto-generated node ID will be stored in this file and read after restarts. It is a good idea @@ -109,8 +111,8 @@ private static class DummyConfiguration { @Parameter(value = "password_secret", required = true, validators = StringNotBlankValidator.class) private String passwordSecret; + @DocumentationSection(heading = "OpenSearch JWT token usage", description = "Communication between Graylog and OpenSearch is secured by JWT.") @Documentation(""" - communication between Graylog and OpenSearch is secured by JWT. This configuration defines interval between token regenerations. """) @Parameter(value = "indexer_jwt_auth_token_caching_duration") diff --git a/graylog2-server/src/main/java/org/graylog2/configuration/DocumentationSection.java b/graylog2-server/src/main/java/org/graylog2/configuration/DocumentationSection.java new file mode 100644 index 000000000000..c048e0e4afc4 --- /dev/null +++ b/graylog2-server/src/main/java/org/graylog2/configuration/DocumentationSection.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog2.configuration; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE, ElementType.FIELD}) +public @interface DocumentationSection { + String heading(); + String description(); +}