Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Automatically generate datanode.conf.example #19141

Open
wants to merge 12 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions data-node/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -673,6 +673,34 @@
<artifactId>exec-maven-plugin</artifactId>
<groupId>org.codehaus.mojo</groupId>
<executions>
<execution>
<id>generate-csv-docs</id>
<phase>prepare-package</phase>
<goals>
<goal>java</goal>
</goals>
<configuration>
<mainClass>org.graylog.datanode.docs.ConfigurationDocsGenerator</mainClass>
<arguments>
<argument>csv</argument>
<argument>${project.build.directory}/datanode-conf-docs.csv</argument>
</arguments>
</configuration>
</execution>
<execution>
<id>generate-conf-example</id>
<phase>prepare-package</phase>
<goals>
<goal>java</goal>
</goals>
<configuration>
<mainClass>org.graylog.datanode.docs.ConfigurationDocsGenerator</mainClass>
<arguments>
<argument>conf</argument>
<argument>${project.build.directory}/datanode.conf.example</argument>
</arguments>
</configuration>
</execution>
<execution>
<id>install-required-opensearch-plugins-x64</id>
<phase>prepare-package</phase>
Expand Down
2 changes: 1 addition & 1 deletion data-node/src/main/assembly/datanode.xml
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@
<outputDirectory>.</outputDirectory>
</file>
<file>
<source>${project.basedir}/../misc/datanode.conf</source>
<source>${project.build.directory}/datanode.conf.example</source>
<destName>datanode.conf.example</destName>
<outputDirectory>.</outputDirectory>
</file>
Expand Down
82 changes: 60 additions & 22 deletions data-node/src/main/java/org/graylog/datanode/Configuration.java
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,10 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.net.InetAddresses;
import org.graylog.datanode.configuration.DatanodeDirectories;
import org.graylog.datanode.docs.DocumentationConstants;
import org.graylog2.Configuration.SafeClassesValidator;
import org.graylog2.configuration.Documentation;
import org.graylog2.configuration.DocumentationSection;
import org.graylog2.plugin.Tools;
import org.graylog2.shared.SuppressForbidden;
import org.slf4j.Logger;
Expand All @@ -57,7 +59,9 @@
* Helper class to hold configuration of DataNode
*/
@SuppressWarnings("FieldMayBeFinal")
@DocumentationSection(heading = "GRAYLOG DATANODE CONFIGURATION FILE", description = DocumentationConstants.DATANODE_DOCUMENTATION_DESCRIPTION)
public class Configuration {

private static final Logger LOG = LoggerFactory.getLogger(Configuration.class);
public static final String TRANSPORT_CERTIFICATE_PASSWORD_PROPERTY = "transport_certificate_password";
public static final String HTTP_CERTIFICATE_PASSWORD_PROPERTY = "http_certificate_password";
Expand Down Expand Up @@ -86,17 +90,22 @@ public class Configuration {
@Parameter(value = "opensearch_location")
private String opensearchDistributionRoot = "dist";

@Documentation("Data directory of the embedded opensearch. Contains indices of the opensearch. May be pointed to an existing" +
"opensearch directory during in-place migration to Datanode")
@Documentation("""
Data directory of the embedded opensearch. Contains indices of the opensearch.
May be pointed to an existing opensearch directory during in-place migration to Datanode
""")
@Parameter(value = "opensearch_data_location", required = true)
private Path opensearchDataLocation = Path.of("datanode/data");

@Documentation("Logs directory of the embedded opensearch")
@Parameter(value = "opensearch_logs_location", required = true, validators = DirectoryWritableValidator.class)
private Path opensearchLogsLocation = Path.of("datanode/logs");

@Documentation("Configuration directory of the embedded opensearch. This is the directory where the opensearch" +
"process will store its configuration files. Caution, each start of the Datanode will regenerate the complete content of the directory!")
@Documentation("""
Configuration directory of the embedded opensearch. This is the directory where the opensearch
process will store its configuration files. Caution, each start of the Datanode will regenerate
the complete content of the directory!
""")
@Parameter(value = "opensearch_config_location", required = true, validators = DirectoryWritableValidator.class)
private Path opensearchConfigLocation = Path.of("datanode/config");

Expand All @@ -113,7 +122,10 @@ public class Configuration {
private Integer opensearchProcessLogsBufferSize = 500;


@Documentation("Unique name of this Datanode instance. use this, if your node name should be different from the hostname that's found by programmatically looking it up")
@Documentation("""
Unique name of this Datanode instance. use this, if your node name should be different from the hostname
that's found by programmatically looking it up.
""")
@Parameter(value = "node_name")
private String datanodeNodeName;

Expand All @@ -122,7 +134,10 @@ public class Configuration {
@Parameter(value = "initial_cluster_manager_nodes")
private String initialClusterManagerNodes;

@Documentation("Opensearch heap memory. Initial and maxmium heap must be identical for OpenSearch, otherwise the boot fails. So it's only one config option")
@Documentation("""
Opensearch heap memory. Initial and maxmium heap must be identical for OpenSearch, otherwise the boot fails.
So it's only one config option.
""")
@Parameter(value = "opensearch_heap")
private String opensearchHeap = "1g";

Expand All @@ -138,7 +153,10 @@ public class Configuration {
@Parameter(value = "opensearch_discovery_seed_hosts", converter = StringListConverter.class)
private List<String> opensearchDiscoverySeedHosts = Collections.emptyList();

@Documentation("Binds an OpenSearch node to an address. Use 0.0.0.0 to include all available network interfaces, or specify an IP address assigned to a specific interface. ")
@Documentation("""
Binds an OpenSearch node to an address. Use 0.0.0.0 to include all available network interfaces,
or specify an IP address assigned to a specific interface.
""")
@Parameter(value = "opensearch_network_host")
private String opensearchNetworkHost = null;

Expand All @@ -158,23 +176,36 @@ public class Configuration {
@Parameter(value = HTTP_CERTIFICATE_PASSWORD_PROPERTY)
private String datanodeHttpCertificatePassword;

@Documentation("You MUST set a secret to secure/pepper the stored user passwords here. Use at least 64 characters." +
"Generate one by using for example: pwgen -N 1 -s 96 \n" +
"ATTENTION: This value must be the same on all Graylog and Datanode nodes in the cluster. " +
"Changing this value after installation will render all user sessions and encrypted values in the database invalid. (e.g. encrypted access tokens)")
@Documentation("""
You MUST set a secret to secure/pepper the stored user passwords here. Use at least 64 characters.
Generate one by using for example: pwgen -N 1 -s 96
ATTENTION: This value must be the same on all Graylog and Datanode nodes in the cluster.
Changing this value after installation will render all user sessions and encrypted values
in the database invalid. (e.g. encrypted access tokens)
""")
@Parameter(value = "password_secret", required = true, validators = StringNotBlankValidator.class)
private String passwordSecret;

@Documentation("communication between Graylog and OpenSearch is secured by JWT. This configuration defines interval between token regenerations.")
@DocumentationSection(heading = "OpenSearch JWT token usage",description = """
communication between Graylog and OpenSearch is secured by JWT. These are the defaults used for the token usage
adjust them, if you have special needs.
""")
@Documentation(value = "This configuration defines interval between token regenerations.")
@Parameter(value = "indexer_jwt_auth_token_caching_duration")
Duration indexerJwtAuthTokenCachingDuration = Duration.seconds(60);

@Documentation("communication between Graylog and OpenSearch is secured by JWT. This configuration defines validity interval of JWT tokens.")
@DocumentationSection(heading = "OpenSearch JWT token usage",description = """
communication between Graylog and OpenSearch is secured by JWT. These are the defaults used for the token usage
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think repeating the section headings and descriptions is not optimal. Maybe it would make sense to introduce an id to the sections which can be referenced (or reference by heading as it is and make description optional?)

adjust them, if you have special needs.
""")
@Documentation("This configuration defines validity interval of JWT tokens")
@Parameter(value = "indexer_jwt_auth_token_expiration_duration")
Duration indexerJwtAuthTokenExpirationDuration = Duration.seconds(180);

@Documentation("The auto-generated node ID will be stored in this file and read after restarts. It is a good idea " +
"to use an absolute file path here if you are starting Graylog DataNode from init scripts or similar.")
@Documentation("""
The auto-generated node ID will be stored in this file and read after restarts. It is a good idea
to use an absolute file path here if you are starting Graylog DataNode from init scripts or similar.
""")
@Parameter(value = "node_id_file", validators = NodeIdFileValidator.class)
private String nodeIdFile = "data/node-id";

Expand All @@ -195,15 +226,19 @@ public class Configuration {
@Parameter(value = "clustername")
private String clustername = "datanode-cluster";

@Documentation("This configuration should be used if you want to connect to this Graylog DataNode's REST API and it is available on " +
"another network interface than $http_bind_address, " +
"for example if the machine has multiple network interfaces or is behind a NAT gateway.")
@Parameter(value = "http_publish_uri", validators = URIAbsoluteValidator.class)
@Documentation("""
This configuration should be used if you want to connect to this Graylog DataNode's REST API
and it is available on another network interface than $http_bind_address,
for example if the machine has multiple network interfaces or is behind a NAT gateway.
""")
@Parameter(value = "http_publish_uri", validators = URIAbsoluteValidator.class)
private URI httpPublishUri;


@Documentation("Enable GZIP support for HTTP interface. This compresses API responses and therefore helps to reduce " +
" overall round trip times.")
@Documentation("""
Enable GZIP support for HTTP interface. This compresses API responses and therefore helps to reduce
overall round trip times.
""")
@Parameter(value = "http_enable_gzip")
private boolean httpEnableGzip = true;

Expand Down Expand Up @@ -286,7 +321,10 @@ public Integer getIndicesQueryBoolMaxClauseCount() {
return indicesQueryBoolMaxClauseCount;
}

@Documentation("Configures verbosity of embedded opensearch logs. Possible values OFF, FATAL, ERROR, WARN, INFO, DEBUG, and TRACE, default is INFO")
@Documentation("""
Configures verbosity of embedded opensearch logs.
Possible values OFF, FATAL, ERROR, WARN, INFO, DEBUG, and TRACE, default is INFO
""")
@Parameter(value = "opensearch_logger_org_opensearch")
private String opensearchDebug;

Expand Down
Loading
Loading