ISPN-13512 Configuration converter

* Add CLI `config convert` command * Complete server config serialization * Proper serialization API for secret masking * Implement JGroups stack serialization * Delay JGroups stack initialization * Fix parsing of YAML attribute lists * Don't include implict authorization roles
infinispan · Dec 3, 2021 · 75332ff · 75332ff
1 parent f38b497
commit 75332ff
Show file tree

Hide file tree

Showing 78 changed files with 1,582 additions and 820 deletions.
diff --git a/cli/pom.xml b/cli/pom.xml
@@ -39,6 +39,11 @@
          <artifactId>kubernetes-client</artifactId>
          <optional>true</optional>
       </dependency>
+      <dependency>
+         <groupId>org.infinispan</groupId>
+         <artifactId>infinispan-core</artifactId>
+         <optional>true</optional> <!-- Configuration converter -->
+      </dependency>
       <dependency>
          <groupId>org.yaml</groupId>
          <artifactId>snakeyaml</artifactId>
@@ -102,11 +107,6 @@
          <artifactId>junit</artifactId>
          <scope>test</scope>
       </dependency>
-      <dependency>
-         <groupId>org.infinispan</groupId>
-         <artifactId>infinispan-core</artifactId>
-         <scope>test</scope>
-      </dependency>
       <dependency>
          <groupId>org.infinispan</groupId>
          <artifactId>infinispan-commons-test</artifactId>

diff --git a/cli/src/main/java/org/infinispan/cli/activators/ConfigConversionAvailable.java b/cli/src/main/java/org/infinispan/cli/activators/ConfigConversionAvailable.java
@@ -0,0 +1,20 @@
+package org.infinispan.cli.activators;
+
+import org.aesh.command.activator.CommandActivator;
+import org.aesh.command.impl.internal.ParsedCommand;
+import org.infinispan.commons.util.Util;
+
+/**
+ * @since 14.0
+ **/
+public class ConfigConversionAvailable implements CommandActivator {
+   @Override
+   public boolean isActivated(ParsedCommand command) {
+      try {
+         Util.loadClass("org.infinispan.configuration.parsing.ParserRegistry", this.getClass().getClassLoader());
+         return true;
+      } catch (Exception e) {
+         return false;
+      }
+   }
+}
diff --git a/cli/src/main/java/org/infinispan/cli/commands/CliCommand.java b/cli/src/main/java/org/infinispan/cli/commands/CliCommand.java
@@ -31,7 +31,7 @@ public CommandResult execute(ContextAwareCommandInvocation invocation) throws Co
       } catch (Throwable e) {
          // These are unhandled
          Throwable cause = Util.getRootCause(e);
-         invocation.getShell().writeln(ANSI.RED_TEXT + cause.getClass().getSimpleName() +": " + cause.getLocalizedMessage() + ANSI.DEFAULT_TEXT);
+         System.err.println(ANSI.RED_TEXT + cause.getClass().getSimpleName() +": " + cause.getLocalizedMessage() + ANSI.DEFAULT_TEXT);
          return CommandResult.FAILURE;
       }
    }

diff --git a/cli/src/main/java/org/infinispan/cli/commands/Config.java b/cli/src/main/java/org/infinispan/cli/commands/Config.java
@@ -1,26 +1,46 @@
 package org.infinispan.cli.commands;
 
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
+import java.util.Map;
 
 import org.aesh.command.Command;
 import org.aesh.command.CommandDefinition;
+import org.aesh.command.CommandException;
 import org.aesh.command.CommandResult;
 import org.aesh.command.GroupCommandDefinition;
+import org.aesh.command.impl.completer.FileOptionCompleter;
 import org.aesh.command.option.Argument;
 import org.aesh.command.option.Arguments;
 import org.aesh.command.option.Option;
+import org.aesh.io.Resource;
 import org.infinispan.cli.Context;
+import org.infinispan.cli.activators.ConfigConversionAvailable;
 import org.infinispan.cli.completers.ConfigPropertyCompleter;
+import org.infinispan.cli.completers.MediaTypeCompleter;
 import org.infinispan.cli.impl.ContextAwareCommandInvocation;
 import org.infinispan.cli.logging.Messages;
+import org.infinispan.commons.configuration.io.ConfigurationWriter;
+import org.infinispan.commons.dataconversion.MediaType;
+import org.infinispan.commons.util.Util;
+import org.infinispan.configuration.cache.Configuration;
+import org.infinispan.configuration.cache.ConfigurationBuilder;
+import org.infinispan.configuration.parsing.ConfigurationBuilderHolder;
+import org.infinispan.configuration.parsing.ParserRegistry;
 import org.kohsuke.MetaInfServices;
 
 /**
  * @author Tristan Tarrant &lt;tristan@infinispan.org&gt;
  * @since 11.0
  **/
 @MetaInfServices(Command.class)
-@GroupCommandDefinition(name = "config", description = "Configuration operations", groupCommands = {Config.Set.class, Config.Get.class})
+@GroupCommandDefinition(name = "config", description = "Configuration operations", groupCommands = {Config.Set.class, Config.Get.class, Config.Convert.class})
 public class Config extends CliCommand {
 
    @Option(shortName = 'h', hasValue = false, overrideRequired = true)
@@ -92,4 +112,68 @@ public CommandResult exec(ContextAwareCommandInvocation invocation) {
          return CommandResult.SUCCESS;
       }
    }
+
+   @CommandDefinition(name = "convert", description = "Converts configuration to different formats.", activator = ConfigConversionAvailable.class)
+   public static class Convert extends CliCommand {
+
+      @Argument(description = "Specifies the path to a configuration file to convert. Uses standard input (stdin) if you do not specify a path.", completer = FileOptionCompleter.class)
+      Resource input;
+
+      @Option(description = "Specifies the path to the output configuration file. Uses standard output (stdout) if you do not specify a path.", completer = FileOptionCompleter.class, shortName = 'o')
+      Resource output;
+
+      @Option(description = "Sets the format of the output configuration.", required = true, completer = MediaTypeCompleter.class, shortName = 'f')
+      String format;
+
+      @Option(shortName = 'h', hasValue = false, overrideRequired = true)
+      protected boolean help;
+
+      @Override
+      public boolean isHelp() {
+         return help;
+      }
+
+      @Override
+      public CommandResult exec(ContextAwareCommandInvocation invocation) throws CommandException {
+         InputStream is = null;
+         OutputStream os = null;
+         try {
+            ParserRegistry registry = new ParserRegistry();
+            is = input == null ? System.in : new FileInputStream(input.getAbsolutePath());
+            ConfigurationBuilderHolder holder = registry.parse(is, null, null); // Auto-detect type
+            os = output == null ? System.out : new FileOutputStream(output.getAbsolutePath());
+            Map<String, Configuration> configurations = new HashMap<>();
+            for (Map.Entry<String, ConfigurationBuilder> configuration : holder.getNamedConfigurationBuilders().entrySet()) {
+               configurations.put(configuration.getKey(), configuration.getValue().build());
+            }
+            MediaType mediaType;
+            switch (MediaTypeCompleter.MediaType.valueOf(format.toUpperCase(Locale.ROOT))) {
+               case XML:
+                  mediaType = MediaType.APPLICATION_XML;
+                  break;
+               case YAML:
+                  mediaType = MediaType.APPLICATION_YAML;
+                  break;
+               case JSON:
+                  mediaType = MediaType.APPLICATION_JSON;
+                  break;
+               default:
+                  throw new CommandException("Invalid output format: " + format);
+            }
+            try (ConfigurationWriter writer = ConfigurationWriter.to(os).withType(mediaType).clearTextSecrets(true).prettyPrint(true).build()) {
+               registry.serialize(writer, holder.getGlobalConfigurationBuilder().build(), configurations);
+            }
+            return CommandResult.SUCCESS;
+         } catch (FileNotFoundException e) {
+            throw new CommandException(e);
+         } finally {
+            if (input != null) {
+               Util.close(is);
+            }
+            if (output != null) {
+               Util.close(os);
+            }
+         }
+      }
+   }
 }
diff --git a/cli/src/main/java/org/infinispan/cli/completers/MediaTypeCompleter.java b/cli/src/main/java/org/infinispan/cli/completers/MediaTypeCompleter.java
@@ -0,0 +1,16 @@
+package org.infinispan.cli.completers;
+
+/**
+ * @since 14.0
+ */
+public class MediaTypeCompleter extends EnumCompleter<MediaTypeCompleter.MediaType> {
+   public enum MediaType {
+      XML,
+      JSON,
+      YAML
+   }
+
+   public MediaTypeCompleter() {
+      super(MediaType.class);
+   }
+}
diff --git a/cli/src/main/resources/help/config.adoc b/cli/src/main/resources/help/config.adoc
@@ -16,9 +16,11 @@ SYNOPSIS
 
 *config get* 'name'
 
+*config convert* --outputFormat=[xml|json|yaml] [-o outputFile] [inputFile]
+
 DESCRIPTION
 -----------
-Manage (list, set, get) CLI configuration properties.
+Manage (list, set, get) CLI configuration properties and provide configuration conversion between the different formats (XML, JSON, YAML)
 
 
 COMMAND SYNOPSIS
@@ -33,6 +35,9 @@ Sets the value of a specific property. If you do not specify a value, the proper
 *config get* 'name'::
 Retrieves the value of a specific property.
 
+*config convert* --format=[xml|json|yaml] [-o outputFile] [inputFile]::
+Converts a configuration file to a different format.
+
 
 COMMON OPTIONS
 --------------
@@ -43,6 +48,18 @@ These options apply to all commands:
 Displays a help page for the command or sub-command.
 
 
+CONVERT OPTIONS
+---------------
+
+The following options apply to the `convert` command:
+
+*-f, --format*='xml|json|yaml'::
+Specifies the format for the conversion.
+
+*-o, --output*='path'::
+Specifies the path to the output file. Uses standard output (`stdout`) if you do not specify a path.
+
+
 PROPERTIES
 ----------
 
@@ -83,6 +100,12 @@ Specifies the path of a keystore named "my-trust-store.jks".
 `config set truststore-password secret` +
 Sets the keystore password, if required.
 
+`config convert -f yaml -o infinispan.yaml infinispan.xml` +
+Converts the `infinispan.xml` file to YAML and writes the output to the `infinispan.yaml` file.
+
+`config convert -f json` +
+Converts the configuration from standard input to JSON, and writes the output to standard output.
+
 
 SEE ALSO
 --------

diff --git a/.../main/java/org/infinispan/commons/configuration/AbstractTypedPropertiesConfiguration.java b/.../main/java/org/infinispan/commons/configuration/AbstractTypedPropertiesConfiguration.java
@@ -6,18 +6,14 @@
 
 import org.infinispan.commons.configuration.attributes.Attribute;
 import org.infinispan.commons.configuration.attributes.AttributeDefinition;
-import org.infinispan.commons.configuration.attributes.AttributeInitializer;
 import org.infinispan.commons.configuration.attributes.AttributeSet;
+import org.infinispan.commons.configuration.attributes.PropertiesAttributeSerializer;
 import org.infinispan.commons.configuration.attributes.TypedPropertiesAttributeCopier;
 import org.infinispan.commons.util.TypedProperties;
 
 public abstract class AbstractTypedPropertiesConfiguration {
-   public static final AttributeDefinition<TypedProperties> PROPERTIES = AttributeDefinition.builder("properties", null, TypedProperties.class).copier(TypedPropertiesAttributeCopier.INSTANCE).initializer(new AttributeInitializer<TypedProperties>() {
-      @Override
-      public TypedProperties initialize() {
-         return new TypedProperties();
-      }
-   }).build();
+   public static final AttributeDefinition<TypedProperties> PROPERTIES = AttributeDefinition.builder("properties", null, TypedProperties.class)
+         .copier(TypedPropertiesAttributeCopier.INSTANCE).initializer(() -> new TypedProperties()).serializer(PropertiesAttributeSerializer.PROPERTIES).build();
    public static AttributeSet attributeSet() {
       return new AttributeSet(AbstractTypedPropertiesConfiguration.class, PROPERTIES);
    };

diff --git a/...ll/src/main/java/org/infinispan/commons/configuration/attributes/AttributeSerializer.java b/...ll/src/main/java/org/infinispan/commons/configuration/attributes/AttributeSerializer.java
@@ -11,7 +11,6 @@
  * @since 10.0
  */
 public interface AttributeSerializer<T> {
-
    AttributeSerializer<Object> DEFAULT = (writer, name, value) -> {
       if (Boolean.class == value.getClass()) {
          writer.writeAttribute(name, (Boolean) value);
@@ -20,7 +19,7 @@ public interface AttributeSerializer<T> {
       }
    };
    AttributeSerializer<Supplier<char[]>> SECRET = (writer, name, value) -> {
-      if (Boolean.getBoolean("org.infinispan.configuration.clear-text-secrets")) {
+      if (writer.clearTextSecrets()) {
          writer.writeAttribute(name, new String(value.get()));
       } else {
          writer.writeAttribute(name, "***");

diff --git a/...l/src/main/java/org/infinispan/commons/configuration/attributes/ConfigurationElement.java b/...l/src/main/java/org/infinispan/commons/configuration/attributes/ConfigurationElement.java
@@ -154,27 +154,36 @@ public boolean isModified() {
     */
    public void write(ConfigurationWriter writer) {
       if (isModified()) {
-         writer.writeStartElement(element);
-         attributes.write(writer);
-         String repeatElement = null;
-         for (ConfigurationElement<?> child : children) {
-            if (child.repeated) {
-               if (!child.element.equals(repeatElement)) {
-                  if (repeatElement != null) {
-                     writer.writeEndListElement();
+         if (attributes.attributes().isEmpty() && children.length > 0 && Arrays.stream(children).allMatch(c -> children[0].element.equals(c.element))) {
+            // Simple array: all children are homogeneous
+            writer.writeStartListElement(element, true);
+            for (ConfigurationElement<?> child : children) {
+               child.write(writer);
+            }
+            writer.writeEndListElement();
+         } else {
+            writer.writeStartElement(element);
+            attributes.write(writer);
+            String repeatElement = null;
+            for (ConfigurationElement<?> child : children) {
+               if (child.repeated) {
+                  if (!child.element.equals(repeatElement)) {
+                     if (repeatElement != null) {
+                        writer.writeEndListElement();
+                     }
+                     repeatElement = child.element;
+                     writer.writeStartListElement(repeatElement, false);
                   }
-                  repeatElement = child.element;
-                  writer.writeStartListElement(repeatElement, true);
+               } else {
+                  repeatElement = null;
                }
-            } else {
-               repeatElement = null;
+               child.write(writer);
             }
-            child.write(writer);
-         }
-         if (repeatElement != null) {
-            writer.writeEndListElement();
+            if (repeatElement != null) {
+               writer.writeEndListElement();
+            }
+            writer.writeEndElement();
          }
-         writer.writeEndElement();
       }
    }
 

diff --git a/...n/java/org/infinispan/commons/configuration/attributes/PropertiesAttributeSerializer.java b/...n/java/org/infinispan/commons/configuration/attributes/PropertiesAttributeSerializer.java
@@ -0,0 +1,43 @@
+package org.infinispan.commons.configuration.attributes;
+
+import java.util.Map;
+
+import org.infinispan.commons.configuration.io.ConfigurationFormatFeature;
+import org.infinispan.commons.configuration.io.ConfigurationWriter;
+
+public class PropertiesAttributeSerializer implements AttributeSerializer<Map<?, ?>> {
+   public static final PropertiesAttributeSerializer PROPERTIES = new PropertiesAttributeSerializer("properties", "property", "name");
+   private final String collectionElement;
+   private final String itemElement;
+   private final String nameAttribute;
+
+   public PropertiesAttributeSerializer(Enum<?> collectionElement, Enum<?> itemElement, Enum<?> nameAttribute) {
+      this(collectionElement.toString(), itemElement.toString(), nameAttribute.toString());
+   }
+
+   public PropertiesAttributeSerializer(String collectionElement, String itemElement, String nameAttribute) {
+      this.collectionElement = collectionElement;
+      this.itemElement = itemElement;
+      this.nameAttribute = nameAttribute;
+   }
+
+   @Override
+   public void serialize(ConfigurationWriter writer, String name, Map<?, ?> properties) {
+      if (!properties.isEmpty()) {
+         if (writer.hasFeature(ConfigurationFormatFeature.BARE_COLLECTIONS)) {
+            for (Map.Entry<?, ?> entry : properties.entrySet()) {
+               writer.writeStartElement(itemElement);
+               writer.writeAttribute(nameAttribute, entry.getKey().toString());
+               writer.writeCharacters(entry.getValue().toString());
+               writer.writeEndElement();
+            }
+         } else {
+            writer.writeStartMap(collectionElement);
+            for (Map.Entry<?, ?> entry : properties.entrySet()) {
+               writer.writeMapItem(itemElement, nameAttribute, entry.getKey().toString(), entry.getValue().toString());
+            }
+            writer.writeEndMap();
+         }
+      }
+   }
+}