IGNITE-17322 [ducktests] Add ability to configure more options for ignite nodes

modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/self_test/TestAffinityMapper.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.ducktest.tests.self_test;
+
+import org.apache.ignite.cache.affinity.AffinityKeyMapper;
+
+/**
+ * Affinity mapper used to test configuration of the custom affinity mapper in the server node config XML.
+ */
+public class TestAffinityMapper implements AffinityKeyMapper {
+    /** {@inheritDoc} */
+    @Override public Object affinityKey(Object key) {
+        return key;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void reset() {
+    }
+}

modules/ducktests/tests/ignitetest/services/utils/config_template.py

@@ -35,6 +35,7 @@ class ConfigTemplate:
     """
     def __init__(self, path):
         env = Environment(loader=FileSystemLoader(searchpath=TEMPLATE_PATHES))
+        env.filters["snake_to_camel"] = snake_to_camel
 
         self.template = env.get_template(path)
         self.default_params = {}
@@ -49,6 +50,16 @@ def render(self, **kwargs):
         return '\n'.join(filter(lambda line: line.strip(), unfiltered.split('\n')))
 
 
+def snake_to_camel(snake_name):
+    """
+    Custom jinja2 filter to convert named from smake to camel format
+    :param snake_name: name in snake format
+    :return: name in camel format
+    """
+    components = snake_name.split('_')
+    return components[0] + ''.join(x.title() for x in components[1:])
+
+
 class IgniteServerConfigTemplate(ConfigTemplate):
     """
     Ignite server node configuration.

modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/__init__.py

@@ -27,6 +27,7 @@
 from ignitetest.services.utils.ignite_configuration.data_storage import DataStorageConfiguration
 from ignitetest.services.utils.ignite_configuration.discovery import DiscoverySpi, TcpDiscoverySpi
 from ignitetest.services.utils.ignite_configuration.binary_configuration import BinaryConfiguration
+from ignitetest.services.utils.ignite_configuration.transaction import TransactionConfiguration
 from ignitetest.services.utils.ssl.ssl_params import SslParams, is_ssl_enabled, get_ssl_params, IGNITE_CLIENT_ALIAS, \
     IGNITE_SERVER_ALIAS
 from ignitetest.utils.version import IgniteVersion, DEV_BRANCH
@@ -68,6 +69,9 @@ class IgniteConfiguration(NamedTuple):
     include_event_types: list = []
     event_storage_spi: str = None
     log4j_config: str = IgnitePathAware.IGNITE_LOG_CONFIG_NAME
+    sql_schemas: list = []
+    auto_activation_enabled: bool = None
+    transaction_configuration: TransactionConfiguration = None
 
     def __prepare_ssl(self, test_globals, shared_root):
         """
@@ -81,11 +85,13 @@ def __prepare_ssl(self, test_globals, shared_root):
                 IGNITE_CLIENT_ALIAS if self.client_mode else IGNITE_SERVER_ALIAS
             )
         if ssl_params:
+            connector_configuration = self.connector_configuration or ConnectorConfiguration()
+            client_connector_configuration = self.client_connector_configuration or ClientConnectorConfiguration()
             return self._replace(ssl_params=ssl_params,
-                                 connector_configuration=ConnectorConfiguration(ssl_enabled=True,
-                                                                                ssl_params=ssl_params),
-                                 client_connector_configuration=ClientConnectorConfiguration(ssl_enabled=True,
-                                                                                             ssl_params=ssl_params))
+                                 connector_configuration=connector_configuration._replace(
+                                     ssl_enabled=True, ssl_params=ssl_params),
+                                 client_connector_configuration=client_connector_configuration._replace(
+                                     ssl_enabled=True, ssl_params=ssl_params))
         return self
 
     def __prepare_discovery(self, cluster, node):

modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/cache.py

@@ -18,6 +18,8 @@
 """
 from typing import NamedTuple
 
+from ignitetest.utils.bean import Bean
+
 
 class CacheConfiguration(NamedTuple):
     """
@@ -28,3 +30,5 @@ class CacheConfiguration(NamedTuple):
     atomicity_mode: str = 'ATOMIC'
     backups: int = 0
     statistics_enabled: bool = True
+    affinity: Bean = None
+    affinity_mapper: Bean = None

modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/communication.py

@@ -31,14 +31,40 @@ def type(self):
         Type of CommunicationSpi.
         """
 
+    @property
+    @abstractmethod
+    def class_name(self):
+        """
+        Class name of CommunicationSpi.
+        """
+
 
 class TcpCommunicationSpi(CommunicationSpi):
     """
     TcpCommunicationSpi.
     """
-    def __init__(self, port=47100, port_range=100):
-        self.port = port
-        self.port_range = port_range
+
+    def __init__(self,
+                 local_port=47100,
+                 local_port_range=100,
+                 idle_connection_timeout: int = None,
+                 socket_write_timeout: int = None,
+                 selectors_count: int = None,
+                 connections_per_node: int = None,
+                 use_paired_connections: bool = None,
+                 message_queue_limit: int = None):
+        self.local_port = local_port
+        self.local_port_range = local_port_range
+        self.idle_connection_timeout: int = idle_connection_timeout
+        self.socket_write_timeout: int = socket_write_timeout
+        self.selectors_count: int = selectors_count
+        self.connections_per_node: int = connections_per_node
+        self.use_paired_connections: bool = use_paired_connections
+        self.message_queue_limit: int = message_queue_limit
+
+    @property
+    def class_name(self):
+        return "org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi"
 
     @property
     def type(self):

modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py

@@ -25,8 +25,8 @@ class DataRegionConfiguration(NamedTuple):
     Ignite DataRegion Configuration
     """
     name: str = "default"
-    persistent: bool = False
-    init_size: int = 100 * 1024 * 1024
+    persistence_enabled: bool = False
+    initial_size: int = 100 * 1024 * 1024
     max_size: int = 512 * 1024 * 1024
     metrics_enabled: bool = True
     metrics_rate_time_interval: int = None
@@ -36,7 +36,14 @@ class DataStorageConfiguration(NamedTuple):
     """
     Ignite DataStorage configuration
     """
+    checkpoint_frequency: int = None
     default: DataRegionConfiguration = DataRegionConfiguration()
     max_wal_archive_size: int = None
     metrics_enabled: bool = True
+    metrics_rate_time_interval: int = None
     regions: list = []
+    wal_buffer_size: int = None
+    wal_compaction_enabled: bool = None
+    wal_history_size: int = None
+    wal_mode: str = None
+    wal_segment_size: int = None

modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/transaction.py

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+"""
+This module contains classes and utilities for Ignite Cache configuration.
+"""
+from typing import NamedTuple
+
+
+class TransactionConfiguration(NamedTuple):
+    """
+    Transaction configuration.
+    """
+    default_tx_isolation: str = None
+    default_tx_timeout: int = None
+    tx_timeout_on_partition_map_exchange: int = None

modules/ducktests/tests/ignitetest/services/utils/ssl/client_connector_configuration.py

@@ -38,3 +38,4 @@ class ClientConnectorConfiguration(NamedTuple):
     ssl_client_auth: bool = False
     ssl_params: SslParams = None
     thin_client_configuration: ThinClientConfiguration = None
+    thread_pool_size: int = None

modules/ducktests/tests/ignitetest/services/utils/ssl/connector_configuration.py

@@ -27,6 +27,7 @@ class ConnectorConfiguration(NamedTuple):
     Ignite ConnectorConfiguration.
     Used to connect from ControlUtility (control.sh).
     """
+    idle_timeout: int = None
     ssl_enabled: bool = False
     ssl_client_auth: bool = False
     ssl_params: SslParams = None

modules/ducktests/tests/ignitetest/services/utils/templates/cache_macro.j2

@@ -15,6 +15,8 @@
  limitations under the License.
 #}
 
+{% import 'misc_macro.j2' as misc_utils %}
+
 {% macro cache_configs(caches) %}
     {% if caches %}
         <property name="cacheConfiguration">
@@ -27,6 +29,16 @@
                     {% endif %}
                     <property name="atomicityMode" value="{{ cache.atomicity_mode or 'ATOMIC' }}"/>
                     <property name="statisticsEnabled" value="{{ cache.statistics_enabled }}"/>
+                    {% if cache.affinity %}
+                    <property name="affinity">
+                        {{ misc_utils.bean(cache.affinity) }}
+                    </property>
+                    {% endif %}
+                    {% if cache.affinity_mapper %}
+                    <property name="affinityMapper">
+                        {{ misc_utils.bean(cache.affinity_mapper) }}
+                    </property>
+                    {% endif %}
                 </bean>
                 {% endfor %}
             </list>

modules/ducktests/tests/ignitetest/services/utils/templates/client_connector_configuration.j2

@@ -0,0 +1,47 @@
+{#
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+#}
+
+{% import 'ssl_params_macro.j2' as ssl_params_util %}
+
+{% macro client_connector_configuration(config) %}
+{% if config %}
+   <property name="clientConnectorConfiguration">
+        <bean class="org.apache.ignite.configuration.ClientConnectorConfiguration">
+            <property name="port" value="{{ config.port }}"/>
+            {% if config.thread_pool_size %}
+            <property name="threadPoolSize" value="{{ config.thread_pool_size }}"/>
+            {% endif %}
+            {% if config.ssl_enabled %}
+                <property name="sslEnabled" value="true"/>
+                <property name="sslContextFactory">
+                    {{ ssl_params_util.ssl_params(config.ssl_params) }}
+                </property>
+                <property name="useIgniteSslContextFactory" value="{{ config.use_ignite_ssl_context_factory }}"/>
+                <property name="sslClientAuth" value="{{ config.ssl_client_auth }}"/>
+            {% endif %}
+            {% if config.thin_client_configuration %}
+            <property name="thinClientConfiguration">
+                <bean class="org.apache.ignite.configuration.ThinClientConfiguration">
+                    <property name="maxActiveComputeTasksPerConnection" value="{{ config.thin_client_configuration.max_active_compute_tasks_per_connection }}" />
+                    <property name="maxActiveTxPerConnection" value="{{ config.thin_client_configuration.max_active_tx_per_connection }}" />
+                </bean>
+            </property>
+            {% endif %}
+        </bean>
+   </property>
+{% endif %}
+{% endmacro %}

modules/ducktests/tests/ignitetest/services/utils/templates/communication_macro.j2

@@ -17,9 +17,12 @@
 
 {% macro communication_spi(spi) %}
     <property name="communicationSpi">
-        <bean class="org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi">
-            <property name="localPort" value="{{ spi.port }}"/>
-            <property name="localPortRange" value="{{ spi.port_range }}"/>
+        <bean class="{{ spi.class_name }}">
+            {% for name, value in spi.__dict__.items() %}
+            {% if value %}
+            <property name="{{ name | snake_to_camel }}" value="{{ value }}"/>
+            {% endif %}
+            {% endfor %}
         </bean>
     </property>
 {% endmacro %}

modules/ducktests/tests/ignitetest/services/utils/templates/connector_configuration.j2

@@ -18,13 +18,20 @@
 {% import 'ssl_params_macro.j2' as ssl_params_util %}
 
 {% macro connector_configuration(config) %}
+{% if config %}
+    <property name="connectorConfiguration">
         <bean class="org.apache.ignite.configuration.ConnectorConfiguration">
-                <property name="sslEnabled" value="{{ config.ssl_enabled }}"/>
-                {% if config.ssl_enabled %}
-                <property name="sslFactory">
-                    {{ ssl_params_util.ssl_params(config.ssl_params) }}
-                </property>
-                <property name="sslClientAuth" value="{{ config.ssl_client_auth }}"/>
-                {% endif %}
+            {% if config.idle_timeout %}
+            <property name="idleTimeout" value="{{ config.idle_timeout }}"/>
+            {% endif %}
+            <property name="sslEnabled" value="{{ config.ssl_enabled }}"/>
+            {% if config.ssl_enabled %}
+            <property name="sslFactory">
+                {{ ssl_params_util.ssl_params(config.ssl_params) }}
+            </property>
+            <property name="sslClientAuth" value="{{ config.ssl_client_auth }}"/>
+            {% endif %}
         </bean>
+    </property>
+{% endif %}
 {% endmacro %}

modules/ducktests/tests/ignitetest/services/utils/templates/datastorage_macro.j2

@@ -19,36 +19,36 @@
     {% if config %}
         <property name="dataStorageConfiguration">
             <bean class="org.apache.ignite.configuration.DataStorageConfiguration">
-                {% if config.max_wal_archive_size %}
-                    <property name="maxWalArchiveSize" value="{{ config.max_wal_archive_size }}"/>
-                {% endif %}
-                <property name="metricsEnabled" value="{{ config.metrics_enabled }}"/>
-                <property name="defaultDataRegionConfiguration">
-                    {{ data_region(config.default) }}
-                </property>
-                {% if config.regions %}
-                    <property name="dataRegionConfigurations">
-                        <list>
-                        {% for region in config.regions %}
-                            {{ data_region(region) }}
-                        {% endfor %}
-                        </list>
-                    </property>
+                {% for name, value in config._asdict().items() %}
+                {% if value %}
+                    {% if name == 'default' %}
+                        <property name="defaultDataRegionConfiguration">
+                            {{ data_region(value) }}
+                        </property>
+                    {% elif name == 'regions' %}
+                        <property name="dataRegionConfigurations">
+                            <list>
+                            {% for region in value %}
+                                {{ data_region(region) }}
+                            {% endfor %}
+                            </list>
+                        </property>
+                    {% else %}
+                        <property name="{{ name | snake_to_camel }}" value="{{ value }}"/>
+                    {% endif %}
                 {% endif %}
+                {% endfor %}
             </bean>
         </property>
     {% endif %}
 {% endmacro %}
 
 {% macro data_region(config) %}
     <bean class="org.apache.ignite.configuration.DataRegionConfiguration">
-        <property name="name" value="{{ config.name }}"/>
-        <property name="persistenceEnabled" value="{{ config.persistent }}"/>
-        <property name="initialSize" value="{{ config.init_size }}"/>
-        <property name="maxSize" value="{{ config.max_size }}"/>
-        <property name="metricsEnabled" value="{{ config.metrics_enabled }}"/>
-        {% if config.metrics_rate_time_interval %}
-        <property name="metricsRateTimeInterval" value="{{ config.metrics_rate_time_interval }}"/>
+        {% for name, value in config._asdict().items() %}
+        {% if value %}
+        <property name="{{ name | snake_to_camel }}" value="{{ value }}"/>
         {% endif %}
+        {% endfor %}
     </bean>
 {% endmacro %}