diff --git a/docs/_docs/services/services.adoc b/docs/_docs/services/services.adoc index 46801d4e66173..d59cdaece888f 100644 --- a/docs/_docs/services/services.adoc +++ b/docs/_docs/services/services.adoc @@ -62,6 +62,8 @@ The `Service` interface has three methods: You can deploy your service either programmatically at runtime, or by providing a service configuration as part of the node configuration. In the latter case, the service is deployed when the cluster starts. +If deploying several related services you can specify start order with the `ServiceConfiguration.setLocalStartOrder(int)`. +Start order applied on node level during service initialization. Services with lower `localStartOrder` will be inited first. === Deploying Services at Runtime diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteServices.java b/modules/core/src/main/java/org/apache/ignite/IgniteServices.java index a3d64d45e7046..b3f2e3d0f6c86 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteServices.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteServices.java @@ -444,11 +444,14 @@ public void deployMultiple(String name, Service svc, int totalCnt, int maxPerNod * of failed services will be thrown. It is guaranteed that all services that were provided to this method and are * not present in the list of failed services are successfully deployed by the moment of the exception being thrown. * Note that if exception is thrown, then partial deployment may have occurred. + * Note, start order guarantees not provided, by default. + * Node local start order can be forced with the {@link ServiceConfiguration#setLocalStartOrder(int)}. * * @param cfgs {@link Collection} of service configurations to be deployed. * @throws ServiceDeploymentException If failed to deploy services. * @see IgniteServices#deploy(ServiceConfiguration) * @see IgniteServices#deployAllAsync(Collection) + * @see ServiceConfiguration#setLocalStartOrder(int) */ public void deployAll(Collection cfgs) throws ServiceDeploymentException; @@ -463,11 +466,14 @@ public void deployMultiple(String name, Service svc, int totalCnt, int maxPerNod * guaranteed that all services, that were provided to this method and are not present in the list of failed * services, are successfully deployed by the moment of the exception being thrown. Note that if exception is * thrown, then partial deployment may have occurred. + * Note, start order guarantees not provided, by default. + * Node local start order can be forced with the {@link ServiceConfiguration#setLocalStartOrder(int)}. * * @param cfgs {@link Collection} of service configurations to be deployed. * @return a Future representing pending completion of the operation. * @see IgniteServices#deploy(ServiceConfiguration) * @see IgniteServices#deployAll(Collection) + * @see ServiceConfiguration#setLocalStartOrder(int) */ public IgniteFuture deployAllAsync(Collection cfgs); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/LazyServiceConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/LazyServiceConfiguration.java index c5c35b2f1fce6..859106ee73b56 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/LazyServiceConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/LazyServiceConfiguration.java @@ -95,6 +95,7 @@ public LazyServiceConfiguration( isStatisticsEnabled = cfg.isStatisticsEnabled(); interceptors = cfg.getInterceptors(); this.interceptorsBytes = interceptorsBytes; + locStartOrder = cfg.getLocalStartOrder(); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceDeploymentTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceDeploymentTask.java index cdb866d5aa273..dfb12ecc61460 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceDeploymentTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/ServiceDeploymentTask.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -307,33 +308,38 @@ private void processDeploymentActions(@NotNull ServiceDeploymentActions depActio if (!depActions.servicesToDeploy().isEmpty()) { final Collection evtTopNodes = nodeIds(ctx.discovery().nodes(evtTopVer)); - depActions.servicesToDeploy().forEach((srvcId, desc) -> { - try { - ServiceConfiguration cfg = desc.configuration(); + depActions.servicesToDeploy().entrySet().stream() + .sorted(Comparator.comparingInt(e -> e.getValue().configuration().getLocalStartOrder())) + .forEach(entry -> { + IgniteUuid srvcId = entry.getKey(); + ServiceInfo desc = entry.getValue(); - TreeMap oldTop = filterDeadNodes(evtTopNodes, desc.topologySnapshot()); + try { + ServiceConfiguration cfg = desc.configuration(); - Map top = reassign(srvcId, cfg, evtTopVer, oldTop); + TreeMap oldTop = filterDeadNodes(evtTopNodes, desc.topologySnapshot()); - expDeps.put(srvcId, top); + Map top = reassign(srvcId, cfg, evtTopVer, oldTop); - Integer expCnt = top.getOrDefault(ctx.localNodeId(), 0); + expDeps.put(srvcId, top); - if (expCnt > srvcProc.localInstancesCount(srvcId)) { - srvcProc.deployment().deployerBlockingSectionBegin(); + Integer expCnt = top.getOrDefault(ctx.localNodeId(), 0); - try { - srvcProc.redeploy(srvcId, cfg, top); - } - finally { - srvcProc.deployment().deployerBlockingSectionEnd(); + if (expCnt > srvcProc.localInstancesCount(srvcId)) { + srvcProc.deployment().deployerBlockingSectionBegin(); + + try { + srvcProc.redeploy(srvcId, cfg, top); + } + finally { + srvcProc.deployment().deployerBlockingSectionEnd(); + } } } - } - catch (IgniteCheckedException e) { - depErrors.computeIfAbsent(srvcId, c -> new ArrayList<>()).add(e); - } - }); + catch (IgniteCheckedException e) { + depErrors.computeIfAbsent(srvcId, c -> new ArrayList<>()).add(e); + } + }); } createAndSendSingleDeploymentsMessage(depId, depErrors); @@ -484,25 +490,28 @@ protected void onReceiveFullDeploymentsMessage(ServiceClusterDeploymentResultBat final Map services = srvcProc.deployedServices(); - fullTops.forEach((srvcId, top) -> { - Integer expCnt = top.snapshot().getOrDefault(ctx.localNodeId(), 0); + fullTops.entrySet().stream() + .sorted(Comparator.comparingInt(e -> services.get(e.getKey()).configuration().getLocalStartOrder())) + .forEach(entry -> { + IgniteUuid srvcId = entry.getKey(); + ServiceTopology top = entry.getValue(); - if (expCnt < srvcProc.localInstancesCount(srvcId)) { // Undeploy exceed instances - ServiceInfo desc = services.get(srvcId); + Integer expCnt = top.snapshot().getOrDefault(ctx.localNodeId(), 0); - assert desc != null; + if (expCnt < srvcProc.localInstancesCount(srvcId)) { // Undeploy exceed instances + ServiceInfo desc = services.get(srvcId); - ServiceConfiguration cfg = desc.configuration(); + ServiceConfiguration cfg = desc.configuration(); - try { - srvcProc.redeploy(srvcId, cfg, top.snapshot()); + try { + srvcProc.redeploy(srvcId, cfg, top.snapshot()); + } + catch (IgniteCheckedException e) { + log.error("Error occured during cancel exceed service instances: " + + "[srvcId=" + srvcId + ", name=" + desc.name() + ']', e); + } } - catch (IgniteCheckedException e) { - log.error("Error occured during cancel exceed service instances: " + - "[srvcId=" + srvcId + ", name=" + desc.name() + ']', e); - } - } - }); + }); completeSuccess(); } diff --git a/modules/core/src/main/java/org/apache/ignite/services/ServiceConfiguration.java b/modules/core/src/main/java/org/apache/ignite/services/ServiceConfiguration.java index 77d40ae83c333..bd42651e91831 100644 --- a/modules/core/src/main/java/org/apache/ignite/services/ServiceConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/services/ServiceConfiguration.java @@ -20,8 +20,10 @@ import java.io.Externalizable; import java.io.Serializable; import java.util.Arrays; +import java.util.Collection; import org.apache.ignite.IgniteServices; import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.processors.service.IgniteServiceProcessor; import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.typedef.internal.S; @@ -35,14 +37,14 @@ * 
  * IgniteConfiguration gridCfg = new IgniteConfiguration();
  *
- * GridServiceConfiguration svcCfg1 = new GridServiceConfiguration();
+ * ServiceConfiguration svcCfg1 = new ServiceConfiguration();
  *
  * svcCfg1.setName("myClusterSingletonService");
  * svcCfg1.setMaxPerNodeCount(1);
  * svcCfg1.setTotalCount(1);
  * svcCfg1.setService(new MyClusterSingletonService());
  *
- * GridServiceConfiguration svcCfg2 = new GridServiceConfiguration();
+ * ServiceConfiguration svcCfg2 = new ServiceConfiguration();
  *
  * svcCfg2.setName("myNodeSingletonService");
  * svcCfg2.setMaxPerNodeCount(1);
@@ -88,6 +90,19 @@ public class ServiceConfiguration implements Serializable {
     @GridToStringExclude
     protected ServiceCallInterceptor[] interceptors;
 
+    /**
+     * Node local start order.
+     * Note:
+     * 

+     * In case static service configuration {@link IgniteConfiguration#setServiceConfiguration(ServiceConfiguration...)}
+     * order will be applied on node start.
+     * 

+     * 

+     * In case deploying by the {@link IgniteServices#deployAll(Collection)}, order will be applied for deployed services.
+     * 

+     */
+    protected int locStartOrder;
+
     /**
      * Gets service name.
      * 

@@ -318,6 +333,34 @@ public ServiceConfiguration setInterceptors(ServiceCallInterceptor... intercepto
         return this;
     }
 
+    /**
+     * 

+     * In case static service configuration {@link IgniteConfiguration#setServiceConfiguration(ServiceConfiguration...)}
+     * order will be applied on node start.
+     * 

+     * 

+     * In case deploying by the {@link IgniteServices#deployAll(Collection)}, order will be applied for deployed services.
+     * 

+     *
+     * @return Node local start order. Greater value means service started later.
+     */
+    public int getLocalStartOrder() {
+        return locStartOrder;
+    }
+
+    /**
+     * Sets node local start order.
+     * Greater value means service started later.
+     *
+     * @param locStartOrder Node local start order.
+     * @return {@code this} for chaining.
+     */
+    public ServiceConfiguration setLocalStartOrder(int locStartOrder) {
+        this.locStartOrder = locStartOrder;
+
+        return this;
+    }
+
     /** {@inheritDoc} */
     @Override public boolean equals(Object o) {
         if (!equalsIgnoreNodeFilter(o))
@@ -370,6 +413,9 @@ public boolean equalsIgnoreNodeFilter(Object o) {
         if (svc != null ? !svc.getClass().equals(that.svc.getClass()) : that.svc != null)
             return false;
 
+        if (locStartOrder != that.locStartOrder)
+            return false;
+
         return Arrays.deepEquals(interceptors, that.interceptors);
     }
 
@@ -383,6 +429,9 @@ public boolean equalsIgnoreNodeFilter(Object o) {
         String svcCls = svc == null ? "" : svc.getClass().getSimpleName();
         String nodeFilterCls = nodeFilter == null ? "" : nodeFilter.getClass().getSimpleName();
 
-        return S.toString(ServiceConfiguration.class, this, "svcCls", svcCls, "nodeFilterCls", nodeFilterCls);
+        return S.toString(ServiceConfiguration.class, this,
+            "svcCls", svcCls,
+            "nodeFilterCls", nodeFilterCls,
+            "localStartOrder", locStartOrder);
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/ServiceLocalStartOrderTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/ServiceLocalStartOrderTest.java
new file mode 100644
index 0000000000000..11f84ab84805c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/ServiceLocalStartOrderTest.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.service;
+
+import java.util.Arrays;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceConfiguration;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
+
+/** */
+public class ServiceLocalStartOrderTest extends GridCommonAbstractTest {
+    /** */
+    private final int gridCnt = 3;
+
+    /** */
+    private boolean staticConfig;
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        if (staticConfig)
+            cfg.setServiceConfiguration(serviceConfigs());
+
+        return cfg;
+    }
+
+    /** */
+    @Test
+    public void testStaticConfigDeployment() throws Exception {
+        staticConfig = true;
+
+        try {
+            startGrids(gridCnt);
+
+            check();
+        }
+        finally {
+            staticConfig = false;
+        }
+    }
+
+    /** */
+    @Test
+    public void testRuntimeDeployment() throws Exception {
+        IgniteEx ignite = startGrids(gridCnt);
+
+        doTest(ignite);
+    }
+
+    /** */
+    @Test
+    public void testRuntimeDeploymentFromClient() throws Exception {
+        startGrids(gridCnt);
+
+        doTest(startClientGrid());
+    }
+
+    /** */
+    private void doTest(IgniteEx deployFrom) throws Exception {
+        deployFrom.services().deployAll(Arrays.asList(serviceConfigs()));
+
+        check();
+    }
+
+    /** */
+    private void check() throws Exception {
+        for (int i = 0; i < 5; i++) {
+            for (int node = 0; node < gridCnt; node++) {
+                IgniteEx ign = grid(node);
+
+                String srvcName = name(i);
+
+                assertTrue(waitForCondition(() -> ign.services().service(srvcName) != null, 10_000));
+            }
+        }
+    }
+
+    /** */
+    private ServiceConfiguration[] serviceConfigs() {
+        ServiceConfiguration[] cfgs = new ServiceConfiguration[5];
+
+        for (int i = 0; i < 5; i++) {
+            cfgs[i] = new ServiceConfiguration()
+                .setName(name(i))
+                .setService(new OrderedServiceImpl(i))
+                .setLocalStartOrder(i)
+                .setMaxPerNodeCount(1)
+                .setTotalCount(gridCnt);
+        }
+
+        return cfgs;
+    }
+
+    /** */
+    private static class OrderedServiceImpl implements OrderedService {
+        /** */
+        private final int order;
+
+        /** */
+        private boolean started;
+
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** */
+        public OrderedServiceImpl(int order) {
+            this.order = order;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void init() throws Exception {
+            for (int i = order - 1; i >= 0; i--) {
+                OrderedService srvc = ignite.services().service(name(i));
+
+                assertNotNull("Must be deployed previously: " + i, srvc);
+                assertTrue(srvc.started());
+            }
+
+            started = true;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean started() {
+            return started;
+        }
+    }
+
+    /** */
+    private interface OrderedService extends Service {
+        /** */
+        boolean started();
+    }
+
+    /** */
+    private static String name(int i) {
+        return "Ordered" + i;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteServiceGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteServiceGridTestSuite.java
index 034c8ff5c86aa..63f16fbbf7067 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteServiceGridTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteServiceGridTestSuite.java
@@ -60,6 +60,7 @@
 import org.apache.ignite.internal.processors.service.ServiceDeploymentProcessingOnNodesLeftTest;
 import org.apache.ignite.internal.processors.service.ServiceHotRedeploymentViaDeploymentSpiTest;
 import org.apache.ignite.internal.processors.service.ServiceInfoSelfTest;
+import org.apache.ignite.internal.processors.service.ServiceLocalStartOrderTest;
 import org.apache.ignite.internal.processors.service.ServicePredicateAccessCacheTest;
 import org.apache.ignite.internal.processors.service.ServiceReassignmentFunctionSelfTest;
 import org.apache.ignite.internal.processors.service.ServiceRedeploymentOnNodeLeftTest;
@@ -125,6 +126,7 @@
     GridServiceMetricsTest.class,
     IgniteServiceCallInterceptorTest.class,
     ServiceRedeploymentOnNodeLeftTest.class,
+    ServiceLocalStartOrderTest.class
 })
 public class IgniteServiceGridTestSuite {
     /** */