Merge branch 'master' into ignite-8926-9200

examples/src/main/java/org/apache/ignite/examples/ml/preprocessing/ImputingExampleWithMostFrequentValues.java → examples/src/main/java/org/apache/ignite/examples/ml/preprocessing/ImputingWithMostFrequentValuesExample.java

@@ -35,7 +35,7 @@
 /**
  * Example that shows how to use Imputing preprocessor to impute the missing values in the given data.
  */
-public class ImputingExampleWithMostFrequentValues {
+public class ImputingWithMostFrequentValuesExample {
     /** Run example. */
     public static void main(String[] args) throws Exception {
         try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {

examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/binary/LogisticRegressionSGDTrainerSample.java → examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/binary/LogisticRegressionSGDTrainerExample.java

@@ -41,7 +41,7 @@
  *
  * @see LogisticRegressionSGDTrainer
  */
-public class LogisticRegressionSGDTrainerSample {
+public class LogisticRegressionSGDTrainerExample {
     /** Run example. */
     public static void main(String[] args) throws InterruptedException {
         System.out.println();
@@ -50,7 +50,7 @@ public static void main(String[] args) throws InterruptedException {
         try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
             System.out.println(">>> Ignite grid started.");
             IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
-                LogisticRegressionSGDTrainerSample.class.getSimpleName(), () -> {
+                LogisticRegressionSGDTrainerExample.class.getSimpleName(), () -> {
 
                 IgniteCache<Integer, double[]> dataCache = getTestCache(ignite);
 

examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GRBOnTreesRegressionTrainerExample.java → examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesRegressionTrainerExample.java

@@ -37,7 +37,7 @@
  *
  * In this example dataset is creating automatically by parabolic function f(x) = x^2.
  */
-public class GRBOnTreesRegressionTrainerExample {
+public class GDBOnTreesRegressionTrainerExample {
     /**
      * Run example.
      *
@@ -49,7 +49,7 @@ public static void main(String... args) throws InterruptedException {
             System.out.println(">>> Ignite grid started.");
 
             IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
-                GRBOnTreesRegressionTrainerExample.class.getSimpleName(), () -> {
+                GDBOnTreesRegressionTrainerExample.class.getSimpleName(), () -> {
 
                 // Create cache with training data.
                 CacheConfiguration<Integer, double[]> trainingSetCfg = createCacheConfiguration();

modules/ml/src/main/java/org/apache/ignite/ml/math/exceptions/NonPositiveDefiniteMatrixException.java → examples/src/main/java/org/apache/ignite/examples/ml/tutorial/TutorialStepByStepExample.java

@@ -15,21 +15,25 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.ml.math.exceptions;
+package org.apache.ignite.examples.ml.tutorial;
 
 /**
- * This exception is used to indicate error condition of matrix elements failing the positivity check.
+ * Run all the tutorial examples step by step with primary purpose to provide
+ * automatic execution from IgniteExamplesMLTestSuite.
  */
-public class NonPositiveDefiniteMatrixException extends MathIllegalArgumentException {
-    /**
-     * Construct an exception.
-     *
-     * @param wrong Value that fails the positivity check.
-     * @param idx Row (and column) index.
-     * @param threshold Absolute positivity threshold.
-     */
-    public NonPositiveDefiniteMatrixException(double wrong, int idx, double threshold) {
-        super("Matrix must be positive, wrong element located on diagonal with index %d and has value %f with this threshold %f",
-            idx, wrong, threshold);
+public class TutorialStepByStepExample {
+    /** Run example. */
+    public static void main(String[] args) throws InterruptedException {
+        Step_1_Read_and_Learn.main(args);
+        Step_2_Imputing.main(args);
+        Step_3_Categorial.main(args);
+        Step_3_Categorial_with_One_Hot_Encoder.main(args);
+        Step_4_Add_age_fare.main(args);
+        Step_5_Scaling.main(args);
+        Step_6_KNN.main(args);
+        Step_7_Split_train_test.main(args);
+        Step_8_CV.main(args);
+        Step_8_CV_with_Param_Grid.main(args);
+        Step_9_Go_to_LogReg.main(args);
     }
 }

modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/tree/BPlusTreeBenchmark.java

@@ -176,7 +176,7 @@ protected static class TestTree extends BPlusTree<Long, Long> {
         TestTree(ReuseList reuseList, int cacheId, PageMemory pageMem, long metaPageId)
             throws IgniteCheckedException {
             super("test", cacheId, pageMem, null, new AtomicLong(), metaPageId, reuseList,
-                new IOVersions<>(new LongInnerIO()), new IOVersions<>(new LongLeafIO()));
+                new IOVersions<>(new LongInnerIO()), new IOVersions<>(new LongLeafIO()), null);
 
             PageIO.registerTest(latestInnerIO(), latestLeafIO());
 

modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/AbstractRestProcessorSelfTest.java

@@ -43,6 +43,7 @@ public abstract class AbstractRestProcessorSelfTest extends GridCommonAbstractTe
 
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
+        cleanPersistenceDir();
         startGrids(gridCount());
     }
 
@@ -58,6 +59,12 @@ public abstract class AbstractRestProcessorSelfTest extends GridCommonAbstractTe
         assertEquals(0, jcache().localSize());
     }
 
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+        cleanPersistenceDir();
+    }
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);

modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java

@@ -246,23 +246,8 @@ public GridCacheSharedContext(
      * @throws IgniteCheckedException If failed.
      */
     public void activate() throws IgniteCheckedException {
-        if (!kernalCtx.clientNode())
-            dbMgr.lock();
-
-        boolean success = false;
-
-        try {
-            for (IgniteChangeGlobalStateSupport mgr : stateAwareMgrs)
-                mgr.onActivate(kernalCtx);
-
-            success = true;
-        }
-        finally {
-            if (!success) {
-                if (!kernalCtx.clientNode())
-                    dbMgr.unLock();
-            }
-        }
+        for (IgniteChangeGlobalStateSupport mgr : stateAwareMgrs)
+            mgr.onActivate(kernalCtx);
     }
 
     /**

modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java

@@ -472,14 +472,21 @@ private DataRegionConfiguration createDataRegionConfiguration(DataStorageConfigu
             if (!U.mkdirs(cpDir))
                 throw new IgniteCheckedException("Could not create directory for checkpoint metadata: " + cpDir);
 
-            cleanupTempCheckpointDirectory();
-
             final FileLockHolder preLocked = kernalCtx.pdsFolderResolver()
-                .resolveFolders()
-                .getLockedFileLockHolder();
+                    .resolveFolders()
+                    .getLockedFileLockHolder();
+
+            fileLockHolder = preLocked == null ?
+                        new FileLockHolder(storeMgr.workDir().getPath(), kernalCtx, log) : preLocked;
 
-            if (preLocked == null)
-                fileLockHolder = new FileLockHolder(storeMgr.workDir().getPath(), kernalCtx, log);
+            if (log.isDebugEnabled())
+                log.debug("Try to capture file lock [nodeId=" +
+                        cctx.localNodeId() + " path=" + fileLockHolder.lockPath() + "]");
+
+            if (!fileLockHolder.isLocked())
+                fileLockHolder.tryLock(lockWaitTime);
+
+            cleanupTempCheckpointDirectory();
 
             persStoreMetrics.wal(cctx.wal());
 
@@ -710,16 +717,10 @@ else if (regCfg.getMaxSize() < 8 * GB)
 
         onKernalStop0(false);
 
-        stop0(false);
+        super.onDeActivate(kctx);
 
         /* Must be here, because after deactivate we can invoke activate and file lock must be already configured */
         stopping = false;
-
-        if (!cctx.localNode().isClient()) {
-            //we replace lock with new instance (only if we're responsible for locking folders)
-            if (fileLockHolder != null)
-                fileLockHolder = new FileLockHolder(storeMgr.workDir().getPath(), cctx.kernalContext(), log);
-        }
     }
 
     /**
@@ -957,28 +958,6 @@ public List<T2<Long, WALPointer>> nodeStartedPointers() throws IgniteCheckedExce
         return res;
     }
 
-    /** {@inheritDoc} */
-    @Override public void lock() throws IgniteCheckedException {
-        if (fileLockHolder != null) {
-            if (log.isDebugEnabled())
-                log.debug("Try to capture file lock [nodeId=" +
-                    cctx.localNodeId() + " path=" + fileLockHolder.lockPath() + "]");
-
-            fileLockHolder.tryLock(lockWaitTime);
-        }
-    }
-
-    /** {@inheritDoc} */
-    @Override public void unLock() {
-        if (fileLockHolder != null) {
-            if (log.isDebugEnabled())
-                log.debug("Release file lock [nodeId=" +
-                    cctx.localNodeId() + " path=" + fileLockHolder.lockPath() + "]");
-
-            fileLockHolder.release();
-        }
-    }
-
     /** {@inheritDoc} */
     @Override protected void onKernalStop0(boolean cancel) {
         checkpointLock.writeLock().lock();
@@ -996,14 +975,22 @@ public List<T2<Long, WALPointer>> nodeStartedPointers() throws IgniteCheckedExce
 
         super.onKernalStop0(cancel);
 
+        unRegistrateMetricsMBean();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void stop0(boolean cancel) {
+        super.stop0(cancel);
+
         if (!cctx.kernalContext().clientNode()) {
-            unLock();
+            if (fileLockHolder != null) {
+                if (log.isDebugEnabled())
+                    log.debug("Release file lock [nodeId=" +
+                            cctx.localNodeId() + " path=" + fileLockHolder.lockPath() + "]");
 
-            if (fileLockHolder != null)
                 fileLockHolder.close();
+            }
         }
-
-        unRegistrateMetricsMBean();
     }
 
     /** */
@@ -4110,7 +4097,7 @@ public static class FileLockHolder implements AutoCloseable {
         private RandomAccessFile lockFile;
 
         /** Lock. */
-        private FileLock lock;
+        private volatile FileLock lock;
 
         /** Kernal context to generate Id of locked node in file. */
         @NotNull private GridKernalContext ctx;
@@ -4183,6 +4170,7 @@ public void tryLock(long lockWaitTimeMillis) throws IgniteCheckedException {
                 for (int i = 0; i < lockWaitTimeMillis; i += 1000) {
                     try {
                         lock = ch.tryLock(0, 1, false);
+
                         if (lock != null && lock.isValid()) {
                             writeContent(sb.toString());
 
@@ -4253,13 +4241,20 @@ private String readContent() throws IOException {
             return content;
         }
 
+        /** Locked or not. */
+        public boolean isLocked() {
+            return lock != null && lock.isValid();
+        }
+
         /** Releases file lock */
         public void release() {
             U.releaseQuiet(lock);
         }
 
         /** Closes file channel */
-        public void close() {
+        @Override public void close() {
+            release();
+
             U.closeQuiet(lockFile);
         }
 

modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java

@@ -132,7 +132,8 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
             PageIdAllocator.FLAG_IDX,
             reuseList,
             metastoreRoot.pageId().pageId(),
-            metastoreRoot.isAllocated());
+            metastoreRoot.isAllocated(),
+            ctx.kernalContext().failure());
 
         ((GridCacheDatabaseSharedManager)ctx.database()).addCheckpointListener(this);
     }

modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java

@@ -659,21 +659,7 @@ public ReuseList reuseList(String memPlcName) {
 
     /** {@inheritDoc} */
     @Override protected void stop0(boolean cancel) {
-        if (dataRegionMap != null) {
-            for (DataRegion memPlc : dataRegionMap.values()) {
-                memPlc.pageMemory().stop();
-
-                memPlc.evictionTracker().stop();
-
-                unregisterMBean(memPlc.memoryMetrics().getName());
-            }
-
-            dataRegionMap.clear();
-
-            dataRegionMap = null;
-
-            dataRegionsInitialized = false;
-        }
+        onDeActivate(cctx.kernalContext());
     }
 
     /**
@@ -707,20 +693,6 @@ private void unregisterMBean(String name) {
         return true;
     }
 
-    /**
-     *
-     */
-    public void lock() throws IgniteCheckedException {
-
-    }
-
-    /**
-     *
-     */
-    public void unLock() {
-
-    }
-
     /**
      * No-op for non-persistent storage.
      */
@@ -1071,7 +1043,21 @@ protected File buildPath(String path, String consId) throws IgniteCheckedExcepti
 
     /** {@inheritDoc} */
     @Override public void onDeActivate(GridKernalContext kctx) {
-        stop0(false);
+        if (dataRegionMap != null) {
+            for (DataRegion memPlc : dataRegionMap.values()) {
+                memPlc.pageMemory().stop();
+
+                memPlc.evictionTracker().stop();
+
+                unregisterMBean(memPlc.memoryMetrics().getName());
+            }
+
+            dataRegionMap.clear();
+
+            dataRegionMap = null;
+
+            dataRegionsInitialized = false;
+        }
     }
 
     /**