1. First version of MT profiler.

2. Issue #5503 was fixed.

core/src/main/java/com/orientechnologies/orient/core/config/OGlobalConfiguration.java

@@ -93,8 +93,7 @@ public enum OGlobalConfiguration {
    * default value is 10000.
    */
   COMPONENTS_LOCK_CACHE("storage.componentsLock.cache",
-      "Amount of cached locks is used for component lock to avoid constant creation of new lock instances", Integer.class,
-      10000),
+      "Amount of cached locks is used for component lock to avoid constant creation of new lock instances", Integer.class, 10000),
 
   DISK_CACHE_PINNED_PAGES("storage.diskCache.pinnedPages", "Maximum amount of pinned pages which may be contained in cache,"
       + " if this percent is reached next pages will be left in unpinned state. You can not set value more than 50", Integer.class,
@@ -466,6 +465,12 @@ public void change(final Object iCurrentValue, final Object iNewValue) {
 
   PROFILER_MAXVALUES("profiler.maxValues", "Maximum values to store. Values are managed in a LRU", Integer.class, 200),
 
+  /**
+   * Interval between snapshots of profiler state in milliseconds, default value is 100.
+   */
+  STORAGE_PROFILER_SNAPSHOT_INTERVAL("storageProfiler.intervalBetweenSnapshots",
+      "Interval between snapshots of profiler state in milliseconds", Integer.class, 100),
+
   // LOG
   LOG_CONSOLE_LEVEL("log.console.level", "Console logging level.", String.class, "info", new OConfigurationChangeCallback() {
     public void change(final Object iCurrentValue, final Object iNewValue) {

core/src/main/java/com/orientechnologies/orient/core/storage/cache/OWriteCache.java

@@ -23,6 +23,7 @@
 import com.orientechnologies.common.types.OModifiableBoolean;
 import com.orientechnologies.orient.core.command.OCommandOutputListener;
 import com.orientechnologies.orient.core.storage.impl.local.OLowDiskSpaceListener;
+import com.orientechnologies.orient.core.storage.impl.local.statistic.OPerformanceStatisticManager;
 
 import java.io.File;
 import java.io.IOException;
@@ -127,4 +128,6 @@ OCachePointer[] load(long fileId, long startPageIndex, int pageCount, boolean ad
    * @see #getId()
    */
   long externalFileId(int fileId);
+
+  OPerformanceStatisticManager getPerformanceStatisticManager();
 }

core/src/main/java/com/orientechnologies/orient/core/storage/cache/local/OWOWCache.java

@@ -48,6 +48,7 @@
 import com.orientechnologies.orient.core.storage.impl.local.paginated.base.ODurablePage;
 import com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OLogSequenceNumber;
 import com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OWriteAheadLog;
+import com.orientechnologies.orient.core.storage.impl.local.statistic.OPerformanceStatisticManager;
 import com.orientechnologies.orient.core.storage.impl.local.statistic.OSessionStoragePerformanceStatistic;
 
 import javax.management.*;
@@ -128,6 +129,8 @@ public class OWOWCache extends OAbstractWriteCache implements OWriteCache, OCach
 
   private final int id;
 
+  private final OPerformanceStatisticManager performanceStatisticManager;
+
   private final AtomicReference<Date> lastFuzzyCheckpointDate  = new AtomicReference<Date>();
   private final AtomicLong            lastAmountOfFlushedPages = new AtomicLong();
   private final AtomicLong            durationOfLastFlush      = new AtomicLong();
@@ -165,6 +168,7 @@ public OWOWCache(boolean syncOnPageFlush, int pageSize, OByteBufferPool bufferPo
       this.storageLocal = storageLocal;
 
       this.storagePath = storageLocal.getVariableParser().resolveVariables(storageLocal.getStoragePath());
+      this.performanceStatisticManager = storageLocal.getPerformanceStatisticManager();
 
       final OBinarySerializerFactory binarySerializerFactory = storageLocal.getComponentsFactory().binarySerializerFactory;
       this.stringSerializer = binarySerializerFactory.getObjectSerializer(OType.STRING);
@@ -201,6 +205,11 @@ public File getRootDirectory() {
     return new File(storagePath);
   }
 
+  @Override
+  public OPerformanceStatisticManager getPerformanceStatisticManager() {
+    return performanceStatisticManager;
+  }
+
   public void startFuzzyCheckpoints() {
     if (writeAheadLog != null) {
       final long fuzzyCheckPointInterval = OGlobalConfiguration.WAL_FUZZY_CHECKPOINT_INTERVAL.getValueAsInteger();
@@ -1271,8 +1280,8 @@ private OCachePointer[] cacheFileContent(final long fileId, final int intId, fin
     final long firstPageEndPosition = firstPageStartPosition + pageSize;
 
     if (fileClassic.getFileSize() >= firstPageEndPosition) {
-      final OSessionStoragePerformanceStatistic sessionStoragePerformanceStatistic = OSessionStoragePerformanceStatistic
-          .getStatisticInstance();
+      final OSessionStoragePerformanceStatistic sessionStoragePerformanceStatistic = performanceStatisticManager
+          .getSessionPerformanceStatistic();
       if (sessionStoragePerformanceStatistic != null) {
         sessionStoragePerformanceStatistic.startPageReadFromFileTimer();
       }

core/src/main/java/com/orientechnologies/orient/core/storage/cache/local/twoq/O2QCache.java

@@ -377,8 +377,8 @@ public void changeMaximumAmountOfMemory(final long readCacheMaxMemory) throws Il
   public OCacheEntry load(long fileId, final long pageIndex, final boolean checkPinnedPages, OWriteCache writeCache,
       final int pageCount) throws IOException {
 
-    final OSessionStoragePerformanceStatistic sessionStoragePerformanceStatistic = OSessionStoragePerformanceStatistic
-        .getStatisticInstance();
+    final OSessionStoragePerformanceStatistic sessionStoragePerformanceStatistic = writeCache.getPerformanceStatisticManager()
+        .getSessionPerformanceStatistic();
 
     if (sessionStoragePerformanceStatistic != null) {
       sessionStoragePerformanceStatistic.startPageReadFromCacheTimer();
@@ -473,8 +473,8 @@ private UpdateCacheResult doLoad(long fileId, long pageIndex, boolean checkPinne
 
   @Override
   public OCacheEntry allocateNewPage(long fileId, OWriteCache writeCache) throws IOException {
-    final OSessionStoragePerformanceStatistic sessionStoragePerformanceStatistic = OSessionStoragePerformanceStatistic
-        .getStatisticInstance();
+    final OSessionStoragePerformanceStatistic sessionStoragePerformanceStatistic = writeCache.getPerformanceStatisticManager()
+        .getSessionPerformanceStatistic();
 
     if (sessionStoragePerformanceStatistic != null) {
       sessionStoragePerformanceStatistic.startPageReadFromCacheTimer();
@@ -537,8 +537,8 @@ public void release(OCacheEntry cacheEntry, OWriteCache writeCache) {
           assert cacheEntry.getUsagesCount() >= 0;
 
           if (cacheEntry.getUsagesCount() == 0 && cacheEntry.isDirty()) {
-            final OSessionStoragePerformanceStatistic sessionStoragePerformanceStatistic = OSessionStoragePerformanceStatistic
-                .getStatisticInstance();
+            final OSessionStoragePerformanceStatistic sessionStoragePerformanceStatistic = writeCache.getPerformanceStatisticManager()
+                .getSessionPerformanceStatistic();
 
             if (sessionStoragePerformanceStatistic != null) {
               sessionStoragePerformanceStatistic.startPageWriteInCacheTimer();

core/src/main/java/com/orientechnologies/orient/core/storage/impl/local/OAbstractPaginatedStorage.java

@@ -73,6 +73,7 @@
 import com.orientechnologies.orient.core.storage.impl.local.paginated.atomicoperations.OAtomicOperationsManager;
 import com.orientechnologies.orient.core.storage.impl.local.paginated.base.ODurablePage;
 import com.orientechnologies.orient.core.storage.impl.local.paginated.wal.*;
+import com.orientechnologies.orient.core.storage.impl.local.statistic.OPerformanceStatisticManager;
 import com.orientechnologies.orient.core.storage.impl.local.statistic.OSessionStoragePerformanceStatistic;
 import com.orientechnologies.orient.core.tx.OTransaction;
 import com.orientechnologies.orient.core.tx.OTransactionAbstract;
@@ -114,9 +115,13 @@ public abstract class OAbstractPaginatedStorage extends OStorageAbstract
 
   private volatile ThreadLocal<OStorageTransaction> transaction          = new ThreadLocal<OStorageTransaction>();
   private final    AtomicBoolean                    checkpointInProgress = new AtomicBoolean();
-  protected final    OSBTreeCollectionManagerShared sbTreeCollectionManager;
-  protected volatile OWriteAheadLog                 writeAheadLog;
-  private            OStorageRecoverListener        recoverListener;
+  protected final OSBTreeCollectionManagerShared sbTreeCollectionManager;
+
+  private final OPerformanceStatisticManager performanceStatisticManager = new OPerformanceStatisticManager(
+      OGlobalConfiguration.STORAGE_PROFILER_SNAPSHOT_INTERVAL.getValueAsInteger() * 1000000L);
+
+  protected volatile OWriteAheadLog          writeAheadLog;
+  private            OStorageRecoverListener recoverListener;
 
   protected volatile OReadCache  readCache;
   protected volatile OWriteCache writeCache;
@@ -1154,15 +1159,18 @@ public OStorageOperationResult<Boolean> hideRecord(final ORecordId rid, final in
     }
   }
 
+  public OPerformanceStatisticManager getPerformanceStatisticManager() {
+    return performanceStatisticManager;
+  }
+
   /**
    * Starts to gather information about storage performance for current thread. Details which performance characteristics are
    * gathered can be found at {@link OSessionStoragePerformanceStatistic}.
    *
    * @see #completeGatheringPerformanceStatisticForCurrentThread()
    */
   public void startGatheringPerformanceStatisticForCurrentThread() {
-    OSessionStoragePerformanceStatistic
-        .initThreadLocalInstance(OGlobalConfiguration.DISK_CACHE_PAGE_SIZE.getValueAsInteger() * 1024);
+    performanceStatisticManager.startThreadMonitoring();
   }
 
   /**
@@ -1173,7 +1181,7 @@ public void startGatheringPerformanceStatisticForCurrentThread() {
    * <code>null</code> if profiling of storage was not started.
    */
   public OSessionStoragePerformanceStatistic completeGatheringPerformanceStatisticForCurrentThread() {
-    return OSessionStoragePerformanceStatistic.clearThreadLocalInstance();
+    return performanceStatisticManager.stopThreadMonitoring();
   }
 
   @Override
@@ -3255,7 +3263,6 @@ private void doClose(boolean force, boolean onDelete) {
 
       preCloseSteps();
 
-
       sbTreeCollectionManager.close();
 
       closeClusters(onDelete);

core/src/main/java/com/orientechnologies/orient/core/storage/impl/local/paginated/atomicoperations/OAtomicOperation.java

@@ -26,6 +26,7 @@
 import com.orientechnologies.orient.core.storage.cache.OWriteCache;
 import com.orientechnologies.orient.core.storage.impl.local.paginated.base.ODurablePage;
 import com.orientechnologies.orient.core.storage.impl.local.paginated.wal.*;
+import com.orientechnologies.orient.core.storage.impl.local.statistic.OPerformanceStatisticManager;
 import com.orientechnologies.orient.core.storage.impl.local.statistic.OSessionStoragePerformanceStatistic;
 
 import java.io.IOException;
@@ -55,13 +56,16 @@ public class OAtomicOperation {
   private OReadCache  readCache;
   private OWriteCache writeCache;
 
+  private final OPerformanceStatisticManager performanceStatisticManager;
+
   private final Map<String, OAtomicOperationMetadata<?>> metadata = new LinkedHashMap<String, OAtomicOperationMetadata<?>>();
 
   public OAtomicOperation(OLogSequenceNumber startLSN, OOperationUnitId operationUnitId, OReadCache readCache,
-      OWriteCache writeCache, int storageId) {
+      OWriteCache writeCache, int storageId, OPerformanceStatisticManager performanceStatisticManager) {
     this.storageId = storageId;
     this.startLSN = startLSN;
     this.operationUnitId = operationUnitId;
+    this.performanceStatisticManager = performanceStatisticManager;
     startCounter = 1;
     this.readCache = readCache;
     this.writeCache = writeCache;
@@ -350,8 +354,9 @@ public void truncateFile(long fileId) {
   }
 
   public void commitChanges(OWriteAheadLog writeAheadLog) throws IOException {
-    final OSessionStoragePerformanceStatistic sessionStoragePerformanceStatistic = OSessionStoragePerformanceStatistic
-        .getStatisticInstance();
+    final OSessionStoragePerformanceStatistic sessionStoragePerformanceStatistic = performanceStatisticManager
+        .getSessionPerformanceStatistic();
+
     if (sessionStoragePerformanceStatistic != null) {
       sessionStoragePerformanceStatistic.startCommitTimer();
       sessionStoragePerformanceStatistic.startComponentOperation("atomic operation");

core/src/main/java/com/orientechnologies/orient/core/storage/impl/local/paginated/atomicoperations/OAtomicOperationsManager.java

@@ -38,21 +38,17 @@
 import com.orientechnologies.orient.core.storage.impl.local.paginated.wal.ONonTxOperationPerformedWALRecord;
 import com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OOperationUnitId;
 import com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OWriteAheadLog;
+import com.orientechnologies.orient.core.storage.impl.local.statistic.OPerformanceStatisticManager;
 
-import javax.management.InstanceAlreadyExistsException;
-import javax.management.InstanceNotFoundException;
-import javax.management.MBeanRegistrationException;
-import javax.management.MBeanServer;
-import javax.management.MalformedObjectNameException;
-import javax.management.NotCompliantMBeanException;
-import javax.management.ObjectName;
+import javax.management.*;
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.lang.management.ManagementFactory;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
-import java.util.*;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -83,6 +79,7 @@ public class OAtomicOperationsManager implements OAtomicOperationsMangerMXBean {
   private final AtomicReference<WaitingListNode> waitingTail = new AtomicReference<WaitingListNode>();
 
   private static volatile ThreadLocal<OAtomicOperation> currentOperation = new ThreadLocal<OAtomicOperation>();
+  private final OPerformanceStatisticManager performanceStatisticManager;
 
   /**
    * Flag which indicates whether we work in unsafe mode for current thread.
@@ -112,9 +109,10 @@ public void onShutdown() {
     });
   }
 
-  private final OAbstractPaginatedStorage    storage;
-  private final OWriteAheadLog               writeAheadLog;
-  private final OLockManager<String> lockManager = new OLockManager<String>(true, -1, OGlobalConfiguration.COMPONENTS_LOCK_CACHE.getValueAsInteger());
+  private final OAbstractPaginatedStorage storage;
+  private final OWriteAheadLog            writeAheadLog;
+  private final OLockManager<String> lockManager = new OLockManager<String>(true, -1,
+      OGlobalConfiguration.COMPONENTS_LOCK_CACHE.getValueAsInteger());
   private final OReadCache  readCache;
   private final OWriteCache writeCache;
 
@@ -125,6 +123,7 @@ public OAtomicOperationsManager(OAbstractPaginatedStorage storage) {
     this.writeAheadLog = storage.getWALInstance();
     this.readCache = storage.getReadCache();
     this.writeCache = storage.getWriteCache();
+    this.performanceStatisticManager = storage.getPerformanceStatisticManager();
   }
 
   /**
@@ -198,7 +197,7 @@ public OAtomicOperation startAtomicOperation(String fullName, boolean trackNonTx
     final OOperationUnitId unitId = OOperationUnitId.generateId();
     final OLogSequenceNumber lsn = writeAheadLog.logAtomicOperationStartRecord(true, unitId);
 
-    operation = new OAtomicOperation(lsn, unitId, readCache, writeCache, storage.getId());
+    operation = new OAtomicOperation(lsn, unitId, readCache, writeCache, storage.getId(), performanceStatisticManager);
     currentOperation.set(operation);
 
     if (trackAtomicOperations) {
@@ -458,9 +457,9 @@ public void registerMBean() {
           server.registerMBean(this, mbeanName);
         } else {
           mbeanIsRegistered.set(false);
-          OLogManager.instance().warn(this, "MBean with name %s has already registered. Probably your system was not shutdown correctly "
-              + "or you have several running applications which use OrientDB engine inside",
-              mbeanName.getCanonicalName());
+          OLogManager.instance().warn(this,
+              "MBean with name %s has already registered. Probably your system was not shutdown correctly "
+                  + "or you have several running applications which use OrientDB engine inside", mbeanName.getCanonicalName());
         }
 
       } catch (MalformedObjectNameException e) {

core/src/main/java/com/orientechnologies/orient/core/storage/impl/local/paginated/base/ODurableComponent.java

@@ -28,6 +28,7 @@
 import com.orientechnologies.orient.core.storage.impl.local.paginated.atomicoperations.OAtomicOperation;
 import com.orientechnologies.orient.core.storage.impl.local.paginated.atomicoperations.OAtomicOperationsManager;
 import com.orientechnologies.orient.core.storage.impl.local.paginated.wal.OWALChanges;
+import com.orientechnologies.orient.core.storage.impl.local.statistic.OPerformanceStatisticManager;
 import com.orientechnologies.orient.core.storage.impl.local.statistic.OSessionStoragePerformanceStatistic;
 
 import java.io.IOException;
@@ -57,17 +58,17 @@
  * @since 8/27/13
  */
 public abstract class ODurableComponent extends OSharedResourceAdaptive {
-  protected final OAtomicOperationsManager  atomicOperationsManager;
-  protected final OAbstractPaginatedStorage storage;
-  protected final OReadCache                readCache;
-  protected final OWriteCache               writeCache;
+  protected final OAtomicOperationsManager     atomicOperationsManager;
+  protected final OAbstractPaginatedStorage    storage;
+  protected final OReadCache                   readCache;
+  protected final OWriteCache                  writeCache;
+  private final   OPerformanceStatisticManager performanceStatisticManager;
 
   private volatile String name;
   private volatile String fullName;
 
   protected final String extension;
 
-
   public ODurableComponent(OAbstractPaginatedStorage storage, String name, String extension) {
     super(true);
 
@@ -79,6 +80,7 @@ public ODurableComponent(OAbstractPaginatedStorage storage, String name, String
     this.atomicOperationsManager = storage.getAtomicOperationsManager();
     this.readCache = storage.getReadCache();
     this.writeCache = storage.getWriteCache();
+    this.performanceStatisticManager = storage.getPerformanceStatisticManager();
   }
 
   public String getName() {
@@ -212,16 +214,16 @@ protected void truncateFile(OAtomicOperation atomicOperation, long filedId) thro
   }
 
   protected void startOperation() {
-    OSessionStoragePerformanceStatistic sessionStoragePerformanceStatistic = OSessionStoragePerformanceStatistic
-        .getStatisticInstance();
+    OSessionStoragePerformanceStatistic sessionStoragePerformanceStatistic = performanceStatisticManager
+        .getSessionPerformanceStatistic();
     if (sessionStoragePerformanceStatistic != null) {
       sessionStoragePerformanceStatistic.startComponentOperation(getFullName());
     }
   }
 
   protected void completeOperation() {
-    OSessionStoragePerformanceStatistic sessionStoragePerformanceStatistic = OSessionStoragePerformanceStatistic
-        .getStatisticInstance();
+    OSessionStoragePerformanceStatistic sessionStoragePerformanceStatistic = performanceStatisticManager
+        .getSessionPerformanceStatistic();
     if (sessionStoragePerformanceStatistic != null) {
       sessionStoragePerformanceStatistic.completeComponentOperation();
     }