diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCache.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCache.java index e1d3a8737c4ee..aa4f271977b0a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCache.java @@ -71,9 +71,6 @@ public class VisorCache extends VisorDataTransferObject { /** Number of backup entries in cache. */ private long backupSize; - /** Number of cache entries stored in heap memory. */ - private long onHeapEntriesCnt; - /** Number of partitions. */ private int partitions; @@ -112,8 +109,6 @@ public VisorCache(IgniteEx ignite, GridCacheAdapter ca) throws IgniteCheckedExce partitions = ca.affinity().partitions(); near = cctx.isNear(); - onHeapEntriesCnt = 0; // TODO GG-11148 How to get this metric? - metrics = new VisorCacheMetrics(ignite, name); } @@ -131,7 +126,6 @@ public VisorCache history() { c.nearSize = nearSize; c.backupSize = backupSize; c.primarySize = primarySize; - c.onHeapEntriesCnt = onHeapEntriesCnt; c.partitions = partitions; c.metrics = metrics; c.near = near; @@ -211,13 +205,6 @@ public long getPrimarySize() { return primarySize; } - /** - * @return Number of cache entries stored in heap memory. - */ - public long getOnHeapEntriesCount() { - return onHeapEntriesCnt; - } - /** * @return Number of partitions. */ @@ -250,7 +237,6 @@ public boolean isNear() { out.writeInt(nearSize); out.writeLong(primarySize); out.writeLong(backupSize); - out.writeLong(onHeapEntriesCnt); out.writeInt(partitions); out.writeBoolean(near); out.writeObject(metrics); @@ -267,7 +253,6 @@ public boolean isNear() { nearSize = in.readInt(); primarySize = in.readLong(); backupSize = in.readLong(); - onHeapEntriesCnt = in.readLong(); partitions = in.readInt(); near = in.readBoolean(); metrics = (VisorCacheMetrics)in.readObject(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetrics.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetrics.java index 5c1dfcb3f26ba..e67f5f8c92f4a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetrics.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetrics.java @@ -157,6 +157,9 @@ public class VisorCacheMetrics extends VisorDataTransferObject { /** Number of cached rolled back DHT transaction IDs. */ private int txDhtRolledbackVersionsSize; + /** Number of cache entries stored in heap memory. */ + private long heapEntriesCnt; + /** Memory size allocated in off-heap. */ private long offHeapAllocatedSize; @@ -259,6 +262,7 @@ public VisorCacheMetrics(IgniteEx ignite, String cacheName) { txDhtCommittedVersionsSize = m.getTxDhtCommittedVersionsSize(); txDhtRolledbackVersionsSize = m.getTxDhtRolledbackVersionsSize(); + heapEntriesCnt = m.getHeapEntriesCount(); offHeapAllocatedSize = m.getOffHeapAllocatedSize(); offHeapEntriesCnt = m.getOffHeapEntriesCount(); @@ -560,6 +564,13 @@ public int getTxDhtRolledbackVersionsSize() { return txDhtRolledbackVersionsSize; } + /** + * @return Number of entries in heap memory. + */ + public long getHeapEntriesCount() { + return heapEntriesCnt; + } + /** * @return Memory size allocated in off-heap. */ @@ -653,6 +664,7 @@ public long getRebalancingBytesRate() { out.writeInt(txDhtCommittedVersionsSize); out.writeInt(txDhtRolledbackVersionsSize); + out.writeLong(heapEntriesCnt); out.writeLong(offHeapAllocatedSize); out.writeLong(offHeapEntriesCnt); @@ -708,6 +720,7 @@ public long getRebalancingBytesRate() { txDhtCommittedVersionsSize = in.readInt(); txDhtRolledbackVersionsSize = in.readInt(); + heapEntriesCnt = in.readLong(); offHeapAllocatedSize = in.readLong(); offHeapEntriesCnt = in.readLong(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorMemoryMetrics.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorMemoryMetrics.java new file mode 100644 index 0000000000000..14eaba66fe677 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorMemoryMetrics.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.visor.cache; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import org.apache.ignite.MemoryMetrics; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.internal.visor.VisorDataTransferObject; + +/** + * Data transfer object for {@link MemoryMetrics} + */ +public class VisorMemoryMetrics extends VisorDataTransferObject { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + private String name; + + /** */ + private long totalAllocatedPages; + + /** */ + private float allocationRate; + + /** */ + private float evictionRate; + + /** */ + private float largeEntriesPagesPercentage; + + /** */ + private float pagesFillFactor; + + /** */ + private long dirtyPages; + + /** */ + private float pageReplaceRate; + + /** */ + private long physicalMemoryPages; + + /** + * Default constructor. + */ + public VisorMemoryMetrics() { + // No-op. + } + + /** + * @param m Metrics instance to create DTO. + */ + public VisorMemoryMetrics(MemoryMetrics m) { + name = m.getName(); + totalAllocatedPages = m.getTotalAllocatedPages(); + allocationRate = m.getAllocationRate(); + evictionRate = m.getEvictionRate(); + largeEntriesPagesPercentage = m.getLargeEntriesPagesPercentage(); + pagesFillFactor = m.getPagesFillFactor(); + dirtyPages = m.getDirtyPages(); + pageReplaceRate = m.getPagesReplaceRate(); + physicalMemoryPages = m.getPhysicalMemoryPages(); + } + + /** + * @return Name of the memory region. + */ + public String getName() { + return name; + } + + /** + * @return Total number of allocated pages. + */ + public long getTotalAllocatedPages() { + return totalAllocatedPages; + } + + /** + * @return Number of allocated pages per second. + */ + public float getAllocationRate() { + return allocationRate; + } + + /** {@inheritDoc} */ + public float getEvictionRate() { + return evictionRate; + } + + /** + * @return Number of evicted pages per second. + */ + public float getLargeEntriesPagesPercentage() { + return largeEntriesPagesPercentage; + } + + /** + * @return Percentage of pages fully occupied by large entities. + */ + public float getPagesFillFactor() { + return pagesFillFactor; + } + + /** + * @return Current number of dirty pages. + */ + public long getDirtyPages() { + return dirtyPages; + } + + /** + * @return Pages per second replace rate. + */ + public float getPagesReplaceRate() { + return pageReplaceRate; + } + + /** + * @return Total number of pages loaded to RAM. + */ + public long getPhysicalMemoryPages() { + return physicalMemoryPages; + } + + /** {@inheritDoc} */ + @Override protected void writeExternalData(ObjectOutput out) throws IOException { + U.writeString(out, name); + out.writeLong(totalAllocatedPages); + out.writeFloat(allocationRate); + out.writeFloat(evictionRate); + out.writeFloat(largeEntriesPagesPercentage); + out.writeFloat(pagesFillFactor); + out.writeLong(dirtyPages); + out.writeFloat(pageReplaceRate); + out.writeLong(physicalMemoryPages); + } + + /** {@inheritDoc} */ + @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException { + name = U.readString(in); + totalAllocatedPages = in.readLong(); + allocationRate = in.readFloat(); + evictionRate = in.readFloat(); + largeEntriesPagesPercentage = in.readFloat(); + pagesFillFactor = in.readFloat(); + dirtyPages = in.readLong(); + pageReplaceRate = in.readFloat(); + physicalMemoryPages = in.readLong(); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(VisorMemoryMetrics.class, this); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java index 02d3b6594216e..f5ea5670484ba 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java @@ -18,8 +18,10 @@ package org.apache.ignite.internal.visor.node; import java.util.Collection; +import java.util.List; import java.util.concurrent.ConcurrentMap; import org.apache.ignite.IgniteFileSystem; +import org.apache.ignite.MemoryMetrics; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.FileSystemConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; @@ -33,6 +35,7 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.cache.VisorCache; +import org.apache.ignite.internal.visor.cache.VisorMemoryMetrics; import org.apache.ignite.internal.visor.compute.VisorComputeMonitoringHolder; import org.apache.ignite.internal.visor.igfs.VisorIgfs; import org.apache.ignite.internal.visor.igfs.VisorIgfsEndpoint; @@ -115,8 +118,8 @@ protected void events(VisorNodeDataCollectorJobResult res, VisorNodeDataCollecto events0(res, arg.getEventsOrderKey(), arg.getEventsThrottleCounterKey(), arg.isTaskMonitoringEnabled()); } - catch (Exception eventsEx) { - res.setEventsEx(eventsEx); + catch (Exception e) { + res.setEventsEx(e); } } @@ -144,6 +147,23 @@ private boolean proxyCache(String cacheName) { return !(discovery.cacheAffinityNode(locNode, cacheName) || discovery.cacheNearNode(locNode, cacheName)); } + /** + * Collect memory metrics. + * + * @param res Job result. + */ + protected void memoryMetrics(VisorNodeDataCollectorJobResult res) { + try { + List memoryMetrics = res.getMemoryMetrics(); + + for (MemoryMetrics m : ignite.memoryMetrics()) + memoryMetrics.add(new VisorMemoryMetrics(m)); + } + catch (Exception e) { + res.setMemoryMetricsEx(e); + } +} + /** * Collect caches. * @@ -156,6 +176,8 @@ protected void caches(VisorNodeDataCollectorJobResult res, VisorNodeDataCollecto GridCacheProcessor cacheProc = ignite.context().cache(); + List resCaches = res.getCaches(); + for (String cacheName : cacheProc.cacheNames()) { if (proxyCache(cacheName)) continue; @@ -169,7 +191,7 @@ protected void caches(VisorNodeDataCollectorJobResult res, VisorNodeDataCollecto if (ca == null || !ca.context().started()) continue; - res.getCaches().add(new VisorCache(ignite, ca)); + resCaches.add(new VisorCache(ignite, ca)); } catch(IllegalStateException | IllegalArgumentException e) { if (debug && ignite.log() != null) @@ -182,8 +204,8 @@ protected void caches(VisorNodeDataCollectorJobResult res, VisorNodeDataCollecto } } } - catch (Exception cachesEx) { - res.setCachesEx(cachesEx); + catch (Exception e) { + res.setCachesEx(e); } } @@ -222,8 +244,8 @@ protected void igfs(VisorNodeDataCollectorJobResult res) { } } } - catch (Exception igfssEx) { - res.setIgfssEx(igfssEx); + catch (Exception e) { + res.setIgfssEx(e); } } @@ -257,6 +279,11 @@ protected VisorNodeDataCollectorJobResult run(VisorNodeDataCollectorJobResult re if (debug) start0 = log(ignite.log(), "Collected events", getClass(), start0); + memoryMetrics(res); + + if (debug) + start0 = log(ignite.log(), "Collected memory metrics", getClass(), start0); + caches(res, arg); if (debug) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJobResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJobResult.java index 761d0ffe1d723..ce4f9fcbdf72d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJobResult.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJobResult.java @@ -26,6 +26,7 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorDataTransferObject; import org.apache.ignite.internal.visor.cache.VisorCache; +import org.apache.ignite.internal.visor.cache.VisorMemoryMetrics; import org.apache.ignite.internal.visor.event.VisorGridEvent; import org.apache.ignite.internal.visor.igfs.VisorIgfs; import org.apache.ignite.internal.visor.igfs.VisorIgfsEndpoint; @@ -52,6 +53,12 @@ public class VisorNodeDataCollectorJobResult extends VisorDataTransferObject { /** Exception while collecting node events. */ private Throwable evtsEx; + /** Node memory metrics. */ + private List memoryMetrics = new ArrayList<>(); + + /** Exception while collecting memory metrics. */ + private Throwable memoryMetricsEx; + /** Node caches. */ private List caches = new ArrayList<>(); @@ -146,6 +153,27 @@ public void setEventsEx(Throwable evtsEx) { this.evtsEx = evtsEx; } + /** + * @return Collected memory metrics. + */ + public List getMemoryMetrics() { + return memoryMetrics; + } + + /** + * @return Exception caught during collecting memory metrics. + */ + public Throwable getMemoryMetricsEx() { + return memoryMetricsEx; + } + + /** + * @param memoryMetricsEx Exception caught during collecting memory metrics. + */ + public void setMemoryMetricsEx(Throwable memoryMetricsEx) { + this.memoryMetricsEx = memoryMetricsEx; + } + /** * @return Collected cache metrics. */ @@ -244,6 +272,8 @@ public void setHasPendingExchange(boolean hasPendingExchange) { out.writeBoolean(taskMonitoringEnabled); U.writeCollection(out, evts); out.writeObject(evtsEx); + U.writeCollection(out, memoryMetrics); + out.writeObject(memoryMetricsEx); U.writeCollection(out, caches); out.writeObject(cachesEx); U.writeCollection(out, igfss); @@ -261,6 +291,8 @@ public void setHasPendingExchange(boolean hasPendingExchange) { taskMonitoringEnabled = in.readBoolean(); evts = U.readList(in); evtsEx = (Throwable)in.readObject(); + memoryMetrics = U.readList(in); + memoryMetricsEx = (Throwable)in.readObject(); caches = U.readList(in); cachesEx = (Throwable)in.readObject(); igfss = U.readList(in); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTask.java index 2790dec689473..56b3718bb2f4b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTask.java @@ -99,6 +99,12 @@ protected void reduceJobResult(VisorNodeDataCollectorTaskResult taskRes, if (jobRes.getEventsEx() != null) taskRes.getEventsEx().put(nid, new VisorExceptionWrapper(jobRes.getEventsEx())); + if (!jobRes.getMemoryMetrics().isEmpty()) + taskRes.getMemoryMetrics().put(nid, jobRes.getMemoryMetrics()); + + if (jobRes.getMemoryMetricsEx() != null) + taskRes.getMemoryMetricsEx().put(nid, new VisorExceptionWrapper(jobRes.getMemoryMetricsEx())); + if (!jobRes.getCaches().isEmpty()) taskRes.getCaches().put(nid, jobRes.getCaches()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskResult.java index 8d1fe8e90ebef..cef3a29054ce9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskResult.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskResult.java @@ -30,6 +30,7 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.VisorDataTransferObject; import org.apache.ignite.internal.visor.cache.VisorCache; +import org.apache.ignite.internal.visor.cache.VisorMemoryMetrics; import org.apache.ignite.internal.visor.event.VisorGridEvent; import org.apache.ignite.internal.visor.igfs.VisorIgfs; import org.apache.ignite.internal.visor.igfs.VisorIgfsEndpoint; @@ -66,6 +67,12 @@ public class VisorNodeDataCollectorTaskResult extends VisorDataTransferObject { /** Exceptions caught during collecting events from nodes. */ private Map evtsEx = new HashMap<>(); + /** All memory metrics collected from nodes. */ + private Map> memoryMetrics = new HashMap<>(); + + /** Exceptions caught during collecting memory metrics from nodes. */ + private Map memoryMetricsEx = new HashMap<>(); + /** All caches collected from nodes. */ private Map> caches = new HashMap<>(); @@ -105,6 +112,8 @@ public boolean isEmpty() { taskMonitoringEnabled.isEmpty() && evts.isEmpty() && evtsEx.isEmpty() && + memoryMetrics.isEmpty() && + memoryMetricsEx.isEmpty() && caches.isEmpty() && cachesEx.isEmpty() && igfss.isEmpty() && @@ -170,6 +179,20 @@ public Map getEventsEx() { return evtsEx; } + /** + * @return All memory metrics collected from nodes. + */ + public Map> getMemoryMetrics() { + return memoryMetrics; + } + + /** + * @return Exceptions caught during collecting memory metrics from nodes. + */ + public Map getMemoryMetricsEx() { + return memoryMetricsEx; + } + /** * @return All caches collected from nodes. */ @@ -236,6 +259,8 @@ public Map getPendingExchanges() { U.writeMap(out, errCnts); U.writeCollection(out, evts); U.writeMap(out, evtsEx); + U.writeMap(out, memoryMetrics); + U.writeMap(out, memoryMetricsEx); U.writeMap(out, caches); U.writeMap(out, cachesEx); U.writeMap(out, igfss); @@ -255,6 +280,8 @@ public Map getPendingExchanges() { errCnts = U.readMap(in); evts = U.readList(in); evtsEx = U.readMap(in); + memoryMetrics = U.readMap(in); + memoryMetricsEx = U.readMap(in); caches = U.readMap(in); cachesEx = U.readMap(in); igfss = U.readMap(in);