From ce9048c995d732f129a41a96d6e7ffcc177cc9d8 Mon Sep 17 00:00:00 2001 From: Hossein Taleghani Date: Mon, 30 Jul 2018 19:15:35 +0430 Subject: [PATCH] Add fields to info metricset of Redis module (#7695) Add memory fragmentation ratio, active defragmentation stats, slave status and some other fields that were missing from info metricset. --- CHANGELOG.asciidoc | 4 +- metricbeat/docs/fields.asciidoc | 524 ++++++++++++++++-- metricbeat/module/redis/fields.go | 2 +- metricbeat/module/redis/info/_meta/fields.yml | 336 ++++++++--- metricbeat/module/redis/info/data.go | 82 ++- metricbeat/module/redis/info/info.go | 7 + .../redis/info/info_integration_test.go | 2 +- metricbeat/module/redis/redis.go | 11 + metricbeat/tests/system/test_redis.py | 2 +- 9 files changed, 839 insertions(+), 131 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 44eeeb329d4..0ca8c0626fc 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -61,8 +61,9 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff] *Heartbeat* *Metricbeat* - - Add metrics about cache size to memcached module {pull}7740[7740] +- Add fields for mermory fragmentation, memory allocator stats, copy on write, master-slave status, and active defragmentation to `info` metricset of Redis module. {pull}7695[7695] + *Packetbeat* @@ -78,6 +79,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff] *Heartbeat* *Metricbeat* +- Redis `info` `replication.master_offset` has been deprecated in favor of `replication.master.offset`.{pull}7695[7695] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 93266bcfd09..12e23bfb714 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -14612,7 +14612,7 @@ Redis memory stats. -- type: long -format: bytes Used memory. +format: bytes Total number of bytes allocated by Redis. @@ -14625,7 +14625,7 @@ type: long format: bytes -Used memory rss. +Number of bytes that Redis allocated as seen by the operating system (a.k.a resident set size). -- @@ -14637,7 +14637,7 @@ type: long format: bytes -Used memory peak. +Peak memory consumed by Redis. -- @@ -14649,7 +14649,19 @@ type: long format: bytes -Used memory lua. +Used memory by the Lua engine. + + +-- + +*`redis.info.memory.used.dataset`*:: ++ +-- +type: long + +format: bytes + +The size in bytes of the dataset -- @@ -14673,6 +14685,39 @@ type: keyword Eviction policy to use when memory limit is reached. + +-- + +*`redis.info.memory.fragmentation.ratio`*:: ++ +-- +type: float + +Ratio between used_memory_rss and used_memory + + +-- + +*`redis.info.memory.fragmentation.bytes`*:: ++ +-- +type: long + +format: bytes + +Bytes between used_memory_rss and used_memory + + +-- + +*`redis.info.memory.active_defrag.is_running`*:: ++ +-- +type: boolean + +Flag indicating if active defragmentation is active + + -- *`redis.info.memory.allocator`*:: @@ -14683,6 +14728,87 @@ type: keyword Memory allocator. +-- + + +*`redis.info.memory.allocator_stats.allocated`*:: ++ +-- +type: long + +format: bytes + +Allocated memory + + +-- + +*`redis.info.memory.allocator_stats.active`*:: ++ +-- +type: long + +format: bytes + +Active memeory + + +-- + +*`redis.info.memory.allocator_stats.resident`*:: ++ +-- +type: long + +format: bytes + +Resident memory + + +-- + +*`redis.info.memory.allocator_stats.fragmentation.ratio`*:: ++ +-- +type: float + +Fragmentation ratio + + +-- + +*`redis.info.memory.allocator_stats.fragmentation.bytes`*:: ++ +-- +type: long + +format: bytes + +Fragmented bytes + + +-- + +*`redis.info.memory.allocator_stats.rss.ratio`*:: ++ +-- +type: float + +Resident ratio + + +-- + +*`redis.info.memory.allocator_stats.rss.bytes`*:: ++ +-- +type: long + +format: bytes + +Resident bytes + + -- [float] @@ -14697,14 +14823,16 @@ Redis CPU stats. -- type: boolean -None +Flag indicating if the load of a dump file is on-going + -- [float] == rdb fields -None +Provides information about RDB persistence + *`redis.info.persistence.rdb.last_save.changes_since`*:: @@ -14712,25 +14840,28 @@ None -- type: long -None +Number of changes since the last dump + -- -*`redis.info.persistence.rdb.bgsave.in_progress`*:: +*`redis.info.persistence.rdb.last_save.time`*:: + -- -type: boolean +type: long + +Epoch-based timestamp of last successful RDB save -None -- -*`redis.info.persistence.rdb.last_save.time`*:: +*`redis.info.persistence.rdb.bgsave.in_progress`*:: + -- -type: long +type: boolean + +Flag indicating a RDB save is on-going -None -- @@ -14739,7 +14870,8 @@ None -- type: keyword -None +Status of the last RDB save operation + -- @@ -14748,7 +14880,10 @@ None -- type: long -None +format: duration + +Duration of the last RDB save operation in seconds + -- @@ -14757,14 +14892,30 @@ None -- type: long -None +format: duration + +Duration of the on-going RDB save operation if any + + +-- + +*`redis.info.persistence.rdb.copy_on_write.last_size`*:: ++ +-- +type: long + +format: bytes + +The size in bytes of copy-on-write allocations during the last RBD save operation + -- [float] == aof fields -None +Provides information about AOF persitence + *`redis.info.persistence.aof.enabled`*:: @@ -14772,7 +14923,8 @@ None -- type: boolean -None +Flag indicating AOF logging is activated + -- @@ -14781,7 +14933,8 @@ None -- type: boolean -None +Flag indicating a AOF rewrite operation is on-going + -- @@ -14790,7 +14943,8 @@ None -- type: boolean -None +Flag indicating an AOF rewrite operation will be scheduled once the on-going RDB save is complete. + -- @@ -14799,7 +14953,10 @@ None -- type: long -None +format: duration + +Duration of the last AOF rewrite operation in seconds + -- @@ -14808,7 +14965,22 @@ None -- type: long -None +format: duration + +Duration of the on-going AOF rewrite operation if any + + +-- + +*`redis.info.persistence.aof.rewrite.buffer.size`*:: ++ +-- +type: long + +format: bytes + +Size of the AOF rewrite buffer + -- @@ -14817,7 +14989,8 @@ None -- type: keyword -None +Status of the last AOF rewrite operatio + -- @@ -14826,7 +14999,76 @@ None -- type: keyword -None +Status of the last write operation to the AOF + + +-- + +*`redis.info.persistence.aof.copy_on_write.last_size`*:: ++ +-- +type: long + +format: bytes + +The size in bytes of copy-on-write allocations during the last RBD save operation + + +-- + +*`redis.info.persistence.aof.buffer.size`*:: ++ +-- +type: long + +format: bytes + +Size of the AOF buffer + + +-- + +*`redis.info.persistence.aof.size.current`*:: ++ +-- +type: long + +format: bytes + +AOF current file size + + +-- + +*`redis.info.persistence.aof.size.base`*:: ++ +-- +type: long + +format: bytes + +AOF file size on latest startup or rewrite + + +-- + +*`redis.info.persistence.aof.fsync.pending`*:: ++ +-- +type: long + +Number of fsync pending jobs in background I/O queue + + +-- + +*`redis.info.persistence.aof.fsync.delayed`*:: ++ +-- +type: long + +Delayed fsync counter + -- @@ -14842,7 +15084,8 @@ Replication -- type: keyword -None +Role of the instance (can be "master", or "slave"). + -- @@ -14851,16 +15094,21 @@ None -- type: long -None +Number of connected slaves + -- *`redis.info.replication.master_offset`*:: + -- + +deprecated[6.5] + type: long -None +The server's current replication offset + -- @@ -14869,7 +15117,8 @@ None -- type: long -None +Flag indicating replication backlog is active + -- @@ -14878,7 +15127,10 @@ None -- type: long -None +format: bytes + +Total size in bytes of the replication backlog buffer + -- @@ -14887,7 +15139,8 @@ None -- type: long -None +The master offset of the replication backlog buffer + -- @@ -14896,7 +15149,114 @@ None -- type: long -None +Size in bytes of the data in the replication backlog buffer + + +-- + +*`redis.info.replication.master.offset`*:: ++ +-- +type: long + +The server's current replication offset + + +-- + +*`redis.info.replication.master.second_offset`*:: ++ +-- +type: long + +The offset up to which replication IDs are accepted + + +-- + +*`redis.info.replication.master.link_status`*:: ++ +-- +type: keyword + +Status of the link (up/down) + + +-- + +*`redis.info.replication.master.last_io_seconds_ago`*:: ++ +-- +type: long + +format: duration + +Number of seconds since the last interaction with master + + +-- + +*`redis.info.replication.master.sync.in_progress`*:: ++ +-- +type: boolean + +Indicate the master is syncing to the slave + + +-- + +*`redis.info.replication.master.sync.left_bytes`*:: ++ +-- +type: long + +format: bytes + +Number of bytes left before syncing is complete + + +-- + +*`redis.info.replication.master.sync.last_io_seconds_ago`*:: ++ +-- +type: long + +format: duration + +Number of seconds since last transfer I/O during a SYNC operation + + +-- + +*`redis.info.replication.slave.offset`*:: ++ +-- +type: long + +The replication offset of the slave instance + + +-- + +*`redis.info.replication.slave.priority`*:: ++ +-- +type: long + +The priority of the instance as a candidate for failover + + +-- + +*`redis.info.replication.slave.is_readonly`*:: ++ +-- +type: boolean + +Flag indicating if the slave is read-only + -- @@ -15108,7 +15468,8 @@ Total network output in bytes. -- type: long -None +Number of commands processed per second + -- @@ -15117,7 +15478,8 @@ None -- type: scaled_float -None +The network's read rate per second in KB/sec + -- @@ -15126,7 +15488,8 @@ None -- type: scaled_float -None +The network's write rate per second in KB/sec + -- @@ -15135,7 +15498,8 @@ None -- type: long -None +The number of full resyncs with slaves + -- @@ -15144,7 +15508,8 @@ None -- type: long -None +The number of accepted partial resync requests + -- @@ -15153,7 +15518,8 @@ None -- type: long -None +The number of denied partial resync requests + -- @@ -15162,7 +15528,8 @@ None -- type: long -None +Total number of key expiration events + -- @@ -15171,7 +15538,8 @@ None -- type: long -None +Number of evicted keys due to maxmemory limit + -- @@ -15180,7 +15548,8 @@ None -- type: long -None +Number of successful lookup of keys in the main dictionary + -- @@ -15189,7 +15558,8 @@ None -- type: long -None +Number of failed lookup of keys in the main dictionary + -- @@ -15198,7 +15568,8 @@ None -- type: long -None +Global number of pub/sub channels with client subscriptions + -- @@ -15207,7 +15578,8 @@ None -- type: long -None +Global number of pub/sub pattern with client subscriptions + -- @@ -15216,7 +15588,8 @@ None -- type: long -None +Duration of the latest fork operation in microseconds + -- @@ -15225,7 +15598,68 @@ None -- type: long -None +The number of sockets open for MIGRATE purposes + + +-- + +*`redis.info.stats.slave_expires_tracked_keys`*:: ++ +-- +type: long + +The number of keys tracked for expiry purposes (applicable only to writable slaves) + + +-- + +*`redis.info.stats.active_defrag.hits`*:: ++ +-- +type: long + +Number of value reallocations performed by active the defragmentation process + + +-- + +*`redis.info.stats.active_defrag.misses`*:: ++ +-- +type: long + +Number of aborted value reallocations started by the active defragmentation process + + +-- + +*`redis.info.stats.active_defrag.key_hits`*:: ++ +-- +type: long + +Number of keys that were actively defragmented + + +-- + +*`redis.info.stats.active_defrag.key_misses`*:: ++ +-- +type: long + +Number of keys that were skipped by the active defragmentation process + + +-- + +*`redis.info.slowlog.count`*:: ++ +-- +type: long + +Count of slow operations + -- diff --git a/metricbeat/module/redis/fields.go b/metricbeat/module/redis/fields.go index d831b638bc2..a2cdd21ba07 100644 --- a/metricbeat/module/redis/fields.go +++ b/metricbeat/module/redis/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzEms2O2zYQx+/7FEROKdAI6HUPBZqiRYOmySJpzgxFjWzWFClwht51nr4gZflrJUu2KVmXxdoy/78ZzYyGH+/YCjaPzEGh8IExUqThkb35Ev5/88BYASidqklZ88h+fWCMsfgdq4Ccksik1RokQcFKZ6vmy+yBMQcaBMIjW4gHxkoFusDH+Pt3zIgK9prhok0dbnXW19tPOoTD9T3+6juT1pBQBhktgSlTWleJcC8TpmBIghRSwDuGCtchyiFOGGT3YRfRGapIFgYYD+aAvDNQsHwTb/3+4dOfn8PPq0qYIjsY+tiT7XVqxqEpUiswhEff9Vk0YNX+gTeDRhMwO7mnC+YIyBoTg+TVHS2WtmbR8eUAWbg++SoHx2zZEm7FlDXI3sKL1L5QZnH0cYwK1GIN+NPell78AAdI3HqqPXGtkCYw5GOjwhoVFlSYqGxA985Fy14ZOAI+V4sIr0xgz305Afr7RoNFDZb7sgSXhF1buZolapDVYGKchFRthGPUCK3Z2/cfnz4//czef9n/+fj07etffcGzT0SPBC51IsZBD4vLpfkIRuT6jF9zazUIc51rP5hCSUEQSqCgWNxOwLEFGHJf7dO67venb00Bu9BfHqHIcHP6sz0RSqGh4KW2oqs0jPDa1w0SVJFQWoO+2r8aGnYEtwY3ImlaWC6XShcOuh5jKupvCC4Zs8dXqZKS9YyHcyFXIahMwWpnJSDCmPK0o76rqy+Ab8ErqKzbpM2tZszr+oPoyLXQHpKX+qZGPrJ8Q9CVv9GxxZZ+7DN32F8KekCHOMZFQAvKHI4O0BrE6s60AWEsrvbizrTaixGwlXi5LmQT0P6zBVWVopGotdVKnqb8nnUFm2frujqCQ5xOmD/WKjZ0rJFgZMODZM9LMDuPBtDw3ncg5LL3vX/ILLS2UpDtfyOMQz7vwZ3KQMGswaFCAiNPH3eqjuTSkqmtCO3qjS3c8HNwRd6r0WV1r8KwaUfmCSSOYg2ZXIowL+KoXjv/GKcn38bbfAiQL6K6Mrx2duGgs+KzEe6+Tn5vP6lqfrMbeRLkz5vdn4O3ywfDMwQ5u/HbKetMALuCZ/tn5hMmWv9UkE0W3A6enaL7JVerj+Fd5O9l/Z2CvJWfN8rZUZodueAOZWZe9b3na61kXLBJ10T0DTnUPzir+zvXC7vBXpHdui9vFltvmd2NaXORwHFblgg3LdAOS4U5t7aLTEhS65smreO1UP2YSalUDomH+cmszlwqJH1mDeVqsVaoWY5KlXtf42in+0ZsRO6tw2ziVcay5Om3UMRxKX6ZRahQjpLNL/tjxStdcNW/dJ1KqLLF9PXR9lfEVBLCySXP1asl7/RKldekag0vyiy4qNX0USclnyuVtiur5yIvSS103swR3SRrXls3cWH3dc/kNaHG8se042vnudRWXryEenGnVKoFL9VEXdnuFdix+3XjEtYtJwLiPrADCWp92zZvZ3P+ryWhmdlv9R4cAWhFRyxFHrP+d/NBhitYG9FRrPH8CPJ2K2gO0kZyt/s0htMAZfG0QNa3DJ4EEujZutX2XIIyzZr7SLzmHMYsfNsjHxcAKoMkDAkD1mNma+Q1ON49jU9Ypo5lmyMlq7xOsjF+sdHNaZz55HFjZFZ6raf1cZSphSMldGYnfu8ciYHr321JoraCDWbwUiuXqC4NKK1Vslp9VqkWErLluSY7rVSlEKdeTal9jj6P2y8G9DxitSACZyYW04IAiZfWrbifvGRWauEEAZdx55OjlStIHSinAfJwOuol51jbQc6dZc2tb052tTdPe6D11YzoXEM80LT+vSVmqgBDqlS9B5R2U+j1ghOdVv3exzWg/9sanFgAI9IDusG3iUT3xx3DoKHROHx4AxxNwb4W5f8AAAD//+K17E4=" + return "eJzkXEuT2zbyv/tTdPl/yKTKQ9f/sHuYw1Z54jjrihO7ZuzDnmgQbEqIQIABQI2VT7+FBymJIkjqQXlSq4vLGgr960Y/gW7ewgo3d6AwZ/oFgGGG4x28fLD/f/kCIEdNFasMk+IO/vUCAMD9DUo0ilENVHKO1GAOhZKl/2PyAkAhR6LxDhbkBUDBkOf6zv3+FgQpcUvTfsymso8qWVfhmx7C9vPV/eorUCkMYUKDWSIwUUhVEvssEJGDNsQwbSy8fVD2swtlF45dpP2yD9EAKofMLjAdmEJTK4E5ZBv36Nf3v7/7aH9elkTkyc7S+5JsPl02dlmhnKEweu9vMY5GuNpuuF/UsaCTzjN9YPYASSGckhw80cDiUix6/jiCzH5+r8sMFciiQRiIMSk03OA3yuucicXe104rNCdr1D92edmitphQm1TWpqpNypk2M+D/4KmApwKWCpBSWsS1Uo6hA77imDO2cJiZsJCzupgB8b2nAY4GZHVRoDoHMpd0dRXV0FChcMpg7dETdqpBOIeb+w+fPn56BfcP238+fPry+O8dDYlYW60Nqktbm1t014Mca3QoSMYH5JpJyZGI00T7XuSMEoPWzxHjPFgHuG4AjImvqi8rup8+ffFe6kh51RrzRG+6P9si0pRwzNOCS9LnCCZI7XGjDZYOIZVC1+XW/3vsGtUaVdxWGowpXTKeK+zbvUuB/aJRnQu11geGcSV5ZoSurAqJHColKWqNAz6oBftdBTuMudd6Siyl2lzWgPyap0V6J8g14TVe3J97R3gH2cZgn5ECfJaGcBCt13dPAuFcWlflxLyXCkbgKx33ARHwY9iOClYetvOrfkO2HBBrdygahZEVKmJsCNPeEm5IskoIKNQsd9kaGtDsLxzIchzLFZLVd+D5E5JVo267xjBll3hNvgPiLxrzBnHYhA81ARQLJjBpnhpGnhNDNB6dS14A/eclOnUAJoKWycLxECBBv7PZ5aAk306z7wvA/83LnbOSmbh2WISV5Ix23eIW4go3T1L1pUYTUPy8Zi6lBU8EjLTbCk9LFI1mOIQ2BVJI6DKaAu2iLhRZlCiMT/WsVcso/DNiz4NdGDI0T9aNWHVMPeZUae1q1J3vJoKN7ejc6nDvNPhYXqJMEWrYGtMcLXcJ06mqhWC94C+QQL/jZAHMZ9HWgbMiAAAPoBWvVSP/lwks+EAh4znXWZof7K+lMkGt22fTvpQcBpMVGMg3emj0ljowpocwKbOYJB77edOG6ogJwYHGPQfUXvNKLHEMc5NaPAPUD02WM0HU09wrjLrYI+C927PiOMnjnCpcWcgNFy4xi63WKofW15Ftu/XjYrWYnoswW9j7a/VWdhUqzbRBQbse4lLnI8fWdlyS/Jrx0CamlqZNUgnkdVlBwTjagCjF7UL2Y/k/+CzfSijlGuFrgPzVJmnNf5JwLvXVpQgkz0GaJaqGPS8byNxW+SrsRhuiDBhW4iswrsh0G/jK/aaxjFeQJMmP4yFR5dnRYXBKLaXkmuWo9y8dMlkbeHh7P6BOMDHMcqJNqskaE7okYoE61ax/NZhiVhNNZucM11MFR9UrB9HG6cVE3HYDZ4b7cyXp8jYjtky05LQhZWXRO6y6phS1Lmru9sSCiuvLLg/ZwjHARFopuVDYe0IBEyzxCFa6FklazCMWeADb74Ahph6GHU9Oj4D96Og0ha0Te4s7nJfIfsn0obZ7mGik5waPvB4gPJm3t2GVEe5sea+RSpEPh+vAaLi0eea8NgrXy28BRPQngIMCoLLapFKkT4qZRk3ZX88hI+89qbFwb6W4dXCbssddaOa1sqLZ6sT9266MYnKJl24yfnk4T5x68/Gdj1PnhKn4tRfM6x8tei4XC5e9hJp9ryodqau8En5nF2+ZCFB27Wuiv2+Y0HSJef1ddoGICA9PjHPIEFpsIJs84tC1MA1UlhVHg4cHjX0c/12CRWR/p8WLhtm/WcCI8ByPGV1+fYtD8kxCw6MNC4HHXdY8yqn55J7ifsfcrG9zpvHwPPB31crIZmOmcfE/lIJMVM1nbWwdI9v9DHJl2Wn85jNgy7LS9Gq5MxW30cdxY4vcZ8LKlgUpgBOD2jUqKlNXIFXjX6bpX6E3giahUexqhxuOatue9ofMtDO7bUPI+9cf4c8ae+9du+Bz5GRz+nXI1HjrqQToVNbCxMLPNqBWnNG+WH/GmWZsybHjTCV5/BL7rHuyB8lbp8GENsQmmjeUCJt/viyJNqhevrKa+dI1ob6MtRhCXw9t6htXo9Av0i/ZEIMOsSg8z1Uqi+KE1oYcK4Xu2uwO/pn84zT0Ll65prQfdOvcdjQOItiiLFnr43KRRK/pzpV3t3bZBRuI99z+xmBGIubsPSXuPLy3q6SPn0gA7TJTMKVNalc7WaUmKYxX26Aa47i3Px7lYMm04QN9hKfjfox18NgvJ8p9xI6TWYV+ipV28PlydV7dCEpRVzapf1oyutwD+v6tBqIQCKVY9XUfdCBzJlbxSuUCYadTnTCxgpu6ep3LJ/HjKDhbeDCZhmOAlCzi7UcjvmSgmj8qBgUo3fseZtMMQsOZjlkGDkYVxiZGw4drF7jFbPviHd7gXJgGS91VR746dEF1EmKOhXeDz6El1YKBDAupsOVo56RsGkPPXdGckhlFhLb+3qbdobIl8Pif338auEOK8u+2e16XeuhBGz/giLdp6AjGSjGpmIl3Tp6Hsln+IDkmGghQInKWW+MppIKCMC7XA4btETOdKiS5FDwOeo7GhCBW1+KZ3+6R7y19fMy7VNXz6FbrzizChKpnjUr328wxEWg8j1gwk+ol+f+rEMqZGtDZS1HKasbzlMUnqi5FqJT5uZXpOBF5bhYyToIoukwzNtD2eTGJ1dywiuM3JhYpqdj8Wkdpei1TCrNAQ5o3wS2P01G1uIZ2G1qllVRnRcJxKnUV6fa5II3lX/Ouz1WdUi7p0aM5x5GhUhRskRbs7POwkRDY0wF+Zi/jOdPobipZIUW2Pm/6uD/d6cyi7Y6fN0TjMyz7EP84e3b+BIie6KSDSffKAp02M4vXQOpJtmOSU3AKNIkbWT9tYGUqSDRPUq3CcHxzShPfaYvKj/5fBVZ4y8AhrihAn6MbIlDWOpGVTitUaf+t/yUPoLs7DBWqUKpNxOrfgbDKqjlHum1ZE4T7gy8GQNkSZovWCvvX+9d9EovI2L9v4qrA/X3uOPJ4NWaL+6LmfKbScWv8lggotAS1PwGKXIjsQauIMozwRJ4VTCcBbA4EIdAMYEHhnzXqnmy4Fyiq+BTXpZDmKNgEnFHAK9zoBL9VTM3y/o6u21/hBhw1f86B65633HTBrdns750JNBxByGsEI6Ek33ZnQQ9WGERdEYrJcqhuugTsnWZ0LuWqroKIdXONUBImIPfDrmRgGrSFXDKtZ76YLAjjmB8JOF5Z1ZmuMzfPIJDPgfwXLrM9Ha7q7LWuM2hoeg/WvFypztoV44odUFfEGFQ9z82IOtCcADpe1Lj2jLSQapXW86QRh52PriOkcAnQbtNjyaiS3c7H+CEDW9gAmVI3051qSVc4i43uO+lAxyIX7mT0t/e/PLz5/DNUtaqknnI57wJk6h21To0idIV5ak1ndvTOPgNFh96h2LTg4YZU7ug64whScDdPb7MR90V4R9c4h/vD27P7TvcGBJvt7TTAVagKqcIbLMIst7uZ7cxzh4x2IitX8Kkkk8rGrz6mXBfV9g01kRH141ha4SadfYe83i2JgSdUDXC+2YE+cGN7iPcK29BBrFesqk6S/Nbq5ROXi8S1ZvWeu/TAHoH8k13LeSUun7bOtMcHdbODF13ix7xbsVlk6P2KboTEfts8PO9LFg9OSocOykak+mtADG46mBUMY+9WaDV0vUiN6RZbp+7qmzUqskAwho/Q7YkZpxLtaH7I4dpkchhHiGUnQvlvAAAA//+DAhOI" } diff --git a/metricbeat/module/redis/info/_meta/fields.yml b/metricbeat/module/redis/info/_meta/fields.yml index 46775c91a63..aa31e962447 100644 --- a/metricbeat/module/redis/info/_meta/fields.yml +++ b/metricbeat/module/redis/info/_meta/fields.yml @@ -13,17 +13,14 @@ type: long description: > Number of client connections (excluding connections from slaves). - - name: longest_output_list type: long description: > Longest output list among current client connections. - - name: biggest_input_buf type: long description: > Biggest input buffer among current client connections. - - name: blocked type: long description: > @@ -48,17 +45,14 @@ type: scaled_float description: > System CPU consumed by the Redis server. - - name: used.sys_children type: scaled_float description: > User CPU consumed by the Redis server. - - name: used.user type: scaled_float description: > System CPU consumed by the background processes. - - name: used.user_children type: scaled_float description: > @@ -73,42 +67,95 @@ type: long description: > format: bytes - Used memory. - + Total number of bytes allocated by Redis. - name: used.rss type: long format: bytes description: > - Used memory rss. - + Number of bytes that Redis allocated as seen by the operating system (a.k.a resident set size). - name: used.peak type: long format: bytes description: > - Used memory peak. - + Peak memory consumed by Redis. - name: used.lua type: long format: bytes description: > - Used memory lua. + Used memory by the Lua engine. + - name: used.dataset + type: long + format: bytes + description: > + The size in bytes of the dataset - name: max.value type: long format: bytes description: > Memory limit. - - name: max.policy type: keyword - description: + description: > Eviction policy to use when memory limit is reached. + - name: fragmentation.ratio + type: float + description: > + Ratio between used_memory_rss and used_memory + - name: fragmentation.bytes + type: long + format: bytes + description: > + Bytes between used_memory_rss and used_memory + + - name: active_defrag.is_running + type: boolean + description: > + Flag indicating if active defragmentation is active + - name: allocator type: keyword description: > Memory allocator. + - name: allocator_stats + type: group + fields: + - name: allocated + type: long + format: bytes + description: > + Allocated memory + - name: active + type: long + format: bytes + description: > + Active memeory + - name: resident + type: long + format: bytes + description: > + Resident memory + - name: fragmentation.ratio + type: float + description: > + Fragmentation ratio + - name: fragmentation.bytes + type: long + format: bytes + description: > + Fragmented bytes + - name: rss.ratio + type: float + description: > + Resident ratio + - name: rss.bytes + type: long + format: bytes + description: > + Resident bytes + - name: persistence type: group description: > @@ -116,67 +163,124 @@ fields: - name: loading type: boolean - description: + description: > + Flag indicating if the load of a dump file is on-going + # ToDo move `loading` to `loading.enabled` and add other loading stats beside that (start time, total bytes, loaded bytes, ...) - name: rdb type: group - description: + description: > + Provides information about RDB persistence fields: - name: last_save.changes_since type: long - description: - - - name: bgsave.in_progress - type: boolean - description: - + description: > + Number of changes since the last dump - name: last_save.time type: long - description: + description: > + Epoch-based timestamp of last successful RDB save + - name: bgsave.in_progress + type: boolean + description: > + Flag indicating a RDB save is on-going - name: bgsave.last_status type: keyword - description: - + description: > + Status of the last RDB save operation - name: bgsave.last_time.sec type: long - description: - + format: duration + description: > + Duration of the last RDB save operation in seconds - name: bgsave.current_time.sec type: long - description: + format: duration + description: > + Duration of the on-going RDB save operation if any + + - name: copy_on_write.last_size + type: long + format: bytes + description: > + The size in bytes of copy-on-write allocations during the last RBD save operation - name: aof type: group - description: + description: > + Provides information about AOF persitence fields: - name: enabled type: boolean - description: + description: > + Flag indicating AOF logging is activated - name: rewrite.in_progress type: boolean - description: - + description: > + Flag indicating a AOF rewrite operation is on-going - name: rewrite.scheduled type: boolean - description: - + description: > + Flag indicating an AOF rewrite operation will be scheduled once the on-going RDB save is complete. - name: rewrite.last_time.sec type: long - description: - + format: duration + description: > + Duration of the last AOF rewrite operation in seconds - name: rewrite.current_time.sec type: long - description: + format: duration + description: > + Duration of the on-going AOF rewrite operation if any + - name: rewrite.buffer.size + type: long + format: bytes + description: > + Size of the AOF rewrite buffer - name: bgrewrite.last_status type: keyword - description: + description: > + Status of the last AOF rewrite operatio - name: write.last_status type: keyword - description: + description: > + Status of the last write operation to the AOF + + - name: copy_on_write.last_size + type: long + format: bytes + description: > + The size in bytes of copy-on-write allocations during the last RBD save operation + + - name: buffer.size + type: long + format: bytes + description: > + Size of the AOF buffer + + - name: size.current + type: long + format: bytes + description: > + AOF current file size + - name: size.base + type: long + format: bytes + description: > + AOF file size on latest startup or rewrite + + - name: fsync.pending + type: long + description: > + Number of fsync pending jobs in background I/O queue + - name: fsync.delayed + type: long + description: > + Delayed fsync counter - name: replication type: group @@ -185,31 +289,82 @@ fields: - name: role type: keyword - description: + description: > + Role of the instance (can be "master", or "slave"). - name: connected_slaves type: long - description: + description: > + Number of connected slaves - name: master_offset type: long - description: - + deprecated: 6.5 + description: > + The server's current replication offset + - name: backlog.active type: long - description: - + description: > + Flag indicating replication backlog is active - name: backlog.size type: long - description: - + format: bytes + description: > + Total size in bytes of the replication backlog buffer - name: backlog.first_byte_offset type: long - description: - + description: > + The master offset of the replication backlog buffer - name: backlog.histlen type: long - description: + description: > + Size in bytes of the data in the replication backlog buffer + + - name: master.offset + type: long + description: > + The server's current replication offset + - name: master.second_offset + type: long + description: > + The offset up to which replication IDs are accepted + - name: master.link_status + type: keyword + description: > + Status of the link (up/down) + - name: master.last_io_seconds_ago + type: long + format: duration + description: > + Number of seconds since the last interaction with master + - name: master.sync.in_progress + type: boolean + description: > + Indicate the master is syncing to the slave + - name: master.sync.left_bytes + type: long + format: bytes + description: > + Number of bytes left before syncing is complete + - name: master.sync.last_io_seconds_ago + type: long + format: duration + description: > + Number of seconds since last transfer I/O during a SYNC operation + + - name: slave.offset + type: long + description: > + The replication offset of the slave instance + - name: slave.priority + type: long + description: > + The priority of the instance as a candidate for failover + - name: slave.is_readonly + type: boolean + description: > + Flag indicating if the slave is read-only - name: server type: group @@ -289,7 +444,6 @@ type: long description: Total number of connections received. - - name: connections.rejected type: long description: @@ -304,7 +458,6 @@ type: long description: Total network input in bytes. - - name: net.output.bytes type: long description: @@ -312,57 +465,90 @@ - name: instantaneous.ops_per_sec type: long - description: - + description: > + Number of commands processed per second - name: instantaneous.input_kbps type: scaled_float - description: - + description: > + The network's read rate per second in KB/sec - name: instantaneous.output_kbps type: scaled_float - description: + description: > + The network's write rate per second in KB/sec - name: sync.full type: long - description: - + description: > + The number of full resyncs with slaves - name: sync.partial.ok type: long - description: - + description: > + The number of accepted partial resync requests - name: sync.partial.err type: long - description: + description: > + The number of denied partial resync requests - name: keys.expired type: long - description: - + description: > + Total number of key expiration events - name: keys.evicted type: long - description: - + description: > + Number of evicted keys due to maxmemory limit + - name: keyspace.hits type: long - description: - + description: > + Number of successful lookup of keys in the main dictionary - name: keyspace.misses type: long - description: + description: > + Number of failed lookup of keys in the main dictionary - name: pubsub.channels type: long - description: - + description: > + Global number of pub/sub channels with client subscriptions - name: pubsub.patterns type: long - description: + description: > + Global number of pub/sub pattern with client subscriptions - name: latest_fork_usec type: long - description: + description: > + Duration of the latest fork operation in microseconds - name: migrate_cached_sockets type: long - description: + description: > + The number of sockets open for MIGRATE purposes + + - name: slave_expires_tracked_keys + type: long + description: > + The number of keys tracked for expiry purposes (applicable only to writable slaves) + + - name: active_defrag.hits + type: long + description: > + Number of value reallocations performed by active the defragmentation process + - name: active_defrag.misses + type: long + description: > + Number of aborted value reallocations started by the active defragmentation process + - name: active_defrag.key_hits + type: long + description: > + Number of keys that were actively defragmented + - name: active_defrag.key_misses + type: long + description: > + Number of keys that were skipped by the active defragmentation process + - name: slowlog.count + type: long + description: > + Count of slow operations diff --git a/metricbeat/module/redis/info/data.go b/metricbeat/module/redis/info/data.go index 64d055a82df..7e98a40e7b9 100644 --- a/metricbeat/module/redis/info/data.go +++ b/metricbeat/module/redis/info/data.go @@ -44,16 +44,37 @@ var ( }, "memory": s.Object{ "used": s.Object{ - "value": c.Int("used_memory"), // As it is a top key, this goes into value - "rss": c.Int("used_memory_rss"), - "peak": c.Int("used_memory_peak"), - "lua": c.Int("used_memory_lua"), + "value": c.Int("used_memory"), // As it is a top key, this goes into value + "rss": c.Int("used_memory_rss"), + "peak": c.Int("used_memory_peak"), + "lua": c.Int("used_memory_lua"), + "dataset": c.Int("used_memory_dataset"), }, "max": s.Object{ "value": c.Int("maxmemory"), "policy": c.Str("maxmemory_policy"), }, + "fragmentation": s.Object{ + "ratio": c.Float("mem_fragmentation_ratio"), + "bytes": c.Int("mem_fragmentation_bytes"), + }, + "active_defrag": s.Object{ + "is_running": c.Bool("active_defrag_running"), + }, "allocator": c.Str("mem_allocator"), // Could be moved to server as it rarely changes + "allocator_stats": s.Object{ + "allocated": c.Int("allocator_allocated"), + "active": c.Int("allocator_active"), + "resident": c.Int("allocator_resident"), + "fragmentation": s.Object{ + "ratio": c.Float("allocator_frag_ratio"), + "bytes": c.Int("allocator_frag_bytes"), + }, + "rss": s.Object{ + "ratio": c.Float("allocator_rss_ratio"), + "bytes": c.Int("allocator_rss_bytes"), + }, + }, }, "persistence": s.Object{ "loading": c.Bool("loading"), @@ -72,6 +93,9 @@ var ( "sec": c.Int("rdb_current_bgsave_time_sec"), }, }, + "copy_on_write": s.Object{ + "last_size": c.Int("rdb_last_cow_size"), + }, }, "aof": s.Object{ "enabled": c.Bool("aof_enabled"), @@ -84,6 +108,9 @@ var ( "current_time": s.Object{ "sec": c.Int("aof_current_rewrite_time_sec"), }, + "buffer": s.Object{ + "size": c.Int("aof_rewrite_buffer_length"), + }, }, "bgrewrite": s.Object{ "last_status": c.Str("aof_last_bgrewrite_status"), @@ -91,18 +118,49 @@ var ( "write": s.Object{ "last_status": c.Str("aof_last_write_status"), }, + "copy_on_write": s.Object{ + "last_size": c.Int("aof_last_cow_size"), + }, + "buffer": s.Object{ + "size": c.Int("aof_buffer_length"), + }, + "size": s.Object{ + "current": c.Int("aof_current_size"), + "base": c.Int("aof_base_size"), + }, + "fsync": s.Object{ + "pending": c.Int("aof_pending_bio_fsync"), + "delayed": c.Int("aof_delayed_fsync"), + }, }, }, "replication": s.Object{ "role": c.Str("role"), "connected_slaves": c.Int("connected_slaves"), - "master_offset": c.Int("master_repl_offset"), + "master_offset": c.Int("master_repl_offset"), // ToDo Should be deprectad as master.offset introduced "backlog": s.Object{ "active": c.Int("repl_backlog_active"), "size": c.Int("repl_backlog_size"), "first_byte_offset": c.Int("repl_backlog_first_byte_offset"), "histlen": c.Int("repl_backlog_histlen"), }, + "master": s.Object{ + "offset": c.Int("master_repl_offset"), + "second_offset": c.Int("second_repl_offset"), + "link_status": c.Str("master_link_status", s.Optional), + "last_io_seconds_ago": c.Int("master_last_io_seconds_ago", s.Optional), + "sync": s.Object{ + "in_progress": c.Bool("master_sync_in_progress", s.Optional), + "left_bytes": c.Int("master_sync_left_bytes", s.Optional), + "last_io_seconds_ago": c.Int("master_sync_last_io_seconds_ago", s.Optional), + }, + }, + "slave": s.Object{ + "offset": c.Int("slave_repl_offset", s.Optional), + "priority": c.Int("slave_priority", s.Optional), + "is_readonly": c.Bool("slave_read_only", s.Optional), + }, + // ToDo find a way to add dynamic object of slaves: "slaves": s.Str("slaveXXX") }, "server": s.Object{ "version": c.Str("redis_version"), @@ -160,8 +218,18 @@ var ( "channels": c.Int("pubsub_channels"), "patterns": c.Int("pubsub_patterns"), }, - "latest_fork_usec": c.Int("latest_fork_usec"), - "migrate_cached_sockets": c.Int("migrate_cached_sockets"), + "latest_fork_usec": c.Int("latest_fork_usec"), + "migrate_cached_sockets": c.Int("migrate_cached_sockets"), + "slave_expires_tracked_keys": c.Int("slave_expires_tracked_keys"), + "active_defrag": s.Object{ + "hits": c.Int("active_defrag_hits"), + "misses": c.Int("active_defrag_misses"), + "key_hits": c.Int("active_defrag_key_hits"), + "key_misses": c.Int("active_defrag_key_misses"), + }, + }, + "slowlog": s.Object{ + "count": c.Int("slowlog_len"), }, } ) diff --git a/metricbeat/module/redis/info/info.go b/metricbeat/module/redis/info/info.go index d48ec511943..1d78baad1d2 100644 --- a/metricbeat/module/redis/info/info.go +++ b/metricbeat/module/redis/info/info.go @@ -18,6 +18,7 @@ package info import ( + "strconv" "time" "github.com/elastic/beats/libbeat/common" @@ -79,6 +80,12 @@ func (m *MetricSet) Fetch() (common.MapStr, error) { return nil, err } + slowLogLength, err := redis.FetchSlowLogLength(m.pool.Get()) + if err != nil { + return nil, err + } + info["slowlog_len"] = strconv.FormatInt(slowLogLength, 10) + debugf("Redis INFO from %s: %+v", m.Host(), info) return eventMapping(info), nil } diff --git a/metricbeat/module/redis/info/info_integration_test.go b/metricbeat/module/redis/info/info_integration_test.go index 5378057c25b..bbcfd1b8601 100644 --- a/metricbeat/module/redis/info/info_integration_test.go +++ b/metricbeat/module/redis/info/info_integration_test.go @@ -49,7 +49,7 @@ func TestFetch(t *testing.T) { t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) // Check fields - assert.Equal(t, 8, len(event)) + assert.Equal(t, 9, len(event)) server := event["server"].(common.MapStr) assert.Equal(t, "standalone", server["mode"]) } diff --git a/metricbeat/module/redis/redis.go b/metricbeat/module/redis/redis.go index 061eeee5bf9..0b1e1a5df0a 100644 --- a/metricbeat/module/redis/redis.go +++ b/metricbeat/module/redis/redis.go @@ -65,6 +65,17 @@ func FetchRedisInfo(stat string, c rd.Conn) (map[string]string, error) { return ParseRedisInfo(out), nil } +// FetchSlowLogLength returns count of slow operations +func FetchSlowLogLength(c rd.Conn) (int64, error) { + count, err := rd.Int64(c.Do("SLOWLOG", "len")) + if err != nil { + logp.Err("Error retrieving slowlog len: %v", err) + return 0, err + } + + return count, nil +} + // CreatePool creates a redis connection pool func CreatePool( host, password, network string, diff --git a/metricbeat/tests/system/test_redis.py b/metricbeat/tests/system/test_redis.py index 73cc548a3a5..0f2219feecb 100644 --- a/metricbeat/tests/system/test_redis.py +++ b/metricbeat/tests/system/test_redis.py @@ -7,7 +7,7 @@ REDIS_FIELDS = metricbeat.COMMON_FIELDS + ["redis"] REDIS_INFO_FIELDS = ["clients", "cluster", "cpu", "memory", - "persistence", "replication", "server", "stats"] + "persistence", "replication", "server", "stats", "slowlog"] REDIS_KEYSPACE_FIELDS = ["keys", "expires", "id", "avg_ttl"]