From 499717f6c82d00f0c6cdde1b570d53cecc73c342 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 7 Apr 2020 19:52:05 -0600 Subject: [PATCH 1/5] add more metric collection into pubsub metricset --- .../metricbeat/module/googlecloud/fields.go | 2 +- .../module/googlecloud/pubsub/_meta/data.json | 36 +++++- .../pubsub/_meta/data_subscription.json | 53 --------- .../googlecloud/pubsub/_meta/data_topic.json | 61 ----------- .../googlecloud/pubsub/_meta/fields.yml | 103 ++++++++++++++++++ .../module/googlecloud/pubsub/manifest.yml | 33 ++++++ .../pubsub/pubsub_integration_test.go | 3 - 7 files changed, 169 insertions(+), 122 deletions(-) delete mode 100644 x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_subscription.json delete mode 100644 x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_topic.json diff --git a/x-pack/metricbeat/module/googlecloud/fields.go b/x-pack/metricbeat/module/googlecloud/fields.go index 3f8bc6b15c1..94dfe71b097 100644 --- a/x-pack/metricbeat/module/googlecloud/fields.go +++ b/x-pack/metricbeat/module/googlecloud/fields.go @@ -19,5 +19,5 @@ func init() { // AssetGooglecloud returns asset data. // This is the base64 encoded gzipped contents of module/googlecloud. func AssetGooglecloud() string { - return "eJzsXE1z2zgSvedX9C3JlqMc9ubamipHszuTWmfHtXZyVYFgS8QaBLj4sEb59VMASIqURBKUKE08pZstkujXDeC9RgPkB3jGzS2spFxxpFza9A2AYYbjLbz9xf8Kc/czPHBillLlb98AKORINN5Cgoa8AUhRU8UKw6S4hZ/eAAD8Mn+AXKaW4xuAJUOe6lt/4QMIkmPL5IyTBLn2lwHMpsBbkMn/kJryp+bzzTasRjX7W/3zwWe3d+doSEoMGfWEYlTHP6A32mAefz+VeWENNm7fDW27mZWStmj82gp8q7vmoeXKhcYzu8Fs4mFCGyIoti52Ge9qrNngkilcE873buhrtK/hZuOpkkWB6SLZGNQLKq0wB++vbHEpVh03tAL5WVCZM7EC33BlBpINmAz7XNqHVhD6jOYs4Mqmo+HVY66wZ+kNhRrVC6YLKhXqXl9TaRO+O8gOevsfmyeoQC7Bt1obASm8t5nUxl11f3cM3jZKaxhn34lrfSKITy7oilD3XwWFcC4pMZjC/OErmIwYYBqoVQqF4RtgwnFX5UQccE1WuDAs7wI1FvdX1yAspXJoy/AyARqpFOl+/9Ujm+nnM40fMv1UnruWXK+EqexswFLJvMuNHTiyOAsYZzxg+fwbyAKVH4+H50yFZ62YwXPHxxkxKMDI4QAFQOeMkLcwEKJ6ehQdU6MXQMv4r3Lt7/Nz8tsXyIiGBFGAskIwsbqJmR4CzVqqc80QiuzlDIK3N0uCnTBTXDS6vTqAbnrNq/FVmnccQo3CnDt2zgbIF1TjUF0gZv3IKjRckjQhnAjKWnamykvvJUnhU2VgZHqaGVPsTr3uaXUCiC4gTTCJi6tIF5wYFJQdTHlaEO4cqxnFEttMFcLjG6CEU8t9zuAH9TrDkB0o/L9FbWBNyj4sE71Cyd83jqndPyUYsMIw3ri8P1GqO4Ntbfy4dWAU6kIKjbOzcNcJA3u7GiP9OdtgC4l1UyFMsv4MNbYpWXQr91DYYCB0TXO/9dqJsxVjr2nzn78XUqAwjPBP3ts+8/EQYmE0oayUXJtssSTUSDX4VFQuvG9EUxJxdwQP7zctbL5YMuEzuKhQDtjZ5aCSIwbELT4VcmsaUS+9GspGdMVHOlDKr09PDx8fvWxA0A0nMLJCpvfZpAv71KhrnOWKMdnUoNzlQ8BjwAaSvFCkg7Ey1DX6d1IBJTTD9y7S4xxZKimM88TQYqHMIfARovXfpydHx9oqJy1SARKaAZVCYFgHJ2jWLnWmnHlnRBoE6SouV3G5istg4z+EuDBhUAkyZe12TO7818mehwdv/5CNobmIkRNBdSNaiaG78c31U14cCwzP/3jii+edONY5mv7GEGA8BR5PgiNpcDwRjqLCE8lwwNa2whWTZQ8CPzLTDqmUjk/59oGfA/KBNHs8vKikesrAthLrrjVMGfE+6EYawi8sY7UqtaRsV7tW0sDd/N/NsQNSBNGq/PdhuSrWSc1dFetkIPFg4KpYEbbG6dU5qkKjtSpWqSYsBo0F9ucVfkbrU7w6vUJtupaLOpq6lotGQ4iFAddy0UE79a7x32cd1aILb89WgHClUOtesu4JXRxRe/b6fP9pWxSqKBneLaWCp/kDLLlca2DmrQ6ssz1VKAWQouCM+oMtoI1CkoMUfPN+l/V2nOo/K3CcW61DAj2OlZrg3OpCycSlYl9iMrIF9jyxr7y6TPAPetYFTRnTo/EDCt/ayfGnQ1zotjs52m/vOBQ+nPuC3Ee0w4XgI9f6gzIc9fSwBI9ppk9+p1lNDwnvtEvNkZJ7nlXnOLEdIbXjhDZaZk8Q2QGJNbRYaM0XPoP+c0W2MX251P7Ed00XJ62Jmqe9twTkj0+vUSEYVDkTfglSUdXHx8f7rq3lEcnAcSh3Benbl4ZUWu1P6fdDm2Y7XudSmgxTz+bvmIBcv9+yenMp9lZ7fteG0OebsGufM2ENtgSRkw2q0ruC6HIZWm+ZBQeva7Pr2uy6Nhts/AfZyr8UBW4z129f4ihQ4PpiAkIVjlYPWaCYDOA8vALUSP6lNdoQkbpItUEraVeZp+4OpBXCwibaJg3TUx3WfrDJo01GHtLWNqnbOy5PebT1P84V32PH5SeEPi9y1P7VqdNH/dzmlhPDXjDIlOu+snXtTAm55piuguTebf+vq883wfdwQ4qcvaDaeAD9Jw+5XIWpe1rFWRrCw4EVzb5jlTxY0YJeO/SOzJ5nZFYBqC+8ByaAtLq5Z3LbfNEY4VVnTDXLa7BlMNHXKtrg3moorM4ARVpIJswNJNaAkAY2aFrd1u+GFbWRqd04axdInqI2C29jC31BVqe9LXW3Qp9qli9Eva+GUzDX4VKXR6McKizni+bMrl8PO9Mcb3hSO7J9Ja01qRVqy80M/iUVEEjRi3mZqR96VGODaz9uybYVio+5TL3DKZKUM4FdnkfEbLoNtqFIHSS9Mj79OC/Vr87Yq+7Q83amD8+Rvaizs4PTGRBjMC8Og4OvgrNn9F7om7ACds/4DWAFLC845ihMWPGmErXXhIQYmvkPS9RsO4NHGZbK1U6nFHzjkjVDmNAgBbYemPnybtOYckMg1J1RKancOHEytWIvKFrPAiW+No1EQW65YQVHMCzHnl3eVrT7NnvjI/7zTpHBO1N5X5sIpQZGlaxEYNQo8S8+Xig7q47+tqemkdW0dEJcbqcfk6SFHQSX4FxcnB4r0w9uuv4FpOpALM9LJbXBNuPBmpkMhBQfHMdsWqFl6ThC3PHpUmNjx7NLDYh/UJniT0cNi9gI1sdxLjQkyvM4g52+PXlTMHrc6vfJPXriuveVRAvaapBeMPNLONPZq6XIad+b7ItQVP73obPi5L8u5ncbUuWE9IRiVPwXrbazQKr2MneqwthjaHlkZYwULJoS+o+39Pd97CmEn5Ebsu3yu4fPQInPlht97RJndyVHk8nU2/evc9bn9BzZHyZBYk32/USPm7U8vkiIxnQRPqa2IJSinqKavROHOgHwdeMw1n19QpSfcYMEHdmtFBGOogMO0JIj30Bq0eWV5Z138/uezN25tOWPCTwJ33RyfTm/bzATJEo+o1vlrIXv095ktgyvLpCyJaMLhzK3Zio+3ok2zYhwGXpO0mbgKusHItj/qZ3jB1jk53WO9nTn4zq7n2I5ddbBuC/dnOjGwa/JTEUc+8R9Ws+WI3qCQ9u+hN7YvPENayhQlTu9rRCUfgDlROsZPGVMwwvhFoHpxqkrQdG3kJLNjQ+RC119n8IibFoRE6oQmuQFdwTkj1y+EF7tFUlr/KMp6dnOCmfCXScuur5l9TrG+NaRKbZFqh0Rwnndqyycb//BOvaPAAAA//+LJduy" + return "eJzsXc1y2zgSvucp+pZky3EOe0ttTZXHszuT2mTHtfbMVQWCLQlrEODgxxrN028BIClS4g9IkYqd8s0WSfTXDaC/7gZAfoBH3H+CjZQbjpRLm74BMMxw/ARvf/a/wq37Ge44MWupsrdvABRyJBo/QYKGvAFIUVPFcsOk+AQ/vAEA+Pn2DjKZWo5vANYMeao/+QsfQJAMGyKvOUmQa38ZwOxz/AQy+R9SU/xUf77ehtWorv9W/dz67OHuDA1JiSGjnlCM6vgH9F4bzOLvpzLLrcHa7cembTazUdLmtV8bhm90121ouVSh9syxMet4mNCGCIqNi13CuxqrN7hmCneE85Mb+hrta7jeeKpknmO6SvYG9YpKK0zr/aUsLsWm44aGIT8LKjMmNuAbLsVAsgezxT6VTqHlhD6iWQRc0XQ0vGrM5XaR3lCoUT1huqJSoe7VNZU24ceDrFXb/9gsQQVyDb7VSghI4bXdSm3cVfd3x+BtorSGcfYXca3PBPHBGV0R6v4roRDOJSUGU7i9+w3MlhhgGqhVCoXhe2DC+a5SiTjgmmxwZVjWBWos7t9cg7CWyqEtzMsEaKRSpKf9V41sph8XGj9k/ql861pyvRKmspMBayWzLjWO4Mh8ETBOeMDy+VeQOSo/HtvnTIlnp5jBpe3jhBgUYOSwgQKgJS3kJQyYqJoeecfU6AXQEP6L3Pn7/Jz8/StsiYYEUYCyQjCxuYqZHgLNTqqlZghF9rQA4Z3MkiAnzBRnjW6tWtDNz3kVvpLzpiHUKMzStnMyQD6hGofqAjbrR1ai4ZKkCeFEUNaQM1dc+kWSFH4sBYwMT7fG5MdTr3tanQGiC0gdTOLsKtIVJwYFZa0hTwPCjfNqRrHE1kOF8PgeKOHUch8z+EG922KIDhT+YVEb2JGiD4tAL1fyz73z1O6fAgxYYRivXT6dKOWdQbY2ftw6MAp1LoXG60V81xkD+5CNkf6YbbCFxLqpECZZf4Qa25TMu5l7yGwwYLq6uF975cTJipFXl/nPP3MpUBhG+I9e2z7x8RBiYdShbJTcme1qTaiRavCpqFj4VIimJOLuCD982rSw2WrNhI/gokw5IOfYBxU+YoDc4kMhl9OIKvWqMRvRpT/SwaX88vBw9/He0wYE3nAEI0tk+tSbdGGfG3WFs8gYk30Fyl1uAx4DNjjJC1k6CCtMXaF/JxVQQrf43ll6nCJrJYVxmhiar5RpAx9BWv99eHDuWFvlqEUqQEK3QKUQGPLgBM3Ohc6UM6+MSAMhvZLLK7m8kstg48+CXJgwqASZs3Y7Jnb+fqLn4cHbP2Rj3FzEyIlwdSNaiXF345vrd3lxXmB4/sc7vni/E+d1Jru/MQ4w3gVOd4Ij3eB4RzjKFZ7pDAdkHSpcMVH2IPCJkXYIpXR8yHcKfAnILWH2eHhRQfWchm0E1l05TGHxPuhGGsIvTGMVKzWo7Ji7NtLAze2/62MHpAikVervzfLKWGc198pYZwOJBwOvjBUhaxxfLVEVGs1VsUw1YzFoLLBvV/gZzU/x7PQCuem1XNTR1Gu5aDSEWBjwWi5qlVOtGv/9uqNadOHl2RIQbhRq3euse0wX56i99/r85cdDUah0yfBuLRU83N7BmsudBmbe6uB1DrsKpQCS55xRv7EFtFFIMpCC798fe70jpfr3CkxTq7FJoEexghOcWl0ombiU7QtMRjbALmP7UqvLGL9Vsy5oypgejh9g+MZKjt8d4kx3WMnRfnnHofDmPCXkPkc7XAiemOsP0nDU08MUPKaZPvqdJ5seIt55U82RlLtM1jmObEdQ7TiijabZM0h2gGINzVda85WPoL8tydamL5fa7/iu3MVZOVF9t/fBAfnt0ztUCAZVxoRPQUpX9fH+/kvX0vKIYGAaymNC+v1rjSqt9rv0+6HNsxyvMynNFlPvzd8xAZl+f/Dq9VTsrfb+XRtCH6/Cqn3GhDXYIERO9qgK7XKiizS0WjILCr7mZq+52WtuNtj4M1nKv5QLPESuv3+Nc4ECdxcjEKpwNHvIHMVsAG/DEaBa8C+t0YaI1FmqCVpJu9l6192BtESY20TbpCZ6rs3adza5t8nITdraJlV70+KUe1v941TxPTYtPiH0cZWh9kenzh/1tzaznBj2hIGmXPcVrWsnSsgdx3QTKPfm8H9Vfb4KuocbUuTsCdXeA+jfecjlJkzd8yrO0hAeNqxo9heWwYMVDeiVQu/I9eM1uS4BVBfeAxNAGt3cM7lttqqN8LIz5prlFdjCmOhrFU1wbzXkVm8BRZpLJswVJNaAkAb2aBrd1q+GFZWQudVYtAskT1GblZdxgL4im/NOS91s0IeaxYGo9+VwCuI6VOrSaJRCueV8VZ/Z1fGwheZ4TZNKkcORtMakVqgtN9fwL6mAQIqezItIve1RjTVf+/HgbBum+JjJ1CucIkk5E9ileYTN5ltgG7JUq9Mr7NOP81L96oS96A5dtjO9eSb2ot4uDk5vgRiDWd4ODn4TnD2i10JfhQzYPeMXgBWwLOeYoTAh400las8JCTF0618sUXnba7iXIVUuVzql4HsXrBnChAYpsPHAtS/v1oUpNwRC3RmVksqNE0dTG/aEovEsUOJr00gUZJYblnMEwzLsWeVtWLtvsTfe4j8dFRm8MqX2lYhQamBUyZIERo0Sf/DxQtFZufW3OTWNLKelI+JiOX1KkBZWEFyAc3Fyui9F37np+h1QVYstl3UllcCmx4MdM1sQUnxwPmbfMC1LxznEI50uNTaONLvUgPgHlSn+MGlYxFqw2o5zoSFR7McZ1elO1xVHY1Bdys3lNuFMb0Ma5ORDkA9G5oyOQt/ZYwursNtKjVBK9ZuKbJ76GkCyh68yZev9DX38qbzhnNx68qg8V9tTNeLmZrwmy7rMyG4YQl2kpgpdIIVeg8tlqK35aQllQoLdrcUq2a8UbtpfOfSt9LmCRMlHdKHvThxCo4BzSjXhMlp21BXmUu+oExcrurXXe6aMvjbEM3XFTNgndoWwWcecmqvsNl8n9GCdqS+mo55ofo24eOCN+Nibw0fH09+Kx5vJ1ykrTmTzIfWeRT50qm1YjRmlqXMuKyr1jFqEFwLWLV/thGAirFdew8OWaWAarA6xcnEL/GGlIfWXBHZDp1Ks2WYVgtM5FlXb+iIIsUEVoFsi3GSv3rjQnOc1u1fqe/n+/QtDXVFS+gUYpGu9Y07qOIpQ5vfDSyhxOLiRMzpt8fTBPXrmsukLTLY1ivSCCwc+zX6xFbZ5X7vTZ6Fl0sJnk1AVZZXvLpM6R6/nmZCco9Fzi+7P0WWmgOu8KKsWX/lYxkddPStc3y7U+l6CqnOH/7eNpsair4hOkFxv5fFYid2GVjx9ZjBV+PkLVTU7Cn+FKoNsdHEW6sc71cktvWeuq+wyaOYGsmX9wQDGM5zBUpv4xhqzjmV2h7SM+V4Ok33o3EXsvxjjT5Ckij1hfZ/92A3G8V8pOaSmUjVd6Fybne9DyyN3O5OcRbNL/5Hl/oQs9mTpT8gNOQyFm7vPQInfAVXrfuce3JUMzVamXn45BsK7F6hMsZ1SiTXbv87UuL4/m68SojFdhQ/krAilqOeYF0d2qIqY/ixAGOt+Qovi0zyQIBMb2CgiHNUHHKAlR76H1KILX4s7b26/9MSqTqVDWDyDJuE7Ha4vb7/UAu5j/9O/0l2YV+dI2ZrRlUOZWTNXkeTI2mWdMiNp3XCl9BYLHg6DtL0wf/oAi/xkwmRNjz6YcPx6/XNnHRxvl1tUjdYvBMzlOE4d93k9W4zoGV7E48On2oEc37CGHFVxeq9hgkIPoJzoMsV9ItyiS3QPJ+kFRd9CSvZX3kTOdNV9CvNwEImYopBPspw7B+Rfo/FEeHn+R1rjH01JzxGl8J4fn9p3fZ/kZYzxgyJzhO1lNEw4r3qVhXcWPbOO/X8AAAD//+FPrA0=" } diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json index 42516375e59..e2cf2be2b97 100644 --- a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json @@ -13,15 +13,43 @@ }, "googlecloud": { "labels": { + "metrics": { + "region": "us-west1" + }, "resource": { - "topic_id": "test-ks" + "subscription_id": "test-ks" } }, "pubsub": { + "snapshot": { + "backlog_bytes": 19, + "backlog_bytes_by_region": 19, + "num_messages": 4, + "num_messages_by_region": 4, + "oldest_message_age": 389, + "oldest_message_age_by_region": 389 + }, "subscription": { - "backlog_bytes": 20, - "num_undelivered_messages": 4, - "oldest_unacked_message_age": 435752 + "backlog_bytes": 0, + "num_retained_acked_messages": 0, + "num_retained_acked_messages_by_region": 0, + "num_unacked_messages_by_region": 4, + "num_undelivered_messages": 0, + "oldest_retained_acked_message_age": 0, + "oldest_retained_acked_message_age_by_region": 0, + "oldest_unacked_message_age": 0, + "oldest_unacked_message_age_by_region": 362, + "retained_acked_bytes": 0, + "retained_acked_bytes_by_region": 0, + "unacked_bytes_by_region": 19 + }, + "topic": { + "num_retained_acked_messages_by_region": 0, + "num_unacked_messages_by_region": 16, + "oldest_retained_acked_message_age_by_region": 0, + "oldest_unacked_message_age_by_region": 389, + "retained_acked_bytes_by_region": 0, + "unacked_bytes_by_region": 76 } } }, diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_subscription.json b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_subscription.json deleted file mode 100644 index 1503d6aa80f..00000000000 --- a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_subscription.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "@timestamp": "2020-01-14T11:46:00.000Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0" - }, - "googlecloud": { - "labels": { - "resource": { - "subscription_id": "test-subscription" - } - }, - "pubsub": { - "subscription": { - "num_undelivered_messages": 0, - "oldest_unacked_message_age": 0, - "backlog_bytes": 0 - } - } - }, - "service": { - "type": "googlecloud" - }, - "cloud": { - "account": { - "id": "elastic-metricbeat" - }, - "provider": "googlecloud" - }, - "host": { - "name": "mcastro" - }, - "agent": { - "version": "8.0.0", - "type": "metricbeat", - "ephemeral_id": "b3b8d9da-0094-4d54-b1dc-4d5e103d38aa", - "hostname": "mcastro", - "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228" - }, - "ecs": { - "version": "1.4.0" - }, - "event": { - "dataset": "googlecloud.pubsub", - "module": "googlecloud", - "duration": 933264720 - }, - "metricset": { - "name": "pubsub", - "period": 300000 - } -} diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_topic.json b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_topic.json deleted file mode 100644 index f85d8148782..00000000000 --- a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_topic.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "@timestamp": "2020-01-14T11:42:00.000Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0" - }, - "ecs": { - "version": "1.4.0" - }, - "cloud": { - "provider": "googlecloud", - "account": { - "id": "elastic-metricbeat" - } - }, - "metricset": { - "name": "pubsub", - "period": 300000 - }, - "event": { - "duration": 932060852, - "dataset": "googlecloud.pubsub", - "module": "googlecloud" - }, - "googlecloud": { - "labels": { - "resource": { - "subscription_id": "test-subscription" - } - }, - "pubsub": { - "topic": { - "message_sizes": { - "bucket_options": { - "Options": { - "ExponentialBuckets": { - "growth_factor": 4, - "scale": 1, - "num_finite_buckets": 16 - } - } - } - } - } - } - }, - "service": { - "type": "googlecloud" - }, - "host": { - "name": "mcastro" - }, - "agent": { - "ephemeral_id": "b3b8d9da-0094-4d54-b1dc-4d5e103d38aa", - "hostname": "mcastro", - "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228", - "version": "8.0.0", - "type": "metricbeat" - } -} diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml index 64063c44497..4e05ae476cf 100644 --- a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml @@ -55,6 +55,60 @@ - name: streaming_pull_response_count type: long description: Cumulative count of streaming pull responses, grouped by result. + - name: dead_letter_message_count + type: long + description: Cumulative count of messages published to dead letter topic, grouped by result. + - name: mod_ack_deadline_message_count + type: long + description: Cumulative count of messages whose deadline was updated by ModifyAckDeadline requests, grouped by delivery type. + - name: mod_ack_deadline_message_operation_count + type: long + description: Cumulative count of ModifyAckDeadline message operations, grouped by result. + - name: mod_ack_deadline_request_count + type: long + description: Cumulative count of ModifyAckDeadline requests, grouped by result. + - name: oldest_retained_acked_message_age + type: long + description: Age (in seconds) of the oldest acknowledged message retained in a subscription. + - name: oldest_retained_acked_message_age_by_region + type: long + description: Age (in seconds) of the oldest acknowledged message retained in a subscription, broken down by Cloud region. + - name: oldest_unacked_message_age_by_region + type: long + description: Age (in seconds) of the oldest unacknowledged message in a subscription, broken down by Cloud region. + - name: retained_acked_bytes + type: long + description: Total byte size of the acknowledged messages retained in a subscription. + - name: retained_acked_bytes_by_region + type: long + description: Total byte size of the acknowledged messages retained in a subscription, broken down by Cloud region. + - name: num_retained_acked_messages + type: long + description: Number of acknowledged messages retained in a subscription. + - name: num_retained_acked_messages_by_region + type: long + description: Number of acknowledged messages retained in a subscription, broken down by Cloud region. + - name: seek_request_count + type: long + description: Cumulative count of seek attempts, grouped by result. + - name: streaming_pull_mod_ack_deadline_message_operation_count + type: long + description: Cumulative count of StreamingPull ModifyAckDeadline operations, grouped by result. + - name: streaming_pull_mod_ack_deadline_request_count + type: long + description: Cumulative count of streaming pull requests with non-empty ModifyAckDeadline fields, grouped by result. + - name: byte_cost + type: long + description: Cumulative cost of operations, measured in bytes. This is used to measure quota utilization. + - name: config_updates_count + type: long + description: Cumulative count of configuration changes for each subscription, grouped by operation type and result. + - name: unacked_bytes_by_region + type: long + description: Total byte size of the unacknowledged messages in a subscription, broken down by Cloud region. + - name: num_unacked_messages_by_region + type: long + description: Number of unacknowledged messages in a subscription, broken down by Cloud region. - name: topic type: group description: Topic related metrics @@ -68,3 +122,52 @@ - name: send_request_count type: long description: Cumulative count of publish requests, grouped by result. + - name: oldest_retained_acked_message_age_by_region + type: long + description: Age (in seconds) of the oldest acknowledged message retained in a topic, broken down by Cloud region. + - name: oldest_unacked_message_age_by_region + type: long + description: Age (in seconds) of the oldest unacknowledged message in a topic, broken down by Cloud region. + - name: retained_acked_bytes_by_region + type: long + description: Total byte size of the acknowledged messages retained in a topic, broken down by Cloud region. + - name: num_retained_acked_messages_by_region + type: long + description: Number of acknowledged messages retained in a topic, broken down by Cloud region. + - name: byte_cost + type: long + description: Cost of operations, measured in bytes. This is used to measure utilization for quotas. + - name: config_updates_count + type: long + description: Cumulative count of configuration changes, grouped by operation type and result. + - name: unacked_bytes_by_region + type: long + description: Total byte size of the unacknowledged messages in a topic, broken down by Cloud region. + - name: num_unacked_messages_by_region + type: long + description: Number of unacknowledged messages in a topic, broken down by Cloud region. + - name: snapshot + type: group + description: Snapshot related metrics + fields: + - name: oldest_message_age + type: long + description: Age (in seconds) of the oldest message retained in a snapshot. + - name: oldest_message_age_by_region + type: long + description: Age (in seconds) of the oldest message retained in a snapshot, broken down by Cloud region. + - name: backlog_bytes + type: long + description: Total byte size of the messages retained in a snapshot. + - name: backlog_bytes_by_region + type: long + description: Total byte size of the messages retained in a snapshot, broken down by Cloud region. + - name: num_messages + type: long + description: Number of messages retained in a snapshot. + - name: num_messages_by_region + type: long + description: Number of messages retained in a snapshot, broken down by Cloud region. + - name: config_updates_count + type: long + description: Cumulative count of configuration changes, grouped by operation type and result. diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml b/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml index e7e3e032b71..307879450ff 100644 --- a/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml +++ b/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml @@ -22,6 +22,39 @@ input: - "pubsub.googleapis.com/subscription/streaming_pull_ack_request_count" - "pubsub.googleapis.com/subscription/streaming_pull_message_operation_count" - "pubsub.googleapis.com/subscription/streaming_pull_response_count" + - "pubsub.googleapis.com/subscription/dead_letter_message_count" + - "pubsub.googleapis.com/subscription/mod_ack_deadline_message_count" + - "pubsub.googleapis.com/subscription/mod_ack_deadline_message_operation_count" + - "pubsub.googleapis.com/subscription/mod_ack_deadline_request_count" + - "pubsub.googleapis.com/subscription/oldest_retained_acked_message_age" + - "pubsub.googleapis.com/subscription/oldest_retained_acked_message_age_by_region" + - "pubsub.googleapis.com/subscription/oldest_unacked_message_age_by_region" + - "pubsub.googleapis.com/subscription/retained_acked_bytes" + - "pubsub.googleapis.com/subscription/retained_acked_bytes_by_region" + - "pubsub.googleapis.com/subscription/num_retained_acked_messages" + - "pubsub.googleapis.com/subscription/num_retained_acked_messages_by_region" + - "pubsub.googleapis.com/subscription/seek_request_count" + - "pubsub.googleapis.com/subscription/streaming_pull_mod_ack_deadline_message_operation_count" + - "pubsub.googleapis.com/subscription/streaming_pull_mod_ack_deadline_request_count" + - "pubsub.googleapis.com/subscription/byte_cost" + - "pubsub.googleapis.com/subscription/config_updates_count" + - "pubsub.googleapis.com/subscription/unacked_bytes_by_region" + - "pubsub.googleapis.com/subscription/num_unacked_messages_by_region" - "pubsub.googleapis.com/topic/message_sizes" - "pubsub.googleapis.com/topic/send_message_operation_count" - "pubsub.googleapis.com/topic/send_request_count" + - "pubsub.googleapis.com/topic/oldest_retained_acked_message_age_by_region" + - "pubsub.googleapis.com/topic/oldest_unacked_message_age_by_region" + - "pubsub.googleapis.com/topic/retained_acked_bytes_by_region" + - "pubsub.googleapis.com/topic/num_retained_acked_messages_by_region" + - "pubsub.googleapis.com/topic/byte_cost" + - "pubsub.googleapis.com/topic/config_updates_count" + - "pubsub.googleapis.com/topic/unacked_bytes_by_region" + - "pubsub.googleapis.com/topic/num_unacked_messages_by_region" + - "pubsub.googleapis.com/snapshot/oldest_message_age" + - "pubsub.googleapis.com/snapshot/oldest_message_age_by_region" + - "pubsub.googleapis.com/snapshot/backlog_bytes" + - "pubsub.googleapis.com/snapshot/backlog_bytes_by_region" + - "pubsub.googleapis.com/snapshot/num_messages" + - "pubsub.googleapis.com/snapshot/num_messages_by_region" + - "pubsub.googleapis.com/snapshot/config_updates_count" diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/pubsub_integration_test.go b/x-pack/metricbeat/module/googlecloud/pubsub/pubsub_integration_test.go index 90032d50310..e20d229b534 100644 --- a/x-pack/metricbeat/module/googlecloud/pubsub/pubsub_integration_test.go +++ b/x-pack/metricbeat/module/googlecloud/pubsub/pubsub_integration_test.go @@ -2,9 +2,6 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -// +build integration -// +build googlecloud - package pubsub import ( From 44eaa0027ee170f86594013d9278fdf0ea65a6e8 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 8 Apr 2020 07:21:00 -0600 Subject: [PATCH 2/5] undo change for unit test --- .../module/googlecloud/pubsub/pubsub_integration_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/pubsub_integration_test.go b/x-pack/metricbeat/module/googlecloud/pubsub/pubsub_integration_test.go index e20d229b534..90032d50310 100644 --- a/x-pack/metricbeat/module/googlecloud/pubsub/pubsub_integration_test.go +++ b/x-pack/metricbeat/module/googlecloud/pubsub/pubsub_integration_test.go @@ -2,6 +2,9 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +// +build integration +// +build googlecloud + package pubsub import ( From cbd754121382fe0d695ad1cf0fd8b75220930000 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 8 Apr 2020 12:19:22 -0600 Subject: [PATCH 3/5] update documentation --- .../metricbeat/module/googlecloud/fields.go | 2 +- .../module/googlecloud/pubsub/_meta/data.json | 16 +-- .../googlecloud/pubsub/_meta/docs.asciidoc | 100 +++++++++++++++++- .../googlecloud/pubsub/_meta/fields.yml | 15 --- .../module/googlecloud/pubsub/manifest.yml | 53 +++++----- 5 files changed, 127 insertions(+), 59 deletions(-) diff --git a/x-pack/metricbeat/module/googlecloud/fields.go b/x-pack/metricbeat/module/googlecloud/fields.go index 94dfe71b097..8fac4947787 100644 --- a/x-pack/metricbeat/module/googlecloud/fields.go +++ b/x-pack/metricbeat/module/googlecloud/fields.go @@ -19,5 +19,5 @@ func init() { // AssetGooglecloud returns asset data. // This is the base64 encoded gzipped contents of module/googlecloud. func AssetGooglecloud() string { - return "eJzsXc1y2zgSvucp+pZky3EOe0ttTZXHszuT2mTHtfbMVQWCLQlrEODgxxrN028BIClS4g9IkYqd8s0WSfTXDaC/7gZAfoBH3H+CjZQbjpRLm74BMMxw/ARvf/a/wq37Ge44MWupsrdvABRyJBo/QYKGvAFIUVPFcsOk+AQ/vAEA+Pn2DjKZWo5vANYMeao/+QsfQJAMGyKvOUmQa38ZwOxz/AQy+R9SU/xUf77ehtWorv9W/dz67OHuDA1JiSGjnlCM6vgH9F4bzOLvpzLLrcHa7cembTazUdLmtV8bhm90121ouVSh9syxMet4mNCGCIqNi13CuxqrN7hmCneE85Mb+hrta7jeeKpknmO6SvYG9YpKK0zr/aUsLsWm44aGIT8LKjMmNuAbLsVAsgezxT6VTqHlhD6iWQRc0XQ0vGrM5XaR3lCoUT1huqJSoe7VNZU24ceDrFXb/9gsQQVyDb7VSghI4bXdSm3cVfd3x+BtorSGcfYXca3PBPHBGV0R6v4roRDOJSUGU7i9+w3MlhhgGqhVCoXhe2DC+a5SiTjgmmxwZVjWBWos7t9cg7CWyqEtzMsEaKRSpKf9V41sph8XGj9k/ql861pyvRKmspMBayWzLjWO4Mh8ETBOeMDy+VeQOSo/HtvnTIlnp5jBpe3jhBgUYOSwgQKgJS3kJQyYqJoeecfU6AXQEP6L3Pn7/Jz8/StsiYYEUYCyQjCxuYqZHgLNTqqlZghF9rQA4Z3MkiAnzBRnjW6tWtDNz3kVvpLzpiHUKMzStnMyQD6hGofqAjbrR1ai4ZKkCeFEUNaQM1dc+kWSFH4sBYwMT7fG5MdTr3tanQGiC0gdTOLsKtIVJwYFZa0hTwPCjfNqRrHE1kOF8PgeKOHUch8z+EG922KIDhT+YVEb2JGiD4tAL1fyz73z1O6fAgxYYRivXT6dKOWdQbY2ftw6MAp1LoXG60V81xkD+5CNkf6YbbCFxLqpECZZf4Qa25TMu5l7yGwwYLq6uF975cTJipFXl/nPP3MpUBhG+I9e2z7x8RBiYdShbJTcme1qTaiRavCpqFj4VIimJOLuCD982rSw2WrNhI/gokw5IOfYBxU+YoDc4kMhl9OIKvWqMRvRpT/SwaX88vBw9/He0wYE3nAEI0tk+tSbdGGfG3WFs8gYk30Fyl1uAx4DNjjJC1k6CCtMXaF/JxVQQrf43ll6nCJrJYVxmhiar5RpAx9BWv99eHDuWFvlqEUqQEK3QKUQGPLgBM3Ohc6UM6+MSAMhvZLLK7m8kstg48+CXJgwqASZs3Y7Jnb+fqLn4cHbP2Rj3FzEyIlwdSNaiXF345vrd3lxXmB4/sc7vni/E+d1Jru/MQ4w3gVOd4Ij3eB4RzjKFZ7pDAdkHSpcMVH2IPCJkXYIpXR8yHcKfAnILWH2eHhRQfWchm0E1l05TGHxPuhGGsIvTGMVKzWo7Ji7NtLAze2/62MHpAikVervzfLKWGc198pYZwOJBwOvjBUhaxxfLVEVGs1VsUw1YzFoLLBvV/gZzU/x7PQCuem1XNTR1Gu5aDSEWBjwWi5qlVOtGv/9uqNadOHl2RIQbhRq3euse0wX56i99/r85cdDUah0yfBuLRU83N7BmsudBmbe6uB1DrsKpQCS55xRv7EFtFFIMpCC798fe70jpfr3CkxTq7FJoEexghOcWl0ombiU7QtMRjbALmP7UqvLGL9Vsy5oypgejh9g+MZKjt8d4kx3WMnRfnnHofDmPCXkPkc7XAiemOsP0nDU08MUPKaZPvqdJ5seIt55U82RlLtM1jmObEdQ7TiijabZM0h2gGINzVda85WPoL8tydamL5fa7/iu3MVZOVF9t/fBAfnt0ztUCAZVxoRPQUpX9fH+/kvX0vKIYGAaymNC+v1rjSqt9rv0+6HNsxyvMynNFlPvzd8xAZl+f/Dq9VTsrfb+XRtCH6/Cqn3GhDXYIERO9qgK7XKiizS0WjILCr7mZq+52WtuNtj4M1nKv5QLPESuv3+Nc4ECdxcjEKpwNHvIHMVsAG/DEaBa8C+t0YaI1FmqCVpJu9l6192BtESY20TbpCZ6rs3adza5t8nITdraJlV70+KUe1v941TxPTYtPiH0cZWh9kenzh/1tzaznBj2hIGmXPcVrWsnSsgdx3QTKPfm8H9Vfb4KuocbUuTsCdXeA+jfecjlJkzd8yrO0hAeNqxo9heWwYMVDeiVQu/I9eM1uS4BVBfeAxNAGt3cM7lttqqN8LIz5prlFdjCmOhrFU1wbzXkVm8BRZpLJswVJNaAkAb2aBrd1q+GFZWQudVYtAskT1GblZdxgL4im/NOS91s0IeaxYGo9+VwCuI6VOrSaJRCueV8VZ/Z1fGwheZ4TZNKkcORtMakVqgtN9fwL6mAQIqezItIve1RjTVf+/HgbBum+JjJ1CucIkk5E9ileYTN5ltgG7JUq9Mr7NOP81L96oS96A5dtjO9eSb2ot4uDk5vgRiDWd4ODn4TnD2i10JfhQzYPeMXgBWwLOeYoTAh400las8JCTF0618sUXnba7iXIVUuVzql4HsXrBnChAYpsPHAtS/v1oUpNwRC3RmVksqNE0dTG/aEovEsUOJr00gUZJYblnMEwzLsWeVtWLtvsTfe4j8dFRm8MqX2lYhQamBUyZIERo0Sf/DxQtFZufW3OTWNLKelI+JiOX1KkBZWEFyAc3Fyui9F37np+h1QVYstl3UllcCmx4MdM1sQUnxwPmbfMC1LxznEI50uNTaONLvUgPgHlSn+MGlYxFqw2o5zoSFR7McZ1elO1xVHY1Bdys3lNuFMb0Ma5ORDkA9G5oyOQt/ZYwursNtKjVBK9ZuKbJ76GkCyh68yZev9DX38qbzhnNx68qg8V9tTNeLmZrwmy7rMyG4YQl2kpgpdIIVeg8tlqK35aQllQoLdrcUq2a8UbtpfOfSt9LmCRMlHdKHvThxCo4BzSjXhMlp21BXmUu+oExcrurXXe6aMvjbEM3XFTNgndoWwWcecmqvsNl8n9GCdqS+mo55ofo24eOCN+Nibw0fH09+Kx5vJ1ykrTmTzIfWeRT50qm1YjRmlqXMuKyr1jFqEFwLWLV/thGAirFdew8OWaWAarA6xcnEL/GGlIfWXBHZDp1Ks2WYVgtM5FlXb+iIIsUEVoFsi3GSv3rjQnOc1u1fqe/n+/QtDXVFS+gUYpGu9Y07qOIpQ5vfDSyhxOLiRMzpt8fTBPXrmsukLTLY1ivSCCwc+zX6xFbZ5X7vTZ6Fl0sJnk1AVZZXvLpM6R6/nmZCco9Fzi+7P0WWmgOu8KKsWX/lYxkddPStc3y7U+l6CqnOH/7eNpsair4hOkFxv5fFYid2GVjx9ZjBV+PkLVTU7Cn+FKoNsdHEW6sc71cktvWeuq+wyaOYGsmX9wQDGM5zBUpv4xhqzjmV2h7SM+V4Ok33o3EXsvxjjT5Ckij1hfZ/92A3G8V8pOaSmUjVd6Fybne9DyyN3O5OcRbNL/5Hl/oQs9mTpT8gNOQyFm7vPQInfAVXrfuce3JUMzVamXn45BsK7F6hMsZ1SiTXbv87UuL4/m68SojFdhQ/krAilqOeYF0d2qIqY/ixAGOt+Qovi0zyQIBMb2CgiHNUHHKAlR76H1KILX4s7b26/9MSqTqVDWDyDJuE7Ha4vb7/UAu5j/9O/0l2YV+dI2ZrRlUOZWTNXkeTI2mWdMiNp3XCl9BYLHg6DtL0wf/oAi/xkwmRNjz6YcPx6/XNnHRxvl1tUjdYvBMzlOE4d93k9W4zoGV7E48On2oEc37CGHFVxeq9hgkIPoJzoMsV9ItyiS3QPJ+kFRd9CSvZX3kTOdNV9CvNwEImYopBPspw7B+Rfo/FEeHn+R1rjH01JzxGl8J4fn9p3fZ/kZYzxgyJzhO1lNEw4r3qVhXcWPbOO/X8AAAD//+FPrA0=" + return "eJzsXUuT2zYSvvtX9M321nh82JtrK1WTyW7iWnsztTPJVQWCLQk7IMDgMYry67cAkBQp8QFSpPwo3WYkEv11A/i+bgCk3sEz7j/ARsoNR8qlTV8BGGY4foDXP/tP4d59DA+cmLVU2etXAAo5Eo0fIEFDXgGkqKliuWFSfIAfXgEA/Hz/AJlMLcdXAGuGPNUf/BfvQJAMGyZvOUmQa/81gNnn+AFk8j+kpviofn+9DatR3f6t+rj13sPVGRqSEkNG3aEY1fE36L02mMVfT2WWW4O1y49D22xmo6TNa582At/orvvQculC7Z7jYNbxMKENERQbX3YZ72qs3uCaKdwRzk8u6Gu0r+F646mSeY7pKtkb1CsqrTCt15e2uBSbjgsagfwoqMyY2IBvuDQDyR7MFvtcOoWWE/qMZhFwRdPR8Koxl9tFekOhRvWC6YpKhbrX11TahB8PslZv/2OzBBXINfhWKyMghfd2K7Vx37q/OwZvE6U1jLO/iGt9JohPLuiKUPdfCYVwLikxmML9w29gtsQA00CtUigM3wMTjrtKJ+KAa7LBlWFZF6ixuH9zDcJaKoe2CC8ToJFKkZ72XzWymX5eaPyQ+afyvWvJ9UqYys4GrJXMutw4giPzRcA44wHLx19B5qj8eGyfMyWenWIGl46PM2JQgJHDAQqAloyQtzAQomp65B1ToxdAw/gvcuev83Py98+wJRoSRAHKCsHE5iZmegg0O6mWmiEU2csCgncyS4KdMFNcNLq9akE3v+ZV+ErNm4ZQozBLx87ZAPmCahyqC8SsH1mJhkuSJoQTQVnDzlx56SdJUvixNDAyPd0akx9Pve5pdQaILiB1MImLq0hXnBgUlLWmPA0Id47VjGKJracK4fY9UMKp5T5n8IN6t8WQHSj8w6I2sCNFHxaJXq7kn3vH1O6fAgxYYRivfX06Ucorg21t/Lh1YBTqXAqNt4tw1xkD+1CNkf6cbbCFxLqpECZZf4Ya25TMu5V7KGwwELq6uV977cTZirFXt/nPP3MpUBhG+I/e2z7z8RBiYdShbJTcme1qTaiRavCuqFz41IimJOLqCB4+bVrYbLVmwmdwUaEcsHPMQQVHDIhbfCrkahpRlV41ZSO65CMdKOWXp6eH949eNiDohhMYWSLTp2zShX1u1BXOomJM9hUo93Ub8BiwgSQvFOlgrAh1hf6NVEAJ3eJbF+lxjqyVFMZ5Ymi+UqYNfIRo/ffpydGxtspJi1SAhG6BSiEw1MEJmp1LnSln3hmRBkG6istVXK7iMtj4VyEuTBhUgsy5djsmd/5+sufhwds/ZGNoLmLkRFDdiFZi6G58c/2UF8cCw/M/nvjieSeOdSbT3xgCjKfA6SQ4kgbHE+EoKjyTDAdsHVa4YrLsQeATM+2QSun4lO8U+BKQW9Ls8fCikuo5A9tIrLtqmCLifdCNNIRfWMYqVWpI2bF2baSBu/t/18cOSBFEq/Tfh+WqWGc1d1Wss4HEg4GrYkXYGqdXS6wKjdaqWKWacTFoLLAvt/AzWp/i1ekb1KbrclFHU9flotEQYmHAdbmo1U61a/z3247Vogtvz5aAcKNQ616y7gldHFF79vr46cfDolBJyfBmLRU83T/AmsudBmZe68A6h1OFUgDJc86oP9gC2igkGUjB92+PWe/Iqf6zAtPcahwS6HGs0ATnVhdKJi4V+wKTkQ2wy8S+9OoywW/1rAuaMqZH4wcUvrGT40+HuNAddnK0395xKHw4TwW5j2iHF4In1vqDMhx197AEj2mmT37nqaaHhHfeUnOk5C5TdY4T2xFSO05oo2X2DJEdkFhD85XWfOUz6C8rsrXpy6X2J74rujirJqqf9j4QkD8+vUOFYFBlTPgSpKSq94+Pn7q2lkckA9NQHgvS759rUmm1P6XfD22e7XidSWm2mHo2f8MEZPrtgdXrpdhr7fldG0Kfb8KufcaENdgQRE72qArvcqKLMrTaMgsOXmuza212rc0GG/9KtvIvRYGHzPX3z3EUKHB3MQGhCkerh8xRzAbwPjwCVEv+pTXaEJG6SDVBK2k3W0/dHUhLhLlNtE1qpuc6rP1gk0ebjDykrW1StTctT3m01T/OFd9j0/ITQp9XGWr/6NT5o/7eZpYTw14wyJTrvqJ17UwJueOYboLk3h3+r1afb4Lv4YIUOXtBtfcA+k8ecrkJU/e8FWdpCA8HVjT7C8vkwYoG9MqhN+T2+ZbclgCqL94CE0Aa3dwzuW22qo3wsjPmmuUV2CKY6NcqmuBea8it3gKKNJdMmBtIrAEhDezRNLqt3w0rKiNzu7FoF0ieojYrb+MAfUU25z0tdbdBn2oWD0S9LYdTMNfhUpdHoxzKLeer+syuHg9baI7XPKkcOTyS1pjUCrXl5hb+JRUQSNGLeZGpt92qsca17w9k2wjF+0ym3uEUScqZwC7PI2I23wbbUKRaSa+ITz/OS/WrM/ZNd+iynenDM7EX9XZxcHoLxBjM8nZw8Jvg7Bm9F/omVMDuHr8BrIBlOccMhQkVbypRe01IiKFb/2KJim1v4VGGUrnc6ZSC712yZggTGqTAxg23fnm3bky5IRDWnVEpqdw4cTK1YS8oGvcCJX5tGomCzHLDco5gWIY9u7yNaPdt9sZH/KejRQbvTOl9ZSIsNTCqZCkCo0aJf/DxQtlZefS3OTWNLKelE+JiO31KkhZ2EFyCc3FxeixNP7jp+h1IVUssl6WSymCT8WDHzBaEFO8cx+wboWXpOEI88ulSY+PIs0sNiH9QmeIPk4ZFbASr4zgXGhLFeZxRne58XXE0BtWlaC63CWd6G8ogZx+CfTAyZ3QU+s4eW9iF3VZqhNKqP1Rk89SvASR7+CxTtt7f0eefygvOqa0nj8pzvT11I25uxnuyLGVGdsMQ6qI0VegSKfQeXK5Cba1PSygTCuxuL1bJfqVw0/7KoS/lzw0kSj6jS3134pAaBZxTVhMu42XHusJc7h114mKLbu3rPVNGXxvimbpiJuwTu0IjLp75IT73FpHRCd2XEpJm9n9KyxPlZMi9ryIhP/U2bAeM8tSN7hWVekYvwhvp6pGvtuKZCBtmt/C0ZRqYdsW/T9aKS+APKw2pv6WuGzqVYs02q5AdzbGr19YXwYgNrgDdEuEme/XIf3Oe1+Jeue/t+xcADHVFqSkXoLCuBfdzuOtw6D5ndNrG15O79cwtr2+wUNIo0gsu+voS6ZtdHZn3lSl9EVompf9qkuGiJP7usuBz/Po6k8lzPJpJ388T9Zqce+n0It+zov/llP170fCxQ6aiV0FyvZXH0Y49uFLcfaaEF+xyoXWQjqWCwpVBDrw49/XjnUoTS5+y6aqTB8PcQLbsjBrAODG0wmaLHfsZG8w6lpliGY1qYvi+HS1413nu0P/GhD9znir2gvWTuWOPJMb/rsGhIJKqSaFzHY98DC2PPB9JchatLv0POfaXAbHPov2E3JDDULh7+AiU+DMTte539OC+ydBsZertl2MgPK1NZYrtkkqs2f51psf1E518lRCN6Sr8pMaKUIp6jnlxFIdq1cmfHg5j3U9oUfyYByTo6t6NIsJJfcABWnLke0gtugSwuPLu/lNPtudcOiSWM3gS3uzv+vL+Uy1lPeaf/r2xIrw6R8rWjK4cysyauUrzo2iXC0sZSeuBK623RPBwfLztFdvTB1jkS9Yne3r0ivXjF3KfO+vg+IDNom60vlN8LuI4Je7zerYY0TO8usOnT7Uj/L5hDTmq4nmfRggKP4Byossi8YVwi65UPDx7Kyj6FlKyv/EhcqGrrlOYh0cXiClWXkmWc0dA/sH7F8LLJwakNf7WlPQ81BDeDOKL465fNPg2xvjBkTnS9jIbJpxXvcrCW06+so79fwAAAP//qGqEJg==" } diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json index e2cf2be2b97..86da8d924e5 100644 --- a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json @@ -13,9 +13,6 @@ }, "googlecloud": { "labels": { - "metrics": { - "region": "us-west1" - }, "resource": { "subscription_id": "test-ks" } @@ -26,28 +23,23 @@ "backlog_bytes_by_region": 19, "num_messages": 4, "num_messages_by_region": 4, - "oldest_message_age": 389, - "oldest_message_age_by_region": 389 + "oldest_message_age": 69319, + "oldest_message_age_by_region": 69319 }, "subscription": { "backlog_bytes": 0, - "num_retained_acked_messages": 0, - "num_retained_acked_messages_by_region": 0, - "num_unacked_messages_by_region": 4, "num_undelivered_messages": 0, "oldest_retained_acked_message_age": 0, "oldest_retained_acked_message_age_by_region": 0, "oldest_unacked_message_age": 0, - "oldest_unacked_message_age_by_region": 362, + "oldest_unacked_message_age_by_region": 69277, "retained_acked_bytes": 0, "retained_acked_bytes_by_region": 0, "unacked_bytes_by_region": 19 }, "topic": { - "num_retained_acked_messages_by_region": 0, - "num_unacked_messages_by_region": 16, "oldest_retained_acked_message_age_by_region": 0, - "oldest_unacked_message_age_by_region": 389, + "oldest_unacked_message_age_by_region": 69319, "retained_acked_bytes_by_region": 0, "unacked_bytes_by_region": 76 } diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc index 9fa3ea4f50c..1e012f42293 100644 --- a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc @@ -1,25 +1,121 @@ PubSub Metricset to fetch metrics from https://cloud.google.com/pubsub/[Pub/Sub] topics and subscriptions in Google Cloud Platform. -The `pubsub` Metricset contains all metrics exported from the https://cloud.google.com/monitoring/api/metrics_gcp#gcp-pubsub[Stackdriver API]. The field names have been left untouched for people already familiar with them. +The `pubsub` Metricset contains all GA stage metrics exported from the https://cloud.google.com/monitoring/api/metrics_gcp#gcp-pubsub[Stackdriver API]. The field names have been left untouched for people already familiar with them. No special permissions are needed apart from the ones detailed in the module section of the docs. +[float] +==== Snapshot Metrics +- `pubsub.snapshot.backlog_bytes`: Total byte size of the messages retained in a snapshot. +- `pubsub.snapshot.backlog_bytes_by_region`: Total byte size of the messages retained in a snapshot, broken down by Cloud region. +- `pubsub.snapshot.config_updates_count`: Cumulative count of configuration changes, grouped by operation type and result. +- `pubsub.snapshot.num_messages`: Number of messages retained in a snapshot. Sampled every 60 seconds. +- `pubsub.snapshot.num_messages_by_region`: Number of messages retained in a snapshot, broken down by Cloud region. +- `pubsub.snapshot.oldest_message_age`: Age (in seconds) of the oldest message retained in a snapshot. +- `pubsub.snapshot.oldest_message_age_by_region`: Age (in seconds) of the oldest message retained in a snapshot, broken down by Cloud region. + +[float] +==== Subscription Metrics +- `pubsub.subscription.ack_message_count`: Cumulative count of messages acknowledged by Acknowledge requests, grouped by delivery type. +- `pubsub.subscription.backlog_bytes`: Total byte size of the unacknowledged messages (a.k.a. backlog messages) in a subscription. +- `pubsub.subscription.byte_cost`: Cumulative cost of operations, measured in bytes. This is used to measure quota utilization. +- `pubsub.subscription.config_updates_count`: Cumulative count of configuration changes for each subscription, grouped by operation type and result. +- `pubsub.subscription.dead_letter_message_count`: Cumulative count of messages published to dead letter topic, grouped by result. +- `pubsub.subscription.mod_ack_deadline_message_count`: Cumulative count of messages whose deadline was updated by ModifyAckDeadline requests, grouped by delivery type. +- `pubsub.subscription.mod_ack_deadline_message_operation_count`: Cumulative count of ModifyAckDeadline message operations, grouped by result. +- `pubsub.subscription.mod_ack_deadline_request_count`: Cumulative count of ModifyAckDeadline requests, grouped by result. +- `pubsub.subscription.num_outstanding_messages`: Number of messages delivered to a subscription's push endpoint, but not yet acknowledged. +- `pubsub.subscription.num_undelivered_messages`: Number of unacknowledged messages (a.k.a. backlog messages) in a subscription. +- `pubsub.subscription.oldest_retained_acked_message_age`: Age (in seconds) of the oldest acknowledged message retained in a subscription. +- `pubsub.subscription.oldest_retained_acked_message_age_by_region`: Age (in seconds) of the oldest acknowledged message retained in a subscription, broken down by Cloud region. +- `pubsub.subscription.oldest_unacked_message_age`: Age (in seconds) of the oldest unacknowledged message (a.k.a. backlog message) in a subscription. +- `pubsub.subscription.oldest_unacked_message_age_by_region`: Age (in seconds) of the oldest unacknowledged message in a subscription, broken down by Cloud region. +- `pubsub.subscription.pull_ack_message_operation_count`: Cumulative count of acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. +- `pubsub.subscription.pull_ack_request_count`: Cumulative count of acknowledge requests, grouped by result. +- `pubsub.subscription.pull_message_operation_count`: Cumulative count of pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. +- `pubsub.subscription.pull_request_count`: Cumulative count of pull requests, grouped by result. +- `pubsub.subscription.push_request_count`: Cumulative count of push attempts, grouped by result. Unlike pulls, the push server implementation does not batch user messages. So each request only contains one user message. The push server retries on errors, so a given user message can appear multiple times. +- `pubsub.subscription.push_request_latencies`: Distribution of push request latencies (in microseconds), grouped by result. +- `pubsub.subscription.retained_acked_bytes`: otal byte size of the acknowledged messages retained in a subscription. +- `pubsub.subscription.retained_acked_bytes_by_region`: Total byte size of the acknowledged messages retained in a subscription, broken down by Cloud region. +- `pubsub.subscription.seek_request_count`: Cumulative count of seek attempts, grouped by result. +- `pubsub.subscription.sent_message_count`: Cumulative count of messages sent by Cloud Pub/Sub to subscriber clients, grouped by delivery type. +- `pubsub.subscription.streaming_pull_ack_message_operation_count`: Cumulative count of StreamingPull acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. +- `pubsub.subscription.streaming_pull_ack_request_count`: Cumulative count of streaming pull requests with non-empty acknowledge ids, grouped by result. +- `pubsub.subscription.streaming_pull_message_operation_count`: Cumulative count of streaming pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count +- `pubsub.subscription.streaming_pull_mod_ack_deadline_message_operation_count`: Cumulative count of StreamingPull ModifyAckDeadline operations, grouped by result. +- `pubsub.subscription.streaming_pull_mod_ack_deadline_request_count`: Cumulative count of streaming pull requests with non-empty ModifyAckDeadline fields, grouped by result. +- `pubsub.subscription.streaming_pull_response_count`: Cumulative count of streaming pull responses, grouped by result. +- `pubsub.subscription.unacked_bytes_by_region`: Total byte size of the unacknowledged messages in a subscription, broken down by Cloud region. + +[float] +==== Topic Metrics +- `pubsub.topic.byte_cost`: Cost of operations, measured in bytes. This is used to measure utilization for quotas. +- `pubsub.topic.config_updates_count`: Cumulative count of configuration changes, grouped by operation type and result. +- `pubsub.topic.message_sizes`: Distribution of publish message sizes (in bytes). +- `pubsub.topic.oldest_retained_acked_message_age_by_region`: Age (in seconds) of the oldest acknowledged message retained in a topic, broken down by Cloud region. +- `pubsub.topic.oldest_unacked_message_age_by_region`: Age (in seconds) of the oldest unacknowledged message in a topic, broken down by Cloud region. +- `pubsub.topic.retained_acked_bytes_by_region`: Total byte size of the acknowledged messages retained in a topic, broken down by Cloud region. +- `pubsub.topic.send_message_operation_count`: Cumulative count of publish message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. +- `pubsub.topic.send_request_count`: Cumulative count of publish requests, grouped by result. +- `pubsub.topic.streaming_pull_responsPubSub Metricset to fetch metrics from https://cloud.google.com/pubsub/[Pub/Sub] topics and subscriptions in Google Cloud Platform. + +The `pubsub` Metricset contains all GA stage metrics exported from the https://cloud.google.com/monitoring/api/metrics_gcp#gcp-pubsub[Stackdriver API]. The field names have been left untouched for people already familiar with them. + +No special permissions are needed apart from the ones detailed in the module section of the docs. + +[float] +==== Snapshot Metrics +- `pubsub.snapshot.backlog_bytes`: Total byte size of the messages retained in a snapshot. +- `pubsub.snapshot.backlog_bytes_by_region`: Total byte size of the messages retained in a snapshot, broken down by Cloud region. +- `pubsub.snapshot.config_updates_count`: Cumulative count of configuration changes, grouped by operation type and result. +- `pubsub.snapshot.num_messages`: Number of messages retained in a snapshot. Sampled every 60 seconds. +- `pubsub.snapshot.num_messages_by_region`: Number of messages retained in a snapshot, broken down by Cloud region. +- `pubsub.snapshot.oldest_message_age`: Age (in seconds) of the oldest message retained in a snapshot. +- `pubsub.snapshot.oldest_message_age_by_region`: Age (in seconds) of the oldest message retained in a snapshot, broken down by Cloud region. + +[float] +==== Subscription Metrics - `pubsub.subscription.ack_message_count`: Cumulative count of messages acknowledged by Acknowledge requests, grouped by delivery type. - `pubsub.subscription.backlog_bytes`: Total byte size of the unacknowledged messages (a.k.a. backlog messages) in a subscription. +- `pubsub.subscription.byte_cost`: Cumulative cost of operations, measured in bytes. This is used to measure quota utilization. +- `pubsub.subscription.config_updates_count`: Cumulative count of configuration changes for each subscription, grouped by operation type and result. +- `pubsub.subscription.dead_letter_message_count`: Cumulative count of messages published to dead letter topic, grouped by result. +- `pubsub.subscription.mod_ack_deadline_message_count`: Cumulative count of messages whose deadline was updated by ModifyAckDeadline requests, grouped by delivery type. +- `pubsub.subscription.mod_ack_deadline_message_operation_count`: Cumulative count of ModifyAckDeadline message operations, grouped by result. +- `pubsub.subscription.mod_ack_deadline_request_count`: Cumulative count of ModifyAckDeadline requests, grouped by result. - `pubsub.subscription.num_outstanding_messages`: Number of messages delivered to a subscription's push endpoint, but not yet acknowledged. - `pubsub.subscription.num_undelivered_messages`: Number of unacknowledged messages (a.k.a. backlog messages) in a subscription. +- `pubsub.subscription.oldest_retained_acked_message_age`: Age (in seconds) of the oldest acknowledged message retained in a subscription. +- `pubsub.subscription.oldest_retained_acked_message_age_by_region`: Age (in seconds) of the oldest acknowledged message retained in a subscription, broken down by Cloud region. - `pubsub.subscription.oldest_unacked_message_age`: Age (in seconds) of the oldest unacknowledged message (a.k.a. backlog message) in a subscription. +- `pubsub.subscription.oldest_unacked_message_age_by_region`: Age (in seconds) of the oldest unacknowledged message in a subscription, broken down by Cloud region. - `pubsub.subscription.pull_ack_message_operation_count`: Cumulative count of acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. - `pubsub.subscription.pull_ack_request_count`: Cumulative count of acknowledge requests, grouped by result. - `pubsub.subscription.pull_message_operation_count`: Cumulative count of pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. - `pubsub.subscription.pull_request_count`: Cumulative count of pull requests, grouped by result. - `pubsub.subscription.push_request_count`: Cumulative count of push attempts, grouped by result. Unlike pulls, the push server implementation does not batch user messages. So each request only contains one user message. The push server retries on errors, so a given user message can appear multiple times. - `pubsub.subscription.push_request_latencies`: Distribution of push request latencies (in microseconds), grouped by result. +- `pubsub.subscription.retained_acked_bytes`: otal byte size of the acknowledged messages retained in a subscription. +- `pubsub.subscription.retained_acked_bytes_by_region`: Total byte size of the acknowledged messages retained in a subscription, broken down by Cloud region. +- `pubsub.subscription.seek_request_count`: Cumulative count of seek attempts, grouped by result. - `pubsub.subscription.sent_message_count`: Cumulative count of messages sent by Cloud Pub/Sub to subscriber clients, grouped by delivery type. - `pubsub.subscription.streaming_pull_ack_message_operation_count`: Cumulative count of StreamingPull acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. - `pubsub.subscription.streaming_pull_ack_request_count`: Cumulative count of streaming pull requests with non-empty acknowledge ids, grouped by result. - `pubsub.subscription.streaming_pull_message_operation_count`: Cumulative count of streaming pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count +- `pubsub.subscription.streaming_pull_mod_ack_deadline_message_operation_count`: Cumulative count of StreamingPull ModifyAckDeadline operations, grouped by result. +- `pubsub.subscription.streaming_pull_mod_ack_deadline_request_count`: Cumulative count of streaming pull requests with non-empty ModifyAckDeadline fields, grouped by result. - `pubsub.subscription.streaming_pull_response_count`: Cumulative count of streaming pull responses, grouped by result. -- `pubsub.topic.streaming_pull_response_count`: Cumulative count of streaming pull responses, grouped by result. +- `pubsub.subscription.unacked_bytes_by_region`: Total byte size of the unacknowledged messages in a subscription, broken down by Cloud region. + +[float] +==== Topic Metrics +- `pubsub.topic.byte_cost`: Cost of operations, measured in bytes. This is used to measure utilization for quotas. +- `pubsub.topic.config_updates_count`: Cumulative count of configuration changes, grouped by operation type and result. +- `pubsub.topic.message_sizes`: Distribution of publish message sizes (in bytes). +- `pubsub.topic.oldest_retained_acked_message_age_by_region`: Age (in seconds) of the oldest acknowledged message retained in a topic, broken down by Cloud region. +- `pubsub.topic.oldest_unacked_message_age_by_region`: Age (in seconds) of the oldest unacknowledged message in a topic, broken down by Cloud region. +- `pubsub.topic.retained_acked_bytes_by_region`: Total byte size of the acknowledged messages retained in a topic, broken down by Cloud region. - `pubsub.topic.send_message_operation_count`: Cumulative count of publish message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. - `pubsub.topic.send_request_count`: Cumulative count of publish requests, grouped by result. +- `pubsub.topic.unacked_bytes_by_region`: Total byte size of the unacknowledged messages in a topic, broken down by Cloud region. diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml index 4e05ae476cf..32e4acbb521 100644 --- a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml @@ -82,12 +82,6 @@ - name: retained_acked_bytes_by_region type: long description: Total byte size of the acknowledged messages retained in a subscription, broken down by Cloud region. - - name: num_retained_acked_messages - type: long - description: Number of acknowledged messages retained in a subscription. - - name: num_retained_acked_messages_by_region - type: long - description: Number of acknowledged messages retained in a subscription, broken down by Cloud region. - name: seek_request_count type: long description: Cumulative count of seek attempts, grouped by result. @@ -106,9 +100,6 @@ - name: unacked_bytes_by_region type: long description: Total byte size of the unacknowledged messages in a subscription, broken down by Cloud region. - - name: num_unacked_messages_by_region - type: long - description: Number of unacknowledged messages in a subscription, broken down by Cloud region. - name: topic type: group description: Topic related metrics @@ -131,9 +122,6 @@ - name: retained_acked_bytes_by_region type: long description: Total byte size of the acknowledged messages retained in a topic, broken down by Cloud region. - - name: num_retained_acked_messages_by_region - type: long - description: Number of acknowledged messages retained in a topic, broken down by Cloud region. - name: byte_cost type: long description: Cost of operations, measured in bytes. This is used to measure utilization for quotas. @@ -143,9 +131,6 @@ - name: unacked_bytes_by_region type: long description: Total byte size of the unacknowledged messages in a topic, broken down by Cloud region. - - name: num_unacked_messages_by_region - type: long - description: Number of unacknowledged messages in a topic, broken down by Cloud region. - name: snapshot type: group description: Snapshot related metrics diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml b/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml index 307879450ff..3d8cdb0949c 100644 --- a/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml +++ b/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml @@ -6,55 +6,50 @@ input: stackdriver: service: pubsub metrics: + - "pubsub.googleapis.com/snapshot/backlog_bytes" + - "pubsub.googleapis.com/snapshot/backlog_bytes_by_region" + - "pubsub.googleapis.com/snapshot/config_updates_count" + - "pubsub.googleapis.com/snapshot/num_messages" + - "pubsub.googleapis.com/snapshot/num_messages_by_region" + - "pubsub.googleapis.com/snapshot/oldest_message_age" + - "pubsub.googleapis.com/snapshot/oldest_message_age_by_region" - "pubsub.googleapis.com/subscription/ack_message_count" - "pubsub.googleapis.com/subscription/backlog_bytes" + - "pubsub.googleapis.com/subscription/byte_cost" + - "pubsub.googleapis.com/subscription/config_updates_count" + - "pubsub.googleapis.com/subscription/dead_letter_message_count" + - "pubsub.googleapis.com/subscription/mod_ack_deadline_message_count" + - "pubsub.googleapis.com/subscription/mod_ack_deadline_message_operation_count" + - "pubsub.googleapis.com/subscription/mod_ack_deadline_request_count" - "pubsub.googleapis.com/subscription/num_outstanding_messages" - "pubsub.googleapis.com/subscription/num_undelivered_messages" + - "pubsub.googleapis.com/subscription/oldest_retained_acked_message_age" + - "pubsub.googleapis.com/subscription/oldest_retained_acked_message_age_by_region" - "pubsub.googleapis.com/subscription/oldest_unacked_message_age" + - "pubsub.googleapis.com/subscription/oldest_unacked_message_age_by_region" - "pubsub.googleapis.com/subscription/pull_ack_message_operation_count" - "pubsub.googleapis.com/subscription/pull_ack_request_count" - "pubsub.googleapis.com/subscription/pull_message_operation_count" - "pubsub.googleapis.com/subscription/pull_request_count" - "pubsub.googleapis.com/subscription/push_request_count" - "pubsub.googleapis.com/subscription/push_request_latencies" + - "pubsub.googleapis.com/subscription/retained_acked_bytes" + - "pubsub.googleapis.com/subscription/retained_acked_bytes_by_region" + - "pubsub.googleapis.com/subscription/seek_request_count" - "pubsub.googleapis.com/subscription/sent_message_count" - "pubsub.googleapis.com/subscription/streaming_pull_ack_message_operation_count" - "pubsub.googleapis.com/subscription/streaming_pull_ack_request_count" - "pubsub.googleapis.com/subscription/streaming_pull_message_operation_count" - - "pubsub.googleapis.com/subscription/streaming_pull_response_count" - - "pubsub.googleapis.com/subscription/dead_letter_message_count" - - "pubsub.googleapis.com/subscription/mod_ack_deadline_message_count" - - "pubsub.googleapis.com/subscription/mod_ack_deadline_message_operation_count" - - "pubsub.googleapis.com/subscription/mod_ack_deadline_request_count" - - "pubsub.googleapis.com/subscription/oldest_retained_acked_message_age" - - "pubsub.googleapis.com/subscription/oldest_retained_acked_message_age_by_region" - - "pubsub.googleapis.com/subscription/oldest_unacked_message_age_by_region" - - "pubsub.googleapis.com/subscription/retained_acked_bytes" - - "pubsub.googleapis.com/subscription/retained_acked_bytes_by_region" - - "pubsub.googleapis.com/subscription/num_retained_acked_messages" - - "pubsub.googleapis.com/subscription/num_retained_acked_messages_by_region" - - "pubsub.googleapis.com/subscription/seek_request_count" - "pubsub.googleapis.com/subscription/streaming_pull_mod_ack_deadline_message_operation_count" - "pubsub.googleapis.com/subscription/streaming_pull_mod_ack_deadline_request_count" - - "pubsub.googleapis.com/subscription/byte_cost" - - "pubsub.googleapis.com/subscription/config_updates_count" + - "pubsub.googleapis.com/subscription/streaming_pull_response_count" - "pubsub.googleapis.com/subscription/unacked_bytes_by_region" - - "pubsub.googleapis.com/subscription/num_unacked_messages_by_region" + - "pubsub.googleapis.com/topic/byte_cost" + - "pubsub.googleapis.com/topic/config_updates_count" - "pubsub.googleapis.com/topic/message_sizes" - - "pubsub.googleapis.com/topic/send_message_operation_count" - - "pubsub.googleapis.com/topic/send_request_count" - "pubsub.googleapis.com/topic/oldest_retained_acked_message_age_by_region" - "pubsub.googleapis.com/topic/oldest_unacked_message_age_by_region" - "pubsub.googleapis.com/topic/retained_acked_bytes_by_region" - - "pubsub.googleapis.com/topic/num_retained_acked_messages_by_region" - - "pubsub.googleapis.com/topic/byte_cost" - - "pubsub.googleapis.com/topic/config_updates_count" + - "pubsub.googleapis.com/topic/send_message_operation_count" + - "pubsub.googleapis.com/topic/send_request_count" - "pubsub.googleapis.com/topic/unacked_bytes_by_region" - - "pubsub.googleapis.com/topic/num_unacked_messages_by_region" - - "pubsub.googleapis.com/snapshot/oldest_message_age" - - "pubsub.googleapis.com/snapshot/oldest_message_age_by_region" - - "pubsub.googleapis.com/snapshot/backlog_bytes" - - "pubsub.googleapis.com/snapshot/backlog_bytes_by_region" - - "pubsub.googleapis.com/snapshot/num_messages" - - "pubsub.googleapis.com/snapshot/num_messages_by_region" - - "pubsub.googleapis.com/snapshot/config_updates_count" From ab90fbd349d4b3f0d380c5b32febca86736611aa Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 8 Apr 2020 12:21:25 -0600 Subject: [PATCH 4/5] add changelog entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8726aea0113..7d3aed28d8d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -150,6 +150,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix Unix socket path in memcached. {pull}17512[17512] - Fix vsphere VM dashboard host aggregation visualizations. {pull}17555[17555] - Metricbeat no longer needs to be started strictly after Logstash for `logstash-xpack` module to report correct data. {issue}17261[17261] {pull}17497[17497] +- Fix pubsub metricset to collect all GA stage metrics from gcp stackdriver. {issue}17154[17154] {pull}17600[17600] *Packetbeat* From 224bbe04839a775a4746fb413572aca855b7c80c Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 9 Apr 2020 09:27:19 -0600 Subject: [PATCH 5/5] fix docs.asciidoc for pubsub --- .../googlecloud/pubsub/_meta/docs.asciidoc | 60 ------------------- 1 file changed, 60 deletions(-) diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc index 1e012f42293..cd6e94083ef 100644 --- a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc @@ -48,66 +48,6 @@ No special permissions are needed apart from the ones detailed in the module sec - `pubsub.subscription.streaming_pull_response_count`: Cumulative count of streaming pull responses, grouped by result. - `pubsub.subscription.unacked_bytes_by_region`: Total byte size of the unacknowledged messages in a subscription, broken down by Cloud region. -[float] -==== Topic Metrics -- `pubsub.topic.byte_cost`: Cost of operations, measured in bytes. This is used to measure utilization for quotas. -- `pubsub.topic.config_updates_count`: Cumulative count of configuration changes, grouped by operation type and result. -- `pubsub.topic.message_sizes`: Distribution of publish message sizes (in bytes). -- `pubsub.topic.oldest_retained_acked_message_age_by_region`: Age (in seconds) of the oldest acknowledged message retained in a topic, broken down by Cloud region. -- `pubsub.topic.oldest_unacked_message_age_by_region`: Age (in seconds) of the oldest unacknowledged message in a topic, broken down by Cloud region. -- `pubsub.topic.retained_acked_bytes_by_region`: Total byte size of the acknowledged messages retained in a topic, broken down by Cloud region. -- `pubsub.topic.send_message_operation_count`: Cumulative count of publish message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. -- `pubsub.topic.send_request_count`: Cumulative count of publish requests, grouped by result. -- `pubsub.topic.streaming_pull_responsPubSub Metricset to fetch metrics from https://cloud.google.com/pubsub/[Pub/Sub] topics and subscriptions in Google Cloud Platform. - -The `pubsub` Metricset contains all GA stage metrics exported from the https://cloud.google.com/monitoring/api/metrics_gcp#gcp-pubsub[Stackdriver API]. The field names have been left untouched for people already familiar with them. - -No special permissions are needed apart from the ones detailed in the module section of the docs. - -[float] -==== Snapshot Metrics -- `pubsub.snapshot.backlog_bytes`: Total byte size of the messages retained in a snapshot. -- `pubsub.snapshot.backlog_bytes_by_region`: Total byte size of the messages retained in a snapshot, broken down by Cloud region. -- `pubsub.snapshot.config_updates_count`: Cumulative count of configuration changes, grouped by operation type and result. -- `pubsub.snapshot.num_messages`: Number of messages retained in a snapshot. Sampled every 60 seconds. -- `pubsub.snapshot.num_messages_by_region`: Number of messages retained in a snapshot, broken down by Cloud region. -- `pubsub.snapshot.oldest_message_age`: Age (in seconds) of the oldest message retained in a snapshot. -- `pubsub.snapshot.oldest_message_age_by_region`: Age (in seconds) of the oldest message retained in a snapshot, broken down by Cloud region. - -[float] -==== Subscription Metrics -- `pubsub.subscription.ack_message_count`: Cumulative count of messages acknowledged by Acknowledge requests, grouped by delivery type. -- `pubsub.subscription.backlog_bytes`: Total byte size of the unacknowledged messages (a.k.a. backlog messages) in a subscription. -- `pubsub.subscription.byte_cost`: Cumulative cost of operations, measured in bytes. This is used to measure quota utilization. -- `pubsub.subscription.config_updates_count`: Cumulative count of configuration changes for each subscription, grouped by operation type and result. -- `pubsub.subscription.dead_letter_message_count`: Cumulative count of messages published to dead letter topic, grouped by result. -- `pubsub.subscription.mod_ack_deadline_message_count`: Cumulative count of messages whose deadline was updated by ModifyAckDeadline requests, grouped by delivery type. -- `pubsub.subscription.mod_ack_deadline_message_operation_count`: Cumulative count of ModifyAckDeadline message operations, grouped by result. -- `pubsub.subscription.mod_ack_deadline_request_count`: Cumulative count of ModifyAckDeadline requests, grouped by result. -- `pubsub.subscription.num_outstanding_messages`: Number of messages delivered to a subscription's push endpoint, but not yet acknowledged. -- `pubsub.subscription.num_undelivered_messages`: Number of unacknowledged messages (a.k.a. backlog messages) in a subscription. -- `pubsub.subscription.oldest_retained_acked_message_age`: Age (in seconds) of the oldest acknowledged message retained in a subscription. -- `pubsub.subscription.oldest_retained_acked_message_age_by_region`: Age (in seconds) of the oldest acknowledged message retained in a subscription, broken down by Cloud region. -- `pubsub.subscription.oldest_unacked_message_age`: Age (in seconds) of the oldest unacknowledged message (a.k.a. backlog message) in a subscription. -- `pubsub.subscription.oldest_unacked_message_age_by_region`: Age (in seconds) of the oldest unacknowledged message in a subscription, broken down by Cloud region. -- `pubsub.subscription.pull_ack_message_operation_count`: Cumulative count of acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. -- `pubsub.subscription.pull_ack_request_count`: Cumulative count of acknowledge requests, grouped by result. -- `pubsub.subscription.pull_message_operation_count`: Cumulative count of pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. -- `pubsub.subscription.pull_request_count`: Cumulative count of pull requests, grouped by result. -- `pubsub.subscription.push_request_count`: Cumulative count of push attempts, grouped by result. Unlike pulls, the push server implementation does not batch user messages. So each request only contains one user message. The push server retries on errors, so a given user message can appear multiple times. -- `pubsub.subscription.push_request_latencies`: Distribution of push request latencies (in microseconds), grouped by result. -- `pubsub.subscription.retained_acked_bytes`: otal byte size of the acknowledged messages retained in a subscription. -- `pubsub.subscription.retained_acked_bytes_by_region`: Total byte size of the acknowledged messages retained in a subscription, broken down by Cloud region. -- `pubsub.subscription.seek_request_count`: Cumulative count of seek attempts, grouped by result. -- `pubsub.subscription.sent_message_count`: Cumulative count of messages sent by Cloud Pub/Sub to subscriber clients, grouped by delivery type. -- `pubsub.subscription.streaming_pull_ack_message_operation_count`: Cumulative count of StreamingPull acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. -- `pubsub.subscription.streaming_pull_ack_request_count`: Cumulative count of streaming pull requests with non-empty acknowledge ids, grouped by result. -- `pubsub.subscription.streaming_pull_message_operation_count`: Cumulative count of streaming pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count -- `pubsub.subscription.streaming_pull_mod_ack_deadline_message_operation_count`: Cumulative count of StreamingPull ModifyAckDeadline operations, grouped by result. -- `pubsub.subscription.streaming_pull_mod_ack_deadline_request_count`: Cumulative count of streaming pull requests with non-empty ModifyAckDeadline fields, grouped by result. -- `pubsub.subscription.streaming_pull_response_count`: Cumulative count of streaming pull responses, grouped by result. -- `pubsub.subscription.unacked_bytes_by_region`: Total byte size of the unacknowledged messages in a subscription, broken down by Cloud region. - [float] ==== Topic Metrics - `pubsub.topic.byte_cost`: Cost of operations, measured in bytes. This is used to measure utilization for quotas.