diff --git a/.circleci/checksum.sh b/.circleci/checksum.sh index af2e0f293e..09d820d1cc 100644 --- a/.circleci/checksum.sh +++ b/.circleci/checksum.sh @@ -21,6 +21,6 @@ fi openssl md5 package.json >> $FILE -find packages/*/package.json | xargs -I{} openssl md5 {} >> $FILE +find plugins/node/*/package.json && find plugins/web/*/package.json && find packages/*/package.json | xargs -I{} openssl md5 {} >> $FILE sort -o $FILE $FILE diff --git a/.circleci/config.yml b/.circleci/config.yml index ef65339723..fbe23a4cd8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,6 +54,9 @@ cache_1: &cache_1 - plugins/node/opentelemetry-plugin-mysql/node_modules - plugins/node/opentelemetry-plugin-express/node_modules - propagators/opentelemetry-propagator-jaeger/node_modules + - propagators/opentelemetry-propagator-grpc-census-binary/node_modules + - packages/opentelemetry-rca-metrics/node_modules + - packages/opentelemetry-test-utils/node_modules node_unit_tests: &node_unit_tests resource_class: large @@ -74,7 +77,7 @@ node_unit_tests: &node_unit_tests echo "CIRCLE_NODE_VERSION=${CIRCLE_NODE_VERSION}" - restore_cache: keys: - - npm-cache-01-{{ .Environment.CIRCLE_JOB }}-{{ checksum "/tmp/checksums.txt" }}-F267A71D + - npm-cache-01-{{ .Environment.CIRCLE_JOB }}-{{ checksum "/tmp/checksums.txt" }} - run: name: Install Root Dependencies command: npm install --ignore-scripts @@ -109,7 +112,7 @@ browsers_unit_tests: &browsers_unit_tests echo "CIRCLE_NODE_VERSION=${CIRCLE_NODE_VERSION}" - restore_cache: keys: - - npm-cache-01-{{ .Environment.CIRCLE_JOB }}-{{ checksum "/tmp/checksums.txt" }}-F267A71D + - npm-cache-01-{{ .Environment.CIRCLE_JOB }}-{{ checksum "/tmp/checksums.txt" }} - run: name: Install Root Dependencies command: npm install --ignore-scripts @@ -125,6 +128,40 @@ browsers_unit_tests: &browsers_unit_tests name: report coverage command: if [ "$CIRCLE_NODE_VERSION" = "v12" ]; then npm run codecov:browser; fi +build_metrics: &build_metrics + resource_class: large + steps: + - checkout + - run: + name: Create Checksum + command: sh .circleci/checksum.sh /tmp/checksums.txt + - run: + name: Setup environment variables + command: | + echo "export CIRCLE_NODE_VERSION=\$(node --version | grep -oE 'v[0-9]+')" >> $BASH_ENV + source $BASH_ENV + - run: + name: Log out node.js version + command: | + node --version + echo "CIRCLE_NODE_VERSION=${CIRCLE_NODE_VERSION}" + - restore_cache: + keys: + - npm-cache-01-{{ .Environment.CIRCLE_JOB }}-{{ checksum "/tmp/checksums.txt" }} + - run: + name: Install Root Dependencies + command: npm install --ignore-scripts + - run: + name: Boostrap dependencies + command: npx lerna bootstrap --scope @opentelemetry/rca-metrics --include-filtered-dependencies --ignore-scripts + - run: + name: Create prebuilds for native stats + command: npm run build:rca-metrics + - store_artifacts: + path: ./packages/opentelemetry-rca-metrics/artifacts/prebuilds.tgz + - store_artifacts: + path: ./packages/opentelemetry-rca-metrics/artifacts/prebuilds.tgz.sha1 + jobs: node8: docker: @@ -157,6 +194,11 @@ jobs: docker: - image: circleci/node:12-browsers <<: *browsers_unit_tests + build-native-stats: + docker: + - image: node:14 + environment: *node_test_env + <<: *build_metrics workflows: version: 2 @@ -166,4 +208,8 @@ workflows: - node10 - node12 - node12-browsers - + - build-native-stats: + filters: + branches: + only: + - master diff --git a/.gitignore b/.gitignore index ee4e11fb4b..8738ce6646 100644 --- a/.gitignore +++ b/.gitignore @@ -81,4 +81,3 @@ package.json.lerna_backup *.iml .idea - diff --git a/.nycrc b/.nycrc index f8e60152fa..0238b48d74 100644 --- a/.nycrc +++ b/.nycrc @@ -9,7 +9,10 @@ "karma.conf.js", "src/platform/browser/*.ts", "test/index-webpack.ts", - "webpack/*.js" + "webpack/*.js", + "packages/opentelemetry-rca-metrics/scripts/*.js", + ".eslintrc.js", + "version.ts" ], "all": true } diff --git a/codecov.yml b/codecov.yml index 1beb17b8de..ef9d36d826 100644 --- a/codecov.yml +++ b/codecov.yml @@ -13,3 +13,5 @@ coverage: default: target: auto threshold: 1% +ignore: + - "packages/opentelemetry-rca-metrics/scripts" diff --git a/examples/rca-metrics/node.js b/examples/rca-metrics/node.js new file mode 100644 index 0000000000..90b1b002b8 --- /dev/null +++ b/examples/rca-metrics/node.js @@ -0,0 +1,19 @@ +'use strict'; + +const { RCAMetrics } = require('@opentelemetry/rca-metrics'); +const { PrometheusExporter } = require('@opentelemetry/exporter-prometheus'); + +const exporter = new PrometheusExporter( + { + startServer: true, + }, + () => { + console.log('prometheus scrape endpoint: http://localhost:9464/metrics'); + }, +); + +const rcaMetrics = new RCAMetrics({ + exporter, + interval: 2000, +}); +rcaMetrics.start(); diff --git a/examples/rca-metrics/package.json b/examples/rca-metrics/package.json new file mode 100644 index 0000000000..9aa8426d40 --- /dev/null +++ b/examples/rca-metrics/package.json @@ -0,0 +1,36 @@ +{ + "name": "rca-metrics-example", + "private": true, + "version": "0.9.0", + "description": "Example of using @opentelemetry/rca-metrics", + "main": "index.js", + "scripts": { + "start": "node node.js" + }, + "repository": { + "type": "git", + "url": "git+ssh://git@github.com/open-telemetry/opentelemetry-js-contrib.git" + }, + "keywords": [ + "opentelemetry", + "http", + "tracing", + "metrics" + ], + "engines": { + "node": ">=8" + }, + "author": "OpenTelemetry Authors", + "license": "Apache-2.0", + "bugs": { + "url": "https://github.com/open-telemetry/opentelemetry-js/issues" + }, + "dependencies": { + "@opentelemetry/api": "^0.10.2", + "@opentelemetry/core": "^0.10.2", + "@opentelemetry/exporter-prometheus": "^0.10.2", + "@opentelemetry/metrics": "^0.10.2", + "@opentelemetry/rca-metrics": "^0.9.0" + }, + "homepage": "https://github.com/open-telemetry/opentelemetry-js#readme" +} diff --git a/package.json b/package.json index 34a0bdd33e..510d2c0fc7 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "precompile": "tsc --version", "version:update": "lerna run version:update", "compile": "lerna run compile", + "build:rca-metrics": "lerna run build:rca-metrics", "test": "lerna run test", "test:browser": "lerna run test:browser", "bootstrap": "lerna bootstrap", diff --git a/packages/opentelemetry-rca-metrics/.eslintignore b/packages/opentelemetry-rca-metrics/.eslintignore new file mode 100644 index 0000000000..378eac25d3 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/.eslintignore @@ -0,0 +1 @@ +build diff --git a/packages/opentelemetry-rca-metrics/.eslintrc.js b/packages/opentelemetry-rca-metrics/.eslintrc.js new file mode 100644 index 0000000000..f726f3becb --- /dev/null +++ b/packages/opentelemetry-rca-metrics/.eslintrc.js @@ -0,0 +1,7 @@ +module.exports = { + "env": { + "mocha": true, + "node": true + }, + ...require('../../eslint.config.js') +} diff --git a/packages/opentelemetry-rca-metrics/.gitignore b/packages/opentelemetry-rca-metrics/.gitignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/opentelemetry-rca-metrics/.npmignore b/packages/opentelemetry-rca-metrics/.npmignore new file mode 100644 index 0000000000..9505ba9450 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/.npmignore @@ -0,0 +1,4 @@ +/bin +/coverage +/doc +/test diff --git a/packages/opentelemetry-rca-metrics/LICENSE b/packages/opentelemetry-rca-metrics/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/packages/opentelemetry-rca-metrics/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/opentelemetry-rca-metrics/README.md b/packages/opentelemetry-rca-metrics/README.md new file mode 100644 index 0000000000..d4823352e8 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/README.md @@ -0,0 +1,71 @@ +#OpenTelemetry RCA Metrics for Node.js +[![Gitter chat][gitter-image]][gitter-url] +[![NPM Published Version][npm-img]][npm-url] +[![dependencies][dependencies-image]][dependencies-url] +[![devDependencies][devDependencies-image]][devDependencies-url] +[![Apache License][license-image]][license-url] + +This module provides automatic collection of RCA Metrics + +## Installation + +```bash +npm install --save @opentelemetry/rca-metrics +``` + +## Usage + +```javascript +const { RCAMetrics } = require('@opentelemetry/rca-metrics'); +const { PrometheusExporter } = require('@opentelemetry/exporter-prometheus'); + +const exporter = new PrometheusExporter( + { startServer: true },() => { + console.log('prometheus scrape endpoint: http://localhost:9464/metrics'); + } +); + +const rcaMetrics = new RCAMetrics({ + exporter, + interval: 5000, // default 60000 (60s) +}); +rcaMetrics.start(); +``` + +### Install native stats for active platform +```shell script +npm run build:install +``` + +### Install native stats for custom platform arch and version +```shell script +#for example +npm run build:install platform=darwin arch=x64 version=10 +npm run build:install platform=win32 arch=x64 version=10 +``` + +### Build native stats for all platforms (osx, linux, windows) - this is done on circleci only - you should not need that +```shell script +npm run build:all +``` + +## Useful links + +- For more information on OpenTelemetry, visit: +- For more about OpenTelemetry JavaScript: +- For help or feedback on this project, join us on [gitter][gitter-url] + +## License + +APACHE 2.0 - See [LICENSE][license-url] for more information. + +[gitter-image]: https://badges.gitter.im/open-telemetry/opentelemetry-js-contrib.svg +[gitter-url]: https://gitter.im/open-telemetry/opentelemetry-node?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge +[license-url]: https://github.com/open-telemetry/opentelemetry-js-contrib/blob/master/LICENSE +[license-image]: https://img.shields.io/badge/license-Apache_2.0-green.svg?style=flat +[dependencies-image]: https://david-dm.org/open-telemetry/opentelemetry-js-contrib.svg?path=packages%2Fopentelemetry-rca-metrics +[dependencies-url]: https://david-dm.org/open-telemetry/opentelemetry-js-contrib?path=packages%2Fopentelemetry-rca-metrics +[devDependencies-image]: https://david-dm.org/open-telemetry/opentelemetry-js-contrib.svg?path=packages%2Fopentelemetry-rca-metrics&type=dev +[devDependencies-url]: https://david-dm.org/open-telemetry/opentelemetry-js-contrib?path=packages%2Fopentelemetry-rca-metrics&type=dev +[npm-url]: https://www.npmjs.com/package/@opentelemetry/rca-metrics +[npm-img]: https://badge.fury.io/js/%40opentelemetry%2Frca-metrics.svg diff --git a/packages/opentelemetry-rca-metrics/artifacts/prebuilds.tgz b/packages/opentelemetry-rca-metrics/artifacts/prebuilds.tgz new file mode 100644 index 0000000000..2d8843cf28 Binary files /dev/null and b/packages/opentelemetry-rca-metrics/artifacts/prebuilds.tgz differ diff --git a/packages/opentelemetry-rca-metrics/artifacts/prebuilds.tgz.sha1 b/packages/opentelemetry-rca-metrics/artifacts/prebuilds.tgz.sha1 new file mode 100644 index 0000000000..b9d65fb4fa --- /dev/null +++ b/packages/opentelemetry-rca-metrics/artifacts/prebuilds.tgz.sha1 @@ -0,0 +1 @@ +15650db2e6cb69f2f7f36a5bbe0896c7398e7d77 \ No newline at end of file diff --git a/packages/opentelemetry-rca-metrics/binding.gyp b/packages/opentelemetry-rca-metrics/binding.gyp new file mode 100644 index 0000000000..bc8ce65b43 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/binding.gyp @@ -0,0 +1,44 @@ +{ + "targets": [{ + "target_name": "metrics", + "sources": [ + "native/metrics/Collector.cpp", + "native/metrics/EventLoop.cpp", + "native/metrics/GarbageCollection.cpp", + "native/metrics/Heap.cpp", + "native/metrics/Histogram.cpp", + "native/metrics/Object.cpp", + "native/metrics/main.cpp" + ], + "include_dirs": [ + "native", + " +#include + +#include "Object.hpp" + +namespace opentelemetry { + class Collector { + public: + virtual void inject(Object carrier) = 0; + protected: + virtual uint64_t time_to_micro(uv_timeval_t timeval); + }; +} diff --git a/packages/opentelemetry-rca-metrics/native/metrics/EventLoop.cpp b/packages/opentelemetry-rca-metrics/native/metrics/EventLoop.cpp new file mode 100644 index 0000000000..8428523dff --- /dev/null +++ b/packages/opentelemetry-rca-metrics/native/metrics/EventLoop.cpp @@ -0,0 +1,60 @@ +#include "EventLoop.hpp" + +namespace opentelemetry { + // http://docs.libuv.org/en/v1.x/design.html#the-i-o-loop + EventLoop::EventLoop() { + uv_check_init(uv_default_loop(), &check_handle_); + uv_prepare_init(uv_default_loop(), &prepare_handle_); + uv_unref(reinterpret_cast(&check_handle_)); + uv_unref(reinterpret_cast(&prepare_handle_)); + + check_handle_.data = (void*)this; + prepare_handle_.data = (void*)this; + + check_time_ = uv_hrtime(); + } + + EventLoop::~EventLoop() { + uv_check_stop(&check_handle_); + uv_prepare_stop(&prepare_handle_); + } + + void EventLoop::check_cb (uv_check_t* handle) { + EventLoop* self = (EventLoop*)handle->data; + + uint64_t check_time = uv_hrtime(); + uint64_t poll_time = check_time - self->prepare_time_; + uint64_t latency = self->prepare_time_ - self->check_time_; + uint64_t timeout = self->timeout_ * 1000 * 1000; + + if (poll_time > timeout) { + latency += poll_time - timeout; + } + + self->histogram_.add(latency); + self->check_time_ = check_time; + } + + void EventLoop::prepare_cb (uv_prepare_t* handle) { + EventLoop* self = (EventLoop*)handle->data; + + self->prepare_time_ = uv_hrtime(); + self->timeout_ = uv_backend_timeout(uv_default_loop()); + } + + void EventLoop::enable() { + uv_check_start(&check_handle_, &EventLoop::check_cb); + uv_prepare_start(&prepare_handle_, &EventLoop::prepare_cb); + } + + void EventLoop::disable() { + uv_check_stop(&check_handle_); + uv_prepare_stop(&prepare_handle_); + histogram_.reset(); + } + + void EventLoop::inject(Object carrier) { + carrier.set("eventLoop", histogram_); + histogram_.reset(); + } +} diff --git a/packages/opentelemetry-rca-metrics/native/metrics/EventLoop.hpp b/packages/opentelemetry-rca-metrics/native/metrics/EventLoop.hpp new file mode 100644 index 0000000000..e6417d76f7 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/native/metrics/EventLoop.hpp @@ -0,0 +1,33 @@ +#pragma once + +#include +#include + +#include "Collector.hpp" +#include "Histogram.hpp" + +namespace opentelemetry { + class EventLoop : public Collector { + public: + EventLoop(); + ~EventLoop(); + EventLoop(const EventLoop&) = delete; + void operator=(const EventLoop&) = delete; + + void enable(); + void disable(); + void inject(Object carrier); + protected: + static void check_cb (uv_check_t* handle); + static void prepare_cb (uv_prepare_t* handle); + private: + uv_check_t check_handle_; + uv_prepare_t prepare_handle_; + uint64_t check_time_; + uint64_t prepare_time_; + uint64_t timeout_; + Histogram histogram_; + + uint64_t usage(); + }; +} diff --git a/packages/opentelemetry-rca-metrics/native/metrics/GarbageCollection.cpp b/packages/opentelemetry-rca-metrics/native/metrics/GarbageCollection.cpp new file mode 100644 index 0000000000..3705490130 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/native/metrics/GarbageCollection.cpp @@ -0,0 +1,42 @@ +#include + +#include "GarbageCollection.hpp" + +namespace opentelemetry { + GarbageCollection::GarbageCollection() { + types_[1] = "scavenge"; + types_[2] = "markSweepCompact"; + types_[3] = "all"; + types_[4] = "incrementalMarking"; + types_[8] = "processWeakCallbacks"; + types_[15] = "all"; + + pause_[v8::GCType::kGCTypeAll] = Histogram(); + } + + void GarbageCollection::before(v8::GCType type) { + start_time_ = uv_hrtime(); + } + + void GarbageCollection::after(v8::GCType type) { + uint64_t usage = uv_hrtime() - start_time_; + + if (pause_.find(type) == pause_.end()) { + pause_[type] = Histogram(); + } + + pause_[type].add(usage); + pause_[v8::GCType::kGCTypeAll].add(usage); + } + + void GarbageCollection::inject(Object carrier) { + Object value; + + for (auto &it : pause_) { + value.set(types_[it.first], it.second); + it.second.reset(); + } + + carrier.set("gc", value); + } +} diff --git a/packages/opentelemetry-rca-metrics/native/metrics/GarbageCollection.hpp b/packages/opentelemetry-rca-metrics/native/metrics/GarbageCollection.hpp new file mode 100644 index 0000000000..f09d9afe0d --- /dev/null +++ b/packages/opentelemetry-rca-metrics/native/metrics/GarbageCollection.hpp @@ -0,0 +1,25 @@ +#pragma once + +#include +#include +#include +#include + +#include "Collector.hpp" +#include "Histogram.hpp" +#include "Object.hpp" + +namespace opentelemetry { + class GarbageCollection : public Collector { + public: + GarbageCollection(); + + void before(v8::GCType type); + void after(v8::GCType type); + void inject(Object carrier); + private: + std::map pause_; + std::map types_; + uint64_t start_time_; + }; +} diff --git a/packages/opentelemetry-rca-metrics/native/metrics/Heap.cpp b/packages/opentelemetry-rca-metrics/native/metrics/Heap.cpp new file mode 100644 index 0000000000..12088a9830 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/native/metrics/Heap.cpp @@ -0,0 +1,31 @@ +#include +#include +#include + +#include "Heap.hpp" + +namespace opentelemetry { + void Heap::inject(Object carrier) { + v8::Isolate *isolate = v8::Isolate::GetCurrent(); + Object heap; + std::vector spaces; + + for (unsigned int i = 0; i < isolate->NumberOfHeapSpaces(); i++) { + Object space; + v8::HeapSpaceStatistics stats; + + if (isolate->GetHeapSpaceStatistics(&stats, i)) { + space.set("spaceName", std::string(stats.space_name())); + space.set("size", stats.space_size()); + space.set("usedSize", stats.space_used_size()); + space.set("availableSize", stats.space_available_size()); + space.set("physicalSize", stats.physical_space_size()); + + spaces.push_back(space); + } + } + + heap.set("spaces", spaces); + carrier.set("heap", heap); + } +} diff --git a/packages/opentelemetry-rca-metrics/native/metrics/Heap.hpp b/packages/opentelemetry-rca-metrics/native/metrics/Heap.hpp new file mode 100644 index 0000000000..29d9de8c90 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/native/metrics/Heap.hpp @@ -0,0 +1,11 @@ +#pragma once + +#include "Collector.hpp" +#include "Object.hpp" + +namespace opentelemetry { + class Heap : public Collector { + public: + void inject(Object carrier); + }; +} diff --git a/packages/opentelemetry-rca-metrics/native/metrics/Histogram.cpp b/packages/opentelemetry-rca-metrics/native/metrics/Histogram.cpp new file mode 100644 index 0000000000..f001340a94 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/native/metrics/Histogram.cpp @@ -0,0 +1,39 @@ +#include "Histogram.hpp" + +namespace opentelemetry { + Histogram::Histogram() { + reset(); + } + + uint64_t Histogram::min() { return min_; } + uint64_t Histogram::max() { return max_; } + uint64_t Histogram::sum() { return sum_; } + uint64_t Histogram::avg() { return count_ == 0 ? 0 : sum_ / count_; } + uint64_t Histogram::count() { return count_; } + uint64_t Histogram::percentile(double value) { + return count_ == 0 ? 0 : static_cast(std::round(digest_->quantile(value))); + } + + void Histogram::reset() { + min_ = 0; + max_ = 0; + sum_ = 0; + count_ = 0; + + digest_ = std::make_shared(1000); + } + + void Histogram::add(uint64_t value) { + if (count_ == 0) { + min_ = max_ = value; + } else { + min_ = (std::min)(min_, value); + max_ = (std::max)(max_, value); + } + + count_ += 1; + sum_ += value; + + digest_->add(static_cast(value)); + } +} diff --git a/packages/opentelemetry-rca-metrics/native/metrics/Histogram.hpp b/packages/opentelemetry-rca-metrics/native/metrics/Histogram.hpp new file mode 100644 index 0000000000..d98b962737 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/native/metrics/Histogram.hpp @@ -0,0 +1,36 @@ +#pragma once + +// windows.h defines min and max macros. +#define NOMINMAX +#include +#undef min +#undef max +#undef NOMINMAX + +#include + +#include +#include + +namespace opentelemetry { + class Histogram { + public: + Histogram(); + + uint64_t min(); + uint64_t max(); + uint64_t sum(); + uint64_t avg(); + uint64_t count(); + uint64_t percentile(double percentile); + + void reset(); + void add(uint64_t value); + private: + uint64_t min_; + uint64_t max_; + uint64_t sum_; + uint64_t count_; + std::shared_ptr digest_; + }; +} diff --git a/packages/opentelemetry-rca-metrics/native/metrics/Object.cpp b/packages/opentelemetry-rca-metrics/native/metrics/Object.cpp new file mode 100644 index 0000000000..5161c42bf5 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/native/metrics/Object.cpp @@ -0,0 +1,89 @@ +#include + +#include "Object.hpp" + +namespace opentelemetry { + Object::Object() { + target_ = Nan::New(); + } + + Object::Object(v8::Local target) { + target_ = target; + } + + void Object::set(std::string key, std::string value) { + Nan::Set( + target_, + Nan::New(key).ToLocalChecked(), + Nan::New(value).ToLocalChecked() + ); + } + + void Object::set(std::string key, uint64_t value) { + Nan::Set( + target_, + Nan::New(key).ToLocalChecked(), + Nan::New(static_cast(value)) + ); + } + + void Object::set(std::string key, v8::Local value) { + Nan::Set( + target_, + Nan::New(key).ToLocalChecked(), + value + ); + } + + void Object::set(std::string key, Object value) { + Nan::Set( + target_, + Nan::New(key).ToLocalChecked(), + value.to_json() + ); + } + + void Object::set(std::string key, std::vector value) { + v8::Local array = Nan::New(value.size()); + + for (unsigned int i = 0; i < array->Length(); i++) { + Nan::Set(array, i, value.at(i).to_json()); + } + + Nan::Set( + target_, + Nan::New(key).ToLocalChecked(), + array + ); + } + + void Object::set(std::string key, Histogram value) { + Object obj; + + obj.set("min", value.min()); + obj.set("max", value.max()); + obj.set("sum", value.sum()); + obj.set("avg", value.avg()); + obj.set("count", value.count()); + obj.set("median", value.percentile(0.50)); + obj.set("p95", value.percentile(0.95)); + + Nan::Set( + target_, + Nan::New(key).ToLocalChecked(), + obj.to_json() + ); + } + + void Object::set(std::string key, Nan::FunctionCallback value) { + Nan::Set( + target_, + Nan::New(key).ToLocalChecked(), + Nan::GetFunction(Nan::New(value)).ToLocalChecked() + ); + } + + v8::Local Object::to_json() { + return target_; + } +} diff --git a/packages/opentelemetry-rca-metrics/native/metrics/Object.hpp b/packages/opentelemetry-rca-metrics/native/metrics/Object.hpp new file mode 100644 index 0000000000..b9ca75df86 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/native/metrics/Object.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include +#include +#include +#include + +#include "Histogram.hpp" + +namespace opentelemetry { + class Object { + public: + Object(); + Object(v8::Local target); + + void set(std::string key, std::string value); + void set(std::string key, uint64_t value); + void set(std::string key, v8::Local value); + void set(std::string key, Object value); + void set(std::string key, std::vector value); + void set(std::string key, Histogram value); + void set(std::string key, Nan::FunctionCallback value); + + v8::Local to_json(); + private: + v8::Local target_; + }; +} diff --git a/packages/opentelemetry-rca-metrics/native/metrics/main.cpp b/packages/opentelemetry-rca-metrics/native/metrics/main.cpp new file mode 100644 index 0000000000..19d4c57cd9 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/native/metrics/main.cpp @@ -0,0 +1,57 @@ +#include + +#include "EventLoop.hpp" +#include "GarbageCollection.hpp" +#include "Heap.hpp" +#include "Object.hpp" + +namespace opentelemetry { + namespace { + EventLoop eventLoop; + GarbageCollection gc; + Heap heap; + + NAN_GC_CALLBACK(before_gc) { + gc.before(type); + } + + NAN_GC_CALLBACK(after_gc) { + gc.after(type); + } + + NAN_METHOD(start) { + eventLoop.enable(); + + Nan::AddGCPrologueCallback(before_gc); + Nan::AddGCEpilogueCallback(after_gc); + } + + NAN_METHOD(stop) { + eventLoop.disable(); + + Nan::RemoveGCPrologueCallback(before_gc); + Nan::RemoveGCEpilogueCallback(after_gc); + } + + NAN_METHOD(stats) { + Object obj; + + eventLoop.inject(obj); + gc.inject(obj); + heap.inject(obj); + + info.GetReturnValue().Set(obj.to_json()); + } + + } + + NAN_MODULE_INIT(init) { + Object obj = Object(target); + + obj.set("start", start); + obj.set("stop", stop); + obj.set("stats", stats); + } + + NODE_MODULE(metrics, init); +} diff --git a/packages/opentelemetry-rca-metrics/native/tdigest/LICENSE-2.0.txt b/packages/opentelemetry-rca-metrics/native/tdigest/LICENSE-2.0.txt new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/native/tdigest/LICENSE-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/opentelemetry-rca-metrics/native/tdigest/NOTICES b/packages/opentelemetry-rca-metrics/native/tdigest/NOTICES new file mode 100644 index 0000000000..5797229d67 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/native/tdigest/NOTICES @@ -0,0 +1,8 @@ +The Java version of the t-digest was originally authored by Ted Dunning + +A number of small but very helpful changes have been contributed by Adrien Grand (https://github.com/jpountz) to the Java version. + +The C++ version herein is a derivative of the Java version. It was written by Derrick R. Burns (https:://github.com/derrickburns). +The main modifications are 1) higher performance multi- t-digest merging and 2) faster quantile() and cdf() computation. + +Dependencies have been removed and Windows compatibility has been added by Datadog Inc. diff --git a/packages/opentelemetry-rca-metrics/native/tdigest/TDigest.h b/packages/opentelemetry-rca-metrics/native/tdigest/TDigest.h new file mode 100644 index 0000000000..efad5ac90c --- /dev/null +++ b/packages/opentelemetry-rca-metrics/native/tdigest/TDigest.h @@ -0,0 +1,596 @@ +/* + * Licensed to Derrick R. Burns under one or more + * contributor license agreements. See the NOTICES file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef M_PI +#define M_PI 3.14159265358979323846264338327950288 +#endif + +#ifndef TDIGEST2_TDIGEST_H_ +#define TDIGEST2_TDIGEST_H_ + +#include +#include +#include +#include +#include +#include + +namespace tdigest { + +using Value = double; +using Weight = double; +using Index = size_t; + +const size_t kHighWater = 40000; + +class Centroid { + public: + Centroid() : Centroid(0.0, 0.0) {} + + Centroid(Value mean, Weight weight) : mean_(mean), weight_(weight) {} + + inline Value mean() const noexcept { return mean_; } + + inline Weight weight() const noexcept { return weight_; } + + inline void add(const Centroid& c) { + if( weight_ != 0.0 ) { + weight_ += c.weight_; + mean_ += c.weight_ * (c.mean_ - mean_) / weight_; + } else { + weight_ = c.weight_; + mean_ = c.mean_; + } + } + + private: + Value mean_ = 0; + Weight weight_ = 0; +}; + +struct CentroidList { + CentroidList(const std::vector& s) : iter(s.cbegin()), end(s.cend()) {} + std::vector::const_iterator iter; + std::vector::const_iterator end; + + bool advance() { return ++iter != end; } +}; + +class CentroidListComparator { + public: + CentroidListComparator() {} + + bool operator()(const CentroidList& left, const CentroidList& right) const { + return left.iter->mean() > right.iter->mean(); + } +}; + +using CentroidListQueue = std::priority_queue, CentroidListComparator>; + +struct CentroidComparator { + bool operator()(const Centroid& a, const Centroid& b) const { return a.mean() < b.mean(); } +}; + +class TDigest { + class TDigestComparator { + public: + TDigestComparator() {} + + bool operator()(const TDigest* left, const TDigest* right) const { return left->totalSize() > right->totalSize(); } + }; + + using TDigestQueue = std::priority_queue, TDigestComparator>; + + public: + TDigest() : TDigest(1000) {} + + explicit TDigest(Value compression) : TDigest(compression, 0) {} + + TDigest(Value compression, Index bufferSize) : TDigest(compression, bufferSize, 0) {} + + TDigest(Value compression, Index unmergedSize, Index mergedSize) + : compression_(compression), + maxProcessed_(processedSize(mergedSize, compression)), + maxUnprocessed_(unprocessedSize(unmergedSize, compression)) { + processed_.reserve(maxProcessed_); + unprocessed_.reserve(maxUnprocessed_ + 1); + } + + TDigest(std::vector&& processed, std::vector&& unprocessed, Value compression, + Index unmergedSize, Index mergedSize) + : TDigest(compression, unmergedSize, mergedSize) { + processed_ = std::move(processed); + unprocessed_ = std::move(unprocessed); + + processedWeight_ = weight(processed_); + unprocessedWeight_ = weight(unprocessed_); + if( processed_.size() > 0 ) { + min_ = std::min(min_, processed_[0].mean()); + max_ = std::max(max_, (processed_.cend() - 1)->mean()); + } + updateCumulative(); + } + + static Weight weight(std::vector& centroids) noexcept { + Weight w = 0.0; + for (auto centroid : centroids) { + w += centroid.weight(); + } + return w; + } + + TDigest& operator=(TDigest&& o) { + compression_ = o.compression_; + maxProcessed_ = o.maxProcessed_; + maxUnprocessed_ = o.maxUnprocessed_; + processedWeight_ = o.processedWeight_; + unprocessedWeight_ = o.unprocessedWeight_; + processed_ = std::move(o.processed_); + unprocessed_ = std::move(o.unprocessed_); + cumulative_ = std::move(o.cumulative_); + min_ = o.min_; + max_ = o.max_; + return *this; + } + + TDigest(TDigest&& o) + : TDigest(std::move(o.processed_), std::move(o.unprocessed_), o.compression_, o.maxUnprocessed_, + o.maxProcessed_) {} + + static inline Index processedSize(Index size, Value compression) noexcept { + return (size == 0) ? static_cast(2 * std::ceil(compression)) : size; + } + + static inline Index unprocessedSize(Index size, Value compression) noexcept { + return (size == 0) ? static_cast(8 * std::ceil(compression)) : size; + } + + // merge in another t-digest + inline void merge(const TDigest* other) { + std::vector others{other}; + add(others.cbegin(), others.cend()); + } + + const std::vector& processed() const { return processed_; } + + const std::vector& unprocessed() const { return unprocessed_; } + + Index maxUnprocessed() const { return maxUnprocessed_; } + + Index maxProcessed() const { return maxProcessed_; } + + inline void add(std::vector digests) { add(digests.cbegin(), digests.cend()); } + + // merge in a vector of tdigests in the most efficient manner possible + // in constant space + // works for any value of kHighWater + void add(std::vector::const_iterator iter, std::vector::const_iterator end) { + if (iter != end) { + auto size = std::distance(iter, end); + TDigestQueue pq(TDigestComparator{}); + for (; iter != end; iter++) { + pq.push((*iter)); + } + std::vector batch; + batch.reserve(size); + + size_t totalSize = 0; + while (!pq.empty()) { + auto td = pq.top(); + batch.push_back(td); + pq.pop(); + totalSize += td->totalSize(); + if (totalSize >= kHighWater || pq.empty()) { + mergeProcessed(batch); + mergeUnprocessed(batch); + processIfNecessary(); + batch.clear(); + totalSize = 0; + } + } + updateCumulative(); + } + } + + Weight processedWeight() const { return processedWeight_; } + + Weight unprocessedWeight() const { return unprocessedWeight_; } + + bool haveUnprocessed() const { return unprocessed_.size() > 0; } + + size_t totalSize() const { return processed_.size() + unprocessed_.size(); } + + long totalWeight() const { return static_cast(processedWeight_ + unprocessedWeight_); } + + // return the cdf on the t-digest + Value cdf(Value x) { + if (haveUnprocessed() || isDirty()) process(); + return cdfProcessed(x); + } + + bool isDirty() { return processed_.size() > maxProcessed_ || unprocessed_.size() > maxUnprocessed_; } + + // return the cdf on the processed values + Value cdfProcessed(Value x) const { + if (processed_.size() == 0) { + // no data to examin_e + + return 0.0; + } else if (processed_.size() == 1) { + // exactly one centroid, should have max_==min_ + auto width = max_ - min_; + if (x < min_) { + return 0.0; + } else if (x > max_) { + return 1.0; + } else if (x - min_ <= width) { + // min_ and max_ are too close together to do any viable interpolation + return 0.5; + } else { + // interpolate if somehow we have weight > 0 and max_ != min_ + return (x - min_) / (max_ - min_); + } + } else { + auto n = processed_.size(); + if (x <= min_) { + return 0; + } + + if (x >= max_) { + return 1; + } + + // check for the left tail + if (x <= mean(0)) { + // note that this is different than mean(0) > min_ ... this guarantees interpolation works + if (mean(0) - min_ > 0) { + return (x - min_) / (mean(0) - min_) * weight(0) / processedWeight_ / 2.0; + } else { + return 0; + } + } + + // and the right tail + if (x >= mean(n - 1)) { + if (max_ - mean(n - 1) > 0) { + return 1.0 - (max_ - x) / (max_ - mean(n - 1)) * weight(n - 1) / processedWeight_ / 2.0; + } else { + return 1; + } + } + + CentroidComparator cc; + auto iter = std::upper_bound(processed_.cbegin(), processed_.cend(), Centroid(x, 0), cc); + + auto i = std::distance(processed_.cbegin(), iter); + auto z1 = x - (iter - 1)->mean(); + auto z2 = (iter)->mean() - x; + + return weightedAverage(cumulative_[i - 1], z2, cumulative_[i], z1) / processedWeight_; + } + } + + // this returns a quantile on the t-digest + Value quantile(Value q) { + if (haveUnprocessed() || isDirty()) process(); + return quantileProcessed(q); + } + + // this returns a quantile on the currently processed values without changing the t-digest + // the value will not represent the unprocessed values + Value quantileProcessed(Value q) const { + if (q < 0 || q > 1) { + return NAN; + } + + if (processed_.size() == 0) { + // no sorted means no data, no way to get a quantile + return NAN; + } else if (processed_.size() == 1) { + // with one data point, all quantiles lead to Rome + + return mean(0); + } + + // we know that there are at least two sorted now + auto n = processed_.size(); + + // if values were stored in a sorted array, index would be the offset we are Weighterested in + const auto index = q * processedWeight_; + + // at the boundaries, we return min_ or max_ + if (index <= weight(0) / 2.0) { + return min_ + 2.0 * index / weight(0) * (mean(0) - min_); + } + + auto iter = std::lower_bound(cumulative_.cbegin(), cumulative_.cend(), index); + + if (iter + 1 != cumulative_.cend()) { + auto i = std::distance(cumulative_.cbegin(), iter); + auto z1 = index - *(iter - 1); + auto z2 = *(iter)-index; + // LOG(INFO) << "z2 " << z2 << " index " << index << " z1 " << z1; + return weightedAverage(mean(i - 1), z2, mean(i), z1); + } + + auto z1 = index - processedWeight_ - weight(n - 1) / 2.0; + auto z2 = weight(n - 1) / 2 - z1; + return weightedAverage(mean(n - 1), z1, max_, z2); + } + + Value compression() const { return compression_; } + + void add(Value x) { add(x, 1); } + + inline void compress() { process(); } + + // add a single centroid to the unprocessed vector, processing previously unprocessed sorted if our limit has + // been reached. + inline bool add(Value x, Weight w) { + if (std::isnan(x)) { + return false; + } + unprocessed_.push_back(Centroid(x, w)); + unprocessedWeight_ += w; + processIfNecessary(); + return true; + } + + inline void add(std::vector::const_iterator iter, std::vector::const_iterator end) { + while (iter != end) { + const size_t diff = std::distance(iter, end); + const size_t room = maxUnprocessed_ - unprocessed_.size(); + auto mid = iter + std::min(diff, room); + while (iter != mid) unprocessed_.push_back(*(iter++)); + if (unprocessed_.size() >= maxUnprocessed_) { + process(); + } + } + } + + private: + Value compression_; + + Value min_ = std::numeric_limits::max(); + + Value max_ = std::numeric_limits::min(); + + Index maxProcessed_; + + Index maxUnprocessed_; + + Value processedWeight_ = 0.0; + + Value unprocessedWeight_ = 0.0; + + std::vector processed_; + + std::vector unprocessed_; + + std::vector cumulative_; + + // return mean of i-th centroid + inline Value mean(int i) const noexcept { return processed_[i].mean(); } + + // return weight of i-th centroid + inline Weight weight(int i) const noexcept { return processed_[i].weight(); } + + // append all unprocessed centroids into current unprocessed vector + void mergeUnprocessed(const std::vector& tdigests) { + if (tdigests.size() == 0) return; + + size_t total = unprocessed_.size(); + for (auto& td : tdigests) { + total += td->unprocessed_.size(); + } + + unprocessed_.reserve(total); + for (auto& td : tdigests) { + unprocessed_.insert(unprocessed_.end(), td->unprocessed_.cbegin(), td->unprocessed_.cend()); + unprocessedWeight_ += td->unprocessedWeight_; + } + } + + // merge all processed centroids together into a single sorted vector + void mergeProcessed(const std::vector& tdigests) { + if (tdigests.size() == 0) return; + + size_t total = 0; + CentroidListQueue pq(CentroidListComparator{}); + for (auto& td : tdigests) { + auto& sorted = td->processed_; + auto size = sorted.size(); + if (size > 0) { + pq.push(CentroidList(sorted)); + total += size; + processedWeight_ += td->processedWeight_; + } + } + if (total == 0) return; + + if (processed_.size() > 0) { + pq.push(CentroidList(processed_)); + total += processed_.size(); + } + + std::vector sorted; + sorted.reserve(total); + + while (!pq.empty()) { + auto best = pq.top(); + pq.pop(); + sorted.push_back(*(best.iter)); + if (best.advance()) pq.push(best); + } + processed_ = std::move(sorted); + if( processed_.size() > 0 ) { + min_ = std::min(min_, processed_[0].mean()); + max_ = std::max(max_, (processed_.cend() - 1)->mean()); + } + } + + inline void processIfNecessary() { + if (isDirty()) { + process(); + } + } + + void updateCumulative() { + const auto n = processed_.size(); + cumulative_.clear(); + cumulative_.reserve(n + 1); + auto previous = 0.0; + for (Index i = 0; i < n; i++) { + auto current = weight(i); + auto halfCurrent = current / 2.0; + cumulative_.push_back(previous + halfCurrent); + previous = previous + current; + } + cumulative_.push_back(previous); + } + + // merges unprocessed_ centroids and processed_ centroids together and processes them + // when complete, unprocessed_ will be empty and processed_ will have at most maxProcessed_ centroids + inline void process() { + CentroidComparator cc; + std::sort(unprocessed_.begin(), unprocessed_.end(), cc); + auto count = unprocessed_.size(); + unprocessed_.insert(unprocessed_.end(), processed_.cbegin(), processed_.cend()); + std::inplace_merge(unprocessed_.begin(), unprocessed_.begin() + count, unprocessed_.end(), cc); + + processedWeight_ += unprocessedWeight_; + unprocessedWeight_ = 0; + processed_.clear(); + + processed_.push_back(unprocessed_[0]); + Weight wSoFar = unprocessed_[0].weight(); + Weight wLimit = processedWeight_ * integratedQ(1.0); + + auto end = unprocessed_.end(); + for (auto iter = unprocessed_.cbegin() + 1; iter < end; iter++) { + auto& centroid = *iter; + Weight projectedW = wSoFar + centroid.weight(); + if (projectedW <= wLimit) { + wSoFar = projectedW; + (processed_.end() - 1)->add(centroid); + } else { + auto k1 = integratedLocation(wSoFar / processedWeight_); + wLimit = processedWeight_ * integratedQ(k1 + 1.0); + wSoFar += centroid.weight(); + processed_.emplace_back(centroid); + } + } + unprocessed_.clear(); + min_ = std::min(min_, processed_[0].mean()); + max_ = std::max(max_, (processed_.cend() - 1)->mean()); + updateCumulative(); + } + + inline int checkWeights() { return checkWeights(processed_, processedWeight_); } + + size_t checkWeights(const std::vector& sorted, Value total) { + size_t badWeight = 0; + auto k1 = 0.0; + auto q = 0.0; + for (auto iter = sorted.cbegin(); iter != sorted.cend(); iter++) { + auto w = iter->weight(); + auto dq = w / total; + auto k2 = integratedLocation(q + dq); + if (k2 - k1 > 1 && w != 1) { + badWeight++; + } + if (k2 - k1 > 1.5 && w != 1) { + badWeight++; + } + q += dq; + k1 = k2; + } + + return badWeight; + } + + /** + * Converts a quantile into a centroid scale value. The centroid scale is nomin_ally + * the number k of the centroid that a quantile point q should belong to. Due to + * round-offs, however, we can't align things perfectly without splitting points + * and sorted. We don't want to do that, so we have to allow for offsets. + * In the end, the criterion is that any quantile range that spans a centroid + * scale range more than one should be split across more than one centroid if + * possible. This won't be possible if the quantile range refers to a single point + * or an already existing centroid. + *

+ * This mapping is steep near q=0 or q=1 so each centroid there will correspond to + * less q range. Near q=0.5, the mapping is flatter so that sorted there will + * represent a larger chunk of quantiles. + * + * @param q The quantile scale value to be mapped. + * @return The centroid scale value corresponding to q. + */ + inline Value integratedLocation(Value q) const { + return compression_ * (std::asin(2.0 * q - 1.0) + M_PI / 2) / M_PI; + } + + inline Value integratedQ(Value k) const { + return (std::sin(std::min(k, compression_) * M_PI / compression_ - M_PI / 2) + 1) / 2; + } + + /** + * Same as {@link #weightedAverageSorted(Value, Value, Value, Value)} but flips + * the order of the variables if x2 is greater than + * x1. + */ + static Value weightedAverage(Value x1, Value w1, Value x2, Value w2) { + return (x1 <= x2) ? weightedAverageSorted(x1, w1, x2, w2) : weightedAverageSorted(x2, w2, x1, w1); + } + + /** + * Compute the weighted average between x1 with a weight of + * w1 and x2 with a weight of w2. + * This expects x1 to be less than or equal to x2 + * and is guaranteed to return a number between x1 and + * x2. + */ + static Value weightedAverageSorted(Value x1, Value w1, Value x2, Value w2) { + const Value x = (x1 * w1 + x2 * w2) / (w1 + w2); + return std::max(x1, std::min(x, x2)); + } + + static Value interpolate(Value x, Value x0, Value x1) { return (x - x0) / (x1 - x0); } + + /** + * Computes an interpolated value of a quantile that is between two sorted. + * + * Index is the quantile desired multiplied by the total number of samples - 1. + * + * @param index Denormalized quantile desired + * @param previousIndex The denormalized quantile corresponding to the center of the previous centroid. + * @param nextIndex The denormalized quantile corresponding to the center of the following centroid. + * @param previousMean The mean of the previous centroid. + * @param nextMean The mean of the following centroid. + * @return The interpolated mean. + */ + static Value quantile(Value index, Value previousIndex, Value nextIndex, Value previousMean, Value nextMean) { + const auto delta = nextIndex - previousIndex; + const auto previousWeight = (nextIndex - index) / delta; + const auto nextWeight = (index - previousIndex) / delta; + return previousMean * previousWeight + nextMean * nextWeight; + } +}; + +} // namespace tdigest2 + +#endif // TDIGEST2_TDIGEST_H_ diff --git a/packages/opentelemetry-rca-metrics/package.json b/packages/opentelemetry-rca-metrics/package.json new file mode 100644 index 0000000000..55475869c2 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/package.json @@ -0,0 +1,81 @@ +{ + "name": "@opentelemetry/rca-metrics", + "version": "0.9.0", + "description": "OpenTelemetry RCA Metrics for Node.js", + "main": "build/src/index.js", + "types": "build/src/index.d.ts", + "repository": "open-telemetry/opentelemetry-js-contrib", + "scripts": { + "build:install": "node scripts/build_install.js", + "build:rca-metrics": "node scripts/build_all.js", + "clean": "rimraf build/*", + "codecov": "nyc report --reporter=json && codecov -f coverage/*.json -p ../../", + "compile": "npm run version:update && tsc -p .", + "lint": "gts check", + "lint:fix": "gts fix", + "postcompile": "npm run build:install", + "get:prebuilds": "node scripts/get_prebuilds.js", + "precompile": "tsc --version", + "prepare": "npm run compile", + "tdd": "npm run test -- --watch-extensions ts --watch", + "test": "nyc ts-mocha -p tsconfig.json test/**/*.test.ts", + "version:update": "node ../../scripts/version-update.js", + "watch": "tsc -w" + }, + "keywords": [ + "opentelemetry", + "metrics", + "nodejs", + "tracing", + "profiling", + "plugin" + ], + "author": "OpenTelemetry Authors", + "license": "Apache-2.0", + "engines": { + "node": ">=8.5.0" + }, + "files": [ + "build/src/**/*.js", + "build/src/**/*.d.ts", + "artifacts/prebuilds.tgz", + "doc", + "LICENSE", + "README.md" + ], + "publishConfig": { + "access": "public" + }, + "devDependencies": { + "@opentelemetry/exporter-prometheus": "^0.10.2", + "@types/mocha": "8.0.2", + "@types/node": "14.0.27", + "@types/sinon": "9.0.4", + "axios": "^0.20.0", + "checksum": "^0.1.1", + "codecov": "^3.7.2", + "glob": "^7.1.6", + "gts": "^2.0.2", + "mkdirp": "^1.0.4", + "mocha": "7.2.0", + "mock-require": "^3.0.3", + "nan": "^2.14.1", + "node-gyp": "^7.1.0", + "nyc": "^15.1.0", + "rimraf": "^3.0.2", + "semver": "^7.3.2", + "sinon": "^9.0.3", + "tar": "^6.0.5", + "ts-loader": "^8.0.3", + "ts-mocha": "^7.0.0", + "ts-node": "^8.10.2", + "typescript": "^3.9.7" + }, + "dependencies": { + "@opentelemetry/api": "^0.10.2", + "@opentelemetry/core": "^0.10.2", + "@opentelemetry/metrics": "^0.10.2", + "node-gyp-build": "^4.2.3", + "systeminformation": "^4.27.1" + } +} diff --git a/packages/opentelemetry-rca-metrics/scripts/build_all.js b/packages/opentelemetry-rca-metrics/scripts/build_all.js new file mode 100644 index 0000000000..d7052c84e1 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/scripts/build_all.js @@ -0,0 +1,185 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const execSync = require('child_process').execSync; +const fs = require('fs'); +const rimraf = require('rimraf'); +const mkdirp = require('mkdirp'); +const path = require('path'); +const tar = require('tar'); +const glob = require('glob'); +const os = require('os'); +const checksum = require('checksum'); + +const MAIN_FOLDER = path.resolve(__dirname, '..'); +const BUILD_FOLDER = path.resolve(MAIN_FOLDER, 'build/Release'); +const ARTIFACTS_FOLDER = path.resolve(MAIN_FOLDER, 'artifacts'); +const CACHE_FOLDER = path.join(os.tmpdir(), 'cache'); +const PREBUILDS_FOLDER_NAME = 'prebuilds'; + +const start = new Date().getTime(); +let filesToBeBuild = 0; + +cleanBefore(); +buildAll(); +zipBuilds(); +cleanAfter(); +extractFromBuild(); +validate(); +createChecksum(); +cleanAfter(); + +function buildAll() { + const platforms = ['darwin', 'linux', 'win32']; + const arch = ['arm', 'x64', 'arm64']; + // const platforms = ['darwin']; + // const arch = ['x64']; + // https://nodejs.org/en/download/releases/ + const targets = [ + { version: '8.0.0', abi: '57' }, + { version: '10.0.0', abi: '64' }, + { version: '11.0.0', abi: '67' }, + { version: '12.0.0', abi: '72' }, + { version: '13.0.0', abi: '79' }, + { version: '14.0.0', abi: '83' }, + ]; + filesToBeBuild = platforms.length * arch.length * targets.length; + + console.log(`Files to be build: (${filesToBeBuild})`); + + let count = 0; + for (let i = 0, j = platforms.length; i < j; i++) { + for (let k = 0, l = arch.length; k < l; k++) { + for (let m = 0, n = targets.length; m < n; m++) { + build(platforms[i], arch[k], targets[m]); + count++; + const progress = ((count / filesToBeBuild) * 100).toFixed(2); + const last = new Date().getTime() - start; + const totalTime = last / (count / filesToBeBuild); + const left = Math.round((totalTime - last) / 1000); + console.log(`Progress: ${progress}%, left: ${left}s`); + } + } + } +} + +function build(platform, arch, target) { + console.log( + `Building: platform: ${platform}, arch: ${arch}, node version: ${target.version}` + ); + + mkdirp.sync(BUILD_FOLDER); + mkdirp.sync(CACHE_FOLDER); + mkdirp.sync(`${PREBUILDS_FOLDER_NAME}/${platform}-${arch}`); + + const output = `${PREBUILDS_FOLDER_NAME}/${platform}-${arch}/node-${target.abi}.node`; + const cmd = [ + 'node-gyp rebuild', + `--target=${target.version}`, + `--target_arch=${arch}`, + `--devdir=${CACHE_FOLDER}`, + '--release', + '--build_v8_with_gn=false', + '--enable_lto=false', + ].join(' '); + + execSync(cmd, { stdio: [0, 1, 2] }); + + fs.copyFileSync(`${BUILD_FOLDER}/metrics.node`, output); + + const sum = checksum(fs.readFileSync(`${BUILD_FOLDER}/metrics.node`)); + fs.writeFileSync(`${output}.sha1`, sum); +} + +function zipBuilds() { + rimraf.sync(ARTIFACTS_FOLDER); + mkdirp.sync(ARTIFACTS_FOLDER); + + tar.create( + { + gzip: true, + sync: true, + portable: true, + strict: true, + noDirRecurse: true, + file: path.join(ARTIFACTS_FOLDER, `${PREBUILDS_FOLDER_NAME}.tgz`), + }, + glob.sync(path.join(PREBUILDS_FOLDER_NAME, '**/*.*')) + ); +} + +function extractFromBuild() { + mkdirp.sync(CACHE_FOLDER); + tar.extract({ + sync: true, + strict: true, + file: path.join(ARTIFACTS_FOLDER, `${PREBUILDS_FOLDER_NAME}.tgz`), + cwd: CACHE_FOLDER, + }); +} + +function validate() { + const folderPath = path.join(CACHE_FOLDER, PREBUILDS_FOLDER_NAME); + const filesChecked = []; + fs.readdirSync(folderPath).forEach(folder => { + fs.readdirSync(path.join(folderPath, folder)) + .filter(file => /^node-\d+\.node$/.test(file)) + .forEach(file => { + const content = fs.readFileSync(path.join(folderPath, folder, file)); + const sum = fs.readFileSync( + path.join(folderPath, folder, `${file}.sha1`), + 'ascii' + ); + if (sum !== checksum(content)) { + throw new Error( + `Invalid checksum for "${PREBUILDS_FOLDER_NAME}/${folder}/${file}".` + ); + } + filesChecked.push(`${folder}/${file}`); + }); + }); + if (filesToBeBuild !== filesChecked.length) { + throw new Error( + `Not all files have been checked, files to be build (${filesToBeBuild}), files to be checked (${filesChecked.length})` + ); + } + const time = Math.round((new Date().getTime() - start) / 1000); + + console.log(`All went fine, it took: ${time}s to build all.`); + console.log( + `Number of files generated and checked: (${filesChecked.length}) ->`, + filesChecked + ); +} + +function createChecksum() { + const file = path.join(ARTIFACTS_FOLDER, `${PREBUILDS_FOLDER_NAME}.tgz`); + const sum = checksum(fs.readFileSync(file)); + fs.writeFileSync(`${file}.sha1`, sum); +} + +function cleanBefore() { + rimraf.sync(CACHE_FOLDER); + rimraf.sync(BUILD_FOLDER); + rimraf.sync(PREBUILDS_FOLDER_NAME); +} + +function cleanAfter() { + rimraf.sync(CACHE_FOLDER); + rimraf.sync(PREBUILDS_FOLDER_NAME); +} diff --git a/packages/opentelemetry-rca-metrics/scripts/build_install.js b/packages/opentelemetry-rca-metrics/scripts/build_install.js new file mode 100644 index 0000000000..bd93c5d17f --- /dev/null +++ b/packages/opentelemetry-rca-metrics/scripts/build_install.js @@ -0,0 +1,99 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const fs = require('fs'); +const rimraf = require('rimraf'); +const mkdirp = require('mkdirp'); +const os = require('os'); +const path = require('path'); +const semver = require('semver'); +const tar = require('tar'); + +const args = process.argv.slice(2); +const obj = {}; +args.forEach(arg => { + const arr = arg.split('='); + if (arr.length === 2) { + obj[arr[0]] = arr[1]; + } +}); + +const platform = obj.platform || os.platform(); +const arch = obj.arch || process.env.ARCH || os.arch(); +const nodeVersion = + obj.version || process.version.replace(/v/g, '').split('.')[0]; + +const MAIN_FOLDER = path.resolve(__dirname, '..'); +const BUILD_FOLDER = path.resolve(MAIN_FOLDER, 'build/Release'); +const ARTIFACTS_FOLDER = path.resolve(MAIN_FOLDER, 'artifacts'); +const CACHE_FOLDER = path.join(MAIN_FOLDER, 'cache'); +const PREBUILDS_FOLDER_NAME = 'prebuilds'; + +// https://nodejs.org/en/download/releases/ +const targets = [ + { version: '8.0.0', abi: '57' }, + { version: '10.0.0', abi: '64' }, + { version: '11.0.0', abi: '67' }, + { version: '12.0.0', abi: '72' }, + { version: '13.0.0', abi: '79' }, + { version: '14.0.0', abi: '83' }, +]; +const targetToCopy = targets.filter(target => + semver.satisfies(target.version, `=${nodeVersion}`) +)[0]; + +if (!targetToCopy) { + throw new Error( + `Provided version for node (${nodeVersion}) is not supported` + ); +} + +cleanBefore(); +extractFromBuild(); +copyToBuildFolder(); +cleanAfter(); + +function extractFromBuild() { + mkdirp.sync(CACHE_FOLDER); + tar.extract({ + sync: true, + strict: true, + file: path.join(ARTIFACTS_FOLDER, `${PREBUILDS_FOLDER_NAME}.tgz`), + cwd: CACHE_FOLDER, + }); +} + +function copyToBuildFolder() { + mkdirp.sync(BUILD_FOLDER); + const src = `${CACHE_FOLDER}/${PREBUILDS_FOLDER_NAME}/${platform}-${arch}/node-${targetToCopy.abi}.node`; + const info = `platform: (${platform}), arch: (${arch}), node version: (${targetToCopy.version})`; + if (!fs.existsSync(src)) { + throw new Error(`No precompiled file found for ${info}`); + } + const dest = `${BUILD_FOLDER}/node-${targetToCopy.abi}.node`; + fs.copyFileSync(src, dest); + console.log(`File for ${info} installed successfully`); +} + +function cleanBefore() { + rimraf.sync(BUILD_FOLDER); +} + +function cleanAfter() { + rimraf.sync(CACHE_FOLDER); +} diff --git a/packages/opentelemetry-rca-metrics/scripts/exec.js b/packages/opentelemetry-rca-metrics/scripts/exec.js new file mode 100644 index 0000000000..9d6a6949eb --- /dev/null +++ b/packages/opentelemetry-rca-metrics/scripts/exec.js @@ -0,0 +1,34 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const execSync = require('child_process').execSync; + +function exec(command, options) { + options = Object.assign({ stdio: [0, 1, 2] }, options); + return execSync(command, options); +} + +function pipe(command, options) { + return exec(command, Object.assign({ stdio: 'pipe' }, options)) + .toString() + .replace(/\n$/, ''); +} + +exec.pipe = pipe; + +module.exports = exec; diff --git a/packages/opentelemetry-rca-metrics/scripts/get_prebuilds.js b/packages/opentelemetry-rca-metrics/scripts/get_prebuilds.js new file mode 100644 index 0000000000..68fb108f2d --- /dev/null +++ b/packages/opentelemetry-rca-metrics/scripts/get_prebuilds.js @@ -0,0 +1,201 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +'use strict'; + +/* eslint-disable no-console */ + +const axios = require('axios'); +const checksum = require('checksum'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const exec = require('./exec.js'); + +const MAIN_FOLDER = path.resolve(__dirname, '..'); +const ARTIFACTS_FOLDER = path.resolve(MAIN_FOLDER, 'artifacts'); + +console.log('Downloading rca-metrics compiled files for release.'); +const TOKEN = process.env.CIRCLE_TOKEN; + +if (!TOKEN) { + throw new Error( + [ + 'The prepublish script needs to authenticate to CircleCI.', + 'Please set the CIRCLE_TOKEN environment variable.', + ].join(' ') + ); +} + +const revision = exec.pipe('git rev-parse HEAD'); + +console.log(revision); + +// const branch = exec.pipe(`git symbolic-ref --short HEAD`); +const branch = 'master'; + +console.log(branch); + +const client = axios.create({ + baseURL: 'https://circleci.com/api/v2/', + timeout: 5000, + headers: { + 'Circle-Token': TOKEN, + }, +}); + +const fetch = (url, options) => { + console.log(`GET ${url}`); + + return client + .get(url, options) + .catch(() => client.get(url, options)) + .catch(() => client.get(url, options)); +}; + +getPipeline() + .then(getWorkflow) + .then(getPrebuildsJob) + .then(getPrebuildArtifacts) + .then(downloadArtifacts) + .then(validatePrebuilds) + .then(copyPrebuilds) + .catch(e => { + process.exitCode = 1; + console.error(e); + }); + +function getPipeline() { + return fetch( + `project/gh/open-telemetry/opentelemetry-js-contrib/pipeline?branch=${branch}` + ).then(response => { + const pipeline = response.data.items.find( + item => item.vcs.revision === revision + ); + + if (!pipeline) { + throw new Error( + `Unable to find CircleCI pipeline for ${branch}@${revision}.` + ); + } + + return pipeline; + }); +} + +function getWorkflow(pipeline) { + return fetch(`pipeline/${pipeline.id}/workflow`).then(response => { + const workflows = response.data.items.sort((a, b) => + a.stopped_at < b.stopped_at ? 1 : -1 + ); + const running = workflows.find(workflow => !workflow.stopped_at); + + if (running) { + throw new Error( + `Workflow ${running.id} is still running for pipeline ${pipeline.id}.` + ); + } + + const workflow = workflows[0]; + + if (!workflow) { + throw new Error( + `Unable to find CircleCI workflow for pipeline ${pipeline.id}.` + ); + } + + if (workflow.status !== 'success') { + throw new Error( + `Aborting because CircleCI workflow ${workflow.id} did not succeed.` + ); + } + + return workflow; + }); +} + +function getPrebuildsJob(workflow) { + return fetch(`workflow/${workflow.id}/job`).then(response => { + const job = response.data.items.find( + item => item.name === 'build-native-stats' + ); + + if (!job) { + throw new Error(`Missing prebuild jobs in workflow ${workflow.id}.`); + } + + return job; + }); +} + +function getPrebuildArtifacts(job) { + return fetch( + `project/github/open-telemetry/opentelemetry-js-contrib/${job.job_number}/artifacts` + ).then(response => { + const artifacts = response.data.items.filter(artifact => + /\/prebuilds\.tgz/.test(artifact.url) + ); + + if (artifacts.length === 0) { + throw new Error(`Missing artifacts in job ${job.job_number}.`); + } + + return artifacts; + }); +} + +function downloadArtifacts(artifacts) { + const files = artifacts.map(artifact => artifact.url); + + return Promise.all(files.map(downloadArtifact)); +} + +function downloadArtifact(file) { + return fetch(file, { responseType: 'stream' }).then(response => { + const parts = file.split('/'); + const basename = os.tmpdir(); + const filename = parts.slice(-1)[0]; + + return new Promise((resolve, reject) => { + response.data + .pipe(fs.createWriteStream(path.join(basename, filename))) + .on('finish', () => resolve()) + .on('error', reject); + }); + }); +} + +function validatePrebuilds() { + const file = path.join(os.tmpdir(), 'prebuilds.tgz'); + const content = fs.readFileSync(file); + const sum = fs.readFileSync(path.join(`${file}.sha1`), 'ascii'); + + if (sum !== checksum(content)) { + throw new Error('Invalid checksum for "prebuilds.tgz".'); + } +} + +function copyPrebuilds() { + const filename = 'prebuilds.tgz'; + + fs.copyFileSync( + path.join(os.tmpdir(), filename), + path.join(ARTIFACTS_FOLDER, filename) + ); + fs.copyFileSync( + path.join(os.tmpdir(), `${filename}.sha1`), + path.join(ARTIFACTS_FOLDER, `${filename}.sha1`) + ); +} diff --git a/packages/opentelemetry-rca-metrics/src/enum.ts b/packages/opentelemetry-rca-metrics/src/enum.ts new file mode 100644 index 0000000000..757eed4066 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/src/enum.ts @@ -0,0 +1,92 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export enum METRIC_NAMES { + CPU = 'cpu', + EVENT_LOOP_DELAY = 'runtime.node.eventLoop.delay', + EVENT_LOOP_DELAY_COUNTER = 'runtime.node.eventLoop.delayCounter', + GC = 'runtime.node.gc.pause', + GC_BY_TYPE = 'runtime.node.gc.pause.by.type', + HEAP = 'runtime.node.heap', + HEAP_SPACE = 'runtime.node.heapSpace', + NETWORK = 'net', + MEMORY = 'mem', + MEMORY_RUNTIME = 'runtime.node.mem', + NATIVE = 'native', + PROCESS = 'runtime.node.process', +} + +export enum CPU_LABELS { + USER = 'user', + SYSTEM = 'sys', + USAGE = 'usage', + TOTAL = 'total', +} + +export enum NETWORK_LABELS { + BYTES_SENT = 'bytesSent', + BYTES_RECEIVED = 'bytesRecv', +} + +export enum MEMORY_LABELS_RUNTIME { + EXTERNAL = 'external', + FREE = 'free', + HEAP_TOTAL = 'heapTotal', + HEAP_USED = 'heapUsed', + RSS = 'rss', +} + +export enum MEMORY_LABELS { + AVAILABLE = 'available', + TOTAL = 'total', +} + +export enum HEAP_LABELS { + TOTAL_HEAP_SIZE = 'totalHeapSize', + TOTAL_HEAP_SIZE_EXECUTABLE = 'totalHeapSizeExecutable', + TOTAL_PHYSICAL_SIZE = 'totalPhysicalSize', + TOTAL_AVAILABLE_SIZE = 'totalAvailableSize', + USED_HEAP_SIZE = 'usedHeapSize', + HEAP_SIZE_LIMIT = 'heapSizeLimit', + MALLOCED_MEMORY = 'mallocedMemory', + PEAK_MALLOCED_MEMORY = 'peakMallocedMemory', + DOES_ZAP_GARBAGE = 'doesZapGarbage', +} + +export enum PROCESS_LABELS { + UP_TIME = 'upTime', +} + +export enum NATIVE_STATS_ITEM { + MIN = 'min', + MAX = 'max', + AVG = 'avg', + MEDIAN = 'median', + P95 = 'p95', +} + +export enum NATIVE_STATS_ITEM_COUNTER { + SUM = 'sum', + TOTAL = 'total', + COUNT = 'count', +} + +export enum NATIVE_SPACE_ITEM { + SPACE_SIZE = 'size', + SPACE_USED_SIZE = 'usedSize', + SPACE_AVAILABLE_SIZE = 'availableSize', + PHYSICAL_SPACE_SIZE = 'physicalSize', +} diff --git a/packages/opentelemetry-rca-metrics/src/index.ts b/packages/opentelemetry-rca-metrics/src/index.ts new file mode 100644 index 0000000000..f494c1353e --- /dev/null +++ b/packages/opentelemetry-rca-metrics/src/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './metric'; diff --git a/packages/opentelemetry-rca-metrics/src/metric.ts b/packages/opentelemetry-rca-metrics/src/metric.ts new file mode 100644 index 0000000000..7b6f5fd74e --- /dev/null +++ b/packages/opentelemetry-rca-metrics/src/metric.ts @@ -0,0 +1,549 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as api from '@opentelemetry/api'; +import * as metrics from '@opentelemetry/metrics'; +import * as enums from './enum'; + +import { + getCpuUsageData, + getHeapData, + getMemoryData, + getProcessData, +} from './stats/common'; +import { getStats } from './stats/native'; +import { getNetworkData } from './stats/si'; + +import * as types from './types'; + +/** + * Metrics Collector Configuration + */ +interface MetricsCollectorConfig { + logger?: api.Logger; + exporter: metrics.MetricExporter; + // maximum timeout to wait for stats collection default is 500ms + maxTimeoutUpdateMS?: number; + // Character to be used to join metrics - default is "." + metricNameSeparator?: string; + // Name of component + name: string; + // metric export endpoint + url: string; + // How often the metrics should be exported + interval?: number; +} + +const DEFAULT_INTERVAL = 60 * 1000; +const DEFAULT_MAX_TIMEOUT_UPDATE_MS = 500; +const DEFAULT_NAME = 'opentelemetry-metrics-collector'; +const DEFAULT_METRIC_NAME_SEPARATOR = '.'; + +// default label name to be used to store metric name +const DEFAULT_KEY = 'name'; + +/** + * Metrics Collector - collects metrics for CPU, Memory, Heap, Network, Event + * Loop, Garbage Collector, Heap Space + * the default label name for metric name is "name" + */ +export class RCAMetrics { + private _intervalExport: number | undefined; + private _exporter: metrics.MetricExporter; + private _logger: api.Logger | undefined; + private _maxTimeoutUpdateMS: number; + private _meter: metrics.Meter; + private _name: string; + private _boundCounters: { [key: string]: api.BoundCounter } = {}; + private _metricNameSeparator: string; + + private _memValueObserver: types.ValueObserverWithObservations | undefined; + private _memRuntimeValueObserver: + | types.ValueObserverWithObservations + | undefined; + private _heapValueObserver: types.ValueObserverWithObservations | undefined; + private _procesUptimeValueObserver: + | types.ValueObserverWithObservations + | undefined; + private _eventLoopValueObserver: + | types.ValueObserverWithObservations + | undefined; + private _gcValueObserver: types.ValueObserverWithObservations | undefined; + private _gcByTypeValueObserver: + | types.ValueObserverWithObservations + | undefined; + private _heapSpaceValueObserver: + | types.ValueObserverWithObservations + | undefined; + + constructor(config: MetricsCollectorConfig) { + this._intervalExport = + typeof config.interval === 'number' ? config.interval : DEFAULT_INTERVAL; + this._exporter = config.exporter; + this._logger = config.logger; + this._name = config.name || DEFAULT_NAME; + this._maxTimeoutUpdateMS = + config.maxTimeoutUpdateMS || DEFAULT_MAX_TIMEOUT_UPDATE_MS; + this._metricNameSeparator = + config.metricNameSeparator || DEFAULT_METRIC_NAME_SEPARATOR; + this._meter = new metrics.MeterProvider({ + interval: this._intervalExport, + exporter: this._exporter, + }).getMeter(this._name); + } + + /** + * Creates a metric key name based on metric name and a key + * @param metricName + * @param key + */ + private _boundKey(metricName: string, key: string) { + if (!key) { + return metricName; + } + return `${metricName}${this._metricNameSeparator}${key}`; + } + + /** + * Updates counter based on boundkey + * @param metricName + * @param key + * @param value + */ + private _counterUpdate(metricName: string, key: string, value = 0) { + const boundKey = this._boundKey(metricName, key); + this._boundCounters[boundKey].add(value); + } + + /** + * @param metricName metric name - this will be added as label under name + * "name" + * @param values values to be used to generate bound counters for each + * value prefixed with metricName + * @param description metric description + */ + private _createCounter( + metricName: string, + values: string[], + description?: string + ) { + const keys = values.map(key => this._boundKey(metricName, key)); + const counter = this._meter.createCounter(metricName, { + description: description || metricName, + }); + keys.forEach(key => { + this._boundCounters[key] = counter.bind({ [DEFAULT_KEY]: key }); + }); + } + + /** + * @param metricName metric name - this will be added as label under name + * "name" + * @param values values to be used to generate full metric name + * (metricName + value) + * value prefixed with metricName + * @param description metric description + * @param labelKey extra label to be observed + * @param labelValues label values to be observed based on labelKey + * @param afterKey extra name to be added to full metric name + */ + private _createValueObserver( + metricName: string, + values: string[], + description: string, + labelKey = '', + labelValues: string[] = [], + afterKey = '' + ): types.ValueObserverWithObservations { + const labelKeys = [DEFAULT_KEY]; + if (labelKey) { + labelKeys.push(labelKey); + } + const observer = this._meter.createValueObserver(metricName, { + description: description || metricName, + }); + + const observations: types.Observations[] = []; + values.forEach(value => { + const boundKey = this._boundKey( + this._boundKey(metricName, value), + afterKey + ); + if (labelKey) { + // there is extra label to be observed mixed with default one + // for example we want to be able to observe "name" and "gc_type" + labelValues.forEach(label => { + const observedLabels = Object.assign( + {}, + { [DEFAULT_KEY]: boundKey }, + { + [labelKey]: label, + } + ); + observations.push({ + key: value, + labels: observedLabels, + labelKey, + }); + }); + } else { + observations.push({ + key: value, + labels: { [DEFAULT_KEY]: boundKey }, + }); + } + }); + + return { observer, observations }; + } + + // MEMORY + private _createMemValueObserver() { + this._memValueObserver = this._createValueObserver( + enums.METRIC_NAMES.MEMORY, + Object.values(enums.MEMORY_LABELS), + 'Memory' + ); + } + + // MEMORY RUNTIME + private _createMemRuntimeValueObserver() { + this._memRuntimeValueObserver = this._createValueObserver( + enums.METRIC_NAMES.MEMORY_RUNTIME, + Object.values(enums.MEMORY_LABELS_RUNTIME), + 'Memory Runtime' + ); + } + + // HEAP + private _createHeapValueObserver() { + this._heapValueObserver = this._createValueObserver( + enums.METRIC_NAMES.HEAP, + Object.values(enums.HEAP_LABELS), + 'Heap Data' + ); + } + + // PROCESS + private _createProcesUptimeValueObserver() { + this._procesUptimeValueObserver = this._createValueObserver( + enums.METRIC_NAMES.PROCESS, + Object.values(enums.PROCESS_LABELS), + 'Process UpTime' + ); + } + + // EVENT LOOP + private _createEventLoopValueObserver() { + this._eventLoopValueObserver = this._createValueObserver( + enums.METRIC_NAMES.EVENT_LOOP_DELAY, + Object.values(enums.NATIVE_STATS_ITEM), + 'Event Loop' + ); + } + + // GC ALL + private _createGCValueObserver() { + this._gcValueObserver = this._createValueObserver( + enums.METRIC_NAMES.GC, + Object.values(enums.NATIVE_STATS_ITEM), + 'GC for all' + ); + } + + // GC BY TYPE + private _createGCByTypeValueObserver() { + this._gcByTypeValueObserver = this._createValueObserver( + enums.METRIC_NAMES.GC_BY_TYPE, + Object.values(enums.NATIVE_STATS_ITEM), + 'GC by type', + 'gc_type', + [ + 'scavenge', + 'markSweepCompact', + 'incrementalMarking', + 'processWeakCallbacks', + ] + ); + } + + // HEAP SPACE + private _createHeapSpaceValueObserver() { + const stats = getStats(); + const spacesLabels = stats?.heap.spaces.map(space => space.spaceName); + this._heapSpaceValueObserver = this._createValueObserver( + enums.METRIC_NAMES.HEAP_SPACE, + Object.values(enums.NATIVE_SPACE_ITEM), + 'Heap Spaces', + 'heap_space', + spacesLabels, + this._boundKey('by', 'space') + ); + } + + /** + * Updates observer + * @param observerBatchResult + * @param data + * @param observerWithObservations + */ + private _updateObserver( + observerBatchResult: api.BatchObserverResult, + data: DataType, + observerWithObservations?: types.ValueObserverWithObservations + ) { + if (observerWithObservations) { + observerWithObservations.observations.forEach(observation => { + const value = data[observation.key as keyof DataType]; + if (typeof value === 'number') { + observerBatchResult.observe(observation.labels, [ + observerWithObservations.observer.observation(value), + ]); + } + }); + } + } + + /** + * Updates observer with heap spaces + * @param observerBatchResult + * @param stats + * @param observerWithObservations + */ + private _updateObserverSpaces( + observerBatchResult: api.BatchObserverResult, + stats: types.NativeStats | undefined, + observerWithObservations?: types.ValueObserverWithObservations + ) { + if (observerWithObservations && stats) { + observerWithObservations.observations.forEach(observation => { + const stat = stats?.heap.spaces.find(space => { + return space.spaceName === observation.labels['heap_space']; + }); + let value; + if (stat) { + value = + stat[observation.key as keyof types.NativeStatsSpaceItemNumbers]; + } + if (typeof value === 'number') { + observerBatchResult.observe(observation.labels, [ + observerWithObservations.observer.observation(value), + ]); + } + }); + } + } + + /** + * Updates observer with gc types + * @param observerBatchResult + * @param stats + * @param observerWithObservations + */ + private _updateObserverGCByType( + observerBatchResult: api.BatchObserverResult, + stats: types.NativeStats | undefined, + observerWithObservations?: types.ValueObserverWithObservations + ) { + if (observerWithObservations && stats) { + observerWithObservations.observations.forEach(observation => { + const type = observation.labelKey; + if (!type) { + return; + } + const stat = stats?.gc[observation.labels[type]]; + let value; + if (stat) { + value = stat[observation.key as keyof types.NativeStatsItem]; + } + if (typeof value === 'number') { + observerBatchResult.observe(observation.labels, [ + observerWithObservations.observer.observation(value), + ]); + } + }); + } + } + + /** + * Creates metrics + */ + private _createMetrics() { + // CPU COUNTER + this._createCounter( + enums.METRIC_NAMES.CPU, + Object.values(enums.CPU_LABELS), + 'CPU Usage' + ); + + // NETWORK COUNTER + this._createCounter( + enums.METRIC_NAMES.NETWORK, + Object.values(enums.NETWORK_LABELS), + 'Network Usage' + ); + + // EVENT LOOP COUNTERS + this._createCounter( + enums.METRIC_NAMES.EVENT_LOOP_DELAY_COUNTER, + Object.values(enums.NATIVE_STATS_ITEM_COUNTER), + 'Event Loop' + ); + + // MEMORY + this._createMemValueObserver(); + // MEMORY RUNTIME + this._createMemRuntimeValueObserver(); + // HEAP + this._createHeapValueObserver(); + // PROCESS + this._createProcesUptimeValueObserver(); + // EVENT LOOP + this._createEventLoopValueObserver(); + // GC ALL + this._createGCValueObserver(); + // GC BY TYPE + this._createGCByTypeValueObserver(); + // HEAP SPACE + this._createHeapSpaceValueObserver(); + + this._meter.createBatchObserver( + 'metric_batch_observer', + observerBatchResult => { + Promise.all([ + getMemoryData(), + getHeapData(), + getProcessData(), + getStats(), + getCpuUsageData(), + getNetworkData(), + ]).then( + ([ + memoryData, + heapData, + processData, + stats, + cpuUsage, + networkData, + ]) => { + // CPU COUNTER + Object.values(enums.CPU_LABELS).forEach(value => { + this._counterUpdate( + enums.METRIC_NAMES.CPU, + value, + cpuUsage[value] + ); + }); + + // NETWORK COUNTER + Object.values(enums.NETWORK_LABELS).forEach(value => { + this._counterUpdate( + enums.METRIC_NAMES.NETWORK, + value, + networkData[value] + ); + }); + + // EVENT LOOP COUNTERS + Object.values(enums.NATIVE_STATS_ITEM_COUNTER).forEach(value => { + this._counterUpdate( + enums.METRIC_NAMES.EVENT_LOOP_DELAY_COUNTER, + value, + stats?.eventLoop[value] + ); + }); + + // MEMORY + this._updateObserver( + observerBatchResult, + memoryData, + this._memValueObserver + ); + + // MEMORY RUNTIME + this._updateObserver( + observerBatchResult, + memoryData, + this._memRuntimeValueObserver + ); + + // HEAP + this._updateObserver( + observerBatchResult, + heapData, + this._heapValueObserver + ); + + // PROCESS + this._updateObserver( + observerBatchResult, + processData, + this._procesUptimeValueObserver + ); + // EVENT LOOP + this._updateObserver( + observerBatchResult, + stats?.eventLoop, + this._eventLoopValueObserver + ); + + // GC ALL + this._updateObserver( + observerBatchResult, + stats?.gc.all, + this._gcValueObserver + ); + + // GC BY TYPE + this._updateObserverGCByType( + observerBatchResult, + stats, + this._gcByTypeValueObserver + ); + + // HEAP SPACE + this._updateObserverSpaces( + observerBatchResult, + stats, + this._heapSpaceValueObserver + ); + } + ); + }, + { + maxTimeoutUpdateMS: this._maxTimeoutUpdateMS, + logger: this._logger, + } + ); + } + + /** + * Starts collecting stats + */ + start() { + // initial collection + Promise.all([ + getMemoryData(), + getHeapData(), + getProcessData(), + getStats(), + getCpuUsageData(), + getNetworkData(), + ]).then(() => { + this._createMetrics(); + }); + } +} diff --git a/packages/opentelemetry-rca-metrics/src/stats/common.ts b/packages/opentelemetry-rca-metrics/src/stats/common.ts new file mode 100644 index 0000000000..f9a90d9406 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/src/stats/common.ts @@ -0,0 +1,86 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as v8 from 'v8'; +import * as os from 'os'; +import { + CPU_LABELS, + HEAP_LABELS, + MEMORY_LABELS, + MEMORY_LABELS_RUNTIME, + PROCESS_LABELS, +} from '../enum'; +import { CpuUsageData, HeapData, MemoryData, ProcessData } from '../types'; + +const MICROSECOND = 1 / 1e6; +let cpuUsage: NodeJS.CpuUsage | undefined; + +/** + * It returns cpu load delta from last time + */ +export function getCpuUsageData(): CpuUsageData { + const elapsedUsage = process.cpuUsage(cpuUsage); + cpuUsage = process.cpuUsage(); + return { + [CPU_LABELS.USER]: elapsedUsage.user * MICROSECOND, + [CPU_LABELS.SYSTEM]: elapsedUsage.system * MICROSECOND, + [CPU_LABELS.USAGE]: (elapsedUsage.user + elapsedUsage.system) * MICROSECOND, + [CPU_LABELS.TOTAL]: (cpuUsage.user + cpuUsage.system) * MICROSECOND, + }; +} + +/** + * Returns memory data stats + */ +export function getMemoryData(): MemoryData { + const memoryUsage = process.memoryUsage(); + return { + [MEMORY_LABELS.AVAILABLE]: os.freemem(), + [MEMORY_LABELS_RUNTIME.EXTERNAL]: memoryUsage.external, + [MEMORY_LABELS_RUNTIME.FREE]: os.freemem(), + [MEMORY_LABELS_RUNTIME.HEAP_TOTAL]: memoryUsage.heapTotal, + [MEMORY_LABELS_RUNTIME.HEAP_USED]: memoryUsage.heapUsed, + [MEMORY_LABELS_RUNTIME.RSS]: memoryUsage.rss, + [MEMORY_LABELS.TOTAL]: os.totalmem(), + }; +} + +/** + * Returns heap data stats + */ +export function getHeapData(): HeapData { + const stats = v8.getHeapStatistics(); + return { + [HEAP_LABELS.TOTAL_HEAP_SIZE]: stats.total_heap_size, + [HEAP_LABELS.TOTAL_HEAP_SIZE_EXECUTABLE]: stats.total_heap_size_executable, + [HEAP_LABELS.TOTAL_PHYSICAL_SIZE]: stats.total_physical_size, + [HEAP_LABELS.TOTAL_AVAILABLE_SIZE]: stats.total_available_size, + [HEAP_LABELS.USED_HEAP_SIZE]: stats.used_heap_size, + [HEAP_LABELS.HEAP_SIZE_LIMIT]: stats.heap_size_limit, + [HEAP_LABELS.MALLOCED_MEMORY]: stats.malloced_memory, + [HEAP_LABELS.PEAK_MALLOCED_MEMORY]: stats.peak_malloced_memory, + [HEAP_LABELS.DOES_ZAP_GARBAGE]: stats.does_zap_garbage, + }; +} + +/** + * Returns process uptime stats stats + */ +export function getProcessData(): ProcessData { + return { + [PROCESS_LABELS.UP_TIME]: Math.round(process.uptime()), + }; +} diff --git a/packages/opentelemetry-rca-metrics/src/stats/native.ts b/packages/opentelemetry-rca-metrics/src/stats/native.ts new file mode 100644 index 0000000000..c793a0b413 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/src/stats/native.ts @@ -0,0 +1,46 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// eslint-disable-next-line @typescript-eslint/no-var-requires +const nodeGypBuild = require('node-gyp-build'); +import * as path from 'path'; +import { NativeStats, NativeStatsObj } from '../types'; + +const base = path.resolve(`${__dirname}/../../..`); +let nativeMetrics: NativeStatsObj; + +/** + * Returns native stats (event loop, gc, heap spaces) + */ +export function getStats(): NativeStats | undefined { + if (!nativeMetrics) { + try { + nativeMetrics = nodeGypBuild(base); + nativeMetrics.start(); + } catch (e) { + console.log(e.message); + } + } + const stats: NativeStats | undefined = nativeMetrics + ? nativeMetrics.stats() + : undefined; + if (stats) { + stats.eventLoop.total = stats.eventLoop.sum; + Object.keys(stats.gc).forEach(key => { + stats.gc[key].total = stats.gc[key].sum; + }); + } + return stats; +} diff --git a/packages/opentelemetry-rca-metrics/src/stats/si.ts b/packages/opentelemetry-rca-metrics/src/stats/si.ts new file mode 100644 index 0000000000..4fb8c1f292 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/src/stats/si.ts @@ -0,0 +1,51 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as SI from 'systeminformation'; +import { NetworkData } from '../types'; +import { ObjectKeys } from '../util'; + +let previousNetworkStats: Partial = {}; + +/** + * It returns network usage delta from last time + */ +export function getNetworkData() { + return new Promise(resolve => { + const stats: NetworkData = { + bytesRecv: 0, + bytesSent: 0, + }; + SI.networkStats() + .then(results => { + results.forEach(result => { + stats.bytesRecv += result.rx_bytes; + stats.bytesSent += result.tx_bytes; + }); + const lastStats = Object.assign({}, stats); + + ObjectKeys(stats).forEach(key => { + stats[key] = stats[key] - (previousNetworkStats[key] || 0); + }); + + previousNetworkStats = lastStats; + resolve(stats); + }) + .catch(() => { + resolve(stats); + }); + }); +} diff --git a/packages/opentelemetry-rca-metrics/src/types.ts b/packages/opentelemetry-rca-metrics/src/types.ts new file mode 100644 index 0000000000..2a791c5706 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/src/types.ts @@ -0,0 +1,137 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import type { ValueObserver } from '@opentelemetry/api'; +import { DoesZapCodeSpaceFlag } from 'v8'; + +/** + * Network data + */ +export interface NetworkData { + // bytes received + bytesRecv: number; + // bytes sent + bytesSent: number; +} + +/** + * Process data + */ +export interface ProcessData { + upTime: number; +} + +/** + * CPU usage data + */ +export interface CpuUsageData { + sys: number; + usage: number; + user: number; + total: number; +} + +/** + * Memory data + */ +export interface MemoryData { + available: number; + external: number; + free: number; + heapTotal: number; + heapUsed: number; + rss: number; + total: number; +} + +/** + * Heap Data + */ +export interface HeapData { + doesZapGarbage: DoesZapCodeSpaceFlag; + heapSizeLimit: number; + mallocedMemory: number; + peakMallocedMemory: number; + totalAvailableSize: number; + totalHeapSize: number; + totalHeapSizeExecutable: number; + totalPhysicalSize: number; + usedHeapSize: number; +} + +/** + * Native stats interface + */ +export interface NativeStatsObj { + // returns native stats + stats: () => NativeStats; + // start collecting stats + start: () => void; + // stops collecting stats + stop: () => void; +} + +/** + * Native stats (event loop, gc, heap spaces) + */ +export interface NativeStats { + eventLoop: NativeStatsItem; + gc: { [key: string]: NativeStatsItem }; + heap: { + spaces: (NativeStatsSpaceItem & NativeStatsSpaceItemNumbers)[]; + }; +} + +/** + * Native stats space item that with string values only + */ +export interface NativeStatsSpaceItem { + spaceName: string; +} + +/** + * Native stats space item that with number values only + */ +export interface NativeStatsSpaceItemNumbers { + availableSize: number; + physicalSpaceSize: number; + size: number; + usedSize: number; +} + +/** + * Native stats item + */ +export interface NativeStatsItem { + avg: number; + count: number; + max: number; + median: number; + min: number; + p95: number; + sum: number; + total: number; +} + +export interface Observations { + key: string; + labels: Record; + labelKey?: string; +} + +export interface ValueObserverWithObservations { + observer: ValueObserver; + observations: Observations[]; +} diff --git a/packages/opentelemetry-rca-metrics/src/util.ts b/packages/opentelemetry-rca-metrics/src/util.ts new file mode 100644 index 0000000000..e26b0f7610 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/src/util.ts @@ -0,0 +1,19 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function ObjectKeys(t: T) { + return Object.keys(t) as (keyof T)[]; +} diff --git a/packages/opentelemetry-rca-metrics/src/version.ts b/packages/opentelemetry-rca-metrics/src/version.ts new file mode 100644 index 0000000000..2c92beb616 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/src/version.ts @@ -0,0 +1,18 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// this is autogenerated file, see scripts/version-update.js +export const VERSION = '0.9.0'; diff --git a/packages/opentelemetry-rca-metrics/test/metric.test.ts b/packages/opentelemetry-rca-metrics/test/metric.test.ts new file mode 100644 index 0000000000..e9f8df6b8a --- /dev/null +++ b/packages/opentelemetry-rca-metrics/test/metric.test.ts @@ -0,0 +1,522 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const mock = require('mock-require'); +const v8 = require('v8'); +const SI = require('systeminformation'); +import { ExportResult } from '@opentelemetry/core'; +import { + Distribution, + MetricExporter, + MetricRecord, +} from '@opentelemetry/metrics'; +import * as assert from 'assert'; +import * as os from 'os'; +import * as sinon from 'sinon'; + +const cpuJson = require('./mocks/cpu.json'); +const networkJson = require('./mocks/network.json'); +const nativeJson = require('./mocks/native.json'); +const memoryJson = require('./mocks/memory.json'); +const heapJson = require('./mocks/heap.json'); + +class NoopExporter implements MetricExporter { + export( + metrics: MetricRecord[], + resultCallback: (result: ExportResult) => void + ): void { + // console.log('>>>>>>>>>>>>> EXPORTING', metrics.length); + } + + shutdown(): void {} +} + +const originalSetTimeout = setTimeout; + +const GC_VALUES = ['min', 'max', 'avg', 'median', 'p95']; +const HEAP_SPACE_VALUES = ['size', 'usedSize', 'availableSize', 'physicalSize']; +const mockedNative = { + start: function () {}, + stats: function () { + return nativeJson; + }, +}; + +let countSI = 0; +const mockedSI = { + networkStats: function () { + return new Promise((resolve, reject) => { + countSI++; + const stats: any[] = networkJson + .slice() + .map((obj: any) => Object.assign({}, obj)); + + for (let i = 0, j = networkJson.length; i < j; i++) { + Object.keys(stats[i]).forEach(key => { + if (typeof stats[i][key] === 'number' && stats[i][key] > 0) { + stats[i][key] = stats[i][key] * countSI; + } + }); + } + resolve(stats); + }); + }, +}; + +const mockedOS = { + freemem: function () { + return 7179869184; + }, + totalmem: function () { + return 17179869184; + }, +}; +const mockedUptime = 1405; + +const INTERVAL = 3000; + +let metrics: any; + +// @TODO uncomment this once +// https://github.com/open-telemetry/opentelemetry-js/pull/1470 is merged and +// released +xdescribe('RCA Metrics', () => { + let sandbox: sinon.SinonSandbox; + let rcaMetrics: any; + let exporter: MetricExporter; + let exportSpy: any; + + beforeEach(done => { + sandbox = sinon.createSandbox(); + sandbox.useFakeTimers(); + + mock('node-gyp-build', () => { + return mockedNative; + }); + + sandbox.stub(os, 'freemem').returns(mockedOS.freemem()); + sandbox.stub(os, 'totalmem').returns(mockedOS.totalmem()); + sandbox.stub(v8, 'getHeapStatistics').returns(heapJson); + sandbox.stub(process, 'cpuUsage').returns(cpuJson); + sandbox.stub(process, 'memoryUsage').returns(memoryJson); + sandbox.stub(process, 'uptime').returns(mockedUptime); + const spyNetworkStats = sandbox + .stub(SI, 'networkStats') + .returns(mockedSI.networkStats()); + + exporter = new NoopExporter(); + exportSpy = sandbox.stub(exporter, 'export'); + + // it seems like this is the only way to be able to mock + // `node-gyp-build` before metrics are being loaded, if import them before + // the first pass on unit tests will not mock correctly + metrics = require('../src'); + rcaMetrics = new metrics.RCAMetrics({ + exporter, + interval: INTERVAL, + name: 'opentelemetry-rca-metrics', + url: '', + }); + rcaMetrics.start(); + + // because networkStats mock simulates the network with every call it + // returns the data that is bigger then previous, it needs to stub it again + // as network is also called in initial start to start counting from 0 + spyNetworkStats.restore(); + sandbox.stub(SI, 'networkStats').returns(mockedSI.networkStats()); + + // sinon fake doesn't work fine with setImmediate + originalSetTimeout(() => { + // move the clock with the same value as interval + sandbox.clock.tick(INTERVAL); + // move to "real" next tick so that async batcher observer will start + // processing metrics + originalSetTimeout(() => { + // allow all calbacks to finish correctly as they are finishing in + // next tick due to async + sandbox.clock.tick(1); + originalSetTimeout(() => { + done(); + }); + }); + }); + }); + afterEach(() => { + sandbox.restore(); + }); + + it('should create a new instance', () => { + assert.ok(rcaMetrics instanceof metrics.RCAMetrics); + }); + + it('should export CPU metrics', () => { + const records = getRecords(exportSpy.args[0][0], 'cpu'); + assert.strictEqual(records.length, 4); + ensureValue(records[0], 'cpu.user', 1.899243); + ensureValue(records[1], 'cpu.sys', 0.258553); + ensureValue(records[2], 'cpu.usage', 2.157796); + ensureValue(records[3], 'cpu.total', 2.157796); + }); + + it('should export Network metrics', done => { + const records = getRecords(exportSpy.args[0][0], 'net'); + assert.strictEqual(records.length, 2); + ensureValue(records[0], 'net.bytesSent', 14207163202); + ensureValue(records[1], 'net.bytesRecv', 60073930753); + done(); + }); + + it('should export Memory metrics', done => { + const records = getRecords(exportSpy.args[0][0], 'mem'); + assert.strictEqual(records.length, 2); + ensureValue(records[0], 'mem.available', mockedOS.freemem()); + ensureValue(records[1], 'mem.total', mockedOS.totalmem()); + done(); + }); + + it('should export Memory runtime metrics', () => { + const recordsRuntime = getRecords(exportSpy.args[0][0], 'runtime.node.mem'); + assert.strictEqual(recordsRuntime.length, 5); + ensureValue( + recordsRuntime[0], + 'runtime.node.mem.external', + memoryJson.external + ); + ensureValue(recordsRuntime[1], 'runtime.node.mem.free', mockedOS.freemem()); + ensureValue( + recordsRuntime[2], + 'runtime.node.mem.heapTotal', + memoryJson.heapTotal + ); + ensureValue( + recordsRuntime[3], + 'runtime.node.mem.heapUsed', + memoryJson.heapUsed + ); + ensureValue(recordsRuntime[4], 'runtime.node.mem.rss', memoryJson.rss); + }); + + it('should export Heap metrics', () => { + const records = getRecords(exportSpy.args[0][0], 'runtime.node.heap'); + assert.strictEqual(records.length, 9); + ensureValue( + records[0], + 'runtime.node.heap.totalHeapSize', + heapJson.total_heap_size + ); + ensureValue( + records[1], + 'runtime.node.heap.totalHeapSizeExecutable', + heapJson.total_heap_size_executable + ); + ensureValue( + records[2], + 'runtime.node.heap.totalPhysicalSize', + heapJson.total_physical_size + ); + ensureValue( + records[3], + 'runtime.node.heap.totalAvailableSize', + heapJson.total_available_size + ); + ensureValue( + records[4], + 'runtime.node.heap.usedHeapSize', + heapJson.used_heap_size + ); + ensureValue( + records[5], + 'runtime.node.heap.heapSizeLimit', + heapJson.heap_size_limit + ); + ensureValue( + records[6], + 'runtime.node.heap.mallocedMemory', + heapJson.malloced_memory + ); + ensureValue( + records[7], + 'runtime.node.heap.peakMallocedMemory', + heapJson.peak_malloced_memory + ); + ensureValue( + records[8], + 'runtime.node.heap.doesZapGarbage', + heapJson.does_zap_garbage + ); + }); + + it('should export Uptime metrics', () => { + const records = getRecords(exportSpy.args[0][0], 'runtime.node.process'); + assert.strictEqual(records.length, 1); + ensureValue(records[0], 'runtime.node.process.upTime', mockedUptime); + }); + + it('should export Event Loop metrics', () => { + const records = getRecords( + exportSpy.args[0][0], + 'runtime.node.eventLoop.delay' + ); + assert.strictEqual(records.length, 5); + ensureValue( + records[0], + 'runtime.node.eventLoop.delay.min', + nativeJson.eventLoop.min + ); + ensureValue( + records[1], + 'runtime.node.eventLoop.delay.max', + nativeJson.eventLoop.max + ); + ensureValue( + records[2], + 'runtime.node.eventLoop.delay.avg', + nativeJson.eventLoop.avg + ); + ensureValue( + records[3], + 'runtime.node.eventLoop.delay.median', + nativeJson.eventLoop.median + ); + ensureValue( + records[4], + 'runtime.node.eventLoop.delay.p95', + nativeJson.eventLoop.p95 + ); + }); + + it('should export Event Loop metrics', () => { + const records = getRecords( + exportSpy.args[0][0], + 'runtime.node.eventLoop.delay' + ); + assert.strictEqual(records.length, 5); + ensureValue( + records[0], + 'runtime.node.eventLoop.delay.min', + nativeJson.eventLoop.min + ); + ensureValue( + records[1], + 'runtime.node.eventLoop.delay.max', + nativeJson.eventLoop.max + ); + ensureValue( + records[2], + 'runtime.node.eventLoop.delay.avg', + nativeJson.eventLoop.avg + ); + ensureValue( + records[3], + 'runtime.node.eventLoop.delay.median', + nativeJson.eventLoop.median + ); + ensureValue( + records[4], + 'runtime.node.eventLoop.delay.p95', + nativeJson.eventLoop.p95 + ); + }); + + it('should export Event Loop counter metrics', () => { + const records = getRecords( + exportSpy.args[0][0], + 'runtime.node.eventLoop.delayCounter' + ); + assert.strictEqual(records.length, 3); + ensureValue( + records[0], + 'runtime.node.eventLoop.delayCounter.sum', + nativeJson.eventLoop.sum + ); + ensureValue( + records[1], + 'runtime.node.eventLoop.delayCounter.total', + nativeJson.eventLoop.total + ); + ensureValue( + records[2], + 'runtime.node.eventLoop.delayCounter.count', + nativeJson.eventLoop.count + ); + }); + + it('should export Garbage Collector metrics "all"', () => { + const records = getRecords(exportSpy.args[0][0], 'runtime.node.gc.pause'); + assert.strictEqual(records.length, 5); + ensureValue(records[0], 'runtime.node.gc.pause.min', nativeJson.gc.all.min); + ensureValue(records[1], 'runtime.node.gc.pause.max', nativeJson.gc.all.max); + ensureValue(records[2], 'runtime.node.gc.pause.avg', nativeJson.gc.all.avg); + ensureValue( + records[3], + 'runtime.node.gc.pause.median', + nativeJson.gc.all.median + ); + ensureValue(records[4], 'runtime.node.gc.pause.p95', nativeJson.gc.all.p95); + }); + + it('should export Garbage Collector metrics by type', () => { + const records = getRecords( + exportSpy.args[0][0], + 'runtime.node.gc.pause.by.type' + ); + assert.strictEqual(records.length, 20); + }); + + it('should export Garbage Collector metrics by type "scavenge"', () => { + const records = getRecords( + exportSpy.args[0][0], + 'runtime.node.gc.pause.by.type' + ); + ensureGCValues(records, 0, 4, 'scavenge', GC_VALUES); + }); + + it('should export Garbage Collector metrics by type "markSweepCompact"', () => { + const records = getRecords( + exportSpy.args[0][0], + 'runtime.node.gc.pause.by.type' + ); + ensureGCValues(records, 1, 4, 'markSweepCompact', GC_VALUES); + }); + + it('should export Garbage Collector metrics by type "incrementalMarking"', () => { + const records = getRecords( + exportSpy.args[0][0], + 'runtime.node.gc.pause.by.type' + ); + ensureGCValues(records, 2, 4, 'incrementalMarking', GC_VALUES); + }); + + it('should export Garbage Collector metrics by type "processWeakCallbacks"', () => { + const records = getRecords( + exportSpy.args[0][0], + 'runtime.node.gc.pause.by.type' + ); + ensureGCValues(records, 3, 4, 'processWeakCallbacks', GC_VALUES); + }); + + it('should export heap spaces metrics', () => { + const records = getRecords(exportSpy.args[0][0], 'runtime.node.heapSpace'); + assert.strictEqual(records.length, 32); + }); + + it('should export heap spaces metrics for type "read_only_space"', () => { + const records = getRecords(exportSpy.args[0][0], 'runtime.node.heapSpace'); + ensureHeapSpaceValues(records, 0, 8, 'read_only_space', HEAP_SPACE_VALUES); + }); + + it('should export heap spaces metrics for type "new_space"', () => { + const records = getRecords(exportSpy.args[0][0], 'runtime.node.heapSpace'); + ensureHeapSpaceValues(records, 1, 8, 'new_space', HEAP_SPACE_VALUES); + }); + + it('should export heap spaces metrics for type "old_space"', () => { + const records = getRecords(exportSpy.args[0][0], 'runtime.node.heapSpace'); + ensureHeapSpaceValues(records, 2, 8, 'old_space', HEAP_SPACE_VALUES); + }); + + it('should export heap spaces metrics for type "code_space"', () => { + const records = getRecords(exportSpy.args[0][0], 'runtime.node.heapSpace'); + ensureHeapSpaceValues(records, 3, 8, 'code_space', HEAP_SPACE_VALUES); + }); + + it('should export heap spaces metrics for type "map_space"', () => { + const records = getRecords(exportSpy.args[0][0], 'runtime.node.heapSpace'); + ensureHeapSpaceValues(records, 4, 8, 'map_space', HEAP_SPACE_VALUES); + }); + + it('should export heap spaces metrics for type "large_object_space"', () => { + const records = getRecords(exportSpy.args[0][0], 'runtime.node.heapSpace'); + ensureHeapSpaceValues( + records, + 5, + 8, + 'large_object_space', + HEAP_SPACE_VALUES + ); + }); + + it('should export heap spaces metrics for type "code_large_object_space"', () => { + const records = getRecords(exportSpy.args[0][0], 'runtime.node.heapSpace'); + ensureHeapSpaceValues( + records, + 6, + 8, + 'code_large_object_space', + HEAP_SPACE_VALUES + ); + }); + + it('should export heap spaces metrics for type "new_large_object_space"', () => { + const records = getRecords(exportSpy.args[0][0], 'runtime.node.heapSpace'); + ensureHeapSpaceValues( + records, + 7, + 8, + 'new_large_object_space', + HEAP_SPACE_VALUES + ); + }); +}); + +function getRecords(records: MetricRecord[], name: string): MetricRecord[] { + return records.filter(record => record.descriptor.name === name); +} + +function ensureValue(record: MetricRecord, name: string, value: number) { + assert.strictEqual(record.labels.name, name); + const point = record.aggregator.toPoint(); + const aggValue = + typeof point.value === 'number' + ? point.value + : (point.value as Distribution).min; + assert.strictEqual(aggValue, value); +} + +function ensureGCValues( + records: MetricRecord[], + start: number, + step: number, + name: string, + values: string[] +) { + for (let i = 0, j = values.length; i < j; i++) { + ensureValue( + records[i * step + start], + `runtime.node.gc.pause.by.type.${values[i]}`, + nativeJson.gc[name][values[i]] + ); + } +} + +function ensureHeapSpaceValues( + records: MetricRecord[], + start: number, + step: number, + name: string, + values: string[] +) { + const space = nativeJson.heap.spaces.find( + (space: any) => space.spaceName === name + ); + for (let i = 0, j = values.length; i < j; i++) { + ensureValue( + records[i * step + start], + `runtime.node.heapSpace.${values[i]}.by.space`, + space[values[i]] + ); + } +} diff --git a/packages/opentelemetry-rca-metrics/test/mocks/cpu.json b/packages/opentelemetry-rca-metrics/test/mocks/cpu.json new file mode 100644 index 0000000000..ee4ae3bf1f --- /dev/null +++ b/packages/opentelemetry-rca-metrics/test/mocks/cpu.json @@ -0,0 +1,4 @@ +{ + "user": 1899243, + "system": 258553 +} \ No newline at end of file diff --git a/packages/opentelemetry-rca-metrics/test/mocks/heap.json b/packages/opentelemetry-rca-metrics/test/mocks/heap.json new file mode 100644 index 0000000000..8afc2f37e6 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/test/mocks/heap.json @@ -0,0 +1,13 @@ +{ + "total_heap_size": 82182144, + "total_heap_size_executable": 2408448, + "total_physical_size": 81304464, + "total_available_size": 2142200152, + "used_heap_size": 55252160, + "heap_size_limit": 2197815296, + "malloced_memory": 65592, + "peak_malloced_memory": 9617696, + "does_zap_garbage": 0, + "number_of_native_contexts": 2, + "number_of_detached_contexts": 0 +} \ No newline at end of file diff --git a/packages/opentelemetry-rca-metrics/test/mocks/memory.json b/packages/opentelemetry-rca-metrics/test/mocks/memory.json new file mode 100644 index 0000000000..2a565068ca --- /dev/null +++ b/packages/opentelemetry-rca-metrics/test/mocks/memory.json @@ -0,0 +1,6 @@ +{ + "rss": 152535045, + "heapTotal": 82182145, + "heapUsed": 53736445, + "external": 1543181 +} \ No newline at end of file diff --git a/packages/opentelemetry-rca-metrics/test/mocks/native.json b/packages/opentelemetry-rca-metrics/test/mocks/native.json new file mode 100644 index 0000000000..24a26ff9a8 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/test/mocks/native.json @@ -0,0 +1,120 @@ +{ + "eventLoop": { + "min": 28630, + "max": 32331252, + "sum": 32848385, + "avg": 8212096, + "count": 4, + "median": 244252, + "p95": 32331252, + "total": 32848385 + }, + "gc": { + "all": { + "min": 1, + "max": 10, + "sum": 3, + "avg": 4, + "count": 2, + "median": 3, + "p95": 11, + "total": 123 + }, + "scavenge": { + "min": 5273842, + "max": 5280777, + "sum": 10554619, + "avg": 5277309, + "count": 2, + "median": 5277310, + "p95": 5280777 + }, + "markSweepCompact": { + "min": 15273842, + "max": 15280777, + "sum": 110554619, + "avg": 15277309, + "count": 3, + "median": 15277310, + "p95": 15280777 + }, + "incrementalMarking": { + "min": 25273842, + "max": 25280777, + "sum": 210554619, + "avg": 25277309, + "count": 4, + "median": 25277310, + "p95": 25280777 + }, + "processWeakCallbacks": { + "min": 35273842, + "max": 35280777, + "sum": 310554619, + "avg": 35277309, + "count": 5, + "median": 35277310, + "p95": 35280777 + } + }, + "heap": { + "spaces": [ + { + "spaceName": "read_only_space", + "size": 262144, + "usedSize": 32296, + "availableSize": 229576, + "physicalSize": 32568 + }, + { + "spaceName": "new_space", + "size": 33554432, + "usedSize": 9957432, + "availableSize": 6802376, + "physicalSize": 33541880 + }, + { + "spaceName": "old_space", + "size": 15314944, + "usedSize": 14430952, + "availableSize": 792496, + "physicalSize": 14753904 + }, + { + "spaceName": "code_space", + "size": 688128, + "usedSize": 602752, + "availableSize": 8736, + "physicalSize": 634688 + }, + { + "spaceName": "map_space", + "size": 1314816, + "usedSize": 1117920, + "availableSize": 35152, + "physicalSize": 1154512 + }, + { + "spaceName": "large_object_space", + "size": 17375232, + "usedSize": 17353888, + "availableSize": 12, + "physicalSize": 17375232 + }, + { + "spaceName": "code_large_object_space", + "size": 49152, + "usedSize": 3552, + "availableSize": 0, + "physicalSize": 49152 + }, + { + "spaceName": "new_large_object_space", + "size": 123, + "usedSize": 456, + "availableSize": 16759808, + "physicalSize": 789 + } + ] + } +} diff --git a/packages/opentelemetry-rca-metrics/test/mocks/nativeStart.json b/packages/opentelemetry-rca-metrics/test/mocks/nativeStart.json new file mode 100644 index 0000000000..da25ded584 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/test/mocks/nativeStart.json @@ -0,0 +1,84 @@ +{ + "eventLoop": { + "min": 0, + "max": 0, + "sum": 0, + "avg": 0, + "count": 0, + "median": 0, + "p95": 0, + "total": 0 + }, + "gc": { + "all": { + "min": 0, + "max": 0, + "sum": 0, + "avg": 0, + "count": 0, + "median": 0, + "p95": 0, + "total": 0 + } + }, + "heap": { + "spaces": [ + { + "spaceName": "read_only_space", + "size": 0, + "usedSize": 0, + "availableSize": 0, + "physicalSize": 0 + }, + { + "spaceName": "new_space", + "size": 0, + "usedSize": 0, + "availableSize": 0, + "physicalSize": 0 + }, + { + "spaceName": "old_space", + "size": 0, + "usedSize": 0, + "availableSize": 0, + "physicalSize": 0 + }, + { + "spaceName": "code_space", + "size": 0, + "usedSize": 0, + "availableSize": 0, + "physicalSize": 0 + }, + { + "spaceName": "map_space", + "size": 0, + "usedSize": 0, + "availableSize": 0, + "physicalSize": 0 + }, + { + "spaceName": "large_object_space", + "size": 0, + "usedSize": 0, + "availableSize": 0, + "physicalSize": 0 + }, + { + "spaceName": "code_large_object_space", + "size": 0, + "usedSize": 0, + "availableSize": 0, + "physicalSize": 0 + }, + { + "spaceName": "new_large_object_space", + "size": 0, + "usedSize": 0, + "availableSize": 0, + "physicalSize": 0 + } + ] + } +} diff --git a/packages/opentelemetry-rca-metrics/test/mocks/network.json b/packages/opentelemetry-rca-metrics/test/mocks/network.json new file mode 100644 index 0000000000..77caed7d72 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/test/mocks/network.json @@ -0,0 +1,15 @@ +[ + { + "iface": "en0", + "operstate": "up", + "rx_bytes": 60073930753, + "rx_dropped": 1200, + "rx_errors": 0, + "tx_bytes": 14207163202, + "tx_dropped": 1200, + "tx_errors": 21104, + "rx_sec": -1, + "tx_sec": -1, + "ms": 0 + } +] \ No newline at end of file diff --git a/packages/opentelemetry-rca-metrics/tsconfig.json b/packages/opentelemetry-rca-metrics/tsconfig.json new file mode 100644 index 0000000000..4b1645af66 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../tsconfig.base", + "compilerOptions": { + "rootDir": ".", + "outDir": "build" + }, + "include": [ + "src/**/*.ts", + "scripts/**/*.js", + "test/**/*.ts" + ] +} diff --git a/packages/opentelemetry-rca-metrics/tslint.json b/packages/opentelemetry-rca-metrics/tslint.json new file mode 100644 index 0000000000..0710b135d0 --- /dev/null +++ b/packages/opentelemetry-rca-metrics/tslint.json @@ -0,0 +1,4 @@ +{ + "rulesDirectory": ["node_modules/tslint-microsoft-contrib"], + "extends": ["../../tslint.base.js", "./node_modules/tslint-consistent-codestyle"] +}