Merge branch 'master' into kav/ValueToUpperLower

.github/workflows/push_pr.yml

@@ -15,7 +15,7 @@ on:
 env:
   REPO_FULL_NAME: ${{ github.event.repository.full_name }}
   ORIGINAL_REPO_NAME: "newrelic/nri-flex"
-  GO_VERSION: '1.18'
+  GO_VERSION: '1.19'
 
 jobs:
 
@@ -75,7 +75,7 @@ jobs:
   test-windows:
     strategy:
       matrix:
-        go: [ '1.18' ]
+        go: [ '1.19' ]
         os: [ windows-2019 ]
     name: Run unit and integration tests in ${{matrix.os}} with go ${{matrix.go}}
     runs-on: ${{matrix.os}}

.github/workflows/release.yml

@@ -9,7 +9,7 @@ on:
       - 'v*'
 
 env:
-  GO_VERSION: '1.18'
+  GO_VERSION: '1.19'
   TAG: ${{ github.event.release.tag_name }}
   IS_RELEASE: true
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # needed for goreleaser
@@ -72,7 +72,7 @@ jobs:
   test-windows:
     strategy:
       matrix:
-        go: [ '1.18' ]
+        go: [ '1.19' ]
         os: [ windows-2019 ]
     name: Run unit and integration tests in ${{matrix.os}} with go ${{matrix.go}}
     runs-on: ${{matrix.os}}

CONTRIBUTING.md

@@ -15,8 +15,9 @@ Before submitting an Issue, please search for similar ones in the
 ## Pull Requests
 
 1. Ensure any install or build dependencies are removed before the end of the layer when doing a build.
-2. Increase the version numbers in any examples files and the README.md to the new version that this Pull Request would represent. The versioning scheme we use is [SemVer](http://semver.org/).
-3. You may merge the Pull Request in once you have the sign-off of two other developers, or if you do not have permission to do that, you may request the second reviewer to merge it for you.
+2. If the pull request modifies code used in the released artifact, ensure to add the corresponding Go tests and [end-to-end fixtures](./test/testbed/README.md).
+3. Increase the version numbers in any examples files and the README.md to the new version that this Pull Request would represent. The versioning scheme we use is [SemVer](http://semver.org/).
+4. You may merge the Pull Request in once you have the sign-off of two other developers, or if you do not have permission to do that, you may request the second reviewer to merge it for you.
 
 ## Contributor License Agreement
 
@@ -27,4 +28,4 @@ For more information about CLAs, please check out Alex Russell’s excellent pos
 
 ## Slack
 
-We host a public Slack with a dedicated channel for contributors and maintainers of open source projects hosted by New Relic.  If you are contributing to this project, you're welcome to request access to the #oss-contributors channel in the newrelicusers.slack.com workspace.  To request access, see https://newrelicusers-signup.herokuapp.com/.
+We host a public Slack with a dedicated channel for contributors and maintainers of open source projects hosted by New Relic. If you are contributing to this project, you're welcome to request access to the #oss-contributors channel in the newrelicusers.slack.com workspace. To request access, please use this [link](https://join.slack.com/t/newrelicusers/shared_invite/zt-1ayj69rzm-~go~Eo1whIQGYnu3qi15ng).

Makefile

@@ -6,7 +6,7 @@ SRCDIR           ?= .
 BUILD_DIR        ?= $(CURDIR)/bin
 COVERAGE_FILE    ?= coverage.out
 
-GO_VERSION       ?= 1.18
+GO_VERSION       ?= 1.19
 GO_CMD           ?= go
 GODOC            ?= godocdown
 

README.md

@@ -1,4 +1,4 @@
-[![Community Plus header](https://github.com/newrelic/opensource-website/raw/master/src/images/categories/Community_Plus.png)](https://opensource.newrelic.com/oss-category/#community-plus)
+[![Community Plus header](https://github.com/newrelic/opensource-website/raw/main/src/images/categories/Community_Plus.png)](https://opensource.newrelic.com/oss-category/#community-plus)
 
 ![Build Status](https://github.com/newrelic/nri-flex/actions/workflows/push_pr.yml/badge.svg)
 [![Coverage Status](https://coveralls.io/repos/github/newrelic/nri-flex/badge.svg?branch=master)](https://coveralls.io/github/newrelic/nri-flex?branch=master)

build/Dockerfile

@@ -1,4 +1,4 @@
-ARG GO_VERSION=1.18
+ARG GO_VERSION=1.19
 
 FROM golang:$GO_VERSION
 

docs/apis/README.md

@@ -30,6 +30,6 @@ integrations:
 ```
 The following APIs are still experimental. 'Experimental' here means that New Relic does not yet provides support for them:
 
-- [Database queries](experimental/db.md)
-- [Net dial](experimental/dial.md)
-- [Git configuration synchronization](experimental/git_sync.md)
+- [Database queries](../experimental/db.md)
+- [Net dial](../experimental/dial.md)
+- [Git configuration synchronization](../experimental/git_sync.md)

examples/DORA/README.md

@@ -0,0 +1,66 @@
+# DORA Metrics Integrations 
+
+## Flex Integrations
+- GitLab DORA API
+    - Group: query dora metrics in all repos in a single group  
+    - Repo: query dora metrics in a single repo
+
+## Requirements
+[DORA Entity Definition](https://github.com/newrelic/entity-definitions/blob/main/definitions/ext-dora/definition.yml)
+- Entity Creation
+    - eventType: DoraMetricsSample
+    - doraName: user defined doraName
+        - doraName is used as an encapsulation, send metrics from mulitple sources to create a complete view on DORA metrics
+    - must contain 1 of the 4 DORA Metrics to create an entity (recommended to send an event with all 4)
+        - leadTimeForChanges
+        - deploymentFrequency
+        - timeToRestoreService
+        - changeFailureRate
+    - add a source attribute to allow filtering of different doraSources under the same entity
+    - Optional: 
+        - Send supporting DORA metrics from different sources.
+            - i.e. ci/cd tools, custom scripts, different repositories
+
+### **IMPORTANT** 
+**doraName** needs to be a unique identifier for your entity creation
+
+### Example Payloads
+``` json
+[
+	{
+		"eventType": "DoraMetricsSample",
+		"doraName": "Opencart",
+		"projectId": 123456,
+		"repoUrl": "https://gitlab.com/your/repo",
+		"repoName": "opencart-tf",
+		"deploymentFrequency": 15,
+		"leadTimeForChanges": 12,
+		"timeToRestoreService": 0.4,
+		"changeFailureRate": 12.6,
+		"team": "dev-rel-apj",
+		"pipeline": "production-opencart",
+		"org": "dev-rel",
+		"source": "GitLab"
+	},
+	{
+		"eventType": "DoraMetricsSample",
+		"doraName": "Opencart",
+		"repoName": "opencart-tf-infra",
+		"deploymentFrequency": 15,
+		"team": "dev-rel-sre",
+		"pipeline": "production-opencart-infra",
+		"org": "dev-rel",
+		"source": "GitLab"
+	},
+		{
+		"eventType": "DoraMetricsSample",
+		"doraName": "Opencart",
+		"deploymentFrequency": 1,
+		"team": "dev-rel-sre",
+		"pipeline": "production-opencart-ansible",
+		"org": "dev-rel",
+		"source": "ansible"
+	}
+]
+```
+

examples/DORA/gitlab-dora-groups.yml

@@ -0,0 +1,98 @@
+#group level dora metrics
+---
+integrations:
+  - name: nri-flex
+    interval: 300s
+    env:
+      INSIGHTS_URL: https://insights-collector.newrelic.com/v1/accounts/<accountid>/events
+      INSIGHTS_API_KEY: <insertkey>
+    config:
+      name: gitlabDoraMetrics
+      global:
+        base_url: https://gitlab.com/api/v4      
+        headers:
+          PRIVATE-TOKEN: <accesstoken>
+      apis:
+        - name: doraGroup
+          url: /groups/<groupid>/projects
+          jq: .[]|with_entries(select([.key] | inside(["id", "namespace.path", "path"])))
+          store_lookups:
+            projectId: id
+          rename_keys:
+            namespace.name: org
+            path: repoName
+          add_attribute:
+            doraName: "${org}-${repoName}"
+          value_parser:
+            id: '[0-9]+'
+          metric_parser:
+            metrics:
+              id: ATTRIBUTE
+            namespace:
+              custom_attr: 'dora'
+
+          value_parser:
+            .*: 'nil'
+
+          merge: DoraMetricsSample
+          join_key: id
+
+        - name: DoraDeploymentSample
+          run_async: true        
+          url: /projects/${lookup:projectId}/dora/metrics?metric=deployment_frequency&start_date=${timestamp:dateutc-24hr}
+          jq: >
+            .[-1]|{"deploymentFrequency":.value,"projectid":"${lookup:projectId}"}
+          metric_parser:
+            metrics:
+              projectid: ATTRIBUTE
+            namespace:
+              custom_attr: 'dora'
+
+          merge: DoraMetricsSample
+          join_key: projectid
+
+        - name: change_failure_rate
+          run_async: true        
+          url: /projects/${lookup:projectId}/dora/metrics?metric=change_failure_rate&start_date=${timestamp:dateutc-24hr}
+          jq: >
+            .[-1]|{"changeFailureRate":.value,"projectid":"${lookup:projectId}"}
+
+          metric_parser:
+            metrics:
+              projectid: ATTRIBUTE
+            namespace:
+              custom_attr: 'dora'
+
+          merge: DoraMetricsSample
+          join_key: projectid
+
+        - name: time_to_restore_service
+          run_async: true             
+          url: /projects/${lookup:projectId}/dora/metrics?metric=time_to_restore_service&start_date=${timestamp:dateutc-24hr}
+          jq: >
+            .[-1]|{"timeToRestoreService":.value,"projectid":"${lookup:projectId}"}
+
+          metric_parser:
+            metrics:
+              projectid: ATTRIBUTE
+            namespace:
+              custom_attr: 'dora'
+
+          merge: DoraMetricsSample
+          join_key: projectid
+
+
+        - name: lead_time_for_changes
+          run_async: true             
+          url: /projects/${lookup:projectId}/dora/metrics?metric=lead_time_for_changes&start_date=${timestamp:dateutc-24hr}
+          jq: >
+            .[-1]|{"leadTimeForChanges":.value,"projectid":"${lookup:projectId}"}
+
+          metric_parser:
+            metrics:
+              projectid: ATTRIBUTE
+            namespace:
+              custom_attr: 'dora'
+
+          merge: DoraMetricsSample
+          join_key: projectid

examples/DORA/gitlab-dora-single.yml

@@ -0,0 +1,50 @@
+#repo level dora metrics
+---
+integrations:
+  - name: nri-flex
+    interval: 300s
+    env:
+      INSIGHTS_URL: https://insights-collector.newrelic.com/v1/accounts/<accountid>/events
+      INSIGHTS_API_KEY: <insertkey>
+    config:
+      name: gitlabDoraMetrics
+      variable_store:
+        projectId: <projectId> #your gitlab projectId
+      custom_attributes:
+        doraName: <doraName> # required for entity creation needs to be unique
+        pipeline: <pipeline>
+        team: <team>
+        repoName: <repoName>
+        org: <org>
+      global:
+        base_url: https://gitlab.com/api/v4      
+        headers:
+          PRIVATE-TOKEN: <accesstoken>
+      apis:
+        - name: deployment_frequency
+          run_async: true        
+          url: /projects/${var:projectId}/dora/metrics?metric=deployment_frequency&start_date=${timestamp:dateutc-24hr}
+          jq: >
+            .[-1]|{"deploymentFrequency":.value}
+          merge: DoraMetricsSample
+
+        - name: change_failure_rate
+          run_async: true        
+          url: /projects/${var:projectId}/dora/metrics?metric=change_failure_rate&start_date=${timestamp:dateutc-24hr}
+          jq: >
+            .[-1]|{"changeFailureRate":.value}
+          merge: DoraMetricsSample
+
+        - name: time_to_restore_service
+          run_async: true             
+          url: /projects/${var:projectId}/dora/metrics?metric=time_to_restore_service&start_date=${timestamp:dateutc-24hr}
+          jq: >
+            .[-1]|{"timeToRestoreService":.value}
+          merge: DoraMetricsSample
+
+        - name: lead_time_for_changes
+          run_async: true             
+          url: /projects/${var:projectId}/dora/metrics?metric=lead_time_for_changes&start_date=${timestamp:dateutc-24hr}
+          jq: >
+            .[-1]|{"leadTimeForChanges":.value}
+          merge: DoraMetricsSample

examples/linux/db2/db2-appl-performance.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+cd /home/$1/sqllib
+
+. ./db2profile
+
+db2 connect to $2
+
+db2 -x "SELECT SNAPSHOT_TIMESTAMP, AUTHID, APPL_NAME, AGENT_ID, PERCENT_ROWS_SELECTED, DBPARTITIONNUM, MEMBER FROM SYSIBMADM.APPL_PERFORMANCE"
+
+exit 0

examples/linux/db2/db2-bp-hitratio.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+cd /home/$1/sqllib
+
+. ./db2profile
+
+db2 connect to $2
+
+db2 -x "SELECT SNAPSHOT_TIMESTAMP, DB_NAME, BP_NAME, TOTAL_LOGICAL_READS, TOTAL_PHYSICAL_READS, TOTAL_HIT_RATIO_PERCENT, DATA_LOGICAL_READS, DATA_PHYSICAL_READS, DATA_HIT_RATIO_PERCENT, INDEX_LOGICAL_READS, INDEX_PHYSICAL_READS, INDEX_HIT_RATIO_PERCENT, XDA_LOGICAL_READS, XDA_PHYSICAL_READS, XDA_HIT_RATIO_PERCENT, DBPARTITIONNUM, MEMBER FROM SYSIBMADM.BP_HITRATIO"
+
+exit 0

examples/linux/db2/db2-bp-read-io.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+cd /home/$1/sqllib
+
+. ./db2profile
+
+db2 connect to $2
+
+db2 -x "SELECT SNAPSHOT_TIMESTAMP, BP_NAME, TOTAL_PHYSICAL_READS, AVERAGE_READ_TIME_MS, TOTAL_ASYNC_READS, AVERAGE_ASYNC_READ_TIME_MS, TOTAL_SYNC_READS, AVERAGE_SYNC_READ_TIME_MS, PERCENT_SYNC_READS, ASYNC_NOT_READ_PERCENT, DBPARTITIONNUM, MEMBER FROM SYSIBMADM.BP_READ_IO"
+
+exit 0

examples/linux/db2/db2-cpu-utilization.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+cd /home/$1/sqllib
+
+. ./db2profile
+
+db2 connect to $2
+
+db2 -x "SELECT APPLICATION_HANDLE, TOTAL_CPU_TIME, ROWS_READ, ROWS_RETURNED, TOTAL_WAIT_TIME, TOTAL_SORTS FROM TABLE(MON_GET_UNIT_OF_WORK(NULL,-1)) AS t ORDER BY TOTAL_CPU_TIME DESC"
+
+exit 0

examples/linux/db2/db2-lock-waits.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+cd /home/$1/sqllib
+
+. ./db2profile
+
+db2 connect to $2
+
+db2 -x "SELECT SNAPSHOT_TIMESTAMP, DB_NAME, AGENT_ID, APPL_NAME, AUTHID, TBSP_NAME, TABSCHEMA, TABNAME, SUBSECTION_NUMBER, LOCK_OBJECT_TYPE, LOCK_WAIT_START_TIME, LOCK_NAME, LOCK_MODE, LOCK_MODE_REQUESTED, AGENT_ID_HOLDING_LK, APPL_ID_HOLDING_LK, LOCK_ESCALATION, DBPARTITIONNUM, MEMBER FROM SYSIBMADM.LOCKWAITS"
+
+exit 0

examples/linux/db2/db2-locks-held.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+cd /home/$1/sqllib
+
+. ./db2profile
+
+db2 connect to $2
+
+db2 -x "SELECT SNAPSHOT_TIMESTAMP, DB_NAME, AGENT_ID, APPL_NAME, AUTHID, TBSP_NAME, TABSCHEMA, TABNAME, TAB_FILE_ID, LOCK_OBJECT_TYPE, LOCK_NAME, LOCK_MODE, LOCK_STATUS, LOCK_ESCALATION, DBPARTITIONNUM, MEMBER FROM SYSIBMADM.LOCKS_HELD"
+
+exit 0

examples/linux/db2/db2-log-utilization-percent.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+cd /home/$1/sqllib
+
+. ./db2profile
+
+db2 connect to $2
+
+db2 -x "SELECT LOG_UTILIZATION_PERCENT, cast(( TOTAL_LOG_USED_KB/1024) as Integer) as TOTAL_LOG_USED_MB, cast((TOTAL_LOG_AVAILABLE_KB/1024) as Integer) as TOTAL_LOG_AVAILABLE_MB, cast((TOTAL_LOG_USED_TOP_KB/1024) as integer) as TOTAL_LOG_USED_TOP_MB from SYSIBMADM.MON_TRANSACTION_LOG_UTILIZATION"
+
+exit 0