Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: report metrics related to pvm #238

Merged
merged 8 commits into from
Oct 18, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions setup.py
Expand Up @@ -40,6 +40,7 @@
'grpcio-tools',
'packaging',
'wrapt',
'psutil',
],
extras_require={
'test': [
Expand Down
1 change: 1 addition & 0 deletions skywalking/config.py
Expand Up @@ -90,6 +90,7 @@
meter_reporter_active: bool = os.getenv('SW_AGENT_METER_REPORTER_ACTIVE') == 'True'
meter_reporter_max_buffer_size: int = int(os.getenv('SW_AGENT_METER_REPORTER_BUFFER_SIZE') or '10000')
meter_reporter_peroid: int = int(os.getenv('SW_AGENT_METER_REPORTER_PEROID') or '20')
pvm_meter_reporter_active: bool = os.getenv('SW_AGENT_PVM_METER_REPORTER_ACTIVE') == 'True'

options = {key for key in globals() if key not in options} # THIS MUST FOLLOW DIRECTLY AFTER LIST OF CONFIG OPTIONS!

Expand Down
13 changes: 13 additions & 0 deletions skywalking/meter/__init__.py
Expand Up @@ -15,6 +15,8 @@
# limitations under the License.
#

from skywalking import config

_meter_service = None


Expand All @@ -27,3 +29,14 @@ def init():

_meter_service = MeterService()
_meter_service.start()

if config.pvm_meter_reporter_active:
from skywalking.meter.pvm.cpu_usage import CPUUsageDataSource
from skywalking.meter.pvm.gc_data import GCDataSource
from skywalking.meter.pvm.mem_usage import MEMUsageDataSource
from skywalking.meter.pvm.thread_data import ThreadDataSource

MEMUsageDataSource().registry()
CPUUsageDataSource().registry()
GCDataSource().registry()
ThreadDataSource().registry()
16 changes: 16 additions & 0 deletions skywalking/meter/pvm/__init__.py
@@ -0,0 +1,16 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
32 changes: 32 additions & 0 deletions skywalking/meter/pvm/cpu_usage.py
@@ -0,0 +1,32 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import psutil
from skywalking.meter.pvm.data_source import DataSource


class CPUUsageDataSource(DataSource):
def __init__(self):
self.cur_process = psutil.Process()

def total_cpu_utilization_generator(self):
while (True):
yield psutil.cpu_percent()

def process_cpu_utilization_generator(self):
while (True):
yield self.cur_process.cpu_percent()
26 changes: 26 additions & 0 deletions skywalking/meter/pvm/data_source.py
@@ -0,0 +1,26 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from skywalking.meter.gauge import Gauge


class DataSource:
def registry(self):
for name in dir(self):
if name.endswith('generator'):
generator = getattr(self, name)()
Gauge.Builder(name[:-10], generator).build()
48 changes: 48 additions & 0 deletions skywalking/meter/pvm/gc_data.py
@@ -0,0 +1,48 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import gc
import time

from skywalking.meter.pvm.data_source import DataSource


class GCDataSource(DataSource):
def gc_g0_generator(self):
while (True):
yield gc.get_stats()[0]['collected']

def gc_g1_generator(self):
while (True):
yield gc.get_stats()[1]['collected']

def gc_g2_generator(self):
while (True):
yield gc.get_stats()[2]['collected']

def gc_callback(self, phase, info):
if phase == 'start':
self.start_time = time.time()
elif phase == 'stop':
self.gc_time = time.time() - self.start_time

def gc_time_generator(self):
if hasattr(gc, 'callbacks'):
gc.callbacks.append(self.gc_callback)

while (True):
yield self.gc_time
32 changes: 32 additions & 0 deletions skywalking/meter/pvm/mem_usage.py
@@ -0,0 +1,32 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from skywalking.meter.pvm.data_source import DataSource
import psutil


class MEMUsageDataSource(DataSource):
def __init__(self):
self.cur_process = psutil.Process()

def total_mem_utilization_generator(self):
while (True):
yield psutil.virtual_memory().percent

def process_mem_utilization_generator(self):
while (True):
yield self.cur_process.memory_info().rss / psutil.virtual_memory().total
37 changes: 37 additions & 0 deletions skywalking/meter/pvm/thread_data.py
@@ -0,0 +1,37 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import psutil
from skywalking.meter.pvm.data_source import DataSource


class ThreadDataSource(DataSource):
def __init__(self):
self.cur_process = psutil.Process()

def thread_active_count_generator(self):
while (True):
ps = [self.cur_process]
count = 0

while len(ps) > 0:
p = ps[0]
ps.pop(0)
count += p.num_threads()
ps += p.children()

yield count
5 changes: 5 additions & 0 deletions tests/e2e/base/docker-compose.base.yml
Expand Up @@ -23,6 +23,9 @@ services:
expose:
- 11800 # gRPC
- 12800 # HTTP
volumes: # only for temporary use and should be removed after related backend PR accepted
- ./tmp/python.yaml:/skywalking/config/meter-analyzer-config/python.yaml
- ./tmp/application.yml:/skywalking/config/application.yml
networks:
- e2e
healthcheck: # python base image has no nc command
Expand All @@ -46,6 +49,8 @@ services:
SW_AGENT_LOG_REPORTER_ACTIVE: "True"
SW_AGENT_LOG_REPORTER_LEVEL: WARNING
SW_AGENT_LOG_REPORTER_SAFE_MODE: "True"
SW_AGENT_METER_REPORTER_ACTIVE: "True"
SW_AGENT_PVM_METER_REPORTER_ACTIVE: "True"
healthcheck:
test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9090" ]
interval: 5s
Expand Down