From 1055f11867b71624e1299bd50a6b5e1559bed26c Mon Sep 17 00:00:00 2001 From: S0okJu Date: Sun, 19 Jan 2025 19:45:37 +0900 Subject: [PATCH 1/7] =?UTF-8?q?[Feat]=20=EB=B6=88=ED=95=84=EC=9A=94?= =?UTF-8?q?=ED=95=9C=20=EB=A1=9C=EA=B7=B8=20=ED=98=95=ED=83=9C=20=EC=A0=9C?= =?UTF-8?q?=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/factory.py | 26 ++------------------------ requirements.txt | 3 ++- 2 files changed, 4 insertions(+), 25 deletions(-) diff --git a/app/factory.py b/app/factory.py index fa76e6e..497ff0d 100644 --- a/app/factory.py +++ b/app/factory.py @@ -80,7 +80,6 @@ def _get_request_context(self) -> Dict[str, Any]: "remote_addr": request.remote_addr, "user_agent": request.user_agent.string, "request_id": request.headers.get('X-Request-ID', 'unknown'), - "timestamp": datetime.utcnow().isoformat() } except Exception as e: # 요청 컨텍스트 추출 실패 시 기본값 @@ -103,7 +102,6 @@ def _log_request(self, response, processing_time: float): "request_id": context.get("request_id", "unknown"), "status_code": str(getattr(response, 'status_code', 'unknown')), "method": context.get("method", "UNKNOWN"), - "path": context.get("path", "/") } # Prepare log content @@ -111,19 +109,10 @@ def _log_request(self, response, processing_time: float): "processing_time_ms": round(processing_time * 1000, 2), "remote_addr": context.get("remote_addr", ""), "user_agent": context.get("user_agent", ""), - "method": context.get("method", ""), "path": context.get("path", ""), - "status_code": str(getattr(response, 'status_code', 'unknown')), - "timestamp": context.get("timestamp", datetime.utcnow().isoformat()) } - # 추가 정보 안전하게 포함 - try: - if request.is_json: - log_content["request_body"] = request.get_json() - except Exception as e: - log_content["request_body_error"] = str(e) - + self.logger.info( "HTTP Request", extra={ @@ -139,30 +128,19 @@ def _log_request(self, response, processing_time: float): def _log_error(self, error: CustomBaseException): """에러 로깅""" try: - # 요청 컨텍스트 안전하게 추출 - context = { - "method": getattr(request, 'method', 'UNKNOWN'), - "path": getattr(request, 'path', '/'), - "remote_addr": getattr(request, 'remote_addr', ''), - "user_agent": str(getattr(request, 'user_agent', '')), - "request_id": request.headers.get('X-Request-ID', 'unknown') if request else 'unknown' - } - # 로깅 self.logger.error( "Application Error", extra={ "labels": { "error_type": str(error.error_type.value), - "request_id": context.get('request_id', 'unknown') + "request_id": request.headers.get('X-Request-ID', 'unknown') if request else 'unknown' }, "content": { - **context, "error_type": str(error.error_type.value), "error_message": str(error.message), "error_msg": str(error.error_msg or ''), "status_code": error.status_code, - "timestamp": datetime.utcnow().isoformat() } } ) diff --git a/requirements.txt b/requirements.txt index 5e95feb..acca2fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,5 @@ Flask-SQLAlchemy==3.0.2 mariadb>=1.0.11 prometheus-client==0.19.0 python-logging-loki==0.3.1 -flask-prometheus-metrics==1.0.0 \ No newline at end of file +flask-prometheus-metrics==1.0.0 +prometheus_client==0.17.1 \ No newline at end of file From 4d74b42836401611118f39ce0784c69e4429cb00 Mon Sep 17 00:00:00 2001 From: S0okJu Date: Sun, 19 Jan 2025 21:26:59 +0900 Subject: [PATCH 2/7] =?UTF-8?q?[Feat]=20Prometheus=20System&Challenge=20Co?= =?UTF-8?q?llector=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/api/challenge.py | 27 +++- app/extensions_manager.py | 2 +- app/factory.py | 14 ++ app/monitoring/ctf_metrics_collector.py | 17 +++ app/monitoring/system_metrics_collector.py | 146 +++++++++++++++++++++ configs/prometheus/prometheus.yml | 13 +- requirements.txt | 2 +- 7 files changed, 216 insertions(+), 5 deletions(-) create mode 100644 app/monitoring/ctf_metrics_collector.py create mode 100644 app/monitoring/system_metrics_collector.py diff --git a/app/api/challenge.py b/app/api/challenge.py index 24a8414..10ba44a 100644 --- a/app/api/challenge.py +++ b/app/api/challenge.py @@ -1,5 +1,6 @@ from json import JSONDecodeError from logging import log +from app.monitoring.ctf_metrics_collector import ChallengeMetricsCollector from flask import Blueprint, jsonify, request from app.exceptions.api import InvalidRequest @@ -8,6 +9,7 @@ from app.extensions.k8s.client import K8sClient challenge_bp = Blueprint('challenge', __name__) +metrics = ChallengeMetricsCollector() @challenge_bp.route('', methods=['POST']) def create_challenge(): @@ -15,21 +17,31 @@ def create_challenge(): # Challenge 관련 정보 가져오기 res = request.get_json() if not res: - raise InvalidRequest(error_msg="Request body is empty or not valid JSON") + metrics.challenge_operations.labels(operation='create',result='error').inc() + raise InvalidRequest(error_msg="Request body is empty or not valid JSON") if 'challenge_id' not in res: + metrics.challenge_operations.labels(operation='create',result='error').inc() raise InvalidRequest(error_msg="Required field 'challenge_id' is missing in request") challenge_id = res['challenge_id'] if 'username' not in res: + metrics.challenge_operations.labels(operation='create',result='error').inc() raise InvalidRequest(error_msg="Required field 'username' is missing in request") username = res['username'] # 챌린지 생성 client = K8sClient() endpoint = client.create_challenge_resource(challenge_id, username) if not endpoint: + metrics.challenge_operations.labels(operation='create',result='error').inc() raise UserChallengeCreationError(error_msg=f"Faile to create challenge {challenge_id} for user {username}") + metrics.challenge_state.labels( + challenge_id=challenge_id, + username=username + ).set(1) + + metrics.challenge_operations.labels(operation='create',result='success').inc() return jsonify({'data' : {'port': endpoint}}), 200 @challenge_bp.route('/delete', methods=['POST']) @@ -42,15 +54,18 @@ def delete_userchallenges(): res = request.get_json() if not res: log.error("No data provided") + metrics.challenge_operations.labels(operation='delete',result='error').inc() raise UserChallengeDeletionError(error_msg="Request body is empty or not valid JSON") if 'challenge_id' not in res: log.error("No challenge_id provided") + metrics.challenge_operations.labels(operation='delete',result='error').inc() raise InvalidRequest(error_msg="Required field 'challenge_id' is missing in request") challenge_id = res['challenge_id'] if 'username' not in res: log.error("No username provided") + metrics.challenge_operations.labels(operation='delete',result='error').inc() raise InvalidRequest(error_msg="Required field 'username' is missing in request") username = res['username'] @@ -58,9 +73,19 @@ def delete_userchallenges(): client = K8sClient() client.delete_userchallenge(username, challenge_id) + # Metrics + metrics.challenge_state.labels( + challenge_id=challenge_id, + username=username + ).set(0) + metrics.challenge_operations.labels( + operation='delete', + result='success' + ).inc() return jsonify({'message' : '챌린지가 정상적으로 삭제되었습니다.'}), 200 except JSONDecodeError as e: log.error("Invalid request format") + metrics.challenge_operations.labels(operation='delete',result='error').inc() raise InvalidRequest(error_msg=str(e)) from e @challenge_bp.route('/status', methods=['POST']) diff --git a/app/extensions_manager.py b/app/extensions_manager.py index b37099d..a07c0d1 100644 --- a/app/extensions_manager.py +++ b/app/extensions_manager.py @@ -2,7 +2,7 @@ import sys from threading import Lock, Thread, Event from typing import Optional, Callable -from flask import Flask, +from flask import Flask from flask_sqlalchemy import SQLAlchemy from app.extensions.kafka import KafkaConfig, KafkaEventConsumer diff --git a/app/factory.py b/app/factory.py index 497ff0d..ce68fb7 100644 --- a/app/factory.py +++ b/app/factory.py @@ -1,9 +1,13 @@ import sys + +from requests import Response from app.monitoring.loki_logger import FlaskLokiLogger +from app.monitoring.system_metrics_collector import SystemMetricsCollector from flask import Flask, g, request import threading from datetime import datetime from typing import Any, Dict, Type +from prometheus_client import generate_latest, CONTENT_TYPE_LATEST from app.api.challenge import challenge_bp from app.config import Config @@ -28,6 +32,7 @@ def __init__(self, config_class: Type[Config] = Config): self._setup_middleware() self._register_error_handlers() self._setup_blueprints() + self._init_metrics_collector() def _init_extensions(self): """Extensions 초기화""" @@ -40,6 +45,15 @@ def _init_extensions(self): with self.app.app_context(): db.create_all() + def _init_metrics_collector(self): + system_collector = SystemMetricsCollector(self.app) + system_collector.start_collecting() + + # CTF metrics + @self.app.route('/metrics/ctf') + def challenge_metrics(): + return generate_latest(), 200, {'Content-Type': CONTENT_TYPE_LATEST} + def _setup_middleware(self): """미들웨어 설정""" @self.app.before_request diff --git a/app/monitoring/ctf_metrics_collector.py b/app/monitoring/ctf_metrics_collector.py new file mode 100644 index 0000000..b9f9fd4 --- /dev/null +++ b/app/monitoring/ctf_metrics_collector.py @@ -0,0 +1,17 @@ +from prometheus_client import Gauge, Counter + +class ChallengeMetricsCollector: + def __init__(self): + # 챌린지 상태 Gauge + self.challenge_state = Gauge( + 'challenge_state', + 'Current state of challenges', + ['challenge_id', 'username'] # 1: active, 0: inactive + ) + + # API 요청 결과 카운터 + self.challenge_operations = Counter( + 'challenge_operations_total', + 'Challenge operation results', + ['operation', 'result'] # operation: create/delete/status, result: success/error + ) diff --git a/app/monitoring/system_metrics_collector.py b/app/monitoring/system_metrics_collector.py new file mode 100644 index 0000000..bc8ff79 --- /dev/null +++ b/app/monitoring/system_metrics_collector.py @@ -0,0 +1,146 @@ +import socket +from flask import Flask, Response +import psutil +from prometheus_client import Gauge, Counter, generate_latest, CONTENT_TYPE_LATEST +import time +from threading import Thread + +class SystemMetricsCollector: + def __init__(self, app: Flask = None): + # CPU 메트릭 + self.cpu_usage = Gauge( + 'system_cpu_usage_percent', + 'CPU Usage in Percent', + ['cpu_type'] # user, system, idle + ) + + # 메모리 메트릭 + self.memory_usage = Gauge( + 'system_memory_bytes', + 'Memory Usage in Bytes', + ['type'] # used, free, cached, total + ) + + # 디스크 메트릭 + self.disk_usage = Gauge( + 'system_disk_bytes', + 'Disk Usage in Bytes', + ['mount_point', 'type'] # used, free, total + ) + + self.disk_io = Counter( + 'system_disk_io_bytes', + 'Disk I/O in Bytes', + ['operation'] # read, write + ) + + # 네트워크 메트릭 + self.network_traffic = Counter( + 'system_network_traffic_bytes', + 'Network Traffic in Bytes', + ['interface', 'direction'] # received, transmitted + ) + + self.network_connections = Gauge( + 'system_network_connections', + 'Number of Network Connections', + ['protocol', 'status'] # tcp/udp, ESTABLISHED/LISTEN/etc + ) + + if app is not None: + self.init_app(app) + + def init_app(self, app: Flask): + """Flask 애플리케이션에 메트릭 엔드포인트 등록""" + @app.route('/metrics') + def metrics(): + return Response(generate_latest(), mimetype=CONTENT_TYPE_LATEST) + + # 메트릭 수집 시작 + self.start_collecting() + + def collect_cpu_metrics(self): + """CPU 메트릭 수집""" + cpu_times = psutil.cpu_times_percent() + self.cpu_usage.labels(cpu_type='user').set(cpu_times.user) + self.cpu_usage.labels(cpu_type='system').set(cpu_times.system) + self.cpu_usage.labels(cpu_type='idle').set(cpu_times.idle) + + def collect_memory_metrics(self): + """메모리 메트릭 수집""" + mem = psutil.virtual_memory() + self.memory_usage.labels(type='total').set(mem.total) + self.memory_usage.labels(type='used').set(mem.used) + self.memory_usage.labels(type='free').set(mem.free) + self.memory_usage.labels(type='cached').set(mem.cached) + + def collect_disk_metrics(self): + """디스크 메트릭 수집""" + # 디스크 사용량 + for partition in psutil.disk_partitions(): + if partition.fstype: + usage = psutil.disk_usage(partition.mountpoint) + self.disk_usage.labels( + mount_point=partition.mountpoint, + type='total' + ).set(usage.total) + self.disk_usage.labels( + mount_point=partition.mountpoint, + type='used' + ).set(usage.used) + self.disk_usage.labels( + mount_point=partition.mountpoint, + type='free' + ).set(usage.free) + + # 디스크 I/O + disk_io = psutil.disk_io_counters() + self.disk_io.labels(operation='read').inc(disk_io.read_bytes) + self.disk_io.labels(operation='write').inc(disk_io.write_bytes) + + def collect_network_metrics(self): + """네트워크 메트릭 수집""" + # 네트워크 트래픽 + net_io = psutil.net_io_counters(pernic=True) + for interface, counters in net_io.items(): + self.network_traffic.labels( + interface=interface, + direction='received' + ).inc(counters.bytes_recv) + self.network_traffic.labels( + interface=interface, + direction='transmitted' + ).inc(counters.bytes_sent) + + # 네트워크 연결 + connections = psutil.net_connections() + conn_count = {'tcp': {}, 'udp': {}} + for conn in connections: + proto = 'tcp' if conn.type == socket.SOCK_STREAM else 'udp' + status = conn.status + conn_count[proto][status] = conn_count[proto].get(status, 0) + 1 + + for proto in conn_count: + for status, count in conn_count[proto].items(): + self.network_connections.labels( + protocol=proto, + status=status + ).set(count) + + def collect_metrics(self): + """모든 메트릭 수집""" + while True: + try: + self.collect_cpu_metrics() + self.collect_memory_metrics() + self.collect_disk_metrics() + self.collect_network_metrics() + except Exception as e: + print(f"Error collecting metrics: {e}") + time.sleep(15) # 15초 간격으로 수집 + + def start_collecting(self): + """메트릭 수집 스레드 시작""" + thread = Thread(target=self.collect_metrics) + thread.daemon = True + thread.start() \ No newline at end of file diff --git a/configs/prometheus/prometheus.yml b/configs/prometheus/prometheus.yml index f3ec85c..2c5bb70 100644 --- a/configs/prometheus/prometheus.yml +++ b/configs/prometheus/prometheus.yml @@ -7,6 +7,15 @@ scrape_configs: static_configs: - targets: ['localhost:9090'] - - job_name: 'flask-app' + - job_name: 'flask_system' static_configs: - - targets: ['localhost:8000'] \ No newline at end of file + - targets: ['192.168.80.2:5001'] + metrics_path: '/metrics' + scrape_interval: 15s + + # CTF 메트릭 수집 (/metrics/ctf) + - job_name: 'flask_ctf' + static_configs: + - targets: ['192.168.80.2:5001'] + metrics_path: '/metrics/ctf' + scrape_interval: 15s \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index acca2fa..f9370c2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,4 @@ mariadb>=1.0.11 prometheus-client==0.19.0 python-logging-loki==0.3.1 flask-prometheus-metrics==1.0.0 -prometheus_client==0.17.1 \ No newline at end of file +psutil==5.9.8 \ No newline at end of file From 4193faf2e5f0c1805542fe514578691d0bb9cce3 Mon Sep 17 00:00:00 2001 From: S0okJu Date: Sun, 19 Jan 2025 21:27:20 +0900 Subject: [PATCH 3/7] =?UTF-8?q?[Feat]=20docker-compose=20=EC=88=98?= =?UTF-8?q?=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 07e2815..8db6d2e 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -140,7 +140,7 @@ services: - SECRET_KEY=your-super-secret-key-here # Kafka - KAFKA_BOOTSTRAP_SERVERS=127.0.0.1:9093 - - KAFKA_TOPIC=challenge-status + - KAFKA_TOPIC=challenge-statustem_metrics_collectos - KAFKA_GROUP_ID=challenge-consumer-group # Loki 설정 추가 - LOKI_URL=http://127.0.0.1:3100/loki/api/v1/push From 6b917dd20d42c86c45c51f6ef607b4ab0e42a357 Mon Sep 17 00:00:00 2001 From: S0okJu <65109465+D7MeKz@users.noreply.github.com> Date: Mon, 20 Jan 2025 17:14:08 +0900 Subject: [PATCH 4/7] =?UTF-8?q?[Feat]=20prometheus=20challenge=20metrics?= =?UTF-8?q?=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- a.json | 105 +++++++++++++++++++++ app/api/challenge.py | 11 ++- app/factory.py | 11 ++- app/monitoring/ctf_metrics_collector.py | 47 ++++++++- app/monitoring/system_metrics_collector.py | 40 ++++---- configs/prometheus/prometheus.yml | 12 +-- 6 files changed, 188 insertions(+), 38 deletions(-) create mode 100644 a.json diff --git a/a.json b/a.json new file mode 100644 index 0000000..9cbd3d0 --- /dev/null +++ b/a.json @@ -0,0 +1,105 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "type": "table", + "options": { + "showHeader": true, + "footer": { + "show": false + } + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "center", + "displayMode": "color-background", + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Inactive", + "color": "red" + }, + "1": { + "text": "Active", + "color": "green" + } + } + } + ] + } + }, + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "username": { + "operation": "groupby" + }, + "challenge_id": { + "operation": "groupby" + }, + "Value": { + "operation": "aggregate", + "aggregations": ["lastNotNull"] + } + } + } + } + ] + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "New dashboard", + "version": 0, + "weekStart": "" + } \ No newline at end of file diff --git a/app/api/challenge.py b/app/api/challenge.py index 10ba44a..36a6101 100644 --- a/app/api/challenge.py +++ b/app/api/challenge.py @@ -9,11 +9,12 @@ from app.extensions.k8s.client import K8sClient challenge_bp = Blueprint('challenge', __name__) -metrics = ChallengeMetricsCollector() + @challenge_bp.route('', methods=['POST']) def create_challenge(): """사용자 챌린지 생성""" + metrics = ChallengeMetricsCollector() # Challenge 관련 정보 가져오기 res = request.get_json() if not res: @@ -38,7 +39,8 @@ def create_challenge(): metrics.challenge_state.labels( challenge_id=challenge_id, - username=username + username=username, + state='active' ).set(1) metrics.challenge_operations.labels(operation='create',result='success').inc() @@ -46,6 +48,7 @@ def create_challenge(): @challenge_bp.route('/delete', methods=['POST']) def delete_userchallenges(): + metrics = ChallengeMetricsCollector() try: """ 사용자 챌린지 삭제 @@ -76,7 +79,8 @@ def delete_userchallenges(): # Metrics metrics.challenge_state.labels( challenge_id=challenge_id, - username=username + username=username, + state='inactive' ).set(0) metrics.challenge_operations.labels( operation='delete', @@ -91,6 +95,7 @@ def delete_userchallenges(): @challenge_bp.route('/status', methods=['POST']) def get_userchallenge_status(): """ 사용자 챌린지 상태 조회 """ + metrics = ChallengeMetricsCollector() try: # Challenge 관련 정보 가져오기 res = request.get_json() diff --git a/app/factory.py b/app/factory.py index ce68fb7..2223b78 100644 --- a/app/factory.py +++ b/app/factory.py @@ -1,13 +1,14 @@ import sys from requests import Response +from app.monitoring.ctf_metrics_collector import ChallengeMetricsCollector from app.monitoring.loki_logger import FlaskLokiLogger from app.monitoring.system_metrics_collector import SystemMetricsCollector from flask import Flask, g, request import threading from datetime import datetime from typing import Any, Dict, Type -from prometheus_client import generate_latest, CONTENT_TYPE_LATEST +from prometheus_client import REGISTRY, generate_latest, CONTENT_TYPE_LATEST from app.api.challenge import challenge_bp from app.config import Config @@ -46,13 +47,13 @@ def _init_extensions(self): db.create_all() def _init_metrics_collector(self): + + challenge_collector = ChallengeMetricsCollector() + + # System 메트릭 수집기 초기화 system_collector = SystemMetricsCollector(self.app) system_collector.start_collecting() - # CTF metrics - @self.app.route('/metrics/ctf') - def challenge_metrics(): - return generate_latest(), 200, {'Content-Type': CONTENT_TYPE_LATEST} def _setup_middleware(self): """미들웨어 설정""" diff --git a/app/monitoring/ctf_metrics_collector.py b/app/monitoring/ctf_metrics_collector.py index b9f9fd4..96d7e62 100644 --- a/app/monitoring/ctf_metrics_collector.py +++ b/app/monitoring/ctf_metrics_collector.py @@ -1,4 +1,4 @@ -from prometheus_client import Gauge, Counter +from prometheus_client import REGISTRY, Gauge, Counter, CONTENT_TYPE_LATEST class ChallengeMetricsCollector: def __init__(self): @@ -6,12 +6,53 @@ def __init__(self): self.challenge_state = Gauge( 'challenge_state', 'Current state of challenges', - ['challenge_id', 'username'] # 1: active, 0: inactive + ['challenge_id', 'username','state'] # 1: active, 0: inactive ) - + # API 요청 결과 카운터 self.challenge_operations = Counter( 'challenge_operations_total', 'Challenge operation results', ['operation', 'result'] # operation: create/delete/status, result: success/error ) + + # 레지스트리에 메트릭 등록 + # self.register_metrics() + + def register_metrics(self): + """ + Prometheus 레지스트리에 메트릭을 등록 + """ + REGISTRY.register(self) + + def collect(self): + """ + Prometheus Collector 인터페이스 구현 메서드 + 실제 메트릭을 수집하여 반환 + """ + yield self.challenge_state + yield self.challenge_operations + + def update_challenge_state(self, challenge_id: str, username: str, state: int): + """ + 챌린지 상태 업데이트 + :param challenge_id: 챌린지 ID + :param username: 사용자 이름 + :param state: 상태 (1: active, 0: inactive) + """ + self.challenge_state.labels( + challenge_id=challenge_id, + username=username + ).set(state) + + def record_challenge_operation(self, operation: str, result: str): + """ + 챌린지 작업 결과 기록 + :param operation: 작업 유형 (create/delete/status) + :param result: 결과 (success/error) + """ + self.challenge_operations.labels( + operation=operation, + result=result + ).inc() + diff --git a/app/monitoring/system_metrics_collector.py b/app/monitoring/system_metrics_collector.py index bc8ff79..03294c0 100644 --- a/app/monitoring/system_metrics_collector.py +++ b/app/monitoring/system_metrics_collector.py @@ -7,21 +7,21 @@ class SystemMetricsCollector: def __init__(self, app: Flask = None): - # CPU 메트릭 + # CPU metrics self.cpu_usage = Gauge( 'system_cpu_usage_percent', 'CPU Usage in Percent', ['cpu_type'] # user, system, idle ) - # 메모리 메트릭 + # Memory metrics self.memory_usage = Gauge( 'system_memory_bytes', 'Memory Usage in Bytes', ['type'] # used, free, cached, total ) - # 디스크 메트릭 + # Disk metrics self.disk_usage = Gauge( 'system_disk_bytes', 'Disk Usage in Bytes', @@ -34,7 +34,7 @@ def __init__(self, app: Flask = None): ['operation'] # read, write ) - # 네트워크 메트릭 + # Network metrics self.network_traffic = Counter( 'system_network_traffic_bytes', 'Network Traffic in Bytes', @@ -46,28 +46,26 @@ def __init__(self, app: Flask = None): 'Number of Network Connections', ['protocol', 'status'] # tcp/udp, ESTABLISHED/LISTEN/etc ) - + + # Register Flask app endpoint if provided if app is not None: self.init_app(app) def init_app(self, app: Flask): - """Flask 애플리케이션에 메트릭 엔드포인트 등록""" + """Register the metrics endpoint with a Flask app""" @app.route('/metrics') def metrics(): return Response(generate_latest(), mimetype=CONTENT_TYPE_LATEST) - - # 메트릭 수집 시작 - self.start_collecting() def collect_cpu_metrics(self): - """CPU 메트릭 수집""" + """Collect CPU metrics""" cpu_times = psutil.cpu_times_percent() self.cpu_usage.labels(cpu_type='user').set(cpu_times.user) self.cpu_usage.labels(cpu_type='system').set(cpu_times.system) self.cpu_usage.labels(cpu_type='idle').set(cpu_times.idle) def collect_memory_metrics(self): - """메모리 메트릭 수집""" + """Collect memory metrics""" mem = psutil.virtual_memory() self.memory_usage.labels(type='total').set(mem.total) self.memory_usage.labels(type='used').set(mem.used) @@ -75,8 +73,8 @@ def collect_memory_metrics(self): self.memory_usage.labels(type='cached').set(mem.cached) def collect_disk_metrics(self): - """디스크 메트릭 수집""" - # 디스크 사용량 + """Collect disk metrics""" + # Disk usage for partition in psutil.disk_partitions(): if partition.fstype: usage = psutil.disk_usage(partition.mountpoint) @@ -93,14 +91,14 @@ def collect_disk_metrics(self): type='free' ).set(usage.free) - # 디스크 I/O + # Disk I/O disk_io = psutil.disk_io_counters() self.disk_io.labels(operation='read').inc(disk_io.read_bytes) self.disk_io.labels(operation='write').inc(disk_io.write_bytes) def collect_network_metrics(self): - """네트워크 메트릭 수집""" - # 네트워크 트래픽 + """Collect network metrics""" + # Network traffic net_io = psutil.net_io_counters(pernic=True) for interface, counters in net_io.items(): self.network_traffic.labels( @@ -112,7 +110,7 @@ def collect_network_metrics(self): direction='transmitted' ).inc(counters.bytes_sent) - # 네트워크 연결 + # Network connections connections = psutil.net_connections() conn_count = {'tcp': {}, 'udp': {}} for conn in connections: @@ -128,7 +126,7 @@ def collect_network_metrics(self): ).set(count) def collect_metrics(self): - """모든 메트릭 수집""" + """Collect all metrics periodically""" while True: try: self.collect_cpu_metrics() @@ -137,10 +135,10 @@ def collect_metrics(self): self.collect_network_metrics() except Exception as e: print(f"Error collecting metrics: {e}") - time.sleep(15) # 15초 간격으로 수집 + time.sleep(15) # Collect metrics every 15 seconds def start_collecting(self): - """메트릭 수집 스레드 시작""" + """Start a thread to collect metrics""" thread = Thread(target=self.collect_metrics) thread.daemon = True - thread.start() \ No newline at end of file + thread.start() diff --git a/configs/prometheus/prometheus.yml b/configs/prometheus/prometheus.yml index 2c5bb70..77f697a 100644 --- a/configs/prometheus/prometheus.yml +++ b/configs/prometheus/prometheus.yml @@ -9,13 +9,13 @@ scrape_configs: - job_name: 'flask_system' static_configs: - - targets: ['192.168.80.2:5001'] + - targets: ['192.168.67.2:5001'] metrics_path: '/metrics' scrape_interval: 15s # CTF 메트릭 수집 (/metrics/ctf) - - job_name: 'flask_ctf' - static_configs: - - targets: ['192.168.80.2:5001'] - metrics_path: '/metrics/ctf' - scrape_interval: 15s \ No newline at end of file + # - job_name: 'flask_ctf' + # static_configs: + # - targets: ['192.168.67.2:5001'] + # metrics_path: '/metrics/ctf' + # scrape_interval: 15s \ No newline at end of file From c5bdf489ddba61e699dde184e4f015aa5d1299dd Mon Sep 17 00:00:00 2001 From: S0okJu <65109465+D7MeKz@users.noreply.github.com> Date: Mon, 20 Jan 2025 17:33:06 +0900 Subject: [PATCH 5/7] =?UTF-8?q?[Feat]=20prometheus=20logging=20client=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/api/challenge.py | 29 +++++++++++-------------- app/factory.py | 4 +--- app/monitoring/ctf_metrics_collector.py | 2 ++ 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/app/api/challenge.py b/app/api/challenge.py index 36a6101..1bae1a3 100644 --- a/app/api/challenge.py +++ b/app/api/challenge.py @@ -7,48 +7,46 @@ from app.exceptions.userchallenge import UserChallengeCreationError, UserChallengeDeletionError, UserChallengeNotFoundError from app.extensions.db.repository import UserChallengesRepository from app.extensions.k8s.client import K8sClient +from app.monitoring.ctf_metrics_collector import challenge_metrics_collector challenge_bp = Blueprint('challenge', __name__) - @challenge_bp.route('', methods=['POST']) def create_challenge(): """사용자 챌린지 생성""" - metrics = ChallengeMetricsCollector() # Challenge 관련 정보 가져오기 res = request.get_json() if not res: - metrics.challenge_operations.labels(operation='create',result='error').inc() + challenge_metrics_collector.challenge_operations.labels(operation='create',result='error').inc() raise InvalidRequest(error_msg="Request body is empty or not valid JSON") if 'challenge_id' not in res: - metrics.challenge_operations.labels(operation='create',result='error').inc() + challenge_metrics_collector.challenge_operations.labels(operation='create',result='error').inc() raise InvalidRequest(error_msg="Required field 'challenge_id' is missing in request") challenge_id = res['challenge_id'] if 'username' not in res: - metrics.challenge_operations.labels(operation='create',result='error').inc() + challenge_metrics_collector.challenge_operations.labels(operation='create',result='error').inc() raise InvalidRequest(error_msg="Required field 'username' is missing in request") username = res['username'] # 챌린지 생성 client = K8sClient() endpoint = client.create_challenge_resource(challenge_id, username) if not endpoint: - metrics.challenge_operations.labels(operation='create',result='error').inc() + challenge_metrics_collector.challenge_operations.labels(operation='create',result='error').inc() raise UserChallengeCreationError(error_msg=f"Faile to create challenge {challenge_id} for user {username}") - metrics.challenge_state.labels( + challenge_metrics_collector.challenge_state.labels( challenge_id=challenge_id, username=username, state='active' ).set(1) - metrics.challenge_operations.labels(operation='create',result='success').inc() + challenge_metrics_collector.challenge_operations.labels(operation='create',result='success').inc() return jsonify({'data' : {'port': endpoint}}), 200 @challenge_bp.route('/delete', methods=['POST']) def delete_userchallenges(): - metrics = ChallengeMetricsCollector() try: """ 사용자 챌린지 삭제 @@ -57,18 +55,18 @@ def delete_userchallenges(): res = request.get_json() if not res: log.error("No data provided") - metrics.challenge_operations.labels(operation='delete',result='error').inc() + challenge_metrics_collector.challenge_operations.labels(operation='delete',result='error').inc() raise UserChallengeDeletionError(error_msg="Request body is empty or not valid JSON") if 'challenge_id' not in res: log.error("No challenge_id provided") - metrics.challenge_operations.labels(operation='delete',result='error').inc() + challenge_metrics_collector.challenge_operations.labels(operation='delete',result='error').inc() raise InvalidRequest(error_msg="Required field 'challenge_id' is missing in request") challenge_id = res['challenge_id'] if 'username' not in res: log.error("No username provided") - metrics.challenge_operations.labels(operation='delete',result='error').inc() + challenge_metrics_collector.challenge_operations.labels(operation='delete',result='error').inc() raise InvalidRequest(error_msg="Required field 'username' is missing in request") username = res['username'] @@ -77,25 +75,24 @@ def delete_userchallenges(): client.delete_userchallenge(username, challenge_id) # Metrics - metrics.challenge_state.labels( + challenge_metrics_collector.challenge_state.labels( challenge_id=challenge_id, username=username, state='inactive' ).set(0) - metrics.challenge_operations.labels( + challenge_metrics_collector.challenge_operations.labels( operation='delete', result='success' ).inc() return jsonify({'message' : '챌린지가 정상적으로 삭제되었습니다.'}), 200 except JSONDecodeError as e: log.error("Invalid request format") - metrics.challenge_operations.labels(operation='delete',result='error').inc() + challenge_metrics_collector.challenge_operations.labels(operation='delete',result='error').inc() raise InvalidRequest(error_msg=str(e)) from e @challenge_bp.route('/status', methods=['POST']) def get_userchallenge_status(): """ 사용자 챌린지 상태 조회 """ - metrics = ChallengeMetricsCollector() try: # Challenge 관련 정보 가져오기 res = request.get_json() diff --git a/app/factory.py b/app/factory.py index 2223b78..bc19ae5 100644 --- a/app/factory.py +++ b/app/factory.py @@ -27,7 +27,7 @@ def __init__(self, config_class: Type[Config] = Config): self.app = Flask(__name__) self.app.config.from_object(config_class) self.logger = FlaskLokiLogger(app_name="challenge-api", loki_url=self.app.config['LOKI_URL']).logger - + # 초기 설정 self._init_extensions() self._setup_middleware() @@ -48,8 +48,6 @@ def _init_extensions(self): def _init_metrics_collector(self): - challenge_collector = ChallengeMetricsCollector() - # System 메트릭 수집기 초기화 system_collector = SystemMetricsCollector(self.app) system_collector.start_collecting() diff --git a/app/monitoring/ctf_metrics_collector.py b/app/monitoring/ctf_metrics_collector.py index 96d7e62..b17b69a 100644 --- a/app/monitoring/ctf_metrics_collector.py +++ b/app/monitoring/ctf_metrics_collector.py @@ -56,3 +56,5 @@ def record_challenge_operation(self, operation: str, result: str): result=result ).inc() + +challenge_metrics_collector = ChallengeMetricsCollector() From 06c680061c72556c3f68ed4241655c71ac2b1495 Mon Sep 17 00:00:00 2001 From: S0okJu <65109465+D7MeKz@users.noreply.github.com> Date: Mon, 20 Jan 2025 17:34:18 +0900 Subject: [PATCH 6/7] =?UTF-8?q?[Chore]=20=EB=B6=88=ED=95=84=EC=9A=94?= =?UTF-8?q?=ED=95=9C=20=ED=8C=8C=EC=9D=BC=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- a.json | 105 --------------------------------------------------------- 1 file changed, 105 deletions(-) delete mode 100644 a.json diff --git a/a.json b/a.json deleted file mode 100644 index 9cbd3d0..0000000 --- a/a.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "panels": [ - { - "type": "table", - "options": { - "showHeader": true, - "footer": { - "show": false - } - }, - "fieldConfig": { - "defaults": { - "custom": { - "align": "center", - "displayMode": "color-background", - "filterable": true - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "Inactive", - "color": "red" - }, - "1": { - "text": "Active", - "color": "green" - } - } - } - ] - } - }, - "transformations": [ - { - "id": "groupBy", - "options": { - "fields": { - "username": { - "operation": "groupby" - }, - "challenge_id": { - "operation": "groupby" - }, - "Value": { - "operation": "aggregate", - "aggregations": ["lastNotNull"] - } - } - } - } - ] - } - ], - "preload": false, - "refresh": "", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "New dashboard", - "version": 0, - "weekStart": "" - } \ No newline at end of file From a3cde91702deab46bb475184ef8aba87dd57b654 Mon Sep 17 00:00:00 2001 From: S0okJu <65109465+D7MeKz@users.noreply.github.com> Date: Mon, 20 Jan 2025 17:35:08 +0900 Subject: [PATCH 7/7] =?UTF-8?q?[Chore]=20=EB=B6=88=ED=95=84=EC=9A=94?= =?UTF-8?q?=ED=95=9C=20=ED=8C=8C=EC=9D=BC=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 8db6d2e..07e2815 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -140,7 +140,7 @@ services: - SECRET_KEY=your-super-secret-key-here # Kafka - KAFKA_BOOTSTRAP_SERVERS=127.0.0.1:9093 - - KAFKA_TOPIC=challenge-statustem_metrics_collectos + - KAFKA_TOPIC=challenge-status - KAFKA_GROUP_ID=challenge-consumer-group # Loki 설정 추가 - LOKI_URL=http://127.0.0.1:3100/loki/api/v1/push