From 5bc9f8729757fa0ccc0ca88567bc53619f971b76 Mon Sep 17 00:00:00 2001 From: S0okJu Date: Wed, 22 Jan 2025 10:43:48 +0900 Subject: [PATCH 1/2] =?UTF-8?q?[Feat]=20log=20error=20=EC=A7=81=EB=A0=AC?= =?UTF-8?q?=ED=99=94=20=EB=B0=8F=20=EB=A1=9C=EA=B9=85=20=EA=B5=AC=EC=A1=B0?= =?UTF-8?q?=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/api/challenge.py | 2 ++ app/factory.py | 32 ++++++++++++++++++++++++++------ app/monitoring/loki_logger.py | 6 ------ 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/app/api/challenge.py b/app/api/challenge.py index 2d7613f..12ac4de 100644 --- a/app/api/challenge.py +++ b/app/api/challenge.py @@ -81,10 +81,12 @@ def delete_userchallenges(): username=username, state='inactive' ).set(0) + challenge_metrics_collector.challenge_operations.labels( operation='delete', result='success' ).inc() + return jsonify({'message' : '챌린지가 정상적으로 삭제되었습니다.'}), 200 except JSONDecodeError as e: log.error("Invalid request format") diff --git a/app/factory.py b/app/factory.py index bc19ae5..5a79d74 100644 --- a/app/factory.py +++ b/app/factory.py @@ -1,3 +1,4 @@ +import json import sys from requests import Response @@ -111,7 +112,7 @@ def _log_request(self, response, processing_time: float): context = self._get_request_context() # Prepare labels (these will be indexed by Loki) - labels = { + tags = { "request_id": context.get("request_id", "unknown"), "status_code": str(getattr(response, 'status_code', 'unknown')), "method": context.get("method", "UNKNOWN"), @@ -124,12 +125,11 @@ def _log_request(self, response, processing_time: float): "user_agent": context.get("user_agent", ""), "path": context.get("path", ""), } - self.logger.info( "HTTP Request", extra={ - "labels": labels, + "tags": tags, "content": log_content } ) @@ -138,22 +138,40 @@ def _log_request(self, response, processing_time: float): self.logger.error(f"Logging error: {str(e)}") + def _log_error(self, error: CustomBaseException): """에러 로깅""" try: + # error_msg가 JSON 형태라면 파싱 + parsed_error_msg = {} + if error.error_msg: + try: + parsed_error_msg = json.loads(error.error_msg.split("HTTP response body: ", 1)[-1]) + except (json.JSONDecodeError, IndexError): + parsed_error_msg = {"raw_error_msg": error.error_msg} + + # 중요한 정보만 추출 + simplified_error_msg = { + "status": parsed_error_msg.get("status", "Unknown"), + "message": parsed_error_msg.get("message", "No message provided"), + "reason": parsed_error_msg.get("reason", "Unknown"), + "code": parsed_error_msg.get("code", "Unknown"), + "details": parsed_error_msg.get("details", {}), + } + # 로깅 self.logger.error( "Application Error", extra={ - "labels": { + "tags": { "error_type": str(error.error_type.value), + "status_code": error.status_code, "request_id": request.headers.get('X-Request-ID', 'unknown') if request else 'unknown' }, "content": { "error_type": str(error.error_type.value), "error_message": str(error.message), - "error_msg": str(error.error_msg or ''), - "status_code": error.status_code, + "error_details": simplified_error_msg, } } ) @@ -163,6 +181,8 @@ def _log_error(self, error: CustomBaseException): print(f"[DEBUG] Logging error: {log_error}", file=sys.stderr) self.logger.error(f"Error logging failed: {str(log_error)}") + + def run(self, **kwargs): """애플리케이션 실행""" self.app.run(**kwargs) diff --git a/app/monitoring/loki_logger.py b/app/monitoring/loki_logger.py index 74d195c..a3c41f0 100644 --- a/app/monitoring/loki_logger.py +++ b/app/monitoring/loki_logger.py @@ -16,18 +16,12 @@ def __init__(self, app_name,loki_url: str): def _setup_logger(self, loki_url: str) -> logging.Logger: """Loki 로거 설정""" - # Define static tags for Loki indexing - tags = { - "app": self.app_name, - } handler = LokiHandler( url=loki_url, - tags=tags, version="1", ) - handler.setFormatter(LokiJsonFormatter()) async_handler = AsyncHandler(handler) logger = logging.getLogger(self.app_name) From 1bfb420ce7c815252c9bf395e420776abbb97f2a Mon Sep 17 00:00:00 2001 From: S0okJu Date: Wed, 22 Jan 2025 15:29:13 +0900 Subject: [PATCH 2/2] =?UTF-8?q?[Feat]=20=EB=A1=9C=EA=B7=B8=20=ED=98=95?= =?UTF-8?q?=ED=83=9C=20=EA=B0=84=EC=86=8C=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/api/challenge.py | 36 +---------------- app/factory.py | 67 ++++++++++++++---------------- app/monitoring/async_handler.py | 29 ++++++++----- app/monitoring/loki_logger.py | 72 +++++++++++++-------------------- 4 files changed, 80 insertions(+), 124 deletions(-) diff --git a/app/api/challenge.py b/app/api/challenge.py index 12ac4de..f55e0b6 100644 --- a/app/api/challenge.py +++ b/app/api/challenge.py @@ -17,33 +17,21 @@ def create_challenge(): # Challenge 관련 정보 가져오기 res = request.get_json() if not res: - challenge_metrics_collector.challenge_operations.labels(operation='create',result='error').inc() raise InvalidRequest(error_msg="Request body is empty or not valid JSON") if 'challenge_id' not in res: - challenge_metrics_collector.challenge_operations.labels(operation='create',result='error').inc() raise InvalidRequest(error_msg="Required field 'challenge_id' is missing in request") challenge_id = res['challenge_id'] if 'username' not in res: - challenge_metrics_collector.challenge_operations.labels(operation='create',result='error').inc() raise InvalidRequest(error_msg="Required field 'username' is missing in request") username = res['username'] # 챌린지 생성 client = K8sClient() endpoint = client.create_challenge_resource(challenge_id, username) if not endpoint: - challenge_metrics_collector.challenge_operations.labels(operation='create',result='error').inc() raise UserChallengeCreationError(error_msg=f"Faile to create challenge {challenge_id} for user {username}") - challenge_metrics_collector.challenge_state.labels( - challenge_id=challenge_id, - username=username, - state='active' - ).set(1) - - challenge_metrics_collector.challenge_operations.labels(operation='create',result='success').inc() - return jsonify({'data' : {'port': endpoint}}), 200 @challenge_bp.route('/delete', methods=['POST']) @@ -55,42 +43,23 @@ def delete_userchallenges(): # Challenge 관련 정보 가져오기 res = request.get_json() if not res: - log.error("No data provided") - challenge_metrics_collector.challenge_operations.labels(operation='delete',result='error').inc() raise UserChallengeDeletionError(error_msg="Request body is empty or not valid JSON") if 'challenge_id' not in res: - log.error("No challenge_id provided") - challenge_metrics_collector.challenge_operations.labels(operation='delete',result='error').inc() raise InvalidRequest(error_msg="Required field 'challenge_id' is missing in request") challenge_id = res['challenge_id'] if 'username' not in res: - log.error("No username provided") - challenge_metrics_collector.challenge_operations.labels(operation='delete',result='error').inc() raise InvalidRequest(error_msg="Required field 'username' is missing in request") username = res['username'] # 사용자 챌린지 삭제 client = K8sClient() client.delete_userchallenge(username, challenge_id) - - # Metrics - challenge_metrics_collector.challenge_state.labels( - challenge_id=challenge_id, - username=username, - state='inactive' - ).set(0) - - challenge_metrics_collector.challenge_operations.labels( - operation='delete', - result='success' - ).inc() - + return jsonify({'message' : '챌린지가 정상적으로 삭제되었습니다.'}), 200 except JSONDecodeError as e: log.error("Invalid request format") - challenge_metrics_collector.challenge_operations.labels(operation='delete',result='error').inc() raise InvalidRequest(error_msg=str(e)) from e @challenge_bp.route('/status', methods=['POST']) @@ -100,16 +69,13 @@ def get_userchallenge_status(): # Challenge 관련 정보 가져오기 res = request.get_json() if not res: - log.error("No data provided") raise UserChallengeDeletionError(error_msg="Request body is empty or not valid JSON") if 'challenge_id' not in res: - log.error("No challenge_id provided") raise InvalidRequest(error_msg="Required field 'challenge_id' is missing in request") challenge_id = res['challenge_id'] if 'username' not in res: - log.error("No username provided") raise InvalidRequest(error_msg="Required field 'username' is missing in request") username = res['username'] diff --git a/app/factory.py b/app/factory.py index 5a79d74..884a579 100644 --- a/app/factory.py +++ b/app/factory.py @@ -1,4 +1,3 @@ -import json import sys from requests import Response @@ -105,7 +104,7 @@ def _get_request_context(self) -> Dict[str, Any]: "request_id": "unknown", "context_error": str(e) } - + def _log_request(self, response, processing_time: float): """HTTP 요청 로깅""" try: @@ -113,65 +112,63 @@ def _log_request(self, response, processing_time: float): # Prepare labels (these will be indexed by Loki) tags = { - "request_id": context.get("request_id", "unknown"), - "status_code": str(getattr(response, 'status_code', 'unknown')), - "method": context.get("method", "UNKNOWN"), + "request_id": str(context.get("request_id", "unknown")), + "status_code": str(response.status_code), + "method": str(context.get("method", "UNKNOWN")), } # Prepare log content log_content = { - "processing_time_ms": round(processing_time * 1000, 2), "remote_addr": context.get("remote_addr", ""), "user_agent": context.get("user_agent", ""), "path": context.get("path", ""), } - - self.logger.info( - "HTTP Request", - extra={ - "tags": tags, - "content": log_content - } - ) + + if response.status_code >= 500: + self.logger.error( + "HTTP Request", + extra={ + "tags": tags, + "content": log_content + } + ) + elif response.status_code >= 400: + self.logger.warning( + "HTTP Request", + extra={ + "tags": tags, + "content": log_content + } + ) + else: + self.logger.info( + "HTTP Request", + extra={ + "tags": tags, + "content": log_content + } + ) except Exception as e: # 로깅 중 오류 발생 시 기본 로깅 self.logger.error(f"Logging error: {str(e)}") - def _log_error(self, error: CustomBaseException): """에러 로깅""" try: - # error_msg가 JSON 형태라면 파싱 - parsed_error_msg = {} - if error.error_msg: - try: - parsed_error_msg = json.loads(error.error_msg.split("HTTP response body: ", 1)[-1]) - except (json.JSONDecodeError, IndexError): - parsed_error_msg = {"raw_error_msg": error.error_msg} - - # 중요한 정보만 추출 - simplified_error_msg = { - "status": parsed_error_msg.get("status", "Unknown"), - "message": parsed_error_msg.get("message", "No message provided"), - "reason": parsed_error_msg.get("reason", "Unknown"), - "code": parsed_error_msg.get("code", "Unknown"), - "details": parsed_error_msg.get("details", {}), - } - # 로깅 self.logger.error( "Application Error", extra={ "tags": { "error_type": str(error.error_type.value), - "status_code": error.status_code, "request_id": request.headers.get('X-Request-ID', 'unknown') if request else 'unknown' }, "content": { "error_type": str(error.error_type.value), "error_message": str(error.message), - "error_details": simplified_error_msg, + "error_msg": str(error.error_msg or ''), + "status_code": error.status_code, } } ) @@ -181,8 +178,6 @@ def _log_error(self, error: CustomBaseException): print(f"[DEBUG] Logging error: {log_error}", file=sys.stderr) self.logger.error(f"Error logging failed: {str(log_error)}") - - def run(self, **kwargs): """애플리케이션 실행""" self.app.run(**kwargs) diff --git a/app/monitoring/async_handler.py b/app/monitoring/async_handler.py index 48dfd0c..dee0c64 100644 --- a/app/monitoring/async_handler.py +++ b/app/monitoring/async_handler.py @@ -8,29 +8,40 @@ def __init__(self, handler): self.handler = handler self.log_queue = queue.Queue() self.stop_event = threading.Event() - self.thread = threading.Thread(target=self._log_worker) + self.thread = threading.Thread(target=self._log_worker, name="AsyncLogWorker") self.thread.daemon = True self.thread.start() def emit(self, record): + """Put a log record in the queue.""" try: - self.log_queue.put(record) - except Exception: - self.handleError(record) + self.log_queue.put(record, block=False) + except queue.Full: + # If the queue is full, we drop the log to avoid blocking. + print("Log queue is full. Dropping log record.") def _log_worker(self): + """Process log records from the queue.""" while not self.stop_event.is_set(): try: + # Wait for a log record from the queue record = self.log_queue.get(timeout=0.2) - self.handler.emit(record) - self.log_queue.task_done() + try: + self.handler.emit(record) + except Exception as handler_error: + # If an error occurs in the handler, we log it + print(f"Error emitting log record: {handler_error}") + finally: + self.log_queue.task_done() except queue.Empty: + # Continue if the queue is empty continue except Exception as e: - # 로깅 중 오류 처리 - print(f"Async logging error: {e}") + # Handle unexpected exceptions + print(f"Unexpected error in async logging: {e}") def close(self): + """Shutdown the logging thread gracefully.""" self.stop_event.set() self.thread.join() - super().close() \ No newline at end of file + super().close() diff --git a/app/monitoring/loki_logger.py b/app/monitoring/loki_logger.py index a3c41f0..55b77c0 100644 --- a/app/monitoring/loki_logger.py +++ b/app/monitoring/loki_logger.py @@ -16,57 +16,23 @@ def __init__(self, app_name,loki_url: str): def _setup_logger(self, loki_url: str) -> logging.Logger: """Loki 로거 설정""" + tags = { + "app": self.app_name + } handler = LokiHandler( url=loki_url, + tags=tags, version="1", ) handler.setFormatter(LokiJsonFormatter()) async_handler = AsyncHandler(handler) logger = logging.getLogger(self.app_name) - logger.setLevel(logging.INFO) + logger.setLevel(logging.DEBUG) logger.addHandler(async_handler) return logger - def log_info(self, message: str, labels: dict = None, content: dict = None): - """ - INFO 레벨 로깅 메서드 - - Args: - message (str): 로깅할 메시지 - labels (dict, optional): 로그 인덱싱을 위한 라벨 - content (dict, optional): 추가 로그 컨텍스트 정보 - """ - try: - # 기본 labels 설정 - default_labels = { - "app": self.app_name, - "level": "INFO" - } - - # 제공된 labels와 병합 - if labels: - default_labels.update(labels) - - # 기본 content 설정 - default_content = { - "message": message - } - - # 제공된 content와 병합 - if content: - default_content.update(content) - - self.logger.info( - message, - extra={ - "labels": default_labels, - "content": default_content - } - ) - except Exception as e: - print(f"Logging error: {e}", file=sys.stderr) class LokiJsonFormatter(logging.Formatter): @@ -76,14 +42,13 @@ def format(self, record): timestamp_ns = str(int(time.time() * 1e9)) # record에서 직접 labels와 content 추출 - labels = getattr(record, 'labels', {}) + # tags = getattr(record, 'tags', {}) content = getattr(record, 'content', {}) # 기본 로그 정보 추가 base_content = { - "level": record.levelname, "message": record.getMessage(), - "logger": record.name + "level": record.levelname, } # 예외 정보 추가 (있는 경우) @@ -93,6 +58,7 @@ def format(self, record): "message": str(record.exc_info[1]), "traceback": traceback.format_exception(*record.exc_info) } + # content에 기본 로그 정보 병합 full_content = {**base_content, **content} @@ -100,7 +66,6 @@ def format(self, record): # 로그 구조 생성 log_entry = { "timestamp": timestamp_ns, - "labels": labels, "content": full_content } @@ -118,4 +83,23 @@ def format(self, record): "record_details": str(getattr(record, '__dict__', 'No __dict__')) } } - return json.dumps(fallback_entry) \ No newline at end of file + return json.dumps(fallback_entry) + + def _serialize_dict(self, data, max_depth=3, current_depth=0): + """재귀적으로 dict을 직렬화""" + if current_depth >= max_depth: + return "" + if isinstance(data, dict): + return { + key: self._serialize_dict(value, max_depth, current_depth + 1) + for key, value in data.items() + } + elif isinstance(data, (list, tuple, set)): + return [ + self._serialize_dict(item, max_depth, current_depth + 1) + for item in data + ] + elif hasattr(data, "__dict__"): + return self._serialize_dict(data.__dict__, max_depth, current_depth + 1) + else: + return data \ No newline at end of file