Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 1 addition & 33 deletions app/api/challenge.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,33 +17,21 @@ def create_challenge():
# Challenge 관련 정보 가져오기
res = request.get_json()
if not res:
challenge_metrics_collector.challenge_operations.labels(operation='create',result='error').inc()
raise InvalidRequest(error_msg="Request body is empty or not valid JSON")
if 'challenge_id' not in res:
challenge_metrics_collector.challenge_operations.labels(operation='create',result='error').inc()
raise InvalidRequest(error_msg="Required field 'challenge_id' is missing in request")

challenge_id = res['challenge_id']

if 'username' not in res:
challenge_metrics_collector.challenge_operations.labels(operation='create',result='error').inc()
raise InvalidRequest(error_msg="Required field 'username' is missing in request")
username = res['username']
# 챌린지 생성
client = K8sClient()
endpoint = client.create_challenge_resource(challenge_id, username)
if not endpoint:
challenge_metrics_collector.challenge_operations.labels(operation='create',result='error').inc()
raise UserChallengeCreationError(error_msg=f"Faile to create challenge {challenge_id} for user {username}")

challenge_metrics_collector.challenge_state.labels(
challenge_id=challenge_id,
username=username,
state='active'
).set(1)

challenge_metrics_collector.challenge_operations.labels(operation='create',result='success').inc()

return jsonify({'data' : {'port': endpoint}}), 200

@challenge_bp.route('/delete', methods=['POST'])
Expand All @@ -55,40 +43,23 @@ def delete_userchallenges():
# Challenge 관련 정보 가져오기
res = request.get_json()
if not res:
log.error("No data provided")
challenge_metrics_collector.challenge_operations.labels(operation='delete',result='error').inc()
raise UserChallengeDeletionError(error_msg="Request body is empty or not valid JSON")

if 'challenge_id' not in res:
log.error("No challenge_id provided")
challenge_metrics_collector.challenge_operations.labels(operation='delete',result='error').inc()
raise InvalidRequest(error_msg="Required field 'challenge_id' is missing in request")
challenge_id = res['challenge_id']

if 'username' not in res:
log.error("No username provided")
challenge_metrics_collector.challenge_operations.labels(operation='delete',result='error').inc()
raise InvalidRequest(error_msg="Required field 'username' is missing in request")
username = res['username']

# 사용자 챌린지 삭제
client = K8sClient()
client.delete_userchallenge(username, challenge_id)

# Metrics
challenge_metrics_collector.challenge_state.labels(
challenge_id=challenge_id,
username=username,
state='inactive'
).set(0)
challenge_metrics_collector.challenge_operations.labels(
operation='delete',
result='success'
).inc()

return jsonify({'message' : '챌린지가 정상적으로 삭제되었습니다.'}), 200
except JSONDecodeError as e:
log.error("Invalid request format")
challenge_metrics_collector.challenge_operations.labels(operation='delete',result='error').inc()
raise InvalidRequest(error_msg=str(e)) from e

@challenge_bp.route('/status', methods=['POST'])
Expand All @@ -98,16 +69,13 @@ def get_userchallenge_status():
# Challenge 관련 정보 가져오기
res = request.get_json()
if not res:
log.error("No data provided")
raise UserChallengeDeletionError(error_msg="Request body is empty or not valid JSON")

if 'challenge_id' not in res:
log.error("No challenge_id provided")
raise InvalidRequest(error_msg="Required field 'challenge_id' is missing in request")
challenge_id = res['challenge_id']

if 'username' not in res:
log.error("No username provided")
raise InvalidRequest(error_msg="Required field 'username' is missing in request")
username = res['username']

Expand Down
45 changes: 30 additions & 15 deletions app/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,35 +104,50 @@ def _get_request_context(self) -> Dict[str, Any]:
"request_id": "unknown",
"context_error": str(e)
}

def _log_request(self, response, processing_time: float):
"""HTTP 요청 로깅"""
try:
context = self._get_request_context()

# Prepare labels (these will be indexed by Loki)
labels = {
"request_id": context.get("request_id", "unknown"),
"status_code": str(getattr(response, 'status_code', 'unknown')),
"method": context.get("method", "UNKNOWN"),
tags = {
"request_id": str(context.get("request_id", "unknown")),
"status_code": str(response.status_code),
"method": str(context.get("method", "UNKNOWN")),
}

# Prepare log content
log_content = {
"processing_time_ms": round(processing_time * 1000, 2),
"remote_addr": context.get("remote_addr", ""),
"user_agent": context.get("user_agent", ""),
"path": context.get("path", ""),
}


self.logger.info(
"HTTP Request",
extra={
"labels": labels,
"content": log_content
}
)
if response.status_code >= 500:
self.logger.error(
"HTTP Request",
extra={
"tags": tags,
"content": log_content
}
)
elif response.status_code >= 400:
self.logger.warning(
"HTTP Request",
extra={
"tags": tags,
"content": log_content
}
)
else:
self.logger.info(
"HTTP Request",
extra={
"tags": tags,
"content": log_content
}
)
except Exception as e:
# 로깅 중 오류 발생 시 기본 로깅
self.logger.error(f"Logging error: {str(e)}")
Expand All @@ -145,7 +160,7 @@ def _log_error(self, error: CustomBaseException):
self.logger.error(
"Application Error",
extra={
"labels": {
"tags": {
"error_type": str(error.error_type.value),
"request_id": request.headers.get('X-Request-ID', 'unknown') if request else 'unknown'
},
Expand Down
29 changes: 20 additions & 9 deletions app/monitoring/async_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,29 +8,40 @@ def __init__(self, handler):
self.handler = handler
self.log_queue = queue.Queue()
self.stop_event = threading.Event()
self.thread = threading.Thread(target=self._log_worker)
self.thread = threading.Thread(target=self._log_worker, name="AsyncLogWorker")
self.thread.daemon = True
self.thread.start()

def emit(self, record):
"""Put a log record in the queue."""
try:
self.log_queue.put(record)
except Exception:
self.handleError(record)
self.log_queue.put(record, block=False)
except queue.Full:
# If the queue is full, we drop the log to avoid blocking.
print("Log queue is full. Dropping log record.")

def _log_worker(self):
"""Process log records from the queue."""
while not self.stop_event.is_set():
try:
# Wait for a log record from the queue
record = self.log_queue.get(timeout=0.2)
self.handler.emit(record)
self.log_queue.task_done()
try:
self.handler.emit(record)
except Exception as handler_error:
# If an error occurs in the handler, we log it
print(f"Error emitting log record: {handler_error}")
finally:
self.log_queue.task_done()
except queue.Empty:
# Continue if the queue is empty
continue
except Exception as e:
# 로깅 중 오류 처리
print(f"Async logging error: {e}")
# Handle unexpected exceptions
print(f"Unexpected error in async logging: {e}")

def close(self):
"""Shutdown the logging thread gracefully."""
self.stop_event.set()
self.thread.join()
super().close()
super().close()
72 changes: 25 additions & 47 deletions app/monitoring/loki_logger.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,8 @@ def __init__(self, app_name,loki_url: str):

def _setup_logger(self, loki_url: str) -> logging.Logger:
"""Loki 로거 설정"""
# Define static tags for Loki indexing
tags = {
"app": self.app_name,
"app": self.app_name
}

handler = LokiHandler(
Expand All @@ -27,52 +26,13 @@ def _setup_logger(self, loki_url: str) -> logging.Logger:
version="1",
)


handler.setFormatter(LokiJsonFormatter())
async_handler = AsyncHandler(handler)
logger = logging.getLogger(self.app_name)
logger.setLevel(logging.INFO)
logger.setLevel(logging.DEBUG)
logger.addHandler(async_handler)
return logger

def log_info(self, message: str, labels: dict = None, content: dict = None):
"""
INFO 레벨 로깅 메서드

Args:
message (str): 로깅할 메시지
labels (dict, optional): 로그 인덱싱을 위한 라벨
content (dict, optional): 추가 로그 컨텍스트 정보
"""
try:
# 기본 labels 설정
default_labels = {
"app": self.app_name,
"level": "INFO"
}

# 제공된 labels와 병합
if labels:
default_labels.update(labels)

# 기본 content 설정
default_content = {
"message": message
}

# 제공된 content와 병합
if content:
default_content.update(content)

self.logger.info(
message,
extra={
"labels": default_labels,
"content": default_content
}
)
except Exception as e:
print(f"Logging error: {e}", file=sys.stderr)


class LokiJsonFormatter(logging.Formatter):
Expand All @@ -82,14 +42,13 @@ def format(self, record):
timestamp_ns = str(int(time.time() * 1e9))

# record에서 직접 labels와 content 추출
labels = getattr(record, 'labels', {})
# tags = getattr(record, 'tags', {})
content = getattr(record, 'content', {})

# 기본 로그 정보 추가
base_content = {
"level": record.levelname,
"message": record.getMessage(),
"logger": record.name
"level": record.levelname,
}

# 예외 정보 추가 (있는 경우)
Expand All @@ -99,14 +58,14 @@ def format(self, record):
"message": str(record.exc_info[1]),
"traceback": traceback.format_exception(*record.exc_info)
}


# content에 기본 로그 정보 병합
full_content = {**base_content, **content}

# 로그 구조 생성
log_entry = {
"timestamp": timestamp_ns,
"labels": labels,
"content": full_content
}

Expand All @@ -124,4 +83,23 @@ def format(self, record):
"record_details": str(getattr(record, '__dict__', 'No __dict__'))
}
}
return json.dumps(fallback_entry)
return json.dumps(fallback_entry)

def _serialize_dict(self, data, max_depth=3, current_depth=0):
"""재귀적으로 dict을 직렬화"""
if current_depth >= max_depth:
return "<Max depth reached>"
if isinstance(data, dict):
return {
key: self._serialize_dict(value, max_depth, current_depth + 1)
for key, value in data.items()
}
elif isinstance(data, (list, tuple, set)):
return [
self._serialize_dict(item, max_depth, current_depth + 1)
for item in data
]
elif hasattr(data, "__dict__"):
return self._serialize_dict(data.__dict__, max_depth, current_depth + 1)
else:
return data