Skip to content
Permalink
Browse files
feat: support AuditLog and RequestLog protos (#274)
  • Loading branch information
daniel-sanche committed Jun 9, 2021
1 parent cda1e61 commit 5d91be9f121c364cbd53c6a9fffc4fb6ca6bd324
Showing with 138 additions and 23 deletions.
  1. +10 −2 google/cloud/logging_v2/entries.py
  2. +2 −0 setup.py
  3. +112 −21 tests/system/test_system.py
  4. +14 −0 tests/unit/test_entries.py
@@ -27,6 +27,9 @@
from google.cloud._helpers import _rfc3339_nanos_to_datetime
from google.cloud._helpers import _datetime_to_rfc3339

# import officially supported proto definitions
import google.cloud.audit.audit_log_pb2 # noqa: F401
import google.cloud.appengine_logging # noqa: F401

_GLOBAL_RESOURCE = Resource(type="global", labels={})

@@ -316,13 +319,18 @@ def payload_pb(self):

@property
def payload_json(self):
if not isinstance(self.payload, Any):
if isinstance(self.payload, collections.abc.Mapping):
return self.payload

def to_api_repr(self):
"""API repr (JSON format) for entry."""
info = super(ProtobufEntry, self).to_api_repr()
info["protoPayload"] = MessageToDict(self.payload)
proto_payload = None
if self.payload_json:
proto_payload = dict(self.payload_json)
elif self.payload_pb:
proto_payload = MessageToDict(self.payload_pb)
info["protoPayload"] = proto_payload
return info

def parse_message(self, message):
@@ -30,6 +30,8 @@
release_status = "Development Status :: 5 - Production/Stable"
dependencies = [
"google-api-core[grpc] >= 1.22.2, < 2.0.0dev",
"google-cloud-appengine-logging >= 0.1.0, < 1.0.0dev",
"google-cloud-audit-log >= 0.1.0, < 1.0.0dev",
"google-cloud-core >= 1.4.1, < 2.0dev",
"proto-plus >= 1.11.0",
"packaging >= 14.3",
@@ -16,9 +16,11 @@
from datetime import timedelta
from datetime import timezone
import logging
import numbers
import os
import pytest
import unittest
import uuid

from google.api_core.exceptions import BadGateway
from google.api_core.exceptions import Conflict
@@ -36,6 +38,8 @@
from google.cloud.logging_v2 import client
from google.cloud.logging_v2.resource import Resource

from google.protobuf.struct_pb2 import Struct, Value, ListValue, NullValue

from test_utils.retry import RetryErrors
from test_utils.retry import RetryResult
from test_utils.system import unique_resource_id
@@ -142,32 +146,119 @@ def tearDown(self):
def _logger_name(prefix):
return prefix + unique_resource_id("-")

def test_list_entry_with_unregistered(self):
from google.protobuf import any_pb2
@staticmethod
def _to_value(data):
if data is None:
return Value(null_value=NullValue.NULL_VALUE)
elif isinstance(data, numbers.Number):
return Value(number_value=data)
elif isinstance(data, str):
return Value(string_value=data)
elif isinstance(data, bool):
return Value(bool_value=data)
elif isinstance(data, (list, tuple, set)):
return Value(
list_value=ListValue(values=(TestLogging._to_value(e) for e in data))
)
elif isinstance(data, dict):
return Value(struct_value=TestLogging._dict_to_struct(data))
else:
raise TypeError("Unknown data type: %r" % type(data))

@staticmethod
def _dict_to_struct(data):
return Struct(fields={k: TestLogging._to_value(v) for k, v in data.items()})

def test_list_entry_with_auditlog(self):
"""
Test emitting and listing logs containing a google.cloud.audit.AuditLog proto message
"""
from google.protobuf import descriptor_pool
from google.cloud.logging_v2 import entries

pool = descriptor_pool.Default()
type_name = "google.cloud.audit.AuditLog"
# Make sure the descriptor is not known in the registry.
with self.assertRaises(KeyError):
pool.FindMessageTypeByName(type_name)

type_url = "type.googleapis.com/" + type_name
filter_ = self.TYPE_FILTER.format(type_url) + f" AND {_time_filter}"
entry_iter = iter(Config.CLIENT.list_entries(page_size=1, filter_=filter_))
# Make sure the descriptor is known in the registry.
# Raises KeyError if unknown
pool.FindMessageTypeByName(type_name)

# create log
audit_dict = {
"@type": type_url,
"methodName": "test",
"requestMetadata": {"callerIp": "::1", "callerSuppliedUserAgent": "test"},
"resourceName": "test",
"serviceName": "test",
"status": {"code": 0},
}
audit_struct = self._dict_to_struct(audit_dict)

logger = Config.CLIENT.logger(f"audit-proto-{uuid.uuid1()}")
logger.log_proto(audit_struct)

# retrieve log
retry = RetryErrors((TooManyRequests, StopIteration), max_tries=8)
protobuf_entry = retry(lambda: next(logger.list_entries()))()

retry = RetryErrors(TooManyRequests)
protobuf_entry = retry(lambda: next(entry_iter))()
self.assertIsInstance(protobuf_entry, entries.ProtobufEntry)
self.assertIsNone(protobuf_entry.payload_pb)
self.assertIsInstance(protobuf_entry.payload_json, dict)
self.assertEqual(protobuf_entry.payload_json["@type"], type_url)
self.assertEqual(
protobuf_entry.payload_json["methodName"], audit_dict["methodName"]
)
self.assertEqual(
protobuf_entry.to_api_repr()["protoPayload"]["@type"], type_url
)
self.assertEqual(
protobuf_entry.to_api_repr()["protoPayload"]["methodName"],
audit_dict["methodName"],
)

def test_list_entry_with_requestlog(self):
"""
Test emitting and listing logs containing a google.appengine.logging.v1.RequestLog proto message
"""
from google.protobuf import descriptor_pool
from google.cloud.logging_v2 import entries

pool = descriptor_pool.Default()
type_name = "google.appengine.logging.v1.RequestLog"
type_url = "type.googleapis.com/" + type_name
# Make sure the descriptor is known in the registry.
# Raises KeyError if unknown
pool.FindMessageTypeByName(type_name)

# create log
req_dict = {
"@type": type_url,
"ip": "0.0.0.0",
"appId": "test",
"versionId": "test",
"requestId": "12345",
"latency": "500.0s",
"method": "GET",
"status": 500,
"resource": "test",
"httpVersion": "HTTP/1.1",
}
req_struct = self._dict_to_struct(req_dict)

logger = Config.CLIENT.logger(f"req-proto-{uuid.uuid1()}")
logger.log_proto(req_struct)

# retrieve log
retry = RetryErrors((TooManyRequests, StopIteration), max_tries=8)
protobuf_entry = retry(lambda: next(logger.list_entries()))()

self.assertIsInstance(protobuf_entry, entries.ProtobufEntry)
if Config.CLIENT._use_grpc:
self.assertIsNone(protobuf_entry.payload_json)
self.assertIsInstance(protobuf_entry.payload_pb, any_pb2.Any)
self.assertEqual(protobuf_entry.payload_pb.type_url, type_url)
else:
self.assertIsNone(protobuf_entry.payload_pb)
self.assertEqual(protobuf_entry.payload_json["@type"], type_url)
self.assertIsNone(protobuf_entry.payload_pb)
self.assertIsInstance(protobuf_entry.payload_json, dict)
self.assertEqual(protobuf_entry.payload_json["@type"], type_url)
self.assertEqual(
protobuf_entry.to_api_repr()["protoPayload"]["@type"], type_url
)

def test_log_text(self):
TEXT_PAYLOAD = "System test: test_log_text"
@@ -288,7 +379,7 @@ def test_log_handler_async(self):

cloud_logger = logging.getLogger(handler.name)
cloud_logger.addHandler(handler)
cloud_logger.warn(LOG_MESSAGE)
cloud_logger.warning(LOG_MESSAGE)
handler.flush()
entries = _list_entries(logger)
expected_payload = {"message": LOG_MESSAGE, "python_logger": handler.name}
@@ -310,7 +401,7 @@ def test_log_handler_sync(self):
LOGGER_NAME = "mylogger"
cloud_logger = logging.getLogger(LOGGER_NAME)
cloud_logger.addHandler(handler)
cloud_logger.warn(LOG_MESSAGE)
cloud_logger.warning(LOG_MESSAGE)

entries = _list_entries(logger)
expected_payload = {"message": LOG_MESSAGE, "python_logger": LOGGER_NAME}
@@ -342,7 +433,7 @@ def test_handlers_w_extras(self):
"resource": Resource(type="cloudiot_device", labels={}),
"labels": {"test-label": "manual"},
}
cloud_logger.warn(LOG_MESSAGE, extra=extra)
cloud_logger.warning(LOG_MESSAGE, extra=extra)

entries = _list_entries(logger)
self.assertEqual(len(entries), 1)
@@ -363,7 +454,7 @@ def test_log_root_handler(self):
self.to_delete.append(logger)

google.cloud.logging.handlers.handlers.setup_logging(handler)
logging.warn(LOG_MESSAGE)
logging.warning(LOG_MESSAGE)

entries = _list_entries(logger)
expected_payload = {"message": LOG_MESSAGE, "python_logger": "root"}
@@ -503,6 +503,20 @@ def test_to_api_repr_defaults(self):
}
self.assertEqual(entry.to_api_repr(), expected)

def test_to_api_repr_struct(self):
from google.protobuf.struct_pb2 import Struct, Value
from google.cloud.logging_v2.logger import _GLOBAL_RESOURCE

LOG_NAME = "struct.log"
message = Struct(fields={"foo": Value(bool_value=True)})
entry = self._make_one(log_name=LOG_NAME, payload=message)
expected = {
"logName": LOG_NAME,
"jsonPayload": message,
"resource": _GLOBAL_RESOURCE._to_dict(),
}
self.assertEqual(entry.to_api_repr(), expected)

def test_to_api_repr_explicit(self):
import datetime
from google.cloud.logging import Resource

0 comments on commit 5d91be9

Please sign in to comment.