Skip to content
Permalink
Browse files
feat: detect monitored resources on all GCP environments (#200)
  • Loading branch information
daniel-sanche committed Mar 5, 2021
1 parent 22f836a commit 4eda6813d19df8a119f1dcd47ff79389310d4a6f
@@ -29,7 +29,6 @@
from google.cloud.client import ClientWithProject
from google.cloud.environment_vars import DISABLE_GRPC
from google.cloud.logging_v2._helpers import _add_defaults_to_filter
from google.cloud.logging_v2._helpers import retrieve_metadata_server
from google.cloud.logging_v2._http import Connection
from google.cloud.logging_v2._http import _LoggingAPI as JSONLoggingAPI
from google.cloud.logging_v2._http import _MetricsAPI as JSONMetricsAPI
@@ -39,6 +38,9 @@
from google.cloud.logging_v2.handlers import ContainerEngineHandler
from google.cloud.logging_v2.handlers import setup_logging
from google.cloud.logging_v2.handlers.handlers import EXCLUDED_LOGGER_DEFAULTS
from google.cloud.logging_v2.resource import Resource
from google.cloud.logging_v2.handlers._monitored_resources import detect_resource


from google.cloud.logging_v2.logger import Logger
from google.cloud.logging_v2.metric import Metric
@@ -48,14 +50,8 @@
_DISABLE_GRPC = os.getenv(DISABLE_GRPC, False)
_USE_GRPC = _HAVE_GRPC and not _DISABLE_GRPC

_APPENGINE_FLEXIBLE_ENV_VM = "GAE_APPENGINE_HOSTNAME"
"""Environment variable set in App Engine when vm:true is set."""

_APPENGINE_INSTANCE_ID = "GAE_INSTANCE"
"""Environment variable set in App Engine standard and flexible environment."""

_GKE_CLUSTER_NAME = "instance/attributes/cluster-name"
"""Attribute in metadata server when in GKE environment."""
_GAE_RESOURCE_TYPE = "gae_app"
_GKE_RESOURCE_TYPE = "k8s_container"


class Client(ClientWithProject):
@@ -348,17 +344,20 @@ def get_default_handler(self, **kw):
Returns:
logging.Handler: The default log handler based on the environment
"""
gke_cluster_name = retrieve_metadata_server(_GKE_CLUSTER_NAME)
monitored_resource = kw.pop("resource", detect_resource(self.project))

if (
_APPENGINE_FLEXIBLE_ENV_VM in os.environ
or _APPENGINE_INSTANCE_ID in os.environ
isinstance(monitored_resource, Resource)
and monitored_resource.type == _GAE_RESOURCE_TYPE
):
return AppEngineHandler(self, **kw)
elif gke_cluster_name is not None:
elif (
isinstance(monitored_resource, Resource)
and monitored_resource.type == _GKE_RESOURCE_TYPE
):
return ContainerEngineHandler(**kw)
else:
return CloudLoggingHandler(self, **kw)
return CloudLoggingHandler(self, resource=monitored_resource, **kw)

def setup_logging(
self, *, log_level=logging.INFO, excluded_loggers=EXCLUDED_LOGGER_DEFAULTS, **kw
@@ -0,0 +1,195 @@
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os

from google.cloud.logging_v2.resource import Resource
from google.cloud.logging_v2._helpers import retrieve_metadata_server

_GAE_SERVICE_ENV = "GAE_SERVICE"
_GAE_VERSION_ENV = "GAE_VERSION"
_GAE_INSTANCE_ENV = "GAE_INSTANCE"
_GAE_ENV_VARS = [_GAE_SERVICE_ENV, _GAE_VERSION_ENV, _GAE_INSTANCE_ENV]
"""Environment variables set in App Engine environment."""

_CLOUD_RUN_SERVICE_ID = "K_SERVICE"
_CLOUD_RUN_REVISION_ID = "K_REVISION"
_CLOUD_RUN_CONFIGURATION_ID = "K_CONFIGURATION"
_CLOUD_RUN_ENV_VARS = [
_CLOUD_RUN_SERVICE_ID,
_CLOUD_RUN_REVISION_ID,
_CLOUD_RUN_CONFIGURATION_ID,
]
"""Environment variables set in Cloud Run environment."""

_FUNCTION_TARGET = "FUNCTION_TARGET"
_FUNCTION_SIGNATURE = "FUNCTION_SIGNATURE_TYPE"
_FUNCTION_NAME = "FUNCTION_NAME"
_FUNCTION_REGION = "FUNCTION_REGION"
_FUNCTION_ENTRY = "ENTRY_POINT"
_FUNCTION_ENV_VARS = [_FUNCTION_TARGET, _FUNCTION_SIGNATURE, _CLOUD_RUN_SERVICE_ID]
_LEGACY_FUNCTION_ENV_VARS = [_FUNCTION_NAME, _FUNCTION_REGION, _FUNCTION_ENTRY]
"""Environment variables set in Cloud Functions environments."""


_REGION_ID = "instance/region"
_ZONE_ID = "instance/zone"
_GCE_INSTANCE_ID = "instance/id"
"""Attribute in metadata server for compute region and instance."""

_GKE_CLUSTER_NAME = "instance/attributes/cluster-name"
"""Attribute in metadata server when in GKE environment."""


def _create_functions_resource(project):
"""Create a standardized Cloud Functions resource.
Args:
project (str): The project ID to pass on to the resource
Returns:
google.cloud.logging.Resource
"""
region = retrieve_metadata_server(_REGION_ID)
if _FUNCTION_NAME in os.environ:
function_name = os.environ.get(_FUNCTION_NAME)
elif _CLOUD_RUN_SERVICE_ID in os.environ:
function_name = os.environ.get(_CLOUD_RUN_SERVICE_ID)
else:
function_name = ""
resource = Resource(
type="cloud_function",
labels={
"project_id": project,
"function_name": function_name,
"region": region if region else "",
},
)
return resource


def _create_kubernetes_resource(project):
"""Create a standardized Kubernetes resource.
Args:
project (str): The project ID to pass on to the resource
Returns:
google.cloud.logging.Resource
"""
zone = retrieve_metadata_server(_ZONE_ID)
cluster_name = retrieve_metadata_server(_GKE_CLUSTER_NAME)

resource = Resource(
type="k8s_container",
labels={
"project_id": project,
"location": zone if zone else "",
"cluster_name": cluster_name if cluster_name else "",
},
)
return resource


def _create_compute_resource(project):
"""Create a standardized Compute Engine resource.
Args:
project (str): The project ID to pass on to the resource
Returns:
google.cloud.logging.Resource
"""
instance = retrieve_metadata_server(_GCE_INSTANCE_ID)
zone = retrieve_metadata_server(_ZONE_ID)
resource = Resource(
type="gce_instance",
labels={
"project_id": project,
"instance_id": instance if instance else "",
"zone": zone if zone else "",
},
)
return resource


def _create_cloud_run_resource(project):
"""Create a standardized Cloud Run resource.
Args:
project (str): The project ID to pass on to the resource
Returns:
google.cloud.logging.Resource
"""
region = retrieve_metadata_server(_REGION_ID)
resource = Resource(
type="cloud_run_revision",
labels={
"project_id": project,
"service_name": os.environ.get(_CLOUD_RUN_SERVICE_ID, ""),
"revision_name": os.environ.get(_CLOUD_RUN_REVISION_ID, ""),
"location": region if region else "",
"configuration_name": os.environ.get(_CLOUD_RUN_CONFIGURATION_ID, ""),
},
)
return resource


def _create_app_engine_resource(project):
"""Create a standardized App Engine resource.
Args:
project (str): The project ID to pass on to the resource
Returns:
google.cloud.logging.Resource
"""
zone = retrieve_metadata_server(_ZONE_ID)
resource = Resource(
type="gae_app",
labels={
"project_id": project,
"module_id": os.environ.get(_GAE_SERVICE_ENV, ""),
"version_id": os.environ.get(_GAE_VERSION_ENV, ""),
"zone": zone if zone else "",
},
)
return resource


def _create_global_resource(project):
return Resource(type="global", labels={"project_id": project})


def detect_resource(project):
"""Return the default monitored resource based on the local environment.
Args:
project (str): The project ID to pass on to the resource
Returns:
google.cloud.logging.Resource: The default resource based on the environment
"""
gke_cluster_name = retrieve_metadata_server(_GKE_CLUSTER_NAME)
gce_instance_name = retrieve_metadata_server(_GCE_INSTANCE_ID)

if all([env in os.environ for env in _GAE_ENV_VARS]):
# App Engine Flex or Standard
return _create_app_engine_resource(project)
elif gke_cluster_name is not None:
# Kubernetes Engine
return _create_kubernetes_resource(project)
elif all([env in os.environ for env in _LEGACY_FUNCTION_ENV_VARS]) or all(
[env in os.environ for env in _FUNCTION_ENV_VARS]
):
# Cloud Functions
return _create_functions_resource(project)
elif all([env in os.environ for env in _CLOUD_RUN_ENV_VARS]):
# Cloud Run
return _create_cloud_run_resource(project)
elif gce_instance_name is not None:
# Compute Engine
return _create_compute_resource(project)
else:
# use generic global resource
return _create_global_resource(project)
@@ -22,8 +22,10 @@
import os

from google.cloud.logging_v2.handlers._helpers import get_request_data
from google.cloud.logging_v2.handlers._monitored_resources import (
_create_app_engine_resource,
)
from google.cloud.logging_v2.handlers.transports import BackgroundThreadTransport
from google.cloud.logging_v2.resource import Resource

_DEFAULT_GAE_LOGGER_NAME = "app"

@@ -75,15 +77,7 @@ def get_gae_resource(self):
Returns:
google.cloud.logging_v2.resource.Resource: Monitored resource for GAE.
"""
gae_resource = Resource(
type="gae_app",
labels={
"project_id": self.project_id,
"module_id": self.module_id,
"version_id": self.version_id,
},
)
return gae_resource
return _create_app_engine_resource(self.project_id)

def get_gae_labels(self):
"""Return the labels for GAE app.
@@ -17,7 +17,7 @@
import logging

from google.cloud.logging_v2.handlers.transports import BackgroundThreadTransport
from google.cloud.logging_v2.logger import _GLOBAL_RESOURCE
from google.cloud.logging_v2.handlers._monitored_resources import detect_resource

DEFAULT_LOGGER_NAME = "python"

@@ -59,7 +59,7 @@ def __init__(
*,
name=DEFAULT_LOGGER_NAME,
transport=BackgroundThreadTransport,
resource=_GLOBAL_RESOURCE,
resource=None,
labels=None,
stream=None,
):
@@ -78,12 +78,15 @@ def __init__(
:class:`.BackgroundThreadTransport`. The other
option is :class:`.SyncTransport`.
resource (~logging_v2.resource.Resource):
Resource for this Handler. Defaults to ``GLOBAL_RESOURCE``.
Resource for this Handler. If not given, will be inferred from the environment.
labels (Optional[dict]): Monitored resource of the entry, defaults
to the global resource type.
stream (Optional[IO]): Stream to be used by the handler.
"""
super(CloudLoggingHandler, self).__init__(stream)
if not resource:
# infer the correct monitored resource from the local environment
resource = detect_resource(client.project)
self.name = name
self.client = client
self.transport = transport(client, name)

0 comments on commit 4eda681

Please sign in to comment.