[VOL-634] Remove fluentd logging reference from voltha services

This update removes the "hardcoded" logging framework (fluentd)
from the voltha, ofagent and netconf services.  The logging is
now determined at deployment time.  Testing was done in kubernetes
environment and minimally in docker swarm mode environment (docker
swarm mode is not required for voltha 1.3).  A separate Jira will
be raised to adjust the docker swarm deployment files, i.e. direct
the logs to fluentd/fluentd-agg listen ports instead of writing locally.

Change-Id: Ic780fc9633f0f833f961c97dc59d89eed2112a53

Jenkinsfile

@@ -13,9 +13,6 @@ node('build') {
             stage 'Remove the pre-created venv-linux'
             sh 'vagrant ssh -c "rm -rf /cord/incubator/voltha/venv-linux"'
 
-            stage 'Use the jenkins voltha.yml file'
-            sh 'vagrant ssh -c "cp /cord/incubator/voltha/voltha/voltha.jenkins.yml /cord/incubator/voltha/voltha/voltha.yml"'
-
             stage 'Build voltha'
             sh 'vagrant ssh -c "cd /cord/incubator/voltha && source env.sh && make fetch && make build" voltha'
 

common/structlog_setup.py

@@ -29,12 +29,12 @@
     from dummy_thread import get_ident as _get_ident
 
 
-class FluentRenderer(object):
+class StructuredLogRenderer(object):
     def __call__(self, logger, name, event_dict):
         # in order to keep structured log data in event_dict to be forwarded as
-        # is to the fluent logger, we need to pass it into the logger framework
-        # as the first positional argument.
-        args = (event_dict, )
+        # is, we need to pass it into the logger framework as the first
+        # positional argument.
+        args = (event_dict,)
         kwargs = {}
         return args, kwargs
 
@@ -58,14 +58,12 @@ def __repr__(self, _repr_running={}):
             del _repr_running[call_key]
 
 
-def setup_logging(log_config, instance_id, verbosity_adjust=0, fluentd=None):
+def setup_logging(log_config, instance_id, verbosity_adjust=0):
     """
     Set up logging such that:
     - The primary logging entry method is structlog
       (see http://structlog.readthedocs.io/en/stable/index.html)
     - By default, the logging backend is Python standard lib logger
-    - Alternatively, fluentd can be configured with to be the backend,
-      providing direct bridge to a fluent logging agent.
     """
 
     def add_exc_info_flag_for_exception(_, name, event_dict):
@@ -77,20 +75,6 @@ def add_instance_id(_, __, event_dict):
         event_dict['instance_id'] = instance_id
         return event_dict
 
-    # if fluentd is specified, we need to override the config data with
-    # its host and port info
-    if fluentd is not None:
-        fluentd_host = fluentd.split(':')[0].strip()
-        fluentd_port = int(fluentd.split(':')[1].strip())
-
-        handlers = log_config.get('handlers', None)
-        if isinstance(handlers, dict):
-            for _, defs in handlers.iteritems():
-                if isinstance(defs, dict):
-                    if defs.get('class', '').endswith('FluentHandler'):
-                        defs['host'] = fluentd_host
-                        defs['port'] = fluentd_port
-
     # Configure standard logging
     logging.config.dictConfig(log_config)
     logging.root.level -= 10 * verbosity_adjust
@@ -100,7 +84,7 @@ def add_instance_id(_, __, event_dict):
         structlog.processors.StackInfoRenderer(),
         structlog.processors.format_exc_info,
         add_instance_id,
-        FluentRenderer(),
+        StructuredLogRenderer(),
     ]
     structlog.configure(logger_factory=structlog.stdlib.LoggerFactory(),
                         context_class=PlainRenderedOrderedDict,
@@ -138,7 +122,7 @@ def add_vcore_id(_, __, event_dict):
         structlog.processors.format_exc_info,
         add_instance_id,
         add_vcore_id,
-        FluentRenderer(),
+        StructuredLogRenderer(),
     ]
     structlog.configure(processors=processors)
 

compose/docker-compose-all.yml.j2

@@ -130,7 +130,6 @@ services:
       - voltha/voltha/main.py
       - -v
       - --consul=vconsul:8500
-      - --fluentd=fluentd:24224
       - --kafka=kafka
       - --rest-port=8880
       - --grpc-port=50556
@@ -162,7 +161,6 @@ services:
       - /ofagent/ofagent/main.py
       - -v
       - --consul=vconsul:8500
-      - --fluentd=fluentd:24224
       - --controller=onos:6653
       - --grpc-endpoint=vcore:50556
       - --instance-id-is-container-name
@@ -242,7 +240,6 @@ services:
       - /netconf/netconf/main.py
       - -v
       - --consul=vconsul:8500
-      - --fluentd=fluentd:24224
       - --grpc-endpoint=voltha:50555
       - --instance-id-is-container-name
     networks:

compose/docker-compose-docutests.yml

@@ -76,11 +76,15 @@ services:
   #
   voltha:
     image: "${REGISTRY}${REPOSITORY}voltha-voltha:latest"
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
     command: [
       "/voltha/voltha/main.py",
       "-v",
       "--consul=${DOCKER_HOST_IP}:8500",
-      "--fluentd=fluentd:24224",
       "--rest-port=8880",
       "--grpc-port=50555",
       "--kafka=@kafka",
@@ -97,7 +101,6 @@ services:
     - vconsul
     links:
     - vconsul
-    - fluentd
     environment:
       SERVICE_8880_NAME: "voltha-health"
       SERVICE_8880_CHECK_HTTP: "/health"
@@ -116,11 +119,15 @@ services:
   #
   ofagent:
     image: "${REGISTRY}${REPOSITORY}voltha-ofagent:latest"
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
     command: [
       "/ofagent/ofagent/main.py",
       "-v",
       "--consul=${DOCKER_HOST_IP}:8500",
-      "--fluentd=fluentd:24224",
       "--controller=${DOCKER_HOST_IP}:6653",
       "--grpc-endpoint=@voltha-grpc",
       "--instance-id-is-container-name",
@@ -131,7 +138,6 @@ services:
     - voltha
     links:
     - vconsul
-    - fluentd
     volumes:
     - "/var/run/docker.sock:/tmp/docker.sock"
     restart: unless-stopped
@@ -141,12 +147,16 @@ services:
   #
   netconf:
     image: "${REGISTRY}${REPOSITORY}voltha-netconf:latest"
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
     privileged: true
     command: [
       "/netconf/netconf/main.py",
       "-v",
       "--consul=${DOCKER_HOST_IP}:8500",
-      "--fluentd=fluentd:24224",
       "--grpc-endpoint=@voltha-grpc",
       "--instance-id-is-container-name",
       "-v"
@@ -158,7 +168,6 @@ services:
     - voltha
     links:
     - vconsul
-    - fluentd
     environment:
       SERVICE_1830_NAME: "netconf-server"
     volumes:

compose/docker-compose-fixed-port.yml

@@ -116,7 +116,6 @@ services:
       "/voltha/voltha/main.py",
       "-v",
       "--consul=${DOCKER_HOST_IP}:8500",
-      "--fluentd=fluentd:24224",
       "--rest-port=8880",
       "--grpc-port=50555",
       "--kafka=@kafka",
@@ -132,7 +131,6 @@ services:
     - vconsul
     links:
     - vconsul
-    - fluentd
     environment:
       SERVICE_8880_NAME: "voltha-health"
       SERVICE_8880_CHECK_HTTP: "/health"
@@ -159,7 +157,6 @@ services:
       "/ofagent/ofagent/main.py",
       "-v",
       "--consul=${DOCKER_HOST_IP}:8500",
-      "--fluentd=fluentd:24224",
       "--controller=${DOCKER_HOST_IP}:6653",
       "--grpc-endpoint=@voltha-grpc",
       "--instance-id-is-container-name",
@@ -170,7 +167,6 @@ services:
     - voltha
     links:
     - vconsul
-    - fluentd
     volumes:
     - "/var/run/docker.sock:/tmp/docker.sock"
     restart: unless-stopped
@@ -185,7 +181,6 @@ services:
       "/netconf/netconf/main.py",
       "-v",
       "--consul=${DOCKER_HOST_IP}:8500",
-      "--fluentd=fluentd:24224",
       "--grpc-endpoint=@voltha-grpc",
       "--instance-id-is-container-name",
       "-v"
@@ -197,7 +192,6 @@ services:
     - voltha
     links:
     - vconsul
-    - fluentd
     environment:
       SERVICE_1830_NAME: "netconf-server"
     volumes:

compose/docker-compose-netconf-swarm.yml

@@ -27,7 +27,6 @@ services:
       - /netconf/netconf/main.py
       - -v
       - --consul=vconsul:8500
-      - --fluentd=fluentd:24224
       - --grpc-endpoint=voltha:50555
       - --instance-id-is-container-name
     networks:

compose/docker-compose-ofagent-swarm.yml

@@ -21,7 +21,6 @@ services:
             - /ofagent/ofagent/main.py
             - -v
             - --consul=vconsul:8500
-            - --fluentd=fluentd:24224
             - --controller=onos:6653
             - --grpc-endpoint=vcore:50556
             - --instance-id-is-container-name

compose/docker-compose-ofagent-test.yml

@@ -55,7 +55,6 @@ services:
       "/voltha/voltha/main.py",
       "-v",
       "--consul=${DOCKER_HOST_IP}:8500",
-      "--fluentd=fluentd:24224",
       "--rest-port=8880",
       "--grpc-port=50556",
       "--kafka=@kafka",
@@ -73,7 +72,6 @@ services:
     - vconsul
     links:
     - vconsul
-    - fluentd
     environment:
       SERVICE_8880_NAME: "voltha-health"
       SERVICE_8880_CHECK_HTTP: "/health"
@@ -182,13 +180,12 @@ services:
   #
   ofagent:
     image: "${REGISTRY}${REPOSITORY}voltha-ofagent:${TAG:-latest}"
-    command: /ofagent/ofagent/main.py -v --consul=${DOCKER_HOST_IP}:8500 --fluentd=fluentd:24224 --controller ${DOCKER_HOST_IP}:6633 ${DOCKER_HOST_IP}:6644 ${DOCKER_HOST_IP}:6655 --grpc-endpoint=@voltha-grpc --instance-id-is-container-name --enable-tls --key-file=/ofagent/pki/voltha.key --cert-file=/ofagent/pki/voltha.crt
+    command: /ofagent/ofagent/main.py -v --consul=${DOCKER_HOST_IP}:8500 --controller ${DOCKER_HOST_IP}:6633 ${DOCKER_HOST_IP}:6644 ${DOCKER_HOST_IP}:6655 --grpc-endpoint=@voltha-grpc --instance-id-is-container-name --enable-tls --key-file=/ofagent/pki/voltha.key --cert-file=/ofagent/pki/voltha.crt
     depends_on:
     - vconsul
     - voltha
     links:
     - vconsul
-    - fluentd
     volumes:
     - "/var/run/docker.sock:/tmp/docker.sock"
     restart: unless-stopped

compose/docker-compose-system-test-dispatcher.yml

@@ -114,11 +114,15 @@ services:
   #
   voltha:
     image: "${REGISTRY}${REPOSITORY}voltha-voltha${TAG}"
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
     command: [
       "/voltha/voltha/main.py",
       "-v",
       "--consul=${DOCKER_HOST_IP}:8500",
-      "--fluentd=fluentd:24224",
       "--rest-port=8880",
       "--grpc-port=50556",
       "--kafka=@kafka",
@@ -135,7 +139,6 @@ services:
     - vconsul
     links:
     - vconsul
-    - fluentd
     environment:
       SERVICE_8880_NAME: "voltha-health"
       SERVICE_8880_CHECK_HTTP: "/health"
@@ -202,11 +205,15 @@ services:
   #
   ofagent:
     image: "${REGISTRY}${REPOSITORY}voltha-ofagent${TAG}"
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
     command: [
       "/ofagent/ofagent/main.py",
       "-v",
       "--consul=${DOCKER_HOST_IP}:8500",
-      "--fluentd=fluentd:24224",
       "--controller=${DOCKER_HOST_IP}:6653",
       "--grpc-endpoint=@voltha-grpc",
       "--instance-id-is-container-name",
@@ -217,7 +224,6 @@ services:
     - voltha
     links:
     - vconsul
-    - fluentd
     volumes:
     - "/var/run/docker.sock:/tmp/docker.sock"
     restart: unless-stopped
@@ -227,12 +233,16 @@ services:
   #
   netconf:
     image: "${REGISTRY}${REPOSITORY}voltha-netconf${TAG}"
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
     privileged: true
     command: [
       "/netconf/netconf/main.py",
       "-v",
       "--consul=${DOCKER_HOST_IP}:8500",
-      "--fluentd=fluentd:24224",
       "--grpc-endpoint=@voltha-grpc",
       "--instance-id-is-container-name",
       "-v"
@@ -244,7 +254,6 @@ services:
     - voltha
     links:
     - vconsul
-    - fluentd
     environment:
       SERVICE_1830_NAME: "netconf-server"
     volumes: