Metricbeat docker.container.size.root_fs and docker.container.size.rw are always 0

[vshypyguzov]

• Version: 7.5.0
• Operating System: Docker image running on Ubuntu
• Steps to Reproduce:
  Gather docker.container module metrics

Sample document produced by metricbeat:
{
"_index": "metricbeat-7.5.0-2019.12.03-000001",
"_type": "_doc",
"_id": "6y4d224BbVD047gaVWms",
"_version": 1,
"_score": null,
"_source": {
"@timestamp": "2019-12-06T12:09:22.701Z",
"service": {
"address": "/var/run/docker.sock",
"type": "docker"
},
"ecs": {
"version": "1.1.0"
},
"host": {
"name": "knqprvc042"
},
"agent": {
"type": "metricbeat",
"ephemeral_id": "776fe9b8-f85e-47d4-acfc-d7a941793f68",
"id": "83b0eb2c-15cb-4029-8f9e-4720f06e32e9",
"version": "7.5.0"
},
"container": {
"name": "portainer_agent.jnj8a8pfoy2duncbscipner85.ws7f6ttst404jhekpqpst5iy8",
"runtime": "docker",
"id": "09bb5fe3297ad1c5290b8b40ef6212e5d77040b9fa233d3d1df788e480e0814e",
"image": {
"name": "portainer/agent:latest@sha256:88989d2dd0783524ab5af4bb8f4f4be2ac82c10188e17c6c6bda862bb7a5676d"
}
},
"docker": {
"container": {
"created": "2019-12-02T15:57:57.000Z",
"command": "./agent",
"ip_addresses": [
"10.0.2.10"
],
"size": {
"rw": 0,
"root_fs": 0
},
"status": "Up 3 days",
"labels": {
"com_docker_swarm_task": "",
"com_docker_swarm_task_id": "ws7f6ttst404jhekpqpst5iy8",
"com_docker_swarm_task_name": "portainer_agent.jnj8a8pfoy2duncbscipner85.ws7f6ttst404jhekpqpst5iy8",
"com_docker_stack_namespace": "portainer",
"com_docker_swarm_node_id": "jnj8a8pfoy2duncbscipner85",
"com_docker_swarm_service_id": "je7vhimsgjioj2ubm3a2recg9",
"com_docker_swarm_service_name": "portainer_agent"
}
}
},
"event": {
"dataset": "docker.container",
"module": "docker",
"duration": 3062657
},
"metricset": {
"name": "container",
"period": 10000
}
},
"fields": {
"docker.container.created": [
"2019-12-02T15:57:57.000Z"
],
"@timestamp": [
"2019-12-06T12:09:22.701Z"
]
},
"sort": [
1575634162701
]
}

Docker API response for :
curl --unix-socket /var/run/docker.sock http://localhost/containers/09bb5fe3297ad1c5290b8b40ef6212e5d77040b9fa233d3d1df788e480e0814e/json?size=1
{
"Id": "09bb5fe3297ad1c5290b8b40ef6212e5d77040b9fa233d3d1df788e480e0814e",
"Name": "/portainer_agent.jnj8a8pfoy2duncbscipner85.ws7f6ttst404jhekpqpst5iy8",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "docker-default",
"ExecIDs": null,
"HostConfig": {
"Binds": null,
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {
"max-file": "3",
"max-size": "1g"
}
},
"NetworkMode": "default",
"PortBindings": {},
"RestartPolicy": {
"Name": "",
"MaximumRetryCount": 0
},
"Data": {
"LowerDir": "/var/lib/docker/overlay2/c556175d493cb339919641feb92a2bb1c79d1b4c5a15068449e06cc4a5c516a2-init/diff:/var/lib/docker/overlay2/d040c49af92c49f181af52ae8f39ba5beccc788f8693f6fa0d098a2663840f57/diff:/var/lib/docker/overlay2/05d7e8e5f14ac27316fb4e154edb654134704d7522d48a550fe04d53fe328d65/diff:/var/lib/docker/overlay2/7b886828185561b0107c3f52672d6a1418ec38d2dc0159eefe906c79379b1dd6/diff:/var/lib/docker/overlay2/0a27fba2d111ef6e92bcda707ae0143f0b78e8ac338b0c7d44fd580d52be0105/diff",
"MergedDir": "/var/lib/docker/overlay2/c556175d493cb339919641feb92a2bb1c79d1b4c5a15068449e06cc4a5c516a2/merged",
"UpperDir": "/var/lib/docker/overlay2/c556175d493cb339919641feb92a2bb1c79d1b4c5a15068449e06cc4a5c516a2/diff",
"WorkDir": "/var/lib/docker/overlay2/c556175d493cb339919641feb92a2bb1c79d1b4c5a15068449e06cc4a5c516a2/work"
},
"Name": "overlay2"
},
"SizeRw": 1198405,
"SizeRootFs": 15116747,
"Mounts": [
{
"Type": "bind",
"Source": "/var/run/docker.sock",
"Destination": "/var/run/docker.sock",
"Mode": "",
"RW": true,
"Propagation": "rprivate"
},
{
"Type": "bind",
"Source": "/var/lib/docker/volumes",
"Destination": "/var/lib/docker/volumes",
"Mode": "",
"RW": true,
"Propagation": "rslave"
}
],
"Config": {
"Hostname": "09bb5fe3297a",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
],
"Cmd": null,
"Image": "portainer/agent:latest@sha256:88989d2dd0783524ab5af4bb8f4f4be2ac82c10188e17c6c6bda862bb7a5676d",
"Volumes": null,
"WorkingDir": "/app",
"Entrypoint": [
"./agent"
],
"OnBuild": null,
"Labels": {
"com.docker.stack.namespace": "portainer",
"com.docker.swarm.node.id": "jnj8a8pfoy2duncbscipner85",
"com.docker.swarm.service.id": "je7vhimsgjioj2ubm3a2recg9",
"com.docker.swarm.service.name": "portainer_agent",
"com.docker.swarm.task": "",
"com.docker.swarm.task.id": "ws7f6ttst404jhekpqpst5iy8",
"com.docker.swarm.task.name": "portainer_agent.jnj8a8pfoy2duncbscipner85.ws7f6ttst404jhekpqpst5iy8"
}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "429492c7baa92b2b31d7e1e4610bc44bdbbf4b4437b825d18717940a90e01e56",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {},
"SandboxKey": "/var/run/docker/netns/429492c7baa9",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "",
"Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"MacAddress": "",
"Networks": {
"portainer_agent_network": {
"IPAMConfig": {
"IPv4Address": "10.0.2.10"
},
"Links": null,
"Aliases": [
"09bb5fe3297a"
],
"NetworkID": "06h0m4i8fwchvh2sug1hlxro2",
"EndpointID": "b6bcece614bc123ab1cba9801c32eafb7f611eaf1d5f0e492b616118b95b740d",
"Gateway": "",
"IPAddress": "10.0.2.10",
"IPPrefixLen": 24,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"MacAddress": "02:42:0a:00:02:0a",
"DriverOpts": null
}
}
}
}

[vshypyguzov]

Metricbeat config:

metricbeat.config:
  modules:
    path: ${path.config}/modules.d/*.yml
    reload.enabled: false

metricbeat.autodiscover:
  providers:
    - type: docker
      hints.enabled: true

metricbeat.modules:
- module: docker
  metricsets:
    - container
    - cpu
    - diskio
    - healthcheck
    - info
    - memory
    - network
  hosts: 
  - unix:///var/run/docker.sock
  period: 10s
  enabled: true
- module: system
  metricsets:
  - cpu
  - load
  - memory
  - network
  - uptime
  - filesystem
  - diskio
  enabled: true
  period: 10s
  cpu.metrics:  ["percentages","normalized_percentages"]
  diskio.include_devices: ["sda2", "sdb1"]
processors:
  - add_docker_metadata:
      host: "unix:///var/run/docker.sock"  
  - add_cloud_metadata: ~
  - drop_event:
      when:
        and:
          - equals:
              metricset.name: filesystem
          - not:
              regexp:
                system.filesystem.mount_point: '^(/hostfs|/hostfs/opt/data)$'

[kaiyan-sheng]

I haven't tried to reproduce this yet. This needs more investigation.

[odacremolbap]

@vshypyguzov can you add the docker version you are using?
I'm not able to reproduce because I'm not getting any value at all for SizeRw not SizeRootFs from the docker daemon socket.

[vshypyguzov]

@odacremolbap Docker version is 19.03.5
You will not get this info unless you specify ?size=1 query string parameter

[odacremolbap]

You will not get this info unless you specify ?size=1 query string parameter

that was helpful, thank so much

[odacremolbap]

@vshypyguzov

looks like the size parameter was also disabled at code.
Also, it looks like some other docker metrics are informing that same size and that they are not enabling it.

Fixing

[varsraja]

Hi ,
currently using metricbeat 8.3.3 , still facing the same issue as reported. Can you explain what is the exact solution to get data for rootfs and rw.

[nyetwurk]

Please re-open, as #16600 re-broke this.

[varsraja]

I dont have an option to reopen this issue. Should I open a new ticket ?

[thekofimensah]

Yes, please. Still broken

[thekofimensah]

@varsraja CC @odacremolbap

[nyetwurk]

bump