sendfile returning -EAGAIN on top of ZFS/OVL

[aither64]

chef-client on linux 5.6 ends up in an loop on

poll([{fd=12, events=POLLOUT}], 1, -1)  = 1 ([{fd=12, revents=POLLOUT}])
sendfile(12, 11, NULL, 7)               = -1 EAGAIN (Resource temporarily unavailable)
poll([{fd=12, events=POLLOUT}], 1, -1)  = 1 ([{fd=12, revents=POLLOUT}])
sendfile(12, 11, NULL, 7)               = -1 EAGAIN (Resource temporarily unavailable)

where

ls -l /proc/995449/fd
total 0
lrwx------ 1 509739008 509739008 64 Jun 16 22:22 0 -> /dev/null
l-wx------ 1 509739008 509739008 64 Jun 16 22:22 1 -> 'pipe:[40105052]'
lrwx------ 1 509739008 509739008 64 Jun 16 22:22 10 -> /var/opt/gitlab/gitlab-rails/.chef-VERSION20200616-26-19ldlsx
lr-x------ 1 509739008 509739008 64 Jun 16 22:22 11 -> /opt/gitlab/embedded/service/gitlab-rails/VERSION
l-wx------ 1 509739008 509739008 64 Jun 16 22:22 12 -> /var/opt/gitlab/gitlab-rails/.chef-VERSION20200616-26-19ldlsx
l-wx------ 1 509739008 509739008 64 Jun 16 22:22 2 -> 'pipe:[40105053]'
lrwx------ 1 509739008 509739008 64 Jun 16 22:22 3 -> 'anon_inode:[eventfd]'
lrwx------ 1 509739008 509739008 64 Jun 16 22:22 4 -> 'anon_inode:[eventfd]'
l-wx------ 1 509739008 509739008 64 Jun 16 22:22 5 -> 'pipe:[40105052]'
lr-x------ 1 509739008 509739008 64 Jun 16 22:22 6 -> 'pipe:[40100375]'
l-wx------ 1 509739008 509739008 64 Jun 16 22:22 7 -> 'pipe:[40100375]'
l-wx------ 1 509739008 509739008 64 Jun 16 22:22 8 -> /var/log/gitlab/reconfigure/1592338634.log
lrwx------ 1 509739008 509739008 64 Jun 16 22:22 9 -> /opt/gitlab/embedded/cookbooks/cache/chef-client-running.pid

so far unverified reproducer:

export GITLAB_HOME=/srv/gitlab
docker run --detach \
  --hostname gitlab.example.com \
  --publish 443:443 --publish 80:80 --publish 22:22 \
  --name gitlab \
  --restart always \
  --volume $GITLAB_HOME/config:/etc/gitlab \
  --volume $GITLAB_HOME/logs:/var/log/gitlab \
  --volume $GITLAB_HOME/data:/var/opt/gitlab \
  gitlab/gitlab-ce:latest

[snajpa]

lr-x------. 1 100000 100000 64 Jun 17 15:21 11 -> /opt/gitlab/embedded/service/gitlab-rails/VERSION
l-wx------. 1 100000 100000 64 Jun 17 15:21 12 -> /var/opt/gitlab/gitlab-rails/.chef-VERSION20200617-25-md7qii

ssize_t sendfile(int out_fd, int in_fd, off_t *offset, size_t count);

sendfile(12, 11, NULL, 7)               = -1 EAGAIN (Resource temporarily unavailable)

overlay / overlay rw,relatime,lowerdir=/var/lib/docker/overlay2/l/TFVFQDTTXXXHJMX5R6LDGYYD2Q:/var/lib/docker/overlay2/l/HDCKICGKRSCRBKSD7JMPALKLY3:/var/lib/docker/overlay2/l/FXD7VLCW3TEBSZSJBPNZN6PWEI:/var/lib/docker/overlay2/l/EEMEO2BX3FR5NIH4TWFVGZARPB:/var/lib/docker/overlay2/l/5UIEL3RS2HEXFI2RMW6LIZGV7N:/var/lib/docker/overlay2/l/IGMNCMJF4XY3IPR4NWCM3XCDM3:/var/lib/docker/overlay2/l/BLPQF3G2A3OSZIJQDRPJL66GSC:/var/lib/docker/overlay2/l/Z7GZCDSMCP655PTEIT366B64FF:/var/lib/docker/overlay2/l/U6CKIJDIKIJLX4ONBG5U3L3TXH:/var/lib/docker/overlay2/l/RJUUQYFZBVGDTBVLRBUCI5CMID:/var/lib/docker/overlay2/l/GCIHGVCTGVXMZRGLCABK63MIGH,upperdir=/var/lib/docker/overlay2/7a4fcf2bdb66f75c5234c4d58b5c2ea0c02ea788aca8edf7e43d95a2bdae09a5/diff,workdir=/var/lib/docker/overlay2/7a4fcf2bdb66f75c5234c4d58b5c2ea0c02ea788aca8edf7e43d95a2bdae09a5/work 0 0

[snajpa]

openzfs/zfs#1156

[snajpa]

-EAGAIN coming from here: https://elixir.bootlin.com/linux/v5.7.4/source/fs/splice.c#L326

[snajpa]

temporary workaround: vpsfreecz/linux@c2b6cff

[snajpa]

more proper fix needs UIO to recognize and work with pipes... openzfs/zfs#10018 should be merged first