wrone file size when sync file via filer.sync

[wang-zi-tao]

Sponsors SeaweedFS via Patreon https://www.patreon.com/seaweedfs

Describe the bug
write a file and sync via filer.sync, then the FileSize of file on remote node will be zero. Can not read the content of file on remote node via fuse!!!

The steps:
node1:

[root@node1/]# cd /mnt/weed/mount/huawei-ecs
[root@node1/mnt/weed/mount/huawei-ecs]# echo "test file content" > test
[root@node1/mnt/weed/mount/huawei-ecs]# cat test
test file content

node2:

[root@node2/]# cd /mnt/weed/mount/aliyun-hk
[root@node2/mnt/weed/mount/aliyun-hk]# ls -l 
.rw-r--r-- 0 root  4 5月  12:04  test
drwxr-xr-x - 1000  4 5月  11:24  workspace
[root@node2/mnt/weed/mount/aliyun-hk]# # file size is 0!!!
[root@node2/mnt/weed/mount/aliyun-hk]# cat test
[root@node2/mnt/weed/mount/aliyun-hk]# # Can not read the content of file on remote node via fuse!!!
[root@node2/mnt/weed/mount/aliyun-hk]# curl "http://192.168.16.2:302/test"
test file content
[root@node2/mnt/weed/mount/aliyun-hk]# curl -H "Accept: application/json" "http://192.168.16.2:302/?pretty=y"
{
  "Path": "",
  "Entries": [
    {
      "FullPath": "/test",
      "Mtime": "2022-05-04T12:04:58+08:00",
      "Crtime": "2022-05-04T12:04:58+08:00",
      "Mode": 420,
      "Uid": 0,
      "Gid": 0,
      "Mime": "",
      "Replication": "",
      "Collection": "",
      "TtlSec": 0,
      "DiskType": "",
      "UserName": "",
      "GroupNames": null,
      "SymlinkTarget": "",
      "Md5": null,
      "FileSize": 0,
      "Rdev": 0,
      "Inode": 4988074225482950118,
      "Extended": null,
      "chunks": [
        {
          "file_id": "1,33207ce732",
          "size": 18,
          "mtime": 1651637098396528000,
          "e_tag": "oML6X6uTobQVh44mMqfE/A==",
          "source_file_id": "4,3b8a2fedda",
          "fid": {
            "volume_id": 1,
            "file_key": 51,
            "cookie": 545056562
          },
          "source_fid": {
            "volume_id": 4,
            "file_key": 59,
            "cookie": 2318396890
          },
          "is_compressed": true
        }
      ],
      "HardLinkId": null,
      "HardLinkCounter": 0,
      "Content": null,
      "Remote": null,
      "Quota": 0
    },
    {
      "FullPath": "/topics",
      "Mtime": "2022-05-04T11:24:26+08:00",
      "Crtime": "2022-05-04T11:24:26+08:00",
      "Mode": 2147484141,
      "Uid": 0,
      "Gid": 0,
      "Mime": "",
      "Replication": "",
      "Collection": "",
      "TtlSec": 0,
      "DiskType": "",
      "UserName": "",
      "GroupNames": null,
      "SymlinkTarget": "",
      "Md5": null,
      "FileSize": 0,
      "Rdev": 0,
      "Inode": 0,
      "Extended": null,
      "HardLinkId": null,
      "HardLinkCounter": 0,
      "Content": null,
      "Remote": null,
      "Quota": 0
    },
    {
      "FullPath": "/workspace",
      "Mtime": "2022-05-04T11:24:01+08:00",
      "Crtime": "2022-05-04T11:24:01+08:00",
      "Mode": 2147484141,
      "Uid": 1000,
      "Gid": 996,
      "Mime": "",
      "Replication": "",
      "Collection": "",
      "TtlSec": 0,
      "DiskType": "",
      "UserName": "",
      "GroupNames": null,
      "SymlinkTarget": "",
      "Md5": null,
      "FileSize": 0,
      "Rdev": 0,
      "Inode": 0,
      "Extended": null,
      "HardLinkId": null,
      "HardLinkCounter": 0,
      "Content": null,
      "Remote": null,
      "Quota": 0
    }
  ],
  "Limit": 100,
  "LastFileName": "workspace",
  "ShouldDisplayLoadMore": false,
  "EmptyFolder": false
}
[root@node2/mnt/weed/mount/aliyun-hk]# # FileSize of '/test' is 0!!!

on node2, the size of '/test' via fuse become to zero, and can not read any content from fuse.
System Setup
weed server on node1

/nix/store/fflmps8kvqpfar8kqy87yimi5xmcshjq-seaweedfs-3.00/bin/weed server \
            -resumeState \
            -ip=192.168.16.1 \
            -dir=/mnt/weed/server/weed \
            -master.port=301 \
            -master.dir=/mnt/weed/server/master \
            -master.defaultReplication=000 \
            -master.volumePreallocate=false \
            -volume.dir.idx=/mnt/weed/server/volume \
            -filer \
            -filer.port=302 \
            -volume \
            -volume.index=leveldb \
            -volume.port=303 \
            -volume.max=256 \
            -volume.publicUrl=192.168.16.1:303 \
            -metricsPort=9101

weed mount on node1

/nix/store/fflmps8kvqpfar8kqy87yimi5xmcshjq-seaweedfs-3.00/bin/weed mount \
            -filer=192.168.16.1:302 \
            -dir=/mnt/weed/mount/huawei-ecs \
            -dirAutoCreate \
            -cacheDir=/tmp/weed-cache/huawei-ecs \
            -cacheCapacityMB=1024

weed filer.sync on node1

weed filer.sync -a=192.168.16.1:302 -b=192.168.16.2:302 -a.debug -b.debug

weed server on node2

/nix/store/fflmps8kvqpfar8kqy87yimi5xmcshjq-seaweedfs-3.00/bin/weed server \
            -resumeState \
            -ip=192.168.16.2 \
            -dir=/mnt/weed/server/weed \
            -master.port=301 \
            -master.dir=/mnt/weed/server/master \
            -master.defaultReplication=000 \
            -master.volumePreallocate=false \
            -volume.dir.idx=/mnt/weed/server/volume \
            -filer \
            -filer.port=302 \
            -volume \
            -volume.index=leveldb \
            -volume.port=303 \
            -volume.max=256 \
            -volume.publicUrl=192.168.16.2:303 \
            -metricsPort=9101

weed mount on node2

/nix/store/fflmps8kvqpfar8kqy87yimi5xmcshjq-seaweedfs-3.00/bin/weed mount \
            -filer=192.168.16.2:302 \
            -dir=/mnt/weed/mount/aliyun-hk \
            -dirAutoCreate \
            -cacheDir=/tmp/weed-cache/aliyun-hk \
            -cacheCapacityMB=1024

• OS version

  OS: NixOS 21.11 (Porcupine) x86_64
  node1: OpenStack Nova 13.2.1 on HUAWEI cloud
  node2: Alibaba Cloud ECS pc-i440fx-2.1 on ALIYUN cloud

• output of weed version
version 30GB 3.00 linux amd64
  Expected behavior
  the test can be read from fuse filesystem with content.

Additional context
output of filer.sync:

I0504 12:19:47 96228 filer_sync.go:257] received directory:"/"  event_notification:{old_entry:{name:"test"  attributes:{mtime:1651637987  file_mode:420  crtime:1651637987  inode:4988074225482983011}}  new_entry:{name:"test"  chunks:{size:18  mtime:1651637987008244864  e_tag:"oML6X6uTobQVh44mMqfE/A=="  fid:{volume_id:2  file_key:63  cookie:3717584227}  is_compressed:true}  attributes:{file_size:18  mtime:1651637987  file_mode:420  crtime:1651637987  mime:"text/plain; charset=utf-8"  replication:"000"  inode:4988074225482983011}}  delete_chunks:true  new_parent_path:"/"  signatures:2071815694  signatures:-524889372}  ts_ns:1651637987009207514

output of filer.meta.tail on node1

{"directory":"/","eventNotification":{"newEntry":{"name":"test","attributes":{"mtime":"1651637987","fileMode":420,"crtime":"1651637987","inode":"4988074225482983011"}},"deleteChunks":true,"newParentPath":"/","signatures":[2071815694,-524889372]},"tsNs":"1651637987006878537"}
{"directory":"/","eventNotification":{"oldEntry":{"name":"test","attributes":{"mtime":"1651637987","fileMode":420,"crtime":"1651637987","inode":"4988074225482983011"}},"newEntry":{"name":"test","chunks":[{"size":"18","mtime":"1651637987008244864","eTag":"oML6X6uTobQVh44mMqfE/A==","fid":{"volumeId":2,"fileKey":"63","cookie":3717584227},"isCompressed":true}],"attributes":{"fileSize":"18","mtime":"1651637987","fileMode":420,"crtime":"1651637987","mime":"text/plain; charset=utf-8","replication":"000","inode":"4988074225482983011"}},"deleteChunks":true,"newParentPath":"/","signatures":[2071815694,-524889372]},"tsNs":"1651637987009207514"}

output of filer.meta.tail on node2

{"directory":"/","eventNotification":{"newEntry":{"name":"test","attributes":{"mtime":"1651637987","fileMode":420,"crtime":"1651637987","inode":"4988074225482983011"}},"deleteChunks":true,"newParentPath":"/","isFromOtherCluster":true,"signatures":[2071815694,-524889372,1364791943]},"tsNs":"1651637987013178283"}
{"directory":"/","eventNotification":{"oldEntry":{"name":"test","attributes":{"mtime":"1651637987","fileMode":420,"crtime":"1651637987","inode":"4988074225482983011"}},"newEntry":{"name":"test","chunks":[{"size":"18","mtime":"1651637987008244864","eTag":"oML6X6uTobQVh44mMqfE/A==","fid":{"volumeId":5,"fileKey":"56","cookie":326924095},"sourceFid":{"volumeId":2,"fileKey":"63","cookie":3717584227},"isCompressed":true}],"attributes":{"mtime":"1651637987","fileMode":420,"crtime":"1651637987","inode":"4988074225482983011"}},"deleteChunks":true,"newParentPath":"/","isFromOtherCluster":true,"signatures":[2071815694,-524889372,1364791943]},"tsNs":"1651637987123813690"}

file can be sync correctly when using filer's http api to upload file.

[chrislusf]

can not reproduce. Please try to recreate it with some docker compose file, so that I can reproduce this.

[wang-zi-tao]

docker-compose.yml

version: '3.9'
services:
  node1:
    image: chrislusf/seaweedfs 
    command: "server -master -volume -filer"
  mount1:
    image: chrislusf/seaweedfs 
    privileged: true
    command: "mount -filer=node1:8888 -dir=/mnt -dirAutoCreate"
  node2:
    image: chrislusf/seaweedfs 
    ports:
      - 7888:8888
    command: "server -master -volume -filer"
  mount2:
    image: chrislusf/seaweedfs 
    privileged: true
    command: "mount -filer=node2:8888 -dir=/mnt -dirAutoCreate"
  sync:
    image: chrislusf/seaweedfs 
    command: "filer.sync -a=node1:8888 -b=node2:8888"

run.sh

set -x

docker-compose exec mount1 sh -c "echo test file >> /mnt/test"
docker-compose exec mount1 cat /mnt/test
docker-compose exec mount1 ls -l /mnt/
sleep 16
docker-compose exec mount2 cat /mnt/test
docker-compose exec mount2 ls -l /mnt/
curl -H "Accept: application/json" "http://localhost:7888/?pretty=y"
curl -H "Accept: application/json" "http://localhost:7888/?pretty=y" | grep FileSize

[chrislusf]

thanks for the docker compose file! Added a fix.