basic_cas.json pointless re_uploads

[lukts30]

Since commit f9f7908 I see the following behavior:

1. RE-Client start JobA
2. downloads all artifacts
3. JobB depends on JobA but the artifacts are not used from the CAS instead the RE-Client uploads what it just downloaded.

I have only tested with basic_cas.json unsure what the status of the other config are.
Noticed this originally with buck2 but could reproduce the problem with bazel as well.

genrule(
    name = "zero_data",
    outs = ["large_file"],
    cmd =  "fallocate -l 1G $(OUTS)"
)

genrule(
    name = "dummy_copy",
    srcs = [":zero_data"],
    outs = ["file2"],
    cmd =  "cp $(SRCS) $(OUTS)"
)

bazel clean && killall java
sudo systemd-run --uid=1000 --gid=1000 -p IPAccounting=1 -d --wait -t $(which bazel) build //:dummy_copy  --remote_instance_name=main --remote_cache=grpc://127.0.0.1:50051 --remote_executor=grpc://127.0.0.1:50051

With: 2a89ce6 (GOOD)

Finished with result: success
Main processes terminated with: code=exited/status=0
Service runtime: 14.330s
CPU time consumed: 14.493s
Memory peak: 1.7G
Memory swap peak: 0B
IP traffic received: 1.0G
IP traffic sent: 1.1M

With: f9f7908 (BAD)

Finished with result: success
Main processes terminated with: code=exited/status=0
Service runtime: 19.630s
CPU time consumed: 23.279s
Memory peak: 2.8G
Memory swap peak: 0B
IP traffic received: 2.0G
IP traffic sent: 1.0G

Notice: IP traffic sent: 1.0G

[aaronmondal]

cc @chrisstaite-menlo

[allada]

Oh this is very interesting and not something I'd ever thought would be an issue.

The problem is that in the basic_cas.json example we only use a fast store and the slow store is sent to a noop (/dev/null store). Because this change made fast_slow store always bypass the fast store when checking if an item exists it ends up always getting a 404 not found error from the noop store.

We require a fast_slow store for a store that is wired up to a worker, since production systems would have the CAS and workers live in separate instances (at least separate processes), we simplified the code a lot by recycling the fast_slow store logic to ensure things are placed into a filesystem store part of the fast store.

We have a couple options here:

1. We could change fast_slow_store to check if the slow store is a noop store and query fast_store in such case.
2. We could change the logic in the worker to allow it to accept a filesystem store as well then change the basic_cas.json accordingly.

1 is much easier to implement quickly, 2 requires a bit of thought, since we expose special APIs for workers to interact with on the FastSlowStore.

I think for now we can do 1 then create a ticket to do 2 later.

@lukts30, thanks for pointing this out. If you are blocked by this here's a config that should get you over this hurdle for now (untested, so please let me know if it doesn't work). I don't believe there's any significant happy-path performance penalties for this config:

{
  "stores": {
    "AC_MAIN_STORE": {
      "memory": {
        "eviction_policy": {
          // 100mb.
          "max_bytes": 100000000,
        }
      }
    },
    "FILESYSTEM_STORE": {
      "filesystem": {
        "content_path": "/tmp/nativelink/data-worker-test/content_path-cas",
        "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas",
        "eviction_policy": {
          // 10gb.
          "max_bytes": 10000000000,
        }
      }
    },
    "WORKER_FAST_SLOW_STORE": {
      "fast_slow": {
        // "fast" must be a "filesystem" store because the worker uses it to make
        // hardlinks on disk to a directory where the jobs are running.
        "fast": {
          "ref_store": {
            "name": "FILESYSTEM_STORE"
          }
        },
        "slow": {
          "ref_store": {
            // Also forward to the same filesystem store.
            "name": "FILESYSTEM_STORE"
          }
        }
      }
    }
  },
  "schedulers": {
    "MAIN_SCHEDULER": {
      "simple": {
        "supported_platform_properties": {
          "cpu_count": "minimum",
          "memory_kb": "minimum",
          "network_kbps": "minimum",
          "disk_read_iops": "minimum",
          "disk_read_bps": "minimum",
          "disk_write_iops": "minimum",
          "disk_write_bps": "minimum",
          "shm_size": "minimum",
          "gpu_count": "minimum",
          "gpu_model": "exact",
          "cpu_vendor": "exact",
          "cpu_arch": "exact",
          "cpu_model": "exact",
          "kernel_version": "exact",
          "OSFamily": "priority",
          "container-image": "priority",
          // Example of how to set which docker images are available and set
          // them in the platform properties.
          // "docker_image": "priority",
        }
      }
    }
  },
  "workers": [{
    "local": {
      "worker_api_endpoint": {
        "uri": "grpc://127.0.0.1:50061",
      },
      "cas_fast_slow_store": "WORKER_FAST_SLOW_STORE",
      "upload_action_result": {
        "ac_store": "AC_MAIN_STORE",
      },
      "work_directory": "/tmp/nativelink/work",
      "platform_properties": {
        "cpu_count": {
          "values": ["16"],
        },
        "memory_kb": {
          "values": ["500000"],
        },
        "network_kbps": {
          "values": ["100000"],
        },
        "cpu_arch": {
          "values": ["x86_64"],
        },
        "OSFamily": {
          "values": [""]
        },
        "container-image": {
          "values": [""]
        },
        // Example of how to set which docker images are available and set
        // them in the platform properties.
        // "docker_image": {
        //   "query_cmd": "docker images --format {{.Repository}}:{{.Tag}}",
        // }
      }
    }
  }],
  "servers": [{
    "name": "public",
    "listener": {
      "http": {
        "socket_address": "0.0.0.0:50051"
      }
    },
    "services": {
      "cas": {
        "main": {
          "cas_store": "FILESYSTEM_STORE"
        }
      },
      "ac": {
        "main": {
          "ac_store": "AC_MAIN_STORE"
        }
      },
      "execution": {
        "main": {
          "cas_store": "FILESYSTEM_STORE",
          "scheduler": "MAIN_SCHEDULER",
        }
      },
      "capabilities": {
        "main": {
          "remote_execution": {
            "scheduler": "MAIN_SCHEDULER",
          }
        }
      },
      "bytestream": {
        "cas_stores": {
          "main": "FILESYSTEM_STORE",
        }
      }
    }
  }, {
    "name": "private_workers_servers",
    "listener": {
      "http": {
        "socket_address": "0.0.0.0:50061"
      }
    },
    "services": {
      "experimental_prometheus": {
        "path": "/metrics"
      },
      // Note: This should be served on a different port, because it has
      // a different permission set than the other services.
      // In other words, this service is a backend api. The ones above
      // are a frontend api.
      "worker_api": {
        "scheduler": "MAIN_SCHEDULER",
      },
      "admin": {}
    }
  }],
  "global": {
    "max_open_files": 512
  }
}

[allada]

@blakehatch, do you want to take a swing at this?

[chrisstaite]

What if you just change the no-op store to a ref store and point both fast and slow to the same file store?

Just realised that's what you suggested in your config. That seems reasonable.

[allada]

What if you just change the no-op store to a ref store and point both fast and slow to the same file store?

The reason I don't want to do this as an official suggestion is because if anything except workers use that fast_slow store, it'd copy the data to disk twice. In the case of how it is being used here (workers are the only thing pointed to it), it's fine, since the store works more like a this-then-that store, but I don't want to do this as an official in the official documentation due to the useless overhead it might cause people to copy-pasta.

[blakehatch]

@allada yes I’ll take a crack at approach 1:
“We could change fast_slow_store to check if the slow store is a noop store and query fast_store in such case.”