From 581b09d84f7d4666735c6866af563d6dd5e26003 Mon Sep 17 00:00:00 2001 From: Marcus Date: Fri, 8 Dec 2023 22:32:07 -0800 Subject: [PATCH] renames everything but need to generate new protos. --- .bazelignore | 4 +- .dockerignore | 2 +- .github/workflows/main.yml | 12 +- BUILD.bazel | 12 +- Cargo.Bazel.lock | 66 +- Cargo.lock | 54 +- Cargo.toml | 14 +- README.md | 38 +- SECURITY.md | 10 +- WORKSPACE.bazel | 2 +- deployment-examples/docker-compose/Dockerfile | 10 +- deployment-examples/docker-compose/README.md | 36 +- .../docker-compose/docker-compose.yml | 28 +- .../docker-compose/local-storage-cas.json | 10 +- .../docker-compose/worker.json | 6 +- deployment-examples/terraform/AWS/ami.tf | 48 +- .../terraform/AWS/auto_scaling_groups.tf | 34 +- deployment-examples/terraform/AWS/iam.tf | 48 +- deployment-examples/terraform/AWS/lambdas.tf | 6 +- .../terraform/AWS/load_balancers.tf | 8 +- deployment-examples/terraform/AWS/s3.tf | 4 +- .../terraform/AWS/scripts/cas.json | 18 +- ...native-link.service => nativelink.service} | 2 +- .../terraform/AWS/scripts/scheduler.json | 18 +- .../AWS/scripts/start_native_link.sh | 30 +- .../terraform/AWS/scripts/worker.json | 12 +- .../terraform/AWS/security_groups.tf | 18 +- deployment-examples/terraform/AWS/ssh_keys.tf | 8 +- .../terraform/AWS/variables.tf | 2 +- .../terraform/GCP/deployments/dev/main.tf | 2 +- .../terraform/GCP/module/base_image.tf | 34 +- .../terraform/GCP/module/cloud_metrics.tf | 10 +- .../terraform/GCP/module/instance_browser.tf | 8 +- .../terraform/GCP/module/instance_cas.tf | 8 +- .../GCP/module/instance_scheduler.tf | 8 +- .../GCP/module/instance_x86_cpu_worker.tf | 14 +- .../terraform/GCP/module/s3.tf | 2 +- .../GCP/module/scripts/browser_proxy.json | 8 +- .../terraform/GCP/module/scripts/cas.json | 16 +- .../GCP/module/scripts/cloud_publisher.py | 10 +- ...native-link.service => nativelink.service} | 2 +- .../GCP/module/scripts/scheduler.json | 16 +- .../GCP/module/scripts/start_native_link.sh | 82 +- .../terraform/GCP/module/scripts/worker.json | 16 +- .../terraform/GCP/module/ssh_keys.tf | 2 +- flake.nix | 16 +- integration_tests/simple_prometheus_test.sh | 22 +- .../BUILD.bazel | 6 +- .../Cargo.toml | 2 +- .../README.md | 4 +- .../examples/basic_cas.json | 6 +- .../examples/filesystem_cas.json | 14 +- .../s3_backend_with_local_fast_cas.json | 12 +- .../src/cas_server.rs | 6 +- .../src/lib.rs | 0 .../src/schedulers.rs | 0 .../src/serde_utils.rs | 0 .../src/stores.rs | 0 .../BUILD.bazel | 20 +- .../Cargo.toml | 8 +- .../src/action_scheduler.rs | 4 +- .../src/cache_lookup_scheduler.rs | 10 +- .../src/default_scheduler_factory.rs | 6 +- .../src/grpc_scheduler.rs | 12 +- .../src/lib.rs | 0 .../src/platform_property_manager.rs | 4 +- .../src/property_modifier_scheduler.rs | 8 +- .../src/simple_scheduler.rs | 12 +- .../src/worker.rs | 8 +- .../src/worker_scheduler.rs | 4 +- .../tests/action_messages_test.rs | 8 +- .../tests/cache_lookup_scheduler_test.rs | 18 +- .../tests/property_modifier_scheduler_test.rs | 20 +- .../tests/simple_scheduler_test.rs | 52 +- .../tests/utils/mock_scheduler.rs | 6 +- .../tests/utils/scheduler_utils.rs | 8 +- .../BUILD.bazel | 24 +- .../Cargo.toml | 10 +- .../src/ac_server.rs | 12 +- .../src/bytestream_server.rs | 16 +- .../src/capabilities_server.rs | 6 +- .../src/cas_server.rs | 10 +- .../src/execution_server.rs | 18 +- .../src/lib.rs | 0 .../src/worker_api_server.rs | 16 +- .../tests/ac_server_test.rs | 16 +- .../tests/bytestream_server_test.rs | 12 +- .../tests/cas_server_test.rs | 12 +- .../tests/worker_api_server_test.rs | 26 +- .../BUILD.bazel | 16 +- .../Cargo.toml | 6 +- .../src/ac_utils.rs | 8 +- .../src/completeness_checking_store.rs | 6 +- .../src/compression_store.rs | 14 +- .../src/dedup_store.rs | 10 +- .../src/default_store_factory.rs | 6 +- .../src/existence_cache_store.rs | 10 +- .../src/fast_slow_store.rs | 10 +- .../src/filesystem_store.rs | 16 +- .../src/grpc_store.rs | 38 +- .../src/lib.rs | 0 .../src/memory_store.rs | 14 +- .../src/noop_store.rs | 6 +- .../src/ref_store.rs | 8 +- .../src/s3_store.rs | 18 +- .../src/shard_store.rs | 10 +- .../src/size_partitioning_store.rs | 8 +- .../src/store_manager.rs | 2 +- .../src/verify_store.rs | 10 +- .../tests/ac_utils_test.rs | 10 +- .../tests/completeness_checking_store_test.rs | 14 +- .../tests/compression_store_test.rs | 82 +- .../tests/dedup_store_test.rs | 70 +- .../tests/existence_store_test.rs | 16 +- .../tests/fast_slow_store_test.rs | 18 +- .../tests/filesystem_store_test.rs | 44 +- .../tests/memory_store_test.rs | 16 +- .../tests/ref_store_test.rs | 20 +- .../tests/s3_store_test.rs | 28 +- .../tests/shard_store_test.rs | 16 +- .../tests/size_partitioning_store_test.rs | 22 +- .../tests/verify_store_test.rs | 58 +- .../BUILD.bazel | 12 +- .../Cargo.toml | 4 +- .../src/action_messages.rs | 0 .../src/async_fixed_buffer.rs | 0 .../src/buf_channel.rs | 0 .../src/common.rs | 0 .../src/digest_hasher.rs | 2 +- .../src/evicting_map.rs | 2 +- .../src/fastcdc.rs | 0 .../src/fs.rs | 0 .../src/lib.rs | 0 .../src/metrics_utils.rs | 0 .../src/platform_properties.rs | 0 .../src/resource_info.rs | 0 .../src/retry.rs | 0 .../src/store_trait.rs | 0 .../src/write_counter.rs | 0 .../src/write_request_stream_wrapper.rs | 0 .../tests/async_fixed_buffer_test.rs | 2 +- .../tests/buf_channel_test.rs | 2 +- .../tests/data/SekienAkashita.jpg | Bin .../tests/evicting_map_test.rs | 6 +- .../tests/fastcdc_test.rs | 2 +- .../tests/fs_test.rs | 2 +- .../tests/resource_info_test.rs | 2 +- .../tests/retry_test.rs | 2 +- .../BUILD.bazel | 24 +- .../Cargo.toml | 10 +- .../src/lib.rs | 0 .../src/local_worker.rs | 20 +- .../src/running_actions_manager.rs | 24 +- .../src/worker_api_client_wrapper.rs | 4 +- .../src/worker_utils.rs | 4 +- .../tests/local_worker_test.rs | 54 +- .../tests/running_actions_manager_test.rs | 116 +-- .../tests/utils/local_worker_test_utils.rs | 10 +- .../utils/mock_running_actions_manager.rs | 10 +- proto/BUILD.bazel | 4 +- .../remote_execution/worker_api.proto | 4 +- ..._machina.nativelink.remote_execution.pb.rs | 755 ++++++++++++++++++ proto/genproto/lib.rs | 4 +- run_integration_tests.sh | 24 +- src/bin/cas.rs | 40 +- tools/local-image-test.nix | 4 +- 166 files changed, 1903 insertions(+), 1148 deletions(-) rename deployment-examples/terraform/AWS/scripts/{native-link.service => nativelink.service} (82%) rename deployment-examples/terraform/GCP/module/scripts/{native-link.service => nativelink.service} (82%) rename {native-link-config => nativelink-config}/BUILD.bazel (82%) rename {native-link-config => nativelink-config}/Cargo.toml (83%) rename {native-link-config => nativelink-config}/README.md (96%) rename {native-link-config => nativelink-config}/examples/basic_cas.json (95%) rename {native-link-config => nativelink-config}/examples/filesystem_cas.json (90%) rename {native-link-config => nativelink-config}/examples/s3_backend_with_local_fast_cas.json (90%) rename {native-link-config => nativelink-config}/src/cas_server.rs (99%) rename {native-link-config => nativelink-config}/src/lib.rs (100%) rename {native-link-config => nativelink-config}/src/schedulers.rs (100%) rename {native-link-config => nativelink-config}/src/serde_utils.rs (100%) rename {native-link-config => nativelink-config}/src/stores.rs (100%) rename {native-link-scheduler => nativelink-scheduler}/BUILD.bazel (84%) rename {native-link-scheduler => nativelink-scheduler}/Cargo.toml (78%) rename {native-link-scheduler => nativelink-scheduler}/src/action_scheduler.rs (92%) rename {native-link-scheduler => nativelink-scheduler}/src/cache_lookup_scheduler.rs (96%) rename {native-link-scheduler => nativelink-scheduler}/src/default_scheduler_factory.rs (97%) rename {native-link-scheduler => nativelink-scheduler}/src/grpc_scheduler.rs (94%) rename {native-link-scheduler => nativelink-scheduler}/src/lib.rs (100%) rename {native-link-scheduler => nativelink-scheduler}/src/platform_property_manager.rs (95%) rename {native-link-scheduler => nativelink-scheduler}/src/property_modifier_scheduler.rs (93%) rename {native-link-scheduler => nativelink-scheduler}/src/simple_scheduler.rs (99%) rename {native-link-scheduler => nativelink-scheduler}/src/worker.rs (97%) rename {native-link-scheduler => nativelink-scheduler}/src/worker_scheduler.rs (95%) rename {native-link-scheduler => nativelink-scheduler}/tests/action_messages_test.rs (97%) rename {native-link-scheduler => nativelink-scheduler}/tests/cache_lookup_scheduler_test.rs (91%) rename {native-link-scheduler => nativelink-scheduler}/tests/property_modifier_scheduler_test.rs (93%) rename {native-link-scheduler => nativelink-scheduler}/tests/simple_scheduler_test.rs (97%) rename {native-link-scheduler => nativelink-scheduler}/tests/utils/mock_scheduler.rs (96%) rename {native-link-scheduler => nativelink-scheduler}/tests/utils/scheduler_utils.rs (86%) rename {native-link-service => nativelink-service}/BUILD.bazel (80%) rename {native-link-service => nativelink-service}/Cargo.toml (70%) rename {native-link-service => nativelink-service}/src/ac_server.rs (95%) rename {native-link-service => nativelink-service}/src/bytestream_server.rs (98%) rename {native-link-service => nativelink-service}/src/capabilities_server.rs (96%) rename {native-link-service => nativelink-service}/src/cas_server.rs (98%) rename {native-link-service => nativelink-service}/src/execution_server.rs (94%) rename {native-link-service => nativelink-service}/src/lib.rs (100%) rename {native-link-service => nativelink-service}/src/worker_api_server.rs (95%) rename {native-link-service => nativelink-service}/tests/ac_server_test.rs (93%) rename {native-link-service => nativelink-service}/tests/bytestream_server_test.rs (98%) rename {native-link-service => nativelink-service}/tests/cas_server_test.rs (97%) rename {native-link-service => nativelink-service}/tests/worker_api_server_test.rs (94%) rename {native-link-store => nativelink-store}/BUILD.bazel (92%) rename {native-link-store => nativelink-store}/Cargo.toml (89%) rename {native-link-store => nativelink-store}/src/ac_utils.rs (96%) rename {native-link-store => nativelink-store}/src/completeness_checking_store.rs (98%) rename {native-link-store => nativelink-store}/src/compression_store.rs (98%) rename {native-link-store => nativelink-store}/src/dedup_store.rs (97%) rename {native-link-store => nativelink-store}/src/default_store_factory.rs (97%) rename {native-link-store => nativelink-store}/src/existence_cache_store.rs (94%) rename {native-link-store => nativelink-store}/src/fast_slow_store.rs (97%) rename {native-link-store => nativelink-store}/src/filesystem_store.rs (97%) rename {native-link-store => nativelink-store}/src/grpc_store.rs (94%) rename {native-link-store => nativelink-store}/src/lib.rs (100%) rename {native-link-store => nativelink-store}/src/memory_store.rs (89%) rename {native-link-store => nativelink-store}/src/noop_store.rs (91%) rename {native-link-store => nativelink-store}/src/ref_store.rs (94%) rename {native-link-store => nativelink-store}/src/s3_store.rs (97%) rename {native-link-store => nativelink-store}/src/shard_store.rs (95%) rename {native-link-store => nativelink-store}/src/size_partitioning_store.rs (94%) rename {native-link-store => nativelink-store}/src/store_manager.rs (97%) rename {native-link-store => nativelink-store}/src/verify_store.rs (94%) rename {native-link-store => nativelink-store}/tests/ac_utils_test.rs (91%) rename {native-link-store => nativelink-store}/tests/completeness_checking_store_test.rs (95%) rename {native-link-store => nativelink-store}/tests/compression_store_test.rs (82%) rename {native-link-store => nativelink-store}/tests/dedup_store_test.rs (79%) rename {native-link-store => nativelink-store}/tests/existence_store_test.rs (85%) rename {native-link-store => nativelink-store}/tests/fast_slow_store_test.rs (91%) rename {native-link-store => nativelink-store}/tests/filesystem_store_test.rs (95%) rename {native-link-store => nativelink-store}/tests/memory_store_test.rs (93%) rename {native-link-store => nativelink-store}/tests/ref_store_test.rs (90%) rename {native-link-store => nativelink-store}/tests/s3_store_test.rs (96%) rename {native-link-store => nativelink-store}/tests/shard_store_test.rs (94%) rename {native-link-store => nativelink-store}/tests/size_partitioning_store_test.rs (90%) rename {native-link-store => nativelink-store}/tests/verify_store_test.rs (77%) rename {native-link-util => nativelink-util}/BUILD.bazel (92%) rename {native-link-util => nativelink-util}/Cargo.toml (89%) rename {native-link-util => nativelink-util}/src/action_messages.rs (100%) rename {native-link-util => nativelink-util}/src/async_fixed_buffer.rs (100%) rename {native-link-util => nativelink-util}/src/buf_channel.rs (100%) rename {native-link-util => nativelink-util}/src/common.rs (100%) rename {native-link-util => nativelink-util}/src/digest_hasher.rs (98%) rename {native-link-util => nativelink-util}/src/evicting_map.rs (99%) rename {native-link-util => nativelink-util}/src/fastcdc.rs (100%) rename {native-link-util => nativelink-util}/src/fs.rs (100%) rename {native-link-util => nativelink-util}/src/lib.rs (100%) rename {native-link-util => nativelink-util}/src/metrics_utils.rs (100%) rename {native-link-util => nativelink-util}/src/platform_properties.rs (100%) rename {native-link-util => nativelink-util}/src/resource_info.rs (100%) rename {native-link-util => nativelink-util}/src/retry.rs (100%) rename {native-link-util => nativelink-util}/src/store_trait.rs (100%) rename {native-link-util => nativelink-util}/src/write_counter.rs (100%) rename {native-link-util => nativelink-util}/src/write_request_stream_wrapper.rs (100%) rename {native-link-util => nativelink-util}/tests/async_fixed_buffer_test.rs (99%) rename {native-link-util => nativelink-util}/tests/buf_channel_test.rs (99%) rename {native-link-util => nativelink-util}/tests/data/SekienAkashita.jpg (100%) rename {native-link-util => nativelink-util}/tests/evicting_map_test.rs (99%) rename {native-link-util => nativelink-util}/tests/fastcdc_test.rs (99%) rename {native-link-util => nativelink-util}/tests/fs_test.rs (99%) rename {native-link-util => nativelink-util}/tests/resource_info_test.rs (99%) rename {native-link-util => nativelink-util}/tests/retry_test.rs (99%) rename {native-link-worker => nativelink-worker}/BUILD.bazel (83%) rename {native-link-worker => nativelink-worker}/Cargo.toml (78%) rename {native-link-worker => nativelink-worker}/src/lib.rs (100%) rename {native-link-worker => nativelink-worker}/src/local_worker.rs (97%) rename {native-link-worker => nativelink-worker}/src/running_actions_manager.rs (98%) rename {native-link-worker => nativelink-worker}/src/worker_api_client_wrapper.rs (93%) rename {native-link-worker => nativelink-worker}/src/worker_utils.rs (96%) rename {native-link-worker => nativelink-worker}/tests/local_worker_test.rs (91%) rename {native-link-worker => nativelink-worker}/tests/running_actions_manager_test.rs (95%) rename {native-link-worker => nativelink-worker}/tests/utils/local_worker_test_utils.rs (95%) rename {native-link-worker => nativelink-worker}/tests/utils/mock_running_actions_manager.rs (97%) create mode 100644 proto/genproto/com.github.trace_machina.nativelink.remote_execution.pb.rs diff --git a/.bazelignore b/.bazelignore index e981efaed..d5b677a81 100644 --- a/.bazelignore +++ b/.bazelignore @@ -5,7 +5,7 @@ bazel-out bazel-remote-bin bazel-remote-out bazel-remote-testlogs -bazel-remote-native-link +bazel-remote-nativelink bazel-root bazel-testlogs -bazel-native-link +bazel-nativelink diff --git a/.dockerignore b/.dockerignore index 59ad40101..df99d5c3f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,5 @@ target/ -*/*/.terraform-native-link-builder +*/*/.terraform-nativelink-builder */*/.update_scheduler_ips.zip */*/terraform.tfstate */*/.terraform diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1d38b3f8f..9c5cd057a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,7 +8,7 @@ on: branches: [ main ] jobs: - docker-compose-compiles-native-link: + docker-compose-compiles-nativelink: # The type of runner that the job will run on. runs-on: ubuntu-22.04 strategy: @@ -30,7 +30,7 @@ jobs: OS_VERSION=${{ matrix.os_version }} ADDITIONAL_SETUP_WORKER_CMD=DEBIAN_FRONTEND=noninteractive apt-get install -y gcc g++ lld pkg-config python3 load: true # This brings the build into `docker images` from buildx. - tags: trace_machina/native-link:latest + tags: trace_machina/nativelink:latest - uses: docker/build-push-action@v4 with: context: . @@ -39,7 +39,7 @@ jobs: OPT_LEVEL=opt OS_VERSION=${{ matrix.os_version }} load: true # This brings the build into `docker images` from buildx. - tags: trace_machina/native-link:builder + tags: trace_machina/nativelink:builder target: builder - name: Compile Native Link with Native Link @@ -48,7 +48,7 @@ jobs: cd deployment-examples/docker-compose && \ docker-compose up -d && \ cd ../../ && \ - docker run --rm --net=host -w /root/native-link -v $PWD:/root/native-link trace_machina/native-link:builder sh -c ' \ + docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \ bazel clean && \ bazel test //... \ --remote_instance_name=main \ @@ -56,7 +56,7 @@ jobs: --remote_executor=grpc://127.0.0.1:50052 \ --remote_default_exec_properties=cpu_count=1 \ ' && \ - docker run --rm --net=host -w /root/native-link -v $PWD:/root/native-link trace_machina/native-link:builder sh -c ' \ + docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \ bazel clean && \ bazel test //... \ --remote_instance_name=main \ @@ -85,7 +85,7 @@ jobs: OPT_LEVEL=fastbuild OS_VERSION=${{ matrix.os_version }} load: true # This brings the build into `docker images` from buildx. - tags: trace_machina/native-link:latest + tags: trace_machina/nativelink:latest - name: Run tests run: ./run_integration_tests.sh diff --git a/BUILD.bazel b/BUILD.bazel index 93b3ee5e6..9173be0fe 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -14,12 +14,12 @@ rust_binary( ], deps = [ "//error", - "//native-link-config", - "//native-link-scheduler", - "//native-link-service", - "//native-link-store", - "//native-link-util", - "//native-link-worker", + "//nativelink-config", + "//nativelink-scheduler", + "//nativelink-service", + "//nativelink-store", + "//nativelink-util", + "//nativelink-worker", "//proto", "@crate_index//:async-lock", "@crate_index//:axum", diff --git a/Cargo.Bazel.lock b/Cargo.Bazel.lock index c76fba625..3e3b6f6d0 100644 --- a/Cargo.Bazel.lock +++ b/Cargo.Bazel.lock @@ -7968,8 +7968,8 @@ }, "license": "MIT/Apache-2.0" }, - "native-link 0.0.0": { - "name": "native-link", + "nativelink 0.0.0": { + "name": "nativelink", "version": "0.0.0", "repository": null, "targets": [], @@ -8052,14 +8052,14 @@ }, "license": null }, - "native-link-config 0.0.0": { - "name": "native-link-config", + "nativelink-config 0.0.0": { + "name": "nativelink-config", "version": "0.0.0", "repository": null, "targets": [ { "Library": { - "crate_name": "native_link_config", + "crate_name": "nativelink_config", "crate_root": "src/lib.rs", "srcs": [ "**/*.rs" @@ -8067,7 +8067,7 @@ } } ], - "library_target_name": "native_link_config", + "library_target_name": "nativelink_config", "common_attrs": { "compile_data_glob": [ "**" @@ -8090,14 +8090,14 @@ }, "license": null }, - "native-link-scheduler 0.0.0": { - "name": "native-link-scheduler", + "nativelink-scheduler 0.0.0": { + "name": "nativelink-scheduler", "version": "0.0.0", "repository": null, "targets": [ { "Library": { - "crate_name": "native_link_scheduler", + "crate_name": "nativelink_scheduler", "crate_root": "src/lib.rs", "srcs": [ "**/*.rs" @@ -8105,7 +8105,7 @@ } } ], - "library_target_name": "native_link_scheduler", + "library_target_name": "nativelink_scheduler", "common_attrs": { "compile_data_glob": [ "**" @@ -8190,14 +8190,14 @@ }, "license": null }, - "native-link-service 0.0.0": { - "name": "native-link-service", + "nativelink-service 0.0.0": { + "name": "nativelink-service", "version": "0.0.0", "repository": null, "targets": [ { "Library": { - "crate_name": "native_link_service", + "crate_name": "nativelink_service", "crate_root": "src/lib.rs", "srcs": [ "**/*.rs" @@ -8205,7 +8205,7 @@ } } ], - "library_target_name": "native_link_service", + "library_target_name": "nativelink_service", "common_attrs": { "compile_data_glob": [ "**" @@ -8289,14 +8289,14 @@ }, "license": null }, - "native-link-store 0.0.0": { - "name": "native-link-store", + "nativelink-store 0.0.0": { + "name": "nativelink-store", "version": "0.0.0", "repository": null, "targets": [ { "Library": { - "crate_name": "native_link_store", + "crate_name": "nativelink_store", "crate_root": "src/lib.rs", "srcs": [ "**/*.rs" @@ -8304,7 +8304,7 @@ } } ], - "library_target_name": "native_link_store", + "library_target_name": "nativelink_store", "common_attrs": { "compile_data_glob": [ "**" @@ -8457,14 +8457,14 @@ }, "license": null }, - "native-link-util 0.0.0": { - "name": "native-link-util", + "nativelink-util 0.0.0": { + "name": "nativelink-util", "version": "0.0.0", "repository": null, "targets": [ { "Library": { - "crate_name": "native_link_util", + "crate_name": "nativelink_util", "crate_root": "src/lib.rs", "srcs": [ "**/*.rs" @@ -8472,7 +8472,7 @@ } } ], - "library_target_name": "native_link_util", + "library_target_name": "nativelink_util", "common_attrs": { "compile_data_glob": [ "**" @@ -8585,14 +8585,14 @@ }, "license": null }, - "native-link-worker 0.0.0": { - "name": "native-link-worker", + "nativelink-worker 0.0.0": { + "name": "nativelink-worker", "version": "0.0.0", "repository": null, "targets": [ { "Library": { - "crate_name": "native_link_worker", + "crate_name": "nativelink_worker", "crate_root": "src/lib.rs", "srcs": [ "**/*.rs" @@ -8600,7 +8600,7 @@ } } ], - "library_target_name": "native_link_worker", + "library_target_name": "nativelink_worker", "common_attrs": { "compile_data_glob": [ "**" @@ -17387,13 +17387,13 @@ "binary_crates": [], "workspace_members": { "error 0.0.0": "error", - "native-link 0.0.0": "", - "native-link-config 0.0.0": "native-link-config", - "native-link-scheduler 0.0.0": "native-link-scheduler", - "native-link-service 0.0.0": "native-link-service", - "native-link-store 0.0.0": "native-link-store", - "native-link-util 0.0.0": "native-link-util", - "native-link-worker 0.0.0": "native-link-worker", + "nativelink 0.0.0": "", + "nativelink-config 0.0.0": "nativelink-config", + "nativelink-scheduler 0.0.0": "nativelink-scheduler", + "nativelink-service 0.0.0": "nativelink-service", + "nativelink-store 0.0.0": "nativelink-store", + "nativelink-util 0.0.0": "nativelink-util", + "nativelink-worker 0.0.0": "nativelink-worker", "proto 0.0.0": "proto" }, "conditions": { diff --git a/Cargo.lock b/Cargo.lock index 312564f87..2490d07cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1544,7 +1544,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" [[package]] -name = "native-link" +name = "nativelink" version = "0.0.0" dependencies = [ "async-lock", @@ -1553,12 +1553,12 @@ dependencies = [ "error", "futures", "hyper", - "native-link-config", - "native-link-scheduler", - "native-link-service", - "native-link-store", - "native-link-util", - "native-link-worker", + "nativelink-config", + "nativelink-scheduler", + "nativelink-service", + "nativelink-store", + "nativelink-util", + "nativelink-worker", "parking_lot", "prometheus-client", "proto", @@ -1574,7 +1574,7 @@ dependencies = [ ] [[package]] -name = "native-link-config" +name = "nativelink-config" version = "0.0.0" dependencies = [ "serde", @@ -1582,7 +1582,7 @@ dependencies = [ ] [[package]] -name = "native-link-scheduler" +name = "nativelink-scheduler" version = "0.0.0" dependencies = [ "async-trait", @@ -1591,9 +1591,9 @@ dependencies = [ "futures", "hashbrown 0.14.3", "lru", - "native-link-config", - "native-link-store", - "native-link-util", + "nativelink-config", + "nativelink-store", + "nativelink-util", "parking_lot", "pretty_assertions", "prost", @@ -1608,7 +1608,7 @@ dependencies = [ ] [[package]] -name = "native-link-service" +name = "nativelink-service" version = "0.0.0" dependencies = [ "bytes", @@ -1617,10 +1617,10 @@ dependencies = [ "hyper", "log", "maplit", - "native-link-config", - "native-link-scheduler", - "native-link-store", - "native-link-util", + "nativelink-config", + "nativelink-scheduler", + "nativelink-store", + "nativelink-util", "parking_lot", "pretty_assertions", "prometheus-client", @@ -1636,7 +1636,7 @@ dependencies = [ ] [[package]] -name = "native-link-store" +name = "nativelink-store" version = "0.0.0" dependencies = [ "async-lock", @@ -1658,8 +1658,8 @@ dependencies = [ "hyper-rustls", "lz4_flex", "memory-stats", - "native-link-config", - "native-link-util", + "nativelink-config", + "nativelink-util", "once_cell", "parking_lot", "pretty_assertions", @@ -1678,7 +1678,7 @@ dependencies = [ ] [[package]] -name = "native-link-util" +name = "nativelink-util" version = "0.0.0" dependencies = [ "async-lock", @@ -1692,7 +1692,7 @@ dependencies = [ "log", "lru", "mock_instant", - "native-link-config", + "nativelink-config", "parking_lot", "pin-project-lite", "pretty_assertions", @@ -1709,7 +1709,7 @@ dependencies = [ ] [[package]] -name = "native-link-worker" +name = "nativelink-worker" version = "0.0.0" dependencies = [ "async-lock", @@ -1721,10 +1721,10 @@ dependencies = [ "futures", "hex", "hyper", - "native-link-config", - "native-link-scheduler", - "native-link-store", - "native-link-util", + "nativelink-config", + "nativelink-scheduler", + "nativelink-store", + "nativelink-util", "once_cell", "parking_lot", "pretty_assertions", diff --git a/Cargo.toml b/Cargo.toml index cb6139a58..33752f61a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ resolver = "2" [package] -name = "native-link" +name = "nativelink" version = "0.0.0" edition = "2021" @@ -21,12 +21,12 @@ name = "cas" [dependencies] error = { path = "error" } proto = { path = "proto" } -native-link-config = { path = "native-link-config" } -native-link-scheduler = { path = "native-link-scheduler" } -native-link-service = { path = "native-link-service" } -native-link-store = { path = "native-link-store" } -native-link-util = { path = "native-link-util" } -native-link-worker = { path = "native-link-worker" } +nativelink-config = { path = "nativelink-config" } +nativelink-scheduler = { path = "nativelink-scheduler" } +nativelink-service = { path = "nativelink-service" } +nativelink-store = { path = "nativelink-store" } +nativelink-util = { path = "nativelink-util" } +nativelink-worker = { path = "nativelink-worker" } async-lock = "2.8.0" axum = "0.6.20" diff --git a/README.md b/README.md index 2259efb34..aedbef603 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Native Link -[![CI](https://github.com/tracemachina/native-link/workflows/CI/badge.svg)](https://github.com/tracemachina/native-link/actions/workflows/main.yml) +[![CI](https://github.com/tracemachina/nativelink/workflows/CI/badge.svg)](https://github.com/tracemachina/nativelink/actions/workflows/main.yml) Native link is an extremely (blazingly?) fast and efficient build cache and remote executor for systems that communicate using the [Remote execution @@ -16,37 +16,37 @@ Supports Unix-based operating systems and Windows. * Nix with [flakes](https://nixos.wiki/wiki/Flakes) enabled This build does not require cloning the repository, but you need to provide a -config file, for instance the one at [native-link-config/examples/basic_cas.json](./native-link-config/examples/basic_cas.json). +config file, for instance the one at [nativelink-config/examples/basic_cas.json](./nativelink-config/examples/basic_cas.json). The following command builds and runs Native Link in release (optimized) mode: ```sh -nix run github:TraceMachina/native-link ./basic_cas.json +nix run github:TraceMachina/nativelink ./basic_cas.json ``` For use in production pin the executable to a specific revision: ```sh -nix run github:TraceMachina/native-link/ ./basic_cas.json +nix run github:TraceMachina/nativelink/ ./basic_cas.json ``` ## ๐Ÿ“ฆ Using the OCI image -See the published [OCI images](https://github.com/TraceMachina/native-link/pkgs/container/native-link) +See the published [OCI images](https://github.com/TraceMachina/nativelink/pkgs/container/nativelink) for pull commands. Images are tagged by nix derivation hash. The most recently pushed image corresponds to the `main` branch. Images are signed by the GitHub action that -produced the image. Note that the [OCI workflow](https://github.com/TraceMachina/native-link/actions/workflows/image.yaml) +produced the image. Note that the [OCI workflow](https://github.com/TraceMachina/nativelink/actions/workflows/image.yaml) might take a few minutes to publish the latest image. ```sh # Get the tag for the latest commit -export LATEST=$(nix eval github:TraceMachina/native-link#image.imageTag --raw) +export LATEST=$(nix eval github:TraceMachina/nativelink#image.imageTag --raw) # Verify the signature -cosign verify ghcr.io/tracemachina/native-link:${LATEST} \ - --certificate-identity=https://github.com/TraceMachina/native-link/.github/workflows/image.yaml@refs/heads/main \ +cosign verify ghcr.io/tracemachina/nativelink:${LATEST} \ + --certificate-identity=https://github.com/TraceMachina/nativelink/.github/workflows/image.yaml@refs/heads/main \ --certificate-oidc-issuer=https://token.actions.githubusercontent.com ``` @@ -54,11 +54,11 @@ For use in production pin the image to a specific revision: ```sh # Get the tag for a specific commit -export PINNED_TAG=$(nix eval github:TraceMachina/native-link/#image.imageTag --raw) +export PINNED_TAG=$(nix eval github:TraceMachina/nativelink/#image.imageTag --raw) # Verify the signature -cosign verify ghcr.io/tracemachina/native-link:${PINNED_TAG} \ - --certificate-identity=https://github.com/TraceMachina/native-link/.github/workflows/image.yaml@refs/heads/main \ +cosign verify ghcr.io/tracemachina/nativelink:${PINNED_TAG} \ + --certificate-identity=https://github.com/TraceMachina/nativelink/.github/workflows/image.yaml@refs/heads/main \ --certificate-oidc-issuer=https://token.actions.githubusercontent.com ``` @@ -85,16 +85,16 @@ the service: ```sh # Unoptimized development build on Unix -bazel run cas -- ./native-link-config/examples/basic_cas.json +bazel run cas -- ./nativelink-config/examples/basic_cas.json # Optimized release build on Unix -bazel run -c opt cas -- ./native-link-config/examples/basic_cas.json +bazel run -c opt cas -- ./nativelink-config/examples/basic_cas.json # Unoptimized development build on Windows -bazel run --config=windows cas -- ./native-link-config/examples/basic_cas.json +bazel run --config=windows cas -- ./nativelink-config/examples/basic_cas.json # Optimized release build on Windows -bazel run --config=windows -c opt cas -- ./native-link-config/examples/basic_cas.json +bazel run --config=windows -c opt cas -- ./nativelink-config/examples/basic_cas.json ``` ## ๐Ÿฆ€ Building with Cargo @@ -110,10 +110,10 @@ bazel run --config=windows -c opt cas -- ./native-link-config/examples/basic_cas ```bash # Unoptimized development build -cargo run --bin cas -- ./native-link-config/examples/basic_cas.json +cargo run --bin cas -- ./nativelink-config/examples/basic_cas.json # Optimized release build -cargo run --release --bin cas -- ./native-link-config/examples/basic_cas.json +cargo run --release --bin cas -- ./nativelink-config/examples/basic_cas.json ``` ## ๐Ÿงช Evaluating Native Link @@ -136,7 +136,7 @@ and `worker`. ## โš™๏ธ Configuration -The `cas` executable reads a JSON file as it's only parameter. See [native-link-config](./native-link-config) +The `cas` executable reads a JSON file as it's only parameter. See [nativelink-config](./nativelink-config) for more details and examples. ## ๐Ÿš€ Example Deployments diff --git a/SECURITY.md b/SECURITY.md index 3bbb421c3..5af432da9 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,25 +1,25 @@ # Security policy -GitHub's bots crawl `native-link` to detect security vulnerabilities wherever +GitHub's bots crawl `nativelink` to detect security vulnerabilities wherever possible. -TraceMachina and the `native-link` authors place a high emphasis on fixing any +TraceMachina and the `nativelink` authors place a high emphasis on fixing any vulnerabilities. Please send a report if something doesn't look right. ## Supported versions -At the moment no version of `native-link` is officially supported. Consider +At the moment no version of `nativelink` is officially supported. Consider using the latest commit on the `main` branch until official production binaries are released. ## Reporting a vulnerability -Prefer reporting vulnerabilities via [GitHub](https://github.com/TraceMachina/native-link/security). +Prefer reporting vulnerabilities via [GitHub](https://github.com/TraceMachina/nativelink/security). If you'd rather communicate via email please contact , , or . ## Vulnerability disclosure and advisories -See [Advisories](https://github.com/TraceMachina/native-link/security/advisories) +See [Advisories](https://github.com/TraceMachina/nativelink/security/advisories) for publicly disclosed vulnerabilities. diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index d22f0f7ea..7c3e9a7c4 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -1,4 +1,4 @@ -workspace(name = "native-link") +workspace(name = "nativelink") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") diff --git a/deployment-examples/docker-compose/Dockerfile b/deployment-examples/docker-compose/Dockerfile index f1ba3d300..9ca3dc031 100644 --- a/deployment-examples/docker-compose/Dockerfile +++ b/deployment-examples/docker-compose/Dockerfile @@ -38,16 +38,16 @@ RUN apt update && \ # Build the binary. FROM dependencies AS builder -WORKDIR /root/native-link +WORKDIR /root/nativelink ADD . . ARG OPT_LEVEL ARG ADDITIONAL_BAZEL_FLAGS RUN bazel build -c ${OPT_LEVEL} ${ADDITIONAL_BAZEL_FLAGS} cas && \ - cp ./bazel-bin/cas /root/native-link-bin + cp ./bazel-bin/cas /root/nativelink-bin # Go back to a fresh ubuntu container and copy only the compiled binary. FROM ubuntu:${OS_VERSION} as final -COPY --from=builder /root/native-link-bin /usr/local/bin/native-link +COPY --from=builder /root/nativelink-bin /usr/local/bin/nativelink # Install any specific dependencies needed by the user. # Install runtime packages. RUN apt-get update && \ @@ -56,6 +56,6 @@ RUN apt-get update && \ ARG ADDITIONAL_SETUP_WORKER_CMD RUN bash -ueo pipefail -c "${ADDITIONAL_SETUP_WORKER_CMD}" -RUN mkdir -p /root/.cache/native-link +RUN mkdir -p /root/.cache/nativelink EXPOSE 50051/tcp 50052/tcp -CMD ["native-link"] +CMD ["nativelink"] diff --git a/deployment-examples/docker-compose/README.md b/deployment-examples/docker-compose/README.md index 5f26878bb..f25f931d5 100644 --- a/deployment-examples/docker-compose/README.md +++ b/deployment-examples/docker-compose/README.md @@ -27,15 +27,15 @@ bazel test //... \ ## Instances -All instances use the same Docker image, trace_machina/native-link:latest, built from the Dockerfile located at ./deployment-examples/docker-compose/Dockerfile. +All instances use the same Docker image, trace_machina/nativelink:latest, built from the Dockerfile located at ./deployment-examples/docker-compose/Dockerfile. ### CAS -The CAS (Content Addressable Storage) service is used as a local cache for the Native Link system. It is configured in the docker-compose.yml file under the native_link_local_cas service. +The CAS (Content Addressable Storage) service is used as a local cache for the Native Link system. It is configured in the docker-compose.yml file under the nativelink_local_cas service. ```yml - native_link_local_cas: - image: trace_machina/native-link:latest + nativelink_local_cas: + image: trace_machina/nativelink:latest build: context: ../.. dockerfile: ./deployment-examples/docker-compose/Dockerfile @@ -43,7 +43,7 @@ The CAS (Content Addressable Storage) service is used as a local cache for the N args: - ADDITIONAL_SETUP_WORKER_CMD=${ADDITIONAL_SETUP_WORKER_CMD:-} volumes: - - ${NATIVE_LINK_DIR:-~/.cache/native-link}:/root/.cache/native-link + - ${nativelink_DIR:-~/.cache/nativelink}:/root/.cache/nativelink - type: bind source: . target: /root @@ -56,17 +56,17 @@ The CAS (Content Addressable Storage) service is used as a local cache for the N "50071":50071/tcp", ] command: | - native-link /root/local-storage-cas.json + nativelink /root/local-storage-cas.json ``` ### Scheduler The scheduler is currently the only single point of failure in the system. We currently only support one scheduler at a time. -The scheduler service is responsible for scheduling tasks in the Native Link system. It is configured in the docker-compose.yml file under the native_link_scheduler service. +The scheduler service is responsible for scheduling tasks in the Native Link system. It is configured in the docker-compose.yml file under the nativelink_scheduler service. ```yml - native_link_scheduler: - image: trace_machina/native-link:latest + nativelink_scheduler: + image: trace_machina/nativelink:latest build: context: ../.. dockerfile: ./deployment-examples/docker-compose/Dockerfile @@ -79,19 +79,19 @@ The scheduler service is responsible for scheduling tasks in the Native Link sys target: /root environment: RUST_LOG: ${RUST_LOG:-warn} - CAS_ENDPOINT: native_link_local_cas + CAS_ENDPOINT: nativelink_local_cas ports: [ "50052:50052/tcp" ] command: | - native-link /root/scheduler.json + nativelink /root/scheduler.json ``` ### Workers -Worker instances are responsible for executing tasks. They are configured in the docker-compose.yml file under the native_link_executor service. +Worker instances are responsible for executing tasks. They are configured in the docker-compose.yml file under the nativelink_executor service. ```yml - native_link_executor: - image: trace_machina/native-link:latest + nativelink_executor: + image: trace_machina/nativelink:latest build: context: ../.. dockerfile: ./deployment-examples/docker-compose/Dockerfile @@ -99,16 +99,16 @@ Worker instances are responsible for executing tasks. They are configured in the args: - ADDITIONAL_SETUP_WORKER_CMD=${ADDITIONAL_SETUP_WORKER_CMD:-} volumes: - - ${NATIVE_LINK_DIR:-~/.cache/native-link}:/root/.cache/native-link + - ${nativelink_DIR:-~/.cache/nativelink}:/root/.cache/nativelink - type: bind source: . target: /root environment: RUST_LOG: ${RUST_LOG:-warn} - CAS_ENDPOINT: native_link_local_cas - SCHEDULER_ENDPOINT: native_link_scheduler + CAS_ENDPOINT: nativelink_local_cas + SCHEDULER_ENDPOINT: nativelink_scheduler command: | - native-link /root/worker.json + nativelink /root/worker.json ``` ## Security diff --git a/deployment-examples/docker-compose/docker-compose.yml b/deployment-examples/docker-compose/docker-compose.yml index c4d04e9e5..f2d3224b4 100644 --- a/deployment-examples/docker-compose/docker-compose.yml +++ b/deployment-examples/docker-compose/docker-compose.yml @@ -15,8 +15,8 @@ version: '3.4' services: - native_link_local_cas: - image: trace_machina/native-link:latest + nativelink_local_cas: + image: trace_machina/nativelink:latest build: context: ../.. dockerfile: ./deployment-examples/docker-compose/Dockerfile @@ -24,7 +24,7 @@ services: args: - ADDITIONAL_SETUP_WORKER_CMD=${ADDITIONAL_SETUP_WORKER_CMD:-} volumes: - - ${NATIVE_LINK_DIR:-~/.cache/native-link}:/root/.cache/native-link + - ${nativelink_DIR:-~/.cache/nativelink}:/root/.cache/nativelink - type: bind source: . target: /root @@ -32,10 +32,10 @@ services: RUST_LOG: ${RUST_LOG:-warn} ports: [ "50051:50051/tcp", "127.0.0.1:50061:50061", "50071:50071/tcp", ] command: | - native-link /root/local-storage-cas.json + nativelink /root/local-storage-cas.json - native_link_scheduler: - image: trace_machina/native-link:latest + nativelink_scheduler: + image: trace_machina/nativelink:latest build: context: ../.. dockerfile: ./deployment-examples/docker-compose/Dockerfile @@ -48,13 +48,13 @@ services: target: /root environment: RUST_LOG: ${RUST_LOG:-warn} - CAS_ENDPOINT: native_link_local_cas + CAS_ENDPOINT: nativelink_local_cas ports: [ "50052:50052/tcp" ] command: | - native-link /root/scheduler.json + nativelink /root/scheduler.json - native_link_executor: - image: trace_machina/native-link:latest + nativelink_executor: + image: trace_machina/nativelink:latest build: context: ../.. dockerfile: ./deployment-examples/docker-compose/Dockerfile @@ -62,13 +62,13 @@ services: args: - ADDITIONAL_SETUP_WORKER_CMD=${ADDITIONAL_SETUP_WORKER_CMD:-} volumes: - - ${NATIVE_LINK_DIR:-~/.cache/native-link}:/root/.cache/native-link + - ${nativelink_DIR:-~/.cache/nativelink}:/root/.cache/nativelink - type: bind source: . target: /root environment: RUST_LOG: ${RUST_LOG:-warn} - CAS_ENDPOINT: native_link_local_cas - SCHEDULER_ENDPOINT: native_link_scheduler + CAS_ENDPOINT: nativelink_local_cas + SCHEDULER_ENDPOINT: nativelink_scheduler command: | - native-link /root/worker.json + nativelink /root/worker.json diff --git a/deployment-examples/docker-compose/local-storage-cas.json b/deployment-examples/docker-compose/local-storage-cas.json index 79f368509..23f9be14e 100644 --- a/deployment-examples/docker-compose/local-storage-cas.json +++ b/deployment-examples/docker-compose/local-storage-cas.json @@ -1,5 +1,5 @@ // This configuration will place objects in various folders in -// `~/.cache/native-link`. It will store all data on disk and +// `~/.cache/nativelink`. It will store all data on disk and // allows for restarts of the underlying service. It is optimized // so objects are compressed, deduplicated and uses some in-memory // optimizations for certain hot paths. @@ -14,8 +14,8 @@ }, "backend": { "filesystem": { - "content_path": "~/.cache/native-link/content_path-cas", - "temp_path": "~/.cache/native-link/tmp_path-cas", + "content_path": "~/.cache/nativelink/content_path-cas", + "temp_path": "~/.cache/nativelink/tmp_path-cas", "eviction_policy": { // 10gb. "max_bytes": 10000000000, @@ -30,8 +30,8 @@ }, "AC_MAIN_STORE": { "filesystem": { - "content_path": "~/.cache/native-link/content_path-ac", - "temp_path": "~/.cache/native-link/tmp_path-ac", + "content_path": "~/.cache/nativelink/content_path-ac", + "temp_path": "~/.cache/nativelink/tmp_path-ac", "eviction_policy": { // 500mb. "max_bytes": 500000000, diff --git a/deployment-examples/docker-compose/worker.json b/deployment-examples/docker-compose/worker.json index c1f9ca0bb..a34a9e12c 100644 --- a/deployment-examples/docker-compose/worker.json +++ b/deployment-examples/docker-compose/worker.json @@ -20,8 +20,8 @@ "fast_slow": { "fast": { "filesystem": { - "content_path": "/root/.cache/native-link/data-worker-test/content_path-cas", - "temp_path": "/root/.cache/native-link/data-worker-test/tmp_path-cas", + "content_path": "/root/.cache/nativelink/data-worker-test/content_path-cas", + "temp_path": "/root/.cache/nativelink/data-worker-test/tmp_path-cas", "eviction_policy": { // 10gb. "max_bytes": 10000000000, @@ -45,7 +45,7 @@ "upload_action_result": { "ac_store": "GRPC_LOCAL_AC_STORE", }, - "work_directory": "/root/.cache/native-link/work", + "work_directory": "/root/.cache/nativelink/work", "platform_properties": { "cpu_count": { "query_cmd": "nproc" diff --git a/deployment-examples/terraform/AWS/ami.tf b/deployment-examples/terraform/AWS/ami.tf index 9dd652d4a..33af5a99c 100644 --- a/deployment-examples/terraform/AWS/ami.tf +++ b/deployment-examples/terraform/AWS/ami.tf @@ -14,7 +14,7 @@ # -- Begin Base AMI --- -resource "aws_instance" "build_native_link_instance" { +resource "aws_instance" "build_nativelink_instance" { for_each = { arm = { "instance_type": var.build_arm_instance_type, @@ -29,7 +29,7 @@ resource "aws_instance" "build_native_link_instance" { ami = each.value["ami"] instance_type = each.value["instance_type"] associate_public_ip_address = true - key_name = aws_key_pair.native_link_key.key_name + key_name = aws_key_pair.nativelink_key.key_name iam_instance_profile = aws_iam_instance_profile.builder_profile.name vpc_security_group_ids = [ @@ -44,7 +44,7 @@ resource "aws_instance" "build_native_link_instance" { } tags = { - "native_link:instance_type" = "ami_builder", + "nativelink:instance_type" = "ami_builder", } connection { @@ -52,7 +52,7 @@ resource "aws_instance" "build_native_link_instance" { agent = true type = "ssh" user = "ubuntu" - private_key = data.tls_public_key.native_link_pem.private_key_openssh + private_key = data.tls_public_key.nativelink_pem.private_key_openssh } provisioner "local-exec" { @@ -60,9 +60,9 @@ resource "aws_instance" "build_native_link_instance" { set -ex SELF_DIR=$(pwd) cd ../../.. - rm -rf $SELF_DIR/.terraform-native-link-builder - mkdir -p $SELF_DIR/.terraform-native-link-builder - find . ! -ipath '*/target*' -and ! \( -ipath '*/.*' -and ! -name '.rustfmt.toml' -and ! -name '.bazelrc' \) -and ! -ipath './bazel-*' -type f -print0 | tar cvf $SELF_DIR/.terraform-native-link-builder/file.tar.gz --null -T - + rm -rf $SELF_DIR/.terraform-nativelink-builder + mkdir -p $SELF_DIR/.terraform-nativelink-builder + find . ! -ipath '*/target*' -and ! \( -ipath '*/.*' -and ! -name '.rustfmt.toml' -and ! -name '.bazelrc' \) -and ! -ipath './bazel-*' -type f -print0 | tar cvf $SELF_DIR/.terraform-nativelink-builder/file.tar.gz --null -T - EOT } @@ -102,7 +102,7 @@ resource "aws_instance" "build_native_link_instance" { } provisioner "file" { - source = "./.terraform-native-link-builder/file.tar.gz" + source = "./.terraform-nativelink-builder/file.tar.gz" destination = "/tmp/file.tar.gz" } @@ -110,27 +110,27 @@ resource "aws_instance" "build_native_link_instance" { inline = [ <&2 + echo "Unknown nativelink type: $type" >&2 echo 1 fi diff --git a/deployment-examples/terraform/GCP/module/scripts/worker.json b/deployment-examples/terraform/GCP/module/scripts/worker.json index 578182866..b7e6c17f8 100644 --- a/deployment-examples/terraform/GCP/module/scripts/worker.json +++ b/deployment-examples/terraform/GCP/module/scripts/worker.json @@ -4,8 +4,8 @@ "compression": { "backend": { "s3_store": { - "region": "${NATIVE_LINK_AC_S3_BUCKET_REGION:-}", - "bucket": "${NATIVE_LINK_AC_S3_BUCKET:-}", + "region": "${nativelink_AC_S3_BUCKET_REGION:-}", + "bucket": "${nativelink_AC_S3_BUCKET:-}", "retry": { "max_retries": 7, "delay": 0.1, @@ -22,8 +22,8 @@ "compression": { "backend": { "s3_store": { - "region": "${NATIVE_LINK_CAS_S3_BUCKET_REGION:-}", - "bucket": "${NATIVE_LINK_CAS_S3_BUCKET:-}", + "region": "${nativelink_CAS_S3_BUCKET_REGION:-}", + "bucket": "${nativelink_CAS_S3_BUCKET:-}", "retry": { "max_retries": 7, "delay": 0.1, @@ -43,7 +43,7 @@ "content_path": "/worker/cas/content", "temp_path": "/worker/cas/temp", "eviction_policy": { - "max_bytes": "${NATIVE_LINK_CAS_FS_CONTENT_LIMIT:-100000000}" + "max_bytes": "${nativelink_CAS_FS_CONTENT_LIMIT:-100000000}" } } }, @@ -66,14 +66,14 @@ "workers": [{ "local": { "worker_api_endpoint": { - "uri": "grpc://${NATIVE_LINK_SCHEDULER_ENDPOINT:-127.0.0.1}:50061" + "uri": "grpc://${nativelink_SCHEDULER_ENDPOINT:-127.0.0.1}:50061" }, "cas_fast_slow_store": "CAS_STORE", "upload_action_result": { "ac_store": "AC_STORE", "historical_results_store": "S3_CAS_STORE", - "success_message_template": "https://${NATIVE_LINK_BROWSER_ENDPOINT:-example.com}/main/blobs/{digest_function}/action/{action_digest_hash}-{action_digest_size}/", - "failure_message_template": "https://${NATIVE_LINK_BROWSER_ENDPOINT:-example.com}/main/blobs/{digest_function}/historical_execute_response/{historical_results_hash}-{historical_results_size}/" + "success_message_template": "https://${nativelink_BROWSER_ENDPOINT:-example.com}/main/blobs/{digest_function}/action/{action_digest_hash}-{action_digest_size}/", + "failure_message_template": "https://${nativelink_BROWSER_ENDPOINT:-example.com}/main/blobs/{digest_function}/historical_execute_response/{historical_results_hash}-{historical_results_size}/" }, "work_directory": "/worker/work", "platform_properties": { diff --git a/deployment-examples/terraform/GCP/module/ssh_keys.tf b/deployment-examples/terraform/GCP/module/ssh_keys.tf index f44d68bf0..7ea36ff2f 100644 --- a/deployment-examples/terraform/GCP/module/ssh_keys.tf +++ b/deployment-examples/terraform/GCP/module/ssh_keys.tf @@ -16,6 +16,6 @@ resource "tls_private_key" "ssh_key" { algorithm = "ED25519" } -data "tls_public_key" "native_link_pem" { +data "tls_public_key" "nativelink_pem" { private_key_openssh = tls_private_key.ssh_key.private_key_openssh } diff --git a/flake.nix b/flake.nix index 29dd75f05..69859d6e5 100644 --- a/flake.nix +++ b/flake.nix @@ -1,5 +1,5 @@ { - description = "native-link"; + description = "nativelink"; inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; @@ -58,7 +58,7 @@ # Additional target for external dependencies to simplify caching. cargoArtifacts = craneLib.buildDepsOnly commonArgs; - native-link = craneLib.buildPackage (commonArgs + nativelink = craneLib.buildPackage (commonArgs // { inherit cargoArtifacts; }); @@ -73,26 +73,26 @@ apps = { default = { type = "app"; - program = "${native-link}/bin/cas"; + program = "${nativelink}/bin/cas"; }; }; packages = { inherit publish-ghcr local-image-test; - default = native-link; + default = nativelink; image = pkgs.dockerTools.streamLayeredImage { - name = "native-link"; + name = "nativelink"; contents = [ - native-link + nativelink pkgs.dockerTools.caCertificates ]; config = { Entrypoint = [ "/bin/cas" ]; Labels = { "org.opencontainers.image.description" = "An RBE compatible, high-performance cache and remote executor."; - "org.opencontainers.image.documentation" = "https://github.com/TraceMachina/native-link"; + "org.opencontainers.image.documentation" = "https://github.com/TraceMachina/nativelink"; "org.opencontainers.image.licenses" = "Apache-2.0"; "org.opencontainers.image.revision" = "${self.rev or self.dirtyRev or "dirty"}"; - "org.opencontainers.image.source" = "https://github.com/TraceMachina/native-link"; + "org.opencontainers.image.source" = "https://github.com/TraceMachina/nativelink"; "org.opencontainers.image.title" = "Native Link"; "org.opencontainers.image.vendor" = "Trace Machina, Inc."; }; diff --git a/integration_tests/simple_prometheus_test.sh b/integration_tests/simple_prometheus_test.sh index 4ebf83e16..f588b4697 100644 --- a/integration_tests/simple_prometheus_test.sh +++ b/integration_tests/simple_prometheus_test.sh @@ -32,15 +32,15 @@ echo "$all_contents" # Check static metrics in some of the stores. These settings are set # in the config file of integration tests for the CAS. -echo 'Checking: native_link_stores_AC_MAIN_STORE_evicting_map_max_bytes 500000000' -grep -q 'native_link_stores_AC_MAIN_STORE_evicting_map_max_bytes 500000000' <<< "$all_contents" -echo 'Checking: native_link_stores_AC_MAIN_STORE_read_buff_size_bytes 32768' -grep -q 'native_link_stores_AC_MAIN_STORE_read_buff_size_bytes 32768' <<< "$all_contents" -echo 'Checking: native_link_stores_AC_MAIN_STORE_evicting_map_max_bytes 500000000' -grep -q 'native_link_stores_AC_MAIN_STORE_evicting_map_max_bytes 500000000' <<< "$all_contents" +echo 'Checking: nativelink_stores_AC_MAIN_STORE_evicting_map_max_bytes 500000000' +grep -q 'nativelink_stores_AC_MAIN_STORE_evicting_map_max_bytes 500000000' <<< "$all_contents" +echo 'Checking: nativelink_stores_AC_MAIN_STORE_read_buff_size_bytes 32768' +grep -q 'nativelink_stores_AC_MAIN_STORE_read_buff_size_bytes 32768' <<< "$all_contents" +echo 'Checking: nativelink_stores_AC_MAIN_STORE_evicting_map_max_bytes 500000000' +grep -q 'nativelink_stores_AC_MAIN_STORE_evicting_map_max_bytes 500000000' <<< "$all_contents" # Ensure our store metrics are only published once. -count=$(grep 'native_link_stores_AC_MAIN_STORE_evicting_map_max_bytes 500000000' <<< "$all_contents" | wc -l) +count=$(grep 'nativelink_stores_AC_MAIN_STORE_evicting_map_max_bytes 500000000' <<< "$all_contents" | wc -l) if [[ $count -ne 1 ]]; then echo "Expected to find 1 instance of CAS_MAIN_STORE, but found $count" exit 1 @@ -48,7 +48,7 @@ fi # Check dynamic metrics in some of the stores. # These are the most stable settings to test that are dymaic. -echo 'Checking: native_link_stores_AC_MAIN_STORE_evicting_map_item_size_bytes{quantile="0.99"}' -grep -q 'native_link_stores_AC_MAIN_STORE_evicting_map_item_size_bytes{quantile="0.99"}' <<< "$all_contents" -echo 'Checking: native_link_stores_AC_MAIN_STORE_evicting_map_items_in_store_total 3' -grep -q 'native_link_stores_AC_MAIN_STORE_evicting_map_items_in_store_total 3' <<< "$all_contents" +echo 'Checking: nativelink_stores_AC_MAIN_STORE_evicting_map_item_size_bytes{quantile="0.99"}' +grep -q 'nativelink_stores_AC_MAIN_STORE_evicting_map_item_size_bytes{quantile="0.99"}' <<< "$all_contents" +echo 'Checking: nativelink_stores_AC_MAIN_STORE_evicting_map_items_in_store_total 3' +grep -q 'nativelink_stores_AC_MAIN_STORE_evicting_map_items_in_store_total 3' <<< "$all_contents" diff --git a/native-link-config/BUILD.bazel b/nativelink-config/BUILD.bazel similarity index 82% rename from native-link-config/BUILD.bazel rename to nativelink-config/BUILD.bazel index 64399a4cd..5221131f3 100644 --- a/native-link-config/BUILD.bazel +++ b/nativelink-config/BUILD.bazel @@ -6,7 +6,7 @@ load( ) rust_library( - name = "native-link-config", + name = "nativelink-config", srcs = [ "src/cas_server.rs", "src/lib.rs", @@ -23,10 +23,10 @@ rust_library( rust_doc( name = "docs", - crate = ":native-link-config", + crate = ":nativelink-config", ) rust_doc_test( name = "doc_test", - crate = ":native-link-config", + crate = ":nativelink-config", ) diff --git a/native-link-config/Cargo.toml b/nativelink-config/Cargo.toml similarity index 83% rename from native-link-config/Cargo.toml rename to nativelink-config/Cargo.toml index 862c6cd05..807d7bd99 100644 --- a/native-link-config/Cargo.toml +++ b/nativelink-config/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "native-link-config" +name = "nativelink-config" version = "0.0.0" edition = "2021" diff --git a/native-link-config/README.md b/nativelink-config/README.md similarity index 96% rename from native-link-config/README.md rename to nativelink-config/README.md index 02cf812e4..880b9b8e8 100644 --- a/native-link-config/README.md +++ b/nativelink-config/README.md @@ -3,14 +3,14 @@ This service uses a JSON file as the configuration format. The JSON format is compatible with how protobuf's JSON format is structed. -To view the available fields please refer to [stores.rs](https://github.com/tracemachina/native-link/blob/master/native-link-config/stores.rs) and [cas_server](https://github.com/tracemachina/native-link/blob/master/native-link-config/cas_server.rs). +To view the available fields please refer to [stores.rs](https://github.com/tracemachina/nativelink/blob/master/nativelink-config/stores.rs) and [cas_server](https://github.com/tracemachina/nativelink/blob/master/nativelink-config/cas_server.rs). These two files should have enough documentation in them on what each field does and where each field goes. ## Examples -The [examples directory](https://github.com/tracemachina/native-link/tree/master/native-link-config/examples) contains a few examples of configuration files. +The [examples directory](https://github.com/tracemachina/nativelink/tree/master/nativelink-config/examples) contains a few examples of configuration files. A very basic configuration that is a pure in-memory store is: diff --git a/native-link-config/examples/basic_cas.json b/nativelink-config/examples/basic_cas.json similarity index 95% rename from native-link-config/examples/basic_cas.json rename to nativelink-config/examples/basic_cas.json index 6d53bc3d4..62aa193fe 100644 --- a/native-link-config/examples/basic_cas.json +++ b/nativelink-config/examples/basic_cas.json @@ -14,8 +14,8 @@ // hardlinks on disk to a directory where the jobs are running. "fast": { "filesystem": { - "content_path": "/tmp/native_link/data-worker-test/content_path-cas", - "temp_path": "/tmp/native_link/data-worker-test/tmp_path-cas", + "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", + "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", "eviction_policy": { // 10gb. "max_bytes": 10000000000, @@ -67,7 +67,7 @@ "upload_action_result": { "ac_store": "AC_MAIN_STORE", }, - "work_directory": "/tmp/native_link/work", + "work_directory": "/tmp/nativelink/work", "platform_properties": { "cpu_count": { "values": ["16"], diff --git a/native-link-config/examples/filesystem_cas.json b/nativelink-config/examples/filesystem_cas.json similarity index 90% rename from native-link-config/examples/filesystem_cas.json rename to nativelink-config/examples/filesystem_cas.json index dc5df4fb9..e9113f6db 100644 --- a/native-link-config/examples/filesystem_cas.json +++ b/nativelink-config/examples/filesystem_cas.json @@ -1,5 +1,5 @@ // This configuration will place objects in various folders in -// `/tmp/native_link/data`. It will store all data on disk and +// `/tmp/nativelink/data`. It will store all data on disk and // allows for restarts of the underlying service. It is optimized // so objects are compressed, deduplicated and uses some in-memory // optimizations for certain hot paths. @@ -12,8 +12,8 @@ }, "backend": { "filesystem": { - "content_path": "/tmp/native_link/data/content_path-cas", - "temp_path": "/tmp/native_link/data/tmp_path-cas", + "content_path": "/tmp/nativelink/data/content_path-cas", + "temp_path": "/tmp/nativelink/data/tmp_path-cas", "eviction_policy": { // 2gb. "max_bytes": 2000000000, @@ -56,8 +56,8 @@ }, "slow": { "filesystem": { - "content_path": "/tmp/native_link/data/content_path-index", - "temp_path": "/tmp/native_link/data/tmp_path-index", + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", "eviction_policy": { // 500mb. "max_bytes": 500000000, @@ -81,8 +81,8 @@ }, "AC_MAIN_STORE": { "filesystem": { - "content_path": "/tmp/native_link/data/content_path-ac", - "temp_path": "/tmp/native_link/data/tmp_path-ac", + "content_path": "/tmp/nativelink/data/content_path-ac", + "temp_path": "/tmp/nativelink/data/tmp_path-ac", "eviction_policy": { // 500mb. "max_bytes": 500000000, diff --git a/native-link-config/examples/s3_backend_with_local_fast_cas.json b/nativelink-config/examples/s3_backend_with_local_fast_cas.json similarity index 90% rename from native-link-config/examples/s3_backend_with_local_fast_cas.json rename to nativelink-config/examples/s3_backend_with_local_fast_cas.json index bf90b7877..c4d438a32 100644 --- a/native-link-config/examples/s3_backend_with_local_fast_cas.json +++ b/nativelink-config/examples/s3_backend_with_local_fast_cas.json @@ -8,8 +8,8 @@ "fast_slow": { "fast": { "filesystem": { - "content_path": "/tmp/native_link/data/content_path-index", - "temp_path": "/tmp/native_link/data/tmp_path-index", + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", "eviction_policy": { // 500mb. "max_bytes": 500000000, @@ -40,8 +40,8 @@ "fast_slow": { "fast": { "filesystem": { - "content_path": "/tmp/native_link/data/content_path-content", - "temp_path": "/tmp/native_link/data/tmp_path-content", + "content_path": "/tmp/nativelink/data/content_path-content", + "temp_path": "/tmp/nativelink/data/tmp_path-content", "eviction_policy": { // 2gb. "max_bytes": 2000000000, @@ -81,8 +81,8 @@ } }, "filesystem": { - "content_path": "/tmp/native_link/data/content_path-ac", - "temp_path": "/tmp/native_link/data/tmp_path-ac", + "content_path": "/tmp/nativelink/data/content_path-ac", + "temp_path": "/tmp/nativelink/data/tmp_path-ac", "eviction_policy": { // 500mb. "max_bytes": 500000000, diff --git a/native-link-config/src/cas_server.rs b/nativelink-config/src/cas_server.rs similarity index 99% rename from native-link-config/src/cas_server.rs rename to nativelink-config/src/cas_server.rs index 6a54f3b62..c9b007ada 100644 --- a/native-link-config/src/cas_server.rs +++ b/nativelink-config/src/cas_server.rs @@ -53,7 +53,7 @@ pub struct CompressionConfig { /// responses to clients. Enabling this will likely save a lot of /// data transfer, but will consume a lot of CPU and add a lot of /// latency. - /// see: + /// see: /// /// Default: CompressionAlgorithm::None pub send_compression_algorithm: Option, @@ -63,7 +63,7 @@ pub struct CompressionConfig { /// clients and the client will choose which compression algorithm to /// use. Enabling this will likely save a lot of data transfer, but /// will consume a lot of CPU and add a lot of latency. - /// see: + /// see: /// /// Defaults: {no supported compression} pub accepted_compression_algorithms: Vec, @@ -553,7 +553,7 @@ pub struct GlobalConfig { /// are collected at runtime (performance metrics) from being tallied. The /// overhead of collecting metrics is very low, so this flag should only be /// used if there is a very good reason to disable metrics. - /// This flag can be forcably set using the `NATIVE_LINK_DISABLE_METRICS` variable. + /// This flag can be forcably set using the `nativelink_DISABLE_METRICS` variable. /// If the variable is set it will always disable metrics regardless of what /// this flag is set to. /// diff --git a/native-link-config/src/lib.rs b/nativelink-config/src/lib.rs similarity index 100% rename from native-link-config/src/lib.rs rename to nativelink-config/src/lib.rs diff --git a/native-link-config/src/schedulers.rs b/nativelink-config/src/schedulers.rs similarity index 100% rename from native-link-config/src/schedulers.rs rename to nativelink-config/src/schedulers.rs diff --git a/native-link-config/src/serde_utils.rs b/nativelink-config/src/serde_utils.rs similarity index 100% rename from native-link-config/src/serde_utils.rs rename to nativelink-config/src/serde_utils.rs diff --git a/native-link-config/src/stores.rs b/nativelink-config/src/stores.rs similarity index 100% rename from native-link-config/src/stores.rs rename to nativelink-config/src/stores.rs diff --git a/native-link-scheduler/BUILD.bazel b/nativelink-scheduler/BUILD.bazel similarity index 84% rename from native-link-scheduler/BUILD.bazel rename to nativelink-scheduler/BUILD.bazel index e4a107093..cb3f3583c 100644 --- a/native-link-scheduler/BUILD.bazel +++ b/nativelink-scheduler/BUILD.bazel @@ -7,7 +7,7 @@ load( ) rust_library( - name = "native-link-scheduler", + name = "nativelink-scheduler", srcs = [ "src/action_scheduler.rs", "src/cache_lookup_scheduler.rs", @@ -26,9 +26,9 @@ rust_library( visibility = ["//visibility:public"], deps = [ "//error", - "//native-link-config", - "//native-link-store", - "//native-link-util", + "//nativelink-config", + "//nativelink-store", + "//nativelink-util", "//proto", "@crate_index//:blake3", "@crate_index//:futures", @@ -62,11 +62,11 @@ rust_test_suite( "@crate_index//:async-trait", ], deps = [ - ":native-link-scheduler", + ":nativelink-scheduler", "//error", - "//native-link-config", - "//native-link-store", - "//native-link-util", + "//nativelink-config", + "//nativelink-store", + "//nativelink-util", "//proto", "@crate_index//:futures", "@crate_index//:pretty_assertions", @@ -78,10 +78,10 @@ rust_test_suite( rust_doc( name = "docs", - crate = ":native-link-scheduler", + crate = ":nativelink-scheduler", ) rust_doc_test( name = "doc_test", - crate = ":native-link-scheduler", + crate = ":nativelink-scheduler", ) diff --git a/native-link-scheduler/Cargo.toml b/nativelink-scheduler/Cargo.toml similarity index 78% rename from native-link-scheduler/Cargo.toml rename to nativelink-scheduler/Cargo.toml index 5702d5ea0..b104a7a25 100644 --- a/native-link-scheduler/Cargo.toml +++ b/nativelink-scheduler/Cargo.toml @@ -1,17 +1,17 @@ [package] -name = "native-link-scheduler" +name = "nativelink-scheduler" version = "0.0.0" edition = "2021" [dependencies] error = { path = "../error" } -native-link-config = { path = "../native-link-config" } -native-link-util = { path = "../native-link-util" } +nativelink-config = { path = "../nativelink-config" } +nativelink-util = { path = "../nativelink-util" } proto = { path = "../proto" } # TODO(aaronmondal): This should not be a dependency. Move the corresponding # files somewhere else. -native-link-store = { path = "../native-link-store" } +nativelink-store = { path = "../nativelink-store" } async-trait = "0.1.74" blake3 = "1.5.0" diff --git a/native-link-scheduler/src/action_scheduler.rs b/nativelink-scheduler/src/action_scheduler.rs similarity index 92% rename from native-link-scheduler/src/action_scheduler.rs rename to nativelink-scheduler/src/action_scheduler.rs index a4075ab7b..d22461956 100644 --- a/native-link-scheduler/src/action_scheduler.rs +++ b/nativelink-scheduler/src/action_scheduler.rs @@ -16,8 +16,8 @@ use std::sync::Arc; use async_trait::async_trait; use error::Error; -use native_link_util::action_messages::{ActionInfo, ActionInfoHashKey, ActionState}; -use native_link_util::metrics_utils::Registry; +use nativelink_util::action_messages::{ActionInfo, ActionInfoHashKey, ActionState}; +use nativelink_util::metrics_utils::Registry; use tokio::sync::watch; use crate::platform_property_manager::PlatformPropertyManager; diff --git a/native-link-scheduler/src/cache_lookup_scheduler.rs b/nativelink-scheduler/src/cache_lookup_scheduler.rs similarity index 96% rename from native-link-scheduler/src/cache_lookup_scheduler.rs rename to nativelink-scheduler/src/cache_lookup_scheduler.rs index 5d2633e3b..dac666c95 100644 --- a/native-link-scheduler/src/cache_lookup_scheduler.rs +++ b/nativelink-scheduler/src/cache_lookup_scheduler.rs @@ -19,11 +19,11 @@ use std::sync::Arc; use async_trait::async_trait; use error::Error; use futures::stream::StreamExt; -use native_link_store::ac_utils::get_and_decode_digest; -use native_link_store::grpc_store::GrpcStore; -use native_link_util::action_messages::{ActionInfo, ActionInfoHashKey, ActionResult, ActionStage, ActionState}; -use native_link_util::common::DigestInfo; -use native_link_util::store_trait::Store; +use nativelink_store::ac_utils::get_and_decode_digest; +use nativelink_store::grpc_store::GrpcStore; +use nativelink_util::action_messages::{ActionInfo, ActionInfoHashKey, ActionResult, ActionStage, ActionState}; +use nativelink_util::common::DigestInfo; +use nativelink_util::store_trait::Store; use parking_lot::{Mutex, MutexGuard}; use proto::build::bazel::remote::execution::v2::{ digest_function, ActionResult as ProtoActionResult, GetActionResultRequest, diff --git a/native-link-scheduler/src/default_scheduler_factory.rs b/nativelink-scheduler/src/default_scheduler_factory.rs similarity index 97% rename from native-link-scheduler/src/default_scheduler_factory.rs rename to nativelink-scheduler/src/default_scheduler_factory.rs index cdd05cb11..8d9724d9e 100644 --- a/native-link-scheduler/src/default_scheduler_factory.rs +++ b/nativelink-scheduler/src/default_scheduler_factory.rs @@ -17,9 +17,9 @@ use std::sync::Arc; use std::time::Duration; use error::{Error, ResultExt}; -use native_link_config::schedulers::SchedulerConfig; -use native_link_store::store_manager::StoreManager; -use native_link_util::metrics_utils::Registry; +use nativelink_config::schedulers::SchedulerConfig; +use nativelink_store::store_manager::StoreManager; +use nativelink_util::metrics_utils::Registry; use tokio::time::interval; use crate::action_scheduler::ActionScheduler; diff --git a/native-link-scheduler/src/grpc_scheduler.rs b/nativelink-scheduler/src/grpc_scheduler.rs similarity index 94% rename from native-link-scheduler/src/grpc_scheduler.rs rename to nativelink-scheduler/src/grpc_scheduler.rs index 78255d9f3..b7bdfbc86 100644 --- a/native-link-scheduler/src/grpc_scheduler.rs +++ b/nativelink-scheduler/src/grpc_scheduler.rs @@ -21,8 +21,8 @@ use async_trait::async_trait; use error::{make_err, Code, Error, ResultExt}; use futures::stream::unfold; use futures::TryFutureExt; -use native_link_util::action_messages::{ActionInfo, ActionInfoHashKey, ActionState, DEFAULT_EXECUTION_PRIORITY}; -use native_link_util::retry::{ExponentialBackoff, Retrier, RetryResult}; +use nativelink_util::action_messages::{ActionInfo, ActionInfoHashKey, ActionState, DEFAULT_EXECUTION_PRIORITY}; +use nativelink_util::retry::{ExponentialBackoff, Retrier, RetryResult}; use parking_lot::Mutex; use proto::build::bazel::remote::execution::v2::capabilities_client::CapabilitiesClient; use proto::build::bazel::remote::execution::v2::execution_client::ExecutionClient; @@ -46,12 +46,12 @@ pub struct GrpcScheduler { execution_client: ExecutionClient, platform_property_managers: Mutex>>, jitter_fn: Box Duration + Send + Sync>, - retry: native_link_config::stores::Retry, + retry: nativelink_config::stores::Retry, retrier: Retrier, } impl GrpcScheduler { - pub fn new(config: &native_link_config::schedulers::GrpcScheduler) -> Result { + pub fn new(config: &nativelink_config::schedulers::GrpcScheduler) -> Result { let jitter_amt = config.retry.jitter; Self::new_with_jitter( config, @@ -67,7 +67,7 @@ impl GrpcScheduler { } pub fn new_with_jitter( - config: &native_link_config::schedulers::GrpcScheduler, + config: &nativelink_config::schedulers::GrpcScheduler, jitter_fn: Box Duration + Send + Sync>, ) -> Result { let endpoint = transport::Channel::balance_list(std::iter::once( @@ -173,7 +173,7 @@ impl ActionScheduler for GrpcScheduler { .err_tip(|| "Unable to get execution properties in GrpcScheduler")? .supported_node_properties .iter() - .map(|property| (property.clone(), native_link_config::schedulers::PropertyType::Exact)) + .map(|property| (property.clone(), nativelink_config::schedulers::PropertyType::Exact)) .collect(), )); diff --git a/native-link-scheduler/src/lib.rs b/nativelink-scheduler/src/lib.rs similarity index 100% rename from native-link-scheduler/src/lib.rs rename to nativelink-scheduler/src/lib.rs diff --git a/native-link-scheduler/src/platform_property_manager.rs b/nativelink-scheduler/src/platform_property_manager.rs similarity index 95% rename from native-link-scheduler/src/platform_property_manager.rs rename to nativelink-scheduler/src/platform_property_manager.rs index 967eb03f8..6aec19b1d 100644 --- a/native-link-scheduler/src/platform_property_manager.rs +++ b/nativelink-scheduler/src/platform_property_manager.rs @@ -15,8 +15,8 @@ use std::collections::HashMap; use error::{make_input_err, Code, Error, ResultExt}; -use native_link_config::schedulers::PropertyType; -use native_link_util::platform_properties::PlatformPropertyValue; +use nativelink_config::schedulers::PropertyType; +use nativelink_util::platform_properties::PlatformPropertyValue; /// Helps manage known properties and conversion into `PlatformPropertyValue`. pub struct PlatformPropertyManager { diff --git a/native-link-scheduler/src/property_modifier_scheduler.rs b/nativelink-scheduler/src/property_modifier_scheduler.rs similarity index 93% rename from native-link-scheduler/src/property_modifier_scheduler.rs rename to nativelink-scheduler/src/property_modifier_scheduler.rs index 4a09114a6..3741f776b 100644 --- a/native-link-scheduler/src/property_modifier_scheduler.rs +++ b/nativelink-scheduler/src/property_modifier_scheduler.rs @@ -18,8 +18,8 @@ use std::sync::Arc; use async_trait::async_trait; use error::{Error, ResultExt}; -use native_link_config::schedulers::{PropertyModification, PropertyType}; -use native_link_util::action_messages::{ActionInfo, ActionInfoHashKey, ActionState}; +use nativelink_config::schedulers::{PropertyModification, PropertyType}; +use nativelink_util::action_messages::{ActionInfo, ActionInfoHashKey, ActionState}; use parking_lot::Mutex; use tokio::sync::watch; @@ -27,14 +27,14 @@ use crate::action_scheduler::ActionScheduler; use crate::platform_property_manager::PlatformPropertyManager; pub struct PropertyModifierScheduler { - modifications: Vec, + modifications: Vec, scheduler: Arc, property_managers: Mutex>>, } impl PropertyModifierScheduler { pub fn new( - config: &native_link_config::schedulers::PropertyModifierScheduler, + config: &nativelink_config::schedulers::PropertyModifierScheduler, scheduler: Arc, ) -> Self { Self { diff --git a/native-link-scheduler/src/simple_scheduler.rs b/nativelink-scheduler/src/simple_scheduler.rs similarity index 99% rename from native-link-scheduler/src/simple_scheduler.rs rename to nativelink-scheduler/src/simple_scheduler.rs index bef18343f..55a817608 100644 --- a/native-link-scheduler/src/simple_scheduler.rs +++ b/nativelink-scheduler/src/simple_scheduler.rs @@ -25,14 +25,14 @@ use error::{error_if, make_err, make_input_err, Code, Error, ResultExt}; use futures::Future; use hashbrown::{HashMap, HashSet}; use lru::LruCache; -use native_link_config::schedulers::WorkerAllocationStrategy; -use native_link_util::action_messages::{ +use nativelink_config::schedulers::WorkerAllocationStrategy; +use nativelink_util::action_messages::{ ActionInfo, ActionInfoHashKey, ActionResult, ActionStage, ActionState, ExecutionMetadata, }; -use native_link_util::metrics_utils::{ +use nativelink_util::metrics_utils::{ AsyncCounterWrapper, Collector, CollectorState, CounterWithTime, FuncCounterWrapper, MetricsComponent, Registry, }; -use native_link_util::platform_properties::PlatformPropertyValue; +use nativelink_util::platform_properties::PlatformPropertyValue; use parking_lot::{Mutex, MutexGuard}; use tokio::sync::{watch, Notify}; use tokio::task::JoinHandle; @@ -601,7 +601,7 @@ pub struct SimpleScheduler { impl SimpleScheduler { #[inline] #[must_use] - pub fn new(scheduler_cfg: &native_link_config::schedulers::SimpleScheduler) -> Self { + pub fn new(scheduler_cfg: &nativelink_config::schedulers::SimpleScheduler) -> Self { Self::new_with_callback(scheduler_cfg, || { // The cost of running `do_try_match()` is very high, but constant // in relation to the number of changes that have happened. This means @@ -615,7 +615,7 @@ impl SimpleScheduler { } pub fn new_with_callback + Send, F: Fn() -> Fut + Send + Sync + 'static>( - scheduler_cfg: &native_link_config::schedulers::SimpleScheduler, + scheduler_cfg: &nativelink_config::schedulers::SimpleScheduler, on_matching_engine_run: F, ) -> Self { let platform_property_manager = Arc::new(PlatformPropertyManager::new( diff --git a/native-link-scheduler/src/worker.rs b/nativelink-scheduler/src/worker.rs similarity index 97% rename from native-link-scheduler/src/worker.rs rename to nativelink-scheduler/src/worker.rs index 7656c8199..4096cabaa 100644 --- a/native-link-scheduler/src/worker.rs +++ b/nativelink-scheduler/src/worker.rs @@ -18,10 +18,10 @@ use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; use error::{make_err, make_input_err, Code, Error, ResultExt}; -use native_link_util::action_messages::ActionInfo; -use native_link_util::metrics_utils::{CollectorState, CounterWithTime, FuncCounterWrapper, MetricsComponent}; -use native_link_util::platform_properties::{PlatformProperties, PlatformPropertyValue}; -use proto::com::github::trace_machina::native_link::remote_execution::{ +use nativelink_util::action_messages::ActionInfo; +use nativelink_util::metrics_utils::{CollectorState, CounterWithTime, FuncCounterWrapper, MetricsComponent}; +use nativelink_util::platform_properties::{PlatformProperties, PlatformPropertyValue}; +use proto::com::github::trace_machina::nativelink::remote_execution::{ update_for_worker, ConnectionResult, StartExecute, UpdateForWorker, }; use tokio::sync::mpsc::UnboundedSender; diff --git a/native-link-scheduler/src/worker_scheduler.rs b/nativelink-scheduler/src/worker_scheduler.rs similarity index 95% rename from native-link-scheduler/src/worker_scheduler.rs rename to nativelink-scheduler/src/worker_scheduler.rs index aaf5ce45a..8ea55edf3 100644 --- a/native-link-scheduler/src/worker_scheduler.rs +++ b/nativelink-scheduler/src/worker_scheduler.rs @@ -16,8 +16,8 @@ use std::sync::Arc; use async_trait::async_trait; use error::Error; -use native_link_util::action_messages::{ActionInfoHashKey, ActionStage}; -use native_link_util::metrics_utils::Registry; +use nativelink_util::action_messages::{ActionInfoHashKey, ActionStage}; +use nativelink_util::metrics_utils::Registry; use crate::platform_property_manager::PlatformPropertyManager; use crate::worker::{Worker, WorkerId, WorkerTimestamp}; diff --git a/native-link-scheduler/tests/action_messages_test.rs b/nativelink-scheduler/tests/action_messages_test.rs similarity index 97% rename from native-link-scheduler/tests/action_messages_test.rs rename to nativelink-scheduler/tests/action_messages_test.rs index 5434fc521..0bcc5c5ec 100644 --- a/native-link-scheduler/tests/action_messages_test.rs +++ b/nativelink-scheduler/tests/action_messages_test.rs @@ -17,12 +17,12 @@ use std::sync::Arc; use std::time::{Duration, SystemTime}; use error::Error; -use native_link_util::action_messages::{ +use nativelink_util::action_messages::{ ActionInfo, ActionInfoHashKey, ActionResult, ActionStage, ActionState, ExecutionMetadata, }; -use native_link_util::common::DigestInfo; -use native_link_util::digest_hasher::DigestHasherFunc; -use native_link_util::platform_properties::PlatformProperties; +use nativelink_util::common::DigestInfo; +use nativelink_util::digest_hasher::DigestHasherFunc; +use nativelink_util::platform_properties::PlatformProperties; use proto::build::bazel::remote::execution::v2::ExecuteResponse; use proto::google::longrunning::{operation, Operation}; use proto::google::rpc::Status; diff --git a/native-link-scheduler/tests/cache_lookup_scheduler_test.rs b/nativelink-scheduler/tests/cache_lookup_scheduler_test.rs similarity index 91% rename from native-link-scheduler/tests/cache_lookup_scheduler_test.rs rename to nativelink-scheduler/tests/cache_lookup_scheduler_test.rs index 7a721b2d6..714f9217f 100644 --- a/native-link-scheduler/tests/cache_lookup_scheduler_test.rs +++ b/nativelink-scheduler/tests/cache_lookup_scheduler_test.rs @@ -24,13 +24,13 @@ mod utils { use error::{Error, ResultExt}; use futures::join; -use native_link_scheduler::action_scheduler::ActionScheduler; -use native_link_scheduler::cache_lookup_scheduler::CacheLookupScheduler; -use native_link_scheduler::platform_property_manager::PlatformPropertyManager; -use native_link_store::memory_store::MemoryStore; -use native_link_util::action_messages::{ActionInfoHashKey, ActionResult, ActionStage, ActionState, DirectoryInfo}; -use native_link_util::common::DigestInfo; -use native_link_util::store_trait::Store; +use nativelink_scheduler::action_scheduler::ActionScheduler; +use nativelink_scheduler::cache_lookup_scheduler::CacheLookupScheduler; +use nativelink_scheduler::platform_property_manager::PlatformPropertyManager; +use nativelink_store::memory_store::MemoryStore; +use nativelink_util::action_messages::{ActionInfoHashKey, ActionResult, ActionStage, ActionState, DirectoryInfo}; +use nativelink_util::common::DigestInfo; +use nativelink_util::store_trait::Store; use prost::Message; use proto::build::bazel::remote::execution::v2::ActionResult as ProtoActionResult; use tokio::sync::watch; @@ -48,8 +48,8 @@ struct TestContext { fn make_cache_scheduler() -> Result { let mock_scheduler = Arc::new(MockActionScheduler::new()); - let cas_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); - let ac_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); + let cas_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); + let ac_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); let cache_scheduler = CacheLookupScheduler::new(cas_store, ac_store.clone(), mock_scheduler.clone())?; Ok(TestContext { mock_scheduler, diff --git a/native-link-scheduler/tests/property_modifier_scheduler_test.rs b/nativelink-scheduler/tests/property_modifier_scheduler_test.rs similarity index 93% rename from native-link-scheduler/tests/property_modifier_scheduler_test.rs rename to nativelink-scheduler/tests/property_modifier_scheduler_test.rs index 1572361a7..ea623f85f 100644 --- a/native-link-scheduler/tests/property_modifier_scheduler_test.rs +++ b/nativelink-scheduler/tests/property_modifier_scheduler_test.rs @@ -24,13 +24,13 @@ mod utils { use error::Error; use futures::join; -use native_link_config::schedulers::{PlatformPropertyAddition, PropertyModification, PropertyType}; -use native_link_scheduler::action_scheduler::ActionScheduler; -use native_link_scheduler::platform_property_manager::PlatformPropertyManager; -use native_link_scheduler::property_modifier_scheduler::PropertyModifierScheduler; -use native_link_util::action_messages::{ActionInfoHashKey, ActionStage, ActionState}; -use native_link_util::common::DigestInfo; -use native_link_util::platform_properties::PlatformPropertyValue; +use nativelink_config::schedulers::{PlatformPropertyAddition, PropertyModification, PropertyType}; +use nativelink_scheduler::action_scheduler::ActionScheduler; +use nativelink_scheduler::platform_property_manager::PlatformPropertyManager; +use nativelink_scheduler::property_modifier_scheduler::PropertyModifierScheduler; +use nativelink_util::action_messages::{ActionInfoHashKey, ActionStage, ActionState}; +use nativelink_util::common::DigestInfo; +use nativelink_util::platform_properties::PlatformPropertyValue; use tokio::sync::watch; use utils::mock_scheduler::MockActionScheduler; use utils::scheduler_utils::{make_base_action_info, INSTANCE_NAME}; @@ -42,10 +42,10 @@ struct TestContext { fn make_modifier_scheduler(modifications: Vec) -> TestContext { let mock_scheduler = Arc::new(MockActionScheduler::new()); - let config = native_link_config::schedulers::PropertyModifierScheduler { + let config = nativelink_config::schedulers::PropertyModifierScheduler { modifications, - scheduler: Box::new(native_link_config::schedulers::SchedulerConfig::simple( - native_link_config::schedulers::SimpleScheduler::default(), + scheduler: Box::new(nativelink_config::schedulers::SchedulerConfig::simple( + nativelink_config::schedulers::SimpleScheduler::default(), )), }; let modifier_scheduler = PropertyModifierScheduler::new(&config, mock_scheduler.clone()); diff --git a/native-link-scheduler/tests/simple_scheduler_test.rs b/nativelink-scheduler/tests/simple_scheduler_test.rs similarity index 97% rename from native-link-scheduler/tests/simple_scheduler_test.rs rename to nativelink-scheduler/tests/simple_scheduler_test.rs index 927e435c1..cfa713931 100644 --- a/native-link-scheduler/tests/simple_scheduler_test.rs +++ b/nativelink-scheduler/tests/simple_scheduler_test.rs @@ -18,21 +18,21 @@ use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use error::{make_err, Code, Error, ResultExt}; -use native_link_scheduler::action_scheduler::ActionScheduler; -use native_link_util::action_messages::{ +use nativelink_scheduler::action_scheduler::ActionScheduler; +use nativelink_util::action_messages::{ ActionInfoHashKey, ActionResult, ActionStage, ActionState, DirectoryInfo, ExecutionMetadata, FileInfo, NameOrPath, SymlinkInfo, INTERNAL_ERROR_EXIT_CODE, }; -use native_link_util::platform_properties::{PlatformProperties, PlatformPropertyValue}; +use nativelink_util::platform_properties::{PlatformProperties, PlatformPropertyValue}; mod utils { pub(crate) mod scheduler_utils; } -use native_link_scheduler::simple_scheduler::SimpleScheduler; -use native_link_scheduler::worker::{Worker, WorkerId}; -use native_link_scheduler::worker_scheduler::WorkerScheduler; -use native_link_util::common::DigestInfo; +use nativelink_scheduler::simple_scheduler::SimpleScheduler; +use nativelink_scheduler::worker::{Worker, WorkerId}; +use nativelink_scheduler::worker_scheduler::WorkerScheduler; +use nativelink_util::common::DigestInfo; use proto::build::bazel::remote::execution::v2::{digest_function, ExecuteRequest}; -use proto::com::github::trace_machina::native_link::remote_execution::{ +use proto::com::github::trace_machina::nativelink::remote_execution::{ update_for_worker, ConnectionResult, StartExecute, UpdateForWorker, }; use tokio::sync::{mpsc, watch}; @@ -98,7 +98,7 @@ mod scheduler_tests { const WORKER_ID: WorkerId = WorkerId(0x1234_5678_9111); let scheduler = SimpleScheduler::new_with_callback( - &native_link_config::schedulers::SimpleScheduler::default(), + &nativelink_config::schedulers::SimpleScheduler::default(), || async move {}, ); let action_digest = DigestInfo::new([99u8; 32], 512); @@ -150,7 +150,7 @@ mod scheduler_tests { const WORKER_ID: WorkerId = WorkerId(0x1234_5678_9111); let scheduler = SimpleScheduler::new_with_callback( - &native_link_config::schedulers::SimpleScheduler::default(), + &nativelink_config::schedulers::SimpleScheduler::default(), || async move {}, ); let action_digest = DigestInfo::new([99u8; 32], 512); @@ -210,7 +210,7 @@ mod scheduler_tests { const WORKER_ID1: WorkerId = WorkerId(0x0011_1111); const WORKER_ID2: WorkerId = WorkerId(0x0022_2222); let scheduler = SimpleScheduler::new_with_callback( - &native_link_config::schedulers::SimpleScheduler { + &nativelink_config::schedulers::SimpleScheduler { worker_timeout_s: WORKER_TIMEOUT_S, ..Default::default() }, @@ -354,7 +354,7 @@ mod scheduler_tests { const WORKER_ID: WorkerId = WorkerId(0x1234_5678_9111); let scheduler = SimpleScheduler::new_with_callback( - &native_link_config::schedulers::SimpleScheduler::default(), + &nativelink_config::schedulers::SimpleScheduler::default(), || async move {}, ); let action_digest = DigestInfo::new([99u8; 32], 512); @@ -428,7 +428,7 @@ mod scheduler_tests { const WORKER_ID2: WorkerId = WorkerId(0x0010_0002); let scheduler = SimpleScheduler::new_with_callback( - &native_link_config::schedulers::SimpleScheduler::default(), + &nativelink_config::schedulers::SimpleScheduler::default(), || async move {}, ); let action_digest = DigestInfo::new([99u8; 32], 512); @@ -498,7 +498,7 @@ mod scheduler_tests { const WORKER_ID: WorkerId = WorkerId(0x0010_0009); let scheduler = SimpleScheduler::new_with_callback( - &native_link_config::schedulers::SimpleScheduler::default(), + &nativelink_config::schedulers::SimpleScheduler::default(), || async move {}, ); let action_digest = DigestInfo::new([99u8; 32], 512); @@ -589,7 +589,7 @@ mod scheduler_tests { async fn worker_disconnects_does_not_schedule_for_execution_test() -> Result<(), Error> { const WORKER_ID: WorkerId = WorkerId(0x0010_0010); let scheduler = SimpleScheduler::new_with_callback( - &native_link_config::schedulers::SimpleScheduler::default(), + &nativelink_config::schedulers::SimpleScheduler::default(), || async move {}, ); let action_digest = DigestInfo::new([99u8; 32], 512); @@ -626,7 +626,7 @@ mod scheduler_tests { const WORKER_ID1: WorkerId = WorkerId(0x0011_1111); const WORKER_ID2: WorkerId = WorkerId(0x0022_2222); let scheduler = SimpleScheduler::new_with_callback( - &native_link_config::schedulers::SimpleScheduler { + &nativelink_config::schedulers::SimpleScheduler { worker_timeout_s: WORKER_TIMEOUT_S, ..Default::default() }, @@ -724,7 +724,7 @@ mod scheduler_tests { const WORKER_ID: WorkerId = WorkerId(0x1234_5678_9111); let scheduler = SimpleScheduler::new_with_callback( - &native_link_config::schedulers::SimpleScheduler::default(), + &nativelink_config::schedulers::SimpleScheduler::default(), || async move {}, ); let action_digest = DigestInfo::new([99u8; 32], 512); @@ -823,7 +823,7 @@ mod scheduler_tests { const WORKER_ID: WorkerId = WorkerId(0x1234_5678_9111); let scheduler = SimpleScheduler::new_with_callback( - &native_link_config::schedulers::SimpleScheduler::default(), + &nativelink_config::schedulers::SimpleScheduler::default(), || async move {}, ); let action_digest = DigestInfo::new([99u8; 32], 512); @@ -925,7 +925,7 @@ mod scheduler_tests { const ROGUE_WORKER_ID: WorkerId = WorkerId(0x0009_8765_4321); let scheduler = SimpleScheduler::new_with_callback( - &native_link_config::schedulers::SimpleScheduler::default(), + &nativelink_config::schedulers::SimpleScheduler::default(), || async move {}, ); let action_digest = DigestInfo::new([99u8; 32], 512); @@ -1018,7 +1018,7 @@ mod scheduler_tests { const WORKER_ID: WorkerId = WorkerId(0x0010_000f); let scheduler = SimpleScheduler::new_with_callback( - &native_link_config::schedulers::SimpleScheduler::default(), + &nativelink_config::schedulers::SimpleScheduler::default(), || async move {}, ); let action_digest = DigestInfo::new([99u8; 32], 512); @@ -1142,7 +1142,7 @@ mod scheduler_tests { const WORKER_ID: WorkerId = WorkerId(0x1234_5678_9111); let scheduler = SimpleScheduler::new_with_callback( - &native_link_config::schedulers::SimpleScheduler::default(), + &nativelink_config::schedulers::SimpleScheduler::default(), || async move {}, ); let action_digest1 = DigestInfo::new([11u8; 32], 512); @@ -1279,7 +1279,7 @@ mod scheduler_tests { const WORKER_ID: WorkerId = WorkerId(0x1234_5678_9111); let scheduler = SimpleScheduler::new_with_callback( - &native_link_config::schedulers::SimpleScheduler::default(), + &nativelink_config::schedulers::SimpleScheduler::default(), || async move {}, ); let action_digest1 = DigestInfo::new([11u8; 32], 512); @@ -1330,7 +1330,7 @@ mod scheduler_tests { const WORKER_ID: WorkerId = WorkerId(0x1234_5678_9111); let scheduler = SimpleScheduler::new_with_callback( - &native_link_config::schedulers::SimpleScheduler { + &nativelink_config::schedulers::SimpleScheduler { max_job_retries: 2, ..Default::default() }, @@ -1460,7 +1460,7 @@ mod scheduler_tests { // inner spawn was dropped because our callback would be dropped, which dropps our // DropChecker. let scheduler = SimpleScheduler::new_with_callback( - &native_link_config::schedulers::SimpleScheduler::default(), + &nativelink_config::schedulers::SimpleScheduler::default(), move || { // This will ensure dropping happens if this function is ever dropped. let _drop_checker = drop_checker.clone(); @@ -1478,14 +1478,14 @@ mod scheduler_tests { Ok(()) } - /// Regression test for: https://github.com/TraceMachina/native-link/issues/257. + /// Regression test for: https://github.com/TraceMachina/nativelink/issues/257. #[tokio::test] async fn ensure_task_or_worker_change_notification_received_test() -> Result<(), Error> { const WORKER_ID1: WorkerId = WorkerId(0x0011_1111); const WORKER_ID2: WorkerId = WorkerId(0x0022_2222); let scheduler = SimpleScheduler::new_with_callback( - &native_link_config::schedulers::SimpleScheduler::default(), + &nativelink_config::schedulers::SimpleScheduler::default(), || async move {}, ); let action_digest = DigestInfo::new([99u8; 32], 512); diff --git a/native-link-scheduler/tests/utils/mock_scheduler.rs b/nativelink-scheduler/tests/utils/mock_scheduler.rs similarity index 96% rename from native-link-scheduler/tests/utils/mock_scheduler.rs rename to nativelink-scheduler/tests/utils/mock_scheduler.rs index 13b89fa8f..979914eb7 100644 --- a/native-link-scheduler/tests/utils/mock_scheduler.rs +++ b/nativelink-scheduler/tests/utils/mock_scheduler.rs @@ -16,9 +16,9 @@ use std::sync::Arc; use async_trait::async_trait; use error::{make_input_err, Error}; -use native_link_scheduler::action_scheduler::ActionScheduler; -use native_link_scheduler::platform_property_manager::PlatformPropertyManager; -use native_link_util::action_messages::{ActionInfo, ActionInfoHashKey, ActionState}; +use nativelink_scheduler::action_scheduler::ActionScheduler; +use nativelink_scheduler::platform_property_manager::PlatformPropertyManager; +use nativelink_util::action_messages::{ActionInfo, ActionInfoHashKey, ActionState}; use tokio::sync::{mpsc, watch, Mutex}; #[allow(clippy::large_enum_variant)] diff --git a/native-link-scheduler/tests/utils/scheduler_utils.rs b/nativelink-scheduler/tests/utils/scheduler_utils.rs similarity index 86% rename from native-link-scheduler/tests/utils/scheduler_utils.rs rename to nativelink-scheduler/tests/utils/scheduler_utils.rs index eb7081904..e4504bdab 100644 --- a/native-link-scheduler/tests/utils/scheduler_utils.rs +++ b/nativelink-scheduler/tests/utils/scheduler_utils.rs @@ -15,10 +15,10 @@ use std::collections::HashMap; use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use native_link_util::action_messages::{ActionInfo, ActionInfoHashKey}; -use native_link_util::common::DigestInfo; -use native_link_util::digest_hasher::DigestHasherFunc; -use native_link_util::platform_properties::PlatformProperties; +use nativelink_util::action_messages::{ActionInfo, ActionInfoHashKey}; +use nativelink_util::common::DigestInfo; +use nativelink_util::digest_hasher::DigestHasherFunc; +use nativelink_util::platform_properties::PlatformProperties; pub const INSTANCE_NAME: &str = "foobar_instance_name"; diff --git a/native-link-service/BUILD.bazel b/nativelink-service/BUILD.bazel similarity index 80% rename from native-link-service/BUILD.bazel rename to nativelink-service/BUILD.bazel index 1ca2d0c18..241d35f67 100644 --- a/native-link-service/BUILD.bazel +++ b/nativelink-service/BUILD.bazel @@ -7,7 +7,7 @@ load( ) rust_library( - name = "native-link-service", + name = "nativelink-service", srcs = [ "src/ac_server.rs", "src/bytestream_server.rs", @@ -20,10 +20,10 @@ rust_library( visibility = ["//visibility:public"], deps = [ "//error", - "//native-link-config", - "//native-link-scheduler", - "//native-link-store", - "//native-link-util", + "//nativelink-config", + "//nativelink-scheduler", + "//nativelink-store", + "//nativelink-util", "//proto", "@crate_index//:bytes", "@crate_index//:futures", @@ -49,11 +49,11 @@ rust_test_suite( ], deps = [ "//error", - "//native-link-config", - "//native-link-scheduler", - "//native-link-service", - "//native-link-store", - "//native-link-util", + "//nativelink-config", + "//nativelink-scheduler", + "//nativelink-service", + "//nativelink-store", + "//nativelink-util", "//proto", "@crate_index//:bytes", "@crate_index//:futures", @@ -71,10 +71,10 @@ rust_test_suite( rust_doc( name = "docs", - crate = ":native-link-service", + crate = ":nativelink-service", ) rust_doc_test( name = "doc_test", - crate = ":native-link-service", + crate = ":nativelink-service", ) diff --git a/native-link-service/Cargo.toml b/nativelink-service/Cargo.toml similarity index 70% rename from native-link-service/Cargo.toml rename to nativelink-service/Cargo.toml index 51524454c..0493f91e3 100644 --- a/native-link-service/Cargo.toml +++ b/nativelink-service/Cargo.toml @@ -1,15 +1,15 @@ [package] -name = "native-link-service" +name = "nativelink-service" version = "0.0.0" edition = "2021" [dependencies] proto = { path = "../proto" } error = { path = "../error" } -native-link-config = { path = "../native-link-config" } -native-link-util = { path = "../native-link-util" } -native-link-store = { path = "../native-link-store" } -native-link-scheduler = { path = "../native-link-scheduler" } +nativelink-config = { path = "../nativelink-config" } +nativelink-util = { path = "../nativelink-util" } +nativelink-store = { path = "../nativelink-store" } +nativelink-scheduler = { path = "../nativelink-scheduler" } bytes = "1.5.0" futures = "0.3.29" diff --git a/native-link-service/src/ac_server.rs b/nativelink-service/src/ac_server.rs similarity index 95% rename from native-link-service/src/ac_server.rs rename to nativelink-service/src/ac_server.rs index 06471fd08..433e05042 100644 --- a/native-link-service/src/ac_server.rs +++ b/nativelink-service/src/ac_server.rs @@ -20,12 +20,12 @@ use std::time::Instant; use bytes::BytesMut; use error::{make_input_err, Code, Error, ResultExt}; -use native_link_config::cas_server::{AcStoreConfig, InstanceName}; -use native_link_store::ac_utils::{get_and_decode_digest, ESTIMATED_DIGEST_SIZE}; -use native_link_store::grpc_store::GrpcStore; -use native_link_store::store_manager::StoreManager; -use native_link_util::common::DigestInfo; -use native_link_util::store_trait::Store; +use nativelink_config::cas_server::{AcStoreConfig, InstanceName}; +use nativelink_store::ac_utils::{get_and_decode_digest, ESTIMATED_DIGEST_SIZE}; +use nativelink_store::grpc_store::GrpcStore; +use nativelink_store::store_manager::StoreManager; +use nativelink_util::common::DigestInfo; +use nativelink_util::store_trait::Store; use prost::Message; use proto::build::bazel::remote::execution::v2::action_cache_server::{ActionCache, ActionCacheServer as Server}; use proto::build::bazel::remote::execution::v2::{ActionResult, GetActionResultRequest, UpdateActionResultRequest}; diff --git a/native-link-service/src/bytestream_server.rs b/nativelink-service/src/bytestream_server.rs similarity index 98% rename from native-link-service/src/bytestream_server.rs rename to nativelink-service/src/bytestream_server.rs index d39cdd72c..62d820116 100644 --- a/native-link-service/src/bytestream_server.rs +++ b/nativelink-service/src/bytestream_server.rs @@ -24,14 +24,14 @@ use error::{make_err, make_input_err, Code, Error, ResultExt}; use futures::future::{pending, BoxFuture}; use futures::stream::unfold; use futures::{try_join, Future, Stream, TryFutureExt}; -use native_link_config::cas_server::ByteStreamConfig; -use native_link_store::grpc_store::GrpcStore; -use native_link_store::store_manager::StoreManager; -use native_link_util::buf_channel::{make_buf_channel_pair, DropCloserReadHalf, DropCloserWriteHalf}; -use native_link_util::common::DigestInfo; -use native_link_util::resource_info::ResourceInfo; -use native_link_util::store_trait::{Store, UploadSizeInfo}; -use native_link_util::write_request_stream_wrapper::WriteRequestStreamWrapper; +use nativelink_config::cas_server::ByteStreamConfig; +use nativelink_store::grpc_store::GrpcStore; +use nativelink_store::store_manager::StoreManager; +use nativelink_util::buf_channel::{make_buf_channel_pair, DropCloserReadHalf, DropCloserWriteHalf}; +use nativelink_util::common::DigestInfo; +use nativelink_util::resource_info::ResourceInfo; +use nativelink_util::store_trait::{Store, UploadSizeInfo}; +use nativelink_util::write_request_stream_wrapper::WriteRequestStreamWrapper; use parking_lot::Mutex; use proto::google::bytestream::byte_stream_server::{ByteStream, ByteStreamServer as Server}; use proto::google::bytestream::{ diff --git a/native-link-service/src/capabilities_server.rs b/nativelink-service/src/capabilities_server.rs similarity index 96% rename from native-link-service/src/capabilities_server.rs rename to nativelink-service/src/capabilities_server.rs index 10502181f..0442391e4 100644 --- a/native-link-service/src/capabilities_server.rs +++ b/nativelink-service/src/capabilities_server.rs @@ -16,9 +16,9 @@ use std::collections::HashMap; use std::sync::Arc; use error::{Error, ResultExt}; -use native_link_config::cas_server::{CapabilitiesConfig, InstanceName}; -use native_link_scheduler::action_scheduler::ActionScheduler; -use native_link_util::digest_hasher::default_digest_hasher_func; +use nativelink_config::cas_server::{CapabilitiesConfig, InstanceName}; +use nativelink_scheduler::action_scheduler::ActionScheduler; +use nativelink_util::digest_hasher::default_digest_hasher_func; use proto::build::bazel::remote::execution::v2::capabilities_server::{Capabilities, CapabilitiesServer as Server}; use proto::build::bazel::remote::execution::v2::digest_function::Value as DigestFunction; use proto::build::bazel::remote::execution::v2::priority_capabilities::PriorityRange; diff --git a/native-link-service/src/cas_server.rs b/nativelink-service/src/cas_server.rs similarity index 98% rename from native-link-service/src/cas_server.rs rename to nativelink-service/src/cas_server.rs index f0d60680a..cd038f73e 100644 --- a/native-link-service/src/cas_server.rs +++ b/nativelink-service/src/cas_server.rs @@ -22,11 +22,11 @@ use bytes::Bytes; use error::{error_if, make_err, make_input_err, Code, Error, ResultExt}; use futures::stream::{FuturesUnordered, Stream}; use futures::TryStreamExt; -use native_link_config::cas_server::{CasStoreConfig, InstanceName}; -use native_link_store::grpc_store::GrpcStore; -use native_link_store::store_manager::StoreManager; -use native_link_util::common::DigestInfo; -use native_link_util::store_trait::Store; +use nativelink_config::cas_server::{CasStoreConfig, InstanceName}; +use nativelink_store::grpc_store::GrpcStore; +use nativelink_store::store_manager::StoreManager; +use nativelink_util::common::DigestInfo; +use nativelink_util::store_trait::Store; use proto::build::bazel::remote::execution::v2::content_addressable_storage_server::{ ContentAddressableStorage, ContentAddressableStorageServer as Server, }; diff --git a/native-link-service/src/execution_server.rs b/nativelink-service/src/execution_server.rs similarity index 94% rename from native-link-service/src/execution_server.rs rename to nativelink-service/src/execution_server.rs index 00c5cab36..afa9170e1 100644 --- a/native-link-service/src/execution_server.rs +++ b/nativelink-service/src/execution_server.rs @@ -19,15 +19,15 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use error::{make_input_err, Error, ResultExt}; use futures::{Stream, StreamExt}; -use native_link_config::cas_server::{ExecutionConfig, InstanceName}; -use native_link_scheduler::action_scheduler::ActionScheduler; -use native_link_store::ac_utils::get_and_decode_digest; -use native_link_store::store_manager::StoreManager; -use native_link_util::action_messages::{ActionInfo, ActionInfoHashKey, ActionState, DEFAULT_EXECUTION_PRIORITY}; -use native_link_util::common::DigestInfo; -use native_link_util::digest_hasher::DigestHasherFunc; -use native_link_util::platform_properties::PlatformProperties; -use native_link_util::store_trait::Store; +use nativelink_config::cas_server::{ExecutionConfig, InstanceName}; +use nativelink_scheduler::action_scheduler::ActionScheduler; +use nativelink_store::ac_utils::get_and_decode_digest; +use nativelink_store::store_manager::StoreManager; +use nativelink_util::action_messages::{ActionInfo, ActionInfoHashKey, ActionState, DEFAULT_EXECUTION_PRIORITY}; +use nativelink_util::common::DigestInfo; +use nativelink_util::digest_hasher::DigestHasherFunc; +use nativelink_util::platform_properties::PlatformProperties; +use nativelink_util::store_trait::Store; use proto::build::bazel::remote::execution::v2::execution_server::{Execution, ExecutionServer as Server}; use proto::build::bazel::remote::execution::v2::{Action, Command, ExecuteRequest, WaitExecutionRequest}; use proto::google::longrunning::Operation; diff --git a/native-link-service/src/lib.rs b/nativelink-service/src/lib.rs similarity index 100% rename from native-link-service/src/lib.rs rename to nativelink-service/src/lib.rs diff --git a/native-link-service/src/worker_api_server.rs b/nativelink-service/src/worker_api_server.rs similarity index 95% rename from native-link-service/src/worker_api_server.rs rename to nativelink-service/src/worker_api_server.rs index 8c8a3e1eb..d47d83a36 100644 --- a/native-link-service/src/worker_api_server.rs +++ b/nativelink-service/src/worker_api_server.rs @@ -20,16 +20,16 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use error::{make_err, Code, Error, ResultExt}; use futures::stream::unfold; use futures::Stream; -use native_link_config::cas_server::WorkerApiConfig; -use native_link_scheduler::worker::{Worker, WorkerId}; -use native_link_scheduler::worker_scheduler::WorkerScheduler; -use native_link_util::action_messages::ActionInfoHashKey; -use native_link_util::common::DigestInfo; -use native_link_util::platform_properties::PlatformProperties; -use proto::com::github::trace_machina::native_link::remote_execution::worker_api_server::{ +use nativelink_config::cas_server::WorkerApiConfig; +use nativelink_scheduler::worker::{Worker, WorkerId}; +use nativelink_scheduler::worker_scheduler::WorkerScheduler; +use nativelink_util::action_messages::ActionInfoHashKey; +use nativelink_util::common::DigestInfo; +use nativelink_util::platform_properties::PlatformProperties; +use proto::com::github::trace_machina::nativelink::remote_execution::worker_api_server::{ WorkerApi, WorkerApiServer as Server, }; -use proto::com::github::trace_machina::native_link::remote_execution::{ +use proto::com::github::trace_machina::nativelink::remote_execution::{ execute_result, ExecuteResult, GoingAwayRequest, KeepAliveRequest, SupportedProperties, UpdateForWorker, }; use tokio::sync::mpsc; diff --git a/native-link-service/tests/ac_server_test.rs b/nativelink-service/tests/ac_server_test.rs similarity index 93% rename from native-link-service/tests/ac_server_test.rs rename to nativelink-service/tests/ac_server_test.rs index 6ea00701a..9aa42f500 100644 --- a/native-link-service/tests/ac_server_test.rs +++ b/nativelink-service/tests/ac_server_test.rs @@ -18,11 +18,11 @@ use std::sync::Arc; use bytes::BytesMut; use error::Error; use maplit::hashmap; -use native_link_service::ac_server::AcServer; -use native_link_store::default_store_factory::store_factory; -use native_link_store::store_manager::StoreManager; -use native_link_util::common::DigestInfo; -use native_link_util::store_trait::Store; +use nativelink_service::ac_server::AcServer; +use nativelink_store::default_store_factory::store_factory; +use nativelink_store::store_manager::StoreManager; +use nativelink_util::common::DigestInfo; +use nativelink_util::store_trait::Store; use prometheus_client::registry::Registry; use prost::Message; use proto::build::bazel::remote::execution::v2::action_cache_server::ActionCache; @@ -52,7 +52,7 @@ async fn make_store_manager() -> Result, Error> { store_manager.add_store( "main_cas", store_factory( - &native_link_config::stores::StoreConfig::memory(native_link_config::stores::MemoryStore::default()), + &nativelink_config::stores::StoreConfig::memory(nativelink_config::stores::MemoryStore::default()), &store_manager, Some(&mut ::default()), ) @@ -61,7 +61,7 @@ async fn make_store_manager() -> Result, Error> { store_manager.add_store( "main_ac", store_factory( - &native_link_config::stores::StoreConfig::memory(native_link_config::stores::MemoryStore::default()), + &nativelink_config::stores::StoreConfig::memory(nativelink_config::stores::MemoryStore::default()), &store_manager, Some(&mut ::default()), ) @@ -73,7 +73,7 @@ async fn make_store_manager() -> Result, Error> { fn make_ac_server(store_manager: &StoreManager) -> Result { AcServer::new( &hashmap! { - "foo_instance_name".to_string() => native_link_config::cas_server::AcStoreConfig{ + "foo_instance_name".to_string() => nativelink_config::cas_server::AcStoreConfig{ ac_store: "main_ac".to_string(), } }, diff --git a/native-link-service/tests/bytestream_server_test.rs b/nativelink-service/tests/bytestream_server_test.rs similarity index 98% rename from native-link-service/tests/bytestream_server_test.rs rename to nativelink-service/tests/bytestream_server_test.rs index c6c88fd06..59cbb7ac7 100644 --- a/native-link-service/tests/bytestream_server_test.rs +++ b/nativelink-service/tests/bytestream_server_test.rs @@ -21,10 +21,10 @@ use futures::poll; use futures::task::Poll; use hyper::body::Sender; use maplit::hashmap; -use native_link_service::bytestream_server::ByteStreamServer; -use native_link_store::default_store_factory::store_factory; -use native_link_store::store_manager::StoreManager; -use native_link_util::common::{encode_stream_proto, DigestInfo}; +use nativelink_service::bytestream_server::ByteStreamServer; +use nativelink_store::default_store_factory::store_factory; +use nativelink_store::store_manager::StoreManager; +use nativelink_util::common::{encode_stream_proto, DigestInfo}; use prometheus_client::registry::Registry; use proto::google::bytestream::WriteResponse; use tokio::task::{yield_now, JoinHandle}; @@ -38,7 +38,7 @@ async fn make_store_manager() -> Result, Error> { store_manager.add_store( "main_cas", store_factory( - &native_link_config::stores::StoreConfig::memory(native_link_config::stores::MemoryStore::default()), + &nativelink_config::stores::StoreConfig::memory(nativelink_config::stores::MemoryStore::default()), &store_manager, Some(&mut ::default()), ) @@ -49,7 +49,7 @@ async fn make_store_manager() -> Result, Error> { fn make_bytestream_server(store_manager: &StoreManager) -> Result { ByteStreamServer::new( - &native_link_config::cas_server::ByteStreamConfig { + &nativelink_config::cas_server::ByteStreamConfig { cas_stores: hashmap! { "foo_instance_name".to_string() => "main_cas".to_string(), }, diff --git a/native-link-service/tests/cas_server_test.rs b/nativelink-service/tests/cas_server_test.rs similarity index 97% rename from native-link-service/tests/cas_server_test.rs rename to nativelink-service/tests/cas_server_test.rs index a1460c9a8..1887739f9 100644 --- a/native-link-service/tests/cas_server_test.rs +++ b/nativelink-service/tests/cas_server_test.rs @@ -17,10 +17,10 @@ use std::sync::Arc; use error::Error; use maplit::hashmap; -use native_link_service::cas_server::CasServer; -use native_link_store::default_store_factory::store_factory; -use native_link_store::store_manager::StoreManager; -use native_link_util::common::DigestInfo; +use nativelink_service::cas_server::CasServer; +use nativelink_store::default_store_factory::store_factory; +use nativelink_store::store_manager::StoreManager; +use nativelink_util::common::DigestInfo; use prometheus_client::registry::Registry; use proto::build::bazel::remote::execution::v2::content_addressable_storage_server::ContentAddressableStorage; use proto::build::bazel::remote::execution::v2::{compressor, digest_function, Digest}; @@ -38,7 +38,7 @@ async fn make_store_manager() -> Result, Error> { store_manager.add_store( "main_cas", store_factory( - &native_link_config::stores::StoreConfig::memory(native_link_config::stores::MemoryStore::default()), + &nativelink_config::stores::StoreConfig::memory(nativelink_config::stores::MemoryStore::default()), &store_manager, Some(&mut ::default()), ) @@ -50,7 +50,7 @@ async fn make_store_manager() -> Result, Error> { fn make_cas_server(store_manager: &StoreManager) -> Result { CasServer::new( &hashmap! { - "foo_instance_name".to_string() => native_link_config::cas_server::CasStoreConfig{ + "foo_instance_name".to_string() => nativelink_config::cas_server::CasStoreConfig{ cas_store: "main_cas".to_string(), } }, diff --git a/native-link-service/tests/worker_api_server_test.rs b/nativelink-service/tests/worker_api_server_test.rs similarity index 94% rename from native-link-service/tests/worker_api_server_test.rs rename to nativelink-service/tests/worker_api_server_test.rs index e73214253..dc033655e 100644 --- a/native-link-service/tests/worker_api_server_test.rs +++ b/nativelink-service/tests/worker_api_server_test.rs @@ -17,22 +17,22 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use error::{Error, ResultExt}; -use native_link_config::cas_server::WorkerApiConfig; -use native_link_scheduler::action_scheduler::ActionScheduler; -use native_link_scheduler::simple_scheduler::SimpleScheduler; -use native_link_scheduler::worker::WorkerId; -use native_link_scheduler::worker_scheduler::WorkerScheduler; -use native_link_service::worker_api_server::{ConnectWorkerStream, NowFn, WorkerApiServer}; -use native_link_util::action_messages::{ActionInfo, ActionInfoHashKey, ActionStage}; -use native_link_util::common::DigestInfo; -use native_link_util::digest_hasher::DigestHasherFunc; -use native_link_util::platform_properties::PlatformProperties; +use nativelink_config::cas_server::WorkerApiConfig; +use nativelink_scheduler::action_scheduler::ActionScheduler; +use nativelink_scheduler::simple_scheduler::SimpleScheduler; +use nativelink_scheduler::worker::WorkerId; +use nativelink_scheduler::worker_scheduler::WorkerScheduler; +use nativelink_service::worker_api_server::{ConnectWorkerStream, NowFn, WorkerApiServer}; +use nativelink_util::action_messages::{ActionInfo, ActionInfoHashKey, ActionStage}; +use nativelink_util::common::DigestInfo; +use nativelink_util::digest_hasher::DigestHasherFunc; +use nativelink_util::platform_properties::PlatformProperties; use proto::build::bazel::remote::execution::v2::{ ActionResult as ProtoActionResult, ExecuteResponse, ExecutedActionMetadata, LogFile, OutputDirectory, OutputFile, OutputSymlink, }; -use proto::com::github::trace_machina::native_link::remote_execution::worker_api_server::WorkerApi; -use proto::com::github::trace_machina::native_link::remote_execution::{ +use proto::com::github::trace_machina::nativelink::remote_execution::worker_api_server::WorkerApi; +use proto::com::github::trace_machina::nativelink::remote_execution::{ execute_result, update_for_worker, ExecuteResult, KeepAliveRequest, SupportedProperties, }; use proto::google::rpc::Status as ProtoStatus; @@ -56,7 +56,7 @@ fn static_now_fn() -> Result { async fn setup_api_server(worker_timeout: u64, now_fn: NowFn) -> Result { const SCHEDULER_NAME: &str = "DUMMY_SCHEDULE_NAME"; - let scheduler = Arc::new(SimpleScheduler::new(&native_link_config::schedulers::SimpleScheduler { + let scheduler = Arc::new(SimpleScheduler::new(&nativelink_config::schedulers::SimpleScheduler { worker_timeout_s: worker_timeout, ..Default::default() })); diff --git a/native-link-store/BUILD.bazel b/nativelink-store/BUILD.bazel similarity index 92% rename from native-link-store/BUILD.bazel rename to nativelink-store/BUILD.bazel index 1c1acbf57..b6ba69ff0 100644 --- a/native-link-store/BUILD.bazel +++ b/nativelink-store/BUILD.bazel @@ -7,7 +7,7 @@ load( ) rust_library( - name = "native-link-store", + name = "nativelink-store", srcs = [ "src/ac_utils.rs", "src/completeness_checking_store.rs", @@ -34,8 +34,8 @@ rust_library( visibility = ["//visibility:public"], deps = [ "//error", - "//native-link-config", - "//native-link-util", + "//nativelink-config", + "//nativelink-util", "//proto", "@crate_index//:async-lock", "@crate_index//:aws-config", @@ -87,10 +87,10 @@ rust_test_suite( "@crate_index//:async-trait", ], deps = [ - ":native-link-store", + ":nativelink-store", "//error", - "//native-link-config", - "//native-link-util", + "//nativelink-config", + "//nativelink-util", "//proto", "@crate_index//:async-lock", "@crate_index//:aws-sdk-s3", @@ -113,10 +113,10 @@ rust_test_suite( rust_doc( name = "docs", - crate = ":native-link-store", + crate = ":nativelink-store", ) rust_doc_test( name = "doc_test", - crate = ":native-link-store", + crate = ":nativelink-store", ) diff --git a/native-link-store/Cargo.toml b/nativelink-store/Cargo.toml similarity index 89% rename from native-link-store/Cargo.toml rename to nativelink-store/Cargo.toml index 095e95848..0a21dd0cf 100644 --- a/native-link-store/Cargo.toml +++ b/nativelink-store/Cargo.toml @@ -1,12 +1,12 @@ [package] -name = "native-link-store" +name = "nativelink-store" version = "0.0.0" edition = "2021" [dependencies] error = { path = "../error" } -native-link-config = { path = "../native-link-config" } -native-link-util = { path = "../native-link-util" } +nativelink-config = { path = "../nativelink-config" } +nativelink-util = { path = "../nativelink-util" } proto = { path = "../proto" } async-lock = "2.8.0" diff --git a/native-link-store/src/ac_utils.rs b/nativelink-store/src/ac_utils.rs similarity index 96% rename from native-link-store/src/ac_utils.rs rename to nativelink-store/src/ac_utils.rs index ae29fd8ca..e03cb961d 100644 --- a/native-link-store/src/ac_utils.rs +++ b/nativelink-store/src/ac_utils.rs @@ -24,10 +24,10 @@ use bytes::{Bytes, BytesMut}; use error::{Code, Error, ResultExt}; use futures::future::join; use futures::{Future, FutureExt, TryFutureExt}; -use native_link_util::buf_channel::{make_buf_channel_pair, DropCloserWriteHalf}; -use native_link_util::common::{fs, DigestInfo}; -use native_link_util::digest_hasher::DigestHasher; -use native_link_util::store_trait::{Store, UploadSizeInfo}; +use nativelink_util::buf_channel::{make_buf_channel_pair, DropCloserWriteHalf}; +use nativelink_util::common::{fs, DigestInfo}; +use nativelink_util::digest_hasher::DigestHasher; +use nativelink_util::store_trait::{Store, UploadSizeInfo}; use prost::Message; use tokio::io::{AsyncRead, AsyncReadExt}; diff --git a/native-link-store/src/completeness_checking_store.rs b/nativelink-store/src/completeness_checking_store.rs similarity index 98% rename from native-link-store/src/completeness_checking_store.rs rename to nativelink-store/src/completeness_checking_store.rs index 8a21ecc2a..37d084d28 100644 --- a/native-link-store/src/completeness_checking_store.rs +++ b/nativelink-store/src/completeness_checking_store.rs @@ -20,9 +20,9 @@ use async_trait::async_trait; use error::{make_err, Code, Error, ResultExt}; use futures::stream::{FuturesUnordered, StreamExt}; use futures::{select, FutureExt, TryFutureExt}; -use native_link_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; -use native_link_util::common::DigestInfo; -use native_link_util::store_trait::{Store, UploadSizeInfo}; +use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; +use nativelink_util::common::DigestInfo; +use nativelink_util::store_trait::{Store, UploadSizeInfo}; use parking_lot::Mutex; use proto::build::bazel::remote::execution::v2::{ ActionResult as ProtoActionResult, OutputDirectory as ProtoOutputDirectory, Tree as ProtoTree, diff --git a/native-link-store/src/compression_store.rs b/nativelink-store/src/compression_store.rs similarity index 98% rename from native-link-store/src/compression_store.rs rename to nativelink-store/src/compression_store.rs index c7f3aeddb..2813cd6c3 100644 --- a/native-link-store/src/compression_store.rs +++ b/nativelink-store/src/compression_store.rs @@ -24,9 +24,9 @@ use bytes::{Buf, BufMut, BytesMut}; use error::{error_if, make_err, Code, Error, ResultExt}; use futures::future::FutureExt; use lz4_flex::block::{compress_into, decompress_into, get_maximum_output_size}; -use native_link_util::buf_channel::{make_buf_channel_pair, DropCloserReadHalf, DropCloserWriteHalf}; -use native_link_util::common::{DigestInfo, JoinHandleDropGuard}; -use native_link_util::store_trait::{Store, UploadSizeInfo}; +use nativelink_util::buf_channel::{make_buf_channel_pair, DropCloserReadHalf, DropCloserWriteHalf}; +use nativelink_util::common::{DigestInfo, JoinHandleDropGuard}; +use nativelink_util::store_trait::{Store, UploadSizeInfo}; use serde::{Deserialize, Serialize}; // In the event the bytestream format changes this number should be incremented to prevent @@ -107,7 +107,7 @@ pub const CHUNK_FRAME_TYPE: u8 = 0; /// Number representing the footer. pub const FOOTER_FRAME_TYPE: u8 = 1; -/// This is a partial mirror of native_link_config::stores::Lz4Config. +/// This is a partial mirror of nativelink_config::stores::Lz4Config. /// We cannot use that natively here because it could cause our /// serialized format to change if we added more configs. #[derive(Serialize, Deserialize, PartialEq, Debug, Default, Copy, Clone)] @@ -199,17 +199,17 @@ impl UploadState { /// only send the contents requested. pub struct CompressionStore { inner_store: Arc, - config: native_link_config::stores::Lz4Config, + config: nativelink_config::stores::Lz4Config, bincode_options: BincodeOptions, } impl CompressionStore { pub fn new( - compression_config: native_link_config::stores::CompressionStore, + compression_config: nativelink_config::stores::CompressionStore, inner_store: Arc, ) -> Result { let lz4_config = match compression_config.compression_algorithm { - native_link_config::stores::CompressionAlgorithm::LZ4(mut lz4_config) => { + nativelink_config::stores::CompressionAlgorithm::LZ4(mut lz4_config) => { if lz4_config.block_size == 0 { lz4_config.block_size = DEFAULT_BLOCK_SIZE; } diff --git a/native-link-store/src/dedup_store.rs b/nativelink-store/src/dedup_store.rs similarity index 97% rename from native-link-store/src/dedup_store.rs rename to nativelink-store/src/dedup_store.rs index 44e8271b9..772fb907c 100644 --- a/native-link-store/src/dedup_store.rs +++ b/nativelink-store/src/dedup_store.rs @@ -21,10 +21,10 @@ use bincode::config::{FixintEncoding, WithOtherIntEncoding}; use bincode::{self, DefaultOptions, Options}; use error::{make_err, Code, Error, ResultExt}; use futures::stream::{self, FuturesOrdered, StreamExt, TryStreamExt}; -use native_link_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf, StreamReader}; -use native_link_util::common::DigestInfo; -use native_link_util::fastcdc::FastCDC; -use native_link_util::store_trait::{Store, UploadSizeInfo}; +use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf, StreamReader}; +use nativelink_util::common::DigestInfo; +use nativelink_util::fastcdc::FastCDC; +use nativelink_util::store_trait::{Store, UploadSizeInfo}; use serde::{Deserialize, Serialize}; use tokio_util::codec::FramedRead; use tracing::warn; @@ -52,7 +52,7 @@ pub struct DedupStore { impl DedupStore { pub fn new( - config: &native_link_config::stores::DedupStore, + config: &nativelink_config::stores::DedupStore, index_store: Arc, content_store: Arc, ) -> Self { diff --git a/native-link-store/src/default_store_factory.rs b/nativelink-store/src/default_store_factory.rs similarity index 97% rename from native-link-store/src/default_store_factory.rs rename to nativelink-store/src/default_store_factory.rs index c903e4127..4e93f9741 100644 --- a/native-link-store/src/default_store_factory.rs +++ b/nativelink-store/src/default_store_factory.rs @@ -18,9 +18,9 @@ use std::sync::Arc; use error::Error; use futures::stream::FuturesOrdered; use futures::{Future, TryStreamExt}; -use native_link_config::stores::StoreConfig; -use native_link_util::metrics_utils::Registry; -use native_link_util::store_trait::Store; +use nativelink_config::stores::StoreConfig; +use nativelink_util::metrics_utils::Registry; +use nativelink_util::store_trait::Store; use crate::completeness_checking_store::CompletenessCheckingStore; use crate::compression_store::CompressionStore; diff --git a/native-link-store/src/existence_cache_store.rs b/nativelink-store/src/existence_cache_store.rs similarity index 94% rename from native-link-store/src/existence_cache_store.rs rename to nativelink-store/src/existence_cache_store.rs index 4d2513591..ad834792c 100644 --- a/native-link-store/src/existence_cache_store.rs +++ b/nativelink-store/src/existence_cache_store.rs @@ -19,11 +19,11 @@ use std::time::SystemTime; use async_trait::async_trait; use error::{error_if, Error, ResultExt}; -use native_link_config::stores::{EvictionPolicy, ExistenceCacheStore as ExistenceCacheStoreConfig}; -use native_link_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; -use native_link_util::common::DigestInfo; -use native_link_util::evicting_map::{EvictingMap, LenEntry}; -use native_link_util::store_trait::{Store, UploadSizeInfo}; +use nativelink_config::stores::{EvictionPolicy, ExistenceCacheStore as ExistenceCacheStoreConfig}; +use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; +use nativelink_util::common::DigestInfo; +use nativelink_util::evicting_map::{EvictingMap, LenEntry}; +use nativelink_util::store_trait::{Store, UploadSizeInfo}; #[derive(Clone, Debug)] struct ExistanceItem(usize); diff --git a/native-link-store/src/fast_slow_store.rs b/nativelink-store/src/fast_slow_store.rs similarity index 97% rename from native-link-store/src/fast_slow_store.rs rename to nativelink-store/src/fast_slow_store.rs index 7d886e5ad..a30f32761 100644 --- a/native-link-store/src/fast_slow_store.rs +++ b/nativelink-store/src/fast_slow_store.rs @@ -20,10 +20,10 @@ use std::sync::Arc; use async_trait::async_trait; use error::{make_err, Code, Error, ResultExt}; use futures::{join, FutureExt}; -use native_link_util::buf_channel::{make_buf_channel_pair, DropCloserReadHalf, DropCloserWriteHalf}; -use native_link_util::common::DigestInfo; -use native_link_util::metrics_utils::Registry; -use native_link_util::store_trait::{Store, UploadSizeInfo}; +use nativelink_util::buf_channel::{make_buf_channel_pair, DropCloserReadHalf, DropCloserWriteHalf}; +use nativelink_util::common::DigestInfo; +use nativelink_util::metrics_utils::Registry; +use nativelink_util::store_trait::{Store, UploadSizeInfo}; // TODO(blaise.bruer) This store needs to be evaluated for more efficient memory usage, // there are many copies happening internally. @@ -39,7 +39,7 @@ pub struct FastSlowStore { impl FastSlowStore { pub fn new( - _config: &native_link_config::stores::FastSlowStore, + _config: &nativelink_config::stores::FastSlowStore, fast_store: Arc, slow_store: Arc, ) -> Self { diff --git a/native-link-store/src/filesystem_store.rs b/nativelink-store/src/filesystem_store.rs similarity index 97% rename from native-link-store/src/filesystem_store.rs rename to nativelink-store/src/filesystem_store.rs index 646a3fd38..15cd3d190 100644 --- a/native-link-store/src/filesystem_store.rs +++ b/nativelink-store/src/filesystem_store.rs @@ -26,11 +26,11 @@ use error::{make_err, make_input_err, Code, Error, ResultExt}; use filetime::{set_file_atime, FileTime}; use futures::stream::{StreamExt, TryStreamExt}; use futures::{Future, TryFutureExt}; -use native_link_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; -use native_link_util::common::{fs, DigestInfo}; -use native_link_util::evicting_map::{EvictingMap, LenEntry}; -use native_link_util::metrics_utils::{Collector, CollectorState, MetricsComponent, Registry}; -use native_link_util::store_trait::{Store, UploadSizeInfo}; +use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; +use nativelink_util::common::{fs, DigestInfo}; +use nativelink_util::evicting_map::{EvictingMap, LenEntry}; +use nativelink_util::metrics_utils::{Collector, CollectorState, MetricsComponent, Registry}; +use nativelink_util::store_trait::{Store, UploadSizeInfo}; use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt, SeekFrom}; use tokio::task::spawn_blocking; use tokio::time::{sleep, timeout, Sleep}; @@ -441,17 +441,17 @@ pub struct FilesystemStore { } impl FilesystemStore { - pub async fn new(config: &native_link_config::stores::FilesystemStore) -> Result { + pub async fn new(config: &nativelink_config::stores::FilesystemStore) -> Result { Self::new_with_timeout(config, sleep).await } pub async fn new_with_timeout( - config: &native_link_config::stores::FilesystemStore, + config: &nativelink_config::stores::FilesystemStore, sleep_fn: fn(Duration) -> Sleep, ) -> Result { let now = SystemTime::now(); - let empty_policy = native_link_config::stores::EvictionPolicy::default(); + let empty_policy = nativelink_config::stores::EvictionPolicy::default(); let eviction_policy = config.eviction_policy.as_ref().unwrap_or(&empty_policy); let evicting_map = EvictingMap::new(eviction_policy, now); diff --git a/native-link-store/src/grpc_store.rs b/nativelink-store/src/grpc_store.rs similarity index 94% rename from native-link-store/src/grpc_store.rs rename to nativelink-store/src/grpc_store.rs index 1984b3e99..3fdacd069 100644 --- a/native-link-store/src/grpc_store.rs +++ b/nativelink-store/src/grpc_store.rs @@ -22,11 +22,11 @@ use bytes::BytesMut; use error::{error_if, make_input_err, Error, ResultExt}; use futures::stream::{unfold, FuturesUnordered}; use futures::{future, Future, Stream, TryStreamExt}; -use native_link_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; -use native_link_util::common::DigestInfo; -use native_link_util::retry::{ExponentialBackoff, Retrier, RetryResult}; -use native_link_util::store_trait::{Store, UploadSizeInfo}; -use native_link_util::write_request_stream_wrapper::WriteRequestStreamWrapper; +use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; +use nativelink_util::common::DigestInfo; +use nativelink_util::retry::{ExponentialBackoff, Retrier, RetryResult}; +use nativelink_util::store_trait::{Store, UploadSizeInfo}; +use nativelink_util::write_request_stream_wrapper::WriteRequestStreamWrapper; use parking_lot::Mutex; use prost::Message; use proto::build::bazel::remote::execution::v2::action_cache_client::ActionCacheClient; @@ -58,14 +58,14 @@ pub struct GrpcStore { cas_client: ContentAddressableStorageClient, bytestream_client: ByteStreamClient, ac_client: ActionCacheClient, - store_type: native_link_config::stores::StoreType, + store_type: nativelink_config::stores::StoreType, jitter_fn: Box Duration + Send + Sync>, - retry: native_link_config::stores::Retry, + retry: nativelink_config::stores::Retry, retrier: Retrier, } impl GrpcStore { - pub async fn new(config: &native_link_config::stores::GrpcStore) -> Result { + pub async fn new(config: &nativelink_config::stores::GrpcStore) -> Result { let jitter_amt = config.retry.jitter; Self::new_with_jitter( config, @@ -82,7 +82,7 @@ impl GrpcStore { } pub async fn new_with_jitter( - config: &native_link_config::stores::GrpcStore, + config: &nativelink_config::stores::GrpcStore, jitter_fn: Box Duration + Send + Sync>, ) -> Result { error_if!(config.endpoints.is_empty(), "Expected at least 1 endpoint in GrpcStore"); @@ -149,7 +149,7 @@ impl GrpcStore { grpc_request: Request, ) -> Result, Error> { error_if!( - matches!(self.store_type, native_link_config::stores::StoreType::AC), + matches!(self.store_type, nativelink_config::stores::StoreType::AC), "CAS operation on AC store" ); @@ -170,7 +170,7 @@ impl GrpcStore { grpc_request: Request, ) -> Result, Error> { error_if!( - matches!(self.store_type, native_link_config::stores::StoreType::AC), + matches!(self.store_type, nativelink_config::stores::StoreType::AC), "CAS operation on AC store" ); @@ -191,7 +191,7 @@ impl GrpcStore { grpc_request: Request, ) -> Result, Error> { error_if!( - matches!(self.store_type, native_link_config::stores::StoreType::AC), + matches!(self.store_type, nativelink_config::stores::StoreType::AC), "CAS operation on AC store" ); @@ -212,7 +212,7 @@ impl GrpcStore { grpc_request: Request, ) -> Result>, Error> { error_if!( - matches!(self.store_type, native_link_config::stores::StoreType::AC), + matches!(self.store_type, nativelink_config::stores::StoreType::AC), "CAS operation on AC store" ); @@ -233,7 +233,7 @@ impl GrpcStore { grpc_request: impl IntoRequest, ) -> Result>, Error> { error_if!( - matches!(self.store_type, native_link_config::stores::StoreType::AC), + matches!(self.store_type, nativelink_config::stores::StoreType::AC), "CAS operation on AC store" ); @@ -266,7 +266,7 @@ impl GrpcStore { E: Into + 'static, { error_if!( - matches!(self.store_type, native_link_config::stores::StoreType::AC), + matches!(self.store_type, nativelink_config::stores::StoreType::AC), "CAS operation on AC store" ); @@ -358,7 +358,7 @@ impl GrpcStore { grpc_request: Request, ) -> Result, Error> { error_if!( - matches!(self.store_type, native_link_config::stores::StoreType::AC), + matches!(self.store_type, nativelink_config::stores::StoreType::AC), "CAS operation on AC store" ); @@ -492,7 +492,7 @@ impl Store for GrpcStore { digests: &[DigestInfo], results: &mut [Option], ) -> Result<(), Error> { - if matches!(self.store_type, native_link_config::stores::StoreType::AC) { + if matches!(self.store_type, nativelink_config::stores::StoreType::AC) { digests .iter() .zip(results.iter_mut()) @@ -546,7 +546,7 @@ impl Store for GrpcStore { reader: DropCloserReadHalf, _size_info: UploadSizeInfo, ) -> Result<(), Error> { - if matches!(self.store_type, native_link_config::stores::StoreType::AC) { + if matches!(self.store_type, nativelink_config::stores::StoreType::AC) { return self.update_action_result_from_bytes(digest, reader).await; } @@ -617,7 +617,7 @@ impl Store for GrpcStore { offset: usize, length: Option, ) -> Result<(), Error> { - if matches!(self.store_type, native_link_config::stores::StoreType::AC) { + if matches!(self.store_type, nativelink_config::stores::StoreType::AC) { return self.get_action_result_as_part(digest, writer, offset, length).await; } diff --git a/native-link-store/src/lib.rs b/nativelink-store/src/lib.rs similarity index 100% rename from native-link-store/src/lib.rs rename to nativelink-store/src/lib.rs diff --git a/native-link-store/src/memory_store.rs b/nativelink-store/src/memory_store.rs similarity index 89% rename from native-link-store/src/memory_store.rs rename to nativelink-store/src/memory_store.rs index 4a59070c7..378aa57c0 100644 --- a/native-link-store/src/memory_store.rs +++ b/nativelink-store/src/memory_store.rs @@ -20,11 +20,11 @@ use std::time::SystemTime; use async_trait::async_trait; use bytes::{Bytes, BytesMut}; use error::{Code, Error, ResultExt}; -use native_link_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; -use native_link_util::common::DigestInfo; -use native_link_util::evicting_map::{EvictingMap, LenEntry}; -use native_link_util::metrics_utils::{Collector, CollectorState, MetricsComponent, Registry}; -use native_link_util::store_trait::{Store, UploadSizeInfo}; +use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; +use nativelink_util::common::DigestInfo; +use nativelink_util::evicting_map::{EvictingMap, LenEntry}; +use nativelink_util::metrics_utils::{Collector, CollectorState, MetricsComponent, Registry}; +use nativelink_util::store_trait::{Store, UploadSizeInfo}; #[derive(Clone)] pub struct BytesWrapper(Bytes); @@ -52,8 +52,8 @@ pub struct MemoryStore { } impl MemoryStore { - pub fn new(config: &native_link_config::stores::MemoryStore) -> Self { - let empty_policy = native_link_config::stores::EvictionPolicy::default(); + pub fn new(config: &nativelink_config::stores::MemoryStore) -> Self { + let empty_policy = nativelink_config::stores::EvictionPolicy::default(); let eviction_policy = config.eviction_policy.as_ref().unwrap_or(&empty_policy); MemoryStore { evicting_map: EvictingMap::new(eviction_policy, SystemTime::now()), diff --git a/native-link-store/src/noop_store.rs b/nativelink-store/src/noop_store.rs similarity index 91% rename from native-link-store/src/noop_store.rs rename to nativelink-store/src/noop_store.rs index cc5346cc3..368ae21f7 100644 --- a/native-link-store/src/noop_store.rs +++ b/nativelink-store/src/noop_store.rs @@ -17,9 +17,9 @@ use std::sync::Arc; use async_trait::async_trait; use error::{make_err, Code, Error, ResultExt}; -use native_link_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; -use native_link_util::common::DigestInfo; -use native_link_util::store_trait::{Store, UploadSizeInfo}; +use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; +use nativelink_util::common::DigestInfo; +use nativelink_util::store_trait::{Store, UploadSizeInfo}; #[derive(Default)] pub struct NoopStore; diff --git a/native-link-store/src/ref_store.rs b/nativelink-store/src/ref_store.rs similarity index 94% rename from native-link-store/src/ref_store.rs rename to nativelink-store/src/ref_store.rs index 8037339ff..f3a8d1ee9 100644 --- a/native-link-store/src/ref_store.rs +++ b/nativelink-store/src/ref_store.rs @@ -18,9 +18,9 @@ use std::sync::{Arc, Mutex, Weak}; use async_trait::async_trait; use error::{make_err, make_input_err, Code, Error, ResultExt}; -use native_link_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; -use native_link_util::common::DigestInfo; -use native_link_util::store_trait::{Store, UploadSizeInfo}; +use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; +use nativelink_util::common::DigestInfo; +use nativelink_util::store_trait::{Store, UploadSizeInfo}; use tracing::error; use crate::store_manager::StoreManager; @@ -42,7 +42,7 @@ pub struct RefStore { } impl RefStore { - pub fn new(config: &native_link_config::stores::RefStore, store_manager: Weak) -> Self { + pub fn new(config: &nativelink_config::stores::RefStore, store_manager: Weak) -> Self { RefStore { ref_store_name: config.name.clone(), store_manager, diff --git a/native-link-store/src/s3_store.rs b/nativelink-store/src/s3_store.rs similarity index 97% rename from native-link-store/src/s3_store.rs rename to nativelink-store/src/s3_store.rs index ac368981a..69264e13d 100644 --- a/native-link-store/src/s3_store.rs +++ b/nativelink-store/src/s3_store.rs @@ -37,10 +37,10 @@ use hyper::client::connect::HttpConnector; use hyper::service::Service; use hyper::Uri; use hyper_rustls::{HttpsConnector, MaybeHttpsStream}; -use native_link_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; -use native_link_util::common::DigestInfo; -use native_link_util::retry::{ExponentialBackoff, Retrier, RetryResult}; -use native_link_util::store_trait::{Store, UploadSizeInfo}; +use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; +use nativelink_util::common::DigestInfo; +use nativelink_util::retry::{ExponentialBackoff, Retrier, RetryResult}; +use nativelink_util::store_trait::{Store, UploadSizeInfo}; use rand::rngs::OsRng; use rand::Rng; use tokio::net::TcpStream; @@ -60,7 +60,7 @@ const DEFAULT_MULTIPART_MAX_CONCURRENT_UPLOADS: usize = 10; #[derive(Clone)] pub struct TlsConnector { connector: HttpsConnector, - retry: native_link_config::stores::Retry, + retry: nativelink_config::stores::Retry, retrier: Retrier, jitter_fn: Arc Duration + Send + Sync>, } @@ -68,7 +68,7 @@ pub struct TlsConnector { impl TlsConnector { #[must_use] pub fn new( - config: &native_link_config::stores::S3Store, + config: &nativelink_config::stores::S3Store, jitter_fn: Arc Duration + Send + Sync>, ) -> Self { let connector_with_roots = hyper_rustls::HttpsConnectorBuilder::new().with_webpki_roots(); @@ -132,13 +132,13 @@ pub struct S3Store { bucket: String, key_prefix: String, jitter_fn: Arc Duration + Send + Sync>, - retry: native_link_config::stores::Retry, + retry: nativelink_config::stores::Retry, retrier: Retrier, multipart_max_concurrent_uploads: usize, } impl S3Store { - pub async fn new(config: &native_link_config::stores::S3Store) -> Result { + pub async fn new(config: &nativelink_config::stores::S3Store) -> Result { let jitter_amt = config.retry.jitter; let jitter_fn = Arc::new(move |delay: Duration| { if jitter_amt == 0. { @@ -165,7 +165,7 @@ impl S3Store { } pub fn new_with_client_and_jitter( - config: &native_link_config::stores::S3Store, + config: &nativelink_config::stores::S3Store, s3_client: Client, jitter_fn: Arc Duration + Send + Sync>, ) -> Result { diff --git a/native-link-store/src/shard_store.rs b/nativelink-store/src/shard_store.rs similarity index 95% rename from native-link-store/src/shard_store.rs rename to nativelink-store/src/shard_store.rs index 8af662c34..89969c96d 100644 --- a/native-link-store/src/shard_store.rs +++ b/nativelink-store/src/shard_store.rs @@ -19,10 +19,10 @@ use std::sync::Arc; use async_trait::async_trait; use error::{error_if, Error, ResultExt}; use futures::stream::{FuturesUnordered, TryStreamExt}; -use native_link_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; -use native_link_util::common::DigestInfo; -use native_link_util::metrics_utils::Registry; -use native_link_util::store_trait::{Store, UploadSizeInfo}; +use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; +use nativelink_util::common::DigestInfo; +use nativelink_util::metrics_utils::Registry; +use nativelink_util::store_trait::{Store, UploadSizeInfo}; pub struct ShardStore { // The weights will always be in ascending order a specific store is choosen based on the @@ -31,7 +31,7 @@ pub struct ShardStore { } impl ShardStore { - pub fn new(config: &native_link_config::stores::ShardStore, stores: Vec>) -> Result { + pub fn new(config: &nativelink_config::stores::ShardStore, stores: Vec>) -> Result { error_if!( config.stores.len() != stores.len(), "Config shards do not match stores length" diff --git a/native-link-store/src/size_partitioning_store.rs b/nativelink-store/src/size_partitioning_store.rs similarity index 94% rename from native-link-store/src/size_partitioning_store.rs rename to nativelink-store/src/size_partitioning_store.rs index 21b265b35..f28af31b5 100644 --- a/native-link-store/src/size_partitioning_store.rs +++ b/nativelink-store/src/size_partitioning_store.rs @@ -17,9 +17,9 @@ use std::sync::Arc; use async_trait::async_trait; use error::{Error, ResultExt}; -use native_link_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; -use native_link_util::common::DigestInfo; -use native_link_util::store_trait::{Store, UploadSizeInfo}; +use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; +use nativelink_util::common::DigestInfo; +use nativelink_util::store_trait::{Store, UploadSizeInfo}; use tokio::join; pub struct SizePartitioningStore { @@ -30,7 +30,7 @@ pub struct SizePartitioningStore { impl SizePartitioningStore { pub fn new( - config: &native_link_config::stores::SizePartitioningStore, + config: &nativelink_config::stores::SizePartitioningStore, lower_store: Arc, upper_store: Arc, ) -> Self { diff --git a/native-link-store/src/store_manager.rs b/nativelink-store/src/store_manager.rs similarity index 97% rename from native-link-store/src/store_manager.rs rename to nativelink-store/src/store_manager.rs index bc69b31e7..7c42a7140 100644 --- a/native-link-store/src/store_manager.rs +++ b/nativelink-store/src/store_manager.rs @@ -15,7 +15,7 @@ use std::collections::HashMap; use std::sync::{Arc, RwLock}; -use native_link_util::store_trait::Store; +use nativelink_util::store_trait::Store; pub struct StoreManager { stores: RwLock>>, diff --git a/native-link-store/src/verify_store.rs b/nativelink-store/src/verify_store.rs similarity index 94% rename from native-link-store/src/verify_store.rs rename to nativelink-store/src/verify_store.rs index 2470a37ac..8d04584df 100644 --- a/native-link-store/src/verify_store.rs +++ b/nativelink-store/src/verify_store.rs @@ -18,10 +18,10 @@ use std::sync::Arc; use async_trait::async_trait; use error::{make_input_err, Error, ResultExt}; -use native_link_util::buf_channel::{make_buf_channel_pair, DropCloserReadHalf, DropCloserWriteHalf}; -use native_link_util::common::DigestInfo; -use native_link_util::metrics_utils::{Collector, CollectorState, CounterWithTime, MetricsComponent, Registry}; -use native_link_util::store_trait::{Store, UploadSizeInfo}; +use nativelink_util::buf_channel::{make_buf_channel_pair, DropCloserReadHalf, DropCloserWriteHalf}; +use nativelink_util::common::DigestInfo; +use nativelink_util::metrics_utils::{Collector, CollectorState, CounterWithTime, MetricsComponent, Registry}; +use nativelink_util::store_trait::{Store, UploadSizeInfo}; use sha2::{Digest, Sha256}; pub struct VerifyStore { @@ -35,7 +35,7 @@ pub struct VerifyStore { } impl VerifyStore { - pub fn new(config: &native_link_config::stores::VerifyStore, inner_store: Arc) -> Self { + pub fn new(config: &nativelink_config::stores::VerifyStore, inner_store: Arc) -> Self { VerifyStore { inner_store, verify_size: config.verify_size, diff --git a/native-link-store/tests/ac_utils_test.rs b/nativelink-store/tests/ac_utils_test.rs similarity index 91% rename from native-link-store/tests/ac_utils_test.rs rename to nativelink-store/tests/ac_utils_test.rs index 7b1863458..c036ed979 100644 --- a/native-link-store/tests/ac_utils_test.rs +++ b/nativelink-store/tests/ac_utils_test.rs @@ -18,10 +18,10 @@ use std::pin::Pin; use std::sync::Arc; use error::{Error, ResultExt}; -use native_link_store::ac_utils::upload_file_to_store; -use native_link_store::memory_store::MemoryStore; -use native_link_util::common::{fs, DigestInfo}; -use native_link_util::store_trait::Store; +use nativelink_store::ac_utils::upload_file_to_store; +use nativelink_store::memory_store::MemoryStore; +use nativelink_util::common::{fs, DigestInfo}; +use nativelink_util::store_trait::Store; use rand::{thread_rng, Rng}; use tokio::io::AsyncWriteExt; @@ -53,7 +53,7 @@ mod ac_utils_tests { async fn upload_file_to_store_with_large_file() -> Result<(), Error> { let filepath = make_temp_path("test.txt").await; let expected_data = vec![0x88; 1024 * 1024]; // 1MB. - let store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); + let store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); let store_pin = Pin::new(store.as_ref()); let digest = DigestInfo::try_new(HASH1, HASH1_SIZE)?; // Dummy hash data. { diff --git a/native-link-store/tests/completeness_checking_store_test.rs b/nativelink-store/tests/completeness_checking_store_test.rs similarity index 95% rename from native-link-store/tests/completeness_checking_store_test.rs rename to nativelink-store/tests/completeness_checking_store_test.rs index 26d96ae3d..443ae32a7 100644 --- a/native-link-store/tests/completeness_checking_store_test.rs +++ b/nativelink-store/tests/completeness_checking_store_test.rs @@ -16,13 +16,13 @@ use std::pin::Pin; use std::sync::Arc; use error::Error; -use native_link_config::stores::MemoryStore as MemoryStoreConfig; -use native_link_store::ac_utils::serialize_and_upload_message; -use native_link_store::completeness_checking_store::CompletenessCheckingStore; -use native_link_store::memory_store::MemoryStore; -use native_link_util::common::DigestInfo; -use native_link_util::digest_hasher::DigestHasherFunc::Blake3; -use native_link_util::store_trait::Store; +use nativelink_config::stores::MemoryStore as MemoryStoreConfig; +use nativelink_store::ac_utils::serialize_and_upload_message; +use nativelink_store::completeness_checking_store::CompletenessCheckingStore; +use nativelink_store::memory_store::MemoryStore; +use nativelink_util::common::DigestInfo; +use nativelink_util::digest_hasher::DigestHasherFunc::Blake3; +use nativelink_util::store_trait::Store; use proto::build::bazel::remote::execution::v2::{ ActionResult as ProtoActionResult, Directory, DirectoryNode, FileNode, OutputDirectory, OutputFile, Tree, }; diff --git a/native-link-store/tests/compression_store_test.rs b/nativelink-store/tests/compression_store_test.rs similarity index 82% rename from native-link-store/tests/compression_store_test.rs rename to nativelink-store/tests/compression_store_test.rs index 49d752715..17ed811d8 100644 --- a/native-link-store/tests/compression_store_test.rs +++ b/nativelink-store/tests/compression_store_test.rs @@ -20,14 +20,14 @@ use std::sync::Arc; use bincode::{DefaultOptions, Options}; use error::{make_err, Code, Error, ResultExt}; -use native_link_store::compression_store::{ +use nativelink_store::compression_store::{ CompressionStore, Footer, Lz4Config, SliceIndex, CURRENT_STREAM_FORMAT_VERSION, DEFAULT_BLOCK_SIZE, FOOTER_FRAME_TYPE, }; -use native_link_store::memory_store::MemoryStore; -use native_link_util::buf_channel::make_buf_channel_pair; -use native_link_util::common::DigestInfo; -use native_link_util::store_trait::{Store, UploadSizeInfo}; +use nativelink_store::memory_store::MemoryStore; +use nativelink_util::buf_channel::make_buf_channel_pair; +use nativelink_util::common::DigestInfo; +use nativelink_util::store_trait::{Store, UploadSizeInfo}; use rand::rngs::SmallRng; use rand::{Rng, SeedableRng}; use tokio::io::AsyncReadExt; @@ -64,15 +64,15 @@ mod compression_store_tests { #[tokio::test] async fn simple_smoke_test() -> Result<(), Error> { let store_owned = CompressionStore::new( - native_link_config::stores::CompressionStore { - backend: native_link_config::stores::StoreConfig::memory( - native_link_config::stores::MemoryStore::default(), + nativelink_config::stores::CompressionStore { + backend: nativelink_config::stores::StoreConfig::memory( + nativelink_config::stores::MemoryStore::default(), ), - compression_algorithm: native_link_config::stores::CompressionAlgorithm::LZ4( - native_link_config::stores::Lz4Config { ..Default::default() }, + compression_algorithm: nativelink_config::stores::CompressionAlgorithm::LZ4( + nativelink_config::stores::Lz4Config { ..Default::default() }, ), }, - Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())), + Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())), ) .err_tip(|| "Failed to create compression store")?; let store = Pin::new(&store_owned); @@ -93,18 +93,18 @@ mod compression_store_tests { #[tokio::test] async fn partial_reads_test() -> Result<(), Error> { let store_owned = CompressionStore::new( - native_link_config::stores::CompressionStore { - backend: native_link_config::stores::StoreConfig::memory( - native_link_config::stores::MemoryStore::default(), + nativelink_config::stores::CompressionStore { + backend: nativelink_config::stores::StoreConfig::memory( + nativelink_config::stores::MemoryStore::default(), ), - compression_algorithm: native_link_config::stores::CompressionAlgorithm::LZ4( - native_link_config::stores::Lz4Config { + compression_algorithm: nativelink_config::stores::CompressionAlgorithm::LZ4( + nativelink_config::stores::Lz4Config { block_size: 10, ..Default::default() }, ), }, - Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())), + Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())), ) .err_tip(|| "Failed to create compression store")?; let store = Pin::new(&store_owned); @@ -146,15 +146,15 @@ mod compression_store_tests { #[tokio::test] async fn rand_5mb_smoke_test() -> Result<(), Error> { let store_owned = CompressionStore::new( - native_link_config::stores::CompressionStore { - backend: native_link_config::stores::StoreConfig::memory( - native_link_config::stores::MemoryStore::default(), + nativelink_config::stores::CompressionStore { + backend: nativelink_config::stores::StoreConfig::memory( + nativelink_config::stores::MemoryStore::default(), ), - compression_algorithm: native_link_config::stores::CompressionAlgorithm::LZ4( - native_link_config::stores::Lz4Config { ..Default::default() }, + compression_algorithm: nativelink_config::stores::CompressionAlgorithm::LZ4( + nativelink_config::stores::Lz4Config { ..Default::default() }, ), }, - Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())), + Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())), ) .err_tip(|| "Failed to create compression store")?; let store = Pin::new(&store_owned); @@ -177,14 +177,14 @@ mod compression_store_tests { #[tokio::test] async fn sanity_check_zero_bytes_test() -> Result<(), Error> { - let inner_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); + let inner_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); let store_owned = CompressionStore::new( - native_link_config::stores::CompressionStore { - backend: native_link_config::stores::StoreConfig::memory( - native_link_config::stores::MemoryStore::default(), + nativelink_config::stores::CompressionStore { + backend: nativelink_config::stores::StoreConfig::memory( + nativelink_config::stores::MemoryStore::default(), ), - compression_algorithm: native_link_config::stores::CompressionAlgorithm::LZ4( - native_link_config::stores::Lz4Config { ..Default::default() }, + compression_algorithm: nativelink_config::stores::CompressionAlgorithm::LZ4( + nativelink_config::stores::Lz4Config { ..Default::default() }, ), }, inner_store.clone(), @@ -226,14 +226,14 @@ mod compression_store_tests { async fn check_header_test() -> Result<(), Error> { const BLOCK_SIZE: u32 = 150; const MAX_SIZE_INPUT: usize = 1024 * 1024; // 1MB. - let inner_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); + let inner_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); let store_owned = CompressionStore::new( - native_link_config::stores::CompressionStore { - backend: native_link_config::stores::StoreConfig::memory( - native_link_config::stores::MemoryStore::default(), + nativelink_config::stores::CompressionStore { + backend: nativelink_config::stores::StoreConfig::memory( + nativelink_config::stores::MemoryStore::default(), ), - compression_algorithm: native_link_config::stores::CompressionAlgorithm::LZ4( - native_link_config::stores::Lz4Config { + compression_algorithm: nativelink_config::stores::CompressionAlgorithm::LZ4( + nativelink_config::stores::Lz4Config { block_size: BLOCK_SIZE, ..Default::default() }, @@ -305,14 +305,14 @@ mod compression_store_tests { #[tokio::test] async fn check_footer_test() -> Result<(), Error> { const BLOCK_SIZE: u32 = 32 * 1024; - let inner_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); + let inner_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); let store_owned = CompressionStore::new( - native_link_config::stores::CompressionStore { - backend: native_link_config::stores::StoreConfig::memory( - native_link_config::stores::MemoryStore::default(), + nativelink_config::stores::CompressionStore { + backend: nativelink_config::stores::StoreConfig::memory( + nativelink_config::stores::MemoryStore::default(), ), - compression_algorithm: native_link_config::stores::CompressionAlgorithm::LZ4( - native_link_config::stores::Lz4Config { + compression_algorithm: nativelink_config::stores::CompressionAlgorithm::LZ4( + nativelink_config::stores::Lz4Config { block_size: BLOCK_SIZE, ..Default::default() }, diff --git a/native-link-store/tests/dedup_store_test.rs b/nativelink-store/tests/dedup_store_test.rs similarity index 79% rename from native-link-store/tests/dedup_store_test.rs rename to nativelink-store/tests/dedup_store_test.rs index e172fa630..fe642c33d 100644 --- a/native-link-store/tests/dedup_store_test.rs +++ b/nativelink-store/tests/dedup_store_test.rs @@ -16,18 +16,18 @@ use std::pin::Pin; use std::sync::Arc; use error::{Code, Error, ResultExt}; -use native_link_store::dedup_store::DedupStore; -use native_link_store::memory_store::MemoryStore; -use native_link_util::common::DigestInfo; -use native_link_util::store_trait::Store; +use nativelink_store::dedup_store::DedupStore; +use nativelink_store::memory_store::MemoryStore; +use nativelink_util::common::DigestInfo; +use nativelink_util::store_trait::Store; use rand::rngs::SmallRng; use rand::{Rng, SeedableRng}; -fn make_default_config() -> native_link_config::stores::DedupStore { - native_link_config::stores::DedupStore { - index_store: native_link_config::stores::StoreConfig::memory(native_link_config::stores::MemoryStore::default()), - content_store: native_link_config::stores::StoreConfig::memory( - native_link_config::stores::MemoryStore::default(), +fn make_default_config() -> nativelink_config::stores::DedupStore { + nativelink_config::stores::DedupStore { + index_store: nativelink_config::stores::StoreConfig::memory(nativelink_config::stores::MemoryStore::default()), + content_store: nativelink_config::stores::StoreConfig::memory( + nativelink_config::stores::MemoryStore::default(), ), min_size: 8 * 1024, normal_size: 32 * 1024, @@ -57,8 +57,8 @@ mod dedup_store_tests { async fn simple_round_trip_test() -> Result<(), Error> { let store_owned = DedupStore::new( &make_default_config(), - Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())), // Index store. - Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())), // Content store. + Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())), // Index store. + Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())), // Content store. ); let store = Pin::new(&store_owned); @@ -81,10 +81,10 @@ mod dedup_store_tests { #[tokio::test] async fn check_missing_last_chunk_test() -> Result<(), Error> { - let content_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); + let content_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); let store_owned = DedupStore::new( &make_default_config(), - Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())), // Index store. + Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())), // Index store. content_store.clone(), ); let store = Pin::new(&store_owned); @@ -124,8 +124,8 @@ mod dedup_store_tests { async fn fetch_part_test() -> Result<(), Error> { let store_owned = DedupStore::new( &make_default_config(), - Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())), // Index store. - Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())), // Content store. + Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())), // Index store. + Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())), // Content store. ); let store = Pin::new(&store_owned); @@ -156,20 +156,20 @@ mod dedup_store_tests { #[tokio::test] async fn check_length_not_set_with_chunk_read_beyond_first_chunk_regression_test() -> Result<(), Error> { let store_owned = DedupStore::new( - &native_link_config::stores::DedupStore { - index_store: native_link_config::stores::StoreConfig::memory( - native_link_config::stores::MemoryStore::default(), + &nativelink_config::stores::DedupStore { + index_store: nativelink_config::stores::StoreConfig::memory( + nativelink_config::stores::MemoryStore::default(), ), - content_store: native_link_config::stores::StoreConfig::memory( - native_link_config::stores::MemoryStore::default(), + content_store: nativelink_config::stores::StoreConfig::memory( + nativelink_config::stores::MemoryStore::default(), ), min_size: 5, normal_size: 6, max_size: 7, max_concurrent_fetch_per_get: 10, }, - Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())), // Index store. - Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())), // Content store. + Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())), // Index store. + Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())), // Content store. ); let store = Pin::new(&store_owned); @@ -205,20 +205,20 @@ mod dedup_store_tests { #[tokio::test] async fn check_chunk_boundary_reads_test() -> Result<(), Error> { let store_owned = DedupStore::new( - &native_link_config::stores::DedupStore { - index_store: native_link_config::stores::StoreConfig::memory( - native_link_config::stores::MemoryStore::default(), + &nativelink_config::stores::DedupStore { + index_store: nativelink_config::stores::StoreConfig::memory( + nativelink_config::stores::MemoryStore::default(), ), - content_store: native_link_config::stores::StoreConfig::memory( - native_link_config::stores::MemoryStore::default(), + content_store: nativelink_config::stores::StoreConfig::memory( + nativelink_config::stores::MemoryStore::default(), ), min_size: 5, normal_size: 6, max_size: 7, max_concurrent_fetch_per_get: 10, }, - Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())), // Index store. - Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())), // Content store. + Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())), // Index store. + Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())), // Content store. ); let store = Pin::new(&store_owned); @@ -275,9 +275,9 @@ mod dedup_store_tests { /// content items exist instead of just checking the entry in the index store. #[tokio::test] async fn has_checks_content_store() -> Result<(), Error> { - let index_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); - let content_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore { - eviction_policy: Some(native_link_config::stores::EvictionPolicy { + let index_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); + let content_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore { + eviction_policy: Some(nativelink_config::stores::EvictionPolicy { max_count: 10, ..Default::default() }), @@ -330,9 +330,9 @@ mod dedup_store_tests { /// properly return None. #[tokio::test] async fn has_with_no_existing_index_returns_none_test() -> Result<(), Error> { - let index_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); - let content_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore { - eviction_policy: Some(native_link_config::stores::EvictionPolicy { + let index_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); + let content_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore { + eviction_policy: Some(nativelink_config::stores::EvictionPolicy { max_count: 10, ..Default::default() }), diff --git a/native-link-store/tests/existence_store_test.rs b/nativelink-store/tests/existence_store_test.rs similarity index 85% rename from native-link-store/tests/existence_store_test.rs rename to nativelink-store/tests/existence_store_test.rs index 660d59f5d..a0cca5938 100644 --- a/native-link-store/tests/existence_store_test.rs +++ b/nativelink-store/tests/existence_store_test.rs @@ -16,11 +16,11 @@ use std::pin::Pin; use std::sync::Arc; use error::{Error, ResultExt}; -use native_link_config::stores::{ExistenceCacheStore as ExistenceCacheStoreConfig, StoreConfig}; -use native_link_store::existence_cache_store::ExistenceCacheStore; -use native_link_store::memory_store::MemoryStore; -use native_link_util::common::DigestInfo; -use native_link_util::store_trait::Store; +use nativelink_config::stores::{ExistenceCacheStore as ExistenceCacheStoreConfig, StoreConfig}; +use nativelink_store::existence_cache_store::ExistenceCacheStore; +use nativelink_store::memory_store::MemoryStore; +use nativelink_util::common::DigestInfo; +use nativelink_util::store_trait::Store; #[cfg(test)] mod verify_store_tests { @@ -35,7 +35,7 @@ mod verify_store_tests { backend: StoreConfig::noop, // Note: Not used. eviction_policy: Default::default(), }; - let inner_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); + let inner_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); let store_owned = ExistenceCacheStore::new(&config, inner_store.clone()); let store = Pin::new(&store_owned); @@ -71,7 +71,7 @@ mod verify_store_tests { backend: StoreConfig::noop, eviction_policy: Default::default(), }; - let inner_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); + let inner_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); let store = ExistenceCacheStore::new(&config, inner_store.clone()); let digest = DigestInfo::try_new(VALID_HASH1, 3).unwrap(); @@ -94,7 +94,7 @@ mod verify_store_tests { backend: StoreConfig::noop, eviction_policy: Default::default(), }; - let inner_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); + let inner_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); let digest = DigestInfo::try_new(VALID_HASH1, 3).unwrap(); Pin::new(inner_store.as_ref()) .update_oneshot(digest, VALUE.into()) diff --git a/native-link-store/tests/fast_slow_store_test.rs b/nativelink-store/tests/fast_slow_store_test.rs similarity index 91% rename from native-link-store/tests/fast_slow_store_test.rs rename to nativelink-store/tests/fast_slow_store_test.rs index b7d815215..8344267cc 100644 --- a/native-link-store/tests/fast_slow_store_test.rs +++ b/nativelink-store/tests/fast_slow_store_test.rs @@ -16,22 +16,22 @@ use std::pin::Pin; use std::sync::Arc; use error::Error; -use native_link_store::fast_slow_store::FastSlowStore; -use native_link_store::memory_store::MemoryStore; -use native_link_util::common::DigestInfo; -use native_link_util::store_trait::Store; +use nativelink_store::fast_slow_store::FastSlowStore; +use nativelink_store::memory_store::MemoryStore; +use nativelink_util::common::DigestInfo; +use nativelink_util::store_trait::Store; use rand::rngs::SmallRng; use rand::{Rng, SeedableRng}; const MEGABYTE_SZ: usize = 1024 * 1024; fn make_stores() -> (Arc, Arc, Arc) { - let fast_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); - let slow_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); + let fast_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); + let slow_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); let fast_slow_store = Arc::new(FastSlowStore::new( - &native_link_config::stores::FastSlowStore { - fast: native_link_config::stores::StoreConfig::memory(native_link_config::stores::MemoryStore::default()), - slow: native_link_config::stores::StoreConfig::memory(native_link_config::stores::MemoryStore::default()), + &nativelink_config::stores::FastSlowStore { + fast: nativelink_config::stores::StoreConfig::memory(nativelink_config::stores::MemoryStore::default()), + slow: nativelink_config::stores::StoreConfig::memory(nativelink_config::stores::MemoryStore::default()), }, fast_store.clone(), slow_store.clone(), diff --git a/native-link-store/tests/filesystem_store_test.rs b/nativelink-store/tests/filesystem_store_test.rs similarity index 95% rename from native-link-store/tests/filesystem_store_test.rs rename to nativelink-store/tests/filesystem_store_test.rs index 33bdf5095..d0a446711 100644 --- a/native-link-store/tests/filesystem_store_test.rs +++ b/nativelink-store/tests/filesystem_store_test.rs @@ -32,13 +32,13 @@ use filetime::{set_file_atime, FileTime}; use futures::executor::block_on; use futures::task::Poll; use futures::{poll, Future}; -use native_link_store::filesystem_store::{ +use nativelink_store::filesystem_store::{ digest_from_filename, EncodedFilePath, FileEntry, FileEntryImpl, FilesystemStore, }; -use native_link_util::buf_channel::{make_buf_channel_pair, DropCloserReadHalf}; -use native_link_util::common::{fs, DigestInfo, JoinHandleDropGuard}; -use native_link_util::evicting_map::LenEntry; -use native_link_util::store_trait::{Store, UploadSizeInfo}; +use nativelink_util::buf_channel::{make_buf_channel_pair, DropCloserReadHalf}; +use nativelink_util::common::{fs, DigestInfo, JoinHandleDropGuard}; +use nativelink_util::evicting_map::LenEntry; +use nativelink_util::store_trait::{Store, UploadSizeInfo}; use once_cell::sync::Lazy; use rand::{thread_rng, Rng}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -210,7 +210,7 @@ mod filesystem_store_tests { let temp_path = make_temp_path("temp_path"); { let store = Box::pin( - FilesystemStore::::new(&native_link_config::stores::FilesystemStore { + FilesystemStore::::new(&nativelink_config::stores::FilesystemStore { content_path: content_path.clone(), temp_path: temp_path.clone(), eviction_policy: None, @@ -231,7 +231,7 @@ mod filesystem_store_tests { { // With a new store ensure content is still readable (ie: restores from shutdown). let store = Box::pin( - FilesystemStore::::new(&native_link_config::stores::FilesystemStore { + FilesystemStore::::new(&nativelink_config::stores::FilesystemStore { content_path, temp_path, eviction_policy: None, @@ -262,10 +262,10 @@ mod filesystem_store_tests { } let store = Box::pin( - FilesystemStore::>::new(&native_link_config::stores::FilesystemStore { + FilesystemStore::>::new(&nativelink_config::stores::FilesystemStore { content_path: content_path.clone(), temp_path: temp_path.clone(), - eviction_policy: Some(native_link_config::stores::EvictionPolicy { + eviction_policy: Some(nativelink_config::stores::EvictionPolicy { max_count: 3, ..Default::default() }), @@ -336,10 +336,10 @@ mod filesystem_store_tests { } let store = Arc::new( - FilesystemStore::>::new(&native_link_config::stores::FilesystemStore { + FilesystemStore::>::new(&nativelink_config::stores::FilesystemStore { content_path: content_path.clone(), temp_path: temp_path.clone(), - eviction_policy: Some(native_link_config::stores::EvictionPolicy { + eviction_policy: Some(nativelink_config::stores::EvictionPolicy { max_count: 3, ..Default::default() }), @@ -440,10 +440,10 @@ mod filesystem_store_tests { } let store = Arc::new( - FilesystemStore::>::new(&native_link_config::stores::FilesystemStore { + FilesystemStore::>::new(&nativelink_config::stores::FilesystemStore { content_path: content_path.clone(), temp_path: temp_path.clone(), - eviction_policy: Some(native_link_config::stores::EvictionPolicy { + eviction_policy: Some(nativelink_config::stores::EvictionPolicy { max_count: 1, ..Default::default() }), @@ -523,7 +523,7 @@ mod filesystem_store_tests { let digest1 = DigestInfo::try_new(HASH1, VALUE1.len())?; let store = Box::pin( - FilesystemStore::::new(&native_link_config::stores::FilesystemStore { + FilesystemStore::::new(&nativelink_config::stores::FilesystemStore { content_path: make_temp_path("content_path"), temp_path: make_temp_path("temp_path"), eviction_policy: None, @@ -590,10 +590,10 @@ mod filesystem_store_tests { // Load the existing store from disk. let store = Box::pin( - FilesystemStore::::new(&native_link_config::stores::FilesystemStore { + FilesystemStore::::new(&nativelink_config::stores::FilesystemStore { content_path, temp_path: make_temp_path("temp_path"), - eviction_policy: Some(native_link_config::stores::EvictionPolicy { + eviction_policy: Some(nativelink_config::stores::EvictionPolicy { max_bytes: 0, max_seconds: 0, max_count: 1, @@ -620,7 +620,7 @@ mod filesystem_store_tests { let digest1 = DigestInfo::try_new(HASH1, VALUE1.len())?; let store = Box::pin( - FilesystemStore::::new(&native_link_config::stores::FilesystemStore { + FilesystemStore::::new(&nativelink_config::stores::FilesystemStore { content_path: make_temp_path("content_path"), temp_path: make_temp_path("temp_path"), eviction_policy: None, @@ -666,7 +666,7 @@ mod filesystem_store_tests { let digest = DigestInfo::try_new(HASH1, VALUE1.len())?; let store = Box::pin( - FilesystemStore::::new(&native_link_config::stores::FilesystemStore { + FilesystemStore::::new(&nativelink_config::stores::FilesystemStore { content_path: make_temp_path("content_path"), temp_path: make_temp_path("temp_path"), eviction_policy: None, @@ -720,10 +720,10 @@ mod filesystem_store_tests { } let store = Box::pin( - FilesystemStore::>::new(&native_link_config::stores::FilesystemStore { + FilesystemStore::>::new(&nativelink_config::stores::FilesystemStore { content_path: make_temp_path("content_path"), temp_path: make_temp_path("temp_path"), - eviction_policy: Some(native_link_config::stores::EvictionPolicy { + eviction_policy: Some(nativelink_config::stores::EvictionPolicy { max_bytes: 5, ..Default::default() }), @@ -763,7 +763,7 @@ mod filesystem_store_tests { } let store = Box::pin( - FilesystemStore::>::new(&native_link_config::stores::FilesystemStore { + FilesystemStore::>::new(&nativelink_config::stores::FilesystemStore { content_path: content_path.clone(), temp_path: temp_path.clone(), eviction_policy: None, @@ -875,7 +875,7 @@ mod filesystem_store_tests { let store = Arc::new( FilesystemStore::::new_with_timeout( - &native_link_config::stores::FilesystemStore { + &nativelink_config::stores::FilesystemStore { content_path: content_path.clone(), temp_path: temp_path.clone(), read_buffer_size: 1, diff --git a/native-link-store/tests/memory_store_test.rs b/nativelink-store/tests/memory_store_test.rs similarity index 93% rename from native-link-store/tests/memory_store_test.rs rename to nativelink-store/tests/memory_store_test.rs index 39d388c44..f1624348d 100644 --- a/native-link-store/tests/memory_store_test.rs +++ b/nativelink-store/tests/memory_store_test.rs @@ -17,9 +17,9 @@ use std::pin::Pin; use bytes::{BufMut, BytesMut}; use error::{Error, ResultExt}; use memory_stats::memory_stats; -use native_link_store::memory_store::MemoryStore; -use native_link_util::common::DigestInfo; -use native_link_util::store_trait::Store; +use nativelink_store::memory_store::MemoryStore; +use nativelink_util::common::DigestInfo; +use nativelink_util::store_trait::Store; const VALID_HASH1: &str = "0123456789abcdef000000000000000000010000000000000123456789abcdef"; const VALID_HASH2: &str = "0123456789abcdef000000000000000000020000000000000123456789abcdef"; @@ -39,7 +39,7 @@ mod memory_store_tests { async fn insert_one_item_then_update() -> Result<(), Error> { const VALUE1: &str = "13"; const VALUE2: &str = "23"; - let store_owned = MemoryStore::new(&native_link_config::stores::MemoryStore::default()); + let store_owned = MemoryStore::new(&nativelink_config::stores::MemoryStore::default()); let store = Pin::new(&store_owned); // Insert dummy value into store. @@ -81,7 +81,7 @@ mod memory_store_tests { // too low for some kernels/operating systems. const MAXIMUM_MEMORY_USAGE_INCREASE_PERC: f64 = 1.3; // 30% increase. - let store_owned = MemoryStore::new(&native_link_config::stores::MemoryStore::default()); + let store_owned = MemoryStore::new(&nativelink_config::stores::MemoryStore::default()); let store = Pin::new(&store_owned); let initial_virtual_mem = memory_stats().err_tip(|| "Failed to read memory.")?.physical_mem; @@ -115,7 +115,7 @@ mod memory_store_tests { #[tokio::test] async fn read_partial() -> Result<(), Error> { const VALUE1: &str = "1234"; - let store_owned = MemoryStore::new(&native_link_config::stores::MemoryStore::default()); + let store_owned = MemoryStore::new(&nativelink_config::stores::MemoryStore::default()); let store = Pin::new(&store_owned); let digest = DigestInfo::try_new(VALID_HASH1, 4).unwrap(); @@ -138,7 +138,7 @@ mod memory_store_tests { #[tokio::test] async fn read_zero_size_item_test() -> Result<(), Error> { const VALUE: &str = ""; - let store_owned = MemoryStore::new(&native_link_config::stores::MemoryStore::default()); + let store_owned = MemoryStore::new(&nativelink_config::stores::MemoryStore::default()); let store = Pin::new(&store_owned); // Insert dummy value into store. @@ -158,7 +158,7 @@ mod memory_store_tests { #[tokio::test] async fn errors_with_invalid_inputs() -> Result<(), Error> { const VALUE1: &str = "123"; - let store_owned = MemoryStore::new(&native_link_config::stores::MemoryStore::default()); + let store_owned = MemoryStore::new(&nativelink_config::stores::MemoryStore::default()); let store = Pin::new(&store_owned); { // .has() tests. diff --git a/native-link-store/tests/ref_store_test.rs b/nativelink-store/tests/ref_store_test.rs similarity index 90% rename from native-link-store/tests/ref_store_test.rs rename to nativelink-store/tests/ref_store_test.rs index 1a91ece66..e27654571 100644 --- a/native-link-store/tests/ref_store_test.rs +++ b/nativelink-store/tests/ref_store_test.rs @@ -18,11 +18,11 @@ use std::sync::Arc; #[cfg(test)] mod ref_store_tests { use error::Error; - use native_link_store::memory_store::MemoryStore; - use native_link_store::ref_store::RefStore; - use native_link_store::store_manager::StoreManager; - use native_link_util::common::DigestInfo; - use native_link_util::store_trait::Store; + use nativelink_store::memory_store::MemoryStore; + use nativelink_store::ref_store::RefStore; + use nativelink_store::store_manager::StoreManager; + use nativelink_util::common::DigestInfo; + use nativelink_util::store_trait::Store; use pretty_assertions::assert_eq; // Must be declared in every module. use super::*; @@ -32,11 +32,11 @@ mod ref_store_tests { fn setup_stores() -> (Arc, Arc, Arc) { let store_manager = Arc::new(StoreManager::new()); - let memory_store_owned = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); + let memory_store_owned = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); store_manager.add_store("foo", memory_store_owned.clone()); let ref_store_owned = Arc::new(RefStore::new( - &native_link_config::stores::RefStore { + &nativelink_config::stores::RefStore { name: "foo".to_string(), }, Arc::downgrade(&store_manager), @@ -129,11 +129,11 @@ mod ref_store_tests { async fn inner_store_test() -> Result<(), Error> { let store_manager = Arc::new(StoreManager::new()); - let memory_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); + let memory_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); store_manager.add_store("mem_store", memory_store.clone()); let ref_store_inner = Arc::new(RefStore::new( - &native_link_config::stores::RefStore { + &nativelink_config::stores::RefStore { name: "mem_store".to_string(), }, Arc::downgrade(&store_manager), @@ -141,7 +141,7 @@ mod ref_store_tests { store_manager.add_store("ref_store_inner", ref_store_inner.clone()); let ref_store_outer = Arc::new(RefStore::new( - &native_link_config::stores::RefStore { + &nativelink_config::stores::RefStore { name: "ref_store_inner".to_string(), }, Arc::downgrade(&store_manager), diff --git a/native-link-store/tests/s3_store_test.rs b/nativelink-store/tests/s3_store_test.rs similarity index 96% rename from native-link-store/tests/s3_store_test.rs rename to nativelink-store/tests/s3_store_test.rs index 3895e089c..91fbca3a6 100644 --- a/native-link-store/tests/s3_store_test.rs +++ b/nativelink-store/tests/s3_store_test.rs @@ -25,9 +25,9 @@ use futures::join; use http::header; use http::status::StatusCode; use hyper::Body; -use native_link_store::s3_store::S3Store; -use native_link_util::common::DigestInfo; -use native_link_util::store_trait::Store; +use nativelink_store::s3_store::S3Store; +use nativelink_util::common::DigestInfo; +use nativelink_util::store_trait::Store; // TODO(aaronmondal): Figure out how to test the connector retry mechanism. @@ -56,7 +56,7 @@ mod s3_store_tests { .build(); let s3_client = aws_sdk_s3::Client::from_conf(test_config); let store = S3Store::new_with_client_and_jitter( - &native_link_config::stores::S3Store { + &nativelink_config::stores::S3Store { bucket: BUCKET_NAME.to_string(), ..Default::default() }, @@ -86,7 +86,7 @@ mod s3_store_tests { .build(); let s3_client = aws_sdk_s3::Client::from_conf(test_config); let store = S3Store::new_with_client_and_jitter( - &native_link_config::stores::S3Store { + &nativelink_config::stores::S3Store { bucket: BUCKET_NAME.to_string(), ..Default::default() }, @@ -140,9 +140,9 @@ mod s3_store_tests { let s3_client = aws_sdk_s3::Client::from_conf(test_config); let store = S3Store::new_with_client_and_jitter( - &native_link_config::stores::S3Store { + &nativelink_config::stores::S3Store { bucket: BUCKET_NAME.to_string(), - retry: native_link_config::stores::Retry { + retry: nativelink_config::stores::Retry { max_retries: 1024, delay: 0., jitter: 0., @@ -190,7 +190,7 @@ mod s3_store_tests { .build(); let s3_client = aws_sdk_s3::Client::from_conf(test_config); let store = S3Store::new_with_client_and_jitter( - &native_link_config::stores::S3Store { + &nativelink_config::stores::S3Store { bucket: BUCKET_NAME.to_string(), ..Default::default() }, @@ -226,7 +226,7 @@ mod s3_store_tests { .build(); let s3_client = aws_sdk_s3::Client::from_conf(test_config); let store = S3Store::new_with_client_and_jitter( - &native_link_config::stores::S3Store { + &nativelink_config::stores::S3Store { bucket: BUCKET_NAME.to_string(), ..Default::default() }, @@ -270,7 +270,7 @@ mod s3_store_tests { .build(); let s3_client = aws_sdk_s3::Client::from_conf(test_config); let store = S3Store::new_with_client_and_jitter( - &native_link_config::stores::S3Store { + &nativelink_config::stores::S3Store { bucket: BUCKET_NAME.to_string(), ..Default::default() }, @@ -331,9 +331,9 @@ mod s3_store_tests { let s3_client = aws_sdk_s3::Client::from_conf(test_config); let store = S3Store::new_with_client_and_jitter( - &native_link_config::stores::S3Store { + &nativelink_config::stores::S3Store { bucket: BUCKET_NAME.to_string(), - retry: native_link_config::stores::Retry { + retry: nativelink_config::stores::Retry { max_retries: 1024, delay: 0., jitter: 0., @@ -456,7 +456,7 @@ mod s3_store_tests { .build(); let s3_client = aws_sdk_s3::Client::from_conf(test_config); let store = S3Store::new_with_client_and_jitter( - &native_link_config::stores::S3Store { + &nativelink_config::stores::S3Store { bucket: BUCKET_NAME.to_string(), ..Default::default() }, @@ -491,7 +491,7 @@ mod s3_store_tests { .build(); let s3_client = aws_sdk_s3::Client::from_conf(test_config); let store = S3Store::new_with_client_and_jitter( - &native_link_config::stores::S3Store { + &nativelink_config::stores::S3Store { bucket: BUCKET_NAME.to_string(), ..Default::default() }, diff --git a/native-link-store/tests/shard_store_test.rs b/nativelink-store/tests/shard_store_test.rs similarity index 94% rename from native-link-store/tests/shard_store_test.rs rename to nativelink-store/tests/shard_store_test.rs index 4cbf67c5c..ca0ada97d 100644 --- a/native-link-store/tests/shard_store_test.rs +++ b/nativelink-store/tests/shard_store_test.rs @@ -16,28 +16,28 @@ use std::pin::Pin; use std::sync::Arc; use error::Error; -use native_link_store::memory_store::MemoryStore; -use native_link_store::shard_store::ShardStore; -use native_link_util::common::DigestInfo; -use native_link_util::store_trait::Store; +use nativelink_store::memory_store::MemoryStore; +use nativelink_store::shard_store::ShardStore; +use nativelink_util::common::DigestInfo; +use nativelink_util::store_trait::Store; use rand::rngs::SmallRng; use rand::{Rng, SeedableRng}; const MEGABYTE_SZ: usize = 1024 * 1024; fn make_stores(weights: &[u32]) -> (Arc, Vec>) { - let memory_store_config = native_link_config::stores::MemoryStore::default(); - let store_config = native_link_config::stores::StoreConfig::memory(memory_store_config.clone()); + let memory_store_config = nativelink_config::stores::MemoryStore::default(); + let store_config = nativelink_config::stores::StoreConfig::memory(memory_store_config.clone()); let stores: Vec<_> = weights .iter() .map(|_| -> Arc { Arc::new(MemoryStore::new(&memory_store_config)) }) .collect(); let shard_store = Arc::new( ShardStore::new( - &native_link_config::stores::ShardStore { + &nativelink_config::stores::ShardStore { stores: weights .iter() - .map(|weight| native_link_config::stores::ShardConfig { + .map(|weight| nativelink_config::stores::ShardConfig { store: store_config.clone(), weight: Some(*weight), }) diff --git a/native-link-store/tests/size_partitioning_store_test.rs b/nativelink-store/tests/size_partitioning_store_test.rs similarity index 90% rename from native-link-store/tests/size_partitioning_store_test.rs rename to nativelink-store/tests/size_partitioning_store_test.rs index 4d2094edd..9f3d10ec4 100644 --- a/native-link-store/tests/size_partitioning_store_test.rs +++ b/nativelink-store/tests/size_partitioning_store_test.rs @@ -18,10 +18,10 @@ use std::sync::Arc; #[cfg(test)] mod ref_store_tests { use error::Error; - use native_link_store::memory_store::MemoryStore; - use native_link_store::size_partitioning_store::SizePartitioningStore; - use native_link_util::common::DigestInfo; - use native_link_util::store_trait::Store; + use nativelink_store::memory_store::MemoryStore; + use nativelink_store::size_partitioning_store::SizePartitioningStore; + use nativelink_util::common::DigestInfo; + use nativelink_util::store_trait::Store; use pretty_assertions::assert_eq; // Must be declared in every module. use super::*; @@ -35,17 +35,17 @@ mod ref_store_tests { const BIG_VALUE: &str = "123456789"; fn setup_stores(size: u64) -> (SizePartitioningStore, Arc, Arc) { - let lower_memory_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); - let upper_memory_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); + let lower_memory_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); + let upper_memory_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); let size_part_store = SizePartitioningStore::new( - &native_link_config::stores::SizePartitioningStore { + &nativelink_config::stores::SizePartitioningStore { size, - lower_store: native_link_config::stores::StoreConfig::memory( - native_link_config::stores::MemoryStore::default(), + lower_store: nativelink_config::stores::StoreConfig::memory( + nativelink_config::stores::MemoryStore::default(), ), - upper_store: native_link_config::stores::StoreConfig::memory( - native_link_config::stores::MemoryStore::default(), + upper_store: nativelink_config::stores::StoreConfig::memory( + nativelink_config::stores::MemoryStore::default(), ), }, lower_memory_store.clone(), diff --git a/native-link-store/tests/verify_store_test.rs b/nativelink-store/tests/verify_store_test.rs similarity index 77% rename from native-link-store/tests/verify_store_test.rs rename to nativelink-store/tests/verify_store_test.rs index 5cbac7d62..20548404c 100644 --- a/native-link-store/tests/verify_store_test.rs +++ b/nativelink-store/tests/verify_store_test.rs @@ -20,11 +20,11 @@ use futures::try_join; #[cfg(test)] mod verify_store_tests { use error::{Error, ResultExt}; - use native_link_store::memory_store::MemoryStore; - use native_link_store::verify_store::VerifyStore; - use native_link_util::buf_channel::make_buf_channel_pair; - use native_link_util::common::DigestInfo; - use native_link_util::store_trait::{Store, UploadSizeInfo}; + use nativelink_store::memory_store::MemoryStore; + use nativelink_store::verify_store::VerifyStore; + use nativelink_util::buf_channel::make_buf_channel_pair; + use nativelink_util::common::DigestInfo; + use nativelink_util::store_trait::{Store, UploadSizeInfo}; use pretty_assertions::assert_eq; // Must be declared in every module. use super::*; @@ -33,11 +33,11 @@ mod verify_store_tests { #[tokio::test] async fn verify_size_false_passes_on_update() -> Result<(), Error> { - let inner_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); + let inner_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); let store_owned = VerifyStore::new( - &native_link_config::stores::VerifyStore { - backend: native_link_config::stores::StoreConfig::memory( - native_link_config::stores::MemoryStore::default(), + &nativelink_config::stores::VerifyStore { + backend: nativelink_config::stores::StoreConfig::memory( + nativelink_config::stores::MemoryStore::default(), ), verify_size: false, verify_hash: false, @@ -65,11 +65,11 @@ mod verify_store_tests { #[tokio::test] async fn verify_size_true_fails_on_update() -> Result<(), Error> { - let inner_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); + let inner_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); let store_owned = VerifyStore::new( - &native_link_config::stores::VerifyStore { - backend: native_link_config::stores::StoreConfig::memory( - native_link_config::stores::MemoryStore::default(), + &nativelink_config::stores::VerifyStore { + backend: nativelink_config::stores::StoreConfig::memory( + nativelink_config::stores::MemoryStore::default(), ), verify_size: true, verify_hash: false, @@ -105,11 +105,11 @@ mod verify_store_tests { #[tokio::test] async fn verify_size_true_suceeds_on_update() -> Result<(), Error> { - let inner_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); + let inner_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); let store_owned = VerifyStore::new( - &native_link_config::stores::VerifyStore { - backend: native_link_config::stores::StoreConfig::memory( - native_link_config::stores::MemoryStore::default(), + &nativelink_config::stores::VerifyStore { + backend: nativelink_config::stores::StoreConfig::memory( + nativelink_config::stores::MemoryStore::default(), ), verify_size: true, verify_hash: false, @@ -132,11 +132,11 @@ mod verify_store_tests { #[tokio::test] async fn verify_size_true_suceeds_on_multi_chunk_stream_update() -> Result<(), Error> { - let inner_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); + let inner_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); let store_owned = VerifyStore::new( - &native_link_config::stores::VerifyStore { - backend: native_link_config::stores::StoreConfig::memory( - native_link_config::stores::MemoryStore::default(), + &nativelink_config::stores::VerifyStore { + backend: nativelink_config::stores::StoreConfig::memory( + nativelink_config::stores::MemoryStore::default(), ), verify_size: true, verify_hash: false, @@ -168,11 +168,11 @@ mod verify_store_tests { #[tokio::test] async fn verify_hash_true_suceeds_on_update() -> Result<(), Error> { - let inner_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); + let inner_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); let store_owned = VerifyStore::new( - &native_link_config::stores::VerifyStore { - backend: native_link_config::stores::StoreConfig::memory( - native_link_config::stores::MemoryStore::default(), + &nativelink_config::stores::VerifyStore { + backend: nativelink_config::stores::StoreConfig::memory( + nativelink_config::stores::MemoryStore::default(), ), verify_size: false, verify_hash: true, @@ -197,11 +197,11 @@ mod verify_store_tests { #[tokio::test] async fn verify_hash_true_fails_on_update() -> Result<(), Error> { - let inner_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); + let inner_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); let store_owned = VerifyStore::new( - &native_link_config::stores::VerifyStore { - backend: native_link_config::stores::StoreConfig::memory( - native_link_config::stores::MemoryStore::default(), + &nativelink_config::stores::VerifyStore { + backend: nativelink_config::stores::StoreConfig::memory( + nativelink_config::stores::MemoryStore::default(), ), verify_size: false, verify_hash: true, diff --git a/native-link-util/BUILD.bazel b/nativelink-util/BUILD.bazel similarity index 92% rename from native-link-util/BUILD.bazel rename to nativelink-util/BUILD.bazel index 607e827fe..0674b7b44 100644 --- a/native-link-util/BUILD.bazel +++ b/nativelink-util/BUILD.bazel @@ -7,7 +7,7 @@ load( ) rust_library( - name = "native-link-util", + name = "nativelink-util", srcs = [ "src/action_messages.rs", "src/async_fixed_buffer.rs", @@ -32,7 +32,7 @@ rust_library( visibility = ["//visibility:public"], deps = [ "//error", - "//native-link-config", + "//nativelink-config", "//proto", "@crate_index//:async-lock", "@crate_index//:blake3", @@ -73,9 +73,9 @@ rust_test_suite( "@crate_index//:async-trait", ], deps = [ - ":native-link-util", + ":nativelink-util", "//error", - "//native-link-config", + "//nativelink-config", "@crate_index//:bytes", "@crate_index//:futures", "@crate_index//:hex", @@ -90,10 +90,10 @@ rust_test_suite( rust_doc( name = "docs", - crate = ":native-link-util", + crate = ":nativelink-util", ) rust_doc_test( name = "doc_test", - crate = ":native-link-util", + crate = ":nativelink-util", ) diff --git a/native-link-util/Cargo.toml b/nativelink-util/Cargo.toml similarity index 89% rename from native-link-util/Cargo.toml rename to nativelink-util/Cargo.toml index 67652ee16..981d41d77 100644 --- a/native-link-util/Cargo.toml +++ b/nativelink-util/Cargo.toml @@ -1,10 +1,10 @@ [package] -name = "native-link-util" +name = "nativelink-util" version = "0.0.0" edition = "2021" [dependencies] -native-link-config = { path = "../native-link-config" } +nativelink-config = { path = "../nativelink-config" } error = { path = "../error" } proto = { path = "../proto" } diff --git a/native-link-util/src/action_messages.rs b/nativelink-util/src/action_messages.rs similarity index 100% rename from native-link-util/src/action_messages.rs rename to nativelink-util/src/action_messages.rs diff --git a/native-link-util/src/async_fixed_buffer.rs b/nativelink-util/src/async_fixed_buffer.rs similarity index 100% rename from native-link-util/src/async_fixed_buffer.rs rename to nativelink-util/src/async_fixed_buffer.rs diff --git a/native-link-util/src/buf_channel.rs b/nativelink-util/src/buf_channel.rs similarity index 100% rename from native-link-util/src/buf_channel.rs rename to nativelink-util/src/buf_channel.rs diff --git a/native-link-util/src/common.rs b/nativelink-util/src/common.rs similarity index 100% rename from native-link-util/src/common.rs rename to nativelink-util/src/common.rs diff --git a/native-link-util/src/digest_hasher.rs b/nativelink-util/src/digest_hasher.rs similarity index 98% rename from native-link-util/src/digest_hasher.rs rename to nativelink-util/src/digest_hasher.rs index 24d3e591d..a7c69adaa 100644 --- a/native-link-util/src/digest_hasher.rs +++ b/nativelink-util/src/digest_hasher.rs @@ -16,7 +16,7 @@ use std::sync::OnceLock; use blake3::Hasher as Blake3Hasher; use error::{make_err, make_input_err, Code, Error}; -use native_link_config::cas_server::ConfigDigestHashFunction; +use nativelink_config::cas_server::ConfigDigestHashFunction; use proto::build::bazel::remote::execution::v2::digest_function::Value as ProtoDigestFunction; use sha2::{Digest, Sha256}; diff --git a/native-link-util/src/evicting_map.rs b/nativelink-util/src/evicting_map.rs similarity index 99% rename from native-link-util/src/evicting_map.rs rename to nativelink-util/src/evicting_map.rs index 339b54989..01de27456 100644 --- a/native-link-util/src/evicting_map.rs +++ b/nativelink-util/src/evicting_map.rs @@ -22,7 +22,7 @@ use async_trait::async_trait; use futures::stream::FuturesUnordered; use futures::{future, StreamExt}; use lru::LruCache; -use native_link_config::stores::EvictionPolicy; +use nativelink_config::stores::EvictionPolicy; use serde::{Deserialize, Serialize}; use tracing::info; diff --git a/native-link-util/src/fastcdc.rs b/nativelink-util/src/fastcdc.rs similarity index 100% rename from native-link-util/src/fastcdc.rs rename to nativelink-util/src/fastcdc.rs diff --git a/native-link-util/src/fs.rs b/nativelink-util/src/fs.rs similarity index 100% rename from native-link-util/src/fs.rs rename to nativelink-util/src/fs.rs diff --git a/native-link-util/src/lib.rs b/nativelink-util/src/lib.rs similarity index 100% rename from native-link-util/src/lib.rs rename to nativelink-util/src/lib.rs diff --git a/native-link-util/src/metrics_utils.rs b/nativelink-util/src/metrics_utils.rs similarity index 100% rename from native-link-util/src/metrics_utils.rs rename to nativelink-util/src/metrics_utils.rs diff --git a/native-link-util/src/platform_properties.rs b/nativelink-util/src/platform_properties.rs similarity index 100% rename from native-link-util/src/platform_properties.rs rename to nativelink-util/src/platform_properties.rs diff --git a/native-link-util/src/resource_info.rs b/nativelink-util/src/resource_info.rs similarity index 100% rename from native-link-util/src/resource_info.rs rename to nativelink-util/src/resource_info.rs diff --git a/native-link-util/src/retry.rs b/nativelink-util/src/retry.rs similarity index 100% rename from native-link-util/src/retry.rs rename to nativelink-util/src/retry.rs diff --git a/native-link-util/src/store_trait.rs b/nativelink-util/src/store_trait.rs similarity index 100% rename from native-link-util/src/store_trait.rs rename to nativelink-util/src/store_trait.rs diff --git a/native-link-util/src/write_counter.rs b/nativelink-util/src/write_counter.rs similarity index 100% rename from native-link-util/src/write_counter.rs rename to nativelink-util/src/write_counter.rs diff --git a/native-link-util/src/write_request_stream_wrapper.rs b/nativelink-util/src/write_request_stream_wrapper.rs similarity index 100% rename from native-link-util/src/write_request_stream_wrapper.rs rename to nativelink-util/src/write_request_stream_wrapper.rs diff --git a/native-link-util/tests/async_fixed_buffer_test.rs b/nativelink-util/tests/async_fixed_buffer_test.rs similarity index 99% rename from native-link-util/tests/async_fixed_buffer_test.rs rename to nativelink-util/tests/async_fixed_buffer_test.rs index e2de5e960..d461929ff 100644 --- a/native-link-util/tests/async_fixed_buffer_test.rs +++ b/nativelink-util/tests/async_fixed_buffer_test.rs @@ -17,7 +17,7 @@ use std::task::Poll; use error::{make_err, Code, Error, ResultExt}; use futures::{pin_mut, poll}; -use native_link_util::async_fixed_buffer::AsyncFixedBuf; +use nativelink_util::async_fixed_buffer::AsyncFixedBuf; use tokio::io::{AsyncReadExt, AsyncWriteExt}; #[cfg(test)] diff --git a/native-link-util/tests/buf_channel_test.rs b/nativelink-util/tests/buf_channel_test.rs similarity index 99% rename from native-link-util/tests/buf_channel_test.rs rename to nativelink-util/tests/buf_channel_test.rs index 5877ba7e4..be0798727 100644 --- a/native-link-util/tests/buf_channel_test.rs +++ b/nativelink-util/tests/buf_channel_test.rs @@ -14,7 +14,7 @@ use bytes::Bytes; use error::{make_err, Code, Error, ResultExt}; -use native_link_util::buf_channel::make_buf_channel_pair; +use nativelink_util::buf_channel::make_buf_channel_pair; use tokio::try_join; #[cfg(test)] diff --git a/native-link-util/tests/data/SekienAkashita.jpg b/nativelink-util/tests/data/SekienAkashita.jpg similarity index 100% rename from native-link-util/tests/data/SekienAkashita.jpg rename to nativelink-util/tests/data/SekienAkashita.jpg diff --git a/native-link-util/tests/evicting_map_test.rs b/nativelink-util/tests/evicting_map_test.rs similarity index 99% rename from native-link-util/tests/evicting_map_test.rs rename to nativelink-util/tests/evicting_map_test.rs index 7d48ad441..b151040eb 100644 --- a/native-link-util/tests/evicting_map_test.rs +++ b/nativelink-util/tests/evicting_map_test.rs @@ -20,9 +20,9 @@ use async_trait::async_trait; use bytes::Bytes; use error::Error; use mock_instant::{Instant as MockInstant, MockClock}; -use native_link_config::stores::EvictionPolicy; -use native_link_util::common::DigestInfo; -use native_link_util::evicting_map::{EvictingMap, InstantWrapper, LenEntry}; +use nativelink_config::stores::EvictionPolicy; +use nativelink_util::common::DigestInfo; +use nativelink_util::evicting_map::{EvictingMap, InstantWrapper, LenEntry}; #[derive(Clone, PartialEq, Debug)] pub struct BytesWrapper(Bytes); diff --git a/native-link-util/tests/fastcdc_test.rs b/nativelink-util/tests/fastcdc_test.rs similarity index 99% rename from native-link-util/tests/fastcdc_test.rs rename to nativelink-util/tests/fastcdc_test.rs index ef40cf194..0377d0281 100644 --- a/native-link-util/tests/fastcdc_test.rs +++ b/nativelink-util/tests/fastcdc_test.rs @@ -18,7 +18,7 @@ use std::marker::Unpin; use bytes::Bytes; use futures::stream::StreamExt; -use native_link_util::fastcdc::FastCDC; +use nativelink_util::fastcdc::FastCDC; use rand::rngs::SmallRng; use rand::{Rng, SeedableRng}; use sha2::{Digest, Sha256}; diff --git a/native-link-util/tests/fs_test.rs b/nativelink-util/tests/fs_test.rs similarity index 99% rename from native-link-util/tests/fs_test.rs rename to nativelink-util/tests/fs_test.rs index 9cefd3418..904dd19c1 100644 --- a/native-link-util/tests/fs_test.rs +++ b/nativelink-util/tests/fs_test.rs @@ -18,7 +18,7 @@ use std::io::SeekFrom; use std::str::from_utf8; use error::Error; -use native_link_util::common::fs; +use nativelink_util::common::fs; use rand::{thread_rng, Rng}; use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; use tokio::sync::Semaphore; diff --git a/native-link-util/tests/resource_info_test.rs b/nativelink-util/tests/resource_info_test.rs similarity index 99% rename from native-link-util/tests/resource_info_test.rs rename to nativelink-util/tests/resource_info_test.rs index 17025dc73..419719b94 100644 --- a/native-link-util/tests/resource_info_test.rs +++ b/nativelink-util/tests/resource_info_test.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use native_link_util::resource_info::ResourceInfo; +use nativelink_util::resource_info::ResourceInfo; #[cfg(test)] mod resource_info_tests { diff --git a/native-link-util/tests/retry_test.rs b/nativelink-util/tests/retry_test.rs similarity index 99% rename from native-link-util/tests/retry_test.rs rename to nativelink-util/tests/retry_test.rs index bc82ea35a..84caa2f4b 100644 --- a/native-link-util/tests/retry_test.rs +++ b/nativelink-util/tests/retry_test.rs @@ -19,7 +19,7 @@ use std::sync::Arc; use error::{make_err, Code, Error}; use futures::future::ready; use futures::stream::repeat_with; -use native_link_util::retry::{Retrier, RetryResult}; +use nativelink_util::retry::{Retrier, RetryResult}; use tokio::time::Duration; struct MockDurationIterator { diff --git a/native-link-worker/BUILD.bazel b/nativelink-worker/BUILD.bazel similarity index 83% rename from native-link-worker/BUILD.bazel rename to nativelink-worker/BUILD.bazel index d6f018811..f121fa6fd 100644 --- a/native-link-worker/BUILD.bazel +++ b/nativelink-worker/BUILD.bazel @@ -7,7 +7,7 @@ load( ) rust_library( - name = "native-link-worker", + name = "nativelink-worker", srcs = [ "src/lib.rs", "src/local_worker.rs", @@ -21,10 +21,10 @@ rust_library( visibility = ["//visibility:public"], deps = [ "//error", - "//native-link-config", - "//native-link-scheduler", - "//native-link-store", - "//native-link-util", + "//nativelink-config", + "//nativelink-scheduler", + "//nativelink-store", + "//nativelink-util", "//proto", "@crate_index//:bytes", "@crate_index//:filetime", @@ -60,12 +60,12 @@ rust_test_suite( "@crate_index//:async-trait", ], deps = [ - ":native-link-worker", + ":nativelink-worker", "//error", - "//native-link-config", - "//native-link-scheduler", - "//native-link-store", - "//native-link-util", + "//nativelink-config", + "//nativelink-scheduler", + "//nativelink-store", + "//nativelink-util", "//proto", "@crate_index//:async-lock", "@crate_index//:futures", @@ -82,10 +82,10 @@ rust_test_suite( rust_doc( name = "docs", - crate = ":native-link-worker", + crate = ":nativelink-worker", ) rust_doc_test( name = "doc_test", - crate = ":native-link-worker", + crate = ":nativelink-worker", ) diff --git a/native-link-worker/Cargo.toml b/nativelink-worker/Cargo.toml similarity index 78% rename from native-link-worker/Cargo.toml rename to nativelink-worker/Cargo.toml index 69064c5bc..d8f6308c9 100644 --- a/native-link-worker/Cargo.toml +++ b/nativelink-worker/Cargo.toml @@ -1,18 +1,18 @@ [package] -name = "native-link-worker" +name = "nativelink-worker" version = "0.0.0" edition = "2021" [dependencies] error = { path = "../error" } proto = { path = "../proto" } -native-link-config = { path = "../native-link-config" } -native-link-util = { path = "../native-link-util" } -native-link-store = { path = "../native-link-store" } +nativelink-config = { path = "../nativelink-config" } +nativelink-util = { path = "../nativelink-util" } +nativelink-store = { path = "../nativelink-store" } # TODO(aaronmondal): This should not be a dependency. Move corresponding # functionality out of the schedulers. -native-link-scheduler = { path = "../native-link-scheduler" } +nativelink-scheduler = { path = "../nativelink-scheduler" } async-lock = "2.8.0" async-trait = "0.1.74" diff --git a/native-link-worker/src/lib.rs b/nativelink-worker/src/lib.rs similarity index 100% rename from native-link-worker/src/lib.rs rename to nativelink-worker/src/lib.rs diff --git a/native-link-worker/src/local_worker.rs b/nativelink-worker/src/local_worker.rs similarity index 97% rename from native-link-worker/src/local_worker.rs rename to nativelink-worker/src/local_worker.rs index 0a43bfd0e..024218753 100644 --- a/native-link-worker/src/local_worker.rs +++ b/nativelink-worker/src/local_worker.rs @@ -23,18 +23,18 @@ use error::{make_err, make_input_err, Code, Error, ResultExt}; use futures::future::BoxFuture; use futures::stream::FuturesUnordered; use futures::{select, Future, FutureExt, StreamExt, TryFutureExt}; -use native_link_config::cas_server::LocalWorkerConfig; -use native_link_store::fast_slow_store::FastSlowStore; -use native_link_util::action_messages::{ActionResult, ActionStage}; -use native_link_util::common::fs; -use native_link_util::digest_hasher::DigestHasherFunc; -use native_link_util::metrics_utils::{ +use nativelink_config::cas_server::LocalWorkerConfig; +use nativelink_store::fast_slow_store::FastSlowStore; +use nativelink_util::action_messages::{ActionResult, ActionStage}; +use nativelink_util::common::fs; +use nativelink_util::digest_hasher::DigestHasherFunc; +use nativelink_util::metrics_utils::{ AsyncCounterWrapper, Collector, CollectorState, CounterWithTime, MetricsComponent, Registry, }; -use native_link_util::store_trait::Store; -use proto::com::github::trace_machina::native_link::remote_execution::update_for_worker::Update; -use proto::com::github::trace_machina::native_link::remote_execution::worker_api_client::WorkerApiClient; -use proto::com::github::trace_machina::native_link::remote_execution::{ +use nativelink_util::store_trait::Store; +use proto::com::github::trace_machina::nativelink::remote_execution::update_for_worker::Update; +use proto::com::github::trace_machina::nativelink::remote_execution::worker_api_client::WorkerApiClient; +use proto::com::github::trace_machina::nativelink::remote_execution::{ execute_result, ExecuteResult, KeepAliveRequest, UpdateForWorker, }; use tokio::process; diff --git a/native-link-worker/src/running_actions_manager.rs b/nativelink-worker/src/running_actions_manager.rs similarity index 98% rename from native-link-worker/src/running_actions_manager.rs rename to nativelink-worker/src/running_actions_manager.rs index 117ed971c..363b8d5c5 100644 --- a/native-link-worker/src/running_actions_manager.rs +++ b/nativelink-worker/src/running_actions_manager.rs @@ -35,28 +35,28 @@ use filetime::{set_file_mtime, FileTime}; use formatx::Template; use futures::future::{try_join, try_join3, try_join_all, BoxFuture, Future, FutureExt, TryFutureExt}; use futures::stream::{FuturesUnordered, StreamExt, TryStreamExt}; -use native_link_config::cas_server::{EnvironmentSource, UploadActionResultConfig, UploadCacheResultsStrategy}; -use native_link_store::ac_utils::{ +use nativelink_config::cas_server::{EnvironmentSource, UploadActionResultConfig, UploadCacheResultsStrategy}; +use nativelink_store::ac_utils::{ compute_buf_digest, compute_digest, get_and_decode_digest, serialize_and_upload_message, upload_buf_to_store, upload_file_to_store, ESTIMATED_DIGEST_SIZE, }; -use native_link_store::fast_slow_store::FastSlowStore; -use native_link_store::filesystem_store::{FileEntry, FilesystemStore}; -use native_link_store::grpc_store::GrpcStore; -use native_link_util::action_messages::{ +use nativelink_store::fast_slow_store::FastSlowStore; +use nativelink_store::filesystem_store::{FileEntry, FilesystemStore}; +use nativelink_store::grpc_store::GrpcStore; +use nativelink_util::action_messages::{ to_execute_response, ActionInfo, ActionResult, DirectoryInfo, ExecutionMetadata, FileInfo, NameOrPath, SymlinkInfo, }; -use native_link_util::common::{fs, DigestInfo, JoinHandleDropGuard}; -use native_link_util::digest_hasher::DigestHasherFunc; -use native_link_util::metrics_utils::{AsyncCounterWrapper, CollectorState, CounterWithTime, MetricsComponent}; -use native_link_util::store_trait::Store; +use nativelink_util::common::{fs, DigestInfo, JoinHandleDropGuard}; +use nativelink_util::digest_hasher::DigestHasherFunc; +use nativelink_util::metrics_utils::{AsyncCounterWrapper, CollectorState, CounterWithTime, MetricsComponent}; +use nativelink_util::store_trait::Store; use parking_lot::Mutex; use prost::Message; use proto::build::bazel::remote::execution::v2::{ Action, ActionResult as ProtoActionResult, Command as ProtoCommand, Directory as ProtoDirectory, Directory, DirectoryNode, ExecuteResponse, FileNode, SymlinkNode, Tree as ProtoTree, UpdateActionResultRequest, }; -use proto::com::github::trace_machina::native_link::remote_execution::{HistoricalExecuteResponse, StartExecute}; +use proto::com::github::trace_machina::nativelink::remote_execution::{HistoricalExecuteResponse, StartExecute}; use relative_path::RelativePath; use scopeguard::{guard, ScopeGuard}; use serde::Deserialize; @@ -806,7 +806,7 @@ impl RunningActionImpl { // TODO(allada) We should implement stderr/stdout streaming to client here. // If we get killed before the stream is started, then these will lock up. // TODO(allada) There is a significant bug here. If we kill the action and the action creates - // child processes, it can create zombies. See: https://github.com/tracemachina/native-link/issues/225 + // child processes, it can create zombies. See: https://github.com/tracemachina/nativelink/issues/225 let (stdout, stderr) = if killed_action { drop(timer); (Bytes::new(), Bytes::new()) diff --git a/native-link-worker/src/worker_api_client_wrapper.rs b/nativelink-worker/src/worker_api_client_wrapper.rs similarity index 93% rename from native-link-worker/src/worker_api_client_wrapper.rs rename to nativelink-worker/src/worker_api_client_wrapper.rs index ebc079cdc..026a874dd 100644 --- a/native-link-worker/src/worker_api_client_wrapper.rs +++ b/nativelink-worker/src/worker_api_client_wrapper.rs @@ -13,8 +13,8 @@ // limitations under the License. use async_trait::async_trait; -use proto::com::github::trace_machina::native_link::remote_execution::worker_api_client::WorkerApiClient; -use proto::com::github::trace_machina::native_link::remote_execution::{ +use proto::com::github::trace_machina::nativelink::remote_execution::worker_api_client::WorkerApiClient; +use proto::com::github::trace_machina::nativelink::remote_execution::{ ExecuteResult, GoingAwayRequest, KeepAliveRequest, SupportedProperties, UpdateForWorker, }; use tonic::codec::Streaming; diff --git a/native-link-worker/src/worker_utils.rs b/nativelink-worker/src/worker_utils.rs similarity index 96% rename from native-link-worker/src/worker_utils.rs rename to nativelink-worker/src/worker_utils.rs index f6f665fe2..6e7828ee9 100644 --- a/native-link-worker/src/worker_utils.rs +++ b/nativelink-worker/src/worker_utils.rs @@ -20,9 +20,9 @@ use std::str::from_utf8; use error::{make_err, make_input_err, Error, ResultExt}; use futures::future::try_join_all; -use native_link_config::cas_server::WorkerProperty; +use nativelink_config::cas_server::WorkerProperty; use proto::build::bazel::remote::execution::v2::platform::Property; -use proto::com::github::trace_machina::native_link::remote_execution::SupportedProperties; +use proto::com::github::trace_machina::nativelink::remote_execution::SupportedProperties; use tokio::process; use tracing::info; diff --git a/native-link-worker/tests/local_worker_test.rs b/nativelink-worker/tests/local_worker_test.rs similarity index 91% rename from native-link-worker/tests/local_worker_test.rs rename to nativelink-worker/tests/local_worker_test.rs index 5ae9ab8be..9fa6ed608 100644 --- a/native-link-worker/tests/local_worker_test.rs +++ b/nativelink-worker/tests/local_worker_test.rs @@ -30,19 +30,19 @@ mod utils { } use error::{make_err, make_input_err, Code, Error}; -use native_link_config::cas_server::{LocalWorkerConfig, WorkerProperty}; -use native_link_store::fast_slow_store::FastSlowStore; -use native_link_store::filesystem_store::FilesystemStore; -use native_link_store::memory_store::MemoryStore; -use native_link_util::action_messages::{ActionInfo, ActionInfoHashKey, ActionResult, ActionStage, ExecutionMetadata}; -use native_link_util::common::{encode_stream_proto, fs, DigestInfo}; -use native_link_util::digest_hasher::DigestHasherFunc; -use native_link_util::platform_properties::PlatformProperties; -use native_link_worker::local_worker::new_local_worker; +use nativelink_config::cas_server::{LocalWorkerConfig, WorkerProperty}; +use nativelink_store::fast_slow_store::FastSlowStore; +use nativelink_store::filesystem_store::FilesystemStore; +use nativelink_store::memory_store::MemoryStore; +use nativelink_util::action_messages::{ActionInfo, ActionInfoHashKey, ActionResult, ActionStage, ExecutionMetadata}; +use nativelink_util::common::{encode_stream_proto, fs, DigestInfo}; +use nativelink_util::digest_hasher::DigestHasherFunc; +use nativelink_util::platform_properties::PlatformProperties; +use nativelink_worker::local_worker::new_local_worker; use prost::Message; use proto::build::bazel::remote::execution::v2::platform::Property; -use proto::com::github::trace_machina::native_link::remote_execution::update_for_worker::Update; -use proto::com::github::trace_machina::native_link::remote_execution::{ +use proto::com::github::trace_machina::nativelink::remote_execution::update_for_worker::Update; +use proto::com::github::trace_machina::nativelink::remote_execution::{ execute_result, ConnectionResult, ExecuteResult, StartExecute, SupportedProperties, UpdateForWorker, }; use rand::{thread_rng, Rng}; @@ -387,26 +387,26 @@ mod local_worker_tests { #[tokio::test] async fn new_local_worker_creates_work_directory_test() -> Result<(), Box> { let cas_store = Arc::new(FastSlowStore::new( - &native_link_config::stores::FastSlowStore { + &nativelink_config::stores::FastSlowStore { // Note: These are not needed for this test, so we put dummy memory stores here. - fast: native_link_config::stores::StoreConfig::memory( - native_link_config::stores::MemoryStore::default(), + fast: nativelink_config::stores::StoreConfig::memory( + nativelink_config::stores::MemoryStore::default(), ), - slow: native_link_config::stores::StoreConfig::memory( - native_link_config::stores::MemoryStore::default(), + slow: nativelink_config::stores::StoreConfig::memory( + nativelink_config::stores::MemoryStore::default(), ), }, Arc::new( - ::new(&native_link_config::stores::FilesystemStore { + ::new(&nativelink_config::stores::FilesystemStore { content_path: make_temp_path("content_path"), temp_path: make_temp_path("temp_path"), ..Default::default() }) .await?, ), - Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())), + Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())), )); - let ac_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); + let ac_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); let work_directory = make_temp_path("foo"); new_local_worker( Arc::new(LocalWorkerConfig { @@ -430,26 +430,26 @@ mod local_worker_tests { #[tokio::test] async fn new_local_worker_removes_work_directory_before_start_test() -> Result<(), Box> { let cas_store = Arc::new(FastSlowStore::new( - &native_link_config::stores::FastSlowStore { + &nativelink_config::stores::FastSlowStore { // Note: These are not needed for this test, so we put dummy memory stores here. - fast: native_link_config::stores::StoreConfig::memory( - native_link_config::stores::MemoryStore::default(), + fast: nativelink_config::stores::StoreConfig::memory( + nativelink_config::stores::MemoryStore::default(), ), - slow: native_link_config::stores::StoreConfig::memory( - native_link_config::stores::MemoryStore::default(), + slow: nativelink_config::stores::StoreConfig::memory( + nativelink_config::stores::MemoryStore::default(), ), }, Arc::new( - ::new(&native_link_config::stores::FilesystemStore { + ::new(&nativelink_config::stores::FilesystemStore { content_path: make_temp_path("content_path"), temp_path: make_temp_path("temp_path"), ..Default::default() }) .await?, ), - Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())), + Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())), )); - let ac_store = Arc::new(MemoryStore::new(&native_link_config::stores::MemoryStore::default())); + let ac_store = Arc::new(MemoryStore::new(&nativelink_config::stores::MemoryStore::default())); let work_directory = make_temp_path("foo"); fs::create_dir_all(format!("{}/{}", work_directory, "another_dir")).await?; let mut file = fs::create_file(OsString::from(format!("{}/{}", work_directory, "foo.txt"))).await?; diff --git a/native-link-worker/tests/running_actions_manager_test.rs b/nativelink-worker/tests/running_actions_manager_test.rs similarity index 95% rename from native-link-worker/tests/running_actions_manager_test.rs rename to nativelink-worker/tests/running_actions_manager_test.rs index 16b33d111..d5de5395d 100644 --- a/native-link-worker/tests/running_actions_manager_test.rs +++ b/nativelink-worker/tests/running_actions_manager_test.rs @@ -28,19 +28,19 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use error::{make_input_err, Code, Error, ResultExt}; use futures::{FutureExt, TryFutureExt}; -use native_link_config::cas_server::EnvironmentSource; -use native_link_store::ac_utils::{compute_digest, get_and_decode_digest, serialize_and_upload_message}; -use native_link_store::fast_slow_store::FastSlowStore; -use native_link_store::filesystem_store::FilesystemStore; -use native_link_store::memory_store::MemoryStore; +use nativelink_config::cas_server::EnvironmentSource; +use nativelink_store::ac_utils::{compute_digest, get_and_decode_digest, serialize_and_upload_message}; +use nativelink_store::fast_slow_store::FastSlowStore; +use nativelink_store::filesystem_store::FilesystemStore; +use nativelink_store::memory_store::MemoryStore; #[cfg_attr(target_family = "windows", allow(unused_imports))] -use native_link_util::action_messages::{ +use nativelink_util::action_messages::{ ActionResult, DirectoryInfo, ExecutionMetadata, FileInfo, NameOrPath, SymlinkInfo, }; -use native_link_util::common::{fs, DigestInfo}; -use native_link_util::digest_hasher::DigestHasherFunc; -use native_link_util::store_trait::Store; -use native_link_worker::running_actions_manager::{ +use nativelink_util::common::{fs, DigestInfo}; +use nativelink_util::digest_hasher::DigestHasherFunc; +use nativelink_util::store_trait::Store; +use nativelink_worker::running_actions_manager::{ download_to_directory, Callbacks, ExecutionConfiguration, RunningAction, RunningActionImpl, RunningActionsManager, RunningActionsManagerArgs, RunningActionsManagerImpl, }; @@ -52,7 +52,7 @@ use proto::build::bazel::remote::execution::v2::{ Command, Directory, DirectoryNode, ExecuteRequest, ExecuteResponse, FileNode, NodeProperties, Platform, SymlinkNode, Tree, }; -use proto::com::github::trace_machina::native_link::remote_execution::{HistoricalExecuteResponse, StartExecute}; +use proto::com::github::trace_machina::nativelink::remote_execution::{HistoricalExecuteResponse, StartExecute}; use rand::{thread_rng, Rng}; use tokio::sync::oneshot; @@ -84,20 +84,20 @@ async fn setup_stores() -> Result< ), Error, > { - let fast_config = native_link_config::stores::FilesystemStore { + let fast_config = nativelink_config::stores::FilesystemStore { content_path: make_temp_path("content_path"), temp_path: make_temp_path("temp_path"), eviction_policy: None, ..Default::default() }; - let slow_config = native_link_config::stores::MemoryStore::default(); + let slow_config = nativelink_config::stores::MemoryStore::default(); let fast_store = Pin::new(Arc::new(FilesystemStore::new(&fast_config).await?)); let slow_store = Pin::new(Arc::new(MemoryStore::new(&slow_config))); let ac_store = Pin::new(Arc::new(MemoryStore::new(&slow_config))); let cas_store = Pin::new(Arc::new(FastSlowStore::new( - &native_link_config::stores::FastSlowStore { - fast: native_link_config::stores::StoreConfig::filesystem(fast_config), - slow: native_link_config::stores::StoreConfig::memory(slow_config), + &nativelink_config::stores::FastSlowStore { + fast: nativelink_config::stores::StoreConfig::filesystem(fast_config), + slow: nativelink_config::stores::StoreConfig::memory(slow_config), }, Pin::into_inner(fast_store.clone()), Pin::into_inner(slow_store.clone()), @@ -427,8 +427,8 @@ mod running_actions_manager_tests { cas_store: Pin::into_inner(cas_store.clone()), ac_store: Some(Pin::into_inner(ac_store.clone())), historical_store: Pin::into_inner(cas_store.clone()), - upload_action_result_config: &native_link_config::cas_server::UploadActionResultConfig { - upload_ac_results_strategy: native_link_config::cas_server::UploadCacheResultsStrategy::Never, + upload_action_result_config: &nativelink_config::cas_server::UploadActionResultConfig { + upload_ac_results_strategy: nativelink_config::cas_server::UploadCacheResultsStrategy::Never, ..Default::default() }, max_action_timeout: Duration::MAX, @@ -527,8 +527,8 @@ mod running_actions_manager_tests { cas_store: Pin::into_inner(cas_store.clone()), ac_store: Some(Pin::into_inner(ac_store.clone())), historical_store: Pin::into_inner(cas_store.clone()), - upload_action_result_config: &native_link_config::cas_server::UploadActionResultConfig { - upload_ac_results_strategy: native_link_config::cas_server::UploadCacheResultsStrategy::Never, + upload_action_result_config: &nativelink_config::cas_server::UploadActionResultConfig { + upload_ac_results_strategy: nativelink_config::cas_server::UploadCacheResultsStrategy::Never, ..Default::default() }, max_action_timeout: Duration::MAX, @@ -634,8 +634,8 @@ mod running_actions_manager_tests { cas_store: Pin::into_inner(cas_store.clone()), ac_store: Some(Pin::into_inner(ac_store.clone())), historical_store: Pin::into_inner(cas_store.clone()), - upload_action_result_config: &native_link_config::cas_server::UploadActionResultConfig { - upload_ac_results_strategy: native_link_config::cas_server::UploadCacheResultsStrategy::Never, + upload_action_result_config: &nativelink_config::cas_server::UploadActionResultConfig { + upload_ac_results_strategy: nativelink_config::cas_server::UploadCacheResultsStrategy::Never, ..Default::default() }, max_action_timeout: Duration::MAX, @@ -793,8 +793,8 @@ mod running_actions_manager_tests { cas_store: Pin::into_inner(cas_store.clone()), ac_store: Some(Pin::into_inner(ac_store.clone())), historical_store: Pin::into_inner(cas_store.clone()), - upload_action_result_config: &native_link_config::cas_server::UploadActionResultConfig { - upload_ac_results_strategy: native_link_config::cas_server::UploadCacheResultsStrategy::Never, + upload_action_result_config: &nativelink_config::cas_server::UploadActionResultConfig { + upload_ac_results_strategy: nativelink_config::cas_server::UploadCacheResultsStrategy::Never, ..Default::default() }, max_action_timeout: Duration::MAX, @@ -953,8 +953,8 @@ mod running_actions_manager_tests { cas_store: Pin::into_inner(cas_store.clone()), ac_store: Some(Pin::into_inner(ac_store.clone())), historical_store: Pin::into_inner(cas_store.clone()), - upload_action_result_config: &native_link_config::cas_server::UploadActionResultConfig { - upload_ac_results_strategy: native_link_config::cas_server::UploadCacheResultsStrategy::Never, + upload_action_result_config: &nativelink_config::cas_server::UploadActionResultConfig { + upload_ac_results_strategy: nativelink_config::cas_server::UploadCacheResultsStrategy::Never, ..Default::default() }, max_action_timeout: Duration::MAX, @@ -1130,8 +1130,8 @@ mod running_actions_manager_tests { cas_store: Pin::into_inner(cas_store.clone()), ac_store: Some(Pin::into_inner(ac_store.clone())), historical_store: Pin::into_inner(cas_store.clone()), - upload_action_result_config: &native_link_config::cas_server::UploadActionResultConfig { - upload_ac_results_strategy: native_link_config::cas_server::UploadCacheResultsStrategy::Never, + upload_action_result_config: &nativelink_config::cas_server::UploadActionResultConfig { + upload_ac_results_strategy: nativelink_config::cas_server::UploadCacheResultsStrategy::Never, ..Default::default() }, max_action_timeout: Duration::MAX, @@ -1247,8 +1247,8 @@ mod running_actions_manager_tests { cas_store: Pin::into_inner(cas_store.clone()), ac_store: Some(Pin::into_inner(ac_store.clone())), historical_store: Pin::into_inner(cas_store.clone()), - upload_action_result_config: &native_link_config::cas_server::UploadActionResultConfig { - upload_ac_results_strategy: native_link_config::cas_server::UploadCacheResultsStrategy::Never, + upload_action_result_config: &nativelink_config::cas_server::UploadActionResultConfig { + upload_ac_results_strategy: nativelink_config::cas_server::UploadCacheResultsStrategy::Never, ..Default::default() }, max_action_timeout: Duration::MAX, @@ -1383,8 +1383,8 @@ exit 0 cas_store: Pin::into_inner(cas_store.clone()), ac_store: Some(Pin::into_inner(ac_store.clone())), historical_store: Pin::into_inner(cas_store.clone()), - upload_action_result_config: &native_link_config::cas_server::UploadActionResultConfig { - upload_ac_results_strategy: native_link_config::cas_server::UploadCacheResultsStrategy::Never, + upload_action_result_config: &nativelink_config::cas_server::UploadActionResultConfig { + upload_ac_results_strategy: nativelink_config::cas_server::UploadCacheResultsStrategy::Never, ..Default::default() }, max_action_timeout: Duration::MAX, @@ -1519,8 +1519,8 @@ exit 0 cas_store: Pin::into_inner(cas_store.clone()), ac_store: Some(Pin::into_inner(ac_store.clone())), historical_store: Pin::into_inner(cas_store.clone()), - upload_action_result_config: &native_link_config::cas_server::UploadActionResultConfig { - upload_ac_results_strategy: native_link_config::cas_server::UploadCacheResultsStrategy::Never, + upload_action_result_config: &nativelink_config::cas_server::UploadActionResultConfig { + upload_ac_results_strategy: nativelink_config::cas_server::UploadCacheResultsStrategy::Never, ..Default::default() }, max_action_timeout: Duration::MAX, @@ -1656,8 +1656,8 @@ exit 1 cas_store: Pin::into_inner(cas_store.clone()), ac_store: Some(Pin::into_inner(ac_store.clone())), historical_store: Pin::into_inner(cas_store.clone()), - upload_action_result_config: &native_link_config::cas_server::UploadActionResultConfig { - upload_ac_results_strategy: native_link_config::cas_server::UploadCacheResultsStrategy::Never, + upload_action_result_config: &nativelink_config::cas_server::UploadActionResultConfig { + upload_ac_results_strategy: nativelink_config::cas_server::UploadCacheResultsStrategy::Never, ..Default::default() }, max_action_timeout: Duration::MAX, @@ -1719,8 +1719,8 @@ exit 1 cas_store: Pin::into_inner(cas_store.clone()), ac_store: Some(Pin::into_inner(ac_store.clone())), historical_store: Pin::into_inner(cas_store.clone()), - upload_action_result_config: &native_link_config::cas_server::UploadActionResultConfig { - upload_ac_results_strategy: native_link_config::cas_server::UploadCacheResultsStrategy::SuccessOnly, + upload_action_result_config: &nativelink_config::cas_server::UploadActionResultConfig { + upload_ac_results_strategy: nativelink_config::cas_server::UploadCacheResultsStrategy::SuccessOnly, ..Default::default() }, max_action_timeout: Duration::MAX, @@ -1778,8 +1778,8 @@ exit 1 cas_store: Pin::into_inner(cas_store.clone()), ac_store: Some(Pin::into_inner(ac_store.clone())), historical_store: Pin::into_inner(cas_store.clone()), - upload_action_result_config: &native_link_config::cas_server::UploadActionResultConfig { - upload_ac_results_strategy: native_link_config::cas_server::UploadCacheResultsStrategy::Everything, + upload_action_result_config: &nativelink_config::cas_server::UploadActionResultConfig { + upload_ac_results_strategy: nativelink_config::cas_server::UploadCacheResultsStrategy::Everything, ..Default::default() }, max_action_timeout: Duration::MAX, @@ -1837,9 +1837,9 @@ exit 1 cas_store: Pin::into_inner(cas_store.clone()), ac_store: Some(Pin::into_inner(ac_store.clone())), historical_store: Pin::into_inner(cas_store.clone()), - upload_action_result_config: &native_link_config::cas_server::UploadActionResultConfig { + upload_action_result_config: &nativelink_config::cas_server::UploadActionResultConfig { upload_historical_results_strategy: Some( - native_link_config::cas_server::UploadCacheResultsStrategy::SuccessOnly, + nativelink_config::cas_server::UploadCacheResultsStrategy::SuccessOnly, ), success_message_template: "{historical_results_hash}-{historical_results_size}".to_string(), ..Default::default() @@ -1919,9 +1919,9 @@ exit 1 cas_store: Pin::into_inner(cas_store.clone()), ac_store: Some(Pin::into_inner(ac_store.clone())), historical_store: Pin::into_inner(cas_store.clone()), - upload_action_result_config: &native_link_config::cas_server::UploadActionResultConfig { + upload_action_result_config: &nativelink_config::cas_server::UploadActionResultConfig { upload_historical_results_strategy: Some( - native_link_config::cas_server::UploadCacheResultsStrategy::SuccessOnly, + nativelink_config::cas_server::UploadCacheResultsStrategy::SuccessOnly, ), success_message_template: "{historical_results_hash}-{historical_results_size}".to_string(), ..Default::default() @@ -1953,9 +1953,9 @@ exit 1 cas_store: Pin::into_inner(cas_store.clone()), ac_store: Some(Pin::into_inner(ac_store.clone())), historical_store: Pin::into_inner(cas_store.clone()), - upload_action_result_config: &native_link_config::cas_server::UploadActionResultConfig { + upload_action_result_config: &nativelink_config::cas_server::UploadActionResultConfig { upload_historical_results_strategy: Some( - native_link_config::cas_server::UploadCacheResultsStrategy::FailuresOnly, + nativelink_config::cas_server::UploadCacheResultsStrategy::FailuresOnly, ), failure_message_template: "{historical_results_hash}-{historical_results_size}".to_string(), ..Default::default() @@ -2012,8 +2012,8 @@ exit 1 cas_store: Pin::into_inner(cas_store.clone()), ac_store: Some(Pin::into_inner(ac_store.clone())), historical_store: Pin::into_inner(cas_store.clone()), - upload_action_result_config: &native_link_config::cas_server::UploadActionResultConfig { - upload_ac_results_strategy: native_link_config::cas_server::UploadCacheResultsStrategy::SuccessOnly, + upload_action_result_config: &nativelink_config::cas_server::UploadActionResultConfig { + upload_ac_results_strategy: nativelink_config::cas_server::UploadCacheResultsStrategy::SuccessOnly, success_message_template: "{action_digest_hash}-{action_digest_size}".to_string(), ..Default::default() }, @@ -2108,8 +2108,8 @@ exit 1 cas_store: Pin::into_inner(cas_store.clone()), ac_store: Some(Pin::into_inner(ac_store.clone())), historical_store: Pin::into_inner(cas_store.clone()), - upload_action_result_config: &native_link_config::cas_server::UploadActionResultConfig { - upload_ac_results_strategy: native_link_config::cas_server::UploadCacheResultsStrategy::Never, + upload_action_result_config: &nativelink_config::cas_server::UploadActionResultConfig { + upload_ac_results_strategy: nativelink_config::cas_server::UploadCacheResultsStrategy::Never, ..Default::default() }, max_action_timeout: MAX_TIMEOUT_DURATION, @@ -2176,8 +2176,8 @@ exit 1 cas_store: Pin::into_inner(cas_store.clone()), ac_store: Some(Pin::into_inner(ac_store.clone())), historical_store: Pin::into_inner(cas_store.clone()), - upload_action_result_config: &native_link_config::cas_server::UploadActionResultConfig { - upload_ac_results_strategy: native_link_config::cas_server::UploadCacheResultsStrategy::Never, + upload_action_result_config: &nativelink_config::cas_server::UploadActionResultConfig { + upload_ac_results_strategy: nativelink_config::cas_server::UploadCacheResultsStrategy::Never, ..Default::default() }, max_action_timeout: MAX_TIMEOUT_DURATION, @@ -2247,8 +2247,8 @@ exit 1 cas_store: Pin::into_inner(cas_store.clone()), ac_store: Some(Pin::into_inner(ac_store.clone())), historical_store: Pin::into_inner(cas_store.clone()), - upload_action_result_config: &native_link_config::cas_server::UploadActionResultConfig { - upload_ac_results_strategy: native_link_config::cas_server::UploadCacheResultsStrategy::Never, + upload_action_result_config: &nativelink_config::cas_server::UploadActionResultConfig { + upload_ac_results_strategy: nativelink_config::cas_server::UploadCacheResultsStrategy::Never, ..Default::default() }, max_action_timeout: MAX_TIMEOUT_DURATION, @@ -2319,8 +2319,8 @@ exit 1 cas_store: Pin::into_inner(cas_store.clone()), ac_store: Some(Pin::into_inner(ac_store.clone())), historical_store: Pin::into_inner(cas_store.clone()), - upload_action_result_config: &native_link_config::cas_server::UploadActionResultConfig { - upload_ac_results_strategy: native_link_config::cas_server::UploadCacheResultsStrategy::Never, + upload_action_result_config: &nativelink_config::cas_server::UploadActionResultConfig { + upload_ac_results_strategy: nativelink_config::cas_server::UploadCacheResultsStrategy::Never, ..Default::default() }, max_action_timeout: Duration::MAX, @@ -2427,8 +2427,8 @@ exit 1 cas_store: Pin::into_inner(cas_store.clone()), ac_store: Some(Pin::into_inner(ac_store.clone())), historical_store: Pin::into_inner(cas_store.clone()), - upload_action_result_config: &native_link_config::cas_server::UploadActionResultConfig { - upload_ac_results_strategy: native_link_config::cas_server::UploadCacheResultsStrategy::Never, + upload_action_result_config: &nativelink_config::cas_server::UploadActionResultConfig { + upload_ac_results_strategy: nativelink_config::cas_server::UploadCacheResultsStrategy::Never, ..Default::default() }, max_action_timeout: Duration::MAX, diff --git a/native-link-worker/tests/utils/local_worker_test_utils.rs b/nativelink-worker/tests/utils/local_worker_test_utils.rs similarity index 95% rename from native-link-worker/tests/utils/local_worker_test_utils.rs rename to nativelink-worker/tests/utils/local_worker_test_utils.rs index 63f3fa837..6f95d3b75 100644 --- a/native-link-worker/tests/utils/local_worker_test_utils.rs +++ b/nativelink-worker/tests/utils/local_worker_test_utils.rs @@ -19,11 +19,11 @@ use async_lock::Mutex; use async_trait::async_trait; use error::Error; use hyper::body::Sender as HyperSender; -use native_link_config::cas_server::{EndpointConfig, LocalWorkerConfig, WorkerProperty}; -use native_link_util::common::JoinHandleDropGuard; -use native_link_worker::local_worker::LocalWorker; -use native_link_worker::worker_api_client_wrapper::WorkerApiClientTrait; -use proto::com::github::trace_machina::native_link::remote_execution::{ +use nativelink_config::cas_server::{EndpointConfig, LocalWorkerConfig, WorkerProperty}; +use nativelink_util::common::JoinHandleDropGuard; +use nativelink_worker::local_worker::LocalWorker; +use nativelink_worker::worker_api_client_wrapper::WorkerApiClientTrait; +use proto::com::github::trace_machina::nativelink::remote_execution::{ ExecuteResult, GoingAwayRequest, KeepAliveRequest, SupportedProperties, UpdateForWorker, }; use tokio::sync::mpsc; diff --git a/native-link-worker/tests/utils/mock_running_actions_manager.rs b/nativelink-worker/tests/utils/mock_running_actions_manager.rs similarity index 97% rename from native-link-worker/tests/utils/mock_running_actions_manager.rs rename to nativelink-worker/tests/utils/mock_running_actions_manager.rs index 17cb61fb4..25f7e6cee 100644 --- a/native-link-worker/tests/utils/mock_running_actions_manager.rs +++ b/nativelink-worker/tests/utils/mock_running_actions_manager.rs @@ -17,11 +17,11 @@ use std::sync::Arc; use async_lock::Mutex; use async_trait::async_trait; use error::{make_input_err, Error}; -use native_link_util::action_messages::ActionResult; -use native_link_util::common::DigestInfo; -use native_link_util::digest_hasher::DigestHasherFunc; -use native_link_worker::running_actions_manager::{Metrics, RunningAction, RunningActionsManager}; -use proto::com::github::trace_machina::native_link::remote_execution::StartExecute; +use nativelink_util::action_messages::ActionResult; +use nativelink_util::common::DigestInfo; +use nativelink_util::digest_hasher::DigestHasherFunc; +use nativelink_worker::running_actions_manager::{Metrics, RunningAction, RunningActionsManager}; +use proto::com::github::trace_machina::nativelink::remote_execution::StartExecute; use tokio::sync::mpsc; #[derive(Debug)] diff --git a/proto/BUILD.bazel b/proto/BUILD.bazel index b1c7c59bf..d2abf44ce 100644 --- a/proto/BUILD.bazel +++ b/proto/BUILD.bazel @@ -3,7 +3,7 @@ load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library") PROTO_NAMES = [ "build.bazel.remote.execution.v2", "build.bazel.semver", - "com.github.trace_machina.native_link.remote_execution", + "com.github.trace_machina.nativelink.remote_execution", "google.api", "google.bytestream", "google.longrunning", @@ -25,7 +25,7 @@ genrule( srcs = [ "build/bazel/remote/execution/v2/remote_execution.proto", "build/bazel/semver/semver.proto", - "com/github/trace_machina/native_link/remote_execution/worker_api.proto", + "com/github/trace_machina/nativelink/remote_execution/worker_api.proto", "google/api/annotations.proto", "google/api/client.proto", "google/api/http.proto", diff --git a/proto/com/github/trace_machina/native_link/remote_execution/worker_api.proto b/proto/com/github/trace_machina/native_link/remote_execution/worker_api.proto index f2d5d00cc..c5bb2c81c 100644 --- a/proto/com/github/trace_machina/native_link/remote_execution/worker_api.proto +++ b/proto/com/github/trace_machina/native_link/remote_execution/worker_api.proto @@ -14,7 +14,7 @@ syntax = "proto3"; -package com.github.trace_machina.native_link.remote_execution; +package com.github.trace_machina.nativelink.remote_execution; import "build/bazel/remote/execution/v2/remote_execution.proto"; import "google/protobuf/empty.proto"; @@ -85,7 +85,7 @@ message SupportedProperties { /// the unsupported properties. /// /// The details on how to use this property can be found here: - /// https://github.com/TraceMachina/native-link/blob/3147265047544572e3483c985e4aab0f9fdded38/native-link-config/src/cas_server.rs + /// https://github.com/TraceMachina/nativelink/blob/3147265047544572e3483c985e4aab0f9fdded38/nativelink-config/src/cas_server.rs repeated build.bazel.remote.execution.v2.Platform.Property properties = 1; reserved 2; // NextId. } diff --git a/proto/genproto/com.github.trace_machina.nativelink.remote_execution.pb.rs b/proto/genproto/com.github.trace_machina.nativelink.remote_execution.pb.rs new file mode 100644 index 000000000..a7c327d4b --- /dev/null +++ b/proto/genproto/com.github.trace_machina.nativelink.remote_execution.pb.rs @@ -0,0 +1,755 @@ +// Copyright 2022 The Native Link Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// / Request object for keep alive requests. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct KeepAliveRequest { + /// / ID of the worker making the request. + #[prost(string, tag = "1")] + pub worker_id: ::prost::alloc::string::String, +} +/// / Request object for going away requests. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GoingAwayRequest { + /// / ID of the worker making the request. + #[prost(string, tag = "1")] + pub worker_id: ::prost::alloc::string::String, +} +/// / Represents the initial request sent to the scheduler informing the +/// / scheduler about this worker's capabilities. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SupportedProperties { + /// / The list of properties this worker can support. The exact + /// / implementation is driven by the configuration matrix between the + /// / worker and scheduler. + /// / + /// / The scheduler may reject this worker if any property keys that + /// / the scheduler is not configured to support, or may simply ignore + /// / the unsupported properties. + /// / + /// / The details on how to use this property can be found here: + /// / + #[prost(message, repeated, tag = "1")] + pub properties: ::prost::alloc::vec::Vec< + super::super::super::super::super::build::bazel::remote::execution::v2::platform::Property, + >, +} +/// / The result of an ExecutionRequest. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ExecuteResult { + /// / ID of the worker making the request. + #[prost(string, tag = "1")] + pub worker_id: ::prost::alloc::string::String, + /// / The `instance_name` this task was initially assigned to. This is set by the client + /// / that initially sent the job as part of the BRE protocol. + #[prost(string, tag = "6")] + pub instance_name: ::prost::alloc::string::String, + /// / The original execution digest request for this response. The scheduler knows what it + /// / should be, but we do safety checks to ensure it really is the request we expected. + #[prost(message, optional, tag = "2")] + pub action_digest: ::core::option::Option< + super::super::super::super::super::build::bazel::remote::execution::v2::Digest, + >, + /// / The salt originally sent along with the StartExecute request. This salt is used + /// / as a seed for cases where the execution digest should never be cached or merged + /// / with other jobs. This salt is added to the hash function used to compute jobs that + /// / are running or cached. + #[prost(uint64, tag = "3")] + pub salt: u64, + /// / The actual response data. + #[prost(oneof = "execute_result::Result", tags = "4, 5")] + pub result: ::core::option::Option, +} +/// Nested message and enum types in `ExecuteResult`. +pub mod execute_result { + /// / The actual response data. + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Result { + /// / Result of the execution. See `build.bazel.remote.execution.v2.ExecuteResponse` + /// / for details. + #[prost(message, tag = "4")] + ExecuteResponse( + super::super::super::super::super::super::build::bazel::remote::execution::v2::ExecuteResponse, + ), + /// / An internal error. This is only present when an internal error happened that + /// / was not recoverable. If the execution job failed but at no fault of the worker + /// / it should not use this field and should send the error via execute_response. + #[prost(message, tag = "5")] + InternalError(super::super::super::super::super::super::google::rpc::Status), + } +} +/// / Result sent back from the server when a node connects. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ConnectionResult { + /// / The internal ID given to the newly connected node. + #[prost(string, tag = "1")] + pub worker_id: ::prost::alloc::string::String, +} +/// / Communication from the scheduler to the worker. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct UpdateForWorker { + #[prost(oneof = "update_for_worker::Update", tags = "1, 2, 3, 4")] + pub update: ::core::option::Option, +} +/// Nested message and enum types in `UpdateForWorker`. +pub mod update_for_worker { + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Update { + /// / This will be sent only as the first item in the stream after the node + /// / has connected. + #[prost(message, tag = "1")] + ConnectionResult(super::ConnectionResult), + /// / Message used to let the worker know that it is still alive as well + /// / as check to see if the worker is still alive. The worker + /// / may close the connection if the scheduler has not sent any messages + /// / after some amount of time (configured in the scheduler's + /// / configuration). + #[prost(message, tag = "2")] + KeepAlive(()), + /// / Informs the worker about some work it should begin performing the + /// / requested action. + #[prost(message, tag = "3")] + StartAction(super::StartExecute), + /// / Informs the worker that it has been disconnected from the pool. + /// / The worker may discard any outstanding work that is being executed. + #[prost(message, tag = "4")] + Disconnect(()), + } +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StartExecute { + /// / The action information used to execute job. + #[prost(message, optional, tag = "1")] + pub execute_request: ::core::option::Option< + super::super::super::super::super::build::bazel::remote::execution::v2::ExecuteRequest, + >, + /// / See documentation in ExecuteResult::salt. + #[prost(uint64, tag = "2")] + pub salt: u64, + /// / The time at which the command was added to the queue to allow population + /// / of the ActionResult. + #[prost(message, optional, tag = "3")] + pub queued_timestamp: ::core::option::Option<::prost_types::Timestamp>, +} +/// / This is a special message used to save actions into the CAS that can be used +/// / by programs like bb_browswer to inspect the history of a build. +/// / Note: Ensure this is always compatible with: +/// / +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct HistoricalExecuteResponse { + #[prost(message, optional, tag = "1")] + pub action_digest: ::core::option::Option< + super::super::super::super::super::build::bazel::remote::execution::v2::Digest, + >, + #[prost(message, optional, tag = "3")] + pub execute_response: ::core::option::Option< + super::super::super::super::super::build::bazel::remote::execution::v2::ExecuteResponse, + >, +} +/// Generated client implementations. +pub mod worker_api_client { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::*; + use tonic::codegen::http::Uri; + /// / This API describes how schedulers communicate with Worker nodes. + /// / + /// / When a worker node comes online it must be pre-configured with the + /// / endpoint of the scheduler it will register with. Once the worker + /// / connects to the scheduler it must send a `RegisterSupportedProperties` + /// / command to the scheduler. The scheduler will then use this information + /// / to determine which jobs the worker can process. + #[derive(Debug, Clone)] + pub struct WorkerApiClient { + inner: tonic::client::Grpc, + } + impl WorkerApiClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl WorkerApiClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + Send + 'static, + ::Error: Into + Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> WorkerApiClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + , + >>::Error: Into + Send + Sync, + { + WorkerApiClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + /// / Registers this worker and informs the scheduler what properties + /// / this worker supports. The response must be listened on the client + /// / side for updates from the server. The first item sent will always be + /// / a ConnectionResult, after that it is undefined. + pub async fn connect_worker( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response>, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/com.github.trace_machina.nativelink.remote_execution.WorkerApi/ConnectWorker", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "com.github.trace_machina.nativelink.remote_execution.WorkerApi", + "ConnectWorker", + ), + ); + self.inner.server_streaming(req, path, codec).await + } + /// / Message used to let the scheduler know that it is still alive as + /// / well as check to see if the scheduler is still alive. The scheduler + /// / may close the connection if the worker has not sent any messages + /// / after some amount of time (configured in the scheduler's + /// / configuration). + pub async fn keep_alive( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/com.github.trace_machina.nativelink.remote_execution.WorkerApi/KeepAlive", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "com.github.trace_machina.nativelink.remote_execution.WorkerApi", + "KeepAlive", + ), + ); + self.inner.unary(req, path, codec).await + } + /// / Informs the scheduler that the service is going offline and + /// / should stop issuing any new actions on this worker. + /// / + /// / The worker may stay connected even after sending this command + /// / and may even send an `ExecuteResult` after sending this command. + /// / It is up to the scheduler implementation to decide how to handle + /// / this case. + /// / + /// / Any job that was running on this instance likely needs to be + /// / executed again, but up to the scheduler on how or when to handle + /// / this case. + pub async fn going_away( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/com.github.trace_machina.nativelink.remote_execution.WorkerApi/GoingAway", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "com.github.trace_machina.nativelink.remote_execution.WorkerApi", + "GoingAway", + ), + ); + self.inner.unary(req, path, codec).await + } + /// / Informs the scheduler about the result of an execution request. + pub async fn execution_response( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/com.github.trace_machina.nativelink.remote_execution.WorkerApi/ExecutionResponse", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new( + "com.github.trace_machina.nativelink.remote_execution.WorkerApi", + "ExecutionResponse", + ), + ); + self.inner.unary(req, path, codec).await + } + } +} +/// Generated server implementations. +pub mod worker_api_server { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with WorkerApiServer. + #[async_trait] + pub trait WorkerApi: Send + Sync + 'static { + /// Server streaming response type for the ConnectWorker method. + type ConnectWorkerStream: futures_core::Stream< + Item = std::result::Result, + > + + Send + + 'static; + /// / Registers this worker and informs the scheduler what properties + /// / this worker supports. The response must be listened on the client + /// / side for updates from the server. The first item sent will always be + /// / a ConnectionResult, after that it is undefined. + async fn connect_worker( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + /// / Message used to let the scheduler know that it is still alive as + /// / well as check to see if the scheduler is still alive. The scheduler + /// / may close the connection if the worker has not sent any messages + /// / after some amount of time (configured in the scheduler's + /// / configuration). + async fn keep_alive( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + /// / Informs the scheduler that the service is going offline and + /// / should stop issuing any new actions on this worker. + /// / + /// / The worker may stay connected even after sending this command + /// / and may even send an `ExecuteResult` after sending this command. + /// / It is up to the scheduler implementation to decide how to handle + /// / this case. + /// / + /// / Any job that was running on this instance likely needs to be + /// / executed again, but up to the scheduler on how or when to handle + /// / this case. + async fn going_away( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + /// / Informs the scheduler about the result of an execution request. + async fn execution_response( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + } + /// / This API describes how schedulers communicate with Worker nodes. + /// / + /// / When a worker node comes online it must be pre-configured with the + /// / endpoint of the scheduler it will register with. Once the worker + /// / connects to the scheduler it must send a `RegisterSupportedProperties` + /// / command to the scheduler. The scheduler will then use this information + /// / to determine which jobs the worker can process. + #[derive(Debug)] + pub struct WorkerApiServer { + inner: _Inner, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + struct _Inner(Arc); + impl WorkerApiServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + let inner = _Inner(inner); + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for WorkerApiServer + where + T: WorkerApi, + B: Body + Send + 'static, + B::Error: Into + Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + let inner = self.inner.clone(); + match req.uri().path() { + "/com.github.trace_machina.nativelink.remote_execution.WorkerApi/ConnectWorker" => { + #[allow(non_camel_case_types)] + struct ConnectWorkerSvc(pub Arc); + impl< + T: WorkerApi, + > tonic::server::ServerStreamingService + for ConnectWorkerSvc { + type Response = super::UpdateForWorker; + type ResponseStream = T::ConnectWorkerStream; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + (*inner).connect_worker(request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = ConnectWorkerSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.server_streaming(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/com.github.trace_machina.nativelink.remote_execution.WorkerApi/KeepAlive" => { + #[allow(non_camel_case_types)] + struct KeepAliveSvc(pub Arc); + impl< + T: WorkerApi, + > tonic::server::UnaryService + for KeepAliveSvc { + type Response = (); + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { (*inner).keep_alive(request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = KeepAliveSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/com.github.trace_machina.nativelink.remote_execution.WorkerApi/GoingAway" => { + #[allow(non_camel_case_types)] + struct GoingAwaySvc(pub Arc); + impl< + T: WorkerApi, + > tonic::server::UnaryService + for GoingAwaySvc { + type Response = (); + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { (*inner).going_away(request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = GoingAwaySvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/com.github.trace_machina.nativelink.remote_execution.WorkerApi/ExecutionResponse" => { + #[allow(non_camel_case_types)] + struct ExecutionResponseSvc(pub Arc); + impl tonic::server::UnaryService + for ExecutionResponseSvc { + type Response = (); + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + (*inner).execution_response(request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = ExecutionResponseSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => { + Box::pin(async move { + Ok( + http::Response::builder() + .status(200) + .header("grpc-status", "12") + .header("content-type", "application/grpc") + .body(empty_body()) + .unwrap(), + ) + }) + } + } + } + } + impl Clone for WorkerApiServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + impl Clone for _Inner { + fn clone(&self) -> Self { + Self(Arc::clone(&self.0)) + } + } + impl std::fmt::Debug for _Inner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self.0) + } + } + impl tonic::server::NamedService for WorkerApiServer { + const NAME: &'static str = "com.github.trace_machina.nativelink.remote_execution.WorkerApi"; + } +} diff --git a/proto/genproto/lib.rs b/proto/genproto/lib.rs index b3aabbcc0..3a1d95d25 100644 --- a/proto/genproto/lib.rs +++ b/proto/genproto/lib.rs @@ -34,9 +34,9 @@ pub mod build { pub mod com { pub mod github { pub mod trace_machina { - pub mod native_link { + pub mod nativelink { pub mod remote_execution { - include!("com.github.trace_machina.native_link.remote_execution.pb.rs"); + include!("com.github.trace_machina.nativelink.remote_execution.pb.rs"); } } } diff --git a/run_integration_tests.sh b/run_integration_tests.sh index 0d7d55786..71f9e51e7 100755 --- a/run_integration_tests.sh +++ b/run_integration_tests.sh @@ -61,20 +61,20 @@ cd "$SELF_DIR/deployment-examples/docker-compose" export UNDER_TEST_RUNNER=1 # Ensure our cache locations are empty. -sudo rm -rf ~/.cache/native-link -mkdir -p ~/.cache/native-link +sudo rm -rf ~/.cache/nativelink +mkdir -p ~/.cache/nativelink # Ensure our docker compose is not running. sudo docker-compose rm --stop -f -export TMPDIR=$HOME/.cache/native-link/ +export TMPDIR=$HOME/.cache/nativelink/ mkdir -p "$TMPDIR" if [[ "$OSTYPE" == "darwin"* ]]; then - export CACHE_DIR=$(mktemp -d "${TMPDIR}native-link-integration-test") + export CACHE_DIR=$(mktemp -d "${TMPDIR}nativelink-integration-test") else echo "Assumes Linux/WSL" - export CACHE_DIR=$(mktemp -d --tmpdir="$TMPDIR" --suffix="-native-link-integration-test") + export CACHE_DIR=$(mktemp -d --tmpdir="$TMPDIR" --suffix="-nativelink-integration-test") fi export BAZEL_CACHE_DIR="$CACHE_DIR/bazel" @@ -84,18 +84,18 @@ echo "" # New line. DID_FAIL=0 -export NATIVE_LINK_DIR="$CACHE_DIR/native-link" -mkdir -p "$NATIVE_LINK_DIR" +export nativelink_DIR="$CACHE_DIR/nativelink" +mkdir -p "$nativelink_DIR" for pattern in "${TEST_PATTERNS[@]}"; do find "$SELF_DIR/integration_tests/" -name "$pattern" -type f -print0 | while IFS= read -r -d $'\0' fullpath; do # Cleanup. - echo "Cleaning up cache directories TURBOC_CACHE_DIR: $NATIVE_LINK_DIR" - echo "Checking for existince of the NATIVE_LINK_DIR" - if [ -d "$NATIVE_LINK_DIR" ]; then - sudo find "$NATIVE_LINK_DIR" -delete + echo "Cleaning up cache directories TURBOC_CACHE_DIR: $nativelink_DIR" + echo "Checking for existince of the nativelink_DIR" + if [ -d "$nativelink_DIR" ]; then + sudo find "$nativelink_DIR" -delete else - echo "Directory $NATIVE_LINK_DIR does not exist." + echo "Directory $nativelink_DIR does not exist." fi bazel --output_base="$BAZEL_CACHE_DIR" clean diff --git a/src/bin/cas.rs b/src/bin/cas.rs index 90a735919..757573c0b 100644 --- a/src/bin/cas.rs +++ b/src/bin/cas.rs @@ -25,25 +25,25 @@ use futures::future::{select_all, BoxFuture, OptionFuture, TryFutureExt}; use futures::FutureExt; use hyper::server::conn::Http; use hyper::{Response, StatusCode}; -use native_link_config::cas_server::{ +use nativelink_config::cas_server::{ CasConfig, CompressionAlgorithm, ConfigDigestHashFunction, GlobalConfig, ServerConfig, WorkerConfig, }; -use native_link_scheduler::default_scheduler_factory::scheduler_factory; -use native_link_scheduler::worker::WorkerId; -use native_link_service::ac_server::AcServer; -use native_link_service::bytestream_server::ByteStreamServer; -use native_link_service::capabilities_server::CapabilitiesServer; -use native_link_service::cas_server::CasServer; -use native_link_service::execution_server::ExecutionServer; -use native_link_service::worker_api_server::WorkerApiServer; -use native_link_store::default_store_factory::store_factory; -use native_link_store::store_manager::StoreManager; -use native_link_util::common::fs::{set_idle_file_descriptor_timeout, set_open_file_limit}; -use native_link_util::digest_hasher::{set_default_digest_hasher_func, DigestHasherFunc}; -use native_link_util::metrics_utils::{ +use nativelink_scheduler::default_scheduler_factory::scheduler_factory; +use nativelink_scheduler::worker::WorkerId; +use nativelink_service::ac_server::AcServer; +use nativelink_service::bytestream_server::ByteStreamServer; +use nativelink_service::capabilities_server::CapabilitiesServer; +use nativelink_service::cas_server::CasServer; +use nativelink_service::execution_server::ExecutionServer; +use nativelink_service::worker_api_server::WorkerApiServer; +use nativelink_store::default_store_factory::store_factory; +use nativelink_store::store_manager::StoreManager; +use nativelink_util::common::fs::{set_idle_file_descriptor_timeout, set_open_file_limit}; +use nativelink_util::digest_hasher::{set_default_digest_hasher_func, DigestHasherFunc}; +use nativelink_util::metrics_utils::{ set_metrics_enabled_for_this_thread, Collector, CollectorState, Counter, MetricsComponent, Registry, }; -use native_link_worker::local_worker::new_local_worker; +use nativelink_worker::local_worker::new_local_worker; use parking_lot::Mutex; use rustls_pemfile::{certs, pkcs8_private_keys}; use scopeguard::guard; @@ -64,12 +64,12 @@ const DEFAULT_PROMETHEUS_METRICS_PATH: &str = "/metrics"; const DEFAULT_ADMIN_API_PATH: &str = "/admin"; /// Name of environment variable to disable metrics. -const METRICS_DISABLE_ENV: &str = "NATIVE_LINK_DISABLE_METRICS"; +const METRICS_DISABLE_ENV: &str = "nativelink_DISABLE_METRICS"; /// Backend for bazel remote execution / cache API. #[derive(Parser, Debug)] #[clap( - author = "Trace Machina, Inc. ", + author = "Trace Machina, Inc. ", version = "0.0.1", about, long_about = None @@ -81,7 +81,7 @@ struct Args { } async fn inner_main(cfg: CasConfig, server_start_timestamp: u64) -> Result<(), Box> { - let mut root_metrics_registry = ::with_prefix("native_link"); + let mut root_metrics_registry = ::with_prefix("nativelink"); let store_manager = Arc::new(StoreManager::new()); { @@ -372,8 +372,8 @@ async fn inner_main(cfg: CasConfig, server_start_timestamp: u64) -> Result<(), B .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)) .map(|_| { // This is a hack to get around this bug: https://github.com/prometheus/client_rust/issues/155 - buf = buf.replace("native_link_native_link_stores_", ""); - buf = buf.replace("native_link_native_link_workers_", ""); + buf = buf.replace("nativelink_nativelink_stores_", ""); + buf = buf.replace("nativelink_nativelink_workers_", ""); let mut response = Response::new(buf); // Per spec we should probably use `application/openmetrics-text; version=1.0.0; charset=utf-8` // https://github.com/OpenObservability/OpenMetrics/blob/1386544931307dff279688f332890c31b6c5de36/specification/OpenMetrics.md#overall-structure diff --git a/tools/local-image-test.nix b/tools/local-image-test.nix index d36d0d884..74f7fd16e 100644 --- a/tools/local-image-test.nix +++ b/tools/local-image-test.nix @@ -15,10 +15,10 @@ pkgs.writeShellScriptBin "local-image-test" '' | ${pkgs.skopeo}/bin/skopeo \ copy \ docker-archive:/dev/stdin \ - docker-daemon:native-link:''${IMAGE_TAG} + docker-daemon:nativelink:''${IMAGE_TAG} # Ensure that the image has minimal closure size. CI=1 ${pkgs.dive}/bin/dive \ - native-link:''${IMAGE_TAG} \ + nativelink:''${IMAGE_TAG} \ --highestWastedBytes=0 ''