-
Notifications
You must be signed in to change notification settings - Fork 51
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add configuration breakdown page (#725)
- Loading branch information
1 parent
433829c
commit 35daf43
Showing
4 changed files
with
330 additions
and
11 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,5 @@ | ||
| --- | ||
| sidebar_position: 4 | ||
| sidebar_position: 5 | ||
| title: 'About' | ||
| description: 'What is NativeLink?' | ||
| --- | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,319 @@ | ||
| --- | ||
| sidebar_position: 3 | ||
| title: 'Configuration' | ||
| description: 'Learn how to configure NativeLink' | ||
| --- | ||
|
|
||
| import Tabs from '@theme/Tabs'; | ||
| import TabItem from '@theme/TabItem'; | ||
|
|
||
| NativeLink uses JSON files for configuration of the CAS and Scheduler. | ||
|
|
||
| This section will break down all of the customizable fields in these JSON files so | ||
| they can be modified to suit your needs. | ||
|
|
||
| ## ⚙️ Configuration Instructions | ||
|
|
||
| <Tabs> | ||
| <TabItem value="cas" label="CAS Configuration" default> | ||
| ### 1. Copy the [Sample CAS.json](https://github.com/blakehatch/nativelink/blob/main/deployment-examples/kubernetes/cas.json). | ||
| <p style={{ | ||
| maxHeight: '200px', | ||
| overflowY: 'auto', | ||
| borderRadius: '10px 10px 10px 10px' | ||
| }}> | ||
| <!-- vale off --> | ||
| ```json5 | ||
| { | ||
| "stores": { | ||
| "AC_FAST_SLOW_STORE": { | ||
| "fast_slow": { | ||
| "fast": { | ||
| "memory": { | ||
| "eviction_policy": { | ||
| "max_bytes": 1000000000 | ||
| } | ||
| } | ||
| }, | ||
| "slow": { | ||
| "experimental_s3_store": { | ||
| "region": "${NATIVE_LINK_AWS_REGION:-us-east-1}", | ||
| "bucket": "${NATIVE_LINK_AWS_S3_CAS_BUCKET:-not_set}", | ||
| "key_prefix": "ac/", | ||
| "retry": { | ||
| "max_retries": 6, | ||
| "delay": 0.3, | ||
| "jitter": 0.5 | ||
| }, | ||
| "additional_max_concurrent_requests": 10 | ||
| } | ||
| } | ||
| } | ||
| }, | ||
| "AC_S3_STORE": { | ||
| "completeness_checking": { | ||
| "backend": { | ||
| "ref_store": { | ||
| "name": "AC_FAST_SLOW_STORE" | ||
| } | ||
| }, | ||
| "cas_store": { | ||
| "ref_store": { | ||
| "name": "CAS_S3_STORE" | ||
| } | ||
| } | ||
| } | ||
| }, | ||
| "CAS_FAST_SLOW_STORE": { | ||
| "fast_slow": { | ||
| "fast": { | ||
| "memory": { | ||
| "eviction_policy": { | ||
| "max_bytes": 1000000000 | ||
| } | ||
| } | ||
| }, | ||
| "slow": { | ||
| "experimental_s3_store": { | ||
| "region": "${NATIVE_LINK_AWS_REGION:-us-east-1}", | ||
| "bucket": "${NATIVE_LINK_AWS_S3_CAS_BUCKET:-not_set}", | ||
| "key_prefix": "cas/", | ||
| "retry": { | ||
| "max_retries": 6, | ||
| "delay": 0.3, | ||
| "jitter": 0.5 | ||
| }, | ||
| "additional_max_concurrent_requests": 10 | ||
| } | ||
| } | ||
| } | ||
| }, | ||
| "CAS_S3_STORE": { | ||
| "existence_cache": { | ||
| "backend": { | ||
| "compression": { | ||
| "compression_algorithm": { | ||
| "lz4": {} | ||
| }, | ||
| "backend": { | ||
| "ref_store": { | ||
| "name": "CAS_FAST_SLOW_STORE" | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| }, | ||
| "servers": [ | ||
| { | ||
| "listener": { | ||
| "http": { | ||
| "socket_address": "0.0.0.0:{{ .Values.cas.service.grpcPort }}" | ||
| } | ||
| }, | ||
| "services": { | ||
| "cas": { | ||
| "main": { | ||
| "cas_store": "CAS_S3_STORE" | ||
| } | ||
| }, | ||
| "ac": { | ||
| "main": { | ||
| "ac_store": "AC_S3_STORE" | ||
| } | ||
| }, | ||
| "capabilities": {}, | ||
| "bytestream": { | ||
| "cas_stores": { | ||
| "main": "CAS_S3_STORE" | ||
| } | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "listener": { | ||
| "http": { | ||
| "socket_address": "0.0.0.0:50061" | ||
| } | ||
| }, | ||
| "services": { | ||
| "experimental_prometheus": { | ||
| "path": "/metrics" | ||
| } | ||
| } | ||
| } | ||
| ] | ||
| } | ||
| ``` | ||
| <!-- vale on --> | ||
| </p> | ||
|
|
||
|
|
||
| </TabItem> | ||
|
|
||
| <TabItem value="update" label="Scheduler Configuration"> | ||
| ### 1. Copy the [Sample Scheduler.json](https://github.com/blakehatch/nativelink/blob/main/deployment-examples/kubernetes/scheduler.json). | ||
| <p | ||
| style={{ | ||
| maxHeight: '200px', | ||
| overflowY: 'auto' | ||
| }}> | ||
| <!-- vale off --> | ||
| ```json5 | ||
| { | ||
| "stores": { | ||
| "GRPC_LOCAL_STORE": { | ||
| // Note: This file is used to test GRPC store. | ||
| "grpc": { | ||
| "instance_name": "main", | ||
| "endpoints": [ | ||
| {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} | ||
| ], | ||
| "store_type": "cas" | ||
| } | ||
| }, | ||
| "GRPC_LOCAL_AC_STORE": { | ||
| // Note: This file is used to test GRPC store. | ||
| "grpc": { | ||
| "instance_name": "main", | ||
| "endpoints": [ | ||
| {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} | ||
| ], | ||
| "store_type": "ac" | ||
| } | ||
| } | ||
| }, | ||
| "schedulers": { | ||
| "MAIN_SCHEDULER": { | ||
| "simple": { | ||
| "supported_platform_properties": { | ||
| "cpu_count": "minimum", | ||
| "OSFamily": "exact", | ||
| "container-image": "exact" | ||
| } | ||
| } | ||
| } | ||
| }, | ||
| "servers": [{ | ||
| "listener": { | ||
| "http": { | ||
| "socket_address": "0.0.0.0:50052" | ||
| } | ||
| }, | ||
| "services": { | ||
| "ac": { | ||
| "main": { | ||
| "ac_store": "GRPC_LOCAL_AC_STORE" | ||
| } | ||
| }, | ||
| "execution": { | ||
| "main": { | ||
| "cas_store": "GRPC_LOCAL_STORE", | ||
| "scheduler": "MAIN_SCHEDULER", | ||
| } | ||
| }, | ||
| "capabilities": { | ||
| "main": { | ||
| "remote_execution": { | ||
| "scheduler": "MAIN_SCHEDULER", | ||
| } | ||
| } | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "listener": { | ||
| "http": { | ||
| "socket_address": "0.0.0.0:50061", | ||
| } | ||
| }, | ||
| "services": { | ||
| // Note: This should be served on a different port, because it has | ||
| // a different permission set than the other services. | ||
| // In other words, this service is a backend api. The ones above | ||
| // are a frontend api. | ||
| "worker_api": { | ||
| "scheduler": "MAIN_SCHEDULER", | ||
| } | ||
| } | ||
| }] | ||
| } | ||
| ``` | ||
| <!-- vale on --> | ||
| </p> | ||
|
|
||
| </TabItem> | ||
|
|
||
| </Tabs> | ||
|
|
||
| ### 2. Paste default sample where needed in custom [deployment](https://github.com/blakehatch/nativelink/tree/main/deployment-examples). | ||
| ### 3. Change settings to preferred values. | ||
|
|
||
| <Tabs> | ||
| <TabItem value="stores" label="Stores" default> | ||
| | Key | Type | Description | Example | | ||
| |------------------------------------------|---------------|-----------------------------------------------------------------------------|-------------------------------------------| | ||
| | stores | Object | Defines the store configurations for CAS and AC.| `{"stores": {...}}`| | ||
| | AC_FAST_SLOW_STORE | Object | Configuration for an AC store that has both fast and slow storage options.| `{"AC_FAST_SLOW_STORE": {...}}`| | ||
| | CAS_FAST_SLOW_STORE | Object | Configuration for a CAS store with both fast and slow storage options.| `{"CAS_FAST_SLOW_STORE": {...}}`| | ||
| | AC_MAIN_STORE | Object | Main configuration for an AC store.| `{"AC_MAIN_STORE": {...}}`| | ||
| | CAS_MAIN_STORE | Object | Main configuration for a CAS store.| `{"CAS_MAIN_STORE": {...}}`| | ||
| | GRPC_LOCAL_STORE | Object | Configuration for a local GRPC store.| `{"GRPC_LOCAL_STORE": {...}}`| | ||
| | GRPC_LOCAL_AC_STORE | Object | Configuration for a local GRPC AC store.| `{"GRPC_LOCAL_AC_STORE": {...}}`| | ||
| | MAIN_SCHEDULER | Object | Configuration for the main scheduler.| `{"MAIN_SCHEDULER": {...}}`| | ||
| | fast | Object | Configuration for fast storage, typically in-memory.| `{"fast": {...}}`| | ||
| | eviction_policy | Object | Policy for evicting data from memory.| `{"eviction_policy": {...}}`| | ||
| | slow | Object | Configuration for slow storage, typically disk or cloud storage.| `{"slow": {...}}`| | ||
| | cas_stores | Object | Configuration for CAS stores used by the bytestream service.| `{"cas_stores": {...}}`| | ||
| | experimental_prometheus | Object | Configuration for the experimental Prometheus metrics service.| `{"experimental_prometheus": {...}}`| | ||
| | memory | Object | Memory store will store all data in a hashmap in memory.| `{"memory": {"eviction_policy": {...}}}`| | ||
| | experimental_s3_store | Object | S3 store will use Amazon's S3 service as a backend to store the files. This configuration will never delete files, so you are responsible for purging old files in other ways.| `{"experimental_s3_store": {"region": "eu-north-1", "bucket": "...", "key_prefix": "...", "retry": {...}}}`| | ||
| | verify | Object | Verify store is used to apply verifications to an underlying store implementation. It is strongly encouraged to validate as much data as you can before accepting data from a client.| `{"verify": {"backend": {...}, "verify_size": true, "hash_verification_function": "sha256"}}`| | ||
| | completeness_checking | Object | Completeness checking store verifies if the output files & folders exist in the CAS before forwarding the request to the underlying store. Note: This store should only be used on AC stores.| `{"completeness_checking": {"backend": {...}, "cas_store": {"ref_store": {...}}}}`| | ||
| | compression | Object | A compression store that will compress the data inbound and outbound. There will be a non-trivial cost to compress and decompress the data.| `{"compression": {"compression_algorithm": {...}, "backend": {...}}}`| | ||
| | dedup | Object | A dedup store will take the inputs and run a rolling hash algorithm on them to slice the input into smaller parts then run a sha256 algorithm on the slice.| `{"dedup": {"index_store": {"fast_slow": {"fast": {...}, "slow": {...}}}, "content_store": {...}}}`| | ||
| | existence_cache | Object | Existence store will wrap around another store and cache calls to has so that subsequent has_with_results calls will be faster. Note: This store should only be used on CAS stores.| `{"existence_cache": {"backend": {...}, "cas_store": {"ref_store": {...}}}}`| | ||
| | fast_slow | Object | FastSlow store will first try to fetch the data from the `fast` store and then if it does not exist try the `slow` store.| `{"fast_slow": {"fast": {...}, "slow": {...}}}`| | ||
| | shard | Object | Shards the data to multiple stores. This is useful for cases when you want to distribute the load across multiple stores.| `{"shard": {"stores": [{...}]}}`| | ||
| | filesystem | Object | Stores the data on the filesystem. This store is designed for local persistent storage.| `{"filesystem": {"content_path": "...", "temp_path": "...", "eviction_policy": {...}}}`| | ||
| | ref_store | Object | Store used to reference a store in the root store manager. This is useful for cases when you want to share a store in different nested stores.| `{"ref_store": {"name": "FS_CONTENT_STORE"}}`| | ||
| | size_partitioning | Object | Uses the size field of the digest to separate which store to send the data. This is useful for cases when you'd like to put small objects in one store and large objects in another store.| `{"size_partitioning": {"size": 134217728, "lower_store": {...}, "upper_store": {...}}}`| | ||
| | grpc | Object | This store will pass-through calls to another GRPC store. This store is not designed to be used as a sub-store of another store.| `{"grpc": {"instance_name": "main", "endpoints": [{...}], "store_type": "ac"}}`| | ||
| | noop | Object | Noop store is a store that sends streams into the void and all data retrieval will return 404 (NotFound). This can be useful for cases where you may need to partition your data and part of your data needs to be discarded.| `{"noop": {}}`| | ||
| </TabItem> | ||
| <TabItem value="behavior" label="Behavior" default> | ||
| | Key | Type | Description | Example | | ||
| |------------------------------------------|---------------|-----------------------------------------------------------------------------|-------------------------------------------| | ||
| | retry | Object | Configuration for retry behavior on failed requests.| `{"retry": {...}}`| | ||
| | backend | Object | Backend configuration for completeness checking.| `{"backend": {...}}`| | ||
| | listener | Object | Configuration for the server listener.| `{"listener": {...}}`| | ||
| | http | Object | HTTP-specific listener settings.| `{"http": {"socket_address": "0.0.0.0:50071", "tls": {...}}}`| | ||
| | services | Object | Defines the services provided by the server.| `{"services": {...}}`| | ||
| | cas | Object | Configuration for the CAS service.| `{"cas": {...}}`| | ||
| | main | Object | Main configuration for a service.| `{"main": {...}}`| | ||
| | ac | Object | Configuration for the AC service.| `{"ac": {...}}`| | ||
| | capabilities | Object | Configuration for service capabilities.| `{"capabilities": {...}}`| | ||
| | bytestream | Object | Configuration for the bytestream service.| `{"bytestream": {...}}`| | ||
| </TabItem> | ||
| <TabItem value="fields" label="Fields" default> | ||
| | Key | Type | Description | Example | | ||
| |------------------------------------------|---------------|-----------------------------------------------------------------------------|-------------------------------------------| | ||
| | max_bytes | Integer | Maximum bytes that can be stored in memory before eviction occurs.| 1000000000| | ||
| | region | String | AWS region where the S3 bucket is located.| "us-east-2"| | ||
| | bucket | String | Name of the S3 bucket used for storage.| "YOUR-BUCKET-NAME"| | ||
| | key_prefix | String | Prefix for all keys stored in this bucket.| "YOUR-KEY-PREFIX"| | ||
| | max_retries | Integer | Maximum number of retries for a failed request.| 6| | ||
| | delay | Float | Initial delay between retries, in seconds.| 0.3| | ||
| | jitter | Float | Randomized delay added to the retry delay to prevent thundering herd.| 0.5| | ||
| | additional_max_concurrent_requests | Integer | Additional concurrent requests allowed to the store beyond the default.| 10| | ||
| | name | String | Name of the reference store.| "AC_FAST_SLOW_STORE"| | ||
| | socket_address | String | Address and port where the server listens for connections.| "0.0.0.0:50052"| | ||
| | cas_store | String | Reference to the CAS store used by the service.| "CAS_S3_STORE"| | ||
| | ac_store | String | Reference to the AC store used by the service.| "AC_S3_STORE"| | ||
| | path | String | HTTP path where the metrics are served.| "/metrics"| | ||
| | servers | Array | Defines the server configurations.| `{"servers": {...}}`| | ||
| </TabItem> | ||
| </Tabs> | ||
|
|
||
| Additional examples for store configurations and additional store options can be found in the **[store configuration documentation](https://github.com/blakehatch/nativelink/blob/main/nativelink-config/src/stores.rs)**. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters