diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index f3e4380c44..8c346ad626 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -69,6 +69,7 @@ include::third-party:partial$nav.adoc[] ** xref:learn:clusters-and-availability/size-limitations.adoc[Size Limits] ** xref:learn:clusters-and-availability/nodes.adoc[Nodes] *** xref:learn:clusters-and-availability/rebalance.adoc[Rebalance] + **** xref:learn:clusters-and-availability/rebalance-and-index-service.adoc[Index Rebalance] *** xref:learn:clusters-and-availability/removal.adoc[Removal] *** xref:learn:clusters-and-availability/failover.adoc[Failover] **** xref:learn:clusters-and-availability/graceful-failover.adoc[Graceful] diff --git a/modules/install/pages/upgrade-feature-availability.adoc b/modules/install/pages/upgrade-feature-availability.adoc index e105d4b3cc..2f7a31757c 100644 --- a/modules/install/pages/upgrade-feature-availability.adoc +++ b/modules/install/pages/upgrade-feature-availability.adoc @@ -22,7 +22,7 @@ _Not usable in mixed mode_ indicates that cluster-upgrade must be fully complete |=== | Feature | Whether Usable in Mixed Mode -| xref:learn:clusters-and-availability/rebalance.adoc#index-rebalance-methods[File-based Index Rebalancing] +| xref:learn:clusters-and-availability/rebalance-and-index-service.adoc#index-rebalance-methods[] | You cannot enable file-based index rebalancing until all nodes have upgraded to version 7.6 or later. | xref:learn:data/scopes-and-collections.adoc[Scopes & Collections] diff --git a/modules/introduction/partials/new-features-76.adoc b/modules/introduction/partials/new-features-76.adoc index 2d34a11d66..ccb292427b 100644 --- a/modules/introduction/partials/new-features-76.adoc +++ b/modules/introduction/partials/new-features-76.adoc @@ -150,7 +150,7 @@ See xref:manage:monitor/set-up-prometheus-for-monitoring.adoc[Configure Promethe * You can choose to have the rebalance process move an index's files between nodes instead of rebuilding them from scratch. This setting improves rebalance performance as moving the files is faster than rebuilding them. -See xref:learn:clusters-and-availability/rebalance.adoc#index-rebalance-methods[Index Rebalance Methods]. +See xref:learn:clusters-and-availability/rebalance-and-index-service.adoc#index-rebalance-methods[]. === Search Service diff --git a/modules/introduction/partials/new-features-76_2.adoc b/modules/introduction/partials/new-features-76_2.adoc index f4dee51fa7..f285aee8ac 100644 --- a/modules/introduction/partials/new-features-76_2.adoc +++ b/modules/introduction/partials/new-features-76_2.adoc @@ -63,7 +63,7 @@ For details, see xref:n1ql:n1ql-language-reference/createindex.adoc[CREATE INDEX * In Couchbase Server Versions 7.6.0 and 7.6.1, enabling file-based index rebalance prevented you from controlling which Index Service nodes contain an index. Version 7.6.2 removes this restriction. You can now use the `WITH ` clause of the `CREATE INDEX` {sqlpp} statement when your cluster has file-based index rebalancing enabled. -See xref:learn:clusters-and-availability/rebalance.adoc#index-rebalance-methods[Index Rebalance Methods] for more information. +See xref:learn:clusters-and-availability/rebalance-and-index-service.adoc#index-rebalance-methods[] for more information. + NOTE: You still cannot use the `WITH ` clause with the `ALTER INDEX` statement when your cluster has file-based index rebalancing enabled. diff --git a/modules/learn/assets/images/clusters-and-availability/failover-with-replica.png b/modules/learn/assets/images/clusters-and-availability/failover-with-replica.png new file mode 100644 index 0000000000..56aa81cc2d Binary files /dev/null and b/modules/learn/assets/images/clusters-and-availability/failover-with-replica.png differ diff --git a/modules/learn/assets/images/clusters-and-availability/failover-without-replica.png b/modules/learn/assets/images/clusters-and-availability/failover-without-replica.png new file mode 100644 index 0000000000..899814303c Binary files /dev/null and b/modules/learn/assets/images/clusters-and-availability/failover-without-replica.png differ diff --git a/modules/learn/assets/images/clusters-and-availability/fbr-rebalance-out.png b/modules/learn/assets/images/clusters-and-availability/fbr-rebalance-out.png new file mode 100644 index 0000000000..fb4c974921 Binary files /dev/null and b/modules/learn/assets/images/clusters-and-availability/fbr-rebalance-out.png differ diff --git a/modules/learn/assets/images/clusters-and-availability/fbr-scale-out.png b/modules/learn/assets/images/clusters-and-availability/fbr-scale-out.png new file mode 100644 index 0000000000..0b69d4969b Binary files /dev/null and b/modules/learn/assets/images/clusters-and-availability/fbr-scale-out.png differ diff --git a/modules/learn/assets/images/clusters-and-availability/fbr-swap-one-on-one.png b/modules/learn/assets/images/clusters-and-availability/fbr-swap-one-on-one.png new file mode 100644 index 0000000000..568b0c0efe Binary files /dev/null and b/modules/learn/assets/images/clusters-and-availability/fbr-swap-one-on-one.png differ diff --git a/modules/learn/assets/images/clusters-and-availability/full-swap-stage0.png b/modules/learn/assets/images/clusters-and-availability/full-swap-stage0.png new file mode 100644 index 0000000000..a4171c2db4 Binary files /dev/null and b/modules/learn/assets/images/clusters-and-availability/full-swap-stage0.png differ diff --git a/modules/learn/assets/images/clusters-and-availability/full-swap-stage1.png b/modules/learn/assets/images/clusters-and-availability/full-swap-stage1.png new file mode 100644 index 0000000000..a005a3b1c1 Binary files /dev/null and b/modules/learn/assets/images/clusters-and-availability/full-swap-stage1.png differ diff --git a/modules/learn/assets/images/clusters-and-availability/full-swap-stage2.png b/modules/learn/assets/images/clusters-and-availability/full-swap-stage2.png new file mode 100644 index 0000000000..226a4389ff Binary files /dev/null and b/modules/learn/assets/images/clusters-and-availability/full-swap-stage2.png differ diff --git a/modules/learn/assets/images/clusters-and-availability/full-swap-stage3.png b/modules/learn/assets/images/clusters-and-availability/full-swap-stage3.png new file mode 100644 index 0000000000..be7a9501b8 Binary files /dev/null and b/modules/learn/assets/images/clusters-and-availability/full-swap-stage3.png differ diff --git a/modules/learn/assets/images/clusters-and-availability/removal-with-replica.png b/modules/learn/assets/images/clusters-and-availability/removal-with-replica.png new file mode 100644 index 0000000000..79b53db3ca Binary files /dev/null and b/modules/learn/assets/images/clusters-and-availability/removal-with-replica.png differ diff --git a/modules/learn/assets/images/clusters-and-availability/removal-without-replica.png b/modules/learn/assets/images/clusters-and-availability/removal-without-replica.png new file mode 100644 index 0000000000..adc06a5751 Binary files /dev/null and b/modules/learn/assets/images/clusters-and-availability/removal-without-replica.png differ diff --git a/modules/learn/assets/images/clusters-and-availability/swap-without-optimization.png b/modules/learn/assets/images/clusters-and-availability/swap-without-optimization.png new file mode 100644 index 0000000000..ef6a4636fe Binary files /dev/null and b/modules/learn/assets/images/clusters-and-availability/swap-without-optimization.png differ diff --git a/modules/learn/pages/clusters-and-availability/failover.adoc b/modules/learn/pages/clusters-and-availability/failover.adoc index 3d9d014e92..ccd2759f51 100644 --- a/modules/learn/pages/clusters-and-availability/failover.adoc +++ b/modules/learn/pages/clusters-and-availability/failover.adoc @@ -27,9 +27,11 @@ Graceful failover _must_ be manually initiated. Hard failover _can_ be manually Hard failover can also be initiated automatically by Couchbase Server: this is known as _automatic_ failover. The Cluster Manager detects the unavailability of a node, and duly initiates a _hard_ failover, without administrator intervention. - -Note that when a node is failed over (as opposed to _removed_), some replica vBuckets are lost from the surviving nodes; since some are promoted to _active_ status, and are not replaced with new replica-copies. -By contrast, xref:learn:clusters-and-availability/removal.adoc[removal] creates new copies of those replica vBuckets that would otherwise be lost. This maintains the cluster's previous level of data-availability; but results in greater competition for memory resources, across the surviving nodes. +NOTE: When a node is failed over, some replica vBuckets on the remaining nodes are promoted to active status but are not recreated as new replicas. +As a result, the cluster temporarily loses those replica copies. ++ +In contrast, when a node is removed, the cluster automatically restores any replica vBuckets that would otherwise be lost during xref:learn:clusters-and-availability/removal.adoc[removal]. +This preserves the cluster’s data availability level, but increases memory resource usage on the surviving nodes. Ideally, after any failover, xref:learn:clusters-and-availability/rebalance.adoc[rebalance] should be performed. This is especially important when a Data Service node has been failed over, since the rebalance will ensure an optimal ratio of active to replica vBuckets across all the remaining Data Service nodes. @@ -180,7 +182,6 @@ For example, Couchbase Server considers a node running Analytics, Indexing, and If one is available, Couchbase Service always chooses an the cluster's xref:learn:clusters-and-availability/cluster-manager.adoc#master-services[Master Services] to host the active Master Service. Arbiter nodes do not run any services, and are therefore perfect candidates for hosting the Master Service. - [#node-removal] == Node Removal diff --git a/modules/learn/pages/clusters-and-availability/rebalance-and-index-service.adoc b/modules/learn/pages/clusters-and-availability/rebalance-and-index-service.adoc new file mode 100644 index 0000000000..c793a2b1be --- /dev/null +++ b/modules/learn/pages/clusters-and-availability/rebalance-and-index-service.adoc @@ -0,0 +1,815 @@ += Index Rebalance +:description: pass:q[This page explains how rebalance operations impact the Index Service in Couchbase Server, covering file-based rebalance, shard affinity, index redistribution, and node failover.] +:page-aliases: clusters-and-availability:rebalance +:page-toclevels: 3 + +[abstract] +{description} + +Rebalance affects different services differently. +The following sections explain how rebalance operation affects the Index Service. + +== Index Service + +The Index Service maintains cluster-wide index definitions and metadata to <<#index-redistribution,redistribute indexes and replicas>> during rebalance operations. + +The rebalance operation evaluates each node's CPU, RAM, and disk bandwidth to minimize effects on database performance. + +== Index Storage Modes, Shards, and Partitions + +Couchbase Server defines how Global Secondary Indexes (GSI) stores and manages its underlying data structures using the following Index Storage Modes: + +* xref:learn:services-and-indexes/indexes/storage-modes.adoc#standard-index-storage[Standard GSI Storage], which stores index data on disk required for File-Based Rebalance. + +* xref:learn:services-and-indexes/indexes/storage-modes.adoc#memory-optimized-index-storage[Memory Optimized GSI Storage], which stores most index structures in-memory and does not support file-based operations. + +Plasma, the storage engine for GSI, stores index data in shards. +Each shard is a set of on-disk files that can contain one or more index partitions, allowing multiple indexes to share the same physical storage. +Sharing shard files across indexes reduces disk overhead and improves index storage scalability. + +[#shard-affinity] +== Shard Affinity + +In Couchbase Server versions earlier than 7.6, Plasma automatically chose which shard an index partition belonged to. +Couchbase Server 7.6 introduces index-shard affinity and continues to support it in later versions. + +With shard affinity, the GSI layer assigns a shard slot (an alternate shard ID) to each index partition and provides this information to Plasma. +Plasma then places the partition’s data into the designated shard on the node as specified by the GSI layer. +Plasma is unaware of the cluster topology and high-availability (HA) constraints, so the GSI layer makes this decision. + +Only GSI can do the following: + +* The replicas are distributed appropriately across nodes and server groups. +* The corresponding replicas share the same shard slot (shard ID). + +As a result, this design provides the following benefits: + +* Provides deterministic collocation, where all indexes with the same shard ID share the same shard files. +* Prevents replica-on-same-node conflicts. +* Enables efficient data movement through File-Based Rebalance. + +Shard affinity keeps all index partitions with the same shard ID on the same node and moves them as a single unit during index rebalancing. +This behavior also applies to internal data structures associated with those shards, which remain collocated and move together. + +For information about how this affects File-Based Rebalance, see <<#shard-affinity-and-fbr,Relationship Between Shard Affinity and File-Based Rebalance>>. + +[#index-rebalance-methods] +== Index Rebalance Methods + +Couchbase Server provides the following index rebalance methods: + +* *Standard Rebalance*: This is the default DCP-based method used for self-managed clusters. +This method reads data from the Data Service and rebuilds indexes on the destination node. +Couchbase Server uses this method when File-Based Rebalance is disabled. ++ +For an example of DCP-based rebalance operation, see <<#full-swap-rebalance,Full Swap Rebalance (DCP-based Rebuild)>>. + +* *<<#file-based-rebalance-fbr,File-Based Rebalance or Shard Based Rebalance>>*: This is the relocation method. +In this method, Couchbase Server moves index files between nodes instead of rebuilding them during a rebalance. +Copying the index files is faster than having the target node rebuild the index from scratch. ++ +NOTE: For Couchbase Capella (Provisioned Clusters), File-Based Rebalance is enabled by default. +It can be disabled through a support request. +The on and off behavior is the same as in self-managed clusters. + +[#file-based-rebalance-fbr] +=== File-Based Rebalance or Shard Based Rebalance +[.edition]#{enterprise}# + +File-Based Rebalance, called also as Shard Based Rebalance, moves index files between nodes instead of rebuilding them. + +File-Based Rebalance is composed of the following features: + +* xref:learn:clusters-and-availability/rebalance-and-index-service.adoc#shard-affinity[*Shard Affinity*]: It decides shard placement, so partitions and their replicas share the same slot ID. +* *Shard Based Rebalance:* It moves the shard files between nodes during rebalance. + +[NOTE] +==== +* File-Based Rebalance is supported only if you have enabled Standard Global Secondary xref:manage:manage-settings/general-settings.adoc#index-storage-mode[Index Storage Mode] on your cluster. + +* You cannot use File-Based Rebalance if you have enabled xref:learn:services-and-indexes/indexes/storage-modes.adoc#memory-optimized-index-storage[Memory Optimized Index Storage] in the xref:manage:manage-settings/general-settings.adoc#index-storage-mode[Index Storage Mode] settings on your cluster because it does not store index metadata in files. +* Shard Based Rebalance and Rebalance Based on File Transfer are synonyms for File-Based Rebalance. +==== + +For examples of Index Rebalance and File-Based Rebalance in action, see <<#fbr-all-examples,Understanding Index Rebalance with Examples>>. + +[#shard-affinity-and-fbr] +==== Relationship Between Shard Affinity and File-Based Rebalance + +xref:learn:clusters-and-availability/rebalance-and-index-service.adoc#shard-affinity[Shard Affinity] is related to File-Based Rebalance (or Shard Based Rebalance) in the following ways: + +* As partitions with the same shard ID reside in the same shard file, a rebalance can relocate those partitions by copying that file to another node. + +* Without shard affinity, partitions of the same logical shard may be spread across multiple files or nodes. +This distribution makes File-Based Rebalance inefficient forcing it to use the slower DCP-based rebuild method. +For this reason, a single configuration setting enables both shard affinity and File-Based Rebalance. + +[#enabling-fbr] +==== Enabling File-Based Rebalance + +For self-managed clusters, File-Based Rebalance is disabled by default. +You need to enable File-Based Rebalance to use it. + +* To enable File-Based Rebalance from the UI, do the following: + +.. On your cluster, in the xref:manage:manage-settings/general-settings.adoc#index-storage-mode[Index Storage Mode], enable Standard Global Secondary mode. ++ +For more information about Standard Global Secondary index storage mode, see xref:learn:services-and-indexes/indexes/storage-modes.adoc#standard-index-storage[Standard Index Storage]. + +.. Select *Enable File Transfer Based Rebalance* and save the changes. + +* To enable File-Based Rebalance using the REST API, set the `enableShardAffinity` parameter to `true` in the xref:rest-api:post-settings-indexes.adoc#disable-file-transfer-based-rebalance[GSI curl command settings]. + +NOTE: <<#smart-batching,Smart Batching>> does not work if you have enabled File-Based Rebalance on your cluster. + +[#when-fbr-takes-effect] +==== When Does File-Based Rebalance Take Effect? + +The File-Based Rebalance method uses metadata in the index's files during the rebalance process, through relocation. + +IMPORTANT: Couchbase Server can perform File-Based Rebalance for an index only when the index’s files have the required metadata. + +These are the ways to have the required metadata in the index files and to understand when does File-Based Rebalance take effect: + +* *File-Based Rebalance Enabled Before Index Creation*: +The following steps explain what happens when you enable File-Based Rebalance before creating any indexes: + +.. <<#enabling-fbr,Enable File-Based Rebalance>> (or shard affinity) before creating any indexes. + +.. Create the indexes. + +.. Couchbase Server adds shard-affinity metadata into the index files. + +.. Because the metadata is present in the index files from the start, when a Rebalance operation is triggered, +the initial and all subsequent rebalance operations use the File-Based Rebalance method. ++ +NOTE: This is the best practice for using File-Based Rebalance. + +* *File-Based Rebalance Enabled After Index Creation*: +The following steps explain what happens when you enable File-Based Rebalance after creating indexes: + +.. Create the indexes. + +.. <<#enabling-fbr,Enable File-Based Rebalance>> (or shard affinity). + +.. When the Rebalance operation is triggered, Couchbase Server does not use File-Based Rebalance method for those indexes right away, +because those indexes do not have <<#shard-affinity,Shard Affinity>>. ++ +Instead, the rebalance rebuilds the indexes and to establish shard affinity for all existing indexes, you must rebuild them once. +A <<#full-swap-rebalance,full swap rebalance>> cycle is a method for performing this rebuild. +During this rebuild Couchbase Server adds the required metadata to the index files. ++ +In summary, the initial rebuild uses full swap rebalance, which is a DCP-based method to add metadata to the indexes. +Then all subsequent Rebalance operations for those indexes use the File-Based Rebalance method. + +For examples on the working of File-Based Rebalance, see <<#working-of-fbr-operations,Working of File-Based Rebalance in Common Operations>>. + +For examples of Index Rebalance and File-Based Rebalance in action, see <<#fbr-all-examples,Understanding Index Rebalance with Examples>>. + +==== Restarting an File-Based Rebalance + +If a File-Based Rebalance fails, you can start a new rebalance. +The subsequent rebalance does not repeat any shard transfers that completed successfully during the earlier attempt. +The subsequent rebalance only transfers the indexes that were incomplete or still in progress when the failure occurred. + +To restart File-Based Rebalance, enable it. +For more information, see <<#enabling-fbr, Enabling File-Based Rebalance>>. + +===== What happens When You Retry a Cancelled or Failed File-Based Rebalance + +When you retry a failed or cancelled File-Based Rebalance, Couchbase Server performs the following actions: + +. The previous rebalance state is discarded: ++ +When a rebalance is either cancelled or fails, Couchbase Server does not retain the partial movement plan or the ongoing transfer state. +Instead, the cluster returns to a normal operational state, but the planner does not save any details about what was already moved or scheduled. + +. A new rebalance request triggers a full re-planning cycle: ++ +When you retry the rebalance, the Index Service recalculates the placement plan from the current state of the cluster. +The planner rebuilds the entire rebalance plan instead of resuming the previous one (the one that was cancelled or failed). +The planner evaluates the cluster topology, shard affinity, node capacity, and optimization settings as if this were the first time this rebalance is being run. + +. The new plan may move indexes across any eligible nodes: ++ +Because the planner starts fresh, it may select a different set of nodes for index movement compared to the previous attempt. +Even if an index was already transferred during the earlier, incomplete rebalance, the new plan may move that index again if the planner determines that a different placement results in a more balanced or optimal outcome. + +// Remove after the 7.x series. +ifeval::['{page-component-version}' == '7.6'] +NOTE: In Couchbase Server 7.6.0 and 7.6.1, enabling File-Based Rebalance prevented you from specifying Index Service node placement in a `CREATE INDEX` statement. +In Couchbase Server 7.6.2 and later versions, you can use the `WITH ` clause in `CREATE INDEX` to control index placement. +However, the `WITH ` clause is still not supported with `ALTER INDEX` when File-Based Rebalance is enabled. +endif::[] + +[#index-redistribution] +== More about Index Redistribution + +Couchbase Server can redistribute indexes during rebalance. +Redistributing indexes can improve performance by offloading heavily loaded nodes. + +A rebalance automatically redistributes indexes in the following situations: + +* *Rebalance when you add an index node*: +Rebalance always moves indexes off of nodes that you're removing from the cluster to the remaining nodes. +A rebalance does not affect indexes that reside on nodes that you're not removing. + +* *Rebalance when you add and remove index nodes*: +A swap rebalance moves indexes from nodes you're removing and places them on the nodes you're adding. +For more information, see <<#swap-rebalance-for-index-service,Swap Rebalance for Index Service>>. + +You can enable index redistribution or index movements across any index node during a rebalance in the cluster using one of the following ways: + +* From the UI, enable Optimize Index Placement in the xref:manage:manage-settings/general-settings.adoc#index-storage-mode[Index Storage Mode] section. + +* Use the REST API `redistributeIndexes` argument. +For more information, see xref:manage:manage-settings/general-settings.adoc#index-settings-via-rest[Settings REST API]. + +NOTE: In Couchbase Server 7.2 and later versions, the redistribution setting affects both partitioned and non-partitioned indexes. + +Enabling the index redistribution setting causes a rebalance to redistribute indexes in the following situations: + +* *Rebalance after you add an index node*: +Rebalance optimizes index placement across all index nodes in the cluster, including on the new index nodes. + +* *Rebalance after you add or remove a non-index node*: +Rebalance moves indexes from heavily loaded nodes to nodes with free resources to balance distribution. + +* *Rebalance during an index server group repair*: +A group failure in a multiple server group database can force all replicas into a single group. In this case, rebalance redistributes the replicas to support high availability across server groups after the server group repair. + +NOTE: If after you drop Index Service nodes, the remaining nodes cannot handle all of the index replicas, Couchbase Server drops some of the replicas. +If you later add additional Index Service nodes to the cluster, Couchbase Server replaces the dropped replicas. + +For examples of Index Rebalance and File-Based Rebalance in action, see <<#fbr-all-examples,Understanding Index Rebalance with Examples>>. + +=== Index Rebuild Batching + +Couchbase Server can batch index rebuilds during a rebalance to optimize resource usage and minimize performance impacts. + +[#smart-batching] +==== Smart Batching + +When Couchbase Server rebalances indexes by rebuilding them, it groups the rebuilds in batches. +This batching limits the overhead of rebuilding the indexes on the cluster and limits the performance impacts. +This process is called smart batching. + +The default batch size is `3`, which means that a rebalance rebuilds up to 3 indexes at the same time. + +NOTE: Smart batching does not work if you have enabled File-Based Rebalance. + +Users with Full Admin or Cluster Admin roles can xref:rest-api:rest-modify-index-batch-size.adoc[Modify Index Batch Size] using the REST API. + +You can use smart batching for one or more of the following purposes: + +* Reduce further time and resources required to move the index metadata. +* Rebuild indexes at their new locations during rebalance. +* Increase rebalance performance by increasing pipeline parallelism by starting the next batch when the previous batch is only partially completed. +* Allow the increase of overall concurrency by using administrator-determined modifications of the batch size. +* Optimize the sharing of data streams where index relocation takes place. +* Repair index replicas lost due to node failure before relocating indexes. +* Dynamically reassign index-rebuild tasks between batches to improve workload balance. + +NOTE: If at least one node in the cluster is running Server 7.1 or a later version, most smart batching features apply across the cluster, +even when some nodes are running earlier versions. + +==== Empty Node Batching + +Empty node batching is a behavior in rebalance planner that occurs when a newly added Index Service node has no indexes on it. +Because the node is empty, there are no placement conflicts, no replica-placement constraints, and no query workload running on the node. +This allows the planner to group multiple index movements together and schedule them as a single batch, rather than planning each index transfer individually. + +By treating the empty node as a clean target, the system reduces planning overhead and speeds up the process of distributing indexes onto the new node. +This batching behavior improves the efficiency of scale-out operations, especially in clusters with multiple indexes. +As the node is not yet serving application traffic, the system can safely move larger groups of indexes at once without affecting query performance. + +[#fbr-all-examples] +== Understanding Index Rebalance with Examples + +This section provides explanations and examples of how different rebalance methods, and failover and removal operations affect the Index Service. + +* <<#swap-rebalance-for-index-service,Swap Rebalance for Index Service>> +* <<#working-of-fbr-operations,Working of File-Based Rebalance in Common Operations>> +* <<#failover-idx-nodes,Failover of Index Service Nodes>> +* <<#removal-of-idx-nodes,Removal of Index Service Nodes>> + +[#swap-rebalance-for-index-service] +=== Swap Rebalance for Index Service + +A swap rebalance removes one or more source nodes and adds the same number of destination nodes. + +The effect of a swap rebalance operation on Index Service is that it moves indexes from the nodes you're removing and places them on the nodes you're adding in a cluster. +The indexes move only between those source and destination nodes. +All other nodes continue supporting Query Service and no indexes relocate between them. + +The following examples show how Couchbase Server handles swap rebalance operations for the Index Service. + +* <<#full-swap-rebalance,Full Swap Rebalance (DCP-based Rebuild)>> +* <<#swap-rebalance-without-optimize-index-placement,Swap Rebalance without Optimize Index Placement>> + +To understand how swap rebalance works with File-Based Rebalance, see <<#swap-rebalance-fbr,Swap Rebalance (One-for-One Replacement)>>. + +[#full-swap-rebalance] +==== Full Swap Rebalance (DCP-based Rebuild) + +Perform a full swap rebalance cycle as follows: + +. Consider a cluster with 3 nodes N1, N2, and N3. ++ +image::clusters-and-availability/full-swap-stage0.png[,600,align=middle] + +. Select a source node (N1), add a new destination node (S), and perform a swap rebalance (remove N1, add S). ++ +All indexes from N1 are rebuilt on S using the DCP-based method, which adds the required shard affinity metadata. ++ +image::clusters-and-availability/full-swap-stage1.png[,600,align=middle] + +. Add N1 back as an empty node and repeat the swap rebalance with the next node (N2). ++ +image::clusters-and-availability/full-swap-stage2.png[,600,align=middle] + +. Continue this process for all nodes in the cluster until every index has been moved once. ++ +This operation rewrites the indexes with shard affinity. ++ +After the cycle completes, all indexes have shard affinity, and future index movements use File-Based Rebalance. ++ +image::clusters-and-availability/full-swap-stage3.png[,600,align=middle] + +For information about replacing only one node with another using swap rebalance, see <<#swap-rebalance-fbr,Swap Rebalance (One-for-One Replacement)>>. + +[#swap-rebalance-without-optimize-index-placement] +==== Swap Rebalance without Optimize Index Placement + +For example, in a four-node cluster (N1, N2, N3, N4) with Optimize Index Placement disabled, you plan to replace nodes N1 and N2. +Add two new empty nodes, N5 and N6, then perform a swap rebalance by marking N1 and N2 for removal and adding N5 and N6. + +image::clusters-and-availability/swap-without-optimization.png[,720,align=middle] + +. Assume the source node N1 hosts shards S1 that includes index idx_A, S7 and S10. +Assume node N2 hosts shards S2 that also includes idx_A and S9. + +. If shard affinity is enabled, entire shards move between nodes. +During the swap rebalance, the planner initiates index movements for all these shards to the new destination nodes N5 and N6. +For example, S1 and S10 may move from N1 to N6, while S7 moves from N1 to N5. + +. If shard affinity is not enabled, index movements aren’t restricted by shard grouping. + +. There’s no one-to-one mapping between source and destination nodes. +Indexes or shards can move from any of the source nodes to any of the destination nodes involved in the rebalance. ++ +Nodes N3 and N4 aren’t involved in this index movement. + +[#working-of-fbr-operations] +=== Working of File-Based Rebalance in Common Operations + +This section explains the Working of File-Based Rebalance during the following common operations. + +* <<#adding-capacity-scale-out,Adding Capacity (Scale-Out)>> +* <<#planned-node-removal-rebalance-out,Planned Node Removal (Rebalance-Out)>> +* <<#swap-rebalance-fbr,Swap Rebalance (One-for-One Replacement)>> + +[#adding-capacity-scale-out] +==== Adding Capacity (Scale-Out) + +The following procedure shows how File-Based Rebalance works when adding index capacity: + +. When a new index node is added to the cluster and a rebalance is triggered, the system may move indexes to the new node to balance the load. + +. Source nodes remain online and continue supporting Query Service while the index movement occurs in the background. + +. The index files are copied to the destination node, which then catches up on any data changes that occurred during the transfer. + +. After synchronization, the index on the destination node becomes active, and the original index on the source node is removed. + +An example of adding index capacity is as follows: + +image::clusters-and-availability/fbr-scale-out.png[,500,align=middle] + +. A cluster has 2 index nodes, I1 and I2. ++ +Add a new destination node, I3, and trigger a rebalance. + +. During the rebalance, an index such as idx_A on source node I1 is moved to node I3 by copying its shard file. +Other indexes that share the same shard are also moved to node I3 as a part of the same operation. + +. While the copy is in progress, I1 continues to support Query Service for idx_A. +After I3 synchronizes and catches up with recent changes, it takes over query processing for idx_A, and the original index on I1 is removed. + +[#planned-node-removal-rebalance-out] +==== Planned Node Removal (Rebalance-Out) + +The following procedure shows how File-Based Rebalance works when removing a node: + +. When a node is marked for removal and a rebalance is triggered, the rebalance process moves all indexes from the source node to other nodes in the cluster. +Entire shards are transferred as a part of this operation. + +. The process does not interrupt service. +Query Service remains available throughout. + +. After synchronization completes and the destination nodes begin serving the indexes, the source node can be safely removed from the cluster. + +An example of planned node removal is as follows: + +image::clusters-and-availability/fbr-rebalance-out.png[,500,align=middle] + +. A cluster has 3 index nodes, I1, I2, and I3. ++ +You plan to decommission I3. + +. During rebalance, indexes on the source node I3 are moved to the destination nodes I1 and I2. +For example, idx_B on I3 is moved to I1, and idx_C on I3 is moved to I2, along with all indexes that share the same shards. + +. After all index movements are complete, I3 is safely removed from the cluster. + +[#swap-rebalance-fbr] +==== Swap Rebalance (One-for-One Replacement) + +The following procedure shows how File-Based Rebalance works when one node is replaced by another: + +. An existing node O is replaced by a new node R. + +. During rebalance, all indexes on the source node O are moved to the destination node R by copying their shard files. ++ +No other index movements occur when Optimize Index Placement is disabled. + +. The source node O continues to support Query Service during the transfer. + +. After the File-Based Rebalance completes and the destination node R catches up with new mutations, +it takes over serving the indexes, and the source node O is removed from the cluster. + +An example of swap rebalance is as follows: + +image::clusters-and-availability/fbr-swap-one-on-one.png[,540,align=middle] + +. In a cluster having nodes I1, I2, and I3, you plan to replace the source node I1 with a new destination node N1. + +. If I1 hosts indexes idx_A and idx_B, they're moved to N1 using File-Based Rebalance. +During the transfer, I1 continues to support Query Service. + +. After the rebalance completes, N1 becomes fully active, and I1 is removed from the cluster with no downtime. + +[#failover-idx-nodes] +=== Failover of Index Service Nodes + +The following is how node failover affects the Index Service: + +. If a node becomes unhealthy, ns_server can remove it from the cluster through autofailover, or you can manually fail it over. + +. Indexes on the failed node are unavailable when replicas do not exist on other nodes. + +. The Query Service automatically redirects requests to available replicas. + +. If a rebalance is triggered while the node remains failed over, the system performs a replica repair. +When a replica exists, it can be copied using File-Based Rebalance if shard affinity is enabled. +If no replica exists, the index is lost. + +. If the failed node recovers before a rebalance occurs, it can be added back to the cluster, and the next rebalance restores its indexes. +Any duplicate replicas created during the failover are removed during this add-back rebalance. + +The following examples explain how Couchbase Server handles node failover scenarios, both when replicas are available and when they're not. + +* <<#failover-with-replica,Failover With a Replica>> +* <<#failover-without-replica,Failover With No Replica (Index Loss)>> + +[#failover-with-replica] +==== Failover With a Replica + +This example shows how Couchbase Server maintains data availability when a node hosting an index fails and a replica is available: + +image::clusters-and-availability/failover-with-replica.png[,570,align=middle] + +. Node N1 hosts idx_A-r0, the primary replica of idx_A. ++ +Node N2 hosts idx_A-r1, a secondary replica of idx_A. ++ +When node N1 crashes, an administrator triggers a failover. + +. idx_A-r0 is marked as lost. ++ +Query Services are automatically redirected to idx_A-r1 on node N2. + +. Afterwards, a rebalance is triggered. ++ +The system creates a new replica, idx_A-r0, on another node (for example, N3) to restore the required replica count. +This process is known as replica repair. + +NOTE: If the Index Service is under memory pressure, for example, due to inadequate sizing, it may skip replica repair during rebalance. +Replica repair also does not occur if the feature is disabled in the settings or if there are not enough available nodes. + +[#failover-without-replica] +==== Failover With No Replica (Index Loss) + +This example shows what happens when a node hosting an index fails and no replica exists to maintain data availability: + +image::clusters-and-availability/failover-without-replica.png[,570,align=middle] + +. Index idx_B exists only on node N1 (replica count = 0). ++ +Node N1 fails over. +When this happens, the system removes N1 from the active cluster topology. ++ +Because idx_B has no replicas on other nodes, it becomes immediately unavailable. ++ +Any Query Services that depend on idx_B return an error, because no valid copy of the index exists in the cluster. + +. If a rebalance runs, the system marks idx_B as lost and removes its definition from the cluster metadata. + +[#removal-of-idx-nodes] +=== Removal of Index Service Nodes + +The following is how node removal affects the Index Service: + +. A planned, graceful operation that decommissions a healthy node. ++ +The node is marked for removal, and a rebalance is triggered. + +. Before the source node goes offline, all its indexes move to other destination nodes in the cluster by using File-Based Rebalance, if shard affinity is enabled. + +. The node is removed only after all index movements are complete. +This process ensures that ongoing scans on the source node complete before removal, preventing any data loss or service interruption. + +The following examples explain how node removal and rebalance operations affect the Index Service, both when replicas are present and when they're not. + +* <<#removal-with-replica,Removal and Rebalance-Out With Replicas>> +* <<#removal-without-replica,Removal and Rebalance-Out Without Replicas>> + +[#removal-with-replica] +==== Removal and Rebalance-Out With Replicas + +The following example explains removing an Index Service node from a cluster and how the system maintains availability throughout the process. + +image::clusters-and-availability/removal-with-replica.png[,500,align=middle] + +. Node N3 hosts the replica idx_C-r1. +The node is healthy but needs to be decommissioned. ++ +An administrator marks N3 for removal and starts a rebalance. ++ +NOTE: Selecting Remove only marks the node for removal. +You must start a rebalance for the removal to take effect. + +. The system begins moving idx_C-r1 to a destination node, for example, N1. + +. During the transfer, N3 continues to support Query Service for idx_C-r1 to maintain availability. + +. After the transfer completes, N3 is safely removed from the cluster with no service interruption. + +[#removal-without-replica] +==== Removal and Rebalance-Out Without Replicas + +The following example describes the process of removing an Index Service node when the index does not have replicas: + +image::clusters-and-availability/removal-without-replica.png[,500,align=middle] + +. Index idx_D exists only on node N3. ++ +An administrator marks N3 for removal and starts a rebalance. + +. The system begins copying idx_D from the source node N3 to a destination node, for example, N1. + +. Node N3 leaves the cluster only after the copy completes and idx_D is fully available on N1. ++ +Query Service that uses idx_D remain online during the entire operation. + +== Operational Comparisons + +The following sections provide concise, side-by-side comparisons to clarify operational behaviors and choices for Index Service during rebalance, failover, and upgrades. + +[#failover-vs-removal] +=== Failover vs Removal (Rebalance-Out) + +The following table summarizes the key differences between failover and planned removal (rebalance-out) of Index Service nodes: + +[cols="3,3,4", options="header"] +|=== +| Aspect| Failover | Removal (Rebalance-Out) + +| *Immediate availability* +| Potential for index loss +| No loss of service + +| *Data Transfer for File-Based Rebalance* +| No transfer from failed node but relies on replicas +| Graceful copy from source node being removed to destination nodes + +| *Node State* +| Node can be potentially recovered or added back +| Node is permanently decommissioned +|=== + +[#upgrade-idx-service-swap-vs-fbr] +=== Upgrade Index Service Using - Swap Rebalance vs Failover-Based Rebalance + +This section compares 2 approaches for upgrading Index Service nodes in a Couchbase cluster - swap rebalance and failover-based rebalance. + +[cols="1,1,1", options="header"] +|=== +| Aspect | Swap Rebalance | Failover-Based Rebalance + +| *Purpose* +| Replaces old nodes with new ones during a planned upgrade or maintenance. +| Used during an upgrade by failing over a node, adding a new node, and performing a rebalance. + +| *Index Movement* +| Indexes move only between the specified old and new nodes. +Other nodes in the cluster are not involved. +| Indexes are rebuilt or repaired after failover as part of the rebalance process. + +| *Availability* +| Indexes remain available throughout the rebalance operation. +| One replica is lost during failover, and the workload shifts to remaining replicas. + +| *Impact on Replicas* +| Replica distribution remains intact. +| If a node with a single replica fails, the index is lost. + +| *Operation Type* +| Planned and controlled process. +| Reactive process following a node failure or manual failover. +|=== + +[fbr-in-mixed-version-clusters] +=== File-Based Rebalance in Mixed-Version Clusters + +File-Based Rebalance behavior depends on the Couchbase Server version running on the nodes in a mixed-version cluster as follows. + +[cols="1,1,1", options="header"] +|=== +| Source Node Version | Destination Node Version | Index Rebalance Method + +| 7.6 or later +| 7.6 or later +| File-Based Rebalance + +(if Shard Affinity exists) + +| 7.6 or earlier +| Any +| DCP-based rebuild + +| Any +| 7.6 or earlier +| DCP-based rebuild + +|=== + +== Additional Information + +The following sections have additional information about Index Rebalance. + +=== When Index Movement Appears Higher than Anticipated + +The number of indexes that move during a rebalance may appear higher than you expect. +The reasons can be one or more of the following: + +* Rebalance was cancelled or failed, then retried: ++ +A retry triggers a full re-planning cycle and not a continuation. +The new plan may choose different nodes, causing more indexes to move than in the previous attempt. + +* Optimize Index Placement is enabled: ++ +With optimization on, the planner can redistribute indexes across any eligible nodes, instead of just the nodes you assumed were involved. + +* Empty node batching is not used: ++ +If the destination node is not empty or the planner cannot treat it as empty, then movements cannot be grouped efficiently. +The planner schedules smaller, scattered movements, creating the impression that more indexes are moving. + +* Multiple indexes share the same shard (shard affinity): ++ +Indexes that share the same Alternate Shard ID must move together as a single shard, so moving one index results in several indexes moving. + +* Swap rebalance interacting with shard groups: ++ +Even if you expect only the swap node pair to participate, shared shards may cause all indexes in those shards to move. + +* Mixed-version cluster, such as 7.6 and earlier: ++ +File-Based Rebalance is only possible between 7.6 and later version nodes. +Movements involving older nodes fall back to DCP-based rebuild, leading to unexpected index rebuild or movement. + +* Failover with replicas present: ++ +The system may select a replica on a different node than expected based on coverage, partitions, and shard size. +This can cause additional shard movement during repair. + +* Failover for indexes without replicas: ++ +Non-replicated indexes must be rebuilt via DCP, so you will see extra rebuild work even if you expected minimal movement. + +* Large-scale cluster topology changes: ++ +Adding or removing nodes, in particular when multiple nodes are affected, expands the planner’s options. +This may cause broader redistribution than you anticipated. + +=== Verifying Shard Affinity Metadata + +To verify that an index has shard-affinity metadata, use the Index Service status REST API endpoint `/getIndexStatus`. + +curl syntax is as follows: + +---- +curl -X GET http://:9102/getIndexStatus?getAll=true / +-u : +---- + +In the response, look for the `alternateShardIds` field for each index. +The outcome is one of the following: + +* If `alternateShardIds` contains entries for all partitions, the index has shard affinity. +* If `alternateShardIds` is empty or missing, the index does not have shard affinity. + +=== Adding Shard Affinity Metadata Without Full Swap Rebalance Cycle + +A swap-rebalance cycle is the most efficient and recommended method to add Alternate Shard IDs to all indexes across the cluster. + +The alternatives are as follows: + +[cols="1,3", options="header"] +|=== +| Method | Description + +| Swap Rebalance Cycle +| Recommended approach. + +Moves each index exactly once while adding shard affinity metadata. + +| Drop and Recreate +| Create indexes with shard affinity enabled to embed metadata. + +Causes downtime for affected indexes. + +| ALTER INDEX + +(Add Replica) + +| Running `ALTER INDEX` with the `replica_count` action adds shard-affinity metadata when enabled. + +This is applicable only to the newly generated replicas. + +|=== + +NOTE: Any index creation or movement by the rebalance planner while shard affinity is enabled adds the Alternate Shard ID. +Using `ALTER INDEX` with action `replica_count`, which is applicable only to moving replica that are new or moved, also applies the metadata. +A rebalance is not required. + +=== Scenarios Where File-Based Rebalance may be Skipped + +File-Based Rebalance may be skipped and fall back to DCP-based rebuild in the following scenarios. + +[cols="1,1", options="header"] +|=== +| Condition | Why File-Based Rebalance is Skipped + +| Index lacks Alternate Shard IDs +| Existing indexes without shard affinity metadata cannot use File-Based Rebalance. + +| Source or destination node runs a version earlier than 7.6 +| Both nodes must run Couchbase Server 7.6 or a later version for File-Based Rebalance. + +| Shard version incompatibility +| Destination node’s shard format version is older than the source. + +| Non-Plasma storage mode +| File-Based Rebalance is supported only with the Plasma storage engine (Standard GSI). + +| Replica repair with no surviving replica +| When no replica exist to copy, the system rebuilds via DCP-based method. + +|=== + +=== Moving Indexes Within a Shared Shard + +When a shard contains multiple indexes, those indexes cannot be moved independently without breaking shard affinity. +With shard affinity enabled, all indexes that share the same Alternate Shard ID (Slot ID) are treated as a single unit, and the shard is moved in its entirety. +This behavior is intentional and enables efficient file-based transfer during rebalance. + +CAUTION: Breaking shard affinity to move one index from its group is technically possible but not recommended. + +=== Replica Selection During Failover Repair + +When multiple replicas are available during failover, Couchbase Server selects the replica that minimizes repair time. +The system evaluates replicas in the following priority order: + +. *Largest index size*: Prioritizes the index in the shard with largest disk size, indicating that it's the most up-to-date. + +. *Highest coverage for required indexes*: Prioritizes the replica that already has the largest amount of data built for the indexes undergoing repair. + +. *Most matching partition instances*: If coverage is similar, selects the replica with more matching partitions. + +. *Smallest total shard size*: If all other factors are equal, prefers the shard with a smaller total disk size to reduce file copy time. + +This optimization makes sure that repair completes as efficiently as possible by choosing the replica that requires the least amount of work to bring online. + +=== Recovery Behavior for Indexes Without Replicas + +For indexes without replicas, there is no automated recovery mechanism after a failover other than a DCP-based rebuild. +When a node hosting an index without replicas fails over: + +. The index becomes unavailable immediately. + +. Queries that depend on that index fail. + +. If a rebalance is triggered without adding the failed node back to the cluster, Couchbase Server rebuilds the index using the DCP-based rebuild method. \ No newline at end of file diff --git a/modules/learn/pages/clusters-and-availability/rebalance.adoc b/modules/learn/pages/clusters-and-availability/rebalance.adoc index 55894d78ea..40fd0296ad 100644 --- a/modules/learn/pages/clusters-and-availability/rebalance.adoc +++ b/modules/learn/pages/clusters-and-availability/rebalance.adoc @@ -136,97 +136,20 @@ A complete account of the report-content is provided in the xref:rebalance-refer == Rebalance and Other Services Rebalance affects different services differently. -The effects on services other than the Data Service are described below. +The following sections explain the rebalance operation effects on non-Data Services. [#rebalancing-the-index-service] -=== Index Service +== Index Service The Index Service maintains a cluster-wide set of index definitions and metadata, which allows the redistribution of indexes and index replicas during a rebalance. -The rebalance process takes into account the nodes' CPU, RAM, and disk bandwidth to limit its effect on database performance. +The rebalance process considers the node's CPU, RAM, and disk bandwidth to limit its effect on database performance. -[#index-rebalance-methods] -==== Index Rebalance Methods +Couchbase Server either rebuilds indexes (using DCP-based method) in their new locations or +moves index files between nodes (#file-based-rebalance-fbr[File-Based Rebalance]) during a rebalance, +depending on the cluster's index storage mode and on configuration settings. -By default, Couchbase Server rebuilds the indexes in their new locations during a rebalance. -If your cluster uses xref:learn:services-and-indexes/indexes/storage-modes.adoc#standard-index-storage[Standard Index Storage], you can choose to enable file-based rebalance which moves index files between nodes instead of rebuilding them. -Copying the index files is faster than having the target node rebuild the index from scratch. -You cannot use file-based rebalance if you have enabled xref:learn:services-and-indexes/indexes/storage-modes.adoc#memory-optimized-index-storage[Memory Optimized Index Storage] because this option does not store index metadata in files. - -Couchbase Capella defaults to file-based index rebalance. - -To learn how to enable file-based rebalance via the Couchbase Server Web Console, see File Transfer Based Rebalance under xref:manage:manage-settings/general-settings.adoc#index-storage-mode[Index Storage Mode]. To learn how to enable it via the REST API, see the `enableShardAffinity` parameter in xref:rest-api:post-settings-indexes.adoc[]. - -// Remove after the 7.x series. -ifeval::['{page-component-version}' == '7.6'] -NOTE: In Couchbase Server versions 7.6.0 and 7.6.1, when you enabled file-based rebalance you could not choose which Index Service nodes would contain an index when using the `CREATE INDEX` statement. -In Couchbase Server 7.6.2 and later, you can use the `WITH ` clause to set which node contains the index. -You still cannot use the `WITH ` clause with `ALTER INDEX` after you enable file-based rebalance. -endif::[] - -===== When File-Based Rebalance Take Effect - -The file-based index rebalance method uses metadata in the index's files during the relocation process. -When Couchbase Server starts using file-based index rebalance for an index depends on whether the index's files contains this metadata: - -* If you enable file-based index rebalance before you create an index, Couchbase Server adds the metadata to the index files while creating them. -Because this metadata is in the index's files from the start, the initial rebalance of the index as well as all later rebalances use the file-based rebalance method. -If you choose to use file-based index rebalance, your best practice is to enable it before you create any indexes. - -* If you enable file-based index rebalance after you have created an index, Couchbase Server does not use the file-based rebalance method the next time it rebalances that index. -Instead, the next rebalance process still rebuilds the index. -During this rebuild, Couchbase Server adds the metadata to the index files to enable file-based rebalance. -After this initial rebalance, all later rebalances of the index use the file-based rebalance method. - -===== Restarting a Rebalance - -If a file-based rebalance fails, you can try starting a new rebalance. -The second rebalance does not re-transfer any index that the failed rebalance finished moving. -Instead, it moves just the indexes that had not moved or were in the process of moving when the error occurred. - -You enable file-based rebalance using settings located under the under the xref:manage:manage-settings/general-settings.adoc#index-storage-mode[Index Storage Mode] on the Couchbase Server Web Console's Settings page. -You can also enable it using the REST API's xref:manage:manage-settings/general-settings.adoc#index-settings-via-rest[`settings/indexes` endpoint]. - -[#index-redistribution] -==== Index Redistribution - -Couchbase Server can redistribute indexes during rebalance. Redistributing indexes can improve performance by offloading heavily loaded nodes. A rebalance automatically redistributes indexes in the following situations: - -Rebalance when you add an index node:: -Rebalance always moves indexes off of nodes that you're removing from the cluster to the remaining nodes. -A rebalance does not affect indexes that reside on nodes that you're not removing. - -Rebalance when you add and remove index nodes:: -A swap rebalance moves indexes from nodes you're removing and places them on the nodes you're adding. - -You can change a setting to have a rebalance redistribute indexes in more cases. This setting is named **Optimize Index Placement On Rebalance** on the Couchbase Server Web Console's xref:manage:manage-settings/general-settings.adoc#index-storage-mode[Settings page]. -You can also change this setting using the xref:manage:manage-settings/general-settings.adoc#index-settings-via-rest[Settings REST API] `redistributeIndexes` argument. - -NOTE: In Couchbase Server 7.2 and later, the redistribution setting affects both partitioned and non-partitioned indexes. - -Enabling the index redistribution setting causes a rebalance to redistribute indexes in the following situations: - -Rebalance after you add an index node:: -Rebalance optimizes index placement across _all_ index nodes in the cluster, including on the new index nodes. - -Rebalance after you add or remove a non-index node:: -Rebalance moves indexes from heavily loaded nodes to nodes with free resources to balance distribution. - -Rebalance during an index server group repair:: -A group failure in a multiple server group database can force all replicas into a single group. In this case, rebalance redistributes the replicas to support high availability across server groups after the server group repair. - -NOTE: If after you drop Index Service nodes, the remaining nodes cannot handle all of the index replicas, Couchbase Server drops some of the replicas. If you later add additional Index Service nodes to the cluster, Couchbase Server replaces the dropped replicas. - -[#index-rebalance-batch-size] -==== Index Rebuild Batch Size - -When Couchbase Server rebalances indexes by rebuilding them, it groups the rebuilds in _batches_. -This batching limits the overhead of rebuilding the indexes on the cluster. -The default batch size is `3`, which means that a rebalance rebuilds up to three indexes at the same time. -This setting has no effect if you have enabled file-based index rebalancing. - -Users with Full Admin or Cluster Admin roles can change the batch size using the REST API. -See xref:rest-api:rest-modify-index-batch-size.adoc[Modify Index Batch Size]. +For more information about the rebalance operation on Index Service, see xref:learn:clusters-and-availability/rebalance-and-index-service.adoc[Index Rebalance]. [#rebalancing-the-search-service] === Search Service diff --git a/modules/manage/pages/manage-settings/general-settings.adoc b/modules/manage/pages/manage-settings/general-settings.adoc index 0c534e5a32..72f661d520 100644 --- a/modules/manage/pages/manage-settings/general-settings.adoc +++ b/modules/manage/pages/manage-settings/general-settings.adoc @@ -291,7 +291,7 @@ See xref:learn:services-and-indexes/indexes/index-replication.adoc#index-replica * *Indexer Rebalance Settings*: When cleared, *Optimize Index Placement On Rebalance* has Couchbase Server only redistribute indexes during a rebalance when the nodes containing them are leaving the cluster. When selected, Couchbase Server redistribute indexes among the Index Service nodes during a rebalance to optimize performance. -See xref:learn:clusters-and-availability/rebalance.adoc#index-redistribution[Index Redistribution]. +See xref:learn:clusters-and-availability/rebalance-and-index-service.adoc#index-redistribution[]. * *Indexer Log Level*: Sets the logging level. @@ -313,11 +313,14 @@ During an upgrade, mixed mode clusters with nodes that support bloom filters wil [#file-based-index-rebalance] * *File Transfer Based Rebalance*: -Controls whether Couchbase Server rebuilds indexes or copies them between nodes. +Controls whether Couchbase Server rebuilds indexes or copies them between nodes. +Use the **Enable File Transfer Based Rebalance** checkbox to enable or disable the File Transfer Based Rebalance from the UI. The default cleared setting has an Index Server node rebuild any newly assigned index during a rebalance. You cannot enable file-based rebalance when you have enabled Memory Optimized Index Storage. When you select this option, Couchbase Server copies the index files from one Index Server node to another during a rebalance instead of rebuilding them. -See xref:learn:clusters-and-availability/rebalance.adoc#index-rebalance-methods[Index Rebalance Methods]. +See xref:learn:clusters-and-availability/rebalance-and-index-service.adoc#index-rebalance-methods[]. ++ +NOTE: Shard Based Rebalance and Rebalance Based on File Transfer are synonyms for File-based Rebalance. + You can disable this feature from the UI or via REST API. To learn about disabling this feature via REST API, see xref:rest-api:post-settings-indexes.adoc#disable-file-transfer-based-rebalance[Curl Command to Disable the File Transfer Based Rebalance]. This feature is disabled by default. + diff --git a/modules/rest-api/pages/post-settings-indexes.adoc b/modules/rest-api/pages/post-settings-indexes.adoc index bb6e2baf85..36001358dc 100644 --- a/modules/rest-api/pages/post-settings-indexes.adoc +++ b/modules/rest-api/pages/post-settings-indexes.adoc @@ -4,7 +4,7 @@ // Cross-references :index-replication: xref:learn:services-and-indexes/indexes/index-replication.adoc#index-replication -:rebalancing-the-index-service: xref:learn:clusters-and-availability/rebalance.adoc#rebalancing-the-index-service +:rebalancing-the-index-service: xref:learn:clusters-and-availability/rebalance-and-index-service.adoc [abstract] {description} @@ -35,7 +35,7 @@ The parameters are specified as key-value pairs (e.g `key=value`). | Name | Description | Type | `enableShardAffinity` -a| Selects the xref:learn:clusters-and-availability/rebalance.adoc#index-rebalance-methods[index rebalance method]: +a| Selects the xref:learn:clusters-and-availability/rebalance-and-index-service.adoc#index-rebalance-methods[]: * `false` (default value): Index Service nodes rebuild indexes that are newly assigned to them during a rebalance. * `true`: Couchbase Server moves a reassigned index's files between Index Service nodes. @@ -161,6 +161,8 @@ include::example$post-settings-indexes.jsonc[] The following command disables the File Transfer Based Rebalance (`enableShardAffinity`) feature in the xref:manage:manage-settings/general-settings.adoc#index-storage-mode[Index Storage Mode]. +NOTE: Shard Based Rebalance and Rebalance Based on File Transfer are synonyms for File-based Rebalance. + [source#example-curl,bash] ---- include::example$post-settings-indexes.sh[tag=disable-ftb-rebalance] diff --git a/modules/rest-api/pages/rest-modify-index-batch-size.adoc b/modules/rest-api/pages/rest-modify-index-batch-size.adoc index 2f39e7943b..cebb41c518 100644 --- a/modules/rest-api/pages/rest-modify-index-batch-size.adoc +++ b/modules/rest-api/pages/rest-modify-index-batch-size.adoc @@ -5,6 +5,8 @@ {description} +NOTE: Shard Based Rebalance and Rebalance Based on File Transfer are synonyms for File-based Rebalance. + == HTTP Method and URI ---- @@ -54,5 +56,5 @@ If successful, the call returns `200 OK` and no object. == See Also -An overview of rebalance as it affects the Index Service, including an overview of _smart batching_, is provided in xref:learn:clusters-and-availability/rebalance.adoc#rebalancing-the-index-service[Index Service]. +An overview of rebalance as it affects the Index Service, including an overview of _smart batching_, is provided in xref:learn:clusters-and-availability/rebalance-and-index-service.adoc[]. For information on Couchbase-Server ports, see xref:install:install-ports.adoc[Couchbase Server Ports]. diff --git a/preview/DOC-13173-preview.yml b/preview/DOC-13173-preview.yml new file mode 100644 index 0000000000..0e1be9b7dc --- /dev/null +++ b/preview/DOC-13173-preview.yml @@ -0,0 +1,28 @@ +sources: + docs-server: + branches: DOC_13173_Rebalance_doc_enhancements + docs-analytics: + branches: release/7.6 + docs-devex: + url: https://github.com/couchbaselabs/docs-devex.git + branches: master + startPaths: docs/ + couchbase-cli: + # url: ../../docs-includes/couchbase-cli + url: https://github.com/couchbaselabs/couchbase-cli-doc + # branches: HEAD + branches: master + startPaths: docs/ + backup: + # url: ../../docs-includes/backup + url: https://github.com/couchbaselabs/backup-docs.git + #branches: HEAD + branches: master + startPaths: docs/ + #analytics: + # url: ../../docs-includes/docs-analytics + # branches: HEAD + #cb-swagger: + # url: https://github.com/couchbaselabs/cb-swagger + # branches: release/7.6 + # start_path: docs \ No newline at end of file