Implement online index backfill

CHANGELOG.md

@@ -29,6 +29,8 @@
 
 ### Added
 
+- [#7408](https://github.com/ChainSafe/forest/pull/7408): Impl `Forest.IndexBackfill`, `Forest.IndexBackfillStatus` and `Forest.IndexBackfillCancel` RPC methods and the `forest-cli index backfill` command back-fill the chain index (Ethereum mappings, events, block blooms) through the running daemon, so the node no longer needs to be stopped.
+
 ### Changed
 
 ### Removed

docs/docs/users/reference/cli.sh

@@ -73,6 +73,11 @@ generate_markdown_section "forest-cli" "config"
 generate_markdown_section "forest-cli" "snapshot"
 generate_markdown_section "forest-cli" "snapshot export"
 
+generate_markdown_section "forest-cli" "index"
+generate_markdown_section "forest-cli" "index backfill"
+generate_markdown_section "forest-cli" "index backfill-status"
+generate_markdown_section "forest-cli" "index backfill-cancel"
+
 generate_markdown_section "forest-cli" "send"
 generate_markdown_section "forest-cli" "info"
 generate_markdown_section "forest-cli" "shutdown"

docs/openrpc-specs/v0.json

@@ -6024,6 +6024,53 @@
       },
       "paramStructure": "by-position"
     },
+    {
+      "name": "Forest.IndexBackfill",
+      "description": "Starts a chain index backfill (Ethereum mappings, events, block blooms) over an epoch range using the running node, returning immediately. Poll `Forest.IndexBackfillStatus` for progress. Only one backfill may run at a time, and it never overlaps snapshot export or the snapshot GC.",
+      "params": [
+        {
+          "name": "params",
+          "required": true,
+          "schema": {
+            "$ref": "#/components/schemas/IndexBackfillParams"
+          }
+        }
+      ],
+      "result": {
+        "name": "Forest.IndexBackfill.Result",
+        "required": true,
+        "schema": {
+          "type": "null"
+        }
+      },
+      "paramStructure": "by-position"
+    },
+    {
+      "name": "Forest.IndexBackfillCancel",
+      "description": "Cancels the in-progress index backfill, returning whether one was running.",
+      "params": [],
+      "result": {
+        "name": "Forest.IndexBackfillCancel.Result",
+        "required": true,
+        "schema": {
+          "type": "boolean"
+        }
+      },
+      "paramStructure": "by-position"
+    },
+    {
+      "name": "Forest.IndexBackfillStatus",
+      "description": "Returns the progress and status of the in-progress (or last) index backfill.",
+      "params": [],
+      "result": {
+        "name": "Forest.IndexBackfillStatus.Result",
+        "required": true,
+        "schema": {
+          "$ref": "#/components/schemas/ApiIndexBackfillStatus"
+        }
+      },
+      "paramStructure": "by-position"
+    },
     {
       "name": "Forest.NetChainExchange",
       "description": "Internal API for debugging chain exchange.",
@@ -7878,6 +7925,63 @@
           "ValidityTerm"
         ]
       },
+      "ApiIndexBackfillStatus": {
+        "description": "Progress and status of the in-daemon index backfill, returned by [`IndexBackfillStatus`].",
+        "type": "object",
+        "properties": {
+          "currentEpoch": {
+            "type": "integer",
+            "format": "int64"
+          },
+          "error": {
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "indexed": {
+            "type": "integer",
+            "format": "uint64",
+            "minimum": 0
+          },
+          "progress": {
+            "type": "number",
+            "format": "double"
+          },
+          "skipped": {
+            "type": "integer",
+            "format": "uint64",
+            "minimum": 0
+          },
+          "startEpoch": {
+            "type": "integer",
+            "format": "int64"
+          },
+          "startTime": {
+            "type": [
+              "string",
+              "null"
+            ],
+            "format": "date-time"
+          },
+          "state": {
+            "$ref": "#/components/schemas/ChainExportState"
+          },
+          "targetEpoch": {
+            "type": "integer",
+            "format": "int64"
+          }
+        },
+        "required": [
+          "state",
+          "progress",
+          "startEpoch",
+          "currentEpoch",
+          "targetEpoch",
+          "indexed",
+          "skipped"
+        ]
+      },
       "ApiInvocResult": {
         "type": "object",
         "properties": {
@@ -8475,6 +8579,11 @@
             "description": "A lite snapshot export performed by the automatic snapshot GC.",
             "type": "string",
             "const": "SnapshotGc"
+          },
+          {
+            "description": "An index backfill requested via `Forest.IndexBackfill`. It holds the same single-flight\nslot as exports and the snapshot GC so that these heavy DB operations never overlap.",
+            "type": "string",
+            "const": "IndexBackfill"
           }
         ]
       },
@@ -10542,6 +10651,47 @@
           "RebroadcastBackoffMax"
         ]
       },
+      "IndexBackfillParams": {
+        "description": "Parameters for [`IndexBackfill`].",
+        "type": "object",
+        "properties": {
+          "allowNearHead": {
+            "description": "Allow indexing revert-prone tipsets within `CHAIN_FINALITY` of the head.",
+            "type": "boolean",
+            "default": false
+          },
+          "from": {
+            "description": "Starting epoch, inclusive. Defaults to the persisted resume checkpoint if present,\notherwise the chain head.",
+            "type": [
+              "integer",
+              "null"
+            ],
+            "format": "int64"
+          },
+          "nTipsets": {
+            "description": "Number of tipsets to backfill. Mutually exclusive with `to`.",
+            "type": [
+              "integer",
+              "null"
+            ],
+            "format": "uint64",
+            "minimum": 0
+          },
+          "recompute": {
+            "description": "Recompute missing tipset state (expensive) instead of skipping it.",
+            "type": "boolean",
+            "default": false
+          },
+          "to": {
+            "description": "Ending epoch, inclusive. Mutually exclusive with `n_tipsets`.",
+            "type": [
+              "integer",
+              "null"
+            ],
+            "format": "int64"
+          }
+        }
+      },
       "KeyInfo": {
         "type": "object",
         "properties": {