Add command reference doc for ledgerutil

Add command reference doc for ledgerutil.
Includes minor edits to the command's user messages.

Signed-off-by: David Enyeart <enyeart@us.ibm.com>

cmd/ledgerutil/main.go

@@ -17,23 +17,21 @@ import (
 const (
 	compareErrorMessage = "Ledger Compare Error: "
 	outputDirDesc       = "Snapshot comparison json results output directory. Default is the current directory."
-	firstDiffsDesc      = "Number of differences to record in " + ledgerutil.FirstDiffsByHeight +
-		". Generating a file of more than the first 10 differences will result in a large amount " +
-		"of memory usage and is not recommended. Defaults to 10. If set to 0, will not produce " +
+	firstDiffsDesc      = "Maximum number of differences to record in " + ledgerutil.FirstDiffsByHeight +
+		". Requesting a report with many differences may result in a large amount " +
+		"of memory usage. Defaults to 10. If set to 0, will not produce " +
 		ledgerutil.FirstDiffsByHeight + "."
 )
 
 var (
 	app = kingpin.New("ledgerutil", "Ledger Utility Tool")
 
-	compare       = app.Command("compare", "Compare two ledgers via their snapshots.")
+	compare       = app.Command("compare", "Compare channel snapshots from two different peers.")
 	snapshotPath1 = compare.Arg("snapshotPath1", "First ledger snapshot directory.").Required().String()
 	snapshotPath2 = compare.Arg("snapshotPath2", "Second ledger snapshot directory.").Required().String()
 	outputDir     = compare.Flag("outputDir", outputDirDesc).Short('o').String()
 	firstDiffs    = compare.Flag("firstDiffs", firstDiffsDesc).Short('f').Default("10").Int()
 
-	troubleshoot = app.Command("troubleshoot", "Identify potentially divergent transactions.")
-
 	args = os.Args[1:]
 )
 
@@ -48,7 +46,6 @@ func main() {
 
 	// Command logic
 	switch command {
-
 	case compare.FullCommand():
 
 		// Determine result json file location
@@ -72,10 +69,5 @@ func main() {
 		} else {
 			fmt.Printf("Results saved to %s. Total differences found: %d\n", outputDirPath, count)
 		}
-
-	case troubleshoot.FullCommand():
-
-		fmt.Println("Command TBD")
-
 	}
 }

docs/source/command_ref.rst

@@ -15,5 +15,6 @@ Commands Reference
    commands/configtxgen.md
    commands/configtxlator.md
    commands/cryptogen.md
+   commands/ledgerutil.md
    discovery-cli.md
    commands/fabric-ca-commands

docs/source/commands/ledgerutil.md

@@ -0,0 +1,73 @@
+<!---
+ File generated by help_docs.sh. DO NOT EDIT.
+ Please make changes to preamble and postscript wrappers as appropriate.
+ --->
+
+# ledgerutil
+
+The `ledgerutil compare` command allows administrators to compare channel snapshots from two different peers.
+Although channel snapshots from the same height should be identical across peers, if the snapshot
+files indicate different state hashes (as seen in `_snapshot_signable_metadata.json` file from each snapshot)
+then it indicates that at least one of the peers state databases is not correct relative to the blockchain and
+may have gotten corrupted.
+
+The `ledgerutil compare` utility will output a set of JSON files if the snapshots are not identical to assist with troubleshooting in these situations.
+Two output JSON files will include any key/value differences sorted by key (one for public key/value differences and one for private key/value differences),
+another JSON file will include any key/value differences (public or private) sorted by block and transaction height so that you can identify the height where a divergence may have first occurred.
+
+The output files may help an administrator to understand the scope of a state database issue and identify which keys are impacted.
+Snapshots from additional peers can be compared to determine which peer has incorrect state.
+The block and transaction height of the first difference can be used as a reference point when looking at the peer's logs to understand what may have happened to the state database at that time.
+
+## Syntax
+
+The `ledgerutil` command has one subcommand
+
+  * compare
+
+## ledgerutil compare
+```
+usage: ledgerutil compare [<flags>] <snapshotPath1> <snapshotPath2>
+
+Compare channel snapshots from two different peers.
+
+Flags:
+      --help                 Show context-sensitive help (also try --help-long
+                             and --help-man).
+  -o, --outputDir=OUTPUTDIR  Snapshot comparison json results output directory.
+                             Default is the current directory.
+  -f, --firstDiffs=10        Maximum number of differences to record in
+                             first_diffs_by_height.json. Requesting a report
+                             with many differences may result in a large amount
+                             of memory usage. Defaults to 10. If set to 0, will
+                             not produce first_diffs_by_height.json.
+
+Args:
+  <snapshotPath1>  First ledger snapshot directory.
+  <snapshotPath2>  Second ledger snapshot directory.
+```
+
+## Example Usage
+
+### ledgerutil compare example
+
+Here is an example of the `ledgerutil compare` command.
+
+  * Compare snapshots from two different peers for mychannel at snapshot height 5.
+
+    ```
+    ledgerutil compare -o ./compare_output -f 10 ./peer0.org1.example.com/snapshots/completed/mychannel/5 ./peer1.org1.example.com/snapshots/completed/mychannel/5
+
+    Both snapshot public state and private state hashes were the same. No results were generated.
+
+    ```
+
+    The response above indicates that the snapshots are identical. If the snapshots were not identical, the command results will indicate where the comparison output files are written, for example:
+
+    ```
+    Successfully compared snapshots. Results saved to compare_output/mychannel_5_comparison. Total differences found: 3
+    ```
+
+  * Note that both snapshot locations must be accessible by the command, for example by mounting volumes from two different peers, or by copying the snapshots to a common location.
+
+<a rel="license" href="http://creativecommons.org/licenses/by/4.0/"><img alt="Creative Commons License" style="border-width:0" src="https://i.creativecommons.org/l/by/4.0/88x31.png" /></a><br />This work is licensed under a <a rel="license" href="http://creativecommons.org/licenses/by/4.0/">Creative Commons Attribution 4.0 International License</a>.

docs/source/peer_ledger_snapshot.md

@@ -42,7 +42,11 @@ The organization that will use the snapshot to join the channel will then:
 
 If an organization that is already joined to the channel wants to join a new peer using a snapshot, it might decide to skip the process of having other organizations take snapshots and evaluate them, though it is a best practice for an organization to periodically take snapshots of all its peers and compare them to ensure the no ledger forks have occurred. In that case, the organization can take a snapshot immediately and then use the snapshot to join the new peer to the channel.
 
-**Note: snapshots can also be used to verify that the public state between peers is identical (in other words, that no ledger fork has occurred), even if no new peer will use the snapshot to join the channel. This can be done by ensuring that the `snapshot_hash` in the file `_snapshot_additional_metadata.json` in the snapshots generated across peers is the same.**
+## Using snapshots to verify peer integrity
+
+Snapshots can be used to verify that the state between peers is identical (in other words, that no ledger fork has occurred), even if no new peer will use the snapshot to join the channel.
+This can be done by ensuring that the `snapshot_hash` in the file `_snapshot_additional_metadata.json` in the snapshots generated across peers is the same.
+If the hashes are not identical, you can use the [`ledgerutil compare` utility](./commands/ledgerutil.html) to troubleshoot which keys are different across any two snapshots and to understand when a divergence may have occurred.
 
 ## Taking a snapshot
 

docs/wrappers/ledgerutil_postscript.md

@@ -0,0 +1,24 @@
+## Example Usage
+
+### ledgerutil compare example
+
+Here is an example of the `ledgerutil compare` command.
+
+  * Compare snapshots from two different peers for mychannel at snapshot height 5.
+
+    ```
+    ledgerutil compare -o ./compare_output -f 10 ./peer0.org1.example.com/snapshots/completed/mychannel/5 ./peer1.org1.example.com/snapshots/completed/mychannel/5
+
+    Both snapshot public state and private state hashes were the same. No results were generated.
+
+    ```
+
+    The response above indicates that the snapshots are identical. If the snapshots were not identical, the command results will indicate where the comparison output files are written, for example:
+
+    ```
+    Successfully compared snapshots. Results saved to compare_output/mychannel_5_comparison. Total differences found: 3
+    ```
+
+  * Note that both snapshot locations must be accessible by the command, for example by mounting volumes from two different peers, or by copying the snapshots to a common location.
+
+<a rel="license" href="http://creativecommons.org/licenses/by/4.0/"><img alt="Creative Commons License" style="border-width:0" src="https://i.creativecommons.org/l/by/4.0/88x31.png" /></a><br />This work is licensed under a <a rel="license" href="http://creativecommons.org/licenses/by/4.0/">Creative Commons Attribution 4.0 International License</a>.

docs/wrappers/ledgerutil_preamble.md

@@ -0,0 +1,21 @@
+# ledgerutil
+
+The `ledgerutil compare` command allows administrators to compare channel snapshots from two different peers.
+Although channel snapshots from the same height should be identical across peers, if the snapshot
+files indicate different state hashes (as seen in `_snapshot_signable_metadata.json` file from each snapshot)
+then it indicates that at least one of the peers state databases is not correct relative to the blockchain and
+may have gotten corrupted.
+
+The `ledgerutil compare` utility will output a set of JSON files if the snapshots are not identical to assist with troubleshooting in these situations.
+Two output JSON files will include any key/value differences sorted by key (one for public key/value differences and one for private key/value differences),
+another JSON file will include any key/value differences (public or private) sorted by block and transaction height so that you can identify the height where a divergence may have first occurred.
+
+The output files may help an administrator to understand the scope of a state database issue and identify which keys are impacted.
+Snapshots from additional peers can be compared to determine which peer has incorrect state.
+The block and transaction height of the first difference can be used as a reference point when looking at the peer's logs to understand what may have happened to the state database at that time.
+
+## Syntax
+
+The `ledgerutil` command has one subcommand
+
+  * compare

scripts/help_docs.sh

@@ -141,4 +141,11 @@ generateOrCheck \
         docs/wrappers/osnadmin_channel_postscript.md \
         "${commands[@]}"
 
+commands=("ledgerutil compare")
+generateOrCheck \
+        docs/source/commands/ledgerutil.md \
+        docs/wrappers/ledgerutil_preamble.md \
+        docs/wrappers/ledgerutil_postscript.md \
+        "${commands[@]}"
+
 exit