Auto Generate Doc Examples

.gitignore

@@ -92,3 +92,4 @@ release/
 # Ignore release verification Sha256Sums
 SHA256SUMS-*
 doc/.doc_version
+autogenerate-examples-status.log

contrib/pyln-testing/pyln/testing/utils.py

@@ -416,6 +416,7 @@ def __init__(self, bitcoin_dir="/tmp/bitcoind-test", rpcport=None):
             '-debug=mempoolrej',
             '-debug=rpc',
             '-debug=validation',
+            '-rpcthreads=20',
         ]
         # For up to and including 0.16.1, this needs to be in main section.
         BITCOIND_CONFIG['rpcport'] = rpcport

doc/contribute-to-core-lightning/coding-style-guidelines/writing-json-schemas.md

@@ -62,6 +62,21 @@ To add conditional fields:
    it can be defined as `"dependentUpon": { "index": ["start", "limit"] }` in the json and it will
    generate the markdown syntax as `[*index* [*start*] [*limit*]]`.
 
+### Re-generate examples listed in rpc schemas (doc/schemas/lightning-*.json)
+
+1. The `autogenerate-rpc-examples.py` script regenerates RPC examples for methods listed in `doc/schemas/lightning-*.json` files.
+2. It uses our pre-existing pytest suite to perform this task.
+3. The script runs a test named `test_generate_examples`, which sets up test nodes, records RPC requests, and captures responses.
+4. To prevent accidental overwriting of examples with other tests, set the environment variable `GENERATE_EXAMPLES=True` before running the script.
+5. By default, all methods are regenerated. To specify which methods to regenerate, set the `REGENERATE` environment variable with a comma-separated list of method names. Eg. `REGENERATE='getinfo,connect'` will only regenerate examples for the getinfo and connect RPCs.
+6. The dev-bitcoind-poll is set to 3 seconds. Ensure the `TIMEOUT` environment variable is set to more than 3 seconds to avoid test failures due to a short waiting time for bitcoind responses.
+7. To run the script using Poetry, use the following command:
+```
+rm -rf /tmp/ltests* && REGENERATE='getinfo,connect' VALGRIND=0 TIMEOUT=10 TEST_DEBUG=1 pytest -vvv -s -p no:logging -n 6 tests/autogenerate-rpc-examples.py
+```
+8. The script saves logs in a file named `autogenerate-examples-status.log`, located in the root directory.
+9. After running the script, execute make to ensure that the schema has been updated in all relevant locations, such as `...msggen/schema.json`.
+
 ### JSON Drinking Game!
 
 1. Sip whenever you have an additional comma at the end of a sequence.

doc/schemas/lightning-addpsbtoutput.json

@@ -81,17 +81,18 @@
   "examples": [
     {
       "description": [
-        "Here is a command to make a PSBT with a 100,000 sat output that leads to the on-chain wallet."
+        "Here is a command to make a PSBT with a 1,000,000 sat output that leads to the on-chain wallet:"
       ],
       "request": {
         "id": "example:addpsbtoutput#1",
         "method": "addpsbtoutput",
         "params": {
-          "satoshi": 100000
+          "satoshi": 1000000,
+          "locktime": 111
         }
       },
       "response": {
-        "psbt": "cHNidP8BAgQCAAAAAQMEbAAAAAEEAQABBQEBAQYBAwH7BAIAAAAAAQMIoIYBAAAAAAABBCJRIHg2NV/cioLcTLAKdyxVVBUdBjhKTdZejT9orAhWa4S+AA==",
+        "psbt": "cHNidP8BAgQCAAAAAQMEbwAAAAEEAQABBQEBAQYBAwH7BAIAAAAAAQMIQEIPAAAAAAABBCJRIAUY6SzZTg9iwG8Sbf+Yuavnm37YRaFW1SRWeOJlVEddAA==",
         "estimated_added_weight": 172,
         "outnum": 0
       }
@@ -100,31 +101,31 @@
       "request": {
         "id": "example:addpsbtoutput#2",
         "method": "addpsbtoutput",
-        "params": {
-          "satoshi": 1000000,
-          "locktime": 111
-        }
+        "params": [
+          3333333,
+          "cHNidP8BAgQCAAAAAQMEbwAAAAEEAQABBQECAQYBAwH7BAIAAAAAAQMIQEIPAAAAAAABBCJRIAUY6SzZTg9iwG8Sbf+Yuavnm37YRaFW1SRWeOJlVEddAAEDCNXcMgAAAAAAAQQiUSB1Sne1A/y6D9gPChqCJu1nZP+anZu2G0hdQNTJ9L4kWQA="
+        ]
       },
       "response": {
-        "psbt": "cHNidP8BAgQCAAAAAQMEbwAAAAEEAQABBQEBAQYBAwH7BAIAAAAAAQMIQEIPAAAAAAABBCJRIJd6ICNAQALFOMhoUHuSVSuzcaUdkDKlk4K+A+DR9+4uAA==",
+        "psbt": "cHNidP8BAgQCAAAAAQMEbwAAAAEEAQABBQECAQYBAwH7BAIAAAAAAQMIQEIPAAAAAAABBCJRIAUY6SzZTg9iwG8Sbf+Yuavnm37YRaFW1SRWeOJlVEddAAEDCNXcMgAAAAAAAQQiUSB1Sne1A/y6D9gPChqCJu1nZP+anZu2G0hdQNTJ9L4kWQA=",
         "estimated_added_weight": 172,
-        "outnum": 0
+        "outnum": 1
       }
     },
     {
       "request": {
         "id": "example:addpsbtoutput#3",
         "method": "addpsbtoutput",
         "params": {
-          "satoshi": 974343,
-          "initialpsbt": "cHNidP8BAF4CAAAAAfwbEpvpi6D14YV4VLnuVB47Y0uF41kXEyJRL4IusySSAQAAAAD9////ASICAAAAAAAAIlEgeDY1X9yKgtxMsAp3LFVUFR0GOEpN1l6NP2isCFZrhL5nAAAAAAEA9gIAAAAAAQFEkxvLatohY6mw5gr5qG1aiArSrziFPR2YoqD21Hv+RAAAAAAA/f///wJAQg8AAAAAACIAIFuM07kUz2fN2Ppic8kwNT3TZHZzT72WIQLC31O5CIDNrz8PAAAAAAAiUSBj/+5Op9UebK35CG4oaiUnkiqqJbjFOuvzL6MqCmJ/WgJHMEQCIEu1nfVRt9i+rFM219mwhMqdwJsqygWSWTFUS+cemdh6AiBG3Qo8g9J/aAMO2RHDsIBScscj6pTTIwZp7Gw8G3EOKAEhA9dFRFyTYmZfIuDZbp52byc/MmDeo5yKdr+gXdJoTdzPZgAAAAEBK68/DwAAAAAAIlEgY//uTqfVHmyt+QhuKGolJ5IqqiW4xTrr8y+jKgpif1oAAA==",
-          "destination": "bcrt1q9tc6q49l6wrrtp8ul45rj92hsleehwwxty32zu"
+          "satoshi": 3333333,
+          "initialpsbt": "cHNidP8BAgQCAAAAAQMEbwAAAAEEAQABBQECAQYBAwH7BAIAAAAAAQMIQEIPAAAAAAABBCJRIAUY6SzZTg9iwG8Sbf+Yuavnm37YRaFW1SRWeOJlVEddAAEDCNXcMgAAAAAAAQQiUSB1Sne1A/y6D9gPChqCJu1nZP+anZu2G0hdQNTJ9L4kWQA=",
+          "destination": "bcrt1p97zrhgxgm6wscsdx8gjafj4jyqetunat7fynrk4cyg2rz6lzrr9q6dlrp2"
         }
       },
       "response": {
-        "psbt": "cHNidP8BAH0CAAAAAfwbEpvpi6D14YV4VLnuVB47Y0uF41kXEyJRL4IusySSAQAAAAD9////AiICAAAAAAAAIlEgeDY1X9yKgtxMsAp3LFVUFR0GOEpN1l6NP2isCFZrhL4H3g4AAAAAABYAFCrxoFS/04Y1hPz9aDkVV4fzm7nGZwAAAAABAPYCAAAAAAEBRJMby2raIWOpsOYK+ahtWogK0q84hT0dmKKg9tR7/kQAAAAAAP3///8CQEIPAAAAAAAiACBbjNO5FM9nzdj6YnPJMDU902R2c0+9liECwt9TuQiAza8/DwAAAAAAIlEgY//uTqfVHmyt+QhuKGolJ5IqqiW4xTrr8y+jKgpif1oCRzBEAiBLtZ31UbfYvqxTNtfZsITKncCbKsoFklkxVEvnHpnYegIgRt0KPIPSf2gDDtkRw7CAUnLHI+qU0yMGaexsPBtxDigBIQPXRURck2JmXyLg2W6edm8nPzJg3qOcina/oF3SaE3cz2YAAAABASuvPw8AAAAAACJRIGP/7k6n1R5srfkIbihqJSeSKqoluMU66/MvoyoKYn9aAAAA",
+        "psbt": "cHNidP8BAgQCAAAAAQMEbwAAAAEEAQABBQEDAQYBAwH7BAIAAAAAAQMIQEIPAAAAAAABBCJRIAUY6SzZTg9iwG8Sbf+Yuavnm37YRaFW1SRWeOJlVEddAAEDCNXcMgAAAAAAAQQiUSB1Sne1A/y6D9gPChqCJu1nZP+anZu2G0hdQNTJ9L4kWQABAwjV3DIAAAAAAAEEIlEgL4Q7oMjenQxBpjol1MqyIDK+T6vySTHauCIUMWviGMoA",
         "estimated_added_weight": 172,
-        "outnum": 1
+        "outnum": 2
       }
     }
   ]

doc/schemas/lightning-autoclean-once.json

@@ -209,8 +209,8 @@
       "response": {
         "autoclean": {
           "failedpays": {
-            "cleaned": 1,
-            "uncleaned": 1
+            "cleaned": 0,
+            "uncleaned": 7
           }
         }
       }
@@ -227,7 +227,7 @@
       "response": {
         "autoclean": {
           "succeededpays": {
-            "cleaned": 1,
+            "cleaned": 7,
             "uncleaned": 0
           }
         }

doc/schemas/lightning-autoclean-status.json

@@ -438,7 +438,8 @@
       "response": {
         "autoclean": {
           "expiredinvoices": {
-            "enabled": false,
+            "enabled": true,
+            "age": 300,
             "cleaned": 0
           }
         }
@@ -462,7 +463,7 @@
           },
           "succeededpays": {
             "enabled": false,
-            "cleaned": 0
+            "cleaned": 7
           },
           "failedpays": {
             "enabled": false,
@@ -474,7 +475,7 @@
           },
           "expiredinvoices": {
             "enabled": true,
-            "age": 2,
+            "age": 300,
             "cleaned": 0
           }
         }

doc/schemas/lightning-bkpr-channelsapy.json

@@ -214,29 +214,49 @@
       "response": {
         "channels_apy": [
           {
-            "account": "e41b2ec83e9139a9fd8f1d89b01e5d7df73099494e6b91504c39445e37485b0d",
-            "routed_out_msat": 1431440,
+            "account": "252d1b0a1e57895e84137f28cf19ab2c35847e284c112fefdecc7afeaa5c1de7",
+            "routed_out_msat": 1000000,
+            "routed_in_msat": 510081208,
+            "lease_fee_paid_msat": 0,
+            "lease_fee_earned_msat": 0,
+            "pushed_out_msat": 0,
+            "pushed_in_msat": 0,
+            "our_start_balance_msat": 0,
+            "channel_start_balance_msat": 1000000000,
+            "fees_out_msat": 0,
+            "fees_in_msat": 106,
+            "utilization_out": "0.1000%",
+            "utilization_in": "51.0081%",
+            "utilization_in_initial": "51.0081%",
+            "apy_out": "0.0000%",
+            "apy_in": "0.0252%",
+            "apy_in_initial": "0.0252%",
+            "apy_total": "0.0252%"
+          },
+          {
+            "account": "a397dd9b3e44afcb67f3f3ce1d649b74a8ade63e35505985e4cc1828634f69a2",
+            "routed_out_msat": 510181102,
             "routed_in_msat": 0,
             "lease_fee_paid_msat": 0,
             "lease_fee_earned_msat": 0,
             "pushed_out_msat": 0,
             "pushed_in_msat": 0,
             "our_start_balance_msat": 1000000000,
             "channel_start_balance_msat": 1000000000,
-            "fees_out_msat": 0,
+            "fees_out_msat": 106,
             "fees_in_msat": 0,
-            "utilization_out": "0.1431%",
-            "utilization_out_initial": "0.1431%",
+            "utilization_out": "51.0181%",
+            "utilization_out_initial": "51.0181%",
             "utilization_in": "0.0000%",
-            "apy_out": "0.0000%",
-            "apy_out_initial": "0.0000%",
+            "apy_out": "0.0505%",
+            "apy_out_initial": "0.0505%",
             "apy_in": "0.0000%",
-            "apy_total": "0.0000%",
-            "apy_total_initial": "0.0000%"
+            "apy_total": "0.0505%",
+            "apy_total_initial": "0.0505%"
           },
           {
-            "account": "net",
-            "routed_out_msat": 1431440,
+            "account": "f8fc83a432cbfb2fffe222cc06727fdd977b5dd10ebd6707158e799e6f522d9f",
+            "routed_out_msat": 500000000,
             "routed_in_msat": 0,
             "lease_fee_paid_msat": 0,
             "lease_fee_earned_msat": 0,
@@ -246,14 +266,37 @@
             "channel_start_balance_msat": 1000000000,
             "fees_out_msat": 0,
             "fees_in_msat": 0,
-            "utilization_out": "0.1431%",
-            "utilization_out_initial": "0.1431%",
+            "utilization_out": "50.0000%",
+            "utilization_out_initial": "50.0000%",
             "utilization_in": "0.0000%",
             "apy_out": "0.0000%",
             "apy_out_initial": "0.0000%",
             "apy_in": "0.0000%",
             "apy_total": "0.0000%",
             "apy_total_initial": "0.0000%"
+          },
+          {
+            "account": "net",
+            "routed_out_msat": 1011181102,
+            "routed_in_msat": 510081208,
+            "lease_fee_paid_msat": 0,
+            "lease_fee_earned_msat": 0,
+            "pushed_out_msat": 0,
+            "pushed_in_msat": 0,
+            "our_start_balance_msat": 2000000000,
+            "channel_start_balance_msat": 3000000000,
+            "fees_out_msat": 106,
+            "fees_in_msat": 106,
+            "utilization_out": "33.7060%",
+            "utilization_out_initial": "50.5591%",
+            "utilization_in": "17.0027%",
+            "utilization_in_initial": "51.0081%",
+            "apy_out": "0.0084%",
+            "apy_out_initial": "0.0126%",
+            "apy_in": "0.0084%",
+            "apy_in_initial": "0.0252%",
+            "apy_total": "0.0168%",
+            "apy_total_initial": "0.0168%"
           }
         ]
       }

doc/schemas/lightning-bkpr-inspect.json

@@ -211,23 +211,54 @@
       "request": {
         "id": "example:bkpr-inspect#1",
         "method": "bkpr-inspect",
-        "params": [
-          "f30a7bab1ec077622d8fe877634bc6dd38bb08122ad49606199c565e0383b2ab"
-        ]
+        "params": {
+          "account": "a397dd9b3e44afcb67f3f3ce1d649b74a8ade63e35505985e4cc1828634f69a2"
+        }
       },
       "response": {
         "txs": [
           {
-            "txid": "abb283035e569c190696d42a1208bb38ddc64b6377e88f2d6277c01eab7b0af3",
-            "fees_paid_msat": 0,
+            "txid": "670ea3ab29bf182e25bd2697db791e81788ede2723206066815596663c7ab0ea",
+            "blockheight": 110,
+            "fees_paid_msat": 4927000,
+            "outputs": [
+              {
+                "account": "a397dd9b3e44afcb67f3f3ce1d649b74a8ade63e35505985e4cc1828634f69a2",
+                "outnum": 1,
+                "output_tag": "channel_open",
+                "output_value_msat": 1000000000,
+                "credit_msat": 1000000000,
+                "currency": "bcrt",
+                "spend_tag": "channel_close",
+                "spending_txid": "e7c787703fb67013b912ecb4f35d1d93326a5678cf11985ee0b6a6fc092cdb5b",
+                "debit_msat": 489818898
+              }
+            ]
+          },
+          {
+            "txid": "e7c787703fb67013b912ecb4f35d1d93326a5678cf11985ee0b6a6fc092cdb5b",
+            "blockheight": 120,
+            "fees_paid_msat": 2895898,
             "outputs": [
               {
-                "account": "f30a7bab1ec077622d8fe877634bc6dd38bb08122ad49606199c565e0383b2ab",
+                "account": "wallet",
                 "outnum": 0,
-                "output_tag": "channel_proposed",
-                "output_value_msat": 996363000,
-                "credit_msat": 996363000,
-                "currency": "bcrt"
+                "output_tag": "deposit",
+                "output_value_msat": 486923000,
+                "credit_msat": 486923000,
+                "currency": "bcrt",
+                "spend_tag": "withdrawal",
+                "spending_txid": "cfa369a6ba1b817a3c526c1bd80b05fab695091005ca360bb2c19a8cfb55d680",
+                "debit_msat": 486923000
+              },
+              {
+                "account": "external",
+                "outnum": 1,
+                "output_tag": "to_them",
+                "output_value_msat": 510181000,
+                "credit_msat": 510181000,
+                "currency": "bcrt",
+                "originating_account": "a397dd9b3e44afcb67f3f3ce1d649b74a8ade63e35505985e4cc1828634f69a2"
               }
             ]
           }