feat: bal: with --declared, include declared accounts (#1765)

Together with -E, this allows showing a balance for all accounts, both
used and declared. I mainly want this for hledger-ui, but there's no
harm in exposing it in the balance command as well. This is somewhat
consistent with the accounts and payees commands.

hledger-lib/Hledger/Reports/MultiBalanceReport.hs

@@ -1,5 +1,6 @@
 {-# LANGUAGE FlexibleInstances   #-}
 {-# LANGUAGE OverloadedStrings   #-}
+{-# LANGUAGE NamedFieldPuns   #-}
 {-# LANGUAGE RecordWildCards     #-}
 {-# LANGUAGE ScopedTypeVariables #-}
 {-|
@@ -26,7 +27,6 @@ module Hledger.Reports.MultiBalanceReport (
   getPostingsByColumn,
   getPostings,
   startingPostings,
-  startingBalancesFromPostings,
   generateMultiBalanceReport,
   balanceReportTableAsText,
 
@@ -122,8 +122,8 @@ multiBalanceReportWith rspec' j priceoracle = report
 
     -- The matched accounts with a starting balance. All of these should appear
     -- in the report, even if they have no postings during the report period.
-    startbals = dbg5 "startbals" . startingBalancesFromPostings rspec j priceoracle
-                                 $ startingPostings rspec j priceoracle reportspan
+    startbals = dbg5 "startbals" $
+      startingBalances rspec j priceoracle $ startingPostings rspec j priceoracle reportspan
 
     -- Generate and postprocess the report, negating balances and taking percentages if needed
     report = dbg4 "multiBalanceReportWith" $
@@ -166,7 +166,7 @@ compoundBalanceReportWith rspec' j priceoracle subreportspecs = cbr
             -- Filter the column postings according to each subreport
             colps'     = map (second $ filter (matchesPosting q)) colps
             -- We need to filter historical postings directly, rather than their accumulated balances. (#1698)
-            startbals' = startingBalancesFromPostings rspec j priceoracle $ filter (matchesPosting q) startps
+            startbals' = startingBalances rspec j priceoracle $ filter (matchesPosting q) startps
             ropts      = cbcsubreportoptions $ _rsReportOpts rspec
             q          = cbcsubreportquery j
 
@@ -183,10 +183,12 @@ compoundBalanceReportWith rspec' j priceoracle subreportspecs = cbr
 
     cbr = CompoundPeriodicReport "" (map fst colps) subreports overalltotals
 
--- | Calculate starting balances from postings, if needed for -H.
-startingBalancesFromPostings :: ReportSpec -> Journal -> PriceOracle -> [Posting]
+-- XXX seems refactorable
+-- | Calculate accounts' balances on the report start date, from these postings
+-- which should be all postings before that data, and possibly also from account declarations.
+startingBalances :: ReportSpec -> Journal -> PriceOracle -> [Posting]
                              -> HashMap AccountName Account
-startingBalancesFromPostings rspec j priceoracle ps =
+startingBalances rspec j priceoracle ps =
     M.findWithDefault nullacct emptydatespan
       <$> calculateReportMatrix rspec j priceoracle mempty [(emptydatespan, ps)]
 
@@ -261,24 +263,34 @@ getPostings rspec@ReportSpec{_rsQuery=query, _rsReportOpts=ropts} j priceoracle
     -- handles the hledger-ui+future txns case above).
     depthless = dbg3 "depthless" $ filterQuery (not . queryIsDepth) query
 
--- | Given a set of postings, eg for a single report column, gather
--- the accounts that have postings and calculate the change amount for
--- each. Accounts and amounts will be depth-clipped appropriately if
--- a depth limit is in effect.
-acctChangesFromPostings :: ReportSpec -> [Posting] -> HashMap ClippedAccountName Account
-acctChangesFromPostings ReportSpec{_rsQuery=query,_rsReportOpts=ropts} ps =
-    HM.fromList [(aname a, a) | a <- as]
+-- | From set of postings, eg for a single report column, calculate the balance change in each account. 
+-- Accounts and amounts will be depth-clipped appropriately if a depth limit is in effect.
+--
+-- When --declared is used, accounts which have been declared with an account directive are included, 
+-- with a 0 balance change. These are really only needed when calculating starting balances, but 
+-- it's harmless to have them in the column changes as well.
+acctChanges :: ReportSpec -> Journal -> [Posting] -> HashMap ClippedAccountName Account
+acctChanges ReportSpec{_rsQuery=query,_rsReportOpts=ReportOpts{accountlistmode_, declared_}} j ps =
+  HM.fromList [(aname a, a) | a <- accts]
   where
-    as = filterAccounts . drop 1 $ accountsFromPostings ps
-    filterAccounts = case accountlistmode_ ropts of
-        ALTree -> filter ((depthq `matchesAccount`) . aname)      -- exclude deeper balances
-        ALFlat -> clipAccountsAndAggregate (queryDepth depthq) .  -- aggregate deeper balances at the depth limit.
-                      filter ((0<) . anumpostings)
-    depthq = dbg3 "depthq" $ filterQuery queryIsDepth query
+    -- with --declared, add all query-matching declared accounts, with 0 change
+    -- (as dummy postings so they are processed like the rest)
+    ps' = ps ++ if declared_ then declaredacctps else []
+      where 
+        declaredacctps = 
+          [nullposting{paccount=n} | n <- journalAccountNamesDeclared j, acctq `matchesAccount` n]
+          where acctq  = dbg3 "acctq" $ filterQuery queryIsAcct query
+
+    filterbydepth = case accountlistmode_ of
+      ALTree -> filter ((depthq `matchesAccount`) . aname)                -- exclude deeper balances
+      ALFlat -> clipAccountsAndAggregate (queryDepth depthq)              -- aggregate deeper balances at the depth limit
+                . if declared_ then id else filter ((0<) . anumpostings)  -- & exclude unposted accounts, unless --declared
+      where depthq = dbg3 "depthq" $ filterQuery queryIsDepth query
+
+    accts = filterbydepth $ drop 1 $ accountsFromPostings ps'
 
 -- | Gather the account balance changes into a regular matrix, then
 -- accumulate and value amounts, as specified by the report options.
---
 -- Makes sure all report columns have an entry.
 calculateReportMatrix :: ReportSpec -> Journal -> PriceOracle
                       -> HashMap ClippedAccountName Account
@@ -308,11 +320,12 @@ calculateReportMatrix rspec@ReportSpec{_rsReportOpts=ropts} j priceoracle startb
         startingBalance = HM.lookupDefault nullacct name startbals
         valuedStart = avalue (DateSpan Nothing historicalDate) startingBalance
 
-    -- Transpose to get each account's balance changes across all columns, then
-    -- pad with zeros
-    allchanges     = ((<>zeros) <$> acctchanges) <> (zeros <$ startbals)
-    acctchanges    = dbg5 "acctchanges" $ transposeMap colacctchanges
-    colacctchanges = dbg5 "colacctchanges" $ map (second $ acctChangesFromPostings rspec) colps
+    -- In each column, get each account's balance changes
+    colacctchanges = dbg5 "colacctchanges" $ map (second $ acctChanges rspec j) colps :: [(DateSpan, HashMap ClippedAccountName Account)]
+    -- Transpose it to get each account's balance changes across all columns
+    acctchanges = dbg5 "acctchanges" $ transposeMap colacctchanges :: HashMap AccountName (Map DateSpan Account)
+    -- Fill out the matrix with zeros in empty cells
+    allchanges = ((<>zeros) <$> acctchanges) <> (zeros <$ startbals)
 
     avalue = acctApplyBoth . mixedAmountApplyValuationAfterSumFromOptsWith ropts j priceoracle
     acctApplyBoth f a = a{aibalance = f $ aibalance a, aebalance = f $ aebalance a}

hledger-lib/Hledger/Reports/ReportOptions.hs

@@ -148,12 +148,13 @@ data ReportOpts = ReportOpts {
                                         --   (Not a regexp, nor a full hledger query, for now.)
     ,accountlistmode_  :: AccountListMode
     ,drop_             :: Int
+    ,declared_         :: Bool  -- ^ Include accounts declared but not yet posted to ?
     ,row_total_        :: Bool
     ,no_total_         :: Bool
-    ,show_costs_       :: Bool  -- ^ Whether to show costs for reports which normally don't show them
+    ,show_costs_       :: Bool  -- ^ Show costs for reports which normally don't show them ?
     ,sort_amount_      :: Bool
     ,percent_          :: Bool
-    ,invert_           :: Bool  -- ^ if true, flip all amount signs in reports
+    ,invert_           :: Bool  -- ^ Flip all amount signs in reports ?
     ,normalbalance_    :: Maybe NormalSign
       -- ^ This can be set when running balance reports on a set of accounts
       --   with the same normal balance type (eg all assets, or all incomes).
@@ -197,6 +198,7 @@ defreportopts = ReportOpts
     , budgetpat_        = Nothing
     , accountlistmode_  = ALFlat
     , drop_             = 0
+    , declared_         = False
     , row_total_        = False
     , no_total_         = False
     , show_costs_       = False
@@ -250,6 +252,7 @@ rawOptsToReportOpts d rawopts =
           ,budgetpat_        = maybebudgetpatternopt rawopts
           ,accountlistmode_  = accountlistmodeopt rawopts
           ,drop_             = posintopt "drop" rawopts
+          ,declared_         = boolopt "declared" rawopts
           ,row_total_        = boolopt "row-total" rawopts
           ,no_total_         = boolopt "no-total" rawopts
           ,show_costs_       = boolopt "show-costs" rawopts

hledger/Hledger/Cli/Commands/Balance.hs

@@ -308,6 +308,7 @@ balancemode = hledgerCommandMode
     ]
     ++ flattreeflags True ++
     [flagReq  ["drop"] (\s opts -> Right $ setopt "drop" s opts) "N" "omit N leading account name parts (in flat mode)"
+    ,flagNone ["declared"] (setboolopt "declared") "include accounts which have been declared but not yet used"
     ,flagNone ["average","A"] (setboolopt "average") "show a row average column (in multicolumn reports)"
     ,flagNone ["related","r"] (setboolopt "related") "show postings' siblings instead"
     ,flagNone ["row-total","T"] (setboolopt "row-total") "show a row total column (in multicolumn reports)"

hledger/Hledger/Cli/Commands/Balance.md

@@ -267,6 +267,12 @@ Here are some ways to handle that:
 [csv-mode]: https://elpa.gnu.org/packages/csv-mode.html
 [visidata]: https://www.visidata.org
 
+### Showing declared accounts
+
+With `--declared`, accounts which have been declared with an [account directive](#declaring-accounts), 
+even if they have no transactions yet, will be included in the balance report with a zero balance,
+and will be visible with `-E/--empty`.
+
 ### Commodity layout
 
 With `--layout`, you can control how amounts with more than one commodity are displayed:

hledger/test/balance/balance.test

@@ -168,3 +168,17 @@ hledger -f - balance -N --output-format=csv --tree
 "Assets:Cash","$-1"
 >>>= 0
 
+# 9. --declared includes all declared accounts, even if they have no postings.
+# They are filtered, depth-clipped, and form trees like the others.
+hledger -f - balance -NE --declared --tree --depth 2 a
+<<<
+account a
+account a:aa
+account a:ab
+account a:ac:aca
+account b
+>>>
+                   0  a
+                   0    aa
+                   0    ab
+>>>= 0