lib,cli,ui: Generate totally balanced conversion postings for amounts

with cost.

This means there will no longer be unbalanced transactions, but will be
offsetting conversion postings to balance things out. For example.

2000-01-01
  a   1 AAA @@ 2 BBB
  b  -2 BBB

When converting to cost, this is treated the same as before.
When not converting to cost, this is now treated as:

2000-01-01
  a   1 AAA
  equity:conversion:AAA:BBB  -1 AAA
  equity:conversion:BBB:AAA   2 BBB
  b  -2 BBB

hledger-lib/Hledger/Data/Journal.hs

@@ -104,7 +104,7 @@ import Data.Foldable (toList)
 import Data.Function ((&))
 import qualified Data.HashTable.Class as H (toList)
 import qualified Data.HashTable.ST.Cuckoo as H
-import Data.List (find, foldl', sortOn)
+import Data.List (find, foldl', sortBy, sortOn)
 import Data.List.Extra (nubSort)
 import qualified Data.Map.Strict as M
 import Data.Maybe (catMaybes, fromJust, fromMaybe, isJust, mapMaybe, maybeToList)
@@ -122,11 +122,11 @@ import Hledger.Data.Types
 import Hledger.Data.AccountName
 import Hledger.Data.Amount
 import Hledger.Data.Dates
+import Hledger.Data.Posting
 import Hledger.Data.Transaction
 import Hledger.Data.TransactionModifier
-import Hledger.Data.Posting
+import Hledger.Data.Valuation
 import Hledger.Query
-import Data.List (sortBy)
 
 
 -- try to make Journal ppShow-compatible
@@ -1189,10 +1189,9 @@ postingInferredmarketPrice p@Posting{pamount} =
 
 -- | Convert all this journal's amounts to cost using the transaction prices, if any.
 -- The journal's commodity styles are applied to the resulting amounts.
-journalToCost :: Journal -> Journal
-journalToCost j@Journal{jtxns=ts} = j{jtxns=map (transactionToCost styles) ts}
-    where
-      styles = journalCommodityStyles j
+journalToCost :: Costing -> Journal -> Journal
+journalToCost cost j@Journal{jtxns=ts} = j{jtxns=map (transactionToCost styles cost) ts}
+  where styles = journalCommodityStyles j
 
 -- -- | Get this journal's unique, display-preference-canonicalised commodities, by symbol.
 -- journalCanonicalCommodities :: Journal -> M.Map String CommoditySymbol

hledger-lib/Hledger/Data/Posting.hs

@@ -335,8 +335,28 @@ postingApplyValuation priceoracle styles periodlast today v p =
     postingTransformAmount (mixedAmountApplyValuation priceoracle styles periodlast today (postingDate p) v) p
 
 -- | Convert this posting's amount to cost, and apply the appropriate amount styles.
-postingToCost :: M.Map CommoditySymbol AmountStyle -> Posting -> Posting
-postingToCost styles = postingTransformAmount (styleMixedAmount styles . mixedAmountCost)
+postingToCost :: M.Map CommoditySymbol AmountStyle -> Costing -> Posting -> [Posting]
+postingToCost styles Cost           p = [postingTransformAmount (styleMixedAmount styles . mixedAmountCost) p]
+postingToCost styles ConversionCost p = p : concatMap conversionPostings (amountsRaw $ pamount p)
+  where
+    conversionPostings amt = case aprice amt of
+        Nothing -> []
+        Just _  -> [ cp{ paccount = "equity:conversion:" <> amtCommodity <> ":" <> costCommodity
+                       , pamount  = mixedAmount . negate $ amountStripPrices amt }
+                   , cp{ paccount = "equity:conversion:" <> costCommodity <> ":" <> amtCommodity
+                       , pamount  = styleMixedAmount styles $ mixedAmount cost }
+                   ]
+      where
+        cost = amountCost amt
+        amtCommodity  = commodity amt
+        costCommodity = commodity cost
+        cp = p{ pcomment = pcomment p `commentAddTag` ("generated-posting","")
+              , ptags = [("generated-posting", ""), ("_generated-posting", "")]
+              , pbalanceassertion = Nothing
+              , poriginal = Nothing
+              }
+        -- Take the commodity of an amount and collapse consecutive spaces to a single space
+        commodity = T.unwords . filter (not . T.null) . T.words . acommodity
 
 -- | Apply a transform function to this posting's amount.
 postingTransformAmount :: (MixedAmount -> MixedAmount) -> Posting -> Posting

hledger-lib/Hledger/Data/Transaction.hs

@@ -635,8 +635,9 @@ transactionApplyValuation priceoracle styles periodlast today v =
   transactionTransformPostings (postingApplyValuation priceoracle styles periodlast today v)
 
 -- | Convert this transaction's amounts to cost, and apply the appropriate amount styles.
-transactionToCost :: M.Map CommoditySymbol AmountStyle -> Transaction -> Transaction
-transactionToCost styles = transactionTransformPostings (postingToCost styles)
+transactionToCost :: M.Map CommoditySymbol AmountStyle -> Costing -> Transaction -> Transaction
+transactionToCost styles cost t@Transaction{tpostings=ps} =
+    t{tpostings=concatMap (postingToCost styles cost) ps}
 
 -- | Apply some account aliases to all posting account names in the transaction, as described by accountNameApplyAliases.
 -- This can fail due to a bad replacement pattern in a regular expression alias.

hledger-lib/Hledger/Data/Valuation.hs

@@ -51,8 +51,8 @@ import Text.Printf (printf)
 ------------------------------------------------------------------------------
 -- Types
 
--- | Whether to convert amounts to cost.
-data Costing = Cost | NoCost
+-- | Whether to convert amounts to cost or create conversion postings.
+data Costing = Cost | ConversionCost
   deriving (Show,Eq)
 
 -- | What kind of value conversion should be done on amounts ?
@@ -98,8 +98,8 @@ priceDirectiveToMarketPrice PriceDirective{..} =
 -- Converting things to value
 
 -- | Convert all component amounts to cost/selling price if requested, and style them.
-mixedAmountToCost :: Costing -> M.Map CommoditySymbol AmountStyle -> MixedAmount -> MixedAmount
-mixedAmountToCost cost styles = mapMixedAmount (amountToCost cost styles)
+mixedAmountToCost :: M.Map CommoditySymbol AmountStyle -> Costing -> MixedAmount -> MixedAmount
+mixedAmountToCost styles cost = mapMixedAmount (amountToCost styles cost)
 
 -- | Apply a specified valuation to this mixed amount, using the
 -- provided price oracle, commodity styles, and reference dates.
@@ -109,9 +109,9 @@ mixedAmountApplyValuation priceoracle styles periodlast today postingdate v =
   mapMixedAmount (amountApplyValuation priceoracle styles periodlast today postingdate v)
 
 -- | Convert an Amount to its cost if requested, and style it appropriately.
-amountToCost :: Costing -> M.Map CommoditySymbol AmountStyle -> Amount -> Amount
-amountToCost NoCost _      = id
-amountToCost Cost   styles = styleAmount styles . amountCost
+amountToCost :: M.Map CommoditySymbol AmountStyle -> Costing -> Amount -> Amount
+amountToCost styles Cost           = styleAmount styles . amountCost
+amountToCost _      ConversionCost = id
 
 -- | Apply a specified valuation to this amount, using the provided
 -- price oracle, reference dates, and whether this is for a

hledger-lib/Hledger/Reports/AccountTransactionsReport.hs

@@ -79,8 +79,11 @@ type AccountTransactionsReportItem =
   )
 
 accountTransactionsReport :: ReportSpec -> Journal -> Query -> Query -> AccountTransactionsReport
-accountTransactionsReport rspec@ReportSpec{rsOpts=ropts} j reportq thisacctq = items
+accountTransactionsReport rspec' j reportq thisacctq = items
   where
+    rspec = setDefaultCosting ConversionCost rspec'
+    ropts = rsOpts rspec
+
     -- a depth limit should not affect the account transactions report
     -- seems unnecessary for some reason XXX
     reportq'   = reportq -- filterQuery (not . queryIsDepth)

hledger-lib/Hledger/Reports/BalanceReport.hs

@@ -314,7 +314,7 @@ tests_BalanceReport = tests "BalanceReport" [
                 ,"  a:b          10h @ $50"
                 ,"  c:d                   "
                 ]) >>= either error' return
-         let j' = journalCanonicaliseAmounts $ journalToCost j -- enable cost basis adjustment
+         let j' = journalCanonicaliseAmounts $ journalToCost Cost j -- enable cost basis adjustment
          balanceReportAsText defreportopts (balanceReport defreportopts Any j') `is`
            ["                $500  a:b"
            ,"               $-500  c:d"

hledger-lib/Hledger/Reports/BudgetReport.hs

@@ -64,10 +64,11 @@ type BudgetDisplayCell = ((Text, Int), Maybe ((Text, Int), Maybe (Text, Int)))
 -- from the regular transactions, and compare these to get a 'BudgetReport'.
 -- Unbudgeted accounts may be hidden or renamed (see journalWithBudgetAccountNames).
 budgetReport :: ReportSpec -> BalancingOpts -> DateSpan -> Journal -> BudgetReport
-budgetReport rspec bopts reportspan j = dbg4 "sortedbudgetreport" budgetreport
+budgetReport rspec' bopts reportspan j = dbg4 "sortedbudgetreport" budgetreport
   where
     -- Budget report demands ALTree mode to ensure subaccounts and subaccount budgets are properly handled
     -- and that reports with and without --empty make sense when compared side by side
+    rspec = setDefaultCosting ConversionCost rspec'
     ropts = (rsOpts rspec){ accountlistmode_ = ALTree }
     showunbudgeted = empty_ ropts
     budgetedaccts =
@@ -211,8 +212,8 @@ budgetReportAsText ropts@ReportOpts{..} budgetr = TB.toLazyText $
   where
     title = "Budget performance in " <> showDateSpan (periodicReportSpan budgetr)
            <> (case cost_ of
-                 Cost   -> ", converted to cost"
-                 NoCost -> "")
+                 Just Cost -> ", converted to cost"
+                 _         -> "")
            <> (case value_ of
                  Just (AtThen _mc)   -> ", valued at posting date"
                  Just (AtEnd _mc)    -> ", valued at period ends"
@@ -271,8 +272,8 @@ budgetReportAsText ropts@ReportOpts{..} budgetr = TB.toLazyText $
                Nothing
       where
         costedAmounts = case cost_ of
-            Cost   -> amounts . mixedAmountCost
-            NoCost -> amounts
+            Just Cost -> amounts . mixedAmountCost
+            _         -> amounts
 
     maybetranspose | transpose_ = \(Table rh ch vals) -> Table ch rh (transpose vals)
                    | otherwise  = id

hledger-lib/Hledger/Reports/MultiBalanceReport.hs

@@ -108,7 +108,7 @@ multiBalanceReportWith rspec' j priceoracle = report
   where
     -- Queries, report/column dates.
     reportspan = dbg3 "reportspan" $ reportSpan j rspec'
-    rspec      = dbg3 "reportopts" $ makeReportQuery rspec' reportspan
+    rspec      = dbg3 "reportopts" . setDefaultCosting ConversionCost $ makeReportQuery rspec' reportspan
 
     -- Group postings into their columns.
     colps = dbg5 "colps" $ getPostingsByColumn rspec j priceoracle reportspan
@@ -136,7 +136,7 @@ compoundBalanceReportWith rspec' j priceoracle subreportspecs = cbr
   where
     -- Queries, report/column dates.
     reportspan = dbg3 "reportspan" $ reportSpan j rspec'
-    rspec      = dbg3 "reportopts" $ makeReportQuery rspec' reportspan
+    rspec      = dbg3 "reportopts" . setDefaultCosting ConversionCost $ makeReportQuery rspec' reportspan
 
     -- Group postings into their columns.
     colps = dbg5 "colps" $ getPostingsByColumn rspec j priceoracle reportspan

hledger-lib/Hledger/Reports/PostingsReport.hs

@@ -63,8 +63,11 @@ type SummaryPosting = (Posting, Day)
 -- | Select postings from the journal and add running balance and other
 -- information to make a postings report. Used by eg hledger's register command.
 postingsReport :: ReportSpec -> Journal -> PostingsReport
-postingsReport rspec@ReportSpec{rsOpts=ropts@ReportOpts{..}} j = items
+postingsReport rspec' j = items
     where
+      rspec                = setDefaultCosting ConversionCost rspec'
+      ropts@ReportOpts{..} = rsOpts rspec
+
       reportspan  = reportSpanBothDates j rspec
       whichdate   = whichDateFromOpts ropts
       mdepth      = queryDepth $ rsQuery rspec

hledger-lib/Hledger/Reports/ReportOptions.hs

@@ -18,6 +18,7 @@ module Hledger.Reports.ReportOptions (
   defreportopts,
   rawOptsToReportOpts,
   defreportspec,
+  setDefaultCosting,
   reportOptsToSpec,
   updateReportSpec,
   updateReportSpecWith,
@@ -97,7 +98,7 @@ data ReportOpts = ReportOpts {
      period_         :: Period
     ,interval_       :: Interval
     ,statuses_       :: [Status]  -- ^ Zero, one, or two statuses to be matched
-    ,cost_           :: Costing  -- ^ Should we convert amounts to cost, when present?
+    ,cost_           :: Maybe Costing  -- ^ Should we convert amounts to cost, when present?
     ,value_          :: Maybe ValuationType  -- ^ What value should amounts be converted to ?
     ,infer_value_    :: Bool      -- ^ Infer market prices from transactions ?
     ,depth_          :: Maybe Int
@@ -149,7 +150,7 @@ defreportopts = ReportOpts
     { period_          = PeriodAll
     , interval_        = NoInterval
     , statuses_        = []
-    , cost_            = NoCost
+    , cost_            = Nothing
     , value_           = Nothing
     , infer_value_     = False
     , depth_           = Nothing
@@ -249,6 +250,11 @@ defreportspec = ReportSpec
     , rsQueryOpts = []
     }
 
+-- | Set the default Costing strategy.
+setDefaultCosting :: Costing -> ReportSpec -> ReportSpec
+setDefaultCosting def rspec@ReportSpec{rsOpts=ropts} =
+    rspec{rsOpts=ropts{cost_=cost_ ropts <|> Just def}}
+
 -- | Generate a ReportSpec from a set of ReportOpts on a given day.
 reportOptsToSpec :: Day -> ReportOpts -> Either String ReportSpec
 reportOptsToSpec day ropts = do
@@ -435,10 +441,10 @@ reportOptsToggleStatus s ropts@ReportOpts{statuses_=ss}
 -- specified by -B/--cost, -V, -X/--exchange, or --value flags. It is
 -- allowed to combine -B/--cost with any other valuation type. If
 -- there's more than one valuation type, the rightmost flag wins.
-valuationTypeFromRawOpts :: RawOpts -> (Costing, Maybe ValuationType)
+valuationTypeFromRawOpts :: RawOpts -> (Maybe Costing, Maybe ValuationType)
 valuationTypeFromRawOpts rawopts = (costing, valuation)
   where
-    costing   = if (any ((Cost==) . fst) valuationopts) then Cost else NoCost
+    costing   = lastMay $ mapMaybe fst valuationopts
     valuation = case reporttypeopt rawopts of
         ValueChangeReport -> case directval of
             Nothing        -> Just $ AtEnd Nothing  -- If no valuation requested for valuechange, use AtEnd
@@ -449,18 +455,18 @@ valuationTypeFromRawOpts rawopts = (costing, valuation)
 
     valuationopts = collectopts valuationfromrawopt rawopts
     valuationfromrawopt (n,v)  -- option name, value
-      | n == "B"     = Just (Cost,   Nothing)  -- keep supporting --value=cost for now
-      | n == "V"     = Just (NoCost, Just $ AtEnd Nothing)
-      | n == "X"     = Just (NoCost, Just $ AtEnd (Just $ T.pack v))
+      | n == "B"     = Just (Just Cost, Nothing)  -- keep supporting --value=cost for now
+      | n == "V"     = Just (Nothing,   Just $ AtEnd Nothing)
+      | n == "X"     = Just (Nothing,   Just $ AtEnd (Just $ T.pack v))
       | n == "value" = Just $ valueopt v
       | otherwise    = Nothing
     valueopt v
-      | t `elem` ["cost","c"]  = (Cost,   AtEnd . Just <$> mc)  -- keep supporting --value=cost,COMM for now
-      | t `elem` ["then" ,"t"] = (NoCost, Just $ AtThen mc)
-      | t `elem` ["end" ,"e"]  = (NoCost, Just $ AtEnd  mc)
-      | t `elem` ["now" ,"n"]  = (NoCost, Just $ AtNow  mc)
+      | t `elem` ["cost","c"]  = (Just Cost, AtEnd . Just <$> mc)  -- keep supporting --value=cost,COMM for now
+      | t `elem` ["then" ,"t"] = (Nothing,   Just $ AtThen mc)
+      | t `elem` ["end" ,"e"]  = (Nothing,   Just $ AtEnd  mc)
+      | t `elem` ["now" ,"n"]  = (Nothing,   Just $ AtNow  mc)
       | otherwise = case parsedateM t of
-            Just d  -> (NoCost, Just $ AtDate d mc)
+            Just d  -> (Nothing, Just $ AtDate d mc)
             Nothing -> usageError $ "could not parse \""++t++"\" as valuation type, should be: then|end|now|t|e|n|YYYY-MM-DD"
       where
         -- parse --value's value: TYPE[,COMM]
@@ -510,9 +516,7 @@ journalApplyValuationFromOptsWith rspec@ReportSpec{rsOpts=ropts} j priceoracle =
   where
     valuation p = maybe id (postingApplyValuation priceoracle styles (periodEnd p) (rsToday rspec)) (value_ ropts) p
     maybeStripPrices = if show_costs_ ropts then id else postingStripPrices
-    costing = case cost_ ropts of
-        Cost   -> journalToCost
-        NoCost -> id
+    costing = maybe id journalToCost (cost_ ropts)
 
     -- Find the end of the period containing this posting
     periodEnd  = addDays (-1) . fromMaybe err . mPeriodEnd . postingDate
@@ -535,8 +539,9 @@ mixedAmountApplyValuationAfterSumFromOptsWith ropts j priceoracle =
       where err = error "mixedAmountApplyValuationAfterSumFromOptsWith: expected all spans to have an end date"
     maybeStripPrices = if show_costs_ ropts then id else mixedAmountStripPrices
     costing = case cost_ ropts of
-        Cost   -> styleMixedAmount styles . mixedAmountCost
-        NoCost -> id
+        Nothing             -> id
+        Just ConversionCost -> id
+        Just Cost           -> styleMixedAmount styles . mixedAmountCost
     styles = journalCommodityStyles j
 
 -- | If the ReportOpts specify that we are performing valuation after summing amounts,

hledger-lib/Hledger/Reports/TransactionsReport.hs

@@ -62,9 +62,10 @@ triCommodityBalance c = filterMixedAmountByCommodity c  . triBalance
 -- are ordered most recent first. XXX Or an EntriesReport - use that instead ?
 -- This is used by hledger-web's journal view.
 transactionsReport :: ReportSpec -> Journal -> Query -> TransactionsReport
-transactionsReport rspec j q = items
+transactionsReport rspec' j q = items
    where
      -- XXX items' first element should be the full transaction with all postings
+     rspec = setDefaultCosting ConversionCost rspec'
      items = reverse $ accountTransactionsReportItems q None nullmixedamt id ts
      ts    = sortBy (comparing date) $ filter (q `matchesTransaction`) $ jtxns $ journalApplyValuationFromOpts rspec j
      date = transactionDateFn $ rsOpts rspec

hledger-ui/Hledger/UI/TransactionScreen.hs

@@ -82,9 +82,7 @@ tsDraw UIState{aopts=UIOpts{cliopts_=copts@CliOpts{reportspec_=rspec@ReportSpec{
       render . defaultLayout toplabel bottomlabel . str
         . T.unpack . showTransactionOneLineAmounts
         . maybe id (transactionApplyValuation prices styles periodlast (rsToday rspec)) (value_ ropts)
-        $ case cost_ ropts of
-               Cost   -> transactionToCost styles t
-               NoCost -> t
+        $ maybe id (transactionToCost styles) (cost_ ropts) t
         -- (if real_ ropts then filterTransactionPostings (Real True) else id) -- filter postings by --real
       where
         toplabel =

hledger-ui/Hledger/UI/UIState.hs

@@ -115,8 +115,9 @@ toggleCost :: UIState -> UIState
 toggleCost ui@UIState{aopts=uopts@UIOpts{cliopts_=copts@CliOpts{reportspec_=rspec@ReportSpec{rsOpts=ropts}}}} =
     ui{aopts=uopts{cliopts_=copts{reportspec_=rspec{rsOpts=ropts{cost_ = toggle $ cost_ ropts}}}}}
   where
-    toggle Cost   = NoCost
-    toggle NoCost = Cost
+    toggle Nothing               = Just Cost
+    toggle (Just Cost)           = Just ConversionCost
+    toggle (Just ConversionCost) = Just Cost
 
 -- | Toggle between showing primary amounts or default valuation.
 toggleValue :: UIState -> UIState

hledger/Hledger/Cli/Commands/Balance.hs

@@ -594,8 +594,8 @@ multiBalanceReportAsText ropts@ReportOpts{..} r = TB.toLazyText $
         (_,                 HistoricalBalance) -> "Ending balances (historical)"
     valuationdesc =
         (case cost_ of
-            Cost   -> ", converted to cost"
-            NoCost -> "")
+            Just Cost -> ", converted to cost"
+            _         -> "")
         <> (case value_ of
             Just (AtThen _mc)    -> ", valued at posting date"
             Just (AtEnd _mc) | changingValuation -> ""

hledger/Hledger/Cli/Commands/Roi.hs

@@ -64,7 +64,7 @@ roi CliOpts{rawopts_=rawopts, reportspec_=rspec@ReportSpec{rsOpts=ReportOpts{..}
     today = rsToday rspec
     mixedAmountValue periodlast date =
         maybe id (mixedAmountApplyValuation priceOracle styles periodlast today date) value_
-        . mixedAmountToCost cost_ styles
+      . maybe id (mixedAmountToCost styles) cost_
 
   let
     ropts = rsOpts rspec

hledger/Hledger/Cli/CompoundBalanceCommand.hs

@@ -137,8 +137,8 @@ compoundBalanceCommand CompoundBalanceCommandSpec{..} opts@CliOpts{reportspec_=r
 
         valuationdesc =
           (case cost_ of
-               Cost   -> ", converted to cost"
-               NoCost -> "")
+               Just Cost -> ", converted to cost"
+               _         -> "")
           <> (case value_ of
                Just (AtThen _mc)       -> ", valued at posting date"
                Just (AtEnd _mc) | changingValuation -> ""

hledger/test/balance/balance.test

@@ -156,7 +156,7 @@ hledger -f - balance -N --output-format=csv not:equity:conversion
 >>>= 0
 
 # 8. CSV output always shows full account names, even in tree mode (#1565).
-hledger -f - balance -N --output-format=csv --tree
+hledger -f - balance -N --output-format=csv --tree not:equity:conversion
 <<<
 2021-01-01 Test
     Assets:ABC   "AB.C" 1