imp!: lib: Generate totally balanced conversion postings for amounts …

…with costs.

This means there will no longer be unbalanced transactions, but will be
offsetting conversion postings to balance things out. For example.

2000-01-01
  a   1 AAA @@ 2 BBB
  b  -2 BBB

When converting to cost, this is treated the same as before.
When not converting to cost, this is now treated as:

2000-01-01
  a   1 AAA
  equity:conversion:AAA:BBB  -1 AAA
  equity:conversion:BBB:AAA   2 BBB
  b  -2 BBB

This also removes show_costs_ option in ReportOpts, replacing its
functionality with a richer cost_ option.

hledger-lib/Hledger/Data/Journal.hs

@@ -120,9 +120,10 @@ import Hledger.Utils
 import Hledger.Data.Types
 import Hledger.Data.AccountName
 import Hledger.Data.Amount
+import Hledger.Data.Posting
 import Hledger.Data.Transaction
 import Hledger.Data.TransactionModifier
-import Hledger.Data.Posting
+import Hledger.Data.Valuation
 import Hledger.Query
 
 
@@ -865,10 +866,9 @@ postingInferredmarketPrice p@Posting{pamount} =
 
 -- | Convert all this journal's amounts to cost using the transaction prices, if any.
 -- The journal's commodity styles are applied to the resulting amounts.
-journalToCost :: Journal -> Journal
-journalToCost j@Journal{jtxns=ts} = j{jtxns=map (transactionToCost styles) ts}
-    where
-      styles = journalCommodityStyles j
+journalToCost :: Costing -> Journal -> Journal
+journalToCost cost j@Journal{jtxns=ts} = j{jtxns=map (transactionToCost styles cost) ts}
+  where styles = journalCommodityStyles j
 
 -- -- | Get this journal's unique, display-preference-canonicalised commodities, by symbol.
 -- journalCanonicalCommodities :: Journal -> M.Map String CommoditySymbol

hledger-lib/Hledger/Data/Posting.hs

@@ -343,8 +343,29 @@ postingApplyValuation priceoracle styles periodlast today v p =
     postingTransformAmount (mixedAmountApplyValuation priceoracle styles periodlast today (postingDate p) v) p
 
 -- | Convert this posting's amount to cost, and apply the appropriate amount styles.
-postingToCost :: M.Map CommoditySymbol AmountStyle -> Posting -> Posting
-postingToCost styles = postingTransformAmount (styleMixedAmount styles . mixedAmountCost)
+postingToCost :: M.Map CommoditySymbol AmountStyle -> Costing -> Posting -> [Posting]
+postingToCost _      NoCost         p = [p]
+postingToCost styles Cost           p = [postingTransformAmount (styleMixedAmount styles . mixedAmountCost) p]
+postingToCost styles ConversionCost p = postingStripPrices p : concatMap conversionPostings (amountsRaw $ pamount p)
+  where
+    conversionPostings amt = case aprice amt of
+        Nothing -> []
+        Just _  -> [ cp{ paccount = "equity:conversion:" <> amtCommodity <> ":" <> costCommodity
+                       , pamount  = mixedAmount . negate $ amountStripPrices amt }
+                   , cp{ paccount = "equity:conversion:" <> costCommodity <> ":" <> amtCommodity
+                       , pamount  = styleMixedAmount styles $ mixedAmount cost }
+                   ]
+      where
+        cost = amountCost amt
+        amtCommodity  = commodity amt
+        costCommodity = commodity cost
+        cp = p{ pcomment = pcomment p `commentAddTag` ("generated-posting","")
+              , ptags = [("generated-posting", ""), ("_generated-posting", "")]
+              , pbalanceassertion = Nothing
+              , poriginal = Nothing
+              }
+        -- Take the commodity of an amount and collapse consecutive spaces to a single space
+        commodity = T.unwords . filter (not . T.null) . T.words . acommodity
 
 -- | Apply a transform function to this posting's amount.
 postingTransformAmount :: (MixedAmount -> MixedAmount) -> Posting -> Posting

hledger-lib/Hledger/Data/Transaction.hs

@@ -351,8 +351,8 @@ transactionApplyValuation priceoracle styles periodlast today v =
   transactionTransformPostings (postingApplyValuation priceoracle styles periodlast today v)
 
 -- | Convert this transaction's amounts to cost, and apply the appropriate amount styles.
-transactionToCost :: M.Map CommoditySymbol AmountStyle -> Transaction -> Transaction
-transactionToCost styles = transactionTransformPostings (postingToCost styles)
+transactionToCost :: M.Map CommoditySymbol AmountStyle -> Costing -> Transaction -> Transaction
+transactionToCost styles cost t = t{tpostings=concatMap (postingToCost styles cost) $ tpostings t}
 
 -- | Apply some account aliases to all posting account names in the transaction, as described by accountNameApplyAliases.
 -- This can fail due to a bad replacement pattern in a regular expression alias.

hledger-lib/Hledger/Data/Valuation.hs

@@ -51,8 +51,8 @@ import Text.Printf (printf)
 ------------------------------------------------------------------------------
 -- Types
 
--- | Whether to convert amounts to cost.
-data Costing = Cost | NoCost
+-- | Whether to convert amounts to cost or create conversion postings.
+data Costing = Cost | ConversionCost | NoCost
   deriving (Show,Eq)
 
 -- | What kind of value conversion should be done on amounts ?
@@ -98,8 +98,8 @@ priceDirectiveToMarketPrice PriceDirective{..} =
 -- Converting things to value
 
 -- | Convert all component amounts to cost/selling price if requested, and style them.
-mixedAmountToCost :: Costing -> M.Map CommoditySymbol AmountStyle -> MixedAmount -> MixedAmount
-mixedAmountToCost cost styles = mapMixedAmount (amountToCost cost styles)
+mixedAmountToCost :: M.Map CommoditySymbol AmountStyle -> Costing -> MixedAmount -> MixedAmount
+mixedAmountToCost styles cost = mapMixedAmount (amountToCost styles cost)
 
 -- | Apply a specified valuation to this mixed amount, using the
 -- provided price oracle, commodity styles, and reference dates.
@@ -109,9 +109,10 @@ mixedAmountApplyValuation priceoracle styles periodlast today postingdate v =
   mapMixedAmount (amountApplyValuation priceoracle styles periodlast today postingdate v)
 
 -- | Convert an Amount to its cost if requested, and style it appropriately.
-amountToCost :: Costing -> M.Map CommoditySymbol AmountStyle -> Amount -> Amount
-amountToCost NoCost _      = id
-amountToCost Cost   styles = styleAmount styles . amountCost
+amountToCost :: M.Map CommoditySymbol AmountStyle -> Costing -> Amount -> Amount
+amountToCost styles Cost           = styleAmount styles . amountCost
+amountToCost _      ConversionCost = amountStripPrices
+amountToCost _      NoCost         = id
 
 -- | Apply a specified valuation to this amount, using the provided
 -- price oracle, and reference dates. Also fix up its display style

hledger-lib/Hledger/Reports/BalanceReport.hs

@@ -313,7 +313,7 @@ tests_BalanceReport = testGroup "BalanceReport" [
                 ,"  a:b          10h @ $50"
                 ,"  c:d                   "
                 ]) >>= either error' return
-         let j' = journalCanonicaliseAmounts $ journalToCost j -- enable cost basis adjustment
+         let j' = journalCanonicaliseAmounts $ journalToCost Cost j -- enable cost basis adjustment
          balanceReportAsText defreportopts (balanceReport defreportopts Any j') `is`
            ["                $500  a:b"
            ,"               $-500  c:d"

hledger-lib/Hledger/Reports/BudgetReport.hs

@@ -219,8 +219,8 @@ budgetReportAsText ropts@ReportOpts{..} budgetr = TB.toLazyText $
   where
     title = "Budget performance in " <> showDateSpan (periodicReportSpan budgetr)
            <> (case cost_ of
-                 Cost   -> ", converted to cost"
-                 NoCost -> "")
+                 Just Cost -> ", converted to cost"
+                 _         -> "")
            <> (case value_ of
                  Just (AtThen _mc)   -> ", valued at posting date"
                  Just (AtEnd _mc)    -> ", valued at period ends"
@@ -388,8 +388,8 @@ budgetReportAsTable
                Nothing
       where
         costedAmounts = case cost_ of
-            Cost   -> amounts . mixedAmountCost
-            NoCost -> amounts
+            Just Cost -> amounts . mixedAmountCost
+            _         -> amounts
 
     -- | Calculate the percentage of actual change to budget goal for a particular commodity
     percentage' :: Change -> BudgetGoal -> CommoditySymbol -> Maybe Percentage

hledger-lib/Hledger/Reports/EntriesReport.hs

@@ -36,7 +36,7 @@ type EntriesReportItem = Transaction
 entriesReport :: ReportSpec -> Journal -> EntriesReport
 entriesReport rspec@ReportSpec{_rsReportOpts=ropts} =
     sortBy (comparing $ transactionDateFn ropts) . jtxns
-    . journalApplyValuationFromOpts rspec{_rsReportOpts=ropts{show_costs_=True}}
+    . journalApplyValuationFromOpts (setDefaultCosting NoCost rspec)
     . filterJournalTransactions (_rsQuery rspec)
 
 tests_EntriesReport = testGroup "EntriesReport" [

hledger-lib/Hledger/Reports/MultiBalanceReport.hs

@@ -255,7 +255,7 @@ getPostings rspec@ReportSpec{_rsQuery=query, _rsReportOpts=ropts} j priceoracle
   where
     rspec' = rspec{_rsQuery=depthless, _rsReportOpts = ropts'}
     ropts' = if isJust (valuationAfterSum ropts)
-        then ropts{value_=Nothing, cost_=NoCost}  -- If we're valuing after the sum, don't do it now
+        then ropts{value_=Nothing, cost_=Just NoCost}  -- If we're valuing after the sum, don't do it now
         else ropts
 
     -- The user's query with no depth limit, and expanded to the report span
@@ -416,7 +416,7 @@ displayedAccounts ReportSpec{_rsQuery=query,_rsReportOpts=ropts} valuedaccts
         balance = maybeStripPrices . case accountlistmode_ ropts of
             ALTree | d == depth -> aibalance
             _                   -> aebalance
-          where maybeStripPrices = if show_costs_ ropts then id else mixedAmountStripPrices
+          where maybeStripPrices = if cost_ ropts == Just NoCost then id else mixedAmountStripPrices
 
     -- Accounts interesting because they are a fork for interesting subaccounts
     interestingParents = dbg5 "interestingParents" $ case accountlistmode_ ropts of

hledger-lib/Hledger/Reports/ReportOptions.hs

@@ -29,6 +29,7 @@ module Hledger.Reports.ReportOptions (
   defreportopts,
   rawOptsToReportOpts,
   defreportspec,
+  setDefaultCosting,
   reportOptsToSpec,
   updateReportSpec,
   updateReportSpecWith,
@@ -116,7 +117,7 @@ data ReportOpts = ReportOpts {
      period_         :: Period
     ,interval_       :: Interval
     ,statuses_       :: [Status]  -- ^ Zero, one, or two statuses to be matched
-    ,cost_           :: Costing  -- ^ Should we convert amounts to cost, when present?
+    ,cost_           :: Maybe Costing  -- ^ Should we convert amounts to cost, when present?
     ,value_          :: Maybe ValuationType  -- ^ What value should amounts be converted to ?
     ,infer_prices_   :: Bool      -- ^ Infer market prices from transactions ?
     ,depth_          :: Maybe Int
@@ -143,7 +144,6 @@ data ReportOpts = ReportOpts {
     ,drop_           :: Int
     ,row_total_      :: Bool
     ,no_total_       :: Bool
-    ,show_costs_     :: Bool  -- ^ Whether to show costs for reports which normally don't show them
     ,sort_amount_    :: Bool
     ,percent_        :: Bool
     ,invert_         :: Bool  -- ^ if true, flip all amount signs in reports
@@ -171,7 +171,7 @@ defreportopts = ReportOpts
     { period_          = PeriodAll
     , interval_        = NoInterval
     , statuses_        = []
-    , cost_            = NoCost
+    , cost_            = Nothing
     , value_           = Nothing
     , infer_prices_    = False
     , depth_           = Nothing
@@ -192,7 +192,6 @@ defreportopts = ReportOpts
     , drop_            = 0
     , row_total_       = False
     , no_total_        = False
-    , show_costs_      = False
     , sort_amount_     = False
     , percent_         = False
     , invert_          = False
@@ -245,7 +244,6 @@ rawOptsToReportOpts d rawopts =
           ,drop_        = posintopt "drop" rawopts
           ,row_total_   = boolopt "row-total" rawopts
           ,no_total_    = boolopt "no-total" rawopts
-          ,show_costs_  = boolopt "show-costs" rawopts
           ,sort_amount_ = boolopt "sort-amount" rawopts
           ,percent_     = boolopt "percent" rawopts
           ,invert_      = boolopt "invert" rawopts
@@ -279,6 +277,11 @@ defreportspec = ReportSpec
     , _rsQueryOpts  = []
     }
 
+-- | Set the default Costing strategy.
+setDefaultCosting :: Costing -> ReportSpec -> ReportSpec
+setDefaultCosting def rspec@ReportSpec{_rsReportOpts=ropts} =
+    rspec{_rsReportOpts=ropts{cost_=cost_ ropts <|> Just def}}
+
 accountlistmodeopt :: RawOpts -> AccountListMode
 accountlistmodeopt =
   fromMaybe ALFlat . choiceopt parse where
@@ -443,32 +446,32 @@ reportOptsToggleStatus s ropts@ReportOpts{statuses_=ss}
 -- This will fail with a usage error if an invalid argument is passed
 -- to --value, or if --valuechange is called with a valuation type
 -- other than -V/--value=end.
-valuationTypeFromRawOpts :: RawOpts -> (Costing, Maybe ValuationType)
+valuationTypeFromRawOpts :: RawOpts -> (Maybe Costing, Maybe ValuationType)
 valuationTypeFromRawOpts rawopts = case (balancecalcopt rawopts, directcost, directval) of
-    (CalcValueChange, _,      Nothing       ) -> (directcost, Just $ AtEnd Nothing)  -- If no valuation requested for valuechange, use AtEnd
-    (CalcValueChange, _,      Just (AtEnd _)) -> (directcost, directval)             -- If AtEnd valuation requested, use it
-    (CalcValueChange, _,      _             ) -> usageError "--valuechange only produces sensible results with --value=end"
-    (CalcGain,        Cost,   _             ) -> usageError "--gain cannot be combined with --cost"
-    (CalcGain,        NoCost, Nothing       ) -> (directcost, Just $ AtEnd Nothing)  -- If no valuation requested for gain, use AtEnd
-    (_,               _,      _             ) -> (directcost, directval)             -- Otherwise, use requested valuation
+    (CalcValueChange, _,         Nothing       ) -> (directcost, Just $ AtEnd Nothing)  -- If no valuation requested for valuechange, use AtEnd
+    (CalcValueChange, _,         Just (AtEnd _)) -> (directcost, directval)             -- If AtEnd valuation requested, use it
+    (CalcValueChange, _,         _             ) -> usageError "--valuechange only produces sensible results with --value=end"
+    (CalcGain,        Just Cost, _             ) -> usageError "--gain cannot be combined with --cost"
+    (CalcGain,        _,         Nothing       ) -> (directcost, Just $ AtEnd Nothing)  -- If no valuation requested for gain, use AtEnd
+    (_,               _,         _             ) -> (directcost, directval)             -- Otherwise, use requested valuation
   where
-    directcost = if Cost `elem` map fst valuationopts then Cost else NoCost
+    directcost = lastMay $ mapMaybe fst valuationopts
     directval  = lastMay $ mapMaybe snd valuationopts
 
     valuationopts = collectopts valuationfromrawopt rawopts
     valuationfromrawopt (n,v)  -- option name, value
-      | n == "B"     = Just (Cost,   Nothing)  -- keep supporting --value=cost for now
-      | n == "V"     = Just (NoCost, Just $ AtEnd Nothing)
-      | n == "X"     = Just (NoCost, Just $ AtEnd (Just $ T.pack v))
+      | n == "B"     = Just (Just Cost, Nothing)  -- keep supporting --value=cost for now
+      | n == "V"     = Just (Nothing,   Just $ AtEnd Nothing)
+      | n == "X"     = Just (Nothing,   Just $ AtEnd (Just $ T.pack v))
       | n == "value" = Just $ valueopt v
       | otherwise    = Nothing
     valueopt v
-      | t `elem` ["cost","c"]  = (Cost,   AtEnd . Just <$> mc)  -- keep supporting --value=cost,COMM for now
-      | t `elem` ["then" ,"t"] = (NoCost, Just $ AtThen mc)
-      | t `elem` ["end" ,"e"]  = (NoCost, Just $ AtEnd  mc)
-      | t `elem` ["now" ,"n"]  = (NoCost, Just $ AtNow  mc)
+      | t `elem` ["cost","c"]  = (Just Cost, AtEnd . Just <$> mc)  -- keep supporting --value=cost,COMM for now
+      | t `elem` ["then" ,"t"] = (Nothing,   Just $ AtThen mc)
+      | t `elem` ["end" ,"e"]  = (Nothing,   Just $ AtEnd  mc)
+      | t `elem` ["now" ,"n"]  = (Nothing,   Just $ AtNow  mc)
       | otherwise = case parsedateM t of
-            Just d  -> (NoCost, Just $ AtDate d mc)
+            Just d  -> (Nothing, Just $ AtDate d mc)
             Nothing -> usageError $ "could not parse \""++t++"\" as valuation type, should be: then|end|now|t|e|n|YYYY-MM-DD"
       where
         -- parse --value's value: TYPE[,COMM]
@@ -545,9 +548,7 @@ journalApplyValuationFromOptsWith rspec@ReportSpec{_rsReportOpts=ropts} j priceo
   where
     valuation p = maybe id (mixedAmountApplyValuation priceoracle styles (periodEnd p) (_rsDay rspec) (postingDate p)) (value_ ropts)
     gain      p = maybe id (mixedAmountApplyGain      priceoracle styles (periodEnd p) (_rsDay rspec) (postingDate p)) (value_ ropts)
-    costing = case cost_ ropts of
-        Cost   -> journalToCost
-        NoCost -> id
+    costing     = journalToCost (fromMaybe ConversionCost $ cost_ ropts)
 
     -- Find the end of the period containing this posting
     periodEnd  = addDays (-1) . fromMaybe err . mPeriodEnd . postingDate
@@ -572,9 +573,10 @@ mixedAmountApplyValuationAfterSumFromOptsWith ropts j priceoracle =
   where
     valuation mc span = mixedAmountValueAtDate priceoracle styles mc (maybe err (addDays (-1)) $ spanEnd span)
     gain mc span = mixedAmountGainAtDate priceoracle styles mc (maybe err (addDays (-1)) $ spanEnd span)
-    costing = case cost_ ropts of
-        Cost   -> styleMixedAmount styles . mixedAmountCost
-        NoCost -> id
+    costing = case fromMaybe ConversionCost $ cost_ ropts of
+        NoCost         -> id
+        ConversionCost -> mixedAmountStripPrices
+        Cost           -> styleMixedAmount styles . mixedAmountCost
     styles = journalCommodityStyles j
     err = error "mixedAmountApplyValuationAfterSumFromOptsWith: expected all spans to have an end date"
 

hledger-ui/Hledger/UI/TransactionScreen.hs

@@ -86,9 +86,7 @@ tsDraw UIState{aopts=UIOpts{uoCliOpts=copts@CliOpts{reportspec_=rspec@ReportSpec
       render . defaultLayout toplabel bottomlabel . str
         . T.unpack . showTransactionOneLineAmounts
         . maybe id (transactionApplyValuation prices styles periodlast (_rsDay rspec)) (value_ ropts)
-        $ case cost_ ropts of
-               Cost   -> transactionToCost styles t
-               NoCost -> t
+        $ maybe id (transactionToCost styles) (cost_ ropts) t
         -- (if real_ ropts then filterTransactionPostings (Real True) else id) -- filter postings by --real
       where
         toplabel =

hledger-ui/Hledger/UI/UIState.hs

@@ -98,8 +98,10 @@ toggleEmpty = over empty__ not
 toggleCost :: UIState -> UIState
 toggleCost = over cost toggleCostMode
   where
-    toggleCostMode Cost   = NoCost
-    toggleCostMode NoCost = Cost
+    toggleCostMode Nothing               = Just Cost
+    toggleCostMode (Just NoCost)         = Just Cost
+    toggleCostMode (Just ConversionCost) = Just Cost
+    toggleCostMode (Just Cost)           = Just ConversionCost
 
 -- | Toggle between showing primary amounts or default valuation.
 toggleValue :: UIState -> UIState

hledger/Hledger/Cli/Commands/Balance.hs

@@ -634,8 +634,8 @@ multiBalanceReportAsText ropts@ReportOpts{..} r = TB.toLazyText $
         (_,               Historical)  -> "Ending balances (historical)"
     valuationdesc =
         (case cost_ of
-            Cost   -> ", converted to cost"
-            NoCost -> "")
+            Just Cost -> ", converted to cost"
+            _         -> "")
         <> (case value_ of
             Just (AtThen _mc)    -> ", valued at posting date"
             Just (AtEnd _mc) | changingValuation -> ""