Many fixes to both --market and --exchange

src/balance.cc

@@ -163,18 +163,19 @@ balance_t::value(const bool		       primary_only,
 		 const optional<datetime_t>&   moment,
 		 const optional<commodity_t&>& in_terms_of) const
 {
-  optional<balance_t> temp;
+  balance_t temp;
+  bool      resolved = false;
 
   foreach (const amounts_map::value_type& pair, amounts) {
-    if (! temp)
-      temp = balance_t();
     if (optional<amount_t> val = pair.second.value(primary_only, moment,
-						   in_terms_of))
+						   in_terms_of)) {
-      *temp += *val;
+      temp += *val;
-    else
+      resolved = true;
-      *temp += pair.second;
+    } else {
+      temp += pair.second;
     }
-  return temp;
+  }
+  return resolved ? temp : optional<balance_t>();
 }
 
 optional<amount_t>

src/chain.cc

@@ -74,6 +74,15 @@ xact_handler_ptr chain_xact_handlers(report_t&	      report,
       handler.reset(new filter_xacts(handler, display_predicate, report));
     }
 
+    // changed_value_xacts adds virtual xacts to the list to account for
+    // changes in market value of commodities, which otherwise would affect
+    // the running total unpredictably.
+    if (report.HANDLED(revalued))
+      handler.reset(new changed_value_xacts(handler,
+					    report.HANDLER(display_total_).expr,
+					    report,
+					    report.HANDLED(revalued_only)));
+
     // calc_xacts computes the running total.  When this appears will
     // determine, for example, whether filtered xacts are included or excluded
     // from the running total.
@@ -98,14 +107,6 @@ xact_handler_ptr chain_xact_handlers(report_t&	      report,
 	handler.reset(new sort_xacts(handler, report.HANDLER(sort_).str()));
     }
 
-    // changed_value_xacts adds virtual xacts to the list to account for
-    // changes in market value of commodities, which otherwise would affect
-    // the running total unpredictably.
-    if (report.HANDLED(revalued))
-      handler.reset(new changed_value_xacts(handler,
-					    report.HANDLER(total_).expr,
-					    report.HANDLED(revalued_only)));
-
     // collapse_xacts causes entries with multiple xacts to appear as entries
     // with a subtotaled xact for each commodity used.
     if (report.HANDLED(collapse))
@@ -137,7 +138,6 @@ xact_handler_ptr chain_xact_handlers(report_t&	      report,
   if (report.HANDLED(period_)) {
     handler.reset(new interval_xacts(handler, expr,
 				     report.HANDLER(period_).str(),
-				     report.session.master.get(),
 				     report.HANDLED(exact),
 				     report.HANDLED(empty)));
     handler.reset(new sort_xacts(handler, "date"));

src/filters.cc

@@ -33,6 +33,7 @@
 #include "iterators.h"
 #include "compare.h"
 #include "format.h"
+#include "report.h"
 
 namespace ledger {
 
@@ -218,7 +219,8 @@ namespace {
 		    unsigned int	  flags,
 		    std::list<xact_t>&    temps,
 		    item_handler<xact_t>& handler,
-		    const date_t&         date = date_t())
+		    const date_t&         date  = date_t(),
+		    const value_t&        total = value_t())
   {
     temps.push_back(xact_t(account));
     xact_t& xact(temps.back());
@@ -266,6 +268,9 @@ namespace {
       break;
     }
 
+    if (! total.is_null())
+      xdata.total = total;
+
     if (flags)
       xdata.add_flags(flags);
 
@@ -294,7 +299,7 @@ void collapse_xacts::report_subtotal()
     date_t earliest_date;
 
     foreach (xact_t * xact, component_xacts) {
-      date_t reported = xact->reported_date();
+      date_t reported = xact->date();
       if (! is_valid(earliest_date) ||
 	  reported < earliest_date)
 	earliest_date = reported;
@@ -367,35 +372,54 @@ void related_xacts::flush()
   item_handler<xact_t>::flush();
 }
 
-void changed_value_xacts::output_diff(const date_t& date)
+void changed_value_xacts::output_diff(xact_t * xact, const date_t& date)
 {
-  value_t cur_bal;
+  if (is_valid(date))
+    xact->xdata().date = date;
 
-  last_xact->xdata().date = date;
+  value_t repriced_total;
-  cur_bal = total_expr.calc(*last_xact).rounded();
+  try {
+    bind_scope_t bound_scope(report, *xact);
+    repriced_total = total_expr.calc(bound_scope);
+  }
+  catch (...) {
+    xact->xdata().date = date_t();
+    throw;
+  }
+  xact->xdata().date = date_t();
+
+  DEBUG("filter.changed_value",
+	"output_diff(last_balance) = " << last_balance);
+  DEBUG("filter.changed_value",
+	"output_diff(repriced_total) = " << repriced_total);
+
+  if (value_t diff = repriced_total - last_balance) {
+    DEBUG("filter.changed_value", "output_diff(strip(diff)) = "
+	  << diff.strip_annotations(report.what_to_keep()));
 
-  if (value_t diff = cur_bal - last_balance) {
     entry_temps.push_back(entry_t());
     entry_t& entry = entry_temps.back();
     entry.payee = "Commodities revalued";
-    entry._date = date;
+    entry._date = is_valid(date) ? date : xact->date();
 
-    handle_value(diff, NULL, &entry, XACT_EXT_NO_TOTAL, xact_temps,
+    handle_value(diff, &revalued_account, &entry, XACT_EXT_NO_TOTAL,
-		 *handler);
+		 xact_temps, *handler, *entry._date, repriced_total);
   }
 }
 
 void changed_value_xacts::operator()(xact_t& xact)
 {
   if (last_xact)
-    output_diff(last_xact->reported_date());
+    output_diff(last_xact, xact.date());
 
   if (changed_values_only)
     xact.xdata().add_flags(XACT_EXT_DISPLAYED);
 
   item_handler<xact_t>::operator()(xact);
 
-  last_balance = total_expr.calc(xact).rounded();
+  bind_scope_t bound_scope(report, xact);
+  last_balance = total_expr.calc(bound_scope);
+
   last_xact = &xact;
 }
 
@@ -409,7 +433,7 @@ void subtotal_xacts::report_subtotal(const char *  spec_fmt,
   date_t range_start  = start;
   date_t range_finish = finish;
   foreach (xact_t * xact, component_xacts) {
-    date_t date = xact->reported_date();
+    date_t date = xact->date();
     if (! is_valid(range_start) || date < range_start)
       range_start = date;
     if (! is_valid(range_finish) || date > range_finish)
@@ -437,7 +461,7 @@ void subtotal_xacts::report_subtotal(const char *  spec_fmt,
 
   foreach (values_map::value_type& pair, values)
     handle_value(pair.second.value, pair.second.account, &entry, 0,
-		 xact_temps, *handler, range_finish);
+		 xact_temps, *handler);
 
   values.clear();
 }
@@ -484,7 +508,7 @@ void interval_xacts::report_subtotal(const date_t& finish)
 
 void interval_xacts::operator()(xact_t& xact)
 {
-  date_t date = xact.reported_date();
+  date_t date = xact.date();
 
   if ((is_valid(interval.begin) && date < interval.begin) ||
       (is_valid(interval.end)   && date >= interval.end))
@@ -542,7 +566,7 @@ void xacts_as_equity::report_subtotal()
 {
   date_t finish;
   foreach (xact_t * xact, component_xacts) {
-    date_t date = xact->reported_date();
+    date_t date = xact->date();
     if (! is_valid(finish) || date > finish)
       finish = date;
   }
@@ -614,7 +638,7 @@ void transfer_details::operator()(xact_t& xact)
 {
   entry_temps.push_back(*xact.entry);
   entry_t& entry = entry_temps.back();
-  entry._date = xact.reported_date();
+  entry._date = xact.date();
 
   xact_temps.push_back(xact);
   xact_t& temp = xact_temps.back();

src/filters.h

@@ -450,9 +450,11 @@ class changed_value_xacts : public item_handler<xact_t>
   // later in the chain.
 
   expr_t    total_expr;
+  report_t& report;
   bool	    changed_values_only;
   xact_t *  last_xact;
   value_t   last_balance;
+  account_t revalued_account;
 
   std::list<entry_t> entry_temps;
   std::list<xact_t>  xact_temps;
@@ -462,11 +464,13 @@ class changed_value_xacts : public item_handler<xact_t>
 public:
   changed_value_xacts(xact_handler_ptr handler,
 		      const expr_t&    _total_expr,
+		      report_t&        _report,
 		      bool	       _changed_values_only)
     : item_handler<xact_t>(handler), total_expr(_total_expr),
-      changed_values_only(_changed_values_only), last_xact(NULL) {
+      report(_report), changed_values_only(_changed_values_only),
+      last_xact(NULL), revalued_account(NULL, "<Revalued>") {
     TRACE_CTOR(changed_value_xacts,
-	       "xact_handler_ptr, bool");
+	       "xact_handler_ptr, const expr_t&, report_t&, bool");
   }
   virtual ~changed_value_xacts() {
     TRACE_DTOR(changed_value_xacts);
@@ -474,14 +478,14 @@ public:
   }
 
   virtual void flush() {
-    if (last_xact) {
+    if (last_xact && last_xact->date() <= CURRENT_DATE()) {
-      output_diff(CURRENT_DATE());
+      output_diff(last_xact, CURRENT_DATE());
       last_xact = NULL;
     }
     item_handler<xact_t>::flush();
   }
 
-  void output_diff(const date_t& current);
+  void output_diff(xact_t * xact, const date_t& current);
 
   virtual void operator()(xact_t& xact);
 };
@@ -576,11 +580,10 @@ public:
   interval_xacts(xact_handler_ptr  _handler,
 		 expr_t&	   amount_expr,
 		 const interval_t& _interval,
-		 account_t *	   master		 = NULL,
 		 bool		   _exact_periods	 = false,
 		 bool              _generate_empty_xacts = false)
     : subtotal_xacts(_handler, amount_expr), interval(_interval),
-      last_xact(NULL), empty_account(master, "<None>"),
+      last_xact(NULL), empty_account(NULL, "<None>"),
       exact_periods(_exact_periods),
       generate_empty_xacts(_generate_empty_xacts) {
     TRACE_CTOR(interval_xacts,

src/output.cc

@@ -170,7 +170,7 @@ void gather_statistics::operator()(xact_t& xact)
 
     statistics.filenames.insert(xact.pathname);
 
-    date_t date = xact.reported_date();
+    date_t date = xact.date();
 
     if (date.year() == CURRENT_DATE().year() &&
 	date.month() == CURRENT_DATE().month())
@@ -185,11 +185,11 @@ void gather_statistics::operator()(xact_t& xact)
       statistics.total_uncleared_xacts++;
 
     if (! is_valid(statistics.earliest_xact) ||
-	xact.reported_date() < statistics.earliest_xact)
+	xact.date() < statistics.earliest_xact)
-      statistics.earliest_xact = xact.reported_date();
+      statistics.earliest_xact = xact.date();
     if (! is_valid(statistics.latest_xact) ||
-	xact.reported_date() > statistics.latest_xact)
+	xact.date() > statistics.latest_xact)
-      statistics.latest_xact = xact.reported_date();
+      statistics.latest_xact = xact.date();
 
     statistics.accounts_referenced.insert(xact.account->fullname());
     statistics.payees_referenced.insert(xact.entry->payee);

src/report.cc

@@ -120,7 +120,7 @@ value_t report_t::fn_display_total(call_scope_t& scope)
   return HANDLER(display_total_).expr.calc(scope);
 }
 
-value_t report_t::fn_market_value(call_scope_t& scope)
+value_t report_t::fn_market(call_scope_t& scope)
 {
   interactive_t args(scope, "a&ts");
 
@@ -132,14 +132,18 @@ value_t report_t::fn_market_value(call_scope_t& scope)
 	 p;
 	 p = std::strtok(NULL, ",")) {
       if (commodity_t * commodity = amount_t::current_pool->find(trim_ws(p))) {
+	DEBUG("report.market", "Searching for value of " << args.value_at(0)
+	      << " in terms of commodity " << commodity->symbol());
 	value_t result =
 	  args.value_at(0).value(false, args.has(1) ?
 				 args.get<datetime_t>(1) :
 				 optional<datetime_t>(), *commodity);
-	if (! result.is_null())
+	if (! result.is_null()) {
+	  DEBUG("report.market", "Market value is = " << result);
 	  return result;
 	}
       }
+    }
   } else {
     value_t result =
       args.value_at(0).value(true, args.has(1) ?
@@ -621,7 +625,7 @@ expr_t::ptr_op_t report_t::lookup(const string& name)
 
   case 'm':
     if (is_eq(p, "market"))
-      return MAKE_FUNCTOR(report_t::fn_market_value);
+      return MAKE_FUNCTOR(report_t::fn_market);
     break;
 
   case 'o':

src/report.h

@@ -132,7 +132,7 @@ public:
   value_t fn_total_expr(call_scope_t& scope);
   value_t fn_display_amount(call_scope_t& scope);
   value_t fn_display_total(call_scope_t& scope);
-  value_t fn_market_value(call_scope_t& scope);
+  value_t fn_market(call_scope_t& scope);
   value_t fn_strip(call_scope_t& scope);
   value_t fn_scrub(call_scope_t& scope);
   value_t fn_quantity(call_scope_t& scope);

src/xact.cc

@@ -77,6 +77,9 @@ optional<string> xact_t::get_tag(const mask_t& tag_mask,
 
 date_t xact_t::date() const
 {
+  if (xdata_ && is_valid(xdata_->date))
+    return xdata_->date;
+
   if (item_t::use_effective_date) {
     if (_date_eff)
       return *_date_eff;

src/xact.h

@@ -189,12 +189,6 @@ public:
 
   void add_to_value(value_t& value, expr_t& expr);
 
-  date_t reported_date() const {
-    if (xdata_ && is_valid(xdata_->date))
-      return xdata_->date;
-    return date();
-  }
-
   account_t * reported_account() {
     if (xdata_)
       if (account_t * acct = xdata_->account)

test/baseline/opt-exchange.test

@@ -49,16 +49,12 @@ reg --exchange=' C, A '
     Assets:Brokerage               -155 A [2009/01/06]
 >>>1
 09-Jan-01 January 1st, 2009 (1) Assets:Brokerage              100 A        100 A
-                                Assets:Brokerage              -50 A        100 A
+                                Assets:Brokerage              -50 A         50 A
-                                                                          -200 B
+09-Jan-01 January 1st, 2009 (2) Assets:Brokerage              100 A        150 A
-09-Jan-01 January 1st, 2009 (2) Assets:Brokerage              100 A        200 A
+                                Assets:Brokerage              -75 A         75 A
-                                                                          -200 B
+09-Jan-01 January 1st, 2009 (3) Assets:Brokerage              100 A        175 A
-                                Assets:Brokerage              -75 A        200 A
+                                Assets:Brokerage             -100 A         75 A
-                                                                          -500 B
+09-Jan-02 Commodities revalued  <Revalued>                        0        600 C
-09-Jan-01 January 1st, 2009 (3) Assets:Brokerage              100 A        300 A
-                                                                          -500 B
-                                Assets:Brokerage             -100 A        300 A
-                                                                          -900 B
 09-Jan-02 January 2nd, 2009     Assets:Brokerage              500 C       1100 C
                                 Assets:Brokerage             -500 C        600 C
 09-Jan-03 January 3rd, 2009     Assets:Brokerage              600 C       1200 C
@@ -67,12 +63,16 @@ reg --exchange=' C, A '
                                 Assets:Brokerage            -2400 C        600 C
 09-Jan-05 January 5th, 2009     Assets:Brokerage             1280 C       1880 C
                                 Assets:Brokerage            -1280 C        600 C
+09-Jan-06 Commodities revalued  <Revalued>                  -2040 C      -1440 C
 09-Jan-06 January 6th, 2009     Assets:Brokerage              186 C      -1254 C
                                 Assets:Brokerage             -186 C      -1440 C
+09-Jan-07 Commodities revalued  <Revalued>                    -18 C      -1458 C
 09-Jan-07 January 7th, 2009     Assets:Brokerage              200 C      -1258 C
                                 Assets:Brokerage             -200 C      -1458 C
+09-Jan-08 Commodities revalued  <Revalued>                  -5613 C      -7071 C
 09-Jan-08 January 8th, 2009     Assets:Brokerage              200 C      -6871 C
                                 Assets:Brokerage             -200 C      -7071 C
+09-Jan-09 Commodities revalued  <Revalued>                  -2800 C      -9871 C
 09-Jan-09 January 9th, 2009     Assets:Brokerage              200 C      -9671 C
                                 Assets:Brokerage             -200 C      -9871 C
 09-Jan-10 January 10th, 2009    Assets:Brokerage              200 C      -9671 C

test/baseline/opt-market.test

@@ -50,11 +50,17 @@ P 2010/03/01 00:00:00 S 8 P
 P 2010/04/01 00:00:00 S 16 P
 >>>1
 09-Jan-01 Sample 1a             As:Brokerage:Stocks           200 P        200 P
+09-Feb-01 Commodities revalued  <Revalued>                    200 P        400 P
 09-Feb-01 Sample 2a             As:Brokerage:Stocks           400 P        800 P
+09-Mar-01 Commodities revalued  <Revalued>                    800 P       1600 P
 09-Mar-01 Sample 3a             As:Brokerage:Stocks           800 P       2400 P
+09-Apr-01 Commodities revalued  <Revalued>                   2400 P       4800 P
 09-Apr-01 Sample 4a             As:Brokerage:Stocks         -1600 P       3200 P
+10-Feb-01 Commodities revalued  <Revalued>                  -2400 P        800 P
 10-Feb-01 Sample 2b             As:Brokerage:Stocks           400 P       1200 P
+10-Mar-01 Commodities revalued  <Revalued>                   1200 P       2400 P
 10-Mar-01 Sample 3b             As:Brokerage:Stocks           800 P       3200 P
+10-Apr-01 Commodities revalued  <Revalued>                   3200 P       6400 P
 10-Apr-01 Sample 4b             As:Brokerage:Stocks         -1600 P       4800 P
 >>>2
 === 0