pricing history support

Makefile

@@ -4,7 +4,7 @@ OBJS   = $(patsubst %.cc,%.o,$(CODE))
 CXX    = g++
 CFLAGS = #-Wall -ansi -pedantic
 DFLAGS = -O3 -fomit-frame-pointer
-#DFLAGS = -g -DDEBUG=1
+DFLAGS = #-g -DDEBUG=1
 INCS   = -I/sw/include -I/usr/include/gcc/darwin/3.3/c++ -I/usr/include/gcc/darwin/3.3/c++/ppc-darwin
 LIBS   = -L/sw/lib -lgmpxx -lgmp -lpcre
 

README

@@ -819,6 +819,109 @@ If you want to show all accounts but for one account, remember to use
 ledger balance -- -equity
 </example>
 
+** File format
+
+The ledger file format is quite simple, but supports many options.
+These are summarized here.
+
+The initial character of each line determines what that line means,
+and how it should be parsed.  The possibilities are:
+
+NUMBER ::
+  A line starting with a number denotes a regular ledger entry.  It
+  may be followed by any number of lines that beginning whitespace, to
+  denote account transactions.  The format of an entry is:
+<example>
+DATE [*] [(CODE)] DESC
+  ACCOUNT  AMOUNT
+  ACCOUNT  AMOUNT
+  ...
+</example>
+
++ ::
+  If a line begins with plus, it denotes an inclusion regexp that
+  will always be considered, as if it had been specified by the user
+  at the end of their command-line.
+
+- ::
+  If a line begins with minus, it denotes an exclusion regexp that
+  will always be considered, as if it had been specified by the user
+  at the end of their command-line.
+
+<literal>=</literal> ::
+  If a line begins with equals, it denotes an automated transaction.
+  The next item on the line must be a regular expression.  Any number
+  of such lines may appear, with no intervening whitespace.
+  Following this block of lines can be a list of account transactions
+  preceded by whitespace.  The format is:
+<example>
+= REGEXP
+= REGEXP
+= REGEXP
+...
+  ACCOUNT AMOUNT
+  ACCOUNT AMOUNT
+  ...
+</example>
+
+!WORD ::
+  A line beginning with an exclamation mark denotes a command
+  directive.  It must be immediately followed by a word specifying
+  which directories.  At the moment, only =!include= is supported, for
+  including the content of other ledger files into the current one.
+
+whitespace ::
+  A line beginning with whitespace, which is not part of a regular or
+  automated transaction, is ignored.
+
+; ::
+  If a line begins with semicolon it is ignored.  This is the
+  preferred method of entering comments.
+
+Y NUM ::
+  If a line begins with a capital Y, it denotes the year to be used
+  for all subsequent entries that specify a date, whatever their type.
+  This sets the "default year", which ordinarily is the current year
+  at the time the program is run.  Useful at the beginning of a file
+  to specify the file's year.
+
+P DATE SYMBOL PRICE ::
+  Capital P specifies a historical price for a commodity.  Any such
+  number of entries are allowed.  These are usually found in a pricing
+  history file (see the =-Q= option).
+
+C SYMBOL PRICE ::
+  Capital C specifies a conversion price for a commodity.  This has
+  no reference to time, and always takes precedence over any
+  historical price (even very current prices).
+
+N SYMBOL ::
+  Capital N indicates that no implicit price conversions should be
+  obtained for the given symbol.  This means that no quotes will ever
+  be downloaded for that symbol.  Useful for a home currency, such as
+  the dollar ($).  Be aware that these pricing options will set the
+  default reporting characteristics for a commodity.  Thus it is
+  recommended that pricing options occur only after all regular ledger
+  entries have been parsed.
+
+i DATE TIME ACCOUNT  [DESC] ::
+  Lowercase (and capital) i indicate an time-in event.  This will
+  start accumulating hours in the account specified.  Usually these
+  entries are created in a timelog file by the timeclock program,
+  which is distributed with ledger.  There must be two spaces between
+  the account name, and the optional description, if one is used.
+
+o DATE TIME ACCOUNT  [DESC] ::
+  Lowercase (and capital) o indicate an time-out event.  This will
+  accumulate hours in the account specified.  Usually these entries
+  are created in a timelog file by the timeclock program, which is
+  distributed with ledger.  There must be two spaces between the
+  account name, and the optional description, if one is used.
+
+b, h ::
+  Entries beginning with lowercase b and h are ignored.  These are
+  special entries used by timeclock, but ignored by ledger.
+
 ** Command summary
 
 *** balance
@@ -982,6 +1085,13 @@ launches =vi= to let you confirm that the entry looks appropriate.
   Read in the list of patterns to include/exclude from FILE.
   Ordinarily, these are specified as arguments after the command.
 
+-L MINS ::
+  Specifies the number of minutes old that pricing data can be, before
+  the =-Q= and =-P= options will download a new quote from the
+  Internet.  =-P= only downloads the information, while =-Q= maintains
+  the information in a history file.  The default value for this
+  option is one day, or 1440 minutes.
+
 -M ::
   When used with the "register" command, causes only monthly subtotals
   to appear.  This can be useful for looking at spending patterns.
@@ -1004,11 +1114,12 @@ launches =vi= to let you confirm that the entry looks appropriate.
 
 -p ARG ::
   If a string, such as "COMM=$1.20", the commodity COMM will be
-  reported only in terms of its translated dollar value.  This can be
+  reported only in terms of the conversion factor, which supersedes
-  used to perform arbitrary value substitutions.  For example, to
+  all other pricing histories for that commodity.  This can be used to
-  report the value of your dollars in terms of the ounces of gold they
+  perform arbitrary value substitutions.  For example, to report the
-  would buy, use: -p "$=0.00280112 AU" (or whatever the current
+  value of your dollars in terms of the ounces of gold they would buy,
-  exchange rate is).
+  use: -p "$=0.00280112 AU" (or whatever the current exchange rate
+  is).
 
 -P ::
   Download current prices for all commodities by calling the script
@@ -1019,6 +1130,19 @@ launches =vi= to let you confirm that the entry looks appropriate.
   commodity has no price, nothing should be output and the exit code
   should be set to a non-zero value.
 
+-Q FILE ::
+  This option, like =-P=, downloads commodities prices from the
+  Internet as needed, by calling the script "getquote" (see above).
+  However, this option takes a string argument: the file to write the
+  downloaded pricing data to.  On future runs, this pricing data is
+  consulted to see if it's fresh enough, to avoid downloading it from
+  the Internet again.  The freshness period is given by the =-L=
+  option, specifying the maximum allowable age in minutes.  The
+  default is one day.  So, to report the current value of your
+  investments up to the day, add =-Q ~/.pricedb= to your ledger
+  command-line.  Also, it is recommended that the =-Q= option always
+  appear after all uses of =-f=.
+
 -R ::
   Ignore all virtual transactions, and report only the real balance
   for each account.
@@ -1036,6 +1160,21 @@ launches =vi= to let you confirm that the entry looks appropriate.
 -v ::
   Display the version of ledger being used.
 
+** Environment variables
+
+LEDGER ::
+  A colon-separated list of files to be parsed whenever ledger is run.
+  Easier than typing =-f= all the time.
+
+PRICE_HIST ::
+  The ledger file used to hold pricing data.  =~/.pricedb= would be a
+  good choice.
+
+PRICE_EXP ::
+  The number of minutes before pricing data becomes out-of-date.  The
+  default is one day.  Use =-L= to temporarily decrease or increase
+  the value.
+
 Footnotes:
 [1]  In some special cases, it will automatically balance the entry
      for you.

amount.cc

@@ -199,34 +199,10 @@ amount * gmp_amount::value(const amount * pr) const
   }
 }
 
-static bool get_commodity_price(commodity * comm)
-{
-  using namespace std;
-
-  char buf[256];
-  buf[0] = '\0';
-
-  if (FILE * fp = popen((std::string("getquote ") +
-			 comm->symbol).c_str(), "r")) {
-    if (feof(fp) || ! fgets(buf , 255, fp)) {
-      fclose(fp);
-      return false;
-    }
-    fclose(fp);
-  }
-
-  if (buf[0]) {
-    char * p = strchr(buf, '\n');
-    if (p) *p = '\0';
-
-    comm->price = create_amount(buf);
-    return true;
-  }
-  return false;
-}
-
 amount * gmp_amount::street(bool get_quotes) const
 {
+  static std::time_t now = std::time(NULL);
+
   amount * amt = copy();
 
   if (! amt->commdty())
@@ -234,16 +210,12 @@ amount * gmp_amount::street(bool get_quotes) const
 
   int  max = 10;
   while (--max >= 0) {
-    if (! amt->commdty()->price && ! amt->commdty()->sought) {
+    amount * price = amt->commdty()->price(&now, get_quotes);
-      if (get_quotes)
+    if (! price)
-	get_commodity_price(amt->commdty());
+      break;
-      amt->commdty()->sought = true;
-      if (! amt->commdty()->price)
-	break;
-    }
 
     amount * old = amt;
-    amt = amt->value(amt->commdty()->price);
+    amt = amt->value(price);
 
     if (amt->commdty() == old->commdty()) {
       delete old;
@@ -574,8 +546,8 @@ static commodity * parse_amount(mpz_t out, const char * num,
     commodities_map_iterator item =
       main_ledger->commodities.find(symbol.c_str());
     if (item == main_ledger->commodities.end())
-      comm = new commodity(symbol, prefix, separate,
+      comm = new commodity(symbol, prefix, separate, thousands,
-			   thousands, european, precision);
+			   european, precision);
     else
       comm = (*item).second;
   }

ledger.cc

@@ -11,8 +11,82 @@ extern int linenum;
 
 commodity::~commodity()
 {
-  if (price)
+  if (conversion)
-    delete price;
+    delete conversion;
+
+  for (price_map::iterator i = history.begin();
+       i != history.end();
+       i++)
+    delete (*i).second;
+}
+
+void commodity::set_price(amount * price, std::time_t * when)
+{
+  assert(price);
+  if (when)
+    history.insert(price_map_pair(*when, price));
+  else
+    conversion = price;
+}
+
+amount * commodity::price(std::time_t * when, bool download) const
+{
+  if (conversion)
+    return conversion;
+
+  std::time_t age;
+  amount *    price = NULL;
+  
+  for (price_map::reverse_iterator i = history.rbegin();
+       i != history.rend();
+       i++) {
+    if (*when >= (*i).first) {
+      age   = (*i).first;
+      price = (*i).second;
+      break;
+    }
+  }
+
+  extern long pricing_leeway;
+
+  if (download && ! sought &&
+      (! price || (*when - age) > pricing_leeway)) {
+    using namespace std;
+
+    // Only consult the Internet once for any commodity
+    sought = true;
+
+    char buf[256];
+    buf[0] = '\0';
+
+    std::cout << "Consulting the Internet: " << symbol << std::endl;
+    if (FILE * fp = popen((string("getquote ") + symbol).c_str(), "r")) {
+      if (feof(fp) || ! fgets(buf, 255, fp)) {
+	fclose(fp);
+	return price;
+      }
+      fclose(fp);
+    }
+
+    if (buf[0]) {
+      char * p = strchr(buf, '\n');
+      if (p) *p = '\0';
+
+      price = create_amount(buf);
+      const_cast<commodity *>(this)->set_price(price, when);
+
+      extern string price_db;
+      if (! price_db.empty()) {
+	char buf[128];
+	strftime(buf, 127, "%Y/%m/%d", localtime(when));
+	ofstream database(price_db.c_str(), ios_base::out | ios_base::app);
+	database << "P " << buf << " " << symbol << " "
+		 << price->as_str() << endl;
+      }
+    }
+  }
+
+  return price;
 }
 
 const std::string transaction::acct_as_str() const

ledger.h

@@ -35,28 +35,36 @@ class commodity
 {
   commodity(const commodity&);
 
+  typedef std::map<const std::time_t, amount *>  price_map;
+  typedef std::pair<const std::time_t, amount *> price_map_pair;
+
  public:
   std::string name;
   std::string symbol;
 
-  mutable amount * price;       // the current price
   mutable bool sought;
 
   bool prefix;
   bool separate;
   bool thousands;
   bool european;
+  int  precision;
 
-  int precision;
+ protected:
+  mutable price_map history;	// the price history
+  mutable amount *  conversion;	// fixed conversion (ignore history)
 
-  explicit commodity() : price(NULL), sought(false),
+ public:
-    prefix(false), separate(true), thousands(false), european(false) {}
+  explicit commodity() : sought(false), prefix(false), separate(true),
+    thousands(false), european(false), conversion(NULL) {}
 
   explicit commodity(const std::string& sym, bool pre = false,
 		     bool sep = true, bool thou = true,
 		     bool euro = false, int prec = 2);
-
   ~commodity();
+
+  void     set_price(amount * price, std::time_t * when = NULL);
+  amount * price(std::time_t * when = NULL, bool download = false) const;
 };
 
 typedef std::map<const std::string, commodity *>  commodities_map;
@@ -299,8 +307,8 @@ extern book * main_ledger;
 
 inline commodity::commodity(const std::string& sym, bool pre, bool sep,
 			    bool thou, bool euro, int prec)
-  : symbol(sym), price(NULL), sought(false), prefix(pre), separate(sep),
+  : symbol(sym), sought(false), prefix(pre), separate(sep),
-    thousands(thou), european(euro), precision(prec) {
+    thousands(thou), european(euro), precision(prec), conversion(NULL) {
 #ifdef DEBUG
   std::pair<commodities_map_iterator, bool> result =
 #endif

parse.cc

@@ -91,6 +91,20 @@ bool parse_date(const char * date_str, std::time_t * result, const int year)
   return true;
 }
 
+void record_price(const std::string& symbol, amount * price,
+		  std::time_t * date = NULL)
+{
+  commodity * comm = NULL;
+  commodities_map_iterator item = main_ledger->commodities.find(symbol);
+  if (item == main_ledger->commodities.end())
+    comm = new commodity(symbol);
+  else
+    comm = (*item).second;
+
+  assert(comm);
+  comm->set_price(price, date);
+}
+
 void parse_price_setting(const std::string& setting)
 {
   char buf[128];
@@ -104,16 +118,7 @@ void parse_price_setting(const std::string& setting)
     std::cerr << "Warning: Invalid price setting: " << setting << std::endl;
   } else {
     *p++ = '\0';
-
+    record_price(c, create_amount(p));
-    commodity * comm = NULL;
-    commodities_map_iterator item = main_ledger->commodities.find(c);
-    if (item == main_ledger->commodities.end())
-      comm = new commodity(c);
-    else
-      comm = (*item).second;
-
-    assert(comm);
-    comm->price = create_amount(p);
   }
 }
 
@@ -412,6 +417,56 @@ int parse_ledger(book * ledger, std::istream& in,
       break;
 #endif // TIMELOG_SUPPORT
 
+    case 'P': {			// a pricing entry
+      in >> c;
+
+      time_t      date;
+      std::string symbol;
+
+      in >> line;		// the date
+      if (! parse_date(line, &date, ledger->current_year)) {
+	std::cerr << "Error, line " << linenum
+		  << ": Failed to parse date: " << line << std::endl;
+	break;
+      }
+      in >> symbol;		// the commodity
+      in >> line;		// the price
+
+      // Add this pricing entry to the history for the given
+      // commodity.
+      record_price(symbol, create_amount(line), &date);
+      break;
+    }
+
+    case 'N': {			// don't download prices
+      in >> c;
+      in >> line;		// the symbol
+
+      commodity * comm = NULL;
+      commodities_map_iterator item = main_ledger->commodities.find(line);
+      if (item == main_ledger->commodities.end())
+	comm = new commodity(line);
+      else
+	comm = (*item).second;
+
+      assert(comm);
+      if (comm)
+	comm->sought = true;
+      break;
+    }
+
+    case 'C': {			// a flat conversion
+      in >> c;
+
+      std::string symbol;
+      in >> symbol;		// the commodity
+      in >> line;		// the price
+
+      // Add this pricing entry to the given commodity
+      record_price(symbol, create_amount(line));
+      break;
+    }
+
     case 'Y':                   // set the current year
       in >> c;
       in >> ledger->current_year;
@@ -514,16 +569,4 @@ void read_regexps(const std::string& path, regexps_list& regexps)
   }
 }
 
-void read_prices(const std::string& path)
-{
-  std::ifstream file(path.c_str());
-
-  while (! file.eof()) {
-    char buf[80];
-    file.getline(buf, 79);
-    if (*buf && ! std::isspace(*buf))
-      parse_price_setting(buf);
-  }
-}
-
 } // namespace ledger

reports.cc

@@ -1,6 +1,6 @@
 #include "ledger.h"
 
-#define LEDGER_VERSION "1.6"
+#define LEDGER_VERSION "1.7"
 
 #include <cstring>
 #include <unistd.h>
@@ -11,7 +11,6 @@ static bool cleared_only   = false;
 static bool uncleared_only = false;
 static bool cost_basis     = false;
 static bool show_virtual   = true;
-static bool get_quotes     = false;
 static bool show_children  = false;
 static bool show_sorted    = false;
 static bool show_empty     = false;
@@ -20,6 +19,10 @@ static bool full_names     = false;
 static bool print_monthly  = false;
 static bool gnuplot_safe   = false;
 
+static bool get_quotes = false;
+       long pricing_leeway = 24 * 3600;
+       std::string price_db;
+
 static amount * lower_limit = NULL;
 
 static mask * negonly_regexp = NULL;
@@ -789,11 +792,13 @@ int main(int argc, char * argv[])
 
   std::vector<std::string> files;
 
+  main_ledger = new book;
+
   // Parse the command-line options
 
   int c;
   while (-1 != (c = getopt(argc, argv,
-			   "+b:e:d:cCUhBRV:f:i:p:PvsSEnFMGl:N:"))) {
+			   "+b:e:d:cCUhBRV:f:i:p:PL:Q:vsSEnFMGl:N:"))) {
     switch (char(c)) {
     case 'b':
       have_beginning = true;
@@ -849,17 +854,25 @@ int main(int argc, char * argv[])
       break;
 
     // -p "COMMODITY=PRICE"
-    // -p path-to-price-database
     case 'p':
-      prices = optarg;
+      parse_price_setting(optarg);
       break;
 
     case 'P':
       get_quotes = true;
       break;
 
+    case 'L':
+      pricing_leeway = std::atol(optarg) * 60;
+      break;
+
+    case 'Q':
+      get_quotes = true;
+      price_db = optarg;
+      break;
+
     case 'l':
-      limit = optarg;
+      lower_limit = create_amount(optarg);
       break;
 
     case 'v':
@@ -904,12 +917,22 @@ int main(int argc, char * argv[])
     for (; index < argc; index++)
       regexps.push_back(mask(argv[index]));
 
+  // If a price history file is specified with the environment
+  // variable PRICE_HIST, add it to the list of ledger files to read.
+
+  if (price_db.empty())
+    if (char * p = std::getenv("PRICE_HIST")) {
+      get_quotes = true;
+      price_db = p;
+    }
+
+  if (char * p = std::getenv("PRICE_EXP"))
+    pricing_leeway = std::atol(p) * 60;
+
   // A ledger data file must be specified
 
   int entry_count = 0;
   
-  main_ledger = new book;
-
   if (files.empty()) {
     if (char * p = std::getenv("LEDGER")) {
       for (p = std::strtok(p, ":"); p; p = std::strtok(NULL, ":")) {
@@ -932,26 +955,16 @@ int main(int argc, char * argv[])
     }
   }
 
+  if (! price_db.empty())
+    entry_count += parse_ledger_file(main_ledger, price_db,
+				     regexps, command == "equity");
+
   if (entry_count == 0) {
     std::cerr << ("Please specify ledger file(s) using -f option "
 		  "or LEDGER environment variable.") << std::endl;
     return 1;
   }
 
-  // Record any prices specified by the user
-
-  if (! prices.empty()) {
-    if (access(prices.c_str(), R_OK) != -1)
-      read_prices(prices);
-    else
-      parse_price_setting(prices);
-  }
-
-  // Parse the lower limit, if specified
-
-  if (! limit.empty())
-      lower_limit = create_amount(limit);
-
   // Process the command
 
   if (command == "balance" || command == "bal") {