ledger/textual.cc

#include "textual.h"
#include "constraint.h"
#include "error.h"

#include <vector>
#include <fstream>
#include <sstream>
#include <cstring>
#include <ctime>
#include <cctype>

#define TIMELOG_SUPPORT 1

namespace ledger {

#if 0
static const std::string entry1_fmt = "%10d %p";
static const std::string entryn_fmt = "    %-30a  %15t";
#endif

#define MAX_LINE 1024

std::string             path;
unsigned int		linenum;

#ifdef TIMELOG_SUPPORT
static std::time_t	time_in;
static account_t *	last_account;
static std::string	last_desc;
#endif

static std::time_t	now	  = std::time(NULL);
static struct std::tm * now_tm	  = std::localtime(&now);

static std::time_t	base      = -1;
static int		base_year = -1;

static const int	month_days[12] = {
  31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31
};

static const char *	formats[] = {
  "%Y/%m/%d",
  "%m/%d",
  "%Y.%m.%d",
  "%m.%d",
  "%Y-%m-%d",
  "%m-%d",
  "%a",
  "%A",
  "%b",
  "%B",
  "%Y",
  NULL
};

inline char * skip_ws(char * ptr)
{
  while (*ptr == ' ' || *ptr == '\t' || *ptr == '\n')
    ptr++;
  return ptr;
}

inline char * next_element(char * buf, bool variable = false)
{
  for (char * p = buf; *p; p++) {
    if (! (*p == ' ' || *p == '\t'))
      continue;

    if (! variable) {
      *p = '\0';
      return skip_ws(p + 1);
    }
    else if (*p == '\t') {
      *p = '\0';
      return skip_ws(p + 1);
    }
    else if (*(p + 1) == ' ') {
      *p = '\0';
      return skip_ws(p + 2);
    }
  }
  return NULL;
}

bool parse_date_mask(const char * date_str, struct std::tm * result)
{
  for (const char ** f = formats; *f; f++) {
    memset(result, INT_MAX, sizeof(struct std::tm));
    if (strptime(date_str, *f, result))
      return true;
  }
  return false;
}

bool parse_date(const char * date_str, std::time_t * result, const int year)
{
  struct std::tm when;

  if (! parse_date_mask(date_str, &when))
    return false;

  when.tm_hour = 0;
  when.tm_min  = 0;
  when.tm_sec  = 0;

  if (when.tm_year == -1)
    when.tm_year = ((year == -1) ? now_tm->tm_year : (year - 1900));

  if (when.tm_mon == -1)
    when.tm_mon = 0;

  if (when.tm_mday == -1)
    when.tm_mday = 1;

  *result = std::mktime(&when);

  return true;
}

static bool quick_parse_date(char * date_str, std::time_t * result)
{
  int year = -1, month = -1, day, num = 0;

  for (char * p = date_str; *p; p++) {
    if (*p == '/' || *p == '-' || *p == '.') {
      if (year == -1)
	year = num;
      else
	month = num;
      num = 0;
    }
    else if (*p < '0' || *p > '9') {
      return false;
    }
    else {
      num *= 10;
      num += *p - '0';
    }
  }

  day = num;

  if (month == -1) {
    month = year;
    year  = -1;
  }

  if (base == -1 || year != base_year) {
    struct std::tm when;

    when.tm_hour = 0;
    when.tm_min  = 0;
    when.tm_sec  = 0;

    base_year    = year == -1 ? now_tm->tm_year + 1900 : year;
    when.tm_year = year == -1 ? now_tm->tm_year : year - 1900;
    when.tm_mon  = 0;
    when.tm_mday = 1;

    base = std::mktime(&when);
  }

  *result = base;

  --month;
  while (--month >= 0) {
    *result += month_days[month] * 24 * 60 * 60;
    if (month == 1 && year % 4 == 0 && year != 2000) // february in leap years
      *result += 24 * 60 * 60;
  }

  if (--day)
    *result += day * 24 * 60 * 60;

  return true;
}

inline char peek_next_nonws(std::istream& in)
{
  char c = in.peek();
  while (! in.eof() && std::isspace(c) && c != '\n') {
    in.get(c);
    c = in.peek();
  }
  return c;
}

transaction_t * parse_transaction_text(char * line, account_t * account,
				       entry_t * entry)
{
  // The account will be determined later...

  transaction_t * xact = new transaction_t(entry, NULL);

  // The call to `next_element' will skip past the account name,
  // and return a pointer to the beginning of the amount.  Once
  // we know where the amount is, we can strip off any
  // transaction note, and parse it.

  char * p = skip_ws(line);
  if (char * cost_str = next_element(p, true)) {
    if (char * note_str = std::strchr(cost_str, ';')) {
      *note_str++ = '\0';
      xact->note = skip_ws(note_str);
    }

    char * price_str = std::strchr(cost_str, '@');
    if (price_str) {
      *price_str++ = '\0';
      xact->cost.parse(price_str);
    }

    xact->amount.parse(cost_str);

    if (price_str)
      xact->cost *= xact->amount;
    else
      xact->cost = xact->amount;
  }

  if (*p == '[' || *p == '(') {
    xact->flags |= TRANSACTION_VIRTUAL;
    if (*p == '[')
      xact->flags |= TRANSACTION_BALANCE;
    p++;

    char * e = p + (std::strlen(p) - 1);
    assert(*e == ')' || *e == ']');
    *e = '\0';
  }

  xact->account = account->find_account(p);

  if (! xact->amount.commodity)
    xact->amount.commodity = commodity_t::find_commodity("", true);
  if (! xact->cost.commodity)
    xact->cost.commodity = commodity_t::find_commodity("", true);

  return xact;
}

transaction_t * parse_transaction(std::istream& in, account_t * account,
				  entry_t * entry)
{
  static char line[MAX_LINE + 1];
  in.getline(line, MAX_LINE);
  linenum++;

  return parse_transaction_text(line, account, entry);
}

class automated_transaction_t
{
public:
  masks_list        masks;
  transactions_list transactions;

  automated_transaction_t(masks_list& _masks,
			  transactions_list& _transactions) {
    masks.insert(masks.begin(), _masks.begin(), _masks.end());
    transactions.insert(transactions.begin(),
			_transactions.begin(), _transactions.end());
    // Take over ownership of the pointers
    _transactions.clear();
  }

  ~automated_transaction_t() {
    for (transactions_list::iterator i = transactions.begin();
	 i != transactions.end();
	 i++)
      delete *i;
  }

  void extend_entry(entry_t * entry);
};

typedef std::vector<automated_transaction_t *>
  automated_transactions_vector;

void automated_transaction_t::extend_entry(entry_t * entry)
{
  for (transactions_list::iterator i = entry->transactions.begin();
       i != entry->transactions.end();
       i++)
    if (matches(masks, *((*i)->account))) {
      for (transactions_list::iterator t = transactions.begin();
	   t != transactions.end();
	   t++) {
	amount_t amt;
	if ((*t)->amount.commodity->symbol.empty())
	  amt = (*i)->amount * (*t)->amount;
	else
	  amt = (*t)->amount;

	transaction_t * xact
	  = new transaction_t(entry, (*t)->account, amt, amt, (*t)->flags);
	entry->add_transaction(xact);
      }
    }
}

class automated_transactions_t
{
public:
  automated_transactions_vector automated_transactions;

  ~automated_transactions_t() {
    for (automated_transactions_vector::iterator i
	   = automated_transactions.begin();
	 i != automated_transactions.end();
	 i++)
      delete *i;
  }

  void extend_entry(entry_t * entry) {
    for (automated_transactions_vector::iterator i
	   = automated_transactions.begin();
	 i != automated_transactions.end();
	 i++)
      (*i)->extend_entry(entry);
  }

  void add_automated_transaction(automated_transaction_t * auto_xact) {
    automated_transactions.push_back(auto_xact);
  }
  bool remove_automated_transaction(automated_transaction_t * auto_xact) {
    for (automated_transactions_vector::iterator i
	   = automated_transactions.begin();
	 i != automated_transactions.end();
	 i++) {
      if (*i == auto_xact) {
	automated_transactions.erase(i);
	return true;
      }
    }
    return false;
  }
};

void parse_automated_transactions(std::istream& in, account_t * account,
				  automated_transactions_t& auto_xacts)
{
  static char line[MAX_LINE + 1];

  masks_list masks;

  while (! in.eof() && in.peek() == '=') {
    in.getline(line, MAX_LINE);
    linenum++;

    char * p = line + 1;
    p = skip_ws(p);

    masks.push_back(mask_t(p));
  }

  transactions_list xacts;

  while (! in.eof() && (in.peek() == ' ' || in.peek() == '\t')) {
    if (transaction_t * xact = parse_transaction(in, account, NULL)) {
      if (! xact->amount)
	throw parse_error(path, linenum,
			  "All automated transactions must have a value");
      else
	xacts.push_back(xact);
    }
  }

  if (! masks.empty() && ! xacts.empty()) {
    automated_transaction_t * auto_xact
      = new automated_transaction_t(masks, xacts);
    auto_xacts.add_automated_transaction(auto_xact);
  }
}

bool finalize_entry(entry_t * entry)
{
  // Scan through and compute the total balance for the entry.  This
  // is used for auto-calculating the value of entries with no cost,
  // and the per-unit price of unpriced commodities.

  balance_t balance;

  for (transactions_list::const_iterator x = entry->transactions.begin();
       x != entry->transactions.end();
       x++)
    if (! ((*x)->flags & TRANSACTION_VIRTUAL) ||
	((*x)->flags & TRANSACTION_BALANCE))
      balance += (*x)->cost;

  // If one transaction of a two-line transaction is of a different
  // commodity than the others, and it has no per-unit price,
  // determine its price by dividing the unit count into the value of
  // the balance.  This is done for the last eligible commodity.

  if (! balance.amounts.empty() && balance.amounts.size() == 2)
    for (transactions_list::const_iterator x = entry->transactions.begin();
	 x != entry->transactions.end();
	 x++) {
      if ((*x)->cost != (*x)->amount || ((*x)->flags & TRANSACTION_VIRTUAL))
	continue;

      for (amounts_map::const_iterator i = balance.amounts.begin();
	   i != balance.amounts.end();
	   i++)
	if ((*i).second.commodity != (*x)->amount.commodity) {
	  assert((*x)->amount);
	  balance -= (*x)->cost;
	  (*x)->cost = - (*i).second;
	  balance += (*x)->cost;
	  break;
	}

      break;
    }

  // Walk through each of the transactions, fixing up any that we
  // can, and performing any on-the-fly calculations.

  bool empty_allowed = true;

  for (transactions_list::const_iterator x = entry->transactions.begin();
       x != entry->transactions.end();
       x++) {
    if ((*x)->amount || ((*x)->flags & TRANSACTION_VIRTUAL))
      continue;

    if (! empty_allowed || balance.amounts.empty() ||
	balance.amounts.size() != 1)
      return false;

    empty_allowed = false;

    // If one transaction gives no value at all -- and all the
    // rest are of the same commodity -- then its value is the
    // inverse of the computed value of the others.

    amounts_map::const_iterator i = balance.amounts.begin();
    (*x)->amount = (*x)->cost = - balance.amount((*i).first);

    balance = 0;
  }

  return ! balance;
}

entry_t * parse_entry(std::istream& in, account_t * master)
{
  entry_t * curr = new entry_t;

  static char line[MAX_LINE + 1];
  in.getline(line, MAX_LINE);
  linenum++;

  // Parse the date

  char * next = next_element(line);

  if (! quick_parse_date(line, &curr->date))
    throw parse_error(path, linenum, "Failed to parse date");

  // Parse the optional cleared flag: *

  if (*next == '*') {
    curr->state = entry_t::CLEARED;
    next = skip_ws(++next);
  }

  // Parse the optional code: (TEXT)

  if (*next == '(') {
    if (char * p = std::strchr(next++, ')')) {
      *p++ = '\0';
      curr->code = next;
      next = skip_ws(p);
    }
  }

  // Parse the description text

  curr->payee = next;

  // Parse all of the transactions associated with this entry

  while (! in.eof() && (in.peek() == ' ' || in.peek() == '\t'))
    if (transaction_t * xact = parse_transaction(in, master, curr))
      curr->add_transaction(xact);

  // If there were no transactions, throw away the entry

  if (curr->transactions.empty() || ! finalize_entry(curr)) {
    delete curr;
    return NULL;
  }

  return curr;
}

//////////////////////////////////////////////////////////////////////
//
// Textual ledger parser
//

unsigned int parse_textual_ledger(std::istream& in, ledger_t * journal,
				  account_t * master)
{
  static char   line[MAX_LINE + 1];
  char		c;
  unsigned int  count = 0;
  unsigned int  errors = 0;
  commodity_t *	time_commodity = NULL;

  std::list<account_t *>   account_stack;
  automated_transactions_t auto_xacts;

  if (! master)
    master = journal->master;

  account_stack.push_front(master);

  path	  = journal->sources.back();
  linenum = 1;

  while (! in.eof()) {
    try {
      switch (in.peek()) {
      case -1:                    // end of file
	goto done;

      case ' ':
      case '\t':
	if (peek_next_nonws(in) != '\n') {
	  in.getline(line, MAX_LINE);
	  linenum++;
	  throw parse_error(path, linenum,
			    "Ignoring entry beginning with whitespace");
	}
	// fall through...

      case '\n':
	linenum++;
      case '\r':                  // skip blank lines
	in.get(c);
	break;

#ifdef TIMELOG_SUPPORT
      case 'i':
      case 'I': {
	std::string date, time;

	in >> c;
	in >> date;
	in >> time;
	date += " ";
	date += time;

	in.getline(line, MAX_LINE);
	linenum++;

	char * p = skip_ws(line);
	char * n = next_element(p, true);
	last_desc = n ? n : "";

	static struct std::tm when;
	if (strptime(date.c_str(), "%Y/%m/%d %H:%M:%S", &when)) {
	  time_in      = std::mktime(&when);
	  last_account = account_stack.front()->find_account(p);
	} else {
	  last_account = NULL;
	  throw parse_error(path, linenum, "Cannot parse timelog entry date");
	}
	break;
      }

      case 'o':
      case 'O':
	if (last_account) {
	  std::string date, time;

	  in >> c;
	  in >> date;
	  in >> time;

	  in.getline(line, MAX_LINE);
	  linenum++;

	  date += " ";
	  date += time;

	  static struct std::tm when;
	  if (strptime(date.c_str(), "%Y/%m/%d %H:%M:%S", &when)) {
	    entry_t * curr = new entry_t;
	    curr->date  = std::mktime(&when);
	    curr->state = entry_t::CLEARED;
	    curr->code  = "";
	    curr->payee = last_desc;

	    double diff = std::difftime(curr->date, time_in) / 60.0 / 60.0;
	    char   buf[32];
	    std::sprintf(buf, "%fh", diff);
	    amount_t amt;
	    amt.parse(buf);
	    time_commodity = amt.commodity;

	    transaction_t * xact = new transaction_t(curr, last_account, amt, amt,
						     TRANSACTION_VIRTUAL);
	    curr->add_transaction(xact);

	    if (! finalize_entry(curr) || ! journal->add_entry(curr))
	      assert(0);

	    count++;
	  } else {
	    throw parse_error(path, linenum, "Cannot parse timelog entry date");
	  }

	  last_account = NULL;
	} else {
	  in.getline(line, MAX_LINE);
	  linenum++;
	}
	break;
#endif // TIMELOG_SUPPORT

      case 'P': {			// a pricing entry
	in >> c;

	std::time_t date;
	std::string symbol;

	in >> line;		// the date
	if (! quick_parse_date(line, &date))
	  throw parse_error(path, linenum, "Failed to parse date");

	int hour, min, sec;

	in >> hour;		// the time
	in >> c;
	in >> min;
	in >> c;
	in >> sec;

	date = std::time_t(((unsigned long) date) +
			   hour * 3600 + min * 60 + sec);

	amount_t price;

	parse_commodity(in, symbol);
	in >> line;		// the price
	price.parse(line);

	commodity_t * commodity = commodity_t::find_commodity(symbol, true);
	commodity->add_price(date, price);
	break;
      }

      case 'N': {			// don't download prices
	std::string symbol;

	in >> c;
	parse_commodity(in, symbol);

	commodity_t * commodity = commodity_t::find_commodity(line, true);
	commodity->flags |= (COMMODITY_STYLE_CONSULTED |
			     COMMODITY_STYLE_NOMARKET);
	break;
      }

      case 'C': {			// a flat conversion
	in >> c;

	std::string symbol;
	amount_t    price;

	parse_commodity(in, symbol);
	in >> line;		// the price
	price.parse(line);

	commodity_t * commodity = commodity_t::find_commodity(symbol, true);
	commodity->set_conversion(price);
	break;
      }

      case 'Y':                   // set the current year
	in >> c;
	in >> now_tm->tm_year;
	now_tm->tm_year -= 1900;
	break;

#ifdef TIMELOG_SUPPORT
      case 'h':
      case 'b':
#endif
      case ';':                   // a comment line
	in.getline(line, MAX_LINE);
	linenum++;
	break;

      case '=':                   // automated transactions
	parse_automated_transactions(in, account_stack.front(), auto_xacts);
	break;

      case '@': {                 // account specific
	in >> c;
	if (in.peek() == '@') {
	  in.get(c);
	  account_stack.pop_front();
	  break;
	}

	in.getline(line, MAX_LINE);
	linenum++;

	account_t * acct = account_stack.front()->find_account(skip_ws(line));
	account_stack.push_front(acct);
	break;
      }

      case '!':                   // directive
	in >> line;
	if (std::string(line) == "!include") {
	  in.getline(line, MAX_LINE);
	  linenum++;

	  char * p = skip_ws(line);
	  std::ifstream stream(p);

	  journal->sources.push_back(p);

	  unsigned int curr_linenum = linenum;
	  std::string  curr_path    = path;

	  count += parse_textual_ledger(stream, journal, account_stack.front());

	  linenum = curr_linenum;
	  path    = curr_path;
	}
	break;

      default: {
	unsigned int first_line = linenum;
	if (entry_t * entry = parse_entry(in, account_stack.front())) {
	  if (! auto_xacts.automated_transactions.empty())
	    auto_xacts.extend_entry(entry);

	  if (journal->add_entry(entry))
	    count++;
	  else
	    throw parse_error(path, first_line, "Entry does not balance");
	} else {
	  throw parse_error(path, first_line, "Failed to parse entry");
	}
	break;
      }
      }
    }
    catch (const parse_error& err) {
      std::cerr << "Error: " << err.what() << std::endl;
      errors++;
    }
  }

 done:
  if (time_commodity) {
    time_commodity->precision = 2;
    time_commodity->flags |= (COMMODITY_STYLE_CONSULTED |
			      COMMODITY_STYLE_NOMARKET);
  }

  if (errors > 0) {
    std::ostringstream msg;
    msg << "Errors parsing file '" << path << "'";
    throw error(msg.str());
  }

  return count;
}

//////////////////////////////////////////////////////////////////////
//
// Textual ledger printing code
//

void print_transaction(std::ostream& out, transaction_t * xact,
		       bool display_amount, bool display_cost)
{
  std::ostringstream s;
  s << *(xact->account);
  std::string acct_name = s.str();

  if (xact->flags & TRANSACTION_VIRTUAL) {
    if (xact->flags & TRANSACTION_BALANCE)
      acct_name = std::string("[") + acct_name + "]";
    else
      acct_name = std::string("(") + acct_name + ")";
  }

  out.width(30);
  out.fill(' ');
  out << std::left << acct_name;

  if (xact->amount && display_amount) {
    out << "  ";
    out.width(12);
    out.fill(' ');
    std::ostringstream s;
    s << xact->amount;
    out << std::right << s.str();
  }

  if (xact->amount && display_cost &&
      xact->amount != xact->cost) {
    out << " @ ";
    out << xact->cost / xact->amount;
  }

  if (! xact->note.empty())
    out << "  ; " << xact->note;

  out << std::endl;
}

void print_textual_entry(std::ostream& out, entry_t * entry, bool shortcut)
{
  char buf[32];
  std::strftime(buf, 31, "%Y/%m/%d ", std::gmtime(&entry->date));
  out << buf;

  if (entry->state == entry_t::CLEARED)
    out << "* ";
  if (! entry->code.empty())
    out << '(' << entry->code << ") ";
  if (! entry->payee.empty())
    out << entry->payee;

  out << std::endl;

  const commodity_t * comm = NULL;
  int size = 0;

  for (transactions_list::const_iterator x
	 = entry->transactions.begin();
       x != entry->transactions.end();
       x++) {
    if ((*x)->flags & TRANSACTION_VIRTUAL &&
	! ((*x)->flags & TRANSACTION_BALANCE))
      continue;

    if (! comm)
      comm = (*x)->amount.commodity;
    else if (comm != (*x)->amount.commodity)
      shortcut = false;

    size++;
  }

  if (shortcut && size != 2)
    shortcut = false;

  for (transactions_list::const_iterator x
	 = entry->transactions.begin();
       x != entry->transactions.end();
       x++) {
    out << "    ";
    print_transaction(out, *x,
		      (! shortcut || x == entry->transactions.begin() ||
		       ((*x)->flags & TRANSACTION_VIRTUAL &&
			! ((*x)->flags & TRANSACTION_BALANCE))),
		      size != 2);
  }

  out << std::endl;
}

void print_textual_ledger(std::ostream& out, ledger_t * journal,
			  bool shortcut)
{
  for (entries_list::const_iterator i = journal->entries.begin();
       i != journal->entries.end();
       i++)
    print_textual_entry(out, *i, shortcut);
}

} // namespace ledger

#ifdef PARSE_TEST

int main(int argc, char *argv[])
{
  journal.sources.push_back(argv[1]);
  std::ifstream stream(argv[1]);
  ledger::ledger_t journal;
  int count = parse_textual_ledger(stream, &journal, journal.master);
  std::cout << "Read " << count << " entries." << std::endl;
  print_textual_ledger(std::cout, &journal, true);
}

#endif // PARSE_TEST