From 91e8378f04d33137f8e6281928ae70af12be3b2b Mon Sep 17 00:00:00 2001 From: John Wiegley Date: Tue, 10 Nov 2009 14:16:40 -0500 Subject: [PATCH 01/19] Fixes to Python importing; removed "hello" precommand --- python/hello.py | 7 -- python/server.py | 5 +- src/pyinterp.cc | 173 +++++++++++++++++++++++++++++++++------------- src/pyinterp.h | 50 +++----------- src/report.cc | 20 +----- src/report.h | 2 - tools/Makefile.am | 4 +- 7 files changed, 144 insertions(+), 117 deletions(-) delete mode 100644 python/hello.py diff --git a/python/hello.py b/python/hello.py deleted file mode 100644 index b5b072bb..00000000 --- a/python/hello.py +++ /dev/null @@ -1,7 +0,0 @@ -import ledger - -def precmd_hello(): - hello = ledger.Value() - hello.set_string("Well, hello yourself! This is Ledger, coming to you from Python Land.") - print hello - return hello diff --git a/python/server.py b/python/server.py index 64a91ca3..3c6ddb91 100644 --- a/python/server.py +++ b/python/server.py @@ -20,7 +20,7 @@ class LedgerHandler(BaseHTTPRequestHandler): except Exception: print "Saw exception in POST handler" -def cmd_server(): +def main(*args): try: port = 9000 server = HTTPServer(('', port), LedgerHandler) @@ -31,3 +31,6 @@ def cmd_server(): print "Shutting down server" server.socket.close() +print __name__ +if __name__ == '__main__': + main() diff --git a/src/pyinterp.cc b/src/pyinterp.cc index 6a2dea03..394739c4 100644 --- a/src/pyinterp.cc +++ b/src/pyinterp.cc @@ -99,66 +99,75 @@ void python_interpreter_t::initialize() Py_Initialize(); assert(Py_IsInitialized()); + hack_system_paths(); + object main_module = python::import("__main__"); if (! main_module) - throw_(std::logic_error, + throw_(std::runtime_error, _("Python failed to initialize (couldn't find __main__)")); main_nspace = extract(main_module.attr("__dict__")); if (! main_nspace) - throw_(std::logic_error, + throw_(std::runtime_error, _("Python failed to initialize (couldn't find __dict__)")); python::detail::init_module("ledger", &initialize_for_python); is_initialized = true; - - // Hack ledger.__path__ so it points to a real location - python::object module_sys = import("sys"); - python::object sys_dict = module_sys.attr("__dict__"); - - python::list paths(sys_dict["path"]); - - bool path_initialized = false; - int n = python::extract(paths.attr("__len__")()); - for (int i = 0; i < n; i++) { - python::extract str(paths[i]); - path pathname(str); - DEBUG("python.interp", "sys.path = " << pathname); - - if (exists(pathname / "ledger" / "__init__.py")) { - if (python::object module_ledger = import("ledger")) { - DEBUG("python.interp", - "Setting ledger.__path__ = " << (pathname / "ledger")); - - python::object ledger_dict = module_ledger.attr("__dict__"); - python::list temp_list; - temp_list.append((pathname / "ledger").string()); - - ledger_dict["__path__"] = temp_list; - } else { - throw_(std::logic_error, - _("Python failed to initialize (couldn't find ledger)")); - } - path_initialized = true; - break; - } - } -#if defined(DEBUG_ON) - if (! path_initialized) - DEBUG("python.init", - "Ledger failed to find 'ledger/__init__.py' on the PYTHONPATH"); -#endif } catch (const error_already_set&) { PyErr_Print(); - throw_(std::logic_error, _("Python failed to initialize")); + throw_(std::runtime_error, _("Python failed to initialize")); } TRACE_FINISH(python_init, 1); } -object python_interpreter_t::import(const string& str) +void python_interpreter_t::hack_system_paths() +{ + // Hack ledger.__path__ so it points to a real location + python::object sys_module = python::import("sys"); + python::object sys_dict = sys_module.attr("__dict__"); + + python::list paths(sys_dict["path"]); + +#if defined(DEBUG_ON) + bool path_initialized = false; +#endif + int n = python::extract(paths.attr("__len__")()); + for (int i = 0; i < n; i++) { + python::extract str(paths[i]); + path pathname(str); + DEBUG("python.interp", "sys.path = " << pathname); + + if (exists(pathname / "ledger" / "__init__.py")) { + if (python::object module_ledger = python::import("ledger")) { + DEBUG("python.interp", + "Setting ledger.__path__ = " << (pathname / "ledger")); + + python::object ledger_dict = module_ledger.attr("__dict__"); + python::list temp_list; + temp_list.append((pathname / "ledger").string()); + + ledger_dict["__path__"] = temp_list; + } else { + throw_(std::runtime_error, + _("Python failed to initialize (couldn't find ledger)")); + } +#if defined(DEBUG_ON) + path_initialized = true; +#endif + break; + } + } +#if defined(DEBUG_ON) + if (! path_initialized) + DEBUG("python.init", + "Ledger failed to find 'ledger/__init__.py' on the PYTHONPATH"); +#endif +} + +object python_interpreter_t::import_into_main(const string& str) { if (! is_initialized) initialize(); @@ -166,7 +175,8 @@ object python_interpreter_t::import(const string& str) try { object mod = python::import(str.c_str()); if (! mod) - throw_(std::logic_error, _("Failed to import Python module %1") << str); + throw_(std::runtime_error, + _("Failed to import Python module %1") << str); // Import all top-level entries directly into the main namespace main_nspace.update(mod.attr("__dict__")); @@ -179,6 +189,32 @@ object python_interpreter_t::import(const string& str) return object(); } +object python_interpreter_t::import_option(const string& str) +{ + path file(str); + + python::object sys_module = python::import("sys"); + python::object sys_dict = sys_module.attr("__dict__"); + + python::list paths(sys_dict["path"]); + +#if BOOST_VERSION >= 103700 + paths.insert(0, file.parent_path().string()); + sys_dict["path"] = paths; + + string name = file.filename(); + if (contains(name, ".py")) + name = file.stem(); +#else // BOOST_VERSION >= 103700 + paths.insert(0, file.branch_path().string()); + sys_dict["path"] = paths; + + string name = file.leaf(); +#endif // BOOST_VERSION >= 103700 + + return python::import(python::str(name.c_str())); +} + object python_interpreter_t::eval(std::istream& in, py_eval_mode_t mode) { bool first = true; @@ -212,7 +248,7 @@ object python_interpreter_t::eval(std::istream& in, py_eval_mode_t mode) } catch (const error_already_set&) { PyErr_Print(); - throw_(std::logic_error, _("Failed to evaluate Python code")); + throw_(std::runtime_error, _("Failed to evaluate Python code")); } return object(); } @@ -234,7 +270,7 @@ object python_interpreter_t::eval(const string& str, py_eval_mode_t mode) } catch (const error_already_set&) { PyErr_Print(); - throw_(std::logic_error, _("Failed to evaluate Python code")); + throw_(std::runtime_error, _("Failed to evaluate Python code")); } return object(); } @@ -255,7 +291,7 @@ value_t python_interpreter_t::python_command(call_scope_t& args) std::strcpy(argv[i + 1], arg.c_str()); } - int status; + int status = 1; try { status = Py_Main(static_cast(args.size()) + 1, argv); @@ -277,6 +313,44 @@ value_t python_interpreter_t::python_command(call_scope_t& args) return NULL_VALUE; } +value_t python_interpreter_t::server_command(call_scope_t& args) +{ + if (! is_initialized) + initialize(); + + python::object server_module; + + try { + server_module = python::import("ledger.server"); + if (! server_module) + throw_(std::runtime_error, + _("Could not import ledger.server; please check your PYTHONPATH")); + } + catch (const error_already_set&) { + PyErr_Print(); + throw_(std::runtime_error, + _("Could not import ledger.server; please check your PYTHONPATH")); + } + + if (python::object main_function = server_module.attr("main")) { + functor_t func(main_function, "main"); + try { + func(args); + } + catch (const error_already_set&) { + PyErr_Print(); + throw_(std::runtime_error, + _("Error while invoking ledger.server's main() function")); + } + return true; + } else { + throw_(std::runtime_error, + _("The ledger.server module is missing its main() function!")); + } + + return false; +} + option_t * python_interpreter_t::lookup_option(const char * p) { @@ -304,7 +378,7 @@ expr_t::ptr_op_t python_interpreter_t::lookup(const symbol_t::kind_t kind, DEBUG("python.interp", "Python lookup: " << name); if (python::object obj = main_nspace.get(name.c_str())) - return WRAP_FUNCTOR(functor_t(name, obj)); + return WRAP_FUNCTOR(functor_t(obj, name)); } break; @@ -320,6 +394,11 @@ expr_t::ptr_op_t python_interpreter_t::lookup(const symbol_t::kind_t kind, if (is_eq(p, "python")) return MAKE_FUNCTOR(python_interpreter_t::python_command); break; + + case 's': + if (is_eq(p, "server")) + return MAKE_FUNCTOR(python_interpreter_t::server_command); + break; } } @@ -349,7 +428,7 @@ namespace { dynamic_cast(scope)) lst.append(ptr(auto_xact)); else - throw_(std::runtime_error, + throw_(std::logic_error, _("Cannot downcast scoped object to specific type")); } else { lst.append(value); diff --git a/src/pyinterp.h b/src/pyinterp.h index 002e8af1..f2d7b760 100644 --- a/src/pyinterp.h +++ b/src/pyinterp.h @@ -57,8 +57,10 @@ public: } void initialize(); + void hack_system_paths(); - python::object import(const string& name); + python::object import_into_main(const string& name); + python::object import_option(const string& name); enum py_eval_mode_t { PY_EVAL_EXPR, @@ -67,16 +69,17 @@ public: }; python::object eval(std::istream& in, - py_eval_mode_t mode = PY_EVAL_EXPR); + py_eval_mode_t mode = PY_EVAL_EXPR); python::object eval(const string& str, - py_eval_mode_t mode = PY_EVAL_EXPR); + py_eval_mode_t mode = PY_EVAL_EXPR); python::object eval(const char * c_str, - py_eval_mode_t mode = PY_EVAL_EXPR) { + py_eval_mode_t mode = PY_EVAL_EXPR) { string str(c_str); return eval(str, mode); } value_t python_command(call_scope_t& scope); + value_t server_command(call_scope_t& args); class functor_t { functor_t(); @@ -87,9 +90,9 @@ public: public: string name; - functor_t(const string& _name, python::object _func) + functor_t(python::object _func, const string& _name) : func(_func), name(_name) { - TRACE_CTOR(functor_t, "const string&, python::object"); + TRACE_CTOR(functor_t, "python::object, const string&"); } functor_t(const functor_t& other) : func(other.func), name(other.name) { @@ -106,41 +109,10 @@ public: virtual expr_t::ptr_op_t lookup(const symbol_t::kind_t kind, const string& name); -#if BOOST_VERSION >= 103700 OPTION_(python_interpreter_t, import_, DO_(scope) { - interactive_t args(scope, "s"); - - path file(args.get(0)); - - python::object module_sys = parent->import("sys"); - python::object sys_dict = module_sys.attr("__dict__"); - - python::list paths(sys_dict["path"]); - paths.insert(0, file.parent_path().string()); - sys_dict["path"] = paths; - - string name = file.filename(); - if (contains(name, ".py")) - parent->import(file.stem()); - else - parent->import(name); + interactive_t args(scope, "ss"); + parent->import_option(args.get(1)); }); -#else // BOOST_VERSION >= 103700 - OPTION_(python_interpreter_t, import_, DO_(scope) { - interactive_t args(scope, "s"); - - path file(args.get(0)); - - python::object module_sys = parent->import("sys"); - python::object sys_dict = module_sys.attr("__dict__"); - - python::list paths(sys_dict["path"]); - paths.insert(0, file.branch_path().string()); - sys_dict["path"] = paths; - - parent->import(file.leaf()); - }); -#endif // BOOST_VERSION >= 103700 }; extern shared_ptr python_session; diff --git a/src/report.cc b/src/report.cc index e05b4bc1..0f04a212 100644 --- a/src/report.cc +++ b/src/report.cc @@ -453,15 +453,6 @@ value_t report_t::echo_command(call_scope_t& scope) return true; } -bool report_t::maybe_import(const string& module) -{ - if (lookup(symbol_t::OPTION, "import_")) { - expr_t(string("import_(\"") + module + "\")").calc(*this); - return true; - } - return false; -} - option_t * report_t::lookup_option(const char * p) { switch (*p) { @@ -930,8 +921,6 @@ expr_t::ptr_op_t report_t::lookup(const symbol_t::kind_t kind, (reporter (new format_posts(*this, report_format(HANDLER(pricesdb_format_))), *this, "#pricesdb")); - else if (is_eq(p, "python") && maybe_import("ledger.interp")) - return session.lookup(symbol_t::COMMAND, "python"); break; case 'r': @@ -947,9 +936,8 @@ expr_t::ptr_op_t report_t::lookup(const symbol_t::kind_t kind, case 's': if (is_eq(p, "stats") || is_eq(p, "stat")) return WRAP_FUNCTOR(report_statistics); - else - if (is_eq(p, "server") && maybe_import("ledger.server")) - return session.lookup(symbol_t::COMMAND, "server"); + else if (is_eq(p, "server")) + return session.lookup(symbol_t::COMMAND, "server"); break; case 'x': @@ -981,10 +969,6 @@ expr_t::ptr_op_t report_t::lookup(const symbol_t::kind_t kind, (reporter (new format_posts(*this, report_format(HANDLER(print_format_)), false), *this, "#generate")); - case 'h': - if (is_eq(p, "hello") && maybe_import("ledger.hello")) - return session.lookup(symbol_t::PRECOMMAND, "hello"); - break; case 'p': if (is_eq(p, "parse")) return WRAP_FUNCTOR(parse_command); diff --git a/src/report.h b/src/report.h index 38b2b07e..02fd2c8d 100644 --- a/src/report.h +++ b/src/report.h @@ -183,8 +183,6 @@ public: HANDLED(lots_actual)); } - bool maybe_import(const string& module); - void report_options(std::ostream& out) { HANDLER(abbrev_len_).report(out); diff --git a/tools/Makefile.am b/tools/Makefile.am index 8c042a94..8c28277b 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -235,9 +235,7 @@ ledger_la_DEPENDENCIES = $(lib_LTLIBRARIES) ledger_la_LDFLAGS = -avoid-version -module ledger_la_LIBADD = $(LIBOBJS) $(lib_LTLIBRARIES) $(INTLLIBS) -pkgpython_PYTHON = python/__init__.py \ - python/hello.py \ - python/server.py +pkgpython_PYTHON = python/__init__.py python/server.py endif From a345f9edb76ec584b500719ce221c20a81e7276d Mon Sep 17 00:00:00 2001 From: John Wiegley Date: Tue, 10 Nov 2009 14:20:00 -0500 Subject: [PATCH 02/19] Python Unicode objects are now handled --- src/py_utils.cc | 78 +++++++++++++++++++++++++++++++++++++++++------- src/system.hh.in | 1 + 2 files changed, 69 insertions(+), 10 deletions(-) diff --git a/src/py_utils.cc b/src/py_utils.cc index b2b9d0f8..364e575f 100644 --- a/src/py_utils.cc +++ b/src/py_utils.cc @@ -61,7 +61,8 @@ struct bool_from_python static void construct(PyObject* obj_ptr, converter::rvalue_from_python_stage1_data* data) { - void* storage = ((converter::rvalue_from_python_storage*) data)->storage.bytes; + void * storage = + ((converter::rvalue_from_python_storage*) data)->storage.bytes; if (obj_ptr == Py_True) new (storage) bool(true); else @@ -92,23 +93,72 @@ struct string_from_python return obj_ptr; } - static void construct(PyObject* obj_ptr, converter::rvalue_from_python_stage1_data* data) + static void construct(PyObject* obj_ptr, + converter::rvalue_from_python_stage1_data* data) { const char* value = PyString_AsString(obj_ptr); if (value == 0) throw_error_already_set(); void* storage = - reinterpret_cast *>(data)->storage.bytes; + reinterpret_cast *> + (data)->storage.bytes; new (storage) ledger::string(value); data->convertible = storage; } }; -typedef register_python_conversion +typedef register_python_conversion string_python_conversion; #endif // STRING_VERIFY_ON +struct unicode_to_python +{ + static PyObject* convert(const std::string& utf8str) + { + PyObject * pstr = PyString_FromString(utf8str.c_str()); + PyObject * uni = PyUnicode_FromEncodedObject(pstr, "UTF-8", NULL); + return object(handle<>(borrowed(uni))).ptr(); + } +}; + +struct unicode_from_python +{ + static void* convertible(PyObject* obj_ptr) + { + if (!PyUnicode_Check(obj_ptr)) return 0; + return obj_ptr; + } + + static void construct(PyObject* obj_ptr, + converter::rvalue_from_python_stage1_data* data) + { + Py_ssize_t size = PyUnicode_GET_SIZE(obj_ptr); + const Py_UNICODE* value = PyUnicode_AS_UNICODE(obj_ptr); + + std::string str; + if (sizeof(Py_UNICODE) == 2) // UTF-16 + utf8::unchecked::utf16to8(value, value + size, std::back_inserter(str)); + else if (sizeof(Py_UNICODE) == 4) // UTF-32 + utf8::unchecked::utf32to8(value, value + size, std::back_inserter(str)); + else + assert(! "Py_UNICODE has an unexpected size"); + + if (value == 0) throw_error_already_set(); + void* storage = + reinterpret_cast *> + (data)->storage.bytes; + new (storage) std::string(str); + data->convertible = storage; + } +}; + +typedef register_python_conversion + unicode_python_conversion; + + struct istream_to_python { static PyObject* convert(const std::istream&) @@ -125,16 +175,19 @@ struct istream_from_python return obj_ptr; } - static void construct(PyObject* obj_ptr, converter::rvalue_from_python_stage1_data* data) + static void construct(PyObject* obj_ptr, + converter::rvalue_from_python_stage1_data* data) { void* storage = - reinterpret_cast *>(data)->storage.bytes; + reinterpret_cast *> + (data)->storage.bytes; new (storage) pyifstream(reinterpret_cast(obj_ptr)); data->convertible = storage; } }; -typedef register_python_conversion +typedef register_python_conversion istream_python_conversion; @@ -154,15 +207,19 @@ struct ostream_from_python return obj_ptr; } - static void construct(PyObject* obj_ptr, converter::rvalue_from_python_stage1_data* data) + static void construct(PyObject* obj_ptr, + converter::rvalue_from_python_stage1_data* data) { - void* storage = reinterpret_cast *>(data)->storage.bytes; + void* storage = + reinterpret_cast *> + (data)->storage.bytes; new (storage) pyofstream(reinterpret_cast(obj_ptr)); data->convertible = storage; } }; -typedef register_python_conversion +typedef register_python_conversion ostream_python_conversion; @@ -219,6 +276,7 @@ void export_utils() #if defined(STRING_VERIFY_ON) string_python_conversion(); #endif + unicode_python_conversion(); istream_python_conversion(); ostream_python_conversion(); } diff --git a/src/system.hh.in b/src/system.hh.in index 12f257eb..abb823dc 100644 --- a/src/system.hh.in +++ b/src/system.hh.in @@ -246,6 +246,7 @@ void serialize(Archive& ar, istream_pos_type& pos, const unsigned int) #include #include +#include #include From 7cd37b1d50b5d6a0f6c771b18c05471197670bb9 Mon Sep 17 00:00:00 2001 From: John Wiegley Date: Tue, 10 Nov 2009 18:43:38 -0500 Subject: [PATCH 03/19] Moving the #include of unistring.h into format.h --- src/format.cc | 1 - src/format.h | 1 + src/post.cc | 1 - src/report.cc | 1 - src/value.cc | 2 +- 5 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/format.cc b/src/format.cc index b93a42a4..e910ce3b 100644 --- a/src/format.cc +++ b/src/format.cc @@ -33,7 +33,6 @@ #include "format.h" #include "scope.h" -#include "unistring.h" #include "pstream.h" namespace ledger { diff --git a/src/format.h b/src/format.h index b3ae464f..fc1272aa 100644 --- a/src/format.h +++ b/src/format.h @@ -43,6 +43,7 @@ #define _FORMAT_H #include "expr.h" +#include "unistring.h" namespace ledger { diff --git a/src/post.cc b/src/post.cc index 7dd0c9b2..7c27b6c4 100644 --- a/src/post.cc +++ b/src/post.cc @@ -36,7 +36,6 @@ #include "account.h" #include "journal.h" #include "interactive.h" -#include "unistring.h" #include "format.h" namespace ledger { diff --git a/src/report.cc b/src/report.cc index 0f04a212..4e9cfe89 100644 --- a/src/report.cc +++ b/src/report.cc @@ -33,7 +33,6 @@ #include "report.h" #include "session.h" -#include "unistring.h" #include "format.h" #include "query.h" #include "output.h" diff --git a/src/value.cc b/src/value.cc index b4060a1c..021bf957 100644 --- a/src/value.cc +++ b/src/value.cc @@ -35,7 +35,7 @@ #include "commodity.h" #include "annotate.h" #include "pool.h" -#include "unistring.h" +#include "unistring.h" // for justify() namespace ledger { From 0ac2dc28104253faf82a3ed3d12bb16a1a150067 Mon Sep 17 00:00:00 2001 From: John Wiegley Date: Tue, 10 Nov 2009 18:43:53 -0500 Subject: [PATCH 04/19] Fixed an erroneous use of operator>> in textual.cc --- src/textual.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/textual.cc b/src/textual.cc index 1d0d7998..bd7333d2 100644 --- a/src/textual.cc +++ b/src/textual.cc @@ -628,7 +628,7 @@ void instance_t::include_directive(char * line) if (! exists(filename)) throw_(std::runtime_error, - _("File to include was not found: '%1'" << filename)); + _("File to include was not found: '%1'") << filename); ifstream stream(filename); From ac885a907525589aa56266d9a2527cdc7127c9cb Mon Sep 17 00:00:00 2001 From: John Wiegley Date: Tue, 10 Nov 2009 18:44:08 -0500 Subject: [PATCH 05/19] All strings passed to Python are now Unicode objects --- src/py_utils.cc | 105 +++++++++++++++++++----------------------------- src/pyutils.h | 51 ++++++++++++++++++++++- src/utils.cc | 24 ++++++----- src/utils.h | 54 ++++++++++++------------- test/convert.py | 1 + 5 files changed, 132 insertions(+), 103 deletions(-) diff --git a/src/py_utils.cc b/src/py_utils.cc index 364e575f..2736ed3e 100644 --- a/src/py_utils.cc +++ b/src/py_utils.cc @@ -75,13 +75,19 @@ typedef register_python_conversion bool_python_conversion; -#if defined(STRING_VERIFY_ON) - struct string_to_python { - static PyObject* convert(const ledger::string& str) + static PyObject* convert(const string& str) { +#if 1 + // Return a Unicode object + PyObject * pstr = PyString_FromString(str.c_str()); + PyObject * uni = PyUnicode_FromEncodedObject(pstr, "UTF-8", NULL); + return object(handle<>(borrowed(uni))).ptr(); +#else + // Return a 7-bit ASCII string return incref(object(static_cast(str)).ptr()); +#endif } }; @@ -89,75 +95,49 @@ struct string_from_python { static void* convertible(PyObject* obj_ptr) { - if (!PyString_Check(obj_ptr)) return 0; + if (!PyUnicode_Check(obj_ptr) && + !PyString_Check(obj_ptr)) return 0; return obj_ptr; } static void construct(PyObject* obj_ptr, converter::rvalue_from_python_stage1_data* data) { - const char* value = PyString_AsString(obj_ptr); - if (value == 0) throw_error_already_set(); - void* storage = - reinterpret_cast *> - (data)->storage.bytes; - new (storage) ledger::string(value); - data->convertible = storage; + if (PyString_Check(obj_ptr)) { + const char* value = PyString_AsString(obj_ptr); + if (value == 0) throw_error_already_set(); + void* storage = + reinterpret_cast *> + (data)->storage.bytes; + new (storage) string(value); + data->convertible = storage; + } else { + VERIFY(PyUnicode_Check(obj_ptr)); + + Py_ssize_t size = PyUnicode_GET_SIZE(obj_ptr); + const Py_UNICODE* value = PyUnicode_AS_UNICODE(obj_ptr); + + string str; + if (sizeof(Py_UNICODE) == 2) // UTF-16 + utf8::unchecked::utf16to8(value, value + size, std::back_inserter(str)); + else if (sizeof(Py_UNICODE) == 4) // UTF-32 + utf8::unchecked::utf32to8(value, value + size, std::back_inserter(str)); + else + assert(! "Py_UNICODE has an unexpected size"); + + if (value == 0) throw_error_already_set(); + void* storage = + reinterpret_cast *> + (data)->storage.bytes; + new (storage) string(str); + data->convertible = storage; + } } }; -typedef register_python_conversion +typedef register_python_conversion string_python_conversion; -#endif // STRING_VERIFY_ON - - -struct unicode_to_python -{ - static PyObject* convert(const std::string& utf8str) - { - PyObject * pstr = PyString_FromString(utf8str.c_str()); - PyObject * uni = PyUnicode_FromEncodedObject(pstr, "UTF-8", NULL); - return object(handle<>(borrowed(uni))).ptr(); - } -}; - -struct unicode_from_python -{ - static void* convertible(PyObject* obj_ptr) - { - if (!PyUnicode_Check(obj_ptr)) return 0; - return obj_ptr; - } - - static void construct(PyObject* obj_ptr, - converter::rvalue_from_python_stage1_data* data) - { - Py_ssize_t size = PyUnicode_GET_SIZE(obj_ptr); - const Py_UNICODE* value = PyUnicode_AS_UNICODE(obj_ptr); - - std::string str; - if (sizeof(Py_UNICODE) == 2) // UTF-16 - utf8::unchecked::utf16to8(value, value + size, std::back_inserter(str)); - else if (sizeof(Py_UNICODE) == 4) // UTF-32 - utf8::unchecked::utf32to8(value, value + size, std::back_inserter(str)); - else - assert(! "Py_UNICODE has an unexpected size"); - - if (value == 0) throw_error_already_set(); - void* storage = - reinterpret_cast *> - (data)->storage.bytes; - new (storage) std::string(str); - data->convertible = storage; - } -}; - -typedef register_python_conversion - unicode_python_conversion; - struct istream_to_python { @@ -273,10 +253,7 @@ void export_utils() ; bool_python_conversion(); -#if defined(STRING_VERIFY_ON) string_python_conversion(); -#endif - unicode_python_conversion(); istream_python_conversion(); ostream_python_conversion(); } diff --git a/src/pyutils.h b/src/pyutils.h index 5709eb35..a9e968e0 100644 --- a/src/pyutils.h +++ b/src/pyutils.h @@ -36,7 +36,7 @@ template struct object_from_python { object_from_python() { - boost::python::converter::registry::push_back + boost::python::converter::registry::insert (&TfromPy::convertible, &TfromPy::construct, boost::python::type_id()); } @@ -106,6 +106,55 @@ struct register_optional_to_python : public boost::noncopyable } }; +namespace boost { namespace python { + +// Use expr to create the PyObject corresponding to x +# define BOOST_PYTHON_RETURN_TO_PYTHON_BY_VALUE(T, expr, pytype)\ + template <> struct to_python_value \ + : detail::builtin_to_python \ + { \ + inline PyObject* operator()(T const& x) const \ + { \ + return (expr); \ + } \ + inline PyTypeObject const* get_pytype() const \ + { \ + return (pytype); \ + } \ + }; \ + template <> struct to_python_value \ + : detail::builtin_to_python \ + { \ + inline PyObject* operator()(T const& x) const \ + { \ + return (expr); \ + } \ + inline PyTypeObject const* get_pytype() const \ + { \ + return (pytype); \ + } \ + }; + +# define BOOST_PYTHON_ARG_TO_PYTHON_BY_VALUE(T, expr) \ + namespace converter \ + { \ + template <> struct arg_to_python< T > \ + : handle<> \ + { \ + arg_to_python(T const& x) \ + : python::handle<>(expr) {} \ + }; \ + } + +// Specialize argument and return value converters for T using expr +# define BOOST_PYTHON_TO_PYTHON_BY_VALUE(T, expr, pytype) \ + BOOST_PYTHON_RETURN_TO_PYTHON_BY_VALUE(T,expr, pytype) \ + BOOST_PYTHON_ARG_TO_PYTHON_BY_VALUE(T,expr) + +BOOST_PYTHON_TO_PYTHON_BY_VALUE(ledger::string, ::PyUnicode_FromEncodedObject(::PyString_FromString(x.c_str()), "UTF-8", NULL), &PyUnicode_Type) + +} } // namespace boost::python + //boost::python::register_ptr_to_python< boost::shared_ptr >(); #endif // _PY_UTILS_H diff --git a/src/utils.cc b/src/utils.cc index 2f2899fb..6cef1a8c 100644 --- a/src/utils.cc +++ b/src/utils.cc @@ -406,8 +406,16 @@ void report_memory(std::ostream& out, bool report_all) } } +} // namespace ledger -#if defined(STRING_VERIFY_ON) +#endif // VERIFY_ON + +/********************************************************************** + * + * String wrapper + */ + +namespace ledger { string::string() : std::string() { TRACE_CTOR(string, ""); @@ -445,18 +453,10 @@ string::~string() throw() { TRACE_DTOR(string); } -#endif // STRING_VERIFY_ON +string empty_string(""); -} // namespace ledger - -#endif // VERIFY_ON - -ledger::string empty_string(""); - -ledger::strings_list split_arguments(const char * line) +strings_list split_arguments(const char * line) { - using namespace ledger; - strings_list args; char buf[4096]; @@ -506,6 +506,8 @@ ledger::strings_list split_arguments(const char * line) return args; } +} // namespace ledger + /********************************************************************** * * Logging diff --git a/src/utils.h b/src/utils.h index bfdee0b2..8ddc3c44 100644 --- a/src/utils.h +++ b/src/utils.h @@ -62,10 +62,6 @@ #define TIMERS_ON 1 #endif -#if defined(VERIFY_ON) -//#define STRING_VERIFY_ON 1 -#endif - /*@}*/ /** @@ -76,11 +72,7 @@ namespace ledger { using namespace boost; -#if defined(STRING_VERIFY_ON) class string; -#else - typedef std::string string; -#endif typedef std::list strings_list; @@ -162,12 +154,33 @@ void trace_dtor_func(void * ptr, const char * cls_name, std::size_t cls_size); void report_memory(std::ostream& out, bool report_all = false); -#if defined(STRING_VERIFY_ON) +} // namespace ledger + +#else // ! VERIFY_ON + +#define VERIFY(x) +#define DO_VERIFY() true +#define TRACE_CTOR(cls, args) +#define TRACE_DTOR(cls) + +#endif // VERIFY_ON + +#define IF_VERIFY() if (DO_VERIFY()) + +/*@}*/ /** - * This string type is a wrapper around std::string that allows us to - * trace constructor and destructor calls. + * @name String wrapper + * + * This string type is a wrapper around std::string that allows us to trace + * constructor and destructor calls. It also makes ledger's use of strings a + * unique type, that the Boost.Python code can use as the basis for + * transparent Unicode conversions. */ +/*@{*/ + +namespace ledger { + class string : public std::string { public: @@ -240,25 +253,12 @@ inline bool operator!=(const char* __lhs, const string& __rhs) inline bool operator!=(const string& __lhs, const char* __rhs) { return __lhs.compare(__rhs) != 0; } -#endif // STRING_VERIFY_ON +extern string empty_string; + +strings_list split_arguments(const char * line); } // namespace ledger -#else // ! VERIFY_ON - -#define VERIFY(x) -#define DO_VERIFY() true -#define TRACE_CTOR(cls, args) -#define TRACE_DTOR(cls) - -#endif // VERIFY_ON - -extern ledger::string empty_string; - -ledger::strings_list split_arguments(const char * line); - -#define IF_VERIFY() if (DO_VERIFY()) - /*@}*/ /** diff --git a/test/convert.py b/test/convert.py index d61da790..0c64fde4 100755 --- a/test/convert.py +++ b/test/convert.py @@ -150,6 +150,7 @@ for line in fd.readlines(): line = re.sub('set_session_context\(\)', 'set_session_context()\n self.testSession = None', line) line = re.sub('([a-z_]+?)_t\b', '\\1', line) + line = re.sub('("[^"]+")', 'u\\1', line) line = re.sub('std::string\(([^)]+?)\)', '\\1', line) line = re.sub('string\(([^)]+?)\)', '\\1', line) line = re.sub('\.print\(([^)]+?)\)', '.print_(\\1)', line) From 048845184a91daaf6d23c09d43367b6d128fdf2b Mon Sep 17 00:00:00 2001 From: John Wiegley Date: Tue, 10 Nov 2009 18:58:32 -0500 Subject: [PATCH 06/19] Added a --no-git flag to acprep --- acprep | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/acprep b/acprep index 055d2306..245f16a1 100755 --- a/acprep +++ b/acprep @@ -255,6 +255,9 @@ class PrepareBuild(CommandLineApp): op.add_option('', '--no-patch', action='store_true', dest='no_patch', default=False, help='Do not patch the Makefile for prettier output') + op.add_option('', '--no-git', action='store_true', dest='no_git', + default=False, + help='Do not call out to Git; useful for offline builds') op.add_option('', '--output', metavar='DIR', action="callback", callback=self.option_output, help='Build in the specified directory') @@ -382,9 +385,12 @@ class PrepareBuild(CommandLineApp): sys.exit(1) return dirname + def git_working_tree(self): + return exists('.git') and isdir('.git') and not self.options.no_git + def current_version(self): if not self.current_ver: - if exists('.git') and isdir('.git'): + if self.git_working_tree(): #date = self.get_stdout('git', 'log', '--format=%ci', '-1', 'HEAD') #date = re.sub(" [-+][0-9][0-9][0-9][0-9]$", "", date) #when = datetime.datetime.strptime(date, "%Y-%m-%d %H:%M:%S") @@ -394,7 +400,7 @@ class PrepareBuild(CommandLineApp): tag = self.get_stdout('git', 'describe', '--all', '--long') self.current_ver = re.sub('heads/', '', tag) else: - self.current_ver = "unknown" + self.current_ver = "no-git" return self.current_ver def need_to_prepare_autotools(self): @@ -522,17 +528,15 @@ class PrepareBuild(CommandLineApp): def phase_submodule(self, *args): self.log.info('Executing phase: submodule') - if exists('.git') and isdir('.git'): + if self.git_working_tree(): self.execute('git', 'submodule', 'init') self.execute('git', 'submodule', 'update') def phase_pull(self, *args): self.log.info('Executing phase: pull') - if not exists('.git') and not isdir('.git'): - self.log.error("This is not a Git clone.") - sys.exit(1) - self.execute('git', 'pull') - self.phase_submodule() + if self.git_working_tree(): + self.execute('git', 'pull') + self.phase_submodule() ######################################################################### # Automatic installation of build dependencies # @@ -653,7 +657,7 @@ class PrepareBuild(CommandLineApp): self.log.error('Failed to locate the Boost sources') sys.exit(1) - if not exists('cppunit'): + if not exists('cppunit') and self.git_working_tree(): self.execute('git', 'clone', 'git://github.com/jwiegley/cppunit.git') if not exists('cppunit'): @@ -1239,7 +1243,8 @@ class PrepareBuild(CommandLineApp): def phase_gitclean(self, *args): self.log.info('Executing phase: gitclean') - self.execute('git', 'clean', '-dfx') + if self.git_working_tree(): + self.execute('git', 'clean', '-dfx') ######################################################################### # Packaging phases # From 5bb376b3f9d0d103f7d7d5568a4985ea99df2e5c Mon Sep 17 00:00:00 2001 From: John Wiegley Date: Tue, 10 Nov 2009 20:52:36 -0500 Subject: [PATCH 07/19] Added implicit Python conversion of time_duration_t --- src/py_times.cc | 98 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 86 insertions(+), 12 deletions(-) diff --git a/src/py_times.cc b/src/py_times.cc index e140a23c..a62bb9b6 100644 --- a/src/py_times.cc +++ b/src/py_times.cc @@ -33,6 +33,7 @@ #include "pyinterp.h" #include "pyutils.h" +#include "times.h" // jww (2007-05-04): Convert time duration objects to PyDelta @@ -40,8 +41,6 @@ namespace ledger { using namespace boost::python; -typedef boost::gregorian::date date; - #define MY_PyDateTime_IMPORT \ PyDateTimeAPI = (PyDateTime_CAPI*) \ PyCObject_Import(const_cast("datetime"), \ @@ -49,7 +48,7 @@ typedef boost::gregorian::date date; struct date_to_python { - static PyObject* convert(const date& dte) + static PyObject* convert(const date_t& dte) { MY_PyDateTime_IMPORT; return PyDate_FromDate(dte.year(), dte.month(), dte.day()); @@ -77,13 +76,13 @@ struct date_from_python date::day_type d = static_cast(PyDateTime_GET_DAY(obj_ptr)); - date * dte = new date(y, m, d); + date_t * dte = new date_t(y, m, d); data->convertible = (void *) dte; } }; -typedef register_python_conversion +typedef register_python_conversion date_python_conversion; @@ -93,7 +92,7 @@ struct datetime_to_python { MY_PyDateTime_IMPORT; - date dte = moment.date(); + date_t dte = moment.date(); datetime_t::time_duration_type tod = moment.time_of_day(); return PyDateTime_FromDateAndTime @@ -127,28 +126,102 @@ struct datetime_from_python datetime_t::time_duration_type::hour_type h = static_cast - (PyDateTime_DATE_GET_HOUR(obj_ptr)); + (PyDateTime_DATE_GET_HOUR(obj_ptr)); datetime_t::time_duration_type::min_type min = static_cast - (PyDateTime_DATE_GET_MINUTE(obj_ptr)); + (PyDateTime_DATE_GET_MINUTE(obj_ptr)); datetime_t::time_duration_type::sec_type s = static_cast - (PyDateTime_DATE_GET_SECOND(obj_ptr)); + (PyDateTime_DATE_GET_SECOND(obj_ptr)); datetime_t::time_duration_type::fractional_seconds_type ms = static_cast - (PyDateTime_DATE_GET_MICROSECOND(obj_ptr)) * 1000000; + (PyDateTime_DATE_GET_MICROSECOND(obj_ptr)) * 1000000; datetime_t * moment - = new datetime_t(date(y, m, d), + = new datetime_t(date_t(y, m, d), datetime_t::time_duration_type(h, min, s, ms)); data->convertible = (void *) moment; } }; -typedef register_python_conversion +typedef register_python_conversion datetime_python_conversion; + +/* Convert time_duration to/from python */ +struct duration_to_python +{ + static int get_usecs(boost::posix_time::time_duration const& d) + { + static int64_t resolution = + boost::posix_time::time_duration::ticks_per_second(); + int64_t fracsecs = d.fractional_seconds(); + if (resolution > 1000000) + return static_cast(fracsecs / (resolution / 1000000)); + else + return static_cast(fracsecs * (1000000 / resolution)); + } + + static PyObject * convert(posix_time::time_duration d) + { + int days = d.hours() / 24; + if (days < 0) + days --; + int seconds = d.total_seconds() - days*(24*3600); + int usecs = get_usecs(d); + if (days < 0) + usecs = 1000000-1 - usecs; + return PyDelta_FromDSU(days, seconds, usecs); + } +}; + +/* Should support the negative values, but not the special boost time + durations */ +struct duration_from_python +{ + static void* convertible(PyObject * obj_ptr) + { + if ( ! PyDelta_Check(obj_ptr)) + return 0; + return obj_ptr; + } + + static void construct(PyObject* obj_ptr, + python::converter::rvalue_from_python_stage1_data * data) + { + PyDateTime_Delta const* pydelta + = reinterpret_cast(obj_ptr); + + int days = pydelta->days; + bool is_negative = (days < 0); + if (is_negative) + days = -days; + + // Create time duration object + posix_time::time_duration + duration = (posix_time::hours(24) * days + + posix_time::seconds(pydelta->seconds) + + posix_time::microseconds(pydelta->microseconds)); + if (is_negative) + duration = duration.invert_sign(); + + void * storage = + reinterpret_cast *> + (data)->storage.bytes; + + new (storage) posix_time::time_duration(duration); + data->convertible = storage; + } +}; + +typedef register_python_conversion + duration_python_conversion; + + datetime_t py_parse_datetime(const string& str) { return parse_datetime(str); } @@ -161,6 +234,7 @@ void export_times() { datetime_python_conversion(); date_python_conversion(); + duration_python_conversion(); register_optional_to_python(); register_optional_to_python(); From 7d15b1ed5a4d393e48814c4ecbdc3be7ba90b745 Mon Sep 17 00:00:00 2001 From: John Wiegley Date: Tue, 10 Nov 2009 20:56:27 -0500 Subject: [PATCH 08/19] Fixed automated posts not appearing in bal reports --- src/xact.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/xact.cc b/src/xact.cc index 561170bd..d94464a6 100644 --- a/src/xact.cc +++ b/src/xact.cc @@ -542,6 +542,7 @@ void auto_xact_t::extend_xact(xact_base_t& xact, bool post_handler) new_post->add_flags(ITEM_GENERATED); xact.add_post(new_post); + new_post->account->add_post(new_post); } } } From acb69193d8b1edceb1b53725306a1533dbfcb15c Mon Sep 17 00:00:00 2001 From: John Wiegley Date: Tue, 10 Nov 2009 20:57:08 -0500 Subject: [PATCH 09/19] Added a few missing explicit instantiations --- src/amount.cc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/amount.cc b/src/amount.cc index 77a5f8e3..28fa9eaf 100644 --- a/src/amount.cc +++ b/src/amount.cc @@ -1238,17 +1238,24 @@ void serialize(Archive& ar, long unsigned int& integer, BOOST_CLASS_EXPORT(ledger::annotated_commodity_t) +template void boost::serialization::serialize(boost::archive::binary_iarchive&, + MP_INT&, const unsigned int); template void boost::serialization::serialize(boost::archive::binary_oarchive&, MP_INT&, const unsigned int); template void boost::serialization::serialize(boost::archive::binary_iarchive&, MP_RAT&, const unsigned int); +template void boost::serialization::serialize(boost::archive::binary_oarchive&, + MP_RAT&, const unsigned int); template void boost::serialization::serialize(boost::archive::binary_iarchive&, long unsigned int&, const unsigned int); +template void boost::serialization::serialize(boost::archive::binary_oarchive&, + long unsigned int&, + const unsigned int); -template void ledger::amount_t::serialize(boost::archive::binary_oarchive&, - const unsigned int); template void ledger::amount_t::serialize(boost::archive::binary_iarchive&, const unsigned int); +template void ledger::amount_t::serialize(boost::archive::binary_oarchive&, + const unsigned int); #endif // HAVE_BOOST_SERIALIZATION From b62b03335f33a6a0ae422605b8b6271add849aa6 Mon Sep 17 00:00:00 2001 From: John Wiegley Date: Wed, 11 Nov 2009 01:16:42 -0500 Subject: [PATCH 10/19] Removed "account" as a report query keyword --- src/query.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/query.cc b/src/query.cc index e48e65b5..de1c5631 100644 --- a/src/query.cc +++ b/src/query.cc @@ -155,8 +155,6 @@ query_t::lexer_t::token_t query_t::lexer_t::next_token() return token_t(token_t::TOK_PAYEE); else if (ident == "note") return token_t(token_t::TOK_NOTE); - else if (ident == "account") - return token_t(token_t::TOK_ACCOUNT); else if (ident == "tag") return token_t(token_t::TOK_META); else if (ident == "meta") From eb772893b00119ead26d8662d73460651cafe11d Mon Sep 17 00:00:00 2001 From: John Wiegley Date: Wed, 11 Nov 2009 01:17:29 -0500 Subject: [PATCH 11/19] Timeclock entries can now have notes Example of a tagged entry: i 2009/11/01 12:00:00 Account Payee ; :Foo: o 2009/11/01 13:00:00 Two spaces or a tab must separate account from payee, and payee from note. --- src/textual.cc | 22 ++++++++++++++++++---- src/timelog.cc | 23 +++++++++++++++++------ src/timelog.h | 20 ++++++++++++-------- 3 files changed, 47 insertions(+), 18 deletions(-) diff --git a/src/textual.cc b/src/textual.cc index bd7333d2..844edad3 100644 --- a/src/textual.cc +++ b/src/textual.cc @@ -413,11 +413,18 @@ void instance_t::clock_in_directive(char * line, { string datetime(line, 2, 19); - char * p = skip_ws(line + 22); - char * n = next_element(p, true); + char * p = skip_ws(line + 22); + char * n = next_element(p, true); + char * end = n ? next_element(n, true) : NULL; + + if (end && *end == ';') + end = skip_ws(end + 1); + else + end = NULL; timelog.clock_in(parse_datetime(datetime, current_year), - account_stack.front()->find_account(p), n ? n : ""); + account_stack.front()->find_account(p), + n ? n : "", end ? end : ""); } void instance_t::clock_out_directive(char * line, @@ -427,9 +434,16 @@ void instance_t::clock_out_directive(char * line, char * p = skip_ws(line + 22); char * n = next_element(p, true); + char * end = n ? next_element(n, true) : NULL; + + if (end && *end == ';') + end = skip_ws(end + 1); + else + end = NULL; timelog.clock_out(parse_datetime(datetime, current_year), - p ? account_stack.front()->find_account(p) : NULL, n ? n : ""); + p ? account_stack.front()->find_account(p) : NULL, + n ? n : "", end ? end : ""); count++; } diff --git a/src/timelog.cc b/src/timelog.cc index f7b79b9a..dc3d3496 100644 --- a/src/timelog.cc +++ b/src/timelog.cc @@ -44,6 +44,7 @@ namespace { const datetime_t& when, account_t * account, const char * desc, + const char * note, journal_t& journal) { time_xact_t event; @@ -82,11 +83,19 @@ namespace { desc = NULL; } + if (note && event.note.empty()) { + event.note = note; + note = NULL; + } + std::auto_ptr curr(new xact_t); curr->_date = when.date(); curr->code = desc ? desc : ""; curr->payee = event.desc; + if (! event.note.empty()) + curr->append_note(event.note.c_str()); + if (when < event.checkin) throw parse_error (_("Timelog check-out date less than corresponding check-in")); @@ -119,8 +128,8 @@ time_log_t::~time_log_t() accounts.push_back(time_xact.account); foreach (account_t * account, accounts) - clock_out_from_timelog(time_xacts, CURRENT_TIME(), account, NULL, - journal); + clock_out_from_timelog(time_xacts, CURRENT_TIME(), account, + NULL, NULL, journal); assert(time_xacts.empty()); } @@ -128,9 +137,10 @@ time_log_t::~time_log_t() void time_log_t::clock_in(const datetime_t& checkin, account_t * account, - const string& desc) + const string& desc, + const string& note) { - time_xact_t event(checkin, account, desc); + time_xact_t event(checkin, account, desc, note); if (! time_xacts.empty()) { foreach (time_xact_t& time_xact, time_xacts) { @@ -144,13 +154,14 @@ void time_log_t::clock_in(const datetime_t& checkin, void time_log_t::clock_out(const datetime_t& checkin, account_t * account, - const string& desc) + const string& desc, + const string& note) { if (time_xacts.empty()) throw std::logic_error(_("Timelog check-out event without a check-in")); clock_out_from_timelog(time_xacts, checkin, account, desc.c_str(), - journal); + note.c_str(), journal); } } // namespace ledger diff --git a/src/timelog.h b/src/timelog.h index 7d79af3e..d0519ce4 100644 --- a/src/timelog.h +++ b/src/timelog.h @@ -56,19 +56,21 @@ public: datetime_t checkin; account_t * account; string desc; + string note; time_xact_t() : account(NULL) { TRACE_CTOR(time_xact_t, ""); } time_xact_t(const datetime_t& _checkin, - account_t * _account = NULL, - const string& _desc = "") - : checkin(_checkin), account(_account), desc(_desc) { - TRACE_CTOR(time_xact_t, "const datetime_t&, account_t *, const string&"); + account_t * _account = NULL, + const string& _desc = "", + const string& _note = "") + : checkin(_checkin), account(_account), desc(_desc), note(_note) { + TRACE_CTOR(time_xact_t, "const datetime_t&, account_t *, string, string"); } time_xact_t(const time_xact_t& xact) : checkin(xact.checkin), account(xact.account), - desc(xact.desc) { + desc(xact.desc), note(xact.note) { TRACE_CTOR(time_xact_t, "copy"); } ~time_xact_t() throw() { @@ -79,7 +81,7 @@ public: class time_log_t { std::list time_xacts; - journal_t& journal; + journal_t& journal; public: time_log_t(journal_t& _journal) : journal(_journal) { @@ -89,11 +91,13 @@ public: void clock_in(const datetime_t& checkin, account_t * account = NULL, - const string& desc = ""); + const string& desc = "", + const string& note = ""); void clock_out(const datetime_t& checkin, account_t * account = NULL, - const string& desc = ""); + const string& desc = "", + const string& note = ""); }; } // namespace ledger From a4b1e7c5ab70fc846b0ec4762f2e9f8ee242ca11 Mon Sep 17 00:00:00 2001 From: John Wiegley Date: Wed, 11 Nov 2009 02:01:24 -0500 Subject: [PATCH 12/19] Added a --prepend-format option This lets you, for example, debug registers that cull data from many different sources, without having to change the basic formatting string. You can locate each posting's location with this: ledger reg --prepend-format='%-25(filename + ":" + beg_line)' --- src/output.cc | 32 +++++++++++++++++++++++++++----- src/output.h | 8 ++++++-- src/report.cc | 22 +++++++++++++++------- src/report.h | 9 +++++++++ 4 files changed, 57 insertions(+), 14 deletions(-) diff --git a/src/output.cc b/src/output.cc index 2a6f0c20..71ec6d88 100644 --- a/src/output.cc +++ b/src/output.cc @@ -40,9 +40,10 @@ namespace ledger { -format_posts::format_posts(report_t& _report, - const string& format, - bool _print_raw) +format_posts::format_posts(report_t& _report, + const string& format, + bool _print_raw, + const optional& _prepend_format) : report(_report), last_xact(NULL), last_post(NULL), print_raw(_print_raw) { @@ -65,6 +66,9 @@ format_posts::format_posts(report_t& _report, first_line_format.parse_format(format); next_lines_format.parse_format(format); } + + if (_prepend_format) + prepend_format.parse_format(*_prepend_format); } void format_posts::flush() @@ -95,6 +99,10 @@ void format_posts::operator()(post_t& post) else if (! post.has_xdata() || ! post.xdata().has_flags(POST_EXT_DISPLAYED)) { bind_scope_t bound_scope(report, post); + + if (prepend_format) + out << prepend_format(bound_scope); + if (last_xact != post.xact) { if (last_xact) { bind_scope_t xact_scope(report, *last_xact); @@ -115,8 +123,9 @@ void format_posts::operator()(post_t& post) } } -format_accounts::format_accounts(report_t& _report, - const string& format) +format_accounts::format_accounts(report_t& _report, + const string& format, + const optional& _prepend_format) : report(_report), disp_pred() { TRACE_CTOR(format_accounts, "report&, const string&"); @@ -136,6 +145,9 @@ format_accounts::format_accounts(report_t& _report, account_line_format.parse_format(format); total_line_format.parse_format(format, account_line_format); } + + if (_prepend_format) + prepend_format.parse_format(*_prepend_format); } std::size_t format_accounts::post_account(account_t& account, const bool flat) @@ -150,6 +162,11 @@ std::size_t format_accounts::post_account(account_t& account, const bool flat) account.xdata().add_flags(ACCOUNT_EXT_DISPLAYED); bind_scope_t bound_scope(report, account); + + if (prepend_format) + static_cast(report.output_stream) + << prepend_format(bound_scope); + static_cast(report.output_stream) << account_line_format(bound_scope); @@ -217,6 +234,11 @@ void format_accounts::flush() ! report.HANDLED(no_total) && ! report.HANDLED(percent)) { bind_scope_t bound_scope(report, *report.session.journal->master); out << separator_format(bound_scope); + + if (prepend_format) + static_cast(report.output_stream) + << prepend_format(bound_scope); + out << total_line_format(bound_scope); } diff --git a/src/output.h b/src/output.h index fedd08ff..778a9335 100644 --- a/src/output.h +++ b/src/output.h @@ -60,13 +60,15 @@ protected: format_t first_line_format; format_t next_lines_format; format_t between_format; + format_t prepend_format; xact_t * last_xact; post_t * last_post; bool print_raw; public: format_posts(report_t& _report, const string& format, - bool _print_raw = false); + bool _print_raw = false, + const optional& _prepend_format = none); virtual ~format_posts() { TRACE_DTOR(format_posts); } @@ -82,12 +84,14 @@ protected: format_t account_line_format; format_t total_line_format; format_t separator_format; + format_t prepend_format; predicate_t disp_pred; std::list posted_accounts; public: - format_accounts(report_t& _report, const string& _format); + format_accounts(report_t& _report, const string& _format, + const optional& _prepend_format = none); virtual ~format_accounts() { TRACE_DTOR(format_accounts); } diff --git a/src/report.cc b/src/report.cc index 4e9cfe89..4d1a9700 100644 --- a/src/report.cc +++ b/src/report.cc @@ -631,6 +631,7 @@ option_t * report_t::lookup_option(const char * p) else OPT(pricesdb_format_); else OPT(print_format_); else OPT(payee_width_); + else OPT(prepend_format_); break; case 'q': OPT(quantity); @@ -854,7 +855,8 @@ expr_t::ptr_op_t report_t::lookup(const symbol_t::kind_t kind, if (*(p + 1) == '\0' || is_eq(p, "bal") || is_eq(p, "balance")) { return expr_t::op_t::wrap_functor (reporter - (new format_accounts(*this, report_format(HANDLER(balance_format_))), + (new format_accounts(*this, report_format(HANDLER(balance_format_)), + maybe_format(HANDLER(prepend_format_))), *this, "#balance")); } else if (is_eq(p, "budget")) { @@ -866,7 +868,8 @@ expr_t::ptr_op_t report_t::lookup(const symbol_t::kind_t kind, return expr_t::op_t::wrap_functor (reporter - (new format_accounts(*this, report_format(HANDLER(budget_format_))), + (new format_accounts(*this, report_format(HANDLER(budget_format_)), + maybe_format(HANDLER(prepend_format_))), *this, "#budget")); } break; @@ -875,7 +878,8 @@ expr_t::ptr_op_t report_t::lookup(const symbol_t::kind_t kind, if (is_eq(p, "csv")) { return WRAP_FUNCTOR (reporter<> - (new format_posts(*this, report_format(HANDLER(csv_format_))), + (new format_posts(*this, report_format(HANDLER(csv_format_)), + maybe_format(HANDLER(prepend_format_))), *this, "#csv")); } else if (is_eq(p, "cleared")) { @@ -884,7 +888,8 @@ expr_t::ptr_op_t report_t::lookup(const symbol_t::kind_t kind, return expr_t::op_t::wrap_functor (reporter - (new format_accounts(*this, report_format(HANDLER(cleared_format_))), + (new format_accounts(*this, report_format(HANDLER(cleared_format_)), + maybe_format(HANDLER(prepend_format_))), *this, "#cleared")); } break; @@ -913,12 +918,14 @@ expr_t::ptr_op_t report_t::lookup(const symbol_t::kind_t kind, else if (is_eq(p, "prices")) return expr_t::op_t::wrap_functor (reporter - (new format_posts(*this, report_format(HANDLER(prices_format_))), + (new format_posts(*this, report_format(HANDLER(prices_format_)), + maybe_format(HANDLER(prepend_format_))), *this, "#prices")); else if (is_eq(p, "pricesdb")) return expr_t::op_t::wrap_functor (reporter - (new format_posts(*this, report_format(HANDLER(pricesdb_format_))), + (new format_posts(*this, report_format(HANDLER(pricesdb_format_)), + maybe_format(HANDLER(prepend_format_))), *this, "#pricesdb")); break; @@ -926,7 +933,8 @@ expr_t::ptr_op_t report_t::lookup(const symbol_t::kind_t kind, if (*(p + 1) == '\0' || is_eq(p, "reg") || is_eq(p, "register")) return WRAP_FUNCTOR (reporter<> - (new format_posts(*this, report_format(HANDLER(register_format_))), + (new format_posts(*this, report_format(HANDLER(register_format_)), + false, maybe_format(HANDLER(prepend_format_))), *this, "#register")); else if (is_eq(p, "reload")) return MAKE_FUNCTOR(report_t::reload_command); diff --git a/src/report.h b/src/report.h index 02fd2c8d..3afb0107 100644 --- a/src/report.h +++ b/src/report.h @@ -172,6 +172,12 @@ public: return option.str(); } + optional maybe_format(option_t& option) { + if (option) + return option.str(); + return none; + } + value_t reload_command(call_scope_t&); value_t echo_command(call_scope_t& scope); @@ -252,6 +258,7 @@ public: HANDLER(period_).report(out); HANDLER(plot_amount_format_).report(out); HANDLER(plot_total_format_).report(out); + HANDLER(prepend_format_).report(out); HANDLER(price).report(out); HANDLER(prices_format_).report(out); HANDLER(pricesdb_format_).report(out); @@ -693,6 +700,8 @@ public: "%(format_date(date, \"%Y-%m-%d\")) %(quantity(scrub(display_total)))\n"); }); + OPTION(report_t, prepend_format_); + OPTION_(report_t, price, DO() { // -I parent->HANDLER(display_amount_) .set_expr(string("--price"), "price(amount_expr)"); From ed9209cc2794eeef2a363947ab9b89be83aefe6b Mon Sep 17 00:00:00 2001 From: John Wiegley Date: Wed, 11 Nov 2009 02:01:34 -0500 Subject: [PATCH 13/19] Timeclock events now record their file position --- src/textual.cc | 30 ++++++++++++++++++++----- src/timelog.cc | 60 +++++++++++++++++++++----------------------------- src/timelog.h | 30 ++++++++++++------------- 3 files changed, 64 insertions(+), 56 deletions(-) diff --git a/src/textual.cc b/src/textual.cc index 844edad3..f5d0635c 100644 --- a/src/textual.cc +++ b/src/textual.cc @@ -422,9 +422,19 @@ void instance_t::clock_in_directive(char * line, else end = NULL; - timelog.clock_in(parse_datetime(datetime, current_year), - account_stack.front()->find_account(p), - n ? n : "", end ? end : ""); + position_t position; + position.pathname = pathname; + position.beg_pos = line_beg_pos; + position.beg_line = linenum; + position.end_pos = curr_pos; + position.end_line = linenum; + + time_xact_t event(position, parse_datetime(datetime, current_year), + p ? account_stack.front()->find_account(p) : NULL, + n ? n : "", + end ? end : ""); + + timelog.clock_in(event); } void instance_t::clock_out_directive(char * line, @@ -441,9 +451,19 @@ void instance_t::clock_out_directive(char * line, else end = NULL; - timelog.clock_out(parse_datetime(datetime, current_year), + position_t position; + position.pathname = pathname; + position.beg_pos = line_beg_pos; + position.beg_line = linenum; + position.end_pos = curr_pos; + position.end_line = linenum; + + time_xact_t event(position, parse_datetime(datetime, current_year), p ? account_stack.front()->find_account(p) : NULL, - n ? n : "", end ? end : ""); + n ? n : "", + end ? end : ""); + + timelog.clock_out(event); count++; } diff --git a/src/timelog.cc b/src/timelog.cc index dc3d3496..b18c436b 100644 --- a/src/timelog.cc +++ b/src/timelog.cc @@ -41,11 +41,8 @@ namespace ledger { namespace { void clock_out_from_timelog(std::list& time_xacts, - const datetime_t& when, - account_t * account, - const char * desc, - const char * note, - journal_t& journal) + time_xact_t out_event, + journal_t& journal) { time_xact_t event; @@ -56,7 +53,7 @@ namespace { else if (time_xacts.empty()) { throw parse_error(_("Timelog check-out event without a check-in")); } - else if (! account) { + else if (! out_event.account) { throw parse_error (_("When multiple check-ins are active, checking out requires an account")); } @@ -66,7 +63,7 @@ namespace { for (std::list::iterator i = time_xacts.begin(); i != time_xacts.end(); i++) - if (account == (*i).account) { + if (out_event.account == (*i).account) { event = *i; found = true; time_xacts.erase(i); @@ -78,36 +75,37 @@ namespace { (_("Timelog check-out event does not match any current check-ins")); } - if (desc && event.desc.empty()) { - event.desc = desc; - desc = NULL; + if (out_event.checkin < event.checkin) + throw parse_error + (_("Timelog check-out date less than corresponding check-in")); + + if (! out_event.desc.empty() && event.desc.empty()) { + event.desc = out_event.desc; + out_event.desc = empty_string; } - if (note && event.note.empty()) { - event.note = note; - note = NULL; - } + if (! out_event.note.empty() && event.note.empty()) + event.note = out_event.note; std::auto_ptr curr(new xact_t); - curr->_date = when.date(); - curr->code = desc ? desc : ""; + curr->_date = out_event.checkin.date(); + curr->code = out_event.desc; // if it wasn't used above curr->payee = event.desc; + curr->pos = event.position; if (! event.note.empty()) curr->append_note(event.note.c_str()); - if (when < event.checkin) - throw parse_error - (_("Timelog check-out date less than corresponding check-in")); - char buf[32]; - std::sprintf(buf, "%lds", long((when - event.checkin).total_seconds())); + std::sprintf(buf, "%lds", long((out_event.checkin - event.checkin) + .total_seconds())); amount_t amt; amt.parse(buf); VERIFY(amt.valid()); post_t * post = new post_t(event.account, amt, POST_VIRTUAL); post->set_state(item_t::CLEARED); + post->pos = event.position; curr->add_post(post); if (! journal.add_xact(curr.get())) @@ -128,20 +126,16 @@ time_log_t::~time_log_t() accounts.push_back(time_xact.account); foreach (account_t * account, accounts) - clock_out_from_timelog(time_xacts, CURRENT_TIME(), account, - NULL, NULL, journal); + clock_out_from_timelog(time_xacts, + time_xact_t(none, CURRENT_TIME(), account), + journal); assert(time_xacts.empty()); } } -void time_log_t::clock_in(const datetime_t& checkin, - account_t * account, - const string& desc, - const string& note) +void time_log_t::clock_in(time_xact_t event) { - time_xact_t event(checkin, account, desc, note); - if (! time_xacts.empty()) { foreach (time_xact_t& time_xact, time_xacts) { if (event.account == time_xact.account) @@ -152,16 +146,12 @@ void time_log_t::clock_in(const datetime_t& checkin, time_xacts.push_back(event); } -void time_log_t::clock_out(const datetime_t& checkin, - account_t * account, - const string& desc, - const string& note) +void time_log_t::clock_out(time_xact_t event) { if (time_xacts.empty()) throw std::logic_error(_("Timelog check-out event without a check-in")); - clock_out_from_timelog(time_xacts, checkin, account, desc.c_str(), - note.c_str(), journal); + clock_out_from_timelog(time_xacts, event, journal); } } // namespace ledger diff --git a/src/timelog.h b/src/timelog.h index d0519ce4..83256dfa 100644 --- a/src/timelog.h +++ b/src/timelog.h @@ -44,6 +44,7 @@ #include "utils.h" #include "times.h" +#include "item.h" namespace ledger { @@ -57,20 +58,24 @@ public: account_t * account; string desc; string note; + position_t position; time_xact_t() : account(NULL) { TRACE_CTOR(time_xact_t, ""); } - time_xact_t(const datetime_t& _checkin, - account_t * _account = NULL, - const string& _desc = "", - const string& _note = "") - : checkin(_checkin), account(_account), desc(_desc), note(_note) { - TRACE_CTOR(time_xact_t, "const datetime_t&, account_t *, string, string"); + time_xact_t(const optional& _position, + const datetime_t& _checkin, + account_t * _account = NULL, + const string& _desc = "", + const string& _note = "") + : checkin(_checkin), account(_account), desc(_desc), note(_note), + position(_position ? *_position : position_t()) { + TRACE_CTOR(time_xact_t, + "position_t, datetime_t, account_t *, string, string"); } time_xact_t(const time_xact_t& xact) : checkin(xact.checkin), account(xact.account), - desc(xact.desc), note(xact.note) { + desc(xact.desc), note(xact.note), position(xact.position) { TRACE_CTOR(time_xact_t, "copy"); } ~time_xact_t() throw() { @@ -89,15 +94,8 @@ public: } ~time_log_t(); - void clock_in(const datetime_t& checkin, - account_t * account = NULL, - const string& desc = "", - const string& note = ""); - - void clock_out(const datetime_t& checkin, - account_t * account = NULL, - const string& desc = "", - const string& note = ""); + void clock_in(time_xact_t event); + void clock_out(time_xact_t event); }; } // namespace ledger From 4a4ff9d4b21e7ed94524314ca496bd17c84fc4ed Mon Sep 17 00:00:00 2001 From: John Wiegley Date: Wed, 11 Nov 2009 02:58:06 -0500 Subject: [PATCH 14/19] Value.basetype in Python returns a Value's base type --- src/py_value.cc | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/py_value.cc b/src/py_value.cc index 9653b0e1..c8e4de72 100644 --- a/src/py_value.cc +++ b/src/py_value.cc @@ -47,6 +47,22 @@ BOOST_PYTHON_MEMBER_FUNCTION_OVERLOADS(set_string_overloads, set_string, 0, 2) namespace { + PyObject * py_base_type(value_t& value) { + if (value.is_boolean()) { + return (PyObject *)&PyBool_Type; + } + else if (value.is_long()) { + return (PyObject *)&PyInt_Type; + } + else if (value.is_string()) { + return (PyObject *)&PyUnicode_Type; + } + else { + object typeobj(object(value).attr("__class__")); + return typeobj.ptr(); + } + } + expr_t py_value_getattr(const value_t& value, const string& name) { if (value.is_scope()) { @@ -305,6 +321,8 @@ void export_value() .def("label", &value_t::label) .def("valid", &value_t::valid) + + .def("basetype", py_base_type) ; scope().attr("NULL_VALUE") = NULL_VALUE; From afe87280e091d4f094f068c5f21aecccd2d1831b Mon Sep 17 00:00:00 2001 From: John Wiegley Date: Wed, 11 Nov 2009 03:39:53 -0500 Subject: [PATCH 15/19] Added floored() and in_place_floor() methods --- src/amount.cc | 199 ++++++++++++++++++++++++---------------------- src/amount.h | 9 +++ src/balance.h | 12 +++ src/py_amount.cc | 4 + src/py_balance.cc | 4 + src/py_value.cc | 2 + src/value.cc | 25 ++++++ src/value.h | 7 ++ 8 files changed, 169 insertions(+), 93 deletions(-) diff --git a/src/amount.cc b/src/amount.cc index 28fa9eaf..f8406505 100644 --- a/src/amount.cc +++ b/src/amount.cc @@ -114,6 +114,99 @@ shared_ptr amount_t::current_pool; bool amount_t::is_initialized = false; +namespace { + void stream_out_mpq(std::ostream& out, + mpq_t quant, + amount_t::precision_t prec, + int zeros_prec = -1, + const optional& comm = none) + { + char * buf = NULL; + try { + IF_DEBUG("amount.convert") { + char * tbuf = mpq_get_str(NULL, 10, quant); + DEBUG("amount.convert", "Rational to convert = " << tbuf); + std::free(tbuf); + } + + // Convert the rational number to a floating-point, extending the + // floating-point to a large enough size to get a precise answer. + const std::size_t bits = (mpz_sizeinbase(mpq_numref(quant), 2) + + mpz_sizeinbase(mpq_denref(quant), 2)); + mpfr_set_prec(tempfb, bits + amount_t::extend_by_digits*8); + mpfr_set_q(tempfb, quant, GMP_RNDN); + + mpfr_asprintf(&buf, "%.*Rf", prec, tempfb); + DEBUG("amount.convert", + "mpfr_print = " << buf << " (precision " << prec << ")"); + + if (zeros_prec >= 0) { + string::size_type index = std::strlen(buf); + string::size_type point = 0; + for (string::size_type i = 0; i < index; i++) { + if (buf[i] == '.') { + point = i; + break; + } + } + if (point > 0) { + while (--index >= (point + 1 + zeros_prec) && buf[index] == '0') + buf[index] = '\0'; + if (index >= (point + zeros_prec) && buf[index] == '.') + buf[index] = '\0'; + } + } + + if (comm) { + int integer_digits = 0; + if (comm && comm->has_flags(COMMODITY_STYLE_THOUSANDS)) { + // Count the number of integer digits + for (const char * p = buf; *p; p++) { + if (*p == '.') + break; + else if (*p != '-') + integer_digits++; + } + } + + for (const char * p = buf; *p; p++) { + if (*p == '.') { + if (commodity_t::european_by_default || + (comm && comm->has_flags(COMMODITY_STYLE_EUROPEAN))) + out << ','; + else + out << *p; + assert(integer_digits <= 3); + } + else if (*p == '-') { + out << *p; + } + else { + out << *p; + + if (integer_digits > 3 && --integer_digits % 3 == 0) { + if (commodity_t::european_by_default || + (comm && comm->has_flags(COMMODITY_STYLE_EUROPEAN))) + out << '.'; + else + out << ','; + } + } + } + } else { + out << buf; + } + } + catch (...) { + if (buf != NULL) + mpfr_free_str(buf); + throw; + } + if (buf != NULL) + mpfr_free_str(buf); + } +} + void amount_t::initialize(shared_ptr pool) { if (! is_initialized) { @@ -498,6 +591,19 @@ void amount_t::in_place_round() set_keep_precision(false); } +void amount_t::in_place_floor() +{ + if (! quantity) + throw_(amount_error, _("Cannot floor an uninitialized amount")); + + _dup(); + + std::ostringstream out; + stream_out_mpq(out, MP(quantity), 0); + + mpq_set_str(MP(quantity), out.str().c_str(), 10); +} + void amount_t::in_place_unround() { if (! quantity) @@ -612,99 +718,6 @@ int amount_t::sign() const return mpq_sgn(MP(quantity)); } -namespace { - void stream_out_mpq(std::ostream& out, - mpq_t quant, - amount_t::precision_t prec, - int zeros_prec = -1, - const optional& comm = none) - { - char * buf = NULL; - try { - IF_DEBUG("amount.convert") { - char * tbuf = mpq_get_str(NULL, 10, quant); - DEBUG("amount.convert", "Rational to convert = " << tbuf); - std::free(tbuf); - } - - // Convert the rational number to a floating-point, extending the - // floating-point to a large enough size to get a precise answer. - const std::size_t bits = (mpz_sizeinbase(mpq_numref(quant), 2) + - mpz_sizeinbase(mpq_denref(quant), 2)); - mpfr_set_prec(tempfb, bits + amount_t::extend_by_digits*8); - mpfr_set_q(tempfb, quant, GMP_RNDN); - - mpfr_asprintf(&buf, "%.*Rf", prec, tempfb); - DEBUG("amount.convert", - "mpfr_print = " << buf << " (precision " << prec << ")"); - - if (zeros_prec >= 0) { - string::size_type index = std::strlen(buf); - string::size_type point = 0; - for (string::size_type i = 0; i < index; i++) { - if (buf[i] == '.') { - point = i; - break; - } - } - if (point > 0) { - while (--index >= (point + 1 + zeros_prec) && buf[index] == '0') - buf[index] = '\0'; - if (index >= (point + zeros_prec) && buf[index] == '.') - buf[index] = '\0'; - } - } - - if (comm) { - int integer_digits = 0; - if (comm && comm->has_flags(COMMODITY_STYLE_THOUSANDS)) { - // Count the number of integer digits - for (const char * p = buf; *p; p++) { - if (*p == '.') - break; - else if (*p != '-') - integer_digits++; - } - } - - for (const char * p = buf; *p; p++) { - if (*p == '.') { - if (commodity_t::european_by_default || - (comm && comm->has_flags(COMMODITY_STYLE_EUROPEAN))) - out << ','; - else - out << *p; - assert(integer_digits <= 3); - } - else if (*p == '-') { - out << *p; - } - else { - out << *p; - - if (integer_digits > 3 && --integer_digits % 3 == 0) { - if (commodity_t::european_by_default || - (comm && comm->has_flags(COMMODITY_STYLE_EUROPEAN))) - out << '.'; - else - out << ','; - } - } - } - } else { - out << buf; - } - } - catch (...) { - if (buf != NULL) - mpfr_free_str(buf); - throw; - } - if (buf != NULL) - mpfr_free_str(buf); - } -} - bool amount_t::is_zero() const { if (! quantity) diff --git a/src/amount.h b/src/amount.h index 505e2ea7..c75370e3 100644 --- a/src/amount.h +++ b/src/amount.h @@ -354,6 +354,15 @@ public: *this = amount_t(to_string()); } + /** Yields an amount which has lost all of its extra precision, beyond what + the display precision of the commodity would have printed. */ + amount_t floored() const { + amount_t temp(*this); + temp.in_place_floor(); + return temp; + } + void in_place_floor(); + /** Yields an amount whose display precision is never truncated, even though its commodity normally displays only rounded values. */ amount_t unrounded() const { diff --git a/src/balance.h b/src/balance.h index 81a7ff13..8a40dea9 100644 --- a/src/balance.h +++ b/src/balance.h @@ -339,6 +339,18 @@ public: *this = temp; } + balance_t floored() const { + balance_t temp(*this); + temp.in_place_floor(); + return temp; + } + void in_place_floor() { + balance_t temp; + foreach (const amounts_map::value_type& pair, amounts) + temp += pair.second.floored(); + *this = temp; + } + balance_t unrounded() const { balance_t temp(*this); temp.in_place_unround(); diff --git a/src/py_amount.cc b/src/py_amount.cc index 83f5dd29..b44f3716 100644 --- a/src/py_amount.cc +++ b/src/py_amount.cc @@ -220,6 +220,10 @@ internal precision.")) .def("in_place_truncate", &amount_t::in_place_truncate, return_internal_reference<>()) + .def("floored", &amount_t::floored) + .def("in_place_floor", &amount_t::in_place_floor, + return_internal_reference<>()) + .def("unrounded", &amount_t::unrounded) .def("in_place_unround", &amount_t::in_place_unround, return_internal_reference<>()) diff --git a/src/py_balance.cc b/src/py_balance.cc index 73049c99..23a2ff73 100644 --- a/src/py_balance.cc +++ b/src/py_balance.cc @@ -176,6 +176,10 @@ void export_balance() .def("in_place_truncate", &balance_t::in_place_truncate, return_internal_reference<>()) + .def("floored", &balance_t::floored) + .def("in_place_floor", &balance_t::in_place_floor, + return_internal_reference<>()) + .def("unrounded", &balance_t::unrounded) .def("in_place_unround", &balance_t::in_place_unround, return_internal_reference<>()) diff --git a/src/py_value.cc b/src/py_value.cc index c8e4de72..e94e74c1 100644 --- a/src/py_value.cc +++ b/src/py_value.cc @@ -234,6 +234,8 @@ void export_value() .def("in_place_round", &value_t::in_place_round) .def("truncated", &value_t::truncated) .def("in_place_truncate", &value_t::in_place_truncate) + .def("floored", &value_t::floored) + .def("in_place_floor", &value_t::in_place_floor) .def("unrounded", &value_t::unrounded) .def("in_place_unround", &value_t::in_place_unround) .def("reduced", &value_t::reduced) diff --git a/src/value.cc b/src/value.cc index 021bf957..ae86eb7c 100644 --- a/src/value.cc +++ b/src/value.cc @@ -1439,6 +1439,31 @@ void value_t::in_place_truncate() throw_(value_error, _("Cannot truncate %1") << label()); } +void value_t::in_place_floor() +{ + switch (type()) { + case INTEGER: + return; + case AMOUNT: + as_amount_lval().in_place_floor(); + return; + case BALANCE: + as_balance_lval().in_place_floor(); + return; + case SEQUENCE: { + value_t temp; + foreach (const value_t& value, as_sequence()) + temp.push_back(value.floored()); + *this = temp; + return; + } + default: + break; + } + + throw_(value_error, _("Cannot floor %1") << label()); +} + void value_t::in_place_unround() { switch (type()) { diff --git a/src/value.h b/src/value.h index 96a3078a..2a420cd3 100644 --- a/src/value.h +++ b/src/value.h @@ -440,6 +440,13 @@ public: } void in_place_truncate(); + value_t floored() const { + value_t temp(*this); + temp.in_place_floor(); + return temp; + } + void in_place_floor(); + value_t unrounded() const { value_t temp(*this); temp.in_place_unround(); From e8ea2d4938fcbb0988fd1e2021d97a519c67ffd8 Mon Sep 17 00:00:00 2001 From: John Wiegley Date: Wed, 11 Nov 2009 03:41:59 -0500 Subject: [PATCH 16/19] Automated postings defer amount expression calculation This allows for value expressions to be used which reference the incoming posting, for example: = Income:Clients: (Liabilities:Taxes:VAT1) (floor(amount) * 1) (Liabilities:Taxes:VAT2) 0.19 2009/07/27 * Invoice Assets:Bank:Checking $1,190.45 Income:Clients:ACME_Inc The automated posting for VAT1 will use the floored amount multiplied by a factor, while the posting for VAT2 multiples the whole amount as before. --- src/archive.cc | 2 +- src/hooks.h | 4 +- src/journal.cc | 4 +- src/post.h | 2 + src/py_journal.cc | 8 ++-- src/textual.cc | 75 +++++++++++++++++++++++--------------- src/xact.cc | 47 +++++++++++++++--------- src/xact.h | 18 ++++----- test/regress/25A099C9.test | 12 +++--- 9 files changed, 99 insertions(+), 73 deletions(-) diff --git a/src/archive.cc b/src/archive.cc index f76b7543..5ea6cd8e 100644 --- a/src/archive.cc +++ b/src/archive.cc @@ -43,7 +43,7 @@ #include "xact.h" #define LEDGER_MAGIC 0x4c454447 -#define ARCHIVE_VERSION 0x03000004 +#define ARCHIVE_VERSION 0x03000005 //BOOST_IS_ABSTRACT(ledger::scope_t) BOOST_CLASS_EXPORT(ledger::scope_t) diff --git a/src/hooks.h b/src/hooks.h index da6fcf84..20c7750c 100644 --- a/src/hooks.h +++ b/src/hooks.h @@ -70,9 +70,9 @@ public: list.remove(func); } - bool run_hooks(Data& item, bool post) { + bool run_hooks(Data& item) { foreach (T * func, list) - if (! (*func)(item, post)) + if (! (*func)(item)) return false; return true; } diff --git a/src/journal.cc b/src/journal.cc index 550b4f4c..b7ad9a23 100644 --- a/src/journal.cc +++ b/src/journal.cc @@ -126,9 +126,7 @@ bool journal_t::add_xact(xact_t * xact) { xact->journal = this; - if (! xact_finalize_hooks.run_hooks(*xact, false) || - ! xact->finalize() || - ! xact_finalize_hooks.run_hooks(*xact, true)) { + if (! xact->finalize() || ! xact_finalize_hooks.run_hooks(*xact)) { xact->journal = NULL; return false; } diff --git a/src/post.h b/src/post.h index addf0629..d9e50580 100644 --- a/src/post.h +++ b/src/post.h @@ -61,6 +61,7 @@ public: account_t * account; amount_t amount; // can be null until finalization + optional amount_expr; optional cost; optional assigned_amount; @@ -212,6 +213,7 @@ private: ar & xact; ar & account; ar & amount; + ar & amount_expr; ar & cost; ar & assigned_amount; } diff --git a/src/py_journal.cc b/src/py_journal.cc index 17c42c21..88447b92 100644 --- a/src/py_journal.cc +++ b/src/py_journal.cc @@ -136,8 +136,8 @@ namespace { py_xact_finalizer_t(object obj) : pyobj(obj) {} py_xact_finalizer_t(const py_xact_finalizer_t& other) : pyobj(other.pyobj) {} - virtual bool operator()(xact_t& xact, bool post) { - return call(pyobj.ptr(), xact, post); + virtual bool operator()(xact_t& xact) { + return call(pyobj.ptr(), xact); } }; @@ -161,9 +161,9 @@ namespace { } } - void py_run_xact_finalizers(journal_t& journal, xact_t& xact, bool post) + void py_run_xact_finalizers(journal_t& journal, xact_t& xact) { - journal.xact_finalize_hooks.run_hooks(xact, post); + journal.xact_finalize_hooks.run_hooks(xact); } std::size_t py_read(journal_t& journal, const string& pathname) diff --git a/src/textual.cc b/src/textual.cc index f5d0635c..8f0dd89c 100644 --- a/src/textual.cc +++ b/src/textual.cc @@ -132,10 +132,12 @@ namespace { std::streamsize len, account_t * account, xact_t * xact, - bool honor_strict = true); + bool honor_strict = true, + bool defer_expr = false); - bool parse_posts(account_t * account, - xact_base_t& xact); + bool parse_posts(account_t * account, + xact_base_t& xact, + const bool defer_expr = false); xact_t * parse_xact(char * line, std::streamsize len, @@ -145,11 +147,13 @@ namespace { const string& name); }; - void parse_amount_expr(scope_t& scope, - std::istream& in, - amount_t& amount, - post_t * post, - const parse_flags_t& flags = PARSE_DEFAULT) + void parse_amount_expr(scope_t& scope, + std::istream& in, + amount_t& amount, + optional * amount_expr, + post_t * post, + const parse_flags_t& flags = PARSE_DEFAULT, + const bool defer_expr = false) { expr_t expr(in, flags.plus_flags(PARSE_PARTIAL)); @@ -166,17 +170,22 @@ namespace { if (expr) { bind_scope_t bound_scope(scope, *post); - - value_t result(expr.calc(bound_scope)); - if (result.is_long()) { - amount = result.to_amount(); + if (defer_expr) { + assert(amount_expr); + *amount_expr = expr; + (*amount_expr)->compile(bound_scope); } else { - if (! result.is_amount()) - throw_(parse_error, _("Postings may only specify simple amounts")); - - amount = result.as_amount(); + value_t result(expr.calc(bound_scope)); + if (result.is_long()) { + amount = result.to_amount(); + } else { + if (! result.is_amount()) + throw_(amount_error, + _("Amount expressions must result in a simple amount")); + amount = result.as_amount(); + } + DEBUG("textual.parse", "The posting amount is " << amount); } - DEBUG("textual.parse", "The posting amount is " << amount); } } } @@ -548,7 +557,7 @@ void instance_t::automated_xact_directive(char * line) reveal_context = false; - if (parse_posts(account_stack.front(), *ae.get())) { + if (parse_posts(account_stack.front(), *ae.get(), true)) { reveal_context = true; journal.auto_xacts.push_back(ae.get()); @@ -592,7 +601,7 @@ void instance_t::period_xact_directive(char * line) pe->journal = &journal; if (pe->finalize()) { - extend_xact_base(&journal, *pe.get(), true); + extend_xact_base(&journal, *pe.get()); journal.period_xacts.push_back(pe.get()); @@ -817,7 +826,8 @@ post_t * instance_t::parse_post(char * line, std::streamsize len, account_t * account, xact_t * xact, - bool honor_strict) + bool honor_strict, + bool defer_expr) { TRACE_START(post_details, 1, "Time spent parsing postings:"); @@ -919,8 +929,9 @@ post_t * instance_t::parse_post(char * line, if (*next != '(') // indicates a value expression post->amount.parse(stream, PARSE_NO_REDUCE); else - parse_amount_expr(scope, stream, post->amount, post.get(), - PARSE_NO_REDUCE | PARSE_SINGLE | PARSE_NO_ASSIGN); + parse_amount_expr(scope, stream, post->amount, &post->amount_expr, + post.get(), PARSE_NO_REDUCE | PARSE_SINGLE | + PARSE_NO_ASSIGN, defer_expr); if (! post->amount.is_null() && honor_strict && strict && post->amount.has_commodity() && @@ -965,9 +976,9 @@ post_t * instance_t::parse_post(char * line, if (*p != '(') // indicates a value expression post->cost->parse(cstream, PARSE_NO_MIGRATE); else - parse_amount_expr(scope, cstream, *post->cost, post.get(), - PARSE_NO_MIGRATE | PARSE_SINGLE | - PARSE_NO_ASSIGN); + parse_amount_expr(scope, cstream, *post->cost, NULL, post.get(), + PARSE_NO_MIGRATE | PARSE_SINGLE | PARSE_NO_ASSIGN, + defer_expr); if (post->cost->sign() < 0) throw parse_error(_("A posting's cost may not be negative")); @@ -1017,8 +1028,9 @@ post_t * instance_t::parse_post(char * line, if (*p != '(') // indicates a value expression post->assigned_amount->parse(stream, PARSE_NO_MIGRATE); else - parse_amount_expr(scope, stream, *post->assigned_amount, post.get(), - PARSE_SINGLE | PARSE_NO_MIGRATE); + parse_amount_expr(scope, stream, *post->assigned_amount, NULL, + post.get(), PARSE_SINGLE | PARSE_NO_MIGRATE, + defer_expr); if (post->assigned_amount->is_null()) { if (post->amount.is_null()) @@ -1118,8 +1130,9 @@ post_t * instance_t::parse_post(char * line, } } -bool instance_t::parse_posts(account_t * account, - xact_base_t& xact) +bool instance_t::parse_posts(account_t * account, + xact_base_t& xact, + const bool defer_expr) { TRACE_START(xact_posts, 1, "Time spent parsing postings:"); @@ -1130,7 +1143,9 @@ bool instance_t::parse_posts(account_t * account, std::streamsize len = read_line(line); assert(len > 0); - if (post_t * post = parse_post(line, len, account, NULL, false)) { + if (post_t * post = + parse_post(line, len, account, NULL, /* honor_strict= */ false, + defer_expr)) { xact.add_post(post); added = true; } diff --git a/src/xact.cc b/src/xact.cc index d94464a6..8ac5280a 100644 --- a/src/xact.cc +++ b/src/xact.cc @@ -480,7 +480,7 @@ bool xact_t::valid() const return true; } -void auto_xact_t::extend_xact(xact_base_t& xact, bool post_handler) +void auto_xact_t::extend_xact(xact_base_t& xact) { posts_list initial_posts(xact.posts.begin(), xact.posts.end()); @@ -490,20 +490,32 @@ void auto_xact_t::extend_xact(xact_base_t& xact, bool post_handler) if (! initial_post->has_flags(ITEM_GENERATED) && predicate(*initial_post)) { foreach (post_t * post, posts) { - amount_t amt; - assert(post->amount); - if (! post->amount.commodity()) { - if ((post_handler && - ! initial_post->has_flags(POST_CALCULATED)) || - initial_post->amount.is_null()) - continue; - amt = initial_post->amount * post->amount; + amount_t post_amount; + if (post->amount.is_null()) { + if (! post->amount_expr) + throw_(amount_error, + _("Automated transaction's posting has no amount")); + + bind_scope_t bound_scope(*scope_t::default_scope, *initial_post); + value_t result(post->amount_expr->calc(bound_scope)); + if (result.is_long()) { + post_amount = result.to_amount(); + } else { + if (! result.is_amount()) + throw_(amount_error, + _("Amount expressions must result in a simple amount")); + post_amount = result.as_amount(); + } } else { - if (post_handler) - continue; - amt = post->amount; + post_amount = post->amount; } + amount_t amt; + if (! post_amount.commodity()) + amt = initial_post->amount * post_amount; + else + amt = post_amount; + IF_DEBUG("xact.extend") { DEBUG("xact.extend", "Initial post on line " << initial_post->pos->beg_line << ": " @@ -517,12 +529,12 @@ void auto_xact_t::extend_xact(xact_base_t& xact, bool post_handler) DEBUG("xact.extend", "Posting on line " << post->pos->beg_line << ": " - << "amount " << post->amount << ", amt " << amt - << " (precision " << post->amount.precision() + << "amount " << post_amount << ", amt " << amt + << " (precision " << post_amount.precision() << " != " << amt.precision() << ")"); #if defined(DEBUG_ON) - if (post->amount.keep_precision()) + if (post_amount.keep_precision()) DEBUG("xact.extend", " precision is kept"); if (amt.keep_precision()) DEBUG("xact.extend", " amt precision is kept"); @@ -556,11 +568,10 @@ void auto_xact_t::extend_xact(xact_base_t& xact, bool post_handler) } void extend_xact_base(journal_t * journal, - xact_base_t& base, - bool post_handler) + xact_base_t& base) { foreach (auto_xact_t * xact, journal->auto_xacts) - xact->extend_xact(base, post_handler); + xact->extend_xact(base); } void to_xml(std::ostream& out, const xact_t& xact) diff --git a/src/xact.h b/src/xact.h index 9a52fafe..ff1f65fa 100644 --- a/src/xact.h +++ b/src/xact.h @@ -142,7 +142,7 @@ private: struct xact_finalizer_t { virtual ~xact_finalizer_t() {} - virtual bool operator()(xact_t& xact, bool post) = 0; + virtual bool operator()(xact_t& xact) = 0; }; class auto_xact_t : public xact_base_t @@ -167,7 +167,7 @@ public: TRACE_DTOR(auto_xact_t); } - virtual void extend_xact(xact_base_t& xact, bool post); + virtual void extend_xact(xact_base_t& xact); #if defined(HAVE_BOOST_SERIALIZATION) private: @@ -201,7 +201,7 @@ struct auto_xact_finalizer_t : public xact_finalizer_t TRACE_DTOR(auto_xact_finalizer_t); } - virtual bool operator()(xact_t& xact, bool post); + virtual bool operator()(xact_t& xact); #if defined(HAVE_BOOST_SERIALIZATION) private: @@ -258,7 +258,7 @@ class func_finalizer_t : public xact_finalizer_t func_finalizer_t(); public: - typedef function func_t; + typedef function func_t; func_t func; @@ -273,15 +273,15 @@ public: TRACE_DTOR(func_finalizer_t); } - virtual bool operator()(xact_t& xact, bool post) { - return func(xact, post); + virtual bool operator()(xact_t& xact) { + return func(xact); } }; -void extend_xact_base(journal_t * journal, xact_base_t& xact, bool post); +void extend_xact_base(journal_t * journal, xact_base_t& xact); -inline bool auto_xact_finalizer_t::operator()(xact_t& xact, bool post) { - extend_xact_base(journal, xact, post); +inline bool auto_xact_finalizer_t::operator()(xact_t& xact) { + extend_xact_base(journal, xact); return true; } diff --git a/test/regress/25A099C9.test b/test/regress/25A099C9.test index 8aa8954f..604939d8 100644 --- a/test/regress/25A099C9.test +++ b/test/regress/25A099C9.test @@ -4,16 +4,16 @@ >>>2 While parsing file "$sourcepath/src/amount.h", line 67: Error: No quantity specified for amount -While parsing file "$sourcepath/src/amount.h", line 712: +While parsing file "$sourcepath/src/amount.h", line 721: Error: Invalid date/time: line amount_t amoun -While parsing file "$sourcepath/src/amount.h", line 718: +While parsing file "$sourcepath/src/amount.h", line 727: Error: Invalid date/time: line string amount_ -While parsing file "$sourcepath/src/amount.h", line 724: +While parsing file "$sourcepath/src/amount.h", line 733: Error: Invalid date/time: line string amount_ -While parsing file "$sourcepath/src/amount.h", line 730: +While parsing file "$sourcepath/src/amount.h", line 739: Error: Invalid date/time: line string amount_ -While parsing file "$sourcepath/src/amount.h", line 736: +While parsing file "$sourcepath/src/amount.h", line 745: Error: Invalid date/time: line std::ostream& -While parsing file "$sourcepath/src/amount.h", line 743: +While parsing file "$sourcepath/src/amount.h", line 752: Error: Invalid date/time: line std::istream& === 7 From dae24c259ba70cfdba92a5d66bc424eef5773188 Mon Sep 17 00:00:00 2001 From: John Wiegley Date: Wed, 11 Nov 2009 03:45:37 -0500 Subject: [PATCH 17/19] Added floor() value expression function --- src/report.cc | 8 ++++++++ src/report.h | 1 + 2 files changed, 9 insertions(+) diff --git a/src/report.cc b/src/report.cc index 4d1a9700..b866970f 100644 --- a/src/report.cc +++ b/src/report.cc @@ -207,6 +207,12 @@ value_t report_t::fn_quantity(call_scope_t& scope) return args.get(0).number(); } +value_t report_t::fn_floor(call_scope_t& scope) +{ + interactive_t args(scope, "v"); + return args.value_at(0).floored(); +} + value_t report_t::fn_abs(call_scope_t& scope) { interactive_t args(scope, "v"); @@ -735,6 +741,8 @@ expr_t::ptr_op_t report_t::lookup(const symbol_t::kind_t kind, case 'f': if (is_eq(p, "format_date")) return MAKE_FUNCTOR(report_t::fn_format_date); + else if (is_eq(p, "floor")) + return MAKE_FUNCTOR(report_t::fn_floor); break; case 'g': diff --git a/src/report.h b/src/report.h index 3afb0107..e27abbf9 100644 --- a/src/report.h +++ b/src/report.h @@ -143,6 +143,7 @@ public: value_t fn_rounded(call_scope_t& scope); value_t fn_unrounded(call_scope_t& scope); value_t fn_truncated(call_scope_t& scope); + value_t fn_floor(call_scope_t& scope); value_t fn_abs(call_scope_t& scope); value_t fn_justify(call_scope_t& scope); value_t fn_quoted(call_scope_t& scope); From f1b495abfea31f4777b74715631dd45f9f4d8ed1 Mon Sep 17 00:00:00 2001 From: John Wiegley Date: Wed, 11 Nov 2009 04:32:27 -0500 Subject: [PATCH 18/19] Added some missing calls to add_post --- src/draft.cc | 2 ++ src/timelog.cc | 1 + 2 files changed, 3 insertions(+) diff --git a/src/draft.cc b/src/draft.cc index b4e23322..8478a31d 100644 --- a/src/draft.cc +++ b/src/draft.cc @@ -420,6 +420,7 @@ xact_t * draft_t::insert(journal_t& journal) } } } + assert(new_post->account); if (new_post.get() && ! new_post->amount.is_null()) { found_commodity = &new_post->amount.commodity(); @@ -475,6 +476,7 @@ xact_t * draft_t::insert(journal_t& journal) } added->add_post(new_post.release()); + added->posts.back()->account->add_post(added->posts.back()); added->posts.back()->set_state(item_t::UNCLEARED); DEBUG("derive.xact", "Added new posting to derived entry"); diff --git a/src/timelog.cc b/src/timelog.cc index b18c436b..25bf8e94 100644 --- a/src/timelog.cc +++ b/src/timelog.cc @@ -107,6 +107,7 @@ namespace { post->set_state(item_t::CLEARED); post->pos = event.position; curr->add_post(post); + event.account->add_post(post); if (! journal.add_xact(curr.get())) throw parse_error(_("Failed to record 'out' timelog transaction")); From f0f1b0cdfa3a0a73695eda52b25de71bd40adc5a Mon Sep 17 00:00:00 2001 From: John Wiegley Date: Wed, 11 Nov 2009 04:46:38 -0500 Subject: [PATCH 19/19] Value.to_sequence returns a valid Python sequence --- src/py_value.cc | 4 ++++ src/system.hh.in | 1 + 2 files changed, 5 insertions(+) diff --git a/src/py_value.cc b/src/py_value.cc index e94e74c1..ee039519 100644 --- a/src/py_value.cc +++ b/src/py_value.cc @@ -327,6 +327,10 @@ void export_value() .def("basetype", py_base_type) ; + class_< value_t::sequence_t > ("ValueSequence") + .def(vector_indexing_suite< value_t::sequence_t >()); + ; + scope().attr("NULL_VALUE") = NULL_VALUE; scope().attr("string_value") = &string_value; scope().attr("mask_value") = &mask_value; diff --git a/src/system.hh.in b/src/system.hh.in index abb823dc..341ce129 100644 --- a/src/system.hh.in +++ b/src/system.hh.in @@ -249,6 +249,7 @@ void serialize(Archive& ar, istream_pos_type& pos, const unsigned int) #include #include +#include #endif // HAVE_BOOST_PYTHON