sfos/ledger
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Create a --cache option, for using a binary cache

Browse source
This commit is contained in:
John Wiegley 2009-10-30 20:50:57 -04:00
parent 63aa8992a8
commit 2149a8e773
9 changed files with 482 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Makefile.am
View file

@ -59,6 +59,7 @@ libledger_data_la_SOURCES = \
src/timelog.cc \
src/textual.cc \
src/journal.cc \
src/archive.cc \
src/account.cc \
src/xact.cc \
src/post.cc \
@ -118,6 +119,7 @@ pkginclude_HEADERS = \
src/xact.h \
src/account.h \
src/journal.h \
src/archive.h \
src/timelog.h \
src/iterators.h \
src/compare.h \

247
src/archive.cc Normal file
View file

@ -0,0 +1,247 @@
/*
* Copyright (c) 2003-2009, John Wiegley. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* - Neither the name of New Artisans LLC nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <system.hh>
#if defined(HAVE_BOOST_SERIALIZATION)
#include "archive.h"
#include "amount.h"
#include "commodity.h"
#include "pool.h"
#include "scope.h"
#include "account.h"
#include "post.h"
#include "xact.h"
#define ARCHIVE_VERSION 0x03000001
//BOOST_IS_ABSTRACT(ledger::scope_t)
BOOST_CLASS_EXPORT(ledger::scope_t)
BOOST_CLASS_EXPORT(ledger::child_scope_t)
BOOST_CLASS_EXPORT(ledger::symbol_scope_t)
BOOST_CLASS_EXPORT(ledger::call_scope_t)
BOOST_CLASS_EXPORT(ledger::account_t)
BOOST_CLASS_EXPORT(ledger::item_t)
BOOST_CLASS_EXPORT(ledger::post_t)
BOOST_CLASS_EXPORT(ledger::xact_base_t)
BOOST_CLASS_EXPORT(ledger::xact_t)
BOOST_CLASS_EXPORT(ledger::auto_xact_t)
BOOST_CLASS_EXPORT(ledger::period_xact_t)
template void ledger::journal_t::serialize(boost::archive::binary_oarchive&,
const unsigned int);
template void ledger::journal_t::serialize(boost::archive::binary_iarchive&,
const unsigned int);
namespace ledger {
void archive_t::read_header()
{
if (exists(file)) {
// Open the stream, read the version number and the list of sources
ifstream stream(file, std::ios::binary);
boost::archive::binary_iarchive iarchive(stream);
DEBUG("archive.journal", "Reading header from archive");
iarchive >> *this;
DEBUG("archive.journal",
"Version number: " << std::hex << version << std::dec);
DEBUG("archive.journal", "Number of sources: " << sources.size());
foreach (const journal_t::fileinfo_t& i, sources)
DEBUG("archive.journal", "Loaded source: " << *i.filename);
}
}
bool archive_t::should_load(const std::list<path>& data_files)
{
std::size_t found = 0;
DEBUG("archive.journal", "Should the archive be loaded?");
if (! exists(file)) {
DEBUG("archive.journal", "No, it does not exist");
return false;
}
if (version != ARCHIVE_VERSION) {
DEBUG("archive.journal", "No, it fails the version check");
return false;
}
if (data_files.empty()) {
DEBUG("archive.journal", "No, there were no data files!");
return false;
}
if (sources.empty()) {
DEBUG("archive.journal", "No, there were no sources!");
return false;
}
if (data_files.size() != sources.size()) {
DEBUG("archive.journal", "No, number of sources doesn't match: "
<< data_files.size() << " != " << sources.size());
return false;
}
foreach (const path& p, data_files) {
DEBUG("archive.journal", "Scanning for data file: " << p);
if (! exists(p)) {
DEBUG("archive.journal", "No, an input source no longer exists: " << p);
return false;
}
foreach (const journal_t::fileinfo_t& i, sources) {
assert(! i.from_stream);
assert(i.filename);
DEBUG("archive.journal", "Comparing against source file: " << *i.filename);
if (*i.filename == p) {
if (! exists(*i.filename)) {
DEBUG("archive.journal",
"No, a referent source no longer exists: " << *i.filename);
return false;
}
if (i.modtime != posix_time::from_time_t(last_write_time(p))) {
DEBUG("archive.journal", "No, a source's modtime has changed: " << p);
return false;
}
if (i.size != file_size(p)) {
DEBUG("archive.journal", "No, a source's size has changed: " << p);
return false;
}
found++;
}
}
}
if (found != data_files.size()) {
DEBUG("archive.journal", "No, not every source's name matched");
return false;
}
DEBUG("archive.journal", "Yes, it should be loaded!");
return true;
}
bool archive_t::should_save(shared_ptr<journal_t> journal)
{
std::list<path> data_files;
DEBUG("archive.journal", "Should the archive be saved?");
if (journal->was_loaded) {
DEBUG("archive.journal", "No, it's one we loaded before");
return false;
}
if (journal->sources.empty()) {
DEBUG("archive.journal", "No, there were no sources!");
return false;
}
foreach (const journal_t::fileinfo_t& i, journal->sources) {
if (i.from_stream) {
DEBUG("archive.journal", "No, one source was from a stream");
return false;
}
if (! exists(*i.filename)) {
DEBUG("archive.journal",
"No, a source no longer exists: " << *i.filename);
return false;
}
data_files.push_back(*i.filename);
}
if (should_load(data_files)) {
DEBUG("archive.journal", "No, because it's still loadable");
return false;
}
DEBUG("archive.journal", "Yes, it should be saved!");
return true;
}
void archive_t::save(shared_ptr<journal_t> journal)
{
INFO_START(archive, "Saved journal file cache");
ofstream archive(file, std::ios::binary);
boost::archive::binary_oarchive oa(archive);
version = ARCHIVE_VERSION;
sources = journal->sources;
foreach (const journal_t::fileinfo_t& i, sources)
DEBUG("archive.journal", "Saving source: " << *i.filename);
DEBUG("archive.journal",
"Creating archive with version " << std::hex << version << std::dec);
oa << *this;
DEBUG("archive.journal",
"Archiving journal with " << sources.size() << " sources");
oa << *journal;
INFO_FINISH(archive);
}
bool archive_t::load(shared_ptr<journal_t> journal)
{
INFO_START(archive, "Read cached journal file");
ifstream stream(file, std::ios::binary);
boost::archive::binary_iarchive iarchive(stream);
// Skip past the archive header, it was already read in before
archive_t temp;
iarchive >> temp;
iarchive >> *journal.get();
journal->was_loaded = true;
INFO_FINISH(archive);
return true;
}
} // namespace ledger
#endif // HAVE_BOOST_SERIALIZATION

105
src/archive.h Normal file
View file

@ -0,0 +1,105 @@
/*
* Copyright (c) 2003-2009, John Wiegley. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* - Neither the name of New Artisans LLC nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/**
* @defgroup report Reporting
*/
/**
* @file archive.h
* @author John Wiegley
*
* @ingroup report
*
* @brief Brief
*
* Long.
*/
#ifndef _ARCHIVE_H
#define _ARCHIVE_H
#include "journal.h"
namespace ledger {
/**
* @brief Brief
*
* Long.
*/
class archive_t
{
path file;
uint32_t version;
std::list<journal_t::fileinfo_t> sources;
public:
archive_t() {
TRACE_CTOR(archive_t, "");
}
archive_t(const path& _file)
: file(_file), version(0) {
TRACE_CTOR(archive_t, "const path&");
}
archive_t(const archive_t& ar)
: file(ar.file), version(0) {
TRACE_CTOR(archive_t, "copy");
}
~archive_t() {
TRACE_DTOR(archive_t);
}
void read_header();
bool should_load(const std::list<path>& data_files);
bool should_save(shared_ptr<journal_t> journal);
void save(shared_ptr<journal_t> journal);
bool load(shared_ptr<journal_t> journal);
#if defined(HAVE_BOOST_SERIALIZATION)
private:
/** Serialization. */
friend class boost::serialization::access;
template<class Archive>
void serialize(Archive & ar, const unsigned int /* version */) {
ar & version;
ar & sources;
}
#endif // HAVE_BOOST_SERIALIZATION
};
} // namespace ledger
#endif // _ARCHIVE_H

2
src/journal.cc
View file

@ -41,7 +41,7 @@
namespace ledger {
journal_t::journal_t()
: master(new account_t),
: master(new account_t), was_loaded(false),
commodity_pool(new commodity_pool_t)
{
TRACE_CTOR(journal_t, "");

58
src/journal.h
View file

@ -48,11 +48,10 @@
#include "utils.h"
#include "hooks.h"
#include "times.h"
namespace ledger {
typedef std::list<path> paths_list;
class commodity_pool_t;
class xact_t;
class auto_xact_t;
@ -73,11 +72,55 @@ typedef std::list<period_xact_t *> period_xacts_list;
class journal_t : public noncopyable
{
public:
account_t * master;
account_t * basket;
xacts_list xacts;
auto_xacts_list auto_xacts;
period_xacts_list period_xacts;
struct fileinfo_t
{
optional<path> filename;
std::size_t size;
datetime_t modtime;
bool from_stream;
fileinfo_t() : size(0), from_stream(true) {
TRACE_CTOR(journal_t::fileinfo_t, "");
}
fileinfo_t(const path& _filename)
: filename(_filename), from_stream(false) {
TRACE_CTOR(journal_t::fileinfo_t, "const path&");
size = file_size(*filename);
modtime = posix_time::from_time_t(last_write_time(*filename));
}
fileinfo_t(const fileinfo_t& info)
: filename(info.filename), size(info.size),
modtime(info.modtime), from_stream(info.from_stream)
{
TRACE_CTOR(journal_t::fileinfo_t, "copy");
}
~fileinfo_t() throw() {
TRACE_DTOR(journal_t::fileinfo_t);
}
#if defined(HAVE_BOOST_SERIALIZATION)
private:
/** Serialization. */
friend class boost::serialization::access;
template<class Archive>
void serialize(Archive & ar, const unsigned int /* version */) {
ar & filename;
ar & size;
ar & modtime;
ar & from_stream;
}
#endif // HAVE_BOOST_SERIALIZATION
};
account_t * master;
account_t * basket;
xacts_list xacts;
auto_xacts_list auto_xacts;
period_xacts_list period_xacts;
std::list<fileinfo_t> sources;
bool was_loaded;
shared_ptr<commodity_pool_t> commodity_pool;
hooks_t<xact_finalizer_t, xact_t> xact_finalize_hooks;
@ -123,6 +166,7 @@ private:
ar & xacts;
ar & auto_xacts;
ar & period_xacts;
ar & sources;
}
#endif // HAVE_BOOST_SERIALIZATION
};

96
src/session.cc
View file

@ -37,7 +37,7 @@
#include "journal.h"
#include "iterators.h"
#include "filters.h"
#include "pstream.h"
#include "archive.h"
namespace ledger {
@ -105,6 +105,9 @@ std::size_t session_t::read_journal(const path& pathname,
std::size_t session_t::read_data(const string& master_account)
{
bool populated_data_files = false;
bool populated_price_db = false;
if (HANDLER(file_).data_files.empty()) {
path file;
if (const char * home_var = std::getenv("HOME"))
@ -114,6 +117,8 @@ std::size_t session_t::read_data(const string& master_account)
HANDLER(file_).data_files.push_back(file);
else
throw_(parse_error, "No journal file was specified (please use -f)");
populated_data_files = true;
}
std::size_t xact_count = 0;
@ -122,43 +127,75 @@ std::size_t session_t::read_data(const string& master_account)
if (! master_account.empty())
acct = journal->find_account(master_account);
if (HANDLED(price_db_)) {
path price_db_path = resolve_path(HANDLER(price_db_).str());
if (exists(price_db_path) && read_journal(price_db_path) > 0)
optional<path> price_db_path;
if (HANDLED(price_db_))
price_db_path = resolve_path(HANDLER(price_db_).str());
optional<archive_t> cache;
if (HANDLED(cache_) && master_account.empty()) {
cache = archive_t(HANDLED(cache_).str());
cache->read_header();
if (price_db_path) {
HANDLER(file_).data_files.push_back(*price_db_path);
populated_price_db = true;
}
}
if (! (cache &&
cache->should_load(HANDLER(file_).data_files) &&
cache->load(journal))) {
if (price_db_path) {
if (exists(*price_db_path) && read_journal(*price_db_path) > 0)
throw_(parse_error, _("Transactions not allowed in price history file"));
}
journal->sources.push_back(journal_t::fileinfo_t(*price_db_path));
HANDLER(file_).data_files.remove(*price_db_path);
}
foreach (const path& pathname, HANDLER(file_).data_files) {
path filename = resolve_path(pathname);
if (filename == "-") {
// To avoid problems with stdin and pipes, etc., we read the entire
// file in beforehand into a memory buffer, and then parcel it out
// from there.
std::ostringstream buffer;
foreach (const path& pathname, HANDLER(file_).data_files) {
path filename = resolve_path(pathname);
if (filename == "-") {
// To avoid problems with stdin and pipes, etc., we read the entire
// file in beforehand into a memory buffer, and then parcel it out
// from there.
std::ostringstream buffer;
while (std::cin.good() && ! std::cin.eof()) {
char line[8192];
std::cin.read(line, 8192);
std::streamsize count = std::cin.gcount();
buffer.write(line, count);
while (std::cin.good() && ! std::cin.eof()) {
char line[8192];
std::cin.read(line, 8192);
std::streamsize count = std::cin.gcount();
buffer.write(line, count);
}
buffer.flush();
std::istringstream buf_in(buffer.str());
xact_count += read_journal(buf_in, "/dev/stdin", acct);
journal->sources.push_back(journal_t::fileinfo_t());
}
buffer.flush();
else if (exists(filename)) {
xact_count += read_journal(filename, acct);
journal->sources.push_back(journal_t::fileinfo_t(filename));
}
else {
throw_(parse_error, _("Could not read journal file '%1'") << filename);
}
}
std::istringstream buf_in(buffer.str());
assert(xact_count == journal->xacts.size());
xact_count += read_journal(buf_in, "/dev/stdin", acct);
}
else if (exists(filename)) {
xact_count += read_journal(filename, acct);
}
else {
throw_(parse_error, _("Could not read journal file '%1'") << filename);
}
if (cache && cache->should_save(journal))
cache->save(journal);
}
if (populated_data_files)
HANDLER(file_).data_files.clear();
else if (populated_price_db)
HANDLER(file_).data_files.remove(*price_db_path);
VERIFY(journal->valid());
return xact_count;
return journal->xacts.size();
}
void session_t::read_journal_files()
@ -219,6 +256,9 @@ option_t<session_t> * session_t::lookup_option(const char * p)
case 'a':
OPT_(account_); // -a
break;
case 'c':
OPT(cache_);
break;
case 'd':
OPT(download); // -Q
break;

2
src/session.h
View file

@ -103,6 +103,7 @@ public:
void report_options(std::ostream& out)
{
HANDLER(account_).report(out);
HANDLER(cache_).report(out);
HANDLER(download).report(out);
HANDLER(file_).report(out);
HANDLER(input_date_format_).report(out);
@ -120,6 +121,7 @@ public:
*/
OPTION(session_t, account_); // -a
OPTION(session_t, cache_);
OPTION(session_t, download); // -Q
OPTION__

14
src/system.hh.in
View file

@ -189,20 +189,14 @@ typedef std::ostream::pos_type ostream_pos_type;
#include <boost/date_time/posix_time/time_serialize.hpp>
#include <boost/date_time/gregorian/greg_serialize.hpp>
BOOST_CLASS_IMPLEMENTATION(boost::filesystem::path, boost::serialization::primitive_type)
#ifndef BOOST_NO_STD_WSTRING
BOOST_CLASS_IMPLEMENTATION(boost::filesystem::wpath, boost::serialization::primitive_type)
#endif
namespace boost {
namespace serialization {
template <class Archive, class String, class Traits>
void serialize(Archive& ar, boost::filesystem::basic_path<String, Traits>& p,
const unsigned int)
template <class Archive>
void serialize(Archive& ar, boost::filesystem::path& p, const unsigned int)
{
String s;
if (Archive::is_saving::value)
std::string s;
if (Archive::is_saving::value)
s = p.string();
ar & s;

2
tools/Makefile.am
View file

@ -59,6 +59,7 @@ libledger_data_la_SOURCES = \
src/timelog.cc \
src/textual.cc \
src/journal.cc \
src/archive.cc \
src/account.cc \
src/xact.cc \
src/post.cc \
@ -118,6 +119,7 @@ pkginclude_HEADERS = \
src/xact.h \
src/account.h \
src/journal.h \
src/archive.h \
src/timelog.h \
src/iterators.h \
src/compare.h \