[doc] Add CheckTests to ctest

to check whether all available ledger options documented
and are being tested.

Signed-off-by: Alexis Hildebrandt <afh@surryhill.net>

doc/ledger.1

@@ -50,7 +50,6 @@ Report in terms of cost basis, not amount or value.  This is the only form of
 report which is guaranteed to always balance to zero, when no
 .Ar report-query
 is specified.
-.It Fl \-collapse Pq Fl n
 Only show totals for the top-most accounts.
 .It Fl \-empty Pq Fl E
 Also show accounts whose total is zero.
@@ -330,7 +329,8 @@ report.
 .It Fl \-anon
 Anonymize registry output, mostly for sending in bug reports.
 .It Fl \-args-only
-.It Fl \-auto-match
+Ignore init files and environment variables for the ledger run.
+.\".It Fl \-auto-match
 .It Fl \-aux-date
 Show auxiliary dates for all calculations.
 Alias for
@@ -342,7 +342,7 @@ running totals.
 Specify the format to use for the
 .Nm balance
 report.
-.It Fl \-base
+.\".It Fl \-base
 .It Fl \-basis Pq Fl B
 Report the cost basis on all posting.
 Alias for
@@ -364,7 +364,7 @@ Specify the format to use for the
 report.
 .It Fl \-by-payee Pq Fl P
 Group postings in the register report by common payee names.
-.It Fl \-cache Ar FILE
+.\".It Fl \-cache Ar FILE
 .It Fl \-check-payees
 Enable strict and pedantic checking for payees as well as accounts,
 commodities and tags.
@@ -419,7 +419,7 @@ Specify the format ledger should use to print dates.
 Specify the width, in characters, of the date column in the
 .Nm register
 report.
-.It Fl \-day-break
+.\".It Fl \-day-break
 .It Fl \-dc
 Display register or balance in debit/credit format If you use
 .Fl \-dc
@@ -486,16 +486,17 @@ Related to the
 .Nm equity
 command.  Gives current account balances in the form of a register
 report.
-.It Fl \-exact
+.\".It Fl \-exact
 .It Fl \-exchange Ar COMMODITY Oo , COMM, ... Oc Pq Fl X
 Display values in terms of the given
 .Ar COMMODITY .
 The latest available price is used.
-.It Fl \-explicit
+.\".It Fl \-explicit
 .It Fl \-file Ar FILE
 Read
 .Ar FILE
 as a ledger file.
+.\".It Fl \-full-help
 .It Fl \-first Ar INT
 Print the first
 .Ar INT
@@ -559,8 +560,12 @@ Alias for
 Print a summary of all the options, and what they are used for.  This
 can be a handy way to remember which options do what.  This help screen
 is also printed if ledger is run without a command.
+.\".It Fl \-help-calc
+.\".It Fl \-help-comm
+.\".It Fl \-help-disp
 .It Fl \-immediate
 Instruct ledger to evaluate calculations immediately rather than lazily.
+.\".It Fl \-import
 .It Fl \-init-file Ar FILE
 Causes
 .Nm FILE
@@ -569,7 +574,11 @@ This file may not contain any postings, but it may contain option
 settings.  To specify options in the init file, use the same syntax as
 the command-line, but put each option on its own line.
 .It Fl \-inject Ar STR
-TODO
+Use
+.Ar STR
+amounts in calculations.  In case you know
+what amount a transaction should be, but the actual transaction has the
+wrong value you can use metadata STR to specify the expected amount.
 .It Fl \-input-date-format Ar DATEFMT
 Specify the input date format for journal entries.
 .It Fl \-invert
@@ -597,7 +606,7 @@ purchased.
 .It Fl \-lots
 Report the date and price at which each commodity was purchased in
 a balance report.
-.It Fl \-lots-actual
+.\".It Fl \-lots-actual
 .It Fl \-market Pq Fl V
 Use the latest market value for all commodities.
 .It Fl \-master-account Ar STR
@@ -617,6 +626,8 @@ Shorthand for
 Aliases are completely ignored.
 .It Fl \-no-color
 Suppress any color TTY output.
+.\".It Fl \-no-pager
+Suppress any color TTY output.
 .It Fl \-no-rounding
 Don't output
 .Nm <Rounding>
@@ -672,7 +683,7 @@ For a balance report only those transactions will be accounted in the
 final balances.
 .It Fl \-period-sort
 Sort the posting within transactions using the given value expression.
-.It Fl \-permissive
+.\".It Fl \-permissive
 .It Fl \-pivot Ar STR
 Produce a balance pivot report
 .Nm around
@@ -706,7 +717,11 @@ considered to be fresh enough.
 Alias for
 .Fl \-leeway Ar INT Pq Fl Z
 .It Fl \-prices-format Ar FMT
+Set the format for the
+.Nm prices
+report.
 .It Fl \-pricedb-format Ar FMT
+Set the format expected for the historical price file.
 .It Fl \-primary-date
 Show primary dates for all calculations. Alias for
 .Fl \-actual-dates
@@ -742,10 +757,10 @@ Show all postings in a transaction, similar to
 but show
 .Nm both sides
 of each transaction.
-.It Fl \-revalued
-.It Fl \-revalued-only
-.It Fl \-revalued-total Ar EXPR
-.It Fl \-rich-data
+.\".It Fl \-revalued
+.\".It Fl \-revalued-only
+.\".It Fl \-revalued-total Ar EXPR
+.\".It Fl \-rich-data
 .It Fl \-seed Ar INT
 Set the random seed to
 .Ar INT
@@ -756,7 +771,7 @@ command.  Used as part of development testing.
 Execute a ledger script.
 .It Fl \-sort Ar EXPR Pq Fl S
 Sort the register report based on the value expression given to sort.
-.It Fl \-sort-all
+.\".It Fl \-sort-all
 .It Fl \-sort-xacts
 Sort the posting within transactions using the given value expression.
 .It Fl \-start-of-week Ar STR
@@ -773,7 +788,10 @@ Report only the last
 .Ar INT
 entries. Only useful on a register report. Alias for
 .Fl \-last Ar INT
-.It Fl \-time-report
+.It Fl \-time-colon
+Display the value for a seconds based commodity as real hours and minutes,
+thus 8100 seconds will be displayed as 2:15h instead of 2.25h.
+.\".It Fl \-time-report
 .It Fl \-total Ar EXPR Pq Fl T
 Define a value expression used to calculate the total in reports.
 .It Fl \-total-data Pq Fl J
@@ -821,7 +839,7 @@ precision.
 Shows the values used by each tag when used in combination with the
 .Nm tags
 command.
-.It Fl \-value-expr Ar EXPR
+.\".It Fl \-value-expr Ar EXPR
 .It Fl \-verbose
 Print detailed information on the execution of Ledger.
 .It Fl \-verify
@@ -829,7 +847,7 @@ Enable additional assertions during run-time. This causes a significant
 slowdown.  When combined with
 .Fl \-debug Ar CODE
 ledger will produce memory trace information.
-.It Fl \-verify-memory
+.\".It Fl \-verify-memory
 .It Fl \-version
 Print version information and exit.
 .It Fl \-weekly Pq Fl W

src/global.cc

@@ -321,9 +321,6 @@ option_t<global_scope_t> * global_scope_t::lookup_option(const char * p)
     break;
   case 'h':
     OPT_(help);
-    else OPT(help_calc);
-    else OPT(help_comm);
-    else OPT(help_disp);
     break;
   case 'i':
     OPT(init_file_);

src/global.h

@@ -147,9 +147,6 @@ See LICENSE file included with the distribution for details and disclaimer.");
 
   OPTION_(global_scope_t, full_help, DO() { parent->visit_man_page(); }); // -H
   OPTION_(global_scope_t, help,      DO() { parent->visit_man_page(); }); // -h
-  OPTION_(global_scope_t, help_calc, DO() { parent->visit_man_page(); });
-  OPTION_(global_scope_t, help_comm, DO() { parent->visit_man_page(); });
-  OPTION_(global_scope_t, help_disp, DO() { parent->visit_man_page(); });
 
   OPTION__
   (global_scope_t, init_file_, // -i

test/CMakeLists.txt

@@ -42,10 +42,15 @@ if (PYTHONINTERP_FOUND)
   foreach(TestFile ${${_class}_TESTS})
     get_filename_component(TestFile_Name ${TestFile} NAME_WE)
     add_test(NAME ${_class}Test_${TestFile_Name}
-      COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/test/DocTests.py
+      COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/test/${_class}.py
       --ledger $<TARGET_FILE:ledger> --file ${TestFile})
     set_target_properties(check PROPERTIES DEPENDS ${_class}Test_${TestFile_Name})
   endforeach()
+
+  set(_class CheckTests)
+  add_test(NAME ${_class}
+    COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/test/${_class}.py
+    --ledger $<TARGET_FILE:ledger> --source ${PROJECT_SOURCE_DIR})
 endif()
 
 ### CMakeLists.txt ends here

test/CheckTests.py

@@ -1,87 +1,136 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function
 
 import sys
 import re
 import os
+import argparse
 
 from os.path import *
 from subprocess import Popen, PIPE
 
-ledger_binary = sys.argv[1]
-source_topdir = sys.argv[2]
+class CheckTests (object):
+  def __init__(self, args):
+    self.ledger     = os.path.abspath(args.ledger)
+    self.source     = os.path.abspath(args.source)
 
-documented_options = []
-for line in open(join(source_topdir, 'doc', 'ledger.1')):
-    match = re.match('\.It Fl \\\\-([-A-Za-z]+)', line)
-    if match:
-        option = match.group(1)
-        if option not in documented_options:
-            documented_options.append(option)
+    self.missing_baseline_tests = set()
+    self.missing_texi_options = set()
+    self.unknown_texi_options = set()
+    self.missing_man_options = set()
+    self.unknown_man_options = set()
 
-pipe   = Popen('%s --debug option.names parse true' % ledger_binary,
-               shell=True, stdout=PIPE, stderr=PIPE)
-errors = 0
+    self.untested_options = [
+        'anon',
+        'args-only',
+        'cache',
+        'debug',
+        'download',
+        'file',
+        'force-color',
+        'force-pager',
+        'full-help',
+        'help',
+        'help-calc',
+        'help-comm',
+        'help-disp',
+        'import',
+        'init-file',
+        'no-color',
+        'options',
+        'price-db',
+        'price-exp',
+        'revalued-total',
+        'script',
+        'seed',
+        'trace',
+        'verbose',
+        'verify',
+        'version'
+    ]
 
-untested_options = [
-    'anon',
-    'args-only',
-    'cache',
-    'debug',
-    'download',
-    'file',
-    'force-color',
-    'force-pager',
-    'full-help',
-    'help',
-    'help-calc',
-    'help-comm',
-    'help-disp',
-    'import',
-    'init-file',
-    'no-color',
-    'options',
-    'price-db',
-    'price-exp',
-    'revalued-total',
-    'script',
-    'seed',
-    'trace',
-    'verbose',
-    'verify',
-    'version'
-]
+    self.known_alternates = [
+        'cost',
+        'first',
+        'import',
+        'last',
+        'leeway',
+        'period-sort'
+    ]
 
-for line in pipe.stderr:
-    match = re.search('\[DEBUG\] Option: (.*)', line)
-    if match:
-        option = match.group(1)
+  def find_options(self, pattern, filename):
+    regex = re.compile(pattern)
+    return {match.group(1) for match in {regex.match(line) for line in open(filename)} if match}
 
-        option = re.sub('_', '-', option)
-        option = re.sub('-$', '', option)
+  def main(self):
+    man_options = self.find_options('\.It Fl \\\\-([-A-Za-z]+)',
+                                    join(self.source, 'doc', 'ledger.1'))
 
-        if option not in untested_options and \
-           not exists(join(source_topdir, 'test', 'baseline',
-                           'opt-%s.test' % option)):
-            print "Baseline test missing for --%s" % option
-            errors += 1
+    texi_options = self.find_options('@item --([-A-Za-z]+).*@c option',
+                                     join(self.source, 'doc', 'ledger3.texi'))
 
-        if option not in documented_options:
-            print "Man page entry missing for --%s" % option
-            errors += 1
-        else:
-            documented_options.remove(option)
+    pipe   = Popen('%s --debug option.names parse true' % self.ledger,
+                   shell=True, stdout=PIPE, stderr=PIPE)
+    regex = re.compile('\[DEBUG\] Option: (.*)')
+    for line in filter(regex.search, [line.decode() for line in pipe.stderr]):
+      match = regex.search(line)
+      option = match.group(1)
+      option = re.sub('_', '-', option)
+      option = re.sub('-$', '', option)
 
-known_alternates = [
-    'cost',
-    'first',
-    'import',
-    'last',
-    'leeway',
-    'period-sort'
-]
+      if option not in self.untested_options and \
+         not exists(join(self.source, 'test', 'baseline',
+                         'opt-%s.test' % option)):
+          self.missing_baseline_tests.add(option)
 
-for option in documented_options:
-    if option not in known_alternates:
-        print "Man page entry for unknown option --%s" % option
+      if option not in man_options:
+          self.missing_man_options.add(option)
+      else:
+          man_options.remove(option)
 
-sys.exit(errors)
+      if option not in texi_options:
+          self.missing_texi_options.add(option)
+      else:
+          texi_options.remove(option)
+
+    self.unknown_man_options = [option for option in man_options if option not in self.known_alternates]
+    self.unknown_texi_options = [option for option in texi_options if option not in self.known_alternates]
+
+    sep = "\n  --"
+    if len(self.missing_baseline_tests):
+      print("Missing Baseline test for:%s%s\n" % (sep, sep.join(sorted(list(self.missing_baseline_tests)))))
+    if len(self.missing_man_options):
+      print("Missing man page entries for:%s%s\n" % (sep, sep.join(sorted(list(self.missing_man_options)))))
+    if len(self.missing_texi_options):
+      print("Missing texi entries for:%s%s\n" % (sep, sep.join(sorted(list(self.missing_texi_options)))))
+    if len(self.unknown_man_options):
+      print("Man page entry for unknown options:%s%s" % (sep, sep.join(sorted(list(self.unknown_man_options)))))
+    if len(self.unknown_texi_options):
+      print("Texi entry for unknown option:%s%s" % (sep, sep.join(sorted(list(self.unknown_texi_options)))))
+
+    errors = len(self.missing_baseline_tests) + len(self.missing_man_options) + len(self.missing_baseline_tests)
+    return errors
+
+if __name__ == "__main__":
+  def getargs():
+    parser = argparse.ArgumentParser(prog='CheckTests', description='Check that ledger options are tested and documented', prefix_chars='-')
+    parser.add_argument('-l', '--ledger',
+        dest='ledger',
+        type=str,
+        action='store',
+        required=True,
+        help='the path to the ledger executable to test with')
+    parser.add_argument('-s', '--source',
+        dest='source',
+        type=str,
+        action='store',
+        required=True,
+        help='the path to the top level ledger source directory')
+    return parser.parse_args()
+
+  args = getargs()
+  script = CheckTests(args)
+  status = script.main()
+  sys.exit(status)

test/GenerateTests.py

@@ -79,6 +79,8 @@ def generation_test(seed):
     p_cout_bal.stdin.close()
 
     cout_lines = harness.readlines(p_cout_bal.stdout)
+    if len(cout_lines) == 0:
+      return False
     #norm_cout_lines = [normalize(line) for line in cout_lines]
 
     if not harness.wait(p_cout_bal, msg=("Stdout balance for seed %d failed:" % seed)):
@@ -89,6 +91,8 @@ def generation_test(seed):
     p_print_bal.stdin.close()
 
     print_lines = harness.readlines(p_print_bal.stdout)
+    if len(print_lines) == 0:
+      return False
 
     if not harness.wait(p_print_bal, msg=("Print balance for seed %d failed:" % seed)):
         return False

test/RegressTests.py

@@ -57,6 +57,9 @@ class RegressFile(object):
         in_error  = False
 
         line = self.fd.readline()
+        if not line:
+          print >>sys.stderr, "WARNING: Empty testfile detected: %s" % (self.filename)
+          return False
         #print "line =", line
         while line:
             if line.startswith("test "):

test/baseline/opt-no-titles.test

@@ -0,0 +1,10 @@
+test reg -f test/input/drewr3.dat --no-titles --group-by payee reg food
+11-Jan-02 Grocery Store         Expense:Food:Groceries      $ 65.00      $ 65.00
+11-Jan-19 Grocery Store         Expense:Food:Groceries      $ 44.00     $ 109.00
+10-Dec-20 Organic Co-op         Expense:Food:Groceries      $ 37.50      $ 37.50
+                                Expense:Food:Groceries      $ 37.50      $ 75.00
+                                Expense:Food:Groceries      $ 37.50     $ 112.50
+                                Expense:Food:Groceries      $ 37.50     $ 150.00
+                                Expense:Food:Groceries      $ 37.50     $ 187.50
+                                Expense:Food:Groceries      $ 37.50     $ 225.00
+end test

test/baseline/opt-values.test

@@ -0,0 +1,6 @@
+test tags -f test/input/drewr3.dat --values
+hastag: not block
+hastag: true
+nestedtag: true
+nobudget
+end test