0e691e76db
With this change, 97% of the tests pass. See the build on appveyor for more info: https://ci.appveyor.com/project/Evan/ledger/build/build-49 I'll follow up with another PR to fix some of the remaining broken tests
268 lines
9.2 KiB
Python
Executable file
268 lines
9.2 KiB
Python
Executable file
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
|
|
import os
|
|
import re
|
|
import sys
|
|
import shlex
|
|
import hashlib
|
|
import argparse
|
|
import subprocess
|
|
|
|
from difflib import unified_diff
|
|
|
|
class DocTests:
|
|
def __init__(self, args):
|
|
scriptpath = os.path.dirname(os.path.realpath(__file__))
|
|
self.ledger = os.path.abspath(args.ledger)
|
|
self.sourcepath = os.path.abspath(args.file)
|
|
self.verbose = args.verbose
|
|
self.tests = args.examples
|
|
|
|
self.examples = dict()
|
|
self.test_files = list()
|
|
self.testin_token = 'command'
|
|
self.testout_token = 'output'
|
|
self.testdat_token = 'input'
|
|
self.testfile_token = 'file'
|
|
self.validate_token = 'validate'
|
|
self.validate_cmd_token = 'validate-command'
|
|
self.validate_dat_token = 'validate-data'
|
|
self.testwithdat_token = 'with_input'
|
|
self.testwithfile_token = 'with_file'
|
|
|
|
def read_example(self):
|
|
endexample = re.compile(r'^@end\s+smallexample\s*$')
|
|
example = str()
|
|
while True:
|
|
line = self.file.readline()
|
|
self.current_line += 1
|
|
if len(line) <= 0 or endexample.match(line): break
|
|
# Replace special texinfo character sequences with their ASCII counterpart
|
|
example += re.sub(r'@([@{}])', r'\1', line)
|
|
return example
|
|
|
|
def test_id(self, example):
|
|
return hashlib.sha1(example.rstrip()).hexdigest()[0:7].upper()
|
|
|
|
def find_examples(self):
|
|
startexample = re.compile(r'^@smallexample\s+@c\s+(%s|%s|%s|%s)(?::([\dA-Fa-f]+|validate))?(?:,(.*))?'
|
|
% (self.testin_token, self.testout_token, self.testdat_token, self.testfile_token))
|
|
while True:
|
|
line = self.file.readline()
|
|
self.current_line += 1
|
|
if len(line) <= 0: break
|
|
|
|
startmatch = startexample.match(line)
|
|
if (startmatch):
|
|
test_begin_pos = self.file.tell()
|
|
test_begin_line = self.current_line
|
|
test_kind = startmatch.group(1)
|
|
test_id = startmatch.group(2)
|
|
test_options = dict()
|
|
for pair in re.split(r',\s*', str(startmatch.group(3))):
|
|
kv = re.split(r':\s*', pair, 2)
|
|
try:
|
|
test_options[kv[0]] = kv[1]
|
|
except IndexError:
|
|
pass
|
|
example = self.read_example()
|
|
test_end_pos = self.file.tell()
|
|
test_end_line = self.current_line
|
|
|
|
if not test_id:
|
|
print >> sys.stderr, 'Example', test_kind, 'in line', test_begin_line, 'is missing id.'
|
|
test_id = self.test_id(example)
|
|
if test_kind == self.testin_token:
|
|
print >> sys.stderr, 'Use', self.test_id(example)
|
|
elif test_kind == self.testin_token and test_id != self.validate_token and test_id != self.test_id(example):
|
|
print >> sys.stderr, 'Expected test id', test_id, 'for example' \
|
|
, test_kind, 'on line', test_begin_line, 'to be', self.test_id(example)
|
|
|
|
if test_id == self.validate_token:
|
|
test_id = "Val-" + str(test_begin_line)
|
|
if test_kind == self.testin_token:
|
|
test_kind = self.validate_cmd_token
|
|
elif test_kind == self.testdat_token:
|
|
test_kind = self.validate_dat_token
|
|
try:
|
|
self.examples[test_id]
|
|
except KeyError:
|
|
self.examples[test_id] = dict()
|
|
|
|
try:
|
|
example = self.examples[test_id][test_kind][test_kind] + example
|
|
except KeyError:
|
|
pass
|
|
|
|
self.examples[test_id][test_kind] = {
|
|
'bpos': test_begin_pos,
|
|
'epos': test_end_pos,
|
|
'blin': test_begin_line,
|
|
'elin': test_end_line,
|
|
'opts': test_options,
|
|
test_kind: example,
|
|
}
|
|
|
|
def parse_command(self, test_id, example):
|
|
validate_command = False
|
|
try:
|
|
command = example[self.testin_token][self.testin_token]
|
|
command = re.sub(r'\\\n', '', command)
|
|
except KeyError:
|
|
if self.validate_dat_token in example:
|
|
command = '$ ledger bal'
|
|
elif self.validate_cmd_token in example:
|
|
validate_command = True
|
|
command = example[self.validate_cmd_token][self.validate_cmd_token]
|
|
else:
|
|
return None
|
|
|
|
command = filter(lambda x: x != '\n', shlex.split(command))
|
|
if command[0] == '$': command.remove('$')
|
|
index = command.index('ledger')
|
|
command[index] = self.ledger
|
|
for i,argument in enumerate(shlex.split('--args-only --columns 80')):
|
|
command.insert(index+i+1, argument)
|
|
|
|
try:
|
|
findex = command.index('-f')
|
|
except ValueError:
|
|
try:
|
|
findex = command.index('--file')
|
|
except ValueError:
|
|
findex = index+1
|
|
command.insert(findex, '--file')
|
|
if validate_command:
|
|
command.insert(findex+1, 'sample.dat')
|
|
else:
|
|
command.insert(findex+1, test_id + '.dat')
|
|
return (command, findex+1)
|
|
|
|
def test_examples(self):
|
|
failed = set()
|
|
tests = self.examples.keys()
|
|
if self.tests:
|
|
tests = list(set(self.tests).intersection(tests))
|
|
temp = list(set(self.tests).difference(tests))
|
|
if len(temp) > 0:
|
|
print >> sys.stderr, 'Skipping non-existent examples: %s' % ', '.join(temp)
|
|
|
|
for test_id in tests:
|
|
validation = False
|
|
if self.validate_dat_token in self.examples[test_id] or self.validate_cmd_token in self.examples[test_id]:
|
|
validation = True
|
|
example = self.examples[test_id]
|
|
try:
|
|
(command, findex) = self.parse_command(test_id, example)
|
|
except TypeError:
|
|
failed.add(test_id)
|
|
continue
|
|
|
|
output = example.get(self.testout_token, {}).get(self.testout_token)
|
|
input = example.get(self.testdat_token, {}).get(self.testdat_token)
|
|
if not input:
|
|
with_input = example.get(self.testin_token, {}).get('opts', {}).get(self.testwithdat_token)
|
|
input = self.examples.get(with_input, {}).get(self.testdat_token, {}).get(self.testdat_token)
|
|
if not input:
|
|
input = example.get(self.validate_dat_token, {}).get(self.validate_dat_token)
|
|
|
|
if command and (output != None or validation):
|
|
test_file_created = False
|
|
if findex:
|
|
scriptpath = os.path.dirname(os.path.realpath(__file__))
|
|
test_input_dir = os.path.join(scriptpath, '..', 'test', 'input')
|
|
test_file = command[findex]
|
|
if not os.path.exists(test_file):
|
|
if input:
|
|
test_file_created = True
|
|
with open(test_file, 'w') as f:
|
|
f.write(input)
|
|
elif os.path.exists(os.path.join(test_input_dir, test_file)):
|
|
command[findex] = os.path.join(test_input_dir, test_file)
|
|
try:
|
|
convert_idx = command.index('convert')
|
|
convert_file = command[convert_idx+1]
|
|
convert_data = example[self.testfile_token][self.testfile_token]
|
|
if not os.path.exists(convert_file):
|
|
with open(convert_file, 'w') as f:
|
|
f.write(convert_data)
|
|
except ValueError:
|
|
pass
|
|
error = None
|
|
try:
|
|
verify = subprocess.check_output(command, stderr=subprocess.STDOUT)
|
|
if sys.platform == 'win32':
|
|
verify = verify.replace('\r\n', '\n')
|
|
valid = (output == verify) or (not error and validation)
|
|
except subprocess.CalledProcessError, e:
|
|
error = e.output
|
|
valid = False
|
|
failed.add(test_id)
|
|
if valid and test_file_created:
|
|
os.remove(test_file)
|
|
if self.verbose > 0:
|
|
print test_id, ':', 'Passed' if valid else 'FAILED: {}'.format(error) if error else 'FAILED'
|
|
else:
|
|
sys.stdout.write('.' if valid else 'E')
|
|
|
|
if not (valid or error):
|
|
failed.add(test_id)
|
|
if self.verbose > 1:
|
|
print ' '.join(command)
|
|
if not validation:
|
|
for line in unified_diff(output.split('\n'), verify.split('\n'), fromfile='generated', tofile='expected'):
|
|
print(line)
|
|
print
|
|
else:
|
|
if self.verbose > 0:
|
|
print test_id, ':', 'Skipped'
|
|
else:
|
|
sys.stdout.write('X')
|
|
|
|
if not self.verbose:
|
|
print
|
|
if len(failed) > 0:
|
|
print "\nThe following examples failed:"
|
|
print " ", "\n ".join(failed)
|
|
return len(failed)
|
|
|
|
def main(self):
|
|
self.file = open(self.sourcepath)
|
|
self.current_line = 0
|
|
self.find_examples()
|
|
failed_examples = self.test_examples()
|
|
self.file.close()
|
|
return failed_examples
|
|
|
|
if __name__ == "__main__":
|
|
def getargs():
|
|
parser = argparse.ArgumentParser(prog='DocTests',
|
|
description='Test and validate ledger examples from the texinfo manual')
|
|
parser.add_argument('-v', '--verbose',
|
|
dest='verbose',
|
|
action='count',
|
|
help='be verbose. Add -vv for more verbosity')
|
|
parser.add_argument('-l', '--ledger',
|
|
dest='ledger',
|
|
type=str,
|
|
action='store',
|
|
required=True,
|
|
help='the path to the ledger executable to test with')
|
|
parser.add_argument('-f', '--file',
|
|
dest='file',
|
|
type=str,
|
|
action='store',
|
|
required=True,
|
|
help='the texinfo documentation file to run the examples from')
|
|
parser.add_argument('examples',
|
|
metavar='EXAMPLE',
|
|
type=str,
|
|
nargs='*',
|
|
help='the examples to test')
|
|
return parser.parse_args()
|
|
|
|
args = getargs()
|
|
script = DocTests(args)
|
|
status = script.main()
|
|
sys.exit(status)
|