X-Git-Url: https://oss.titaniummirror.com/gitweb?p=msp430-binutils.git;a=blobdiff_plain;f=debian%2Ftest-suite-compare.py;fp=debian%2Ftest-suite-compare.py;h=f81ec4ee737ed1060fd2eb3dfbad048f35290eb6;hp=0000000000000000000000000000000000000000;hb=47f73751c91775676bb03af53e83c7f49dfda44f;hpb=6df9443a374e2b81278c61b8afc0a1eef7db280b diff --git a/debian/test-suite-compare.py b/debian/test-suite-compare.py new file mode 100644 index 0000000..f81ec4e --- /dev/null +++ b/debian/test-suite-compare.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python + +# Quick'n'dirty regression check for dejagnu testsuites +# Copyright (C) 2003, 2004, 2005, 2006, 2007 James Troup + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU;5B General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +################################################################################ + +import optparse +import os +import sys + +################################################################################ + +def fubar(msg, exit_code=1): + sys.stderr.write("E: %s\n" % (msg)) + sys.exit(exit_code) + +def warn(msg): + sys.stderr.write("W: %s\n" % (msg)) + +def info(msg): + sys.stderr.write("I: %s\n" % (msg)) + +################################################################################ + +def read_testsummary(filename): + results = {} + file = open(filename) + for line in file.readlines(): + if not line: + continue + if line.startswith("Running"): + s = line.split() + if "/" in s[1]: + x = s[1] + if x.find("/testsuite/") == -1: + fubar("Can't find /testsuite/ in '%s'." % (x)) + # 'Running /home/james/debian/packages/binutils/binutils-2.14.90.0.7/gas/testsuite/gas/hppa/unsorted/unsorted.exp ...' -> 'gas/hppa/unsorted/unsorted.exp' + # ... since using basename() isn't dupe safe. + section = x[x.find("/testsuite/"):].replace("/testsuite/","").split()[0] + + # Tests can be duplicated, e.g. hppa/basic/basic.exp + # is run twice, once for hppa-linux and once for + # hppa64-linux. This is of course a horrible bodge, + # but I can't think of anything trivial and better off + # hand. + + if results.has_key(section): + extra = 1 + too_many = 10 + while results.has_key(section) and extra < too_many: + section = "%s.%s" % (section, extra) + extra += 1 + if extra >= too_many: + fubar("gave up trying to unduplicate %s." % (section)) + + results[section] = {} + continue + + got_state = 0 + for state in [ "PASS", "XPASS", "FAIL", "XFAIL", "UNRESOLVED", + "UNTESTED", "UNSUPPORTED" ]: + if line.startswith(state): + s = line.split(':') + state = s[0] + test = ':'.join(s[1:]).strip() + if results.has_key(test): + warn("%s/%s is duplicated." % (section, test)) + results[section][test] = state + got_state = 1 + break + + if got_state: + continue + + return results + +################################################################################ + +def compare_results(old, new): + total_num = 0 + pass_count = 0 + fail_count = 0 + xfail_count = 0 + untested_count = 0 + regression_count = 0 + progression_count = 0 + change_count = 0 + + for section in new.keys(): + for test in new[section].keys(): + state = new[section][test] + + # Stats pr0n + total_num += 1 + if state == "PASS" or state == "XPASS": + pass_count += 1 + elif state == "FAIL" or state == "UNRESOLVED": + fail_count += 1 + elif state == "XFAIL": + xfail_count += 1 + elif state == "UNTESTED": + untested_count += 1 + + # Compare to old + if not old.has_key(section): + continue + if not old[section].has_key(test): + continue + old_state = old[section][test] + if state == "PASS": + if old_state != "PASS": + progression_count += 1 + info("[%s] progression (%s -> %s): %s" % (section, old_state, state, test)) + elif state == "XPASS": + if old_state != "XPASS" and old_state != "PASS": + progression_count += 1 + warn("[%s] %s: %s" % (section, state, test)) + elif state == "FAIL": + if old_state != "FAIL": + regression_count += 1 + warn("[%s] REGRESSION (%s -> %s): %s" % (section, old_state, state, test)) + elif state == "XFAIL": + if old_state != "XFAIL": + change_count += 1 + info("[%s] change (%s -> %s): %s" % (section, old_state, state, test)) + elif state == "UNRESOLVED": + if old_state != "UNRESOLVED" and old_state != "FAIL": + regression_count += 1 + warn("[%s] REGRESSION (%s -> %s): %s" % (section, old_state, state, test)) + if old_state == "FAIL": + change_count += 1 + info("[%s] change (%s -> %s): %s" % (section, old_state, state, test)) + elif state == "UNTESTED": + if old_state != "UNTESTED": + regression_count += 1 + warn("[%s] REGRESSION (%s -> %s): %s" % (section, old_state, state, test)) + + if regression_count: + print "%d REGRESSIONS (%.2f%%)." % (regression_count, (float(regression_count)/total_num)*100) + if progression_count: + print "%d progressions (%.2f%%)." % (progression_count, (float(progression_count)/total_num)*100) + + if change_count: + print "%d changes (%.2f%%)." % (change_count, (float(change_count)/total_num)*100) + + print "%d tests: %d pass (%.2f%%), %d fail (%.2f%%), %d xfail (%.2f%%) %d untested (%.2f%%)." \ + % (total_num, pass_count, (float(pass_count)/total_num)*100, + fail_count, (float(fail_count)/total_num)*100, + xfail_count, (float(xfail_count)/total_num)*100, + untested_count, (float(untested_count)/total_num)*100) + +################################################################################ + +def compare_multiple(directory, first_version, second_version): + architectures = [ "alpha", "arm", "hppa", "i386", "ia64", "mips", + "m68k", "mipsel", "powerpc", "s390", "sparc" ] + + for arch in architectures: + print "*********************************** %s ******************************" % (arch) + second_filename = "%s/%s_%s" % (directory, second_version, arch) + if not os.path.exists(second_filename): + print " -- NOT AVAILABLE --" + continue + + new = read_testsummary(second_filename) + first_filename = "%s/%s_%s" % (directory, first_version, arch) + old = read_testsummary(first_filename) + compare_results(old, new) + +################################################################################ + +def init(): + """Initalization, including parsing of options.""" + + usage = """usage: %prog [OPTIONS] +compare (binutils) dejagnu testsuite results. + +Example usage: + + test-suite-compare.py binutils-2.17/test-summary binutils-2.18/test-summary + +Or to compare across all architectures (with test results stored in a +'test-summary' directory): + + test-suite-compare.py -mtest-summary 2.17-3 2.18-1""" + parser = optparse.OptionParser(usage) + parser.add_option("-m", "--multiple", dest="multiple", + nargs=1, type="string", + help="compare multiple architectures") + (options, args) = parser.parse_args() + + if len(args) > 2 or len(args) < 2: + parser.error("takes 2 arguments (old and new)") + (old_version, new_version) = args + + return options, old_version, new_version + +################################################################################ + +def main(): + (options, old_version, new_version) = init() + if options.multiple: + compare_multiple(options.multiple, old_version, new_version) + else: + old = read_testsummary(old_version) + new = read_testsummary(new_version) + compare_results(old, new) + +################################################################################ + +if __name__ == '__main__': + main()