"""Yet another unit testing framework for Python."""

# created 2000/03/17, Greg Ward

__revision__ = "$Id: unittest.py,v 1.1 2000/10/20 19:37:18 akuchlin Exp $"

import sys, string, re
from types import *
from indented_file import IndentedFile
from traceback import print_exc, print_tb


class TestScenario:

    def __init__ (self, outfile=None):
        self.total_count = 0
        self.total_run = 0
        self.total_ok = 0
        
        self.tests_seen = None
        self.tests_run = 0
        self.tests_ok = 0
        self.test_reports = None
        self.options = None

        if outfile:
            self.out = outfile
        else:
            self.out = IndentedFile (sys.stdout, 0, 2)

    def setup (self):
        raise NotImplementedError, "subclasses must override 'setup()'"

    def shutdown (self):
        raise NotImplementedError, "subclasses must override 'shutdown()'"

    def warn (self, msg):
        sys.stderr.write ("warning: " + msg + "\n")


    def _resolve_case_names (self, case_names):
        """Given a list of test case names, resolves them into (unbound)
        method objects.  This just means look up each string in
        'case_names' in the current instance's class, prepending
        "check_" if needed.  Returns a list of unbound method objects.
        Raises ValueError if any of the names in 'case_names' don't
        resolve to a method in the current class."""
        
        klass = self.__class__
        klass_name = klass.__name__

        methods = []
        for name in case_names:
            if name[0:6] == 'check_':
                try_names = [name]
            else:
                try_names = [name, 'check_' + name]

            for try_name in try_names:
                try:
                    func = getattr (klass, try_name)
                    break
                except AttributeError:
                    continue

            else:                       # all getattr attempts failed
                if len (try_names) == 1:
                    reason = "'%s' method not found" % try_names[0]
                elif len (try_names) == 2:
                    reason = "neither '%s' nor '%s' methods found" % \
                             tuple(try_names)
                else:
                    raise RuntimeError, "this can't happen!"

                raise ValueError, \
                      ("bad test case name '%s': " + reason +
                       " in class %s") % \
                       (name, klass_name)

            methods.append (func)

        return methods

    # _resolve_case_names ()


    def _find_test_cases (self):
        """Examine the current class (ie. the class of 'self') to find
        all test cases in it -- that is, all methods whose names
        begin with 'check_'.  Returns a list of unbound method objects,
        sorted by name."""

        methods = []
        klass = self.__class__

        # Get a list of (name, method) tuples (so we can sort on name)
        for name in vars(klass).keys():
            if name[0:6] == "check_":
                member = getattr (klass, name)
                if type(member) is UnboundMethodType:
                    methods.append ((name, member))
        methods.sort ()

        # Throw out the names -- all we really need are the method
        # objects (unbound!)        
        for i in range(len(methods)):
            methods[i] = methods[i][1]

        return methods

    # _find_test_cases ()
    

    def _determine_test_cases (self, cases=None):
        """Figure out which set of test cases we are supposed to run.
        If 'cases' is supplied, it should be a list (or tuple) of
        strings that name test case methods in the current class (with
        or without the leading 'check_').  Otherwise, if there is an
        attribute 'test_cases' on 'self' (usually a class attribute, but
        you never know), then it is treated the same -- it should be a
        list of method names.  Otherwise, use '_find_test_cases()' to
        find all test cases from the class dictionary."""

        klass = self.__class__
        klass_name = klass.__name__

        if cases is None:

            # Generate 'cases', a list of method names.  Either it comes
            # from the class attribute 'test_cases', or we figure it out
            # ourselves by inspecting the class dictionary.
            if hasattr (self, "test_cases"):
                case_methods = self._resolve_case_names(self.test_cases)
                #cases = self.test_cases
            else:
                case_methods = self._find_test_cases()

        elif type (cases) in (ListType, TupleType):
            case_methods = self._resolve_case_names(cases)

        else:
            raise TypeError, "'cases' must be a list, tuple, or None"

        return case_methods

    # _determine_test_cases ()


    def _get_testcase_info (self, method):

        case_name = method.__name__
        class_name = self.__class__.__name__

        doc = getattr (method, '__doc__')
        if doc is None:
            self.warn ("test case '%s' in class %s: no doc string" %
                       (case_name, class_name))
            msg = case_name
            count = "???"
        else:
            match = re.match (r'(.*):\s*(\d+)$', method.__doc__)
            if match is None:
                self.warn (("test case '%s' in class %s: " +
                            "badly-formatted doc string") %
                           (case_name, class_name))
                msg = doc
                count = "???"
            else:
                (msg, count) = match.group (1, 2)
                count = int (count)

        return (msg, count)

    # _get_testcase_info ()


    def run (self, cases=None, options=None):
        """Run the appropriate test cases in this test scenario.  Test
        cases are specified either by 'cases', a 'test_cases' attribute
        on 'self' (usually a class attribute), or by searching for all
        methods whose names start with 'check_'.  See
        '_determine_test_cases()' for details.

        Every test case is implemented by a method of 'self'.  Those
        methods are called in turn, and the results are reported.  If an
        exception was caught from the body of the test case itself (ie.,
        not from a particular test), that is reported.  The number of
        tests run, passed, and failed is always reported."""

        if options is not None:
            self.options = options
        if self.options is None:
            self.options = TestOptions()

        case_methods = self._determine_test_cases (cases)
        for method in case_methods:
            case_name = method.__name__
            if case_name[0:6] == 'check_':
                abbr_case_name = case_name[6:]
            else:
                abbr_case_name = case_name
            class_name = self.__class__.__name__
            
            (descr, test_count) = self._get_testcase_info (method)
            if test_count == "???" or self.total_count == "???":
                self.total_count = "???"
            else:
                self.total_count = self.total_count + test_count
            if self.options.preface_testcase:
                self.out.write ("%s ('%s'):\n" % (descr, abbr_case_name))

            self.tests_run = 0
            self.tests_ok = 0
            self.test_reports = []
            try:
                self.setup ()
                self.out.indent_more ()
                method (self)
                self.out.indent_less ()
                self.shutdown ()
                testcase_finished = 1
            except:
                self.out.indent_less ()
                exc_info = sys.exc_info()
                exc_name = exception_name(exc_info[0])
                exc_msg = str(exc_info[1])
                self.warn (("test case '%s' in class %s raised\n" +
                            "         %s: %s") %
                           (case_name, class_name, exc_name, exc_msg))
                print_exc (file=self.out)
                testcase_finished = 0
                self.shutdown ()

            # Print a summary of the test case; will be slightly
            # different depending on whether we've already printed the
            # test case description.
            (status, failed, summary) = \
                gen_summary (test_count, self.tests_run, self.tests_ok,
                             testcase_finished,
                             "test", "tests")
            if self.options.show_testcase_summary:
                self.out.write (status + ": ")
                if self.options.preface_testcase:
                    self.out.write (summary + "\n")
                else:
                    self.out.write ("%s ('%s') (%s)\n" %
                                    (descr, abbr_case_name, summary))

            self.total_run = self.total_run + self.tests_run
            self.total_ok = self.total_ok + self.tests_ok

            # If we *didn't* report test results (success/failure) as we
            # ran each test, do so now.
            if not self.options.report_immediate:
                self.out.indent_more()
                for report in self.test_reports:
                    self.print_report(report)
                self.out.indent_less()

            # Carefully consider whether we should print a newline.
            if (method is not case_methods[-1] and
                (self.options.preface_testcase and
                 self.options.show_testcase_summary)
                or
                (failed and
                 not self.options.preface_testcase and
                 self.options.show_testcase_summary)
               ):
                self.out.write ("\n")

        # for case_methods

    # run ()

    def get_stats (self):
        return (self.total_count, self.total_run, self.total_ok)


    # ------------------------------------------------------------------
    # Everything after this point relates to running individual test
    # cases, while everything previous deals with multiple test cases,
    # i.e. the collection of test cases in a particular test scenario.
    # This is a pretty natural dividing line; if I could figure out a
    # good way to divide these into two classes, I would!
    # ------------------------------------------------------------------


    def record (self, status, message, extra=None, traceback=None):
        self.test_reports.append ((status, message, extra, traceback))

    def print_report (self, report=None):
        if report is None:
            report = self.test_reports[-1]
        (status, message, extra, traceback) = report
        self.out.write ("%s: %s\n" % (status, message))
        self.out.indent_more()
        if extra:
            for line in extra:
                self.out.write (line + "\n")
        if traceback and self.options.show_tracebacks:
            print_tb (traceback, file=self.out)
        self.out.indent_less()
            

    def report_pass (self, msg):
        self.tests_ok = self.tests_ok + 1
        if not self.options.report_passes:
            return
        self.record ("ok", msg)
        if self.options.report_immediate:
            self.print_report ()

    def report_fail (self, msg, traceback=None):
        if not self.options.report_failures:
            return
        info = get_caller_info (2)
        failed_at = ["failed at %s, line %s (in '%s()'):" % info[0:3],
                     "  " + info[3]]
        self.record ("not ok", msg, failed_at, traceback)

        if self.options.report_immediate:
            self.print_report ()
        #if self.options.report_immediate:
        #    self.out.write ("not ok: " + msg + "\n")
        #    self.out.indent_more()
        #    self.out.write (failed_at[0] + "\n")
        #    self.out.write (failed_at[1] + "\n")
        #    if traceback and self.options.show_tracebacks:
        #        print_tb (traceback, file=self.out)
        #    self.out.indent_less()


    def test_val (self, code, value=None, match_ident=0, match_types=0):
        (globals, locals) = get_caller_env ()
        self.tests_run = self.tests_run + 1
        try:
            val = eval (code, globals, locals)
        except:
            (exc_seen, exc_value, exc_tb) = sys.exc_info()
            self.report_fail \
                ("%s != %s (raised %s)" %
                 (code, `value`, describe_exc (exc_seen, exc_value)),
                 exc_tb)
        else:
            if match_ident:
                if val is value:
                    self.report_pass ("%s is %s@%x" %
                                      (code, `value`, id(value)))
                else:
                    self.report_fail ("%s is not %s@%x (got %s@%x)" %
                                      (code,
                                       `value`, id(value),
                                       `val`, id(val)))

            else:                       # just match on equality


                if match_types:
                    if type(value) is InstanceType:
                        if isinstance(val, value.__class__):
                            type_msg = (" (and is an instance of %s)" %
                                        value.__class__.__name__)
                            types_ok = 1
                        else:
                            type_msg = (" (type mismatch: not instance of %s)"%
                                        value.__class__.__name__)
                            types_ok = 0
                    elif type(val) is type(value):
                        type_msg = " (and types match: %s)" % type(value)
                        types_ok = 1
                    else:
                        type_msg = (" (type mismatch: expected %s, got %s)" %
                                      (type(value), type(val)))
                        types_ok = 0
                else:
                    type_msg = ""
                    types_ok = 1

                if not types_ok:
                    self.report_fail ("%s != %s" % (code, `value`) +
                                      type_msg)
                else:
                    if val == value:
                        self.report_pass ("%s == %s" % (code, `value`) +
                                          type_msg)
                    else:
                        self.report_fail ("%s != %s (got %s)" %
                                          (code, `value`, `val`))

    # test_val ()

    # Alias for test_val()
    test = test_val

    def test_bool (self, code, want_true=1):
        (globals, locals) = get_caller_env ()
        self.tests_run = self.tests_run + 1
        try:
            val = eval (code, globals, locals)
        except:
            (exc_seen, exc_value, exc_tb) = sys.exc_info()
            self.report_fail ("%s not %s (raised %s)" %
                              (code, want_true and "true" or "false",
                               describe_exc (exc_seen, exc_value)),
                              exc_tb)
        else:
            if want_true and val:
                self.report_pass ("%s is true (%s)" % (code, `val`))
            elif not want_true and not val:
                self.report_pass ("%s is false (%s)" % (code, `val`))
            elif want_true and not val:
                self.report_fail ("%s not true (got %s)" % (code, `val`))
            elif not want_true and val:
                self.report_fail ("%s not false (got %s)" % (code, `val`))
            else:
                raise RuntimeError, "this can't happen!"
                
    # test_bool ()

    def test_seq (self, code, seqval, match_types=0, match_order=1):
        (globals, locals) = get_caller_env ()
        self.tests_run = self.tests_run + 1
        try:
            val = eval (code, globals, locals)
        except:
            (exc_seen, exc_value, exc_tb) = sys.exc_info()
            self.report_fail ("%s != %s (raised %s)" %
                              (code, `seqval`,
                               describe_exc (exc_seen, exc_value)),
                              exc_tb)
        else:

            # If match_order false, then we put the two sequence objects
            # into real lists and sort them before comparing, so that
            # differences in the ordering of elements disappear.
            if not match_order:
                try:
                    val = list (val)
                except TypeError:
                    self.report_fail ("%s not a sequence (got %s)" %
                                      (code, `val`))
                    return

                seqval = list (seqval)
                val.sort()
                seqval.sort()

            # If match_types true, then both the type and elements of the
            # two sequences have to be the same -- ie. we can just use
            # the normal comparison operator.  (Note that "match_types
            # and match_order" is effectively the same as just calling
            # 'test_val()'.)
            if match_types:
                passed = (val == seqval)

            # Otherwise, have to compare element-by-element -- ie. do
            # just what the normal comparison operator does, but don't
            # care about the nature of the sequence objects.  Note that
            # we rely on elements of the two sequences to be equal in
            # the conventional sense -- that is, [1,2,3] and (1,2,3) are
            # equal here, as are [1,2,("foo","bar")] and (1,2,("foo","bar"))
            # -- but NOT [1,2,("foo","bar")] and (1,2,["foo","bar"]).
            else:
                if len (val) == len (seqval):
                    for i in range (len (val)):
                        if val[i] != seqval[i]:
                            passed = 0
                            break
                    else:
                        passed = 1
                else:
                    passed = 0

            # OK, we have now (possibly) reordered the two sequences and
            # then compared them.  'passed' is true if they compared OK.
            ordered = (match_order) and "(in order)" or "(ignoring order)"
            if passed:
                self.report_pass ("%s == %s %s" %
                                  (code, `seqval`, ordered))
            else:
                self.report_fail ("%s != %s %s (got %s)" %
                                  (code, `seqval`, ordered, `val`))


    def test_stmt(self, stmt):
        "Test whether the statement 'stmt' completes without problems."

        (globals, locals) = get_caller_env ()
        self.tests_run = self.tests_run + 1
        try:
            exec stmt in globals, locals
        except:
            # Raised an exception
            (exc_seen, exc_value, exc_tb) = sys.exc_info()
            self.report_fail \
                ("%s raised %s" % 
                 (stmt, 
                  describe_exc (exc_seen, exc_value)),
                 exc_tb)
        else:
            # No exception raised
            self.report_pass ("%s ran quietly" % (stmt))

    # test_stmt ()

    def test_exc(self, stmt, exc):
        """Test whether the statement 'stmt' raises the expected exception."""

        (globals, locals) = get_caller_env ()
        self.tests_run = self.tests_run + 1
        try:
            exec stmt in globals, locals
        except:
            (exc_seen, exc_value, exc_tb) = sys.exc_info()
            if exc_seen is exc:         # good, got the expected exception
                self.report_pass \
                    ("%s raised %s" % (stmt, describe_exc (exc, exc_value)))

            else:                       # exception, but not the expected one
                self.report_fail \
                    ("%s didn't raise %s (raised %s)" % 
                     (stmt, exception_name (exc),
                      describe_exc (exc_seen, exc_value)),
                     exc_tb)
            return exc_value
        else:
            self.report_fail \
                ("%s didn't raise %s" % 
                 (stmt, exception_name (exc)))
            return None

    # test_exc ()
        
# class TestScenario


class TestOptions:
    """Contains the options that control the output and verbosity of
    running a test scenario.  This class exists purely as a holder for
    instance attributes:

      [...]
      report_passes (boolean, default false)
        print a line for each individual test that passes
      report_failures (boolean, default true)
        print a couple of lines for each individual test that fails
      show_tracebacks (boolean, default true)
        print a traceback for each failure that involves an unexpected
        exception
      [...]

    """

    # verbosity levels
    LOW = 0
    MEDIUM = 1                              # the default
    HIGH = 2


    def __init__ (self, verbosity=MEDIUM, show_coverage=0):

        self.show_coverage = show_coverage
        self.show_coverage_lines = 0
        if verbosity == TestOptions.MEDIUM:
            self.preface_testcase = 0
            self.report_immediate = 0
            self.report_passes = 0
            self.report_failures = 1
            self.show_tracebacks = 1
            self.show_testcase_summary = 1
            self.show_scenario_banner = 1
            self.show_scenario_summary = 1
            self.show_script_summary = 1

        elif verbosity == TestOptions.HIGH:
            self.preface_testcase = 1
            self.report_immediate = 1
            self.report_passes = 1
            self.report_failures = 1
            self.show_tracebacks = 1
            self.show_testcase_summary = 1
            self.show_scenario_banner = 1
            self.show_scenario_summary = 1
            self.show_script_summary = 1
            if self.show_coverage:
                self.show_coverage_lines = 1

        elif verbosity == TestOptions.LOW:
            self.preface_testcase = 0
            self.report_immediate = 0
            self.report_passes = 0
            self.report_failures = 1
            self.show_tracebacks = 1
            self.show_testcase_summary = 0
            self.show_scenario_banner = 0
            self.show_scenario_summary = 0
            self.show_script_summary = 1

    # __init__ ()

# class TestOptions


def exception_name (exc):
    """Returns the name of an exception without any package or module
    names cluttering it up.  'exc' must be a string or exception
    class)."""
    if type(exc) is ClassType:
        return exc.__name__
    else:
        return exc


def describe_exc (exc_type, exc_value):
    return "%s: \"%s\"" % (exception_name (exc_type), exc_value)


def gen_summary (num_expected, num_run, num_passed, testcase_finished,
                 what, what_plural):
    if num_expected > 1:
        what = what_plural

    if num_expected == num_run == num_passed and testcase_finished:
        return ("ok", 0, "%d %s passed" % (num_passed, what))
    else:
        # ugh: use %s for num_expected because it can be
        # an integer or "???"
        text = ("%s %s expected, %d run, %d failed" %
                (num_expected, what, num_run, num_run-num_passed))
        if not testcase_finished:
            text = text + ", and test case crashed"

        return ("not ok", 1, text)


def caller (level=1):
    """Return the execution frame for some function up the call stack,
       by default (level=1) the immediate caller.  level=0 gives your
       own execution frame (that of the function that called 'caller');
       level=2 gives that of your caller's caller, and so on.  Raises
       RuntimeError if level is greater than the depth of the call stack."""

    try:
        try:
            jhasdgfjhasdf
        except NameError:
            tb = sys.exc_info()[2]

        # Now 'tb' is the traceback containing the frame for *this* function;
        # at the very least, we want our *caller's* frame, so...
        frame = tb.tb_frame.f_back
        if frame is None:
            raise RuntimeError, "'caller' has no caller!"

        # Now follow 'next' pointers until we've reached the desired level
        while level > 0:
            frame = frame.f_back
            if frame is None:
                raise RuntimeError, 'call stack not deep enough'
            level = level - 1

        return frame

    finally:
        del tb                          # break cycle, prevent leak

# caller ()


def get_caller_env (level=1):
    frame = caller (level+1)
    return (frame.f_globals, frame.f_locals)


def get_frame_info (frame):
    """Extracts and returns a tuple of handy information from an
       execution frame:
         (filename, lineno, function_name, code_line)
    """
    import linecache
    filename = frame.f_code.co_filename
    lineno = frame.f_lineno
    function_name = frame.f_code.co_name
    code_line = string.strip (linecache.getline (filename, lineno))

    return (filename, lineno, function_name, code_line)


def get_caller_info (level=1):
    """Extracts and returns a tuple of handy information from some
       caller's execution frame:
         (filename, lineno, function_name, code_line)
       If level is 1 (the default), gets this information about the
       caller of the function that is calling 'get_caller_info()'."""

    return get_frame_info (caller (level+1))



# eg. [FooTest, BarTest]
# or  [(FooTest, ['check_ding', 'check_dong']), BarTest]

def run_scenarios (scenarios, options=None):
    """Run a bunch of test scenarios.  'scenarios' is a list of test
    scenario descriptions.  A test scenario description is either a
    single class object, in which case all test cases in that test
    scenario are run; or it is a tuple (test_scenario, [case1, case2,
    ...])  where 'test_scenario' is again a class object and 'case1'
    etc.  are the names of the test cases to run.  Test case names are
    the same as the method that implements the test case, with the
    leading "check_" optionally lopped off."""

    # Empty list of test scenarios?  Then get the list of scenarios from
    # the global variable 'test_scenarios' in the main module (test script)
    # or, if that variable isn't found, just find all test scenarios in the
    # main module and run them.
    if not scenarios:
        scenarios = find_scenarios (sys.modules['__main__'])
    
    out = IndentedFile(sys.stdout, 0, 2)

    if options.show_coverage:
        module_dict = vars(sys.modules['__main__'])
        tested_modules = module_dict.get('tested_modules')
        if tested_modules:
            from code_coverage import Coverage
            coverage = Coverage(modules=tested_modules)
            sys.settrace(coverage.trace)

    # List of (count, run, ok) counts: for each test scenario run, the
    # number of tests expected, run, and passed.  At the end we tally
    # these up and return the totals (useful for the "run_tests" script,
    # which un turn tallies up the totals for many test scripts.  Yow!
    scenario_stats = []

    # Now loop over the list of test scenario descriptions.  If we just
    # built our own, then most of this code is unused -- it's only useful
    # if the caller passed in a list of scenario descriptions, and we have
    # to figure out what the nature of the decription is.  Of course, the
    # code at the end -- instantiate and run the test scenario -- is always
    # important!
    for scenario in scenarios:
        if type (scenario) is ClassType:
            klass = scenario
            cases = None

        elif type (scenario) is TupleType and len (scenario) == 2:
            (klass, cases) = scenario

        else:
            raise TypeError, \
                  "every element in 'scenarios' must be either a " + \
                  "class object or 2-tuple: '%s' invalid" % `scenario`

        if not (type(klass) is ClassType and issubclass (klass, TestScenario)):
            raise TypeError, \
                 "test scenario classes must be subclasses of TestScenario: "+\
                 "%s invalid" % klass.__name__


        if options.show_scenario_banner:
            scenario_name = klass.__name__
            if hasattr(klass, 'description'):
                banner = "%s (%s):\n" % (scenario_name, klass.description)
            else:
                banner = "%s:\n" % scenario_name
            out.write(banner)
            out.indent_more()

        scenario_obj = klass (outfile=out)
        scenario_obj.run (cases, options=options)
        out.indent_less()

        counts = scenario_obj.get_stats()
        scenario_stats.append(counts)
        if options.show_scenario_summary:
            out.write("%s: %s\n" % summarize_results(counts))

        if scenario is not scenarios[-1] and options.show_scenario_banner:
            out.write("\n")

    if options.show_coverage:
        if tested_modules is None:
            out.write('Coverage analysis requested but '
                      '"tested_modules" not defined\n')
        else:
            # reload tested modules to get coverage of imports, etc.
            for name in tested_modules:
                module = sys.modules.get(name)
                if module:
                    reload(module)
            sys.settrace(None)
            out.write("code coverage:\n")
            out.indent_more()
            coverage.write_results(out, show_lines=options.show_coverage_lines)
            out.indent_less()

    return sum_test_counts(scenario_stats)

# run_scenarios ()


def sum_test_counts (count_list):
    """sum_test_counts(count_list : [(expected : int,
                                      run : int,
                                      passed : int)])
       -> (total_expected : int, total_run : int, total_passed : int)

    Compute sums of a list of test counts.  'count_list' is a list of
    3-tuples (expected, run, passed): the number of tests expected, run,
    and passed.  Returns a 3-tuple that is the sum of all the tuples in
    'count_list'.
    """
    totals = [0, 0, 0]
    for counts in count_list:
        for i in (0, 1, 2):
            if counts[i] == '???':
                totals[i] = '???'
            elif totals[i] != '???':
                totals[i] = totals[i] + counts[i]
    return tuple(totals)

def tests_successful (counts):
    """Returns true if all tests completed successfully."""
    (expected, run, passed) = counts
    return expected == run == passed

def summarize_results (counts, preface=None):
    """summarize_results(counts : [(int, int, int)],
                         preface : string)
       -> (string, string)

    Return a pair of strings summarizing the test counts statistics in
    'counts', a 3-tuple of (expected, run, passed) test counts.  The
    first string is either "ok" or "not ok"; the second gives more
    detail (possibly including 'preface').  Generally, you can just join
    them together with ":" and print them.
    """
    (expected, run, passed) = counts
    if tests_successful(counts):
        summary = "%d tests passed" % expected
        if preface:
            return ("ok", "%s: %s" % (preface, summary))
        else:
            return ("ok", summary)
    else:
        summary = ("%s tests expected, %d run, %d failed" %
                   (expected, run, run-passed))
        if preface:
            return ("not ok", "%s: %s" % (preface, summary))
        else:
            return ("not ok", summary)


def find_test_scenario (script, name, module=None):

    if module is None:
        module = sys.modules['__main__']
    module_dict = vars(module)

    if name[-4:] != "Test":
        try_names = [name, name + "Test"]
    else:
        try_names = [name]

    for try_name in try_names:
        klass = module_dict.get(try_name)
        if klass:
            break
    else:
        if len (try_names) == 1:
            reason = "'%s' class not found" % try_names[0]
        elif len (try_names) == 2:
            reason = "neither '%s' nor '%s' classes found" % tuple(try_names)
        else:
            raise RuntimeError, "this can't happen!"

        raise ValueError, \
              ("unknown test scenario '%s': " + reason +
               " in script %s") % \
              (name, script)

    if not issubclass (klass, TestScenario):
        raise RuntimeError, \
              ("bad test scenario '%s': " +
               "'%s' not a subclass of TestScenario") % \
              (name, name)

    return klass

# find_test_scenario()
    

def find_scenarios (module):

    """Finds all test scenarios (subclasses of TestScenario) made available
    by module 'module' and returns the list of class objects found.
    "Available" means "listed in the module global variable
    'test_scenarios' if it is defined, or defined in that module if
    not"."""

    mod_dict = vars (module)

    if hasattr (module, 'test_scenarios'):
        scenarios = module.test_scenarios
    else:
        scenarios = []
        for val in mod_dict.values():
            if (type (val) is ClassType and
                issubclass (val, TestScenario) and
                val.__module__ == module.__name__):
                scenarios.append(val)

    return scenarios

# find_scenarios ()


def has_test_case (scenario, casename):
    if hasattr (scenario, casename):
        return casename
    elif casename[0:6] != "check_" and hasattr (scenario, "check_" + casename):
        return "check_" + casename
    else:
        return None


def parse_args (argv=sys.argv):
    """Parse the command-line arguments to a test script.
    Syntax is:
      [-c] [-q|-v] [scenario1][:case1,...] ...
    ie. user may specify any number of test scenarios and any number
    of test cases for each scenario.  Scenarios are specified by the
    name of the class that implements them, and cases by the name
    of the method.  Trailing "Test" is optional for scenario names,
    as is leading "check_" for case names.

    If no scenarios or cases are supplied, all found in the test script
    (__main__ module) are run.  If a scenario is specified without any test
    cases, all cases ("check_" methods or methods listed in class attribute
    'test_cases') are run.  If a test case is specified without any test
    scenario, that test case is run in all test scenarios (this is mainly a
    shortcut for scripts with a single test scenario class).

    Exits with appropriate error message and non-zero exit status if any
    errors found in command-line.  Exits with help message and zero exit
    status if "-h" or "--help" seen on command-line.

    Returns a tuple (scenarios, options); 'options' is a TestOptions
    instance recording the -q/-v/etc. options; scenarios is a list
    of test scenario descriptions as required by 'run_scenarios()'."""

    from getopt import getopt

    script = argv[0]
    args = argv[1:]
    usage = "usage: %s: [-q] [-v] [scenario1][:case1,...] ..." % script

    # Check for help options
    if "-h" in argv or "--help" in argv:
        print usage
        sys.exit()

    # Look for -q and -v; modify default option settings appropriately
    (opts, args) = getopt (args, 'cqv')
    verbosity = TestOptions.MEDIUM
    show_coverage = 0
    for (opt,_) in opts:
        if opt == '-c':
            show_coverage = 1
        elif opt == '-q':
            verbosity = TestOptions.LOW
        elif opt == '-v':
            verbosity = TestOptions.HIGH
    options = TestOptions (verbosity, show_coverage)

    all_scenarios = find_scenarios (sys.modules['__main__'])

    # No arguments -- we'll just run all tests
    if not args:
        return (all_scenarios, options)

    scenarios = []
    for arg in args:

        colon = string.find (arg, ':')

        # No colon in 'arg' -- it's just a bare word.
        if colon == -1:

            # Special case for lazy typists running test scripts that only
            # define one test scenario: a lone word on the command line is
            # first checked to see if it's the name of a test case (method)
            # in that test scenario.
            if len(all_scenarios) == 1:
                casename = has_test_case (all_scenarios[0], arg)
                if casename:
                    scenarios.append ((all_scenarios[0],[casename]))
                    continue

            # Otherwise, assume the lone word is the name of a test
            # scenario class; dig up and validate the class object.
            klass = find_test_scenario (script, arg)
            scenarios.append (klass)

        # Found a colon: split into scenario name and list of test cases.
        else:
            scenario = arg[:colon]
            klass = find_test_scenario (script, scenario)

            # If no cases were listed (ie. there's nothing after the
            # colon), then treat this like the no-colon case above: will
            # run all test cases
            if colon+1 == len(arg):
                scenarios.append (klass)

            # Otherwise, pull out the list of case names after the colon
            else:
                cases = string.split (arg[colon+1:], ',')
                scenarios.append ((klass, cases))

        # if/else colon
        
    # for args

    return (scenarios, options)

# parse_args ()


def run_scenario (test_class):
    """Run all test cases in a single test scenario.  Deprecated."""
    scenario = test_class ()
    if not isinstance (scenario, TestScenario):
        raise TypeError, "'test_class' must be a subclass of TestScenario"

    scenario.run ()
