"""$URL: svn+ssh://svn.mems-exchange.org/repos/trunk/grouch/lib/schema_util.py $
$Id: schema_util.py 24750 2004-07-21 15:26:51Z dbinger $

Search a set of directories for Python modules and classes, and parse
the class docstrings to generate an object schema.  Lots of
project-specific information (eg. directories to search, classes to
exclude, ...) can be supplied via a project description file.
"""

import sys, os, string, re
import getopt
import types
import traceback
from fnmatch import fnmatch
from cPickle import dump

from grouch.schema import ObjectSchema, ClassDefinition, InvalidAlias
from grouch.script_util import announce, error, warn


# -- Mid-level workers -------------------------------------------------
# (called by generate_schema())

def find_modules (dirs, base_dir=None, prefix=None, exclude=None):
    """find_modules(dirs : [string],
                    base_dir : string = None,
                    prefix : string = None,
                    exclude : [string] = None)
       -> [(modname : string, filename : string)]

    Searches a list of directories for Python module files (*.py).
    Returns a list of (modname, filename) tuples.  If 'prefix' is
    supplied, it is prepended (with a dot interpolated) to each module
    name.  If 'exclude' is supplied, it must be a list of
    fully-qualified module names; if a module is found in 'exclude'
    (after adding prefix), it will not be included in the returned list
    of modules.

    Example: dirs == ["foo"] and directory "foo" contains "bar.py" and
    "baz.py".  If no prefix or base_dir is supplied, returns
      [("foo.bar", "foo/bar.py"),
       ("foo.baz", "foo/baz.py")]

    If base_dir == "d" and prefix == "p", returns
      [("p.foo.bar", "d/foo/bar.py"),
       ("p.foo.baz", "d/foo/baz.py")]
    """
    modules = []                        # list of (modname, filename) tuples
    for dir in dirs:
        components = os.path.normpath(dir).split(os.sep)
        if os.pardir in components:     # disallow "../foo" after normpath()
            raise ValueError, \
                  ("invalid directory '%s': cannot contain '%s'" %
                   (dir, os.pardir))
        if components == [os.curdir]: # eg. dir == "." or ""
            components = []
        if prefix:
            components.insert(0, prefix)

        if base_dir:
            real_dir = os.path.normpath(os.path.join(base_dir, dir))
        else:
            real_dir = os.path.normpath(dir)

        for basename in os.listdir(real_dir):
            # Skip files that aren't Python modules
            if not fnmatch(basename, "*.py") or basename == "__init__.py":
                continue

            real_filename = os.path.join(real_dir, basename)
            bare_modname = os.path.splitext(basename)[0]
            modname = ".".join(components + [bare_modname])

            if modname not in exclude:
                modules.append((modname, real_filename))

    return modules


class ClassInfo:
    """
    Encapsulates all the information we need about a class in order to
    add it to an object schema.  find_classes() returns a list of
    ClassInfo instances to find_all_classes(), which incorporates that
    in a dictionary mapping module names to lists of ClassInfo
    instances.

    Instance attributes:
      bare_name : string
        the bare class name as seen in the "class" statement
      full_name :
        the fully-qualified class name (ie. including module name)
      base_classes : [string]
        the list of base class names from the class statement
      docstring : string
        the class docstring as a single (probably multi-line) string
    """

    def __init__ (self, bare_name, full_name, base_classes, docstring):
        self.bare_name = bare_name
        self.full_name = full_name
        self.base_classes = base_classes
        self.docstring = docstring

    def __str__ (self):
        return self.full_name

    def __repr__ (self):
        return "<%s at %08x: %s>" % (self.__class__.__name__, id(self), self)

    def expand_base_classes (self, schema):
        """
        Ensure that every base class listed in self.base_classes is
        either a class or an alias for a class in 'schema'.  Any base
        class names that are aliases are expanded to the full class
        name.  Raises ValueError if any base class names are bad.
        """
        for i in range(len(self.base_classes)):
            base_name = self.base_classes[i]
            if not schema.get_class_definition(base_name):
                alias = schema.get_alias(base_name)
                if not alias:
                    raise ValueError(
                        "%s: invalid base class %r (no such class or alias)"
                        % (self.full_name, base_name))
                elif not alias.is_plain_instance_type():
                    raise ValueError(
                        "%s: invalid base class %r (alias to non-class)"
                        % (self.full_name, base_name))
                else:
                    self.base_classes[i] = alias.klass_name


def find_all_classes (modules, schema, exclude_classes=None):
    """find_all_classes(modules : [(modname : string, filename : string)],
                        schema : ObjectSchema,
                        exclude_classes : [string] = None)
       -> { modname:string : [ClassInfo] }

    Find all classes in a list of modules.  Returns a dictionary mapping
    module name to list of ClassInfo objects.  For each class found,
    adds two things to schema: an empty ClassDefinition, and an alias
    mapping the bare class name to its full name (eg. for class
    foo.bar.FooBar, the alias maps "FooBar" to "foo.bar.FooBar").
    """

    # Map module name to list of class objects
    module_classes = {}
    num_classes = 0

    announce("looking for classes...\n")
    for (modname, filename) in modules:
        announce("module %s:\n" % modname, threshold=2)
        klasses = find_classes(filename, modname, exclude_classes)
        num_classes += len(klasses)

        ok_klasses = []
        for klass in klasses:
            bare_name = klass.bare_name
            full_name = klass.full_name

            # create new alias mapping bare_name to full_name and add
            # it to the schema?
            add_alias = 1

            alias = schema.get_alias(bare_name)
            if isinstance(alias, InvalidAlias):
                # We've already had one name collision on 'bare_name' and
                # invalidated that alias, so don't try to create it again!
                warn("can't alias %r to %r: name collision"
                     % (bare_name, full_name))
                add_alias = 0

            elif alias:
                # The alias is actually for this class, so it's not a
                # problem after all.
                if (alias.is_instance_type() and
                    alias.get_class_name() == full_name):
                    add_alias = 0

                # Uh-oh, alias already exists and it maps 'bare_name'
                # to something else.  Ambiguity is bad, so invalidate
                # this alias.
                else:
                    warn("invalidating ambiguous alias %r"
                         % bare_name)
                    schema.invalidate_alias(bare_name)
                    add_alias = 0

            # If we need to, add a new class definition to the schema.
            klass_def = schema.get_class_definition(full_name)
            if klass_def is None:
                klass_def = ClassDefinition(full_name, schema)
                schema.add_class(klass_def)

            # The above code decided there's no problem aliasing
            # 'bare_name' to 'full_name', so do it now.
            if add_alias:
                schema.add_alias(bare_name, full_name)

        module_classes[modname] = klasses
        for klass in klasses:
            announce("  %s\n" % klass, threshold=2)

    # This has to be done after scanning all modules, because it depends
    # on us having seen all classes in the application.
    for klasses in module_classes.values():
        for klass in klasses:
            try:
                klass.expand_base_classes(schema)
            except ValueError, err:
                # Should we exclude the class from the schema entirely
                # if this happens?  Right now we just refrain from giving
                # it any base classes if one of its base classes has
                # a problem.
                error(str(err))
            else:
                klass_def = schema.get_class_definition(klass.full_name)
                klass_def.set_bases(klass.base_classes)

    announce("found %d classes\n" % num_classes)
    return module_classes


def get_names (nodes):
    from compiler import ast
    names = []
    for node in nodes:
        if isinstance(node, ast.Name):
            names.append(node.name)
        elif isinstance(node, ast.Getattr):
            # Dotted name: "foo.bar.baz" in source becomes
            # Getattr(Getattr(Name(foo),bar),baz) in the AST.  Unwind
            # this stack to reconstruct the original string.
            cur_node = node
            stack = []
            while isinstance(cur_node, ast.Getattr):
                stack.append(cur_node.attrname)
                cur_node = cur_node.expr
            assert isinstance(cur_node, ast.Name), \
                   "expected Name at bottom of Getattr stack"
            stack.append(cur_node.name)
            stack.reverse()
            name = string.join(stack, ".")
            names.append(name)

    return names


def find_classes (filename, modname, exclude=None):
    """find_classes(filename : string,
                    modname : string,
                    exclude : [string] = None)
       -> [ClassInfo]

    Parses the Python source file 'filename' looking for classes.
    Assumes this source file contains the module 'modname'.  Returns a
    list of ClassInfo instances, one for each class in the file.

    Eg. if parsing a file foo.py which (according to 'modname') contains
    a module 'foo', this code:
      class Foo (bar.Bar):
        '''
        foo
        bar
        '''
    results in a ClassInfo instance like:
      bare_name = "Foo"
      full_name = "foo.Foo"
      base_classes = ["bar.Bar"]
      docstring = "\n    foo\n    bar\n    "

    No attempt is made to determine what "bar.Bar" really refers to --
    that would require interpreting the module, and in that case we
    might as well just import the damn thing.  Note that the
    ClassInfo.expand_base_classes() method, called by find_all_classes()
    just before returning, attempts to expand all base class names to
    their true names.
    """
    from compiler import parseFile, ast
    from parser import ParserError

    klasses = []

    try:
        module_ast = parseFile(filename)
    except ParserError:
        error("%s: unable to parse module (try importing it for more details)"
              % filename)

    for node in module_ast.node.nodes:
        if isinstance(node, ast.Class):
            if modname:
                fullname = modname + "." + node.name
            else:
                fullname = node.name

            if fullname in exclude:
                continue

            base_classes = get_names(node.bases)
            klasses.append(ClassInfo(node.name, fullname,
                                     base_classes, node.doc))

    return klasses


def parse_class_docstrings (modules, module_classes, schema):
    """parse_class_docstrings(modules : [(string, string)],
                              module_classes : {string : [ClassInfo]},
                              schema : ObjectSchema)
    """
    announce("parsing class docstrings...\n")
    for (modname, _) in modules:
        klasses = module_classes.get(modname, [])
        announce("  module: %s (%d classes)\n" % (modname, len(klasses)),
                 threshold=2)
        for klass in klasses:
            try:
                errors = parse_docstring(klass, schema)
            except ValueError, exc:
                error(str(exc))
                announce("   failed to parse %s docstring\n" % klass, threshold=2)
            else:
                for e in errors:
                    warn(e)
                announce("   parsed %s docstring\n" % klass, threshold=2)


# -- Parsing code ------------------------------------------------------
# (parse_class_docstrings() calls parse_docstring(), which uses
#  everything else in this section)

leading_ws_re = re.compile('^(\s*)')

def get_indent_level (s):
    """Return the number of spaces that 's' starts with."""
    m = leading_ws_re.match(s)
    return len(m.group(1))


def clean_docstring (doc, klass_name):
    lines = (doc or '').split("\n")
    assert lines, "string.split() returned empty list"
    if lines[0] and lines[0][0] == ' ':              # first line indented
        leading_indent = get_indent_level(lines[0])
        start_line = 0
    elif len(lines) > 1:                # look for first "real" line
        i = 1
        while i < len(lines) and not lines[i]: # skip over blank lines
            i += 1
        assert i < len(lines), "arg, screwy docstring"
        leading_indent = get_indent_level(lines[i])
        start_line = 1
    else:                               # single-line docstring
        leading_indent = 0
        start_line = 0

    for i in range(start_line, len(lines)):
        if not lines[i]:                # skip blanks
            continue
        if len(lines[i]) < leading_indent:
            raise ValueError, \
                  ("class %s: inconsistent indent in docstring "
                   "(line %d too short)" % (klass_name, i))
        if string.lstrip(lines[i][:leading_indent]) != "":
            raise ValueError, \
                  ("class %s: inconsistent indent in docstring "
                   "(line %d dedented relative to line %d)" %
                   (klass_name, i, start_line))

        lines[i] = re.sub(r'#.*', '', lines[i])
        lines[i] = string.rstrip(lines[i][leading_indent:])

    return lines

def find_attrs (lines, klass_name):
    i = 0
    while i < len(lines):
        line = lines[i]
        if line.startswith("Instance attributes:"):
            if line.endswith("none"):
                return None
            return i+1
        i += 1
    else:
        raise ValueError, \
              ("class %s: no \"Instance attributes:\" line in docstring" %
               klass_name)


_name_pat = r'[a-zA-Z_][a-zA-Z0-9_]*'
_dotted_name_pat = r'%s(?:\.%s)*' % (_name_pat, _name_pat)
_attr_line_re = re.compile(r'\s*(%s)\s*:\s*(.*)' % _name_pat)
_element_name_re = re.compile(r'(%s):(%s)' % (_name_pat, _dotted_name_pat))

def massage_typespec (typespec):
    # Now we have to massage the typespec so it can be parsed as
    # a ValueType.  Luckily, the docstring type specification
    # language is pretty similar to the ValueType type
    # specification language; the differences are:
    #   - container elements can have names as well as types,
    #     eg.  { key:keytype : value:valuetype }
    #     or   (val1:type1, val2:type2)
    #   - plain container types can include the name of the
    #     container, eg.
    #       dictionary {string : int}
    #   - typespecs can be trailed by default value description, eg.
    #       foo : int = 37
    #       bar : [string] = ["hello"]

    # Note that for named container elements, whitespace
    # matters!  {key:keytype : value:valuetype} is *not* the
    # same as {key : keytype : value : valuetype} -- the latter
    # is illegal.

    # Deal with the second exception first, since it's easiest.
    words = string.split(typespec, None, 1)
    if len(words) > 1:
        remainder = words[1]
        if words[0] in ('list', 'tuple', 'dictionary'):
            typespec = remainder
    else:
        remainder = None

    # Strip anything that looks like a default value, ie. " = ..."
    typespec = re.sub(r'\s*=.*', '', typespec)

    # Deal with the named-elements thing.  This is a kludge, but it
    # should work given the tight syntax constraints on named elements.
    if typespec[0] in "[({" or (remainder and remainder[0] in "[({"):
        typespec = _element_name_re.sub(r'\2', typespec)

    return typespec


def parse_docstring(klass, schema):
    """(klass : ClassInfo, schema : ObjectSchema) -> errors : [string]

    Parse the class docstring in 'klass' and use the docstring to update
    the class definition already in 'schema'.  Return a list of error
    messages resulting from parsing the docstring which should be presented
    to the user.  Raises ValueError if the docstring is a lost cause,
    ie. missing or completely unparseable.
    """

    klass_name = klass.full_name        # for error messages

    klass_def = schema.get_class_definition(klass_name)
    if klass_def.attrs or klass_def.all_attrs:
        raise ValueError, "class %s: already seen" % klass_name

    doc = klass.docstring
    if not doc:
        raise ValueError, "class %s: no docstring" % klass_name

    errors, attrs = parse_docstring_helper(schema, doc, klass_name)

    for name, type in attrs:
        if type.is_any_type():
            type.set_allow_any_instance(0)
        klass_def.add_attribute(name, type)

    # XXX ClassDefinition should expose len() of its attrs list
    if len(klass_def.attrs) == 0:
        errors.append("class %s: no attributes successfully parsed" %
                      klass_def.name)

    return errors


def parse_docstring_helper(schema, doc, klass_name):
    """
    This is separated so that it can be used by outside code.
    """

    errors = []
    attrs = []

    lines = clean_docstring(doc, klass_name)
    # Now 'lines' is a list of consistently-indented lines.  Find the
    # one that says "Instance attributes:", and start parsing attribute
    # type specifications from there.  Stop when we get back to
    # zero-indent level.
    start_line = find_attrs(lines, klass_name)
    if start_line is None:              # class is declared to have no
        return errors, attrs            # instance attributes

    i = start_line
    while i < len(lines):
        line = lines[i]
        if not line:                    # skip blanks
            i += 1
            continue
        indent = get_indent_level(line)
        if indent == 0:                 # out of the attribute list
            break

        m = _attr_line_re.match(line)
        if m:
            (name, typespec) = m.group(1,2)
            typespec = massage_typespec(typespec)
            try:
                attrs.append((name, schema.parse_type(typespec)))
            except ValueError, exc:
                errors.append("class %s, attribute %s: %s" %
                              (klass_name, name, exc))
        else:
            errors.append("class %s: couldn't parse line %d of docstring: %s" %
                          (klass_name, i, `line`))

        # Read lines until back at the indent level of this line
        # (ie. skip the indented attribute description).
        i += 1
        while i < len(lines):
            if get_indent_level(lines[i]) <= indent:
                break
            i += 1
    return errors, attrs


# -- High-level workers ------------------------------------------------
# (called from main())

def generate_schema (project, base_dir):
    schema = ObjectSchema()

    # Preparatory work -- stuff that needs to go in the schema, but
    # can't be discovered by searching for and parsing *.py files.
    for atomic_type in project.atomic_types:
        if type(atomic_type) is types.TupleType and len(atomic_type) == 2:
            schema.add_atomic_type(*atomic_type)
        else:
            schema.add_atomic_type(atomic_type)

    for name in project.forward_classes:
        cdef = ClassDefinition(name, schema)
        schema.add_class(cdef)

    for (name, value) in project.type_aliases:
        schema.add_alias(name, value)

    # The meat of the schema: class definitions for classes in whatever
    # *.py files we can find in project.dirs.
    if project.dirs:
        announce("searching for modules...")
        modules = find_modules(project.dirs,
                               base_dir=base_dir,
                               prefix=project.prefix,
                               exclude=project.exclude_modules)
        announce("found %d modules\n" % len(modules))
    else:
        modules = []

    project.add_extra_modules(modules, base_dir)

    module_classes = find_all_classes(modules, schema, project.exclude_classes)
    parse_class_docstrings(modules, module_classes, schema)

    if project.post_parse_hook:
        project.post_parse_hook(schema)

    # Finish up all class definitions -- ie. look at the inheritance tree
    # and gather up the list of all attributes that should be in instances
    # of a class, including those inherited from superclasses.
    for klass_name in schema.get_class_names():
        klass_def = schema.get_class_definition(klass_name)
        klass_def.finish_definition()

    return schema


def write_schema (schema, text_filename, pickle_filename):

    if text_filename:
        announce("writing object schema to %s..." % text_filename)
        schema_file = open(text_filename, "w")
        schema.write_aliases(schema_file)
        schema_file.write("\n\n")

        for klass_name in schema.get_class_names():
            klass_def = schema.get_class_definition(klass_name)
            klass_def.write(schema_file)
            schema_file.write("\n")
        schema_file.close()
        announce("\n")

    if pickle_filename:
        announce("pickling object schema to %s..." % pickle_filename)
        schema_file = open(pickle_filename, "w")
        dump(schema, schema_file, 1)
        schema_file.close()
        announce("\n")


# -- Main program ------------------------------------------------------

class ProjectDescription:
    """
    Instance attributes:
      atomic_types : [(any, string) | any]
      forward_classes : [string]
      type_aliases : [(alias:string, alias_expansion:string)]
      prefix : string
      dirs : [string]
      extra_modules : [string | (string, string)]
      exclude_modules : [string]
      exclude_classes : [string]

      post_parse_hook : function
    """

    def __init__ (self):
        self.atomic_types = []
        self.forward_classes = []
        self.type_aliases = []
        self.prefix = None
        self.dirs = []
        self.extra_modules = []
        self.exclude_modules = []
        self.exclude_classes = []

        self.post_parse_hook = None

    def read (self, filename):
        data = {}
        if not os.path.isfile(filename):
            raise ValueError, "no such file: %s" % filename
        try:
            execfile(filename, data)
        except:
            (t, v, tb) = sys.exc_info()
            error("error in %s:" % filename)
            exc = "".join(traceback.format_exception_only(t, v))
            sys.stderr.write(exc)
            sys.exit(1)

        for (name, value) in data.items():
            if hasattr(self, name):
                setattr(self, name, value)

        self.check_types(filename)

    def check_types (self, filename):
        from grouch.context import TypecheckContext

        schema = ObjectSchema()
        schema.add_atomic_type(lambda: None) # add 'function' type
        cdef = ClassDefinition(self.__class__.__name__, schema)
        schema.add_class(cdef)
        klass = ClassInfo("ProjectDescription",
                          "ProjectDescription",
                          [], self.__class__.__doc__)
        errors = parse_docstring(klass, schema)
        assert not errors, "errors in my own docstring!"
        cdef.finish_definition()
        context = TypecheckContext(report_errors=0)
        schema.check_value(self, context)
        if context.num_errors() > 0:
            error("type errors in %s:" % filename)
            context.write_errors(sys.stderr)
            sys.exit(1)

    def add_extra_modules (self, modules, base_dir):
        """add_extra_modules(modules : [(string, string)])

        Adds the modules listed in self.extra_modules to modules.  A bit
        tricky because extra_modules might not include filenames, in
        which case we have to go out and find the files.
        """

        # extra_modules is a list of any of the following
        #   string
        #     fully-qualified module name -- we'll have to search
        #     sys.path to find the file
        #   (modname : string, filename : string)
        #     module name with filename; we have to make sure the
        #     file exists, and try it relative to base_dir if not

        # yow! this is hairy and undertested!

        for module in self.extra_modules:
            if type(module) is types.StringType:
                modname = module
                comps = module.split(".")
                tail = os.path.join(*comps) + ".py"
                for dir in sys.path:
                    filename = os.path.join(dir, tail)
                    if os.path.exists(filename):
                        break
                else:
                    raise ValueError(
                        "no such module %s: %s not found in sys.path"
                        % (module, tail))

            elif type(module) is types.TupleType and len(module) == 2:
                (modname, filename) = module
                if os.path.exists(filename):
                    pass                # found it, good
                elif base_dir:          # we have a second chance
                    filename2 = os.path.join(base_dir, filename)
                    if os.path.exists(filename2):
                        filename = filename2
                    else:
                        raise ValueError(
                            "no such module %s: neither %s nor %s exist"
                            % (modname, filename, filename2))
                else:
                    raise ValueError(
                        "no such module %s: %s does not exist"
                        % (modname, filename))
            else:
                raise TypeError("bad extra_modules")

            assert os.path.exists(filename)
            modules.append((modname, filename))

        # for module

    # add_extra_modules ()


def main ():
    global VERBOSITY                    # global because announce(

    prog = os.path.basename(sys.argv[0])
    args = sys.argv[1:]

    usage = """\
usage: %s [options]
options:
  -v                        verbose: run noisily (repeat for more noise)
  -q                        run quietly
  -p FILE, --project=FILE   read project description info from FILE
  -o FILE, --output=FILE    output file: write pickled schema to FILE
                            [default: schema.pkl]
  -t FILE, --text=FILE      text output: write human-readable schema to FILE
                            [default: none]
  -d DIR, --base-dir=DIR    interpret DIRS in project description file
                            relative to DIR
""" % prog

    try:
        (opts, args) = getopt.getopt(args, "p:o:t:d:vq",
                                     ["project=",
                                      "output=",
                                      "text=",
                                      "base-dir=",])
    except getopt.error, msg:
        sys.exit(usage + str(msg))

    project = ProjectDescription()
    VERBOSITY = 1
    text_filename = None
    pickle_filename = "schema.pkl"
    base_dir = None
    prefix = None
    for (opt, val) in opts:
        if opt in ("-p", "project"):
            project.read(val)
        elif opt in ("-o", "--output"):
            pickle_filename = val
        elif opt in ("-t", "text"):
            text_filename = val
        elif opt in ("-d", "--base-dir"):
            base_dir = val
        elif opt == "-v":
            VERBOSITY += 1
        elif opt == "-q":
            VERBOSITY = 0

    if len(args) != 0:
        raise SystemExit, usage + "error: too many arguments"

    # Find modules in project.dirs, classes in those modules, and parse
    # the docstring of each class.  The result is an ObjectSchema, a
    # collection of atomic types, type aliases, and class definitions.
    schema = generate_schema(project, base_dir)

    # Write the schema out to either or both of a text and pickle file.
    write_schema(schema, text_filename, pickle_filename)

# main ()


if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
        sys.exit("interrupted")
