"""$URL: svn://svn/repos/trunk/sitereaper/lib/page_parser.py $
$Id: page_parser.py 20952 2003-03-05 16:51:11Z dbinger $

Simple HTML parser that finds two kinds of links ("A HREF" and "IMG
SRC"), and stores non-tag text for later spell-checking.
"""

import re
from sitereaper.verbose import Verbose
from sitereaper.utils import warn
from HTMLParser import HTMLParser

HTML_ENTITY = { "quot"    : '"',
                "amp"     : '&',
                "lt"      : '<',
                "gt"      : '>',
                "nbsp"    : ' ',
                "copy"    : '(C)',
                "reg"     : '(R)',
                "micro"   : 'u',        # as in micrometres
                "Aring"   : 'A',        # as in Angstroms
                "Eacute"  : 'E',
                "eacute"  : 'e',
                "Omega"   : 'ohm',      # ugh!
                "deg"     : 'deg',

              }

url_re = re.compile(r'(https?|ftp)://\S+')
# space allowed around the '@' because of how our /feedback page
# formats the recipient email address
email_re = re.compile(r'[\w\!\#\$\%\&\'\*\+\-\/\=\?\^\`\{\|\}\~\.]+'
                      r'\s*\@\s*'
                      r'[\w\!\#\$\%\&\'\*\+\-\/\=\?\^\`\{\|\}\~\.]+')

# A "numword" is a gibberish combination of letters and digits which we
# do not want to spellcheck, eg. PXC200, 200SL, NT3300, nght2html.
numword_re = re.compile(r'([a-zA-Z]*? \d+ [a-zA-Z]+ | '
                        r' [a-zA-Z]+? \d+ [a-zA-Z]*)', re.VERBOSE)

# Also want to skip spellchecking TLAs (three-letter acronyms) and
# XTLAs (extended three-letter acronyms, aka four-letter acronyms)
tla_re = re.compile(r'\b[A-Z]{3,4}\b')


class PageParser (HTMLParser, Verbose):
    """
    Instance attributes:
      __links : [string]
        list of URLs found in "<A HREF=...>"
      __images : [string]
        list of URLs found in "<IMG SRC=...>"
      __text_file : file
        file to which all non-tag text found in the document will
        be written.
      __suppress_text : boolean = false
        when true, non-tag text will be skipped (not written to
        __text_file) -- used for <code> and <pre> tags
    """

    SUPPRESS_TEXT_TAGS = ("pre", "code", "script", "style")

    def __init__ (self, verbose=0):
        Verbose.__init__(self, verbose)
        HTMLParser.__init__(self)

    def reset (self):
        HTMLParser.reset(self)
        self.__links = []               # list of URLs (<A HREF> links)
        self.__images = []              # list of URLs (<IMG SRC> links)
        self.__text_file = None
        self.__suppress_text = 0

    def warn (self, msg):
        (line, offset) = self.getpos()
        text = []
        if line is not None:
            text.append("line %d" % line)
        if offset is not None:
            text.append("columnd %d" % offset)
        if text:
            text = ", ".join(text) + ": "
        else:
            text = ""
        warn(text + msg)

    def set_text_file (self, file):
        self.__text_file = file

    def _write_text (self, text):
        if not self.__suppress_text:
            self.__text_file.write(text)

    def handle_starttag (self, tag, attrs):
        self.note(10, "starttag: %r" % tag)
        if tag == "a":                  # looking for "<A HREF ...>"
            for (attr, val) in attrs:
                if attr == "href":
                    self.note(7, "adding link %r" % val)
                    self.__links.append(val)

        elif tag == "img":              # looking for "<IMG SRC ...>"
            for (attr, val) in attrs:
                if attr == "src":
                    self.note(7, "adding image %r" % val)
                    self.__images.append(val)

        elif tag in ("sup", "td", "th"):
            # Turning <sup> into a space is a nasty little hack to
            # ensure that "X<sup>sm</sup>" is treated as
            # "X sm".
            self._write_text(" ")
        elif tag in ("br", "tr"):
            self._write_text("\n")

        elif tag in self.SUPPRESS_TEXT_TAGS:
            self.__suppress_text += 1

    def handle_endtag (self, tag):
        if tag in self.SUPPRESS_TEXT_TAGS:
            if self.__suppress_text == 0:
                self.error("unbalanced tag: </%s>" % tag)
            self.__suppress_text -= 1
        elif tag in ("h1", "h2", "h3", "h4", "h5", "h6"):
            self._write_text("\n")

    def handle_data (self, data):
        assert self.__text_file is not None, "text_file not set"

        if self.get_verbose() >= 10:
            if len(data) > 50:
                self.note(10, "data: '%s ... %s'" % (data[0:40], data[-5:]))
            else:
                self.note(10, "data: %r" % data)

        data = url_re.sub('', data)
        data = email_re.sub('', data)
        data = numword_re.sub('', data)
        data = tla_re.sub('', data)

        self._write_text(data)

    def handle_entityref (self, name):
        text = HTML_ENTITY.get(name)
        if text is None:
            self.warn("HTML entity &%s; not recognized" % name)
        else:
            self._write_text(text)

    def handle_charref (self, ref):
        try:
            ref = int(ref)
        except ValueError:
            self.warn("bogus HTML character reference &#%s;" % ref)
            return
        else:
            if 32 <= ref <= 127:
                self._write_text(chr(ref))

    def get_links (self):
        return self.__links

    def get_images (self):
        return self.__images


if __name__ == "__main__":
    import sys
    infile = sys.argv[1]

    file = open(infile)
    parser = PageParser(verbose=0)
    parser.set_text_file(sys.stdout)
    print "-- parsing %s ---------" % infile
    parser.feed(file.read())
    file.close()
    parser.close()
    print "--------------------------------"

    print "links:"
    for url in parser.get_links():
        print "  " + url
    print "images:"
    for url in parser.get_images():
        print "  " + url
