"""$URL: svn://svn/repos/trunk/sitereaper/lib/reaper.py $
$Id: reaper.py 20952 2003-03-05 16:51:11Z dbinger $

   Provides Reaper, a class for traversing web sites and
   reporting on the results.
   Provides PageParser, a class used by Reaper to
   parse html pages.
"""

import sys
import posixpath
from cPickle import dump
from types import ListType, StringType, FileType
from urlparse import urlparse, urlunparse
from time import time, localtime, strftime, timezone

from sitereaper.verbose import Verbose
from sitereaper.email_validator import EmailValidator
from sitereaper.page_cache import PageCache
from sitereaper.link_report import LinkReport


def timestamp ():
    return strftime("%Y-%m-%d %H:%M:%S", localtime(time()))

class Reaper (Verbose):
    """A Reaper manages a traversal of a web tree, parsing
    pages, checking the spelling of html files, checking
    all links, and generating a report.

    Instance attributes:
      local_urls : [string]
        a list of URLs to help identify pages that will get extra
        scrutiny.  Any page with an initial substring that matches one
        of the local_urls will get a chance to be parsed, unless it also
        has matches one of excluded_urls
      excluded_urls : [string]
        list of URLs to exclude from further scrutiny (any URL
        with a prefix in excluded_urls is excluded)
      deprecated_urls : [string]
        list of URLs to warn about if seen in the wild
      ignore_url_schemes : [string]
        list of URL schemes to ignore

      check_external : boolean = true
        enable checking external URLs with a HEAD request
      max_errors : int = None
        stop checking after this many "internal server errors" (5xx
        response code) from a local URL.  If None (default), no limit.
      _num_errors : int
        internal counter: how many server errors we've seen so far
      spellcheck_skiplist : [string]
        skip spellchecking any document whose URL starts with one of the
        strings in this list.  If the list is empty, will spellcheck the
        non-tag text of every HTML document seen from a "local" URL; if
        the list contains the empty string, no documents will be
        spellchecked.
      validation_skiplist : [string]
        skip HTML validation of any document whose URL starts with one
        of the strings in this list.  Like with spellcheck_skiplist, []
        means validate all local documents, and [''] means validate
        nothing.
      email_validator : EmailValidator
        handles validating email addresses found in mailto: links
      page_cache : PageCache
        Used for contacting and retrieving pages.
      link_report : LinkReport
        Records links from and to each page.
      spell_dictionary : string
        Name of file containing extra spelling dictionary.
        Format is one word per line.

      report_file : file


    Besides the constructor, the most useful methods here
    are traverse() and report().
    """

    def __init__ (self,
                  local_urls=None,
                  excluded_urls=None,
                  deprecated_urls=None,
                  ignore_url_schemes=None,
                  report_file=None,
                  spell_dictionary=None,
                  verbose=0):
        """Reaper(local_urls : [string] = None,
                  excluded_urls : [string] = None,
                  deprecated_urls : [string] = None,
                  report_file : file = sys.stdout,
                  spell_dictionary : string = None,
                  verbose : int = 0)
        """
        assert local_urls is None or type(local_urls) is ListType, \
               "bad local_urls: %r" % local_urls
        assert excluded_urls is None or type(excluded_urls) is ListType, \
               "bad excluded_urls: %r" % excluded_urls
        assert deprecated_urls is None or type(deprecated_urls) is ListType, \
               "bad deprecated_urls: %r" % deprecated_urls
        assert report_file is None or type(report_file) is FileType, \
               "bad report_file: %r" % report_file
        assert (spell_dictionary is None
                or type(spell_dictionary) is StringType), \
               "bad spell_dictionary: %r" % spell_dictionary

        Verbose.__init__(self, verbose)
        self.local_urls = local_urls
        self.excluded_urls = excluded_urls
        self.deprecated_urls = deprecated_urls
        self.ignore_url_schemes = ignore_url_schemes or []
        self.report_file = report_file or sys.stdout
        self.check_external = 1
        self.max_errors = None
        self._num_errors = None
        self.spellcheck_skiplist = []
        self.validation_skiplist = []
        self.email_validator = EmailValidator()
        self.page_cache = PageCache(verbose)
        self.link_report = LinkReport()
        self.spell_dictionary = spell_dictionary


    # -- Trivial modifiers ---------------------------------------------

    def disable_external_checks (self):
        self.check_external = 0

    def set_max_errors (self, max):
        self.max_errors = max

    def set_spellcheck_skiplist (self, skiplist):
        self.spellcheck_skiplist = skiplist

    def set_validation_skiplist (self, skiplist):
        self.validation_skiplist = skiplist


    # -- Trivial delegate methods --------------------------------------

    def is_local (self, page):
        return page.url_matches(self.local_urls)

    def is_excluded (self, page):
        return page.url_matches(self.excluded_urls)

    def is_deprecated (self, page):
        return page.url_matches(self.deprecated_urls)

    def get_page(self, url):
        return self.page_cache.get_page(url)

    def should_spellcheck (self, page):
        return not page.url_matches(self.spellcheck_skiplist)

    def should_validate (self, page):
        return not page.url_matches(self.validation_skiplist)


    # -- Traversal methods ---------------------------------------------

    def traverse (self, roots):
        """traverse(roots : [string], out : file)

        'roots' is a list of URLs to use as starting points.

        A report is generated and written to 'out'.
        """

        # If no local URLs provided, derive a list from the roots.
        if not self.local_urls:
            self.local_urls = []
            for url in roots:
                (scheme, netloc, path, _, _, _) = urlparse(url)
                path = posixpath.dirname(path)
                url = urlunparse((scheme, netloc, path, '', '', ''))
                self.local_urls.append(url)

        start_time = time()
        self.report("""\
sitereaper traversal started at %s

traversal parameters:
  roots:         %s
  local_urls:    %s
  excluded_urls: %s
  deprecated:    %s

dictionary: %s
""" % (timestamp(),
       ", ".join(roots),
       ", ".join(self.local_urls),
       ", ".join(self.excluded_urls),
       ", ".join(self.deprecated_urls),
       self.spell_dictionary))

        already_processed = {}          # maps Page objects to 1
        self._num_errors = 0

        self.note(5, 'roots = ', str(roots))
        for root in roots:
            self.note(1, "traversing pages from %s" % root)
            remaining = [self.page_cache.get_page(root, retrieve=1)]

            while remaining:
                page = remaining.pop(0)
                if already_processed.has_key(page):
                    continue

                self.note(3, "examining %s (%d pages in queue)"
                             % (page, len(remaining)))

                assert page.retrieved, "page %s not retrieved" % page

                if page.page_ok() and page.get_type() == 'text/html':
                    more_pages = self.traverse_page(page)
                    remaining.extend(more_pages)

                self.note(3, "generating report for %s" % page)
                self.report_page(page)
                page.cleanup()
                already_processed[page] = 1

        self.report_all_suspicious_words()

    def traverse_page (self, page):
        """traverse_page(page : Page)
           -> [Page]

        Examine the Page, follow links, check spelling, and print a
        report.

        Return a list of Pages found in links that might need more
        examination.
        """
        self.note(3, 'traversing ', page)
        page.parse()
        if self.should_spellcheck(page):
            page.spellcheck(self.spell_dictionary)
        if self.should_validate(page):
            page.validate()
        self.check_images(page)
        remaining = self.follow_links(page)
        return remaining

    def follow_links (self, page):
        """follow_links(page : Page) -> [ Page ]

        Return a list of Pages for the hrefs of anchor tags
        in the given page.
        """
        self.note(4, "following links in %s" % page)
        remaining = []
        for link_url in page.get_links():
            scheme = link_url.split(":", 1)[0]
            if scheme == "http":
                link_page = self.get_page(link_url)
                self.link_report.record_link(page.url, link_url)
                self.examine_link (page, link_page, remaining)
            elif scheme == "mailto":
                link_url = link_url.replace("&#64;", "@")
                addr = link_url[7:]
                self.note(4, "validating email address %s" % addr)
                if not self.email_validator.check(addr):
                    err = self.email_validator.get_error(addr)
                    page.add_link_warning(link_url, err)
            elif scheme not in self.ignore_url_schemes:
                page.add_link_warning(link_url, "unknown URL type")

        return remaining

    def examine_link (self, source_page, link_page, remaining):
        """examine_link(source_page : Page,
                        link_page : Page,
                        remaining : [Page])

        Examine a single link found in source_page.  Skips link
        (with appropriate logging or warning) if it is excluded
        or deprecated; calls follow_link() otherwise.
        """
        if self.is_excluded(link_page):
            self.note(5, "skipping %s (excluded)" % link_page)

        elif self.is_deprecated(link_page):
            source_page.add_link_warning(link_url, "deprecated")

        else:
            self.follow_link(source_page, link_page, remaining)

    def follow_link (self, source_page, link_page, remaining):
        """follow_link(source_page : Page,
                       link_page : Page,
                       remaining : [Page])

        Try to follow a link from 'source_page' that is not explicitly
        excluded or deprecated.  If it's local, retrieve the page
        and add it to 'remaining' (which ends up being appended to
        the master page queue); if it's external, either check it or
        skip it as appropriate.
        """
        if self.is_local(link_page):
            if not link_page.retrieved:
                # XXX it might be better just to check link_page now;
                # this would let us avoid retrieving potentially large
                # non-HTML files.  The downside is that we would do both
                # HEAD and GET on all local HTML files.
                self.note(5, "retrieving %s" % link_page)
                rpage = self.page_cache.retrieve_page(link_page)
                if rpage.is_server_error():
                    self._num_errors += 1
                    if self.max_errors and self._num_errors >= self.max_errors:
                        raise RuntimeError("too many server errors")

                if rpage is not link_page: # redirected
                    self.follow_link(source_page, rpage, remaining)
                else:
                    self.note(4, "queuing %s" % rpage)
                    remaining.append(rpage)

        elif self.check_external:
            self.note(5, "checking %s" % link_page)
            rpage = self.page_cache.check_page(link_page)

        else:
            self.note(5, "skipping %s (external)" % link_page)
            return

        # Error retrieving or checking link_page -- save a warning
        # in source_page.
        if not link_page.page_ok():
            source_page.add_link_warning(link_page.url, link_page.get_error())


    def check_images (self, page):
        """check_images(page : Page)

        """
        self.note(4, "checking images in %s" % page)
        for image_url in page.get_images():
            image_page = self.get_page(image_url)
            self.note(5, "checking image %s" % image_page)
            self.page_cache.check_page(image_page)
            if not image_page.page_ok():
                page.add_link_warning(image_url,
                                      image_page.get_error())

            self.link_report.record_link(page.url, image_url)


    # -- Per-page reporting --------------------------------------------

    def report (self, *args):
        if not args or len(args) == 1 and not args[0]:
            return
        for arg in args:
            self.report_file.write(arg)
        if args[-1][-1] != "\n":
            self.report_file.write("\n")
        self.report_file.flush()

    def report_page (self, page):
        """report_page(page : Page, root : string)

        Print a report of any problems (network error, HTTP error, bad
        links, bad words, bad HTML, ...)  found in 'page' to the report
        file.
        """

        errors = []
        if page.page_ok():
            if page.parse_error:
                errors.append("HTML parse error: %s" % page.parse_error)

            link_warnings = ["link: " + warning
                             for warning in page.format_link_warnings()]
            errors.extend(link_warnings)

            words = page.get_suspicious_words()
            if words:
                errors.extend(["bad word: %s" % word
                               for word in words])
            vresults = page.get_validation_results()
            if vresults:
                errors.extend(vresults)
        else:
            status = page.get_response_status()
            # 3xx or 4xx responses will have been reported with the
            # page that links to this one, so don't report them again.
            # 5xx responses and network errors will be reported twice --
            # that's deliberate!
            if status is None or 500 <= status <= 599:
                errors.append(page.get_error())

        if errors:
            size = page.get_size()
            time = page.get_elapsed_time()
            stats = []
            if size is not None:
                stats.append("%d bytes" % size)
            if time is not None:
                stats.append("%0.2f sec" % time)
            if stats:
                stats = " (%s)" % (", ".join(stats))
            else:
                stats = ""

            self.report(page.url + stats)
            for error in errors:
                self.report("  " + error)


    # -- After-the-fact reporting --------------------------------------

    def get_suspicious_words(self):
        """get_suspicious_words() -> { word : string  -> urls : [ string ] }

        Returns a dictionary mapping each suspicious word to a list
        of the URLs that contain the word."""
        r = {}
        for (url, page) in self.page_cache.get_urls_pages():
            pwords = page.get_suspicious_words()
            if pwords:
                for word in pwords:
                    r.setdefault(word, []).append(url)

        self.note(7, 'suspicious words=', ", ".join(r.keys()))
        return r

    def report_all_suspicious_words (self):
        """report_all_suspicious_words() -> string

        Writes a report listing all suspicious words
        found in all pages.
        """
        word_urls = self.get_suspicious_words()
        words = word_urls.keys()
        words.sort()

        if not words:
            return

        self.report("\nMisspelled or unrecognized words:")
        for word in words:
            self.report("  %s:" % word)
            for url in word_urls[word]:
                self.report('    %s' % url)

    def report_stats (self):
        s = 'Search Statistics:\n'
        s += 'Date: %s\n' % timestamp()
        s += 'Pages in Cache: %s\n\n' % str(self.page_cache.get_size())
        return s

    def report_links (self, filename):
        file = open(filename, "w")
        dump(self.link_report, file, 1)


