"""$URL: svn://svn/repos/trunk/sitereaper/lib/page.py $
$Id: page.py 20952 2003-03-05 16:51:11Z dbinger $

Provides the Page class, used to store everything we know about
a single web resource (URL, HTTP response, content, etc.).
"""

import sys, os
import time
import socket
from urlparse import urlparse, urlunparse, urljoin
from httplib import HTTPConnection
from HTMLParser import HTMLParseError
from timeoutsocket import Timeout
import sitereaper
from sitereaper.utils import std_repr, warn
from sitereaper.verbose import Verbose
from sitereaper.spell import Speller
from sitereaper.validate import Validator
from sitereaper.page_parser import PageParser


FILE_EXTENSION = {
    'text/plain' : '.txt',
    'text/html'  : '.html',
    'image/jpeg' : '.jpeg',
    'image/gif' : '.gif',
    'image/png' : '.png',
    }



# Used by Page to inform PageCache of a redirect.
class Redirect (Exception):
    def __init__ (self, status, reason, url):
        self.status = status
        self.reason = reason
        self.url = url

    def __str__ (self):
        return "%d %s: %s" % (self.status, self.reason, self.url)


class Page(Verbose):
    """
    A Page represents a web resource (ie. the object at the other
    end of an HTTP URL), and everything that went wrong while
    retrieving or inspecting it.  This can include:
      * network problems (hostname didn't resolve, connection
        refused or timed out)
      * HTTP problems (3xx, 4xx, 5xx response) (note that a 3xx
        response isn't really a problem, but if some other Page
        links to a URL that just does a redirect, we want to
        know about it)
      * bad links (if it's an HTML document, it links to some
        resource that had either network or HTTP problems)
      * bad text (eg. spell-checking or HTML validation found
        problems with the content of this Page -- again only for
        HTML docs)


    Instance attributes:
      url : string
        the URL given in the constructor, with the fragment stripped
      parsed_url : ( scheme : string, \
                     netloc : string, \
                     path : string, \
                     params : sting, \
                     query : string, \
                     fragment : string )
        <scheme>://<netloc>/<path>;<params>?<query>#<fragment>

      network_error : string
        string containing any error message that resulted from
        attempting to connect to the HTTP server; None means
        no error
      response : httplib.HTTPResponse
        the response received from the server when this page was
        requested

      filename : string
        name of the file to which the page's content was written;
        None if we didn't save the content of the page (eg. on error
        or redirect)
      text_filename : string
        name of file with page's non-tag text -- only used for
        HTML documents
      parser : PageParser
        A PageParser that has been used to parse the contents of this page.
      parse_error : HTMLParser.HTMLParseError
        if there were errors parsing the page, the exception object
        that signaled the error and records the information about it
      size : int
        The number of bytes in the content for this web page.
      elapsed_time : float
        number of seconds it took to retrieve this page

      checked : boolean
        true if we have done either a HEAD or a GET for this URL
        (regardless of success)
      retrieved : boolean
        true if we have done a GET for this URL
        (regardless of success)

      spellcheck_results : { string : int }
        dictionary of words reported by ispell from the non-tag content
        of this web page; the dict records how many times each word
        occurs in the page
      validation_results : [ string ]
        list of error messages from running an HTML validator
        on the page's HTML content

      link_warnings : [(string, string)]
        a list of (url, message) tuples, one for each link in the
        page that had some sort of problem (eg. a network error,
        HTTP error, permanent HTTP redirect, unchecked link, ...)
    """

    def __init__ (self, url, verbose=0):
        """Page(url : string, verbose : int = 0)"""
        Verbose.__init__(self, verbose)
        self.set_url(url)               # sets url, parsed_url

        self.network_error = None
        self.response = None

        self.base_filename = None
        self.filename = None
        self.text_filename = None
        self.parser = None
        self.parse_error = None
        self.size = None
        self.elapsed_time = None

        self.checked = 0
        self.retrieved = 0

        self.spellcheck_results = None
        self.validation_results = None
        self.link_warnings = []

    def __str__ (self):
        return self.url

    __repr__ = std_repr

    def set_url (self, url):
        self.url = url
        self.parsed_url = urlparse(url)
        if self.parsed_url[0] != "http":
            raise ValueError, "Page class is only for HTTP URLs"

    def url_matches (self, url_prefixes):
        """url_matches(urls : [string]) -> boolean

        Return true if the URL of this page matches any of
        url_prefixes.
        """
        if url_prefixes is None:        # treat None like an empty list
            return 0
        url = self.url
        for prefix in url_prefixes:
            if url.startswith(prefix):
                return 1
        return 0

    def get_suspicious_words (self):
        if self.spellcheck_results:
            words = self.spellcheck_results.keys()
            words.sort()
            return words
        else:
            return None

    def get_validation_results (self):
        return self.validation_results

    def get_url (self): return self.url

    def get_scheme (self):   return self.parsed_url[0]
    def get_netloc (self):   return self.parsed_url[1]
    def get_path (self):     return self.parsed_url[2]
    def get_params (self):   return self.parsed_url[3]
    def get_query (self):    return self.parsed_url[4]
    def get_fragment (self): return self.parsed_url[5]

    def _make_request (self, headers, method="GET"):
        netloc = self.parsed_url[1]
        selector = urlunparse((None, None) + self.parsed_url[2:])
        try:
            conn = HTTPConnection(netloc)
            conn.putrequest(method, selector)
            conn.putheader("User-agent",
                           "sitereaper/%s" % sitereaper.__version__)
            if headers:
                for (key, value) in headers.items():
                    conn.putheader(key, value)
            conn.endheaders()
            response = conn.getresponse()
        except socket.error, err:
            self.network_error = "%s: %s" % (netloc, err[1])
            return (None, None)
        except Timeout, err:
            self.network_error = str(err)
            return (None, None)
        else:
            return (conn, response)

    def _interpret_response (self, response):
        read_content = 0
        if 100 <= response.status <= 199:
            # 1xx responses shouldn't be sent to HTTP 1.0 clients,
            # but clients are allowed to ignore them.
            ok = 1
        if 200 <= response.status <= 299:     # successful completion
            # should only get 200 responses, so warn if anything else
            if response.status != 200:
                warn("%s: unexpected success response: %s %s"
                     % (self.url, response.status, response.reason))
            read_content = 1
            ok = 1

        if 300 <= response.status <= 399:     # redirect of some sort
            if response.status not in (301, 302, 307):
                warn("%s: unexpected redirect response: %s %s"
                     % (self.url, response.status, response.reason))
            url = response.getheader("location")
            # Raise an exception because PageCache needs to cook
            # up a new cache filename for the new URL.
            self.note(5, "redirecting to %s" % url)
            raise Redirect(response.status, response.reason, url)

        elif 400 <= response.status <= 499:   # request error
            ok = 0
        elif 500 <= response.status <= 599:   # server error
            warn("%s: server error: %s %s"
                 % (self.url, response.status, response.reason))
            ok = 0

        return (ok, read_content)

    def _read_content (self, response, base_filename):

        headers = response.msg
        ext = FILE_EXTENSION.get(headers.gettype(), '')

        self.filename = filename = base_filename + ext
        self.note(8, "response received, writing content to %s" % filename)
        file = open(filename, "w")
        size = 0
        while 1:
            data = response.read(8*1024)
            if not data:
                break
            size += len(data)
            file.write(data)
        file.close()
        self.size = size

        hdr_filename = base_filename + ".hdr"
        self.note(8, "writing response headers to %s" % hdr_filename)
        hfile = open(hdr_filename, "w")
        print >>hfile, headers
        hfile.close()

    def retrieve (self, headers=None, base_filename=None):
        """retrieve(headers : { string : string } = None,
                    base_filename : string = None)

        Retrieve the content of this page and store it in a cache file.
        This should probably only be called from PageCache, to ensure
        that cookies are handled properly.

        Returns true if we successfully connected to the server, got a
        2xx response, and read the whole body.  Raises Redirect if we
        got a 3xx response.  Returns false otherwise.
        """
        self.note(7, "fetching %s" % self.url)
        start_time = time.time()
        (connection, response) = self._make_request(headers)
        self.response = response
        if connection and response:     # no network errors
            self.note(7, "request started OK, response = %s %s"
                         % (response.status, response.reason))

            try:
                (ok, read_content) = self._interpret_response(response)

                if read_content:
                    # Should only get here for a 2xx response.  Copy the
                    # response content to the cache file.
                    self.base_filename = base_filename
                    try:
                        self._read_content(response, base_filename)
                    except Timeout, err:
                        self.network_error = "%s: %s" % (self.url, err)
                        self.note(7, "network error reading content")
                        ok = 0
                    else:
                        clen = self.response.getheader("Content-length")
                        if clen and int(clen) != self.size:
                            warn("%s: content-length said %d bytes, "
                                 "but got %d bytes"
                                 % (self.url, clen, self.size))
            finally:
                self.checked = self.retrieved = 1
                connection.close()
        else:
            self.checked = self.retrieved = 1
            self.note(7, "network problem: %s" % self.network_error)
            ok = 0

        stop_time = time.time()
        self.elapsed_time = stop_time - start_time

        return ok

    def check (self, headers=None):
        self.note(7, "checking %s" % self.url)
        start_time = time.time()
        (connection, response) = self._make_request(headers, method="HEAD")
        self.response = response
        if connection and response:     # no network errors
            self.note(7, "made request, response = %s %s"
                         % (response.status, response.reason))
            try:
                (ok, _) = self._interpret_response(response)
            finally:
                self.checked = 1
                connection.close()
        else:
            self.checked = 1
            ok = 0

        self.elapsed_time = time.time() - start_time
        return ok

    def get_type (self):
        """get_type() -> string

        Return the MIME type reported when we requested this page.
        """
        if self.response:
            return self.response.msg.gettype()

    def get_size (self):
        """get_size() -> int

        Report the best information available about the number of bytes
        in the content of this web page.
        """
        return self.size

    def get_elapsed_time (self):
        return self.elapsed_time

    def get_response_status (self):
        return self.response and self.response.status

    def get_response_reason (self):
        return self.response and self.response.reason

    def get_error (self):
        if self.network_error:
            return self.network_error
        elif self.response:
            return "%03d %s" % (self.response.status, self.response.reason)
        else:
            return None

    def page_ok (self):
        return (self.network_error is None and
                self.response is not None and
                (200 <= self.response.status <= 299 or
                 self.response.status in (302, 307)))

    def is_request_error (self):
        return self.response and 400 <= self.response.status <= 499

    def is_server_error (self):
        return self.response and 500 <= self.response.status <= 599

    # XXX not thread-safe -- but who cares?
    def is_parsed (self):
        return self.parser is not None

    def parse (self):
        """Parse the content of the page: extract links and non-tag
        text for later spell-checking.
        """
        self.parser = PageParser(verbose=self.get_verbose())
        file = open(self.filename)
        self.text_filename = self.base_filename + ".txt"
        text_file = open(self.text_filename, "w")
        self.parser.set_text_file(text_file)
        try:
            self.parser.feed(file.read())
            self.parser.close()
        except HTMLParseError, err:
            self.parse_error = err
        text_file.close()
        file.close()

    def get_links (self):
        return [urljoin(self.url, url) for url in self.parser.get_links()]

    def get_images (self):
        return [urljoin(self.url, url) for url in self.parser.get_images()]

    def add_link_warning (self, url, message):
        assert message, \
               ("page %s: adding empty message %r for link %r" %
                (self, message, url))
        self.link_warnings.append((url, message))

    def format_link_warnings (self):
        return ["%s: %s" % (warning) for warning in self.link_warnings]


    def spellcheck (self, dictionary):
        speller = Speller(dictionary)
        speller.start(self.text_filename)
        self.spellcheck_results = speller.get_results()

    def validate (self):
        val = Validator(self.filename)
        val.validate()
        self.validation_results = val.format_results()

    def cleanup (self):
        try:
            if self.filename is not None and os.path.exists(self.filename):
                os.remove(self.filename)
            if self.text_filename is not None:
                os.remove(self.text_filename)
        except OSError, err:
            warn("%s: %s" % (err.filename, err.strerror))
