diff -r 000000000000 -r ae805ac0140d python-2.5.2/win32/Lib/robotparser.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/python-2.5.2/win32/Lib/robotparser.py Fri Apr 03 17:19:34 2009 +0100 @@ -0,0 +1,297 @@ +""" robotparser.py + + Copyright (C) 2000 Bastian Kleineidam + + You can choose between two licenses when using this package: + 1) GNU GPLv2 + 2) PSF license for Python 2.2 + + The robots.txt Exclusion Protocol is implemented as specified in + http://info.webcrawler.com/mak/projects/robots/norobots-rfc.html +""" +import urlparse,urllib + +__all__ = ["RobotFileParser"] + +debug = 0 + +def _debug(msg): + if debug: print msg + + +class RobotFileParser: + """ This class provides a set of methods to read, parse and answer + questions about a single robots.txt file. + + """ + + def __init__(self, url=''): + self.entries = [] + self.default_entry = None + self.disallow_all = False + self.allow_all = False + self.set_url(url) + self.last_checked = 0 + + def mtime(self): + """Returns the time the robots.txt file was last fetched. + + This is useful for long-running web spiders that need to + check for new robots.txt files periodically. + + """ + return self.last_checked + + def modified(self): + """Sets the time the robots.txt file was last fetched to the + current time. + + """ + import time + self.last_checked = time.time() + + def set_url(self, url): + """Sets the URL referring to a robots.txt file.""" + self.url = url + self.host, self.path = urlparse.urlparse(url)[1:3] + + def read(self): + """Reads the robots.txt URL and feeds it to the parser.""" + opener = URLopener() + f = opener.open(self.url) + lines = [] + line = f.readline() + while line: + lines.append(line.strip()) + line = f.readline() + self.errcode = opener.errcode + if self.errcode == 401 or self.errcode == 403: + self.disallow_all = True + _debug("disallow all") + elif self.errcode >= 400: + self.allow_all = True + _debug("allow all") + elif self.errcode == 200 and lines: + _debug("parse lines") + self.parse(lines) + + def _add_entry(self, entry): + if "*" in entry.useragents: + # the default entry is considered last + self.default_entry = entry + else: + self.entries.append(entry) + + def parse(self, lines): + """parse the input lines from a robots.txt file. + We allow that a user-agent: line is not preceded by + one or more blank lines.""" + state = 0 + linenumber = 0 + entry = Entry() + + for line in lines: + linenumber = linenumber + 1 + if not line: + if state==1: + _debug("line %d: warning: you should insert" + " allow: or disallow: directives below any" + " user-agent: line" % linenumber) + entry = Entry() + state = 0 + elif state==2: + self._add_entry(entry) + entry = Entry() + state = 0 + # remove optional comment and strip line + i = line.find('#') + if i>=0: + line = line[:i] + line = line.strip() + if not line: + continue + line = line.split(':', 1) + if len(line) == 2: + line[0] = line[0].strip().lower() + line[1] = urllib.unquote(line[1].strip()) + if line[0] == "user-agent": + if state==2: + _debug("line %d: warning: you should insert a blank" + " line before any user-agent" + " directive" % linenumber) + self._add_entry(entry) + entry = Entry() + entry.useragents.append(line[1]) + state = 1 + elif line[0] == "disallow": + if state==0: + _debug("line %d: error: you must insert a user-agent:" + " directive before this line" % linenumber) + else: + entry.rulelines.append(RuleLine(line[1], False)) + state = 2 + elif line[0] == "allow": + if state==0: + _debug("line %d: error: you must insert a user-agent:" + " directive before this line" % linenumber) + else: + entry.rulelines.append(RuleLine(line[1], True)) + else: + _debug("line %d: warning: unknown key %s" % (linenumber, + line[0])) + else: + _debug("line %d: error: malformed line %s"%(linenumber, line)) + if state==2: + self.entries.append(entry) + _debug("Parsed rules:\n%s" % str(self)) + + + def can_fetch(self, useragent, url): + """using the parsed robots.txt decide if useragent can fetch url""" + _debug("Checking robots.txt allowance for:\n user agent: %s\n url: %s" % + (useragent, url)) + if self.disallow_all: + return False + if self.allow_all: + return True + # search for given user agent matches + # the first match counts + url = urllib.quote(urlparse.urlparse(urllib.unquote(url))[2]) or "/" + for entry in self.entries: + if entry.applies_to(useragent): + return entry.allowance(url) + # try the default entry last + if self.default_entry: + return self.default_entry.allowance(url) + # agent not found ==> access granted + return True + + + def __str__(self): + ret = "" + for entry in self.entries: + ret = ret + str(entry) + "\n" + return ret + + +class RuleLine: + """A rule line is a single "Allow:" (allowance==True) or "Disallow:" + (allowance==False) followed by a path.""" + def __init__(self, path, allowance): + if path == '' and not allowance: + # an empty value means allow all + allowance = True + self.path = urllib.quote(path) + self.allowance = allowance + + def applies_to(self, filename): + return self.path=="*" or filename.startswith(self.path) + + def __str__(self): + return (self.allowance and "Allow" or "Disallow")+": "+self.path + + +class Entry: + """An entry has one or more user-agents and zero or more rulelines""" + def __init__(self): + self.useragents = [] + self.rulelines = [] + + def __str__(self): + ret = "" + for agent in self.useragents: + ret = ret + "User-agent: "+agent+"\n" + for line in self.rulelines: + ret = ret + str(line) + "\n" + return ret + + def applies_to(self, useragent): + """check if this entry applies to the specified agent""" + # split the name token and make it lower case + useragent = useragent.split("/")[0].lower() + for agent in self.useragents: + if agent=='*': + # we have the catch-all agent + return True + agent = agent.lower() + if agent in useragent: + return True + return False + + def allowance(self, filename): + """Preconditions: + - our agent applies to this entry + - filename is URL decoded""" + for line in self.rulelines: + _debug((filename, str(line), line.allowance)) + if line.applies_to(filename): + return line.allowance + return True + +class URLopener(urllib.FancyURLopener): + def __init__(self, *args): + urllib.FancyURLopener.__init__(self, *args) + self.errcode = 200 + + def prompt_user_passwd(self, host, realm): + ## If robots.txt file is accessible only with a password, + ## we act as if the file wasn't there. + return None, None + + def http_error_default(self, url, fp, errcode, errmsg, headers): + self.errcode = errcode + return urllib.FancyURLopener.http_error_default(self, url, fp, errcode, + errmsg, headers) + +def _check(a,b): + if not b: + ac = "access denied" + else: + ac = "access allowed" + if a!=b: + print "failed" + else: + print "ok (%s)" % ac + print + +def _test(): + global debug + rp = RobotFileParser() + debug = 1 + + # robots.txt that exists, gotten to by redirection + rp.set_url('http://www.musi-cal.com/robots.txt') + rp.read() + + # test for re.escape + _check(rp.can_fetch('*', 'http://www.musi-cal.com/'), 1) + # this should match the first rule, which is a disallow + _check(rp.can_fetch('', 'http://www.musi-cal.com/'), 0) + # various cherry pickers + _check(rp.can_fetch('CherryPickerSE', + 'http://www.musi-cal.com/cgi-bin/event-search' + '?city=San+Francisco'), 0) + _check(rp.can_fetch('CherryPickerSE/1.0', + 'http://www.musi-cal.com/cgi-bin/event-search' + '?city=San+Francisco'), 0) + _check(rp.can_fetch('CherryPickerSE/1.5', + 'http://www.musi-cal.com/cgi-bin/event-search' + '?city=San+Francisco'), 0) + # case sensitivity + _check(rp.can_fetch('ExtractorPro', 'http://www.musi-cal.com/blubba'), 0) + _check(rp.can_fetch('extractorpro', 'http://www.musi-cal.com/blubba'), 0) + # substring test + _check(rp.can_fetch('toolpak/1.1', 'http://www.musi-cal.com/blubba'), 0) + # tests for catch-all * agent + _check(rp.can_fetch('spam', 'http://www.musi-cal.com/search'), 0) + _check(rp.can_fetch('spam', 'http://www.musi-cal.com/Musician/me'), 1) + _check(rp.can_fetch('spam', 'http://www.musi-cal.com/'), 1) + _check(rp.can_fetch('spam', 'http://www.musi-cal.com/'), 1) + + # robots.txt that does not exist + rp.set_url('http://www.lycos.com/robots.txt') + rp.read() + _check(rp.can_fetch('Mozilla', 'http://www.lycos.com/search'), 1) + +if __name__ == '__main__': + _test()