--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/python-2.5.2/win32/Lib/robotparser.py Fri Apr 03 17:19:34 2009 +0100
@@ -0,0 +1,297 @@
+""" robotparser.py
+
+ Copyright (C) 2000 Bastian Kleineidam
+
+ You can choose between two licenses when using this package:
+ 1) GNU GPLv2
+ 2) PSF license for Python 2.2
+
+ The robots.txt Exclusion Protocol is implemented as specified in
+ http://info.webcrawler.com/mak/projects/robots/norobots-rfc.html
+"""
+import urlparse,urllib
+
+__all__ = ["RobotFileParser"]
+
+debug = 0
+
+def _debug(msg):
+ if debug: print msg
+
+
+class RobotFileParser:
+ """ This class provides a set of methods to read, parse and answer
+ questions about a single robots.txt file.
+
+ """
+
+ def __init__(self, url=''):
+ self.entries = []
+ self.default_entry = None
+ self.disallow_all = False
+ self.allow_all = False
+ self.set_url(url)
+ self.last_checked = 0
+
+ def mtime(self):
+ """Returns the time the robots.txt file was last fetched.
+
+ This is useful for long-running web spiders that need to
+ check for new robots.txt files periodically.
+
+ """
+ return self.last_checked
+
+ def modified(self):
+ """Sets the time the robots.txt file was last fetched to the
+ current time.
+
+ """
+ import time
+ self.last_checked = time.time()
+
+ def set_url(self, url):
+ """Sets the URL referring to a robots.txt file."""
+ self.url = url
+ self.host, self.path = urlparse.urlparse(url)[1:3]
+
+ def read(self):
+ """Reads the robots.txt URL and feeds it to the parser."""
+ opener = URLopener()
+ f = opener.open(self.url)
+ lines = []
+ line = f.readline()
+ while line:
+ lines.append(line.strip())
+ line = f.readline()
+ self.errcode = opener.errcode
+ if self.errcode == 401 or self.errcode == 403:
+ self.disallow_all = True
+ _debug("disallow all")
+ elif self.errcode >= 400:
+ self.allow_all = True
+ _debug("allow all")
+ elif self.errcode == 200 and lines:
+ _debug("parse lines")
+ self.parse(lines)
+
+ def _add_entry(self, entry):
+ if "*" in entry.useragents:
+ # the default entry is considered last
+ self.default_entry = entry
+ else:
+ self.entries.append(entry)
+
+ def parse(self, lines):
+ """parse the input lines from a robots.txt file.
+ We allow that a user-agent: line is not preceded by
+ one or more blank lines."""
+ state = 0
+ linenumber = 0
+ entry = Entry()
+
+ for line in lines:
+ linenumber = linenumber + 1
+ if not line:
+ if state==1:
+ _debug("line %d: warning: you should insert"
+ " allow: or disallow: directives below any"
+ " user-agent: line" % linenumber)
+ entry = Entry()
+ state = 0
+ elif state==2:
+ self._add_entry(entry)
+ entry = Entry()
+ state = 0
+ # remove optional comment and strip line
+ i = line.find('#')
+ if i>=0:
+ line = line[:i]
+ line = line.strip()
+ if not line:
+ continue
+ line = line.split(':', 1)
+ if len(line) == 2:
+ line[0] = line[0].strip().lower()
+ line[1] = urllib.unquote(line[1].strip())
+ if line[0] == "user-agent":
+ if state==2:
+ _debug("line %d: warning: you should insert a blank"
+ " line before any user-agent"
+ " directive" % linenumber)
+ self._add_entry(entry)
+ entry = Entry()
+ entry.useragents.append(line[1])
+ state = 1
+ elif line[0] == "disallow":
+ if state==0:
+ _debug("line %d: error: you must insert a user-agent:"
+ " directive before this line" % linenumber)
+ else:
+ entry.rulelines.append(RuleLine(line[1], False))
+ state = 2
+ elif line[0] == "allow":
+ if state==0:
+ _debug("line %d: error: you must insert a user-agent:"
+ " directive before this line" % linenumber)
+ else:
+ entry.rulelines.append(RuleLine(line[1], True))
+ else:
+ _debug("line %d: warning: unknown key %s" % (linenumber,
+ line[0]))
+ else:
+ _debug("line %d: error: malformed line %s"%(linenumber, line))
+ if state==2:
+ self.entries.append(entry)
+ _debug("Parsed rules:\n%s" % str(self))
+
+
+ def can_fetch(self, useragent, url):
+ """using the parsed robots.txt decide if useragent can fetch url"""
+ _debug("Checking robots.txt allowance for:\n user agent: %s\n url: %s" %
+ (useragent, url))
+ if self.disallow_all:
+ return False
+ if self.allow_all:
+ return True
+ # search for given user agent matches
+ # the first match counts
+ url = urllib.quote(urlparse.urlparse(urllib.unquote(url))[2]) or "/"
+ for entry in self.entries:
+ if entry.applies_to(useragent):
+ return entry.allowance(url)
+ # try the default entry last
+ if self.default_entry:
+ return self.default_entry.allowance(url)
+ # agent not found ==> access granted
+ return True
+
+
+ def __str__(self):
+ ret = ""
+ for entry in self.entries:
+ ret = ret + str(entry) + "\n"
+ return ret
+
+
+class RuleLine:
+ """A rule line is a single "Allow:" (allowance==True) or "Disallow:"
+ (allowance==False) followed by a path."""
+ def __init__(self, path, allowance):
+ if path == '' and not allowance:
+ # an empty value means allow all
+ allowance = True
+ self.path = urllib.quote(path)
+ self.allowance = allowance
+
+ def applies_to(self, filename):
+ return self.path=="*" or filename.startswith(self.path)
+
+ def __str__(self):
+ return (self.allowance and "Allow" or "Disallow")+": "+self.path
+
+
+class Entry:
+ """An entry has one or more user-agents and zero or more rulelines"""
+ def __init__(self):
+ self.useragents = []
+ self.rulelines = []
+
+ def __str__(self):
+ ret = ""
+ for agent in self.useragents:
+ ret = ret + "User-agent: "+agent+"\n"
+ for line in self.rulelines:
+ ret = ret + str(line) + "\n"
+ return ret
+
+ def applies_to(self, useragent):
+ """check if this entry applies to the specified agent"""
+ # split the name token and make it lower case
+ useragent = useragent.split("/")[0].lower()
+ for agent in self.useragents:
+ if agent=='*':
+ # we have the catch-all agent
+ return True
+ agent = agent.lower()
+ if agent in useragent:
+ return True
+ return False
+
+ def allowance(self, filename):
+ """Preconditions:
+ - our agent applies to this entry
+ - filename is URL decoded"""
+ for line in self.rulelines:
+ _debug((filename, str(line), line.allowance))
+ if line.applies_to(filename):
+ return line.allowance
+ return True
+
+class URLopener(urllib.FancyURLopener):
+ def __init__(self, *args):
+ urllib.FancyURLopener.__init__(self, *args)
+ self.errcode = 200
+
+ def prompt_user_passwd(self, host, realm):
+ ## If robots.txt file is accessible only with a password,
+ ## we act as if the file wasn't there.
+ return None, None
+
+ def http_error_default(self, url, fp, errcode, errmsg, headers):
+ self.errcode = errcode
+ return urllib.FancyURLopener.http_error_default(self, url, fp, errcode,
+ errmsg, headers)
+
+def _check(a,b):
+ if not b:
+ ac = "access denied"
+ else:
+ ac = "access allowed"
+ if a!=b:
+ print "failed"
+ else:
+ print "ok (%s)" % ac
+ print
+
+def _test():
+ global debug
+ rp = RobotFileParser()
+ debug = 1
+
+ # robots.txt that exists, gotten to by redirection
+ rp.set_url('http://www.musi-cal.com/robots.txt')
+ rp.read()
+
+ # test for re.escape
+ _check(rp.can_fetch('*', 'http://www.musi-cal.com/'), 1)
+ # this should match the first rule, which is a disallow
+ _check(rp.can_fetch('', 'http://www.musi-cal.com/'), 0)
+ # various cherry pickers
+ _check(rp.can_fetch('CherryPickerSE',
+ 'http://www.musi-cal.com/cgi-bin/event-search'
+ '?city=San+Francisco'), 0)
+ _check(rp.can_fetch('CherryPickerSE/1.0',
+ 'http://www.musi-cal.com/cgi-bin/event-search'
+ '?city=San+Francisco'), 0)
+ _check(rp.can_fetch('CherryPickerSE/1.5',
+ 'http://www.musi-cal.com/cgi-bin/event-search'
+ '?city=San+Francisco'), 0)
+ # case sensitivity
+ _check(rp.can_fetch('ExtractorPro', 'http://www.musi-cal.com/blubba'), 0)
+ _check(rp.can_fetch('extractorpro', 'http://www.musi-cal.com/blubba'), 0)
+ # substring test
+ _check(rp.can_fetch('toolpak/1.1', 'http://www.musi-cal.com/blubba'), 0)
+ # tests for catch-all * agent
+ _check(rp.can_fetch('spam', 'http://www.musi-cal.com/search'), 0)
+ _check(rp.can_fetch('spam', 'http://www.musi-cal.com/Musician/me'), 1)
+ _check(rp.can_fetch('spam', 'http://www.musi-cal.com/'), 1)
+ _check(rp.can_fetch('spam', 'http://www.musi-cal.com/'), 1)
+
+ # robots.txt that does not exist
+ rp.set_url('http://www.lycos.com/robots.txt')
+ rp.read()
+ _check(rp.can_fetch('Mozilla', 'http://www.lycos.com/search'), 1)
+
+if __name__ == '__main__':
+ _test()